cclaw-cli 0.25.0 → 0.27.0

package/dist/cli.d.ts

@@ -26,6 +26,10 @@ interface ParsedArgs {
     evalNoWrite?: boolean;
     evalUpdateBaseline?: boolean;
     evalConfirm?: boolean;
+    /** Optional subcommand after `eval`. Currently only `diff` is supported. */
+    evalSubcommand?: "diff";
+    /** Positional arguments for eval subcommands (e.g. `diff <old> <new>`). */
+    evalArgs?: string[];
     showHelp?: boolean;
     showVersion?: boolean;
 }

package/dist/cli.js

@@ -16,6 +16,7 @@ import { HARNESS_ADAPTERS } from "./harness-adapters.js";
 import { runEval } from "./eval/runner.js";
 import { writeBaselinesFromReport } from "./eval/baseline.js";
 import { writeJsonReport, writeMarkdownReport } from "./eval/report.js";
+import { formatDiffMarkdown, runEvalDiff } from "./eval/diff.js";
 import { EVAL_TIERS } from "./eval/types.js";
 import { FLOW_STAGES } from "./types.js";
 const INSTALLER_COMMANDS = [
@@ -55,16 +56,22 @@ Commands:
                     --skip-retro       Bypass mandatory retro gate (requires --retro-reason).
                     --retro-reason=<t> Reason for bypassing retro gate.
   eval       Run cclaw evals against .cclaw/evals/corpus (Phase 7: structural verifier + baselines).
-             Flags: --stage=<id>         Limit to one flow stage (${FLOW_STAGES.join("|")}).
-                    --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=workflow).
+             Flags: --stage=<id>         Limit to one flow stage (${FLOW_STAGES.join("|")}) for Tier A/B.
+                    --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=multi-stage workflow).
                     --schema-only        Run only structural verifiers (default).
                     --rules              Also run rule-based verifiers (keywords, regex, counts, uniqueness, traceability).
-                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; Tier A also runs the single-shot agent-under-test.
+                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; Tier A runs the single-shot agent, Tier B/C the sandbox tool-using agent (read_file/write_file/glob/grep).
                     --dry-run            Validate config + corpus, print summary, do not execute.
                     --json               Emit machine-readable JSON on stdout.
                     --no-write           Skip writing the report to .cclaw/evals/reports/.
                     --update-baseline    Overwrite baselines from the current run (requires --confirm).
                     --confirm            Acknowledge --update-baseline (prevents accidental resets).
+
+             Subcommands:
+                    diff <old> <new>     Compare two reports under .cclaw/evals/reports/.
+                                         Each argument is a cclawVersion (e.g. 0.26.0), a filename,
+                                         or the literal "latest". Exit code 1 when the diff shows a
+                                         regression. Accepts --json to emit machine-readable output.
   upgrade    Refresh generated files in .cclaw without modifying user artifacts.
   uninstall  Remove .cclaw runtime and the generated harness shim files.
 
@@ -79,6 +86,9 @@ Examples:
   cclaw eval --dry-run
   cclaw eval --stage=brainstorm --schema-only
   cclaw eval --judge --tier=A --stage=brainstorm
+  cclaw eval --judge --tier=B --stage=spec
+  cclaw eval --tier=C --judge
+  cclaw eval diff 0.26.0 latest
 
 Docs:   https://github.com/zuevrs/cclaw
 Issues: https://github.com/zuevrs/cclaw/issues
@@ -372,10 +382,42 @@ function parseArgs(argv) {
     if (versionFlag) {
         parsed.showVersion = true;
     }
-    const [commandRaw, ...flags] = argv.filter((arg) => arg !== "--help" && arg !== "-h" && arg !== "--version" && arg !== "-v");
+    const filteredArgv = argv.filter((arg) => arg !== "--help" && arg !== "-h" && arg !== "--version" && arg !== "-v");
+    const [commandRaw, ...rest] = filteredArgv;
     parsed.command = INSTALLER_COMMANDS.includes(commandRaw)
         ? commandRaw
         : undefined;
+    // For `eval`, the next non-flag argument is an optional subcommand. Any
+    // subsequent non-flag tokens are captured as evalArgs (consumed by the
+    // subcommand handler). This preserves backwards compat: callers that run
+    // `cclaw eval --dry-run` see no subcommand and no positional args.
+    let flags = rest;
+    if (parsed.command === "eval") {
+        const evalArgs = [];
+        const remainder = [];
+        let sawSubcommand = false;
+        for (const token of rest) {
+            if (token.startsWith("--")) {
+                remainder.push(token);
+                continue;
+            }
+            if (!sawSubcommand) {
+                if (token === "diff") {
+                    parsed.evalSubcommand = "diff";
+                    sawSubcommand = true;
+                }
+                else {
+                    // Treat unknown positional as an eval arg for forward compat.
+                    evalArgs.push(token);
+                }
+                continue;
+            }
+            evalArgs.push(token);
+        }
+        if (evalArgs.length > 0)
+            parsed.evalArgs = evalArgs;
+        flags = remainder;
+    }
     for (const flag of flags) {
         if (flag.startsWith("--harnesses=")) {
             parsed.harnesses = parseHarnesses(flag.replace("--harnesses=", ""));
@@ -566,6 +608,33 @@ async function runCommand(parsed, ctx) {
         info(ctx, "Upgraded .cclaw runtime and regenerated generated files");
         return 0;
     }
+    if (command === "eval" && parsed.evalSubcommand === "diff") {
+        const args = parsed.evalArgs ?? [];
+        if (args.length !== 2) {
+            error(ctx, `\`cclaw eval diff\` requires two arguments: <old> <new>. ` +
+                `Example: cclaw eval diff 0.26.0 latest`);
+            return 1;
+        }
+        const [oldSel, newSel] = args;
+        try {
+            const diff = await runEvalDiff({
+                projectRoot: ctx.cwd,
+                old: oldSel,
+                new: newSel
+            });
+            if (parsed.evalJson === true) {
+                ctx.stdout.write(`${JSON.stringify(diff, null, 2)}\n`);
+            }
+            else {
+                ctx.stdout.write(formatDiffMarkdown(diff));
+            }
+            return diff.regressed ? 1 : 0;
+        }
+        catch (err) {
+            error(ctx, err instanceof Error ? err.message : String(err));
+            return 1;
+        }
+    }
     if (command === "eval") {
         const result = await runEval({
             projectRoot: ctx.cwd,
@@ -592,6 +661,12 @@ async function runCommand(parsed, ctx) {
             for (const [stage, count] of Object.entries(result.corpus.byStage)) {
                 ctx.stdout.write(`    - ${stage}: ${count}\n`);
             }
+            if (result.workflowCorpus.total > 0 || result.plannedTier === "C") {
+                ctx.stdout.write(`  workflow corpus: ${result.workflowCorpus.total} case(s)\n`);
+                for (const wf of result.workflowCorpus.cases) {
+                    ctx.stdout.write(`    - ${wf.id}: ${wf.stages.join(" → ")}\n`);
+                }
+            }
             ctx.stdout.write(`  verifiers available:\n`);
             for (const [key, value] of Object.entries(result.verifiersAvailable)) {
                 ctx.stdout.write(`    - ${key}: ${value ? "yes" : "no"}\n`);

package/dist/eval/agents/with-tools.d.ts

@@ -0,0 +1,44 @@
+import type { ChatUsage, EvalLlmClient } from "../llm-client.js";
+import { createSandbox, type Sandbox } from "../sandbox.js";
+import type { SandboxTool } from "../tools/index.js";
+import type { EvalCase, ResolvedEvalConfig, ToolUseSummary } from "../types.js";
+export declare class MaxTurnsExceededError extends Error {
+    readonly turns: number;
+    constructor(turns: number);
+}
+export interface WithToolsInput {
+    caseEntry: EvalCase;
+    config: Pick<ResolvedEvalConfig, "model" | "agentTemperature" | "timeoutMs" | "tokenPricing" | "toolMaxTurns" | "toolMaxArgumentsBytes" | "toolMaxResultBytes">;
+    projectRoot: string;
+    client: EvalLlmClient;
+    tools?: SandboxTool[];
+    /** Override for the SKILL.md loader (test hook). */
+    loadSkill?: (stage: EvalCase["stage"]) => Promise<string>;
+    /** Override for the sandbox factory (test hook). */
+    createSandboxFn?: typeof createSandbox;
+    /**
+     * Reuse an externally-managed sandbox instead of creating + disposing a
+     * per-call one. Tier C workflow orchestration uses this so every stage
+     * shares the same sandbox and earlier artifacts remain visible. When
+     * set, the caller is responsible for `dispose()`.
+     */
+    externalSandbox?: Sandbox;
+    /**
+     * Optional override of the default user prompt prefix. Tier C uses this
+     * to tell the model which stage it is on and where the prior artifacts
+     * are located.
+     */
+    promptPreamble?: string;
+}
+export interface WithToolsOutput {
+    artifact: string;
+    usage: ChatUsage;
+    usageUsd: number;
+    model: string;
+    attempts: number;
+    durationMs: number;
+    toolUse: ToolUseSummary;
+    systemPrompt: string;
+    userPrompt: string;
+}
+export declare function runWithTools(input: WithToolsInput): Promise<WithToolsOutput>;

package/dist/eval/agents/with-tools.js

@@ -0,0 +1,261 @@
+/**
+ * Tier B with-tools agent.
+ *
+ * Multi-turn loop with OpenAI-style function-calling over a set of
+ * sandbox-confined tools. The AUT is given:
+ *
+ *  - System prompt = stage SKILL.md (same contract as Tier A so the
+ *    single-shot baseline is comparable).
+ *  - User prompt = task description + a short "tools available" hint
+ *    that names the sandbox root and the four built-in tools.
+ *  - Tools = `read_file`, `write_file`, `glob`, `grep` (see
+ *    `src/eval/tools/`).
+ *
+ * The loop runs up to `config.toolMaxTurns` turns (default 8). Each
+ * turn:
+ *
+ *  1. Send the current transcript to the model with tools enabled.
+ *  2. Commit token usage against the wrapped client (cost guard sees
+ *     every call).
+ *  3. If the model returned tool_calls, execute each sandbox tool and
+ *     append a `role: "tool"` message with the JSON-serialized result.
+ *  4. If the model produced assistant content with `finish_reason: stop`,
+ *     treat that as the artifact and exit.
+ *
+ * When the turn budget is exhausted without a terminal stop, the agent
+ * throws `MaxTurnsExceededError`. The runner surfaces the error as a
+ * failed workflow verifier so the case counts as a regression.
+ *
+ * Artifact resolution: the final assistant content is the artifact. If
+ * the model used `write_file` to stage the artifact at
+ * `artifact.md` (or `artifact/<stage>.md`), we prefer that file — it
+ * mirrors the Tier C workflow where writes are the deliverable. The
+ * fallback is the terminal assistant message so prompts that don't
+ * call write_file still produce something judgable.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { computeUsageUsd } from "../cost-guard.js";
+import { createSandbox } from "../sandbox.js";
+import { BUILTIN_TOOLS, toolsByName, toolsForRequest, truncatePayload } from "../tools/index.js";
+import { loadStageSkill } from "./single-shot.js";
+export class MaxTurnsExceededError extends Error {
+    turns;
+    constructor(turns) {
+        super(`Tier B agent exceeded the ${turns}-turn budget without a terminal stop.`);
+        this.name = "MaxTurnsExceededError";
+        this.turns = turns;
+    }
+}
+const DEFAULT_MAX_TURNS = 8;
+const DEFAULT_MAX_ARG_BYTES = 64 * 1024;
+const DEFAULT_MAX_RESULT_BYTES = 32 * 1024;
+const ARTIFACT_CANDIDATES = ["artifact.md", "artifact.txt", "ARTIFACT.md"];
+export async function runWithTools(input) {
+    const { caseEntry, config, projectRoot, client } = input;
+    const maxTurns = clampPositive(config.toolMaxTurns, DEFAULT_MAX_TURNS);
+    const maxArgBytes = clampPositive(config.toolMaxArgumentsBytes, DEFAULT_MAX_ARG_BYTES);
+    const maxResultBytes = clampPositive(config.toolMaxResultBytes, DEFAULT_MAX_RESULT_BYTES);
+    const loader = input.loadSkill ?? ((stage) => loadStageSkill(projectRoot, stage));
+    const systemPrompt = await loader(caseEntry.stage);
+    const tools = input.tools ?? BUILTIN_TOOLS;
+    const toolMap = toolsByName(tools);
+    const toolsBody = toolsForRequest(tools);
+    const sandboxFactory = input.createSandboxFn ?? createSandbox;
+    const externalSandbox = input.externalSandbox;
+    const sandbox = externalSandbox ??
+        (await sandboxFactory({
+            projectRoot,
+            ...(caseEntry.contextFiles ? { contextFiles: caseEntry.contextFiles } : {})
+        }));
+    const toolUse = {
+        turns: 0,
+        calls: 0,
+        errors: 0,
+        deniedPaths: [],
+        byTool: {}
+    };
+    const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
+    let lastModel = config.model;
+    let totalAttempts = 0;
+    const userPrompt = buildUserPrompt(caseEntry, sandbox, tools, input.promptPreamble);
+    const messages = [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: userPrompt }
+    ];
+    const started = Date.now();
+    try {
+        for (let turn = 0; turn < maxTurns; turn += 1) {
+            toolUse.turns = turn + 1;
+            const response = await client.chat({
+                model: config.model,
+                messages,
+                temperature: config.agentTemperature ?? 0.2,
+                timeoutMs: config.timeoutMs,
+                tools: toolsBody,
+                toolChoice: "auto"
+            });
+            usage.promptTokens += response.usage.promptTokens;
+            usage.completionTokens += response.usage.completionTokens;
+            usage.totalTokens += response.usage.totalTokens;
+            lastModel = response.model;
+            totalAttempts += response.attempts;
+            const hasToolCalls = response.toolCalls && response.toolCalls.length > 0;
+            messages.push(rememberAssistant(response.content, response.toolCalls));
+            if (!hasToolCalls) {
+                const artifact = await resolveArtifact(sandbox, response.content);
+                return finalize(artifact, usage, lastModel, totalAttempts, started, toolUse, systemPrompt, userPrompt, config);
+            }
+            for (const call of response.toolCalls) {
+                const tool = toolMap.get(call.name);
+                const argBytes = Buffer.byteLength(call.arguments ?? "", "utf8");
+                if (argBytes > maxArgBytes) {
+                    toolUse.errors += 1;
+                    bumpToolCount(toolUse, call.name);
+                    messages.push(toolResponseMessage(call.id, {
+                        ok: false,
+                        name: call.name,
+                        error: `arguments payload exceeds ${maxArgBytes} bytes`
+                    }));
+                    continue;
+                }
+                if (!tool) {
+                    toolUse.errors += 1;
+                    bumpToolCount(toolUse, call.name);
+                    messages.push(toolResponseMessage(call.id, {
+                        ok: false,
+                        name: call.name,
+                        error: `unknown tool "${call.name}"`
+                    }));
+                    continue;
+                }
+                bumpToolCount(toolUse, call.name);
+                const result = await tool.invoke(call.arguments ?? "", {
+                    sandbox,
+                    maxResultBytes
+                });
+                if (!result.ok) {
+                    toolUse.errors += 1;
+                    const denied = result.details && typeof result.details.deniedPath === "string"
+                        ? result.details.deniedPath
+                        : undefined;
+                    if (denied && !toolUse.deniedPaths.includes(denied)) {
+                        toolUse.deniedPaths.push(denied);
+                    }
+                }
+                else {
+                    toolUse.calls += 1;
+                }
+                messages.push(toolResponseMessage(call.id, result));
+            }
+        }
+        throw new MaxTurnsExceededError(maxTurns);
+    }
+    finally {
+        if (!externalSandbox)
+            await sandbox.dispose();
+    }
+}
+function finalize(artifact, usage, model, attempts, started, toolUse, systemPrompt, userPrompt, config) {
+    const usageUsd = computeUsageUsd(model, usage, {
+        tokenPricing: config.tokenPricing
+    });
+    return {
+        artifact: artifact.trim(),
+        usage,
+        usageUsd,
+        model,
+        attempts,
+        durationMs: Date.now() - started,
+        toolUse,
+        systemPrompt,
+        userPrompt
+    };
+}
+function rememberAssistant(content, toolCalls) {
+    const base = { role: "assistant", content };
+    if (toolCalls && toolCalls.length > 0)
+        base.toolCalls = toolCalls;
+    return base;
+}
+function toolResponseMessage(callId, result) {
+    const payload = result.ok
+        ? { ok: true, content: result.content, details: result.details ?? {} }
+        : { ok: false, error: result.error, details: result.details ?? {} };
+    return {
+        role: "tool",
+        content: truncatePayload(JSON.stringify(payload), 32 * 1024),
+        toolCallId: callId,
+        name: result.name
+    };
+}
+function bumpToolCount(summary, name) {
+    summary.byTool[name] = (summary.byTool[name] ?? 0) + 1;
+}
+function clampPositive(value, fallback) {
+    if (value === undefined)
+        return fallback;
+    if (!Number.isFinite(value) || value <= 0)
+        return fallback;
+    return Math.floor(value);
+}
+function buildUserPrompt(caseEntry, sandbox, tools, preamble) {
+    const toolList = tools.map((t) => `- ${t.descriptor.name}: ${t.descriptor.description}`);
+    const files = caseEntry.contextFiles ?? [];
+    const contextLines = files.length > 0
+        ? files.map((f) => `- ${f}`).join("\n")
+        : "(no files seeded)";
+    const lines = [];
+    if (preamble && preamble.trim().length > 0) {
+        lines.push(preamble.trim(), ``);
+    }
+    lines.push(`Stage: ${caseEntry.stage}`, `Case id: ${caseEntry.id}`, ``);
+    const rest = [
+        `Sandbox root: ${sandbox.root}`,
+        `You may call the following tools to read or modify files inside the sandbox.`,
+        `All paths are relative to the sandbox root.`,
+        ``,
+        `Tools:`,
+        ...toolList,
+        ``,
+        `Seeded context files (available under the sandbox root):`,
+        contextLines,
+        ``,
+        `Task:`,
+        caseEntry.inputPrompt.trim(),
+        ``,
+        `When you are done, reply with the artifact as the final assistant message.`,
+        `Output the artifact directly (markdown with optional YAML frontmatter).`,
+        `Do not wrap in code fences, do not add commentary before or after.`,
+        `You may optionally write the artifact to \`artifact.md\` in the sandbox; ` +
+            `if you do, the last written \`artifact.md\` is preferred over the chat reply.`
+    ];
+    lines.push(...rest);
+    return lines.join("\n");
+}
+async function resolveArtifact(sandbox, fallback) {
+    for (const candidate of ARTIFACT_CANDIDATES) {
+        try {
+            const abs = await sandbox.resolve(candidate);
+            const stat = await fs.stat(abs);
+            if (stat.isFile()) {
+                return await fs.readFile(abs, "utf8");
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    try {
+        const dir = path.join(sandbox.root);
+        const entries = (await fs.readdir(dir, { withFileTypes: true }));
+        const match = entries.find((entry) => entry.isFile() && /^artifact\./i.test(entry.name));
+        if (match) {
+            return await fs.readFile(path.join(dir, match.name), "utf8");
+        }
+    }
+    catch {
+        // fall through to fallback
+    }
+    return fallback;
+}

package/dist/eval/agents/workflow.d.ts

@@ -0,0 +1,24 @@
+import type { EvalLlmClient } from "../llm-client.js";
+import { createSandbox } from "../sandbox.js";
+import type { SandboxTool } from "../tools/index.js";
+import type { ResolvedEvalConfig, WorkflowCase, WorkflowStageName, WorkflowStageResult } from "../types.js";
+export interface WorkflowInput {
+    workflow: WorkflowCase;
+    config: Pick<ResolvedEvalConfig, "model" | "agentTemperature" | "timeoutMs" | "tokenPricing" | "toolMaxTurns" | "toolMaxArgumentsBytes" | "toolMaxResultBytes" | "workflowMaxTotalTurns">;
+    projectRoot: string;
+    client: EvalLlmClient;
+    tools?: SandboxTool[];
+    /** Override for the SKILL.md loader (test hook). */
+    loadSkill?: (stage: WorkflowStageName) => Promise<string>;
+    /** Override for the sandbox factory (test hook). */
+    createSandboxFn?: typeof createSandbox;
+}
+export interface WorkflowOutput {
+    caseId: string;
+    stages: WorkflowStageResult[];
+    /** Map from stage name to produced artifact (also persisted in sandbox). */
+    artifacts: Map<WorkflowStageName, string>;
+    totalUsageUsd: number;
+    totalDurationMs: number;
+}
+export declare function runWorkflow(input: WorkflowInput): Promise<WorkflowOutput>;

package/dist/eval/agents/workflow.js

@@ -0,0 +1,133 @@
+/**
+ * Tier C workflow agent.
+ *
+ * Runs the Tier B with-tools loop once per stage in a workflow case,
+ * sharing a single sandbox across stages so every new stage can read
+ * the earlier artifacts the model produced. The shape of the run is:
+ *
+ *   1. Create one sandbox seeded with `contextFiles`.
+ *   2. For each stage in `workflow.stages`:
+ *      a. Delete any leftover `artifact.md` so the resolver doesn't
+ *         accidentally pick the previous stage's output.
+ *      b. Invoke `runWithTools({ externalSandbox: sandbox, promptPreamble })`.
+ *         The preamble tells the model which stage it is on and lists the
+ *         `stages/*.md` files available for reading.
+ *      c. Persist the returned artifact to `stages/<stage>.md` inside the
+ *         sandbox (deterministic, regardless of whether the model wrote
+ *         `artifact.md` itself).
+ *      d. Record `WorkflowStageResult` with usage, duration, and tool use.
+ *   3. Dispose the sandbox in a `finally` so temp directories never leak.
+ *
+ * Errors bubble up from `runWithTools`:
+ *   - `MaxTurnsExceededError` stops the workflow at the current stage.
+ *   - `DailyCostCapExceededError` (surfaced by the cost-guard wrapper in
+ *     the runner) aborts immediately.
+ *   - Generic `EvalLlmError` subclasses propagate as-is so the runner can
+ *     record a workflow-level verifier failure.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { createSandbox } from "../sandbox.js";
+import { loadStageSkill } from "./single-shot.js";
+import { runWithTools } from "./with-tools.js";
+const STAGES_SUBDIR = "stages";
+const ARTIFACT_CANDIDATES = ["artifact.md", "artifact.txt", "ARTIFACT.md"];
+export async function runWorkflow(input) {
+    const { workflow, config, projectRoot, client } = input;
+    const sandboxFactory = input.createSandboxFn ?? createSandbox;
+    const sandbox = await sandboxFactory({
+        projectRoot,
+        ...(workflow.contextFiles ? { contextFiles: workflow.contextFiles } : {})
+    });
+    const stageResults = [];
+    const artifacts = new Map();
+    let totalUsageUsd = 0;
+    let totalDurationMs = 0;
+    try {
+        await fs.mkdir(await sandbox.resolve(STAGES_SUBDIR, { allowMissing: true }), { recursive: true });
+        for (const step of workflow.stages) {
+            await clearArtifactFile(sandbox);
+            const priorStages = stageResults.map((r) => r.stage);
+            const preamble = buildStagePreamble(workflow, step.name, priorStages);
+            const caseEntry = {
+                id: `${workflow.id}/${step.name}`,
+                stage: step.name,
+                inputPrompt: step.inputPrompt,
+                ...(workflow.contextFiles ? { contextFiles: workflow.contextFiles } : {})
+            };
+            const result = await runWithTools({
+                caseEntry,
+                config,
+                projectRoot,
+                client,
+                ...(input.tools ? { tools: input.tools } : {}),
+                ...(input.loadSkill
+                    ? { loadSkill: input.loadSkill }
+                    : {
+                        loadSkill: (stage) => loadStageSkill(projectRoot, stage)
+                    }),
+                externalSandbox: sandbox,
+                promptPreamble: preamble
+            });
+            await persistStageArtifact(sandbox, step.name, result.artifact);
+            artifacts.set(step.name, result.artifact);
+            const stageResult = {
+                stage: step.name,
+                artifact: result.artifact,
+                durationMs: result.durationMs,
+                usageUsd: result.usageUsd,
+                toolUse: result.toolUse,
+                attempts: result.attempts,
+                model: result.model,
+                promptTokens: result.usage.promptTokens,
+                completionTokens: result.usage.completionTokens
+            };
+            stageResults.push(stageResult);
+            totalUsageUsd += result.usageUsd;
+            totalDurationMs += result.durationMs;
+        }
+        return {
+            caseId: workflow.id,
+            stages: stageResults,
+            artifacts,
+            totalUsageUsd: Number(totalUsageUsd.toFixed(6)),
+            totalDurationMs
+        };
+    }
+    finally {
+        await sandbox.dispose();
+    }
+}
+async function clearArtifactFile(sandbox) {
+    for (const candidate of ARTIFACT_CANDIDATES) {
+        try {
+            const abs = await sandbox.resolve(candidate);
+            await fs.rm(abs, { force: true });
+        }
+        catch {
+            // candidate did not exist — resolve threw SandboxEscapeError for
+            // missing realpath; safe to ignore.
+        }
+    }
+}
+async function persistStageArtifact(sandbox, stage, artifact) {
+    const rel = `${STAGES_SUBDIR}/${stage}.md`;
+    const abs = await sandbox.resolve(rel, { allowMissing: true });
+    await fs.mkdir(path.dirname(abs), { recursive: true });
+    await fs.writeFile(abs, artifact.endsWith("\n") ? artifact : `${artifact}\n`, "utf8");
+}
+function buildStagePreamble(workflow, current, priorStages) {
+    const lines = [];
+    lines.push(`You are running stage "${current}" of the Tier C workflow "${workflow.id}".`);
+    if (workflow.description) {
+        lines.push(`Case description: ${workflow.description}`);
+    }
+    if (priorStages.length === 0) {
+        lines.push(`This is the first stage. Any context_files have been seeded into the sandbox root.`);
+    }
+    else {
+        lines.push(`Earlier stage artifacts are available via read_file:`, ...priorStages.map((name) => `  - ${STAGES_SUBDIR}/${name}.md`), `Read the prior artifacts before drafting your output so decisions and ` +
+            `ids carry through.`);
+    }
+    return lines.join("\n");
+}

package/dist/eval/config-loader.js

@@ -32,7 +32,11 @@ const NUMERIC_ENVS = new Set([
     "CCLAW_EVAL_MAX_RETRIES",
     "CCLAW_EVAL_JUDGE_SAMPLES",
     "CCLAW_EVAL_JUDGE_TEMPERATURE",
-    "CCLAW_EVAL_AGENT_TEMPERATURE"
+    "CCLAW_EVAL_AGENT_TEMPERATURE",
+    "CCLAW_EVAL_TOOL_MAX_TURNS",
+    "CCLAW_EVAL_TOOL_MAX_ARG_BYTES",
+    "CCLAW_EVAL_TOOL_MAX_RESULT_BYTES",
+    "CCLAW_EVAL_WORKFLOW_MAX_TOTAL_TURNS"
 ]);
 function evalConfigError(configFilePath, reason) {
     return new Error(`Invalid cclaw eval config at ${configFilePath}: ${reason}\n` +
@@ -152,6 +156,18 @@ function validateFileConfig(raw, configFilePath) {
         }
         out.tokenPricing = pricing;
     }
+    const assignPositiveInt = (key, value, label) => {
+        if (value === undefined)
+            return;
+        if (!Number.isInteger(value) || value < 1) {
+            throw evalConfigError(configFilePath, `"${label}" must be a positive integer`);
+        }
+        out[key] = value;
+    };
+    assignPositiveInt("toolMaxTurns", raw.toolMaxTurns, "toolMaxTurns");
+    assignPositiveInt("toolMaxArgumentsBytes", raw.toolMaxArgumentsBytes, "toolMaxArgumentsBytes");
+    assignPositiveInt("toolMaxResultBytes", raw.toolMaxResultBytes, "toolMaxResultBytes");
+    assignPositiveInt("workflowMaxTotalTurns", raw.workflowMaxTotalTurns, "workflowMaxTotalTurns");
     if (raw.regression !== undefined) {
         if (!isRecord(raw.regression)) {
             throw evalConfigError(configFilePath, `"regression" must be a mapping`);
@@ -186,7 +202,11 @@ function validateFileConfig(raw, configFilePath) {
         "judgeSamples",
         "judgeTemperature",
         "agentTemperature",
-        "tokenPricing"
+        "tokenPricing",
+        "toolMaxTurns",
+        "toolMaxArgumentsBytes",
+        "toolMaxResultBytes",
+        "workflowMaxTotalTurns"
     ]);
     const unknown = Object.keys(raw).filter((key) => !knownKeys.has(key));
     if (unknown.length > 0) {
@@ -296,6 +316,22 @@ function applyEnvOverrides(base, env) {
         patched.agentTemperature = value;
         overridden = true;
     }
+    const readPositiveInt = (name, key, label) => {
+        const raw = read(name);
+        if (!raw)
+            return;
+        const value = parseNumericEnv(name, raw);
+        if (!Number.isInteger(value) || value < 1) {
+            throw new Error(`Environment variable ${name} must be a positive integer, got: ${raw}`);
+        }
+        patched[key] = value;
+        overridden = true;
+        void label;
+    };
+    readPositiveInt("CCLAW_EVAL_TOOL_MAX_TURNS", "toolMaxTurns", "toolMaxTurns");
+    readPositiveInt("CCLAW_EVAL_WORKFLOW_MAX_TOTAL_TURNS", "workflowMaxTotalTurns", "workflowMaxTotalTurns");
+    readPositiveInt("CCLAW_EVAL_TOOL_MAX_ARG_BYTES", "toolMaxArgumentsBytes", "toolMaxArgumentsBytes");
+    readPositiveInt("CCLAW_EVAL_TOOL_MAX_RESULT_BYTES", "toolMaxResultBytes", "toolMaxResultBytes");
     const apiKey = read("CCLAW_EVAL_API_KEY");
     return { patched, overridden, apiKey };
 }