cclaw-cli 0.26.0 → 0.28.0

package/dist/eval/diff.d.ts

@@ -0,0 +1,64 @@
+import type { EvalReport } from "./types.js";
+export interface EvalDiffInput {
+    projectRoot: string;
+    /** Version string, filename, or "latest". */
+    old: string;
+    /** Version string, filename, or "latest". */
+    new: string;
+}
+export interface EvalDiffCaseEntry {
+    caseId: string;
+    stage: string;
+    /** Pass/fail transition: `same`, `regressed`, `recovered`, `added`, `removed`. */
+    transition: "same" | "regressed" | "recovered" | "added" | "removed";
+    previousPassed?: boolean;
+    currentPassed?: boolean;
+    durationDeltaMs?: number;
+    costDeltaUsd?: number;
+    verifierDeltas: EvalDiffVerifierEntry[];
+    stageDeltas?: EvalDiffStageEntry[];
+}
+export interface EvalDiffVerifierEntry {
+    verifierId: string;
+    kind: string;
+    transition: "same" | "regressed" | "recovered" | "added" | "removed" | "score-drop";
+    previousScore?: number;
+    currentScore?: number;
+    previousOk?: boolean;
+    currentOk?: boolean;
+}
+export interface EvalDiffStageEntry {
+    stage: string;
+    durationDeltaMs: number;
+    costDeltaUsd: number;
+    turnsDelta: number;
+    callsDelta: number;
+}
+export interface EvalDiffReport {
+    old: EvalDiffReportMeta;
+    new: EvalDiffReportMeta;
+    summaryDelta: {
+        totalCasesDelta: number;
+        passedDelta: number;
+        failedDelta: number;
+        skippedDelta: number;
+        totalCostUsdDelta: number;
+        totalDurationMsDelta: number;
+    };
+    cases: EvalDiffCaseEntry[];
+    /** True when any case regressed or any verifier dropped. */
+    regressed: boolean;
+}
+export interface EvalDiffReportMeta {
+    runId: string;
+    cclawVersion: string;
+    generatedAt: string;
+    mode: string;
+    model: string;
+    sourcePath: string;
+}
+export declare function resolveReportPath(projectRoot: string, selector: string): Promise<string>;
+export declare function diffReports(previous: EvalReport, current: EvalReport, prevPath: string, currPath: string): EvalDiffReport;
+export declare function runEvalDiff(input: EvalDiffInput): Promise<EvalDiffReport>;
+/** Render the diff as a terse human-readable Markdown block. */
+export declare function formatDiffMarkdown(diff: EvalDiffReport): string;

package/dist/eval/diff.js

@@ -0,0 +1,323 @@
+/**
+ * `cclaw eval diff <old> <new>` — side-by-side report comparison.
+ *
+ * Loads two JSON reports under `.cclaw/evals/reports/` (by version tag or
+ * explicit filename) and emits a compact human-readable + JSON diff:
+ *
+ *   - summary-level deltas (passed/failed/cost/duration)
+ *   - per-case pass/fail transitions
+ *   - per-verifier score drops (only the drops — new passes are noted in
+ *     the summary line, not repeated per verifier)
+ *   - Workflow-mode stage-level cost & duration deltas when both reports
+ *     carry a `workflow` summary for the same case id
+ *
+ * The resolver accepts three shapes for the `<old>` / `<new>` arguments:
+ *
+ *   1. A bare version string (`0.26.0`) — matched against any report JSON
+ *      whose `cclawVersion` field equals the string.
+ *   2. A full or relative filename (`eval-2026-04-17T...-abc123.json`).
+ *   3. The literal `latest` — picks the most recent report on disk by
+ *      mtime.
+ *
+ * The diff is deterministic: sorted by case id, then verifier id. Missing
+ * cases in one report show up as `added` or `removed` so callers can see
+ * which corpus changes slipped in between versions.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+const SCORE_DROP_EPSILON = 0.0001;
+export async function resolveReportPath(projectRoot, selector) {
+    const dir = path.join(projectRoot, EVALS_ROOT, "reports");
+    if (!(await exists(dir))) {
+        throw new Error(`No reports directory at ${path.relative(projectRoot, dir)}. ` +
+            `Run \`cclaw eval\` at least once before comparing reports.`);
+    }
+    const trimmed = selector.trim();
+    if (trimmed.length === 0) {
+        throw new Error(`Empty report selector. Pass a version like "0.26.0" or "latest".`);
+    }
+    // 1. Explicit filename (absolute or relative).
+    const asPath = path.isAbsolute(trimmed) ? trimmed : path.join(dir, trimmed);
+    if (await exists(asPath))
+        return asPath;
+    if (trimmed.endsWith(".json") && (await exists(asPath)))
+        return asPath;
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    const jsonFiles = entries
+        .filter((e) => e.isFile() && e.name.endsWith(".json"))
+        .map((e) => path.join(dir, e.name));
+    if (jsonFiles.length === 0) {
+        throw new Error(`No JSON reports found under ${path.relative(projectRoot, dir)}.`);
+    }
+    if (trimmed === "latest") {
+        let latest = jsonFiles[0];
+        let latestMtime = (await fs.stat(latest)).mtimeMs;
+        for (const f of jsonFiles.slice(1)) {
+            const stat = await fs.stat(f);
+            if (stat.mtimeMs > latestMtime) {
+                latest = f;
+                latestMtime = stat.mtimeMs;
+            }
+        }
+        return latest;
+    }
+    // 3. Version match — pick most recent by mtime among matches.
+    const matches = [];
+    for (const file of jsonFiles) {
+        try {
+            const raw = await fs.readFile(file, "utf8");
+            const parsed = JSON.parse(raw);
+            if (parsed.cclawVersion === trimmed) {
+                const stat = await fs.stat(file);
+                matches.push({ file, mtimeMs: stat.mtimeMs });
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    if (matches.length === 0) {
+        throw new Error(`No report matched selector "${selector}". ` +
+            `Pass a filename under ${path.relative(projectRoot, dir)} or a cclawVersion present in one of the reports.`);
+    }
+    matches.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    return matches[0].file;
+}
+async function loadReport(filePath) {
+    const raw = await fs.readFile(filePath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (parsed.schemaVersion !== 1 || !Array.isArray(parsed.cases)) {
+        throw new Error(`File at ${filePath} is not a valid cclaw eval report (missing schemaVersion or cases).`);
+    }
+    return parsed;
+}
+function meta(report, sourcePath) {
+    return {
+        runId: report.runId,
+        cclawVersion: report.cclawVersion,
+        generatedAt: report.generatedAt,
+        mode: report.mode,
+        model: report.model,
+        sourcePath
+    };
+}
+function verifierMap(results) {
+    const out = new Map();
+    for (const v of results)
+        out.set(v.id, v);
+    return out;
+}
+function diffCase(caseId, previous, current) {
+    const stage = (current ?? previous).stage;
+    if (!previous) {
+        return {
+            caseId,
+            stage,
+            transition: "added",
+            currentPassed: current?.passed,
+            verifierDeltas: []
+        };
+    }
+    if (!current) {
+        return {
+            caseId,
+            stage,
+            transition: "removed",
+            previousPassed: previous.passed,
+            verifierDeltas: []
+        };
+    }
+    const transition = previous.passed === current.passed
+        ? "same"
+        : previous.passed && !current.passed
+            ? "regressed"
+            : "recovered";
+    const prevMap = verifierMap(previous.verifierResults);
+    const currMap = verifierMap(current.verifierResults);
+    const verifierDeltas = [];
+    const allIds = new Set([...prevMap.keys(), ...currMap.keys()]);
+    for (const id of [...allIds].sort((a, b) => a.localeCompare(b))) {
+        const p = prevMap.get(id);
+        const c = currMap.get(id);
+        const kind = (c ?? p).kind;
+        if (!p && c) {
+            verifierDeltas.push({
+                verifierId: id,
+                kind,
+                transition: "added",
+                currentOk: c.ok,
+                ...(c.score !== undefined ? { currentScore: c.score } : {})
+            });
+            continue;
+        }
+        if (p && !c) {
+            verifierDeltas.push({
+                verifierId: id,
+                kind,
+                transition: "removed",
+                previousOk: p.ok,
+                ...(p.score !== undefined ? { previousScore: p.score } : {})
+            });
+            continue;
+        }
+        if (!p || !c)
+            continue;
+        const okChanged = p.ok !== c.ok;
+        const scoreChanged = typeof p.score === "number" &&
+            typeof c.score === "number" &&
+            Math.abs(p.score - c.score) > SCORE_DROP_EPSILON;
+        if (!okChanged && !scoreChanged)
+            continue;
+        const entry = {
+            verifierId: id,
+            kind,
+            transition: okChanged
+                ? p.ok
+                    ? "regressed"
+                    : "recovered"
+                : typeof p.score === "number" &&
+                    typeof c.score === "number" &&
+                    c.score < p.score
+                    ? "score-drop"
+                    : "same",
+            previousOk: p.ok,
+            currentOk: c.ok
+        };
+        if (typeof p.score === "number")
+            entry.previousScore = p.score;
+        if (typeof c.score === "number")
+            entry.currentScore = c.score;
+        if (entry.transition !== "same")
+            verifierDeltas.push(entry);
+    }
+    const caseEntry = {
+        caseId,
+        stage,
+        transition,
+        previousPassed: previous.passed,
+        currentPassed: current.passed,
+        durationDeltaMs: current.durationMs - previous.durationMs,
+        verifierDeltas
+    };
+    const costDelta = (current.costUsd ?? 0) - (previous.costUsd ?? 0);
+    if (Math.abs(costDelta) > SCORE_DROP_EPSILON) {
+        caseEntry.costDeltaUsd = Number(costDelta.toFixed(6));
+    }
+    if (previous.workflow && current.workflow) {
+        const prevStages = new Map();
+        for (const s of previous.workflow.stages)
+            prevStages.set(s.stage, s);
+        const stageDeltas = [];
+        for (const curStage of current.workflow.stages) {
+            const prevStage = prevStages.get(curStage.stage);
+            if (!prevStage)
+                continue;
+            stageDeltas.push({
+                stage: curStage.stage,
+                durationDeltaMs: curStage.durationMs - prevStage.durationMs,
+                costDeltaUsd: Number((curStage.usageUsd - prevStage.usageUsd).toFixed(6)),
+                turnsDelta: curStage.toolUse.turns - prevStage.toolUse.turns,
+                callsDelta: curStage.toolUse.calls - prevStage.toolUse.calls
+            });
+        }
+        if (stageDeltas.length > 0)
+            caseEntry.stageDeltas = stageDeltas;
+    }
+    return caseEntry;
+}
+export function diffReports(previous, current, prevPath, currPath) {
+    const prevMap = new Map();
+    const currMap = new Map();
+    for (const c of previous.cases)
+        prevMap.set(c.caseId, c);
+    for (const c of current.cases)
+        currMap.set(c.caseId, c);
+    const allIds = new Set([...prevMap.keys(), ...currMap.keys()]);
+    const cases = [...allIds]
+        .sort((a, b) => a.localeCompare(b))
+        .map((id) => diffCase(id, prevMap.get(id), currMap.get(id)));
+    const regressed = cases.some((c) => c.transition === "regressed" ||
+        c.transition === "removed" ||
+        c.verifierDeltas.some((v) => v.transition === "regressed" || v.transition === "score-drop"));
+    return {
+        old: meta(previous, prevPath),
+        new: meta(current, currPath),
+        summaryDelta: {
+            totalCasesDelta: current.summary.totalCases - previous.summary.totalCases,
+            passedDelta: current.summary.passed - previous.summary.passed,
+            failedDelta: current.summary.failed - previous.summary.failed,
+            skippedDelta: current.summary.skipped - previous.summary.skipped,
+            totalCostUsdDelta: Number((current.summary.totalCostUsd - previous.summary.totalCostUsd).toFixed(6)),
+            totalDurationMsDelta: current.summary.totalDurationMs - previous.summary.totalDurationMs
+        },
+        cases,
+        regressed
+    };
+}
+export async function runEvalDiff(input) {
+    const [oldPath, newPath] = await Promise.all([
+        resolveReportPath(input.projectRoot, input.old),
+        resolveReportPath(input.projectRoot, input.new)
+    ]);
+    const [oldReport, newReport] = await Promise.all([
+        loadReport(oldPath),
+        loadReport(newPath)
+    ]);
+    return diffReports(oldReport, newReport, oldPath, newPath);
+}
+/** Render the diff as a terse human-readable Markdown block. */
+export function formatDiffMarkdown(diff) {
+    const lines = [];
+    lines.push(`# cclaw eval diff`);
+    lines.push(``);
+    lines.push(`- old: ${diff.old.cclawVersion} (${path.basename(diff.old.sourcePath)})`);
+    lines.push(`- new: ${diff.new.cclawVersion} (${path.basename(diff.new.sourcePath)})`);
+    lines.push(`- regressed: ${diff.regressed ? "yes" : "no"}`);
+    lines.push(``);
+    lines.push(`## Summary delta`);
+    lines.push(``);
+    const sd = diff.summaryDelta;
+    lines.push(`| metric | delta |`);
+    lines.push(`| --- | --- |`);
+    lines.push(`| total cases | ${sd.totalCasesDelta >= 0 ? "+" : ""}${sd.totalCasesDelta} |`);
+    lines.push(`| passed | ${sd.passedDelta >= 0 ? "+" : ""}${sd.passedDelta} |`);
+    lines.push(`| failed | ${sd.failedDelta >= 0 ? "+" : ""}${sd.failedDelta} |`);
+    lines.push(`| skipped | ${sd.skippedDelta >= 0 ? "+" : ""}${sd.skippedDelta} |`);
+    lines.push(`| cost (USD) | ${sd.totalCostUsdDelta >= 0 ? "+" : ""}${sd.totalCostUsdDelta.toFixed(4)} |`);
+    lines.push(`| duration (ms) | ${sd.totalDurationMsDelta >= 0 ? "+" : ""}${sd.totalDurationMsDelta} |`);
+    lines.push(``);
+    const noisyCases = diff.cases.filter((c) => c.transition !== "same" || c.verifierDeltas.length > 0);
+    if (noisyCases.length === 0) {
+        lines.push(`No case-level changes.`);
+        lines.push(``);
+        return `${lines.join("\n")}\n`;
+    }
+    lines.push(`## Case changes`);
+    lines.push(``);
+    lines.push(`| case id | stage | transition | prev | curr |`);
+    lines.push(`| --- | --- | --- | --- | --- |`);
+    for (const c of noisyCases) {
+        const prev = c.previousPassed === undefined ? "-" : c.previousPassed ? "pass" : "fail";
+        const curr = c.currentPassed === undefined ? "-" : c.currentPassed ? "pass" : "fail";
+        lines.push(`| ${c.caseId} | ${c.stage} | ${c.transition} | ${prev} | ${curr} |`);
+    }
+    lines.push(``);
+    const withVerifiers = noisyCases.filter((c) => c.verifierDeltas.length > 0);
+    if (withVerifiers.length > 0) {
+        lines.push(`## Verifier changes`);
+        lines.push(``);
+        lines.push(`| case id | verifier | kind | transition | prev score | curr score |`);
+        lines.push(`| --- | --- | --- | --- | --- | --- |`);
+        for (const c of withVerifiers) {
+            for (const v of c.verifierDeltas) {
+                const prev = v.previousScore !== undefined ? v.previousScore.toFixed(2) : "-";
+                const curr = v.currentScore !== undefined ? v.currentScore.toFixed(2) : "-";
+                lines.push(`| ${c.caseId} | ${v.verifierId} | ${v.kind} | ${v.transition} | ${prev} | ${curr} |`);
+            }
+        }
+        lines.push(``);
+    }
+    return `${lines.join("\n")}\n`;
+}

package/dist/eval/llm-client.d.ts

@@ -7,7 +7,7 @@ export interface ChatMessage {
     toolCallId?: string;
     /**
      * OpenAI-style tool calls carried on a preceding assistant message.
-     * Populated by the Tier B loop so the wire transcript stays
+     * Populated by the with-tools loop so the wire transcript stays
      * consistent (assistant message → tool responses).
      */
     toolCalls?: Array<{
@@ -35,7 +35,7 @@ export interface ChatRequest {
     seed?: number;
     /**
      * Tool/function-calling definitions in OpenAI wire format. Populated only
-     * by Tier B. Ignored by the Tier A single-shot path.
+     * by agent/workflow modes. Ignored by the single-shot path.
      */
     tools?: unknown[];
     toolChoice?: "auto" | "none";
@@ -111,6 +111,17 @@ export interface CreateEvalClientOptions {
     retryPolicy?: RetryPolicy;
     /** Deterministic sleep used by the retry loop. Defaults to `setTimeout`. */
     sleep?: (ms: number) => Promise<void>;
+    /**
+     * Observer invoked when a chat() call is about to sleep before the next
+     * retry attempt. Use this to surface "we are retrying" status via the
+     * progress logger so long, silent backoff windows become visible.
+     */
+    onRetry?: (event: {
+        attempt: number;
+        maxAttempts: number;
+        waitMs: number;
+        error: EvalLlmError;
+    }) => void;
 }
 export interface RetryPolicy {
     /** Max retries *on top of* the initial attempt. 0 = single attempt. */

package/dist/eval/llm-client.js

@@ -251,7 +251,14 @@ export function createEvalClient(config, options = {}) {
                     const isLastAttempt = attempt === maxAttempts - 1;
                     if (!normalized.retryable || isLastAttempt)
                         throw normalized;
-                    await sleep(backoffDelay(attempt, retryPolicy));
+                    const waitMs = backoffDelay(attempt, retryPolicy);
+                    options.onRetry?.({
+                        attempt: attempt + 1,
+                        maxAttempts,
+                        waitMs,
+                        error: normalized
+                    });
+                    await sleep(waitMs);
                 }
             }
             throw lastError ?? new EvalLlmTransportError(new Error("unknown"));

package/dist/eval/mode.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Helpers that translate between the legacy `Tier A/B/C` naming and the
+ * current `EvalMode` identifiers (`fixture` / `agent` / `workflow`).
+ *
+ * The names we actually carry in reports, config, CLI flags, and verifier
+ * messages are the `EvalMode` ones; legacy tier inputs are accepted with a
+ * single deprecation warning per process so existing scripts keep working
+ * through the 0.28.x line.
+ */
+import { type EvalMode } from "./types.js";
+/**
+ * Reset the per-process "already warned about legacy tier" flag. Used by
+ * tests so each test file gets a deterministic warning surface.
+ */
+export declare function __resetLegacyWarningForTests(): void;
+export interface LegacyTierInput {
+    source: "cli" | "env" | "config";
+    raw: string;
+}
+/**
+ * Normalize a raw string from the CLI / env / config into an `EvalMode`.
+ * Accepts both new (`fixture|agent|workflow`) and legacy (`A|B|C`) names.
+ * Emits a deprecation warning to stderr at most once per process when a
+ * legacy tier name is seen.
+ */
+export declare function parseModeInput(raw: string, input: LegacyTierInput, writeWarning?: (message: string) => void): EvalMode;
+/** @deprecated kept for callers that still need to serialize as legacy. */
+export declare function modeToLegacyTier(mode: EvalMode): "A" | "B" | "C";

package/dist/eval/mode.js

@@ -0,0 +1,61 @@
+/**
+ * Helpers that translate between the legacy `Tier A/B/C` naming and the
+ * current `EvalMode` identifiers (`fixture` / `agent` / `workflow`).
+ *
+ * The names we actually carry in reports, config, CLI flags, and verifier
+ * messages are the `EvalMode` ones; legacy tier inputs are accepted with a
+ * single deprecation warning per process so existing scripts keep working
+ * through the 0.28.x line.
+ */
+import { EVAL_MODES } from "./types.js";
+const LEGACY_TIER_TO_MODE = {
+    A: "fixture",
+    B: "agent",
+    C: "workflow"
+};
+const MODE_TO_LEGACY_TIER = {
+    fixture: "A",
+    agent: "B",
+    workflow: "C"
+};
+const DEPRECATED_NAMES = new Set(Object.keys(LEGACY_TIER_TO_MODE));
+let legacyWarningEmitted = false;
+/**
+ * Reset the per-process "already warned about legacy tier" flag. Used by
+ * tests so each test file gets a deterministic warning surface.
+ */
+export function __resetLegacyWarningForTests() {
+    legacyWarningEmitted = false;
+}
+/**
+ * Normalize a raw string from the CLI / env / config into an `EvalMode`.
+ * Accepts both new (`fixture|agent|workflow`) and legacy (`A|B|C`) names.
+ * Emits a deprecation warning to stderr at most once per process when a
+ * legacy tier name is seen.
+ */
+export function parseModeInput(raw, input, writeWarning = defaultWriteWarning) {
+    const trimmed = raw.trim();
+    if (trimmed.length === 0) {
+        throw new Error(`Evaluation mode must be one of: ${EVAL_MODES.join("|")} (or legacy A|B|C).`);
+    }
+    if (EVAL_MODES.includes(trimmed)) {
+        return trimmed;
+    }
+    if (DEPRECATED_NAMES.has(trimmed)) {
+        const replacement = LEGACY_TIER_TO_MODE[trimmed];
+        if (!legacyWarningEmitted) {
+            legacyWarningEmitted = true;
+            writeWarning(`[cclaw] "${input.source}: ${input.raw}" is using the legacy tier name "${trimmed}". ` +
+                `Please switch to --mode=${replacement} (legacy --tier=A|B|C will be removed in the next major release).`);
+        }
+        return replacement;
+    }
+    throw new Error(`Evaluation mode must be one of: ${EVAL_MODES.join("|")} (or legacy A|B|C), got: ${raw}`);
+}
+/** @deprecated kept for callers that still need to serialize as legacy. */
+export function modeToLegacyTier(mode) {
+    return MODE_TO_LEGACY_TIER[mode];
+}
+function defaultWriteWarning(message) {
+    process.stderr.write(`${message}\n`);
+}

package/dist/eval/progress.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Lightweight progress logger for `cclaw eval`.
+ *
+ * The runner is otherwise silent: a full workflow-mode run can easily take
+ * a few minutes and the user would see nothing until the Markdown report
+ * hits disk. We emit structured events here so the CLI can print concise
+ * one-line status updates to stderr (stdout stays reserved for the final
+ * report + `--json` output).
+ *
+ * The logger is intentionally minimal: no ANSI colors, no spinners, no
+ * carriage-return rewrites. Those do not survive `tee`, CI log viewers,
+ * or the background `runs/tail` path (which copies the stream to a log
+ * file), and users also told us "nothing is clear now, everything is
+ * long" — so we optimize for log-friendly line-by-line readability.
+ */
+import type { EvalMode, WorkflowStageName } from "./types.js";
+export type ProgressEvent = {
+    kind: "run-start";
+    mode: EvalMode;
+    totalCases: number;
+} | {
+    kind: "case-start";
+    caseId: string;
+    stage: string;
+    index: number;
+    total: number;
+} | {
+    kind: "case-end";
+    caseId: string;
+    stage: string;
+    index: number;
+    total: number;
+    passed: boolean;
+    durationMs: number;
+    costUsd?: number;
+} | {
+    kind: "stage-start";
+    caseId: string;
+    stage: WorkflowStageName;
+    index: number;
+    total: number;
+} | {
+    kind: "stage-end";
+    caseId: string;
+    stage: WorkflowStageName;
+    index: number;
+    total: number;
+    passed: boolean;
+    durationMs: number;
+    costUsd?: number;
+} | {
+    kind: "retry";
+    caseId: string;
+    stage?: string;
+    attempt: number;
+    maxAttempts: number;
+    waitMs: number;
+    reason: string;
+} | {
+    kind: "run-end";
+    totalCases: number;
+    passed: number;
+    failed: number;
+    durationMs: number;
+};
+export interface ProgressLogger {
+    emit(event: ProgressEvent): void;
+}
+export declare function noopProgressLogger(): ProgressLogger;
+export interface StderrProgressLoggerOptions {
+    /** Override the underlying write target; defaults to `process.stderr.write`. */
+    writer?: (message: string) => void;
+    /** Return wall-clock in ms. Injectable for tests. */
+    now?: () => number;
+}
+/**
+ * Emit a one-line status update per event to stderr.
+ *
+ * Format is deliberately boring: `[cclaw eval] <message>` so users can grep
+ * for the prefix in combined logs. Costs are rendered with up to 4 decimals
+ * so sub-cent runs still show a non-zero value.
+ */
+export declare function createStderrProgressLogger(opts?: StderrProgressLoggerOptions): ProgressLogger;

package/dist/eval/progress.js

@@ -0,0 +1,59 @@
+const NOOP_LOGGER = { emit() { } };
+export function noopProgressLogger() {
+    return NOOP_LOGGER;
+}
+/**
+ * Emit a one-line status update per event to stderr.
+ *
+ * Format is deliberately boring: `[cclaw eval] <message>` so users can grep
+ * for the prefix in combined logs. Costs are rendered with up to 4 decimals
+ * so sub-cent runs still show a non-zero value.
+ */
+export function createStderrProgressLogger(opts = {}) {
+    const writer = opts.writer ?? ((s) => process.stderr.write(s));
+    return {
+        emit(event) {
+            writer(`[cclaw eval] ${formatEvent(event)}\n`);
+        }
+    };
+}
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    const s = ms / 1000;
+    if (s < 60)
+        return `${s.toFixed(1)}s`;
+    const m = Math.floor(s / 60);
+    const rem = Math.round(s - m * 60);
+    return `${m}m${rem.toString().padStart(2, "0")}s`;
+}
+function formatCost(usd) {
+    if (usd === undefined || usd <= 0)
+        return "";
+    return ` $${usd.toFixed(4)}`;
+}
+function formatEvent(event) {
+    switch (event.kind) {
+        case "run-start":
+            return `start mode=${event.mode} cases=${event.totalCases}`;
+        case "case-start":
+            return `[${event.index}/${event.total}] ${event.caseId} (${event.stage}) ...`;
+        case "case-end": {
+            const status = event.passed ? "PASS" : "FAIL";
+            return (`[${event.index}/${event.total}] ${event.caseId} (${event.stage}) ${status} ` +
+                `in ${formatDuration(event.durationMs)}${formatCost(event.costUsd)}`);
+        }
+        case "stage-start":
+            return `  stage ${event.stage} ...`;
+        case "stage-end": {
+            const status = event.passed ? "ok" : "fail";
+            return `  stage ${event.stage} ${status} in ${formatDuration(event.durationMs)}${formatCost(event.costUsd)}`;
+        }
+        case "retry":
+            return (`  retry ${event.caseId}${event.stage ? `/${event.stage}` : ""} ` +
+                `attempt ${event.attempt}/${event.maxAttempts} in ${formatDuration(event.waitMs)} (${event.reason})`);
+        case "run-end":
+            return (`done pass=${event.passed} fail=${event.failed} total=${event.totalCases} ` +
+                `in ${formatDuration(event.durationMs)}`);
+    }
+}