cclaw-cli 0.27.0 → 0.29.0

package/dist/eval/mode.js

@@ -0,0 +1,61 @@
+/**
+ * Helpers that translate between the legacy `Tier A/B/C` naming and the
+ * current `EvalMode` identifiers (`fixture` / `agent` / `workflow`).
+ *
+ * The names we actually carry in reports, config, CLI flags, and verifier
+ * messages are the `EvalMode` ones; legacy tier inputs are accepted with a
+ * single deprecation warning per process so existing scripts keep working
+ * through the 0.28.x line.
+ */
+import { EVAL_MODES } from "./types.js";
+const LEGACY_TIER_TO_MODE = {
+    A: "fixture",
+    B: "agent",
+    C: "workflow"
+};
+const MODE_TO_LEGACY_TIER = {
+    fixture: "A",
+    agent: "B",
+    workflow: "C"
+};
+const DEPRECATED_NAMES = new Set(Object.keys(LEGACY_TIER_TO_MODE));
+let legacyWarningEmitted = false;
+/**
+ * Reset the per-process "already warned about legacy tier" flag. Used by
+ * tests so each test file gets a deterministic warning surface.
+ */
+export function __resetLegacyWarningForTests() {
+    legacyWarningEmitted = false;
+}
+/**
+ * Normalize a raw string from the CLI / env / config into an `EvalMode`.
+ * Accepts both new (`fixture|agent|workflow`) and legacy (`A|B|C`) names.
+ * Emits a deprecation warning to stderr at most once per process when a
+ * legacy tier name is seen.
+ */
+export function parseModeInput(raw, input, writeWarning = defaultWriteWarning) {
+    const trimmed = raw.trim();
+    if (trimmed.length === 0) {
+        throw new Error(`Evaluation mode must be one of: ${EVAL_MODES.join("|")} (or legacy A|B|C).`);
+    }
+    if (EVAL_MODES.includes(trimmed)) {
+        return trimmed;
+    }
+    if (DEPRECATED_NAMES.has(trimmed)) {
+        const replacement = LEGACY_TIER_TO_MODE[trimmed];
+        if (!legacyWarningEmitted) {
+            legacyWarningEmitted = true;
+            writeWarning(`[cclaw] "${input.source}: ${input.raw}" is using the legacy tier name "${trimmed}". ` +
+                `Please switch to --mode=${replacement} (legacy --tier=A|B|C will be removed in the next major release).`);
+        }
+        return replacement;
+    }
+    throw new Error(`Evaluation mode must be one of: ${EVAL_MODES.join("|")} (or legacy A|B|C), got: ${raw}`);
+}
+/** @deprecated kept for callers that still need to serialize as legacy. */
+export function modeToLegacyTier(mode) {
+    return MODE_TO_LEGACY_TIER[mode];
+}
+function defaultWriteWarning(message) {
+    process.stderr.write(`${message}\n`);
+}

package/dist/eval/progress.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Lightweight progress logger for `cclaw eval`.
+ *
+ * The runner is otherwise silent: a full workflow-mode run can easily take
+ * a few minutes and the user would see nothing until the Markdown report
+ * hits disk. We emit structured events here so the CLI can print concise
+ * one-line status updates to stderr (stdout stays reserved for the final
+ * report + `--json` output).
+ *
+ * The logger is intentionally minimal: no ANSI colors, no spinners, no
+ * carriage-return rewrites. Those do not survive `tee`, CI log viewers,
+ * or the background `runs/tail` path (which copies the stream to a log
+ * file), and users also told us "nothing is clear now, everything is
+ * long" — so we optimize for log-friendly line-by-line readability.
+ */
+import type { EvalMode, WorkflowStageName } from "./types.js";
+export type ProgressEvent = {
+    kind: "run-start";
+    mode: EvalMode;
+    totalCases: number;
+} | {
+    kind: "case-start";
+    caseId: string;
+    stage: string;
+    index: number;
+    total: number;
+} | {
+    kind: "case-end";
+    caseId: string;
+    stage: string;
+    index: number;
+    total: number;
+    passed: boolean;
+    durationMs: number;
+    costUsd?: number;
+} | {
+    kind: "stage-start";
+    caseId: string;
+    stage: WorkflowStageName;
+    index: number;
+    total: number;
+} | {
+    kind: "stage-end";
+    caseId: string;
+    stage: WorkflowStageName;
+    index: number;
+    total: number;
+    passed: boolean;
+    durationMs: number;
+    costUsd?: number;
+} | {
+    kind: "retry";
+    caseId: string;
+    stage?: string;
+    attempt: number;
+    maxAttempts: number;
+    waitMs: number;
+    reason: string;
+} | {
+    kind: "run-end";
+    totalCases: number;
+    passed: number;
+    failed: number;
+    durationMs: number;
+};
+export interface ProgressLogger {
+    emit(event: ProgressEvent): void;
+}
+export declare function noopProgressLogger(): ProgressLogger;
+export interface StderrProgressLoggerOptions {
+    /** Override the underlying write target; defaults to `process.stderr.write`. */
+    writer?: (message: string) => void;
+    /** Return wall-clock in ms. Injectable for tests. */
+    now?: () => number;
+}
+/**
+ * Emit a one-line status update per event to stderr.
+ *
+ * Format is deliberately boring: `[cclaw eval] <message>` so users can grep
+ * for the prefix in combined logs. Costs are rendered with up to 4 decimals
+ * so sub-cent runs still show a non-zero value.
+ */
+export declare function createStderrProgressLogger(opts?: StderrProgressLoggerOptions): ProgressLogger;

package/dist/eval/progress.js

@@ -0,0 +1,59 @@
+const NOOP_LOGGER = { emit() { } };
+export function noopProgressLogger() {
+    return NOOP_LOGGER;
+}
+/**
+ * Emit a one-line status update per event to stderr.
+ *
+ * Format is deliberately boring: `[cclaw eval] <message>` so users can grep
+ * for the prefix in combined logs. Costs are rendered with up to 4 decimals
+ * so sub-cent runs still show a non-zero value.
+ */
+export function createStderrProgressLogger(opts = {}) {
+    const writer = opts.writer ?? ((s) => process.stderr.write(s));
+    return {
+        emit(event) {
+            writer(`[cclaw eval] ${formatEvent(event)}\n`);
+        }
+    };
+}
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    const s = ms / 1000;
+    if (s < 60)
+        return `${s.toFixed(1)}s`;
+    const m = Math.floor(s / 60);
+    const rem = Math.round(s - m * 60);
+    return `${m}m${rem.toString().padStart(2, "0")}s`;
+}
+function formatCost(usd) {
+    if (usd === undefined || usd <= 0)
+        return "";
+    return ` $${usd.toFixed(4)}`;
+}
+function formatEvent(event) {
+    switch (event.kind) {
+        case "run-start":
+            return `start mode=${event.mode} cases=${event.totalCases}`;
+        case "case-start":
+            return `[${event.index}/${event.total}] ${event.caseId} (${event.stage}) ...`;
+        case "case-end": {
+            const status = event.passed ? "PASS" : "FAIL";
+            return (`[${event.index}/${event.total}] ${event.caseId} (${event.stage}) ${status} ` +
+                `in ${formatDuration(event.durationMs)}${formatCost(event.costUsd)}`);
+        }
+        case "stage-start":
+            return `  stage ${event.stage} ...`;
+        case "stage-end": {
+            const status = event.passed ? "ok" : "fail";
+            return `  stage ${event.stage} ${status} in ${formatDuration(event.durationMs)}${formatCost(event.costUsd)}`;
+        }
+        case "retry":
+            return (`  retry ${event.caseId}${event.stage ? `/${event.stage}` : ""} ` +
+                `attempt ${event.attempt}/${event.maxAttempts} in ${formatDuration(event.waitMs)} (${event.reason})`);
+        case "run-end":
+            return (`done pass=${event.passed} fail=${event.failed} total=${event.totalCases} ` +
+                `in ${formatDuration(event.durationMs)}`);
+    }
+}

package/dist/eval/report.js

@@ -24,7 +24,7 @@ export function formatMarkdownReport(report) {
     lines.push(`- cclaw version: ${report.cclawVersion}`);
     lines.push(`- provider: ${report.provider}`);
     lines.push(`- model: ${report.model}`);
-    lines.push(`- tier: ${report.tier}`);
+    lines.push(`- mode: ${report.mode}`);
     lines.push(`- stages: ${stages}`);
     lines.push(``);
     lines.push(`## Summary`);

package/dist/eval/runner.d.ts

@@ -1,10 +1,11 @@
 import type { FlowStage } from "../types.js";
 import { type EvalLlmClient } from "./llm-client.js";
-import type { EvalReport, EvalTier, ResolvedEvalConfig, WorkflowStageName } from "./types.js";
+import { type ProgressLogger } from "./progress.js";
+import type { EvalMode, EvalReport, ResolvedEvalConfig, WorkflowStageName } from "./types.js";
 export interface RunEvalOptions {
     projectRoot: string;
     stage?: FlowStage;
-    tier?: EvalTier;
+    mode?: EvalMode;
     /** When true, run only structural verifiers (Step 1). */
     schemaOnly?: boolean;
     /** When true, run structural + rule-based verifiers. Step 2 wires rules. */
@@ -21,6 +22,25 @@ export interface RunEvalOptions {
      * without hitting the network.
      */
     llmClient?: EvalLlmClient;
+    /**
+     * Optional progress logger. The CLI wires a stderr-backed logger by
+     * default so users see one-line updates during long runs; tests and
+     * programmatic callers can inject a silent (noop) logger or capture
+     * events for assertions. When omitted, progress is silenced.
+     */
+    progress?: ProgressLogger;
+    /**
+     * Per-run USD cap. Enforced in-memory; independent from the daily cap
+     * (`dailyUsdCap` / `CCLAW_EVAL_DAILY_USD_CAP`) that persists across
+     * invocations. Undefined means no cap.
+     */
+    maxCostUsd?: number;
+    /**
+     * Override the configured `model` (and `judgeModel`) for this run.
+     * Used by `cclaw eval --compare-model` to replay the same corpus
+     * against an alternative model without editing `config.yaml`.
+     */
+    modelOverride?: string;
 }
 export interface DryRunSummary {
     kind: "dry-run";
@@ -33,7 +53,7 @@ export interface DryRunSummary {
             stage: FlowStage;
         }>;
     };
-    /** Tier C-only workflow corpus summary. Empty for Tier A/B planned runs. */
+    /** Only populated in `workflow` mode; empty for fixture / agent modes. */
     workflowCorpus: {
         total: number;
         cases: Array<{
@@ -41,7 +61,7 @@ export interface DryRunSummary {
             stages: WorkflowStageName[];
         }>;
     };
-    plannedTier: EvalTier;
+    plannedMode: EvalMode;
     verifiersAvailable: {
         structural: boolean;
         rules: boolean;
@@ -52,10 +72,10 @@ export interface DryRunSummary {
     notes: string[];
 }
 /**
- * Structural runner. When `schemaOnly` is set (or no other verifier flags are
- * active), runs structural verifiers against fixture-backed cases and loads
- * per-stage baselines for regression comparison. Tier A/B/C agent loops
- * arrive in later steps; until then cases without `fixture` are marked as
- * skipped rather than failing.
+ * Main eval runner. Dispatches between fixture-backed verification, the
+ * single-stage agent-with-tools loop, and the multi-stage workflow
+ * orchestrator based on `options.mode`. Per-stage baselines are loaded for
+ * regression comparison. Cases without a `fixture` path in the yaml are
+ * marked skipped (not failed) when no LLM drafting runs.
  */
 export declare function runEval(options: RunEvalOptions): Promise<DryRunSummary | EvalReport>;