cclaw-cli 0.27.0 → 0.29.0

package/dist/eval/agents/single-shot.js

@@ -1,5 +1,5 @@
 /**
- * Tier A single-shot agent.
+ * Single-shot agent used by fixture mode when `--judge` is set.
  *
  * Simplest realistic AUT: one LLM call with the stage's SKILL.md as the
  * system prompt and the case's `inputPrompt` as the user message. Output
@@ -9,7 +9,7 @@
  * Design notes:
  *
  * - No tools. No multi-turn. No reads of the project beyond the one
- *   SKILL.md. Tier B/C layer complexity on top in later steps.
+ *   SKILL.md. agent/workflow modes layer complexity on top.
  * - Errors are propagated as-is (`EvalLlmError` subclasses) so the
  *   runner can surface them as verifier failures without swallowing the
  *   cause.
@@ -27,7 +27,7 @@ export async function loadStageSkill(projectRoot, stage) {
     const file = path.join(projectRoot, RUNTIME_ROOT, "skills", folder, "SKILL.md");
     if (!(await exists(file))) {
         throw new Error(`Stage skill not found: ${path.relative(projectRoot, file)}. ` +
-            `Run \`cclaw init\` (or \`cclaw sync\`) before \`cclaw eval --tier=A --judge\`.`);
+            `Run \`cclaw init\` (or \`cclaw sync\`) before \`cclaw eval --mode=fixture --judge\`.`);
     }
     return fs.readFile(file, "utf8");
 }
@@ -50,7 +50,7 @@ function buildUserPrompt(caseEntry) {
         `Do not wrap in code fences, do not add commentary before or after.`);
     return lines.join("\n");
 }
-/** Run the Tier A single-shot AUT and return the produced artifact. */
+/** Run the single-shot AUT (fixture mode + --judge) and return the produced artifact. */
 export async function runSingleShot(input) {
     const { caseEntry, config, projectRoot, client } = input;
     const started = Date.now();

package/dist/eval/agents/with-tools.d.ts

@@ -18,15 +18,15 @@ export interface WithToolsInput {
     createSandboxFn?: typeof createSandbox;
     /**
      * Reuse an externally-managed sandbox instead of creating + disposing a
-     * per-call one. Tier C workflow orchestration uses this so every stage
-     * shares the same sandbox and earlier artifacts remain visible. When
-     * set, the caller is responsible for `dispose()`.
+     * per-call one. Workflow mode uses this so every stage shares the same
+     * sandbox and earlier artifacts remain visible. When set, the caller is
+     * responsible for `dispose()`.
      */
     externalSandbox?: Sandbox;
     /**
-     * Optional override of the default user prompt prefix. Tier C uses this
-     * to tell the model which stage it is on and where the prior artifacts
-     * are located.
+     * Optional override of the default user prompt prefix. Workflow mode uses
+     * this to tell the model which stage it is on and where the prior
+     * artifacts are located.
      */
     promptPreamble?: string;
 }

package/dist/eval/agents/with-tools.js

@@ -1,11 +1,11 @@
 /**
- * Tier B with-tools agent.
+ * Multi-turn with-tools agent (agent mode, reused by workflow mode).
  *
  * Multi-turn loop with OpenAI-style function-calling over a set of
  * sandbox-confined tools. The AUT is given:
  *
- *  - System prompt = stage SKILL.md (same contract as Tier A so the
- *    single-shot baseline is comparable).
+ *  - System prompt = stage SKILL.md (same contract as the single-shot path
+ *    so the baseline is comparable).
  *  - User prompt = task description + a short "tools available" hint
  *    that names the sandbox root and the four built-in tools.
  *  - Tools = `read_file`, `write_file`, `glob`, `grep` (see
@@ -29,7 +29,7 @@
  * Artifact resolution: the final assistant content is the artifact. If
  * the model used `write_file` to stage the artifact at
  * `artifact.md` (or `artifact/<stage>.md`), we prefer that file — it
- * mirrors the Tier C workflow where writes are the deliverable. The
+ * mirrors workflow mode where writes are the deliverable. The
  * fallback is the terminal assistant message so prompts that don't
  * call write_file still produce something judgable.
  */
@@ -42,7 +42,7 @@ import { loadStageSkill } from "./single-shot.js";
 export class MaxTurnsExceededError extends Error {
     turns;
     constructor(turns) {
-        super(`Tier B agent exceeded the ${turns}-turn budget without a terminal stop.`);
+        super(`Agent loop exceeded the ${turns}-turn budget without a terminal stop.`);
         this.name = "MaxTurnsExceededError";
         this.turns = turns;
     }

package/dist/eval/agents/workflow.d.ts

@@ -12,6 +12,13 @@ export interface WorkflowInput {
     loadSkill?: (stage: WorkflowStageName) => Promise<string>;
     /** Override for the sandbox factory (test hook). */
     createSandboxFn?: typeof createSandbox;
+    /**
+     * Optional per-stage lifecycle hooks. The runner uses these to emit
+     * progress events to stderr so workflow-mode runs surface real-time
+     * status rather than going silent for minutes.
+     */
+    onStageStart?: (stage: WorkflowStageName) => void;
+    onStageEnd?: (stage: WorkflowStageName, result: WorkflowStageResult) => void;
 }
 export interface WorkflowOutput {
     caseId: string;

package/dist/eval/agents/workflow.js

@@ -1,7 +1,7 @@
 /**
- * Tier C workflow agent.
+ * Workflow-mode agent.
  *
- * Runs the Tier B with-tools loop once per stage in a workflow case,
+ * Runs the with-tools loop once per stage in a workflow case,
  * sharing a single sandbox across stages so every new stage can read
  * the earlier artifacts the model produced. The shape of the run is:
  *
@@ -46,6 +46,7 @@ export async function runWorkflow(input) {
     try {
         await fs.mkdir(await sandbox.resolve(STAGES_SUBDIR, { allowMissing: true }), { recursive: true });
         for (const step of workflow.stages) {
+            input.onStageStart?.(step.name);
             await clearArtifactFile(sandbox);
             const priorStages = stageResults.map((r) => r.stage);
             const preamble = buildStagePreamble(workflow, step.name, priorStages);
@@ -83,6 +84,7 @@ export async function runWorkflow(input) {
                 completionTokens: result.usage.completionTokens
             };
             stageResults.push(stageResult);
+            input.onStageEnd?.(step.name, stageResult);
             totalUsageUsd += result.usageUsd;
             totalDurationMs += result.durationMs;
         }
@@ -118,7 +120,7 @@ async function persistStageArtifact(sandbox, stage, artifact) {
 }
 function buildStagePreamble(workflow, current, priorStages) {
     const lines = [];
-    lines.push(`You are running stage "${current}" of the Tier C workflow "${workflow.id}".`);
+    lines.push(`You are running stage "${current}" of the workflow "${workflow.id}".`);
     if (workflow.description) {
         lines.push(`Case description: ${workflow.description}`);
     }

package/dist/eval/baseline.d.ts

@@ -1,6 +1,30 @@
 import type { FlowStage } from "../types.js";
 import type { BaselineDelta, BaselineSnapshot, EvalReport } from "./types.js";
 export declare const BASELINE_SCHEMA_VERSION = 1;
+/**
+ * Thrown when a signed baseline's on-disk digest does not match the
+ * canonical encoding of its `{ schemaVersion, stage, cases }` block.
+ * Callers should treat this as a hard failure: the baseline was either
+ * hand-edited or corrupted and cannot be trusted for regression gating.
+ */
+export declare class BaselineSignatureError extends Error {
+    readonly file: string;
+    readonly expected: string;
+    readonly actual: string;
+    constructor(opts: {
+        file: string;
+        expected: string;
+        actual: string;
+    });
+}
+/**
+ * Produce a deterministic sha256 digest over the signable portion of a
+ * baseline. We intentionally exclude `generatedAt` and `cclawVersion`
+ * from the digest so that rebuilding the same baseline from identical
+ * case results on a new CLI version doesn't invalidate the signature —
+ * only changes to the observed pass/ok/score payloads do.
+ */
+export declare function computeBaselineDigest(snapshot: Pick<BaselineSnapshot, "schemaVersion" | "stage" | "cases">): string;
 export declare function loadBaseline(projectRoot: string, stage: FlowStage): Promise<BaselineSnapshot | null>;
 export declare function loadBaselinesByStage(projectRoot: string, stages: readonly FlowStage[]): Promise<Map<FlowStage, BaselineSnapshot>>;
 export declare function buildBaselineForStage(stage: FlowStage, report: EvalReport): BaselineSnapshot;

package/dist/eval/baseline.js

@@ -14,15 +14,67 @@
  * Writes are gated behind an explicit `--update-baseline --confirm` pair at
  * the CLI layer so accidental resets do not slip into PRs.
  */
+import { createHash } from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { EVALS_ROOT, CCLAW_VERSION } from "../constants.js";
 import { exists } from "../fs-utils.js";
 import { FLOW_STAGES } from "../types.js";
 export const BASELINE_SCHEMA_VERSION = 1;
+/**
+ * Thrown when a signed baseline's on-disk digest does not match the
+ * canonical encoding of its `{ schemaVersion, stage, cases }` block.
+ * Callers should treat this as a hard failure: the baseline was either
+ * hand-edited or corrupted and cannot be trusted for regression gating.
+ */
+export class BaselineSignatureError extends Error {
+    file;
+    expected;
+    actual;
+    constructor(opts) {
+        super(`Baseline signature mismatch at ${opts.file}: expected ${opts.expected}, got ${opts.actual}. ` +
+            `The file was modified outside of \`cclaw eval --update-baseline\`. ` +
+            `Re-run with --update-baseline --confirm to re-sign a known-good snapshot.`);
+        this.name = "BaselineSignatureError";
+        this.file = opts.file;
+        this.expected = opts.expected;
+        this.actual = opts.actual;
+    }
+}
 function baselinePath(projectRoot, stage) {
     return path.join(projectRoot, EVALS_ROOT, "baselines", `${stage}.json`);
 }
+/**
+ * Produce a deterministic sha256 digest over the signable portion of a
+ * baseline. We intentionally exclude `generatedAt` and `cclawVersion`
+ * from the digest so that rebuilding the same baseline from identical
+ * case results on a new CLI version doesn't invalidate the signature —
+ * only changes to the observed pass/ok/score payloads do.
+ */
+export function computeBaselineDigest(snapshot) {
+    const canonical = canonicalJson({
+        schemaVersion: snapshot.schemaVersion,
+        stage: snapshot.stage,
+        cases: snapshot.cases
+    });
+    return createHash("sha256").update(canonical).digest("hex");
+}
+/**
+ * JSON.stringify with object keys sorted recursively so the digest is
+ * stable across filesystem / serializer variations.
+ */
+function canonicalJson(value) {
+    if (value === null || typeof value !== "object") {
+        return JSON.stringify(value);
+    }
+    if (Array.isArray(value)) {
+        return `[${value.map((v) => canonicalJson(v)).join(",")}]`;
+    }
+    const record = value;
+    const keys = Object.keys(record).sort();
+    const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(record[k])}`);
+    return `{${parts.join(",")}}`;
+}
 export async function loadBaseline(projectRoot, stage) {
     const filePath = baselinePath(projectRoot, stage);
     if (!(await exists(filePath)))
@@ -38,6 +90,20 @@ export async function loadBaseline(projectRoot, stage) {
     if (!isBaseline(parsed, stage)) {
         throw new Error(`Invalid baseline at ${filePath}: shape mismatch (expected schemaVersion=${BASELINE_SCHEMA_VERSION}, stage=${stage})`);
     }
+    const signature = parsed.signature;
+    if (signature) {
+        if (signature.algorithm !== "sha256") {
+            throw new Error(`Invalid baseline at ${filePath}: unsupported signature algorithm "${signature.algorithm}".`);
+        }
+        const actual = computeBaselineDigest(parsed);
+        if (actual !== signature.digest) {
+            throw new BaselineSignatureError({
+                file: filePath,
+                expected: signature.digest,
+                actual
+            });
+        }
+    }
     return parsed;
 }
 function isBaseline(value, stage) {
@@ -80,13 +146,20 @@ export function buildBaselineForStage(stage, report) {
     for (const c of stageCases) {
         cases[c.caseId] = entryFromResult(c);
     }
-    return {
+    const now = new Date().toISOString();
+    const unsigned = {
         schemaVersion: BASELINE_SCHEMA_VERSION,
         stage,
-        generatedAt: new Date().toISOString(),
+        generatedAt: now,
         cclawVersion: CCLAW_VERSION,
         cases
     };
+    unsigned.signature = {
+        algorithm: "sha256",
+        digest: computeBaselineDigest(unsigned),
+        signedAt: now
+    };
+    return unsigned;
 }
 export async function writeBaselinesFromReport(projectRoot, report) {
     const written = [];

package/dist/eval/config-loader.js

@@ -3,7 +3,8 @@ import path from "node:path";
 import { parse } from "yaml";
 import { EVALS_CONFIG_PATH } from "../constants.js";
 import { exists } from "../fs-utils.js";
-import { EVAL_TIERS } from "./types.js";
+import { EVAL_MODES } from "./types.js";
+import { parseModeInput } from "./mode.js";
 /**
  * Default eval config. Optimized for the z.ai OpenAI-compatible coding endpoint
  * with GLM 5.1 per the roadmap locked decisions (D-EVAL-01..05). Any field can
@@ -14,7 +15,7 @@ export const DEFAULT_EVAL_CONFIG = {
     provider: "zai",
     baseUrl: "https://api.z.ai/api/coding/paas/v4",
     model: "glm-5.1",
-    defaultTier: "A",
+    defaultMode: "fixture",
     regression: {
         failIfDeltaBelow: -0.15,
         failIfCriticalBelow: 3.0
@@ -25,7 +26,6 @@ export const DEFAULT_EVAL_CONFIG = {
     judgeTemperature: 0,
     agentTemperature: 0.2
 };
-const EVAL_TIER_SET = new Set(EVAL_TIERS);
 const NUMERIC_ENVS = new Set([
     "CCLAW_EVAL_DAILY_USD_CAP",
     "CCLAW_EVAL_TIMEOUT_MS",
@@ -40,7 +40,7 @@ const NUMERIC_ENVS = new Set([
 ]);
 function evalConfigError(configFilePath, reason) {
     return new Error(`Invalid cclaw eval config at ${configFilePath}: ${reason}\n` +
-        `Supported tiers: ${EVAL_TIERS.join(", ")}\n` +
+        `Supported modes: ${EVAL_MODES.join(", ")} (legacy tier values A|B|C also accepted).\n` +
         `See docs/evals.md for the full schema. After fixing, run: cclaw eval --dry-run`);
 }
 function isRecord(value) {
@@ -53,12 +53,11 @@ function parseNumericEnv(name, raw) {
     }
     return value;
 }
-function parseTierEnv(raw) {
-    const trimmed = raw.trim().toUpperCase();
-    if (!EVAL_TIER_SET.has(trimmed)) {
-        throw new Error(`Environment variable CCLAW_EVAL_TIER must be one of ${EVAL_TIERS.join("/")}, got: ${raw}`);
-    }
-    return trimmed;
+function parseModeEnv(raw, envName) {
+    return parseModeInput(envName === "CCLAW_EVAL_TIER" ? raw.toUpperCase() : raw, {
+        source: "env",
+        raw: `${envName}=${raw}`
+    });
 }
 function validateFileConfig(raw, configFilePath) {
     if (raw === undefined || raw === null)
@@ -79,11 +78,33 @@ function validateFileConfig(raw, configFilePath) {
     assignString("baseUrl", raw.baseUrl);
     assignString("model", raw.model);
     assignString("judgeModel", raw.judgeModel);
-    if (raw.defaultTier !== undefined) {
-        if (typeof raw.defaultTier !== "string" || !EVAL_TIER_SET.has(raw.defaultTier)) {
-            throw evalConfigError(configFilePath, `"defaultTier" must be one of: ${EVAL_TIERS.join(", ")}`);
+    if (raw.defaultMode !== undefined) {
+        if (typeof raw.defaultMode !== "string") {
+            throw evalConfigError(configFilePath, `"defaultMode" must be one of: ${EVAL_MODES.join(", ")}`);
+        }
+        try {
+            out.defaultMode = parseModeInput(raw.defaultMode, {
+                source: "config",
+                raw: `defaultMode: ${raw.defaultMode}`
+            });
+        }
+        catch (err) {
+            throw evalConfigError(configFilePath, err instanceof Error ? err.message : String(err));
+        }
+    }
+    else if (raw.defaultTier !== undefined) {
+        if (typeof raw.defaultTier !== "string") {
+            throw evalConfigError(configFilePath, `"defaultTier" must be a string (legacy; prefer "defaultMode")`);
+        }
+        try {
+            out.defaultMode = parseModeInput(raw.defaultTier, {
+                source: "config",
+                raw: `defaultTier: ${raw.defaultTier}`
+            });
+        }
+        catch (err) {
+            throw evalConfigError(configFilePath, err instanceof Error ? err.message : String(err));
         }
-        out.defaultTier = raw.defaultTier;
     }
     if (raw.dailyUsdCap !== undefined) {
         if (typeof raw.dailyUsdCap !== "number" || raw.dailyUsdCap < 0) {
@@ -194,6 +215,7 @@ function validateFileConfig(raw, configFilePath) {
         "baseUrl",
         "model",
         "judgeModel",
+        "defaultMode",
         "defaultTier",
         "dailyUsdCap",
         "timeoutMs",
@@ -266,11 +288,18 @@ function applyEnvOverrides(base, env) {
         patched.provider = provider;
         overridden = true;
     }
-    const tier = read("CCLAW_EVAL_TIER");
-    if (tier) {
-        patched.defaultTier = parseTierEnv(tier);
+    const modeEnv = read("CCLAW_EVAL_MODE");
+    if (modeEnv) {
+        patched.defaultMode = parseModeEnv(modeEnv, "CCLAW_EVAL_MODE");
         overridden = true;
     }
+    else {
+        const legacyTier = read("CCLAW_EVAL_TIER");
+        if (legacyTier) {
+            patched.defaultMode = parseModeEnv(legacyTier, "CCLAW_EVAL_TIER");
+            overridden = true;
+        }
+    }
     const cap = read("CCLAW_EVAL_DAILY_USD_CAP");
     if (cap) {
         patched.dailyUsdCap = parseNumericEnv("CCLAW_EVAL_DAILY_USD_CAP", cap);

package/dist/eval/cost-guard.d.ts

@@ -35,6 +35,22 @@ export declare class DailyCostCapExceededError extends Error {
         currentUsd: number;
     });
 }
+/**
+ * Per-run cost cap — enforced in-memory, no ledger file. Complements the
+ * daily cap so a single long workflow run can't blow the whole day's
+ * budget even if the daily cap is generous. Opt-in via
+ * `--max-cost-usd=<n>` on the CLI or `CCLAW_EVAL_MAX_COST_USD`.
+ */
+export declare class RunCostCapExceededError extends Error {
+    readonly capUsd: number;
+    readonly projectedUsd: number;
+    readonly currentUsd: number;
+    constructor(opts: {
+        capUsd: number;
+        projectedUsd: number;
+        currentUsd: number;
+    });
+}
 declare function utcDate(now?: Date): string;
 declare function pricingFor(model: string, config: Pick<ResolvedEvalConfig, "tokenPricing">): TokenPricing;
 /**
@@ -67,6 +83,12 @@ export interface CreateCostGuardOptions {
     now?: () => Date;
     /** Override the default filesystem root for the ledger. */
     ledgerPath?: string;
+    /**
+     * Per-run (in-memory) USD cap. Independent from the persisted daily
+     * cap so a single `cclaw eval` invocation can be budgeted without
+     * touching the shared nightly ledger. Undefined = unlimited.
+     */
+    runCapUsd?: number;
 }
 export declare function createCostGuard(projectRoot: string, config: Pick<ResolvedEvalConfig, "dailyUsdCap" | "tokenPricing">, options?: CreateCostGuardOptions): CostGuard;
 /** Exposed for tests. */

package/dist/eval/cost-guard.js

@@ -52,6 +52,28 @@ export class DailyCostCapExceededError extends Error {
         this.currentUsd = opts.currentUsd;
     }
 }
+/**
+ * Per-run cost cap — enforced in-memory, no ledger file. Complements the
+ * daily cap so a single long workflow run can't blow the whole day's
+ * budget even if the daily cap is generous. Opt-in via
+ * `--max-cost-usd=<n>` on the CLI or `CCLAW_EVAL_MAX_COST_USD`.
+ */
+export class RunCostCapExceededError extends Error {
+    capUsd;
+    projectedUsd;
+    currentUsd;
+    constructor(opts) {
+        super(`Run cost cap would be exceeded: ` +
+            `current=$${opts.currentUsd.toFixed(4)}, ` +
+            `projected=$${opts.projectedUsd.toFixed(4)}, ` +
+            `cap=$${opts.capUsd.toFixed(4)}. ` +
+            `Raise --max-cost-usd or drop it to run uncapped.`);
+        this.name = "RunCostCapExceededError";
+        this.capUsd = opts.capUsd;
+        this.projectedUsd = opts.projectedUsd;
+        this.currentUsd = opts.currentUsd;
+    }
+}
 function utcDate(now = new Date()) {
     return now.toISOString().slice(0, 10);
 }
@@ -109,11 +131,25 @@ export function createCostGuard(projectRoot, config, options = {}) {
     const now = options.now ?? (() => new Date());
     const currentDate = () => utcDate(now());
     const file = () => options.ledgerPath ?? ledgerPath(projectRoot, currentDate());
+    const runCap = options.runCapUsd;
+    let runTotalUsd = 0;
     return {
         async commit(model, usage) {
             const usd = computeUsageUsd(model, usage, config);
-            if (config.dailyUsdCap === undefined)
+            if (runCap !== undefined) {
+                const projected = Number((runTotalUsd + usd).toFixed(6));
+                if (projected > runCap) {
+                    throw new RunCostCapExceededError({
+                        capUsd: runCap,
+                        projectedUsd: projected,
+                        currentUsd: runTotalUsd
+                    });
+                }
+            }
+            if (config.dailyUsdCap === undefined) {
+                runTotalUsd = Number((runTotalUsd + usd).toFixed(6));
                 return usd;
+            }
             const date = currentDate();
             const target = file();
             const ledger = await readLedger(target, date);
@@ -133,6 +169,7 @@ export function createCostGuard(projectRoot, config, options = {}) {
             byModel.usd = Number((byModel.usd + usd).toFixed(6));
             ledger.byModel[model] = byModel;
             await writeLedger(target, ledger);
+            runTotalUsd = Number((runTotalUsd + usd).toFixed(6));
             return usd;
         },
         async snapshot() {

package/dist/eval/diff.d.ts

@@ -53,7 +53,7 @@ export interface EvalDiffReportMeta {
     runId: string;
     cclawVersion: string;
     generatedAt: string;
-    tier: string;
+    mode: string;
     model: string;
     sourcePath: string;
 }

package/dist/eval/diff.js

@@ -8,8 +8,8 @@
  *   - per-case pass/fail transitions
  *   - per-verifier score drops (only the drops — new passes are noted in
  *     the summary line, not repeated per verifier)
- *   - Tier C stage-level cost & duration deltas when both reports carry a
- *     `workflow` summary for the same case id
+ *   - Workflow-mode stage-level cost & duration deltas when both reports
+ *     carry a `workflow` summary for the same case id
  *
  * The resolver accepts three shapes for the `<old>` / `<new>` arguments:
  *
@@ -98,7 +98,7 @@ function meta(report, sourcePath) {
         runId: report.runId,
         cclawVersion: report.cclawVersion,
         generatedAt: report.generatedAt,
-        tier: report.tier,
+        mode: report.mode,
         model: report.model,
         sourcePath
     };

package/dist/eval/llm-client.d.ts

@@ -7,7 +7,7 @@ export interface ChatMessage {
     toolCallId?: string;
     /**
      * OpenAI-style tool calls carried on a preceding assistant message.
-     * Populated by the Tier B loop so the wire transcript stays
+     * Populated by the with-tools loop so the wire transcript stays
      * consistent (assistant message → tool responses).
      */
     toolCalls?: Array<{
@@ -35,7 +35,7 @@ export interface ChatRequest {
     seed?: number;
     /**
      * Tool/function-calling definitions in OpenAI wire format. Populated only
-     * by Tier B. Ignored by the Tier A single-shot path.
+     * by agent/workflow modes. Ignored by the single-shot path.
      */
     tools?: unknown[];
     toolChoice?: "auto" | "none";
@@ -111,6 +111,17 @@ export interface CreateEvalClientOptions {
     retryPolicy?: RetryPolicy;
     /** Deterministic sleep used by the retry loop. Defaults to `setTimeout`. */
     sleep?: (ms: number) => Promise<void>;
+    /**
+     * Observer invoked when a chat() call is about to sleep before the next
+     * retry attempt. Use this to surface "we are retrying" status via the
+     * progress logger so long, silent backoff windows become visible.
+     */
+    onRetry?: (event: {
+        attempt: number;
+        maxAttempts: number;
+        waitMs: number;
+        error: EvalLlmError;
+    }) => void;
 }
 export interface RetryPolicy {
     /** Max retries *on top of* the initial attempt. 0 = single attempt. */

package/dist/eval/llm-client.js

@@ -251,7 +251,14 @@ export function createEvalClient(config, options = {}) {
                     const isLastAttempt = attempt === maxAttempts - 1;
                     if (!normalized.retryable || isLastAttempt)
                         throw normalized;
-                    await sleep(backoffDelay(attempt, retryPolicy));
+                    const waitMs = backoffDelay(attempt, retryPolicy);
+                    options.onRetry?.({
+                        attempt: attempt + 1,
+                        maxAttempts,
+                        waitMs,
+                        error: normalized
+                    });
+                    await sleep(waitMs);
                 }
             }
             throw lastError ?? new EvalLlmTransportError(new Error("unknown"));

package/dist/eval/mode.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Helpers that translate between the legacy `Tier A/B/C` naming and the
+ * current `EvalMode` identifiers (`fixture` / `agent` / `workflow`).
+ *
+ * The names we actually carry in reports, config, CLI flags, and verifier
+ * messages are the `EvalMode` ones; legacy tier inputs are accepted with a
+ * single deprecation warning per process so existing scripts keep working
+ * through the 0.28.x line.
+ */
+import { type EvalMode } from "./types.js";
+/**
+ * Reset the per-process "already warned about legacy tier" flag. Used by
+ * tests so each test file gets a deterministic warning surface.
+ */
+export declare function __resetLegacyWarningForTests(): void;
+export interface LegacyTierInput {
+    source: "cli" | "env" | "config";
+    raw: string;
+}
+/**
+ * Normalize a raw string from the CLI / env / config into an `EvalMode`.
+ * Accepts both new (`fixture|agent|workflow`) and legacy (`A|B|C`) names.
+ * Emits a deprecation warning to stderr at most once per process when a
+ * legacy tier name is seen.
+ */
+export declare function parseModeInput(raw: string, input: LegacyTierInput, writeWarning?: (message: string) => void): EvalMode;
+/** @deprecated kept for callers that still need to serialize as legacy. */
+export declare function modeToLegacyTier(mode: EvalMode): "A" | "B" | "C";