cclaw-cli 0.24.0 → 0.25.0

package/dist/cli.js

@@ -59,7 +59,7 @@ Commands:
                     --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=workflow).
                     --schema-only        Run only structural verifiers (default).
                     --rules              Also run rule-based verifiers (keywords, regex, counts, uniqueness, traceability).
-                    --judge              Include LLM judging (not wired yet; requires API key).
+                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; Tier A also runs the single-shot agent-under-test.
                     --dry-run            Validate config + corpus, print summary, do not execute.
                     --json               Emit machine-readable JSON on stdout.
                     --no-write           Skip writing the report to .cclaw/evals/reports/.
@@ -78,6 +78,7 @@ Examples:
   cclaw archive --name=payments-revamp
   cclaw eval --dry-run
   cclaw eval --stage=brainstorm --schema-only
+  cclaw eval --judge --tier=A --stage=brainstorm
 
 Docs:   https://github.com/zuevrs/cclaw
 Issues: https://github.com/zuevrs/cclaw/issues

package/dist/content/eval-scaffold.d.ts

@@ -6,6 +6,10 @@
  */
 export declare const EVAL_CONFIG_YAML = "# cclaw eval config\n# See docs/evals.md for the full schema and rollout plan.\n#\n# All values can be overridden at runtime with CCLAW_EVAL_* environment\n# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.\nprovider: zai\nbaseUrl: https://api.z.ai/api/coding/paas/v4\nmodel: glm-5.1\n\n# Default fidelity tier when --tier is not supplied.\n#   A = single-shot API call (cheap)\n#   B = SDK with tool use     (realistic)\n#   C = multi-stage workflow  (end-to-end)\ndefaultTier: A\n\n# Per-call timeout and retry budget.\ntimeoutMs: 120000\nmaxRetries: 2\n\n# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.\n# dailyUsdCap: 5\n\n# Regression thresholds used by CI.\nregression:\n  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).\n  failIfDeltaBelow: -0.15\n  # Fail when any single critical rubric drops below this absolute score.\n  failIfCriticalBelow: 3.0\n";
 export declare const EVAL_CORPUS_README = "# Eval Corpus\n\nSeed cases live in `./<stage>/<id>.yaml`, one file per case.\nSee `docs/evals.md` for the schema.\n\nMinimal shape:\n\n```yaml\nid: brainstorm-01\nstage: brainstorm\ninput_prompt: |\n  One short paragraph describing the user's task.\ncontext_files: []\nexpected:\n  # verifier-specific hints; optional\n```\n\nStart with 3 structural cases per stage (24 total), then expand to 5 per\nstage (40 total) once rule verifiers land. Tier B/C runs may add\n`context_files` pulled from real projects to exercise the sandbox.\n";
-export declare const EVAL_RUBRICS_README = "# Eval Rubrics\n\nLLM-judge rubrics. Each rubric is a short list of checks scored on a\n`1\u20135` scale with a rationale:\n\n```yaml\nstage: brainstorm\nchecks:\n  - id: distinctness\n    prompt: \"Are the proposed directions genuinely distinct (not rephrasings)?\"\n    scale: \"1-5 where 5=fully distinct approaches\"\n    weight: 1.0\n```\n\nRubric authoring happens when Tier A runs start producing artifacts, so we\nscore the *right* properties rather than retrofitting generic quality checks.\nSee `docs/evals.md` for the full schema.\n";
+export declare const EVAL_RUBRICS_README = "# Eval Rubrics\n\nLLM-judge rubrics. Each rubric is a short list of checks scored on a\n`1\u20135` scale with a rationale. The runner picks `<stage>.yaml` when\n`cclaw eval --judge` is invoked; every stage ships a starter rubric\nbelow \u2014 edit the checks to match what your team cares about, and add\n`critical: true` to the checks that should hard-fail nightly CI on\nregression.\n\n```yaml\nstage: brainstorm\nchecks:\n  - id: distinctness\n    prompt: \"Are the proposed directions genuinely distinct (not rephrasings)?\"\n    scale: \"1-5 where 5=fully distinct approaches\"\n    weight: 1.0\n    critical: false\n```\n\nSee `docs/evals.md` for the full schema.\n";
+export declare const EVAL_RUBRIC_FILES: ReadonlyArray<{
+    stage: string;
+    contents: string;
+}>;
 export declare const EVAL_BASELINES_README = "# Eval Baselines\n\nFrozen score snapshots used by regression gates. Baselines are committed to\ngit and updated explicitly via `cclaw eval --update-baseline --confirm`.\n\nEach baseline file is a JSON document keyed by stage and case id. Do not edit\nby hand; CI will flag baseline churn.\n";
 export declare const EVAL_REPORTS_README = "# Eval Reports\n\nGenerated reports (JSON + Markdown) land here. This directory is gitignored.\nRun `cclaw eval --dry-run` to preview configuration without producing a\nreport.\n";

package/dist/content/eval-scaffold.js

@@ -57,7 +57,11 @@ stage (40 total) once rule verifiers land. Tier B/C runs may add
 export const EVAL_RUBRICS_README = `# Eval Rubrics
 
 LLM-judge rubrics. Each rubric is a short list of checks scored on a
-\`1–5\` scale with a rationale:
+\`1–5\` scale with a rationale. The runner picks \`<stage>.yaml\` when
+\`cclaw eval --judge\` is invoked; every stage ships a starter rubric
+below — edit the checks to match what your team cares about, and add
+\`critical: true\` to the checks that should hard-fail nightly CI on
+regression.
 
 \`\`\`yaml
 stage: brainstorm
@@ -66,12 +70,289 @@ checks:
     prompt: "Are the proposed directions genuinely distinct (not rephrasings)?"
     scale: "1-5 where 5=fully distinct approaches"
     weight: 1.0
+    critical: false
 \`\`\`
 
-Rubric authoring happens when Tier A runs start producing artifacts, so we
-score the *right* properties rather than retrofitting generic quality checks.
 See \`docs/evals.md\` for the full schema.
 `;
+const STARTER_RUBRICS = [
+    {
+        stage: "brainstorm",
+        checks: [
+            {
+                id: "distinctness",
+                prompt: "Are the proposed directions genuinely distinct (different approaches, not rephrasings of one idea)?",
+                scale: "1-5 where 5 = every direction uses a materially different approach",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "coverage",
+                prompt: "Do the directions cover the problem space (at least one tackling cost, one velocity, one risk)?",
+                scale: "1-5 where 5 = each major trade-off dimension has a direction",
+                weight: 1.0
+            },
+            {
+                id: "actionability",
+                prompt: "Could a reader pick one direction and start a scope doc tomorrow without asking clarifying questions?",
+                scale: "1-5 where 5 = every direction is concrete enough to scope immediately",
+                weight: 1.0
+            },
+            {
+                id: "recommendation-clarity",
+                prompt: "Is the Recommendation section explicit, single-voiced, and consistent with the highest-ranked direction?",
+                scale: "1-5 where 5 = recommendation names the chosen direction and the decisive trade-off",
+                weight: 1.0,
+                critical: true
+            }
+        ]
+    },
+    {
+        stage: "scope",
+        checks: [
+            {
+                id: "problem-statement",
+                prompt: "Is the problem statement anchored on user/system behavior (not on a proposed solution)?",
+                scale: "1-5 where 5 = problem is described independently of any implementation choice",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "non-goals",
+                prompt: "Are non-goals explicit and mutually-exclusive with the goals (no overlap, no vague 'we might' entries)?",
+                scale: "1-5 where 5 = every non-goal is a crisp decision a future reader can defend",
+                weight: 1.0
+            },
+            {
+                id: "decision-ids",
+                prompt: "Does the Decisions section use stable D-NN ids and name who (or what) owns each decision?",
+                scale: "1-5 where 5 = every decision has a D-NN id and an explicit owner",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "risks",
+                prompt: "Are risks concrete (named system, threshold, or scenario) rather than generic hedges?",
+                scale: "1-5 where 5 = each risk is testable by observing a specific signal",
+                weight: 0.8
+            }
+        ]
+    },
+    {
+        stage: "design",
+        checks: [
+            {
+                id: "decision-trace",
+                prompt: "Does the design doc restate every scope D-NN that drives the architecture, and call out the ones it rejects?",
+                scale: "1-5 where 5 = full D-NN trace with explicit kept/rejected markers",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "diagram-or-flow",
+                prompt: "Is there at least one diagram or clearly labeled flow section that shows data and control moving across the system?",
+                scale: "1-5 where 5 = diagram covers read path, write path, and failure path",
+                weight: 1.0
+            },
+            {
+                id: "alternatives-considered",
+                prompt: "Are concrete alternatives considered with explicit trade-offs (cost, complexity, latency)?",
+                scale: "1-5 where 5 = at least two alternatives are rejected with reasons tied to measurable properties",
+                weight: 0.8
+            },
+            {
+                id: "interface-stability",
+                prompt: "Are public interfaces (APIs, queues, tables) named, typed, and marked as SEMVER-stable or experimental?",
+                scale: "1-5 where 5 = every interface has a name, a type/shape, and a stability tag",
+                weight: 1.0
+            }
+        ]
+    },
+    {
+        stage: "spec",
+        checks: [
+            {
+                id: "acceptance-criteria",
+                prompt: "Does the spec have explicit Acceptance Criteria bullets that are unambiguously verifiable?",
+                scale: "1-5 where 5 = each AC states an observable condition with clear pass/fail",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "edge-cases",
+                prompt: "Are failure modes and edge cases enumerated (empty input, concurrent writers, partial outage)?",
+                scale: "1-5 where 5 = at least three distinct edge cases with expected behavior",
+                weight: 1.0
+            },
+            {
+                id: "test-plan-hooks",
+                prompt: "Does the spec name the test surfaces (unit, integration, e2e, synthetic probe) that will validate each AC?",
+                scale: "1-5 where 5 = every AC maps to at least one test surface",
+                weight: 1.0
+            },
+            {
+                id: "traceability",
+                prompt: "Does the spec cite the originating scope decisions (D-NN) and design sections so future engineers can trace back?",
+                scale: "1-5 where 5 = every material choice links to a D-NN or design heading",
+                weight: 0.8,
+                critical: true
+            }
+        ]
+    },
+    {
+        stage: "plan",
+        checks: [
+            {
+                id: "task-granularity",
+                prompt: "Are tasks sized so one engineer can land each in a single PR (<1 day of work)?",
+                scale: "1-5 where 5 = every T-NN fits in a single reviewable PR",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "tdd-loop",
+                prompt: "Does each task have explicit RED/GREEN/REFACTOR expectations or an equivalent TDD-compatible exit condition?",
+                scale: "1-5 where 5 = every task says what test fails first and what code makes it pass",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "dependency-graph",
+                prompt: "Is the dependency order between tasks explicit (and minimal), so parallelizable work is called out?",
+                scale: "1-5 where 5 = every task lists its blockers and independent tasks are marked parallelizable",
+                weight: 0.8
+            },
+            {
+                id: "scope-traceability",
+                prompt: "Does the plan reference the scope D-NN ids that drive each task, and does coverage leave no decision orphaned?",
+                scale: "1-5 where 5 = every D-NN appears in at least one task and every task names its D-NN",
+                weight: 1.0
+            }
+        ]
+    },
+    {
+        stage: "tdd",
+        checks: [
+            {
+                id: "red-first",
+                prompt: "Does the artifact show a failing test (RED) before the implementation change (GREEN)?",
+                scale: "1-5 where 5 = RED command output is quoted and the fix lands after",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "refactor-evidence",
+                prompt: "Is there a REFACTOR step with a diff or named improvement (not just passing tests)?",
+                scale: "1-5 where 5 = REFACTOR names a specific code-quality win and cites the affected file(s)",
+                weight: 0.8
+            },
+            {
+                id: "gate-evidence",
+                prompt: "Does the artifact quote the output of the required gates (lint, typecheck, tests) after the change?",
+                scale: "1-5 where 5 = every gate command is reproduced with its exit status",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "learnings",
+                prompt: "Does the artifact capture at least one durable learning (pattern, pitfall, follow-up) for future runs?",
+                scale: "1-5 where 5 = learning is specific, filed under knowledge.jsonl or an equivalent store",
+                weight: 0.6
+            }
+        ]
+    },
+    {
+        stage: "review",
+        checks: [
+            {
+                id: "two-layer-structure",
+                prompt: "Does the review show both layers (automated gates + human judgment) with distinct evidence?",
+                scale: "1-5 where 5 = Layer 1 cites tool outputs, Layer 2 cites reviewer reasoning",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "blocker-severity",
+                prompt: "Are issues classified by severity (blocker / major / minor) with one-line rationales?",
+                scale: "1-5 where 5 = every finding names severity + consequence if not fixed",
+                weight: 1.0
+            },
+            {
+                id: "security-posture",
+                prompt: "Does the review cover security-relevant areas explicitly (secrets, authz, PII, deps)?",
+                scale: "1-5 where 5 = each security dimension is addressed (with 'n/a' counted as a deliberate pass)",
+                weight: 0.8,
+                critical: true
+            },
+            {
+                id: "follow-ups",
+                prompt: "Are non-blocking follow-ups filed as explicit tickets or knowledge-log entries (not left as prose)?",
+                scale: "1-5 where 5 = every follow-up has a home and an owner",
+                weight: 0.8
+            }
+        ]
+    },
+    {
+        stage: "ship",
+        checks: [
+            {
+                id: "release-readiness",
+                prompt: "Does the artifact prove release readiness (gates green, changelog, version bump)?",
+                scale: "1-5 where 5 = each readiness item is linked to concrete evidence",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "rollback",
+                prompt: "Is there an explicit rollback path (command, feature-flag, migration reversal)?",
+                scale: "1-5 where 5 = rollback is reproducible from the doc with no context rehydration",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "monitoring",
+                prompt: "Are monitoring and alerting hooks named (dashboards, logs, SLO tripwires)?",
+                scale: "1-5 where 5 = each hook has a canonical URL or query",
+                weight: 0.8
+            },
+            {
+                id: "retro-seed",
+                prompt: "Does the artifact leave a retro seed (what went well, what to change for the next run)?",
+                scale: "1-5 where 5 = at least one distinct 'keep' and one 'change' statement",
+                weight: 0.6
+            }
+        ]
+    }
+];
+function renderRubric(rubric) {
+    const lines = [];
+    lines.push(`# Starter rubric for the \`${rubric.stage}\` stage.`);
+    lines.push(`# Edit the checks to reflect your team's bar before running`);
+    lines.push(`# \`cclaw eval --judge\`. Every check id is used verbatim in`);
+    lines.push(`# report output and baseline files, so keep slugs stable once`);
+    lines.push(`# they start appearing in CI.`);
+    lines.push(`stage: ${rubric.stage}`);
+    lines.push(`checks:`);
+    for (const check of rubric.checks) {
+        lines.push(`  - id: ${check.id}`);
+        lines.push(`    prompt: >-`);
+        lines.push(`      ${check.prompt}`);
+        if (check.scale !== undefined) {
+            lines.push(`    scale: ${JSON.stringify(check.scale)}`);
+        }
+        if (check.weight !== undefined) {
+            lines.push(`    weight: ${check.weight}`);
+        }
+        if (check.critical === true) {
+            lines.push(`    critical: true`);
+        }
+    }
+    return `${lines.join("\n")}\n`;
+}
+export const EVAL_RUBRIC_FILES = STARTER_RUBRICS.map((rubric) => ({
+    stage: rubric.stage,
+    contents: renderRubric(rubric)
+}));
 export const EVAL_BASELINES_README = `# Eval Baselines
 
 Frozen score snapshots used by regression gates. Baselines are committed to

package/dist/eval/agents/single-shot.d.ts

@@ -0,0 +1,27 @@
+import type { FlowStage } from "../../types.js";
+import type { ChatUsage, EvalLlmClient } from "../llm-client.js";
+import type { EvalCase, ResolvedEvalConfig } from "../types.js";
+export interface SingleShotInput {
+    caseEntry: EvalCase;
+    config: Pick<ResolvedEvalConfig, "model" | "agentTemperature" | "timeoutMs" | "tokenPricing">;
+    projectRoot: string;
+    client: EvalLlmClient;
+    /**
+     * Override the SKILL.md loader. Primarily a test hook so unit tests
+     * can swap a canned system prompt without creating fixtures on disk.
+     */
+    loadSkill?: (stage: FlowStage) => Promise<string>;
+}
+export interface SingleShotOutput {
+    artifact: string;
+    usage: ChatUsage;
+    usageUsd: number;
+    model: string;
+    durationMs: number;
+    attempts: number;
+    systemPrompt: string;
+    userPrompt: string;
+}
+export declare function loadStageSkill(projectRoot: string, stage: FlowStage): Promise<string>;
+/** Run the Tier A single-shot AUT and return the produced artifact. */
+export declare function runSingleShot(input: SingleShotInput): Promise<SingleShotOutput>;

package/dist/eval/agents/single-shot.js

@@ -0,0 +1,79 @@
+/**
+ * Tier A single-shot agent.
+ *
+ * Simplest realistic AUT: one LLM call with the stage's SKILL.md as the
+ * system prompt and the case's `inputPrompt` as the user message. Output
+ * is the raw assistant content, returned as the artifact for the judge
+ * pipeline.
+ *
+ * Design notes:
+ *
+ * - No tools. No multi-turn. No reads of the project beyond the one
+ *   SKILL.md. Tier B/C layer complexity on top in later steps.
+ * - Errors are propagated as-is (`EvalLlmError` subclasses) so the
+ *   runner can surface them as verifier failures without swallowing the
+ *   cause.
+ * - Usage and USD cost are surfaced so the runner can commit them to
+ *   the cost guard + case-level `costUsd`.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { RUNTIME_ROOT } from "../../constants.js";
+import { stageSkillFolder } from "../../content/skills.js";
+import { exists } from "../../fs-utils.js";
+import { computeUsageUsd } from "../cost-guard.js";
+export async function loadStageSkill(projectRoot, stage) {
+    const folder = stageSkillFolder(stage);
+    const file = path.join(projectRoot, RUNTIME_ROOT, "skills", folder, "SKILL.md");
+    if (!(await exists(file))) {
+        throw new Error(`Stage skill not found: ${path.relative(projectRoot, file)}. ` +
+            `Run \`cclaw init\` (or \`cclaw sync\`) before \`cclaw eval --tier=A --judge\`.`);
+    }
+    return fs.readFile(file, "utf8");
+}
+function buildMessages(systemPrompt, userPrompt) {
+    return [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: userPrompt }
+    ];
+}
+function buildUserPrompt(caseEntry) {
+    const lines = [];
+    lines.push(`Stage: ${caseEntry.stage}`);
+    lines.push(`Case id: ${caseEntry.id}`);
+    lines.push(``);
+    lines.push(`Task:`);
+    lines.push(caseEntry.inputPrompt.trim());
+    lines.push(``);
+    lines.push(`Produce the artifact required by this stage using the SKILL.md above. ` +
+        `Output the artifact directly (markdown with optional YAML frontmatter). ` +
+        `Do not wrap in code fences, do not add commentary before or after.`);
+    return lines.join("\n");
+}
+/** Run the Tier A single-shot AUT and return the produced artifact. */
+export async function runSingleShot(input) {
+    const { caseEntry, config, projectRoot, client } = input;
+    const started = Date.now();
+    const loader = input.loadSkill ?? ((stage) => loadStageSkill(projectRoot, stage));
+    const systemPrompt = await loader(caseEntry.stage);
+    const userPrompt = buildUserPrompt(caseEntry);
+    const response = await client.chat({
+        model: config.model,
+        messages: buildMessages(systemPrompt, userPrompt),
+        temperature: config.agentTemperature ?? 0.2,
+        timeoutMs: config.timeoutMs
+    });
+    const usageUsd = computeUsageUsd(response.model, response.usage, {
+        tokenPricing: config.tokenPricing
+    });
+    return {
+        artifact: response.content.trim(),
+        usage: response.usage,
+        usageUsd,
+        model: response.model,
+        attempts: response.attempts,
+        durationMs: Date.now() - started,
+        systemPrompt,
+        userPrompt
+    };
+}

package/dist/eval/config-loader.js

@@ -20,13 +20,19 @@ export const DEFAULT_EVAL_CONFIG = {
         failIfCriticalBelow: 3.0
     },
     timeoutMs: 120_000,
-    maxRetries: 2
+    maxRetries: 2,
+    judgeSamples: 3,
+    judgeTemperature: 0,
+    agentTemperature: 0.2
 };
 const EVAL_TIER_SET = new Set(EVAL_TIERS);
 const NUMERIC_ENVS = new Set([
     "CCLAW_EVAL_DAILY_USD_CAP",
     "CCLAW_EVAL_TIMEOUT_MS",
-    "CCLAW_EVAL_MAX_RETRIES"
+    "CCLAW_EVAL_MAX_RETRIES",
+    "CCLAW_EVAL_JUDGE_SAMPLES",
+    "CCLAW_EVAL_JUDGE_TEMPERATURE",
+    "CCLAW_EVAL_AGENT_TEMPERATURE"
 ]);
 function evalConfigError(configFilePath, reason) {
     return new Error(`Invalid cclaw eval config at ${configFilePath}: ${reason}\n` +
@@ -93,6 +99,59 @@ function validateFileConfig(raw, configFilePath) {
         }
         out.maxRetries = raw.maxRetries;
     }
+    if (raw.judgeSamples !== undefined) {
+        const value = raw.judgeSamples;
+        if (!Number.isInteger(value) || value < 1) {
+            throw evalConfigError(configFilePath, `"judgeSamples" must be a positive integer`);
+        }
+        if (value % 2 === 0) {
+            throw evalConfigError(configFilePath, `"judgeSamples" must be odd (so median-of-N is a true integer)`);
+        }
+        out.judgeSamples = value;
+    }
+    if (raw.judgeTemperature !== undefined) {
+        if (typeof raw.judgeTemperature !== "number" || !Number.isFinite(raw.judgeTemperature)) {
+            throw evalConfigError(configFilePath, `"judgeTemperature" must be a finite number`);
+        }
+        if (raw.judgeTemperature < 0 || raw.judgeTemperature > 2) {
+            throw evalConfigError(configFilePath, `"judgeTemperature" must be within [0, 2]`);
+        }
+        out.judgeTemperature = raw.judgeTemperature;
+    }
+    if (raw.agentTemperature !== undefined) {
+        if (typeof raw.agentTemperature !== "number" || !Number.isFinite(raw.agentTemperature)) {
+            throw evalConfigError(configFilePath, `"agentTemperature" must be a finite number`);
+        }
+        if (raw.agentTemperature < 0 || raw.agentTemperature > 2) {
+            throw evalConfigError(configFilePath, `"agentTemperature" must be within [0, 2]`);
+        }
+        out.agentTemperature = raw.agentTemperature;
+    }
+    if (raw.tokenPricing !== undefined) {
+        if (!isRecord(raw.tokenPricing)) {
+            throw evalConfigError(configFilePath, `"tokenPricing" must be a mapping`);
+        }
+        const pricing = {};
+        for (const [model, value] of Object.entries(raw.tokenPricing)) {
+            if (!isRecord(value)) {
+                throw evalConfigError(configFilePath, `"tokenPricing.${model}" must be a mapping with numeric input + output keys`);
+            }
+            const input = value.input;
+            const output = value.output;
+            if (typeof input !== "number" || input < 0) {
+                throw evalConfigError(configFilePath, `"tokenPricing.${model}.input" must be a non-negative number`);
+            }
+            if (typeof output !== "number" || output < 0) {
+                throw evalConfigError(configFilePath, `"tokenPricing.${model}.output" must be a non-negative number`);
+            }
+            const extraneous = Object.keys(value).filter((key) => key !== "input" && key !== "output");
+            if (extraneous.length > 0) {
+                throw evalConfigError(configFilePath, `"tokenPricing.${model}" has unknown key(s): ${extraneous.join(", ")}`);
+            }
+            pricing[model] = { input, output };
+        }
+        out.tokenPricing = pricing;
+    }
     if (raw.regression !== undefined) {
         if (!isRecord(raw.regression)) {
             throw evalConfigError(configFilePath, `"regression" must be a mapping`);
@@ -123,7 +182,11 @@ function validateFileConfig(raw, configFilePath) {
         "dailyUsdCap",
         "timeoutMs",
         "maxRetries",
-        "regression"
+        "regression",
+        "judgeSamples",
+        "judgeTemperature",
+        "agentTemperature",
+        "tokenPricing"
     ]);
     const unknown = Object.keys(raw).filter((key) => !knownKeys.has(key));
     if (unknown.length > 0) {
@@ -203,6 +266,36 @@ function applyEnvOverrides(base, env) {
         patched.maxRetries = parseNumericEnv("CCLAW_EVAL_MAX_RETRIES", retries);
         overridden = true;
     }
+    const judgeSamples = read("CCLAW_EVAL_JUDGE_SAMPLES");
+    if (judgeSamples) {
+        const value = parseNumericEnv("CCLAW_EVAL_JUDGE_SAMPLES", judgeSamples);
+        if (!Number.isInteger(value) || value < 1) {
+            throw new Error(`Environment variable CCLAW_EVAL_JUDGE_SAMPLES must be a positive integer, got: ${judgeSamples}`);
+        }
+        if (value % 2 === 0) {
+            throw new Error(`Environment variable CCLAW_EVAL_JUDGE_SAMPLES must be odd, got: ${judgeSamples}`);
+        }
+        patched.judgeSamples = value;
+        overridden = true;
+    }
+    const judgeTemp = read("CCLAW_EVAL_JUDGE_TEMPERATURE");
+    if (judgeTemp) {
+        const value = parseNumericEnv("CCLAW_EVAL_JUDGE_TEMPERATURE", judgeTemp);
+        if (value < 0 || value > 2) {
+            throw new Error(`Environment variable CCLAW_EVAL_JUDGE_TEMPERATURE must be within [0, 2], got: ${judgeTemp}`);
+        }
+        patched.judgeTemperature = value;
+        overridden = true;
+    }
+    const agentTemp = read("CCLAW_EVAL_AGENT_TEMPERATURE");
+    if (agentTemp) {
+        const value = parseNumericEnv("CCLAW_EVAL_AGENT_TEMPERATURE", agentTemp);
+        if (value < 0 || value > 2) {
+            throw new Error(`Environment variable CCLAW_EVAL_AGENT_TEMPERATURE must be within [0, 2], got: ${agentTemp}`);
+        }
+        patched.agentTemperature = value;
+        overridden = true;
+    }
     const apiKey = read("CCLAW_EVAL_API_KEY");
     return { patched, overridden, apiKey };
 }

package/dist/eval/cost-guard.d.ts

@@ -0,0 +1,80 @@
+import type { ChatUsage } from "./llm-client.js";
+import type { ResolvedEvalConfig, TokenPricing } from "./types.js";
+/**
+ * Builtin pricing fallback. Intentionally conservative: when the user
+ * hasn't configured pricing and we don't know the model, we default to a
+ * "small model" USD schedule so the cap can still do something useful.
+ *
+ * Values are USD per 1K tokens. Sources are public pricing pages as of
+ * 2026-04; update by editing this constant, not the guard logic.
+ */
+export declare const DEFAULT_TOKEN_PRICING: Readonly<Record<string, TokenPricing>>;
+/** Hard default when neither config nor builtins know the model. */
+export declare const UNKNOWN_MODEL_PRICING: TokenPricing;
+export interface SpendLedger {
+    /** ISO date (`YYYY-MM-DD` in UTC) — also embedded in the file name. */
+    date: string;
+    /** USD spent so far today across every call that hit the guard. */
+    totalUsd: number;
+    /** Number of `chat()` calls accounted for. */
+    calls: number;
+    /** Per-model breakdown for the report. */
+    byModel: Record<string, {
+        tokensIn: number;
+        tokensOut: number;
+        usd: number;
+    }>;
+}
+export declare class DailyCostCapExceededError extends Error {
+    readonly capUsd: number;
+    readonly projectedUsd: number;
+    readonly currentUsd: number;
+    constructor(opts: {
+        capUsd: number;
+        projectedUsd: number;
+        currentUsd: number;
+    });
+}
+declare function utcDate(now?: Date): string;
+declare function pricingFor(model: string, config: Pick<ResolvedEvalConfig, "tokenPricing">): TokenPricing;
+/**
+ * Compute USD cost of a single `ChatUsage` using the given `model` pricing
+ * schedule. Returns 0 when `usage.totalTokens` is 0 (e.g. transport error
+ * before first token).
+ */
+export declare function computeUsageUsd(model: string, usage: ChatUsage, config: Pick<ResolvedEvalConfig, "tokenPricing">): number;
+declare function ledgerPath(projectRoot: string, date: string): string;
+declare function readLedger(file: string, date: string): Promise<SpendLedger>;
+declare function writeLedger(file: string, ledger: SpendLedger): Promise<void>;
+/**
+ * Guard a single LLM call against the daily USD cap. Returns the updated
+ * ledger on success; throws `DailyCostCapExceededError` when the projected
+ * total would cross the cap. When `config.dailyUsdCap` is unset, the guard
+ * is a no-op — no file writes, no ledger — so non-judge runs never touch
+ * the filesystem.
+ */
+export interface CostGuard {
+    /**
+     * Commit the USD cost of a finished call to the ledger. When `dailyUsdCap`
+     * is set, refuses the commit if the projected total would exceed the cap.
+     */
+    commit(model: string, usage: ChatUsage): Promise<number>;
+    /** Snapshot the current ledger (or undefined when no cap is set). */
+    snapshot(): Promise<SpendLedger | undefined>;
+}
+export interface CreateCostGuardOptions {
+    /** Clock injection for tests. */
+    now?: () => Date;
+    /** Override the default filesystem root for the ledger. */
+    ledgerPath?: string;
+}
+export declare function createCostGuard(projectRoot: string, config: Pick<ResolvedEvalConfig, "dailyUsdCap" | "tokenPricing">, options?: CreateCostGuardOptions): CostGuard;
+/** Exposed for tests. */
+export declare const __internal: {
+    utcDate: typeof utcDate;
+    pricingFor: typeof pricingFor;
+    ledgerPath: typeof ledgerPath;
+    readLedger: typeof readLedger;
+    writeLedger: typeof writeLedger;
+};
+export {};