cclaw-cli 0.21.2 → 0.23.0

package/dist/cli.d.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import type { FlowTrack, HarnessId, InitProfile } from "./types.js";
-type CommandName = "init" | "sync" | "doctor" | "upgrade" | "uninstall" | "archive";
+import type { EvalTier } from "./eval/types.js";
+type CommandName = "init" | "sync" | "doctor" | "upgrade" | "uninstall" | "archive" | "eval";
 interface ParsedArgs {
     command?: CommandName;
     harnesses?: HarnessId[];
@@ -16,6 +17,15 @@ interface ParsedArgs {
     archiveName?: string;
     archiveSkipRetro?: boolean;
     archiveSkipRetroReason?: string;
+    evalStage?: string;
+    evalTier?: EvalTier;
+    evalSchemaOnly?: boolean;
+    evalRules?: boolean;
+    evalJudge?: boolean;
+    evalJson?: boolean;
+    evalNoWrite?: boolean;
+    evalUpdateBaseline?: boolean;
+    evalConfirm?: boolean;
     showHelp?: boolean;
     showVersion?: boolean;
 }

package/dist/cli.js

@@ -13,7 +13,20 @@ import { RUNTIME_ROOT } from "./constants.js";
 import { createDefaultConfig, createProfileConfig } from "./config.js";
 import { detectHarnesses } from "./init-detect.js";
 import { HARNESS_ADAPTERS } from "./harness-adapters.js";
-const INSTALLER_COMMANDS = ["init", "sync", "doctor", "upgrade", "uninstall", "archive"];
+import { runEval } from "./eval/runner.js";
+import { writeBaselinesFromReport } from "./eval/baseline.js";
+import { writeJsonReport, writeMarkdownReport } from "./eval/report.js";
+import { EVAL_TIERS } from "./eval/types.js";
+import { FLOW_STAGES } from "./types.js";
+const INSTALLER_COMMANDS = [
+    "init",
+    "sync",
+    "doctor",
+    "upgrade",
+    "uninstall",
+    "archive",
+    "eval"
+];
 export function usage() {
     return `cclaw - installer-first flow toolkit
 
@@ -41,6 +54,17 @@ Commands:
              Flags: --name=<feature>    Feature slug (default: inferred from 00-idea.md).
                     --skip-retro       Bypass mandatory retro gate (requires --retro-reason).
                     --retro-reason=<t> Reason for bypassing retro gate.
+  eval       Run cclaw evals against .cclaw/evals/corpus (Phase 7, Wave 7.1: structural verifier).
+             Flags: --stage=<id>         Limit to one flow stage (${FLOW_STAGES.join("|")}).
+                    --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=workflow).
+                    --schema-only        Run only structural verifiers (Wave 7.1, default).
+                    --rules              Run structural + rule verifiers (Wave 7.2).
+                    --judge              Include LLM judging (Wave 7.3; requires API key).
+                    --dry-run            Validate config + corpus, print summary, do not execute.
+                    --json               Emit machine-readable JSON on stdout.
+                    --no-write           Skip writing the report to .cclaw/evals/reports/.
+                    --update-baseline    Overwrite baselines from the current run (requires --confirm).
+                    --confirm            Acknowledge --update-baseline (prevents accidental resets).
   upgrade    Refresh generated files in .cclaw without modifying user artifacts.
   uninstall  Remove .cclaw runtime and the generated harness shim files.
 
@@ -52,6 +76,8 @@ Examples:
   cclaw init --harnesses=claude,cursor
   cclaw doctor --reconcile-gates
   cclaw archive --name=payments-revamp
+  cclaw eval --dry-run
+  cclaw eval --stage=brainstorm --schema-only
 
 Docs:   https://github.com/zuevrs/cclaw
 Issues: https://github.com/zuevrs/cclaw/issues
@@ -107,6 +133,20 @@ function parseProfile(raw) {
     }
     return trimmed;
 }
+function parseEvalTier(raw) {
+    const trimmed = raw.trim().toUpperCase();
+    if (!EVAL_TIERS.includes(trimmed)) {
+        throw new Error(`Unknown eval tier: ${raw}. Supported: ${EVAL_TIERS.join(", ")}`);
+    }
+    return trimmed;
+}
+function parseEvalStage(raw) {
+    const trimmed = raw.trim();
+    if (!FLOW_STAGES.includes(trimmed)) {
+        throw new Error(`Unknown eval stage: ${raw}. Supported: ${FLOW_STAGES.join(", ")}`);
+    }
+    return trimmed;
+}
 function isInitPromptAllowed(ctx) {
     return Boolean(process.stdin.isTTY && ctx.stdout.isTTY);
 }
@@ -390,7 +430,45 @@ function parseArgs(argv) {
         }
         if (flag.startsWith("--retro-reason=")) {
             parsed.archiveSkipRetroReason = flag.replace("--retro-reason=", "").trim();
+            continue;
+        }
+        if (flag.startsWith("--stage=")) {
+            parsed.evalStage = parseEvalStage(flag.replace("--stage=", ""));
+            continue;
+        }
+        if (flag.startsWith("--tier=")) {
+            parsed.evalTier = parseEvalTier(flag.replace("--tier=", ""));
+            continue;
+        }
+        if (flag === "--schema-only") {
+            parsed.evalSchemaOnly = true;
+            continue;
+        }
+        if (flag === "--rules") {
+            parsed.evalRules = true;
+            continue;
+        }
+        if (flag === "--judge") {
+            parsed.evalJudge = true;
+            continue;
+        }
+        if (flag === "--no-write") {
+            parsed.evalNoWrite = true;
+            continue;
+        }
+        if (flag === "--update-baseline") {
+            parsed.evalUpdateBaseline = true;
+            continue;
         }
+        if (flag === "--confirm") {
+            parsed.evalConfirm = true;
+            continue;
+        }
+    }
+    // `--json` is shared between doctor and eval. Disambiguate by command.
+    if (parsed.command === "eval" && parsed.doctorJson === true) {
+        parsed.evalJson = true;
+        parsed.doctorJson = undefined;
     }
     return parsed;
 }
@@ -487,6 +565,81 @@ async function runCommand(parsed, ctx) {
         info(ctx, "Upgraded .cclaw runtime and regenerated generated files");
         return 0;
     }
+    if (command === "eval") {
+        const result = await runEval({
+            projectRoot: ctx.cwd,
+            stage: parsed.evalStage,
+            tier: parsed.evalTier,
+            schemaOnly: parsed.evalSchemaOnly === true,
+            rules: parsed.evalRules === true,
+            judge: parsed.evalJudge === true,
+            dryRun: parsed.dryRun === true
+        });
+        if ("kind" in result) {
+            if (parsed.evalJson === true) {
+                ctx.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+                return 0;
+            }
+            ctx.stdout.write(`cclaw eval dry-run\n`);
+            ctx.stdout.write(`  provider: ${result.config.provider}\n`);
+            ctx.stdout.write(`  baseUrl: ${result.config.baseUrl}\n`);
+            ctx.stdout.write(`  model: ${result.config.model}\n`);
+            ctx.stdout.write(`  source: ${result.config.source}\n`);
+            ctx.stdout.write(`  apiKey: ${result.config.apiKey ? "set" : "unset"}\n`);
+            ctx.stdout.write(`  tier: ${result.plannedTier}\n`);
+            ctx.stdout.write(`  corpus: ${result.corpus.total} case(s)\n`);
+            for (const [stage, count] of Object.entries(result.corpus.byStage)) {
+                ctx.stdout.write(`    - ${stage}: ${count}\n`);
+            }
+            ctx.stdout.write(`  verifiers available:\n`);
+            for (const [key, value] of Object.entries(result.verifiersAvailable)) {
+                ctx.stdout.write(`    - ${key}: ${value ? "yes" : "no"}\n`);
+            }
+            if (result.notes.length > 0) {
+                ctx.stdout.write(`  notes:\n`);
+                for (const note of result.notes) {
+                    ctx.stdout.write(`    - ${note}\n`);
+                }
+            }
+            return 0;
+        }
+        if (parsed.evalUpdateBaseline === true && parsed.evalConfirm !== true) {
+            error(ctx, "--update-baseline requires --confirm to prevent accidental baseline resets.");
+            return 1;
+        }
+        if (parsed.evalUpdateBaseline === true) {
+            if (result.summary.failed > 0) {
+                error(ctx, `Refusing to update baselines: ${result.summary.failed} case(s) currently failing. Fix structural checks first.`);
+                return 1;
+            }
+            const written = await writeBaselinesFromReport(ctx.cwd, result);
+            for (const file of written) {
+                info(ctx, `Baseline written: ${path.relative(ctx.cwd, file)}`);
+            }
+        }
+        if (parsed.evalNoWrite !== true) {
+            const jsonPath = await writeJsonReport(ctx.cwd, result);
+            const mdPath = await writeMarkdownReport(ctx.cwd, result);
+            info(ctx, `Report written: ${path.relative(ctx.cwd, jsonPath)}`);
+            info(ctx, `Report written: ${path.relative(ctx.cwd, mdPath)}`);
+        }
+        const regressionCount = result.baselineDelta?.criticalFailures ?? 0;
+        if (parsed.evalJson === true) {
+            ctx.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        }
+        else {
+            const regressionNote = regressionCount > 0 ? `, ${regressionCount} regression(s)` : "";
+            ctx.stdout.write(`cclaw eval: ${result.summary.totalCases} case(s), ` +
+                `${result.summary.passed} passed, ` +
+                `${result.summary.failed} failed, ` +
+                `${result.summary.skipped} skipped${regressionNote}\n`);
+        }
+        if (result.summary.failed > 0)
+            return 1;
+        if (regressionCount > 0)
+            return 1;
+        return 0;
+    }
     if (command === "archive") {
         const archived = await archiveRun(ctx.cwd, parsed.archiveName, {
             skipRetro: parsed.archiveSkipRetro === true,

package/dist/constants.d.ts

@@ -4,8 +4,17 @@ export declare const RUNTIME_ROOT = ".cclaw";
 export declare const CCLAW_VERSION = "0.1.1";
 export declare const FLOW_VERSION = "1.0.0";
 export declare const DEFAULT_HARNESSES: HarnessId[];
-export declare const REQUIRED_DIRS: readonly [".cclaw", ".cclaw/commands", ".cclaw/skills", ".cclaw/contexts", ".cclaw/templates", ".cclaw/artifacts", ".cclaw/worktrees", ".cclaw/state", ".cclaw/runs", ".cclaw/rules", ".cclaw/adapters", ".cclaw/agents", ".cclaw/hooks", ".cclaw/custom-skills"];
-export declare const REQUIRED_GITIGNORE_PATTERNS: readonly ["# cclaw generated artifacts", ".cclaw/", ".claude/commands/cc-*.md", ".claude/commands/cc.md", ".cursor/commands/cc-*.md", ".cursor/commands/cc.md", ".opencode/commands/cc-*.md", ".opencode/commands/cc.md", ".codex/commands/cc-*.md", ".codex/commands/cc.md", ".claude/hooks/hooks.json", ".cursor/hooks.json", ".codex/hooks.json", ".opencode/plugins/cclaw-plugin.mjs", ".cursor/rules/cclaw-workflow.mdc"];
+/**
+ * Evals subtree. Wave 7.0 scaffolds the directory layout and a default config.yaml;
+ * verifiers and LLM wiring arrive in Waves 7.1–7.5. Keeping this separate from the
+ * main REQUIRED_DIRS list makes it explicit that the evals runtime is additive and
+ * does not affect non-eval cclaw behavior.
+ */
+export declare const EVALS_ROOT = ".cclaw/evals";
+export declare const EVALS_CONFIG_PATH = ".cclaw/evals/config.yaml";
+export declare const EVALS_DIRS: readonly [".cclaw/evals", ".cclaw/evals/corpus", ".cclaw/evals/rubrics", ".cclaw/evals/baselines", ".cclaw/evals/reports"];
+export declare const REQUIRED_DIRS: readonly [".cclaw", ".cclaw/commands", ".cclaw/skills", ".cclaw/contexts", ".cclaw/templates", ".cclaw/artifacts", ".cclaw/worktrees", ".cclaw/state", ".cclaw/runs", ".cclaw/rules", ".cclaw/adapters", ".cclaw/agents", ".cclaw/hooks", ".cclaw/custom-skills", ".cclaw/evals", ".cclaw/evals/corpus", ".cclaw/evals/rubrics", ".cclaw/evals/baselines", ".cclaw/evals/reports"];
+export declare const REQUIRED_GITIGNORE_PATTERNS: readonly ["# cclaw generated artifacts", ".cclaw/", "# cclaw evals: user-owned, track in git", "!.cclaw/evals/", "!.cclaw/evals/config.yaml", "!.cclaw/evals/corpus/", "!.cclaw/evals/corpus/**", "!.cclaw/evals/rubrics/", "!.cclaw/evals/rubrics/**", "!.cclaw/evals/baselines/", "!.cclaw/evals/baselines/**", ".claude/commands/cc-*.md", ".claude/commands/cc.md", ".cursor/commands/cc-*.md", ".cursor/commands/cc.md", ".opencode/commands/cc-*.md", ".opencode/commands/cc.md", ".codex/commands/cc-*.md", ".codex/commands/cc.md", ".claude/hooks/hooks.json", ".cursor/hooks.json", ".codex/hooks.json", ".opencode/plugins/cclaw-plugin.mjs", ".cursor/rules/cclaw-workflow.mdc"];
 export declare const COMMAND_FILE_ORDER: FlowStage[];
 export declare const UTILITY_COMMANDS: readonly ["learn", "next", "ideate", "view", "status", "tree", "diff", "ops", "feature", "tdd-log", "retro", "compound", "archive", "rewind"];
 export declare const SUBAGENT_SKILL_FOLDERS: readonly ["subagent-dev", "parallel-dispatch"];

package/dist/constants.js

@@ -8,6 +8,21 @@ export const DEFAULT_HARNESSES = [
     "opencode",
     "codex"
 ];
+/**
+ * Evals subtree. Wave 7.0 scaffolds the directory layout and a default config.yaml;
+ * verifiers and LLM wiring arrive in Waves 7.1–7.5. Keeping this separate from the
+ * main REQUIRED_DIRS list makes it explicit that the evals runtime is additive and
+ * does not affect non-eval cclaw behavior.
+ */
+export const EVALS_ROOT = `${RUNTIME_ROOT}/evals`;
+export const EVALS_CONFIG_PATH = `${EVALS_ROOT}/config.yaml`;
+export const EVALS_DIRS = [
+    EVALS_ROOT,
+    `${EVALS_ROOT}/corpus`,
+    `${EVALS_ROOT}/rubrics`,
+    `${EVALS_ROOT}/baselines`,
+    `${EVALS_ROOT}/reports`
+];
 export const REQUIRED_DIRS = [
     RUNTIME_ROOT,
     `${RUNTIME_ROOT}/commands`,
@@ -22,11 +37,21 @@ export const REQUIRED_DIRS = [
     `${RUNTIME_ROOT}/adapters`,
     `${RUNTIME_ROOT}/agents`,
     `${RUNTIME_ROOT}/hooks`,
-    `${RUNTIME_ROOT}/custom-skills`
+    `${RUNTIME_ROOT}/custom-skills`,
+    ...EVALS_DIRS
 ];
 export const REQUIRED_GITIGNORE_PATTERNS = [
     "# cclaw generated artifacts",
     `${RUNTIME_ROOT}/`,
+    "# cclaw evals: user-owned, track in git",
+    `!${EVALS_ROOT}/`,
+    `!${EVALS_ROOT}/config.yaml`,
+    `!${EVALS_ROOT}/corpus/`,
+    `!${EVALS_ROOT}/corpus/**`,
+    `!${EVALS_ROOT}/rubrics/`,
+    `!${EVALS_ROOT}/rubrics/**`,
+    `!${EVALS_ROOT}/baselines/`,
+    `!${EVALS_ROOT}/baselines/**`,
     ".claude/commands/cc-*.md",
     ".claude/commands/cc.md",
     ".cursor/commands/cc-*.md",

package/dist/content/eval-scaffold.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Static scaffold for `.cclaw/evals/`. Written on `cclaw init` and refreshed
+ * on `cclaw sync` only if the files are missing (user content wins). The
+ * scaffold is intentionally minimal: a usable default config plus short
+ * READMEs that point at `docs/evals.md` for authoring guidance.
+ */
+export declare const EVAL_CONFIG_YAML = "# cclaw eval config\n# See docs/evals.md for the full schema and Wave 7.1\u20137.6 rollout plan.\n#\n# All values can be overridden at runtime with CCLAW_EVAL_* environment\n# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.\nprovider: zai\nbaseUrl: https://api.z.ai/api/coding/paas/v4\nmodel: glm-5.1\n\n# Default fidelity tier when --tier is not supplied.\n#   A = single-shot API call (cheap, Wave 7.3)\n#   B = SDK with tool use     (realistic, Wave 7.4)\n#   C = multi-stage workflow  (end-to-end, Wave 7.5)\ndefaultTier: A\n\n# Per-call timeout and retry budget.\ntimeoutMs: 120000\nmaxRetries: 2\n\n# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.\n# dailyUsdCap: 5\n\n# Regression thresholds used by CI (Wave 7.3+).\nregression:\n  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).\n  failIfDeltaBelow: -0.15\n  # Fail when any single critical rubric drops below this absolute score.\n  failIfCriticalBelow: 3.0\n";
+export declare const EVAL_CORPUS_README = "# Eval Corpus\n\nSeed cases live in `./<stage>/<id>.yaml`, one file per case.\nSee `docs/evals.md` for the schema; authoring begins in Wave 7.1.\n\nMinimal shape:\n\n```yaml\nid: brainstorm-01\nstage: brainstorm\ninput_prompt: |\n  One short paragraph describing the user's task.\ncontext_files: []\nexpected:\n  # verifier-specific hints; optional in Wave 7.0\n```\n\nWave 7.1 will add 3 cases per stage (24 total). Wave 7.2 will expand to 5 per\nstage (40 total). Wave 7.4/7.5 may add `context_files` pulled from real\nprojects to exercise Tier B/C sandboxes.\n";
+export declare const EVAL_RUBRICS_README = "# Eval Rubrics\n\nLLM-judge rubrics land in Wave 7.3. Each rubric is a short list of checks\nscored on a `1\u20135` scale with a rationale:\n\n```yaml\nstage: brainstorm\nchecks:\n  - id: distinctness\n    prompt: \"Are the proposed directions genuinely distinct (not rephrasings)?\"\n    scale: \"1-5 where 5=fully distinct approaches\"\n    weight: 1.0\n```\n\nRubric authoring happens when Tier A runs start producing artifacts, so we\nscore the *right* properties rather than retrofitting generic quality checks.\nSee `docs/evals.md` for the full schema.\n";
+export declare const EVAL_BASELINES_README = "# Eval Baselines\n\nFrozen score snapshots used by regression gates. Baselines are committed to\ngit and updated explicitly via `cclaw eval --update-baseline --confirm`\n(wired in Wave 7.1).\n\nEach baseline file is a JSON document keyed by stage and case id. Do not edit\nby hand; CI will flag baseline churn.\n";
+export declare const EVAL_REPORTS_README = "# Eval Reports\n\nGenerated reports (JSON + Markdown) land here. This directory is gitignored.\nRun `cclaw eval --dry-run` to preview configuration without producing a\nreport.\n";

package/dist/content/eval-scaffold.js

@@ -0,0 +1,89 @@
+/**
+ * Static scaffold for `.cclaw/evals/`. Written on `cclaw init` and refreshed
+ * on `cclaw sync` only if the files are missing (user content wins). The
+ * scaffold is intentionally minimal: a usable default config plus short
+ * READMEs that point at `docs/evals.md` for authoring guidance.
+ */
+export const EVAL_CONFIG_YAML = `# cclaw eval config
+# See docs/evals.md for the full schema and Wave 7.1–7.6 rollout plan.
+#
+# All values can be overridden at runtime with CCLAW_EVAL_* environment
+# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.
+provider: zai
+baseUrl: https://api.z.ai/api/coding/paas/v4
+model: glm-5.1
+
+# Default fidelity tier when --tier is not supplied.
+#   A = single-shot API call (cheap, Wave 7.3)
+#   B = SDK with tool use     (realistic, Wave 7.4)
+#   C = multi-stage workflow  (end-to-end, Wave 7.5)
+defaultTier: A
+
+# Per-call timeout and retry budget.
+timeoutMs: 120000
+maxRetries: 2
+
+# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.
+# dailyUsdCap: 5
+
+# Regression thresholds used by CI (Wave 7.3+).
+regression:
+  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).
+  failIfDeltaBelow: -0.15
+  # Fail when any single critical rubric drops below this absolute score.
+  failIfCriticalBelow: 3.0
+`;
+export const EVAL_CORPUS_README = `# Eval Corpus
+
+Seed cases live in \`./<stage>/<id>.yaml\`, one file per case.
+See \`docs/evals.md\` for the schema; authoring begins in Wave 7.1.
+
+Minimal shape:
+
+\`\`\`yaml
+id: brainstorm-01
+stage: brainstorm
+input_prompt: |
+  One short paragraph describing the user's task.
+context_files: []
+expected:
+  # verifier-specific hints; optional in Wave 7.0
+\`\`\`
+
+Wave 7.1 will add 3 cases per stage (24 total). Wave 7.2 will expand to 5 per
+stage (40 total). Wave 7.4/7.5 may add \`context_files\` pulled from real
+projects to exercise Tier B/C sandboxes.
+`;
+export const EVAL_RUBRICS_README = `# Eval Rubrics
+
+LLM-judge rubrics land in Wave 7.3. Each rubric is a short list of checks
+scored on a \`1–5\` scale with a rationale:
+
+\`\`\`yaml
+stage: brainstorm
+checks:
+  - id: distinctness
+    prompt: "Are the proposed directions genuinely distinct (not rephrasings)?"
+    scale: "1-5 where 5=fully distinct approaches"
+    weight: 1.0
+\`\`\`
+
+Rubric authoring happens when Tier A runs start producing artifacts, so we
+score the *right* properties rather than retrofitting generic quality checks.
+See \`docs/evals.md\` for the full schema.
+`;
+export const EVAL_BASELINES_README = `# Eval Baselines
+
+Frozen score snapshots used by regression gates. Baselines are committed to
+git and updated explicitly via \`cclaw eval --update-baseline --confirm\`
+(wired in Wave 7.1).
+
+Each baseline file is a JSON document keyed by stage and case id. Do not edit
+by hand; CI will flag baseline churn.
+`;
+export const EVAL_REPORTS_README = `# Eval Reports
+
+Generated reports (JSON + Markdown) land here. This directory is gitignored.
+Run \`cclaw eval --dry-run\` to preview configuration without producing a
+report.
+`;

package/dist/eval/baseline.d.ts

@@ -0,0 +1,14 @@
+import type { FlowStage } from "../types.js";
+import type { BaselineDelta, BaselineSnapshot, EvalReport } from "./types.js";
+export declare const BASELINE_SCHEMA_VERSION = 1;
+export declare function loadBaseline(projectRoot: string, stage: FlowStage): Promise<BaselineSnapshot | null>;
+export declare function loadBaselinesByStage(projectRoot: string, stages: readonly FlowStage[]): Promise<Map<FlowStage, BaselineSnapshot>>;
+export declare function buildBaselineForStage(stage: FlowStage, report: EvalReport): BaselineSnapshot;
+export declare function writeBaselinesFromReport(projectRoot: string, report: EvalReport): Promise<string[]>;
+/**
+ * Compare a freshly computed report against loaded baselines. If no baseline
+ * exists for a stage covered by the report, that stage contributes zero
+ * regressions (first run of that stage). Current is the source of truth.
+ */
+export declare function compareAgainstBaselines(report: EvalReport, baselines: Map<FlowStage, BaselineSnapshot>): BaselineDelta | undefined;
+export declare function listBaselineStages(projectRoot: string): Promise<FlowStage[]>;

package/dist/eval/baseline.js

@@ -0,0 +1,209 @@
+/**
+ * Baseline I/O + regression comparison (Wave 7.1).
+ *
+ * Layout on disk (committed):
+ *
+ *   .cclaw/evals/baselines/<stage>.json
+ *
+ * Each file contains a `BaselineSnapshot` keyed by `EvalCase.id`. We compute
+ * regressions by comparing per-verifier `ok` flags across runs: any verifier
+ * that was `ok:true` in the baseline and is `ok:false` now counts as a
+ * critical failure. A case whose aggregate `passed` flipped from true to
+ * false is flagged as `case-now-failing` regardless of per-verifier churn.
+ *
+ * Writes are gated behind an explicit `--update-baseline --confirm` pair at
+ * the CLI layer so accidental resets do not slip into PRs.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { EVALS_ROOT, CCLAW_VERSION } from "../constants.js";
+import { exists } from "../fs-utils.js";
+import { FLOW_STAGES } from "../types.js";
+export const BASELINE_SCHEMA_VERSION = 1;
+function baselinePath(projectRoot, stage) {
+    return path.join(projectRoot, EVALS_ROOT, "baselines", `${stage}.json`);
+}
+export async function loadBaseline(projectRoot, stage) {
+    const filePath = baselinePath(projectRoot, stage);
+    if (!(await exists(filePath)))
+        return null;
+    const raw = await fs.readFile(filePath, "utf8");
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        throw new Error(`Invalid baseline at ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!isBaseline(parsed, stage)) {
+        throw new Error(`Invalid baseline at ${filePath}: shape mismatch (expected schemaVersion=${BASELINE_SCHEMA_VERSION}, stage=${stage})`);
+    }
+    return parsed;
+}
+function isBaseline(value, stage) {
+    if (!value || typeof value !== "object")
+        return false;
+    const candidate = value;
+    if (candidate.schemaVersion !== BASELINE_SCHEMA_VERSION)
+        return false;
+    if (candidate.stage !== stage)
+        return false;
+    if (typeof candidate.generatedAt !== "string")
+        return false;
+    if (typeof candidate.cclawVersion !== "string")
+        return false;
+    if (!candidate.cases || typeof candidate.cases !== "object")
+        return false;
+    return true;
+}
+export async function loadBaselinesByStage(projectRoot, stages) {
+    const out = new Map();
+    for (const stage of stages) {
+        const snapshot = await loadBaseline(projectRoot, stage);
+        if (snapshot)
+            out.set(stage, snapshot);
+    }
+    return out;
+}
+function entryFromResult(result) {
+    const verifierResults = result.verifierResults.map((v) => ({
+        id: v.id,
+        kind: v.kind,
+        ok: v.ok,
+        ...(v.score !== undefined ? { score: v.score } : {})
+    }));
+    return { passed: result.passed, verifierResults };
+}
+export function buildBaselineForStage(stage, report) {
+    const stageCases = report.cases.filter((c) => c.stage === stage);
+    const cases = {};
+    for (const c of stageCases) {
+        cases[c.caseId] = entryFromResult(c);
+    }
+    return {
+        schemaVersion: BASELINE_SCHEMA_VERSION,
+        stage,
+        generatedAt: new Date().toISOString(),
+        cclawVersion: CCLAW_VERSION,
+        cases
+    };
+}
+export async function writeBaselinesFromReport(projectRoot, report) {
+    const written = [];
+    const stages = new Set(report.cases.map((c) => c.stage));
+    for (const stage of stages) {
+        const snapshot = buildBaselineForStage(stage, report);
+        const file = baselinePath(projectRoot, stage);
+        await fs.mkdir(path.dirname(file), { recursive: true });
+        await fs.writeFile(file, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
+        written.push(file);
+    }
+    return written.sort();
+}
+function verifierMap(entries) {
+    const out = new Map();
+    for (const entry of entries) {
+        out.set(entry.id, entry);
+    }
+    return out;
+}
+function computePassRate(cases) {
+    if (cases.length === 0)
+        return 1;
+    const passed = cases.filter((c) => c.passed).length;
+    return passed / cases.length;
+}
+function baselinePassRate(snapshot) {
+    const entries = Object.values(snapshot.cases);
+    if (entries.length === 0)
+        return 1;
+    const passed = entries.filter((e) => e.passed).length;
+    return passed / entries.length;
+}
+/**
+ * Compare a freshly computed report against loaded baselines. If no baseline
+ * exists for a stage covered by the report, that stage contributes zero
+ * regressions (first run of that stage). Current is the source of truth.
+ */
+export function compareAgainstBaselines(report, baselines) {
+    if (baselines.size === 0)
+        return undefined;
+    const regressions = [];
+    const caseResultsByStage = new Map();
+    for (const c of report.cases) {
+        const bucket = caseResultsByStage.get(c.stage) ?? [];
+        bucket.push(c);
+        caseResultsByStage.set(c.stage, bucket);
+    }
+    let baselineTotalPassRate = 0;
+    let baselineStagesCounted = 0;
+    for (const [stage, snapshot] of baselines) {
+        const current = caseResultsByStage.get(stage) ?? [];
+        baselineTotalPassRate += baselinePassRate(snapshot);
+        baselineStagesCounted += 1;
+        for (const caseResult of current) {
+            const baselineEntry = snapshot.cases[caseResult.caseId];
+            if (!baselineEntry)
+                continue;
+            if (baselineEntry.passed && !caseResult.passed) {
+                regressions.push({
+                    caseId: caseResult.caseId,
+                    stage,
+                    verifierId: "<case>",
+                    reason: "case-now-failing",
+                    previousScore: 1,
+                    currentScore: 0
+                });
+            }
+            const baselineVerifiers = verifierMap(baselineEntry.verifierResults);
+            for (const currentVerifier of caseResult.verifierResults) {
+                const prev = baselineVerifiers.get(currentVerifier.id);
+                if (!prev)
+                    continue;
+                if (prev.ok && !currentVerifier.ok) {
+                    regressions.push({
+                        caseId: caseResult.caseId,
+                        stage,
+                        verifierId: currentVerifier.id,
+                        reason: "newly-failing",
+                        previousScore: prev.score ?? 1,
+                        currentScore: currentVerifier.score ?? 0
+                    });
+                }
+                else if (prev.score !== undefined &&
+                    currentVerifier.score !== undefined &&
+                    currentVerifier.score < prev.score) {
+                    regressions.push({
+                        caseId: caseResult.caseId,
+                        stage,
+                        verifierId: currentVerifier.id,
+                        reason: "score-drop",
+                        previousScore: prev.score,
+                        currentScore: currentVerifier.score
+                    });
+                }
+            }
+        }
+    }
+    const currentPassRate = computePassRate(report.cases);
+    const baselineAveragePassRate = baselineStagesCounted === 0 ? currentPassRate : baselineTotalPassRate / baselineStagesCounted;
+    const scoreDelta = Number((currentPassRate - baselineAveragePassRate).toFixed(4));
+    const criticalFailures = regressions.filter((r) => r.reason === "newly-failing" || r.reason === "case-now-failing").length;
+    const baselineStages = [...baselines.keys()].sort().join(",");
+    return {
+        baselineId: baselineStages.length > 0 ? baselineStages : "(empty)",
+        scoreDelta,
+        criticalFailures,
+        regressions
+    };
+}
+export function listBaselineStages(projectRoot) {
+    const root = path.join(projectRoot, EVALS_ROOT, "baselines");
+    return fs
+        .readdir(root, { withFileTypes: true })
+        .then((entries) => entries
+        .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+        .map((entry) => entry.name.replace(/\.json$/, ""))
+        .filter((name) => FLOW_STAGES.includes(name)))
+        .catch(() => []);
+}

package/dist/eval/config-loader.d.ts

@@ -0,0 +1,14 @@
+import type { EvalConfig, ResolvedEvalConfig } from "./types.js";
+/**
+ * Default eval config. Optimized for the z.ai OpenAI-compatible coding endpoint
+ * with GLM 5.1 per the roadmap locked decisions (D-EVAL-01..05). Any field can
+ * be overridden by `.cclaw/evals/config.yaml` and then by `CCLAW_EVAL_*` env
+ * variables (env wins last).
+ */
+export declare const DEFAULT_EVAL_CONFIG: EvalConfig;
+/**
+ * Resolve eval config in layered order: defaults -> config.yaml -> env vars.
+ * Returns a fully-populated config plus a provenance marker so `--dry-run` can
+ * surface where each setting came from.
+ */
+export declare function loadEvalConfig(projectRoot: string, env?: NodeJS.ProcessEnv): Promise<ResolvedEvalConfig>;