cclaw-cli 0.21.2 → 0.22.0

package/dist/cli.d.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import type { FlowTrack, HarnessId, InitProfile } from "./types.js";
-type CommandName = "init" | "sync" | "doctor" | "upgrade" | "uninstall" | "archive";
+import type { EvalTier } from "./eval/types.js";
+type CommandName = "init" | "sync" | "doctor" | "upgrade" | "uninstall" | "archive" | "eval";
 interface ParsedArgs {
     command?: CommandName;
     harnesses?: HarnessId[];
@@ -16,6 +17,13 @@ interface ParsedArgs {
     archiveName?: string;
     archiveSkipRetro?: boolean;
     archiveSkipRetroReason?: string;
+    evalStage?: string;
+    evalTier?: EvalTier;
+    evalSchemaOnly?: boolean;
+    evalRules?: boolean;
+    evalJudge?: boolean;
+    evalJson?: boolean;
+    evalNoWrite?: boolean;
     showHelp?: boolean;
     showVersion?: boolean;
 }

package/dist/cli.js

@@ -13,7 +13,19 @@ import { RUNTIME_ROOT } from "./constants.js";
 import { createDefaultConfig, createProfileConfig } from "./config.js";
 import { detectHarnesses } from "./init-detect.js";
 import { HARNESS_ADAPTERS } from "./harness-adapters.js";
-const INSTALLER_COMMANDS = ["init", "sync", "doctor", "upgrade", "uninstall", "archive"];
+import { runEval } from "./eval/runner.js";
+import { writeJsonReport, writeMarkdownReport } from "./eval/report.js";
+import { EVAL_TIERS } from "./eval/types.js";
+import { FLOW_STAGES } from "./types.js";
+const INSTALLER_COMMANDS = [
+    "init",
+    "sync",
+    "doctor",
+    "upgrade",
+    "uninstall",
+    "archive",
+    "eval"
+];
 export function usage() {
     return `cclaw - installer-first flow toolkit
 
@@ -41,6 +53,15 @@ Commands:
              Flags: --name=<feature>    Feature slug (default: inferred from 00-idea.md).
                     --skip-retro       Bypass mandatory retro gate (requires --retro-reason).
                     --retro-reason=<t> Reason for bypassing retro gate.
+  eval       Run cclaw evals against .cclaw/evals/corpus (Phase 7, Wave 7.0 foundations).
+             Flags: --stage=<id>        Limit to one flow stage (${FLOW_STAGES.join("|")}).
+                    --tier=<A|B|C>      Fidelity tier (A=single-shot, B=tools, C=workflow).
+                    --schema-only       Run only structural verifiers (Wave 7.1).
+                    --rules             Run structural + rule verifiers (Wave 7.2).
+                    --judge             Include LLM judging (Wave 7.3; requires API key).
+                    --dry-run           Validate config + corpus, print summary, do not execute.
+                    --json              Emit machine-readable JSON on stdout.
+                    --no-write          Skip writing the report to .cclaw/evals/reports/.
   upgrade    Refresh generated files in .cclaw without modifying user artifacts.
   uninstall  Remove .cclaw runtime and the generated harness shim files.
 
@@ -52,6 +73,8 @@ Examples:
   cclaw init --harnesses=claude,cursor
   cclaw doctor --reconcile-gates
   cclaw archive --name=payments-revamp
+  cclaw eval --dry-run
+  cclaw eval --stage=brainstorm --schema-only
 
 Docs:   https://github.com/zuevrs/cclaw
 Issues: https://github.com/zuevrs/cclaw/issues
@@ -107,6 +130,20 @@ function parseProfile(raw) {
     }
     return trimmed;
 }
+function parseEvalTier(raw) {
+    const trimmed = raw.trim().toUpperCase();
+    if (!EVAL_TIERS.includes(trimmed)) {
+        throw new Error(`Unknown eval tier: ${raw}. Supported: ${EVAL_TIERS.join(", ")}`);
+    }
+    return trimmed;
+}
+function parseEvalStage(raw) {
+    const trimmed = raw.trim();
+    if (!FLOW_STAGES.includes(trimmed)) {
+        throw new Error(`Unknown eval stage: ${raw}. Supported: ${FLOW_STAGES.join(", ")}`);
+    }
+    return trimmed;
+}
 function isInitPromptAllowed(ctx) {
     return Boolean(process.stdin.isTTY && ctx.stdout.isTTY);
 }
@@ -390,7 +427,37 @@ function parseArgs(argv) {
         }
         if (flag.startsWith("--retro-reason=")) {
             parsed.archiveSkipRetroReason = flag.replace("--retro-reason=", "").trim();
+            continue;
+        }
+        if (flag.startsWith("--stage=")) {
+            parsed.evalStage = parseEvalStage(flag.replace("--stage=", ""));
+            continue;
+        }
+        if (flag.startsWith("--tier=")) {
+            parsed.evalTier = parseEvalTier(flag.replace("--tier=", ""));
+            continue;
+        }
+        if (flag === "--schema-only") {
+            parsed.evalSchemaOnly = true;
+            continue;
+        }
+        if (flag === "--rules") {
+            parsed.evalRules = true;
+            continue;
+        }
+        if (flag === "--judge") {
+            parsed.evalJudge = true;
+            continue;
         }
+        if (flag === "--no-write") {
+            parsed.evalNoWrite = true;
+            continue;
+        }
+    }
+    // `--json` is shared between doctor and eval. Disambiguate by command.
+    if (parsed.command === "eval" && parsed.doctorJson === true) {
+        parsed.evalJson = true;
+        parsed.doctorJson = undefined;
     }
     return parsed;
 }
@@ -487,6 +554,61 @@ async function runCommand(parsed, ctx) {
         info(ctx, "Upgraded .cclaw runtime and regenerated generated files");
         return 0;
     }
+    if (command === "eval") {
+        const result = await runEval({
+            projectRoot: ctx.cwd,
+            stage: parsed.evalStage,
+            tier: parsed.evalTier,
+            schemaOnly: parsed.evalSchemaOnly === true,
+            rules: parsed.evalRules === true,
+            judge: parsed.evalJudge === true,
+            dryRun: parsed.dryRun === true
+        });
+        if ("kind" in result) {
+            if (parsed.evalJson === true) {
+                ctx.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+                return 0;
+            }
+            ctx.stdout.write(`cclaw eval dry-run\n`);
+            ctx.stdout.write(`  provider: ${result.config.provider}\n`);
+            ctx.stdout.write(`  baseUrl: ${result.config.baseUrl}\n`);
+            ctx.stdout.write(`  model: ${result.config.model}\n`);
+            ctx.stdout.write(`  source: ${result.config.source}\n`);
+            ctx.stdout.write(`  apiKey: ${result.config.apiKey ? "set" : "unset"}\n`);
+            ctx.stdout.write(`  tier: ${result.plannedTier}\n`);
+            ctx.stdout.write(`  corpus: ${result.corpus.total} case(s)\n`);
+            for (const [stage, count] of Object.entries(result.corpus.byStage)) {
+                ctx.stdout.write(`    - ${stage}: ${count}\n`);
+            }
+            ctx.stdout.write(`  verifiers available:\n`);
+            for (const [key, value] of Object.entries(result.verifiersAvailable)) {
+                ctx.stdout.write(`    - ${key}: ${value ? "yes" : "no"}\n`);
+            }
+            if (result.notes.length > 0) {
+                ctx.stdout.write(`  notes:\n`);
+                for (const note of result.notes) {
+                    ctx.stdout.write(`    - ${note}\n`);
+                }
+            }
+            return 0;
+        }
+        if (parsed.evalNoWrite !== true) {
+            const jsonPath = await writeJsonReport(ctx.cwd, result);
+            const mdPath = await writeMarkdownReport(ctx.cwd, result);
+            info(ctx, `Report written: ${path.relative(ctx.cwd, jsonPath)}`);
+            info(ctx, `Report written: ${path.relative(ctx.cwd, mdPath)}`);
+        }
+        if (parsed.evalJson === true) {
+            ctx.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+        }
+        else {
+            ctx.stdout.write(`cclaw eval: ${result.summary.totalCases} case(s), ` +
+                `${result.summary.passed} passed, ` +
+                `${result.summary.failed} failed, ` +
+                `${result.summary.skipped} skipped (Wave 7.0 skeleton — verifiers land in Wave 7.1+)\n`);
+        }
+        return result.summary.failed > 0 ? 1 : 0;
+    }
     if (command === "archive") {
         const archived = await archiveRun(ctx.cwd, parsed.archiveName, {
             skipRetro: parsed.archiveSkipRetro === true,

package/dist/constants.d.ts

@@ -4,8 +4,17 @@ export declare const RUNTIME_ROOT = ".cclaw";
 export declare const CCLAW_VERSION = "0.1.1";
 export declare const FLOW_VERSION = "1.0.0";
 export declare const DEFAULT_HARNESSES: HarnessId[];
-export declare const REQUIRED_DIRS: readonly [".cclaw", ".cclaw/commands", ".cclaw/skills", ".cclaw/contexts", ".cclaw/templates", ".cclaw/artifacts", ".cclaw/worktrees", ".cclaw/state", ".cclaw/runs", ".cclaw/rules", ".cclaw/adapters", ".cclaw/agents", ".cclaw/hooks", ".cclaw/custom-skills"];
-export declare const REQUIRED_GITIGNORE_PATTERNS: readonly ["# cclaw generated artifacts", ".cclaw/", ".claude/commands/cc-*.md", ".claude/commands/cc.md", ".cursor/commands/cc-*.md", ".cursor/commands/cc.md", ".opencode/commands/cc-*.md", ".opencode/commands/cc.md", ".codex/commands/cc-*.md", ".codex/commands/cc.md", ".claude/hooks/hooks.json", ".cursor/hooks.json", ".codex/hooks.json", ".opencode/plugins/cclaw-plugin.mjs", ".cursor/rules/cclaw-workflow.mdc"];
+/**
+ * Evals subtree. Wave 7.0 scaffolds the directory layout and a default config.yaml;
+ * verifiers and LLM wiring arrive in Waves 7.1–7.5. Keeping this separate from the
+ * main REQUIRED_DIRS list makes it explicit that the evals runtime is additive and
+ * does not affect non-eval cclaw behavior.
+ */
+export declare const EVALS_ROOT = ".cclaw/evals";
+export declare const EVALS_CONFIG_PATH = ".cclaw/evals/config.yaml";
+export declare const EVALS_DIRS: readonly [".cclaw/evals", ".cclaw/evals/corpus", ".cclaw/evals/rubrics", ".cclaw/evals/baselines", ".cclaw/evals/reports"];
+export declare const REQUIRED_DIRS: readonly [".cclaw", ".cclaw/commands", ".cclaw/skills", ".cclaw/contexts", ".cclaw/templates", ".cclaw/artifacts", ".cclaw/worktrees", ".cclaw/state", ".cclaw/runs", ".cclaw/rules", ".cclaw/adapters", ".cclaw/agents", ".cclaw/hooks", ".cclaw/custom-skills", ".cclaw/evals", ".cclaw/evals/corpus", ".cclaw/evals/rubrics", ".cclaw/evals/baselines", ".cclaw/evals/reports"];
+export declare const REQUIRED_GITIGNORE_PATTERNS: readonly ["# cclaw generated artifacts", ".cclaw/", "# cclaw evals: user-owned, track in git", "!.cclaw/evals/", "!.cclaw/evals/config.yaml", "!.cclaw/evals/corpus/", "!.cclaw/evals/corpus/**", "!.cclaw/evals/rubrics/", "!.cclaw/evals/rubrics/**", "!.cclaw/evals/baselines/", "!.cclaw/evals/baselines/**", ".claude/commands/cc-*.md", ".claude/commands/cc.md", ".cursor/commands/cc-*.md", ".cursor/commands/cc.md", ".opencode/commands/cc-*.md", ".opencode/commands/cc.md", ".codex/commands/cc-*.md", ".codex/commands/cc.md", ".claude/hooks/hooks.json", ".cursor/hooks.json", ".codex/hooks.json", ".opencode/plugins/cclaw-plugin.mjs", ".cursor/rules/cclaw-workflow.mdc"];
 export declare const COMMAND_FILE_ORDER: FlowStage[];
 export declare const UTILITY_COMMANDS: readonly ["learn", "next", "ideate", "view", "status", "tree", "diff", "ops", "feature", "tdd-log", "retro", "compound", "archive", "rewind"];
 export declare const SUBAGENT_SKILL_FOLDERS: readonly ["subagent-dev", "parallel-dispatch"];

package/dist/constants.js

@@ -8,6 +8,21 @@ export const DEFAULT_HARNESSES = [
     "opencode",
     "codex"
 ];
+/**
+ * Evals subtree. Wave 7.0 scaffolds the directory layout and a default config.yaml;
+ * verifiers and LLM wiring arrive in Waves 7.1–7.5. Keeping this separate from the
+ * main REQUIRED_DIRS list makes it explicit that the evals runtime is additive and
+ * does not affect non-eval cclaw behavior.
+ */
+export const EVALS_ROOT = `${RUNTIME_ROOT}/evals`;
+export const EVALS_CONFIG_PATH = `${EVALS_ROOT}/config.yaml`;
+export const EVALS_DIRS = [
+    EVALS_ROOT,
+    `${EVALS_ROOT}/corpus`,
+    `${EVALS_ROOT}/rubrics`,
+    `${EVALS_ROOT}/baselines`,
+    `${EVALS_ROOT}/reports`
+];
 export const REQUIRED_DIRS = [
     RUNTIME_ROOT,
     `${RUNTIME_ROOT}/commands`,
@@ -22,11 +37,21 @@ export const REQUIRED_DIRS = [
     `${RUNTIME_ROOT}/adapters`,
     `${RUNTIME_ROOT}/agents`,
     `${RUNTIME_ROOT}/hooks`,
-    `${RUNTIME_ROOT}/custom-skills`
+    `${RUNTIME_ROOT}/custom-skills`,
+    ...EVALS_DIRS
 ];
 export const REQUIRED_GITIGNORE_PATTERNS = [
     "# cclaw generated artifacts",
     `${RUNTIME_ROOT}/`,
+    "# cclaw evals: user-owned, track in git",
+    `!${EVALS_ROOT}/`,
+    `!${EVALS_ROOT}/config.yaml`,
+    `!${EVALS_ROOT}/corpus/`,
+    `!${EVALS_ROOT}/corpus/**`,
+    `!${EVALS_ROOT}/rubrics/`,
+    `!${EVALS_ROOT}/rubrics/**`,
+    `!${EVALS_ROOT}/baselines/`,
+    `!${EVALS_ROOT}/baselines/**`,
     ".claude/commands/cc-*.md",
     ".claude/commands/cc.md",
     ".cursor/commands/cc-*.md",

package/dist/content/eval-scaffold.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Static scaffold for `.cclaw/evals/`. Written on `cclaw init` and refreshed
+ * on `cclaw sync` only if the files are missing (user content wins). The
+ * scaffold is intentionally minimal: a usable default config plus short
+ * READMEs that point at `docs/evals.md` for authoring guidance.
+ */
+export declare const EVAL_CONFIG_YAML = "# cclaw eval config\n# See docs/evals.md for the full schema and Wave 7.1\u20137.6 rollout plan.\n#\n# All values can be overridden at runtime with CCLAW_EVAL_* environment\n# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.\nprovider: zai\nbaseUrl: https://api.z.ai/api/coding/paas/v4\nmodel: glm-5.1\n\n# Default fidelity tier when --tier is not supplied.\n#   A = single-shot API call (cheap, Wave 7.3)\n#   B = SDK with tool use     (realistic, Wave 7.4)\n#   C = multi-stage workflow  (end-to-end, Wave 7.5)\ndefaultTier: A\n\n# Per-call timeout and retry budget.\ntimeoutMs: 120000\nmaxRetries: 2\n\n# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.\n# dailyUsdCap: 5\n\n# Regression thresholds used by CI (Wave 7.3+).\nregression:\n  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).\n  failIfDeltaBelow: -0.15\n  # Fail when any single critical rubric drops below this absolute score.\n  failIfCriticalBelow: 3.0\n";
+export declare const EVAL_CORPUS_README = "# Eval Corpus\n\nSeed cases live in `./<stage>/<id>.yaml`, one file per case.\nSee `docs/evals.md` for the schema; authoring begins in Wave 7.1.\n\nMinimal shape:\n\n```yaml\nid: brainstorm-01\nstage: brainstorm\ninput_prompt: |\n  One short paragraph describing the user's task.\ncontext_files: []\nexpected:\n  # verifier-specific hints; optional in Wave 7.0\n```\n\nWave 7.1 will add 3 cases per stage (24 total). Wave 7.2 will expand to 5 per\nstage (40 total). Wave 7.4/7.5 may add `context_files` pulled from real\nprojects to exercise Tier B/C sandboxes.\n";
+export declare const EVAL_RUBRICS_README = "# Eval Rubrics\n\nLLM-judge rubrics land in Wave 7.3. Each rubric is a short list of checks\nscored on a `1\u20135` scale with a rationale:\n\n```yaml\nstage: brainstorm\nchecks:\n  - id: distinctness\n    prompt: \"Are the proposed directions genuinely distinct (not rephrasings)?\"\n    scale: \"1-5 where 5=fully distinct approaches\"\n    weight: 1.0\n```\n\nRubric authoring happens when Tier A runs start producing artifacts, so we\nscore the *right* properties rather than retrofitting generic quality checks.\nSee `docs/evals.md` for the full schema.\n";
+export declare const EVAL_BASELINES_README = "# Eval Baselines\n\nFrozen score snapshots used by regression gates. Baselines are committed to\ngit and updated explicitly via `cclaw eval --update-baseline --confirm`\n(wired in Wave 7.1).\n\nEach baseline file is a JSON document keyed by stage and case id. Do not edit\nby hand; CI will flag baseline churn.\n";
+export declare const EVAL_REPORTS_README = "# Eval Reports\n\nGenerated reports (JSON + Markdown) land here. This directory is gitignored.\nRun `cclaw eval --dry-run` to preview configuration without producing a\nreport.\n";

package/dist/content/eval-scaffold.js

@@ -0,0 +1,89 @@
+/**
+ * Static scaffold for `.cclaw/evals/`. Written on `cclaw init` and refreshed
+ * on `cclaw sync` only if the files are missing (user content wins). The
+ * scaffold is intentionally minimal: a usable default config plus short
+ * READMEs that point at `docs/evals.md` for authoring guidance.
+ */
+export const EVAL_CONFIG_YAML = `# cclaw eval config
+# See docs/evals.md for the full schema and Wave 7.1–7.6 rollout plan.
+#
+# All values can be overridden at runtime with CCLAW_EVAL_* environment
+# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.
+provider: zai
+baseUrl: https://api.z.ai/api/coding/paas/v4
+model: glm-5.1
+
+# Default fidelity tier when --tier is not supplied.
+#   A = single-shot API call (cheap, Wave 7.3)
+#   B = SDK with tool use     (realistic, Wave 7.4)
+#   C = multi-stage workflow  (end-to-end, Wave 7.5)
+defaultTier: A
+
+# Per-call timeout and retry budget.
+timeoutMs: 120000
+maxRetries: 2
+
+# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.
+# dailyUsdCap: 5
+
+# Regression thresholds used by CI (Wave 7.3+).
+regression:
+  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).
+  failIfDeltaBelow: -0.15
+  # Fail when any single critical rubric drops below this absolute score.
+  failIfCriticalBelow: 3.0
+`;
+export const EVAL_CORPUS_README = `# Eval Corpus
+
+Seed cases live in \`./<stage>/<id>.yaml\`, one file per case.
+See \`docs/evals.md\` for the schema; authoring begins in Wave 7.1.
+
+Minimal shape:
+
+\`\`\`yaml
+id: brainstorm-01
+stage: brainstorm
+input_prompt: |
+  One short paragraph describing the user's task.
+context_files: []
+expected:
+  # verifier-specific hints; optional in Wave 7.0
+\`\`\`
+
+Wave 7.1 will add 3 cases per stage (24 total). Wave 7.2 will expand to 5 per
+stage (40 total). Wave 7.4/7.5 may add \`context_files\` pulled from real
+projects to exercise Tier B/C sandboxes.
+`;
+export const EVAL_RUBRICS_README = `# Eval Rubrics
+
+LLM-judge rubrics land in Wave 7.3. Each rubric is a short list of checks
+scored on a \`1–5\` scale with a rationale:
+
+\`\`\`yaml
+stage: brainstorm
+checks:
+  - id: distinctness
+    prompt: "Are the proposed directions genuinely distinct (not rephrasings)?"
+    scale: "1-5 where 5=fully distinct approaches"
+    weight: 1.0
+\`\`\`
+
+Rubric authoring happens when Tier A runs start producing artifacts, so we
+score the *right* properties rather than retrofitting generic quality checks.
+See \`docs/evals.md\` for the full schema.
+`;
+export const EVAL_BASELINES_README = `# Eval Baselines
+
+Frozen score snapshots used by regression gates. Baselines are committed to
+git and updated explicitly via \`cclaw eval --update-baseline --confirm\`
+(wired in Wave 7.1).
+
+Each baseline file is a JSON document keyed by stage and case id. Do not edit
+by hand; CI will flag baseline churn.
+`;
+export const EVAL_REPORTS_README = `# Eval Reports
+
+Generated reports (JSON + Markdown) land here. This directory is gitignored.
+Run \`cclaw eval --dry-run\` to preview configuration without producing a
+report.
+`;

package/dist/eval/config-loader.d.ts

@@ -0,0 +1,14 @@
+import type { EvalConfig, ResolvedEvalConfig } from "./types.js";
+/**
+ * Default eval config. Optimized for the z.ai OpenAI-compatible coding endpoint
+ * with GLM 5.1 per the roadmap locked decisions (D-EVAL-01..05). Any field can
+ * be overridden by `.cclaw/evals/config.yaml` and then by `CCLAW_EVAL_*` env
+ * variables (env wins last).
+ */
+export declare const DEFAULT_EVAL_CONFIG: EvalConfig;
+/**
+ * Resolve eval config in layered order: defaults -> config.yaml -> env vars.
+ * Returns a fully-populated config plus a provenance marker so `--dry-run` can
+ * surface where each setting came from.
+ */
+export declare function loadEvalConfig(projectRoot: string, env?: NodeJS.ProcessEnv): Promise<ResolvedEvalConfig>;

package/dist/eval/config-loader.js

@@ -0,0 +1,237 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { parse } from "yaml";
+import { EVALS_CONFIG_PATH } from "../constants.js";
+import { exists } from "../fs-utils.js";
+import { EVAL_TIERS } from "./types.js";
+/**
+ * Default eval config. Optimized for the z.ai OpenAI-compatible coding endpoint
+ * with GLM 5.1 per the roadmap locked decisions (D-EVAL-01..05). Any field can
+ * be overridden by `.cclaw/evals/config.yaml` and then by `CCLAW_EVAL_*` env
+ * variables (env wins last).
+ */
+export const DEFAULT_EVAL_CONFIG = {
+    provider: "zai",
+    baseUrl: "https://api.z.ai/api/coding/paas/v4",
+    model: "glm-5.1",
+    defaultTier: "A",
+    regression: {
+        failIfDeltaBelow: -0.15,
+        failIfCriticalBelow: 3.0
+    },
+    timeoutMs: 120_000,
+    maxRetries: 2
+};
+const EVAL_TIER_SET = new Set(EVAL_TIERS);
+const NUMERIC_ENVS = new Set([
+    "CCLAW_EVAL_DAILY_USD_CAP",
+    "CCLAW_EVAL_TIMEOUT_MS",
+    "CCLAW_EVAL_MAX_RETRIES"
+]);
+function evalConfigError(configFilePath, reason) {
+    return new Error(`Invalid cclaw eval config at ${configFilePath}: ${reason}\n` +
+        `Supported tiers: ${EVAL_TIERS.join(", ")}\n` +
+        `See docs/evals.md for the full schema. After fixing, run: cclaw eval --dry-run`);
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function parseNumericEnv(name, raw) {
+    const value = Number(raw);
+    if (!Number.isFinite(value)) {
+        throw new Error(`Environment variable ${name} must be numeric, got: ${raw}`);
+    }
+    return value;
+}
+function parseTierEnv(raw) {
+    const trimmed = raw.trim().toUpperCase();
+    if (!EVAL_TIER_SET.has(trimmed)) {
+        throw new Error(`Environment variable CCLAW_EVAL_TIER must be one of ${EVAL_TIERS.join("/")}, got: ${raw}`);
+    }
+    return trimmed;
+}
+function validateFileConfig(raw, configFilePath) {
+    if (raw === undefined || raw === null)
+        return {};
+    if (!isRecord(raw)) {
+        throw evalConfigError(configFilePath, "top-level value must be a mapping");
+    }
+    const out = {};
+    const assignString = (key, value) => {
+        if (value === undefined)
+            return;
+        if (typeof value !== "string" || value.trim().length === 0) {
+            throw evalConfigError(configFilePath, `"${String(key)}" must be a non-empty string`);
+        }
+        out[key] = value.trim();
+    };
+    assignString("provider", raw.provider);
+    assignString("baseUrl", raw.baseUrl);
+    assignString("model", raw.model);
+    assignString("judgeModel", raw.judgeModel);
+    if (raw.defaultTier !== undefined) {
+        if (typeof raw.defaultTier !== "string" || !EVAL_TIER_SET.has(raw.defaultTier)) {
+            throw evalConfigError(configFilePath, `"defaultTier" must be one of: ${EVAL_TIERS.join(", ")}`);
+        }
+        out.defaultTier = raw.defaultTier;
+    }
+    if (raw.dailyUsdCap !== undefined) {
+        if (typeof raw.dailyUsdCap !== "number" || raw.dailyUsdCap < 0) {
+            throw evalConfigError(configFilePath, `"dailyUsdCap" must be a non-negative number`);
+        }
+        out.dailyUsdCap = raw.dailyUsdCap;
+    }
+    if (raw.timeoutMs !== undefined) {
+        if (typeof raw.timeoutMs !== "number" || raw.timeoutMs <= 0) {
+            throw evalConfigError(configFilePath, `"timeoutMs" must be a positive number`);
+        }
+        out.timeoutMs = raw.timeoutMs;
+    }
+    if (raw.maxRetries !== undefined) {
+        if (!Number.isInteger(raw.maxRetries) || raw.maxRetries < 0) {
+            throw evalConfigError(configFilePath, `"maxRetries" must be a non-negative integer`);
+        }
+        out.maxRetries = raw.maxRetries;
+    }
+    if (raw.regression !== undefined) {
+        if (!isRecord(raw.regression)) {
+            throw evalConfigError(configFilePath, `"regression" must be a mapping`);
+        }
+        const failIfDeltaBelow = raw.regression.failIfDeltaBelow;
+        const failIfCriticalBelow = raw.regression.failIfCriticalBelow;
+        if (failIfDeltaBelow !== undefined && typeof failIfDeltaBelow !== "number") {
+            throw evalConfigError(configFilePath, `"regression.failIfDeltaBelow" must be a number`);
+        }
+        if (failIfCriticalBelow !== undefined && typeof failIfCriticalBelow !== "number") {
+            throw evalConfigError(configFilePath, `"regression.failIfCriticalBelow" must be a number`);
+        }
+        out.regression = {
+            failIfDeltaBelow: typeof failIfDeltaBelow === "number"
+                ? failIfDeltaBelow
+                : DEFAULT_EVAL_CONFIG.regression.failIfDeltaBelow,
+            failIfCriticalBelow: typeof failIfCriticalBelow === "number"
+                ? failIfCriticalBelow
+                : DEFAULT_EVAL_CONFIG.regression.failIfCriticalBelow
+        };
+    }
+    const knownKeys = new Set([
+        "provider",
+        "baseUrl",
+        "model",
+        "judgeModel",
+        "defaultTier",
+        "dailyUsdCap",
+        "timeoutMs",
+        "maxRetries",
+        "regression"
+    ]);
+    const unknown = Object.keys(raw).filter((key) => !knownKeys.has(key));
+    if (unknown.length > 0) {
+        throw evalConfigError(configFilePath, `unknown top-level key(s): ${unknown.join(", ")}`);
+    }
+    return out;
+}
+async function readFileConfig(projectRoot) {
+    const configFilePath = path.join(projectRoot, EVALS_CONFIG_PATH);
+    if (!(await exists(configFilePath))) {
+        return { patch: {}, source: "default" };
+    }
+    let parsed;
+    try {
+        parsed = parse(await fs.readFile(configFilePath, "utf8"));
+    }
+    catch (err) {
+        throw evalConfigError(configFilePath, err instanceof Error ? err.message : String(err));
+    }
+    const patch = validateFileConfig(parsed, configFilePath);
+    return { patch, source: "file" };
+}
+function applyEnvOverrides(base, env) {
+    let overridden = false;
+    const patched = {
+        ...base,
+        regression: { ...base.regression }
+    };
+    for (const name of Object.keys(env)) {
+        if (!name.startsWith("CCLAW_EVAL_"))
+            continue;
+        if (NUMERIC_ENVS.has(name) && typeof env[name] === "string") {
+            // validated below when applied
+        }
+    }
+    const read = (name) => {
+        const value = env[name];
+        return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+    };
+    const baseUrl = read("CCLAW_EVAL_BASE_URL");
+    if (baseUrl) {
+        patched.baseUrl = baseUrl;
+        overridden = true;
+    }
+    const model = read("CCLAW_EVAL_MODEL");
+    if (model) {
+        patched.model = model;
+        overridden = true;
+    }
+    const judgeModel = read("CCLAW_EVAL_JUDGE_MODEL");
+    if (judgeModel) {
+        patched.judgeModel = judgeModel;
+        overridden = true;
+    }
+    const provider = read("CCLAW_EVAL_PROVIDER");
+    if (provider) {
+        patched.provider = provider;
+        overridden = true;
+    }
+    const tier = read("CCLAW_EVAL_TIER");
+    if (tier) {
+        patched.defaultTier = parseTierEnv(tier);
+        overridden = true;
+    }
+    const cap = read("CCLAW_EVAL_DAILY_USD_CAP");
+    if (cap) {
+        patched.dailyUsdCap = parseNumericEnv("CCLAW_EVAL_DAILY_USD_CAP", cap);
+        overridden = true;
+    }
+    const timeout = read("CCLAW_EVAL_TIMEOUT_MS");
+    if (timeout) {
+        patched.timeoutMs = parseNumericEnv("CCLAW_EVAL_TIMEOUT_MS", timeout);
+        overridden = true;
+    }
+    const retries = read("CCLAW_EVAL_MAX_RETRIES");
+    if (retries) {
+        patched.maxRetries = parseNumericEnv("CCLAW_EVAL_MAX_RETRIES", retries);
+        overridden = true;
+    }
+    const apiKey = read("CCLAW_EVAL_API_KEY");
+    return { patched, overridden, apiKey };
+}
+/**
+ * Resolve eval config in layered order: defaults -> config.yaml -> env vars.
+ * Returns a fully-populated config plus a provenance marker so `--dry-run` can
+ * surface where each setting came from.
+ */
+export async function loadEvalConfig(projectRoot, env = process.env) {
+    const { patch, source: fileSource } = await readFileConfig(projectRoot);
+    const merged = {
+        ...DEFAULT_EVAL_CONFIG,
+        ...patch,
+        regression: {
+            ...DEFAULT_EVAL_CONFIG.regression,
+            ...(patch.regression ?? {})
+        }
+    };
+    const { patched, overridden, apiKey } = applyEnvOverrides(merged, env);
+    let source = "default";
+    if (fileSource === "file" && overridden)
+        source = "file+env";
+    else if (fileSource === "file")
+        source = "file";
+    else if (overridden)
+        source = "env";
+    return {
+        ...patched,
+        apiKey,
+        source
+    };
+}

package/dist/eval/corpus.d.ts

@@ -0,0 +1,8 @@
+import type { FlowStage } from "../types.js";
+import type { EvalCase } from "./types.js";
+/**
+ * Load all eval cases under `.cclaw/evals/corpus/**`. Optionally restrict to a
+ * single stage. Returns an empty array for a fresh install (Wave 7.0 ships
+ * without seed cases; corpus is authored in Wave 7.1+).
+ */
+export declare function loadCorpus(projectRoot: string, stage?: FlowStage): Promise<EvalCase[]>;

package/dist/eval/corpus.js

@@ -0,0 +1,91 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { parse } from "yaml";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+import { FLOW_STAGES } from "../types.js";
+const FLOW_STAGE_SET = new Set(FLOW_STAGES);
+function corpusError(filePath, reason) {
+    return new Error(`Invalid eval case at ${filePath}: ${reason}\n` +
+        `Supported stages: ${FLOW_STAGES.join(", ")}`);
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function validateCase(filePath, raw) {
+    if (!isRecord(raw)) {
+        throw corpusError(filePath, "top-level value must be a mapping");
+    }
+    const id = raw.id;
+    if (typeof id !== "string" || id.trim().length === 0) {
+        throw corpusError(filePath, `"id" must be a non-empty string`);
+    }
+    const stageRaw = raw.stage;
+    if (typeof stageRaw !== "string" || !FLOW_STAGE_SET.has(stageRaw)) {
+        throw corpusError(filePath, `"stage" must be one of: ${FLOW_STAGES.join(", ")}`);
+    }
+    const inputPrompt = raw.input_prompt ?? raw.inputPrompt;
+    if (typeof inputPrompt !== "string" || inputPrompt.trim().length === 0) {
+        throw corpusError(filePath, `"input_prompt" must be a non-empty string`);
+    }
+    const contextFilesRaw = raw.context_files ?? raw.contextFiles;
+    let contextFiles;
+    if (contextFilesRaw !== undefined) {
+        if (!Array.isArray(contextFilesRaw) || contextFilesRaw.some((f) => typeof f !== "string")) {
+            throw corpusError(filePath, `"context_files" must be an array of strings`);
+        }
+        contextFiles = contextFilesRaw;
+    }
+    const expected = raw.expected !== undefined && isRecord(raw.expected)
+        ? raw.expected
+        : undefined;
+    const fixture = typeof raw.fixture === "string" ? raw.fixture : undefined;
+    return {
+        id: id.trim(),
+        stage: stageRaw,
+        inputPrompt: inputPrompt.trim(),
+        contextFiles,
+        expected,
+        fixture
+    };
+}
+/**
+ * Load all eval cases under `.cclaw/evals/corpus/**`. Optionally restrict to a
+ * single stage. Returns an empty array for a fresh install (Wave 7.0 ships
+ * without seed cases; corpus is authored in Wave 7.1+).
+ */
+export async function loadCorpus(projectRoot, stage) {
+    const corpusRoot = path.join(projectRoot, EVALS_ROOT, "corpus");
+    if (!(await exists(corpusRoot))) {
+        return [];
+    }
+    const cases = [];
+    const stageDirs = stage
+        ? [path.join(corpusRoot, stage)]
+        : (await fs.readdir(corpusRoot, { withFileTypes: true }))
+            .filter((entry) => entry.isDirectory())
+            .filter((entry) => FLOW_STAGE_SET.has(entry.name))
+            .map((entry) => path.join(corpusRoot, entry.name));
+    for (const stageDir of stageDirs) {
+        if (!(await exists(stageDir)))
+            continue;
+        const entries = await fs.readdir(stageDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isFile())
+                continue;
+            if (!entry.name.endsWith(".yaml") && !entry.name.endsWith(".yml"))
+                continue;
+            const filePath = path.join(stageDir, entry.name);
+            let parsed;
+            try {
+                parsed = parse(await fs.readFile(filePath, "utf8"));
+            }
+            catch (err) {
+                throw corpusError(filePath, err instanceof Error ? err.message : String(err));
+            }
+            cases.push(validateCase(filePath, parsed));
+        }
+    }
+    cases.sort((a, b) => a.stage.localeCompare(b.stage) || a.id.localeCompare(b.id));
+    return cases;
+}

package/dist/eval/llm-client.d.ts

@@ -0,0 +1,62 @@
+/**
+ * LLM client skeleton for the cclaw eval subsystem.
+ *
+ * Wave 7.0 declares the shape of the client without pulling in the `openai`
+ * runtime dependency. The real implementation is wired in Wave 7.3 when
+ * single-shot (Tier A) evals and LLM judging come online. Keeping this stub
+ * separate means users of Waves 7.0–7.2 (structural + rule-based verifiers)
+ * never install an extra dependency or receive network egress warnings.
+ */
+import type { ResolvedEvalConfig } from "./types.js";
+/**
+ * Minimal chat interface the rest of the eval code will depend on. It is
+ * intentionally a subset of OpenAI's Chat Completions surface so that the
+ * Wave 7.3 implementation is a thin adapter around `OpenAI.chat.completions.create`.
+ */
+export interface ChatMessage {
+    role: "system" | "user" | "assistant" | "tool";
+    content: string;
+    name?: string;
+    toolCallId?: string;
+}
+export interface ChatRequest {
+    model: string;
+    messages: ChatMessage[];
+    maxTokens?: number;
+    temperature?: number;
+    timeoutMs?: number;
+    /**
+     * Tool/function-calling definitions in OpenAI wire format. Populated only by
+     * Wave 7.4 (Tier B). Ignored by the Wave 7.3 single-shot path.
+     */
+    tools?: unknown[];
+    toolChoice?: "auto" | "none";
+}
+export interface ChatUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+}
+export interface ChatResponse {
+    content: string;
+    toolCalls?: Array<{
+        id: string;
+        name: string;
+        arguments: string;
+    }>;
+    usage: ChatUsage;
+    finishReason: "stop" | "length" | "tool_calls" | "content_filter";
+}
+/** Lightweight client abstraction shared across eval runners. */
+export interface EvalLlmClient {
+    chat(request: ChatRequest): Promise<ChatResponse>;
+}
+export declare class EvalLlmNotWiredError extends Error {
+    constructor(wave: string);
+}
+/**
+ * Factory stub. Throws with a clear message so accidental Wave 7.0 usage is
+ * easy to diagnose. The Wave 7.3 implementation will replace this body with
+ * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ */
+export declare function createEvalClient(_config: ResolvedEvalConfig): EvalLlmClient;

package/dist/eval/llm-client.js

@@ -0,0 +1,19 @@
+export class EvalLlmNotWiredError extends Error {
+    constructor(wave) {
+        super(`LLM client is not wired in Wave 7.0. It arrives in Wave ${wave}.\n` +
+            `Run \`cclaw eval --dry-run\` or \`cclaw eval --schema-only\` for offline evals.`);
+        this.name = "EvalLlmNotWiredError";
+    }
+}
+/**
+ * Factory stub. Throws with a clear message so accidental Wave 7.0 usage is
+ * easy to diagnose. The Wave 7.3 implementation will replace this body with
+ * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ */
+export function createEvalClient(_config) {
+    return {
+        async chat() {
+            throw new EvalLlmNotWiredError("7.3");
+        }
+    };
+}

package/dist/eval/report.d.ts

@@ -0,0 +1,11 @@
+import type { EvalReport } from "./types.js";
+export declare function reportsDir(projectRoot: string): string;
+export declare function defaultReportBasename(report: EvalReport): string;
+/**
+ * Format a report as a human-readable Markdown document. Keeping the layout
+ * stable matters: CI posts diffs against earlier reports, and unit tests use
+ * the output as a regression guard.
+ */
+export declare function formatMarkdownReport(report: EvalReport): string;
+export declare function writeJsonReport(projectRoot: string, report: EvalReport, basename?: string): Promise<string>;
+export declare function writeMarkdownReport(projectRoot: string, report: EvalReport, basename?: string): Promise<string>;

package/dist/eval/report.js

@@ -0,0 +1,88 @@
+import path from "node:path";
+import { EVALS_ROOT } from "../constants.js";
+import { writeFileSafe } from "../fs-utils.js";
+export function reportsDir(projectRoot) {
+    return path.join(projectRoot, EVALS_ROOT, "reports");
+}
+export function defaultReportBasename(report) {
+    const ts = report.generatedAt.replace(/[:.]/g, "-");
+    return `eval-${ts}-${report.runId.slice(0, 8)}`;
+}
+/**
+ * Format a report as a human-readable Markdown document. Keeping the layout
+ * stable matters: CI posts diffs against earlier reports, and unit tests use
+ * the output as a regression guard.
+ */
+export function formatMarkdownReport(report) {
+    const { summary } = report;
+    const stages = report.stages.length > 0 ? report.stages.join(", ") : "all";
+    const lines = [];
+    lines.push(`# cclaw eval report`);
+    lines.push(``);
+    lines.push(`- generated: ${report.generatedAt}`);
+    lines.push(`- runId: ${report.runId}`);
+    lines.push(`- cclaw version: ${report.cclawVersion}`);
+    lines.push(`- provider: ${report.provider}`);
+    lines.push(`- model: ${report.model}`);
+    lines.push(`- tier: ${report.tier}`);
+    lines.push(`- stages: ${stages}`);
+    lines.push(``);
+    lines.push(`## Summary`);
+    lines.push(``);
+    lines.push(`| metric | value |`);
+    lines.push(`| --- | --- |`);
+    lines.push(`| total cases | ${summary.totalCases} |`);
+    lines.push(`| passed | ${summary.passed} |`);
+    lines.push(`| failed | ${summary.failed} |`);
+    lines.push(`| skipped | ${summary.skipped} |`);
+    lines.push(`| total cost (USD) | ${summary.totalCostUsd.toFixed(4)} |`);
+    lines.push(`| total duration (ms) | ${summary.totalDurationMs} |`);
+    lines.push(``);
+    if (report.baselineDelta) {
+        lines.push(`## Baseline delta`);
+        lines.push(``);
+        lines.push(`- baseline: ${report.baselineDelta.baselineId}`);
+        lines.push(`- score delta: ${report.baselineDelta.scoreDelta.toFixed(4)}`);
+        lines.push(`- critical failures: ${report.baselineDelta.criticalFailures}`);
+        lines.push(``);
+    }
+    if (report.cases.length === 0) {
+        lines.push(`## Cases`);
+        lines.push(``);
+        lines.push(`No cases were executed. See \`docs/evals.md\` for the Wave rollout plan.`);
+        lines.push(``);
+        return `${lines.join("\n")}\n`;
+    }
+    lines.push(`## Cases`);
+    lines.push(``);
+    lines.push(`| stage | case id | passed | duration (ms) | cost (USD) |`);
+    lines.push(`| --- | --- | --- | --- | --- |`);
+    for (const item of report.cases) {
+        const cost = item.costUsd !== undefined ? item.costUsd.toFixed(4) : "-";
+        lines.push(`| ${item.stage} | ${item.caseId} | ${item.passed ? "yes" : "no"} | ${item.durationMs} | ${cost} |`);
+    }
+    lines.push(``);
+    lines.push(`## Verifier details`);
+    lines.push(``);
+    for (const item of report.cases) {
+        lines.push(`### ${item.stage} / ${item.caseId}`);
+        lines.push(``);
+        for (const verifier of item.verifierResults) {
+            const score = verifier.score !== undefined ? ` (score=${verifier.score.toFixed(2)})` : "";
+            lines.push(`- ${verifier.kind} / ${verifier.id}: ${verifier.ok ? "ok" : "fail"}${score}` +
+                (verifier.message ? ` — ${verifier.message}` : ""));
+        }
+        lines.push(``);
+    }
+    return `${lines.join("\n")}\n`;
+}
+export async function writeJsonReport(projectRoot, report, basename = defaultReportBasename(report)) {
+    const outPath = path.join(reportsDir(projectRoot), `${basename}.json`);
+    await writeFileSafe(outPath, `${JSON.stringify(report, null, 2)}\n`);
+    return outPath;
+}
+export async function writeMarkdownReport(projectRoot, report, basename = defaultReportBasename(report)) {
+    const outPath = path.join(reportsDir(projectRoot), `${basename}.md`);
+    await writeFileSafe(outPath, formatMarkdownReport(report));
+    return outPath;
+}

package/dist/eval/runner.d.ts

@@ -0,0 +1,53 @@
+import type { FlowStage } from "../types.js";
+import type { EvalReport, EvalTier, ResolvedEvalConfig } from "./types.js";
+export interface RunEvalOptions {
+    projectRoot: string;
+    stage?: FlowStage;
+    tier?: EvalTier;
+    /** When true, run only structural verifiers. Wave 7.1 wires actual verifiers. */
+    schemaOnly?: boolean;
+    /** When true, run structural + rule-based verifiers. Wave 7.2 wires rules. */
+    rules?: boolean;
+    /** When true, also run LLM judge verifiers. Wave 7.3 wires judging. */
+    judge?: boolean;
+    /** When true, load config + corpus and return a summary without running any verifier. */
+    dryRun?: boolean;
+    /** Override process.env during tests. */
+    env?: NodeJS.ProcessEnv;
+}
+export interface DryRunSummary {
+    kind: "dry-run";
+    config: ResolvedEvalConfig;
+    corpus: {
+        total: number;
+        byStage: Record<string, number>;
+        cases: Array<{
+            id: string;
+            stage: FlowStage;
+        }>;
+    };
+    plannedTier: EvalTier;
+    /**
+     * Waves 7.1–7.3 progressively flip these to `true`. Wave 7.0 is `false`
+     * across the board because no verifier is implemented yet.
+     */
+    verifiersAvailable: {
+        structural: boolean;
+        rules: boolean;
+        judge: boolean;
+        workflow: boolean;
+    };
+    notes: string[];
+}
+/**
+ * Wave 7.0 runner. Responsibilities:
+ * - Load resolved config (defaults + file + env).
+ * - Load corpus (empty on a fresh install).
+ * - Validate that no verifier flag asks for a capability that does not exist yet.
+ * - Return either a dry-run summary or an empty report.
+ *
+ * Waves 7.1+ will replace the "no verifiers available" branch with the real
+ * verifier dispatch pipeline. The signature stays stable so CLI wiring does
+ * not churn.
+ */
+export declare function runEval(options: RunEvalOptions): Promise<DryRunSummary | EvalReport>;

package/dist/eval/runner.js

@@ -0,0 +1,96 @@
+import { randomUUID } from "node:crypto";
+import { CCLAW_VERSION } from "../constants.js";
+import { loadCorpus } from "./corpus.js";
+import { loadEvalConfig } from "./config-loader.js";
+function groupByStage(cases) {
+    return cases.reduce((acc, item) => {
+        acc[item.stage] = (acc[item.stage] ?? 0) + 1;
+        return acc;
+    }, {});
+}
+/**
+ * Wave 7.0 runner. Responsibilities:
+ * - Load resolved config (defaults + file + env).
+ * - Load corpus (empty on a fresh install).
+ * - Validate that no verifier flag asks for a capability that does not exist yet.
+ * - Return either a dry-run summary or an empty report.
+ *
+ * Waves 7.1+ will replace the "no verifiers available" branch with the real
+ * verifier dispatch pipeline. The signature stays stable so CLI wiring does
+ * not churn.
+ */
+export async function runEval(options) {
+    const config = await loadEvalConfig(options.projectRoot, options.env ?? process.env);
+    const corpus = await loadCorpus(options.projectRoot, options.stage);
+    const plannedTier = options.tier ?? config.defaultTier;
+    const notes = [];
+    if (corpus.length === 0) {
+        notes.push("Corpus is empty. Seed cases land in Wave 7.1 (`.cclaw/evals/corpus/<stage>/*.yaml`).");
+    }
+    if (options.schemaOnly) {
+        notes.push("--schema-only is accepted; structural verifiers wire up in Wave 7.1.");
+    }
+    if (options.rules) {
+        notes.push("--rules is accepted; rule verifiers wire up in Wave 7.2.");
+    }
+    if (options.judge) {
+        notes.push("--judge is accepted; LLM judging wires up in Wave 7.3.");
+    }
+    if (options.dryRun === true) {
+        const summary = {
+            kind: "dry-run",
+            config,
+            corpus: {
+                total: corpus.length,
+                byStage: groupByStage(corpus),
+                cases: corpus.map((item) => ({ id: item.id, stage: item.stage }))
+            },
+            plannedTier,
+            verifiersAvailable: {
+                structural: false,
+                rules: false,
+                judge: false,
+                workflow: false
+            },
+            notes
+        };
+        return summary;
+    }
+    const now = new Date().toISOString();
+    const caseResults = corpus.map((item) => ({
+        caseId: item.id,
+        stage: item.stage,
+        tier: plannedTier,
+        passed: false,
+        durationMs: 0,
+        verifierResults: [
+            {
+                kind: "structural",
+                id: "wave-7-0-skeleton",
+                ok: false,
+                message: "Verifiers are not implemented in Wave 7.0; run with --dry-run.",
+                details: { skipped: true }
+            }
+        ]
+    }));
+    const report = {
+        schemaVersion: 1,
+        generatedAt: now,
+        runId: randomUUID(),
+        cclawVersion: CCLAW_VERSION,
+        provider: config.provider,
+        model: config.model,
+        tier: plannedTier,
+        stages: options.stage ? [options.stage] : [],
+        cases: caseResults,
+        summary: {
+            totalCases: caseResults.length,
+            passed: 0,
+            failed: 0,
+            skipped: caseResults.length,
+            totalCostUsd: 0,
+            totalDurationMs: 0
+        }
+    };
+    return report;
+}

package/dist/eval/types.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Core types for the cclaw eval subsystem (Phase 7).
+ *
+ * The eval subsystem lets us measure whether a change to a prompt, skill, or
+ * stage contract improves or regresses the quality of agent output. It is
+ * deliberately decoupled from the main cclaw runtime so that:
+ *
+ * - Users who never run `cclaw eval` pay zero runtime cost.
+ * - The verifier / rubric / LLM stack evolves on its own release cadence (Waves 7.0-7.6).
+ * - Any OpenAI-compatible endpoint can be swapped in via config (z.ai, OpenAI, vLLM, etc.).
+ */
+import type { FlowStage } from "../types.js";
+/**
+ * Fidelity tier for the agent-under-test.
+ *
+ * - `A` — single-shot API call, no tools. Cheap, validates core prompt behavior.
+ * - `B` — SDK loop with function-calling for Read/Write/Glob/Grep inside a sandbox.
+ * - `C` — multi-stage workflow run (brainstorm -> scope -> ... -> plan) with threaded
+ *   artifacts. Most realistic tier we ship in Phase 7; literal IDE-harness runs
+ *   (claude-code / cursor-agent proxied to OpenAI-compat) are deferred to Phase 8.
+ */
+export declare const EVAL_TIERS: readonly ["A", "B", "C"];
+export type EvalTier = (typeof EVAL_TIERS)[number];
+/**
+ * Verifier kinds, in increasing cost and decreasing determinism:
+ * structural and rules run without LLM; judge and workflow use the configured model.
+ */
+export declare const VERIFIER_KINDS: readonly ["structural", "rules", "judge", "workflow"];
+export type VerifierKind = (typeof VERIFIER_KINDS)[number];
+/**
+ * A single eval case describes one input scenario for one stage. Cases live in
+ * `.cclaw/evals/corpus/<stage>/<id>.yaml` and may reference a pre-generated
+ * fixture artifact for verifier development (Wave 7.1) before the agent loop
+ * exists (Wave 7.3+).
+ */
+export interface EvalCase {
+    id: string;
+    stage: FlowStage;
+    inputPrompt: string;
+    /** Project files copied into the Tier B/C sandbox before the agent runs. */
+    contextFiles?: string[];
+    /**
+     * Optional expected-shape hints consumed by structural/rule verifiers.
+     * Left intentionally loose; verifiers in Waves 7.1–7.2 will narrow this.
+     */
+    expected?: Record<string, unknown>;
+    /**
+     * Path (relative to the corpus case file) of a pre-generated artifact used
+     * when verifiers are exercised without a live agent loop. Primarily a Wave
+     * 7.1 development aid.
+     */
+    fixture?: string;
+}
+/** Result of one verifier applied to one case. */
+export interface VerifierResult {
+    kind: VerifierKind;
+    id: string;
+    ok: boolean;
+    /** Normalized 0..1 score when the verifier produces a numeric signal. */
+    score?: number;
+    message?: string;
+    details?: Record<string, unknown>;
+}
+/** Aggregate result for one case after all verifiers run. */
+export interface EvalCaseResult {
+    caseId: string;
+    stage: FlowStage;
+    tier: EvalTier;
+    passed: boolean;
+    durationMs: number;
+    costUsd?: number;
+    verifierResults: VerifierResult[];
+}
+/** Top-level eval report, serialized to JSON and rendered to Markdown. */
+export interface EvalReport {
+    schemaVersion: 1;
+    generatedAt: string;
+    runId: string;
+    cclawVersion: string;
+    provider: string;
+    model: string;
+    tier: EvalTier;
+    stages: FlowStage[];
+    cases: EvalCaseResult[];
+    summary: {
+        totalCases: number;
+        passed: number;
+        failed: number;
+        skipped: number;
+        totalCostUsd: number;
+        totalDurationMs: number;
+    };
+    /** Present when comparing against a saved baseline (Wave 7.1+). */
+    baselineDelta?: {
+        baselineId: string;
+        scoreDelta: number;
+        criticalFailures: number;
+    };
+}
+/**
+ * Eval configuration, persisted to `.cclaw/evals/config.yaml` and mergeable
+ * with `CCLAW_EVAL_*` environment variables at runtime.
+ */
+export interface EvalConfig {
+    /**
+     * Free-form provider name used in reports. The actual HTTP protocol is
+     * determined by `baseUrl`, which is expected to be OpenAI-compatible.
+     */
+    provider: string;
+    /** OpenAI-compatible base URL, e.g. `https://api.z.ai/api/coding/paas/v4`. */
+    baseUrl: string;
+    /** Model identifier for both agent-under-test and judge unless `judgeModel` overrides. */
+    model: string;
+    /** Optional separate model for the judge role. Defaults to `model`. */
+    judgeModel?: string;
+    /** Default tier when `--tier` is not supplied. */
+    defaultTier: EvalTier;
+    /** Optional hard stop on estimated USD spend per day. Unset = no cap. */
+    dailyUsdCap?: number;
+    /** Regression thresholds for CI gates. */
+    regression: {
+        /** Fail when overall score drops by more than this fraction (e.g. 0.15 = 15%). */
+        failIfDeltaBelow: number;
+        /** Fail when any single critical rubric drops below this absolute score. */
+        failIfCriticalBelow: number;
+    };
+    /** Per-agent-run timeout in milliseconds. */
+    timeoutMs: number;
+    /** Max retries per API call on transient failures. */
+    maxRetries: number;
+}
+/** Resolved config with env overrides applied. */
+export interface ResolvedEvalConfig extends EvalConfig {
+    apiKey?: string;
+    source: "default" | "file" | "env" | "file+env";
+}

package/dist/eval/types.js

@@ -0,0 +1,15 @@
+/**
+ * Fidelity tier for the agent-under-test.
+ *
+ * - `A` — single-shot API call, no tools. Cheap, validates core prompt behavior.
+ * - `B` — SDK loop with function-calling for Read/Write/Glob/Grep inside a sandbox.
+ * - `C` — multi-stage workflow run (brainstorm -> scope -> ... -> plan) with threaded
+ *   artifacts. Most realistic tier we ship in Phase 7; literal IDE-harness runs
+ *   (claude-code / cursor-agent proxied to OpenAI-compat) are deferred to Phase 8.
+ */
+export const EVAL_TIERS = ["A", "B", "C"];
+/**
+ * Verifier kinds, in increasing cost and decreasing determinism:
+ * structural and rules run without LLM; judge and workflow use the configured model.
+ */
+export const VERIFIER_KINDS = ["structural", "rules", "judge", "workflow"];

package/dist/install.js

@@ -28,6 +28,7 @@ import { contextMonitorScript, promptGuardScript, workflowGuardScript } from "./
 import { META_SKILL_NAME, usingCclawSkillMarkdown } from "./content/meta-skill.js";
 import { decisionProtocolMarkdown, completionProtocolMarkdown, ethosProtocolMarkdown } from "./content/protocols.js";
 import { ARTIFACT_TEMPLATES, CURSOR_WORKFLOW_RULE_MDC, RULEBOOK_MARKDOWN, buildRulesJson } from "./content/templates.js";
+import { EVAL_BASELINES_README, EVAL_CONFIG_YAML, EVAL_CORPUS_README, EVAL_REPORTS_README, EVAL_RUBRICS_README } from "./content/eval-scaffold.js";
 import { TDD_WAVE_WALKTHROUGH_MARKDOWN, stageSkillFolder, stageSkillMarkdown } from "./content/skills.js";
 import { stageCommonGuidanceMarkdown } from "./content/stage-common-guidance.js";
 import { STAGE_EXAMPLES_REFERENCE_DIR, stageExamplesReferenceMarkdown } from "./content/examples.js";
@@ -184,6 +185,26 @@ async function writeArtifactTemplates(projectRoot) {
         await writeFileSafe(runtimePath(projectRoot, "templates", fileName), content);
     }
 }
+/**
+ * Seed the `.cclaw/evals/` scaffold. Only writes files that do not already
+ * exist so that user-authored config.yaml / corpus / rubrics / baselines are
+ * never clobbered by `cclaw sync`.
+ */
+async function writeEvalScaffold(projectRoot) {
+    const targets = [
+        { rel: "evals/config.yaml", content: EVAL_CONFIG_YAML },
+        { rel: "evals/corpus/README.md", content: EVAL_CORPUS_README },
+        { rel: "evals/rubrics/README.md", content: EVAL_RUBRICS_README },
+        { rel: "evals/baselines/README.md", content: EVAL_BASELINES_README },
+        { rel: "evals/reports/README.md", content: EVAL_REPORTS_README }
+    ];
+    for (const target of targets) {
+        const absolute = runtimePath(projectRoot, ...target.rel.split("/"));
+        if (await exists(absolute))
+            continue;
+        await writeFileSafe(absolute, target.content);
+    }
+}
 async function writeSkills(projectRoot, config) {
     for (const stage of COMMAND_FILE_ORDER) {
         const folder = stageSkillFolder(stage);
@@ -1044,6 +1065,7 @@ async function materializeRuntime(projectRoot, config, forceStateReset) {
     await writeSkills(projectRoot, config);
     await writeContextModes(projectRoot);
     await writeArtifactTemplates(projectRoot);
+    await writeEvalScaffold(projectRoot);
     await writeRulebook(projectRoot);
     await writeState(projectRoot, config, forceStateReset);
     await ensureRunSystem(projectRoot, { createIfMissing: false });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cclaw-cli",
-  "version": "0.21.2",
+  "version": "0.22.0",
   "description": "Installer-first flow toolkit for coding agents",
   "type": "module",
   "bin": {