cclaw-cli 0.21.2 → 0.23.0

package/dist/eval/config-loader.js

@@ -0,0 +1,237 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { parse } from "yaml";
+import { EVALS_CONFIG_PATH } from "../constants.js";
+import { exists } from "../fs-utils.js";
+import { EVAL_TIERS } from "./types.js";
+/**
+ * Default eval config. Optimized for the z.ai OpenAI-compatible coding endpoint
+ * with GLM 5.1 per the roadmap locked decisions (D-EVAL-01..05). Any field can
+ * be overridden by `.cclaw/evals/config.yaml` and then by `CCLAW_EVAL_*` env
+ * variables (env wins last).
+ */
+export const DEFAULT_EVAL_CONFIG = {
+    provider: "zai",
+    baseUrl: "https://api.z.ai/api/coding/paas/v4",
+    model: "glm-5.1",
+    defaultTier: "A",
+    regression: {
+        failIfDeltaBelow: -0.15,
+        failIfCriticalBelow: 3.0
+    },
+    timeoutMs: 120_000,
+    maxRetries: 2
+};
+const EVAL_TIER_SET = new Set(EVAL_TIERS);
+const NUMERIC_ENVS = new Set([
+    "CCLAW_EVAL_DAILY_USD_CAP",
+    "CCLAW_EVAL_TIMEOUT_MS",
+    "CCLAW_EVAL_MAX_RETRIES"
+]);
+function evalConfigError(configFilePath, reason) {
+    return new Error(`Invalid cclaw eval config at ${configFilePath}: ${reason}\n` +
+        `Supported tiers: ${EVAL_TIERS.join(", ")}\n` +
+        `See docs/evals.md for the full schema. After fixing, run: cclaw eval --dry-run`);
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function parseNumericEnv(name, raw) {
+    const value = Number(raw);
+    if (!Number.isFinite(value)) {
+        throw new Error(`Environment variable ${name} must be numeric, got: ${raw}`);
+    }
+    return value;
+}
+function parseTierEnv(raw) {
+    const trimmed = raw.trim().toUpperCase();
+    if (!EVAL_TIER_SET.has(trimmed)) {
+        throw new Error(`Environment variable CCLAW_EVAL_TIER must be one of ${EVAL_TIERS.join("/")}, got: ${raw}`);
+    }
+    return trimmed;
+}
+function validateFileConfig(raw, configFilePath) {
+    if (raw === undefined || raw === null)
+        return {};
+    if (!isRecord(raw)) {
+        throw evalConfigError(configFilePath, "top-level value must be a mapping");
+    }
+    const out = {};
+    const assignString = (key, value) => {
+        if (value === undefined)
+            return;
+        if (typeof value !== "string" || value.trim().length === 0) {
+            throw evalConfigError(configFilePath, `"${String(key)}" must be a non-empty string`);
+        }
+        out[key] = value.trim();
+    };
+    assignString("provider", raw.provider);
+    assignString("baseUrl", raw.baseUrl);
+    assignString("model", raw.model);
+    assignString("judgeModel", raw.judgeModel);
+    if (raw.defaultTier !== undefined) {
+        if (typeof raw.defaultTier !== "string" || !EVAL_TIER_SET.has(raw.defaultTier)) {
+            throw evalConfigError(configFilePath, `"defaultTier" must be one of: ${EVAL_TIERS.join(", ")}`);
+        }
+        out.defaultTier = raw.defaultTier;
+    }
+    if (raw.dailyUsdCap !== undefined) {
+        if (typeof raw.dailyUsdCap !== "number" || raw.dailyUsdCap < 0) {
+            throw evalConfigError(configFilePath, `"dailyUsdCap" must be a non-negative number`);
+        }
+        out.dailyUsdCap = raw.dailyUsdCap;
+    }
+    if (raw.timeoutMs !== undefined) {
+        if (typeof raw.timeoutMs !== "number" || raw.timeoutMs <= 0) {
+            throw evalConfigError(configFilePath, `"timeoutMs" must be a positive number`);
+        }
+        out.timeoutMs = raw.timeoutMs;
+    }
+    if (raw.maxRetries !== undefined) {
+        if (!Number.isInteger(raw.maxRetries) || raw.maxRetries < 0) {
+            throw evalConfigError(configFilePath, `"maxRetries" must be a non-negative integer`);
+        }
+        out.maxRetries = raw.maxRetries;
+    }
+    if (raw.regression !== undefined) {
+        if (!isRecord(raw.regression)) {
+            throw evalConfigError(configFilePath, `"regression" must be a mapping`);
+        }
+        const failIfDeltaBelow = raw.regression.failIfDeltaBelow;
+        const failIfCriticalBelow = raw.regression.failIfCriticalBelow;
+        if (failIfDeltaBelow !== undefined && typeof failIfDeltaBelow !== "number") {
+            throw evalConfigError(configFilePath, `"regression.failIfDeltaBelow" must be a number`);
+        }
+        if (failIfCriticalBelow !== undefined && typeof failIfCriticalBelow !== "number") {
+            throw evalConfigError(configFilePath, `"regression.failIfCriticalBelow" must be a number`);
+        }
+        out.regression = {
+            failIfDeltaBelow: typeof failIfDeltaBelow === "number"
+                ? failIfDeltaBelow
+                : DEFAULT_EVAL_CONFIG.regression.failIfDeltaBelow,
+            failIfCriticalBelow: typeof failIfCriticalBelow === "number"
+                ? failIfCriticalBelow
+                : DEFAULT_EVAL_CONFIG.regression.failIfCriticalBelow
+        };
+    }
+    const knownKeys = new Set([
+        "provider",
+        "baseUrl",
+        "model",
+        "judgeModel",
+        "defaultTier",
+        "dailyUsdCap",
+        "timeoutMs",
+        "maxRetries",
+        "regression"
+    ]);
+    const unknown = Object.keys(raw).filter((key) => !knownKeys.has(key));
+    if (unknown.length > 0) {
+        throw evalConfigError(configFilePath, `unknown top-level key(s): ${unknown.join(", ")}`);
+    }
+    return out;
+}
+async function readFileConfig(projectRoot) {
+    const configFilePath = path.join(projectRoot, EVALS_CONFIG_PATH);
+    if (!(await exists(configFilePath))) {
+        return { patch: {}, source: "default" };
+    }
+    let parsed;
+    try {
+        parsed = parse(await fs.readFile(configFilePath, "utf8"));
+    }
+    catch (err) {
+        throw evalConfigError(configFilePath, err instanceof Error ? err.message : String(err));
+    }
+    const patch = validateFileConfig(parsed, configFilePath);
+    return { patch, source: "file" };
+}
+function applyEnvOverrides(base, env) {
+    let overridden = false;
+    const patched = {
+        ...base,
+        regression: { ...base.regression }
+    };
+    for (const name of Object.keys(env)) {
+        if (!name.startsWith("CCLAW_EVAL_"))
+            continue;
+        if (NUMERIC_ENVS.has(name) && typeof env[name] === "string") {
+            // validated below when applied
+        }
+    }
+    const read = (name) => {
+        const value = env[name];
+        return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
+    };
+    const baseUrl = read("CCLAW_EVAL_BASE_URL");
+    if (baseUrl) {
+        patched.baseUrl = baseUrl;
+        overridden = true;
+    }
+    const model = read("CCLAW_EVAL_MODEL");
+    if (model) {
+        patched.model = model;
+        overridden = true;
+    }
+    const judgeModel = read("CCLAW_EVAL_JUDGE_MODEL");
+    if (judgeModel) {
+        patched.judgeModel = judgeModel;
+        overridden = true;
+    }
+    const provider = read("CCLAW_EVAL_PROVIDER");
+    if (provider) {
+        patched.provider = provider;
+        overridden = true;
+    }
+    const tier = read("CCLAW_EVAL_TIER");
+    if (tier) {
+        patched.defaultTier = parseTierEnv(tier);
+        overridden = true;
+    }
+    const cap = read("CCLAW_EVAL_DAILY_USD_CAP");
+    if (cap) {
+        patched.dailyUsdCap = parseNumericEnv("CCLAW_EVAL_DAILY_USD_CAP", cap);
+        overridden = true;
+    }
+    const timeout = read("CCLAW_EVAL_TIMEOUT_MS");
+    if (timeout) {
+        patched.timeoutMs = parseNumericEnv("CCLAW_EVAL_TIMEOUT_MS", timeout);
+        overridden = true;
+    }
+    const retries = read("CCLAW_EVAL_MAX_RETRIES");
+    if (retries) {
+        patched.maxRetries = parseNumericEnv("CCLAW_EVAL_MAX_RETRIES", retries);
+        overridden = true;
+    }
+    const apiKey = read("CCLAW_EVAL_API_KEY");
+    return { patched, overridden, apiKey };
+}
+/**
+ * Resolve eval config in layered order: defaults -> config.yaml -> env vars.
+ * Returns a fully-populated config plus a provenance marker so `--dry-run` can
+ * surface where each setting came from.
+ */
+export async function loadEvalConfig(projectRoot, env = process.env) {
+    const { patch, source: fileSource } = await readFileConfig(projectRoot);
+    const merged = {
+        ...DEFAULT_EVAL_CONFIG,
+        ...patch,
+        regression: {
+            ...DEFAULT_EVAL_CONFIG.regression,
+            ...(patch.regression ?? {})
+        }
+    };
+    const { patched, overridden, apiKey } = applyEnvOverrides(merged, env);
+    let source = "default";
+    if (fileSource === "file" && overridden)
+        source = "file+env";
+    else if (fileSource === "file")
+        source = "file";
+    else if (overridden)
+        source = "env";
+    return {
+        ...patched,
+        apiKey,
+        source
+    };
+}

package/dist/eval/corpus.d.ts

@@ -0,0 +1,19 @@
+import type { FlowStage } from "../types.js";
+import type { EvalCase } from "./types.js";
+/**
+ * Load all eval cases under `.cclaw/evals/corpus/**`. Optionally restrict to a
+ * single stage. Returns an empty array for a fresh install.
+ */
+export declare function loadCorpus(projectRoot: string, stage?: FlowStage): Promise<EvalCase[]>;
+/**
+ * Resolve a case's `fixture` path to an absolute filesystem path. The fixture
+ * field is interpreted relative to the case's stage directory (i.e., a
+ * sibling subdirectory or file inside `.cclaw/evals/corpus/<stage>/`).
+ */
+export declare function fixturePathFor(projectRoot: string, caseEntry: EvalCase): string | undefined;
+/**
+ * Read the fixture artifact text for a case. Returns `undefined` if the case
+ * has no fixture reference. Throws a descriptive error if the path exists in
+ * the case but not on disk — Wave 7.1 fixtures ship alongside cases.
+ */
+export declare function readFixtureArtifact(projectRoot: string, caseEntry: EvalCase): Promise<string | undefined>;

package/dist/eval/corpus.js

@@ -0,0 +1,175 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { parse } from "yaml";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+import { FLOW_STAGES } from "../types.js";
+const FLOW_STAGE_SET = new Set(FLOW_STAGES);
+function corpusError(filePath, reason) {
+    return new Error(`Invalid eval case at ${filePath}: ${reason}\n` +
+        `Supported stages: ${FLOW_STAGES.join(", ")}`);
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function readStringArray(filePath, context, value) {
+    if (value === undefined)
+        return undefined;
+    if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+        throw corpusError(filePath, `"${context}" must be an array of strings`);
+    }
+    return value;
+}
+function readNonNegativeInteger(filePath, context, value) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || !Number.isInteger(value)) {
+        throw corpusError(filePath, `"${context}" must be a non-negative integer`);
+    }
+    return value;
+}
+function parseStructural(filePath, raw) {
+    if (raw === undefined)
+        return undefined;
+    if (!isRecord(raw)) {
+        throw corpusError(filePath, `"expected.structural" must be a mapping`);
+    }
+    const requiredSections = readStringArray(filePath, "expected.structural.required_sections", raw.required_sections ?? raw.requiredSections);
+    const forbiddenPatterns = readStringArray(filePath, "expected.structural.forbidden_patterns", raw.forbidden_patterns ?? raw.forbiddenPatterns);
+    const requiredFrontmatterKeys = readStringArray(filePath, "expected.structural.required_frontmatter_keys", raw.required_frontmatter_keys ?? raw.requiredFrontmatterKeys);
+    const minLines = readNonNegativeInteger(filePath, "expected.structural.min_lines", raw.min_lines ?? raw.minLines);
+    const maxLines = readNonNegativeInteger(filePath, "expected.structural.max_lines", raw.max_lines ?? raw.maxLines);
+    const minChars = readNonNegativeInteger(filePath, "expected.structural.min_chars", raw.min_chars ?? raw.minChars);
+    const maxChars = readNonNegativeInteger(filePath, "expected.structural.max_chars", raw.max_chars ?? raw.maxChars);
+    const structural = {};
+    if (requiredSections)
+        structural.requiredSections = requiredSections;
+    if (forbiddenPatterns)
+        structural.forbiddenPatterns = forbiddenPatterns;
+    if (requiredFrontmatterKeys)
+        structural.requiredFrontmatterKeys = requiredFrontmatterKeys;
+    if (minLines !== undefined)
+        structural.minLines = minLines;
+    if (maxLines !== undefined)
+        structural.maxLines = maxLines;
+    if (minChars !== undefined)
+        structural.minChars = minChars;
+    if (maxChars !== undefined)
+        structural.maxChars = maxChars;
+    return structural;
+}
+function parseExpected(filePath, raw) {
+    if (raw === undefined)
+        return undefined;
+    if (!isRecord(raw)) {
+        throw corpusError(filePath, `"expected" must be a mapping`);
+    }
+    const shape = {};
+    const structural = parseStructural(filePath, raw.structural);
+    if (structural)
+        shape.structural = structural;
+    if (raw.rules !== undefined) {
+        if (!isRecord(raw.rules)) {
+            throw corpusError(filePath, `"expected.rules" must be a mapping`);
+        }
+        shape.rules = raw.rules;
+    }
+    if (raw.judge !== undefined) {
+        if (!isRecord(raw.judge)) {
+            throw corpusError(filePath, `"expected.judge" must be a mapping`);
+        }
+        shape.judge = raw.judge;
+    }
+    return Object.keys(shape).length === 0 ? undefined : shape;
+}
+function validateCase(filePath, raw) {
+    if (!isRecord(raw)) {
+        throw corpusError(filePath, "top-level value must be a mapping");
+    }
+    const id = raw.id;
+    if (typeof id !== "string" || id.trim().length === 0) {
+        throw corpusError(filePath, `"id" must be a non-empty string`);
+    }
+    const stageRaw = raw.stage;
+    if (typeof stageRaw !== "string" || !FLOW_STAGE_SET.has(stageRaw)) {
+        throw corpusError(filePath, `"stage" must be one of: ${FLOW_STAGES.join(", ")}`);
+    }
+    const inputPrompt = raw.input_prompt ?? raw.inputPrompt;
+    if (typeof inputPrompt !== "string" || inputPrompt.trim().length === 0) {
+        throw corpusError(filePath, `"input_prompt" must be a non-empty string`);
+    }
+    const contextFiles = readStringArray(filePath, "context_files", raw.context_files ?? raw.contextFiles);
+    const expected = parseExpected(filePath, raw.expected);
+    const fixture = typeof raw.fixture === "string" ? raw.fixture : undefined;
+    return {
+        id: id.trim(),
+        stage: stageRaw,
+        inputPrompt: inputPrompt.trim(),
+        contextFiles,
+        expected,
+        fixture
+    };
+}
+/**
+ * Load all eval cases under `.cclaw/evals/corpus/**`. Optionally restrict to a
+ * single stage. Returns an empty array for a fresh install.
+ */
+export async function loadCorpus(projectRoot, stage) {
+    const corpusRoot = path.join(projectRoot, EVALS_ROOT, "corpus");
+    if (!(await exists(corpusRoot))) {
+        return [];
+    }
+    const cases = [];
+    const stageDirs = stage
+        ? [path.join(corpusRoot, stage)]
+        : (await fs.readdir(corpusRoot, { withFileTypes: true }))
+            .filter((entry) => entry.isDirectory())
+            .filter((entry) => FLOW_STAGE_SET.has(entry.name))
+            .map((entry) => path.join(corpusRoot, entry.name));
+    for (const stageDir of stageDirs) {
+        if (!(await exists(stageDir)))
+            continue;
+        const entries = await fs.readdir(stageDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isFile())
+                continue;
+            if (!entry.name.endsWith(".yaml") && !entry.name.endsWith(".yml"))
+                continue;
+            const filePath = path.join(stageDir, entry.name);
+            let parsed;
+            try {
+                parsed = parse(await fs.readFile(filePath, "utf8"));
+            }
+            catch (err) {
+                throw corpusError(filePath, err instanceof Error ? err.message : String(err));
+            }
+            cases.push(validateCase(filePath, parsed));
+        }
+    }
+    cases.sort((a, b) => a.stage.localeCompare(b.stage) || a.id.localeCompare(b.id));
+    return cases;
+}
+/**
+ * Resolve a case's `fixture` path to an absolute filesystem path. The fixture
+ * field is interpreted relative to the case's stage directory (i.e., a
+ * sibling subdirectory or file inside `.cclaw/evals/corpus/<stage>/`).
+ */
+export function fixturePathFor(projectRoot, caseEntry) {
+    if (!caseEntry.fixture)
+        return undefined;
+    return path.resolve(projectRoot, EVALS_ROOT, "corpus", caseEntry.stage, caseEntry.fixture);
+}
+/**
+ * Read the fixture artifact text for a case. Returns `undefined` if the case
+ * has no fixture reference. Throws a descriptive error if the path exists in
+ * the case but not on disk — Wave 7.1 fixtures ship alongside cases.
+ */
+export async function readFixtureArtifact(projectRoot, caseEntry) {
+    const fixturePath = fixturePathFor(projectRoot, caseEntry);
+    if (!fixturePath)
+        return undefined;
+    if (!(await exists(fixturePath))) {
+        throw new Error(`Fixture missing for case ${caseEntry.stage}/${caseEntry.id}: ${fixturePath}`);
+    }
+    return fs.readFile(fixturePath, "utf8");
+}

package/dist/eval/llm-client.d.ts

@@ -0,0 +1,62 @@
+/**
+ * LLM client skeleton for the cclaw eval subsystem.
+ *
+ * Wave 7.0 declares the shape of the client without pulling in the `openai`
+ * runtime dependency. The real implementation is wired in Wave 7.3 when
+ * single-shot (Tier A) evals and LLM judging come online. Keeping this stub
+ * separate means users of Waves 7.0–7.2 (structural + rule-based verifiers)
+ * never install an extra dependency or receive network egress warnings.
+ */
+import type { ResolvedEvalConfig } from "./types.js";
+/**
+ * Minimal chat interface the rest of the eval code will depend on. It is
+ * intentionally a subset of OpenAI's Chat Completions surface so that the
+ * Wave 7.3 implementation is a thin adapter around `OpenAI.chat.completions.create`.
+ */
+export interface ChatMessage {
+    role: "system" | "user" | "assistant" | "tool";
+    content: string;
+    name?: string;
+    toolCallId?: string;
+}
+export interface ChatRequest {
+    model: string;
+    messages: ChatMessage[];
+    maxTokens?: number;
+    temperature?: number;
+    timeoutMs?: number;
+    /**
+     * Tool/function-calling definitions in OpenAI wire format. Populated only by
+     * Wave 7.4 (Tier B). Ignored by the Wave 7.3 single-shot path.
+     */
+    tools?: unknown[];
+    toolChoice?: "auto" | "none";
+}
+export interface ChatUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+}
+export interface ChatResponse {
+    content: string;
+    toolCalls?: Array<{
+        id: string;
+        name: string;
+        arguments: string;
+    }>;
+    usage: ChatUsage;
+    finishReason: "stop" | "length" | "tool_calls" | "content_filter";
+}
+/** Lightweight client abstraction shared across eval runners. */
+export interface EvalLlmClient {
+    chat(request: ChatRequest): Promise<ChatResponse>;
+}
+export declare class EvalLlmNotWiredError extends Error {
+    constructor(wave: string);
+}
+/**
+ * Factory stub. Throws with a clear message so accidental Wave 7.0 usage is
+ * easy to diagnose. The Wave 7.3 implementation will replace this body with
+ * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ */
+export declare function createEvalClient(_config: ResolvedEvalConfig): EvalLlmClient;

package/dist/eval/llm-client.js

@@ -0,0 +1,19 @@
+export class EvalLlmNotWiredError extends Error {
+    constructor(wave) {
+        super(`LLM client is not wired in Wave 7.0. It arrives in Wave ${wave}.\n` +
+            `Run \`cclaw eval --dry-run\` or \`cclaw eval --schema-only\` for offline evals.`);
+        this.name = "EvalLlmNotWiredError";
+    }
+}
+/**
+ * Factory stub. Throws with a clear message so accidental Wave 7.0 usage is
+ * easy to diagnose. The Wave 7.3 implementation will replace this body with
+ * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ */
+export function createEvalClient(_config) {
+    return {
+        async chat() {
+            throw new EvalLlmNotWiredError("7.3");
+        }
+    };
+}

package/dist/eval/report.d.ts

@@ -0,0 +1,11 @@
+import type { EvalReport } from "./types.js";
+export declare function reportsDir(projectRoot: string): string;
+export declare function defaultReportBasename(report: EvalReport): string;
+/**
+ * Format a report as a human-readable Markdown document. Keeping the layout
+ * stable matters: CI posts diffs against earlier reports, and unit tests use
+ * the output as a regression guard.
+ */
+export declare function formatMarkdownReport(report: EvalReport): string;
+export declare function writeJsonReport(projectRoot: string, report: EvalReport, basename?: string): Promise<string>;
+export declare function writeMarkdownReport(projectRoot: string, report: EvalReport, basename?: string): Promise<string>;

package/dist/eval/report.js

@@ -0,0 +1,101 @@
+import path from "node:path";
+import { EVALS_ROOT } from "../constants.js";
+import { writeFileSafe } from "../fs-utils.js";
+export function reportsDir(projectRoot) {
+    return path.join(projectRoot, EVALS_ROOT, "reports");
+}
+export function defaultReportBasename(report) {
+    const ts = report.generatedAt.replace(/[:.]/g, "-");
+    return `eval-${ts}-${report.runId.slice(0, 8)}`;
+}
+/**
+ * Format a report as a human-readable Markdown document. Keeping the layout
+ * stable matters: CI posts diffs against earlier reports, and unit tests use
+ * the output as a regression guard.
+ */
+export function formatMarkdownReport(report) {
+    const { summary } = report;
+    const stages = report.stages.length > 0 ? report.stages.join(", ") : "all";
+    const lines = [];
+    lines.push(`# cclaw eval report`);
+    lines.push(``);
+    lines.push(`- generated: ${report.generatedAt}`);
+    lines.push(`- runId: ${report.runId}`);
+    lines.push(`- cclaw version: ${report.cclawVersion}`);
+    lines.push(`- provider: ${report.provider}`);
+    lines.push(`- model: ${report.model}`);
+    lines.push(`- tier: ${report.tier}`);
+    lines.push(`- stages: ${stages}`);
+    lines.push(``);
+    lines.push(`## Summary`);
+    lines.push(``);
+    lines.push(`| metric | value |`);
+    lines.push(`| --- | --- |`);
+    lines.push(`| total cases | ${summary.totalCases} |`);
+    lines.push(`| passed | ${summary.passed} |`);
+    lines.push(`| failed | ${summary.failed} |`);
+    lines.push(`| skipped | ${summary.skipped} |`);
+    lines.push(`| total cost (USD) | ${summary.totalCostUsd.toFixed(4)} |`);
+    lines.push(`| total duration (ms) | ${summary.totalDurationMs} |`);
+    lines.push(``);
+    if (report.baselineDelta) {
+        const delta = report.baselineDelta;
+        lines.push(`## Baseline delta`);
+        lines.push(``);
+        lines.push(`- baseline: ${delta.baselineId}`);
+        lines.push(`- score delta: ${delta.scoreDelta.toFixed(4)}`);
+        lines.push(`- critical failures: ${delta.criticalFailures}`);
+        lines.push(``);
+        if (delta.regressions.length > 0) {
+            lines.push(`### Regressions`);
+            lines.push(``);
+            lines.push(`| stage | case id | verifier | reason | prev | curr |`);
+            lines.push(`| --- | --- | --- | --- | --- | --- |`);
+            for (const reg of delta.regressions) {
+                const prev = reg.previousScore !== undefined ? reg.previousScore.toFixed(2) : "-";
+                const curr = reg.currentScore !== undefined ? reg.currentScore.toFixed(2) : "-";
+                lines.push(`| ${reg.stage} | ${reg.caseId} | ${reg.verifierId} | ${reg.reason} | ${prev} | ${curr} |`);
+            }
+            lines.push(``);
+        }
+    }
+    if (report.cases.length === 0) {
+        lines.push(`## Cases`);
+        lines.push(``);
+        lines.push(`No cases were executed. See \`docs/evals.md\` for the Wave rollout plan.`);
+        lines.push(``);
+        return `${lines.join("\n")}\n`;
+    }
+    lines.push(`## Cases`);
+    lines.push(``);
+    lines.push(`| stage | case id | passed | duration (ms) | cost (USD) |`);
+    lines.push(`| --- | --- | --- | --- | --- |`);
+    for (const item of report.cases) {
+        const cost = item.costUsd !== undefined ? item.costUsd.toFixed(4) : "-";
+        lines.push(`| ${item.stage} | ${item.caseId} | ${item.passed ? "yes" : "no"} | ${item.durationMs} | ${cost} |`);
+    }
+    lines.push(``);
+    lines.push(`## Verifier details`);
+    lines.push(``);
+    for (const item of report.cases) {
+        lines.push(`### ${item.stage} / ${item.caseId}`);
+        lines.push(``);
+        for (const verifier of item.verifierResults) {
+            const score = verifier.score !== undefined ? ` (score=${verifier.score.toFixed(2)})` : "";
+            lines.push(`- ${verifier.kind} / ${verifier.id}: ${verifier.ok ? "ok" : "fail"}${score}` +
+                (verifier.message ? ` — ${verifier.message}` : ""));
+        }
+        lines.push(``);
+    }
+    return `${lines.join("\n")}\n`;
+}
+export async function writeJsonReport(projectRoot, report, basename = defaultReportBasename(report)) {
+    const outPath = path.join(reportsDir(projectRoot), `${basename}.json`);
+    await writeFileSafe(outPath, `${JSON.stringify(report, null, 2)}\n`);
+    return outPath;
+}
+export async function writeMarkdownReport(projectRoot, report, basename = defaultReportBasename(report)) {
+    const outPath = path.join(reportsDir(projectRoot), `${basename}.md`);
+    await writeFileSafe(outPath, formatMarkdownReport(report));
+    return outPath;
+}

package/dist/eval/runner.d.ts

@@ -0,0 +1,45 @@
+import type { FlowStage } from "../types.js";
+import type { EvalReport, EvalTier, ResolvedEvalConfig } from "./types.js";
+export interface RunEvalOptions {
+    projectRoot: string;
+    stage?: FlowStage;
+    tier?: EvalTier;
+    /** When true, run only structural verifiers (Wave 7.1). */
+    schemaOnly?: boolean;
+    /** When true, run structural + rule-based verifiers. Wave 7.2 wires rules. */
+    rules?: boolean;
+    /** When true, also run LLM judge verifiers. Wave 7.3 wires judging. */
+    judge?: boolean;
+    /** When true, load config + corpus and return a summary without running any verifier. */
+    dryRun?: boolean;
+    /** Override process.env during tests. */
+    env?: NodeJS.ProcessEnv;
+}
+export interface DryRunSummary {
+    kind: "dry-run";
+    config: ResolvedEvalConfig;
+    corpus: {
+        total: number;
+        byStage: Record<string, number>;
+        cases: Array<{
+            id: string;
+            stage: FlowStage;
+        }>;
+    };
+    plannedTier: EvalTier;
+    verifiersAvailable: {
+        structural: boolean;
+        rules: boolean;
+        judge: boolean;
+        workflow: boolean;
+    };
+    notes: string[];
+}
+/**
+ * Wave 7.1 runner. When `schemaOnly` is set (or no other verifier flags are
+ * active), runs structural verifiers against fixture-backed cases and loads
+ * per-stage baselines for regression comparison. Tier A/B/C agent loops
+ * still arrive in Waves 7.3+; until then cases without `fixture` are marked
+ * as skipped rather than failing.
+ */
+export declare function runEval(options: RunEvalOptions): Promise<DryRunSummary | EvalReport>;