cclaw-cli 0.21.2 → 0.23.0

package/dist/eval/runner.js

@@ -0,0 +1,178 @@
+import { randomUUID } from "node:crypto";
+import { CCLAW_VERSION } from "../constants.js";
+import { FLOW_STAGES } from "../types.js";
+import { compareAgainstBaselines, loadBaselinesByStage } from "./baseline.js";
+import { loadCorpus, readFixtureArtifact } from "./corpus.js";
+import { loadEvalConfig } from "./config-loader.js";
+import { verifyStructural } from "./verifiers/structural.js";
+function groupByStage(cases) {
+    return cases.reduce((acc, item) => {
+        acc[item.stage] = (acc[item.stage] ?? 0) + 1;
+        return acc;
+    }, {});
+}
+function skeletonVerifierResult(message, details) {
+    return {
+        kind: "structural",
+        id: "wave-7-1-no-structural-expected",
+        ok: true,
+        score: 1,
+        message,
+        ...(details !== undefined ? { details } : {})
+    };
+}
+async function runCaseStructural(projectRoot, caseEntry, plannedTier) {
+    const started = Date.now();
+    const structuralExpected = caseEntry.expected?.structural;
+    const verifierResults = [];
+    if (!structuralExpected || Object.keys(structuralExpected).length === 0) {
+        // No structural expectations declared — case is treated as "N/A" for this
+        // verifier kind; a placeholder pass keeps downstream math simple while
+        // making the situation visible in the report.
+        verifierResults.push(skeletonVerifierResult("No structural expectations declared for this case; structural verifier skipped.", { skipped: true }));
+    }
+    else {
+        let artifact;
+        try {
+            artifact = await readFixtureArtifact(projectRoot, caseEntry);
+        }
+        catch (err) {
+            verifierResults.push({
+                kind: "structural",
+                id: "structural:fixture:missing",
+                ok: false,
+                score: 0,
+                message: err instanceof Error ? err.message : String(err),
+                details: { fixture: caseEntry.fixture }
+            });
+        }
+        if (artifact !== undefined) {
+            const results = verifyStructural(artifact, structuralExpected);
+            if (results.length === 0) {
+                verifierResults.push(skeletonVerifierResult("Structural expectations parsed but produced zero checks.", { skipped: true }));
+            }
+            else {
+                verifierResults.push(...results);
+            }
+        }
+        else if (verifierResults.length === 0) {
+            verifierResults.push({
+                kind: "structural",
+                id: "structural:fixture:absent",
+                ok: false,
+                score: 0,
+                message: "Structural expectations declared but no fixture path provided. Add `fixture: ./<id>/fixture.md`.",
+                details: { fixtureProvided: false }
+            });
+        }
+    }
+    const allOk = verifierResults.every((r) => r.ok);
+    return {
+        caseId: caseEntry.id,
+        stage: caseEntry.stage,
+        tier: plannedTier,
+        passed: allOk,
+        durationMs: Date.now() - started,
+        verifierResults
+    };
+}
+function reduceSummary(caseResults) {
+    let passed = 0;
+    let failed = 0;
+    let skipped = 0;
+    let totalCostUsd = 0;
+    let totalDurationMs = 0;
+    for (const c of caseResults) {
+        totalDurationMs += c.durationMs;
+        if (c.costUsd !== undefined)
+            totalCostUsd += c.costUsd;
+        if (c.verifierResults.length === 1 && c.verifierResults[0]?.details?.skipped === true) {
+            skipped += 1;
+            continue;
+        }
+        if (c.passed)
+            passed += 1;
+        else
+            failed += 1;
+    }
+    return {
+        totalCases: caseResults.length,
+        passed,
+        failed,
+        skipped,
+        totalCostUsd: Number(totalCostUsd.toFixed(6)),
+        totalDurationMs
+    };
+}
+function stagesInResults(caseResults) {
+    const set = new Set();
+    for (const c of caseResults)
+        set.add(c.stage);
+    return FLOW_STAGES.filter((s) => set.has(s));
+}
+/**
+ * Wave 7.1 runner. When `schemaOnly` is set (or no other verifier flags are
+ * active), runs structural verifiers against fixture-backed cases and loads
+ * per-stage baselines for regression comparison. Tier A/B/C agent loops
+ * still arrive in Waves 7.3+; until then cases without `fixture` are marked
+ * as skipped rather than failing.
+ */
+export async function runEval(options) {
+    const config = await loadEvalConfig(options.projectRoot, options.env ?? process.env);
+    const corpus = await loadCorpus(options.projectRoot, options.stage);
+    const plannedTier = options.tier ?? config.defaultTier;
+    const notes = [];
+    if (corpus.length === 0) {
+        notes.push("Corpus is empty. Seed cases live under `.cclaw/evals/corpus/<stage>/*.yaml`.");
+    }
+    if (options.rules) {
+        notes.push("--rules is accepted; rule verifiers wire up in Wave 7.2.");
+    }
+    if (options.judge) {
+        notes.push("--judge is accepted; LLM judging wires up in Wave 7.3.");
+    }
+    if (options.dryRun === true) {
+        const summary = {
+            kind: "dry-run",
+            config,
+            corpus: {
+                total: corpus.length,
+                byStage: groupByStage(corpus),
+                cases: corpus.map((item) => ({ id: item.id, stage: item.stage }))
+            },
+            plannedTier,
+            verifiersAvailable: {
+                structural: true,
+                rules: false,
+                judge: false,
+                workflow: false
+            },
+            notes
+        };
+        return summary;
+    }
+    const now = new Date().toISOString();
+    const caseResults = [];
+    for (const item of corpus) {
+        caseResults.push(await runCaseStructural(options.projectRoot, item, plannedTier));
+    }
+    const stages = stagesInResults(caseResults);
+    const baselines = await loadBaselinesByStage(options.projectRoot, stages);
+    const summary = reduceSummary(caseResults);
+    const report = {
+        schemaVersion: 1,
+        generatedAt: now,
+        runId: randomUUID(),
+        cclawVersion: CCLAW_VERSION,
+        provider: config.provider,
+        model: config.model,
+        tier: plannedTier,
+        stages,
+        cases: caseResults,
+        summary
+    };
+    const baselineDelta = compareAgainstBaselines(report, baselines);
+    if (baselineDelta)
+        report.baselineDelta = baselineDelta;
+    return report;
+}

package/dist/eval/types.d.ts

@@ -0,0 +1,216 @@
+/**
+ * Core types for the cclaw eval subsystem (Phase 7).
+ *
+ * The eval subsystem lets us measure whether a change to a prompt, skill, or
+ * stage contract improves or regresses the quality of agent output. It is
+ * deliberately decoupled from the main cclaw runtime so that:
+ *
+ * - Users who never run `cclaw eval` pay zero runtime cost.
+ * - The verifier / rubric / LLM stack evolves on its own release cadence (Waves 7.0-7.6).
+ * - Any OpenAI-compatible endpoint can be swapped in via config (z.ai, OpenAI, vLLM, etc.).
+ */
+import type { FlowStage } from "../types.js";
+/**
+ * Fidelity tier for the agent-under-test.
+ *
+ * - `A` — single-shot API call, no tools. Cheap, validates core prompt behavior.
+ * - `B` — SDK loop with function-calling for Read/Write/Glob/Grep inside a sandbox.
+ * - `C` — multi-stage workflow run (brainstorm -> scope -> ... -> plan) with threaded
+ *   artifacts. Most realistic tier we ship in Phase 7; literal IDE-harness runs
+ *   (claude-code / cursor-agent proxied to OpenAI-compat) are deferred to Phase 8.
+ */
+export declare const EVAL_TIERS: readonly ["A", "B", "C"];
+export type EvalTier = (typeof EVAL_TIERS)[number];
+/**
+ * Verifier kinds, in increasing cost and decreasing determinism:
+ * structural and rules run without LLM; judge and workflow use the configured model.
+ */
+export declare const VERIFIER_KINDS: readonly ["structural", "rules", "judge", "workflow"];
+export type VerifierKind = (typeof VERIFIER_KINDS)[number];
+/**
+ * Structural expectations — deterministic, LLM-free checks against a single
+ * text artifact. Wave 7.1 implements all fields below; Wave 7.2 adds the
+ * sibling `rules` shape, Wave 7.3 adds `judge`.
+ */
+export interface StructuralExpected {
+    /**
+     * Case-insensitive substrings that must each appear on at least one markdown
+     * heading line (line starting with `#`). Useful for "required sections".
+     */
+    requiredSections?: string[];
+    /**
+     * Case-insensitive substrings that must NOT appear anywhere in the body
+     * (headings or prose). Typical entries: "TBD", "TODO", "placeholder".
+     */
+    forbiddenPatterns?: string[];
+    /** Inclusive minimum line count of the artifact body (frontmatter excluded). */
+    minLines?: number;
+    /** Inclusive maximum line count of the artifact body (frontmatter excluded). */
+    maxLines?: number;
+    /** Inclusive minimum character count of the artifact body. */
+    minChars?: number;
+    /** Inclusive maximum character count of the artifact body. */
+    maxChars?: number;
+    /**
+     * Keys that must appear in the leading YAML frontmatter (between a pair of
+     * `---` delimiters at the very top of the file). An artifact without
+     * frontmatter will fail every entry.
+     */
+    requiredFrontmatterKeys?: string[];
+}
+/** Superset of per-verifier expectation shapes. Only `structural` is wired in Wave 7.1. */
+export interface ExpectedShape {
+    structural?: StructuralExpected;
+    /** Rule-based (keyword/regex/traceability) checks — Wave 7.2. */
+    rules?: Record<string, unknown>;
+    /** LLM-judge rubrics — Wave 7.3. */
+    judge?: Record<string, unknown>;
+}
+/**
+ * A single eval case describes one input scenario for one stage. Cases live in
+ * `.cclaw/evals/corpus/<stage>/<id>.yaml` and may reference a pre-generated
+ * fixture artifact for verifier development (Wave 7.1) before the agent loop
+ * exists (Wave 7.3+).
+ */
+export interface EvalCase {
+    id: string;
+    stage: FlowStage;
+    inputPrompt: string;
+    /** Project files copied into the Tier B/C sandbox before the agent runs. */
+    contextFiles?: string[];
+    /**
+     * Typed expectation hints consumed by the structural/rules/judge verifiers.
+     * Each sub-shape is optional; missing sub-shapes skip that verifier tier.
+     */
+    expected?: ExpectedShape;
+    /**
+     * Path (relative to the corpus case file) of a pre-generated artifact used
+     * when verifiers are exercised without a live agent loop. Primarily a Wave
+     * 7.1 development aid.
+     */
+    fixture?: string;
+}
+/** Result of one verifier applied to one case. */
+export interface VerifierResult {
+    kind: VerifierKind;
+    id: string;
+    ok: boolean;
+    /** Normalized 0..1 score when the verifier produces a numeric signal. */
+    score?: number;
+    message?: string;
+    details?: Record<string, unknown>;
+}
+/** Aggregate result for one case after all verifiers run. */
+export interface EvalCaseResult {
+    caseId: string;
+    stage: FlowStage;
+    tier: EvalTier;
+    passed: boolean;
+    durationMs: number;
+    costUsd?: number;
+    verifierResults: VerifierResult[];
+}
+/** Top-level eval report, serialized to JSON and rendered to Markdown. */
+export interface EvalReport {
+    schemaVersion: 1;
+    generatedAt: string;
+    runId: string;
+    cclawVersion: string;
+    provider: string;
+    model: string;
+    tier: EvalTier;
+    stages: FlowStage[];
+    cases: EvalCaseResult[];
+    summary: {
+        totalCases: number;
+        passed: number;
+        failed: number;
+        skipped: number;
+        totalCostUsd: number;
+        totalDurationMs: number;
+    };
+    /** Present when comparing against a saved baseline (Wave 7.1+). */
+    baselineDelta?: BaselineDelta;
+}
+/**
+ * Eval configuration, persisted to `.cclaw/evals/config.yaml` and mergeable
+ * with `CCLAW_EVAL_*` environment variables at runtime.
+ */
+export interface EvalConfig {
+    /**
+     * Free-form provider name used in reports. The actual HTTP protocol is
+     * determined by `baseUrl`, which is expected to be OpenAI-compatible.
+     */
+    provider: string;
+    /** OpenAI-compatible base URL, e.g. `https://api.z.ai/api/coding/paas/v4`. */
+    baseUrl: string;
+    /** Model identifier for both agent-under-test and judge unless `judgeModel` overrides. */
+    model: string;
+    /** Optional separate model for the judge role. Defaults to `model`. */
+    judgeModel?: string;
+    /** Default tier when `--tier` is not supplied. */
+    defaultTier: EvalTier;
+    /** Optional hard stop on estimated USD spend per day. Unset = no cap. */
+    dailyUsdCap?: number;
+    /** Regression thresholds for CI gates. */
+    regression: {
+        /** Fail when overall score drops by more than this fraction (e.g. 0.15 = 15%). */
+        failIfDeltaBelow: number;
+        /** Fail when any single critical rubric drops below this absolute score. */
+        failIfCriticalBelow: number;
+    };
+    /** Per-agent-run timeout in milliseconds. */
+    timeoutMs: number;
+    /** Max retries per API call on transient failures. */
+    maxRetries: number;
+}
+/** Resolved config with env overrides applied. */
+export interface ResolvedEvalConfig extends EvalConfig {
+    apiKey?: string;
+    source: "default" | "file" | "env" | "file+env";
+}
+/**
+ * Frozen per-stage baseline used by regression gating (Wave 7.1). Baselines
+ * are committed to git; `cclaw eval --update-baseline --confirm` rewrites
+ * them. The shape is intentionally flat so a quick `git diff` reveals what
+ * changed between runs.
+ */
+export interface BaselineSnapshot {
+    schemaVersion: 1;
+    stage: FlowStage;
+    generatedAt: string;
+    cclawVersion: string;
+    /** Keyed by `EvalCase.id` so unchanged cases produce zero diff. */
+    cases: Record<string, BaselineCaseEntry>;
+}
+export interface BaselineCaseEntry {
+    passed: boolean;
+    verifierResults: BaselineVerifierEntry[];
+}
+export interface BaselineVerifierEntry {
+    id: string;
+    kind: VerifierKind;
+    ok: boolean;
+    score?: number;
+}
+/**
+ * Delta between a fresh report and the saved baseline. Populated when
+ * baselines exist on disk and the run covers matching cases.
+ */
+export interface BaselineDelta {
+    baselineId: string;
+    /** Fresh-score − baseline-score, bounded to [-1, 1]. */
+    scoreDelta: number;
+    /** Count of checks that flipped from `ok:true` to `ok:false`. */
+    criticalFailures: number;
+    /** Per-case regression details for the Markdown report. */
+    regressions: BaselineRegression[];
+}
+export interface BaselineRegression {
+    caseId: string;
+    stage: FlowStage;
+    verifierId: string;
+    reason: "newly-failing" | "case-now-failing" | "score-drop";
+    previousScore?: number;
+    currentScore?: number;
+}

package/dist/eval/types.js

@@ -0,0 +1,15 @@
+/**
+ * Fidelity tier for the agent-under-test.
+ *
+ * - `A` — single-shot API call, no tools. Cheap, validates core prompt behavior.
+ * - `B` — SDK loop with function-calling for Read/Write/Glob/Grep inside a sandbox.
+ * - `C` — multi-stage workflow run (brainstorm -> scope -> ... -> plan) with threaded
+ *   artifacts. Most realistic tier we ship in Phase 7; literal IDE-harness runs
+ *   (claude-code / cursor-agent proxied to OpenAI-compat) are deferred to Phase 8.
+ */
+export const EVAL_TIERS = ["A", "B", "C"];
+/**
+ * Verifier kinds, in increasing cost and decreasing determinism:
+ * structural and rules run without LLM; judge and workflow use the configured model.
+ */
+export const VERIFIER_KINDS = ["structural", "rules", "judge", "workflow"];

package/dist/eval/verifiers/structural.d.ts

@@ -0,0 +1,14 @@
+import type { StructuralExpected, VerifierResult } from "../types.js";
+export interface ArtifactSplit {
+    hasFrontmatter: boolean;
+    frontmatterRaw: string;
+    frontmatterParsed?: Record<string, unknown>;
+    body: string;
+}
+export declare function splitFrontmatter(artifact: string): ArtifactSplit;
+/**
+ * Run every configured structural check against the artifact text.
+ * Returns [] when `expected` is undefined/empty so the runner can treat
+ * "no structural expectations" as "no verifier results" rather than "pass".
+ */
+export declare function verifyStructural(artifact: string, expected: StructuralExpected | undefined): VerifierResult[];

package/dist/eval/verifiers/structural.js

@@ -0,0 +1,171 @@
+/**
+ * Structural verifier (Wave 7.1): deterministic, zero-LLM checks against a
+ * single markdown artifact. Each structural expectation produces one
+ * `VerifierResult` so baselines diff cleanly at the check level rather than
+ * lumping everything into a single boolean.
+ *
+ * Design notes:
+ *
+ * - All pattern matching is case-insensitive. Authoring a check as
+ *   `"Directions"` matches `## Directions` and `### directions-suggested`.
+ * - Frontmatter detection is permissive: it must start at byte 0 with `---\n`
+ *   and close on a subsequent `---` line. Anything else is treated as "no
+ *   frontmatter", which fails every `requiredFrontmatterKeys` entry
+ *   deterministically.
+ * - `minLines`/`maxLines` intentionally exclude frontmatter so a rewrite that
+ *   adds metadata does not accidentally drop the body below the floor.
+ * - Scoring: each check scores 0 or 1. The case `passed` becomes the AND of
+ *   all individual `ok` flags. This keeps Wave 7.1 deterministic; the 0..1
+ *   rubric scale shows up in Wave 7.3 (judge).
+ */
+import { parse as parseYaml } from "yaml";
+const FRONTMATTER_OPEN = /^---\r?\n/;
+const FRONTMATTER_CLOSE = /\r?\n---\r?(?:\n|$)/;
+function slugify(input) {
+    return input
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/(^-|-$)/g, "")
+        .slice(0, 64);
+}
+export function splitFrontmatter(artifact) {
+    if (!FRONTMATTER_OPEN.test(artifact)) {
+        return { hasFrontmatter: false, frontmatterRaw: "", body: artifact };
+    }
+    const afterOpen = artifact.replace(FRONTMATTER_OPEN, "");
+    const closeMatch = afterOpen.match(FRONTMATTER_CLOSE);
+    if (!closeMatch || closeMatch.index === undefined) {
+        return { hasFrontmatter: false, frontmatterRaw: "", body: artifact };
+    }
+    const frontmatterRaw = afterOpen.slice(0, closeMatch.index);
+    const body = afterOpen.slice(closeMatch.index + closeMatch[0].length);
+    let frontmatterParsed;
+    try {
+        const parsed = parseYaml(frontmatterRaw);
+        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+            frontmatterParsed = parsed;
+        }
+    }
+    catch {
+        frontmatterParsed = undefined;
+    }
+    return {
+        hasFrontmatter: true,
+        frontmatterRaw,
+        frontmatterParsed,
+        body
+    };
+}
+function extractHeadingLines(body) {
+    return body
+        .split(/\r?\n/)
+        .map((line) => line.trimStart())
+        .filter((line) => /^#{1,6}\s+\S/.test(line));
+}
+function result(id, ok, message, details) {
+    return {
+        kind: "structural",
+        id,
+        ok,
+        score: ok ? 1 : 0,
+        message,
+        ...(details !== undefined ? { details } : {})
+    };
+}
+function checkRequiredSections(sections, body) {
+    const headings = extractHeadingLines(body).map((line) => line.toLowerCase());
+    return sections.map((section) => {
+        const needle = section.toLowerCase().trim();
+        const found = headings.some((heading) => heading.includes(needle));
+        return result(`structural:section:${slugify(section)}`, found, found
+            ? `Section matching "${section}" present.`
+            : `No heading contains "${section}".`, { pattern: section, searchedHeadings: headings.length });
+    });
+}
+function checkForbiddenPatterns(patterns, body) {
+    const bodyLower = body.toLowerCase();
+    return patterns.map((pattern) => {
+        const needle = pattern.toLowerCase();
+        const hits = countOccurrences(bodyLower, needle);
+        const ok = hits === 0;
+        return result(`structural:forbidden:${slugify(pattern)}`, ok, ok
+            ? `Pattern "${pattern}" absent (as required).`
+            : `Pattern "${pattern}" appears ${hits} time(s); remove.`, { pattern, occurrences: hits });
+    });
+}
+function countOccurrences(haystack, needle) {
+    if (needle.length === 0)
+        return 0;
+    let index = 0;
+    let count = 0;
+    while (true) {
+        const at = haystack.indexOf(needle, index);
+        if (at < 0)
+            return count;
+        count += 1;
+        index = at + needle.length;
+    }
+}
+function checkLengthBounds(expected, body) {
+    const results = [];
+    const lineCount = body.length === 0 ? 0 : body.split(/\r?\n/).length;
+    const charCount = body.length;
+    if (expected.minLines !== undefined || expected.maxLines !== undefined) {
+        const min = expected.minLines;
+        const max = expected.maxLines;
+        const withinMin = min === undefined || lineCount >= min;
+        const withinMax = max === undefined || lineCount <= max;
+        const ok = withinMin && withinMax;
+        results.push(result("structural:length:lines", ok, ok
+            ? `Body has ${lineCount} line(s), within bounds.`
+            : buildOutOfRangeMessage("line", lineCount, min, max), { lineCount, minLines: min, maxLines: max }));
+    }
+    if (expected.minChars !== undefined || expected.maxChars !== undefined) {
+        const min = expected.minChars;
+        const max = expected.maxChars;
+        const withinMin = min === undefined || charCount >= min;
+        const withinMax = max === undefined || charCount <= max;
+        const ok = withinMin && withinMax;
+        results.push(result("structural:length:chars", ok, ok
+            ? `Body has ${charCount} char(s), within bounds.`
+            : buildOutOfRangeMessage("char", charCount, min, max), { charCount, minChars: min, maxChars: max }));
+    }
+    return results;
+}
+function buildOutOfRangeMessage(unit, actual, min, max) {
+    const lo = min === undefined ? "0" : String(min);
+    const hi = max === undefined ? "∞" : String(max);
+    return `Body has ${actual} ${unit}(s); expected ${lo}..${hi}.`;
+}
+function checkFrontmatterKeys(keys, split) {
+    if (!split.hasFrontmatter || !split.frontmatterParsed) {
+        return keys.map((key) => result(`structural:frontmatter:${slugify(key)}`, false, `Frontmatter key "${key}" missing (no parseable frontmatter).`, { key, frontmatterPresent: split.hasFrontmatter }));
+    }
+    const present = new Set(Object.keys(split.frontmatterParsed));
+    return keys.map((key) => {
+        const ok = present.has(key);
+        return result(`structural:frontmatter:${slugify(key)}`, ok, ok ? `Frontmatter key "${key}" present.` : `Frontmatter key "${key}" missing.`, { key });
+    });
+}
+/**
+ * Run every configured structural check against the artifact text.
+ * Returns [] when `expected` is undefined/empty so the runner can treat
+ * "no structural expectations" as "no verifier results" rather than "pass".
+ */
+export function verifyStructural(artifact, expected) {
+    if (!expected)
+        return [];
+    const split = splitFrontmatter(artifact);
+    const results = [];
+    if (expected.requiredSections?.length) {
+        results.push(...checkRequiredSections(expected.requiredSections, split.body));
+    }
+    if (expected.forbiddenPatterns?.length) {
+        results.push(...checkForbiddenPatterns(expected.forbiddenPatterns, split.body));
+    }
+    results.push(...checkLengthBounds(expected, split.body));
+    if (expected.requiredFrontmatterKeys?.length) {
+        results.push(...checkFrontmatterKeys(expected.requiredFrontmatterKeys, split));
+    }
+    return results;
+}

package/dist/install.js

@@ -28,6 +28,7 @@ import { contextMonitorScript, promptGuardScript, workflowGuardScript } from "./
 import { META_SKILL_NAME, usingCclawSkillMarkdown } from "./content/meta-skill.js";
 import { decisionProtocolMarkdown, completionProtocolMarkdown, ethosProtocolMarkdown } from "./content/protocols.js";
 import { ARTIFACT_TEMPLATES, CURSOR_WORKFLOW_RULE_MDC, RULEBOOK_MARKDOWN, buildRulesJson } from "./content/templates.js";
+import { EVAL_BASELINES_README, EVAL_CONFIG_YAML, EVAL_CORPUS_README, EVAL_REPORTS_README, EVAL_RUBRICS_README } from "./content/eval-scaffold.js";
 import { TDD_WAVE_WALKTHROUGH_MARKDOWN, stageSkillFolder, stageSkillMarkdown } from "./content/skills.js";
 import { stageCommonGuidanceMarkdown } from "./content/stage-common-guidance.js";
 import { STAGE_EXAMPLES_REFERENCE_DIR, stageExamplesReferenceMarkdown } from "./content/examples.js";
@@ -184,6 +185,26 @@ async function writeArtifactTemplates(projectRoot) {
         await writeFileSafe(runtimePath(projectRoot, "templates", fileName), content);
     }
 }
+/**
+ * Seed the `.cclaw/evals/` scaffold. Only writes files that do not already
+ * exist so that user-authored config.yaml / corpus / rubrics / baselines are
+ * never clobbered by `cclaw sync`.
+ */
+async function writeEvalScaffold(projectRoot) {
+    const targets = [
+        { rel: "evals/config.yaml", content: EVAL_CONFIG_YAML },
+        { rel: "evals/corpus/README.md", content: EVAL_CORPUS_README },
+        { rel: "evals/rubrics/README.md", content: EVAL_RUBRICS_README },
+        { rel: "evals/baselines/README.md", content: EVAL_BASELINES_README },
+        { rel: "evals/reports/README.md", content: EVAL_REPORTS_README }
+    ];
+    for (const target of targets) {
+        const absolute = runtimePath(projectRoot, ...target.rel.split("/"));
+        if (await exists(absolute))
+            continue;
+        await writeFileSafe(absolute, target.content);
+    }
+}
 async function writeSkills(projectRoot, config) {
     for (const stage of COMMAND_FILE_ORDER) {
         const folder = stageSkillFolder(stage);
@@ -1044,6 +1065,7 @@ async function materializeRuntime(projectRoot, config, forceStateReset) {
     await writeSkills(projectRoot, config);
     await writeContextModes(projectRoot);
     await writeArtifactTemplates(projectRoot);
+    await writeEvalScaffold(projectRoot);
     await writeRulebook(projectRoot);
     await writeState(projectRoot, config, forceStateReset);
     await ensureRunSystem(projectRoot, { createIfMissing: false });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cclaw-cli",
-  "version": "0.21.2",
+  "version": "0.23.0",
   "description": "Installer-first flow toolkit for coding agents",
   "type": "module",
   "bin": {