cclaw-cli 0.24.0 → 0.26.0

package/dist/cli.js

@@ -59,7 +59,7 @@ Commands:
                     --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=workflow).
                     --schema-only        Run only structural verifiers (default).
                     --rules              Also run rule-based verifiers (keywords, regex, counts, uniqueness, traceability).
-                    --judge              Include LLM judging (not wired yet; requires API key).
+                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; Tier A runs the single-shot agent, Tier B runs the sandbox tool-using agent (read_file/write_file/glob/grep).
                     --dry-run            Validate config + corpus, print summary, do not execute.
                     --json               Emit machine-readable JSON on stdout.
                     --no-write           Skip writing the report to .cclaw/evals/reports/.
@@ -78,6 +78,8 @@ Examples:
   cclaw archive --name=payments-revamp
   cclaw eval --dry-run
   cclaw eval --stage=brainstorm --schema-only
+  cclaw eval --judge --tier=A --stage=brainstorm
+  cclaw eval --judge --tier=B --stage=spec
 
 Docs:   https://github.com/zuevrs/cclaw
 Issues: https://github.com/zuevrs/cclaw/issues

package/dist/content/eval-scaffold.d.ts

@@ -6,6 +6,10 @@
  */
 export declare const EVAL_CONFIG_YAML = "# cclaw eval config\n# See docs/evals.md for the full schema and rollout plan.\n#\n# All values can be overridden at runtime with CCLAW_EVAL_* environment\n# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.\nprovider: zai\nbaseUrl: https://api.z.ai/api/coding/paas/v4\nmodel: glm-5.1\n\n# Default fidelity tier when --tier is not supplied.\n#   A = single-shot API call (cheap)\n#   B = SDK with tool use     (realistic)\n#   C = multi-stage workflow  (end-to-end)\ndefaultTier: A\n\n# Per-call timeout and retry budget.\ntimeoutMs: 120000\nmaxRetries: 2\n\n# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.\n# dailyUsdCap: 5\n\n# Regression thresholds used by CI.\nregression:\n  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).\n  failIfDeltaBelow: -0.15\n  # Fail when any single critical rubric drops below this absolute score.\n  failIfCriticalBelow: 3.0\n";
 export declare const EVAL_CORPUS_README = "# Eval Corpus\n\nSeed cases live in `./<stage>/<id>.yaml`, one file per case.\nSee `docs/evals.md` for the schema.\n\nMinimal shape:\n\n```yaml\nid: brainstorm-01\nstage: brainstorm\ninput_prompt: |\n  One short paragraph describing the user's task.\ncontext_files: []\nexpected:\n  # verifier-specific hints; optional\n```\n\nStart with 3 structural cases per stage (24 total), then expand to 5 per\nstage (40 total) once rule verifiers land. Tier B/C runs may add\n`context_files` pulled from real projects to exercise the sandbox.\n";
-export declare const EVAL_RUBRICS_README = "# Eval Rubrics\n\nLLM-judge rubrics. Each rubric is a short list of checks scored on a\n`1\u20135` scale with a rationale:\n\n```yaml\nstage: brainstorm\nchecks:\n  - id: distinctness\n    prompt: \"Are the proposed directions genuinely distinct (not rephrasings)?\"\n    scale: \"1-5 where 5=fully distinct approaches\"\n    weight: 1.0\n```\n\nRubric authoring happens when Tier A runs start producing artifacts, so we\nscore the *right* properties rather than retrofitting generic quality checks.\nSee `docs/evals.md` for the full schema.\n";
+export declare const EVAL_RUBRICS_README = "# Eval Rubrics\n\nLLM-judge rubrics. Each rubric is a short list of checks scored on a\n`1\u20135` scale with a rationale. The runner picks `<stage>.yaml` when\n`cclaw eval --judge` is invoked; every stage ships a starter rubric\nbelow \u2014 edit the checks to match what your team cares about, and add\n`critical: true` to the checks that should hard-fail nightly CI on\nregression.\n\n```yaml\nstage: brainstorm\nchecks:\n  - id: distinctness\n    prompt: \"Are the proposed directions genuinely distinct (not rephrasings)?\"\n    scale: \"1-5 where 5=fully distinct approaches\"\n    weight: 1.0\n    critical: false\n```\n\nSee `docs/evals.md` for the full schema.\n";
+export declare const EVAL_RUBRIC_FILES: ReadonlyArray<{
+    stage: string;
+    contents: string;
+}>;
 export declare const EVAL_BASELINES_README = "# Eval Baselines\n\nFrozen score snapshots used by regression gates. Baselines are committed to\ngit and updated explicitly via `cclaw eval --update-baseline --confirm`.\n\nEach baseline file is a JSON document keyed by stage and case id. Do not edit\nby hand; CI will flag baseline churn.\n";
 export declare const EVAL_REPORTS_README = "# Eval Reports\n\nGenerated reports (JSON + Markdown) land here. This directory is gitignored.\nRun `cclaw eval --dry-run` to preview configuration without producing a\nreport.\n";

package/dist/content/eval-scaffold.js

@@ -57,7 +57,11 @@ stage (40 total) once rule verifiers land. Tier B/C runs may add
 export const EVAL_RUBRICS_README = `# Eval Rubrics
 
 LLM-judge rubrics. Each rubric is a short list of checks scored on a
-\`1–5\` scale with a rationale:
+\`1–5\` scale with a rationale. The runner picks \`<stage>.yaml\` when
+\`cclaw eval --judge\` is invoked; every stage ships a starter rubric
+below — edit the checks to match what your team cares about, and add
+\`critical: true\` to the checks that should hard-fail nightly CI on
+regression.
 
 \`\`\`yaml
 stage: brainstorm
@@ -66,12 +70,289 @@ checks:
     prompt: "Are the proposed directions genuinely distinct (not rephrasings)?"
     scale: "1-5 where 5=fully distinct approaches"
     weight: 1.0
+    critical: false
 \`\`\`
 
-Rubric authoring happens when Tier A runs start producing artifacts, so we
-score the *right* properties rather than retrofitting generic quality checks.
 See \`docs/evals.md\` for the full schema.
 `;
+const STARTER_RUBRICS = [
+    {
+        stage: "brainstorm",
+        checks: [
+            {
+                id: "distinctness",
+                prompt: "Are the proposed directions genuinely distinct (different approaches, not rephrasings of one idea)?",
+                scale: "1-5 where 5 = every direction uses a materially different approach",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "coverage",
+                prompt: "Do the directions cover the problem space (at least one tackling cost, one velocity, one risk)?",
+                scale: "1-5 where 5 = each major trade-off dimension has a direction",
+                weight: 1.0
+            },
+            {
+                id: "actionability",
+                prompt: "Could a reader pick one direction and start a scope doc tomorrow without asking clarifying questions?",
+                scale: "1-5 where 5 = every direction is concrete enough to scope immediately",
+                weight: 1.0
+            },
+            {
+                id: "recommendation-clarity",
+                prompt: "Is the Recommendation section explicit, single-voiced, and consistent with the highest-ranked direction?",
+                scale: "1-5 where 5 = recommendation names the chosen direction and the decisive trade-off",
+                weight: 1.0,
+                critical: true
+            }
+        ]
+    },
+    {
+        stage: "scope",
+        checks: [
+            {
+                id: "problem-statement",
+                prompt: "Is the problem statement anchored on user/system behavior (not on a proposed solution)?",
+                scale: "1-5 where 5 = problem is described independently of any implementation choice",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "non-goals",
+                prompt: "Are non-goals explicit and mutually-exclusive with the goals (no overlap, no vague 'we might' entries)?",
+                scale: "1-5 where 5 = every non-goal is a crisp decision a future reader can defend",
+                weight: 1.0
+            },
+            {
+                id: "decision-ids",
+                prompt: "Does the Decisions section use stable D-NN ids and name who (or what) owns each decision?",
+                scale: "1-5 where 5 = every decision has a D-NN id and an explicit owner",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "risks",
+                prompt: "Are risks concrete (named system, threshold, or scenario) rather than generic hedges?",
+                scale: "1-5 where 5 = each risk is testable by observing a specific signal",
+                weight: 0.8
+            }
+        ]
+    },
+    {
+        stage: "design",
+        checks: [
+            {
+                id: "decision-trace",
+                prompt: "Does the design doc restate every scope D-NN that drives the architecture, and call out the ones it rejects?",
+                scale: "1-5 where 5 = full D-NN trace with explicit kept/rejected markers",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "diagram-or-flow",
+                prompt: "Is there at least one diagram or clearly labeled flow section that shows data and control moving across the system?",
+                scale: "1-5 where 5 = diagram covers read path, write path, and failure path",
+                weight: 1.0
+            },
+            {
+                id: "alternatives-considered",
+                prompt: "Are concrete alternatives considered with explicit trade-offs (cost, complexity, latency)?",
+                scale: "1-5 where 5 = at least two alternatives are rejected with reasons tied to measurable properties",
+                weight: 0.8
+            },
+            {
+                id: "interface-stability",
+                prompt: "Are public interfaces (APIs, queues, tables) named, typed, and marked as SEMVER-stable or experimental?",
+                scale: "1-5 where 5 = every interface has a name, a type/shape, and a stability tag",
+                weight: 1.0
+            }
+        ]
+    },
+    {
+        stage: "spec",
+        checks: [
+            {
+                id: "acceptance-criteria",
+                prompt: "Does the spec have explicit Acceptance Criteria bullets that are unambiguously verifiable?",
+                scale: "1-5 where 5 = each AC states an observable condition with clear pass/fail",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "edge-cases",
+                prompt: "Are failure modes and edge cases enumerated (empty input, concurrent writers, partial outage)?",
+                scale: "1-5 where 5 = at least three distinct edge cases with expected behavior",
+                weight: 1.0
+            },
+            {
+                id: "test-plan-hooks",
+                prompt: "Does the spec name the test surfaces (unit, integration, e2e, synthetic probe) that will validate each AC?",
+                scale: "1-5 where 5 = every AC maps to at least one test surface",
+                weight: 1.0
+            },
+            {
+                id: "traceability",
+                prompt: "Does the spec cite the originating scope decisions (D-NN) and design sections so future engineers can trace back?",
+                scale: "1-5 where 5 = every material choice links to a D-NN or design heading",
+                weight: 0.8,
+                critical: true
+            }
+        ]
+    },
+    {
+        stage: "plan",
+        checks: [
+            {
+                id: "task-granularity",
+                prompt: "Are tasks sized so one engineer can land each in a single PR (<1 day of work)?",
+                scale: "1-5 where 5 = every T-NN fits in a single reviewable PR",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "tdd-loop",
+                prompt: "Does each task have explicit RED/GREEN/REFACTOR expectations or an equivalent TDD-compatible exit condition?",
+                scale: "1-5 where 5 = every task says what test fails first and what code makes it pass",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "dependency-graph",
+                prompt: "Is the dependency order between tasks explicit (and minimal), so parallelizable work is called out?",
+                scale: "1-5 where 5 = every task lists its blockers and independent tasks are marked parallelizable",
+                weight: 0.8
+            },
+            {
+                id: "scope-traceability",
+                prompt: "Does the plan reference the scope D-NN ids that drive each task, and does coverage leave no decision orphaned?",
+                scale: "1-5 where 5 = every D-NN appears in at least one task and every task names its D-NN",
+                weight: 1.0
+            }
+        ]
+    },
+    {
+        stage: "tdd",
+        checks: [
+            {
+                id: "red-first",
+                prompt: "Does the artifact show a failing test (RED) before the implementation change (GREEN)?",
+                scale: "1-5 where 5 = RED command output is quoted and the fix lands after",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "refactor-evidence",
+                prompt: "Is there a REFACTOR step with a diff or named improvement (not just passing tests)?",
+                scale: "1-5 where 5 = REFACTOR names a specific code-quality win and cites the affected file(s)",
+                weight: 0.8
+            },
+            {
+                id: "gate-evidence",
+                prompt: "Does the artifact quote the output of the required gates (lint, typecheck, tests) after the change?",
+                scale: "1-5 where 5 = every gate command is reproduced with its exit status",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "learnings",
+                prompt: "Does the artifact capture at least one durable learning (pattern, pitfall, follow-up) for future runs?",
+                scale: "1-5 where 5 = learning is specific, filed under knowledge.jsonl or an equivalent store",
+                weight: 0.6
+            }
+        ]
+    },
+    {
+        stage: "review",
+        checks: [
+            {
+                id: "two-layer-structure",
+                prompt: "Does the review show both layers (automated gates + human judgment) with distinct evidence?",
+                scale: "1-5 where 5 = Layer 1 cites tool outputs, Layer 2 cites reviewer reasoning",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "blocker-severity",
+                prompt: "Are issues classified by severity (blocker / major / minor) with one-line rationales?",
+                scale: "1-5 where 5 = every finding names severity + consequence if not fixed",
+                weight: 1.0
+            },
+            {
+                id: "security-posture",
+                prompt: "Does the review cover security-relevant areas explicitly (secrets, authz, PII, deps)?",
+                scale: "1-5 where 5 = each security dimension is addressed (with 'n/a' counted as a deliberate pass)",
+                weight: 0.8,
+                critical: true
+            },
+            {
+                id: "follow-ups",
+                prompt: "Are non-blocking follow-ups filed as explicit tickets or knowledge-log entries (not left as prose)?",
+                scale: "1-5 where 5 = every follow-up has a home and an owner",
+                weight: 0.8
+            }
+        ]
+    },
+    {
+        stage: "ship",
+        checks: [
+            {
+                id: "release-readiness",
+                prompt: "Does the artifact prove release readiness (gates green, changelog, version bump)?",
+                scale: "1-5 where 5 = each readiness item is linked to concrete evidence",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "rollback",
+                prompt: "Is there an explicit rollback path (command, feature-flag, migration reversal)?",
+                scale: "1-5 where 5 = rollback is reproducible from the doc with no context rehydration",
+                weight: 1.0,
+                critical: true
+            },
+            {
+                id: "monitoring",
+                prompt: "Are monitoring and alerting hooks named (dashboards, logs, SLO tripwires)?",
+                scale: "1-5 where 5 = each hook has a canonical URL or query",
+                weight: 0.8
+            },
+            {
+                id: "retro-seed",
+                prompt: "Does the artifact leave a retro seed (what went well, what to change for the next run)?",
+                scale: "1-5 where 5 = at least one distinct 'keep' and one 'change' statement",
+                weight: 0.6
+            }
+        ]
+    }
+];
+function renderRubric(rubric) {
+    const lines = [];
+    lines.push(`# Starter rubric for the \`${rubric.stage}\` stage.`);
+    lines.push(`# Edit the checks to reflect your team's bar before running`);
+    lines.push(`# \`cclaw eval --judge\`. Every check id is used verbatim in`);
+    lines.push(`# report output and baseline files, so keep slugs stable once`);
+    lines.push(`# they start appearing in CI.`);
+    lines.push(`stage: ${rubric.stage}`);
+    lines.push(`checks:`);
+    for (const check of rubric.checks) {
+        lines.push(`  - id: ${check.id}`);
+        lines.push(`    prompt: >-`);
+        lines.push(`      ${check.prompt}`);
+        if (check.scale !== undefined) {
+            lines.push(`    scale: ${JSON.stringify(check.scale)}`);
+        }
+        if (check.weight !== undefined) {
+            lines.push(`    weight: ${check.weight}`);
+        }
+        if (check.critical === true) {
+            lines.push(`    critical: true`);
+        }
+    }
+    return `${lines.join("\n")}\n`;
+}
+export const EVAL_RUBRIC_FILES = STARTER_RUBRICS.map((rubric) => ({
+    stage: rubric.stage,
+    contents: renderRubric(rubric)
+}));
 export const EVAL_BASELINES_README = `# Eval Baselines
 
 Frozen score snapshots used by regression gates. Baselines are committed to

package/dist/eval/agents/single-shot.d.ts

@@ -0,0 +1,27 @@
+import type { FlowStage } from "../../types.js";
+import type { ChatUsage, EvalLlmClient } from "../llm-client.js";
+import type { EvalCase, ResolvedEvalConfig } from "../types.js";
+export interface SingleShotInput {
+    caseEntry: EvalCase;
+    config: Pick<ResolvedEvalConfig, "model" | "agentTemperature" | "timeoutMs" | "tokenPricing">;
+    projectRoot: string;
+    client: EvalLlmClient;
+    /**
+     * Override the SKILL.md loader. Primarily a test hook so unit tests
+     * can swap a canned system prompt without creating fixtures on disk.
+     */
+    loadSkill?: (stage: FlowStage) => Promise<string>;
+}
+export interface SingleShotOutput {
+    artifact: string;
+    usage: ChatUsage;
+    usageUsd: number;
+    model: string;
+    durationMs: number;
+    attempts: number;
+    systemPrompt: string;
+    userPrompt: string;
+}
+export declare function loadStageSkill(projectRoot: string, stage: FlowStage): Promise<string>;
+/** Run the Tier A single-shot AUT and return the produced artifact. */
+export declare function runSingleShot(input: SingleShotInput): Promise<SingleShotOutput>;

package/dist/eval/agents/single-shot.js

@@ -0,0 +1,79 @@
+/**
+ * Tier A single-shot agent.
+ *
+ * Simplest realistic AUT: one LLM call with the stage's SKILL.md as the
+ * system prompt and the case's `inputPrompt` as the user message. Output
+ * is the raw assistant content, returned as the artifact for the judge
+ * pipeline.
+ *
+ * Design notes:
+ *
+ * - No tools. No multi-turn. No reads of the project beyond the one
+ *   SKILL.md. Tier B/C layer complexity on top in later steps.
+ * - Errors are propagated as-is (`EvalLlmError` subclasses) so the
+ *   runner can surface them as verifier failures without swallowing the
+ *   cause.
+ * - Usage and USD cost are surfaced so the runner can commit them to
+ *   the cost guard + case-level `costUsd`.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { RUNTIME_ROOT } from "../../constants.js";
+import { stageSkillFolder } from "../../content/skills.js";
+import { exists } from "../../fs-utils.js";
+import { computeUsageUsd } from "../cost-guard.js";
+export async function loadStageSkill(projectRoot, stage) {
+    const folder = stageSkillFolder(stage);
+    const file = path.join(projectRoot, RUNTIME_ROOT, "skills", folder, "SKILL.md");
+    if (!(await exists(file))) {
+        throw new Error(`Stage skill not found: ${path.relative(projectRoot, file)}. ` +
+            `Run \`cclaw init\` (or \`cclaw sync\`) before \`cclaw eval --tier=A --judge\`.`);
+    }
+    return fs.readFile(file, "utf8");
+}
+function buildMessages(systemPrompt, userPrompt) {
+    return [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: userPrompt }
+    ];
+}
+function buildUserPrompt(caseEntry) {
+    const lines = [];
+    lines.push(`Stage: ${caseEntry.stage}`);
+    lines.push(`Case id: ${caseEntry.id}`);
+    lines.push(``);
+    lines.push(`Task:`);
+    lines.push(caseEntry.inputPrompt.trim());
+    lines.push(``);
+    lines.push(`Produce the artifact required by this stage using the SKILL.md above. ` +
+        `Output the artifact directly (markdown with optional YAML frontmatter). ` +
+        `Do not wrap in code fences, do not add commentary before or after.`);
+    return lines.join("\n");
+}
+/** Run the Tier A single-shot AUT and return the produced artifact. */
+export async function runSingleShot(input) {
+    const { caseEntry, config, projectRoot, client } = input;
+    const started = Date.now();
+    const loader = input.loadSkill ?? ((stage) => loadStageSkill(projectRoot, stage));
+    const systemPrompt = await loader(caseEntry.stage);
+    const userPrompt = buildUserPrompt(caseEntry);
+    const response = await client.chat({
+        model: config.model,
+        messages: buildMessages(systemPrompt, userPrompt),
+        temperature: config.agentTemperature ?? 0.2,
+        timeoutMs: config.timeoutMs
+    });
+    const usageUsd = computeUsageUsd(response.model, response.usage, {
+        tokenPricing: config.tokenPricing
+    });
+    return {
+        artifact: response.content.trim(),
+        usage: response.usage,
+        usageUsd,
+        model: response.model,
+        attempts: response.attempts,
+        durationMs: Date.now() - started,
+        systemPrompt,
+        userPrompt
+    };
+}

package/dist/eval/agents/with-tools.d.ts

@@ -0,0 +1,31 @@
+import type { ChatUsage, EvalLlmClient } from "../llm-client.js";
+import { createSandbox } from "../sandbox.js";
+import type { SandboxTool } from "../tools/index.js";
+import type { EvalCase, ResolvedEvalConfig, ToolUseSummary } from "../types.js";
+export declare class MaxTurnsExceededError extends Error {
+    readonly turns: number;
+    constructor(turns: number);
+}
+export interface WithToolsInput {
+    caseEntry: EvalCase;
+    config: Pick<ResolvedEvalConfig, "model" | "agentTemperature" | "timeoutMs" | "tokenPricing" | "toolMaxTurns" | "toolMaxArgumentsBytes" | "toolMaxResultBytes">;
+    projectRoot: string;
+    client: EvalLlmClient;
+    tools?: SandboxTool[];
+    /** Override for the SKILL.md loader (test hook). */
+    loadSkill?: (stage: EvalCase["stage"]) => Promise<string>;
+    /** Override for the sandbox factory (test hook). */
+    createSandboxFn?: typeof createSandbox;
+}
+export interface WithToolsOutput {
+    artifact: string;
+    usage: ChatUsage;
+    usageUsd: number;
+    model: string;
+    attempts: number;
+    durationMs: number;
+    toolUse: ToolUseSummary;
+    systemPrompt: string;
+    userPrompt: string;
+}
+export declare function runWithTools(input: WithToolsInput): Promise<WithToolsOutput>;