cclaw-cli 0.23.1 → 0.25.0

package/dist/eval/runner.js

@@ -1,10 +1,17 @@
 import { randomUUID } from "node:crypto";
 import { CCLAW_VERSION } from "../constants.js";
 import { FLOW_STAGES } from "../types.js";
+import { runSingleShot } from "./agents/single-shot.js";
 import { compareAgainstBaselines, loadBaselinesByStage } from "./baseline.js";
-import { loadCorpus, readFixtureArtifact } from "./corpus.js";
+import { loadCorpus, readExtraFixtures, readFixtureArtifact } from "./corpus.js";
 import { loadEvalConfig } from "./config-loader.js";
+import { createCostGuard, DailyCostCapExceededError } from "./cost-guard.js";
+import { createEvalClient, EvalLlmError } from "./llm-client.js";
+import { loadAllRubrics } from "./rubric-loader.js";
+import { judgeResultsToVerifiers, runJudge } from "./verifiers/judge.js";
+import { verifyRules } from "./verifiers/rules.js";
 import { verifyStructural } from "./verifiers/structural.js";
+import { verifyTraceability } from "./verifiers/traceability.js";
 function groupByStage(cases) {
     return cases.reduce((acc, item) => {
         acc[item.stage] = (acc[item.stage] ?? 0) + 1;
@@ -21,33 +28,131 @@ function skeletonVerifierResult(message, details) {
         ...(details !== undefined ? { details } : {})
     };
 }
-async function runCaseStructural(projectRoot, caseEntry, plannedTier) {
+/**
+ * --schema-only narrows to structural. --rules opens up rules + traceability
+ * on top of structural (traceability is a rule-family verifier even though
+ * it lives in its own module). --judge opens up the LLM judge and, for
+ * Tier A, the single-shot agent-under-test. --schema-only always wins so
+ * the LLM-free PR gate never pays for tokens even if stale flags collide.
+ */
+function resolveRunFlags(options) {
+    const rulesRequested = options.rules === true;
+    const schemaOnly = options.schemaOnly === true;
+    const judgeRequested = options.judge === true;
+    const runJudge = judgeRequested && !schemaOnly;
+    const runAgent = runJudge && (options.tier ?? "A") === "A";
+    return {
+        runStructural: true,
+        runRules: rulesRequested && !schemaOnly,
+        runTraceability: rulesRequested && !schemaOnly,
+        runJudge,
+        runAgent
+    };
+}
+/**
+ * Wrap a client so every chat() result is accounted against the cost
+ * guard before being returned. The guard throws
+ * DailyCostCapExceededError if committing the call would cross the
+ * configured cap — the runner surfaces that as a hard failure so
+ * nightly CI fails loud instead of silently overspending.
+ */
+function wrapClientWithCostGuard(client, costGuard, fallbackModel) {
+    return {
+        async chat(request) {
+            const response = await client.chat(request);
+            await costGuard.commit(response.model || fallbackModel, response.usage);
+            return response;
+        }
+    };
+}
+async function loadArtifactOrRecord(projectRoot, caseEntry, verifierResults) {
+    try {
+        return await readFixtureArtifact(projectRoot, caseEntry);
+    }
+    catch (err) {
+        verifierResults.push({
+            kind: "structural",
+            id: "structural:fixture:missing",
+            ok: false,
+            score: 0,
+            message: err instanceof Error ? err.message : String(err),
+            details: { fixture: caseEntry.fixture }
+        });
+        return undefined;
+    }
+}
+async function runCase(ctx) {
+    const { projectRoot, caseEntry, plannedTier, flags, config, client, costGuard, rubrics } = ctx;
     const started = Date.now();
-    const structuralExpected = caseEntry.expected?.structural;
     const verifierResults = [];
-    if (!structuralExpected || Object.keys(structuralExpected).length === 0) {
-        // No structural expectations declared — case is treated as "N/A" for this
-        // verifier kind; a placeholder pass keeps downstream math simple while
-        // making the situation visible in the report.
-        verifierResults.push(skeletonVerifierResult("No structural expectations declared for this case; structural verifier skipped.", { skipped: true }));
-    }
-    else {
-        let artifact;
-        try {
-            artifact = await readFixtureArtifact(projectRoot, caseEntry);
+    const expected = caseEntry.expected;
+    let caseCostUsd = 0;
+    const hasStructural = !!expected?.structural && Object.keys(expected.structural).length > 0;
+    const hasRules = flags.runRules && !!expected?.rules && Object.keys(expected.rules).length > 0;
+    const hasTraceability = flags.runTraceability && !!expected?.traceability;
+    const judgeRequested = flags.runJudge && !!expected?.judge;
+    const needsArtifact = hasStructural || hasRules || hasTraceability || judgeRequested;
+    let artifact;
+    if (needsArtifact) {
+        if (flags.runAgent && judgeRequested && client) {
+            try {
+                const produced = await runSingleShot({
+                    caseEntry,
+                    config,
+                    projectRoot,
+                    client
+                });
+                artifact = produced.artifact;
+                caseCostUsd += produced.usageUsd;
+                verifierResults.push({
+                    kind: "workflow",
+                    id: "agent:single-shot",
+                    ok: true,
+                    score: 1,
+                    message: `single-shot agent produced ${produced.artifact.length} char(s) in ${produced.durationMs}ms`,
+                    details: {
+                        model: produced.model,
+                        tokensIn: produced.usage.promptTokens,
+                        tokensOut: produced.usage.completionTokens,
+                        usageUsd: produced.usageUsd,
+                        attempts: produced.attempts
+                    }
+                });
+            }
+            catch (err) {
+                if (err instanceof DailyCostCapExceededError)
+                    throw err;
+                const retryable = err instanceof EvalLlmError ? err.retryable : false;
+                verifierResults.push({
+                    kind: "workflow",
+                    id: "agent:single-shot",
+                    ok: false,
+                    score: 0,
+                    message: err instanceof Error ? err.message : String(err),
+                    details: { retryable }
+                });
+            }
         }
-        catch (err) {
+        else {
+            artifact = await loadArtifactOrRecord(projectRoot, caseEntry, verifierResults);
+        }
+        if (artifact === undefined && verifierResults.length === 0) {
             verifierResults.push({
                 kind: "structural",
-                id: "structural:fixture:missing",
+                id: "structural:fixture:absent",
                 ok: false,
                 score: 0,
-                message: err instanceof Error ? err.message : String(err),
-                details: { fixture: caseEntry.fixture }
+                message: "Expectations declared but no fixture path provided. Add `fixture: ./<id>/fixture.md`.",
+                details: { fixtureProvided: false }
             });
         }
-        if (artifact !== undefined) {
-            const results = verifyStructural(artifact, structuralExpected);
+    }
+    if (flags.runStructural) {
+        if (!hasStructural) {
+            verifierResults.push(skeletonVerifierResult("No structural expectations declared for this case; structural verifier skipped.", { skipped: true }));
+        }
+        else if (artifact !== undefined) {
+            const results = verifyStructural(artifact, expected.structural);
             if (results.length === 0) {
                 verifierResults.push(skeletonVerifierResult("Structural expectations parsed but produced zero checks.", { skipped: true }));
             }
@@ -55,24 +160,79 @@ async function runCaseStructural(projectRoot, caseEntry, plannedTier) {
                 verifierResults.push(...results);
             }
         }
-        else if (verifierResults.length === 0) {
+    }
+    if (hasRules && artifact !== undefined) {
+        const results = verifyRules(artifact, expected.rules);
+        verifierResults.push(...results);
+    }
+    if (hasTraceability && artifact !== undefined) {
+        try {
+            const extras = await readExtraFixtures(projectRoot, caseEntry);
+            const results = verifyTraceability(artifact, extras, expected.traceability);
+            verifierResults.push(...results);
+        }
+        catch (err) {
             verifierResults.push({
-                kind: "structural",
-                id: "structural:fixture:absent",
+                kind: "rules",
+                id: "traceability:fixture:missing",
                 ok: false,
                 score: 0,
-                message: "Structural expectations declared but no fixture path provided. Add `fixture: ./<id>/fixture.md`.",
-                details: { fixtureProvided: false }
+                message: err instanceof Error ? err.message : String(err),
+                details: { extraFixtures: Object.keys(caseEntry.extraFixtures ?? {}) }
             });
         }
     }
-    const allOk = verifierResults.every((r) => r.ok);
+    if (judgeRequested && artifact !== undefined && client) {
+        const rubric = rubrics.get(caseEntry.stage);
+        if (!rubric) {
+            verifierResults.push({
+                kind: "judge",
+                id: "judge:rubric:missing",
+                ok: false,
+                score: 0,
+                message: `No rubric at .cclaw/evals/rubrics/${caseEntry.stage}.yaml. Add one before running --judge.`,
+                details: { stage: caseEntry.stage }
+            });
+        }
+        else {
+            try {
+                const invocation = await runJudge({
+                    artifact,
+                    rubric,
+                    config,
+                    client,
+                    caseHint: expected.judge
+                });
+                caseCostUsd += invocation.usageUsd;
+                const judgeVerifiers = judgeResultsToVerifiers(rubric, invocation, config, expected.judge);
+                verifierResults.push(...judgeVerifiers);
+            }
+            catch (err) {
+                if (err instanceof DailyCostCapExceededError)
+                    throw err;
+                const retryable = err instanceof EvalLlmError ? err.retryable : false;
+                verifierResults.push({
+                    kind: "judge",
+                    id: "judge:invocation:error",
+                    ok: false,
+                    score: 0,
+                    message: err instanceof Error ? err.message : String(err),
+                    details: { retryable, rubricId: rubric.id }
+                });
+            }
+        }
+    }
+    const nonSkippedResults = verifierResults.filter((r) => r.details?.skipped !== true);
+    const allOk = nonSkippedResults.length === 0
+        ? verifierResults.every((r) => r.ok)
+        : nonSkippedResults.every((r) => r.ok);
     return {
         caseId: caseEntry.id,
         stage: caseEntry.stage,
         tier: plannedTier,
         passed: allOk,
         durationMs: Date.now() - started,
+        costUsd: caseCostUsd > 0 ? Number(caseCostUsd.toFixed(6)) : undefined,
         verifierResults
     };
 }
@@ -125,11 +285,12 @@ export async function runEval(options) {
     if (corpus.length === 0) {
         notes.push("Corpus is empty. Seed cases live under `.cclaw/evals/corpus/<stage>/*.yaml`.");
     }
-    if (options.rules) {
-        notes.push("--rules is accepted; rule verifiers are not wired yet.");
+    const flags = resolveRunFlags(options);
+    if (flags.runJudge && !config.apiKey && !options.llmClient) {
+        notes.push("--judge requires CCLAW_EVAL_API_KEY (or an injected client for tests); judge pipeline will report errors per case.");
     }
-    if (options.judge) {
-        notes.push("--judge is accepted; LLM judging is not wired yet.");
+    if ((options.tier ?? "A") !== "A" && flags.runJudge) {
+        notes.push("Tier B/C agent-under-test is not wired yet; --judge will score the committed fixture as a stand-in.");
     }
     if (options.dryRun === true) {
         const summary = {
@@ -142,19 +303,37 @@ export async function runEval(options) {
             },
             plannedTier,
             verifiersAvailable: {
-                structural: true,
-                rules: false,
-                judge: false,
-                workflow: false
+                structural: flags.runStructural,
+                rules: flags.runRules,
+                judge: flags.runJudge,
+                workflow: flags.runAgent
             },
             notes
         };
         return summary;
     }
+    const costGuard = createCostGuard(options.projectRoot, config);
+    let wrappedClient;
+    if (flags.runJudge) {
+        const base = options.llmClient ?? createEvalClient(config);
+        wrappedClient = wrapClientWithCostGuard(base, costGuard, config.judgeModel ?? config.model);
+    }
+    const rubrics = flags.runJudge
+        ? await loadAllRubrics(options.projectRoot)
+        : new Map();
     const now = new Date().toISOString();
     const caseResults = [];
     for (const item of corpus) {
-        caseResults.push(await runCaseStructural(options.projectRoot, item, plannedTier));
+        caseResults.push(await runCase({
+            projectRoot: options.projectRoot,
+            caseEntry: item,
+            plannedTier,
+            flags,
+            config,
+            client: wrappedClient,
+            costGuard,
+            rubrics
+        }));
     }
     const stages = stagesInResults(caseResults);
     const baselines = await loadBaselinesByStage(options.projectRoot, stages);

package/dist/eval/types.d.ts

@@ -58,13 +58,96 @@ export interface StructuralExpected {
      */
     requiredFrontmatterKeys?: string[];
 }
-/** Superset of per-verifier expectation shapes. Only `structural` is wired in Step 1. */
+/**
+ * Rule-based expectations — zero-LLM content checks that are richer than
+ * structural (regex, numeric bounds, uniqueness). Introduced in Step 2.
+ *
+ * Every array field is optional; an empty `RulesExpected` produces zero
+ * verifier results so authors can enable rules incrementally.
+ */
+export interface RulesExpected {
+    /** Case-insensitive substrings the body must include at least once. */
+    mustContain?: string[];
+    /** Case-insensitive substrings the body must NOT include. */
+    mustNotContain?: string[];
+    /** Regex patterns that must match the body at least once. */
+    regexRequired?: RuleRegex[];
+    /** Regex patterns that must NOT match the body. */
+    regexForbidden?: RuleRegex[];
+    /** For each substring key, the body must contain at least N occurrences. */
+    minOccurrences?: Record<string, number>;
+    /** For each substring key, the body must contain at most N occurrences. */
+    maxOccurrences?: Record<string, number>;
+    /**
+     * For each named section (case-insensitive heading substring), every bullet
+     * (`- ...`) directly under the section must be unique. Catches duplicated
+     * decisions or repeated risks.
+     */
+    uniqueBulletsInSection?: string[];
+}
+export interface RuleRegex {
+    /** Source of the regex. Parsed with `new RegExp(pattern, flags)`. */
+    pattern: string;
+    /** Optional regex flags; defaults to `"i"` for case-insensitive matching. */
+    flags?: string;
+    /** Human-readable label rendered in verifier messages and slugged into the id. */
+    description?: string;
+}
+/**
+ * Cross-stage traceability expectations — assert every ID extracted from
+ * `source` also appears in `self` and/or named `extra_fixtures`. Introduced
+ * in Step 2.
+ */
+export interface TraceabilityExpected {
+    /** Regex applied to the `source` fixture to collect the authoritative ID set. */
+    idPattern: string;
+    /** Optional regex flags (defaults to `"g"`). */
+    idFlags?: string;
+    /**
+     * Where to read the authoritative ID set from. Either `"self"` (the case's
+     * primary `fixture`) or a label present in the case's `extraFixtures` map.
+     */
+    source: string;
+    /**
+     * Where every source ID must also appear. Each entry is `"self"` or an
+     * `extraFixtures` label. Order is preserved for deterministic result ids.
+     */
+    requireIn: string[];
+}
+/**
+ * LLM-judge expectations — Step 3.
+ *
+ * When present, the judge runs against the resolved artifact (live-agent
+ * output in Tier A/B/C, or the pre-generated fixture when `--judge` is
+ * combined with `--schema-only` for smoke tests). Every field below is
+ * optional; the case-level hint overlays the stage-level rubric loaded
+ * from `.cclaw/evals/rubrics/<stage>.yaml`.
+ */
+export interface JudgeExpected {
+    /**
+     * Per-case check ids that MUST be present in the stage rubric. Used when
+     * a case wants to assert the rubric covers scenario-specific properties.
+     */
+    requiredChecks?: string[];
+    /**
+     * Stage rubric identifier when a stage ships multiple rubrics (e.g.
+     * "strict" vs. "lenient"). Defaults to the stage name.
+     */
+    rubric?: string;
+    /** Optional override of `config.judgeSamples` for the case. */
+    samples?: number;
+    /** Per-check minimum score (1..5 scale). Fail when any score drops below. */
+    minimumScores?: Record<string, number>;
+}
+/** Superset of per-verifier expectation shapes. */
 export interface ExpectedShape {
     structural?: StructuralExpected;
-    /** Rule-based (keyword/regex/traceability) checks — Step 2. */
-    rules?: Record<string, unknown>;
+    /** Rule-based (keyword/regex/count/uniqueness) checks — Step 2. */
+    rules?: RulesExpected;
+    /** Cross-stage ID propagation checks — Step 2. */
+    traceability?: TraceabilityExpected;
     /** LLM-judge rubrics — Step 3. */
-    judge?: Record<string, unknown>;
+    judge?: JudgeExpected;
 }
 /**
  * A single eval case describes one input scenario for one stage. Cases live in
@@ -89,6 +172,13 @@ export interface EvalCase {
      * Step 1 development aid.
      */
     fixture?: string;
+    /**
+     * Additional fixture paths loaded alongside the primary `fixture`, keyed
+     * by a free-form label. Consumed by cross-artifact verifiers (e.g.,
+     * traceability) introduced in Step 2. Paths are resolved relative to the
+     * case's stage directory, just like `fixture`.
+     */
+    extraFixtures?: Record<string, string>;
 }
 /** Result of one verifier applied to one case. */
 export interface VerifierResult {
@@ -163,6 +253,26 @@ export interface EvalConfig {
     timeoutMs: number;
     /** Max retries per API call on transient failures. */
     maxRetries: number;
+    /**
+     * Number of judge samples per case (median-of-N). Defaults to 3 when unset.
+     * Must be odd so a true median exists.
+     */
+    judgeSamples?: number;
+    /** Sampling temperature for judge calls. Defaults to 0.0. */
+    judgeTemperature?: number;
+    /** Sampling temperature for the agent-under-test. Defaults to 0.2. */
+    agentTemperature?: number;
+    /**
+     * Optional per-model USD pricing used by the cost guard. Keys match
+     * `model` / `judgeModel`. Values in USD per 1K tokens, so
+     * `{ input: 0.0005, output: 0.0015 }` = $0.50 per 1M input tokens.
+     */
+    tokenPricing?: Record<string, TokenPricing>;
+}
+/** Per-model pricing schedule, expressed as USD per 1K tokens. */
+export interface TokenPricing {
+    input: number;
+    output: number;
 }
 /** Resolved config with env overrides applied. */
 export interface ResolvedEvalConfig extends EvalConfig {
@@ -214,3 +324,60 @@ export interface BaselineRegression {
     previousScore?: number;
     currentScore?: number;
 }
+/**
+ * One rubric check evaluated by the LLM judge. Scored on a 1..5 scale;
+ * 5 means "the artifact fully meets the bar described by `prompt`".
+ */
+export interface RubricCheck {
+    /** Kebab-case slug, unique per rubric. Stable across runs. */
+    id: string;
+    /** Natural-language question posed to the judge. */
+    prompt: string;
+    /** Human-readable scale description rendered in judge prompts. */
+    scale?: string;
+    /** Relative weight for the stage's aggregate score. Defaults to 1.0. */
+    weight?: number;
+    /**
+     * When true, any sample below `config.regression.failIfCriticalBelow`
+     * flips the verifier to `ok:false` (not just a score drop).
+     */
+    critical?: boolean;
+}
+/** Parsed `.cclaw/evals/rubrics/<stage>.yaml`. */
+export interface RubricDoc {
+    stage: FlowStage;
+    /** Optional rubric variant label; defaults to the stage name. */
+    id: string;
+    checks: RubricCheck[];
+}
+/**
+ * Judge response for a single sample (one API call). The judge is asked to
+ * return structured JSON; `scores[id]` maps rubric check id → integer 1..5.
+ * `rationales[id]` is a short plain-text explanation, useful in reports but
+ * never used for gating.
+ */
+export interface JudgeSample {
+    scores: Record<string, number>;
+    rationales: Record<string, string>;
+}
+/** Aggregated judge output across N samples, per rubric check. */
+export interface JudgeAggregate {
+    checkId: string;
+    samples: number[];
+    median: number;
+    mean: number;
+    /** True iff every sample returned a score for this check. */
+    coverage: boolean;
+}
+/**
+ * Judge invocation result. Produced by `runJudge` and consumed by the
+ * runner: the runner converts each aggregate into a `VerifierResult` and
+ * records `usageUsd` toward the per-case cost.
+ */
+export interface JudgeInvocation {
+    rubricId: string;
+    samples: JudgeSample[];
+    aggregates: JudgeAggregate[];
+    usageUsd: number;
+    durationMs: number;
+}

package/dist/eval/verifiers/judge.d.ts

@@ -0,0 +1,40 @@
+/**
+ * LLM judge verifier — Step 3.
+ *
+ * Given an artifact and the stage's rubric, runs N judge samples (default
+ * median-of-3) against the configured LLM, aggregates the per-check
+ * scores, and returns one VerifierResult per rubric check plus one
+ * aggregate result covering the whole stage.
+ *
+ * Deterministic pieces (JSON parsing, aggregation, scoring) are kept pure
+ * so unit tests inject a stub EvalLlmClient and assert on the aggregate
+ * math without touching the network.
+ */
+import { type EvalLlmClient } from "../llm-client.js";
+import type { JudgeExpected, JudgeInvocation, JudgeSample, ResolvedEvalConfig, RubricDoc, VerifierResult } from "../types.js";
+export interface RunJudgeOptions {
+    artifact: string;
+    rubric: RubricDoc;
+    config: Pick<ResolvedEvalConfig, "model" | "judgeModel" | "judgeSamples" | "judgeTemperature" | "timeoutMs" | "tokenPricing">;
+    client: EvalLlmClient;
+    /** Per-case hint that overlays the rubric (sample count, minimums). */
+    caseHint?: JudgeExpected;
+    /** Optional seed seed; incremented per sample for reproducibility. */
+    baseSeed?: number;
+}
+/**
+ * Parse one judge response into a JudgeSample. The parser is intentionally
+ * forgiving with rationales (missing -> empty string) but strict with
+ * scores: missing or non-numeric entries are dropped and the coverage
+ * flag on the aggregate flips to false.
+ */
+export declare function parseJudgeResponse(content: string, rubric: RubricDoc): JudgeSample;
+/** Run the judge against an artifact and return per-sample + aggregate data. */
+export declare function runJudge(options: RunJudgeOptions): Promise<JudgeInvocation>;
+/**
+ * Convert a JudgeInvocation into VerifierResult[] for the runner. One
+ * result per rubric check (score 0..1 normalized from the 1..5 median) +
+ * one "coverage" result that flips to `ok:false` when any sample failed
+ * to emit a score for a check.
+ */
+export declare function judgeResultsToVerifiers(rubric: RubricDoc, invocation: JudgeInvocation, config: Pick<ResolvedEvalConfig, "regression">, caseHint?: JudgeExpected): VerifierResult[];