cclaw-cli 0.49.0 → 0.51.1

package/dist/eval/report.js

@@ -1,181 +0,0 @@
-import path from "node:path";
-import { EVALS_ROOT } from "../constants.js";
-import { writeFileSafe } from "../fs-utils.js";
-export function reportsDir(projectRoot) {
-    return path.join(projectRoot, EVALS_ROOT, "reports");
-}
-export function defaultReportBasename(report) {
-    const ts = report.generatedAt.replace(/[:.]/g, "-");
-    return `eval-${ts}-${report.runId.slice(0, 8)}`;
-}
-/**
- * Format a report as a human-readable Markdown document. Keeping the layout
- * stable matters: CI posts diffs against earlier reports, and unit tests use
- * the output as a regression guard.
- */
-export function formatMarkdownReport(report) {
-    const { summary } = report;
-    const stages = report.stages.length > 0 ? report.stages.join(", ") : "all";
-    const lines = [];
-    lines.push(`# cclaw eval report`);
-    lines.push(``);
-    lines.push(`- generated: ${report.generatedAt}`);
-    lines.push(`- runId: ${report.runId}`);
-    lines.push(`- cclaw version: ${report.cclawVersion}`);
-    lines.push(`- provider: ${report.provider}`);
-    lines.push(`- model: ${report.model}`);
-    lines.push(`- mode: ${report.mode}`);
-    lines.push(`- stages: ${stages}`);
-    lines.push(``);
-    lines.push(`## Summary`);
-    lines.push(``);
-    lines.push(`| metric | value |`);
-    lines.push(`| --- | --- |`);
-    lines.push(`| total cases | ${summary.totalCases} |`);
-    lines.push(`| passed | ${summary.passed} |`);
-    lines.push(`| failed | ${summary.failed} |`);
-    lines.push(`| skipped | ${summary.skipped} |`);
-    lines.push(`| total cost (USD) | ${summary.totalCostUsd.toFixed(4)} |`);
-    lines.push(`| total duration (ms) | ${summary.totalDurationMs} |`);
-    lines.push(``);
-    if (report.baselineDelta) {
-        const delta = report.baselineDelta;
-        lines.push(`## Baseline delta`);
-        lines.push(``);
-        lines.push(`- baseline: ${delta.baselineId}`);
-        lines.push(`- score delta: ${delta.scoreDelta.toFixed(4)}`);
-        lines.push(`- critical failures: ${delta.criticalFailures}`);
-        lines.push(``);
-        if (delta.regressions.length > 0) {
-            lines.push(`### Regressions`);
-            lines.push(``);
-            lines.push(`| stage | case id | verifier | reason | prev | curr |`);
-            lines.push(`| --- | --- | --- | --- | --- | --- |`);
-            for (const reg of delta.regressions) {
-                const prev = reg.previousScore !== undefined ? reg.previousScore.toFixed(2) : "-";
-                const curr = reg.currentScore !== undefined ? reg.currentScore.toFixed(2) : "-";
-                lines.push(`| ${reg.stage} | ${reg.caseId} | ${reg.verifierId} | ${reg.reason} | ${prev} | ${curr} |`);
-            }
-            lines.push(``);
-        }
-    }
-    if (report.cases.length === 0) {
-        lines.push(`## Cases`);
-        lines.push(``);
-        lines.push(`No cases were executed. See \`docs/evals.md\` for the rollout plan.`);
-        lines.push(``);
-        return `${lines.join("\n")}\n`;
-    }
-    lines.push(`## Cases`);
-    lines.push(``);
-    lines.push(`| stage | case id | passed | duration (ms) | cost (USD) |`);
-    lines.push(`| --- | --- | --- | --- | --- |`);
-    for (const item of report.cases) {
-        const cost = item.costUsd !== undefined ? item.costUsd.toFixed(4) : "-";
-        lines.push(`| ${item.stage} | ${item.caseId} | ${item.passed ? "yes" : "no"} | ${item.durationMs} | ${cost} |`);
-    }
-    lines.push(``);
-    const toolCases = report.cases.filter((item) => item.verifierResults.some((r) => r.id === "agent:with-tools" && typeof r.details?.toolUse === "object"));
-    if (toolCases.length > 0) {
-        lines.push(`## Tool use`);
-        lines.push(``);
-        lines.push(`| stage | case id | turns | calls | errors | denied | by tool |`);
-        lines.push(`| --- | --- | --- | --- | --- | --- | --- |`);
-        for (const item of toolCases) {
-            const verifier = item.verifierResults.find((r) => r.id === "agent:with-tools");
-            const toolUse = verifier?.details?.toolUse;
-            if (!toolUse)
-                continue;
-            const byTool = Object.entries(toolUse.byTool)
-                .map(([name, count]) => `${name}=${count}`)
-                .join(", ");
-            const denied = toolUse.deniedPaths.length > 0 ? toolUse.deniedPaths.length : "0";
-            lines.push(`| ${item.stage} | ${item.caseId} | ${toolUse.turns} | ${toolUse.calls} | ${toolUse.errors} | ${denied} | ${byTool || "-"} |`);
-        }
-        lines.push(``);
-    }
-    const judgeCases = report.cases.filter((item) => item.verifierResults.some((r) => r.kind === "judge"));
-    if (judgeCases.length > 0) {
-        lines.push(`## Judge scores`);
-        lines.push(``);
-        lines.push(`| stage | case id | check | median | mean | coverage | ok |`);
-        lines.push(`| --- | --- | --- | --- | --- | --- | --- |`);
-        for (const item of judgeCases) {
-            for (const verifier of item.verifierResults) {
-                if (verifier.kind !== "judge")
-                    continue;
-                if (verifier.id === "judge:required-checks")
-                    continue;
-                if (verifier.id === "judge:rubric:missing")
-                    continue;
-                if (verifier.id === "judge:invocation:error")
-                    continue;
-                const details = verifier.details ?? {};
-                const median = typeof details.median === "number" ? details.median.toFixed(2) : "-";
-                const mean = typeof details.mean === "number" ? details.mean.toFixed(2) : "-";
-                const coverage = details.coverage === true ? "yes" : "no";
-                const checkId = verifier.id.replace(/^judge:/, "");
-                lines.push(`| ${item.stage} | ${item.caseId} | ${checkId} | ${median} | ${mean} | ${coverage} | ${verifier.ok ? "yes" : "no"} |`);
-            }
-        }
-        lines.push(``);
-    }
-    const workflowCases = report.cases.filter((item) => !!item.workflow);
-    if (workflowCases.length > 0) {
-        lines.push(`## Workflow stages`);
-        lines.push(``);
-        lines.push(`| case id | stage | duration (ms) | cost (USD) | turns | tool calls | judge ok |`);
-        lines.push(`| --- | --- | --- | --- | --- | --- | --- |`);
-        for (const item of workflowCases) {
-            const wf = item.workflow;
-            for (const stage of wf.stages) {
-                const cost = stage.usageUsd > 0 ? stage.usageUsd.toFixed(4) : "-";
-                const judgeOk = stage.judgeOk === true ? "yes" : stage.judgeOk === false ? "no" : "-";
-                lines.push(`| ${item.caseId} | ${stage.stage} | ${stage.durationMs} | ${cost} | ` +
-                    `${stage.toolUse.turns} | ${stage.toolUse.calls} | ${judgeOk} |`);
-            }
-        }
-        lines.push(``);
-    }
-    const consistencyCases = report.cases.filter((item) => item.verifierResults.some((r) => r.kind === "consistency"));
-    if (consistencyCases.length > 0) {
-        lines.push(`## Consistency checks`);
-        lines.push(``);
-        lines.push(`| case id | check id | ok | message |`);
-        lines.push(`| --- | --- | --- | --- |`);
-        for (const item of consistencyCases) {
-            for (const verifier of item.verifierResults) {
-                if (verifier.kind !== "consistency")
-                    continue;
-                const message = verifier.message
-                    ? verifier.message.replace(/\|/g, "\\|").slice(0, 160)
-                    : "-";
-                lines.push(`| ${item.caseId} | ${verifier.id} | ${verifier.ok ? "yes" : "no"} | ${message} |`);
-            }
-        }
-        lines.push(``);
-    }
-    lines.push(`## Verifier details`);
-    lines.push(``);
-    for (const item of report.cases) {
-        lines.push(`### ${item.stage} / ${item.caseId}`);
-        lines.push(``);
-        for (const verifier of item.verifierResults) {
-            const score = verifier.score !== undefined ? ` (score=${verifier.score.toFixed(2)})` : "";
-            lines.push(`- ${verifier.kind} / ${verifier.id}: ${verifier.ok ? "ok" : "fail"}${score}` +
-                (verifier.message ? ` — ${verifier.message}` : ""));
-        }
-        lines.push(``);
-    }
-    return `${lines.join("\n")}\n`;
-}
-export async function writeJsonReport(projectRoot, report, basename = defaultReportBasename(report)) {
-    const outPath = path.join(reportsDir(projectRoot), `${basename}.json`);
-    await writeFileSafe(outPath, `${JSON.stringify(report, null, 2)}\n`);
-    return outPath;
-}
-export async function writeMarkdownReport(projectRoot, report, basename = defaultReportBasename(report)) {
-    const outPath = path.join(reportsDir(projectRoot), `${basename}.md`);
-    await writeFileSafe(outPath, formatMarkdownReport(report));
-    return outPath;
-}

package/dist/eval/rubric-loader.d.ts

@@ -1,20 +0,0 @@
-import type { FlowStage } from "../types.js";
-import type { RubricCheck, RubricDoc } from "./types.js";
-export declare function rubricsDir(projectRoot: string): string;
-export declare function rubricPath(projectRoot: string, stage: FlowStage): string;
-declare function validateCheck(raw: unknown, index: number, file: string): RubricCheck;
-declare function validateRubric(raw: unknown, file: string): RubricDoc;
-/**
- * Load the rubric for `stage`. Returns `undefined` when the file is
- * missing so callers can emit a "no rubric" verifier result rather than
- * crashing — authors are expected to grow rubrics incrementally.
- */
-export declare function loadRubric(projectRoot: string, stage: FlowStage): Promise<RubricDoc | undefined>;
-/** Load every rubric present in the given rubrics directory. */
-export declare function loadAllRubrics(projectRoot: string): Promise<Map<FlowStage, RubricDoc>>;
-/** Exposed for tests. */
-export declare const __internal: {
-    validateRubric: typeof validateRubric;
-    validateCheck: typeof validateCheck;
-};
-export {};

package/dist/eval/rubric-loader.js

@@ -1,143 +0,0 @@
-/**
- * Loader + validator for `.cclaw/evals/rubrics/<stage>.yaml`.
- *
- * Each file maps to exactly one `RubricDoc` that drives the LLM judge.
- * Validation is strict: unknown top-level keys, missing required fields,
- * duplicate check ids, and malformed weights all surface as actionable
- * errors rather than turning into silent "judge had nothing to score"
- * passes.
- */
-import fs from "node:fs/promises";
-import path from "node:path";
-import { parse } from "yaml";
-import { EVALS_ROOT } from "../constants.js";
-import { exists } from "../fs-utils.js";
-import { FLOW_STAGES } from "../types.js";
-export function rubricsDir(projectRoot) {
-    return path.join(projectRoot, EVALS_ROOT, "rubrics");
-}
-export function rubricPath(projectRoot, stage) {
-    return path.join(rubricsDir(projectRoot), `${stage}.yaml`);
-}
-function rubricError(file, reason) {
-    return new Error(`Invalid rubric at ${file}: ${reason}\n` +
-        `See docs/evals.md for the rubric schema. Fields: stage (required), id (optional, defaults to stage), checks[] with id + prompt.`);
-}
-function isRecord(value) {
-    return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function validateCheck(raw, index, file) {
-    if (!isRecord(raw)) {
-        throw rubricError(file, `checks[${index}] must be a mapping`);
-    }
-    const id = raw.id;
-    if (typeof id !== "string" || id.trim().length === 0) {
-        throw rubricError(file, `checks[${index}].id must be a non-empty string`);
-    }
-    if (!/^[a-z][a-z0-9-]*$/.test(id)) {
-        throw rubricError(file, `checks[${index}].id "${id}" must be kebab-case (lowercase letters, digits, hyphen; starts with a letter)`);
-    }
-    const prompt = raw.prompt;
-    if (typeof prompt !== "string" || prompt.trim().length === 0) {
-        throw rubricError(file, `checks[${index}].prompt must be a non-empty string`);
-    }
-    const check = {
-        id,
-        prompt: prompt.trim()
-    };
-    if (raw.scale !== undefined) {
-        if (typeof raw.scale !== "string" || raw.scale.trim().length === 0) {
-            throw rubricError(file, `checks[${index}].scale must be a non-empty string when provided`);
-        }
-        check.scale = raw.scale.trim();
-    }
-    if (raw.weight !== undefined) {
-        if (typeof raw.weight !== "number" || !Number.isFinite(raw.weight) || raw.weight < 0) {
-            throw rubricError(file, `checks[${index}].weight must be a non-negative number when provided`);
-        }
-        check.weight = raw.weight;
-    }
-    if (raw.critical !== undefined) {
-        if (typeof raw.critical !== "boolean") {
-            throw rubricError(file, `checks[${index}].critical must be a boolean when provided`);
-        }
-        check.critical = raw.critical;
-    }
-    const known = new Set(["id", "prompt", "scale", "weight", "critical"]);
-    const unknown = Object.keys(raw).filter((key) => !known.has(key));
-    if (unknown.length > 0) {
-        throw rubricError(file, `checks[${index}] has unknown key(s): ${unknown.join(", ")}`);
-    }
-    return check;
-}
-function validateRubric(raw, file) {
-    if (!isRecord(raw)) {
-        throw rubricError(file, "top-level value must be a mapping");
-    }
-    const stage = raw.stage;
-    if (typeof stage !== "string" || !FLOW_STAGES.includes(stage)) {
-        throw rubricError(file, `"stage" must be one of: ${FLOW_STAGES.join(", ")} (got: ${JSON.stringify(stage)})`);
-    }
-    const id = raw.id;
-    let rubricId = stage;
-    if (id !== undefined) {
-        if (typeof id !== "string" || id.trim().length === 0) {
-            throw rubricError(file, `"id" must be a non-empty string when provided`);
-        }
-        rubricId = id.trim();
-    }
-    const checks = raw.checks;
-    if (!Array.isArray(checks) || checks.length === 0) {
-        throw rubricError(file, `"checks" must be a non-empty array`);
-    }
-    const parsed = [];
-    const seen = new Set();
-    for (let i = 0; i < checks.length; i += 1) {
-        const check = validateCheck(checks[i], i, file);
-        if (seen.has(check.id)) {
-            throw rubricError(file, `duplicate check id: "${check.id}"`);
-        }
-        seen.add(check.id);
-        parsed.push(check);
-    }
-    const known = new Set(["stage", "id", "checks"]);
-    const unknown = Object.keys(raw).filter((key) => !known.has(key));
-    if (unknown.length > 0) {
-        throw rubricError(file, `unknown top-level key(s): ${unknown.join(", ")}`);
-    }
-    return {
-        stage: stage,
-        id: rubricId,
-        checks: parsed
-    };
-}
-/**
- * Load the rubric for `stage`. Returns `undefined` when the file is
- * missing so callers can emit a "no rubric" verifier result rather than
- * crashing — authors are expected to grow rubrics incrementally.
- */
-export async function loadRubric(projectRoot, stage) {
-    const file = rubricPath(projectRoot, stage);
-    if (!(await exists(file)))
-        return undefined;
-    let parsed;
-    try {
-        parsed = parse(await fs.readFile(file, "utf8"));
-    }
-    catch (err) {
-        throw rubricError(file, err instanceof Error ? err.message : String(err));
-    }
-    return validateRubric(parsed, file);
-}
-/** Load every rubric present in the given rubrics directory. */
-export async function loadAllRubrics(projectRoot) {
-    const out = new Map();
-    for (const stage of FLOW_STAGES) {
-        const doc = await loadRubric(projectRoot, stage);
-        if (doc)
-            out.set(stage, doc);
-    }
-    return out;
-}
-/** Exposed for tests. */
-export const __internal = { validateRubric, validateCheck };

package/dist/eval/runner.d.ts

@@ -1,81 +0,0 @@
-import type { FlowStage } from "../types.js";
-import { type EvalLlmClient } from "./llm-client.js";
-import { type ProgressLogger } from "./progress.js";
-import type { EvalMode, EvalReport, ResolvedEvalConfig, WorkflowStageName } from "./types.js";
-export interface RunEvalOptions {
-    projectRoot: string;
-    stage?: FlowStage;
-    mode?: EvalMode;
-    /** When true, run only structural verifiers (Step 1). */
-    schemaOnly?: boolean;
-    /** When true, run structural + rule-based verifiers. Step 2 wires rules. */
-    rules?: boolean;
-    /** When true, also run LLM judge verifiers. Step 3 wires judging. */
-    judge?: boolean;
-    /** When true, load config + corpus and return a summary without running any verifier. */
-    dryRun?: boolean;
-    /** Override process.env during tests. */
-    env?: NodeJS.ProcessEnv;
-    /**
-     * Optional LLM client injection. Primary use case: unit and
-     * integration tests that want deterministic judge + agent behavior
-     * without hitting the network.
-     */
-    llmClient?: EvalLlmClient;
-    /**
-     * Optional progress logger. The CLI wires a stderr-backed logger by
-     * default so users see one-line updates during long runs; tests and
-     * programmatic callers can inject a silent (noop) logger or capture
-     * events for assertions. When omitted, progress is silenced.
-     */
-    progress?: ProgressLogger;
-    /**
-     * Per-run USD cap. Enforced in-memory; independent from the daily cap
-     * (`dailyUsdCap` / `CCLAW_EVAL_DAILY_USD_CAP`) that persists across
-     * invocations. Undefined means no cap.
-     */
-    maxCostUsd?: number;
-    /**
-     * Override the configured `model` (and `judgeModel`) for this run.
-     * Used by `cclaw eval --compare-model` to replay the same corpus
-     * against an alternative model without editing `config.yaml`.
-     */
-    modelOverride?: string;
-}
-export interface DryRunSummary {
-    kind: "dry-run";
-    config: ResolvedEvalConfig;
-    corpus: {
-        total: number;
-        byStage: Record<string, number>;
-        cases: Array<{
-            id: string;
-            stage: FlowStage;
-        }>;
-    };
-    /** Only populated in `workflow` mode; empty for fixture / agent modes. */
-    workflowCorpus: {
-        total: number;
-        cases: Array<{
-            id: string;
-            stages: WorkflowStageName[];
-        }>;
-    };
-    plannedMode: EvalMode;
-    verifiersAvailable: {
-        structural: boolean;
-        rules: boolean;
-        judge: boolean;
-        workflow: boolean;
-        consistency: boolean;
-    };
-    notes: string[];
-}
-/**
- * Main eval runner. Dispatches between fixture-backed verification, the
- * single-stage agent-with-tools loop, and the multi-stage workflow
- * orchestrator based on `options.mode`. Per-stage baselines are loaded for
- * regression comparison. Cases without a `fixture` path in the yaml are
- * marked skipped (not failed) when no LLM drafting runs.
- */
-export declare function runEval(options: RunEvalOptions): Promise<DryRunSummary | EvalReport>;