cclaw-cli 0.23.1 → 0.25.0

package/dist/eval/llm-client.js

@@ -1,19 +1,251 @@
-export class EvalLlmNotWiredError extends Error {
+/**
+ * LLM client for the cclaw eval subsystem.
+ *
+ * Thin adapter over the `openai` SDK pointed at any OpenAI-compatible
+ * `baseURL` (z.ai, OpenAI, vLLM, Ollama+openai-proxy, ...). The surface is
+ * deliberately narrow:
+ *
+ *  - `chat()` — one request/response round-trip with timeout, bounded
+ *    retries on transient errors, and a structured error hierarchy so
+ *    callers can react policy-style (cost-guard, judge, agent-under-test).
+ *  - `ChatRequest` / `ChatResponse` — wire format decoupled from the
+ *    OpenAI types so swapping vendors stays a one-file change.
+ *
+ * Factories stay side-effect-free: no network calls are made until `chat()`
+ * is invoked, so CLI help and dry-run paths never need an API key.
+ */
+import OpenAI from "openai";
+/** Base class so callers can `catch (err) { if (err instanceof EvalLlmError) ... }`. */
+export class EvalLlmError extends Error {
+    retryable;
+    status;
+    constructor(message, opts) {
+        super(message);
+        this.name = "EvalLlmError";
+        this.retryable = opts.retryable;
+        if (opts.status !== undefined)
+            this.status = opts.status;
+        if (opts.cause !== undefined)
+            this.cause = opts.cause;
+    }
+}
+export class EvalLlmAuthError extends EvalLlmError {
+    constructor(cause) {
+        super("LLM request rejected (auth). Check CCLAW_EVAL_API_KEY and provider permissions.", {
+            retryable: false,
+            status: 401,
+            cause
+        });
+        this.name = "EvalLlmAuthError";
+    }
+}
+export class EvalLlmConfigError extends EvalLlmError {
+    constructor(message, cause) {
+        super(message, { retryable: false, cause });
+        this.name = "EvalLlmConfigError";
+    }
+}
+export class EvalLlmTimeoutError extends EvalLlmError {
+    constructor(timeoutMs) {
+        super(`LLM request timed out after ${timeoutMs}ms.`, { retryable: true });
+        this.name = "EvalLlmTimeoutError";
+    }
+}
+export class EvalLlmRateLimitedError extends EvalLlmError {
+    constructor(cause) {
+        super("LLM rate limit hit. Retrying with backoff.", {
+            retryable: true,
+            status: 429,
+            cause
+        });
+        this.name = "EvalLlmRateLimitedError";
+    }
+}
+export class EvalLlmTransportError extends EvalLlmError {
+    constructor(cause, status) {
+        super("LLM transport error.", { retryable: true, status, cause });
+        this.name = "EvalLlmTransportError";
+    }
+}
+export class EvalLlmInvalidResponseError extends EvalLlmError {
+    constructor(message, details) {
+        super(message, { retryable: false });
+        this.name = "EvalLlmInvalidResponseError";
+        if (details)
+            this.details = details;
+    }
+}
+export class EvalLlmNotConfiguredError extends EvalLlmError {
     constructor() {
-        super(`LLM client is not wired yet.\n` +
-            `Run \`cclaw eval --dry-run\` or \`cclaw eval --schema-only\` for offline evals.`);
-        this.name = "EvalLlmNotWiredError";
+        super(`LLM client not configured. Set CCLAW_EVAL_API_KEY (and optionally ` +
+            `CCLAW_EVAL_BASE_URL / CCLAW_EVAL_MODEL) or run with --schema-only / --rules.`, { retryable: false });
+        this.name = "EvalLlmNotConfiguredError";
     }
 }
 /**
- * Factory stub. Throws with a clear message so accidental early usage is
- * easy to diagnose. The real implementation will replace this body with
- * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ * Deprecated shim preserved so older wiring keeps compiling. Prefer
+ * `EvalLlmNotConfiguredError` for the "caller forgot to provide an API
+ * key" case.
  */
-export function createEvalClient(_config) {
+export class EvalLlmNotWiredError extends EvalLlmNotConfiguredError {
+}
+export const DEFAULT_RETRY_POLICY = {
+    maxRetries: 2,
+    initialBackoffMs: 500,
+    maxBackoffMs: 8_000
+};
+function isAbortError(err) {
+    if (err === null || typeof err !== "object")
+        return false;
+    const name = err.name;
+    const code = err.code;
+    return (name === "AbortError" || code === "ABORT_ERR" || code === "ERR_CANCELED");
+}
+function statusFromError(err) {
+    if (err === null || typeof err !== "object")
+        return undefined;
+    const status = err.status;
+    return typeof status === "number" ? status : undefined;
+}
+function normalizeError(err, timeoutMs) {
+    if (err instanceof EvalLlmError)
+        return err;
+    if (isAbortError(err))
+        return new EvalLlmTimeoutError(timeoutMs);
+    const status = statusFromError(err);
+    if (status === 401 || status === 403)
+        return new EvalLlmAuthError(err);
+    if (status === 429)
+        return new EvalLlmRateLimitedError(err);
+    if (status !== undefined && status >= 400 && status < 500) {
+        return new EvalLlmError(`LLM request rejected (HTTP ${status}).`, {
+            retryable: false,
+            status,
+            cause: err
+        });
+    }
+    return new EvalLlmTransportError(err, status);
+}
+function normalizeFinishReason(raw) {
+    switch (raw) {
+        case "length":
+            return "length";
+        case "tool_calls":
+        case "function_call":
+            return "tool_calls";
+        case "content_filter":
+            return "content_filter";
+        case "stop":
+        case null:
+        case undefined:
+        default:
+            return "stop";
+    }
+}
+function buildBody(request) {
+    const body = {
+        model: request.model,
+        messages: request.messages.map((m) => ({
+            role: m.role,
+            content: m.content,
+            ...(m.name !== undefined ? { name: m.name } : {}),
+            ...(m.toolCallId !== undefined ? { tool_call_id: m.toolCallId } : {})
+        }))
+    };
+    if (request.maxTokens !== undefined)
+        body.max_tokens = request.maxTokens;
+    if (request.temperature !== undefined)
+        body.temperature = request.temperature;
+    if (request.seed !== undefined)
+        body.seed = request.seed;
+    if (request.tools !== undefined)
+        body.tools = request.tools;
+    if (request.toolChoice !== undefined)
+        body.tool_choice = request.toolChoice;
+    if (request.responseFormatJson === true) {
+        body.response_format = { type: "json_object" };
+    }
+    return body;
+}
+function defaultSleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function backoffDelay(attempt, policy) {
+    const raw = policy.initialBackoffMs * 2 ** attempt;
+    return Math.min(raw, policy.maxBackoffMs);
+}
+/**
+ * Build a real client pointed at the configured endpoint. Throws
+ * `EvalLlmNotConfiguredError` at call time (not construction time) when no
+ * API key is available, so CLI help and dry-run paths stay offline-safe.
+ */
+export function createEvalClient(config, options = {}) {
+    const retryPolicy = options.retryPolicy ?? {
+        ...DEFAULT_RETRY_POLICY,
+        maxRetries: Math.max(0, config.maxRetries ?? DEFAULT_RETRY_POLICY.maxRetries)
+    };
+    const sleep = options.sleep ?? defaultSleep;
+    let cached;
+    const getClient = () => {
+        if (cached)
+            return cached;
+        if (!config.apiKey)
+            throw new EvalLlmNotConfiguredError();
+        const factory = options.openaiFactory ??
+            ((opts) => new OpenAI(opts));
+        cached = factory({ apiKey: config.apiKey, baseURL: config.baseUrl });
+        return cached;
+    };
     return {
-        async chat() {
-            throw new EvalLlmNotWiredError();
+        async chat(request) {
+            const timeoutMs = Math.max(1_000, request.timeoutMs ?? config.timeoutMs);
+            const body = buildBody(request);
+            const client = getClient();
+            let lastError;
+            const maxAttempts = retryPolicy.maxRetries + 1;
+            for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+                const controller = new AbortController();
+                const handle = setTimeout(() => controller.abort(), timeoutMs);
+                try {
+                    const raw = await client.chat.completions.create(body, {
+                        signal: controller.signal
+                    });
+                    clearTimeout(handle);
+                    const choice = raw.choices?.[0];
+                    if (!choice) {
+                        throw new EvalLlmInvalidResponseError("LLM response contained no choices.", { model: raw.model });
+                    }
+                    const content = choice.message?.content ?? "";
+                    const toolCalls = choice.message?.tool_calls?.map((call) => ({
+                        id: call.id,
+                        name: call.function.name,
+                        arguments: call.function.arguments
+                    }));
+                    const usage = {
+                        promptTokens: raw.usage?.prompt_tokens ?? 0,
+                        completionTokens: raw.usage?.completion_tokens ?? 0,
+                        totalTokens: raw.usage?.total_tokens ?? 0
+                    };
+                    return {
+                        content,
+                        ...(toolCalls && toolCalls.length > 0 ? { toolCalls } : {}),
+                        usage,
+                        finishReason: normalizeFinishReason(choice.finish_reason),
+                        model: raw.model ?? request.model,
+                        attempts: attempt + 1
+                    };
+                }
+                catch (err) {
+                    clearTimeout(handle);
+                    const normalized = normalizeError(err, timeoutMs);
+                    lastError = normalized;
+                    const isLastAttempt = attempt === maxAttempts - 1;
+                    if (!normalized.retryable || isLastAttempt)
+                        throw normalized;
+                    await sleep(backoffDelay(attempt, retryPolicy));
+                }
+            }
+            throw lastError ?? new EvalLlmTransportError(new Error("unknown"));
         }
     };
 }

package/dist/eval/report.js

@@ -75,6 +75,32 @@ export function formatMarkdownReport(report) {
         lines.push(`| ${item.stage} | ${item.caseId} | ${item.passed ? "yes" : "no"} | ${item.durationMs} | ${cost} |`);
     }
     lines.push(``);
+    const judgeCases = report.cases.filter((item) => item.verifierResults.some((r) => r.kind === "judge"));
+    if (judgeCases.length > 0) {
+        lines.push(`## Judge scores`);
+        lines.push(``);
+        lines.push(`| stage | case id | check | median | mean | coverage | ok |`);
+        lines.push(`| --- | --- | --- | --- | --- | --- | --- |`);
+        for (const item of judgeCases) {
+            for (const verifier of item.verifierResults) {
+                if (verifier.kind !== "judge")
+                    continue;
+                if (verifier.id === "judge:required-checks")
+                    continue;
+                if (verifier.id === "judge:rubric:missing")
+                    continue;
+                if (verifier.id === "judge:invocation:error")
+                    continue;
+                const details = verifier.details ?? {};
+                const median = typeof details.median === "number" ? details.median.toFixed(2) : "-";
+                const mean = typeof details.mean === "number" ? details.mean.toFixed(2) : "-";
+                const coverage = details.coverage === true ? "yes" : "no";
+                const checkId = verifier.id.replace(/^judge:/, "");
+                lines.push(`| ${item.stage} | ${item.caseId} | ${checkId} | ${median} | ${mean} | ${coverage} | ${verifier.ok ? "yes" : "no"} |`);
+            }
+        }
+        lines.push(``);
+    }
     lines.push(`## Verifier details`);
     lines.push(``);
     for (const item of report.cases) {

package/dist/eval/rubric-loader.d.ts

@@ -0,0 +1,20 @@
+import type { FlowStage } from "../types.js";
+import type { RubricCheck, RubricDoc } from "./types.js";
+export declare function rubricsDir(projectRoot: string): string;
+export declare function rubricPath(projectRoot: string, stage: FlowStage): string;
+declare function validateCheck(raw: unknown, index: number, file: string): RubricCheck;
+declare function validateRubric(raw: unknown, file: string): RubricDoc;
+/**
+ * Load the rubric for `stage`. Returns `undefined` when the file is
+ * missing so callers can emit a "no rubric" verifier result rather than
+ * crashing — authors are expected to grow rubrics incrementally.
+ */
+export declare function loadRubric(projectRoot: string, stage: FlowStage): Promise<RubricDoc | undefined>;
+/** Load every rubric present in the given rubrics directory. */
+export declare function loadAllRubrics(projectRoot: string): Promise<Map<FlowStage, RubricDoc>>;
+/** Exposed for tests. */
+export declare const __internal: {
+    validateRubric: typeof validateRubric;
+    validateCheck: typeof validateCheck;
+};
+export {};

package/dist/eval/rubric-loader.js

@@ -0,0 +1,143 @@
+/**
+ * Loader + validator for `.cclaw/evals/rubrics/<stage>.yaml`.
+ *
+ * Each file maps to exactly one `RubricDoc` that drives the LLM judge.
+ * Validation is strict: unknown top-level keys, missing required fields,
+ * duplicate check ids, and malformed weights all surface as actionable
+ * errors rather than turning into silent "judge had nothing to score"
+ * passes.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { parse } from "yaml";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+import { FLOW_STAGES } from "../types.js";
+export function rubricsDir(projectRoot) {
+    return path.join(projectRoot, EVALS_ROOT, "rubrics");
+}
+export function rubricPath(projectRoot, stage) {
+    return path.join(rubricsDir(projectRoot), `${stage}.yaml`);
+}
+function rubricError(file, reason) {
+    return new Error(`Invalid rubric at ${file}: ${reason}\n` +
+        `See docs/evals.md for the rubric schema. Fields: stage (required), id (optional, defaults to stage), checks[] with id + prompt.`);
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function validateCheck(raw, index, file) {
+    if (!isRecord(raw)) {
+        throw rubricError(file, `checks[${index}] must be a mapping`);
+    }
+    const id = raw.id;
+    if (typeof id !== "string" || id.trim().length === 0) {
+        throw rubricError(file, `checks[${index}].id must be a non-empty string`);
+    }
+    if (!/^[a-z][a-z0-9-]*$/.test(id)) {
+        throw rubricError(file, `checks[${index}].id "${id}" must be kebab-case (lowercase letters, digits, hyphen; starts with a letter)`);
+    }
+    const prompt = raw.prompt;
+    if (typeof prompt !== "string" || prompt.trim().length === 0) {
+        throw rubricError(file, `checks[${index}].prompt must be a non-empty string`);
+    }
+    const check = {
+        id,
+        prompt: prompt.trim()
+    };
+    if (raw.scale !== undefined) {
+        if (typeof raw.scale !== "string" || raw.scale.trim().length === 0) {
+            throw rubricError(file, `checks[${index}].scale must be a non-empty string when provided`);
+        }
+        check.scale = raw.scale.trim();
+    }
+    if (raw.weight !== undefined) {
+        if (typeof raw.weight !== "number" || !Number.isFinite(raw.weight) || raw.weight < 0) {
+            throw rubricError(file, `checks[${index}].weight must be a non-negative number when provided`);
+        }
+        check.weight = raw.weight;
+    }
+    if (raw.critical !== undefined) {
+        if (typeof raw.critical !== "boolean") {
+            throw rubricError(file, `checks[${index}].critical must be a boolean when provided`);
+        }
+        check.critical = raw.critical;
+    }
+    const known = new Set(["id", "prompt", "scale", "weight", "critical"]);
+    const unknown = Object.keys(raw).filter((key) => !known.has(key));
+    if (unknown.length > 0) {
+        throw rubricError(file, `checks[${index}] has unknown key(s): ${unknown.join(", ")}`);
+    }
+    return check;
+}
+function validateRubric(raw, file) {
+    if (!isRecord(raw)) {
+        throw rubricError(file, "top-level value must be a mapping");
+    }
+    const stage = raw.stage;
+    if (typeof stage !== "string" || !FLOW_STAGES.includes(stage)) {
+        throw rubricError(file, `"stage" must be one of: ${FLOW_STAGES.join(", ")} (got: ${JSON.stringify(stage)})`);
+    }
+    const id = raw.id;
+    let rubricId = stage;
+    if (id !== undefined) {
+        if (typeof id !== "string" || id.trim().length === 0) {
+            throw rubricError(file, `"id" must be a non-empty string when provided`);
+        }
+        rubricId = id.trim();
+    }
+    const checks = raw.checks;
+    if (!Array.isArray(checks) || checks.length === 0) {
+        throw rubricError(file, `"checks" must be a non-empty array`);
+    }
+    const parsed = [];
+    const seen = new Set();
+    for (let i = 0; i < checks.length; i += 1) {
+        const check = validateCheck(checks[i], i, file);
+        if (seen.has(check.id)) {
+            throw rubricError(file, `duplicate check id: "${check.id}"`);
+        }
+        seen.add(check.id);
+        parsed.push(check);
+    }
+    const known = new Set(["stage", "id", "checks"]);
+    const unknown = Object.keys(raw).filter((key) => !known.has(key));
+    if (unknown.length > 0) {
+        throw rubricError(file, `unknown top-level key(s): ${unknown.join(", ")}`);
+    }
+    return {
+        stage: stage,
+        id: rubricId,
+        checks: parsed
+    };
+}
+/**
+ * Load the rubric for `stage`. Returns `undefined` when the file is
+ * missing so callers can emit a "no rubric" verifier result rather than
+ * crashing — authors are expected to grow rubrics incrementally.
+ */
+export async function loadRubric(projectRoot, stage) {
+    const file = rubricPath(projectRoot, stage);
+    if (!(await exists(file)))
+        return undefined;
+    let parsed;
+    try {
+        parsed = parse(await fs.readFile(file, "utf8"));
+    }
+    catch (err) {
+        throw rubricError(file, err instanceof Error ? err.message : String(err));
+    }
+    return validateRubric(parsed, file);
+}
+/** Load every rubric present in the given rubrics directory. */
+export async function loadAllRubrics(projectRoot) {
+    const out = new Map();
+    for (const stage of FLOW_STAGES) {
+        const doc = await loadRubric(projectRoot, stage);
+        if (doc)
+            out.set(stage, doc);
+    }
+    return out;
+}
+/** Exposed for tests. */
+export const __internal = { validateRubric, validateCheck };

package/dist/eval/runner.d.ts

@@ -1,4 +1,5 @@
 import type { FlowStage } from "../types.js";
+import { type EvalLlmClient } from "./llm-client.js";
 import type { EvalReport, EvalTier, ResolvedEvalConfig } from "./types.js";
 export interface RunEvalOptions {
     projectRoot: string;
@@ -14,6 +15,12 @@ export interface RunEvalOptions {
     dryRun?: boolean;
     /** Override process.env during tests. */
     env?: NodeJS.ProcessEnv;
+    /**
+     * Optional LLM client injection. Primary use case: unit and
+     * integration tests that want deterministic judge + agent behavior
+     * without hitting the network.
+     */
+    llmClient?: EvalLlmClient;
 }
 export interface DryRunSummary {
     kind: "dry-run";