cclaw-cli 0.24.0 → 0.25.0

package/dist/eval/cost-guard.js

@@ -0,0 +1,153 @@
+/**
+ * Cost guard for the cclaw eval subsystem.
+ *
+ * Two responsibilities:
+ *
+ * 1. Convert `ChatUsage` (prompt/completion token counts) into USD using
+ *    a per-model `TokenPricing` schedule. Pricing comes from
+ *    `config.tokenPricing[model]` first, then from the builtin fallback
+ *    schedule for well-known models (z.ai GLM 5.1 at publish time).
+ * 2. Maintain a per-day running total persisted to
+ *    `.cclaw/evals/.spend-YYYY-MM-DD.json` so that a long eval session
+ *    (or a cron-run nightly) can't blow through the configured
+ *    `dailyUsdCap`. The counter is opt-in: no cap, no writes.
+ *
+ * The guard is deliberately pessimistic — it rounds USD up to 6 decimals
+ * and never subtracts, so a CI run that errors mid-flight still shows the
+ * partial spend in the next report.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+/**
+ * Builtin pricing fallback. Intentionally conservative: when the user
+ * hasn't configured pricing and we don't know the model, we default to a
+ * "small model" USD schedule so the cap can still do something useful.
+ *
+ * Values are USD per 1K tokens. Sources are public pricing pages as of
+ * 2026-04; update by editing this constant, not the guard logic.
+ */
+export const DEFAULT_TOKEN_PRICING = {
+    "glm-5.1": { input: 0.0005, output: 0.0015 },
+    "glm-4.6": { input: 0.0005, output: 0.0015 },
+    "gpt-4o-mini": { input: 0.00015, output: 0.0006 },
+    "gpt-4o": { input: 0.005, output: 0.015 }
+};
+/** Hard default when neither config nor builtins know the model. */
+export const UNKNOWN_MODEL_PRICING = { input: 0.001, output: 0.003 };
+export class DailyCostCapExceededError extends Error {
+    capUsd;
+    projectedUsd;
+    currentUsd;
+    constructor(opts) {
+        super(`Daily cost cap would be exceeded: ` +
+            `current=$${opts.currentUsd.toFixed(4)}, ` +
+            `projected=$${opts.projectedUsd.toFixed(4)}, ` +
+            `cap=$${opts.capUsd.toFixed(4)}. ` +
+            `Unset CCLAW_EVAL_DAILY_USD_CAP or increase the cap to continue.`);
+        this.name = "DailyCostCapExceededError";
+        this.capUsd = opts.capUsd;
+        this.projectedUsd = opts.projectedUsd;
+        this.currentUsd = opts.currentUsd;
+    }
+}
+function utcDate(now = new Date()) {
+    return now.toISOString().slice(0, 10);
+}
+function pricingFor(model, config) {
+    const custom = config.tokenPricing?.[model];
+    if (custom)
+        return custom;
+    const builtin = DEFAULT_TOKEN_PRICING[model];
+    if (builtin)
+        return builtin;
+    return UNKNOWN_MODEL_PRICING;
+}
+/**
+ * Compute USD cost of a single `ChatUsage` using the given `model` pricing
+ * schedule. Returns 0 when `usage.totalTokens` is 0 (e.g. transport error
+ * before first token).
+ */
+export function computeUsageUsd(model, usage, config) {
+    if (!usage || usage.totalTokens <= 0)
+        return 0;
+    const schedule = pricingFor(model, config);
+    const cost = (usage.promptTokens * schedule.input) / 1_000 +
+        (usage.completionTokens * schedule.output) / 1_000;
+    return Math.max(0, Number(cost.toFixed(6)));
+}
+function emptyLedger(date) {
+    return { date, totalUsd: 0, calls: 0, byModel: {} };
+}
+function ledgerPath(projectRoot, date) {
+    return path.join(projectRoot, EVALS_ROOT, `.spend-${date}.json`);
+}
+async function readLedger(file, date) {
+    if (!(await exists(file)))
+        return emptyLedger(date);
+    try {
+        const raw = JSON.parse(await fs.readFile(file, "utf8"));
+        if (raw?.date !== date)
+            return emptyLedger(date);
+        return {
+            date,
+            totalUsd: typeof raw.totalUsd === "number" ? raw.totalUsd : 0,
+            calls: typeof raw.calls === "number" ? raw.calls : 0,
+            byModel: raw.byModel && typeof raw.byModel === "object" ? raw.byModel : {}
+        };
+    }
+    catch {
+        return emptyLedger(date);
+    }
+}
+async function writeLedger(file, ledger) {
+    await fs.mkdir(path.dirname(file), { recursive: true });
+    await fs.writeFile(file, `${JSON.stringify(ledger, null, 2)}\n`, "utf8");
+}
+export function createCostGuard(projectRoot, config, options = {}) {
+    const now = options.now ?? (() => new Date());
+    const currentDate = () => utcDate(now());
+    const file = () => options.ledgerPath ?? ledgerPath(projectRoot, currentDate());
+    return {
+        async commit(model, usage) {
+            const usd = computeUsageUsd(model, usage, config);
+            if (config.dailyUsdCap === undefined)
+                return usd;
+            const date = currentDate();
+            const target = file();
+            const ledger = await readLedger(target, date);
+            const projected = Number((ledger.totalUsd + usd).toFixed(6));
+            if (projected > config.dailyUsdCap) {
+                throw new DailyCostCapExceededError({
+                    capUsd: config.dailyUsdCap,
+                    projectedUsd: projected,
+                    currentUsd: ledger.totalUsd
+                });
+            }
+            ledger.totalUsd = projected;
+            ledger.calls += 1;
+            const byModel = ledger.byModel[model] ?? { tokensIn: 0, tokensOut: 0, usd: 0 };
+            byModel.tokensIn += usage.promptTokens;
+            byModel.tokensOut += usage.completionTokens;
+            byModel.usd = Number((byModel.usd + usd).toFixed(6));
+            ledger.byModel[model] = byModel;
+            await writeLedger(target, ledger);
+            return usd;
+        },
+        async snapshot() {
+            if (config.dailyUsdCap === undefined)
+                return undefined;
+            const date = currentDate();
+            return readLedger(file(), date);
+        }
+    };
+}
+/** Exposed for tests. */
+export const __internal = {
+    utcDate,
+    pricingFor,
+    ledgerPath,
+    readLedger,
+    writeLedger
+};

package/dist/eval/llm-client.d.ts

@@ -1,18 +1,5 @@
-/**
- * LLM client skeleton for the cclaw eval subsystem.
- *
- * This module declares the shape of the client without pulling in the
- * `openai` runtime dependency. The real implementation lands when
- * single-shot (Tier A) evals and LLM judging come online. Keeping this stub
- * separate means users who only run structural + rule-based verifiers never
- * install an extra dependency or receive network egress warnings.
- */
+import type { ClientOptions } from "openai";
 import type { ResolvedEvalConfig } from "./types.js";
-/**
- * Minimal chat interface the rest of the eval code will depend on. It is
- * intentionally a subset of OpenAI's Chat Completions surface so that the
- * real implementation is a thin adapter around `OpenAI.chat.completions.create`.
- */
 export interface ChatMessage {
     role: "system" | "user" | "assistant" | "tool";
     content: string;
@@ -24,7 +11,18 @@ export interface ChatRequest {
     messages: ChatMessage[];
     maxTokens?: number;
     temperature?: number;
+    /** Per-call timeout override. Falls back to `config.timeoutMs`. */
     timeoutMs?: number;
+    /**
+     * Ask the provider for a JSON-object response. The judge pipeline sets
+     * this; the agent-under-test usually leaves it unset.
+     */
+    responseFormatJson?: boolean;
+    /**
+     * Optional deterministic sampling seed. Providers that don't implement
+     * `seed` simply ignore it.
+     */
+    seed?: number;
     /**
      * Tool/function-calling definitions in OpenAI wire format. Populated only
      * by Tier B. Ignored by the Tier A single-shot path.
@@ -46,17 +44,112 @@ export interface ChatResponse {
     }>;
     usage: ChatUsage;
     finishReason: "stop" | "length" | "tool_calls" | "content_filter";
+    model: string;
+    attempts: number;
+}
+/** Base class so callers can `catch (err) { if (err instanceof EvalLlmError) ... }`. */
+export declare class EvalLlmError extends Error {
+    readonly retryable: boolean;
+    readonly status?: number;
+    constructor(message: string, opts: {
+        retryable: boolean;
+        status?: number;
+        cause?: unknown;
+    });
+}
+export declare class EvalLlmAuthError extends EvalLlmError {
+    constructor(cause: unknown);
+}
+export declare class EvalLlmConfigError extends EvalLlmError {
+    constructor(message: string, cause?: unknown);
+}
+export declare class EvalLlmTimeoutError extends EvalLlmError {
+    constructor(timeoutMs: number);
+}
+export declare class EvalLlmRateLimitedError extends EvalLlmError {
+    constructor(cause: unknown);
+}
+export declare class EvalLlmTransportError extends EvalLlmError {
+    constructor(cause: unknown, status?: number);
+}
+export declare class EvalLlmInvalidResponseError extends EvalLlmError {
+    constructor(message: string, details?: Record<string, unknown>);
+}
+export declare class EvalLlmNotConfiguredError extends EvalLlmError {
+    constructor();
 }
 /** Lightweight client abstraction shared across eval runners. */
 export interface EvalLlmClient {
     chat(request: ChatRequest): Promise<ChatResponse>;
 }
-export declare class EvalLlmNotWiredError extends Error {
-    constructor();
+/**
+ * Deprecated shim preserved so older wiring keeps compiling. Prefer
+ * `EvalLlmNotConfiguredError` for the "caller forgot to provide an API
+ * key" case.
+ */
+export declare class EvalLlmNotWiredError extends EvalLlmNotConfiguredError {
+}
+/** `createEvalClient` options — mostly for tests to inject a fake transport. */
+export interface CreateEvalClientOptions {
+    /** Inject an `openai` stand-in. Used by unit tests to avoid real HTTP. */
+    openaiFactory?: (opts: ClientOptions) => OpenAILike;
+    /**
+     * Override the default retry/backoff policy. Honored by the internal
+     * retry loop; transport errors still fall back to the defaults when
+     * unset.
+     */
+    retryPolicy?: RetryPolicy;
+    /** Deterministic sleep used by the retry loop. Defaults to `setTimeout`. */
+    sleep?: (ms: number) => Promise<void>;
+}
+export interface RetryPolicy {
+    /** Max retries *on top of* the initial attempt. 0 = single attempt. */
+    maxRetries: number;
+    /** Initial backoff in ms. Doubles each retry (capped at `maxBackoffMs`). */
+    initialBackoffMs: number;
+    /** Upper bound for a single sleep between attempts. */
+    maxBackoffMs: number;
+}
+export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
+/**
+ * Minimal OpenAI-SDK surface we depend on, declared here so tests can
+ * substitute a plain object without pulling the real SDK into the test
+ * runtime.
+ */
+export interface OpenAILike {
+    chat: {
+        completions: {
+            create(body: Record<string, unknown>, options: {
+                signal: AbortSignal;
+            }): Promise<OpenAILikeChatResponse>;
+        };
+    };
+}
+interface OpenAILikeChatResponse {
+    model?: string;
+    choices: Array<{
+        message?: {
+            content?: string | null;
+            tool_calls?: Array<{
+                id: string;
+                function: {
+                    name: string;
+                    arguments: string;
+                };
+            }>;
+        };
+        finish_reason?: string | null;
+    }>;
+    usage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+        total_tokens?: number;
+    };
 }
 /**
- * Factory stub. Throws with a clear message so accidental early usage is
- * easy to diagnose. The real implementation will replace this body with
- * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ * Build a real client pointed at the configured endpoint. Throws
+ * `EvalLlmNotConfiguredError` at call time (not construction time) when no
+ * API key is available, so CLI help and dry-run paths stay offline-safe.
  */
-export declare function createEvalClient(_config: ResolvedEvalConfig): EvalLlmClient;
+export declare function createEvalClient(config: ResolvedEvalConfig, options?: CreateEvalClientOptions): EvalLlmClient;
+export {};

package/dist/eval/llm-client.js

@@ -1,19 +1,251 @@
-export class EvalLlmNotWiredError extends Error {
+/**
+ * LLM client for the cclaw eval subsystem.
+ *
+ * Thin adapter over the `openai` SDK pointed at any OpenAI-compatible
+ * `baseURL` (z.ai, OpenAI, vLLM, Ollama+openai-proxy, ...). The surface is
+ * deliberately narrow:
+ *
+ *  - `chat()` — one request/response round-trip with timeout, bounded
+ *    retries on transient errors, and a structured error hierarchy so
+ *    callers can react policy-style (cost-guard, judge, agent-under-test).
+ *  - `ChatRequest` / `ChatResponse` — wire format decoupled from the
+ *    OpenAI types so swapping vendors stays a one-file change.
+ *
+ * Factories stay side-effect-free: no network calls are made until `chat()`
+ * is invoked, so CLI help and dry-run paths never need an API key.
+ */
+import OpenAI from "openai";
+/** Base class so callers can `catch (err) { if (err instanceof EvalLlmError) ... }`. */
+export class EvalLlmError extends Error {
+    retryable;
+    status;
+    constructor(message, opts) {
+        super(message);
+        this.name = "EvalLlmError";
+        this.retryable = opts.retryable;
+        if (opts.status !== undefined)
+            this.status = opts.status;
+        if (opts.cause !== undefined)
+            this.cause = opts.cause;
+    }
+}
+export class EvalLlmAuthError extends EvalLlmError {
+    constructor(cause) {
+        super("LLM request rejected (auth). Check CCLAW_EVAL_API_KEY and provider permissions.", {
+            retryable: false,
+            status: 401,
+            cause
+        });
+        this.name = "EvalLlmAuthError";
+    }
+}
+export class EvalLlmConfigError extends EvalLlmError {
+    constructor(message, cause) {
+        super(message, { retryable: false, cause });
+        this.name = "EvalLlmConfigError";
+    }
+}
+export class EvalLlmTimeoutError extends EvalLlmError {
+    constructor(timeoutMs) {
+        super(`LLM request timed out after ${timeoutMs}ms.`, { retryable: true });
+        this.name = "EvalLlmTimeoutError";
+    }
+}
+export class EvalLlmRateLimitedError extends EvalLlmError {
+    constructor(cause) {
+        super("LLM rate limit hit. Retrying with backoff.", {
+            retryable: true,
+            status: 429,
+            cause
+        });
+        this.name = "EvalLlmRateLimitedError";
+    }
+}
+export class EvalLlmTransportError extends EvalLlmError {
+    constructor(cause, status) {
+        super("LLM transport error.", { retryable: true, status, cause });
+        this.name = "EvalLlmTransportError";
+    }
+}
+export class EvalLlmInvalidResponseError extends EvalLlmError {
+    constructor(message, details) {
+        super(message, { retryable: false });
+        this.name = "EvalLlmInvalidResponseError";
+        if (details)
+            this.details = details;
+    }
+}
+export class EvalLlmNotConfiguredError extends EvalLlmError {
     constructor() {
-        super(`LLM client is not wired yet.\n` +
-            `Run \`cclaw eval --dry-run\` or \`cclaw eval --schema-only\` for offline evals.`);
-        this.name = "EvalLlmNotWiredError";
+        super(`LLM client not configured. Set CCLAW_EVAL_API_KEY (and optionally ` +
+            `CCLAW_EVAL_BASE_URL / CCLAW_EVAL_MODEL) or run with --schema-only / --rules.`, { retryable: false });
+        this.name = "EvalLlmNotConfiguredError";
     }
 }
 /**
- * Factory stub. Throws with a clear message so accidental early usage is
- * easy to diagnose. The real implementation will replace this body with
- * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ * Deprecated shim preserved so older wiring keeps compiling. Prefer
+ * `EvalLlmNotConfiguredError` for the "caller forgot to provide an API
+ * key" case.
  */
-export function createEvalClient(_config) {
+export class EvalLlmNotWiredError extends EvalLlmNotConfiguredError {
+}
+export const DEFAULT_RETRY_POLICY = {
+    maxRetries: 2,
+    initialBackoffMs: 500,
+    maxBackoffMs: 8_000
+};
+function isAbortError(err) {
+    if (err === null || typeof err !== "object")
+        return false;
+    const name = err.name;
+    const code = err.code;
+    return (name === "AbortError" || code === "ABORT_ERR" || code === "ERR_CANCELED");
+}
+function statusFromError(err) {
+    if (err === null || typeof err !== "object")
+        return undefined;
+    const status = err.status;
+    return typeof status === "number" ? status : undefined;
+}
+function normalizeError(err, timeoutMs) {
+    if (err instanceof EvalLlmError)
+        return err;
+    if (isAbortError(err))
+        return new EvalLlmTimeoutError(timeoutMs);
+    const status = statusFromError(err);
+    if (status === 401 || status === 403)
+        return new EvalLlmAuthError(err);
+    if (status === 429)
+        return new EvalLlmRateLimitedError(err);
+    if (status !== undefined && status >= 400 && status < 500) {
+        return new EvalLlmError(`LLM request rejected (HTTP ${status}).`, {
+            retryable: false,
+            status,
+            cause: err
+        });
+    }
+    return new EvalLlmTransportError(err, status);
+}
+function normalizeFinishReason(raw) {
+    switch (raw) {
+        case "length":
+            return "length";
+        case "tool_calls":
+        case "function_call":
+            return "tool_calls";
+        case "content_filter":
+            return "content_filter";
+        case "stop":
+        case null:
+        case undefined:
+        default:
+            return "stop";
+    }
+}
+function buildBody(request) {
+    const body = {
+        model: request.model,
+        messages: request.messages.map((m) => ({
+            role: m.role,
+            content: m.content,
+            ...(m.name !== undefined ? { name: m.name } : {}),
+            ...(m.toolCallId !== undefined ? { tool_call_id: m.toolCallId } : {})
+        }))
+    };
+    if (request.maxTokens !== undefined)
+        body.max_tokens = request.maxTokens;
+    if (request.temperature !== undefined)
+        body.temperature = request.temperature;
+    if (request.seed !== undefined)
+        body.seed = request.seed;
+    if (request.tools !== undefined)
+        body.tools = request.tools;
+    if (request.toolChoice !== undefined)
+        body.tool_choice = request.toolChoice;
+    if (request.responseFormatJson === true) {
+        body.response_format = { type: "json_object" };
+    }
+    return body;
+}
+function defaultSleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function backoffDelay(attempt, policy) {
+    const raw = policy.initialBackoffMs * 2 ** attempt;
+    return Math.min(raw, policy.maxBackoffMs);
+}
+/**
+ * Build a real client pointed at the configured endpoint. Throws
+ * `EvalLlmNotConfiguredError` at call time (not construction time) when no
+ * API key is available, so CLI help and dry-run paths stay offline-safe.
+ */
+export function createEvalClient(config, options = {}) {
+    const retryPolicy = options.retryPolicy ?? {
+        ...DEFAULT_RETRY_POLICY,
+        maxRetries: Math.max(0, config.maxRetries ?? DEFAULT_RETRY_POLICY.maxRetries)
+    };
+    const sleep = options.sleep ?? defaultSleep;
+    let cached;
+    const getClient = () => {
+        if (cached)
+            return cached;
+        if (!config.apiKey)
+            throw new EvalLlmNotConfiguredError();
+        const factory = options.openaiFactory ??
+            ((opts) => new OpenAI(opts));
+        cached = factory({ apiKey: config.apiKey, baseURL: config.baseUrl });
+        return cached;
+    };
     return {
-        async chat() {
-            throw new EvalLlmNotWiredError();
+        async chat(request) {
+            const timeoutMs = Math.max(1_000, request.timeoutMs ?? config.timeoutMs);
+            const body = buildBody(request);
+            const client = getClient();
+            let lastError;
+            const maxAttempts = retryPolicy.maxRetries + 1;
+            for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+                const controller = new AbortController();
+                const handle = setTimeout(() => controller.abort(), timeoutMs);
+                try {
+                    const raw = await client.chat.completions.create(body, {
+                        signal: controller.signal
+                    });
+                    clearTimeout(handle);
+                    const choice = raw.choices?.[0];
+                    if (!choice) {
+                        throw new EvalLlmInvalidResponseError("LLM response contained no choices.", { model: raw.model });
+                    }
+                    const content = choice.message?.content ?? "";
+                    const toolCalls = choice.message?.tool_calls?.map((call) => ({
+                        id: call.id,
+                        name: call.function.name,
+                        arguments: call.function.arguments
+                    }));
+                    const usage = {
+                        promptTokens: raw.usage?.prompt_tokens ?? 0,
+                        completionTokens: raw.usage?.completion_tokens ?? 0,
+                        totalTokens: raw.usage?.total_tokens ?? 0
+                    };
+                    return {
+                        content,
+                        ...(toolCalls && toolCalls.length > 0 ? { toolCalls } : {}),
+                        usage,
+                        finishReason: normalizeFinishReason(choice.finish_reason),
+                        model: raw.model ?? request.model,
+                        attempts: attempt + 1
+                    };
+                }
+                catch (err) {
+                    clearTimeout(handle);
+                    const normalized = normalizeError(err, timeoutMs);
+                    lastError = normalized;
+                    const isLastAttempt = attempt === maxAttempts - 1;
+                    if (!normalized.retryable || isLastAttempt)
+                        throw normalized;
+                    await sleep(backoffDelay(attempt, retryPolicy));
+                }
+            }
+            throw lastError ?? new EvalLlmTransportError(new Error("unknown"));
         }
     };
 }

package/dist/eval/report.js

@@ -75,6 +75,32 @@ export function formatMarkdownReport(report) {
         lines.push(`| ${item.stage} | ${item.caseId} | ${item.passed ? "yes" : "no"} | ${item.durationMs} | ${cost} |`);
     }
     lines.push(``);
+    const judgeCases = report.cases.filter((item) => item.verifierResults.some((r) => r.kind === "judge"));
+    if (judgeCases.length > 0) {
+        lines.push(`## Judge scores`);
+        lines.push(``);
+        lines.push(`| stage | case id | check | median | mean | coverage | ok |`);
+        lines.push(`| --- | --- | --- | --- | --- | --- | --- |`);
+        for (const item of judgeCases) {
+            for (const verifier of item.verifierResults) {
+                if (verifier.kind !== "judge")
+                    continue;
+                if (verifier.id === "judge:required-checks")
+                    continue;
+                if (verifier.id === "judge:rubric:missing")
+                    continue;
+                if (verifier.id === "judge:invocation:error")
+                    continue;
+                const details = verifier.details ?? {};
+                const median = typeof details.median === "number" ? details.median.toFixed(2) : "-";
+                const mean = typeof details.mean === "number" ? details.mean.toFixed(2) : "-";
+                const coverage = details.coverage === true ? "yes" : "no";
+                const checkId = verifier.id.replace(/^judge:/, "");
+                lines.push(`| ${item.stage} | ${item.caseId} | ${checkId} | ${median} | ${mean} | ${coverage} | ${verifier.ok ? "yes" : "no"} |`);
+            }
+        }
+        lines.push(``);
+    }
     lines.push(`## Verifier details`);
     lines.push(``);
     for (const item of report.cases) {

package/dist/eval/rubric-loader.d.ts

@@ -0,0 +1,20 @@
+import type { FlowStage } from "../types.js";
+import type { RubricCheck, RubricDoc } from "./types.js";
+export declare function rubricsDir(projectRoot: string): string;
+export declare function rubricPath(projectRoot: string, stage: FlowStage): string;
+declare function validateCheck(raw: unknown, index: number, file: string): RubricCheck;
+declare function validateRubric(raw: unknown, file: string): RubricDoc;
+/**
+ * Load the rubric for `stage`. Returns `undefined` when the file is
+ * missing so callers can emit a "no rubric" verifier result rather than
+ * crashing — authors are expected to grow rubrics incrementally.
+ */
+export declare function loadRubric(projectRoot: string, stage: FlowStage): Promise<RubricDoc | undefined>;
+/** Load every rubric present in the given rubrics directory. */
+export declare function loadAllRubrics(projectRoot: string): Promise<Map<FlowStage, RubricDoc>>;
+/** Exposed for tests. */
+export declare const __internal: {
+    validateRubric: typeof validateRubric;
+    validateCheck: typeof validateCheck;
+};
+export {};