cclaw-cli 0.23.1 → 0.25.0

package/dist/eval/corpus.js

@@ -58,6 +58,128 @@ function parseStructural(filePath, raw) {
         structural.maxChars = maxChars;
     return structural;
 }
+function parseRegexRule(filePath, context, value) {
+    if (typeof value === "string") {
+        return { pattern: value };
+    }
+    if (!isRecord(value)) {
+        throw corpusError(filePath, `"${context}" entries must be either a string or a mapping with "pattern"`);
+    }
+    const pattern = value.pattern;
+    if (typeof pattern !== "string" || pattern.length === 0) {
+        throw corpusError(filePath, `"${context}" mapping entry must include a non-empty "pattern" string`);
+    }
+    const flags = value.flags;
+    if (flags !== undefined && typeof flags !== "string") {
+        throw corpusError(filePath, `"${context}" flags must be a string`);
+    }
+    const description = value.description;
+    if (description !== undefined && typeof description !== "string") {
+        throw corpusError(filePath, `"${context}" description must be a string`);
+    }
+    const rule = { pattern };
+    if (flags !== undefined)
+        rule.flags = flags;
+    if (description !== undefined)
+        rule.description = description;
+    return rule;
+}
+function parseRegexRules(filePath, context, value) {
+    if (value === undefined)
+        return undefined;
+    if (!Array.isArray(value)) {
+        throw corpusError(filePath, `"${context}" must be an array`);
+    }
+    return value.map((entry, index) => parseRegexRule(filePath, `${context}[${index}]`, entry));
+}
+function parseOccurrenceBounds(filePath, context, value) {
+    if (value === undefined)
+        return undefined;
+    if (!isRecord(value)) {
+        throw corpusError(filePath, `"${context}" must be a mapping of phrase → integer`);
+    }
+    const out = {};
+    for (const [phrase, count] of Object.entries(value)) {
+        if (typeof count !== "number" || !Number.isFinite(count) || !Number.isInteger(count) || count < 0) {
+            throw corpusError(filePath, `"${context}.${phrase}" must be a non-negative integer`);
+        }
+        out[phrase] = count;
+    }
+    return out;
+}
+function parseRules(filePath, raw) {
+    if (raw === undefined)
+        return undefined;
+    if (!isRecord(raw)) {
+        throw corpusError(filePath, `"expected.rules" must be a mapping`);
+    }
+    const mustContain = readStringArray(filePath, "expected.rules.must_contain", raw.must_contain ?? raw.mustContain);
+    const mustNotContain = readStringArray(filePath, "expected.rules.must_not_contain", raw.must_not_contain ?? raw.mustNotContain);
+    const regexRequired = parseRegexRules(filePath, "expected.rules.regex_required", raw.regex_required ?? raw.regexRequired);
+    const regexForbidden = parseRegexRules(filePath, "expected.rules.regex_forbidden", raw.regex_forbidden ?? raw.regexForbidden);
+    const minOccurrences = parseOccurrenceBounds(filePath, "expected.rules.min_occurrences", raw.min_occurrences ?? raw.minOccurrences);
+    const maxOccurrences = parseOccurrenceBounds(filePath, "expected.rules.max_occurrences", raw.max_occurrences ?? raw.maxOccurrences);
+    const uniqueBulletsInSection = readStringArray(filePath, "expected.rules.unique_bullets_in_section", raw.unique_bullets_in_section ?? raw.uniqueBulletsInSection);
+    const rules = {};
+    if (mustContain)
+        rules.mustContain = mustContain;
+    if (mustNotContain)
+        rules.mustNotContain = mustNotContain;
+    if (regexRequired)
+        rules.regexRequired = regexRequired;
+    if (regexForbidden)
+        rules.regexForbidden = regexForbidden;
+    if (minOccurrences)
+        rules.minOccurrences = minOccurrences;
+    if (maxOccurrences)
+        rules.maxOccurrences = maxOccurrences;
+    if (uniqueBulletsInSection)
+        rules.uniqueBulletsInSection = uniqueBulletsInSection;
+    return Object.keys(rules).length === 0 ? undefined : rules;
+}
+function parseTraceability(filePath, raw) {
+    if (raw === undefined)
+        return undefined;
+    if (!isRecord(raw)) {
+        throw corpusError(filePath, `"expected.traceability" must be a mapping`);
+    }
+    const idPattern = raw.id_pattern ?? raw.idPattern;
+    if (typeof idPattern !== "string" || idPattern.length === 0) {
+        throw corpusError(filePath, `"expected.traceability.id_pattern" must be a non-empty regex source`);
+    }
+    const idFlags = raw.id_flags ?? raw.idFlags;
+    if (idFlags !== undefined && typeof idFlags !== "string") {
+        throw corpusError(filePath, `"expected.traceability.id_flags" must be a string`);
+    }
+    const source = raw.source;
+    if (typeof source !== "string" || source.length === 0) {
+        throw corpusError(filePath, `"expected.traceability.source" must be "self" or an extra_fixtures label`);
+    }
+    const requireInRaw = raw.require_in ?? raw.requireIn;
+    const requireIn = readStringArray(filePath, "expected.traceability.require_in", requireInRaw);
+    if (!requireIn || requireIn.length === 0) {
+        throw corpusError(filePath, `"expected.traceability.require_in" must be a non-empty array`);
+    }
+    const out = { idPattern, source, requireIn };
+    if (idFlags !== undefined)
+        out.idFlags = idFlags;
+    return out;
+}
+function parseExtraFixtures(filePath, raw) {
+    if (raw === undefined)
+        return undefined;
+    if (!isRecord(raw)) {
+        throw corpusError(filePath, `"extra_fixtures" must be a mapping of label → path`);
+    }
+    const out = {};
+    for (const [label, value] of Object.entries(raw)) {
+        if (typeof value !== "string" || value.length === 0) {
+            throw corpusError(filePath, `"extra_fixtures.${label}" must be a non-empty path string`);
+        }
+        out[label] = value;
+    }
+    return Object.keys(out).length === 0 ? undefined : out;
+}
 function parseExpected(filePath, raw) {
     if (raw === undefined)
         return undefined;
@@ -68,12 +190,12 @@ function parseExpected(filePath, raw) {
     const structural = parseStructural(filePath, raw.structural);
     if (structural)
         shape.structural = structural;
-    if (raw.rules !== undefined) {
-        if (!isRecord(raw.rules)) {
-            throw corpusError(filePath, `"expected.rules" must be a mapping`);
-        }
-        shape.rules = raw.rules;
-    }
+    const rules = parseRules(filePath, raw.rules);
+    if (rules)
+        shape.rules = rules;
+    const traceability = parseTraceability(filePath, raw.traceability);
+    if (traceability)
+        shape.traceability = traceability;
     if (raw.judge !== undefined) {
         if (!isRecord(raw.judge)) {
             throw corpusError(filePath, `"expected.judge" must be a mapping`);
@@ -101,13 +223,15 @@ function validateCase(filePath, raw) {
     const contextFiles = readStringArray(filePath, "context_files", raw.context_files ?? raw.contextFiles);
     const expected = parseExpected(filePath, raw.expected);
     const fixture = typeof raw.fixture === "string" ? raw.fixture : undefined;
+    const extraFixtures = parseExtraFixtures(filePath, raw.extra_fixtures ?? raw.extraFixtures);
     return {
         id: id.trim(),
         stage: stageRaw,
         inputPrompt: inputPrompt.trim(),
         contextFiles,
         expected,
-        fixture
+        fixture,
+        extraFixtures
     };
 }
 /**
@@ -173,3 +297,34 @@ export async function readFixtureArtifact(projectRoot, caseEntry) {
     }
     return fs.readFile(fixturePath, "utf8");
 }
+/**
+ * Resolve an entry from `extraFixtures` to an absolute filesystem path,
+ * relative to the case's stage directory (same convention as `fixture`).
+ */
+export function extraFixturePath(projectRoot, caseEntry, label) {
+    const value = caseEntry.extraFixtures?.[label];
+    if (!value)
+        return undefined;
+    return path.resolve(projectRoot, EVALS_ROOT, "corpus", caseEntry.stage, value);
+}
+/**
+ * Read every declared extra fixture for a case into a `{ label → text }`
+ * map. Missing files throw so authoring mistakes surface immediately rather
+ * than being silently skipped by cross-artifact verifiers.
+ */
+export async function readExtraFixtures(projectRoot, caseEntry) {
+    const out = {};
+    if (!caseEntry.extraFixtures)
+        return out;
+    for (const label of Object.keys(caseEntry.extraFixtures)) {
+        const filePath = extraFixturePath(projectRoot, caseEntry, label);
+        if (!filePath)
+            continue;
+        if (!(await exists(filePath))) {
+            throw new Error(`Extra fixture missing for ${caseEntry.stage}/${caseEntry.id} ` +
+                `(label="${label}"): ${filePath}`);
+        }
+        out[label] = await fs.readFile(filePath, "utf8");
+    }
+    return out;
+}

package/dist/eval/cost-guard.d.ts

@@ -0,0 +1,80 @@
+import type { ChatUsage } from "./llm-client.js";
+import type { ResolvedEvalConfig, TokenPricing } from "./types.js";
+/**
+ * Builtin pricing fallback. Intentionally conservative: when the user
+ * hasn't configured pricing and we don't know the model, we default to a
+ * "small model" USD schedule so the cap can still do something useful.
+ *
+ * Values are USD per 1K tokens. Sources are public pricing pages as of
+ * 2026-04; update by editing this constant, not the guard logic.
+ */
+export declare const DEFAULT_TOKEN_PRICING: Readonly<Record<string, TokenPricing>>;
+/** Hard default when neither config nor builtins know the model. */
+export declare const UNKNOWN_MODEL_PRICING: TokenPricing;
+export interface SpendLedger {
+    /** ISO date (`YYYY-MM-DD` in UTC) — also embedded in the file name. */
+    date: string;
+    /** USD spent so far today across every call that hit the guard. */
+    totalUsd: number;
+    /** Number of `chat()` calls accounted for. */
+    calls: number;
+    /** Per-model breakdown for the report. */
+    byModel: Record<string, {
+        tokensIn: number;
+        tokensOut: number;
+        usd: number;
+    }>;
+}
+export declare class DailyCostCapExceededError extends Error {
+    readonly capUsd: number;
+    readonly projectedUsd: number;
+    readonly currentUsd: number;
+    constructor(opts: {
+        capUsd: number;
+        projectedUsd: number;
+        currentUsd: number;
+    });
+}
+declare function utcDate(now?: Date): string;
+declare function pricingFor(model: string, config: Pick<ResolvedEvalConfig, "tokenPricing">): TokenPricing;
+/**
+ * Compute USD cost of a single `ChatUsage` using the given `model` pricing
+ * schedule. Returns 0 when `usage.totalTokens` is 0 (e.g. transport error
+ * before first token).
+ */
+export declare function computeUsageUsd(model: string, usage: ChatUsage, config: Pick<ResolvedEvalConfig, "tokenPricing">): number;
+declare function ledgerPath(projectRoot: string, date: string): string;
+declare function readLedger(file: string, date: string): Promise<SpendLedger>;
+declare function writeLedger(file: string, ledger: SpendLedger): Promise<void>;
+/**
+ * Guard a single LLM call against the daily USD cap. Returns the updated
+ * ledger on success; throws `DailyCostCapExceededError` when the projected
+ * total would cross the cap. When `config.dailyUsdCap` is unset, the guard
+ * is a no-op — no file writes, no ledger — so non-judge runs never touch
+ * the filesystem.
+ */
+export interface CostGuard {
+    /**
+     * Commit the USD cost of a finished call to the ledger. When `dailyUsdCap`
+     * is set, refuses the commit if the projected total would exceed the cap.
+     */
+    commit(model: string, usage: ChatUsage): Promise<number>;
+    /** Snapshot the current ledger (or undefined when no cap is set). */
+    snapshot(): Promise<SpendLedger | undefined>;
+}
+export interface CreateCostGuardOptions {
+    /** Clock injection for tests. */
+    now?: () => Date;
+    /** Override the default filesystem root for the ledger. */
+    ledgerPath?: string;
+}
+export declare function createCostGuard(projectRoot: string, config: Pick<ResolvedEvalConfig, "dailyUsdCap" | "tokenPricing">, options?: CreateCostGuardOptions): CostGuard;
+/** Exposed for tests. */
+export declare const __internal: {
+    utcDate: typeof utcDate;
+    pricingFor: typeof pricingFor;
+    ledgerPath: typeof ledgerPath;
+    readLedger: typeof readLedger;
+    writeLedger: typeof writeLedger;
+};
+export {};

package/dist/eval/cost-guard.js

@@ -0,0 +1,153 @@
+/**
+ * Cost guard for the cclaw eval subsystem.
+ *
+ * Two responsibilities:
+ *
+ * 1. Convert `ChatUsage` (prompt/completion token counts) into USD using
+ *    a per-model `TokenPricing` schedule. Pricing comes from
+ *    `config.tokenPricing[model]` first, then from the builtin fallback
+ *    schedule for well-known models (z.ai GLM 5.1 at publish time).
+ * 2. Maintain a per-day running total persisted to
+ *    `.cclaw/evals/.spend-YYYY-MM-DD.json` so that a long eval session
+ *    (or a cron-run nightly) can't blow through the configured
+ *    `dailyUsdCap`. The counter is opt-in: no cap, no writes.
+ *
+ * The guard is deliberately pessimistic — it rounds USD up to 6 decimals
+ * and never subtracts, so a CI run that errors mid-flight still shows the
+ * partial spend in the next report.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+/**
+ * Builtin pricing fallback. Intentionally conservative: when the user
+ * hasn't configured pricing and we don't know the model, we default to a
+ * "small model" USD schedule so the cap can still do something useful.
+ *
+ * Values are USD per 1K tokens. Sources are public pricing pages as of
+ * 2026-04; update by editing this constant, not the guard logic.
+ */
+export const DEFAULT_TOKEN_PRICING = {
+    "glm-5.1": { input: 0.0005, output: 0.0015 },
+    "glm-4.6": { input: 0.0005, output: 0.0015 },
+    "gpt-4o-mini": { input: 0.00015, output: 0.0006 },
+    "gpt-4o": { input: 0.005, output: 0.015 }
+};
+/** Hard default when neither config nor builtins know the model. */
+export const UNKNOWN_MODEL_PRICING = { input: 0.001, output: 0.003 };
+export class DailyCostCapExceededError extends Error {
+    capUsd;
+    projectedUsd;
+    currentUsd;
+    constructor(opts) {
+        super(`Daily cost cap would be exceeded: ` +
+            `current=$${opts.currentUsd.toFixed(4)}, ` +
+            `projected=$${opts.projectedUsd.toFixed(4)}, ` +
+            `cap=$${opts.capUsd.toFixed(4)}. ` +
+            `Unset CCLAW_EVAL_DAILY_USD_CAP or increase the cap to continue.`);
+        this.name = "DailyCostCapExceededError";
+        this.capUsd = opts.capUsd;
+        this.projectedUsd = opts.projectedUsd;
+        this.currentUsd = opts.currentUsd;
+    }
+}
+function utcDate(now = new Date()) {
+    return now.toISOString().slice(0, 10);
+}
+function pricingFor(model, config) {
+    const custom = config.tokenPricing?.[model];
+    if (custom)
+        return custom;
+    const builtin = DEFAULT_TOKEN_PRICING[model];
+    if (builtin)
+        return builtin;
+    return UNKNOWN_MODEL_PRICING;
+}
+/**
+ * Compute USD cost of a single `ChatUsage` using the given `model` pricing
+ * schedule. Returns 0 when `usage.totalTokens` is 0 (e.g. transport error
+ * before first token).
+ */
+export function computeUsageUsd(model, usage, config) {
+    if (!usage || usage.totalTokens <= 0)
+        return 0;
+    const schedule = pricingFor(model, config);
+    const cost = (usage.promptTokens * schedule.input) / 1_000 +
+        (usage.completionTokens * schedule.output) / 1_000;
+    return Math.max(0, Number(cost.toFixed(6)));
+}
+function emptyLedger(date) {
+    return { date, totalUsd: 0, calls: 0, byModel: {} };
+}
+function ledgerPath(projectRoot, date) {
+    return path.join(projectRoot, EVALS_ROOT, `.spend-${date}.json`);
+}
+async function readLedger(file, date) {
+    if (!(await exists(file)))
+        return emptyLedger(date);
+    try {
+        const raw = JSON.parse(await fs.readFile(file, "utf8"));
+        if (raw?.date !== date)
+            return emptyLedger(date);
+        return {
+            date,
+            totalUsd: typeof raw.totalUsd === "number" ? raw.totalUsd : 0,
+            calls: typeof raw.calls === "number" ? raw.calls : 0,
+            byModel: raw.byModel && typeof raw.byModel === "object" ? raw.byModel : {}
+        };
+    }
+    catch {
+        return emptyLedger(date);
+    }
+}
+async function writeLedger(file, ledger) {
+    await fs.mkdir(path.dirname(file), { recursive: true });
+    await fs.writeFile(file, `${JSON.stringify(ledger, null, 2)}\n`, "utf8");
+}
+export function createCostGuard(projectRoot, config, options = {}) {
+    const now = options.now ?? (() => new Date());
+    const currentDate = () => utcDate(now());
+    const file = () => options.ledgerPath ?? ledgerPath(projectRoot, currentDate());
+    return {
+        async commit(model, usage) {
+            const usd = computeUsageUsd(model, usage, config);
+            if (config.dailyUsdCap === undefined)
+                return usd;
+            const date = currentDate();
+            const target = file();
+            const ledger = await readLedger(target, date);
+            const projected = Number((ledger.totalUsd + usd).toFixed(6));
+            if (projected > config.dailyUsdCap) {
+                throw new DailyCostCapExceededError({
+                    capUsd: config.dailyUsdCap,
+                    projectedUsd: projected,
+                    currentUsd: ledger.totalUsd
+                });
+            }
+            ledger.totalUsd = projected;
+            ledger.calls += 1;
+            const byModel = ledger.byModel[model] ?? { tokensIn: 0, tokensOut: 0, usd: 0 };
+            byModel.tokensIn += usage.promptTokens;
+            byModel.tokensOut += usage.completionTokens;
+            byModel.usd = Number((byModel.usd + usd).toFixed(6));
+            ledger.byModel[model] = byModel;
+            await writeLedger(target, ledger);
+            return usd;
+        },
+        async snapshot() {
+            if (config.dailyUsdCap === undefined)
+                return undefined;
+            const date = currentDate();
+            return readLedger(file(), date);
+        }
+    };
+}
+/** Exposed for tests. */
+export const __internal = {
+    utcDate,
+    pricingFor,
+    ledgerPath,
+    readLedger,
+    writeLedger
+};

package/dist/eval/llm-client.d.ts

@@ -1,18 +1,5 @@
-/**
- * LLM client skeleton for the cclaw eval subsystem.
- *
- * This module declares the shape of the client without pulling in the
- * `openai` runtime dependency. The real implementation lands when
- * single-shot (Tier A) evals and LLM judging come online. Keeping this stub
- * separate means users who only run structural + rule-based verifiers never
- * install an extra dependency or receive network egress warnings.
- */
+import type { ClientOptions } from "openai";
 import type { ResolvedEvalConfig } from "./types.js";
-/**
- * Minimal chat interface the rest of the eval code will depend on. It is
- * intentionally a subset of OpenAI's Chat Completions surface so that the
- * real implementation is a thin adapter around `OpenAI.chat.completions.create`.
- */
 export interface ChatMessage {
     role: "system" | "user" | "assistant" | "tool";
     content: string;
@@ -24,7 +11,18 @@ export interface ChatRequest {
     messages: ChatMessage[];
     maxTokens?: number;
     temperature?: number;
+    /** Per-call timeout override. Falls back to `config.timeoutMs`. */
     timeoutMs?: number;
+    /**
+     * Ask the provider for a JSON-object response. The judge pipeline sets
+     * this; the agent-under-test usually leaves it unset.
+     */
+    responseFormatJson?: boolean;
+    /**
+     * Optional deterministic sampling seed. Providers that don't implement
+     * `seed` simply ignore it.
+     */
+    seed?: number;
     /**
      * Tool/function-calling definitions in OpenAI wire format. Populated only
      * by Tier B. Ignored by the Tier A single-shot path.
@@ -46,17 +44,112 @@ export interface ChatResponse {
     }>;
     usage: ChatUsage;
     finishReason: "stop" | "length" | "tool_calls" | "content_filter";
+    model: string;
+    attempts: number;
+}
+/** Base class so callers can `catch (err) { if (err instanceof EvalLlmError) ... }`. */
+export declare class EvalLlmError extends Error {
+    readonly retryable: boolean;
+    readonly status?: number;
+    constructor(message: string, opts: {
+        retryable: boolean;
+        status?: number;
+        cause?: unknown;
+    });
+}
+export declare class EvalLlmAuthError extends EvalLlmError {
+    constructor(cause: unknown);
+}
+export declare class EvalLlmConfigError extends EvalLlmError {
+    constructor(message: string, cause?: unknown);
+}
+export declare class EvalLlmTimeoutError extends EvalLlmError {
+    constructor(timeoutMs: number);
+}
+export declare class EvalLlmRateLimitedError extends EvalLlmError {
+    constructor(cause: unknown);
+}
+export declare class EvalLlmTransportError extends EvalLlmError {
+    constructor(cause: unknown, status?: number);
+}
+export declare class EvalLlmInvalidResponseError extends EvalLlmError {
+    constructor(message: string, details?: Record<string, unknown>);
+}
+export declare class EvalLlmNotConfiguredError extends EvalLlmError {
+    constructor();
 }
 /** Lightweight client abstraction shared across eval runners. */
 export interface EvalLlmClient {
     chat(request: ChatRequest): Promise<ChatResponse>;
 }
-export declare class EvalLlmNotWiredError extends Error {
-    constructor();
+/**
+ * Deprecated shim preserved so older wiring keeps compiling. Prefer
+ * `EvalLlmNotConfiguredError` for the "caller forgot to provide an API
+ * key" case.
+ */
+export declare class EvalLlmNotWiredError extends EvalLlmNotConfiguredError {
+}
+/** `createEvalClient` options — mostly for tests to inject a fake transport. */
+export interface CreateEvalClientOptions {
+    /** Inject an `openai` stand-in. Used by unit tests to avoid real HTTP. */
+    openaiFactory?: (opts: ClientOptions) => OpenAILike;
+    /**
+     * Override the default retry/backoff policy. Honored by the internal
+     * retry loop; transport errors still fall back to the defaults when
+     * unset.
+     */
+    retryPolicy?: RetryPolicy;
+    /** Deterministic sleep used by the retry loop. Defaults to `setTimeout`. */
+    sleep?: (ms: number) => Promise<void>;
+}
+export interface RetryPolicy {
+    /** Max retries *on top of* the initial attempt. 0 = single attempt. */
+    maxRetries: number;
+    /** Initial backoff in ms. Doubles each retry (capped at `maxBackoffMs`). */
+    initialBackoffMs: number;
+    /** Upper bound for a single sleep between attempts. */
+    maxBackoffMs: number;
+}
+export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
+/**
+ * Minimal OpenAI-SDK surface we depend on, declared here so tests can
+ * substitute a plain object without pulling the real SDK into the test
+ * runtime.
+ */
+export interface OpenAILike {
+    chat: {
+        completions: {
+            create(body: Record<string, unknown>, options: {
+                signal: AbortSignal;
+            }): Promise<OpenAILikeChatResponse>;
+        };
+    };
+}
+interface OpenAILikeChatResponse {
+    model?: string;
+    choices: Array<{
+        message?: {
+            content?: string | null;
+            tool_calls?: Array<{
+                id: string;
+                function: {
+                    name: string;
+                    arguments: string;
+                };
+            }>;
+        };
+        finish_reason?: string | null;
+    }>;
+    usage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+        total_tokens?: number;
+    };
 }
 /**
- * Factory stub. Throws with a clear message so accidental early usage is
- * easy to diagnose. The real implementation will replace this body with
- * `new OpenAI({ apiKey, baseURL }) ... adapter`.
+ * Build a real client pointed at the configured endpoint. Throws
+ * `EvalLlmNotConfiguredError` at call time (not construction time) when no
+ * API key is available, so CLI help and dry-run paths stay offline-safe.
  */
-export declare function createEvalClient(_config: ResolvedEvalConfig): EvalLlmClient;
+export declare function createEvalClient(config: ResolvedEvalConfig, options?: CreateEvalClientOptions): EvalLlmClient;
+export {};