cclaw-cli 0.48.35 → 0.51.0

package/dist/eval/llm-client.js

@@ -1,267 +0,0 @@
-/**
- * LLM client for the cclaw eval subsystem.
- *
- * Thin adapter over the `openai` SDK pointed at any OpenAI-compatible
- * `baseURL` (z.ai, OpenAI, vLLM, Ollama+openai-proxy, ...). The surface is
- * deliberately narrow:
- *
- *  - `chat()` — one request/response round-trip with timeout, bounded
- *    retries on transient errors, and a structured error hierarchy so
- *    callers can react policy-style (cost-guard, judge, agent-under-test).
- *  - `ChatRequest` / `ChatResponse` — wire format decoupled from the
- *    OpenAI types so swapping vendors stays a one-file change.
- *
- * Factories stay side-effect-free: no network calls are made until `chat()`
- * is invoked, so CLI help and dry-run paths never need an API key.
- */
-import OpenAI from "openai";
-/** Base class so callers can `catch (err) { if (err instanceof EvalLlmError) ... }`. */
-export class EvalLlmError extends Error {
-    retryable;
-    status;
-    constructor(message, opts) {
-        super(message);
-        this.name = "EvalLlmError";
-        this.retryable = opts.retryable;
-        if (opts.status !== undefined)
-            this.status = opts.status;
-        if (opts.cause !== undefined)
-            this.cause = opts.cause;
-    }
-}
-export class EvalLlmAuthError extends EvalLlmError {
-    constructor(cause) {
-        super("LLM request rejected (auth). Check CCLAW_EVAL_API_KEY and provider permissions.", {
-            retryable: false,
-            status: 401,
-            cause
-        });
-        this.name = "EvalLlmAuthError";
-    }
-}
-export class EvalLlmConfigError extends EvalLlmError {
-    constructor(message, cause) {
-        super(message, { retryable: false, cause });
-        this.name = "EvalLlmConfigError";
-    }
-}
-export class EvalLlmTimeoutError extends EvalLlmError {
-    constructor(timeoutMs) {
-        super(`LLM request timed out after ${timeoutMs}ms.`, { retryable: true });
-        this.name = "EvalLlmTimeoutError";
-    }
-}
-export class EvalLlmRateLimitedError extends EvalLlmError {
-    constructor(cause) {
-        super("LLM rate limit hit. Retrying with backoff.", {
-            retryable: true,
-            status: 429,
-            cause
-        });
-        this.name = "EvalLlmRateLimitedError";
-    }
-}
-export class EvalLlmTransportError extends EvalLlmError {
-    constructor(cause, status) {
-        super("LLM transport error.", { retryable: true, status, cause });
-        this.name = "EvalLlmTransportError";
-    }
-}
-export class EvalLlmInvalidResponseError extends EvalLlmError {
-    constructor(message, details) {
-        super(message, { retryable: false });
-        this.name = "EvalLlmInvalidResponseError";
-        if (details)
-            this.details = details;
-    }
-}
-export class EvalLlmNotConfiguredError extends EvalLlmError {
-    constructor() {
-        super(`LLM client not configured. Set CCLAW_EVAL_API_KEY (and optionally ` +
-            `CCLAW_EVAL_BASE_URL / CCLAW_EVAL_MODEL) or run with --schema-only / --rules.`, { retryable: false });
-        this.name = "EvalLlmNotConfiguredError";
-    }
-}
-/**
- * Deprecated shim preserved so older wiring keeps compiling. Prefer
- * `EvalLlmNotConfiguredError` for the "caller forgot to provide an API
- * key" case.
- */
-export class EvalLlmNotWiredError extends EvalLlmNotConfiguredError {
-}
-export const DEFAULT_RETRY_POLICY = {
-    maxRetries: 2,
-    initialBackoffMs: 500,
-    maxBackoffMs: 8_000
-};
-function isAbortError(err) {
-    if (err === null || typeof err !== "object")
-        return false;
-    const name = err.name;
-    const code = err.code;
-    return (name === "AbortError" || code === "ABORT_ERR" || code === "ERR_CANCELED");
-}
-function statusFromError(err) {
-    if (err === null || typeof err !== "object")
-        return undefined;
-    const status = err.status;
-    return typeof status === "number" ? status : undefined;
-}
-function normalizeError(err, timeoutMs) {
-    if (err instanceof EvalLlmError)
-        return err;
-    if (isAbortError(err))
-        return new EvalLlmTimeoutError(timeoutMs);
-    const status = statusFromError(err);
-    if (status === 401 || status === 403)
-        return new EvalLlmAuthError(err);
-    if (status === 429)
-        return new EvalLlmRateLimitedError(err);
-    if (status !== undefined && status >= 400 && status < 500) {
-        return new EvalLlmError(`LLM request rejected (HTTP ${status}).`, {
-            retryable: false,
-            status,
-            cause: err
-        });
-    }
-    return new EvalLlmTransportError(err, status);
-}
-function normalizeFinishReason(raw) {
-    switch (raw) {
-        case "length":
-            return "length";
-        case "tool_calls":
-        case "function_call":
-            return "tool_calls";
-        case "content_filter":
-            return "content_filter";
-        case "stop":
-        case null:
-        case undefined:
-        default:
-            return "stop";
-    }
-}
-function buildBody(request) {
-    const body = {
-        model: request.model,
-        messages: request.messages.map((m) => ({
-            role: m.role,
-            content: m.content,
-            ...(m.name !== undefined ? { name: m.name } : {}),
-            ...(m.toolCallId !== undefined ? { tool_call_id: m.toolCallId } : {}),
-            ...(m.toolCalls && m.toolCalls.length > 0
-                ? {
-                    tool_calls: m.toolCalls.map((call) => ({
-                        id: call.id,
-                        type: "function",
-                        function: { name: call.name, arguments: call.arguments }
-                    }))
-                }
-                : {})
-        }))
-    };
-    if (request.maxTokens !== undefined)
-        body.max_tokens = request.maxTokens;
-    if (request.temperature !== undefined)
-        body.temperature = request.temperature;
-    if (request.seed !== undefined)
-        body.seed = request.seed;
-    if (request.tools !== undefined)
-        body.tools = request.tools;
-    if (request.toolChoice !== undefined)
-        body.tool_choice = request.toolChoice;
-    if (request.responseFormatJson === true) {
-        body.response_format = { type: "json_object" };
-    }
-    return body;
-}
-function defaultSleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}
-function backoffDelay(attempt, policy) {
-    const raw = policy.initialBackoffMs * 2 ** attempt;
-    return Math.min(raw, policy.maxBackoffMs);
-}
-/**
- * Build a real client pointed at the configured endpoint. Throws
- * `EvalLlmNotConfiguredError` at call time (not construction time) when no
- * API key is available, so CLI help and dry-run paths stay offline-safe.
- */
-export function createEvalClient(config, options = {}) {
-    const retryPolicy = options.retryPolicy ?? {
-        ...DEFAULT_RETRY_POLICY,
-        maxRetries: Math.max(0, config.maxRetries ?? DEFAULT_RETRY_POLICY.maxRetries)
-    };
-    const sleep = options.sleep ?? defaultSleep;
-    let cached;
-    const getClient = () => {
-        if (cached)
-            return cached;
-        if (!config.apiKey)
-            throw new EvalLlmNotConfiguredError();
-        const factory = options.openaiFactory ??
-            ((opts) => new OpenAI(opts));
-        cached = factory({ apiKey: config.apiKey, baseURL: config.baseUrl });
-        return cached;
-    };
-    return {
-        async chat(request) {
-            const timeoutMs = Math.max(1_000, request.timeoutMs ?? config.timeoutMs);
-            const body = buildBody(request);
-            const client = getClient();
-            let lastError;
-            const maxAttempts = retryPolicy.maxRetries + 1;
-            for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
-                const controller = new AbortController();
-                const handle = setTimeout(() => controller.abort(), timeoutMs);
-                try {
-                    const raw = await client.chat.completions.create(body, {
-                        signal: controller.signal
-                    });
-                    clearTimeout(handle);
-                    const choice = raw.choices?.[0];
-                    if (!choice) {
-                        throw new EvalLlmInvalidResponseError("LLM response contained no choices.", { model: raw.model });
-                    }
-                    const content = choice.message?.content ?? "";
-                    const toolCalls = choice.message?.tool_calls?.map((call) => ({
-                        id: call.id,
-                        name: call.function.name,
-                        arguments: call.function.arguments
-                    }));
-                    const usage = {
-                        promptTokens: raw.usage?.prompt_tokens ?? 0,
-                        completionTokens: raw.usage?.completion_tokens ?? 0,
-                        totalTokens: raw.usage?.total_tokens ?? 0
-                    };
-                    return {
-                        content,
-                        ...(toolCalls && toolCalls.length > 0 ? { toolCalls } : {}),
-                        usage,
-                        finishReason: normalizeFinishReason(choice.finish_reason),
-                        model: raw.model ?? request.model,
-                        attempts: attempt + 1
-                    };
-                }
-                catch (err) {
-                    clearTimeout(handle);
-                    const normalized = normalizeError(err, timeoutMs);
-                    lastError = normalized;
-                    const isLastAttempt = attempt === maxAttempts - 1;
-                    if (!normalized.retryable || isLastAttempt)
-                        throw normalized;
-                    const waitMs = backoffDelay(attempt, retryPolicy);
-                    options.onRetry?.({
-                        attempt: attempt + 1,
-                        maxAttempts,
-                        waitMs,
-                        error: normalized
-                    });
-                    await sleep(waitMs);
-                }
-            }
-            throw lastError ?? new EvalLlmTransportError(new Error("unknown"));
-        }
-    };
-}

package/dist/eval/mode.d.ts

@@ -1,28 +0,0 @@
-/**
- * Helpers that translate between the legacy `Tier A/B/C` naming and the
- * current `EvalMode` identifiers (`fixture` / `agent` / `workflow`).
- *
- * The names we actually carry in reports, config, CLI flags, and verifier
- * messages are the `EvalMode` ones; legacy tier inputs are accepted with a
- * single deprecation warning per process so existing scripts keep working
- * through the 0.28.x line.
- */
-import { type EvalMode } from "./types.js";
-/**
- * Reset the per-process "already warned about legacy tier" flag. Used by
- * tests so each test file gets a deterministic warning surface.
- */
-export declare function __resetLegacyWarningForTests(): void;
-export interface LegacyTierInput {
-    source: "cli" | "env" | "config";
-    raw: string;
-}
-/**
- * Normalize a raw string from the CLI / env / config into an `EvalMode`.
- * Accepts both new (`fixture|agent|workflow`) and legacy (`A|B|C`) names.
- * Emits a deprecation warning to stderr at most once per process when a
- * legacy tier name is seen.
- */
-export declare function parseModeInput(raw: string, input: LegacyTierInput, writeWarning?: (message: string) => void): EvalMode;
-/** @deprecated kept for callers that still need to serialize as legacy. */
-export declare function modeToLegacyTier(mode: EvalMode): "A" | "B" | "C";

package/dist/eval/mode.js

@@ -1,61 +0,0 @@
-/**
- * Helpers that translate between the legacy `Tier A/B/C` naming and the
- * current `EvalMode` identifiers (`fixture` / `agent` / `workflow`).
- *
- * The names we actually carry in reports, config, CLI flags, and verifier
- * messages are the `EvalMode` ones; legacy tier inputs are accepted with a
- * single deprecation warning per process so existing scripts keep working
- * through the 0.28.x line.
- */
-import { EVAL_MODES } from "./types.js";
-const LEGACY_TIER_TO_MODE = {
-    A: "fixture",
-    B: "agent",
-    C: "workflow"
-};
-const MODE_TO_LEGACY_TIER = {
-    fixture: "A",
-    agent: "B",
-    workflow: "C"
-};
-const DEPRECATED_NAMES = new Set(Object.keys(LEGACY_TIER_TO_MODE));
-let legacyWarningEmitted = false;
-/**
- * Reset the per-process "already warned about legacy tier" flag. Used by
- * tests so each test file gets a deterministic warning surface.
- */
-export function __resetLegacyWarningForTests() {
-    legacyWarningEmitted = false;
-}
-/**
- * Normalize a raw string from the CLI / env / config into an `EvalMode`.
- * Accepts both new (`fixture|agent|workflow`) and legacy (`A|B|C`) names.
- * Emits a deprecation warning to stderr at most once per process when a
- * legacy tier name is seen.
- */
-export function parseModeInput(raw, input, writeWarning = defaultWriteWarning) {
-    const trimmed = raw.trim();
-    if (trimmed.length === 0) {
-        throw new Error(`Evaluation mode must be one of: ${EVAL_MODES.join("|")} (or legacy A|B|C).`);
-    }
-    if (EVAL_MODES.includes(trimmed)) {
-        return trimmed;
-    }
-    if (DEPRECATED_NAMES.has(trimmed)) {
-        const replacement = LEGACY_TIER_TO_MODE[trimmed];
-        if (!legacyWarningEmitted) {
-            legacyWarningEmitted = true;
-            writeWarning(`[cclaw] "${input.source}: ${input.raw}" is using the legacy tier name "${trimmed}". ` +
-                `Please switch to --mode=${replacement} (legacy --tier=A|B|C will be removed in the next major release).`);
-        }
-        return replacement;
-    }
-    throw new Error(`Evaluation mode must be one of: ${EVAL_MODES.join("|")} (or legacy A|B|C), got: ${raw}`);
-}
-/** @deprecated kept for callers that still need to serialize as legacy. */
-export function modeToLegacyTier(mode) {
-    return MODE_TO_LEGACY_TIER[mode];
-}
-function defaultWriteWarning(message) {
-    process.stderr.write(`${message}\n`);
-}

package/dist/eval/progress.d.ts

@@ -1,83 +0,0 @@
-/**
- * Lightweight progress logger for `cclaw eval`.
- *
- * The runner is otherwise silent: a full workflow-mode run can easily take
- * a few minutes and the user would see nothing until the Markdown report
- * hits disk. We emit structured events here so the CLI can print concise
- * one-line status updates to stderr (stdout stays reserved for the final
- * report + `--json` output).
- *
- * The logger is intentionally minimal: no ANSI colors, no spinners, no
- * carriage-return rewrites. Those do not survive `tee`, CI log viewers,
- * or the background `runs/tail` path (which copies the stream to a log
- * file), and users also told us "nothing is clear now, everything is
- * long" — so we optimize for log-friendly line-by-line readability.
- */
-import type { EvalMode, WorkflowStageName } from "./types.js";
-export type ProgressEvent = {
-    kind: "run-start";
-    mode: EvalMode;
-    totalCases: number;
-} | {
-    kind: "case-start";
-    caseId: string;
-    stage: string;
-    index: number;
-    total: number;
-} | {
-    kind: "case-end";
-    caseId: string;
-    stage: string;
-    index: number;
-    total: number;
-    passed: boolean;
-    durationMs: number;
-    costUsd?: number;
-} | {
-    kind: "stage-start";
-    caseId: string;
-    stage: WorkflowStageName;
-    index: number;
-    total: number;
-} | {
-    kind: "stage-end";
-    caseId: string;
-    stage: WorkflowStageName;
-    index: number;
-    total: number;
-    passed: boolean;
-    durationMs: number;
-    costUsd?: number;
-} | {
-    kind: "retry";
-    caseId: string;
-    stage?: string;
-    attempt: number;
-    maxAttempts: number;
-    waitMs: number;
-    reason: string;
-} | {
-    kind: "run-end";
-    totalCases: number;
-    passed: number;
-    failed: number;
-    durationMs: number;
-};
-export interface ProgressLogger {
-    emit(event: ProgressEvent): void;
-}
-export declare function noopProgressLogger(): ProgressLogger;
-export interface StderrProgressLoggerOptions {
-    /** Override the underlying write target; defaults to `process.stderr.write`. */
-    writer?: (message: string) => void;
-    /** Return wall-clock in ms. Injectable for tests. */
-    now?: () => number;
-}
-/**
- * Emit a one-line status update per event to stderr.
- *
- * Format is deliberately boring: `[cclaw eval] <message>` so users can grep
- * for the prefix in combined logs. Costs are rendered with up to 4 decimals
- * so sub-cent runs still show a non-zero value.
- */
-export declare function createStderrProgressLogger(opts?: StderrProgressLoggerOptions): ProgressLogger;

package/dist/eval/progress.js

@@ -1,59 +0,0 @@
-const NOOP_LOGGER = { emit() { } };
-export function noopProgressLogger() {
-    return NOOP_LOGGER;
-}
-/**
- * Emit a one-line status update per event to stderr.
- *
- * Format is deliberately boring: `[cclaw eval] <message>` so users can grep
- * for the prefix in combined logs. Costs are rendered with up to 4 decimals
- * so sub-cent runs still show a non-zero value.
- */
-export function createStderrProgressLogger(opts = {}) {
-    const writer = opts.writer ?? ((s) => process.stderr.write(s));
-    return {
-        emit(event) {
-            writer(`[cclaw eval] ${formatEvent(event)}\n`);
-        }
-    };
-}
-function formatDuration(ms) {
-    if (ms < 1000)
-        return `${ms}ms`;
-    const s = ms / 1000;
-    if (s < 60)
-        return `${s.toFixed(1)}s`;
-    const m = Math.floor(s / 60);
-    const rem = Math.round(s - m * 60);
-    return `${m}m${rem.toString().padStart(2, "0")}s`;
-}
-function formatCost(usd) {
-    if (usd === undefined || usd <= 0)
-        return "";
-    return ` $${usd.toFixed(4)}`;
-}
-function formatEvent(event) {
-    switch (event.kind) {
-        case "run-start":
-            return `start mode=${event.mode} cases=${event.totalCases}`;
-        case "case-start":
-            return `[${event.index}/${event.total}] ${event.caseId} (${event.stage}) ...`;
-        case "case-end": {
-            const status = event.passed ? "PASS" : "FAIL";
-            return (`[${event.index}/${event.total}] ${event.caseId} (${event.stage}) ${status} ` +
-                `in ${formatDuration(event.durationMs)}${formatCost(event.costUsd)}`);
-        }
-        case "stage-start":
-            return `  stage ${event.stage} ...`;
-        case "stage-end": {
-            const status = event.passed ? "ok" : "fail";
-            return `  stage ${event.stage} ${status} in ${formatDuration(event.durationMs)}${formatCost(event.costUsd)}`;
-        }
-        case "retry":
-            return (`  retry ${event.caseId}${event.stage ? `/${event.stage}` : ""} ` +
-                `attempt ${event.attempt}/${event.maxAttempts} in ${formatDuration(event.waitMs)} (${event.reason})`);
-        case "run-end":
-            return (`done pass=${event.passed} fail=${event.failed} total=${event.totalCases} ` +
-                `in ${formatDuration(event.durationMs)}`);
-    }
-}

package/dist/eval/report.d.ts

@@ -1,11 +0,0 @@
-import type { EvalReport } from "./types.js";
-export declare function reportsDir(projectRoot: string): string;
-export declare function defaultReportBasename(report: EvalReport): string;
-/**
- * Format a report as a human-readable Markdown document. Keeping the layout
- * stable matters: CI posts diffs against earlier reports, and unit tests use
- * the output as a regression guard.
- */
-export declare function formatMarkdownReport(report: EvalReport): string;
-export declare function writeJsonReport(projectRoot: string, report: EvalReport, basename?: string): Promise<string>;
-export declare function writeMarkdownReport(projectRoot: string, report: EvalReport, basename?: string): Promise<string>;