cclaw-cli 0.24.0 → 0.26.0

package/dist/eval/runner.js

@@ -1,9 +1,15 @@
 import { randomUUID } from "node:crypto";
 import { CCLAW_VERSION } from "../constants.js";
 import { FLOW_STAGES } from "../types.js";
+import { runSingleShot } from "./agents/single-shot.js";
+import { MaxTurnsExceededError, runWithTools } from "./agents/with-tools.js";
 import { compareAgainstBaselines, loadBaselinesByStage } from "./baseline.js";
 import { loadCorpus, readExtraFixtures, readFixtureArtifact } from "./corpus.js";
 import { loadEvalConfig } from "./config-loader.js";
+import { createCostGuard, DailyCostCapExceededError } from "./cost-guard.js";
+import { createEvalClient, EvalLlmError } from "./llm-client.js";
+import { loadAllRubrics } from "./rubric-loader.js";
+import { judgeResultsToVerifiers, runJudge } from "./verifiers/judge.js";
 import { verifyRules } from "./verifiers/rules.js";
 import { verifyStructural } from "./verifiers/structural.js";
 import { verifyTraceability } from "./verifiers/traceability.js";
@@ -26,16 +32,39 @@ function skeletonVerifierResult(message, details) {
 /**
  * --schema-only narrows to structural. --rules opens up rules + traceability
  * on top of structural (traceability is a rule-family verifier even though
- * it lives in its own module). Default (no flag) matches --schema-only for
- * backwards compatibility with the Step 1 gate.
+ * it lives in its own module). --judge opens up the LLM judge and, for
+ * Tier A, the single-shot agent-under-test. --schema-only always wins so
+ * the LLM-free PR gate never pays for tokens even if stale flags collide.
  */
 function resolveRunFlags(options) {
     const rulesRequested = options.rules === true;
     const schemaOnly = options.schemaOnly === true;
+    const judgeRequested = options.judge === true;
+    const tier = options.tier ?? "A";
+    const runJudge = judgeRequested && !schemaOnly;
+    const runAgent = runJudge && (tier === "A" || tier === "B");
     return {
         runStructural: true,
         runRules: rulesRequested && !schemaOnly,
-        runTraceability: rulesRequested && !schemaOnly
+        runTraceability: rulesRequested && !schemaOnly,
+        runJudge,
+        runAgent
+    };
+}
+/**
+ * Wrap a client so every chat() result is accounted against the cost
+ * guard before being returned. The guard throws
+ * DailyCostCapExceededError if committing the call would cross the
+ * configured cap — the runner surfaces that as a hard failure so
+ * nightly CI fails loud instead of silently overspending.
+ */
+function wrapClientWithCostGuard(client, costGuard, fallbackModel) {
+    return {
+        async chat(request) {
+            const response = await client.chat(request);
+            await costGuard.commit(response.model || fallbackModel, response.usage);
+            return response;
+        }
     };
 }
 async function loadArtifactOrRecord(projectRoot, caseEntry, verifierResults) {
@@ -54,17 +83,107 @@ async function loadArtifactOrRecord(projectRoot, caseEntry, verifierResults) {
         return undefined;
     }
 }
-async function runCase(projectRoot, caseEntry, plannedTier, flags) {
+async function runCase(ctx) {
+    const { projectRoot, caseEntry, plannedTier, flags, config, client, costGuard, rubrics } = ctx;
     const started = Date.now();
     const verifierResults = [];
     const expected = caseEntry.expected;
+    let caseCostUsd = 0;
     const hasStructural = !!expected?.structural && Object.keys(expected.structural).length > 0;
     const hasRules = flags.runRules && !!expected?.rules && Object.keys(expected.rules).length > 0;
     const hasTraceability = flags.runTraceability && !!expected?.traceability;
-    const needsArtifact = hasStructural || hasRules || hasTraceability;
+    const judgeRequested = flags.runJudge && !!expected?.judge;
+    const needsArtifact = hasStructural || hasRules || hasTraceability || judgeRequested;
     let artifact;
     if (needsArtifact) {
-        artifact = await loadArtifactOrRecord(projectRoot, caseEntry, verifierResults);
+        if (flags.runAgent && judgeRequested && client && plannedTier === "A") {
+            try {
+                const produced = await runSingleShot({
+                    caseEntry,
+                    config,
+                    projectRoot,
+                    client
+                });
+                artifact = produced.artifact;
+                caseCostUsd += produced.usageUsd;
+                verifierResults.push({
+                    kind: "workflow",
+                    id: "agent:single-shot",
+                    ok: true,
+                    score: 1,
+                    message: `single-shot agent produced ${produced.artifact.length} char(s) in ${produced.durationMs}ms`,
+                    details: {
+                        model: produced.model,
+                        tokensIn: produced.usage.promptTokens,
+                        tokensOut: produced.usage.completionTokens,
+                        usageUsd: produced.usageUsd,
+                        attempts: produced.attempts
+                    }
+                });
+            }
+            catch (err) {
+                if (err instanceof DailyCostCapExceededError)
+                    throw err;
+                const retryable = err instanceof EvalLlmError ? err.retryable : false;
+                verifierResults.push({
+                    kind: "workflow",
+                    id: "agent:single-shot",
+                    ok: false,
+                    score: 0,
+                    message: err instanceof Error ? err.message : String(err),
+                    details: { retryable }
+                });
+            }
+        }
+        else if (flags.runAgent && judgeRequested && client && plannedTier === "B") {
+            try {
+                const produced = await runWithTools({
+                    caseEntry,
+                    config,
+                    projectRoot,
+                    client
+                });
+                artifact = produced.artifact;
+                caseCostUsd += produced.usageUsd;
+                verifierResults.push({
+                    kind: "workflow",
+                    id: "agent:with-tools",
+                    ok: true,
+                    score: 1,
+                    message: `with-tools agent produced ${produced.artifact.length} char(s) in ` +
+                        `${produced.durationMs}ms across ${produced.toolUse.turns} turn(s) ` +
+                        `(${produced.toolUse.calls} tool call(s))`,
+                    details: {
+                        model: produced.model,
+                        tokensIn: produced.usage.promptTokens,
+                        tokensOut: produced.usage.completionTokens,
+                        usageUsd: produced.usageUsd,
+                        attempts: produced.attempts,
+                        toolUse: produced.toolUse
+                    }
+                });
+            }
+            catch (err) {
+                if (err instanceof DailyCostCapExceededError)
+                    throw err;
+                const retryable = err instanceof EvalLlmError ? err.retryable : false;
+                const maxTurns = err instanceof MaxTurnsExceededError ? err.turns : undefined;
+                verifierResults.push({
+                    kind: "workflow",
+                    id: "agent:with-tools",
+                    ok: false,
+                    score: 0,
+                    message: err instanceof Error ? err.message : String(err),
+                    details: {
+                        retryable,
+                        ...(maxTurns !== undefined ? { maxTurnsExceeded: maxTurns } : {})
+                    }
+                });
+            }
+        }
+        else {
+            artifact = await loadArtifactOrRecord(projectRoot, caseEntry, verifierResults);
+        }
         if (artifact === undefined && verifierResults.length === 0) {
             verifierResults.push({
                 kind: "structural",
@@ -111,6 +230,46 @@ async function runCase(projectRoot, caseEntry, plannedTier, flags) {
             });
         }
     }
+    if (judgeRequested && artifact !== undefined && client) {
+        const rubric = rubrics.get(caseEntry.stage);
+        if (!rubric) {
+            verifierResults.push({
+                kind: "judge",
+                id: "judge:rubric:missing",
+                ok: false,
+                score: 0,
+                message: `No rubric at .cclaw/evals/rubrics/${caseEntry.stage}.yaml. Add one before running --judge.`,
+                details: { stage: caseEntry.stage }
+            });
+        }
+        else {
+            try {
+                const invocation = await runJudge({
+                    artifact,
+                    rubric,
+                    config,
+                    client,
+                    caseHint: expected.judge
+                });
+                caseCostUsd += invocation.usageUsd;
+                const judgeVerifiers = judgeResultsToVerifiers(rubric, invocation, config, expected.judge);
+                verifierResults.push(...judgeVerifiers);
+            }
+            catch (err) {
+                if (err instanceof DailyCostCapExceededError)
+                    throw err;
+                const retryable = err instanceof EvalLlmError ? err.retryable : false;
+                verifierResults.push({
+                    kind: "judge",
+                    id: "judge:invocation:error",
+                    ok: false,
+                    score: 0,
+                    message: err instanceof Error ? err.message : String(err),
+                    details: { retryable, rubricId: rubric.id }
+                });
+            }
+        }
+    }
     const nonSkippedResults = verifierResults.filter((r) => r.details?.skipped !== true);
     const allOk = nonSkippedResults.length === 0
         ? verifierResults.every((r) => r.ok)
@@ -121,6 +280,7 @@ async function runCase(projectRoot, caseEntry, plannedTier, flags) {
         tier: plannedTier,
         passed: allOk,
         durationMs: Date.now() - started,
+        costUsd: caseCostUsd > 0 ? Number(caseCostUsd.toFixed(6)) : undefined,
         verifierResults
     };
 }
@@ -173,10 +333,13 @@ export async function runEval(options) {
     if (corpus.length === 0) {
         notes.push("Corpus is empty. Seed cases live under `.cclaw/evals/corpus/<stage>/*.yaml`.");
     }
-    if (options.judge) {
-        notes.push("--judge is accepted; LLM judging is not wired yet.");
-    }
     const flags = resolveRunFlags(options);
+    if (flags.runJudge && !config.apiKey && !options.llmClient) {
+        notes.push("--judge requires CCLAW_EVAL_API_KEY (or an injected client for tests); judge pipeline will report errors per case.");
+    }
+    if ((options.tier ?? "A") !== "A" && flags.runJudge) {
+        notes.push("Tier B/C agent-under-test is not wired yet; --judge will score the committed fixture as a stand-in.");
+    }
     if (options.dryRun === true) {
         const summary = {
             kind: "dry-run",
@@ -190,17 +353,35 @@ export async function runEval(options) {
             verifiersAvailable: {
                 structural: flags.runStructural,
                 rules: flags.runRules,
-                judge: false,
-                workflow: false
+                judge: flags.runJudge,
+                workflow: flags.runAgent
             },
             notes
         };
         return summary;
     }
+    const costGuard = createCostGuard(options.projectRoot, config);
+    let wrappedClient;
+    if (flags.runJudge) {
+        const base = options.llmClient ?? createEvalClient(config);
+        wrappedClient = wrapClientWithCostGuard(base, costGuard, config.judgeModel ?? config.model);
+    }
+    const rubrics = flags.runJudge
+        ? await loadAllRubrics(options.projectRoot)
+        : new Map();
     const now = new Date().toISOString();
     const caseResults = [];
     for (const item of corpus) {
-        caseResults.push(await runCase(options.projectRoot, item, plannedTier, flags));
+        caseResults.push(await runCase({
+            projectRoot: options.projectRoot,
+            caseEntry: item,
+            plannedTier,
+            flags,
+            config,
+            client: wrappedClient,
+            costGuard,
+            rubrics
+        }));
     }
     const stages = stagesInResults(caseResults);
     const baselines = await loadBaselinesByStage(options.projectRoot, stages);

package/dist/eval/sandbox.d.ts

@@ -0,0 +1,38 @@
+export declare class SandboxEscapeError extends Error {
+    readonly requestedPath: string;
+    constructor(requestedPath: string, reason: string);
+}
+export interface SandboxOptions {
+    /** Project root that `contextFiles` are resolved against. */
+    projectRoot: string;
+    /** Case-relative paths to copy into the sandbox before the agent starts. */
+    contextFiles?: string[];
+    /**
+     * Base directory that will host the per-case tmpdir. Defaults to
+     * `os.tmpdir()`. Tests inject a repo-local path so CI leaves no
+     * traces in `/tmp` when assertions fail.
+     */
+    baseDir?: string;
+    /** Override the per-case suffix. Primarily for deterministic tests. */
+    idOverride?: string;
+}
+export interface Sandbox {
+    /** Absolute path to the sandbox root directory. */
+    root: string;
+    /**
+     * Resolve `requested` relative to the sandbox root and return the
+     * absolute, realpath'd filesystem path. Throws
+     * `SandboxEscapeError` when the resolution crosses the boundary.
+     *
+     * `allowMissing: true` lets callers pre-resolve a destination for a
+     * write where the final component doesn't exist yet — the parent
+     * directory is realpath'd to still catch symlink escapes.
+     */
+    resolve(requested: string, options?: {
+        allowMissing?: boolean;
+    }): Promise<string>;
+    /** Remove the sandbox directory. Idempotent. */
+    dispose(): Promise<void>;
+}
+/** Create and prep a fresh sandbox. Callers own cleanup via `dispose()`. */
+export declare function createSandbox(options: SandboxOptions): Promise<Sandbox>;

package/dist/eval/sandbox.js

@@ -0,0 +1,137 @@
+/**
+ * Per-case sandbox for the Tier B with-tools agent.
+ *
+ * Every case gets its own `os.tmpdir()/cclaw-eval-<uuid>/` directory. Any
+ * `contextFiles` the case declares are copied in relative to the project
+ * root, and every tool invocation resolves paths against the sandbox
+ * root with a defensive check that refuses symlinks and `..` escapes.
+ *
+ * Design notes:
+ *
+ * - The sandbox is intentionally tiny (one directory, no symlink
+ *   creation, no executable bits). We rely on `fs.realpath` on every
+ *   resolved path so hostile tool output that creates a symlink to
+ *   `/etc/passwd` and then tries to read it still trips the boundary
+ *   check.
+ * - Cleanup is handled by `dispose()`; callers (runner, tests) must
+ *   invoke it in a `try/finally` so leftover temp directories never
+ *   accumulate.
+ * - The sandbox does not preserve the project's directory structure
+ *   verbatim. Each entry in `contextFiles` is copied flat into
+ *   `sandboxRoot/<basename>` unless it contains path separators, in
+ *   which case the full relative layout is recreated. That keeps demo
+ *   cases portable while still letting richer cases place files under
+ *   subdirectories (e.g. `.cclaw/skills/brainstorming/SKILL.md`).
+ */
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+export class SandboxEscapeError extends Error {
+    requestedPath;
+    constructor(requestedPath, reason) {
+        super(`Sandbox refused path "${requestedPath}": ${reason}.`);
+        this.name = "SandboxEscapeError";
+        this.requestedPath = requestedPath;
+    }
+}
+/** Create and prep a fresh sandbox. Callers own cleanup via `dispose()`. */
+export async function createSandbox(options) {
+    const baseDir = options.baseDir ?? os.tmpdir();
+    const id = options.idOverride ?? randomUUID();
+    const root = path.join(baseDir, `cclaw-eval-${id}`);
+    await fs.mkdir(root, { recursive: true });
+    const realRoot = await fs.realpath(root);
+    if (options.contextFiles && options.contextFiles.length > 0) {
+        for (const rel of options.contextFiles) {
+            await copyContextFile(options.projectRoot, realRoot, rel);
+        }
+    }
+    async function resolveInside(requested, opts = {}) {
+        if (typeof requested !== "string" || requested.length === 0) {
+            throw new SandboxEscapeError(String(requested), "path must be a non-empty string");
+        }
+        if (path.isAbsolute(requested)) {
+            throw new SandboxEscapeError(requested, "absolute paths are not allowed");
+        }
+        if (requested.includes("\0")) {
+            throw new SandboxEscapeError(requested, "NUL byte in path");
+        }
+        const joined = path.resolve(realRoot, requested);
+        const relative = path.relative(realRoot, joined);
+        if (relative.startsWith("..") || path.isAbsolute(relative)) {
+            throw new SandboxEscapeError(requested, "resolves outside the sandbox");
+        }
+        let finalPath;
+        try {
+            finalPath = await fs.realpath(joined);
+        }
+        catch (err) {
+            if (!opts.allowMissing) {
+                throw new SandboxEscapeError(requested, `realpath failed: ${err.message}`);
+            }
+            const existingAncestor = await findExistingAncestor(joined, realRoot);
+            if (!existingAncestor) {
+                throw new SandboxEscapeError(requested, "no existing ancestor inside the sandbox");
+            }
+            const ancestorRel = path.relative(realRoot, existingAncestor.real);
+            if (ancestorRel.startsWith("..") || path.isAbsolute(ancestorRel)) {
+                throw new SandboxEscapeError(requested, "parent resolves outside the sandbox");
+            }
+            finalPath = path.join(existingAncestor.real, existingAncestor.trailing);
+        }
+        const finalRel = path.relative(realRoot, finalPath);
+        if (finalRel.startsWith("..") || path.isAbsolute(finalRel)) {
+            throw new SandboxEscapeError(requested, "realpath escapes the sandbox");
+        }
+        return finalPath;
+    }
+    return {
+        root: realRoot,
+        resolve: resolveInside,
+        async dispose() {
+            await fs.rm(realRoot, { recursive: true, force: true });
+        }
+    };
+}
+async function findExistingAncestor(target, stopAt) {
+    const segments = [];
+    let current = target;
+    while (true) {
+        try {
+            const real = await fs.realpath(current);
+            return { real, trailing: path.join(...segments.reverse()) };
+        }
+        catch {
+            const parent = path.dirname(current);
+            if (parent === current)
+                return undefined;
+            segments.push(path.basename(current));
+            if (path.relative(stopAt, parent).startsWith(".."))
+                return undefined;
+            current = parent;
+        }
+    }
+}
+async function copyContextFile(projectRoot, sandboxRoot, relPath) {
+    if (path.isAbsolute(relPath)) {
+        throw new Error(`context_files must be project-relative: ${relPath}`);
+    }
+    const src = path.resolve(projectRoot, relPath);
+    const srcReal = await fs.realpath(src);
+    const projectReal = await fs.realpath(projectRoot);
+    const inside = path.relative(projectReal, srcReal);
+    if (inside.startsWith("..") || path.isAbsolute(inside)) {
+        throw new Error(`context_files entry resolves outside the project: ${relPath}`);
+    }
+    const stat = await fs.stat(srcReal);
+    if (stat.isDirectory()) {
+        const dest = path.join(sandboxRoot, relPath);
+        await fs.mkdir(dest, { recursive: true });
+        await fs.cp(srcReal, dest, { recursive: true });
+        return;
+    }
+    const dest = path.join(sandboxRoot, relPath);
+    await fs.mkdir(path.dirname(dest), { recursive: true });
+    await fs.copyFile(srcReal, dest);
+}

package/dist/eval/tools/glob.d.ts

@@ -0,0 +1,2 @@
+import { type SandboxTool } from "./types.js";
+export declare const globTool: SandboxTool;

package/dist/eval/tools/glob.js

@@ -0,0 +1,163 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { SandboxEscapeError } from "../sandbox.js";
+import { parseArgs, requireString, truncatePayload } from "./types.js";
+const DESCRIPTION = "List files inside the sandbox whose relative path matches a glob-style " +
+    "pattern. Supports `*` (any chars within a path segment) and `**` " +
+    "(any number of path segments). Returns matching paths, one per line.";
+const MAX_MATCHES = 500;
+export const globTool = {
+    descriptor: {
+        name: "glob",
+        description: DESCRIPTION,
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            required: ["pattern"],
+            properties: {
+                pattern: {
+                    type: "string",
+                    description: "Glob pattern, relative to the sandbox root."
+                }
+            }
+        }
+    },
+    async invoke(rawArgs, ctx) {
+        let args;
+        try {
+            args = parseArgs(rawArgs);
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        let pattern;
+        try {
+            pattern = requireString(args, "pattern");
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        if (pattern.includes("\0")) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: '"pattern" must not contain NUL bytes'
+            };
+        }
+        let regex;
+        try {
+            regex = globToRegExp(pattern);
+        }
+        catch (err) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: err.message
+            };
+        }
+        const matches = [];
+        try {
+            await walk(ctx.sandbox.root, "", matches, regex);
+        }
+        catch (err) {
+            if (err instanceof SandboxEscapeError) {
+                return {
+                    ok: false,
+                    name: this.descriptor.name,
+                    error: err.message,
+                    details: { deniedPath: pattern }
+                };
+            }
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: `walk failed: ${err.message}`
+            };
+        }
+        matches.sort();
+        const capped = matches.slice(0, MAX_MATCHES);
+        const body = capped.length > 0
+            ? capped.join("\n") +
+                (matches.length > capped.length
+                    ? `\n…[truncated at ${MAX_MATCHES} matches]`
+                    : "")
+            : "(no matches)";
+        return {
+            ok: true,
+            name: this.descriptor.name,
+            content: truncatePayload(body, ctx.maxResultBytes),
+            details: {
+                pattern,
+                matches: capped.length,
+                totalMatches: matches.length,
+                truncated: matches.length > capped.length
+            }
+        };
+    }
+};
+async function walk(root, rel, acc, regex) {
+    const dir = path.join(root, rel);
+    let entries;
+    try {
+        entries = (await fs.readdir(dir, { withFileTypes: true }));
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        const childRel = rel ? path.join(rel, entry.name) : entry.name;
+        if (entry.isSymbolicLink())
+            continue;
+        if (entry.isDirectory()) {
+            await walk(root, childRel, acc, regex);
+            continue;
+        }
+        if (entry.isFile() && regex.test(childRel.replace(/\\/g, "/"))) {
+            acc.push(childRel);
+        }
+    }
+}
+/**
+ * Minimal glob → regex: `**` matches zero or more path segments, `*`
+ * matches anything except `/`, `?` matches a single non-slash char.
+ * Everything else is escaped. Intentionally narrower than full
+ * bash-style expansion so behavior is easy to reason about.
+ */
+function globToRegExp(pattern) {
+    const normalized = pattern.replace(/\\/g, "/");
+    let src = "^";
+    let i = 0;
+    while (i < normalized.length) {
+        const c = normalized[i];
+        if (c === "*") {
+            if (normalized[i + 1] === "*") {
+                if (normalized[i + 2] === "/") {
+                    src += "(?:.*/)?";
+                    i += 3;
+                }
+                else {
+                    src += ".*";
+                    i += 2;
+                }
+            }
+            else {
+                src += "[^/]*";
+                i += 1;
+            }
+        }
+        else if (c === "?") {
+            src += "[^/]";
+            i += 1;
+        }
+        else if ("+()|^$.{}[]\\".includes(c)) {
+            src += `\\${c}`;
+            i += 1;
+        }
+        else {
+            src += c;
+            i += 1;
+        }
+    }
+    src += "$";
+    return new RegExp(src);
+}

package/dist/eval/tools/grep.d.ts

@@ -0,0 +1,2 @@
+import { type SandboxTool } from "./types.js";
+export declare const grepTool: SandboxTool;