cclaw-cli 0.24.0 → 0.26.0

package/dist/eval/tools/grep.js

@@ -0,0 +1,152 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { SandboxEscapeError } from "../sandbox.js";
+import { parseArgs, requireString, optionalNumber, truncatePayload } from "./types.js";
+const DESCRIPTION = "Search the sandbox for a regular expression. Returns matching lines in " +
+    "`path:line:text` form. Accepts optional `caseInsensitive` and a per-call " +
+    "`maxMatches` cap (default 100, hard max 500).";
+const HARD_MAX = 500;
+export const grepTool = {
+    descriptor: {
+        name: "grep",
+        description: DESCRIPTION,
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            required: ["pattern"],
+            properties: {
+                pattern: {
+                    type: "string",
+                    description: "Regular expression compiled with JavaScript semantics."
+                },
+                caseInsensitive: {
+                    type: "boolean",
+                    description: "Match case-insensitively (default false)."
+                },
+                maxMatches: {
+                    type: "integer",
+                    minimum: 1,
+                    description: "Stop after N matches (default 100, hard max 500)."
+                }
+            }
+        }
+    },
+    async invoke(rawArgs, ctx) {
+        let args;
+        try {
+            args = parseArgs(rawArgs);
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        let pattern;
+        try {
+            pattern = requireString(args, "pattern");
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        const caseInsensitive = args.caseInsensitive === true;
+        let maxMatches;
+        try {
+            const raw = optionalNumber(args, "maxMatches");
+            maxMatches = raw === undefined ? 100 : Math.min(HARD_MAX, Math.max(1, Math.floor(raw)));
+        }
+        catch (err) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: err.message
+            };
+        }
+        let regex;
+        try {
+            regex = new RegExp(pattern, caseInsensitive ? "i" : "");
+        }
+        catch (err) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: `invalid regex: ${err.message}`
+            };
+        }
+        let filesScanned = 0;
+        const hits = [];
+        try {
+            await walk(ctx.sandbox.root, "", async (relPath, abs) => {
+                if (hits.length >= maxMatches)
+                    return false;
+                let content;
+                try {
+                    content = await fs.readFile(abs, "utf8");
+                }
+                catch {
+                    return true;
+                }
+                filesScanned += 1;
+                const lines = content.split(/\r?\n/);
+                for (let i = 0; i < lines.length; i += 1) {
+                    const line = lines[i];
+                    if (regex.test(line)) {
+                        hits.push(`${relPath}:${i + 1}:${line}`);
+                        if (hits.length >= maxMatches)
+                            return false;
+                    }
+                }
+                return true;
+            });
+        }
+        catch (err) {
+            if (err instanceof SandboxEscapeError) {
+                return {
+                    ok: false,
+                    name: this.descriptor.name,
+                    error: err.message,
+                    details: { deniedPath: pattern }
+                };
+            }
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: `walk failed: ${err.message}`
+            };
+        }
+        const body = hits.length > 0 ? hits.join("\n") : "(no matches)";
+        return {
+            ok: true,
+            name: this.descriptor.name,
+            content: truncatePayload(body, ctx.maxResultBytes),
+            details: {
+                pattern,
+                caseInsensitive,
+                matches: hits.length,
+                filesScanned,
+                truncated: hits.length >= maxMatches
+            }
+        };
+    }
+};
+async function walk(root, rel, visit) {
+    const dir = path.join(root, rel);
+    let entries;
+    try {
+        entries = (await fs.readdir(dir, { withFileTypes: true }));
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        const childRel = rel ? path.join(rel, entry.name) : entry.name;
+        if (entry.isSymbolicLink())
+            continue;
+        if (entry.isDirectory()) {
+            await walk(root, childRel, visit);
+            continue;
+        }
+        if (entry.isFile()) {
+            const keepGoing = await visit(childRel.replace(/\\/g, "/"), path.join(root, childRel));
+            if (keepGoing === false)
+                return;
+        }
+    }
+}

package/dist/eval/tools/index.d.ts

@@ -0,0 +1,7 @@
+import type { SandboxTool } from "./types.js";
+export { SandboxTool, ToolResult, ToolContext, truncatePayload } from "./types.js";
+export declare const BUILTIN_TOOLS: SandboxTool[];
+/** Build a lookup for the agent loop. */
+export declare function toolsByName(tools?: SandboxTool[]): Map<string, SandboxTool>;
+/** Shape a tool list for OpenAI-style `tools[]` in the chat request. */
+export declare function toolsForRequest(tools?: SandboxTool[]): unknown[];

package/dist/eval/tools/index.js

@@ -0,0 +1,35 @@
+/**
+ * Registry of sandbox-confined tools used by the Tier B with-tools agent.
+ *
+ * The registry order defines the advertised schema order in the
+ * function-calling payload. Keeping it stable means judges reading
+ * generated traces can rely on predictable tool descriptions.
+ */
+import { globTool } from "./glob.js";
+import { grepTool } from "./grep.js";
+import { readTool } from "./read.js";
+import { writeTool } from "./write.js";
+export { truncatePayload } from "./types.js";
+export const BUILTIN_TOOLS = [readTool, writeTool, globTool, grepTool];
+/** Build a lookup for the agent loop. */
+export function toolsByName(tools = BUILTIN_TOOLS) {
+    const map = new Map();
+    for (const tool of tools) {
+        if (map.has(tool.descriptor.name)) {
+            throw new Error(`duplicate tool name: ${tool.descriptor.name}`);
+        }
+        map.set(tool.descriptor.name, tool);
+    }
+    return map;
+}
+/** Shape a tool list for OpenAI-style `tools[]` in the chat request. */
+export function toolsForRequest(tools = BUILTIN_TOOLS) {
+    return tools.map((tool) => ({
+        type: "function",
+        function: {
+            name: tool.descriptor.name,
+            description: tool.descriptor.description,
+            parameters: tool.descriptor.parameters
+        }
+    }));
+}

package/dist/eval/tools/read.d.ts

@@ -0,0 +1,2 @@
+import { type SandboxTool } from "./types.js";
+export declare const readTool: SandboxTool;

package/dist/eval/tools/read.js

@@ -0,0 +1,122 @@
+import fs from "node:fs/promises";
+import { SandboxEscapeError } from "../sandbox.js";
+import { parseArgs, requireString, optionalNumber, truncatePayload } from "./types.js";
+const DESCRIPTION = "Read a UTF-8 text file from the sandbox. Returns the file contents. " +
+    "Supports optional 1-indexed `offset` and `limit` to read a slice.";
+export const readTool = {
+    descriptor: {
+        name: "read_file",
+        description: DESCRIPTION,
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            required: ["path"],
+            properties: {
+                path: {
+                    type: "string",
+                    description: "Path relative to the sandbox root."
+                },
+                offset: {
+                    type: "integer",
+                    minimum: 1,
+                    description: "1-indexed start line (inclusive)."
+                },
+                limit: {
+                    type: "integer",
+                    minimum: 1,
+                    description: "Maximum number of lines to return."
+                }
+            }
+        }
+    },
+    async invoke(rawArgs, ctx) {
+        let args;
+        try {
+            args = parseArgs(rawArgs);
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        let relPath;
+        try {
+            relPath = requireString(args, "path");
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        let offset;
+        let limit;
+        try {
+            offset = optionalNumber(args, "offset");
+            limit = optionalNumber(args, "limit");
+        }
+        catch (err) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: err.message
+            };
+        }
+        if (offset !== undefined && (!Number.isInteger(offset) || offset < 1)) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: '"offset" must be a positive integer'
+            };
+        }
+        if (limit !== undefined && (!Number.isInteger(limit) || limit < 1)) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: '"limit" must be a positive integer'
+            };
+        }
+        let abs;
+        try {
+            abs = await ctx.sandbox.resolve(relPath);
+        }
+        catch (err) {
+            const denied = err instanceof SandboxEscapeError ? relPath : undefined;
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: err.message,
+                details: denied ? { deniedPath: denied } : undefined
+            };
+        }
+        let raw;
+        try {
+            raw = await fs.readFile(abs, "utf8");
+        }
+        catch (err) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: `read failed: ${err.message}`,
+                details: { path: relPath }
+            };
+        }
+        let content = raw;
+        let effectiveLines;
+        if (offset !== undefined || limit !== undefined) {
+            const lines = raw.split(/\r?\n/);
+            const start = Math.max(0, (offset ?? 1) - 1);
+            const end = limit !== undefined ? Math.min(lines.length, start + limit) : lines.length;
+            const slice = lines.slice(start, end);
+            content = slice.join("\n");
+            effectiveLines = slice.length;
+        }
+        const truncated = truncatePayload(content, ctx.maxResultBytes);
+        return {
+            ok: true,
+            name: this.descriptor.name,
+            content: truncated,
+            details: {
+                path: relPath,
+                bytes: Buffer.byteLength(truncated, "utf8"),
+                truncated: truncated !== content,
+                ...(effectiveLines !== undefined ? { lines: effectiveLines } : {})
+            }
+        };
+    }
+};

package/dist/eval/tools/types.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Shared types for Tier B sandbox-confined tools.
+ *
+ * Tools are plain async functions: they take validated arguments and a
+ * sandbox handle and return a structured result. The runner serializes
+ * results for the model as JSON; the `SandboxTool.invoke` wrapper keeps
+ * both the raw structured output (for tests/metrics) and the stringified
+ * model-facing payload.
+ */
+import type { Sandbox } from "../sandbox.js";
+export interface ToolDescriptor {
+    /** Name the model calls (must match the function-calling schema). */
+    name: string;
+    /** Human-readable prompt shown to the model. */
+    description: string;
+    /** JSON schema shipped with the OpenAI-style `tools[]` array. */
+    parameters: Record<string, unknown>;
+}
+export interface ToolContext {
+    sandbox: Sandbox;
+    /**
+     * Maximum bytes the tool may return in `content`. Results longer than
+     * this are truncated with a trailing marker so the model sees the
+     * cutoff.
+     */
+    maxResultBytes: number;
+}
+export interface ToolSuccess {
+    ok: true;
+    name: string;
+    content: string;
+    details?: Record<string, unknown>;
+}
+export interface ToolFailure {
+    ok: false;
+    name: string;
+    error: string;
+    details?: Record<string, unknown>;
+}
+export type ToolResult = ToolSuccess | ToolFailure;
+export interface SandboxTool {
+    descriptor: ToolDescriptor;
+    invoke(rawArgs: string, ctx: ToolContext): Promise<ToolResult>;
+}
+/** Truncate a result payload to `maxBytes` with a visible cutoff marker. */
+export declare function truncatePayload(payload: string, maxBytes: number): string;
+export declare function parseArgs(raw: string): Record<string, unknown>;
+export declare function requireString(args: Record<string, unknown>, key: string): string;
+export declare function optionalNumber(args: Record<string, unknown>, key: string): number | undefined;

package/dist/eval/tools/types.js

@@ -0,0 +1,41 @@
+/** Truncate a result payload to `maxBytes` with a visible cutoff marker. */
+export function truncatePayload(payload, maxBytes) {
+    if (Buffer.byteLength(payload, "utf8") <= maxBytes)
+        return payload;
+    const marker = "\n…[truncated by cclaw sandbox]";
+    const budget = Math.max(0, maxBytes - Buffer.byteLength(marker, "utf8"));
+    const buf = Buffer.from(payload, "utf8").subarray(0, budget);
+    return `${buf.toString("utf8")}${marker}`;
+}
+export function parseArgs(raw) {
+    if (typeof raw !== "string" || raw.trim() === "") {
+        throw new Error("tool arguments missing");
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        throw new Error(`tool arguments are not valid JSON: ${err.message}`);
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+        throw new Error("tool arguments must be a JSON object");
+    }
+    return parsed;
+}
+export function requireString(args, key) {
+    const value = args[key];
+    if (typeof value !== "string" || value.length === 0) {
+        throw new Error(`"${key}" must be a non-empty string`);
+    }
+    return value;
+}
+export function optionalNumber(args, key) {
+    const value = args[key];
+    if (value === undefined || value === null)
+        return undefined;
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        throw new Error(`"${key}" must be a finite number`);
+    }
+    return value;
+}

package/dist/eval/tools/write.d.ts

@@ -0,0 +1,2 @@
+import { type SandboxTool } from "./types.js";
+export declare const writeTool: SandboxTool;

package/dist/eval/tools/write.js

@@ -0,0 +1,92 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { SandboxEscapeError } from "../sandbox.js";
+import { parseArgs, requireString, truncatePayload } from "./types.js";
+const DESCRIPTION = "Write a UTF-8 text file inside the sandbox. Creates parent directories " +
+    "as needed. Overwrites existing files. Only paths inside the sandbox " +
+    "are accepted.";
+export const writeTool = {
+    descriptor: {
+        name: "write_file",
+        description: DESCRIPTION,
+        parameters: {
+            type: "object",
+            additionalProperties: false,
+            required: ["path", "content"],
+            properties: {
+                path: {
+                    type: "string",
+                    description: "Path relative to the sandbox root."
+                },
+                content: {
+                    type: "string",
+                    description: "UTF-8 contents to write."
+                }
+            }
+        }
+    },
+    async invoke(rawArgs, ctx) {
+        let args;
+        try {
+            args = parseArgs(rawArgs);
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        let relPath;
+        try {
+            relPath = requireString(args, "path");
+        }
+        catch (err) {
+            return { ok: false, name: this.descriptor.name, error: err.message };
+        }
+        const rawContent = args.content;
+        if (typeof rawContent !== "string") {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: '"content" must be a string'
+            };
+        }
+        const payloadBytes = Buffer.byteLength(rawContent, "utf8");
+        if (payloadBytes > ctx.maxResultBytes * 4) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: `"content" exceeds per-invocation ceiling (${payloadBytes} bytes).`
+            };
+        }
+        let abs;
+        try {
+            abs = await ctx.sandbox.resolve(relPath, { allowMissing: true });
+        }
+        catch (err) {
+            const denied = err instanceof SandboxEscapeError ? relPath : undefined;
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: err.message,
+                details: denied ? { deniedPath: denied } : undefined
+            };
+        }
+        try {
+            await fs.mkdir(path.dirname(abs), { recursive: true });
+            await fs.writeFile(abs, rawContent, "utf8");
+        }
+        catch (err) {
+            return {
+                ok: false,
+                name: this.descriptor.name,
+                error: `write failed: ${err.message}`,
+                details: { path: relPath }
+            };
+        }
+        const summary = `wrote ${payloadBytes} byte(s) to ${relPath}`;
+        return {
+            ok: true,
+            name: this.descriptor.name,
+            content: truncatePayload(summary, ctx.maxResultBytes),
+            details: { path: relPath, bytes: payloadBytes }
+        };
+    }
+};

package/dist/eval/types.d.ts

@@ -114,6 +114,31 @@ export interface TraceabilityExpected {
      */
     requireIn: string[];
 }
+/**
+ * LLM-judge expectations — Step 3.
+ *
+ * When present, the judge runs against the resolved artifact (live-agent
+ * output in Tier A/B/C, or the pre-generated fixture when `--judge` is
+ * combined with `--schema-only` for smoke tests). Every field below is
+ * optional; the case-level hint overlays the stage-level rubric loaded
+ * from `.cclaw/evals/rubrics/<stage>.yaml`.
+ */
+export interface JudgeExpected {
+    /**
+     * Per-case check ids that MUST be present in the stage rubric. Used when
+     * a case wants to assert the rubric covers scenario-specific properties.
+     */
+    requiredChecks?: string[];
+    /**
+     * Stage rubric identifier when a stage ships multiple rubrics (e.g.
+     * "strict" vs. "lenient"). Defaults to the stage name.
+     */
+    rubric?: string;
+    /** Optional override of `config.judgeSamples` for the case. */
+    samples?: number;
+    /** Per-check minimum score (1..5 scale). Fail when any score drops below. */
+    minimumScores?: Record<string, number>;
+}
 /** Superset of per-verifier expectation shapes. */
 export interface ExpectedShape {
     structural?: StructuralExpected;
@@ -122,7 +147,7 @@ export interface ExpectedShape {
     /** Cross-stage ID propagation checks — Step 2. */
     traceability?: TraceabilityExpected;
     /** LLM-judge rubrics — Step 3. */
-    judge?: Record<string, unknown>;
+    judge?: JudgeExpected;
 }
 /**
  * A single eval case describes one input scenario for one stage. Cases live in
@@ -228,6 +253,44 @@ export interface EvalConfig {
     timeoutMs: number;
     /** Max retries per API call on transient failures. */
     maxRetries: number;
+    /**
+     * Number of judge samples per case (median-of-N). Defaults to 3 when unset.
+     * Must be odd so a true median exists.
+     */
+    judgeSamples?: number;
+    /** Sampling temperature for judge calls. Defaults to 0.0. */
+    judgeTemperature?: number;
+    /** Sampling temperature for the agent-under-test. Defaults to 0.2. */
+    agentTemperature?: number;
+    /**
+     * Optional per-model USD pricing used by the cost guard. Keys match
+     * `model` / `judgeModel`. Values in USD per 1K tokens, so
+     * `{ input: 0.0005, output: 0.0015 }` = $0.50 per 1M input tokens.
+     */
+    tokenPricing?: Record<string, TokenPricing>;
+    /**
+     * Maximum assistant turns (tool_calls → tool result cycles) allowed by
+     * the Tier B with-tools agent. Defaults to 8 when unset. Runs that
+     * exceed the cap fail with a `MaxTurnsExceededError` and surface as a
+     * workflow verifier result.
+     */
+    toolMaxTurns?: number;
+    /**
+     * Per-invocation ceiling on tool call arguments bytes. Defends against
+     * runaway writes. Defaults to 64 KiB.
+     */
+    toolMaxArgumentsBytes?: number;
+    /**
+     * Per-invocation ceiling on tool call result bytes returned to the
+     * model. Defaults to 32 KiB; longer results are truncated with a
+     * marker so the model sees the cutoff.
+     */
+    toolMaxResultBytes?: number;
+}
+/** Per-model pricing schedule, expressed as USD per 1K tokens. */
+export interface TokenPricing {
+    input: number;
+    output: number;
 }
 /** Resolved config with env overrides applied. */
 export interface ResolvedEvalConfig extends EvalConfig {
@@ -279,3 +342,77 @@ export interface BaselineRegression {
     previousScore?: number;
     currentScore?: number;
 }
+/**
+ * One rubric check evaluated by the LLM judge. Scored on a 1..5 scale;
+ * 5 means "the artifact fully meets the bar described by `prompt`".
+ */
+export interface RubricCheck {
+    /** Kebab-case slug, unique per rubric. Stable across runs. */
+    id: string;
+    /** Natural-language question posed to the judge. */
+    prompt: string;
+    /** Human-readable scale description rendered in judge prompts. */
+    scale?: string;
+    /** Relative weight for the stage's aggregate score. Defaults to 1.0. */
+    weight?: number;
+    /**
+     * When true, any sample below `config.regression.failIfCriticalBelow`
+     * flips the verifier to `ok:false` (not just a score drop).
+     */
+    critical?: boolean;
+}
+/** Parsed `.cclaw/evals/rubrics/<stage>.yaml`. */
+export interface RubricDoc {
+    stage: FlowStage;
+    /** Optional rubric variant label; defaults to the stage name. */
+    id: string;
+    checks: RubricCheck[];
+}
+/**
+ * Judge response for a single sample (one API call). The judge is asked to
+ * return structured JSON; `scores[id]` maps rubric check id → integer 1..5.
+ * `rationales[id]` is a short plain-text explanation, useful in reports but
+ * never used for gating.
+ */
+export interface JudgeSample {
+    scores: Record<string, number>;
+    rationales: Record<string, string>;
+}
+/** Aggregated judge output across N samples, per rubric check. */
+export interface JudgeAggregate {
+    checkId: string;
+    samples: number[];
+    median: number;
+    mean: number;
+    /** True iff every sample returned a score for this check. */
+    coverage: boolean;
+}
+/**
+ * Judge invocation result. Produced by `runJudge` and consumed by the
+ * runner: the runner converts each aggregate into a `VerifierResult` and
+ * records `usageUsd` toward the per-case cost.
+ */
+export interface JudgeInvocation {
+    rubricId: string;
+    samples: JudgeSample[];
+    aggregates: JudgeAggregate[];
+    usageUsd: number;
+    durationMs: number;
+}
+/**
+ * Tool-use summary produced by the Tier B with-tools agent. Captured so
+ * the runner can surface per-case tool metrics in the markdown report
+ * (number of calls, depth, error rate, denied paths).
+ */
+export interface ToolUseSummary {
+    /** Turns consumed before the agent produced a terminal assistant message. */
+    turns: number;
+    /** Total successful tool invocations across all turns. */
+    calls: number;
+    /** Tool invocations that returned an error (bad args, denied path, etc.). */
+    errors: number;
+    /** Paths the sandbox refused to resolve (escape attempts, missing files). */
+    deniedPaths: string[];
+    /** Per-tool call counts, keyed by tool name. */
+    byTool: Record<string, number>;
+}