cclaw-cli 0.48.35 → 0.51.0

package/dist/eval/agents/with-tools.js

@@ -1,261 +0,0 @@
-/**
- * Multi-turn with-tools agent (agent mode, reused by workflow mode).
- *
- * Multi-turn loop with OpenAI-style function-calling over a set of
- * sandbox-confined tools. The AUT is given:
- *
- *  - System prompt = stage SKILL.md (same contract as the single-shot path
- *    so the baseline is comparable).
- *  - User prompt = task description + a short "tools available" hint
- *    that names the sandbox root and the four built-in tools.
- *  - Tools = `read_file`, `write_file`, `glob`, `grep` (see
- *    `src/eval/tools/`).
- *
- * The loop runs up to `config.toolMaxTurns` turns (default 8). Each
- * turn:
- *
- *  1. Send the current transcript to the model with tools enabled.
- *  2. Commit token usage against the wrapped client (cost guard sees
- *     every call).
- *  3. If the model returned tool_calls, execute each sandbox tool and
- *     append a `role: "tool"` message with the JSON-serialized result.
- *  4. If the model produced assistant content with `finish_reason: stop`,
- *     treat that as the artifact and exit.
- *
- * When the turn budget is exhausted without a terminal stop, the agent
- * throws `MaxTurnsExceededError`. The runner surfaces the error as a
- * failed workflow verifier so the case counts as a regression.
- *
- * Artifact resolution: the final assistant content is the artifact. If
- * the model used `write_file` to stage the artifact at
- * `artifact.md` (or `artifact/<stage>.md`), we prefer that file — it
- * mirrors workflow mode where writes are the deliverable. The
- * fallback is the terminal assistant message so prompts that don't
- * call write_file still produce something judgable.
- */
-import fs from "node:fs/promises";
-import path from "node:path";
-import { computeUsageUsd } from "../cost-guard.js";
-import { createSandbox } from "../sandbox.js";
-import { BUILTIN_TOOLS, toolsByName, toolsForRequest, truncatePayload } from "../tools/index.js";
-import { loadStageSkill } from "./single-shot.js";
-export class MaxTurnsExceededError extends Error {
-    turns;
-    constructor(turns) {
-        super(`Agent loop exceeded the ${turns}-turn budget without a terminal stop.`);
-        this.name = "MaxTurnsExceededError";
-        this.turns = turns;
-    }
-}
-const DEFAULT_MAX_TURNS = 8;
-const DEFAULT_MAX_ARG_BYTES = 64 * 1024;
-const DEFAULT_MAX_RESULT_BYTES = 32 * 1024;
-const ARTIFACT_CANDIDATES = ["artifact.md", "artifact.txt", "ARTIFACT.md"];
-export async function runWithTools(input) {
-    const { caseEntry, config, projectRoot, client } = input;
-    const maxTurns = clampPositive(config.toolMaxTurns, DEFAULT_MAX_TURNS);
-    const maxArgBytes = clampPositive(config.toolMaxArgumentsBytes, DEFAULT_MAX_ARG_BYTES);
-    const maxResultBytes = clampPositive(config.toolMaxResultBytes, DEFAULT_MAX_RESULT_BYTES);
-    const loader = input.loadSkill ?? ((stage) => loadStageSkill(projectRoot, stage));
-    const systemPrompt = await loader(caseEntry.stage);
-    const tools = input.tools ?? BUILTIN_TOOLS;
-    const toolMap = toolsByName(tools);
-    const toolsBody = toolsForRequest(tools);
-    const sandboxFactory = input.createSandboxFn ?? createSandbox;
-    const externalSandbox = input.externalSandbox;
-    const sandbox = externalSandbox ??
-        (await sandboxFactory({
-            projectRoot,
-            ...(caseEntry.contextFiles ? { contextFiles: caseEntry.contextFiles } : {})
-        }));
-    const toolUse = {
-        turns: 0,
-        calls: 0,
-        errors: 0,
-        deniedPaths: [],
-        byTool: {}
-    };
-    const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
-    let lastModel = config.model;
-    let totalAttempts = 0;
-    const userPrompt = buildUserPrompt(caseEntry, sandbox, tools, input.promptPreamble);
-    const messages = [
-        { role: "system", content: systemPrompt },
-        { role: "user", content: userPrompt }
-    ];
-    const started = Date.now();
-    try {
-        for (let turn = 0; turn < maxTurns; turn += 1) {
-            toolUse.turns = turn + 1;
-            const response = await client.chat({
-                model: config.model,
-                messages,
-                temperature: config.agentTemperature ?? 0.2,
-                timeoutMs: config.timeoutMs,
-                tools: toolsBody,
-                toolChoice: "auto"
-            });
-            usage.promptTokens += response.usage.promptTokens;
-            usage.completionTokens += response.usage.completionTokens;
-            usage.totalTokens += response.usage.totalTokens;
-            lastModel = response.model;
-            totalAttempts += response.attempts;
-            const hasToolCalls = response.toolCalls && response.toolCalls.length > 0;
-            messages.push(rememberAssistant(response.content, response.toolCalls));
-            if (!hasToolCalls) {
-                const artifact = await resolveArtifact(sandbox, response.content);
-                return finalize(artifact, usage, lastModel, totalAttempts, started, toolUse, systemPrompt, userPrompt, config);
-            }
-            for (const call of response.toolCalls) {
-                const tool = toolMap.get(call.name);
-                const argBytes = Buffer.byteLength(call.arguments ?? "", "utf8");
-                if (argBytes > maxArgBytes) {
-                    toolUse.errors += 1;
-                    bumpToolCount(toolUse, call.name);
-                    messages.push(toolResponseMessage(call.id, {
-                        ok: false,
-                        name: call.name,
-                        error: `arguments payload exceeds ${maxArgBytes} bytes`
-                    }));
-                    continue;
-                }
-                if (!tool) {
-                    toolUse.errors += 1;
-                    bumpToolCount(toolUse, call.name);
-                    messages.push(toolResponseMessage(call.id, {
-                        ok: false,
-                        name: call.name,
-                        error: `unknown tool "${call.name}"`
-                    }));
-                    continue;
-                }
-                bumpToolCount(toolUse, call.name);
-                const result = await tool.invoke(call.arguments ?? "", {
-                    sandbox,
-                    maxResultBytes
-                });
-                if (!result.ok) {
-                    toolUse.errors += 1;
-                    const denied = result.details && typeof result.details.deniedPath === "string"
-                        ? result.details.deniedPath
-                        : undefined;
-                    if (denied && !toolUse.deniedPaths.includes(denied)) {
-                        toolUse.deniedPaths.push(denied);
-                    }
-                }
-                else {
-                    toolUse.calls += 1;
-                }
-                messages.push(toolResponseMessage(call.id, result));
-            }
-        }
-        throw new MaxTurnsExceededError(maxTurns);
-    }
-    finally {
-        if (!externalSandbox)
-            await sandbox.dispose();
-    }
-}
-function finalize(artifact, usage, model, attempts, started, toolUse, systemPrompt, userPrompt, config) {
-    const usageUsd = computeUsageUsd(model, usage, {
-        tokenPricing: config.tokenPricing
-    });
-    return {
-        artifact: artifact.trim(),
-        usage,
-        usageUsd,
-        model,
-        attempts,
-        durationMs: Date.now() - started,
-        toolUse,
-        systemPrompt,
-        userPrompt
-    };
-}
-function rememberAssistant(content, toolCalls) {
-    const base = { role: "assistant", content };
-    if (toolCalls && toolCalls.length > 0)
-        base.toolCalls = toolCalls;
-    return base;
-}
-function toolResponseMessage(callId, result) {
-    const payload = result.ok
-        ? { ok: true, content: result.content, details: result.details ?? {} }
-        : { ok: false, error: result.error, details: result.details ?? {} };
-    return {
-        role: "tool",
-        content: truncatePayload(JSON.stringify(payload), 32 * 1024),
-        toolCallId: callId,
-        name: result.name
-    };
-}
-function bumpToolCount(summary, name) {
-    summary.byTool[name] = (summary.byTool[name] ?? 0) + 1;
-}
-function clampPositive(value, fallback) {
-    if (value === undefined)
-        return fallback;
-    if (!Number.isFinite(value) || value <= 0)
-        return fallback;
-    return Math.floor(value);
-}
-function buildUserPrompt(caseEntry, sandbox, tools, preamble) {
-    const toolList = tools.map((t) => `- ${t.descriptor.name}: ${t.descriptor.description}`);
-    const files = caseEntry.contextFiles ?? [];
-    const contextLines = files.length > 0
-        ? files.map((f) => `- ${f}`).join("\n")
-        : "(no files seeded)";
-    const lines = [];
-    if (preamble && preamble.trim().length > 0) {
-        lines.push(preamble.trim(), ``);
-    }
-    lines.push(`Stage: ${caseEntry.stage}`, `Case id: ${caseEntry.id}`, ``);
-    const rest = [
-        `Sandbox root: ${sandbox.root}`,
-        `You may call the following tools to read or modify files inside the sandbox.`,
-        `All paths are relative to the sandbox root.`,
-        ``,
-        `Tools:`,
-        ...toolList,
-        ``,
-        `Seeded context files (available under the sandbox root):`,
-        contextLines,
-        ``,
-        `Task:`,
-        caseEntry.inputPrompt.trim(),
-        ``,
-        `When you are done, reply with the artifact as the final assistant message.`,
-        `Output the artifact directly (markdown with optional YAML frontmatter).`,
-        `Do not wrap in code fences, do not add commentary before or after.`,
-        `You may optionally write the artifact to \`artifact.md\` in the sandbox; ` +
-            `if you do, the last written \`artifact.md\` is preferred over the chat reply.`
-    ];
-    lines.push(...rest);
-    return lines.join("\n");
-}
-async function resolveArtifact(sandbox, fallback) {
-    for (const candidate of ARTIFACT_CANDIDATES) {
-        try {
-            const abs = await sandbox.resolve(candidate);
-            const stat = await fs.stat(abs);
-            if (stat.isFile()) {
-                return await fs.readFile(abs, "utf8");
-            }
-        }
-        catch {
-            continue;
-        }
-    }
-    try {
-        const dir = path.join(sandbox.root);
-        const entries = (await fs.readdir(dir, { withFileTypes: true }));
-        const match = entries.find((entry) => entry.isFile() && /^artifact\./i.test(entry.name));
-        if (match) {
-            return await fs.readFile(path.join(dir, match.name), "utf8");
-        }
-    }
-    catch {
-        // fall through to fallback
-    }
-    return fallback;
-}

package/dist/eval/agents/workflow.d.ts

@@ -1,31 +0,0 @@
-import type { EvalLlmClient } from "../llm-client.js";
-import { createSandbox } from "../sandbox.js";
-import type { SandboxTool } from "../tools/index.js";
-import type { ResolvedEvalConfig, WorkflowCase, WorkflowStageName, WorkflowStageResult } from "../types.js";
-export interface WorkflowInput {
-    workflow: WorkflowCase;
-    config: Pick<ResolvedEvalConfig, "model" | "agentTemperature" | "timeoutMs" | "tokenPricing" | "toolMaxTurns" | "toolMaxArgumentsBytes" | "toolMaxResultBytes" | "workflowMaxTotalTurns">;
-    projectRoot: string;
-    client: EvalLlmClient;
-    tools?: SandboxTool[];
-    /** Override for the SKILL.md loader (test hook). */
-    loadSkill?: (stage: WorkflowStageName) => Promise<string>;
-    /** Override for the sandbox factory (test hook). */
-    createSandboxFn?: typeof createSandbox;
-    /**
-     * Optional per-stage lifecycle hooks. The runner uses these to emit
-     * progress events to stderr so workflow-mode runs surface real-time
-     * status rather than going silent for minutes.
-     */
-    onStageStart?: (stage: WorkflowStageName) => void;
-    onStageEnd?: (stage: WorkflowStageName, result: WorkflowStageResult) => void;
-}
-export interface WorkflowOutput {
-    caseId: string;
-    stages: WorkflowStageResult[];
-    /** Map from stage name to produced artifact (also persisted in sandbox). */
-    artifacts: Map<WorkflowStageName, string>;
-    totalUsageUsd: number;
-    totalDurationMs: number;
-}
-export declare function runWorkflow(input: WorkflowInput): Promise<WorkflowOutput>;

package/dist/eval/agents/workflow.js

@@ -1,155 +0,0 @@
-/**
- * Workflow-mode agent.
- *
- * Runs the with-tools loop once per stage in a workflow case,
- * sharing a single sandbox across stages so every new stage can read
- * the earlier artifacts the model produced. The shape of the run is:
- *
- *   1. Create one sandbox seeded with `contextFiles`.
- *   2. For each stage in `workflow.stages`:
- *      a. Delete any leftover `artifact.md` so the resolver doesn't
- *         accidentally pick the previous stage's output.
- *      b. Invoke `runWithTools({ externalSandbox: sandbox, promptPreamble })`.
- *         The preamble tells the model which stage it is on and lists the
- *         `stages/*.md` files available for reading.
- *      c. Persist the returned artifact to `stages/<stage>.md` inside the
- *         sandbox (deterministic, regardless of whether the model wrote
- *         `artifact.md` itself).
- *      d. Record `WorkflowStageResult` with usage, duration, and tool use.
- *   3. Dispose the sandbox in a `finally` so temp directories never leak.
- *
- * Errors bubble up from `runWithTools`:
- *   - `MaxTurnsExceededError` stops the workflow at the current stage.
- *   - `DailyCostCapExceededError` (surfaced by the cost-guard wrapper in
- *     the runner) aborts immediately.
- *   - Generic `EvalLlmError` subclasses propagate as-is so the runner can
- *     record a workflow-level verifier failure.
- */
-import fs from "node:fs/promises";
-import path from "node:path";
-import { createSandbox } from "../sandbox.js";
-import { loadStageSkill } from "./single-shot.js";
-import { MaxTurnsExceededError, runWithTools } from "./with-tools.js";
-const STAGES_SUBDIR = "stages";
-const ARTIFACT_CANDIDATES = ["artifact.md", "artifact.txt", "ARTIFACT.md"];
-const DEFAULT_WORKFLOW_MAX_TOTAL_TURNS = 40;
-const DEFAULT_STAGE_TURN_CAP = 8;
-function clampPositive(value, fallback) {
-    if (!Number.isInteger(value) || value < 1) {
-        return fallback;
-    }
-    return value;
-}
-export async function runWorkflow(input) {
-    const { workflow, config, projectRoot, client } = input;
-    const sandboxFactory = input.createSandboxFn ?? createSandbox;
-    const sandbox = await sandboxFactory({
-        projectRoot,
-        ...(workflow.contextFiles ? { contextFiles: workflow.contextFiles } : {})
-    });
-    const stageResults = [];
-    const artifacts = new Map();
-    let totalUsageUsd = 0;
-    let totalDurationMs = 0;
-    let totalTurns = 0;
-    const workflowTurnBudget = clampPositive(config.workflowMaxTotalTurns, DEFAULT_WORKFLOW_MAX_TOTAL_TURNS);
-    try {
-        await fs.mkdir(await sandbox.resolve(STAGES_SUBDIR, { allowMissing: true }), { recursive: true });
-        for (const step of workflow.stages) {
-            const remainingWorkflowTurns = workflowTurnBudget - totalTurns;
-            if (remainingWorkflowTurns < 1) {
-                throw new MaxTurnsExceededError(workflowTurnBudget);
-            }
-            const perStageTurnCap = clampPositive(config.toolMaxTurns, DEFAULT_STAGE_TURN_CAP);
-            const stageTurnBudget = Math.min(perStageTurnCap, remainingWorkflowTurns);
-            input.onStageStart?.(step.name);
-            await clearArtifactFile(sandbox);
-            const priorStages = stageResults.map((r) => r.stage);
-            const preamble = buildStagePreamble(workflow, step.name, priorStages);
-            const caseEntry = {
-                id: `${workflow.id}/${step.name}`,
-                stage: step.name,
-                inputPrompt: step.inputPrompt,
-                ...(workflow.contextFiles ? { contextFiles: workflow.contextFiles } : {})
-            };
-            const result = await runWithTools({
-                caseEntry,
-                config: {
-                    ...config,
-                    toolMaxTurns: stageTurnBudget
-                },
-                projectRoot,
-                client,
-                ...(input.tools ? { tools: input.tools } : {}),
-                ...(input.loadSkill
-                    ? { loadSkill: input.loadSkill }
-                    : {
-                        loadSkill: (stage) => loadStageSkill(projectRoot, stage)
-                    }),
-                externalSandbox: sandbox,
-                promptPreamble: preamble
-            });
-            await persistStageArtifact(sandbox, step.name, result.artifact);
-            artifacts.set(step.name, result.artifact);
-            const stageResult = {
-                stage: step.name,
-                artifact: result.artifact,
-                durationMs: result.durationMs,
-                usageUsd: result.usageUsd,
-                toolUse: result.toolUse,
-                attempts: result.attempts,
-                model: result.model,
-                promptTokens: result.usage.promptTokens,
-                completionTokens: result.usage.completionTokens
-            };
-            stageResults.push(stageResult);
-            input.onStageEnd?.(step.name, stageResult);
-            totalUsageUsd += result.usageUsd;
-            totalDurationMs += result.durationMs;
-            totalTurns += result.toolUse.turns;
-        }
-        return {
-            caseId: workflow.id,
-            stages: stageResults,
-            artifacts,
-            totalUsageUsd: Number(totalUsageUsd.toFixed(6)),
-            totalDurationMs
-        };
-    }
-    finally {
-        await sandbox.dispose();
-    }
-}
-async function clearArtifactFile(sandbox) {
-    for (const candidate of ARTIFACT_CANDIDATES) {
-        try {
-            const abs = await sandbox.resolve(candidate);
-            await fs.rm(abs, { force: true });
-        }
-        catch {
-            // candidate did not exist — resolve threw SandboxEscapeError for
-            // missing realpath; safe to ignore.
-        }
-    }
-}
-async function persistStageArtifact(sandbox, stage, artifact) {
-    const rel = `${STAGES_SUBDIR}/${stage}.md`;
-    const abs = await sandbox.resolve(rel, { allowMissing: true });
-    await fs.mkdir(path.dirname(abs), { recursive: true });
-    await fs.writeFile(abs, artifact.endsWith("\n") ? artifact : `${artifact}\n`, "utf8");
-}
-function buildStagePreamble(workflow, current, priorStages) {
-    const lines = [];
-    lines.push(`You are running stage "${current}" of the workflow "${workflow.id}".`);
-    if (workflow.description) {
-        lines.push(`Case description: ${workflow.description}`);
-    }
-    if (priorStages.length === 0) {
-        lines.push(`This is the first stage. Any context_files have been seeded into the sandbox root.`);
-    }
-    else {
-        lines.push(`Earlier stage artifacts are available via read_file:`, ...priorStages.map((name) => `  - ${STAGES_SUBDIR}/${name}.md`), `Read the prior artifacts before drafting your output so decisions and ` +
-            `ids carry through.`);
-    }
-    return lines.join("\n");
-}

package/dist/eval/baseline.d.ts

@@ -1,38 +0,0 @@
-import type { FlowStage } from "../types.js";
-import type { BaselineDelta, BaselineSnapshot, EvalReport } from "./types.js";
-export declare const BASELINE_SCHEMA_VERSION = 1;
-/**
- * Thrown when a signed baseline's on-disk digest does not match the
- * canonical encoding of its `{ schemaVersion, stage, cases }` block.
- * Callers should treat this as a hard failure: the baseline was either
- * hand-edited or corrupted and cannot be trusted for regression gating.
- */
-export declare class BaselineSignatureError extends Error {
-    readonly file: string;
-    readonly expected: string;
-    readonly actual: string;
-    constructor(opts: {
-        file: string;
-        expected: string;
-        actual: string;
-    });
-}
-/**
- * Produce a deterministic sha256 digest over the signable portion of a
- * baseline. We intentionally exclude `generatedAt` and `cclawVersion`
- * from the digest so that rebuilding the same baseline from identical
- * case results on a new CLI version doesn't invalidate the signature —
- * only changes to the observed pass/ok/score payloads do.
- */
-export declare function computeBaselineDigest(snapshot: Pick<BaselineSnapshot, "schemaVersion" | "stage" | "cases">): string;
-export declare function loadBaseline(projectRoot: string, stage: FlowStage): Promise<BaselineSnapshot | null>;
-export declare function loadBaselinesByStage(projectRoot: string, stages: readonly FlowStage[]): Promise<Map<FlowStage, BaselineSnapshot>>;
-export declare function buildBaselineForStage(stage: FlowStage, report: EvalReport): BaselineSnapshot;
-export declare function writeBaselinesFromReport(projectRoot: string, report: EvalReport): Promise<string[]>;
-/**
- * Compare a freshly computed report against loaded baselines. If no baseline
- * exists for a stage covered by the report, that stage contributes zero
- * regressions (first run of that stage). Current is the source of truth.
- */
-export declare function compareAgainstBaselines(report: EvalReport, baselines: Map<FlowStage, BaselineSnapshot>): BaselineDelta | undefined;
-export declare function listBaselineStages(projectRoot: string): Promise<FlowStage[]>;