cclaw-cli 0.26.0 → 0.28.0

package/dist/eval/runs.d.ts

@@ -0,0 +1,41 @@
+export declare const RUNS_DIR = "runs";
+export interface EvalRunStatus {
+    id: string;
+    startedAt: string;
+    endedAt?: string;
+    pid: number;
+    argv: string[];
+    cwd: string;
+    exitCode?: number;
+    state: "running" | "succeeded" | "failed";
+}
+export declare function runsRoot(projectRoot: string): string;
+export declare function runDir(projectRoot: string, id: string): string;
+export declare function runLogPath(projectRoot: string, id: string): string;
+export declare function runStatusPath(projectRoot: string, id: string): string;
+/**
+ * Generate a short, lexicographically-sortable run id. The timestamp
+ * prefix means `ls -1` already returns the runs in chronological order
+ * which keeps the `runs list` subcommand trivial.
+ */
+export declare function generateRunId(now?: Date): string;
+export declare function ensureRunDir(projectRoot: string, id: string): Promise<string>;
+export declare function writeRunStatus(projectRoot: string, status: EvalRunStatus): Promise<void>;
+export declare function readRunStatus(projectRoot: string, id: string): Promise<EvalRunStatus | null>;
+/**
+ * List run ids under `.cclaw/evals/runs/`, most recent first. Directory
+ * entries that don't contain a `run.json` are skipped (half-initialized
+ * or manually mkdir'd folders).
+ */
+export declare function listRuns(projectRoot: string): Promise<EvalRunStatus[]>;
+/**
+ * Resolve `"latest"` (or undefined) to the most recent run id.
+ * Returns `null` when there are no runs.
+ */
+export declare function resolveRunId(projectRoot: string, hint: string | undefined): Promise<string | null>;
+/**
+ * Cheap liveness probe for an EvalRunStatus. A `run.json` can be stale
+ * (process crashed mid-commit), so we double-check with `kill(pid, 0)`
+ * before trusting the `state: "running"` field.
+ */
+export declare function isRunAlive(status: EvalRunStatus): boolean;

package/dist/eval/runs.js

@@ -0,0 +1,114 @@
+/**
+ * Run bookkeeping for backgrounded `cclaw eval` invocations.
+ *
+ * A backgrounded run writes three artifacts under `.cclaw/evals/runs/<id>/`:
+ *
+ *   - `run.json`  — status metadata (pid, started/ended ISO timestamps,
+ *                    exit code, argv, cwd). Updated at start and at exit.
+ *   - `run.log`   — combined stdout+stderr of the child process. This is
+ *                    what `cclaw eval runs tail` streams.
+ *   - `run.pid`   — just the pid, written atomically so `runs status`
+ *                    can probe liveness without parsing JSON.
+ *
+ * The `id` is a short alphanumeric string (8 chars + ISO timestamp prefix)
+ * chosen so sorting directory entries by name produces a chronological
+ * listing without any extra work.
+ */
+import { randomBytes } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+export const RUNS_DIR = "runs";
+export function runsRoot(projectRoot) {
+    return path.join(projectRoot, EVALS_ROOT, RUNS_DIR);
+}
+export function runDir(projectRoot, id) {
+    return path.join(runsRoot(projectRoot), id);
+}
+export function runLogPath(projectRoot, id) {
+    return path.join(runDir(projectRoot, id), "run.log");
+}
+export function runStatusPath(projectRoot, id) {
+    return path.join(runDir(projectRoot, id), "run.json");
+}
+/**
+ * Generate a short, lexicographically-sortable run id. The timestamp
+ * prefix means `ls -1` already returns the runs in chronological order
+ * which keeps the `runs list` subcommand trivial.
+ */
+export function generateRunId(now = new Date()) {
+    const ts = now.toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z");
+    const suffix = randomBytes(3).toString("hex");
+    return `${ts}-${suffix}`;
+}
+export async function ensureRunDir(projectRoot, id) {
+    const dir = runDir(projectRoot, id);
+    await fs.mkdir(dir, { recursive: true });
+    return dir;
+}
+export async function writeRunStatus(projectRoot, status) {
+    await ensureRunDir(projectRoot, status.id);
+    await fs.writeFile(runStatusPath(projectRoot, status.id), `${JSON.stringify(status, null, 2)}\n`, "utf8");
+}
+export async function readRunStatus(projectRoot, id) {
+    const file = runStatusPath(projectRoot, id);
+    if (!(await exists(file)))
+        return null;
+    try {
+        const raw = await fs.readFile(file, "utf8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * List run ids under `.cclaw/evals/runs/`, most recent first. Directory
+ * entries that don't contain a `run.json` are skipped (half-initialized
+ * or manually mkdir'd folders).
+ */
+export async function listRuns(projectRoot) {
+    const root = runsRoot(projectRoot);
+    if (!(await exists(root)))
+        return [];
+    const entries = await fs.readdir(root, { withFileTypes: true });
+    const out = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory())
+            continue;
+        const status = await readRunStatus(projectRoot, entry.name);
+        if (status)
+            out.push(status);
+    }
+    out.sort((a, b) => (a.startedAt < b.startedAt ? 1 : -1));
+    return out;
+}
+/**
+ * Resolve `"latest"` (or undefined) to the most recent run id.
+ * Returns `null` when there are no runs.
+ */
+export async function resolveRunId(projectRoot, hint) {
+    if (hint && hint !== "latest") {
+        const status = await readRunStatus(projectRoot, hint);
+        return status ? hint : null;
+    }
+    const runs = await listRuns(projectRoot);
+    return runs[0]?.id ?? null;
+}
+/**
+ * Cheap liveness probe for an EvalRunStatus. A `run.json` can be stale
+ * (process crashed mid-commit), so we double-check with `kill(pid, 0)`
+ * before trusting the `state: "running"` field.
+ */
+export function isRunAlive(status) {
+    if (status.state !== "running")
+        return false;
+    try {
+        process.kill(status.pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/eval/sandbox.js

@@ -1,5 +1,5 @@
 /**
- * Per-case sandbox for the Tier B with-tools agent.
+ * Per-case sandbox for the with-tools agent (agent/workflow mode).
  *
  * Every case gets its own `os.tmpdir()/cclaw-eval-<uuid>/` directory. Any
  * `contextFiles` the case declares are copied in relative to the project

package/dist/eval/tools/index.js

@@ -1,5 +1,5 @@
 /**
- * Registry of sandbox-confined tools used by the Tier B with-tools agent.
+ * Registry of sandbox-confined tools used by the with-tools agent (agent/workflow mode).
  *
  * The registry order defines the advertised schema order in the
  * function-calling payload. Keeping it stable means judges reading

package/dist/eval/tools/types.d.ts

@@ -1,5 +1,5 @@
 /**
- * Shared types for Tier B sandbox-confined tools.
+ * Shared types for sandbox-confined tools (agent/workflow mode).
  *
  * Tools are plain async functions: they take validated arguments and a
  * sandbox handle and return a structured result. The runner serializes

package/dist/eval/types.d.ts

@@ -11,21 +11,37 @@
  */
 import type { FlowStage } from "../types.js";
 /**
- * Fidelity tier for the agent-under-test.
+ * Evaluation mode — what the agent-under-test actually does.
  *
- * - `A` — single-shot API call, no tools. Cheap, validates core prompt behavior.
- * - `B` — SDK loop with function-calling for Read/Write/Glob/Grep inside a sandbox.
- * - `C` — multi-stage workflow run (brainstorm -> scope -> ... -> plan) with threaded
- *   artifacts. Most realistic tier we ship in Phase 7; literal IDE-harness runs
- *   (claude-code / cursor-agent proxied to OpenAI-compat) are deferred to Phase 8.
+ * - `fixture` — verify an existing artifact against structural/rule/judge
+ *   expectations. No LLM drafting, only verifiers (judge may still invoke
+ *   the API). Cheapest mode.
+ * - `agent` — LLM drafts a single-stage artifact inside a sandbox using the
+ *   function-calling loop (read_file/write_file/glob/grep). Replaces the
+ *   previous single-shot path entirely.
+ * - `workflow` — LLM orchestrates the full multi-stage flow
+ *   (brainstorm → scope → design → spec → plan) with threaded artifacts.
+ *
+ * Legacy `A|B|C` tier names are still accepted by the CLI/config loader with
+ * a deprecation warning — see `src/eval/mode.ts` for the mapping.
+ */
+export declare const EVAL_MODES: readonly ["fixture", "agent", "workflow"];
+export type EvalMode = (typeof EVAL_MODES)[number];
+/**
+ * Legacy tier identifier, kept so on-disk reports generated before v0.28.0
+ * keep parsing. New code should always use `EvalMode`.
+ * @deprecated use `EvalMode` + `toMode()` from `src/eval/mode.ts`.
  */
 export declare const EVAL_TIERS: readonly ["A", "B", "C"];
+/** @deprecated use `EvalMode`. */
 export type EvalTier = (typeof EVAL_TIERS)[number];
 /**
  * Verifier kinds, in increasing cost and decreasing determinism:
  * structural and rules run without LLM; judge and workflow use the configured model.
+ * `consistency` is the workflow-mode cross-artifact family (deterministic but
+ * operates over multiple artifacts at once).
  */
-export declare const VERIFIER_KINDS: readonly ["structural", "rules", "judge", "workflow"];
+export declare const VERIFIER_KINDS: readonly ["structural", "rules", "judge", "workflow", "consistency"];
 export type VerifierKind = (typeof VERIFIER_KINDS)[number];
 /**
  * Structural expectations — deterministic, LLM-free checks against a single
@@ -118,7 +134,7 @@ export interface TraceabilityExpected {
  * LLM-judge expectations — Step 3.
  *
  * When present, the judge runs against the resolved artifact (live-agent
- * output in Tier A/B/C, or the pre-generated fixture when `--judge` is
+ * output in agent/workflow mode, or the pre-generated fixture when `--judge` is
  * combined with `--schema-only` for smoke tests). Every field below is
  * optional; the case-level hint overlays the stage-level rubric loaded
  * from `.cclaw/evals/rubrics/<stage>.yaml`.
@@ -159,7 +175,7 @@ export interface EvalCase {
     id: string;
     stage: FlowStage;
     inputPrompt: string;
-    /** Project files copied into the Tier B/C sandbox before the agent runs. */
+    /** Project files copied into the agent/workflow sandbox before the agent runs. */
     contextFiles?: string[];
     /**
      * Typed expectation hints consumed by the structural/rules/judge verifiers.
@@ -194,11 +210,17 @@ export interface VerifierResult {
 export interface EvalCaseResult {
     caseId: string;
     stage: FlowStage;
-    tier: EvalTier;
+    mode: EvalMode;
     passed: boolean;
     durationMs: number;
     costUsd?: number;
     verifierResults: VerifierResult[];
+    /**
+     * Only populated in `workflow` mode: per-stage breakdown collected by
+     * the workflow orchestrator. Unset for `fixture` / `agent` modes so the
+     * on-disk JSON stays small.
+     */
+    workflow?: WorkflowRunSummary;
 }
 /** Top-level eval report, serialized to JSON and rendered to Markdown. */
 export interface EvalReport {
@@ -208,7 +230,7 @@ export interface EvalReport {
     cclawVersion: string;
     provider: string;
     model: string;
-    tier: EvalTier;
+    mode: EvalMode;
     stages: FlowStage[];
     cases: EvalCaseResult[];
     summary: {
@@ -238,8 +260,8 @@ export interface EvalConfig {
     model: string;
     /** Optional separate model for the judge role. Defaults to `model`. */
     judgeModel?: string;
-    /** Default tier when `--tier` is not supplied. */
-    defaultTier: EvalTier;
+    /** Default mode when `--mode` is not supplied. */
+    defaultMode: EvalMode;
     /** Optional hard stop on estimated USD spend per day. Unset = no cap. */
     dailyUsdCap?: number;
     /** Regression thresholds for CI gates. */
@@ -270,7 +292,7 @@ export interface EvalConfig {
     tokenPricing?: Record<string, TokenPricing>;
     /**
      * Maximum assistant turns (tool_calls → tool result cycles) allowed by
-     * the Tier B with-tools agent. Defaults to 8 when unset. Runs that
+     * the with-tools agent loop (agent/workflow mode). Defaults to 8. Runs that
      * exceed the cap fail with a `MaxTurnsExceededError` and surface as a
      * workflow verifier result.
      */
@@ -286,6 +308,14 @@ export interface EvalConfig {
      * marker so the model sees the cutoff.
      */
     toolMaxResultBytes?: number;
+    /**
+     * Maximum total turns a single workflow-mode case may consume
+     * across all stages combined. Defaults to 40 (stages × toolMaxTurns).
+     * Runs that exceed the cap fail the current stage with a
+     * `MaxTurnsExceededError` propagated from the underlying with-tools
+     * loop rather than a dedicated workflow-level error.
+     */
+    workflowMaxTotalTurns?: number;
 }
 /** Per-model pricing schedule, expressed as USD per 1K tokens. */
 export interface TokenPricing {
@@ -310,6 +340,18 @@ export interface BaselineSnapshot {
     cclawVersion: string;
     /** Keyed by `EvalCase.id` so unchanged cases produce zero diff. */
     cases: Record<string, BaselineCaseEntry>;
+    /**
+     * Tamper-evident signature computed as sha256 over the canonical JSON
+     * encoding of `{ schemaVersion, stage, cases }`. Present on files
+     * written by cclaw >= 0.28.0; older baselines load with `signature`
+     * absent and the loader skips verification.
+     */
+    signature?: {
+        algorithm: "sha256";
+        digest: string;
+        /** ISO timestamp of when the digest was computed. */
+        signedAt: string;
+    };
 }
 export interface BaselineCaseEntry {
     passed: boolean;
@@ -400,7 +442,7 @@ export interface JudgeInvocation {
     durationMs: number;
 }
 /**
- * Tool-use summary produced by the Tier B with-tools agent. Captured so
+ * Tool-use summary produced by the with-tools agent loop. Captured so
  * the runner can surface per-case tool metrics in the markdown report
  * (number of calls, depth, error rate, denied paths).
  */
@@ -416,3 +458,104 @@ export interface ToolUseSummary {
     /** Per-tool call counts, keyed by tool name. */
     byTool: Record<string, number>;
 }
+/**
+ * Cross-stage consistency expectations for a workflow-mode case. Every
+ * sub-check is optional so authors can opt in incrementally; an empty
+ * block produces zero verifier results.
+ */
+export interface WorkflowConsistencyExpected {
+    /**
+     * For each rule, every id extracted from the `from` stage must appear in
+     * every listed `to` stage. Typical entry: `{ idPattern: "D-\\d+", from:
+     * "scope", to: ["plan"] }`. Guards the "decisions flow downstream" rule.
+     */
+    idsFlow?: Array<{
+        idPattern: string;
+        idFlags?: string;
+        from: WorkflowStageName;
+        to: WorkflowStageName[];
+    }>;
+    /**
+     * Stages that must not contain any of the listed case-insensitive
+     * phrases. Defaults to `["TBD", "TODO", "placeholder"]` when set to an
+     * empty array; omit entirely to skip the check.
+     */
+    placeholderFree?: {
+        stages: WorkflowStageName[];
+        phrases?: string[];
+    };
+    /**
+     * Free-form substring pairs: for every entry, if `must` appears in the
+     * named stage, `forbid` must NOT appear anywhere in the listed
+     * `stages`. Useful for "v1 decided in scope, plan must not say v2".
+     */
+    noContradictions?: Array<{
+        stage: WorkflowStageName;
+        must: string;
+        forbid: string;
+        stages: WorkflowStageName[];
+    }>;
+}
+/**
+ * A single stage step inside a workflow-mode case. The stage's
+ * `inputPrompt` is handed to the with-tools agent loop with prior-stage
+ * artifacts seeded into the sandbox under `stages/<name>.md`.
+ */
+export interface WorkflowStageStep {
+    name: WorkflowStageName;
+    inputPrompt: string;
+    /** Per-stage rubric id override (defaults to the stage name). */
+    rubric?: string;
+    /** Per-stage required rubric check ids (mirror of JudgeExpected.requiredChecks). */
+    requiredChecks?: string[];
+    /** Per-stage minimum rubric scores (mirror of JudgeExpected.minimumScores). */
+    minimumScores?: Record<string, number>;
+}
+/**
+ * Supported workflow-mode stages. Deliberately a subset of `FlowStage` —
+ * the workflow mode covers the early "design" arc of a project. TDD/review/ship
+ * are out of scope (they require real code execution).
+ */
+export declare const WORKFLOW_STAGES: readonly ["brainstorm", "scope", "design", "spec", "plan"];
+export type WorkflowStageName = (typeof WORKFLOW_STAGES)[number];
+/**
+ * A workflow-mode case. Lives under
+ * `.cclaw/evals/corpus/workflows/<id>.yaml` and wires a multi-stage run
+ * through the with-tools agent.
+ */
+export interface WorkflowCase {
+    id: string;
+    /** Short human-readable description (rendered in reports). */
+    description?: string;
+    /** Project files seeded into the sandbox before stage 1 runs. */
+    contextFiles?: string[];
+    /** Ordered list of stages to run. Must be non-empty. */
+    stages: WorkflowStageStep[];
+    /** Cross-stage consistency checks (workflow-mode verifier family). */
+    consistency?: WorkflowConsistencyExpected;
+}
+/** Per-stage record inside a workflow-mode run. */
+export interface WorkflowStageResult {
+    stage: WorkflowStageName;
+    artifact: string;
+    durationMs: number;
+    usageUsd: number;
+    toolUse: ToolUseSummary;
+    attempts: number;
+    model: string;
+    promptTokens: number;
+    completionTokens: number;
+    /** True when the judge (when requested) produced `ok:true` for every required check. */
+    judgeOk?: boolean;
+    /** Per-rubric-check medians keyed by check id (for the report). */
+    judgeMedians?: Record<string, number>;
+}
+/** Workflow-mode orchestration output collected by the runner. */
+export interface WorkflowRunSummary {
+    caseId: string;
+    stages: WorkflowStageResult[];
+    totalUsageUsd: number;
+    totalDurationMs: number;
+    /** True when every stage judge was ok (or judge was skipped everywhere). */
+    allJudgeOk: boolean;
+}

package/dist/eval/types.js

@@ -1,15 +1,47 @@
 /**
- * Fidelity tier for the agent-under-test.
+ * Evaluation mode — what the agent-under-test actually does.
  *
- * - `A` — single-shot API call, no tools. Cheap, validates core prompt behavior.
- * - `B` — SDK loop with function-calling for Read/Write/Glob/Grep inside a sandbox.
- * - `C` — multi-stage workflow run (brainstorm -> scope -> ... -> plan) with threaded
- *   artifacts. Most realistic tier we ship in Phase 7; literal IDE-harness runs
- *   (claude-code / cursor-agent proxied to OpenAI-compat) are deferred to Phase 8.
+ * - `fixture` — verify an existing artifact against structural/rule/judge
+ *   expectations. No LLM drafting, only verifiers (judge may still invoke
+ *   the API). Cheapest mode.
+ * - `agent` — LLM drafts a single-stage artifact inside a sandbox using the
+ *   function-calling loop (read_file/write_file/glob/grep). Replaces the
+ *   previous single-shot path entirely.
+ * - `workflow` — LLM orchestrates the full multi-stage flow
+ *   (brainstorm → scope → design → spec → plan) with threaded artifacts.
+ *
+ * Legacy `A|B|C` tier names are still accepted by the CLI/config loader with
+ * a deprecation warning — see `src/eval/mode.ts` for the mapping.
+ */
+export const EVAL_MODES = ["fixture", "agent", "workflow"];
+/**
+ * Legacy tier identifier, kept so on-disk reports generated before v0.28.0
+ * keep parsing. New code should always use `EvalMode`.
+ * @deprecated use `EvalMode` + `toMode()` from `src/eval/mode.ts`.
  */
 export const EVAL_TIERS = ["A", "B", "C"];
 /**
  * Verifier kinds, in increasing cost and decreasing determinism:
  * structural and rules run without LLM; judge and workflow use the configured model.
+ * `consistency` is the workflow-mode cross-artifact family (deterministic but
+ * operates over multiple artifacts at once).
+ */
+export const VERIFIER_KINDS = [
+    "structural",
+    "rules",
+    "judge",
+    "workflow",
+    "consistency"
+];
+/**
+ * Supported workflow-mode stages. Deliberately a subset of `FlowStage` —
+ * the workflow mode covers the early "design" arc of a project. TDD/review/ship
+ * are out of scope (they require real code execution).
  */
-export const VERIFIER_KINDS = ["structural", "rules", "judge", "workflow"];
+export const WORKFLOW_STAGES = [
+    "brainstorm",
+    "scope",
+    "design",
+    "spec",
+    "plan"
+];

package/dist/eval/verifiers/workflow-consistency.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Cross-artifact consistency verifier for workflow mode.
+ *
+ * Operates over a `{ stage → artifact }` map produced by the workflow
+ * agent and emits deterministic verifier results for:
+ *
+ *  - `ids_flow`: every id extracted from `from` must appear in every
+ *    `to` stage. Typical use — `D-\d+` from scope must all land in plan.
+ *  - `placeholder_free`: none of the listed phrases
+ *    (default `TBD`/`TODO`/`placeholder`) appear in any of the named
+ *    stages.
+ *  - `no_contradictions`: for each entry, if `must` is present in the
+ *    declaring stage, `forbid` must not appear in any of the listed
+ *    `stages`.
+ *
+ * Each sub-check contributes zero or more `VerifierResult`s with
+ * `kind: "consistency"`. An empty `WorkflowConsistencyExpected` produces
+ * zero results so authors can opt in incrementally.
+ */
+import type { VerifierResult, WorkflowConsistencyExpected, WorkflowStageName } from "../types.js";
+export declare function verifyWorkflowConsistency(artifacts: Map<WorkflowStageName, string>, expected: WorkflowConsistencyExpected | undefined): VerifierResult[];