@slowdini/slow-powers-opencode 0.1.5 → 0.3.0

package/skills/evaluating-skills/runner/detect-stray-writes.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
+import { join, relative, resolve } from "node:path";
 import { detectRunContext } from "./context";
 import { classifyBash, isUnder, pathArg, WRITE_TOOLS } from "./sandbox-policy";
 import type { ConditionsRecord, RunRecord, ToolInvocation } from "./types";
@@ -71,6 +71,81 @@ export function detectStrayWrites(
   return { violations, warnings };
 }
 
+/** Read-only tools that carry a target path argument (see `pathArg`). */
+const READ_TOOLS = new Set(["Read", "Glob", "Grep"]);
+
+const LIVE_SOURCE_REASON =
+  "reads the live skill source instead of its staged copy — the arm may be contaminated";
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Flag tool invocations that read the **live** skill-under-test directory.
+ *
+ * Eval subagents are only ever meant to see the *staged* copy of the skill
+ * (`.claude/skills/<slug>/`, or the inlined SKILL.md under `--no-stage`). A
+ * read of the live source typically means the Skill tool couldn't resolve the
+ * staged slug yet (mid-session registry refresh race) and the agent improvised
+ * — fatal in revision mode, where the old_skill arm then reads new-skill
+ * content. Reads are detected, not blocked: the guard stays read-permissive,
+ * so this surfaces post-hoc as a validity warning.
+ *
+ * - Read-tool calls (Read/Glob/Grep) whose path arg resolves under the live
+ *   dir are flagged; relative paths resolve against `repoRoot`.
+ * - Bash commands that reference the live dir (absolute, or repo-relative
+ *   text) are flagged. A staged copy under `.claude/skills/` can carry the
+ *   same `skills/<name>` relative text (e.g. via `--stage-name`), so that
+ *   prefix is excluded.
+ */
+export function detectLiveSourceReads(
+  invocations: Array<Pick<ToolInvocation, "name" | "args" | "ordinal">>,
+  liveSkillDir: string,
+  repoRoot: string,
+): StrayFinding[] {
+  const findings: StrayFinding[] = [];
+  const liveDir = resolve(liveSkillDir);
+  const rel = relative(repoRoot, liveDir);
+  const relRe = rel.startsWith("..")
+    ? null
+    : new RegExp(
+        // The lookbehind fires at the boundary char itself, so it checks for a
+        // bare `.claude` — the `/` is consumed by the boundary group.
+        `(?<!\\.claude)(^|[\\s'"=:(/])${escapeRegExp(rel)}(/|[\\s'")]|$)`,
+      );
+
+  for (const inv of invocations) {
+    if (READ_TOOLS.has(inv.name)) {
+      const p = pathArg(inv.args);
+      if (p && isUnder(p, liveDir, repoRoot)) {
+        findings.push({
+          tool: inv.name,
+          path: p,
+          ordinal: inv.ordinal,
+          reason: LIVE_SOURCE_REASON,
+        });
+      }
+      continue;
+    }
+
+    if (inv.name === "Bash") {
+      const args = inv.args as { command?: unknown } | undefined;
+      const command = typeof args?.command === "string" ? args.command : "";
+      if (command.includes(liveDir) || relRe?.test(command)) {
+        findings.push({
+          tool: "Bash",
+          command,
+          ordinal: inv.ordinal,
+          reason: LIVE_SOURCE_REASON,
+        });
+      }
+    }
+  }
+
+  return findings;
+}
+
 if (import.meta.main) {
   const argv = Bun.argv.slice(2);
   const flag = (name: string): string | undefined => {
@@ -127,10 +202,12 @@ if (import.meta.main) {
     condition: string;
     violations: StrayFinding[];
     warnings: StrayFinding[];
+    live_source_reads: StrayFinding[];
   };
   const runs: RunReport[] = [];
   let totalViolations = 0;
   let totalWarnings = 0;
+  let totalLiveReads = 0;
 
   for (const evalDir of evalDirs) {
     const evalId = evalDir.replace(/^eval-/, "");
@@ -149,23 +226,38 @@ if (import.meta.main) {
       const outputsDir =
         outputsByKey.get(`${evalId}:${cond}`) ?? join(condDir, "outputs");
       const findings = detectStrayWrites(invocations, outputsDir, repoRoot);
-      if (findings.violations.length || findings.warnings.length) {
+      const liveReads = detectLiveSourceReads(
+        invocations,
+        ctx.skillSubdir,
+        repoRoot,
+      );
+      if (
+        findings.violations.length ||
+        findings.warnings.length ||
+        liveReads.length
+      ) {
         runs.push({
           eval_id: evalId,
           condition: cond,
           violations: findings.violations,
           warnings: findings.warnings,
+          live_source_reads: liveReads,
         });
       }
       totalViolations += findings.violations.length;
       totalWarnings += findings.warnings.length;
+      totalLiveReads += liveReads.length;
     }
   }
 
   const report = {
     generated: new Date().toISOString(),
     iteration: Number(iteration),
-    totals: { violations: totalViolations, warnings: totalWarnings },
+    totals: {
+      violations: totalViolations,
+      warnings: totalWarnings,
+      live_source_reads: totalLiveReads,
+    },
     runs,
   };
   const outPath = join(iterationDir, "stray-writes.json");
@@ -182,11 +274,15 @@ if (import.meta.main) {
       console.warn(
         `⚠ ${r.eval_id}/${r.condition}: Bash ${w.reason} (ordinal ${w.ordinal}): ${w.command}`,
       );
+    for (const l of r.live_source_reads)
+      console.warn(
+        `⚠ ${r.eval_id}/${r.condition}: ${l.tool} read the live skill source (ordinal ${l.ordinal}): ${l.path ?? l.command}`,
+      );
   }
-  if (totalViolations === 0 && totalWarnings === 0)
-    console.log("✓ No out-of-bounds writes detected.");
+  if (totalViolations === 0 && totalWarnings === 0 && totalLiveReads === 0)
+    console.log("✓ No out-of-bounds writes or live-source reads detected.");
   else
     console.warn(
-      `\n${totalViolations} violation(s), ${totalWarnings} warning(s). Runs with violations edited files outside their sandbox — treat those data points as tainted.`,
+      `\n${totalViolations} violation(s), ${totalWarnings} warning(s), ${totalLiveReads} live-source read(s). Runs with violations edited files outside their sandbox; runs with live-source reads saw the live skill instead of their staged copy — treat those data points as tainted.`,
     );
 }

package/skills/evaluating-skills/runner/guard/policy.test.ts

@@ -68,4 +68,61 @@ describe("guard decide", () => {
       true,
     );
   });
+
+  test("denies git worktree add (working tree outside the sandbox)", () => {
+    const d = decide(
+      "Bash",
+      { command: "git worktree add ../wt -b scratch" },
+      marker(),
+    );
+    expect(d.allow).toBe(false);
+    expect(d.reason).toMatch(/worktree/i);
+  });
+
+  test("denies Bash that creates a path under .claude via a non-redirect verb", () => {
+    expect(
+      decide("Bash", { command: "mkdir -p .claude/foo" }, marker()).allow,
+    ).toBe(false);
+    expect(
+      decide("Bash", { command: "cp out.txt .claude/bar" }, marker()).allow,
+    ).toBe(false);
+  });
+
+  test("denies Bash that creates a bare skills/ dir", () => {
+    expect(decide("Bash", { command: "mkdir skills" }, marker()).allow).toBe(
+      false,
+    );
+    expect(
+      decide("Bash", { command: "cp -r src ./skills" }, marker()).allow,
+    ).toBe(false);
+  });
+
+  test("still allows reads of .claude (no create verb)", () => {
+    expect(
+      decide("Bash", { command: "cat .claude/settings.json" }, marker()).allow,
+    ).toBe(true);
+    expect(decide("Bash", { command: "ls .claude" }, marker()).allow).toBe(
+      true,
+    );
+  });
+
+  test("allows a create scoped to the .claude/skills staging root (allowed-root escape)", () => {
+    expect(
+      decide(
+        "Bash",
+        { command: "mkdir -p /work/.claude/skills/staged-x" },
+        marker(),
+      ).allow,
+    ).toBe(true);
+  });
+
+  test("does not flag skills-workspace as a bare skills/ write", () => {
+    expect(
+      decide(
+        "Bash",
+        { command: "mkdir -p /work/skills-workspace/x/outputs" },
+        marker(),
+      ).allow,
+    ).toBe(true);
+  });
 });

package/skills/evaluating-skills/runner/promote-baseline.test.ts

@@ -8,6 +8,7 @@ import {
 } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { PROMOTED_MARKER } from "./workspace-teardown";
 
 const FIXTURE_ROOT = join(tmpdir(), `slow-powers-promote-test-${process.pid}`);
 const PROMOTE_TS = join(import.meta.dir, "promote-baseline.ts");
@@ -137,6 +138,56 @@ describe("promote-baseline.ts (--skill-dir, isolated CWD)", () => {
     expect(provenance).toContain("Judge model | unspecified");
   });
 
+  test("drops a .promoted.json marker into the iteration dir for teardown", () => {
+    const root = join(FIXTURE_ROOT, "promote-marker");
+
+    const skillDir = join(root, "skill-dir");
+    const skillSub = join(skillDir, "mr-review");
+    mkdirSync(skillSub, { recursive: true });
+    writeFileSync(
+      join(skillSub, "SKILL.md"),
+      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
+    );
+
+    const cwd = join(root, "work");
+    const iterationDir = join(
+      cwd,
+      "skills-workspace",
+      "mr-review",
+      "iteration-3",
+    );
+    mkdirSync(iterationDir, { recursive: true });
+    writeJson(join(iterationDir, "benchmark.json"), {
+      delta: { pass_rate: 0 },
+    });
+
+    const res = Bun.spawnSync(
+      [
+        "bun",
+        "run",
+        PROMOTE_TS,
+        "--skill-dir",
+        skillDir,
+        "--skill",
+        "mr-review",
+        "--iteration",
+        "3",
+      ],
+      { cwd, stdout: "pipe", stderr: "pipe" },
+    );
+    expect(res.stderr.toString()).toBe("");
+    expect(res.exitCode).toBe(0);
+
+    const markerPath = join(iterationDir, PROMOTED_MARKER);
+    expect(existsSync(markerPath)).toBe(true);
+    const marker = JSON.parse(readFileSync(markerPath, "utf8")) as {
+      promoted_at: string;
+      baseline_dir: string;
+    };
+    expect(marker.promoted_at).toBeTruthy();
+    expect(marker.baseline_dir).toBe(join(skillSub, "evals", "baseline"));
+  });
+
   test("records agent and judge models in provenance when flags are passed", () => {
     const root = join(FIXTURE_ROOT, "promote-models");
 

package/skills/evaluating-skills/runner/promote-baseline.ts

@@ -10,6 +10,7 @@ import {
 import { join } from "node:path";
 import { detectRunContext } from "./context";
 import type { ConditionsRecord } from "./types";
+import { PROMOTED_MARKER } from "./workspace-teardown";
 
 function die(msg: string): never {
   console.error(`error: ${msg}`);
@@ -120,7 +121,8 @@ export function promoteBaseline(opts: PromoteOptions): {
     "`bun run evals:promote-baseline -- --skill " +
       `${opts.skillName} --iteration <N>` +
       "` after aggregating. The ephemeral workspace (run records, timing,",
-    "dispatch files, produced outputs) stays gitignored under `skills-workspace/`.",
+    "dispatch files, produced outputs) stays gitignored under `skills-workspace/`",
+    "and is reclaimable by `evals:teardown` once promoted (this commit's marker).",
     "",
     "| Field | Value |",
     "|-------|-------|",
@@ -141,6 +143,22 @@ export function promoteBaseline(opts: PromoteOptions): {
   ].join("\n");
   writeFileSync(join(baselineDir, "BASELINE.md"), `${provenance}\n`);
 
+  // Mark the iteration as committed so `teardown` can safely reclaim its
+  // workspace — without this marker teardown preserves the iteration as
+  // uncommitted results.
+  writeFileSync(
+    join(iterationDir, PROMOTED_MARKER),
+    `${JSON.stringify(
+      {
+        promoted_at: new Date().toISOString(),
+        baseline_dir: baselineDir,
+        commit: head,
+      },
+      null,
+      2,
+    )}\n`,
+  );
+
   return { baselineDir, gradingsCopied };
 }
 

package/skills/evaluating-skills/runner/record-runs.test.ts

@@ -0,0 +1,314 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { recordRuns } from "./record-runs";
+import type { RunRecord, TimingRecord } from "./types";
+
+const ROOT = join(tmpdir(), `record-runs-test-${process.pid}`);
+
+let iterationDir: string;
+let subagentsDir: string;
+
+function jsonl(lines: object[]): string {
+  return `${lines.map((l) => JSON.stringify(l)).join("\n")}\n`;
+}
+
+/** A minimal transcript with usage, timestamps, one tool call, and final text. */
+function transcriptLines(finalText: string): object[] {
+  return [
+    {
+      type: "user",
+      timestamp: "2026-06-04T10:00:00.000Z",
+      message: { role: "user", content: "go" },
+    },
+    {
+      type: "assistant",
+      timestamp: "2026-06-04T10:00:10.000Z",
+      message: {
+        id: "msg_1",
+        role: "assistant",
+        usage: {
+          input_tokens: 100,
+          output_tokens: 20,
+          cache_creation_input_tokens: 30,
+          cache_read_input_tokens: 50,
+        },
+        content: [
+          {
+            type: "tool_use",
+            id: "toolu_1",
+            name: "Bash",
+            input: { command: "ls" },
+          },
+        ],
+      },
+    },
+    {
+      type: "user",
+      timestamp: "2026-06-04T10:00:12.000Z",
+      message: {
+        role: "user",
+        content: [
+          { type: "tool_result", tool_use_id: "toolu_1", content: "ok" },
+        ],
+      },
+    },
+    {
+      type: "assistant",
+      timestamp: "2026-06-04T10:01:00.000Z",
+      message: {
+        id: "msg_2",
+        role: "assistant",
+        usage: {
+          input_tokens: 200,
+          output_tokens: 40,
+          cache_creation_input_tokens: 0,
+          cache_read_input_tokens: 60,
+        },
+        content: [{ type: "text", text: finalText }],
+      },
+    },
+  ];
+}
+
+// Token math for transcriptLines: msg_1 (100+20+30+50) + msg_2 (200+40+0+60) = 500.
+const TRANSCRIPT_TOKENS = 500;
+// 10:00:00.000 → 10:01:00.000
+const TRANSCRIPT_DURATION_MS = 60_000;
+
+function writeSubagent(name: string, description: string, lines: object[]) {
+  writeFileSync(
+    join(subagentsDir, `${name}.meta.json`),
+    JSON.stringify({ agentType: "general-purpose", description }),
+  );
+  writeFileSync(join(subagentsDir, `${name}.jsonl`), jsonl(lines));
+}
+
+type FixtureTask = {
+  eval_id: string;
+  condition: string;
+  finalMessage?: string; // written to outputs/final-message.md when present
+};
+
+/** Builds an iteration dir + dispatch.json shaped like run.ts serializes it. */
+function writeIteration(tasks: FixtureTask[]) {
+  const serialized = tasks.map((t) => {
+    const condDir = join(iterationDir, `eval-${t.eval_id}`, t.condition);
+    const outputsDir = join(condDir, "outputs");
+    mkdirSync(outputsDir, { recursive: true });
+    if (t.finalMessage !== undefined) {
+      writeFileSync(join(outputsDir, "final-message.md"), t.finalMessage);
+    }
+    return {
+      eval_id: t.eval_id,
+      condition: t.condition,
+      skill_path:
+        t.condition === "without_skill" ? null : "/staged/skill/SKILL.md",
+      staged_skill_slug: t.condition === "without_skill" ? null : "test-slug",
+      user_prompt: `Do the ${t.eval_id} task`,
+      fixtures: [join(condDir, "inputs", "fixture.txt")],
+      outputs_dir: outputsDir,
+      run_record_path: join(condDir, "run.json"),
+      timing_path: join(condDir, "timing.json"),
+      agent_description: `${t.eval_id}:${t.condition}:i1-nonce1`,
+      dispatch_prompt_path: join(condDir, "dispatch-prompt.txt"),
+    };
+  });
+  writeFileSync(
+    join(iterationDir, "dispatch.json"),
+    JSON.stringify({ run_nonce: "nonce1", tasks: serialized }, null, 2),
+  );
+  return serialized;
+}
+
+function readRun(evalId: string, condition: string): RunRecord {
+  return JSON.parse(
+    readFileSync(
+      join(iterationDir, `eval-${evalId}`, condition, "run.json"),
+      "utf8",
+    ),
+  );
+}
+
+function readTiming(evalId: string, condition: string): TimingRecord {
+  return JSON.parse(
+    readFileSync(
+      join(iterationDir, `eval-${evalId}`, condition, "timing.json"),
+      "utf8",
+    ),
+  );
+}
+
+beforeEach(() => {
+  iterationDir = join(ROOT, `iter-${Math.random().toString(36).slice(2)}`);
+  subagentsDir = join(ROOT, `sub-${Math.random().toString(36).slice(2)}`);
+  mkdirSync(iterationDir, { recursive: true });
+  mkdirSync(subagentsDir, { recursive: true });
+});
+
+afterEach(() => rmSync(ROOT, { recursive: true, force: true }));
+
+describe("recordRuns", () => {
+  test("assembles run.json and timing.json for every task from disk", () => {
+    writeIteration([
+      { eval_id: "crash", condition: "with_skill", finalMessage: "Fixed it." },
+      {
+        eval_id: "crash",
+        condition: "without_skill",
+        finalMessage: "Done, I think.",
+      },
+    ]);
+    writeSubagent(
+      "agent-a",
+      "crash:with_skill:i1-nonce1",
+      transcriptLines("unused"),
+    );
+    writeSubagent(
+      "agent-b",
+      "crash:without_skill:i1-nonce1",
+      transcriptLines("unused"),
+    );
+
+    const result = recordRuns({ iterationDir, subagentsDir });
+    expect(result.recorded).toBe(2);
+    expect(result.missingTranscript).toBe(0);
+
+    const run = readRun("crash", "with_skill");
+    expect(run.eval_id).toBe("crash");
+    expect(run.condition).toBe("with_skill");
+    expect(run.skill_path).toBe("/staged/skill/SKILL.md");
+    expect(run.prompt).toBe("Do the crash task");
+    expect(run.files).toHaveLength(1);
+    expect(run.final_message).toBe("Fixed it.");
+    expect(run.tool_invocations).toHaveLength(1);
+    expect(run.tool_invocations[0]).toMatchObject({ name: "Bash", ordinal: 0 });
+
+    expect(readRun("crash", "without_skill").skill_path).toBeNull();
+
+    const timing = readTiming("crash", "with_skill");
+    expect(timing.total_tokens).toBe(TRANSCRIPT_TOKENS);
+    expect(timing.duration_ms).toBe(TRANSCRIPT_DURATION_MS);
+    expect(timing.source).toBe("transcript");
+  });
+
+  test("skips existing run.json without --overwrite, replaces with it", () => {
+    const [task] = writeIteration([
+      { eval_id: "crash", condition: "with_skill", finalMessage: "New." },
+    ]);
+    writeSubagent(
+      "agent-a",
+      "crash:with_skill:i1-nonce1",
+      transcriptLines("unused"),
+    );
+    const handWritten = {
+      eval_id: "crash",
+      condition: "with_skill",
+      skill_path: "/staged/skill/SKILL.md",
+      prompt: "Do the crash task",
+      files: [],
+      final_message: "Agent-authored.",
+      tool_invocations: [],
+    };
+    writeFileSync(task.run_record_path, JSON.stringify(handWritten));
+
+    const skipped = recordRuns({ iterationDir, subagentsDir });
+    expect(skipped.recorded).toBe(0);
+    expect(skipped.skippedExisting).toBe(1);
+    expect(readRun("crash", "with_skill").final_message).toBe(
+      "Agent-authored.",
+    );
+
+    const replaced = recordRuns({
+      iterationDir,
+      subagentsDir,
+      overwrite: true,
+    });
+    expect(replaced.recorded).toBe(1);
+    expect(readRun("crash", "with_skill").final_message).toBe("New.");
+  });
+
+  test("backfills timing.json only when absent", () => {
+    const [task] = writeIteration([
+      { eval_id: "crash", condition: "with_skill", finalMessage: "Done." },
+    ]);
+    writeSubagent(
+      "agent-a",
+      "crash:with_skill:i1-nonce1",
+      transcriptLines("unused"),
+    );
+    writeFileSync(
+      task.timing_path,
+      JSON.stringify({ total_tokens: 12345, duration_ms: 9000 }),
+    );
+
+    recordRuns({ iterationDir, subagentsDir });
+
+    // Agent-captured completion-event timing wins; not overwritten.
+    const timing = readTiming("crash", "with_skill");
+    expect(timing.total_tokens).toBe(12345);
+    expect(timing.duration_ms).toBe(9000);
+    expect(timing.source).toBeUndefined();
+  });
+
+  test("falls back to the transcript's final assistant text when final-message.md is missing", () => {
+    writeIteration([{ eval_id: "crash", condition: "with_skill" }]);
+    writeSubagent(
+      "agent-a",
+      "crash:with_skill:i1-nonce1",
+      transcriptLines("Closing summary from transcript."),
+    );
+
+    const result = recordRuns({ iterationDir, subagentsDir });
+    expect(result.recorded).toBe(1);
+    expect(readRun("crash", "with_skill").final_message).toBe(
+      "Closing summary from transcript.",
+    );
+  });
+
+  test("skips the slot entirely when no final-message source exists", () => {
+    writeIteration([{ eval_id: "crash", condition: "with_skill" }]);
+    // No final-message.md, no transcript.
+
+    const result = recordRuns({ iterationDir, subagentsDir });
+    expect(result.recorded).toBe(0);
+    expect(result.skippedNoFinalMessage).toBe(1);
+    expect(
+      existsSync(join(iterationDir, "eval-crash", "with_skill", "run.json")),
+    ).toBe(false);
+    expect(
+      existsSync(join(iterationDir, "eval-crash", "with_skill", "timing.json")),
+    ).toBe(false);
+  });
+
+  test("writes run.json with empty invocations and no timing.json when the transcript is missing", () => {
+    writeIteration([
+      { eval_id: "crash", condition: "with_skill", finalMessage: "Done." },
+    ]);
+    // final-message.md exists but no subagent transcript matches.
+
+    const result = recordRuns({ iterationDir, subagentsDir });
+    expect(result.recorded).toBe(1);
+    expect(result.missingTranscript).toBe(1);
+
+    const run = readRun("crash", "with_skill");
+    expect(run.final_message).toBe("Done.");
+    expect(run.tool_invocations).toEqual([]);
+    expect(
+      existsSync(join(iterationDir, "eval-crash", "with_skill", "timing.json")),
+    ).toBe(false);
+  });
+
+  test("throws when dispatch.json is absent", () => {
+    // Hand-authored/operator runs have no dispatch.json — the manual path owns them.
+    expect(() => recordRuns({ iterationDir, subagentsDir })).toThrow(
+      /dispatch\.json/,
+    );
+  });
+});