@slowdini/slow-powers-opencode 0.3.0 → 0.4.0

package/skills/evaluating-skills/runner/detect-stray-writes.test.ts

@@ -1,396 +0,0 @@
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import {
-  mkdirSync,
-  readFileSync,
-  realpathSync,
-  rmSync,
-  writeFileSync,
-} from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import {
-  detectLiveSourceReads,
-  detectStrayWrites,
-} from "./detect-stray-writes";
-
-const OUTPUTS = "/work/iteration-1/eval-x/with_skill/outputs";
-const REPO = "/work/repo";
-const LIVE_SKILL = join(REPO, "skills", "mr-review");
-
-describe("detectStrayWrites", () => {
-  test("a Write inside the outputs dir is clean", () => {
-    const findings = detectStrayWrites(
-      [
-        {
-          name: "Write",
-          args: { file_path: join(OUTPUTS, "answer.md") },
-          ordinal: 0,
-        },
-      ],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.violations).toHaveLength(0);
-    expect(findings.warnings).toHaveLength(0);
-  });
-
-  test("a Write outside the outputs dir is a violation", () => {
-    const findings = detectStrayWrites(
-      [
-        {
-          name: "Write",
-          args: { file_path: join(REPO, "runner/run.ts") },
-          ordinal: 2,
-        },
-      ],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.violations).toHaveLength(1);
-    expect(findings.violations[0]).toMatchObject({
-      tool: "Write",
-      path: join(REPO, "runner/run.ts"),
-      ordinal: 2,
-    });
-  });
-
-  test("an Edit/MultiEdit/NotebookEdit outside outputs is a violation", () => {
-    const findings = detectStrayWrites(
-      [
-        { name: "Edit", args: { file_path: "/etc/hosts" }, ordinal: 0 },
-        {
-          name: "NotebookEdit",
-          args: { notebook_path: "/tmp/x.ipynb" },
-          ordinal: 1,
-        },
-      ],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.violations.map((v) => v.tool).sort()).toEqual([
-      "Edit",
-      "NotebookEdit",
-    ]);
-  });
-
-  test("an install command is a warning", () => {
-    const findings = detectStrayWrites(
-      [{ name: "Bash", args: { command: "npm install left-pad" }, ordinal: 0 }],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.warnings).toHaveLength(1);
-    expect(findings.warnings[0].tool).toBe("Bash");
-    expect(findings.warnings[0].reason).toMatch(/install/i);
-  });
-
-  test("a mutating Bash command scoped to the outputs dir is not flagged", () => {
-    const findings = detectStrayWrites(
-      [
-        {
-          name: "Bash",
-          args: { command: `echo hi > ${join(OUTPUTS, "log.txt")}` },
-          ordinal: 0,
-        },
-      ],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.warnings).toHaveLength(0);
-  });
-
-  test("git worktree add is a warning (working tree outside the sandbox)", () => {
-    const findings = detectStrayWrites(
-      [
-        {
-          name: "Bash",
-          args: { command: "git worktree add ../wt -b scratch" },
-          ordinal: 0,
-        },
-      ],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.warnings).toHaveLength(1);
-    expect(findings.warnings[0].reason).toMatch(/worktree/i);
-  });
-
-  test("creating a path under .claude is a warning", () => {
-    const findings = detectStrayWrites(
-      [{ name: "Bash", args: { command: "mkdir -p .claude/foo" }, ordinal: 0 }],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.warnings).toHaveLength(1);
-    expect(findings.warnings[0].reason).toMatch(/\.claude/i);
-  });
-
-  test("read-only tools are never flagged", () => {
-    const findings = detectStrayWrites(
-      [
-        { name: "Read", args: { file_path: "/anywhere" }, ordinal: 0 },
-        { name: "Grep", args: { pattern: "x" }, ordinal: 1 },
-        { name: "Bash", args: { command: "ls -la /" }, ordinal: 2 },
-      ],
-      OUTPUTS,
-      REPO,
-    );
-    expect(findings.violations).toHaveLength(0);
-    expect(findings.warnings).toHaveLength(0);
-  });
-});
-
-describe("detectLiveSourceReads", () => {
-  test("a Read of the live SKILL.md is flagged", () => {
-    const findings = detectLiveSourceReads(
-      [
-        {
-          name: "Read",
-          args: { file_path: join(LIVE_SKILL, "SKILL.md") },
-          ordinal: 1,
-        },
-      ],
-      LIVE_SKILL,
-      REPO,
-    );
-    expect(findings).toHaveLength(1);
-    expect(findings[0]).toMatchObject({
-      tool: "Read",
-      path: join(LIVE_SKILL, "SKILL.md"),
-      ordinal: 1,
-    });
-    expect(findings[0].reason).toMatch(/live skill source/i);
-  });
-
-  test("a Read of a staged eval copy is not flagged", () => {
-    const findings = detectLiveSourceReads(
-      [
-        {
-          name: "Read",
-          args: {
-            file_path: join(
-              REPO,
-              ".claude/skills/slow-powers-eval-1-old_skill__mr-review/SKILL.md",
-            ),
-          },
-          ordinal: 0,
-        },
-      ],
-      LIVE_SKILL,
-      REPO,
-    );
-    expect(findings).toHaveLength(0);
-  });
-
-  test("a relative Read path resolving under the live dir is flagged", () => {
-    const findings = detectLiveSourceReads(
-      [
-        {
-          name: "Read",
-          args: { file_path: "skills/mr-review/SKILL.md" },
-          ordinal: 0,
-        },
-      ],
-      LIVE_SKILL,
-      REPO,
-    );
-    expect(findings).toHaveLength(1);
-  });
-
-  test("a Grep scoped to the live dir is flagged", () => {
-    const findings = detectLiveSourceReads(
-      [{ name: "Grep", args: { pattern: "x", path: LIVE_SKILL }, ordinal: 2 }],
-      LIVE_SKILL,
-      REPO,
-    );
-    expect(findings).toHaveLength(1);
-    expect(findings[0].tool).toBe("Grep");
-  });
-
-  test("a Bash command referencing the live dir relatively is flagged", () => {
-    const findings = detectLiveSourceReads(
-      [
-        {
-          name: "Bash",
-          args: { command: "cat skills/mr-review/SKILL.md" },
-          ordinal: 3,
-        },
-      ],
-      LIVE_SKILL,
-      REPO,
-    );
-    expect(findings).toHaveLength(1);
-    expect(findings[0].tool).toBe("Bash");
-    expect(findings[0].command).toBe("cat skills/mr-review/SKILL.md");
-  });
-
-  test("a Bash command referencing the live dir absolutely is flagged", () => {
-    const findings = detectLiveSourceReads(
-      [
-        {
-          name: "Bash",
-          args: { command: `grep -r trigger ${LIVE_SKILL}/` },
-          ordinal: 0,
-        },
-      ],
-      LIVE_SKILL,
-      REPO,
-    );
-    expect(findings).toHaveLength(1);
-  });
-
-  test("a Bash command referencing a staged copy under .claude/skills is not flagged", () => {
-    // --stage-name can stage under the skill's natural name; that path contains
-    // `skills/<name>` but lives under `.claude/`, so it must not match.
-    const findings = detectLiveSourceReads(
-      [
-        {
-          name: "Bash",
-          args: { command: "cat .claude/skills/mr-review/SKILL.md" },
-          ordinal: 0,
-        },
-      ],
-      LIVE_SKILL,
-      REPO,
-    );
-    expect(findings).toHaveLength(0);
-  });
-
-  test("unrelated reads and commands are not flagged", () => {
-    const findings = detectLiveSourceReads(
-      [
-        {
-          name: "Read",
-          args: { file_path: join(OUTPUTS, "x.md") },
-          ordinal: 0,
-        },
-        { name: "Bash", args: { command: "ls skills-workspace" }, ordinal: 1 },
-        {
-          name: "Write",
-          args: { file_path: join(LIVE_SKILL, "SKILL.md") },
-          ordinal: 2,
-        },
-      ],
-      LIVE_SKILL,
-      REPO,
-    );
-    // Write tools are detectStrayWrites' jurisdiction — this check is reads only.
-    expect(findings).toHaveLength(0);
-  });
-});
-
-describe("detect-stray-writes CLI", () => {
-  // realpath: the spawned CLI sees its cwd resolved (macOS /var → /private/var),
-  // so fixture paths must match that form for prefix checks to line up.
-  const FIXTURE_ROOT = join(
-    realpathSync(tmpdir()),
-    `slow-powers-detect-stray-test-${process.pid}`,
-  );
-  const SCRIPT = join(import.meta.dir, "detect-stray-writes.ts");
-
-  beforeAll(() => {
-    mkdirSync(FIXTURE_ROOT, { recursive: true });
-  });
-
-  afterAll(() => {
-    rmSync(FIXTURE_ROOT, { recursive: true, force: true });
-  });
-
-  test("reports live-source reads per run in stray-writes.json", () => {
-    const root = join(FIXTURE_ROOT, "cli-live-reads");
-    const skillDir = join(root, "skill-dir");
-    const skillSub = join(skillDir, "mr-review");
-    mkdirSync(skillSub, { recursive: true });
-    writeFileSync(
-      join(skillSub, "SKILL.md"),
-      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
-    );
-
-    const cwd = join(root, "work");
-    const iterationDir = join(
-      cwd,
-      "skills-workspace",
-      "mr-review",
-      "iteration-1",
-    );
-    const condDir = join(iterationDir, "eval-e1", "old_skill");
-    mkdirSync(condDir, { recursive: true });
-    writeFileSync(
-      join(iterationDir, "conditions.json"),
-      `${JSON.stringify({
-        mode: "revision",
-        conditions: [
-          { name: "old_skill", skill_path: join(skillSub, "SKILL.md") },
-          { name: "new_skill", skill_path: join(skillSub, "SKILL.md") },
-        ],
-        timestamp: new Date().toISOString(),
-        harness: "claude-code",
-      })}\n`,
-    );
-    writeFileSync(
-      join(condDir, "run.json"),
-      `${JSON.stringify({
-        eval_id: "e1",
-        condition: "old_skill",
-        skill_path: join(skillSub, "SKILL.md"),
-        prompt: "do the task",
-        files: [],
-        final_message: "done",
-        tool_invocations: [
-          {
-            name: "Read",
-            args: { file_path: join(skillSub, "SKILL.md") },
-            ordinal: 0,
-          },
-          {
-            name: "Write",
-            args: { file_path: join(condDir, "outputs", "answer.md") },
-            ordinal: 1,
-          },
-        ],
-      })}\n`,
-    );
-
-    const res = Bun.spawnSync(
-      [
-        "bun",
-        "run",
-        SCRIPT,
-        "--skill-dir",
-        skillDir,
-        "--skill",
-        "mr-review",
-        "--iteration",
-        "1",
-      ],
-      { cwd, stdout: "pipe", stderr: "pipe" },
-    );
-    expect(res.exitCode).toBe(0);
-
-    const report = JSON.parse(
-      readFileSync(join(iterationDir, "stray-writes.json"), "utf8"),
-    ) as {
-      totals: {
-        violations: number;
-        warnings: number;
-        live_source_reads: number;
-      };
-      runs: Array<{
-        eval_id: string;
-        condition: string;
-        live_source_reads: Array<{ tool: string; path?: string }>;
-      }>;
-    };
-    expect(report.totals.live_source_reads).toBe(1);
-    expect(report.totals.violations).toBe(0);
-    expect(report.runs).toHaveLength(1);
-    expect(report.runs[0]).toMatchObject({
-      eval_id: "e1",
-      condition: "old_skill",
-    });
-    expect(report.runs[0].live_source_reads[0]).toMatchObject({
-      tool: "Read",
-      path: join(skillSub, "SKILL.md"),
-    });
-  });
-});

package/skills/evaluating-skills/runner/detect-stray-writes.ts

@@ -1,288 +0,0 @@
-#!/usr/bin/env bun
-import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join, relative, resolve } from "node:path";
-import { detectRunContext } from "./context";
-import { classifyBash, isUnder, pathArg, WRITE_TOOLS } from "./sandbox-policy";
-import type { ConditionsRecord, RunRecord, ToolInvocation } from "./types";
-import { validateAgainstSchema } from "./validate-schema";
-
-function die(msg: string): never {
-  console.error(`error: ${msg}`);
-  process.exit(1);
-}
-
-export type StrayFinding = {
-  tool: string;
-  path?: string;
-  command?: string;
-  ordinal: number;
-  reason: string;
-};
-
-export type RunFindings = {
-  violations: StrayFinding[];
-  warnings: StrayFinding[];
-};
-
-/**
- * Classify a run's tool invocations against its allowed outputs dir.
- *
- * - `violations`: file-write tools (Write/Edit/MultiEdit/NotebookEdit) whose
- *   target path resolves outside `outputsDir`. High confidence — a run that
- *   edits the real repo is a tainted data point.
- * - `warnings`: Bash commands matching a mutating pattern that don't reference
- *   `outputsDir`. Heuristic — review before trusting.
- *
- * Relative paths resolve against `repoRoot` (the subagent's working dir);
- * Claude Code's write tools use absolute paths, so this is a best-effort
- * fallback only.
- */
-export function detectStrayWrites(
-  invocations: Array<Pick<ToolInvocation, "name" | "args" | "ordinal">>,
-  outputsDir: string,
-  repoRoot: string,
-): RunFindings {
-  const violations: StrayFinding[] = [];
-  const warnings: StrayFinding[] = [];
-
-  for (const inv of invocations) {
-    if (WRITE_TOOLS.has(inv.name)) {
-      const p = pathArg(inv.args);
-      if (p && !isUnder(p, outputsDir, repoRoot)) {
-        violations.push({
-          tool: inv.name,
-          path: p,
-          ordinal: inv.ordinal,
-          reason: "writes outside the run's outputs dir",
-        });
-      }
-      continue;
-    }
-
-    if (inv.name === "Bash") {
-      const args = inv.args as { command?: unknown } | undefined;
-      const command = typeof args?.command === "string" ? args.command : "";
-      const reason = classifyBash(command, [outputsDir]);
-      if (reason)
-        warnings.push({ tool: "Bash", command, ordinal: inv.ordinal, reason });
-    }
-  }
-
-  return { violations, warnings };
-}
-
-/** Read-only tools that carry a target path argument (see `pathArg`). */
-const READ_TOOLS = new Set(["Read", "Glob", "Grep"]);
-
-const LIVE_SOURCE_REASON =
-  "reads the live skill source instead of its staged copy — the arm may be contaminated";
-
-function escapeRegExp(s: string): string {
-  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-/**
- * Flag tool invocations that read the **live** skill-under-test directory.
- *
- * Eval subagents are only ever meant to see the *staged* copy of the skill
- * (`.claude/skills/<slug>/`, or the inlined SKILL.md under `--no-stage`). A
- * read of the live source typically means the Skill tool couldn't resolve the
- * staged slug yet (mid-session registry refresh race) and the agent improvised
- * — fatal in revision mode, where the old_skill arm then reads new-skill
- * content. Reads are detected, not blocked: the guard stays read-permissive,
- * so this surfaces post-hoc as a validity warning.
- *
- * - Read-tool calls (Read/Glob/Grep) whose path arg resolves under the live
- *   dir are flagged; relative paths resolve against `repoRoot`.
- * - Bash commands that reference the live dir (absolute, or repo-relative
- *   text) are flagged. A staged copy under `.claude/skills/` can carry the
- *   same `skills/<name>` relative text (e.g. via `--stage-name`), so that
- *   prefix is excluded.
- */
-export function detectLiveSourceReads(
-  invocations: Array<Pick<ToolInvocation, "name" | "args" | "ordinal">>,
-  liveSkillDir: string,
-  repoRoot: string,
-): StrayFinding[] {
-  const findings: StrayFinding[] = [];
-  const liveDir = resolve(liveSkillDir);
-  const rel = relative(repoRoot, liveDir);
-  const relRe = rel.startsWith("..")
-    ? null
-    : new RegExp(
-        // The lookbehind fires at the boundary char itself, so it checks for a
-        // bare `.claude` — the `/` is consumed by the boundary group.
-        `(?<!\\.claude)(^|[\\s'"=:(/])${escapeRegExp(rel)}(/|[\\s'")]|$)`,
-      );
-
-  for (const inv of invocations) {
-    if (READ_TOOLS.has(inv.name)) {
-      const p = pathArg(inv.args);
-      if (p && isUnder(p, liveDir, repoRoot)) {
-        findings.push({
-          tool: inv.name,
-          path: p,
-          ordinal: inv.ordinal,
-          reason: LIVE_SOURCE_REASON,
-        });
-      }
-      continue;
-    }
-
-    if (inv.name === "Bash") {
-      const args = inv.args as { command?: unknown } | undefined;
-      const command = typeof args?.command === "string" ? args.command : "";
-      if (command.includes(liveDir) || relRe?.test(command)) {
-        findings.push({
-          tool: "Bash",
-          command,
-          ordinal: inv.ordinal,
-          reason: LIVE_SOURCE_REASON,
-        });
-      }
-    }
-  }
-
-  return findings;
-}
-
-if (import.meta.main) {
-  const argv = Bun.argv.slice(2);
-  const flag = (name: string): string | undefined => {
-    const i = argv.indexOf(`--${name}`);
-    return i === -1 ? undefined : argv[i + 1];
-  };
-  const iteration = flag("iteration");
-  if (!iteration) die("missing --iteration");
-  const ctx = detectRunContext(argv);
-
-  const iterationDir = join(
-    ctx.workspaceRoot,
-    ctx.skillName,
-    `iteration-${iteration}`,
-  );
-  if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
-
-  const conditionsPath = join(iterationDir, "conditions.json");
-  if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
-  const conditions: ConditionsRecord = JSON.parse(
-    readFileSync(conditionsPath, "utf8"),
-  );
-  const conditionNames = conditions.conditions.map((c) => c.name);
-
-  // dispatch.json carries the authoritative outputs_dir per task; fall back to
-  // the conventional <condDir>/outputs when it's absent (hand-authored runs).
-  const dispatchPath = join(iterationDir, "dispatch.json");
-  const outputsByKey = new Map<string, string>();
-  if (existsSync(dispatchPath)) {
-    try {
-      const dispatch = JSON.parse(readFileSync(dispatchPath, "utf8")) as {
-        tasks?: Array<{
-          eval_id: string;
-          condition: string;
-          outputs_dir?: string;
-        }>;
-      };
-      for (const t of dispatch.tasks ?? []) {
-        if (t.outputs_dir)
-          outputsByKey.set(`${t.eval_id}:${t.condition}`, t.outputs_dir);
-      }
-    } catch {
-      // fall through to convention
-    }
-  }
-
-  const repoRoot = process.cwd();
-  const evalDirs = readdirSync(iterationDir).filter((d) =>
-    d.startsWith("eval-"),
-  );
-
-  type RunReport = {
-    eval_id: string;
-    condition: string;
-    violations: StrayFinding[];
-    warnings: StrayFinding[];
-    live_source_reads: StrayFinding[];
-  };
-  const runs: RunReport[] = [];
-  let totalViolations = 0;
-  let totalWarnings = 0;
-  let totalLiveReads = 0;
-
-  for (const evalDir of evalDirs) {
-    const evalId = evalDir.replace(/^eval-/, "");
-    for (const cond of conditionNames) {
-      const condDir = join(iterationDir, evalDir, cond);
-      const runPath = join(condDir, "run.json");
-      if (!existsSync(runPath)) continue;
-      const run = validateAgainstSchema<RunRecord>(
-        "run-record",
-        JSON.parse(readFileSync(runPath, "utf8")),
-        runPath,
-      );
-      const invocations = Array.isArray(run.tool_invocations)
-        ? run.tool_invocations
-        : [];
-      const outputsDir =
-        outputsByKey.get(`${evalId}:${cond}`) ?? join(condDir, "outputs");
-      const findings = detectStrayWrites(invocations, outputsDir, repoRoot);
-      const liveReads = detectLiveSourceReads(
-        invocations,
-        ctx.skillSubdir,
-        repoRoot,
-      );
-      if (
-        findings.violations.length ||
-        findings.warnings.length ||
-        liveReads.length
-      ) {
-        runs.push({
-          eval_id: evalId,
-          condition: cond,
-          violations: findings.violations,
-          warnings: findings.warnings,
-          live_source_reads: liveReads,
-        });
-      }
-      totalViolations += findings.violations.length;
-      totalWarnings += findings.warnings.length;
-      totalLiveReads += liveReads.length;
-    }
-  }
-
-  const report = {
-    generated: new Date().toISOString(),
-    iteration: Number(iteration),
-    totals: {
-      violations: totalViolations,
-      warnings: totalWarnings,
-      live_source_reads: totalLiveReads,
-    },
-    runs,
-  };
-  const outPath = join(iterationDir, "stray-writes.json");
-  validateAgainstSchema("stray-writes", report, outPath);
-  writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`);
-  console.log(`Wrote ${outPath}`);
-
-  for (const r of runs) {
-    for (const v of r.violations)
-      console.warn(
-        `✗ ${r.eval_id}/${r.condition}: ${v.tool} wrote outside outputs dir → ${v.path} (ordinal ${v.ordinal})`,
-      );
-    for (const w of r.warnings)
-      console.warn(
-        `⚠ ${r.eval_id}/${r.condition}: Bash ${w.reason} (ordinal ${w.ordinal}): ${w.command}`,
-      );
-    for (const l of r.live_source_reads)
-      console.warn(
-        `⚠ ${r.eval_id}/${r.condition}: ${l.tool} read the live skill source (ordinal ${l.ordinal}): ${l.path ?? l.command}`,
-      );
-  }
-  if (totalViolations === 0 && totalWarnings === 0 && totalLiveReads === 0)
-    console.log("✓ No out-of-bounds writes or live-source reads detected.");
-  else
-    console.warn(
-      `\n${totalViolations} violation(s), ${totalWarnings} warning(s), ${totalLiveReads} live-source read(s). Runs with violations edited files outside their sandbox; runs with live-source reads saw the live skill instead of their staged copy — treat those data points as tainted.`,
-    );
-}