@slowdini/slow-powers-opencode 0.1.0

package/skills/evaluating-skills/runner/guard/install.test.ts

@@ -0,0 +1,92 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  GUARD_MANIFEST,
+  GUARD_MARKER,
+  installGuard,
+  teardownGuard,
+} from "./install";
+
+const ROOT = join(tmpdir(), `guard-install-test-${process.pid}`);
+
+afterEach(() => rmSync(ROOT, { recursive: true, force: true }));
+
+function setup() {
+  const stageRoot = join(ROOT, `case-${Math.random().toString(36).slice(2)}`);
+  mkdirSync(stageRoot, { recursive: true });
+  const workspaceRoot = join(stageRoot, "skills-workspace");
+  return { stageRoot, workspaceRoot };
+}
+
+const skillsDir = (s: string) => join(s, ".claude", "skills");
+const settingsPath = (s: string) => join(s, ".claude", "settings.local.json");
+
+describe("installGuard / teardownGuard", () => {
+  test("install writes an active marker, hook, and manifest", () => {
+    const { stageRoot, workspaceRoot } = setup();
+    installGuard({ stageRoot, workspaceRoot, guardScriptPath: "/g/guard.ts" });
+
+    const marker = JSON.parse(
+      readFileSync(join(skillsDir(stageRoot), GUARD_MARKER), "utf8"),
+    );
+    expect(marker.active).toBe(true);
+    expect(Date.parse(marker.expiresAt)).toBeGreaterThan(Date.now());
+    expect(
+      marker.allowedRoots.some((r: string) => r.includes("skills-workspace")),
+    ).toBe(true);
+
+    const settings = JSON.parse(readFileSync(settingsPath(stageRoot), "utf8"));
+    expect(settings.hooks.PreToolUse[0].matcher).toContain("Write");
+    expect(settings.hooks.PreToolUse[0].hooks[0].command).toContain("guard.ts");
+
+    expect(existsSync(join(skillsDir(stageRoot), GUARD_MANIFEST))).toBe(true);
+  });
+
+  test("teardown deletes settings.local.json it created", () => {
+    const { stageRoot, workspaceRoot } = setup();
+    installGuard({ stageRoot, workspaceRoot, guardScriptPath: "/g/guard.ts" });
+    expect(existsSync(settingsPath(stageRoot))).toBe(true);
+
+    expect(teardownGuard(stageRoot)).toBe(true);
+    expect(existsSync(settingsPath(stageRoot))).toBe(false);
+    expect(existsSync(join(skillsDir(stageRoot), GUARD_MARKER))).toBe(false);
+    expect(existsSync(join(skillsDir(stageRoot), GUARD_MANIFEST))).toBe(false);
+  });
+
+  test("teardown restores a pre-existing settings.local.json verbatim", () => {
+    const { stageRoot, workspaceRoot } = setup();
+    mkdirSync(join(stageRoot, ".claude"), { recursive: true });
+    const original = `${JSON.stringify({ permissions: { allow: ["Bash(ls)"] } }, null, 2)}\n`;
+    writeFileSync(settingsPath(stageRoot), original);
+
+    installGuard({ stageRoot, workspaceRoot, guardScriptPath: "/g/guard.ts" });
+    // hook present while armed
+    expect(readFileSync(settingsPath(stageRoot), "utf8")).toContain(
+      "PreToolUse",
+    );
+
+    teardownGuard(stageRoot);
+    expect(readFileSync(settingsPath(stageRoot), "utf8")).toBe(original);
+  });
+
+  test("teardown is a safe no-op when nothing is installed", () => {
+    const { stageRoot } = setup();
+    expect(teardownGuard(stageRoot)).toBe(false);
+  });
+
+  test("teardown sweeps a stray marker even without a manifest", () => {
+    const { stageRoot } = setup();
+    mkdirSync(skillsDir(stageRoot), { recursive: true });
+    writeFileSync(join(skillsDir(stageRoot), GUARD_MARKER), "{}");
+    expect(teardownGuard(stageRoot)).toBe(true);
+    expect(existsSync(join(skillsDir(stageRoot), GUARD_MARKER))).toBe(false);
+  });
+});

package/skills/evaluating-skills/runner/guard/install.ts

@@ -0,0 +1,147 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+export const GUARD_MARKER = ".slow-powers-eval-guard.json";
+export const GUARD_MANIFEST = ".slow-powers-eval-guard-manifest.json";
+const GUARD_TTL_MS = 6 * 60 * 60 * 1000; // 6h — bounds a crashed run's lingering hook
+
+const HOOK_MATCHER = "Write|Edit|MultiEdit|NotebookEdit|Bash";
+
+type GuardManifest = {
+  created_at: string;
+  settings_path: string;
+  settings_existed: boolean;
+  settings_backup: string | null;
+  marker_path: string;
+};
+
+type Settings = {
+  hooks?: {
+    PreToolUse?: Array<{ matcher?: string; hooks?: unknown[] }>;
+    [k: string]: unknown;
+  };
+  [k: string]: unknown;
+};
+
+/**
+ * Arm the Claude Code write guard for an eval run. Writes a marker listing the
+ * allowed roots and merges a `PreToolUse` hook into `.claude/settings.local.json`
+ * that runs `guard.ts` on every Write/Edit/Bash. The original settings file is
+ * backed up verbatim in a manifest so {@link teardownGuard} restores it exactly.
+ *
+ * Returns the marker path. The guard is a no-op until this marker exists and is
+ * unexpired (see guard/policy.ts), so the hook is inert outside an active run.
+ */
+export function installGuard(opts: {
+  stageRoot: string;
+  workspaceRoot: string;
+  guardScriptPath: string;
+  ttlMs?: number;
+}): string {
+  const skillsDir = join(opts.stageRoot, ".claude", "skills");
+  mkdirSync(skillsDir, { recursive: true });
+
+  const markerPath = join(skillsDir, GUARD_MARKER);
+  const allowedRoots = [
+    resolve(opts.workspaceRoot),
+    resolve(skillsDir),
+    resolve(tmpdir()),
+  ];
+  writeFileSync(
+    markerPath,
+    `${JSON.stringify(
+      {
+        active: true,
+        allowedRoots,
+        expiresAt: new Date(
+          Date.now() + (opts.ttlMs ?? GUARD_TTL_MS),
+        ).toISOString(),
+      },
+      null,
+      2,
+    )}\n`,
+  );
+
+  const settingsPath = join(opts.stageRoot, ".claude", "settings.local.json");
+  const settingsExisted = existsSync(settingsPath);
+  const backup = settingsExisted ? readFileSync(settingsPath, "utf8") : null;
+
+  let settings: Settings = {};
+  if (backup) {
+    try {
+      settings = JSON.parse(backup);
+    } catch {
+      settings = {};
+    }
+  }
+  settings.hooks ??= {};
+  settings.hooks.PreToolUse ??= [];
+  settings.hooks.PreToolUse.push({
+    matcher: HOOK_MATCHER,
+    hooks: [
+      {
+        type: "command",
+        command: `bun run "${opts.guardScriptPath}" "${markerPath}"`,
+      },
+    ],
+  });
+  writeFileSync(settingsPath, `${JSON.stringify(settings, null, 2)}\n`);
+
+  const manifest: GuardManifest = {
+    created_at: new Date().toISOString(),
+    settings_path: settingsPath,
+    settings_existed: settingsExisted,
+    settings_backup: backup,
+    marker_path: markerPath,
+  };
+  writeFileSync(
+    join(skillsDir, GUARD_MANIFEST),
+    `${JSON.stringify(manifest, null, 2)}\n`,
+  );
+  return markerPath;
+}
+
+/**
+ * Disarm the guard: restore the original `settings.local.json` (or delete it if
+ * we created it) and remove the marker + manifest. Safe to call when no guard is
+ * installed. Returns true if a guard was found and torn down.
+ */
+export function teardownGuard(stageRoot: string): boolean {
+  const skillsDir = join(stageRoot, ".claude", "skills");
+  const manifestPath = join(skillsDir, GUARD_MANIFEST);
+  const markerPath = join(skillsDir, GUARD_MARKER);
+
+  if (!existsSync(manifestPath)) {
+    // No manifest — still sweep a stray marker so the guard can't stay armed.
+    if (existsSync(markerPath)) {
+      rmSync(markerPath, { force: true });
+      return true;
+    }
+    return false;
+  }
+
+  let manifest: GuardManifest;
+  try {
+    manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+  } catch {
+    rmSync(manifestPath, { force: true });
+    rmSync(markerPath, { force: true });
+    return true;
+  }
+
+  if (manifest.settings_existed && manifest.settings_backup !== null) {
+    writeFileSync(manifest.settings_path, manifest.settings_backup);
+  } else if (existsSync(manifest.settings_path)) {
+    rmSync(manifest.settings_path, { force: true });
+  }
+  rmSync(manifest.marker_path, { force: true });
+  rmSync(manifestPath, { force: true });
+  return true;
+}

package/skills/evaluating-skills/runner/guard/policy.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, test } from "bun:test";
+import { decide, type GuardMarker } from "./policy";
+
+const ROOTS = ["/work/skills-workspace", "/work/.claude/skills"];
+const future = () => new Date(Date.now() + 60_000).toISOString();
+const past = () => new Date(Date.now() - 60_000).toISOString();
+
+function marker(over: Partial<GuardMarker> = {}): GuardMarker {
+  return { active: true, allowedRoots: ROOTS, expiresAt: future(), ...over };
+}
+
+describe("guard decide", () => {
+  test("allows everything when marker is null (guard inactive)", () => {
+    expect(decide("Write", { file_path: "/etc/passwd" }, null).allow).toBe(
+      true,
+    );
+  });
+
+  test("allows everything when marker is inactive or expired", () => {
+    expect(
+      decide("Write", { file_path: "/etc/passwd" }, marker({ active: false }))
+        .allow,
+    ).toBe(true);
+    expect(
+      decide(
+        "Write",
+        { file_path: "/etc/passwd" },
+        marker({ expiresAt: past() }),
+      ).allow,
+    ).toBe(true);
+  });
+
+  test("allows a write under an allowed root", () => {
+    expect(
+      decide(
+        "Write",
+        { file_path: "/work/skills-workspace/x/outputs/a.md" },
+        marker(),
+      ).allow,
+    ).toBe(true);
+  });
+
+  test("denies a write outside all allowed roots", () => {
+    const d = decide("Edit", { file_path: "/work/runner/run.ts" }, marker());
+    expect(d.allow).toBe(false);
+    expect(d.reason).toMatch(/outside/i);
+  });
+
+  test("denies an install command", () => {
+    const d = decide("Bash", { command: "npm install left-pad" }, marker());
+    expect(d.allow).toBe(false);
+    expect(d.reason).toMatch(/install/i);
+  });
+
+  test("allows a Bash command scoped to an allowed root", () => {
+    expect(
+      decide(
+        "Bash",
+        { command: "echo hi > /work/skills-workspace/x/outputs/log" },
+        marker(),
+      ).allow,
+    ).toBe(true);
+  });
+
+  test("allows non-mutating Bash and read tools", () => {
+    expect(decide("Bash", { command: "ls -la /" }, marker()).allow).toBe(true);
+    expect(decide("Read", { file_path: "/etc/passwd" }, marker()).allow).toBe(
+      true,
+    );
+  });
+});

package/skills/evaluating-skills/runner/guard/policy.ts

@@ -0,0 +1,74 @@
+import {
+  classifyBash,
+  isUnderAny,
+  pathArg,
+  WRITE_TOOLS,
+} from "../sandbox-policy";
+
+/**
+ * The marker file (`<stageRoot>/.claude/skills/.slow-powers-eval-guard.json`)
+ * that arms the guard. The guard is a no-op unless this file exists, is active,
+ * and has not expired — so a crashed run that never tore the hook down can't
+ * silently block writes in the user's next interactive session.
+ */
+export type GuardMarker = {
+  active?: boolean;
+  allowedRoots?: string[];
+  expiresAt?: string;
+};
+
+export type GuardDecision = { allow: boolean; reason?: string };
+
+const ALLOW: GuardDecision = { allow: true };
+
+function armed(marker: GuardMarker | null, now: number): boolean {
+  if (marker?.active !== true) return false;
+  if (marker.expiresAt && Date.parse(marker.expiresAt) <= now) return false;
+  return true;
+}
+
+/**
+ * Decide whether a tool call should be allowed while the eval guard is armed.
+ *
+ * Write tools targeting a path outside every allowed root are denied; Bash
+ * commands matching a mutation pattern (install/git/sed -i/redirection) that
+ * aren't scoped to an allowed root are denied. Everything else — including all
+ * read-only tools and the orchestrator's own writes under the workspace — is
+ * allowed. When the guard is not armed, every call is allowed.
+ */
+export function decide(
+  toolName: string,
+  toolInput: unknown,
+  marker: GuardMarker | null,
+  now: number = Date.now(),
+): GuardDecision {
+  if (!armed(marker, now)) return ALLOW;
+  const roots = marker?.allowedRoots ?? [];
+  const repoRoot = process.cwd();
+
+  if (WRITE_TOOLS.has(toolName)) {
+    const p = pathArg(toolInput);
+    if (p && !isUnderAny(p, roots, repoRoot)) {
+      return {
+        allow: false,
+        reason: `eval guard: ${toolName} to ${p} is outside the eval sandbox (allowed: ${roots.join(", ")})`,
+      };
+    }
+    return ALLOW;
+  }
+
+  if (toolName === "Bash") {
+    const command =
+      toolInput && typeof toolInput === "object"
+        ? String((toolInput as Record<string, unknown>).command ?? "")
+        : "";
+    const reason = classifyBash(command, roots);
+    if (reason)
+      return {
+        allow: false,
+        reason: `eval guard: blocked Bash (${reason}) — runs outside the eval sandbox`,
+      };
+  }
+
+  return ALLOW;
+}

package/skills/evaluating-skills/runner/promote-baseline.test.ts

@@ -0,0 +1,230 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const FIXTURE_ROOT = join(tmpdir(), `slow-powers-promote-test-${process.pid}`);
+const PROMOTE_TS = join(import.meta.dir, "promote-baseline.ts");
+
+beforeAll(() => {
+  mkdirSync(FIXTURE_ROOT, { recursive: true });
+});
+
+afterAll(() => {
+  rmSync(FIXTURE_ROOT, { recursive: true, force: true });
+});
+
+function writeJson(path: string, value: unknown) {
+  writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+describe("promote-baseline.ts (--skill-dir, isolated CWD)", () => {
+  test("copies benchmark + per-run gradings into the skill's committed baseline/", () => {
+    const root = join(FIXTURE_ROOT, "promote-basic");
+
+    // Skill dir + skill-under-test (detectRunContext validates SKILL.md exists).
+    const skillDir = join(root, "skill-dir");
+    const skillSub = join(skillDir, "mr-review");
+    mkdirSync(skillSub, { recursive: true });
+    writeFileSync(
+      join(skillSub, "SKILL.md"),
+      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
+    );
+
+    // Working dir holding the workspace (mirrors workspaceRoot = <cwd>/skills-workspace).
+    const cwd = join(root, "work");
+    const iterationDir = join(
+      cwd,
+      "skills-workspace",
+      "mr-review",
+      "iteration-2",
+    );
+    mkdirSync(iterationDir, { recursive: true });
+
+    const timestamp = "2026-05-27T00:00:00.000Z";
+    writeJson(join(iterationDir, "conditions.json"), {
+      mode: "new-skill",
+      conditions: [
+        { name: "with_skill", skill_path: join(skillSub, "SKILL.md") },
+        { name: "without_skill", skill_path: null },
+      ],
+      timestamp,
+      harness: "claude-code",
+    });
+    writeJson(join(iterationDir, "benchmark.json"), {
+      run_summary: {
+        with_skill: { pass_rate: { mean: 0.83 } },
+        without_skill: { pass_rate: { mean: 0.33 } },
+      },
+      delta: { pass_rate: 0.5 },
+    });
+
+    const mkGrading = (evalId: string, cond: string, passRate: number) => {
+      const condDir = join(iterationDir, `eval-${evalId}`, cond);
+      mkdirSync(condDir, { recursive: true });
+      writeJson(join(condDir, "grading.json"), {
+        assertion_results: [
+          {
+            id: "a1",
+            passed: passRate > 0,
+            evidence: `${cond} evidence`,
+            confidence: 1,
+          },
+        ],
+        summary: { passed: 1, failed: 0, total: 1, pass_rate: passRate },
+      });
+    };
+    mkGrading("e1", "with_skill", 1);
+    mkGrading("e1", "without_skill", 0);
+
+    const res = Bun.spawnSync(
+      [
+        "bun",
+        "run",
+        PROMOTE_TS,
+        "--skill-dir",
+        skillDir,
+        "--skill",
+        "mr-review",
+        "--iteration",
+        "2",
+      ],
+      { cwd, stdout: "pipe", stderr: "pipe" },
+    );
+    expect(res.stderr.toString()).toBe("");
+    expect(res.exitCode).toBe(0);
+
+    const baselineDir = join(skillSub, "evals", "baseline");
+
+    // benchmark.json copied verbatim.
+    const benchmarkPath = join(baselineDir, "benchmark.json");
+    expect(existsSync(benchmarkPath)).toBe(true);
+    const benchmark = JSON.parse(readFileSync(benchmarkPath, "utf8")) as {
+      delta: { pass_rate: number };
+    };
+    expect(benchmark.delta.pass_rate).toBe(0.5);
+
+    // Per-run gradings copied under grading/<eval-id>__<condition>.json.
+    const withGrading = join(baselineDir, "grading", "e1__with_skill.json");
+    const withoutGrading = join(
+      baselineDir,
+      "grading",
+      "e1__without_skill.json",
+    );
+    expect(existsSync(withGrading)).toBe(true);
+    expect(existsSync(withoutGrading)).toBe(true);
+    const withParsed = JSON.parse(readFileSync(withGrading, "utf8")) as {
+      summary: { pass_rate: number };
+    };
+    expect(withParsed.summary.pass_rate).toBe(1);
+
+    // Provenance file records mode, iteration, harness, timestamp.
+    const provenancePath = join(baselineDir, "BASELINE.md");
+    expect(existsSync(provenancePath)).toBe(true);
+    const provenance = readFileSync(provenancePath, "utf8");
+    expect(provenance).toContain("new-skill");
+    expect(provenance).toContain("iteration-2");
+    expect(provenance).toContain("claude-code");
+    expect(provenance).toContain(timestamp);
+    // Model rows default to "unspecified" when no flags are passed.
+    expect(provenance).toContain("Agent model | unspecified");
+    expect(provenance).toContain("Judge model | unspecified");
+  });
+
+  test("records agent and judge models in provenance when flags are passed", () => {
+    const root = join(FIXTURE_ROOT, "promote-models");
+
+    const skillDir = join(root, "skill-dir");
+    const skillSub = join(skillDir, "mr-review");
+    mkdirSync(skillSub, { recursive: true });
+    writeFileSync(
+      join(skillSub, "SKILL.md"),
+      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
+    );
+
+    const cwd = join(root, "work");
+    const iterationDir = join(
+      cwd,
+      "skills-workspace",
+      "mr-review",
+      "iteration-1",
+    );
+    mkdirSync(iterationDir, { recursive: true });
+    writeJson(join(iterationDir, "conditions.json"), {
+      mode: "new-skill",
+      conditions: [
+        { name: "with_skill", skill_path: join(skillSub, "SKILL.md") },
+        { name: "without_skill", skill_path: null },
+      ],
+      timestamp: "2026-05-27T00:00:00.000Z",
+      harness: "claude-code",
+    });
+    writeJson(join(iterationDir, "benchmark.json"), {
+      delta: { pass_rate: 0 },
+    });
+
+    const res = Bun.spawnSync(
+      [
+        "bun",
+        "run",
+        PROMOTE_TS,
+        "--skill-dir",
+        skillDir,
+        "--skill",
+        "mr-review",
+        "--iteration",
+        "1",
+        "--agent-model",
+        "claude-haiku-4-5-20251001",
+        "--judge-model",
+        "claude-opus-4-7",
+      ],
+      { cwd, stdout: "pipe", stderr: "pipe" },
+    );
+    expect(res.stderr.toString()).toBe("");
+    expect(res.exitCode).toBe(0);
+
+    const provenance = readFileSync(
+      join(skillSub, "evals", "baseline", "BASELINE.md"),
+      "utf8",
+    );
+    expect(provenance).toContain("Agent model | claude-haiku-4-5-20251001");
+    expect(provenance).toContain("Judge model | claude-opus-4-7");
+  });
+
+  test("fails clearly when the iteration directory is missing", () => {
+    const root = join(FIXTURE_ROOT, "promote-missing");
+    const skillDir = join(root, "skill-dir");
+    const skillSub = join(skillDir, "mr-review");
+    mkdirSync(skillSub, { recursive: true });
+    writeFileSync(
+      join(skillSub, "SKILL.md"),
+      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
+    );
+    const cwd = join(root, "work");
+    mkdirSync(cwd, { recursive: true });
+
+    const res = Bun.spawnSync(
+      [
+        "bun",
+        "run",
+        PROMOTE_TS,
+        "--skill-dir",
+        skillDir,
+        "--skill",
+        "mr-review",
+        "--iteration",
+        "9",
+      ],
+      { cwd, stdout: "pipe", stderr: "pipe" },
+    );
+    expect(res.exitCode).not.toBe(0);
+    expect(res.stderr.toString()).toContain("iteration-9");
+  });
+});