@slowdini/slow-powers-opencode 0.1.0

package/skills/evaluating-skills/runner/aggregate.ts

@@ -0,0 +1,228 @@
+#!/usr/bin/env bun
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { detectRunContext } from "./context";
+import type { ConditionsRecord, GradingResult, TimingRecord } from "./types";
+
+function die(msg: string): never {
+  console.error(`error: ${msg}`);
+  process.exit(1);
+}
+
+function parseArgs(argv: string[]) {
+  const flag = (name: string): string | undefined => {
+    const i = argv.indexOf(`--${name}`);
+    if (i === -1) return undefined;
+    return argv[i + 1];
+  };
+  const iteration = flag("iteration");
+  if (!iteration) die("missing --iteration");
+  return { iteration };
+}
+
+type Series = number[];
+
+function mean(values: Series): number {
+  if (values.length === 0) return 0;
+  return values.reduce((a, b) => a + b, 0) / values.length;
+}
+
+function stddev(values: Series, m = mean(values)): number {
+  if (values.length < 2) return 0;
+  const v = values.reduce((s, x) => s + (x - m) ** 2, 0) / values.length;
+  return Math.sqrt(v);
+}
+
+function round(n: number, dp: number): number {
+  const p = 10 ** dp;
+  return Math.round(n * p) / p;
+}
+
+function stats(values: Series, dp: number) {
+  const m = mean(values);
+  return {
+    mean: round(m, dp),
+    stddev: round(stddev(values, m), dp),
+    n: values.length,
+  };
+}
+
+const aggArgv = Bun.argv.slice(2);
+const { iteration } = parseArgs(aggArgv);
+const aggCtx = detectRunContext(aggArgv);
+const iterationDir = join(
+  aggCtx.workspaceRoot,
+  aggCtx.skillName,
+  `iteration-${iteration}`,
+);
+if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
+
+const conditionsPath = join(iterationDir, "conditions.json");
+if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
+const conditions: ConditionsRecord = JSON.parse(
+  readFileSync(conditionsPath, "utf8"),
+);
+const conditionNames = conditions.conditions.map((c) => c.name);
+if (conditionNames.length !== 2)
+  die(`expected exactly 2 conditions, got ${conditionNames.length}`);
+
+const evalDirs = readdirSync(iterationDir).filter((d) => d.startsWith("eval-"));
+if (evalDirs.length === 0) die("no eval directories found");
+
+type Bucket = {
+  passRates: Series;
+  durations: Series;
+  tokens: Series;
+  skillInvoked: boolean[];
+  hadSkillLoaded: boolean;
+};
+const byCondition: Record<string, Bucket> = {};
+const conditionSkillPaths = new Map<string, string | null>();
+for (const c of conditions.conditions) {
+  conditionSkillPaths.set(c.name, c.skill_path);
+  byCondition[c.name] = {
+    passRates: [],
+    durations: [],
+    tokens: [],
+    skillInvoked: [],
+    hadSkillLoaded: !!c.skill_path,
+  };
+}
+
+let missingGradings = 0;
+for (const evalDir of evalDirs) {
+  for (const cond of conditionNames) {
+    const condDir = join(iterationDir, evalDir, cond);
+    const gradingPath = join(condDir, "grading.json");
+    const timingPath = join(condDir, "timing.json");
+    if (!existsSync(gradingPath)) {
+      console.warn(`warn: missing grading for ${evalDir}/${cond}`);
+      missingGradings++;
+      continue;
+    }
+    const grading: GradingResult = JSON.parse(
+      readFileSync(gradingPath, "utf8"),
+    );
+    byCondition[cond].passRates.push(grading.summary.pass_rate);
+    if (grading.meta_summary?.skill_invoked != null)
+      byCondition[cond].skillInvoked.push(grading.meta_summary.skill_invoked);
+    if (existsSync(timingPath)) {
+      const timing: TimingRecord = JSON.parse(readFileSync(timingPath, "utf8"));
+      if (typeof timing.total_tokens === "number")
+        byCondition[cond].tokens.push(timing.total_tokens);
+      if (typeof timing.duration_ms === "number")
+        byCondition[cond].durations.push(timing.duration_ms);
+    }
+  }
+}
+
+type ConditionSummary = {
+  pass_rate: ReturnType<typeof stats>;
+  duration_ms: ReturnType<typeof stats>;
+  total_tokens: ReturnType<typeof stats>;
+  skill_invocation_rate?: number | null;
+  skill_invocation_n?: number;
+};
+
+const runSummary: Record<string, ConditionSummary> = {};
+for (const cond of conditionNames) {
+  const bucket = byCondition[cond];
+  const summary: ConditionSummary = {
+    pass_rate: stats(bucket.passRates, 3),
+    duration_ms: stats(bucket.durations, 0),
+    total_tokens: stats(bucket.tokens, 0),
+  };
+  if (bucket.hadSkillLoaded) {
+    summary.skill_invocation_n = bucket.skillInvoked.length;
+    summary.skill_invocation_rate =
+      bucket.skillInvoked.length === 0
+        ? null
+        : round(
+            bucket.skillInvoked.filter(Boolean).length /
+              bucket.skillInvoked.length,
+            3,
+          );
+  }
+  runSummary[cond] = summary;
+}
+
+const [a, b] = conditionNames;
+const delta = {
+  direction: `${a} - ${b}`,
+  pass_rate: round(
+    runSummary[a].pass_rate.mean - runSummary[b].pass_rate.mean,
+    3,
+  ),
+  duration_ms: round(
+    runSummary[a].duration_ms.mean - runSummary[b].duration_ms.mean,
+    0,
+  ),
+  total_tokens: round(
+    runSummary[a].total_tokens.mean - runSummary[b].total_tokens.mean,
+    0,
+  ),
+};
+
+const validityWarnings: string[] = [];
+for (const cond of conditionNames) {
+  const s = runSummary[cond];
+  if (s.skill_invocation_rate != null && s.skill_invocation_rate < 1) {
+    validityWarnings.push(
+      `condition '${cond}' had skill loaded but invocation rate ${(s.skill_invocation_rate * 100).toFixed(0)}% (${s.skill_invocation_n} runs checked) — substantive results may not reflect skill effectiveness.`,
+    );
+  }
+}
+
+// Stray-write findings (from `evals:detect-stray-writes`, if it ran) taint a
+// run the same way a missed skill invocation does: a subagent that edited the
+// real repo or installed packages is no longer a clean data point.
+const strayPath = join(iterationDir, "stray-writes.json");
+if (existsSync(strayPath)) {
+  try {
+    const stray = JSON.parse(readFileSync(strayPath, "utf8")) as {
+      runs?: Array<{
+        eval_id: string;
+        condition: string;
+        violations?: unknown[];
+      }>;
+    };
+    for (const r of stray.runs ?? []) {
+      const n = r.violations?.length ?? 0;
+      if (n > 0)
+        validityWarnings.push(
+          `${r.eval_id}/${r.condition} wrote ${n} file(s) outside its outputs dir — data point may be tainted (see stray-writes.json).`,
+        );
+    }
+  } catch {
+    // ignore a malformed report rather than failing aggregation
+  }
+}
+
+const benchmark = {
+  generated: new Date().toISOString(),
+  mode: conditions.mode,
+  baseline: conditions.baseline,
+  conditions_compared: [a, b],
+  missing_gradings: missingGradings,
+  validity_warnings: validityWarnings,
+  run_summary: runSummary,
+  delta,
+};
+
+const outPath = join(iterationDir, "benchmark.json");
+writeFileSync(outPath, `${JSON.stringify(benchmark, null, 2)}\n`);
+console.log(`Wrote ${outPath}`);
+if (missingGradings > 0)
+  console.warn(
+    `note: ${missingGradings} grading.json file(s) were missing — benchmark is incomplete.`,
+  );
+for (const warning of validityWarnings) console.warn(`⚠ ${warning}`);
+if (validityWarnings.length === 0) {
+  for (const cond of conditionNames) {
+    const s = runSummary[cond];
+    if (s.skill_invocation_rate === 1)
+      console.log(
+        `✓ ${cond}: skill invocation rate 100% (${s.skill_invocation_n} runs) — substantive results are valid.`,
+      );
+  }
+}

package/skills/evaluating-skills/runner/context.test.ts

@@ -0,0 +1,181 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { detectRunContext } from "./context";
+
+const FIXTURE_ROOT = join(tmpdir(), `slow-powers-context-test-${process.pid}`);
+
+function fixturePath(name: string): string {
+  return join(FIXTURE_ROOT, name);
+}
+
+function makeSkillDir(root: string, skills: string[]): string {
+  const dir = join(root, "skill-dir");
+  mkdirSync(dir, { recursive: true });
+  for (const name of skills) {
+    const sub = join(dir, name);
+    mkdirSync(sub, { recursive: true });
+    writeFileSync(
+      join(sub, "SKILL.md"),
+      `---\nname: ${name}\ndescription: ${name} skill\n---\n\nbody\n`,
+    );
+  }
+  return dir;
+}
+
+beforeAll(() => {
+  mkdirSync(FIXTURE_ROOT, { recursive: true });
+});
+
+afterAll(() => {
+  rmSync(FIXTURE_ROOT, { recursive: true, force: true });
+});
+
+describe("detectRunContext", () => {
+  test("dies when --skill-dir is missing", () => {
+    expect(() => detectRunContext(["--skill", "foo"])).toThrow(/--skill-dir/);
+  });
+
+  test("dies when --skill is missing", () => {
+    const root = fixturePath("missing-skill");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    expect(() => detectRunContext(["--skill-dir", skillDir])).toThrow(
+      /--skill/,
+    );
+  });
+
+  test("dies when --skill-dir is not a directory", () => {
+    expect(() =>
+      detectRunContext([
+        "--skill-dir",
+        "/nonexistent/does-not-exist-12345",
+        "--skill",
+        "foo",
+      ]),
+    ).toThrow(/--skill-dir/);
+  });
+
+  test("dies when skill subdir does not exist", () => {
+    const root = fixturePath("missing-subdir");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    expect(() =>
+      detectRunContext(["--skill-dir", skillDir, "--skill", "bar"]),
+    ).toThrow(/skill not found/);
+  });
+
+  test("dies when --bootstrap path is passed but file does not exist", () => {
+    const root = fixturePath("bad-bootstrap");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    expect(() =>
+      detectRunContext([
+        "--skill-dir",
+        skillDir,
+        "--skill",
+        "foo",
+        "--bootstrap",
+        "/nonexistent/no-bootstrap-12345.md",
+      ]),
+    ).toThrow(/--bootstrap/);
+  });
+
+  test("returns RunContext with absolute paths when --skill-dir and --skill are valid", () => {
+    const root = fixturePath("happy-path");
+    const skillDir = makeSkillDir(root, ["mr-review"]);
+    const ctx = detectRunContext([
+      "--skill-dir",
+      skillDir,
+      "--skill",
+      "mr-review",
+    ]);
+    expect(ctx.skillDir).toBe(resolve(skillDir));
+    expect(ctx.skillName).toBe("mr-review");
+    expect(ctx.skillSubdir).toBe(resolve(skillDir, "mr-review"));
+    expect(ctx.siblingSkillNames).toEqual([]);
+    expect(ctx.bootstrapPath).toBeNull();
+    expect(ctx.harness).toBe("claude-code");
+  });
+
+  test("enumerates siblings excluding the skill-under-test", () => {
+    const root = fixturePath("siblings");
+    const skillDir = makeSkillDir(root, ["alpha", "beta", "gamma"]);
+    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "beta"]);
+    expect(ctx.siblingSkillNames.sort()).toEqual(["alpha", "gamma"]);
+  });
+
+  test("ignores entries in --skill-dir that do not have a SKILL.md", () => {
+    const root = fixturePath("not-skills");
+    const skillDir = makeSkillDir(root, ["real"]);
+    mkdirSync(join(skillDir, "node_modules"), { recursive: true });
+    mkdirSync(join(skillDir, "no-skill-md-here"), { recursive: true });
+    writeFileSync(join(skillDir, "loose-file.txt"), "hello");
+    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "real"]);
+    expect(ctx.siblingSkillNames).toEqual([]);
+  });
+
+  test("workspaceRoot defaults to <CWD>/skills-workspace when --workspace-dir is omitted", () => {
+    const root = fixturePath("workspace-default");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "foo"]);
+    expect(ctx.workspaceRoot).toBe(resolve(process.cwd(), "skills-workspace"));
+  });
+
+  test("workspaceRoot honors --workspace-dir override (resolved absolute)", () => {
+    const root = fixturePath("workspace-override");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    const customWs = join(root, "custom-ws");
+    mkdirSync(customWs, { recursive: true });
+    const ctx = detectRunContext([
+      "--skill-dir",
+      skillDir,
+      "--skill",
+      "foo",
+      "--workspace-dir",
+      customWs,
+    ]);
+    expect(ctx.workspaceRoot).toBe(resolve(customWs));
+  });
+
+  test("stageRoot defaults to CWD", () => {
+    const root = fixturePath("stage-default");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "foo"]);
+    expect(ctx.stageRoot).toBe(resolve(process.cwd()));
+  });
+
+  test("--bootstrap path is resolved absolute when file exists", () => {
+    const root = fixturePath("bootstrap-ok");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    const bootstrapPath = join(root, "my-bootstrap.md");
+    writeFileSync(bootstrapPath, "BOOT");
+    const ctx = detectRunContext([
+      "--skill-dir",
+      skillDir,
+      "--skill",
+      "foo",
+      "--bootstrap",
+      bootstrapPath,
+    ]);
+    expect(ctx.bootstrapPath).toBe(resolve(bootstrapPath));
+  });
+
+  test("unknown --harness value is rejected", () => {
+    const root = fixturePath("harness-bad");
+    const skillDir = makeSkillDir(root, ["foo"]);
+    expect(() =>
+      detectRunContext([
+        "--skill-dir",
+        skillDir,
+        "--skill",
+        "foo",
+        "--harness",
+        "vscode",
+      ]),
+    ).toThrow(/harness/);
+  });
+});
+
+// Sanity: ensure existsSync helper from node:fs is what we expect
+test.skip("smoke: existsSync points at node:fs", () => {
+  expect(typeof existsSync).toBe("function");
+});

package/skills/evaluating-skills/runner/context.ts

@@ -0,0 +1,90 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { resolve } from "node:path";
+
+export type Harness = "claude-code";
+
+export type RunContext = {
+  skillDir: string;
+  skillName: string;
+  skillSubdir: string;
+  siblingSkillNames: string[];
+  workspaceRoot: string;
+  stageRoot: string;
+  bootstrapPath: string | null;
+  harness: Harness;
+};
+
+function die(msg: string): never {
+  throw new Error(msg);
+}
+
+function flag(argv: string[], name: string): string | undefined {
+  const i = argv.indexOf(`--${name}`);
+  if (i === -1) return undefined;
+  const v = argv[i + 1];
+  if (v === undefined || v.startsWith("--")) {
+    die(`flag --${name} requires a value`);
+  }
+  return v;
+}
+
+export function detectRunContext(argv: string[]): RunContext {
+  const skillDirRaw = flag(argv, "skill-dir");
+  if (!skillDirRaw) die("missing required flag --skill-dir <path>");
+  const skillDir = resolve(skillDirRaw);
+  if (!existsSync(skillDir) || !statSync(skillDir).isDirectory()) {
+    die(`--skill-dir is not a directory: ${skillDir}`);
+  }
+
+  const skillName = flag(argv, "skill");
+  if (!skillName) die("missing required flag --skill <name>");
+
+  const skillSubdir = resolve(skillDir, skillName);
+  const skillMd = resolve(skillSubdir, "SKILL.md");
+  if (!existsSync(skillMd)) {
+    die(`skill not found: ${skillMd}`);
+  }
+
+  const bootstrapRaw = flag(argv, "bootstrap");
+  let bootstrapPath: string | null = null;
+  if (bootstrapRaw) {
+    const resolved = resolve(bootstrapRaw);
+    if (!existsSync(resolved)) {
+      die(`--bootstrap file not found: ${resolved}`);
+    }
+    bootstrapPath = resolved;
+  }
+
+  const workspaceRaw = flag(argv, "workspace-dir");
+  const workspaceRoot = workspaceRaw
+    ? resolve(workspaceRaw)
+    : resolve(process.cwd(), "skills-workspace");
+
+  const stageRoot = resolve(process.cwd());
+
+  const harnessRaw = flag(argv, "harness") ?? "claude-code";
+  if (harnessRaw !== "claude-code") {
+    die(`unknown --harness: ${harnessRaw}. Supported: claude-code`);
+  }
+  const harness = harnessRaw as Harness;
+
+  const siblingSkillNames: string[] = [];
+  for (const entry of readdirSync(skillDir)) {
+    if (entry === skillName) continue;
+    const sub = resolve(skillDir, entry);
+    if (!statSync(sub).isDirectory()) continue;
+    if (!existsSync(resolve(sub, "SKILL.md"))) continue;
+    siblingSkillNames.push(entry);
+  }
+
+  return {
+    skillDir,
+    skillName,
+    skillSubdir,
+    siblingSkillNames,
+    workspaceRoot,
+    stageRoot,
+    bootstrapPath,
+    harness,
+  };
+}

package/skills/evaluating-skills/runner/detect-stray-writes.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, test } from "bun:test";
+import { join } from "node:path";
+import { detectStrayWrites } from "./detect-stray-writes";
+
+const OUTPUTS = "/work/iteration-1/eval-x/with_skill/outputs";
+const REPO = "/work/repo";
+
+describe("detectStrayWrites", () => {
+  test("a Write inside the outputs dir is clean", () => {
+    const findings = detectStrayWrites(
+      [
+        {
+          name: "Write",
+          args: { file_path: join(OUTPUTS, "answer.md") },
+          ordinal: 0,
+        },
+      ],
+      OUTPUTS,
+      REPO,
+    );
+    expect(findings.violations).toHaveLength(0);
+    expect(findings.warnings).toHaveLength(0);
+  });
+
+  test("a Write outside the outputs dir is a violation", () => {
+    const findings = detectStrayWrites(
+      [
+        {
+          name: "Write",
+          args: { file_path: join(REPO, "runner/run.ts") },
+          ordinal: 2,
+        },
+      ],
+      OUTPUTS,
+      REPO,
+    );
+    expect(findings.violations).toHaveLength(1);
+    expect(findings.violations[0]).toMatchObject({
+      tool: "Write",
+      path: join(REPO, "runner/run.ts"),
+      ordinal: 2,
+    });
+  });
+
+  test("an Edit/MultiEdit/NotebookEdit outside outputs is a violation", () => {
+    const findings = detectStrayWrites(
+      [
+        { name: "Edit", args: { file_path: "/etc/hosts" }, ordinal: 0 },
+        {
+          name: "NotebookEdit",
+          args: { notebook_path: "/tmp/x.ipynb" },
+          ordinal: 1,
+        },
+      ],
+      OUTPUTS,
+      REPO,
+    );
+    expect(findings.violations.map((v) => v.tool).sort()).toEqual([
+      "Edit",
+      "NotebookEdit",
+    ]);
+  });
+
+  test("an install command is a warning", () => {
+    const findings = detectStrayWrites(
+      [{ name: "Bash", args: { command: "npm install left-pad" }, ordinal: 0 }],
+      OUTPUTS,
+      REPO,
+    );
+    expect(findings.warnings).toHaveLength(1);
+    expect(findings.warnings[0].tool).toBe("Bash");
+    expect(findings.warnings[0].reason).toMatch(/install/i);
+  });
+
+  test("a mutating Bash command scoped to the outputs dir is not flagged", () => {
+    const findings = detectStrayWrites(
+      [
+        {
+          name: "Bash",
+          args: { command: `echo hi > ${join(OUTPUTS, "log.txt")}` },
+          ordinal: 0,
+        },
+      ],
+      OUTPUTS,
+      REPO,
+    );
+    expect(findings.warnings).toHaveLength(0);
+  });
+
+  test("read-only tools are never flagged", () => {
+    const findings = detectStrayWrites(
+      [
+        { name: "Read", args: { file_path: "/anywhere" }, ordinal: 0 },
+        { name: "Grep", args: { pattern: "x" }, ordinal: 1 },
+        { name: "Bash", args: { command: "ls -la /" }, ordinal: 2 },
+      ],
+      OUTPUTS,
+      REPO,
+    );
+    expect(findings.violations).toHaveLength(0);
+    expect(findings.warnings).toHaveLength(0);
+  });
+});