@slowdini/slow-powers-opencode 0.1.0

package/skills/evaluating-skills/runner/detect-stray-writes.ts

@@ -0,0 +1,192 @@
+#!/usr/bin/env bun
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { detectRunContext } from "./context";
+import { classifyBash, isUnder, pathArg, WRITE_TOOLS } from "./sandbox-policy";
+import type { ConditionsRecord, RunRecord, ToolInvocation } from "./types";
+import { validateAgainstSchema } from "./validate-schema";
+
+function die(msg: string): never {
+  console.error(`error: ${msg}`);
+  process.exit(1);
+}
+
+export type StrayFinding = {
+  tool: string;
+  path?: string;
+  command?: string;
+  ordinal: number;
+  reason: string;
+};
+
+export type RunFindings = {
+  violations: StrayFinding[];
+  warnings: StrayFinding[];
+};
+
+/**
+ * Classify a run's tool invocations against its allowed outputs dir.
+ *
+ * - `violations`: file-write tools (Write/Edit/MultiEdit/NotebookEdit) whose
+ *   target path resolves outside `outputsDir`. High confidence — a run that
+ *   edits the real repo is a tainted data point.
+ * - `warnings`: Bash commands matching a mutating pattern that don't reference
+ *   `outputsDir`. Heuristic — review before trusting.
+ *
+ * Relative paths resolve against `repoRoot` (the subagent's working dir);
+ * Claude Code's write tools use absolute paths, so this is a best-effort
+ * fallback only.
+ */
+export function detectStrayWrites(
+  invocations: Array<Pick<ToolInvocation, "name" | "args" | "ordinal">>,
+  outputsDir: string,
+  repoRoot: string,
+): RunFindings {
+  const violations: StrayFinding[] = [];
+  const warnings: StrayFinding[] = [];
+
+  for (const inv of invocations) {
+    if (WRITE_TOOLS.has(inv.name)) {
+      const p = pathArg(inv.args);
+      if (p && !isUnder(p, outputsDir, repoRoot)) {
+        violations.push({
+          tool: inv.name,
+          path: p,
+          ordinal: inv.ordinal,
+          reason: "writes outside the run's outputs dir",
+        });
+      }
+      continue;
+    }
+
+    if (inv.name === "Bash") {
+      const args = inv.args as { command?: unknown } | undefined;
+      const command = typeof args?.command === "string" ? args.command : "";
+      const reason = classifyBash(command, [outputsDir]);
+      if (reason)
+        warnings.push({ tool: "Bash", command, ordinal: inv.ordinal, reason });
+    }
+  }
+
+  return { violations, warnings };
+}
+
+if (import.meta.main) {
+  const argv = Bun.argv.slice(2);
+  const flag = (name: string): string | undefined => {
+    const i = argv.indexOf(`--${name}`);
+    return i === -1 ? undefined : argv[i + 1];
+  };
+  const iteration = flag("iteration");
+  if (!iteration) die("missing --iteration");
+  const ctx = detectRunContext(argv);
+
+  const iterationDir = join(
+    ctx.workspaceRoot,
+    ctx.skillName,
+    `iteration-${iteration}`,
+  );
+  if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
+
+  const conditionsPath = join(iterationDir, "conditions.json");
+  if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
+  const conditions: ConditionsRecord = JSON.parse(
+    readFileSync(conditionsPath, "utf8"),
+  );
+  const conditionNames = conditions.conditions.map((c) => c.name);
+
+  // dispatch.json carries the authoritative outputs_dir per task; fall back to
+  // the conventional <condDir>/outputs when it's absent (hand-authored runs).
+  const dispatchPath = join(iterationDir, "dispatch.json");
+  const outputsByKey = new Map<string, string>();
+  if (existsSync(dispatchPath)) {
+    try {
+      const dispatch = JSON.parse(readFileSync(dispatchPath, "utf8")) as {
+        tasks?: Array<{
+          eval_id: string;
+          condition: string;
+          outputs_dir?: string;
+        }>;
+      };
+      for (const t of dispatch.tasks ?? []) {
+        if (t.outputs_dir)
+          outputsByKey.set(`${t.eval_id}:${t.condition}`, t.outputs_dir);
+      }
+    } catch {
+      // fall through to convention
+    }
+  }
+
+  const repoRoot = process.cwd();
+  const evalDirs = readdirSync(iterationDir).filter((d) =>
+    d.startsWith("eval-"),
+  );
+
+  type RunReport = {
+    eval_id: string;
+    condition: string;
+    violations: StrayFinding[];
+    warnings: StrayFinding[];
+  };
+  const runs: RunReport[] = [];
+  let totalViolations = 0;
+  let totalWarnings = 0;
+
+  for (const evalDir of evalDirs) {
+    const evalId = evalDir.replace(/^eval-/, "");
+    for (const cond of conditionNames) {
+      const condDir = join(iterationDir, evalDir, cond);
+      const runPath = join(condDir, "run.json");
+      if (!existsSync(runPath)) continue;
+      const run = validateAgainstSchema<RunRecord>(
+        "run-record",
+        JSON.parse(readFileSync(runPath, "utf8")),
+        runPath,
+      );
+      const invocations = Array.isArray(run.tool_invocations)
+        ? run.tool_invocations
+        : [];
+      const outputsDir =
+        outputsByKey.get(`${evalId}:${cond}`) ?? join(condDir, "outputs");
+      const findings = detectStrayWrites(invocations, outputsDir, repoRoot);
+      if (findings.violations.length || findings.warnings.length) {
+        runs.push({
+          eval_id: evalId,
+          condition: cond,
+          violations: findings.violations,
+          warnings: findings.warnings,
+        });
+      }
+      totalViolations += findings.violations.length;
+      totalWarnings += findings.warnings.length;
+    }
+  }
+
+  const report = {
+    generated: new Date().toISOString(),
+    iteration: Number(iteration),
+    totals: { violations: totalViolations, warnings: totalWarnings },
+    runs,
+  };
+  const outPath = join(iterationDir, "stray-writes.json");
+  validateAgainstSchema("stray-writes", report, outPath);
+  writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`);
+  console.log(`Wrote ${outPath}`);
+
+  for (const r of runs) {
+    for (const v of r.violations)
+      console.warn(
+        `✗ ${r.eval_id}/${r.condition}: ${v.tool} wrote outside outputs dir → ${v.path} (ordinal ${v.ordinal})`,
+      );
+    for (const w of r.warnings)
+      console.warn(
+        `⚠ ${r.eval_id}/${r.condition}: Bash ${w.reason} (ordinal ${w.ordinal}): ${w.command}`,
+      );
+  }
+  if (totalViolations === 0 && totalWarnings === 0)
+    console.log("✓ No out-of-bounds writes detected.");
+  else
+    console.warn(
+      `\n${totalViolations} violation(s), ${totalWarnings} warning(s). Runs with violations edited files outside their sandbox — treat those data points as tainted.`,
+    );
+}

package/skills/evaluating-skills/runner/fill-transcripts.test.ts

@@ -0,0 +1,73 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { resolveAgentDescription } from "./fill-transcripts";
+
+const ROOT = join(tmpdir(), `fill-transcripts-test-${process.pid}`);
+
+beforeAll(() => mkdirSync(ROOT, { recursive: true }));
+afterAll(() => rmSync(ROOT, { recursive: true, force: true }));
+
+function writeDispatch(iterationDir: string, tasks: unknown[]) {
+  mkdirSync(iterationDir, { recursive: true });
+  writeFileSync(
+    join(iterationDir, "dispatch.json"),
+    JSON.stringify({ run_nonce: "abc123", tasks }, null, 2),
+  );
+}
+
+describe("resolveAgentDescription", () => {
+  test("returns the namespaced agent_description from dispatch.json", () => {
+    const dir = join(ROOT, "iter-canonical");
+    writeDispatch(dir, [
+      {
+        eval_id: "crash",
+        condition: "with_skill",
+        agent_description: "crash:with_skill:i3-abc123",
+      },
+      {
+        eval_id: "crash",
+        condition: "without_skill",
+        agent_description: "crash:without_skill:i3-abc123",
+      },
+    ]);
+    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
+      "crash:with_skill:i3-abc123",
+    );
+    expect(resolveAgentDescription(dir, "crash", "without_skill")).toBe(
+      "crash:without_skill:i3-abc123",
+    );
+  });
+
+  test("falls back to legacy reconstruction when dispatch.json is absent", () => {
+    const dir = join(ROOT, "iter-no-dispatch");
+    mkdirSync(dir, { recursive: true });
+    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
+      "crash:with_skill",
+    );
+  });
+
+  test("falls back when the task is missing from dispatch.json", () => {
+    const dir = join(ROOT, "iter-partial");
+    writeDispatch(dir, [
+      {
+        eval_id: "other",
+        condition: "with_skill",
+        agent_description: "other:with_skill:i1-x",
+      },
+    ]);
+    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
+      "crash:with_skill",
+    );
+  });
+
+  test("falls back when dispatch.json is malformed", () => {
+    const dir = join(ROOT, "iter-malformed");
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, "dispatch.json"), "{ not valid json");
+    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
+      "crash:with_skill",
+    );
+  });
+});

package/skills/evaluating-skills/runner/fill-transcripts.ts

@@ -0,0 +1,154 @@
+#!/usr/bin/env bun
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import * as claudeAdapter from "./adapters/claude-code-transcript";
+import { detectRunContext } from "./context";
+import type { ConditionsRecord, RunRecord } from "./types";
+import { validateAgainstSchema } from "./validate-schema";
+
+function die(msg: string): never {
+  console.error(`error: ${msg}`);
+  process.exit(1);
+}
+
+type DispatchTaskRef = {
+  eval_id: string;
+  condition: string;
+  agent_description?: string;
+};
+
+/**
+ * The canonical dispatch description for an (eval, condition) run.
+ *
+ * The runner writes a unique `agent_description` per task into `dispatch.json`
+ * (namespaced with the iteration + run nonce). Reading it back — rather than
+ * reconstructing `<eval_id>:<condition>` — is what binds each run to the exact
+ * agent that produced it, even when one parent session's shared subagents dir
+ * holds colliding descriptions from other iterations. Falls back to the legacy
+ * reconstruction when dispatch.json is absent (hand-authored/operator runs).
+ */
+export function resolveAgentDescription(
+  iterationDir: string,
+  evalId: string,
+  condition: string,
+): string {
+  const dispatchPath = join(iterationDir, "dispatch.json");
+  if (existsSync(dispatchPath)) {
+    try {
+      const dispatch = JSON.parse(readFileSync(dispatchPath, "utf8")) as {
+        tasks?: DispatchTaskRef[];
+      };
+      const task = dispatch.tasks?.find(
+        (t) => t.eval_id === evalId && t.condition === condition,
+      );
+      if (task?.agent_description) return task.agent_description;
+    } catch {
+      // fall through to legacy reconstruction
+    }
+  }
+  return `${evalId}:${condition}`;
+}
+
+function parseArgs(argv: string[]) {
+  const flag = (name: string): string | undefined => {
+    const i = argv.indexOf(`--${name}`);
+    if (i === -1) return undefined;
+    return argv[i + 1];
+  };
+  const has = (name: string) => argv.includes(`--${name}`);
+  const iteration = flag("iteration");
+  const subagentsDir = flag("subagents-dir");
+  const overwrite = has("overwrite");
+  if (!iteration) die("missing --iteration");
+  if (!subagentsDir)
+    die(
+      "missing --subagents-dir (e.g. ~/.claude/projects/<project-slug>/<parent-session-id>/subagents/)",
+    );
+  return { iteration, subagentsDir, overwrite };
+}
+
+if (import.meta.main) {
+  const fillArgv = Bun.argv.slice(2);
+  const { iteration, subagentsDir, overwrite } = parseArgs(fillArgv);
+  const fillCtx = detectRunContext(fillArgv);
+  const skill = fillCtx.skillName;
+
+  if (!existsSync(subagentsDir))
+    die(`subagents-dir not found: ${subagentsDir}`);
+
+  const adapter = claudeAdapter;
+  console.log("Using harness transcript adapter: claude-code");
+
+  const iterationDir = join(
+    fillCtx.workspaceRoot,
+    skill,
+    `iteration-${iteration}`,
+  );
+  if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
+
+  const conditionsPath = join(iterationDir, "conditions.json");
+  if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
+  const conditions: ConditionsRecord = JSON.parse(
+    readFileSync(conditionsPath, "utf8"),
+  );
+  const conditionNames = conditions.conditions.map((c) => c.name);
+
+  const evalDirs = readdirSync(iterationDir).filter((d) =>
+    d.startsWith("eval-"),
+  );
+
+  let filled = 0;
+  let skipped = 0;
+  let missing = 0;
+
+  for (const evalDir of evalDirs) {
+    const evalId = evalDir.replace(/^eval-/, "");
+    for (const cond of conditionNames) {
+      const condDir = join(iterationDir, evalDir, cond);
+      const runPath = join(condDir, "run.json");
+      if (!existsSync(runPath)) continue;
+
+      const run = validateAgainstSchema<RunRecord>(
+        "run-record",
+        JSON.parse(readFileSync(runPath, "utf8")),
+        runPath,
+      );
+      const existing = Array.isArray(run.tool_invocations)
+        ? run.tool_invocations
+        : [];
+      if (existing.length > 0 && !overwrite) {
+        console.log(
+          `skip ${evalId}/${cond}: already has ${existing.length} tool_invocations (use --overwrite to replace)`,
+        );
+        skipped++;
+        continue;
+      }
+
+      const description = resolveAgentDescription(iterationDir, evalId, cond);
+      const subagent = adapter.findByDescription(subagentsDir, description);
+      if (!subagent) {
+        console.warn(
+          `miss ${evalId}/${cond}: no subagent transcript with description='${description}'`,
+        );
+        missing++;
+        continue;
+      }
+
+      const invocations = adapter.parseTranscript(subagent.jsonlPath);
+      run.tool_invocations = invocations;
+      writeFileSync(runPath, `${JSON.stringify(run, null, 2)}\n`);
+      console.log(
+        `fill ${evalId}/${cond}: wrote ${invocations.length} tool_invocations from ${subagent.jsonlPath}`,
+      );
+      filled++;
+    }
+  }
+
+  console.log(
+    `\nFilled: ${filled}, skipped (already populated): ${skipped}, missing transcript: ${missing}`,
+  );
+  if (missing > 0)
+    console.warn(
+      "Missing transcripts mean the dispatching agent's dispatch `description` did not match the task's `agent_description` in dispatch.json (or dispatch.json is absent and the legacy `eval-id:condition` reconstruction found no match). transcript_check assertions for those runs will be graded unverifiable.",
+    );
+}