@slowdini/slow-powers-opencode 0.2.0 → 0.4.0

package/skills/evaluating-skills/runner/detect-stray-writes.ts

@@ -1,192 +0,0 @@
-#!/usr/bin/env bun
-import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import { detectRunContext } from "./context";
-import { classifyBash, isUnder, pathArg, WRITE_TOOLS } from "./sandbox-policy";
-import type { ConditionsRecord, RunRecord, ToolInvocation } from "./types";
-import { validateAgainstSchema } from "./validate-schema";
-
-function die(msg: string): never {
-  console.error(`error: ${msg}`);
-  process.exit(1);
-}
-
-export type StrayFinding = {
-  tool: string;
-  path?: string;
-  command?: string;
-  ordinal: number;
-  reason: string;
-};
-
-export type RunFindings = {
-  violations: StrayFinding[];
-  warnings: StrayFinding[];
-};
-
-/**
- * Classify a run's tool invocations against its allowed outputs dir.
- *
- * - `violations`: file-write tools (Write/Edit/MultiEdit/NotebookEdit) whose
- *   target path resolves outside `outputsDir`. High confidence — a run that
- *   edits the real repo is a tainted data point.
- * - `warnings`: Bash commands matching a mutating pattern that don't reference
- *   `outputsDir`. Heuristic — review before trusting.
- *
- * Relative paths resolve against `repoRoot` (the subagent's working dir);
- * Claude Code's write tools use absolute paths, so this is a best-effort
- * fallback only.
- */
-export function detectStrayWrites(
-  invocations: Array<Pick<ToolInvocation, "name" | "args" | "ordinal">>,
-  outputsDir: string,
-  repoRoot: string,
-): RunFindings {
-  const violations: StrayFinding[] = [];
-  const warnings: StrayFinding[] = [];
-
-  for (const inv of invocations) {
-    if (WRITE_TOOLS.has(inv.name)) {
-      const p = pathArg(inv.args);
-      if (p && !isUnder(p, outputsDir, repoRoot)) {
-        violations.push({
-          tool: inv.name,
-          path: p,
-          ordinal: inv.ordinal,
-          reason: "writes outside the run's outputs dir",
-        });
-      }
-      continue;
-    }
-
-    if (inv.name === "Bash") {
-      const args = inv.args as { command?: unknown } | undefined;
-      const command = typeof args?.command === "string" ? args.command : "";
-      const reason = classifyBash(command, [outputsDir]);
-      if (reason)
-        warnings.push({ tool: "Bash", command, ordinal: inv.ordinal, reason });
-    }
-  }
-
-  return { violations, warnings };
-}
-
-if (import.meta.main) {
-  const argv = Bun.argv.slice(2);
-  const flag = (name: string): string | undefined => {
-    const i = argv.indexOf(`--${name}`);
-    return i === -1 ? undefined : argv[i + 1];
-  };
-  const iteration = flag("iteration");
-  if (!iteration) die("missing --iteration");
-  const ctx = detectRunContext(argv);
-
-  const iterationDir = join(
-    ctx.workspaceRoot,
-    ctx.skillName,
-    `iteration-${iteration}`,
-  );
-  if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
-
-  const conditionsPath = join(iterationDir, "conditions.json");
-  if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
-  const conditions: ConditionsRecord = JSON.parse(
-    readFileSync(conditionsPath, "utf8"),
-  );
-  const conditionNames = conditions.conditions.map((c) => c.name);
-
-  // dispatch.json carries the authoritative outputs_dir per task; fall back to
-  // the conventional <condDir>/outputs when it's absent (hand-authored runs).
-  const dispatchPath = join(iterationDir, "dispatch.json");
-  const outputsByKey = new Map<string, string>();
-  if (existsSync(dispatchPath)) {
-    try {
-      const dispatch = JSON.parse(readFileSync(dispatchPath, "utf8")) as {
-        tasks?: Array<{
-          eval_id: string;
-          condition: string;
-          outputs_dir?: string;
-        }>;
-      };
-      for (const t of dispatch.tasks ?? []) {
-        if (t.outputs_dir)
-          outputsByKey.set(`${t.eval_id}:${t.condition}`, t.outputs_dir);
-      }
-    } catch {
-      // fall through to convention
-    }
-  }
-
-  const repoRoot = process.cwd();
-  const evalDirs = readdirSync(iterationDir).filter((d) =>
-    d.startsWith("eval-"),
-  );
-
-  type RunReport = {
-    eval_id: string;
-    condition: string;
-    violations: StrayFinding[];
-    warnings: StrayFinding[];
-  };
-  const runs: RunReport[] = [];
-  let totalViolations = 0;
-  let totalWarnings = 0;
-
-  for (const evalDir of evalDirs) {
-    const evalId = evalDir.replace(/^eval-/, "");
-    for (const cond of conditionNames) {
-      const condDir = join(iterationDir, evalDir, cond);
-      const runPath = join(condDir, "run.json");
-      if (!existsSync(runPath)) continue;
-      const run = validateAgainstSchema<RunRecord>(
-        "run-record",
-        JSON.parse(readFileSync(runPath, "utf8")),
-        runPath,
-      );
-      const invocations = Array.isArray(run.tool_invocations)
-        ? run.tool_invocations
-        : [];
-      const outputsDir =
-        outputsByKey.get(`${evalId}:${cond}`) ?? join(condDir, "outputs");
-      const findings = detectStrayWrites(invocations, outputsDir, repoRoot);
-      if (findings.violations.length || findings.warnings.length) {
-        runs.push({
-          eval_id: evalId,
-          condition: cond,
-          violations: findings.violations,
-          warnings: findings.warnings,
-        });
-      }
-      totalViolations += findings.violations.length;
-      totalWarnings += findings.warnings.length;
-    }
-  }
-
-  const report = {
-    generated: new Date().toISOString(),
-    iteration: Number(iteration),
-    totals: { violations: totalViolations, warnings: totalWarnings },
-    runs,
-  };
-  const outPath = join(iterationDir, "stray-writes.json");
-  validateAgainstSchema("stray-writes", report, outPath);
-  writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`);
-  console.log(`Wrote ${outPath}`);
-
-  for (const r of runs) {
-    for (const v of r.violations)
-      console.warn(
-        `✗ ${r.eval_id}/${r.condition}: ${v.tool} wrote outside outputs dir → ${v.path} (ordinal ${v.ordinal})`,
-      );
-    for (const w of r.warnings)
-      console.warn(
-        `⚠ ${r.eval_id}/${r.condition}: Bash ${w.reason} (ordinal ${w.ordinal}): ${w.command}`,
-      );
-  }
-  if (totalViolations === 0 && totalWarnings === 0)
-    console.log("✓ No out-of-bounds writes detected.");
-  else
-    console.warn(
-      `\n${totalViolations} violation(s), ${totalWarnings} warning(s). Runs with violations edited files outside their sandbox — treat those data points as tainted.`,
-    );
-}

package/skills/evaluating-skills/runner/fill-transcripts.test.ts

@@ -1,73 +0,0 @@
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import { mkdirSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { resolveAgentDescription } from "./fill-transcripts";
-
-const ROOT = join(tmpdir(), `fill-transcripts-test-${process.pid}`);
-
-beforeAll(() => mkdirSync(ROOT, { recursive: true }));
-afterAll(() => rmSync(ROOT, { recursive: true, force: true }));
-
-function writeDispatch(iterationDir: string, tasks: unknown[]) {
-  mkdirSync(iterationDir, { recursive: true });
-  writeFileSync(
-    join(iterationDir, "dispatch.json"),
-    JSON.stringify({ run_nonce: "abc123", tasks }, null, 2),
-  );
-}
-
-describe("resolveAgentDescription", () => {
-  test("returns the namespaced agent_description from dispatch.json", () => {
-    const dir = join(ROOT, "iter-canonical");
-    writeDispatch(dir, [
-      {
-        eval_id: "crash",
-        condition: "with_skill",
-        agent_description: "crash:with_skill:i3-abc123",
-      },
-      {
-        eval_id: "crash",
-        condition: "without_skill",
-        agent_description: "crash:without_skill:i3-abc123",
-      },
-    ]);
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill:i3-abc123",
-    );
-    expect(resolveAgentDescription(dir, "crash", "without_skill")).toBe(
-      "crash:without_skill:i3-abc123",
-    );
-  });
-
-  test("falls back to legacy reconstruction when dispatch.json is absent", () => {
-    const dir = join(ROOT, "iter-no-dispatch");
-    mkdirSync(dir, { recursive: true });
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill",
-    );
-  });
-
-  test("falls back when the task is missing from dispatch.json", () => {
-    const dir = join(ROOT, "iter-partial");
-    writeDispatch(dir, [
-      {
-        eval_id: "other",
-        condition: "with_skill",
-        agent_description: "other:with_skill:i1-x",
-      },
-    ]);
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill",
-    );
-  });
-
-  test("falls back when dispatch.json is malformed", () => {
-    const dir = join(ROOT, "iter-malformed");
-    mkdirSync(dir, { recursive: true });
-    writeFileSync(join(dir, "dispatch.json"), "{ not valid json");
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill",
-    );
-  });
-});

package/skills/evaluating-skills/runner/fill-transcripts.ts

@@ -1,154 +0,0 @@
-#!/usr/bin/env bun
-import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import * as claudeAdapter from "./adapters/claude-code-transcript";
-import { detectRunContext } from "./context";
-import type { ConditionsRecord, RunRecord } from "./types";
-import { validateAgainstSchema } from "./validate-schema";
-
-function die(msg: string): never {
-  console.error(`error: ${msg}`);
-  process.exit(1);
-}
-
-type DispatchTaskRef = {
-  eval_id: string;
-  condition: string;
-  agent_description?: string;
-};
-
-/**
- * The canonical dispatch description for an (eval, condition) run.
- *
- * The runner writes a unique `agent_description` per task into `dispatch.json`
- * (namespaced with the iteration + run nonce). Reading it back — rather than
- * reconstructing `<eval_id>:<condition>` — is what binds each run to the exact
- * agent that produced it, even when one parent session's shared subagents dir
- * holds colliding descriptions from other iterations. Falls back to the legacy
- * reconstruction when dispatch.json is absent (hand-authored/operator runs).
- */
-export function resolveAgentDescription(
-  iterationDir: string,
-  evalId: string,
-  condition: string,
-): string {
-  const dispatchPath = join(iterationDir, "dispatch.json");
-  if (existsSync(dispatchPath)) {
-    try {
-      const dispatch = JSON.parse(readFileSync(dispatchPath, "utf8")) as {
-        tasks?: DispatchTaskRef[];
-      };
-      const task = dispatch.tasks?.find(
-        (t) => t.eval_id === evalId && t.condition === condition,
-      );
-      if (task?.agent_description) return task.agent_description;
-    } catch {
-      // fall through to legacy reconstruction
-    }
-  }
-  return `${evalId}:${condition}`;
-}
-
-function parseArgs(argv: string[]) {
-  const flag = (name: string): string | undefined => {
-    const i = argv.indexOf(`--${name}`);
-    if (i === -1) return undefined;
-    return argv[i + 1];
-  };
-  const has = (name: string) => argv.includes(`--${name}`);
-  const iteration = flag("iteration");
-  const subagentsDir = flag("subagents-dir");
-  const overwrite = has("overwrite");
-  if (!iteration) die("missing --iteration");
-  if (!subagentsDir)
-    die(
-      "missing --subagents-dir (e.g. ~/.claude/projects/<project-slug>/<parent-session-id>/subagents/)",
-    );
-  return { iteration, subagentsDir, overwrite };
-}
-
-if (import.meta.main) {
-  const fillArgv = Bun.argv.slice(2);
-  const { iteration, subagentsDir, overwrite } = parseArgs(fillArgv);
-  const fillCtx = detectRunContext(fillArgv);
-  const skill = fillCtx.skillName;
-
-  if (!existsSync(subagentsDir))
-    die(`subagents-dir not found: ${subagentsDir}`);
-
-  const adapter = claudeAdapter;
-  console.log("Using harness transcript adapter: claude-code");
-
-  const iterationDir = join(
-    fillCtx.workspaceRoot,
-    skill,
-    `iteration-${iteration}`,
-  );
-  if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
-
-  const conditionsPath = join(iterationDir, "conditions.json");
-  if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
-  const conditions: ConditionsRecord = JSON.parse(
-    readFileSync(conditionsPath, "utf8"),
-  );
-  const conditionNames = conditions.conditions.map((c) => c.name);
-
-  const evalDirs = readdirSync(iterationDir).filter((d) =>
-    d.startsWith("eval-"),
-  );
-
-  let filled = 0;
-  let skipped = 0;
-  let missing = 0;
-
-  for (const evalDir of evalDirs) {
-    const evalId = evalDir.replace(/^eval-/, "");
-    for (const cond of conditionNames) {
-      const condDir = join(iterationDir, evalDir, cond);
-      const runPath = join(condDir, "run.json");
-      if (!existsSync(runPath)) continue;
-
-      const run = validateAgainstSchema<RunRecord>(
-        "run-record",
-        JSON.parse(readFileSync(runPath, "utf8")),
-        runPath,
-      );
-      const existing = Array.isArray(run.tool_invocations)
-        ? run.tool_invocations
-        : [];
-      if (existing.length > 0 && !overwrite) {
-        console.log(
-          `skip ${evalId}/${cond}: already has ${existing.length} tool_invocations (use --overwrite to replace)`,
-        );
-        skipped++;
-        continue;
-      }
-
-      const description = resolveAgentDescription(iterationDir, evalId, cond);
-      const subagent = adapter.findByDescription(subagentsDir, description);
-      if (!subagent) {
-        console.warn(
-          `miss ${evalId}/${cond}: no subagent transcript with description='${description}'`,
-        );
-        missing++;
-        continue;
-      }
-
-      const invocations = adapter.parseTranscript(subagent.jsonlPath);
-      run.tool_invocations = invocations;
-      writeFileSync(runPath, `${JSON.stringify(run, null, 2)}\n`);
-      console.log(
-        `fill ${evalId}/${cond}: wrote ${invocations.length} tool_invocations from ${subagent.jsonlPath}`,
-      );
-      filled++;
-    }
-  }
-
-  console.log(
-    `\nFilled: ${filled}, skipped (already populated): ${skipped}, missing transcript: ${missing}`,
-  );
-  if (missing > 0)
-    console.warn(
-      "Missing transcripts mean the dispatching agent's dispatch `description` did not match the task's `agent_description` in dispatch.json (or dispatch.json is absent and the legacy `eval-id:condition` reconstruction found no match). transcript_check assertions for those runs will be graded unverifiable.",
-    );
-}