@slowdini/slow-powers-opencode 0.3.0 → 0.4.0

package/skills/evaluating-skills/runner/fill-transcripts.test.ts

@@ -1,73 +0,0 @@
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import { mkdirSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { resolveAgentDescription } from "./fill-transcripts";
-
-const ROOT = join(tmpdir(), `fill-transcripts-test-${process.pid}`);
-
-beforeAll(() => mkdirSync(ROOT, { recursive: true }));
-afterAll(() => rmSync(ROOT, { recursive: true, force: true }));
-
-function writeDispatch(iterationDir: string, tasks: unknown[]) {
-  mkdirSync(iterationDir, { recursive: true });
-  writeFileSync(
-    join(iterationDir, "dispatch.json"),
-    JSON.stringify({ run_nonce: "abc123", tasks }, null, 2),
-  );
-}
-
-describe("resolveAgentDescription", () => {
-  test("returns the namespaced agent_description from dispatch.json", () => {
-    const dir = join(ROOT, "iter-canonical");
-    writeDispatch(dir, [
-      {
-        eval_id: "crash",
-        condition: "with_skill",
-        agent_description: "crash:with_skill:i3-abc123",
-      },
-      {
-        eval_id: "crash",
-        condition: "without_skill",
-        agent_description: "crash:without_skill:i3-abc123",
-      },
-    ]);
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill:i3-abc123",
-    );
-    expect(resolveAgentDescription(dir, "crash", "without_skill")).toBe(
-      "crash:without_skill:i3-abc123",
-    );
-  });
-
-  test("falls back to legacy reconstruction when dispatch.json is absent", () => {
-    const dir = join(ROOT, "iter-no-dispatch");
-    mkdirSync(dir, { recursive: true });
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill",
-    );
-  });
-
-  test("falls back when the task is missing from dispatch.json", () => {
-    const dir = join(ROOT, "iter-partial");
-    writeDispatch(dir, [
-      {
-        eval_id: "other",
-        condition: "with_skill",
-        agent_description: "other:with_skill:i1-x",
-      },
-    ]);
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill",
-    );
-  });
-
-  test("falls back when dispatch.json is malformed", () => {
-    const dir = join(ROOT, "iter-malformed");
-    mkdirSync(dir, { recursive: true });
-    writeFileSync(join(dir, "dispatch.json"), "{ not valid json");
-    expect(resolveAgentDescription(dir, "crash", "with_skill")).toBe(
-      "crash:with_skill",
-    );
-  });
-});

package/skills/evaluating-skills/runner/fill-transcripts.ts

@@ -1,154 +0,0 @@
-#!/usr/bin/env bun
-import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import * as claudeAdapter from "./adapters/claude-code-transcript";
-import { detectRunContext } from "./context";
-import type { ConditionsRecord, RunRecord } from "./types";
-import { validateAgainstSchema } from "./validate-schema";
-
-function die(msg: string): never {
-  console.error(`error: ${msg}`);
-  process.exit(1);
-}
-
-type DispatchTaskRef = {
-  eval_id: string;
-  condition: string;
-  agent_description?: string;
-};
-
-/**
- * The canonical dispatch description for an (eval, condition) run.
- *
- * The runner writes a unique `agent_description` per task into `dispatch.json`
- * (namespaced with the iteration + run nonce). Reading it back — rather than
- * reconstructing `<eval_id>:<condition>` — is what binds each run to the exact
- * agent that produced it, even when one parent session's shared subagents dir
- * holds colliding descriptions from other iterations. Falls back to the legacy
- * reconstruction when dispatch.json is absent (hand-authored/operator runs).
- */
-export function resolveAgentDescription(
-  iterationDir: string,
-  evalId: string,
-  condition: string,
-): string {
-  const dispatchPath = join(iterationDir, "dispatch.json");
-  if (existsSync(dispatchPath)) {
-    try {
-      const dispatch = JSON.parse(readFileSync(dispatchPath, "utf8")) as {
-        tasks?: DispatchTaskRef[];
-      };
-      const task = dispatch.tasks?.find(
-        (t) => t.eval_id === evalId && t.condition === condition,
-      );
-      if (task?.agent_description) return task.agent_description;
-    } catch {
-      // fall through to legacy reconstruction
-    }
-  }
-  return `${evalId}:${condition}`;
-}
-
-function parseArgs(argv: string[]) {
-  const flag = (name: string): string | undefined => {
-    const i = argv.indexOf(`--${name}`);
-    if (i === -1) return undefined;
-    return argv[i + 1];
-  };
-  const has = (name: string) => argv.includes(`--${name}`);
-  const iteration = flag("iteration");
-  const subagentsDir = flag("subagents-dir");
-  const overwrite = has("overwrite");
-  if (!iteration) die("missing --iteration");
-  if (!subagentsDir)
-    die(
-      "missing --subagents-dir (e.g. ~/.claude/projects/<project-slug>/<parent-session-id>/subagents/)",
-    );
-  return { iteration, subagentsDir, overwrite };
-}
-
-if (import.meta.main) {
-  const fillArgv = Bun.argv.slice(2);
-  const { iteration, subagentsDir, overwrite } = parseArgs(fillArgv);
-  const fillCtx = detectRunContext(fillArgv);
-  const skill = fillCtx.skillName;
-
-  if (!existsSync(subagentsDir))
-    die(`subagents-dir not found: ${subagentsDir}`);
-
-  const adapter = claudeAdapter;
-  console.log("Using harness transcript adapter: claude-code");
-
-  const iterationDir = join(
-    fillCtx.workspaceRoot,
-    skill,
-    `iteration-${iteration}`,
-  );
-  if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
-
-  const conditionsPath = join(iterationDir, "conditions.json");
-  if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
-  const conditions: ConditionsRecord = JSON.parse(
-    readFileSync(conditionsPath, "utf8"),
-  );
-  const conditionNames = conditions.conditions.map((c) => c.name);
-
-  const evalDirs = readdirSync(iterationDir).filter((d) =>
-    d.startsWith("eval-"),
-  );
-
-  let filled = 0;
-  let skipped = 0;
-  let missing = 0;
-
-  for (const evalDir of evalDirs) {
-    const evalId = evalDir.replace(/^eval-/, "");
-    for (const cond of conditionNames) {
-      const condDir = join(iterationDir, evalDir, cond);
-      const runPath = join(condDir, "run.json");
-      if (!existsSync(runPath)) continue;
-
-      const run = validateAgainstSchema<RunRecord>(
-        "run-record",
-        JSON.parse(readFileSync(runPath, "utf8")),
-        runPath,
-      );
-      const existing = Array.isArray(run.tool_invocations)
-        ? run.tool_invocations
-        : [];
-      if (existing.length > 0 && !overwrite) {
-        console.log(
-          `skip ${evalId}/${cond}: already has ${existing.length} tool_invocations (use --overwrite to replace)`,
-        );
-        skipped++;
-        continue;
-      }
-
-      const description = resolveAgentDescription(iterationDir, evalId, cond);
-      const subagent = adapter.findByDescription(subagentsDir, description);
-      if (!subagent) {
-        console.warn(
-          `miss ${evalId}/${cond}: no subagent transcript with description='${description}'`,
-        );
-        missing++;
-        continue;
-      }
-
-      const invocations = adapter.parseTranscript(subagent.jsonlPath);
-      run.tool_invocations = invocations;
-      writeFileSync(runPath, `${JSON.stringify(run, null, 2)}\n`);
-      console.log(
-        `fill ${evalId}/${cond}: wrote ${invocations.length} tool_invocations from ${subagent.jsonlPath}`,
-      );
-      filled++;
-    }
-  }
-
-  console.log(
-    `\nFilled: ${filled}, skipped (already populated): ${skipped}, missing transcript: ${missing}`,
-  );
-  if (missing > 0)
-    console.warn(
-      "Missing transcripts mean the dispatching agent's dispatch `description` did not match the task's `agent_description` in dispatch.json (or dispatch.json is absent and the legacy `eval-id:condition` reconstruction found no match). transcript_check assertions for those runs will be graded unverifiable.",
-    );
-}

package/skills/evaluating-skills/runner/grade.test.ts

@@ -1,347 +0,0 @@
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import {
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  rmSync,
-  writeFileSync,
-} from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { checkSkillInvokedFromTranscript } from "./grade";
-import type { ToolInvocation } from "./types";
-
-describe("checkSkillInvokedFromTranscript", () => {
-  test("returns true when transcript contains a Skill call with input.skill matching the slug", () => {
-    const slug =
-      "slow-powers-eval-1-with_skill__verification-before-completion";
-    const invocations: ToolInvocation[] = [
-      { name: "Bash", args: { command: "ls" }, ordinal: 0 },
-      { name: "Skill", args: { skill: slug }, ordinal: 1 },
-      { name: "Read", args: { file_path: "/tmp/x" }, ordinal: 2 },
-    ];
-    expect(checkSkillInvokedFromTranscript(invocations, slug)).toBe(true);
-  });
-
-  test("returns false when transcript has no Skill calls", () => {
-    const invocations: ToolInvocation[] = [
-      { name: "Bash", args: { command: "ls" }, ordinal: 0 },
-      { name: "Read", args: { file_path: "/tmp/x" }, ordinal: 1 },
-    ];
-    expect(
-      checkSkillInvokedFromTranscript(
-        invocations,
-        "slow-powers-eval-1-with_skill__foo",
-      ),
-    ).toBe(false);
-  });
-
-  test("returns false when Skill call references a different slug", () => {
-    const slug =
-      "slow-powers-eval-1-with_skill__verification-before-completion";
-    const invocations: ToolInvocation[] = [
-      {
-        name: "Skill",
-        args: { skill: "slow-powers:writing-skills" },
-        ordinal: 0,
-      },
-      {
-        name: "Skill",
-        args: { skill: "slow-powers-eval-2-old_skill__other" },
-        ordinal: 1,
-      },
-    ];
-    expect(checkSkillInvokedFromTranscript(invocations, slug)).toBe(false);
-  });
-
-  test("returns false on empty invocations array", () => {
-    expect(checkSkillInvokedFromTranscript([], "anything")).toBe(false);
-  });
-
-  test("tolerates Skill invocations whose args are missing or malformed", () => {
-    const slug = "slow-powers-eval-1-with_skill__foo";
-    const invocations: ToolInvocation[] = [
-      { name: "Skill", ordinal: 0 },
-      { name: "Skill", args: "not-an-object", ordinal: 1 },
-      { name: "Skill", args: { other: "field" }, ordinal: 2 },
-    ];
-    expect(checkSkillInvokedFromTranscript(invocations, slug)).toBe(false);
-  });
-});
-
-const GRADE_FIXTURE_ROOT = join(
-  tmpdir(),
-  `slow-powers-grade-test-${process.pid}`,
-);
-const GRADE_TS = join(import.meta.dir, "grade.ts");
-
-beforeAll(() => {
-  mkdirSync(GRADE_FIXTURE_ROOT, { recursive: true });
-});
-
-afterAll(() => {
-  rmSync(GRADE_FIXTURE_ROOT, { recursive: true, force: true });
-});
-
-function writeJsonFile(path: string, value: unknown) {
-  writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
-}
-
-describe("emitJudgeTasks skill-invocation meta-check gating", () => {
-  test("omits the skill-invocation meta-check for evals marked skill_should_trigger: false", () => {
-    const root = join(GRADE_FIXTURE_ROOT, "negative-eval");
-    const skill = "mr-review";
-    const skillDir = join(root, "skill-dir");
-    const skillSub = join(skillDir, skill);
-    mkdirSync(join(skillSub, "evals"), { recursive: true });
-    writeFileSync(
-      join(skillSub, "SKILL.md"),
-      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
-    );
-    // Two evals: a positive one (skill should fire) and a negative one
-    // (skill should NOT fire — non-invocation is the desired behavior).
-    writeJsonFile(join(skillSub, "evals", "evals.json"), {
-      skill_name: skill,
-      evals: [
-        {
-          id: "pos-eval",
-          prompt: "Fix the failing build.",
-          expected_output: "Agent debugs systematically.",
-          assertions: [
-            { id: "a1", type: "llm_judge", rubric: "Did it debug?" },
-          ],
-        },
-        {
-          id: "neg-eval",
-          prompt: "Add a --verbose flag.",
-          expected_output: "Agent treats it as a feature, no debugging.",
-          skill_should_trigger: false,
-          assertions: [
-            { id: "a2", type: "llm_judge", rubric: "Did it avoid debugging?" },
-          ],
-        },
-      ],
-    });
-
-    const cwd = join(root, "work");
-    const iterationDir = join(cwd, "skills-workspace", skill, "iteration-1");
-    mkdirSync(iterationDir, { recursive: true });
-    writeJsonFile(join(iterationDir, "conditions.json"), {
-      mode: "new-skill",
-      conditions: [
-        { name: "with_skill", skill_path: join(skillSub, "SKILL.md") },
-        { name: "without_skill", skill_path: null },
-      ],
-      timestamp: new Date().toISOString(),
-      harness: "claude-code",
-    });
-
-    for (const evalId of ["pos-eval", "neg-eval"]) {
-      for (const cond of ["with_skill", "without_skill"]) {
-        const condDir = join(iterationDir, `eval-${evalId}`, cond);
-        mkdirSync(condDir, { recursive: true });
-        // Empty tool_invocations => meta routed to a judge task (not code-checked).
-        writeJsonFile(join(condDir, "run.json"), {
-          eval_id: evalId,
-          condition: cond,
-          skill_path: cond === "with_skill" ? join(skillSub, "SKILL.md") : null,
-          prompt: "p",
-          files: [],
-          final_message: "done",
-          tool_invocations: [],
-          total_tokens: 100,
-          duration_ms: 1000,
-        });
-      }
-    }
-
-    const res = Bun.spawnSync(
-      [
-        "bun",
-        "run",
-        GRADE_TS,
-        "--skill-dir",
-        skillDir,
-        "--skill",
-        skill,
-        "--iteration",
-        "1",
-      ],
-      { cwd, stdout: "pipe", stderr: "pipe" },
-    );
-    expect(res.exitCode).toBe(0);
-
-    const tasks = JSON.parse(
-      readFileSync(join(iterationDir, "judge-tasks.json"), "utf8"),
-    ) as { tasks: Array<{ eval_id: string; is_meta: boolean }> };
-    const metaTasks = tasks.tasks.filter((t) => t.is_meta);
-    // Exactly one meta-check, and only for the positive eval.
-    expect(metaTasks.map((t) => t.eval_id)).toEqual(["pos-eval"]);
-  });
-});
-
-describe("emitJudgeTasks run.json validation", () => {
-  test("fails fast with a schema error when a run.json is malformed", () => {
-    const root = join(GRADE_FIXTURE_ROOT, "bad-run-record");
-    const skill = "mr-review";
-    const skillDir = join(root, "skill-dir");
-    const skillSub = join(skillDir, skill);
-    mkdirSync(join(skillSub, "evals"), { recursive: true });
-    writeFileSync(
-      join(skillSub, "SKILL.md"),
-      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
-    );
-    writeJsonFile(join(skillSub, "evals", "evals.json"), {
-      skill_name: skill,
-      evals: [
-        {
-          id: "pos-eval",
-          prompt: "Fix the failing build.",
-          expected_output: "Agent debugs systematically.",
-          assertions: [
-            { id: "a1", type: "llm_judge", rubric: "Did it debug?" },
-          ],
-        },
-      ],
-    });
-
-    const cwd = join(root, "work");
-    const iterationDir = join(cwd, "skills-workspace", skill, "iteration-1");
-    mkdirSync(iterationDir, { recursive: true });
-    writeJsonFile(join(iterationDir, "conditions.json"), {
-      mode: "new-skill",
-      conditions: [
-        { name: "with_skill", skill_path: join(skillSub, "SKILL.md") },
-        { name: "without_skill", skill_path: null },
-      ],
-      timestamp: new Date().toISOString(),
-      harness: "claude-code",
-    });
-
-    for (const cond of ["with_skill", "without_skill"]) {
-      const condDir = join(iterationDir, "eval-pos-eval", cond);
-      mkdirSync(condDir, { recursive: true });
-      // Missing required `final_message` and `files` — must be rejected.
-      writeJsonFile(join(condDir, "run.json"), {
-        eval_id: "pos-eval",
-        condition: cond,
-        skill_path: null,
-        prompt: "p",
-        tool_invocations: [],
-      });
-    }
-
-    const res = Bun.spawnSync(
-      [
-        "bun",
-        "run",
-        GRADE_TS,
-        "--skill-dir",
-        skillDir,
-        "--skill",
-        skill,
-        "--iteration",
-        "1",
-      ],
-      { cwd, stdout: "pipe", stderr: "pipe" },
-    );
-    expect(res.exitCode).not.toBe(0);
-    expect(res.stderr.toString()).toContain("run-record schema");
-  });
-});
-
-describe("emitJudgeTasks file-pointer dispatch", () => {
-  test("writes each judge prompt to a file and drops the inline prompt from judge-tasks.json", () => {
-    const root = join(GRADE_FIXTURE_ROOT, "judge-prompt-file");
-    const skill = "mr-review";
-    const skillDir = join(root, "skill-dir");
-    const skillSub = join(skillDir, skill);
-    mkdirSync(join(skillSub, "evals"), { recursive: true });
-    writeFileSync(
-      join(skillSub, "SKILL.md"),
-      "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n",
-    );
-    writeJsonFile(join(skillSub, "evals", "evals.json"), {
-      skill_name: skill,
-      evals: [
-        {
-          id: "pos-eval",
-          prompt: "Fix the failing build.",
-          expected_output: "Agent debugs systematically.",
-          assertions: [
-            { id: "a1", type: "llm_judge", rubric: "Did it debug?" },
-          ],
-        },
-      ],
-    });
-
-    const cwd = join(root, "work");
-    const iterationDir = join(cwd, "skills-workspace", skill, "iteration-1");
-    mkdirSync(iterationDir, { recursive: true });
-    writeJsonFile(join(iterationDir, "conditions.json"), {
-      mode: "new-skill",
-      conditions: [
-        { name: "with_skill", skill_path: join(skillSub, "SKILL.md") },
-        { name: "without_skill", skill_path: null },
-      ],
-      timestamp: new Date().toISOString(),
-      harness: "claude-code",
-    });
-
-    for (const cond of ["with_skill", "without_skill"]) {
-      const condDir = join(iterationDir, "eval-pos-eval", cond);
-      mkdirSync(condDir, { recursive: true });
-      writeJsonFile(join(condDir, "run.json"), {
-        eval_id: "pos-eval",
-        condition: cond,
-        skill_path: cond === "with_skill" ? join(skillSub, "SKILL.md") : null,
-        prompt: "p",
-        files: [],
-        final_message: "done",
-        tool_invocations: [],
-        total_tokens: 100,
-        duration_ms: 1000,
-      });
-    }
-
-    const res = Bun.spawnSync(
-      [
-        "bun",
-        "run",
-        GRADE_TS,
-        "--skill-dir",
-        skillDir,
-        "--skill",
-        skill,
-        "--iteration",
-        "1",
-      ],
-      { cwd, stdout: "pipe", stderr: "pipe" },
-    );
-    expect(res.exitCode).toBe(0);
-
-    const tasks = JSON.parse(
-      readFileSync(join(iterationDir, "judge-tasks.json"), "utf8"),
-    ) as {
-      tasks: Array<{
-        assertion_id: string;
-        response_path: string;
-        dispatch_prompt?: string;
-        dispatch_prompt_path: string;
-      }>;
-    };
-
-    expect(tasks.tasks.length).toBeGreaterThan(0);
-    for (const t of tasks.tasks) {
-      // Nothing inlined; the orchestrator reads the prompt from a file.
-      expect(t.dispatch_prompt).toBeUndefined();
-      expect(t.dispatch_prompt_path.endsWith(`${t.assertion_id}.txt`)).toBe(
-        true,
-      );
-      expect(existsSync(t.dispatch_prompt_path)).toBe(true);
-      const contents = readFileSync(t.dispatch_prompt_path, "utf8");
-      // The judge still learns where to write its verdict from the prompt text.
-      expect(contents).toContain(t.response_path);
-    }
-  });
-});