@slowdini/slow-powers-opencode 0.3.0 → 0.4.0

package/skills/evaluating-skills/runner/record-runs.test.ts

@@ -1,314 +0,0 @@
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import {
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  rmSync,
-  writeFileSync,
-} from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { recordRuns } from "./record-runs";
-import type { RunRecord, TimingRecord } from "./types";
-
-const ROOT = join(tmpdir(), `record-runs-test-${process.pid}`);
-
-let iterationDir: string;
-let subagentsDir: string;
-
-function jsonl(lines: object[]): string {
-  return `${lines.map((l) => JSON.stringify(l)).join("\n")}\n`;
-}
-
-/** A minimal transcript with usage, timestamps, one tool call, and final text. */
-function transcriptLines(finalText: string): object[] {
-  return [
-    {
-      type: "user",
-      timestamp: "2026-06-04T10:00:00.000Z",
-      message: { role: "user", content: "go" },
-    },
-    {
-      type: "assistant",
-      timestamp: "2026-06-04T10:00:10.000Z",
-      message: {
-        id: "msg_1",
-        role: "assistant",
-        usage: {
-          input_tokens: 100,
-          output_tokens: 20,
-          cache_creation_input_tokens: 30,
-          cache_read_input_tokens: 50,
-        },
-        content: [
-          {
-            type: "tool_use",
-            id: "toolu_1",
-            name: "Bash",
-            input: { command: "ls" },
-          },
-        ],
-      },
-    },
-    {
-      type: "user",
-      timestamp: "2026-06-04T10:00:12.000Z",
-      message: {
-        role: "user",
-        content: [
-          { type: "tool_result", tool_use_id: "toolu_1", content: "ok" },
-        ],
-      },
-    },
-    {
-      type: "assistant",
-      timestamp: "2026-06-04T10:01:00.000Z",
-      message: {
-        id: "msg_2",
-        role: "assistant",
-        usage: {
-          input_tokens: 200,
-          output_tokens: 40,
-          cache_creation_input_tokens: 0,
-          cache_read_input_tokens: 60,
-        },
-        content: [{ type: "text", text: finalText }],
-      },
-    },
-  ];
-}
-
-// Token math for transcriptLines: msg_1 (100+20+30+50) + msg_2 (200+40+0+60) = 500.
-const TRANSCRIPT_TOKENS = 500;
-// 10:00:00.000 → 10:01:00.000
-const TRANSCRIPT_DURATION_MS = 60_000;
-
-function writeSubagent(name: string, description: string, lines: object[]) {
-  writeFileSync(
-    join(subagentsDir, `${name}.meta.json`),
-    JSON.stringify({ agentType: "general-purpose", description }),
-  );
-  writeFileSync(join(subagentsDir, `${name}.jsonl`), jsonl(lines));
-}
-
-type FixtureTask = {
-  eval_id: string;
-  condition: string;
-  finalMessage?: string; // written to outputs/final-message.md when present
-};
-
-/** Builds an iteration dir + dispatch.json shaped like run.ts serializes it. */
-function writeIteration(tasks: FixtureTask[]) {
-  const serialized = tasks.map((t) => {
-    const condDir = join(iterationDir, `eval-${t.eval_id}`, t.condition);
-    const outputsDir = join(condDir, "outputs");
-    mkdirSync(outputsDir, { recursive: true });
-    if (t.finalMessage !== undefined) {
-      writeFileSync(join(outputsDir, "final-message.md"), t.finalMessage);
-    }
-    return {
-      eval_id: t.eval_id,
-      condition: t.condition,
-      skill_path:
-        t.condition === "without_skill" ? null : "/staged/skill/SKILL.md",
-      staged_skill_slug: t.condition === "without_skill" ? null : "test-slug",
-      user_prompt: `Do the ${t.eval_id} task`,
-      fixtures: [join(condDir, "inputs", "fixture.txt")],
-      outputs_dir: outputsDir,
-      run_record_path: join(condDir, "run.json"),
-      timing_path: join(condDir, "timing.json"),
-      agent_description: `${t.eval_id}:${t.condition}:i1-nonce1`,
-      dispatch_prompt_path: join(condDir, "dispatch-prompt.txt"),
-    };
-  });
-  writeFileSync(
-    join(iterationDir, "dispatch.json"),
-    JSON.stringify({ run_nonce: "nonce1", tasks: serialized }, null, 2),
-  );
-  return serialized;
-}
-
-function readRun(evalId: string, condition: string): RunRecord {
-  return JSON.parse(
-    readFileSync(
-      join(iterationDir, `eval-${evalId}`, condition, "run.json"),
-      "utf8",
-    ),
-  );
-}
-
-function readTiming(evalId: string, condition: string): TimingRecord {
-  return JSON.parse(
-    readFileSync(
-      join(iterationDir, `eval-${evalId}`, condition, "timing.json"),
-      "utf8",
-    ),
-  );
-}
-
-beforeEach(() => {
-  iterationDir = join(ROOT, `iter-${Math.random().toString(36).slice(2)}`);
-  subagentsDir = join(ROOT, `sub-${Math.random().toString(36).slice(2)}`);
-  mkdirSync(iterationDir, { recursive: true });
-  mkdirSync(subagentsDir, { recursive: true });
-});
-
-afterEach(() => rmSync(ROOT, { recursive: true, force: true }));
-
-describe("recordRuns", () => {
-  test("assembles run.json and timing.json for every task from disk", () => {
-    writeIteration([
-      { eval_id: "crash", condition: "with_skill", finalMessage: "Fixed it." },
-      {
-        eval_id: "crash",
-        condition: "without_skill",
-        finalMessage: "Done, I think.",
-      },
-    ]);
-    writeSubagent(
-      "agent-a",
-      "crash:with_skill:i1-nonce1",
-      transcriptLines("unused"),
-    );
-    writeSubagent(
-      "agent-b",
-      "crash:without_skill:i1-nonce1",
-      transcriptLines("unused"),
-    );
-
-    const result = recordRuns({ iterationDir, subagentsDir });
-    expect(result.recorded).toBe(2);
-    expect(result.missingTranscript).toBe(0);
-
-    const run = readRun("crash", "with_skill");
-    expect(run.eval_id).toBe("crash");
-    expect(run.condition).toBe("with_skill");
-    expect(run.skill_path).toBe("/staged/skill/SKILL.md");
-    expect(run.prompt).toBe("Do the crash task");
-    expect(run.files).toHaveLength(1);
-    expect(run.final_message).toBe("Fixed it.");
-    expect(run.tool_invocations).toHaveLength(1);
-    expect(run.tool_invocations[0]).toMatchObject({ name: "Bash", ordinal: 0 });
-
-    expect(readRun("crash", "without_skill").skill_path).toBeNull();
-
-    const timing = readTiming("crash", "with_skill");
-    expect(timing.total_tokens).toBe(TRANSCRIPT_TOKENS);
-    expect(timing.duration_ms).toBe(TRANSCRIPT_DURATION_MS);
-    expect(timing.source).toBe("transcript");
-  });
-
-  test("skips existing run.json without --overwrite, replaces with it", () => {
-    const [task] = writeIteration([
-      { eval_id: "crash", condition: "with_skill", finalMessage: "New." },
-    ]);
-    writeSubagent(
-      "agent-a",
-      "crash:with_skill:i1-nonce1",
-      transcriptLines("unused"),
-    );
-    const handWritten = {
-      eval_id: "crash",
-      condition: "with_skill",
-      skill_path: "/staged/skill/SKILL.md",
-      prompt: "Do the crash task",
-      files: [],
-      final_message: "Agent-authored.",
-      tool_invocations: [],
-    };
-    writeFileSync(task.run_record_path, JSON.stringify(handWritten));
-
-    const skipped = recordRuns({ iterationDir, subagentsDir });
-    expect(skipped.recorded).toBe(0);
-    expect(skipped.skippedExisting).toBe(1);
-    expect(readRun("crash", "with_skill").final_message).toBe(
-      "Agent-authored.",
-    );
-
-    const replaced = recordRuns({
-      iterationDir,
-      subagentsDir,
-      overwrite: true,
-    });
-    expect(replaced.recorded).toBe(1);
-    expect(readRun("crash", "with_skill").final_message).toBe("New.");
-  });
-
-  test("backfills timing.json only when absent", () => {
-    const [task] = writeIteration([
-      { eval_id: "crash", condition: "with_skill", finalMessage: "Done." },
-    ]);
-    writeSubagent(
-      "agent-a",
-      "crash:with_skill:i1-nonce1",
-      transcriptLines("unused"),
-    );
-    writeFileSync(
-      task.timing_path,
-      JSON.stringify({ total_tokens: 12345, duration_ms: 9000 }),
-    );
-
-    recordRuns({ iterationDir, subagentsDir });
-
-    // Agent-captured completion-event timing wins; not overwritten.
-    const timing = readTiming("crash", "with_skill");
-    expect(timing.total_tokens).toBe(12345);
-    expect(timing.duration_ms).toBe(9000);
-    expect(timing.source).toBeUndefined();
-  });
-
-  test("falls back to the transcript's final assistant text when final-message.md is missing", () => {
-    writeIteration([{ eval_id: "crash", condition: "with_skill" }]);
-    writeSubagent(
-      "agent-a",
-      "crash:with_skill:i1-nonce1",
-      transcriptLines("Closing summary from transcript."),
-    );
-
-    const result = recordRuns({ iterationDir, subagentsDir });
-    expect(result.recorded).toBe(1);
-    expect(readRun("crash", "with_skill").final_message).toBe(
-      "Closing summary from transcript.",
-    );
-  });
-
-  test("skips the slot entirely when no final-message source exists", () => {
-    writeIteration([{ eval_id: "crash", condition: "with_skill" }]);
-    // No final-message.md, no transcript.
-
-    const result = recordRuns({ iterationDir, subagentsDir });
-    expect(result.recorded).toBe(0);
-    expect(result.skippedNoFinalMessage).toBe(1);
-    expect(
-      existsSync(join(iterationDir, "eval-crash", "with_skill", "run.json")),
-    ).toBe(false);
-    expect(
-      existsSync(join(iterationDir, "eval-crash", "with_skill", "timing.json")),
-    ).toBe(false);
-  });
-
-  test("writes run.json with empty invocations and no timing.json when the transcript is missing", () => {
-    writeIteration([
-      { eval_id: "crash", condition: "with_skill", finalMessage: "Done." },
-    ]);
-    // final-message.md exists but no subagent transcript matches.
-
-    const result = recordRuns({ iterationDir, subagentsDir });
-    expect(result.recorded).toBe(1);
-    expect(result.missingTranscript).toBe(1);
-
-    const run = readRun("crash", "with_skill");
-    expect(run.final_message).toBe("Done.");
-    expect(run.tool_invocations).toEqual([]);
-    expect(
-      existsSync(join(iterationDir, "eval-crash", "with_skill", "timing.json")),
-    ).toBe(false);
-  });
-
-  test("throws when dispatch.json is absent", () => {
-    // Hand-authored/operator runs have no dispatch.json — the manual path owns them.
-    expect(() => recordRuns({ iterationDir, subagentsDir })).toThrow(
-      /dispatch\.json/,
-    );
-  });
-});

package/skills/evaluating-skills/runner/record-runs.ts

@@ -1,209 +0,0 @@
-#!/usr/bin/env bun
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import {
-  findByDescription,
-  parseTranscriptFull,
-} from "./adapters/claude-code-transcript";
-import { detectRunContext } from "./context";
-import type { RunRecord, TimingRecord } from "./types";
-import { validateAgainstSchema } from "./validate-schema";
-
-function die(msg: string): never {
-  console.error(`error: ${msg}`);
-  process.exit(1);
-}
-
-/** The dispatch.json task shape record-runs consumes (see DispatchTask in
- * run.ts — `dispatch_prompt` is stripped from the serialized file). */
-type DispatchTask = {
-  eval_id: string;
-  condition: string;
-  skill_path: string | null;
-  user_prompt: string;
-  fixtures: string[];
-  outputs_dir: string;
-  run_record_path: string;
-  timing_path: string;
-  agent_description: string;
-};
-
-export type RecordRunsResult = {
-  recorded: number;
-  skippedExisting: number;
-  skippedNoFinalMessage: number;
-  missingTranscript: number;
-};
-
-/**
- * Assembles a schema-valid `run.json` (and backfills `timing.json`) for every
- * task in the iteration's `dispatch.json`, from sources already on disk:
- *
- * - carry-over fields (`prompt` ← `user_prompt`, `files` ← `fixtures`,
- *   `eval_id`/`condition`/`skill_path`) from `dispatch.json`;
- * - `final_message` from `<outputs_dir>/final-message.md` (the dispatch prompt
- *   instructs the subagent to write it), falling back to the transcript's last
- *   assistant text;
- * - `tool_invocations`, tokens, and duration from the persisted Claude Code
- *   transcript (Claude-Code-tier, like fill-transcripts — transcript-less
- *   harnesses keep hand-authoring these records).
- *
- * Existing records always win: an agent/operator-written `run.json` is skipped
- * without `overwrite`, and `timing.json` is backfill-only — completion-event
- * numbers captured at dispatch time are never replaced by transcript-derived
- * ones, which include cache accounting and are not comparable 1:1.
- */
-export function recordRuns(opts: {
-  iterationDir: string;
-  subagentsDir: string;
-  overwrite?: boolean;
-}): RecordRunsResult {
-  const { iterationDir, subagentsDir, overwrite = false } = opts;
-
-  const dispatchPath = join(iterationDir, "dispatch.json");
-  if (!existsSync(dispatchPath)) {
-    throw new Error(
-      `${dispatchPath} not found — record-runs assembles records from dispatch.json and only supports runner-built iterations. For hand-authored runs, write run.json + timing.json manually (see schema/run-record.schema.json).`,
-    );
-  }
-  const dispatch = JSON.parse(readFileSync(dispatchPath, "utf8")) as {
-    tasks?: DispatchTask[];
-  };
-  const tasks = dispatch.tasks ?? [];
-
-  const result: RecordRunsResult = {
-    recorded: 0,
-    skippedExisting: 0,
-    skippedNoFinalMessage: 0,
-    missingTranscript: 0,
-  };
-
-  for (const task of tasks) {
-    const slot = `${task.eval_id}/${task.condition}`;
-
-    const subagent = findByDescription(subagentsDir, task.agent_description);
-    const summary = subagent ? parseTranscriptFull(subagent.jsonlPath) : null;
-    if (!subagent) {
-      console.warn(
-        `miss ${slot}: no subagent transcript with description='${task.agent_description}'`,
-      );
-      result.missingTranscript++;
-    }
-
-    // run.json — skip if the agent/operator already wrote one.
-    if (existsSync(task.run_record_path) && !overwrite) {
-      console.log(
-        `skip ${slot}: run.json already exists (use --overwrite to replace)`,
-      );
-      result.skippedExisting++;
-    } else {
-      const finalMessagePath = join(task.outputs_dir, "final-message.md");
-      let finalMessage: string | null = null;
-      if (existsSync(finalMessagePath)) {
-        finalMessage = readFileSync(finalMessagePath, "utf8").trim();
-      } else if (summary?.final_text) {
-        console.warn(
-          `warn ${slot}: ${finalMessagePath} missing — using the transcript's last assistant text as final_message`,
-        );
-        finalMessage = summary.final_text;
-      }
-      if (finalMessage === null) {
-        console.warn(
-          `skip ${slot}: no final-message.md and no transcript text — was this task dispatched? Not writing a blank record.`,
-        );
-        result.skippedNoFinalMessage++;
-        continue;
-      }
-
-      const record: RunRecord = {
-        eval_id: task.eval_id,
-        condition: task.condition,
-        skill_path: task.skill_path,
-        prompt: task.user_prompt,
-        files: task.fixtures,
-        final_message: finalMessage,
-        tool_invocations: summary?.tool_invocations ?? [],
-        // Timing lives in timing.json; run.json never carries it.
-        total_tokens: null,
-        duration_ms: null,
-      };
-      validateAgainstSchema<RunRecord>(
-        "run-record",
-        record,
-        task.run_record_path,
-      );
-      writeFileSync(
-        task.run_record_path,
-        `${JSON.stringify(record, null, 2)}\n`,
-      );
-      console.log(
-        `record ${slot}: wrote run.json with ${record.tool_invocations.length} tool_invocations`,
-      );
-      result.recorded++;
-    }
-
-    // timing.json — backfill only; completion-event numbers always win.
-    const timingExists = existsSync(task.timing_path);
-    if (summary && (!timingExists || overwrite)) {
-      const timing: TimingRecord = {
-        total_tokens: summary.total_tokens,
-        duration_ms: summary.duration_ms,
-        source: "transcript",
-      };
-      writeFileSync(task.timing_path, `${JSON.stringify(timing, null, 2)}\n`);
-    }
-  }
-
-  return result;
-}
-
-function parseArgs(argv: string[]) {
-  const flag = (name: string): string | undefined => {
-    const i = argv.indexOf(`--${name}`);
-    if (i === -1) return undefined;
-    return argv[i + 1];
-  };
-  const iteration = flag("iteration");
-  const subagentsDir = flag("subagents-dir");
-  const overwrite = argv.includes("--overwrite");
-  if (!iteration) die("missing --iteration");
-  if (!subagentsDir)
-    die(
-      "missing --subagents-dir (e.g. ~/.claude/projects/<project-slug>/<parent-session-id>/subagents/)",
-    );
-  return { iteration, subagentsDir, overwrite };
-}
-
-if (import.meta.main) {
-  const argv = Bun.argv.slice(2);
-  const { iteration, subagentsDir, overwrite } = parseArgs(argv);
-  const ctx = detectRunContext(argv);
-
-  if (!existsSync(subagentsDir))
-    die(`subagents-dir not found: ${subagentsDir}`);
-
-  const iterationDir = join(
-    ctx.workspaceRoot,
-    ctx.skillName,
-    `iteration-${iteration}`,
-  );
-  if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
-
-  let result: RecordRunsResult;
-  try {
-    result = recordRuns({ iterationDir, subagentsDir, overwrite });
-  } catch (err) {
-    die(err instanceof Error ? err.message : String(err));
-  }
-
-  console.log(
-    `\nRecorded: ${result.recorded}, skipped (existing run.json): ${result.skippedExisting}, skipped (no final message): ${result.skippedNoFinalMessage}, missing transcript: ${result.missingTranscript}`,
-  );
-  if (result.missingTranscript > 0)
-    console.warn(
-      "Missing transcripts mean the dispatching agent's dispatch `description` did not match the task's `agent_description` in dispatch.json. Those slots got empty tool_invocations (transcript_check assertions will grade unverifiable) and no transcript-derived timing.",
-    );
-  console.log(
-    `\nNext: bun run evals:detect-stray-writes -- --skill ${ctx.skillName} --iteration ${iteration}\nThen: bun run evals:grade -- --skill ${ctx.skillName} --iteration ${iteration}`,
-  );
-}