@slowdini/slow-powers-opencode 0.3.0 → 0.4.0

package/skills/evaluating-skills/runner/aggregate.ts

@@ -1,269 +0,0 @@
-#!/usr/bin/env bun
-import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import { detectRunContext } from "./context";
-import {
-  type PluginShadowReport,
-  shadowValidityWarnings,
-} from "./plugin-shadow";
-import type { ConditionsRecord, GradingResult, TimingRecord } from "./types";
-
-function die(msg: string): never {
-  console.error(`error: ${msg}`);
-  process.exit(1);
-}
-
-function parseArgs(argv: string[]) {
-  const flag = (name: string): string | undefined => {
-    const i = argv.indexOf(`--${name}`);
-    if (i === -1) return undefined;
-    return argv[i + 1];
-  };
-  const iteration = flag("iteration");
-  if (!iteration) die("missing --iteration");
-  return { iteration };
-}
-
-type Series = number[];
-
-function mean(values: Series): number {
-  if (values.length === 0) return 0;
-  return values.reduce((a, b) => a + b, 0) / values.length;
-}
-
-function stddev(values: Series, m = mean(values)): number {
-  if (values.length < 2) return 0;
-  const v = values.reduce((s, x) => s + (x - m) ** 2, 0) / values.length;
-  return Math.sqrt(v);
-}
-
-function round(n: number, dp: number): number {
-  const p = 10 ** dp;
-  return Math.round(n * p) / p;
-}
-
-function stats(values: Series, dp: number) {
-  const m = mean(values);
-  return {
-    mean: round(m, dp),
-    stddev: round(stddev(values, m), dp),
-    n: values.length,
-  };
-}
-
-const aggArgv = Bun.argv.slice(2);
-const { iteration } = parseArgs(aggArgv);
-const aggCtx = detectRunContext(aggArgv);
-const iterationDir = join(
-  aggCtx.workspaceRoot,
-  aggCtx.skillName,
-  `iteration-${iteration}`,
-);
-if (!existsSync(iterationDir)) die(`not found: ${iterationDir}`);
-
-const conditionsPath = join(iterationDir, "conditions.json");
-if (!existsSync(conditionsPath)) die(`missing: ${conditionsPath}`);
-const conditions: ConditionsRecord = JSON.parse(
-  readFileSync(conditionsPath, "utf8"),
-);
-const conditionNames = conditions.conditions.map((c) => c.name);
-if (conditionNames.length !== 2)
-  die(`expected exactly 2 conditions, got ${conditionNames.length}`);
-
-const evalDirs = readdirSync(iterationDir).filter((d) => d.startsWith("eval-"));
-if (evalDirs.length === 0) die("no eval directories found");
-
-type Bucket = {
-  passRates: Series;
-  durations: Series;
-  tokens: Series;
-  skillInvoked: boolean[];
-  hadSkillLoaded: boolean;
-};
-const byCondition: Record<string, Bucket> = {};
-const conditionSkillPaths = new Map<string, string | null>();
-for (const c of conditions.conditions) {
-  conditionSkillPaths.set(c.name, c.skill_path);
-  byCondition[c.name] = {
-    passRates: [],
-    durations: [],
-    tokens: [],
-    skillInvoked: [],
-    hadSkillLoaded: !!c.skill_path,
-  };
-}
-
-let missingGradings = 0;
-// Timing provenance across all runs in the comparison. "completion-event"
-// (the agent-captured default, also assumed when `source` is absent) and
-// "transcript" (record-runs backfill, includes cache accounting) measure
-// different things — a delta mixing them is comparing two metrics.
-const timingSources = new Set<string>();
-for (const evalDir of evalDirs) {
-  for (const cond of conditionNames) {
-    const condDir = join(iterationDir, evalDir, cond);
-    const gradingPath = join(condDir, "grading.json");
-    const timingPath = join(condDir, "timing.json");
-    if (!existsSync(gradingPath)) {
-      console.warn(`warn: missing grading for ${evalDir}/${cond}`);
-      missingGradings++;
-      continue;
-    }
-    const grading: GradingResult = JSON.parse(
-      readFileSync(gradingPath, "utf8"),
-    );
-    byCondition[cond].passRates.push(grading.summary.pass_rate);
-    if (grading.meta_summary?.skill_invoked != null)
-      byCondition[cond].skillInvoked.push(grading.meta_summary.skill_invoked);
-    if (existsSync(timingPath)) {
-      const timing: TimingRecord = JSON.parse(readFileSync(timingPath, "utf8"));
-      if (typeof timing.total_tokens === "number")
-        byCondition[cond].tokens.push(timing.total_tokens);
-      if (typeof timing.duration_ms === "number")
-        byCondition[cond].durations.push(timing.duration_ms);
-      if (
-        typeof timing.total_tokens === "number" ||
-        typeof timing.duration_ms === "number"
-      )
-        timingSources.add(timing.source ?? "completion-event");
-    }
-  }
-}
-
-type ConditionSummary = {
-  pass_rate: ReturnType<typeof stats>;
-  duration_ms: ReturnType<typeof stats>;
-  total_tokens: ReturnType<typeof stats>;
-  skill_invocation_rate?: number | null;
-  skill_invocation_n?: number;
-};
-
-const runSummary: Record<string, ConditionSummary> = {};
-for (const cond of conditionNames) {
-  const bucket = byCondition[cond];
-  const summary: ConditionSummary = {
-    pass_rate: stats(bucket.passRates, 3),
-    duration_ms: stats(bucket.durations, 0),
-    total_tokens: stats(bucket.tokens, 0),
-  };
-  if (bucket.hadSkillLoaded) {
-    summary.skill_invocation_n = bucket.skillInvoked.length;
-    summary.skill_invocation_rate =
-      bucket.skillInvoked.length === 0
-        ? null
-        : round(
-            bucket.skillInvoked.filter(Boolean).length /
-              bucket.skillInvoked.length,
-            3,
-          );
-  }
-  runSummary[cond] = summary;
-}
-
-const [a, b] = conditionNames;
-const delta = {
-  direction: `${a} - ${b}`,
-  pass_rate: round(
-    runSummary[a].pass_rate.mean - runSummary[b].pass_rate.mean,
-    3,
-  ),
-  duration_ms: round(
-    runSummary[a].duration_ms.mean - runSummary[b].duration_ms.mean,
-    0,
-  ),
-  total_tokens: round(
-    runSummary[a].total_tokens.mean - runSummary[b].total_tokens.mean,
-    0,
-  ),
-};
-
-const validityWarnings: string[] = [];
-if (timingSources.size > 1) {
-  validityWarnings.push(
-    `runs mix timing sources (${[...timingSources].sort().join(", ")}) — transcript-derived totals include cache accounting, so the token/duration delta compares two different metrics. Re-record one side or read the delta as a rough signal only.`,
-  );
-}
-for (const cond of conditionNames) {
-  const s = runSummary[cond];
-  if (s.skill_invocation_rate != null && s.skill_invocation_rate < 1) {
-    validityWarnings.push(
-      `condition '${cond}' had skill loaded but invocation rate ${(s.skill_invocation_rate * 100).toFixed(0)}% (${s.skill_invocation_n} runs checked) — substantive results may not reflect skill effectiveness.`,
-    );
-  }
-}
-
-// Stray-write findings (from `evals:detect-stray-writes`, if it ran) taint a
-// run the same way a missed skill invocation does: a subagent that edited the
-// real repo or installed packages is no longer a clean data point.
-const strayPath = join(iterationDir, "stray-writes.json");
-if (existsSync(strayPath)) {
-  try {
-    const stray = JSON.parse(readFileSync(strayPath, "utf8")) as {
-      runs?: Array<{
-        eval_id: string;
-        condition: string;
-        violations?: unknown[];
-        live_source_reads?: unknown[];
-      }>;
-    };
-    for (const r of stray.runs ?? []) {
-      const n = r.violations?.length ?? 0;
-      if (n > 0)
-        validityWarnings.push(
-          `${r.eval_id}/${r.condition} wrote ${n} file(s) outside its outputs dir — data point may be tainted (see stray-writes.json).`,
-        );
-      const reads = r.live_source_reads?.length ?? 0;
-      if (reads > 0)
-        validityWarnings.push(
-          `${r.eval_id}/${r.condition} read the live skill source ${reads} time(s) instead of its staged copy — the arm may be contaminated (staged-slug resolution race; see stray-writes.json).`,
-        );
-    }
-  } catch {
-    // ignore a malformed report rather than failing aggregation
-  }
-}
-
-// Plugin-shadow findings (from the runner's build-time preflight, Claude Code)
-// taint a run the same way a missed invocation does: a staged skill also served
-// by an enabled plugin means subagents could discover both copies, so the
-// with/without comparison may not reflect the staged skill alone.
-const shadowPath = join(iterationDir, "plugin-shadow.json");
-if (existsSync(shadowPath)) {
-  try {
-    const report = JSON.parse(
-      readFileSync(shadowPath, "utf8"),
-    ) as PluginShadowReport;
-    for (const w of shadowValidityWarnings(report)) validityWarnings.push(w);
-  } catch {
-    // ignore a malformed report rather than failing aggregation
-  }
-}
-
-const benchmark = {
-  generated: new Date().toISOString(),
-  mode: conditions.mode,
-  baseline: conditions.baseline,
-  conditions_compared: [a, b],
-  missing_gradings: missingGradings,
-  validity_warnings: validityWarnings,
-  run_summary: runSummary,
-  delta,
-};
-
-const outPath = join(iterationDir, "benchmark.json");
-writeFileSync(outPath, `${JSON.stringify(benchmark, null, 2)}\n`);
-console.log(`Wrote ${outPath}`);
-if (missingGradings > 0)
-  console.warn(
-    `note: ${missingGradings} grading.json file(s) were missing — benchmark is incomplete.`,
-  );
-for (const warning of validityWarnings) console.warn(`⚠ ${warning}`);
-if (validityWarnings.length === 0) {
-  for (const cond of conditionNames) {
-    const s = runSummary[cond];
-    if (s.skill_invocation_rate === 1)
-      console.log(
-        `✓ ${cond}: skill invocation rate 100% (${s.skill_invocation_n} runs) — substantive results are valid.`,
-      );
-  }
-}

package/skills/evaluating-skills/runner/context.test.ts

@@ -1,181 +0,0 @@
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-import { detectRunContext } from "./context";
-
-const FIXTURE_ROOT = join(tmpdir(), `slow-powers-context-test-${process.pid}`);
-
-function fixturePath(name: string): string {
-  return join(FIXTURE_ROOT, name);
-}
-
-function makeSkillDir(root: string, skills: string[]): string {
-  const dir = join(root, "skill-dir");
-  mkdirSync(dir, { recursive: true });
-  for (const name of skills) {
-    const sub = join(dir, name);
-    mkdirSync(sub, { recursive: true });
-    writeFileSync(
-      join(sub, "SKILL.md"),
-      `---\nname: ${name}\ndescription: ${name} skill\n---\n\nbody\n`,
-    );
-  }
-  return dir;
-}
-
-beforeAll(() => {
-  mkdirSync(FIXTURE_ROOT, { recursive: true });
-});
-
-afterAll(() => {
-  rmSync(FIXTURE_ROOT, { recursive: true, force: true });
-});
-
-describe("detectRunContext", () => {
-  test("dies when --skill-dir is missing", () => {
-    expect(() => detectRunContext(["--skill", "foo"])).toThrow(/--skill-dir/);
-  });
-
-  test("dies when --skill is missing", () => {
-    const root = fixturePath("missing-skill");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    expect(() => detectRunContext(["--skill-dir", skillDir])).toThrow(
-      /--skill/,
-    );
-  });
-
-  test("dies when --skill-dir is not a directory", () => {
-    expect(() =>
-      detectRunContext([
-        "--skill-dir",
-        "/nonexistent/does-not-exist-12345",
-        "--skill",
-        "foo",
-      ]),
-    ).toThrow(/--skill-dir/);
-  });
-
-  test("dies when skill subdir does not exist", () => {
-    const root = fixturePath("missing-subdir");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    expect(() =>
-      detectRunContext(["--skill-dir", skillDir, "--skill", "bar"]),
-    ).toThrow(/skill not found/);
-  });
-
-  test("dies when --bootstrap path is passed but file does not exist", () => {
-    const root = fixturePath("bad-bootstrap");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    expect(() =>
-      detectRunContext([
-        "--skill-dir",
-        skillDir,
-        "--skill",
-        "foo",
-        "--bootstrap",
-        "/nonexistent/no-bootstrap-12345.md",
-      ]),
-    ).toThrow(/--bootstrap/);
-  });
-
-  test("returns RunContext with absolute paths when --skill-dir and --skill are valid", () => {
-    const root = fixturePath("happy-path");
-    const skillDir = makeSkillDir(root, ["mr-review"]);
-    const ctx = detectRunContext([
-      "--skill-dir",
-      skillDir,
-      "--skill",
-      "mr-review",
-    ]);
-    expect(ctx.skillDir).toBe(resolve(skillDir));
-    expect(ctx.skillName).toBe("mr-review");
-    expect(ctx.skillSubdir).toBe(resolve(skillDir, "mr-review"));
-    expect(ctx.siblingSkillNames).toEqual([]);
-    expect(ctx.bootstrapPath).toBeNull();
-    expect(ctx.harness).toBe("claude-code");
-  });
-
-  test("enumerates siblings excluding the skill-under-test", () => {
-    const root = fixturePath("siblings");
-    const skillDir = makeSkillDir(root, ["alpha", "beta", "gamma"]);
-    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "beta"]);
-    expect(ctx.siblingSkillNames.sort()).toEqual(["alpha", "gamma"]);
-  });
-
-  test("ignores entries in --skill-dir that do not have a SKILL.md", () => {
-    const root = fixturePath("not-skills");
-    const skillDir = makeSkillDir(root, ["real"]);
-    mkdirSync(join(skillDir, "node_modules"), { recursive: true });
-    mkdirSync(join(skillDir, "no-skill-md-here"), { recursive: true });
-    writeFileSync(join(skillDir, "loose-file.txt"), "hello");
-    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "real"]);
-    expect(ctx.siblingSkillNames).toEqual([]);
-  });
-
-  test("workspaceRoot defaults to <CWD>/skills-workspace when --workspace-dir is omitted", () => {
-    const root = fixturePath("workspace-default");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "foo"]);
-    expect(ctx.workspaceRoot).toBe(resolve(process.cwd(), "skills-workspace"));
-  });
-
-  test("workspaceRoot honors --workspace-dir override (resolved absolute)", () => {
-    const root = fixturePath("workspace-override");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    const customWs = join(root, "custom-ws");
-    mkdirSync(customWs, { recursive: true });
-    const ctx = detectRunContext([
-      "--skill-dir",
-      skillDir,
-      "--skill",
-      "foo",
-      "--workspace-dir",
-      customWs,
-    ]);
-    expect(ctx.workspaceRoot).toBe(resolve(customWs));
-  });
-
-  test("stageRoot defaults to CWD", () => {
-    const root = fixturePath("stage-default");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    const ctx = detectRunContext(["--skill-dir", skillDir, "--skill", "foo"]);
-    expect(ctx.stageRoot).toBe(resolve(process.cwd()));
-  });
-
-  test("--bootstrap path is resolved absolute when file exists", () => {
-    const root = fixturePath("bootstrap-ok");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    const bootstrapPath = join(root, "my-bootstrap.md");
-    writeFileSync(bootstrapPath, "BOOT");
-    const ctx = detectRunContext([
-      "--skill-dir",
-      skillDir,
-      "--skill",
-      "foo",
-      "--bootstrap",
-      bootstrapPath,
-    ]);
-    expect(ctx.bootstrapPath).toBe(resolve(bootstrapPath));
-  });
-
-  test("unknown --harness value is rejected", () => {
-    const root = fixturePath("harness-bad");
-    const skillDir = makeSkillDir(root, ["foo"]);
-    expect(() =>
-      detectRunContext([
-        "--skill-dir",
-        skillDir,
-        "--skill",
-        "foo",
-        "--harness",
-        "vscode",
-      ]),
-    ).toThrow(/harness/);
-  });
-});
-
-// Sanity: ensure existsSync helper from node:fs is what we expect
-test.skip("smoke: existsSync points at node:fs", () => {
-  expect(typeof existsSync).toBe("function");
-});

package/skills/evaluating-skills/runner/context.ts

@@ -1,90 +0,0 @@
-import { existsSync, readdirSync, statSync } from "node:fs";
-import { resolve } from "node:path";
-
-export type Harness = "claude-code";
-
-export type RunContext = {
-  skillDir: string;
-  skillName: string;
-  skillSubdir: string;
-  siblingSkillNames: string[];
-  workspaceRoot: string;
-  stageRoot: string;
-  bootstrapPath: string | null;
-  harness: Harness;
-};
-
-function die(msg: string): never {
-  throw new Error(msg);
-}
-
-function flag(argv: string[], name: string): string | undefined {
-  const i = argv.indexOf(`--${name}`);
-  if (i === -1) return undefined;
-  const v = argv[i + 1];
-  if (v === undefined || v.startsWith("--")) {
-    die(`flag --${name} requires a value`);
-  }
-  return v;
-}
-
-export function detectRunContext(argv: string[]): RunContext {
-  const skillDirRaw = flag(argv, "skill-dir");
-  if (!skillDirRaw) die("missing required flag --skill-dir <path>");
-  const skillDir = resolve(skillDirRaw);
-  if (!existsSync(skillDir) || !statSync(skillDir).isDirectory()) {
-    die(`--skill-dir is not a directory: ${skillDir}`);
-  }
-
-  const skillName = flag(argv, "skill");
-  if (!skillName) die("missing required flag --skill <name>");
-
-  const skillSubdir = resolve(skillDir, skillName);
-  const skillMd = resolve(skillSubdir, "SKILL.md");
-  if (!existsSync(skillMd)) {
-    die(`skill not found: ${skillMd}`);
-  }
-
-  const bootstrapRaw = flag(argv, "bootstrap");
-  let bootstrapPath: string | null = null;
-  if (bootstrapRaw) {
-    const resolved = resolve(bootstrapRaw);
-    if (!existsSync(resolved)) {
-      die(`--bootstrap file not found: ${resolved}`);
-    }
-    bootstrapPath = resolved;
-  }
-
-  const workspaceRaw = flag(argv, "workspace-dir");
-  const workspaceRoot = workspaceRaw
-    ? resolve(workspaceRaw)
-    : resolve(process.cwd(), "skills-workspace");
-
-  const stageRoot = resolve(process.cwd());
-
-  const harnessRaw = flag(argv, "harness") ?? "claude-code";
-  if (harnessRaw !== "claude-code") {
-    die(`unknown --harness: ${harnessRaw}. Supported: claude-code`);
-  }
-  const harness = harnessRaw as Harness;
-
-  const siblingSkillNames: string[] = [];
-  for (const entry of readdirSync(skillDir)) {
-    if (entry === skillName) continue;
-    const sub = resolve(skillDir, entry);
-    if (!statSync(sub).isDirectory()) continue;
-    if (!existsSync(resolve(sub, "SKILL.md"))) continue;
-    siblingSkillNames.push(entry);
-  }
-
-  return {
-    skillDir,
-    skillName,
-    skillSubdir,
-    siblingSkillNames,
-    workspaceRoot,
-    stageRoot,
-    bootstrapPath,
-    harness,
-  };
-}