@supersuit/superskill 0.1.0

package/src/run/grade.mjs

@@ -0,0 +1,56 @@
+// Grading one output against one expectation. Machine checks (contains:, regex:,
+// file_exists:) cost nothing; anything else is a plain-language statement graded by one
+// extra headless call that must answer {"pass": boolean, "reason": string}.
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+
+export function isMachineCheck(a) {
+  return /^(contains|regex|file_exists):/.test(String(a));
+}
+
+export function compileRegex(pattern) {
+  let flags = "m";
+  let p = pattern;
+  const inline = p.match(/^\(\?([ims]+)\)/);
+  if (inline) { p = p.slice(inline[0].length); for (const f of inline[1]) if (!flags.includes(f)) flags += f; }
+  return new RegExp(p, flags);
+}
+
+export function gradeMachine(assertion, { output, cwd }) {
+  const a = String(assertion);
+  const i = a.indexOf(":");
+  const kind = a.slice(0, i), arg = a.slice(i + 1);
+  if (kind === "contains") return { pass: output.includes(arg), reason: `contains "${arg}"` };
+  if (kind === "regex") {
+    try { return { pass: compileRegex(arg).test(output), reason: `matches /${arg}/` }; }
+    catch (e) { return { pass: false, reason: `bad regex: ${e.message}` }; }
+  }
+  if (kind === "file_exists") return { pass: Boolean(cwd) && existsSync(join(cwd, arg)), reason: `file ${arg} exists` };
+  return { pass: false, reason: `unknown check ${kind}` };
+}
+
+export function graderPrompt(assertion, { prompt, output }) {
+  return [
+    "You are grading one output from an AI assistant against one expectation.",
+    "Answer with ONLY a JSON object: {\"pass\": true|false, \"reason\": \"<one sentence>\"}.",
+    "", "<request>", prompt, "</request>", "", "<output>", output, "</output>", "",
+    "<expectation>", String(assertion), "</expectation>",
+  ].join("\n");
+}
+
+/** Parse the grader's reply; anything that is not a clear pass is a fail. */
+export function parseVerdict(text) {
+  const m = String(text).match(/\{[\s\S]*\}/);
+  if (!m) return { pass: false, reason: "grader gave no JSON verdict" };
+  try {
+    const v = JSON.parse(m[0]);
+    return { pass: v.pass === true, reason: String(v.reason || "") };
+  } catch {
+    return { pass: false, reason: "grader verdict was not valid JSON" };
+  }
+}
+
+export function grade(assertion, ctx, harness) {
+  if (isMachineCheck(assertion)) return gradeMachine(assertion, ctx);
+  return parseVerdict(harness.ask(graderPrompt(assertion, ctx), { model: ctx.graderModel }));
+}

package/src/run/index.mjs

@@ -0,0 +1,130 @@
+// superskill doctor --run: the only path that spends model calls. Runs every eval case
+// (and every golden) N times with the skill and N times without, grades each output, and
+// writes evals/results/latest.json for the run-evidence and run-fresh rules to read.
+import { spawnSync } from "node:child_process";
+import { mkdirSync, writeFileSync, readFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { createInterface } from "node:readline/promises";
+import { clock, UsageError } from "../args.mjs";
+import { findSkills } from "../doctor.mjs";
+import { parseSkillFile } from "../frontmatter.mjs";
+import { readEvals } from "../evals.mjs";
+import { readGoldens } from "../goldens.mjs";
+import { grade, isMachineCheck } from "./grade.mjs";
+import * as claude from "./claude.mjs";
+import * as codex from "./codex.mjs";
+import { create as createFake } from "./fake.mjs";
+
+export const help = `superskill doctor <skill> --run [--harness claude|codex] [--repeat 3] [--model <id>] [--yes]
+
+Run the skill's evals for real: every case in evals/evals.json and every golden, --repeat
+times with the skill and --repeat times without it, through a headless harness. Machine
+checks (contains:, regex:, file_exists:) are free; each plain-language expectation costs
+one grader call per run. Prints the estimated number of model calls first, and asks
+before starting (or needs --yes when there is no terminal). Writes
+evals/results/latest.json.
+
+Harness: Claude Code (default when \`claude\` is on PATH) or Codex.
+`;
+
+const onPath = (bin) => spawnSync(process.platform === "win32" ? "where" : "which", [bin], { encoding: "utf8" }).status === 0;
+
+export function pickHarness(name) {
+  if (process.env.SUPERSKILL_FAKE_HARNESS) return createFake(process.env.SUPERSKILL_FAKE_HARNESS);
+  const want = name || (onPath("claude") ? "claude" : onPath("codex") ? "codex" : null);
+  if (want === "claude") return claude;
+  if (want === "codex") return codex;
+  if (!want) throw new UsageError("no harness found: install Claude Code (claude) or Codex (codex), or pass --harness");
+  throw new UsageError(`unknown harness "${want}" (claude or codex)`);
+}
+
+/** Every case the run will execute: evals.json cases plus goldens judged against their approved output. */
+export function collectCases(skillDir) {
+  const e = readEvals(skillDir);
+  if (e.error) throw new UsageError(e.error);
+  const cases = e.cases.filter((c) => c.prompt.trim()).map((c) => ({ id: String(c.id), prompt: c.prompt, files: c.files, assertions: c.assertions.length ? c.assertions : [c.expected_output].filter(Boolean) }));
+  for (const g of readGoldens(skillDir)) {
+    if (!g.input || !g.output || !g.output.trim()) continue;
+    cases.push({ id: `golden:${g.id}`, prompt: g.input, files: [], assertions: [`The output matches this approved output in substance (same facts, same shape; wording may differ):\n${g.output}`] });
+  }
+  return cases;
+}
+
+export function estimateCalls(cases, repeat) {
+  const graded = cases.reduce((n, c) => n + c.assertions.filter((a) => !isMachineCheck(a)).length, 0);
+  return { runs: cases.length * repeat * 2, grader: graded * repeat * 2, total: cases.length * repeat * 2 + graded * repeat * 2 };
+}
+
+export function runEvals(skillDir, { harness, repeat = 3, now = new Date(), model, log = () => {} }) {
+  const skillName = parseSkillFile(readFileSync(join(skillDir, "SKILL.md"), "utf8")).data.name || "";
+  const cases = collectCases(skillDir);
+  const side = () => ({ runs: 0, passes: 0, ms: 0, tokens: 0, tokenRuns: 0 });
+  const totals = { with_skill: side(), without_skill: side() };
+  const per_case = [];
+  let seenModel = model || null;
+  for (const c of cases) {
+    const row = { id: c.id, with_skill: { runs: 0, passes: 0 }, without_skill: { runs: 0, passes: 0 }, failures: [] };
+    for (const withSkill of [true, false]) {
+      const key = withSkill ? "with_skill" : "without_skill";
+      for (let i = 0; i < repeat; i++) {
+        log(`${c.id} ${withSkill ? "with" : "without"} skill, run ${i + 1}/${repeat}`);
+        const r = harness.runCase({ skillDir, skillName, prompt: c.prompt, files: c.files, withSkill, model });
+        seenModel ||= r.model;
+        const verdicts = r.failed ? [{ pass: false, reason: "harness reported an error" }] : c.assertions.map((a) => ({ a, ...grade(a, { output: r.output, cwd: r.cwd, prompt: c.prompt, graderModel: model }, harness) }));
+        const pass = verdicts.every((v) => v.pass);
+        if (r.cwd) rmSync(r.cwd, { recursive: true, force: true });
+        const t = totals[key];
+        t.runs++; t.passes += pass ? 1 : 0; t.ms += r.ms || 0;
+        if (Number.isFinite(r.tokens)) { t.tokens += r.tokens; t.tokenRuns++; }
+        row[key].runs++; row[key].passes += pass ? 1 : 0;
+        if (!pass && withSkill) row.failures.push(verdicts.filter((v) => !v.pass).map((v) => v.reason).join("; "));
+      }
+    }
+    per_case.push(row);
+  }
+  const summary = (t) => ({ pass_rate: t.runs ? round(t.passes / t.runs) : 0, mean_ms: t.runs ? Math.round(t.ms / t.runs) : 0, mean_tokens: t.tokenRuns ? Math.round(t.tokens / t.tokenRuns) : null });
+  const result = { run_at: now.toISOString(), harness: harness.name, model: seenModel, cases: cases.length, repeat, with_skill: summary(totals.with_skill), without_skill: summary(totals.without_skill), per_case };
+  mkdirSync(join(skillDir, "evals", "results"), { recursive: true });
+  writeFileSync(join(skillDir, "evals", "results", "latest.json"), JSON.stringify(result, null, 2) + "\n");
+  return result;
+}
+
+const round = (x) => Math.round(x * 1000) / 1000;
+
+export async function runCommand(a) {
+  if (a.flags.help) { process.stdout.write(help); return 0; }
+  if (!a._.length) throw new UsageError("doctor --run needs a skill folder");
+  const repeat = a.flags.repeat === undefined ? 3 : Number(a.flags.repeat);
+  if (!(Number.isInteger(repeat) && repeat > 0)) throw new UsageError("--repeat must be a positive integer");
+  const harness = pickHarness(a.flags.harness);
+  const dirs = a._.flatMap((p) => findSkills(p));
+  if (!dirs.length) throw new UsageError(`no skills found under ${a._.join(", ")}`);
+  const plan = dirs.map((d) => ({ dir: d, cases: collectCases(d) }));
+  const calls = plan.reduce((n, p) => n + estimateCalls(p.cases, repeat).total, 0);
+  for (const p of plan) {
+    const e = estimateCalls(p.cases, repeat);
+    process.stderr.write(`${p.dir}: ${p.cases.length} cases x ${repeat} x 2 = ${e.runs} runs + ${e.grader} grader calls\n`);
+  }
+  process.stderr.write(`estimated model calls: ${calls} through ${harness.name}\n`);
+  if (!a.flags.yes) {
+    if (!(process.stdin.isTTY && process.stdout.isTTY)) {
+      process.stderr.write("superskill: not starting: pass --yes to spend these calls without a terminal\n");
+      return 1;
+    }
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    const ans = (await rl.question("Start? [y/N] ")).trim().toLowerCase();
+    rl.close();
+    if (ans !== "y" && ans !== "yes") { process.stderr.write("not started\n"); return 1; }
+  }
+  const now = clock(a.flags);
+  const results = [];
+  for (const p of plan) {
+    const r = runEvals(p.dir, { harness, repeat, now, model: a.flags.model, log: (m) => process.stderr.write(`  ${m}\n`) });
+    results.push({ path: p.dir, ...r });
+    if (!a.flags.json) process.stdout.write(`${p.dir}\n  with skill ${pct(r.with_skill.pass_rate)}  without ${pct(r.without_skill.pass_rate)}  (${r.cases} cases x ${repeat})\n  wrote evals/results/latest.json\n`);
+  }
+  if (a.flags.json) process.stdout.write(JSON.stringify({ results }, null, 2) + "\n");
+  return results.every((r) => r.with_skill.pass_rate > r.without_skill.pass_rate) ? 0 : 1;
+}
+
+const pct = (x) => `${Math.round(x * 100)}%`;

package/src/run/workspace.mjs

@@ -0,0 +1,25 @@
+import { mkdtempSync, mkdirSync, symlinkSync, cpSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+
+/**
+ * A fresh working folder for one run. With `linkAt` (e.g. ".claude/skills"), the skill is
+ * symlinked in as <linkAt>/<name>; without it the folder has no skill at all. Case input
+ * files (paths relative to the skill) are copied in at the same relative paths.
+ */
+export function prepareWorkspace({ skillDir, skillName, files = [], linkAt = null }) {
+  const cwd = mkdtempSync(join(tmpdir(), "superskill-run-"));
+  if (linkAt) {
+    const target = join(cwd, linkAt, skillName);
+    mkdirSync(dirname(target), { recursive: true });
+    symlinkSync(skillDir, target, "dir");
+  }
+  for (const rel of files) {
+    const src = join(skillDir, rel);
+    if (!existsSync(src)) continue;
+    const dst = join(cwd, rel);
+    mkdirSync(dirname(dst), { recursive: true });
+    cpSync(src, dst, { recursive: true });
+  }
+  return cwd;
+}

package/src/session.mjs

@@ -0,0 +1,34 @@
+// Read the first request and the final answer out of a session record, so the job done
+// by hand once becomes a skill's first eval and first golden candidate.
+import { readFileSync } from "node:fs";
+
+const textOf = (content) => {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content.filter((b) => b && b.type === "text" && typeof b.text === "string").map((b) => b.text).join("\n");
+};
+
+/**
+ * Claude Code .jsonl: prompt = first user message whose content is text (not a tool result),
+ * output = the last assistant message that carries text. Any other file: whole file = prompt.
+ */
+export function readSession(path) {
+  const raw = readFileSync(path, "utf8");
+  if (!/\.jsonl$/i.test(path)) return { prompt: raw.trim(), output: "" };
+  let prompt = "", output = "";
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    let rec;
+    try { rec = JSON.parse(line); } catch { continue; }
+    const msg = rec.message || rec;
+    const role = msg.role || rec.type;
+    if (role === "user" && !prompt) {
+      const t = textOf(msg.content).trim();
+      if (t) prompt = t;
+    } else if (role === "assistant") {
+      const t = textOf(msg.content).trim();
+      if (t) output = t;
+    }
+  }
+  return { prompt, output };
+}

package/src/snippet.md

@@ -0,0 +1,16 @@
+## Skills (superskill)
+
+Skills in this project are held to the superskill standard: https://supersuit.wiki/concepts/superskill
+
+- **After doing a job by hand once, propose a skill for it.** If the person agrees, write the
+  skill, then seed its evidence from this session:
+  `npx @supersuit/superskill init <skill-folder> --from-session <this session's transcript>`
+  The golden it creates waits for the person to approve it; never approve it yourself.
+- **Whenever a skill needed correcting** (the person redirected you, fixed its output, or you
+  worked around it), log it before moving on:
+  `npx @supersuit/superskill miss <skill-folder> "<what happened>" --expected "<what should have>"`
+  When you fix it, add an eval that would catch it again and close it with
+  `npx @supersuit/superskill fix <skill-folder> <miss-id> --eval <eval-id>`.
+- **Before calling a skill done**, run
+  `npx @supersuit/superskill doctor <skill-folder>`
+  and fix everything it lists for the level the skill is meant to reach.