@supersuit/superskill 0.1.0

package/src/rules/skill.mjs

@@ -0,0 +1,359 @@
+// Level 1, "skill": spec-valid (agentskills.io) and hygienic. Every rule is a pure
+// function of the loaded context. A rule returns [] when it has nothing to say.
+
+import { existsSync, statSync } from "node:fs";
+import { join, dirname, normalize } from "node:path";
+import { readText } from "../context.mjs";
+import { defineRules } from "./define.mjs";
+
+const NAME_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/;
+const TEXT_EXT = /\.(md|mdx|txt|mjs|cjs|js|ts|py|sh|bash|zsh|rb|json|ya?ml|toml)$/i;
+const DATA_DIRS = /^(evals|goldens)\//;
+const TEST_FILE = /(^|\/)(tests?|__tests__)\/|(^|\/)test_[^/]*\.py$|_test\.py$|\.test\.[cm]?[jt]s$|\.spec\.[cm]?[jt]s$/;
+const IGNORE = "superskill-ignore";
+const DATA_URI = /data:[a-z]+\/[a-z0-9+.-]+;base64,[A-Za-z0-9+/=]+/gi;
+
+const f = (severity, message, fix, extra = {}) => ({ severity, message, fix, ...extra });
+const FOLD_TOKENS = 5000;
+// A hard rule is shouted (NEVER, ALWAYS, MUST) or bolded as a command (**Never ...**).
+const HARD_RULE = { test: (l) => /\b(NEVER|ALWAYS|MUST|DO NOT|DON'T|REFUSES?)\b/.test(l) || /\*\*(never|always|do not|don't|refuse)\b/i.test(l) };
+
+/** Body lines tagged with whether they sit inside a code fence. */
+function proseLines(body) {
+  let fenced = false;
+  return body.split("\n").map((line) => {
+    if (/^\s*(```|~~~)/.test(line)) { fenced = !fenced; return { line, fenced: true }; }
+    return { line, fenced };
+  });
+}
+const normRule = (line) => line.toLowerCase().replace(/[*_`>#-]/g, "").replace(/\s+/g, " ").trim();
+const broken = (ctx) => Boolean(ctx.error || ctx.parseError);
+const str = (v) => (typeof v === "string" ? v : "");
+
+/** Markdown files that are instructions (not evidence) inside the skill. */
+function instructionDocs(ctx) {
+  return ctx.files.filter((p) => /\.mdx?$/i.test(p) && !DATA_DIRS.test(p) && p !== "MISSES.md");
+}
+
+/** Local link targets in a markdown text, relative to `fromRel`'s folder. */
+export function localLinks(text, fromRel) {
+  const out = [];
+  const re = /\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
+  let m;
+  while ((m = re.exec(text))) {
+    let target = m[1].split("#")[0];
+    if (!target || /^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith("/") || target.startsWith("~")) continue;
+    try { target = decodeURIComponent(target); } catch {}
+    const rel = normalize(join(dirname(fromRel), target)).split("\\").join("/");
+    if (rel.startsWith("..")) continue;
+    out.push(rel);
+  }
+  return out;
+}
+
+const isFile = (dir, rel) => {
+  try { return statSync(join(dir, rel)).isFile(); } catch { return false; }
+};
+
+export const skillRules = defineRules([
+  {
+    id: "frontmatter-valid",
+    level: "skill",
+    check(ctx) {
+      if (ctx.error) return [f("fail", ctx.error, "Add a SKILL.md with YAML frontmatter (name, description).")];
+      if (ctx.parseError) return [f("fail", `SKILL.md ${ctx.parseError}`, "Open SKILL.md with a --- fenced YAML block holding name and description.")];
+      return [];
+    },
+  },
+  {
+    id: "name-format",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const name = str(ctx.data.name);
+      if (!name) return [f("fail", "frontmatter has no name", "Add `name:` matching the folder name.")];
+      if (name.length > 64 || !NAME_RE.test(name))
+        return [f("fail", `name "${name}" is not 1-64 lowercase letters, digits and single hyphens`, "Use lowercase-words-joined-by-single-hyphens, at most 64 characters.")];
+      return [];
+    },
+  },
+  {
+    id: "name-matches-folder",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx) || !str(ctx.data.name)) return [];
+      if (ctx.data.name !== ctx.folderName)
+        return [f("fail", `name "${ctx.data.name}" does not match folder "${ctx.folderName}"`, "Rename the folder or the name so they are identical.")];
+      return [];
+    },
+  },
+  {
+    id: "name-reserved-words",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const name = str(ctx.data.name).toLowerCase();
+      const hit = ["anthropic", "claude"].find((w) => name.includes(w));
+      return hit ? [f("fail", `name contains the reserved word "${hit}"`, "Pick a name without \"anthropic\" or \"claude\".")] : [];
+    },
+  },
+  {
+    id: "description-length",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const d = str(ctx.data.description).trim();
+      if (!d) return [f("fail", "frontmatter has no description", "Add a description that says what the skill does and when to use it.")];
+      if (d.length > 1024) return [f("fail", `description is ${d.length} characters (limit 1024)`, "Cut it to 1024 characters or fewer, main use case first.")];
+      return [];
+    },
+  },
+  {
+    id: "description-has-trigger",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const d = str(ctx.data.description);
+      if (!d.trim()) return [];
+      if (/\bwhen\b|\btrigger(s|ed)?\b|\bfor requests?\b/i.test(d) || str(ctx.data.when_to_use).trim()) return [];
+      return [f("warn", "description never says when to use the skill", "Add a clause like \"Use when ...\" so an agent knows when to load it.")];
+    },
+  },
+  {
+    id: "description-no-xml",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const tag = str(ctx.data.description).match(/<\/?[a-zA-Z][^>]*>/);
+      return tag
+        ? [f("fail", `description contains an XML/HTML tag: ${tag[0]}`, "Remove angle brackets from the description (write a placeholder as {slug} or SLUG, not <slug>); skill descriptions must not contain XML tags.")]
+        : [];
+    },
+  },
+  {
+    id: "compatibility-length",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const c = str(ctx.data.compatibility);
+      return c.length > 500 ? [f("fail", `compatibility is ${c.length} characters (limit 500)`, "Shorten compatibility to 500 characters or fewer.")] : [];
+    },
+  },
+  {
+    id: "metadata-string-map",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx) || !("metadata" in ctx.data)) return [];
+      const m = ctx.data.metadata;
+      if (m === "") return [];
+      if (typeof m !== "object" || Array.isArray(m) || m === null)
+        return [f("fail", "metadata is not a map of string keys to string values", "Write metadata as indented `key: value` lines.")];
+      const bad = Object.entries(m).filter(([, v]) => typeof v !== "string").map(([k]) => k);
+      return bad.length ? [f("fail", `metadata values must be strings: ${bad.join(", ")}`, "Quote each metadata value.")] : [];
+    },
+  },
+  // Length is never a defect on its own. A long skill that is well shaped costs tokens only
+  // when it is invoked; what actually breaks is a long skill in the wrong shape. After
+  // compaction Claude Code keeps only the first ~5,000 tokens of each invoked skill, so the
+  // three rules below check what survives that cut, whether an agent can find its way
+  // around, and whether anything is said twice.
+  {
+    id: "body-size",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const est = Math.round(ctx.body.length / 4);
+      if (est <= FOLD_TOKENS) return [];
+      const n = ctx.body.replace(/\n+$/, "").split("\n").length;
+      return [f("info", `body is ${n} lines, about ${est} tokens; after compaction only the first ~${FOLD_TOKENS} survive`, "Fine if the hard rules sit above that point (see rules-above-the-fold).")];
+    },
+  },
+  {
+    id: "rules-above-the-fold",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const fold = FOLD_TOKENS * 4;
+      if (ctx.body.length <= fold) return [];
+      const above = new Set();
+      const late = [];
+      let offset = 0;
+      let lineNo = 0;
+      for (const { line, fenced } of proseLines(ctx.body)) {
+        lineNo++;
+        const pos = offset;
+        offset += line.length + 1;
+        if (fenced || !HARD_RULE.test(line)) continue;
+        const key = normRule(line);
+        if (!key) continue;
+        if (pos < fold) above.add(key);
+        else if (!above.has(key)) late.push({ lineNo, text: line.trim() });
+      }
+      if (!late.length) return [];
+      const shown = late.slice(0, 3).map((r) => `line ${r.lineNo}: ${r.text.slice(0, 80)}`).join("; ");
+      return [f("warn", `${late.length} hard rule(s) sit past the first ~${FOLD_TOKENS} tokens and would not survive compaction (${shown})`, "Restate them in a short Rules section near the top, or move them up. Length is fine; the rules just need to be above the fold. Step-specific detail can move into step files (steps/<step>.md) that are read fresh when the step comes up.")];
+    },
+  },
+  {
+    id: "navigable",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const lines = proseLines(ctx.body);
+      if (lines.length <= 300) return [];
+      let run = 0, worst = 0, worstStart = 0, start = 1, i = 0;
+      for (const { line, fenced } of lines) {
+        i++;
+        if (!fenced && /^#{1,6}\s/.test(line)) { run = 0; start = i + 1; continue; }
+        run++;
+        if (run > worst) { worst = run; worstStart = start; }
+      }
+      return worst > 150
+        ? [f("warn", `${worst} lines run with no heading (from body line ${worstStart})`, "Break it up with headings, or move a step's detail into its own step file (steps/<step>.md) and leave a pointer that says when to read it: \"Before step 4, read steps/4-reconcile.md.\"")]
+        : [];
+    },
+  },
+  {
+    id: "no-repeated-paragraphs",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const seen = new Map();
+      const dups = [];
+      let para = [];
+      const flush = () => {
+        const text = para.join(" ").toLowerCase().replace(/[*_`>#-]/g, "").replace(/\s+/g, " ").trim();
+        para = [];
+        if (text.length < 100) return;
+        if (seen.has(text)) dups.push(text);
+        else seen.set(text, true);
+      };
+      for (const { line, fenced } of proseLines(ctx.body)) {
+        if (fenced) { flush(); continue; }
+        if (!line.trim() || /^#{1,6}\s/.test(line)) flush();
+        else para.push(line);
+      }
+      flush();
+      return dups.length
+        ? [f("warn", `${dups.length} paragraph(s) appear more than once (first: "${dups[0].slice(0, 70)}...")`, "Keep one copy and point to it; two copies drift apart the first time one is edited.")]
+        : [];
+    },
+  },
+  {
+    id: "references-one-deep",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const out = [];
+      const direct = new Set(localLinks(ctx.body, "SKILL.md").filter((p) => isFile(ctx.dir, p)));
+      for (const ref of direct) {
+        if (!/\.mdx?$/i.test(ref)) continue;
+        const text = readText(ctx.dir, ref) || "";
+        const chained = localLinks(text, ref).filter((p) => p !== "SKILL.md" && !direct.has(p) && isFile(ctx.dir, p));
+        if (chained.length)
+          out.push(f("fail", `${ref} links on to ${chained.join(", ")} (references must be one level deep)`, `Link ${chained[0]} directly from SKILL.md, or fold it into ${ref}.`, { file: ref }));
+      }
+      return out;
+    },
+  },
+  {
+    // A step file only works if the agent knows when to open it: a bare link gets skipped
+    // or read in part. The pointer line has to carry the condition.
+    id: "reference-says-when",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const out = [];
+      for (const { line, fenced } of proseLines(ctx.body)) {
+        if (fenced) continue;
+        for (const target of localLinks(line, "SKILL.md")) {
+          if (!/\.mdx?$/i.test(target) || !isFile(ctx.dir, target)) continue;
+          const words = line.replace(/\[[^\]]*\]\([^)]*\)/g, " ");
+          if (!/\b(when|before|after|if|during|for|to|while|once|read|load|follow)\b/i.test(words))
+            out.push(f("warn", `SKILL.md links ${target} without saying when to read it`, `Put the condition on the pointer line: "Before <step>, read ${target}." or "For <case>, see ${target}."`, { file: target }));
+        }
+      }
+      return out;
+    },
+  },
+  {
+    id: "long-reference-toc",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const out = [];
+      for (const p of instructionDocs(ctx)) {
+        // HDSOP.md is Freedom's workflow map, written for a person reviewing the process,
+        // not a reference an agent loads in part.
+        if (p === "SKILL.md" || p === "HDSOP.md") continue;
+        const lines = (readText(ctx.dir, p) || "").split("\n");
+        if (lines.length <= 100) continue;
+        const head = lines.slice(0, 30);
+        const toc = head.some((l) => /contents/i.test(l)) || head.filter((l) => /^\s*[-*]\s*\[[^\]]+\]\(#/.test(l)).length >= 3;
+        if (!toc) out.push(f("warn", `${p} is ${lines.length} lines with no table of contents`, "Add a Contents list in the first 30 lines so an agent can jump to the part it needs.", { file: p }));
+      }
+      return out;
+    },
+  },
+  {
+    id: "no-absolute-paths",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const out = [];
+      // The path must start the token (not "capture/home/Library") and name something
+      // ("/Users/..." as a placeholder in prose is not a path).
+      const re = /(^|[\s"'`(=:,[{<])(\/Users\/[A-Za-z0-9_]|\/home\/[A-Za-z0-9_]|[A-Z]:\\{1,2}[A-Za-z])/;
+      for (const p of ctx.files) {
+        // Tests use fake machine paths as data; they are not paths the skill depends on.
+        if (!TEXT_EXT.test(p) || DATA_DIRS.test(p) || TEST_FILE.test(p)) continue;
+        const lines = (readText(ctx.dir, p) || "").split("\n");
+        const hits = [];
+        lines.forEach((l, i) => { if (re.test(l) && !l.includes(IGNORE)) hits.push(i + 1); });
+        if (hits.length)
+          out.push(f("fail", `${p}:${hits[0]} hard-codes a machine path${hits.length > 1 ? ` (${hits.length} lines)` : ""}`, "Use a path relative to the skill folder, ~, or an environment variable.", { file: p, line: hits[0] }));
+      }
+      return out;
+    },
+  },
+  {
+    id: "injection-scan",
+    level: "skill",
+    check(ctx) {
+      if (broken(ctx)) return [];
+      const out = [];
+      const patterns = [
+        ["override phrase", /\b(ignore|disregard|forget)\s+(all\s+|any\s+)?(the\s+)?(previous|prior|above|earlier|preceding)?\s*(instructions|prompts?|rules)\b/i, (m) => /(previous|prior|above|earlier|preceding|all|any)/i.test(m)],
+        ["override phrase", /\bdisregard\s+(the\s+)?system\s+prompt\b/i],
+        ["download piped to a shell", /\b(curl|wget)\b[^\n|]*\|\s*(sudo\s+)?(ba|z)?sh\b/i],
+        ["long base64 blob", /[A-Za-z0-9+/]{200,}={0,2}/],
+      ];
+      for (const p of instructionDocs(ctx)) {
+        const text = readText(ctx.dir, p) || "";
+        const lines = text.split("\n");
+        for (const [label, re, confirm] of patterns) {
+          const i = lines.findIndex((l) => {
+            // An inline data: URI (an embedded image) is content, not a hidden payload.
+            const probe = l.replace(DATA_URI, "");
+            const m = probe.match(re);
+            return m && !l.includes(IGNORE) && (!confirm || confirm(m[0]));
+          });
+          if (i >= 0) out.push(f("fail", `${p}:${i + 1} ${label}: "${lines[i].trim().slice(0, 80)}"`, "Remove it, or mark a deliberate example with `superskill-ignore` on the same line.", { file: p, line: i + 1 }));
+        }
+        const comments = text.matchAll(/<!--([\s\S]*?)-->/g);
+        for (const c of comments) {
+          const body = c[1];
+          if (body.includes(IGNORE)) continue;
+          const addressed = /^\s*(assistant|system|ai|agent|claude|model)\s*[:,]/i.test(body);
+          const action = /\b(ignore|disregard|send|upload|exfiltrate|post|curl|wget|read|execute|run|delete)\b/i.test(body);
+          const target = /(https?:\/\/|~\/|\.ssh|\.env\b|credential|token|password|secret|api[_ -]?key)/i.test(body);
+          if (addressed || (action && target)) {
+            const line = text.slice(0, c.index).split("\n").length;
+            out.push(f("fail", `${p}:${line} hidden HTML comment gives the agent instructions`, "Delete the comment; instructions belong in visible text.", { file: p, line }));
+          }
+        }
+      }
+      return out;
+    },
+  },
+]);

package/src/rules/superskill.mjs

@@ -0,0 +1,113 @@
+// Level 3, "superskill": checked against examples a person approved, fixed every time it
+// got something wrong, and proven recently on a current model against the no-skill baseline.
+import { createHash } from "node:crypto";
+import { defineRules } from "./define.mjs";
+import { readGoldens, isApproved } from "../goldens.mjs";
+import { readMisses } from "../misses.mjs";
+import { readEvals, readLatestRun } from "../evals.mjs";
+
+const f = (severity, message, fix) => ({ severity, message, fix });
+const DAY = 86400000;
+export const OPEN_MISS_DAYS = 14;
+/** How long a --run stays fresh, by metadata.cadence. */
+export const FRESH_DAYS = { daily: 30, weekly: 30, monthly: 60, quarterly: 120, yearly: 365 };
+const DEFAULT_FRESH = 30;
+
+const days = (now, date) => Math.floor((now.getTime() - new Date(date).getTime()) / DAY);
+
+export const superskillRules = defineRules([
+  {
+    id: "golden-approved",
+    level: "superskill",
+    check(ctx) {
+      if (ctx.error) return [];
+      const goldens = readGoldens(ctx.dir);
+      const approved = goldens.filter(isApproved);
+      const out = [];
+      for (const g of goldens.filter((x) => x.approvalError))
+        out.push(f("fail", `goldens/${g.id}/APPROVAL.json is not valid JSON`, `Re-record it with \`superskill approve . ${g.id}\`.`));
+      if (!approved.length) {
+        out.push(f("fail", goldens.length ? `${goldens.length} golden${goldens.length === 1 ? "" : "s"}, none approved by a person` : "no goldens", goldens.length ? "A person runs `superskill approve <skill> <golden>` at a terminal after checking the output." : "Save a real input and the output you would sign off on under goldens/<id>/, then `superskill approve`."));
+        return out;
+      }
+      const sha = createHash("sha256").update(ctx.raw).digest("hex");
+      const stale = approved.filter((g) => g.approval.skill_sha && g.approval.skill_sha !== sha).map((g) => g.id);
+      if (stale.length) out.push(f("info", `golden${stale.length === 1 ? "" : "s"} ${stale.join(", ")} approved against an earlier SKILL.md`, "Re-run the golden and re-approve if the output still holds."));
+      return out;
+    },
+  },
+  {
+    id: "misses-log-present",
+    level: "superskill",
+    check(ctx) {
+      if (ctx.error) return [];
+      return readMisses(ctx.dir) === null ? [f("fail", "no MISSES.md", "Run `superskill init` to add one; log each correction with `superskill miss`.")] : [];
+    },
+  },
+  {
+    id: "no-stale-open-miss",
+    level: "superskill",
+    check(ctx, { now = new Date() } = {}) {
+      if (ctx.error) return [];
+      const misses = readMisses(ctx.dir) || [];
+      const out = [];
+      for (const m of misses.filter((x) => x.status === "open")) {
+        const age = days(now, m.date);
+        if (age > OPEN_MISS_DAYS) out.push(f("fail", `miss ${m.id} open ${age} days (limit ${OPEN_MISS_DAYS}): ${m.what.slice(0, 80)}`, `Fix it, add an eval that catches it, then \`superskill fix <skill> ${m.id} --eval <id>\`.`));
+        else out.push(f("info", `miss ${m.id} open ${age} day${age === 1 ? "" : "s"}`, `Fix within ${OPEN_MISS_DAYS} days of ${m.date}.`));
+      }
+      return out;
+    },
+  },
+  {
+    id: "fixed-miss-has-eval",
+    level: "superskill",
+    check(ctx) {
+      if (ctx.error) return [];
+      const misses = (readMisses(ctx.dir) || []).filter((m) => m.status === "fixed");
+      if (!misses.length) return [];
+      const ids = new Set(readEvals(ctx.dir).cases.map((c) => String(c.id)));
+      for (const g of readGoldens(ctx.dir)) ids.add(g.id);
+      return misses
+        .filter((m) => !m.eval || !ids.has(String(m.eval)))
+        .map((m) => f("fail", m.eval ? `miss ${m.id} names eval "${m.eval}", which is not in evals.json or goldens/` : `miss ${m.id} is fixed with no regression eval`, `Add a case to evals/evals.json that would catch ${m.id} again, and name its id on the Eval line.`));
+    },
+  },
+  {
+    id: "run-evidence",
+    level: "superskill",
+    check(ctx) {
+      if (ctx.error) return [];
+      const r = readLatestRun(ctx.dir);
+      if (r.missing) return [f("fail", "no evals/results/latest.json", "Run `superskill doctor <skill> --run` to run the evals with and without the skill.")];
+      if (r.error) return [f("fail", r.error, "Re-run `superskill doctor --run`.")];
+      const w = Number(r.run?.with_skill?.pass_rate), wo = Number(r.run?.without_skill?.pass_rate);
+      if (!Number.isFinite(w) || !Number.isFinite(wo)) return [f("fail", "latest.json has no with_skill / without_skill pass rates", "Re-run `superskill doctor --run`.")];
+      if (!(w > wo)) return [f("fail", `with the skill ${pct(w)} vs without ${pct(wo)}: the skill does not beat the baseline`, "Improve the skill (or its evals) until it clearly beats running the task without it.")];
+      return [];
+    },
+  },
+  {
+    id: "run-fresh",
+    level: "superskill",
+    check(ctx, { now = new Date() } = {}) {
+      if (ctx.error) return [];
+      const r = readLatestRun(ctx.dir);
+      if (!r.run) return [];
+      const at = r.run.run_at;
+      if (!at || Number.isNaN(new Date(at).getTime())) return [f("fail", "latest.json has no valid run_at", "Re-run `superskill doctor --run`.")];
+      const meta = ctx.data.metadata && typeof ctx.data.metadata === "object" ? ctx.data.metadata : {};
+      const cadence = String(meta.cadence || "").toLowerCase();
+      const out = [];
+      if (cadence && !(cadence in FRESH_DAYS)) out.push(f("warn", `unknown metadata.cadence "${cadence}"`, `Use one of ${Object.keys(FRESH_DAYS).join(", ")}.`));
+      if (!cadence) out.push(f("info", "no metadata.cadence; treating the proof as fresh for 30 days", "Declare how often the skill runs: metadata.cadence: daily|weekly|monthly|quarterly|yearly."));
+      const limit = FRESH_DAYS[cadence] ?? DEFAULT_FRESH;
+      const age = days(now, at);
+      if (age > limit) out.push(f("fail", `last --run was ${age} days ago (fresh for ${limit} at ${cadence || "default"} cadence)`, "Run `superskill doctor <skill> --run` again."));
+      else if (cadence === "yearly") out.push(f("warn", `yearly skill: last --run ${age} days ago`, "Run `superskill doctor <skill> --run` before its next real use; a once-a-year skill is otherwise only tested the day it is needed."));
+      return out;
+    },
+  },
+]);
+
+const pct = (x) => `${Math.round(x * 100)}%`;

package/src/rules/tested.mjs

@@ -0,0 +1,52 @@
+// Level 2, "tested": at least three task evals a machine can check, and a trigger set
+// with both should-load requests and near-misses that should not load the skill.
+import { defineRules } from "./define.mjs";
+import { readEvals, readTriggers } from "../evals.mjs";
+import { readGoldens } from "../goldens.mjs";
+
+const f = (severity, message, fix) => ({ severity, message, fix });
+export const MIN_EVALS = 3, MIN_TRIGGERS = 10, MIN_EACH_SIDE = 3;
+
+export const testedRules = defineRules([
+  {
+    id: "evals-present",
+    level: "tested",
+    check(ctx) {
+      if (ctx.error) return [];
+      const e = readEvals(ctx.dir);
+      if (e.error) return [f("fail", e.error, "Fix evals/evals.json so it parses as skill-creator's {skill_name, evals: [...]}.")];
+      const goldenCases = readGoldens(ctx.dir).filter((g) => g.input !== null && g.output !== null).length;
+      const n = e.cases.length + goldenCases;
+      if (n < MIN_EVALS)
+        return [f("fail", `${n} eval case${n === 1 ? "" : "s"} (need ${MIN_EVALS})`, "Add real requests to evals/evals.json (`superskill init` writes an example).")];
+      return [];
+    },
+  },
+  {
+    id: "evals-verifiable",
+    level: "tested",
+    check(ctx) {
+      if (ctx.error) return [];
+      const e = readEvals(ctx.dir);
+      if (e.error || e.missing) return [];
+      const bad = e.cases.filter((c) => !c.prompt.trim() || !c.assertions.length).map((c) => c.id);
+      return bad.length
+        ? [f("fail", `eval case${bad.length === 1 ? "" : "s"} ${bad.join(", ")} missing a prompt or any expectation`, "Give every case a prompt and at least one expectation (contains:, regex:, file_exists:, or a plain statement).")]
+        : [];
+    },
+  },
+  {
+    id: "triggers-present",
+    level: "tested",
+    check(ctx) {
+      if (ctx.error) return [];
+      const t = readTriggers(ctx.dir);
+      if (t.error) return [f("fail", t.error, "Write evals/triggers.json as [{\"query\": ..., \"should_trigger\": true|false}].")];
+      const yes = t.triggers.filter((x) => x.should_trigger).length;
+      const no = t.triggers.length - yes;
+      if (t.triggers.length < MIN_TRIGGERS || yes < MIN_EACH_SIDE || no < MIN_EACH_SIDE)
+        return [f("fail", `trigger set has ${yes} should-load and ${no} should-not (need ${MIN_TRIGGERS} total, at least ${MIN_EACH_SIDE} of each)`, "Add realistic requests to evals/triggers.json, including near-misses that share words with the skill but need something else.")];
+      return [];
+    },
+  },
+]);

package/src/run/claude.mjs

@@ -0,0 +1,36 @@
+// Claude Code headless. With the skill: the run folder carries .claude/skills/<name> as a
+// symlink. Without: the same kind of folder with no skill, and --disable-slash-commands so
+// a copy installed at user level cannot leak into the baseline.
+import { spawnSync } from "node:child_process";
+import { prepareWorkspace } from "./workspace.mjs";
+
+export const name = "claude";
+
+function call(args, cwd) {
+  const t0 = Date.now();
+  const r = spawnSync("claude", args, { cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
+  const ms = Date.now() - t0;
+  if (r.error) throw Object.assign(new Error(`claude failed to start: ${r.error.message}`), { code: "SUPERSKILL" });
+  let doc = null;
+  try { doc = JSON.parse(r.stdout); } catch {}
+  if (!doc) return { output: r.stdout || r.stderr || "", ms, tokens: null, model: null, failed: r.status !== 0 };
+  const u = doc.usage || {};
+  const tokens = ["input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens"].reduce((n, k) => n + (Number(u[k]) || 0), 0) || null;
+  const model = doc.modelUsage ? Object.keys(doc.modelUsage)[0] || null : doc.model || null;
+  return { output: typeof doc.result === "string" ? doc.result : "", ms: doc.duration_ms || ms, tokens, model, failed: doc.is_error === true };
+}
+
+export function runCase({ skillDir, skillName, prompt, files, withSkill, model }) {
+  const cwd = prepareWorkspace({ skillDir, skillName, files, linkAt: withSkill ? ".claude/skills" : null });
+  const args = ["-p", prompt, "--output-format", "json", "--add-dir", cwd];
+  if (!withSkill) args.push("--disable-slash-commands");
+  if (model) args.push("--model", model);
+  return { ...call(args, cwd), cwd };
+}
+
+export function ask(prompt, { model } = {}) {
+  const cwd = prepareWorkspace({ skillDir: null, skillName: null });
+  const args = ["-p", prompt, "--output-format", "json", "--disable-slash-commands"];
+  if (model) args.push("--model", model);
+  return call(args, cwd).output;
+}

package/src/run/codex.mjs

@@ -0,0 +1,32 @@
+// Codex headless. With the skill: .agents/skills/<name> symlinked into the run folder.
+// Without: no skill in the folder. (A copy installed at user level can still load; Codex
+// has no switch to disable skills, so keep the skill out of ~/.agents/skills when proving it.)
+import { spawnSync } from "node:child_process";
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { prepareWorkspace } from "./workspace.mjs";
+
+export const name = "codex";
+
+function call(prompt, cwd, model) {
+  const out = join(cwd, ".superskill-last-message.txt");
+  const args = ["exec", "--skip-git-repo-check", "-C", cwd, "-o", out];
+  if (model) args.push("-m", model);
+  args.push(prompt);
+  const t0 = Date.now();
+  const r = spawnSync("codex", args, { cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 15 * 60 * 1000 });
+  if (r.error) throw Object.assign(new Error(`codex failed to start: ${r.error.message}`), { code: "SUPERSKILL" });
+  const output = existsSync(out) ? readFileSync(out, "utf8") : r.stdout;
+  const tok = (r.stderr + r.stdout).match(/tokens used[:\s]+([\d,]+)/i);
+  return { output, ms: Date.now() - t0, tokens: tok ? Number(tok[1].replace(/,/g, "")) : null, model: model || null, failed: r.status !== 0 };
+}
+
+export function runCase({ skillDir, skillName, prompt, files, withSkill, model }) {
+  const cwd = prepareWorkspace({ skillDir, skillName, files, linkAt: withSkill ? ".agents/skills" : null });
+  return { ...call(prompt, cwd, model), cwd };
+}
+
+export function ask(prompt, { model } = {}) {
+  const cwd = prepareWorkspace({ skillDir: null, skillName: null });
+  return call(prompt, cwd, model).output;
+}

package/src/run/fake.mjs

@@ -0,0 +1,28 @@
+// A stand-in harness for tests: SUPERSKILL_FAKE_HARNESS names a node script that reads one
+// JSON request on stdin ({mode: "case"|"ask", prompt, withSkill, cwd}) and prints
+// {output, tokens?, model?}. Never calls a model.
+import { spawnSync } from "node:child_process";
+import { prepareWorkspace } from "./workspace.mjs";
+
+export const name = "fake";
+
+function call(script, req) {
+  const t0 = Date.now();
+  const r = spawnSync(process.execPath, [script], { input: JSON.stringify(req), encoding: "utf8" });
+  if (r.status !== 0) throw Object.assign(new Error(`fake harness failed: ${r.stderr}`), { code: "SUPERSKILL" });
+  const doc = JSON.parse(r.stdout);
+  return { output: doc.output ?? "", tokens: doc.tokens ?? null, model: doc.model ?? "fake", ms: Date.now() - t0, failed: false };
+}
+
+export function create(script) {
+  return {
+    name,
+    runCase({ skillDir, skillName, prompt, files, withSkill }) {
+      const cwd = prepareWorkspace({ skillDir, skillName, files, linkAt: withSkill ? ".claude/skills" : null });
+      return { ...call(script, { mode: "case", prompt, withSkill, cwd }), cwd };
+    },
+    ask(prompt) {
+      return call(script, { mode: "ask", prompt }).output;
+    },
+  };
+}