@supersuit/superskill 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/LICENSE +21 -0
- package/README.md +103 -0
- package/SPEC.md +239 -0
- package/bin/superskill.mjs +56 -0
- package/package.json +15 -0
- package/src/args.mjs +30 -0
- package/src/changed.mjs +52 -0
- package/src/collection.mjs +139 -0
- package/src/commands/approve.mjs +41 -0
- package/src/commands/collection.mjs +56 -0
- package/src/commands/common.mjs +19 -0
- package/src/commands/doctor.mjs +63 -0
- package/src/commands/fix.mjs +29 -0
- package/src/commands/import.mjs +42 -0
- package/src/commands/init.mjs +74 -0
- package/src/commands/miss.mjs +24 -0
- package/src/commands/snippet.mjs +14 -0
- package/src/context.mjs +60 -0
- package/src/doctor.mjs +65 -0
- package/src/evals.mjs +44 -0
- package/src/frontmatter.mjs +109 -0
- package/src/goldens.mjs +34 -0
- package/src/ledger.mjs +44 -0
- package/src/levels.mjs +38 -0
- package/src/misses.mjs +88 -0
- package/src/report.mjs +27 -0
- package/src/rules/define.mjs +9 -0
- package/src/rules/index.mjs +7 -0
- package/src/rules/interop.mjs +13 -0
- package/src/rules/skill.mjs +359 -0
- package/src/rules/superskill.mjs +113 -0
- package/src/rules/tested.mjs +52 -0
- package/src/run/claude.mjs +36 -0
- package/src/run/codex.mjs +32 -0
- package/src/run/fake.mjs +28 -0
- package/src/run/grade.mjs +56 -0
- package/src/run/index.mjs +130 -0
- package/src/run/workspace.mjs +25 -0
- package/src/session.mjs +34 -0
- package/src/snippet.md +16 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
// Checks that only make sense for a SET of skills: how much of a harness's skill-listing
|
|
2
|
+
// budget they use, which entries get cut off, and which pairs overlap enough that an
|
|
3
|
+
// agent could load the wrong one. Plus the plugin line for a packaged set.
|
|
4
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import { join, relative, sep } from "node:path";
|
|
7
|
+
import { findSkills } from "./doctor.mjs";
|
|
8
|
+
import { loadSkill, listFiles } from "./context.mjs";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Characters of description + when_to_use a harness shows per skill before cutting it off.
|
|
12
|
+
* Claude Code documents this (code.claude.com/docs/en/skills, read 2026-09-28).
|
|
13
|
+
*/
|
|
14
|
+
export const ENTRY_CAP = 1536;
|
|
15
|
+
/**
|
|
16
|
+
* Default whole-listing budgets, in characters. Neither vendor publishes this number (checked
|
|
17
|
+
* 2026-09-28), so it is a conservative default, and --budget overrides it.
|
|
18
|
+
*/
|
|
19
|
+
export const BUDGETS = { "claude-code": 8000, codex: 8000 };
|
|
20
|
+
export const OVERLAP = 0.5;
|
|
21
|
+
|
|
22
|
+
const STOP = new Set(("a an and are as at be by can do does for from has have how i if in into is it its of on or " +
|
|
23
|
+
"so that the their them then there these this to use used uses using via was what when where which who will " +
|
|
24
|
+
"with you your someone asks ask user users wants want skill skills not any all also one more other").split(" "));
|
|
25
|
+
|
|
26
|
+
/** The text a skill costs in the listing: its description plus when_to_use. */
|
|
27
|
+
export function listingText(data) {
|
|
28
|
+
const s = (v) => (typeof v === "string" ? v.trim() : "");
|
|
29
|
+
return [s(data.description), s(data.when_to_use)].filter(Boolean).join(" ");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function listingEntry(data) {
|
|
33
|
+
const name = typeof data.name === "string" ? data.name.trim() : "";
|
|
34
|
+
return `${name}: ${listingText(data)}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function tokens(text) {
|
|
38
|
+
return new Set(String(text).toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 2 && !STOP.has(t)));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function jaccard(a, b) {
|
|
42
|
+
const A = tokens(a), B = tokens(b);
|
|
43
|
+
if (!A.size || !B.size) return 0;
|
|
44
|
+
let inter = 0;
|
|
45
|
+
for (const t of A) if (B.has(t)) inter++;
|
|
46
|
+
return inter / (A.size + B.size - inter);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* collection(paths, {budget, overlap}) -> {skills, total_chars, budgets, truncated, overlaps, plugin}
|
|
51
|
+
* `listed_chars` is what the harness actually shows (capped per entry); totals use it.
|
|
52
|
+
*/
|
|
53
|
+
export function collection(paths, opts = {}) {
|
|
54
|
+
const dirs = [];
|
|
55
|
+
for (const p of paths) dirs.push(...findSkills(p));
|
|
56
|
+
const seen = new Set();
|
|
57
|
+
const skills = [];
|
|
58
|
+
for (const d of dirs) {
|
|
59
|
+
if (seen.has(d)) continue;
|
|
60
|
+
seen.add(d);
|
|
61
|
+
const ctx = loadSkill(d);
|
|
62
|
+
const name = typeof ctx.data.name === "string" && ctx.data.name ? ctx.data.name : ctx.folderName;
|
|
63
|
+
const text = listingText(ctx.data);
|
|
64
|
+
const head = name.length + 2; // "name: "
|
|
65
|
+
skills.push({ name, path: d, chars: head + text.length, listed_chars: head + Math.min(text.length, ENTRY_CAP), description_chars: text.length, description: typeof ctx.data.description === "string" ? ctx.data.description : "" });
|
|
66
|
+
}
|
|
67
|
+
const total = skills.reduce((n, s) => n + s.listed_chars, 0);
|
|
68
|
+
const limits = opts.budget ? Object.fromEntries(Object.keys(BUDGETS).map((k) => [k, Number(opts.budget)])) : BUDGETS;
|
|
69
|
+
const budgets = Object.entries(limits).map(([harness, limit]) => ({ harness, limit, used: total, over: Math.max(0, total - limit) }));
|
|
70
|
+
const truncated = skills.filter((s) => s.description_chars > ENTRY_CAP).map((s) => ({ name: s.name, chars: s.description_chars, cut: s.description_chars - ENTRY_CAP }));
|
|
71
|
+
|
|
72
|
+
const threshold = opts.overlap ?? OVERLAP;
|
|
73
|
+
const overlaps = [];
|
|
74
|
+
for (let i = 0; i < skills.length; i++) {
|
|
75
|
+
for (let j = i + 1; j < skills.length; j++) {
|
|
76
|
+
const a = skills[i], b = skills[j];
|
|
77
|
+
const score = jaccard(a.description, b.description);
|
|
78
|
+
if (score >= threshold) {
|
|
79
|
+
overlaps.push({
|
|
80
|
+
a: a.name, b: b.name, score: Math.round(score * 100) / 100,
|
|
81
|
+
suggest: [
|
|
82
|
+
{ skill: a.name, trigger: { query: nearMiss(b), should_trigger: false } },
|
|
83
|
+
{ skill: b.name, trigger: { query: nearMiss(a), should_trigger: false } },
|
|
84
|
+
],
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
overlaps.sort((x, y) => y.score - x.score);
|
|
90
|
+
const out = { skills: skills.map(({ description, ...s }) => s), total_chars: total, budgets, truncated, overlaps };
|
|
91
|
+
if (paths.length === 1) {
|
|
92
|
+
const plugin = pluginReport(paths[0]);
|
|
93
|
+
if (plugin) out.plugin = plugin;
|
|
94
|
+
}
|
|
95
|
+
out.ok = budgets.every((b) => b.over === 0) && truncated.length === 0;
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** A near-miss request built from the other skill's own description. */
|
|
100
|
+
function nearMiss(other) {
|
|
101
|
+
const first = other.description.split(/(?<=[.!?])\s/)[0].replace(/\.$/, "");
|
|
102
|
+
return `${first.slice(0, 200)} (this is ${other.name}'s job)`;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The plugin line: version, changelog entry for it, helpers copied between skills. */
|
|
106
|
+
export function pluginReport(root) {
|
|
107
|
+
const manifest = join(root, ".claude-plugin", "plugin.json");
|
|
108
|
+
if (!existsSync(manifest)) return null;
|
|
109
|
+
const findings = [];
|
|
110
|
+
let meta = {};
|
|
111
|
+
try { meta = JSON.parse(readFileSync(manifest, "utf8")); }
|
|
112
|
+
catch (e) { findings.push({ rule: "plugin-manifest", severity: "fail", message: `.claude-plugin/plugin.json is not valid JSON: ${e.message}`, fix: "Fix the manifest." }); }
|
|
113
|
+
const version = typeof meta.version === "string" ? meta.version : null;
|
|
114
|
+
if (!version) findings.push({ rule: "plugin-version", severity: "fail", message: "plugin.json has no version", fix: "Add a semver version and bump it on every release." });
|
|
115
|
+
let changelogHas = false;
|
|
116
|
+
const cl = join(root, "CHANGELOG.md");
|
|
117
|
+
if (!existsSync(cl)) findings.push({ rule: "plugin-changelog", severity: "fail", message: "no CHANGELOG.md", fix: "Add a CHANGELOG.md with an entry per release." });
|
|
118
|
+
else if (version) {
|
|
119
|
+
const esc = version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
120
|
+
changelogHas = new RegExp(`^#+\\s*\\[?v?${esc}\\b`, "m").test(readFileSync(cl, "utf8"));
|
|
121
|
+
if (!changelogHas) findings.push({ rule: "plugin-changelog", severity: "fail", message: `CHANGELOG.md has no entry for ${version}`, fix: `Add a "## ${version}" entry saying what changed.` });
|
|
122
|
+
}
|
|
123
|
+
const byHash = new Map();
|
|
124
|
+
for (const rel of listFiles(join(root, "skills"))) {
|
|
125
|
+
if (/(^|\/)SKILL\.md$/.test(rel) || /\.(md|json)$/i.test(rel) || /(^|\/)(evals|goldens)\//.test(rel) || rel.endsWith(".gitkeep")) continue;
|
|
126
|
+
const abs = join(root, "skills", rel);
|
|
127
|
+
let buf;
|
|
128
|
+
try { buf = readFileSync(abs); } catch { continue; }
|
|
129
|
+
if (!buf.length) continue;
|
|
130
|
+
const h = createHash("sha256").update(buf).digest("hex");
|
|
131
|
+
const skillName = rel.split("/")[0];
|
|
132
|
+
const list = byHash.get(h) || [];
|
|
133
|
+
list.push({ skill: skillName, file: relative(root, abs).split(sep).join("/") });
|
|
134
|
+
byHash.set(h, list);
|
|
135
|
+
}
|
|
136
|
+
const duplicates = [...byHash.values()].filter((l) => new Set(l.map((x) => x.skill)).size > 1).map((l) => ({ files: l.map((x) => x.file) }));
|
|
137
|
+
for (const d of duplicates) findings.push({ rule: "plugin-shared-helpers", severity: "warn", message: `same file copied into ${d.files.length} skills: ${d.files.join(", ")}`, fix: "share this helper: keep one copy in a shared scripts/ folder and call it from each skill." });
|
|
138
|
+
return { name: meta.name || null, version, changelog_has_version: changelogHas, duplicates, findings };
|
|
139
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync, writeFileSync, existsSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { createInterface } from "node:readline/promises";
|
|
5
|
+
import { parseArgs, clock, UsageError } from "../args.mjs";
|
|
6
|
+
import { skillDir, refuse } from "./common.mjs";
|
|
7
|
+
import { readGoldens } from "../goldens.mjs";
|
|
8
|
+
|
|
9
|
+
export const help = `superskill approve <skill> <golden-id> [--note "<why it is right>"]
|
|
10
|
+
|
|
11
|
+
Record that a person checked goldens/<id>/ and signs off on its output. Works only at an
|
|
12
|
+
interactive terminal and asks for your name, so an agent cannot approve its own output.
|
|
13
|
+
Writes goldens/<id>/APPROVAL.json with the SKILL.md hash it was approved against.
|
|
14
|
+
`;
|
|
15
|
+
|
|
16
|
+
export async function run(argv) {
|
|
17
|
+
const a = parseArgs(argv);
|
|
18
|
+
if (a.flags.help) { process.stdout.write(help); return 0; }
|
|
19
|
+
const dir = skillDir(a._[0], "approve");
|
|
20
|
+
const id = a._[1];
|
|
21
|
+
if (!id) throw new UsageError("approve needs a golden id");
|
|
22
|
+
if (!(process.stdin.isTTY && process.stdout.isTTY)) return refuse("approval needs a person at a terminal. Run this yourself, not through an agent.");
|
|
23
|
+
const g = readGoldens(dir).find((x) => x.id === id);
|
|
24
|
+
if (!g) return refuse(`no goldens/${id}/`);
|
|
25
|
+
if (g.input === null || g.output === null || !g.output.trim()) return refuse(`goldens/${id}/ needs an input file and a non-empty output file before it can be approved`);
|
|
26
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
27
|
+
try {
|
|
28
|
+
process.stdout.write(`\n--- goldens/${id}/${g.outputFile} ---\n${g.output.slice(0, 2000)}${g.output.length > 2000 ? "\n[...]" : ""}\n---\n`);
|
|
29
|
+
const name = (await rl.question("Your name (blank to cancel): ")).trim();
|
|
30
|
+
if (!name) return refuse("not approved");
|
|
31
|
+
const note = a.flags.note || (await rl.question("Why is this right? (optional): ")).trim();
|
|
32
|
+
const sha = createHash("sha256").update(readFileSync(join(dir, "SKILL.md"))).digest("hex");
|
|
33
|
+
const p = join(dir, "goldens", id, "APPROVAL.json");
|
|
34
|
+
const existed = existsSync(p);
|
|
35
|
+
writeFileSync(p, JSON.stringify({ approved_by: name, approved_at: clock(a.flags).toISOString(), skill_sha: sha, note }, null, 2) + "\n");
|
|
36
|
+
process.stdout.write(`${existed ? "re-approved" : "approved"} goldens/${id}/ by ${name}\n`);
|
|
37
|
+
return 0;
|
|
38
|
+
} finally {
|
|
39
|
+
rl.close();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { parseArgs, UsageError } from "../args.mjs";
|
|
2
|
+
import { collection, ENTRY_CAP } from "../collection.mjs";
|
|
3
|
+
|
|
4
|
+
export const help = `superskill collection <folder...> [--budget <chars>] [--overlap <0-1>] [--json]
|
|
5
|
+
|
|
6
|
+
Check a set of skills together:
|
|
7
|
+
- listing budget: characters of "name: description" the harness loads every turn,
|
|
8
|
+
against claude-code and codex budgets (8000 each by default, a conservative figure:
|
|
9
|
+
neither harness publishes one; --budget overrides)
|
|
10
|
+
- cut-off entries: skills whose description + when_to_use exceeds ${ENTRY_CAP} characters
|
|
11
|
+
(Claude Code's documented per-skill cap)
|
|
12
|
+
- overlap: pairs whose descriptions share enough words (--overlap, default 0.5) that an
|
|
13
|
+
agent could load the wrong one, each with a near-miss trigger to add to the other
|
|
14
|
+
- plugin line (when the folder has .claude-plugin/plugin.json): version, changelog
|
|
15
|
+
entry, helpers copied between skills
|
|
16
|
+
|
|
17
|
+
Exit codes: 0 within budget and nothing cut off, 1 otherwise, 2 usage or IO error.
|
|
18
|
+
`;
|
|
19
|
+
|
|
20
|
+
export async function run(argv) {
|
|
21
|
+
const a = parseArgs(argv);
|
|
22
|
+
if (a.flags.help) { process.stdout.write(help); return 0; }
|
|
23
|
+
if (!a._.length) throw new UsageError("collection needs a folder");
|
|
24
|
+
const opts = {};
|
|
25
|
+
if (a.flags.budget !== undefined) { opts.budget = Number(a.flags.budget); if (!(opts.budget > 0)) throw new UsageError("--budget must be a positive number"); }
|
|
26
|
+
if (a.flags.overlap !== undefined) { opts.overlap = Number(a.flags.overlap); if (!(opts.overlap > 0 && opts.overlap <= 1)) throw new UsageError("--overlap must be between 0 and 1"); }
|
|
27
|
+
const r = collection(a._, opts);
|
|
28
|
+
if (a.flags.json) { process.stdout.write(JSON.stringify(r, null, 2) + "\n"); return r.ok ? 0 : 1; }
|
|
29
|
+
const L = [];
|
|
30
|
+
L.push(`${r.skills.length} skills, ${r.total_chars} listing characters`);
|
|
31
|
+
for (const b of r.budgets) L.push(` ${b.harness.padEnd(12)} ${b.used} / ${b.limit}${b.over ? ` OVER by ${b.over}` : " ok"}`);
|
|
32
|
+
if (r.truncated.length) {
|
|
33
|
+
L.push(`cut off (description over ${ENTRY_CAP} characters):`);
|
|
34
|
+
for (const t of r.truncated) L.push(` ${t.name}: ${t.chars} characters, ${t.cut} not shown`);
|
|
35
|
+
}
|
|
36
|
+
const top = [...r.skills].sort((x, y) => y.chars - x.chars).slice(0, 5);
|
|
37
|
+
L.push("largest entries:");
|
|
38
|
+
for (const s of top) L.push(` ${String(s.chars).padStart(5)} ${s.name}`);
|
|
39
|
+
if (r.overlaps.length) {
|
|
40
|
+
L.push(`overlapping descriptions (${r.overlaps.length}):`);
|
|
41
|
+
for (const o of r.overlaps.slice(0, 25)) {
|
|
42
|
+
L.push(` ${o.score.toFixed(2)} ${o.a} <> ${o.b}`);
|
|
43
|
+
for (const s of o.suggest) L.push(` add to ${s.skill}/evals/triggers.json: ${JSON.stringify(s.trigger)}`);
|
|
44
|
+
}
|
|
45
|
+
if (r.overlaps.length > 25) L.push(` ... ${r.overlaps.length - 25} more (--json for all)`);
|
|
46
|
+
} else L.push("no overlapping descriptions");
|
|
47
|
+
if (r.plugin) L.push(...pluginLines(r.plugin));
|
|
48
|
+
process.stdout.write(L.join("\n") + "\n");
|
|
49
|
+
return r.ok ? 0 : 1;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function pluginLines(p, superplugin) {
|
|
53
|
+
const L = [`plugin ${p.name || "(unnamed)"} ${p.version || "(no version)"}${superplugin === undefined ? "" : superplugin ? " superplugin" : ""}`];
|
|
54
|
+
for (const f of p.findings) L.push(` ${f.severity.padEnd(4)} ${f.rule}: ${f.message}${f.severity === "info" ? "" : `\n fix: ${f.fix}`}`);
|
|
55
|
+
return L;
|
|
56
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { UsageError } from "../args.mjs";
|
|
4
|
+
|
|
5
|
+
/** Resolve a skill folder argument or throw a usage error naming what is wrong. */
|
|
6
|
+
export function skillDir(arg, cmd) {
|
|
7
|
+
if (!arg) throw new UsageError(`${cmd} needs a skill folder`);
|
|
8
|
+
const dir = resolve(arg);
|
|
9
|
+
if (!existsSync(join(dir, "SKILL.md"))) throw new UsageError(`no SKILL.md in ${arg}`);
|
|
10
|
+
return dir;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export const today = (now) => now.toISOString().slice(0, 10);
|
|
14
|
+
|
|
15
|
+
/** A refusal: the command understood the request and declines it (exit 1). */
|
|
16
|
+
export function refuse(message) {
|
|
17
|
+
process.stderr.write(`superskill: ${message}\n`);
|
|
18
|
+
return 1;
|
|
19
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { parseArgs, clock, UsageError } from "../args.mjs";
|
|
3
|
+
import { doctor, findSkills } from "../doctor.mjs";
|
|
4
|
+
import { formatDoctor } from "../report.mjs";
|
|
5
|
+
import { LEVELS } from "../levels.mjs";
|
|
6
|
+
import { changedFiles, touchedSkills, levelDrops } from "../changed.mjs";
|
|
7
|
+
|
|
8
|
+
export const help = `superskill doctor <path...> [options]
|
|
9
|
+
|
|
10
|
+
Score one skill, a folder of skills, or a plugin (skills/*/SKILL.md).
|
|
11
|
+
Levels: skill (spec-valid, hygienic) < tested (evals + triggers) < superskill
|
|
12
|
+
(approved golden, misses fixed with evals, a fresh --run that beats no-skill).
|
|
13
|
+
|
|
14
|
+
Options:
|
|
15
|
+
--level <skill|tested|superskill> target level for the exit code (default skill)
|
|
16
|
+
--json print one JSON document and nothing else
|
|
17
|
+
--changed score only skills a change touched (git): changes
|
|
18
|
+
since --base <ref> plus the working tree
|
|
19
|
+
--base <ref> with --changed, e.g. origin/main in CI
|
|
20
|
+
--baseline-json <file> a previous --json; exit 1 if any skill's level dropped
|
|
21
|
+
--run run the evals with and without the skill (costs
|
|
22
|
+
model calls; see "superskill doctor --run --help")
|
|
23
|
+
--now <iso date> evaluate dates as of this moment
|
|
24
|
+
--help this text
|
|
25
|
+
|
|
26
|
+
Exit codes: 0 every skill meets the target, 1 below target or a level dropped,
|
|
27
|
+
2 usage or IO error.
|
|
28
|
+
`;
|
|
29
|
+
|
|
30
|
+
export async function run(argv) {
|
|
31
|
+
const a = parseArgs(argv, ["changed", "run", "yes"]);
|
|
32
|
+
if (a.flags.run) return (await import("../run/index.mjs")).runCommand(a);
|
|
33
|
+
if (a.flags.help) { process.stdout.write(help); return 0; }
|
|
34
|
+
const level = a.flags.level || "skill";
|
|
35
|
+
if (!LEVELS.includes(level)) throw new UsageError(`--level must be one of ${LEVELS.join(", ")}`);
|
|
36
|
+
if (!a._.length) throw new UsageError("doctor needs a path");
|
|
37
|
+
const opts = { level, now: clock(a.flags) };
|
|
38
|
+
if (a.flags.changed) {
|
|
39
|
+
const all = a._.flatMap((p) => findSkills(p));
|
|
40
|
+
const files = a._.flatMap((p) => changedFiles(p, a.flags.base));
|
|
41
|
+
opts.only = touchedSkills(all, files);
|
|
42
|
+
if (!opts.only.length) {
|
|
43
|
+
if (a.flags.json) process.stdout.write(JSON.stringify({ target: level, ok: true, skills: [], drops: [] }, null, 2) + "\n");
|
|
44
|
+
else process.stdout.write("no changed skills\n");
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const result = doctor(a._, opts);
|
|
49
|
+
if (a.flags["baseline-json"]) {
|
|
50
|
+
let baseline;
|
|
51
|
+
try { baseline = JSON.parse(readFileSync(a.flags["baseline-json"], "utf8")); }
|
|
52
|
+
catch (e) { throw new UsageError(`cannot read --baseline-json: ${e.message}`); }
|
|
53
|
+
result.drops = levelDrops(result, baseline);
|
|
54
|
+
if (result.drops.length) result.ok = false;
|
|
55
|
+
}
|
|
56
|
+
if (a.flags.json) process.stdout.write(JSON.stringify(result, null, 2) + "\n");
|
|
57
|
+
else {
|
|
58
|
+
let out = formatDoctor(result);
|
|
59
|
+
for (const d of result.drops || []) out += `REFUSED: ${d.name} dropped from ${d.from} to ${d.to}\n`;
|
|
60
|
+
process.stdout.write(out);
|
|
61
|
+
}
|
|
62
|
+
return result.ok ? 0 : 1;
|
|
63
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { parseArgs, UsageError } from "../args.mjs";
|
|
2
|
+
import { skillDir, refuse } from "./common.mjs";
|
|
3
|
+
import { readMisses, updateMiss } from "../misses.mjs";
|
|
4
|
+
import { readEvals } from "../evals.mjs";
|
|
5
|
+
import { readGoldens } from "../goldens.mjs";
|
|
6
|
+
|
|
7
|
+
export const help = `superskill fix <skill> <miss-id> --eval <eval-id> [--commit <sha>]
|
|
8
|
+
|
|
9
|
+
Close a miss. Refuses unless the named eval exists in evals/evals.json or goldens/, so
|
|
10
|
+
every fix leaves behind a check that would catch the same miss again.
|
|
11
|
+
`;
|
|
12
|
+
|
|
13
|
+
export async function run(argv) {
|
|
14
|
+
const a = parseArgs(argv);
|
|
15
|
+
if (a.flags.help) { process.stdout.write(help); return 0; }
|
|
16
|
+
const dir = skillDir(a._[0], "fix");
|
|
17
|
+
const id = (a._[1] || "").toLowerCase();
|
|
18
|
+
if (!id) throw new UsageError("fix needs a miss id");
|
|
19
|
+
const evalId = a.flags.eval;
|
|
20
|
+
if (!evalId || evalId === true) throw new UsageError("fix needs --eval <id>: the regression eval that would catch this miss again");
|
|
21
|
+
const miss = (readMisses(dir) || []).find((m) => m.id === id);
|
|
22
|
+
if (!miss) return refuse(`no miss ${id} in MISSES.md`);
|
|
23
|
+
const ids = new Set(readEvals(dir).cases.map((c) => String(c.id)));
|
|
24
|
+
for (const g of readGoldens(dir)) ids.add(g.id);
|
|
25
|
+
if (!ids.has(String(evalId))) return refuse(`eval "${evalId}" is not in evals/evals.json or goldens/. Add the case first, then fix.`);
|
|
26
|
+
updateMiss(dir, id, { status: "fixed", eval: String(evalId), fix: a.flags.commit || miss.fix || "" });
|
|
27
|
+
process.stdout.write(`${id} fixed, guarded by eval ${evalId}\n`);
|
|
28
|
+
return 0;
|
|
29
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { parseArgs, clock, UsageError } from "../args.mjs";
|
|
4
|
+
import { skillDir, today } from "./common.mjs";
|
|
5
|
+
import { readMisses, appendMisses, nextMissId } from "../misses.mjs";
|
|
6
|
+
import { ledgerPaths, ledgerMisses } from "../ledger.mjs";
|
|
7
|
+
import { parseSkillFile } from "../frontmatter.mjs";
|
|
8
|
+
|
|
9
|
+
export const help = `superskill miss import <skill> --freedom-ledger [--ledger <file.jsonl>]
|
|
10
|
+
|
|
11
|
+
Turn Freedom's automatic run record into misses: every run that needed a correction, a
|
|
12
|
+
rescue or a redirect, or that failed, becomes an open miss. Taste corrections and runs
|
|
13
|
+
already imported are skipped. Reads <skill>/invocations.jsonl and
|
|
14
|
+
~/.freedom/ledger/skills/*/<skill>.jsonl, or the file given with --ledger. Freedom does not
|
|
15
|
+
need to be installed; the ledger is read as plain files.
|
|
16
|
+
`;
|
|
17
|
+
|
|
18
|
+
export async function run(argv) {
|
|
19
|
+
const a = parseArgs(argv, ["freedom-ledger"]);
|
|
20
|
+
if (a.flags.help) { process.stdout.write(help); return 0; }
|
|
21
|
+
const dir = skillDir(a._[0], "miss import");
|
|
22
|
+
if (!a.flags["freedom-ledger"]) throw new UsageError("miss import needs a source: --freedom-ledger");
|
|
23
|
+
const name = parseSkillFile(readFileSync(join(dir, "SKILL.md"), "utf8")).data.name || "";
|
|
24
|
+
let paths;
|
|
25
|
+
if (a.flags.ledger) {
|
|
26
|
+
if (!existsSync(a.flags.ledger)) throw new UsageError(`ledger not found: ${a.flags.ledger}`);
|
|
27
|
+
paths = [a.flags.ledger];
|
|
28
|
+
} else paths = ledgerPaths(dir, name);
|
|
29
|
+
if (!paths.length) { process.stdout.write(`no Freedom ledger found for ${name || dir}; nothing imported\n`); return 0; }
|
|
30
|
+
const existing = readMisses(dir) || [];
|
|
31
|
+
const known = new Set(existing.map((m) => (m.source || "").replace(/^freedom-ledger\s+/, "")).filter(Boolean));
|
|
32
|
+
const fresh = paths.flatMap((p) => ledgerMisses(p, known, name));
|
|
33
|
+
const all = [...existing];
|
|
34
|
+
const entries = fresh.map((m) => {
|
|
35
|
+
const e = { ...m, id: nextMissId(all), date: m.date || today(clock(a.flags)) };
|
|
36
|
+
all.push(e);
|
|
37
|
+
return e;
|
|
38
|
+
});
|
|
39
|
+
if (entries.length) appendMisses(dir, entries);
|
|
40
|
+
process.stdout.write(`${entries.length} new miss${entries.length === 1 ? "" : "es"} from ${paths.length} ledger file${paths.length === 1 ? "" : "s"}${entries.length ? `: ${entries.map((e) => e.id).join(", ")}` : ""}\n`);
|
|
41
|
+
return 0;
|
|
42
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { parseArgs, UsageError } from "../args.mjs";
|
|
4
|
+
import { skillDir } from "./common.mjs";
|
|
5
|
+
import { MISSES_HEADER } from "../misses.mjs";
|
|
6
|
+
import { readSession } from "../session.mjs";
|
|
7
|
+
import { parseSkillFile } from "../frontmatter.mjs";
|
|
8
|
+
|
|
9
|
+
export const help = `superskill init <skill> [--from-session <transcript>]
|
|
10
|
+
|
|
11
|
+
Add whatever a skill is missing to climb the levels: evals/evals.json (an example case),
|
|
12
|
+
evals/triggers.json (example should / should-not requests), goldens/, and MISSES.md.
|
|
13
|
+
Never overwrites a file that exists.
|
|
14
|
+
|
|
15
|
+
--from-session <file> build the first eval and golden candidate from the session where
|
|
16
|
+
the job was done by hand. A Claude Code .jsonl gives the first
|
|
17
|
+
request and the final answer; any other file is the request.
|
|
18
|
+
The golden waits for a person: run \`superskill approve\`.
|
|
19
|
+
`;
|
|
20
|
+
|
|
21
|
+
const TRIGGER_EXAMPLE = [
|
|
22
|
+
{ query: "REPLACE: a realistic request that should load this skill", should_trigger: true },
|
|
23
|
+
{ query: "REPLACE: a near-miss that shares words with this skill but needs something else", should_trigger: false },
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
export async function run(argv) {
|
|
27
|
+
const a = parseArgs(argv);
|
|
28
|
+
if (a.flags.help) { process.stdout.write(help); return 0; }
|
|
29
|
+
const dir = skillDir(a._[0], "init");
|
|
30
|
+
const name = parseSkillFile(readFileSync(join(dir, "SKILL.md"), "utf8")).data.name || "";
|
|
31
|
+
const session = a.flags["from-session"];
|
|
32
|
+
if (session !== undefined && (session === true || !existsSync(session))) throw new UsageError(`--from-session file not found: ${session}`);
|
|
33
|
+
const made = [], kept = [];
|
|
34
|
+
const put = (rel, content) => {
|
|
35
|
+
const p = join(dir, rel);
|
|
36
|
+
if (existsSync(p)) { kept.push(rel); return false; }
|
|
37
|
+
mkdirSync(join(p, ".."), { recursive: true });
|
|
38
|
+
writeFileSync(p, content);
|
|
39
|
+
made.push(rel);
|
|
40
|
+
return true;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
let sessionCase = null;
|
|
44
|
+
if (session) {
|
|
45
|
+
const { prompt, output } = readSession(session);
|
|
46
|
+
if (!prompt) throw new UsageError(`no request found in ${session}`);
|
|
47
|
+
sessionCase = { prompt, output };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const evalsPath = join(dir, "evals", "evals.json");
|
|
51
|
+
const example = { id: 1, prompt: "REPLACE: a real request this skill handles", expected_output: "REPLACE: what a good answer looks like", files: [], expectations: ["REPLACE: a statement a grader can check, or contains:<text> / regex:<pattern>"] };
|
|
52
|
+
put("evals/evals.json", JSON.stringify({ skill_name: name, evals: sessionCase ? [] : [example] }, null, 2) + "\n");
|
|
53
|
+
put("evals/triggers.json", JSON.stringify(TRIGGER_EXAMPLE, null, 2) + "\n");
|
|
54
|
+
put("goldens/.gitkeep", "");
|
|
55
|
+
put("MISSES.md", MISSES_HEADER);
|
|
56
|
+
|
|
57
|
+
if (sessionCase) {
|
|
58
|
+
const doc = JSON.parse(readFileSync(evalsPath, "utf8"));
|
|
59
|
+
const list = Array.isArray(doc) ? doc : (doc.evals ||= []);
|
|
60
|
+
const taken = new Set(list.map((c) => String(c.id)));
|
|
61
|
+
let n = 1;
|
|
62
|
+
while (taken.has(`s${n}`) || existsSync(join(dir, "goldens", `s${n}`))) n++;
|
|
63
|
+
const id = `s${n}`;
|
|
64
|
+
list.push({ id, prompt: sessionCase.prompt, expected_output: "Matches the approved output in goldens/" + id + "/", files: [], expectations: ["Output addresses the request in the prompt"] });
|
|
65
|
+
writeFileSync(evalsPath, JSON.stringify(doc, null, 2) + "\n");
|
|
66
|
+
put(`goldens/${id}/input.md`, sessionCase.prompt + "\n");
|
|
67
|
+
put(`goldens/${id}/output.md`, sessionCase.output ? sessionCase.output + "\n" : "");
|
|
68
|
+
process.stdout.write(`eval ${id} and golden candidate goldens/${id}/ written from ${session}\n`);
|
|
69
|
+
process.stdout.write(`a person approves it with: superskill approve ${a._[0]} ${id}\n`);
|
|
70
|
+
}
|
|
71
|
+
for (const m of made) process.stdout.write(`created ${m}\n`);
|
|
72
|
+
for (const k of kept) process.stdout.write(`kept ${k} (exists)\n`);
|
|
73
|
+
return 0;
|
|
74
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { parseArgs, clock, UsageError } from "../args.mjs";
|
|
2
|
+
import { skillDir, today } from "./common.mjs";
|
|
3
|
+
import { readMisses, appendMisses, nextMissId } from "../misses.mjs";
|
|
4
|
+
|
|
5
|
+
export const help = `superskill miss <skill> "<what happened>" [--expected "<what should have>"]
|
|
6
|
+
superskill miss import <skill> --freedom-ledger [--ledger <file>]
|
|
7
|
+
|
|
8
|
+
Log a time the skill got something wrong. The entry opens today; an open miss older than
|
|
9
|
+
14 days blocks the superskill level. Close it with \`superskill fix\`.
|
|
10
|
+
`;
|
|
11
|
+
|
|
12
|
+
export async function run(argv) {
|
|
13
|
+
if (argv[0] === "import") return (await import("./import.mjs")).run(argv.slice(1));
|
|
14
|
+
const a = parseArgs(argv);
|
|
15
|
+
if (a.flags.help) { process.stdout.write(help); return 0; }
|
|
16
|
+
const dir = skillDir(a._[0], "miss");
|
|
17
|
+
const what = (a._[1] || "").trim();
|
|
18
|
+
if (!what) throw new UsageError('miss needs a description: superskill miss <skill> "<what happened>"');
|
|
19
|
+
const misses = readMisses(dir) || [];
|
|
20
|
+
const id = nextMissId(misses);
|
|
21
|
+
appendMisses(dir, [{ id, date: today(clock(a.flags)), status: "open", what, expected: a.flags.expected || "" }]);
|
|
22
|
+
process.stdout.write(`${id} logged (open). Close it with: superskill fix ${a._[0]} ${id} --eval <id>\n`);
|
|
23
|
+
return 0;
|
|
24
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
export const help = `superskill snippet
|
|
4
|
+
|
|
5
|
+
Print a block to paste into AGENTS.md or CLAUDE.md so any agent proposes a skill after doing
|
|
6
|
+
a job once, logs a miss whenever a skill needed correcting, and runs the doctor before
|
|
7
|
+
calling a skill done.
|
|
8
|
+
`;
|
|
9
|
+
|
|
10
|
+
export async function run(argv) {
|
|
11
|
+
if (argv.includes("--help") || argv.includes("-h")) { process.stdout.write(help); return 0; }
|
|
12
|
+
process.stdout.write(readFileSync(new URL("../snippet.md", import.meta.url), "utf8"));
|
|
13
|
+
return 0;
|
|
14
|
+
}
|
package/src/context.mjs
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";
|
|
2
|
+
import { join, basename, resolve, relative, sep } from "node:path";
|
|
3
|
+
import { parseSkillFile } from "./frontmatter.mjs";
|
|
4
|
+
|
|
5
|
+
const SKIP_DIRS = new Set([".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"]);
|
|
6
|
+
const MAX_FILES = 2000;
|
|
7
|
+
|
|
8
|
+
/** Every file under dir, relative, forward slashes. Follows symlinked dirs once, never loops. */
|
|
9
|
+
export function listFiles(dir) {
|
|
10
|
+
const out = [];
|
|
11
|
+
const seen = new Set();
|
|
12
|
+
const walk = (abs) => {
|
|
13
|
+
let real;
|
|
14
|
+
try { real = resolve(abs); } catch { return; }
|
|
15
|
+
if (seen.has(real)) return;
|
|
16
|
+
seen.add(real);
|
|
17
|
+
let entries;
|
|
18
|
+
try { entries = readdirSync(abs, { withFileTypes: true }); } catch { return; }
|
|
19
|
+
for (const e of entries) {
|
|
20
|
+
if (out.length >= MAX_FILES) return;
|
|
21
|
+
const p = join(abs, e.name);
|
|
22
|
+
let isDir = e.isDirectory();
|
|
23
|
+
let isFile = e.isFile();
|
|
24
|
+
if (e.isSymbolicLink()) {
|
|
25
|
+
try { const s = statSync(p); isDir = s.isDirectory(); isFile = s.isFile(); } catch { continue; }
|
|
26
|
+
}
|
|
27
|
+
if (isDir) { if (!SKIP_DIRS.has(e.name)) walk(p); }
|
|
28
|
+
else if (isFile) out.push(relative(dir, p).split(sep).join("/"));
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
walk(dir);
|
|
32
|
+
return out.sort();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function readText(dir, rel) {
|
|
36
|
+
try { return readFileSync(join(dir, rel), "utf8"); } catch { return null; }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Load one skill folder into the context every rule reads.
|
|
41
|
+
* Never throws: an unreadable skill comes back with `error` set.
|
|
42
|
+
*/
|
|
43
|
+
export function loadSkill(dir) {
|
|
44
|
+
const abs = resolve(dir);
|
|
45
|
+
const ctx = { dir: abs, folderName: basename(abs), data: {}, body: "", bodyStartLine: 1, raw: "", files: [] };
|
|
46
|
+
const skillPath = join(abs, "SKILL.md");
|
|
47
|
+
if (!existsSync(skillPath)) return { ...ctx, error: "no SKILL.md in this folder" };
|
|
48
|
+
try {
|
|
49
|
+
ctx.raw = readFileSync(skillPath, "utf8");
|
|
50
|
+
} catch (e) {
|
|
51
|
+
return { ...ctx, error: `cannot read SKILL.md: ${e.code || e.message}` };
|
|
52
|
+
}
|
|
53
|
+
const parsed = parseSkillFile(ctx.raw);
|
|
54
|
+
ctx.data = parsed.data;
|
|
55
|
+
ctx.body = parsed.body;
|
|
56
|
+
ctx.bodyStartLine = parsed.bodyStartLine;
|
|
57
|
+
if (parsed.error) ctx.parseError = parsed.error;
|
|
58
|
+
ctx.files = listFiles(abs);
|
|
59
|
+
return ctx;
|
|
60
|
+
}
|
package/src/doctor.mjs
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { existsSync, statSync, readdirSync } from "node:fs";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { loadSkill } from "./context.mjs";
|
|
4
|
+
import { allRules } from "./rules/index.mjs";
|
|
5
|
+
import { runRules, computeLevel, nextLevel, meets } from "./levels.mjs";
|
|
6
|
+
import { pluginReport } from "./collection.mjs";
|
|
7
|
+
|
|
8
|
+
export class DoctorError extends Error {}
|
|
9
|
+
|
|
10
|
+
const isDir = (p) => { try { return statSync(p).isDirectory(); } catch { return false; } };
|
|
11
|
+
const hasSkill = (p) => existsSync(join(p, "SKILL.md"));
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Resolve a path to skill folders: the path itself if it holds SKILL.md, otherwise every
|
|
15
|
+
* direct child that does, plus skills/<name>/ for a plugin root.
|
|
16
|
+
*/
|
|
17
|
+
export function findSkills(path) {
|
|
18
|
+
const abs = resolve(path);
|
|
19
|
+
if (!existsSync(abs)) throw new DoctorError(`path not found: ${path}`);
|
|
20
|
+
if (!isDir(abs)) throw new DoctorError(`not a folder: ${path}`);
|
|
21
|
+
if (hasSkill(abs)) return [abs];
|
|
22
|
+
const out = [];
|
|
23
|
+
const scan = (dir) => {
|
|
24
|
+
let names = [];
|
|
25
|
+
try { names = readdirSync(dir).sort(); } catch { return; }
|
|
26
|
+
for (const n of names) {
|
|
27
|
+
if (n.startsWith(".")) continue;
|
|
28
|
+
const p = join(dir, n);
|
|
29
|
+
if (isDir(p) && hasSkill(p)) out.push(p);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
scan(abs);
|
|
33
|
+
if (isDir(join(abs, "skills"))) scan(join(abs, "skills"));
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Score one loaded skill. */
|
|
38
|
+
export function scoreSkill(dir, opts = {}) {
|
|
39
|
+
const ctx = loadSkill(dir);
|
|
40
|
+
const findings = runRules(ctx, opts.rules || allRules, opts);
|
|
41
|
+
const level = computeLevel(findings);
|
|
42
|
+
const up = nextLevel(level);
|
|
43
|
+
const next = up ? findings.filter((f) => f.level === up && f.severity === "fail") : [];
|
|
44
|
+
return { path: ctx.dir, name: typeof ctx.data.name === "string" && ctx.data.name ? ctx.data.name : ctx.folderName, level, next, findings };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* doctor(paths, {level, now}) -> {skills, ok, target}
|
|
49
|
+
* `ok` is true when every skill reaches the target level (default "skill").
|
|
50
|
+
*/
|
|
51
|
+
export function doctor(paths, opts = {}) {
|
|
52
|
+
const target = opts.level || "skill";
|
|
53
|
+
let dirs = [];
|
|
54
|
+
for (const p of paths) dirs.push(...findSkills(p));
|
|
55
|
+
if (opts.only) dirs = dirs.filter((d) => opts.only.includes(d));
|
|
56
|
+
else if (!dirs.length) throw new DoctorError(`no skills found under ${paths.join(", ")} (looked for SKILL.md in the folder, its children, and skills/*/)`);
|
|
57
|
+
const skills = [...new Set(dirs)].map((d) => scoreSkill(d, opts));
|
|
58
|
+
const result = { target, ok: skills.every((s) => meets(s.level, target)), skills };
|
|
59
|
+
if (paths.length === 1) {
|
|
60
|
+
const plugin = pluginReport(paths[0]);
|
|
61
|
+
// A plugin is a superplugin only when every skill in it is a superskill.
|
|
62
|
+
if (plugin) result.plugin = { ...plugin, superplugin: skills.every((s) => s.level === "superskill") && !plugin.findings.some((f) => f.severity === "fail") };
|
|
63
|
+
}
|
|
64
|
+
return result;
|
|
65
|
+
}
|