@yogiadhik/skillcheck 0.1.2 → 0.2.0

package/README.md

@@ -53,8 +53,9 @@ case, no re-run — and `--budget 2.50` stops the run once the cap is
 crossed. The check happens between cases, so one long case can still
 overshoot on its own: pair a budget with `--model haiku` to keep
 single cases cheap. skillcheck tells you when no model is set, and
-warns at the end if a run blew past its budget. Lint never spends
-anything.
+warns at the end if a run blew past its budget. `run --dry-run` shows
+which cases would execute and which would replay from cache before
+anything spends a cent. Lint never spends anything.
 
 ## CI
 
@@ -66,6 +67,42 @@ That runs lint with GitHub annotations on every pull request. For
 behavioral runs in CI, call `npx @yogiadhik/skillcheck run` yourself in a job that
 has your agent authenticated; keep a budget flag on it.
 
+## what it finds in the wild
+
+I periodically run lint across public skill collections — official
+vendor skills, a security firm's suite, several large community
+bundles, close to eight thousand skill files so far. No names,
+because the point is the pattern, and the pattern repeats:
+
+- **Skills that silently don't work.** If a skill's frontmatter isn't
+  valid YAML, the skill never loads — and nothing tells you. The usual
+  cause is an unquoted description containing a colon. Every large
+  bundle I've audited shipped a few of these; anyone installing them
+  gets a skill that simply never fires.
+- **Instructions pointing at files that don't exist.** Skills tell the
+  agent "read references/details.md for the full procedure" and that
+  file was never committed. The agent hits a dead end in the middle of
+  someone's task. This is the most common hard error.
+- **Sheer size.** The most common warning by a wide margin. Skills
+  running twenty to forty thousand tokens re-bill that entire document
+  on every single invocation.
+- **Descriptions without trigger language.** The description is the
+  only thing the model reads when deciding whether to use a skill; a
+  description that doesn't say when to use it is a skill that fires
+  rarely and randomly.
+- **Well-maintained collections come back clean.** Several do,
+  official and community both. Quality is achievable; it's just not
+  the default.
+
+Two honest footnotes. First: twice, the loudest "errors" in an audit
+turned out to be this linter's own false positives — links inside
+example code blocks, and templated {baseDir} targets that resolve at
+runtime. Both fixed (0.1.1, 0.1.3), both caught by checking findings
+against the accused code before believing them. Do the same with
+anything this tool tells you. Second: these audits are how skillcheck
+gets tested — every rule here has drawn blood on real files or been
+fixed for crying wolf.
+
 ## a note on safety
 
 `run` executes the agent with permission prompts disabled, inside a

package/dist/cli.js

@@ -3,13 +3,13 @@ import { Command } from 'commander';
 import { initTestFile } from './init.js';
 import { lintPath } from './lint/lint.js';
 import { printGithubAnnotations } from './reporters/github.js';
-import { printFindings, printReports } from './reporters/terminal.js';
-import { discoverTestFiles, reportsPass, runTests } from './runner.js';
+import { printFindings, printPlan, printReports } from './reporters/terminal.js';
+import { discoverTestFiles, planTests, reportsPass, runTests } from './runner.js';
 const program = new Command();
 program
     .name('skillcheck')
     .description('lint and behavioral tests for agent skills')
-    .version('0.1.2');
+    .version('0.2.0');
 program
     .command('lint')
     .argument('[path]', 'skill file or directory', '.')
@@ -39,9 +39,18 @@ program
     .option('--budget <usd>', 'stop once this much has been spent')
     .option('--timeout <seconds>', 'per-case timeout', '300')
     .option('--no-cache', 'ignore and do not write cached results')
+    .option('--dry-run', 'show what would run and what would replay from cache, spending nothing')
     .option('--json', 'machine-readable output')
     .action(async (path, opts) => {
     try {
+        if (opts.dryRun) {
+            const plans = planTests(path, { model: opts.model });
+            if (opts.json)
+                console.log(JSON.stringify(plans, null, 2));
+            else
+                printPlan(plans);
+            return;
+        }
         const files = discoverTestFiles(path);
         if (!opts.json) {
             console.log(`${files.length} test file(s); live agent runs cost money — cap with --budget`);

package/dist/lint/rules.js

@@ -50,6 +50,9 @@ export const relativeLinks = (s) => {
         const target = m[1];
         if (/^(https?:|mailto:|#)/.test(target))
             continue;
+        // templated targets like {baseDir}/x.md resolve at runtime; can't judge them
+        if (target.includes('{'))
+            continue;
         if (!existsSync(resolve(s.dir, target.split('#')[0])))
             out.push(finding(s, 'relative-links', 'error', `broken relative link: ${target}`));
     }

package/dist/reporters/terminal.js

@@ -7,6 +7,25 @@ export function printFindings(findings) {
     const errors = findings.filter((f) => f.severity === 'error').length;
     console.log(`\n${findings.length} finding(s), ${errors} error(s)`);
 }
+export function printPlan(plans) {
+    let live = 0;
+    let knownCost = 0;
+    for (const p of plans) {
+        if (p.cached) {
+            knownCost += p.lastCostUsd ?? 0;
+            console.log(`${pc.dim('cached   ')} ${p.case} ($${(p.lastCostUsd ?? 0).toFixed(4)} last time, replays free)`);
+        }
+        else {
+            live++;
+            console.log(`${pc.yellow('would run')} ${p.case}`);
+        }
+    }
+    console.log(`\n${plans.length} case(s): ${live} would run live, ${plans.length - live} would replay from cache`);
+    if (live > 0)
+        console.log(pc.yellow(`live cases cost real money and their price is unknown until they run — pair --model haiku with --budget`));
+    else if (plans.length > 0)
+        console.log(pc.dim(`nothing to spend — a real run would replay entirely from cache ($${knownCost.toFixed(4)} when first run)`));
+}
 export function printReports(reports, budgetUsd) {
     let cost = 0;
     for (const r of reports) {

package/dist/runner.js

@@ -29,6 +29,22 @@ export function loadTestFile(path) {
         throw new Error(`${path}: needs a "skill" path and a "cases" list`);
     return { path, skill: doc.skill, cases: doc.cases };
 }
+function caseKey(skillDir, fileDir, c, model) {
+    return hashInputs(c.setup ? [skillDir, resolve(fileDir, c.setup)] : skillDir, YAML.stringify(c) + '\0' + (model ?? ''));
+}
+export function planTests(root, opts = {}) {
+    const cache = new ResultCache(join(process.cwd(), '.skillcheck', 'cache.json'));
+    const plans = [];
+    for (const file of discoverTestFiles(root)) {
+        const testFile = loadTestFile(file);
+        const skillDir = resolve(dirname(file), testFile.skill);
+        for (const c of testFile.cases) {
+            const hit = cache.get(caseKey(skillDir, dirname(file), c, opts.model));
+            plans.push({ file, case: c.name, cached: !!hit, lastCostUsd: hit?.costUsd });
+        }
+    }
+    return plans;
+}
 export async function runTests(root, opts = {}) {
     const adapter = opts.adapter ?? claudeCode;
     const cache = new ResultCache(join(process.cwd(), '.skillcheck', 'cache.json'));
@@ -49,7 +65,7 @@ export async function runTests(root, opts = {}) {
                 });
                 continue;
             }
-            const key = hashInputs(c.setup ? [skillDir, resolve(dirname(file), c.setup)] : skillDir, YAML.stringify(c) + '\0' + (opts.model ?? ''));
+            const key = caseKey(skillDir, dirname(file), c, opts.model);
             if (opts.cache !== false) {
                 const hit = cache.get(key);
                 if (hit) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yogiadhik/skillcheck",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "lint and behavioral tests for agent skills",
   "license": "MIT",
   "author": "Yogesh Adhikari",