@yogiadhik/skillcheck 0.1.1 → 0.1.3

package/README.md

@@ -49,8 +49,12 @@ Run `npx @yogiadhik/skillcheck init <skill-dir>` to scaffold a starter file.
 
 Behavioral runs call the real agent, and that costs real money.
 skillcheck caches results by content hash — unchanged skill, unchanged
-case, no re-run — and `--budget 2.50` hard-stops a run at the cap.
-Lint never spends anything.
+case, no re-run — and `--budget 2.50` stops the run once the cap is
+crossed. The check happens between cases, so one long case can still
+overshoot on its own: pair a budget with `--model haiku` to keep
+single cases cheap. skillcheck tells you when no model is set, and
+warns at the end if a run blew past its budget. Lint never spends
+anything.
 
 ## CI
 
@@ -62,6 +66,42 @@ That runs lint with GitHub annotations on every pull request. For
 behavioral runs in CI, call `npx @yogiadhik/skillcheck run` yourself in a job that
 has your agent authenticated; keep a budget flag on it.
 
+## what it finds in the wild
+
+I periodically run lint across public skill collections — official
+vendor skills, a security firm's suite, several large community
+bundles, close to eight thousand skill files so far. No names,
+because the point is the pattern, and the pattern repeats:
+
+- **Skills that silently don't work.** If a skill's frontmatter isn't
+  valid YAML, the skill never loads — and nothing tells you. The usual
+  cause is an unquoted description containing a colon. Every large
+  bundle I've audited shipped a few of these; anyone installing them
+  gets a skill that simply never fires.
+- **Instructions pointing at files that don't exist.** Skills tell the
+  agent "read references/details.md for the full procedure" and that
+  file was never committed. The agent hits a dead end in the middle of
+  someone's task. This is the most common hard error.
+- **Sheer size.** The most common warning by a wide margin. Skills
+  running twenty to forty thousand tokens re-bill that entire document
+  on every single invocation.
+- **Descriptions without trigger language.** The description is the
+  only thing the model reads when deciding whether to use a skill; a
+  description that doesn't say when to use it is a skill that fires
+  rarely and randomly.
+- **Well-maintained collections come back clean.** Several do,
+  official and community both. Quality is achievable; it's just not
+  the default.
+
+Two honest footnotes. First: twice, the loudest "errors" in an audit
+turned out to be this linter's own false positives — links inside
+example code blocks, and templated {baseDir} targets that resolve at
+runtime. Both fixed (0.1.1, 0.1.3), both caught by checking findings
+against the accused code before believing them. Do the same with
+anything this tool tells you. Second: these audits are how skillcheck
+gets tested — every rule here has drawn blood on real files or been
+fixed for crying wolf.
+
 ## a note on safety
 
 `run` executes the agent with permission prompts disabled, inside a

package/dist/cli.js

@@ -9,7 +9,7 @@ const program = new Command();
 program
     .name('skillcheck')
     .description('lint and behavioral tests for agent skills')
-    .version('0.1.1');
+    .version('0.1.3');
 program
     .command('lint')
     .argument('[path]', 'skill file or directory', '.')
@@ -43,18 +43,22 @@ program
     .action(async (path, opts) => {
     try {
         const files = discoverTestFiles(path);
-        if (!opts.json)
+        if (!opts.json) {
             console.log(`${files.length} test file(s); live agent runs cost money — cap with --budget`);
+            if (!opts.model)
+                console.log('no --model given — using the CLI default model; pass --model haiku for cheap runs');
+        }
+        const budgetUsd = opts.budget ? Number(opts.budget) : undefined;
         const reports = await runTests(path, {
             model: opts.model,
-            budgetUsd: opts.budget ? Number(opts.budget) : undefined,
+            budgetUsd,
             timeoutMs: Number(opts.timeout) * 1000,
             cache: opts.cache,
         });
         if (opts.json)
             console.log(JSON.stringify(reports, null, 2));
         else
-            printReports(reports);
+            printReports(reports, budgetUsd);
         process.exitCode = reportsPass(reports) ? 0 : 1;
     }
     catch (e) {

package/dist/lint/rules.js

@@ -50,6 +50,9 @@ export const relativeLinks = (s) => {
         const target = m[1];
         if (/^(https?:|mailto:|#)/.test(target))
             continue;
+        // templated targets like {baseDir}/x.md resolve at runtime; can't judge them
+        if (target.includes('{'))
+            continue;
         if (!existsSync(resolve(s.dir, target.split('#')[0])))
             out.push(finding(s, 'relative-links', 'error', `broken relative link: ${target}`));
     }

package/dist/reporters/terminal.js

@@ -7,7 +7,7 @@ export function printFindings(findings) {
     const errors = findings.filter((f) => f.severity === 'error').length;
     console.log(`\n${findings.length} finding(s), ${errors} error(s)`);
 }
-export function printReports(reports) {
+export function printReports(reports, budgetUsd) {
     let cost = 0;
     for (const r of reports) {
         cost += r.costUsd;
@@ -24,4 +24,6 @@ export function printReports(reports) {
             console.log(pc.red(`     judge: ${r.verdict.score}/10 — ${r.verdict.reasoning}`));
     }
     console.log(`\n${reports.length} case(s), $${cost.toFixed(4)} spent`);
+    if (budgetUsd !== undefined && cost > budgetUsd)
+        console.log(pc.yellow(`budget of $${budgetUsd} exceeded — the cap is checked between cases, so a single case can overshoot it`));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yogiadhik/skillcheck",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "lint and behavioral tests for agent skills",
   "license": "MIT",
   "author": "Yogesh Adhikari",