@yogiadhik/skillcheck 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yogesh Adhikari
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,79 @@
+# skillcheck
+
+Tests for your agent skills.
+
+Skills fail silently. A bad edit to a description and the skill stops
+triggering. A reworded instruction and the agent starts doing something
+you didn't mean. You find out weeks later, in someone else's session.
+
+skillcheck gives skills what code has had for fifty years: a test suite.
+
+    npx @yogiadhik/skillcheck lint .claude/skills/
+    npx @yogiadhik/skillcheck run .claude/skills/commit-helper
+
+`lint` is free and instant: frontmatter problems, weak descriptions
+(the description is the only thing the agent reads when deciding to
+trigger your skill), token cost, broken relative links, dangerous
+patterns. `run` is the real thing: it copies a fixture into a throwaway
+workspace, mounts your skill, runs the agent headless, and tells you
+what actually happened — which files changed, which commands ran,
+whether the skill fired at all.
+
+## writing a test
+
+One YAML file next to your skill. Deterministic checks cost nothing
+extra; a judge rubric is there for the fuzzy parts.
+
+    # commit-helper.test.yaml
+    skill: commit-helper
+    cases:
+      - name: writes conventional commit
+        prompt: "commit my staged changes"
+        setup: fixtures/dirty-repo
+        expect:
+          files_changed: []
+          command_ran: "git commit"
+          transcript_matches: "(feat|fix|chore):"
+        judge: |
+          Did the commit message accurately describe the staged diff?
+          Score 0-10.
+
+`skill:` is the path from the test file to the skill directory.
+`expect:` supports files_changed, command_ran, transcript_matches and
+skill_invoked. `judge:` sends the transcript and your rubric to a cheap
+model; pass is 7/10 by default (set judge_pass to change it).
+
+Run `npx @yogiadhik/skillcheck init <skill-dir>` to scaffold a starter file.
+
+## cost
+
+Behavioral runs call the real agent, and that costs real money.
+skillcheck caches results by content hash — unchanged skill, unchanged
+case, no re-run — and `--budget 2.50` hard-stops a run at the cap.
+Lint never spends anything.
+
+## CI
+
+    - uses: YogiAdhik/skillcheck/action@v0.1.0
+      with:
+        path: .claude/skills
+
+That runs lint with GitHub annotations on every pull request. For
+behavioral runs in CI, call `npx @yogiadhik/skillcheck run` yourself in a job that
+has your agent authenticated; keep a budget flag on it.
+
+## a note on safety
+
+`run` executes the agent with permission prompts disabled, inside a
+throwaway temp directory. That's the point — but it means fixtures
+should be copies, never live data, and skills you don't trust should be
+linted, not run.
+
+## adapters
+
+Claude Code today. An adapter is a name and one function that takes a
+prompt plus a working directory and returns a transcript; if you want
+Codex or Gemini support, `src/adapters/` is where it goes and I'll take
+the pull request.
+
+MIT.

package/dist/adapters/claude-code.js

@@ -0,0 +1,59 @@
+import spawn from 'cross-spawn';
+import { parseStreamJson } from '../transcript.js';
+export function buildArgs(opts) {
+    const args = [
+        '-p',
+        opts.prompt,
+        '--output-format',
+        'stream-json',
+        '--verbose',
+        '--max-turns',
+        String(opts.maxTurns ?? 30),
+    ];
+    if (!opts.bare)
+        args.push('--dangerously-skip-permissions');
+    if (opts.model)
+        args.push('--model', opts.model);
+    return args;
+}
+export const claudeCode = {
+    name: 'claude-code',
+    run(opts) {
+        return new Promise((done, fail) => {
+            const child = spawn('claude', buildArgs(opts), {
+                cwd: opts.cwd,
+                env: process.env,
+                detached: process.platform !== 'win32',
+            });
+            let out = '';
+            let err = '';
+            const timeoutMs = opts.timeoutMs ?? 300_000;
+            const timer = setTimeout(() => {
+                if (process.platform === 'win32') {
+                    child.kill('SIGKILL');
+                }
+                else {
+                    try {
+                        process.kill(-child.pid, 'SIGKILL');
+                    }
+                    catch {
+                        child.kill('SIGKILL');
+                    }
+                }
+                fail(new Error(`agent run timed out after ${Math.round(timeoutMs / 1000)}s`));
+            }, timeoutMs);
+            child.stdout?.on('data', (d) => (out += d));
+            child.stderr?.on('data', (d) => (err += d));
+            child.on('error', (e) => {
+                clearTimeout(timer);
+                fail(e);
+            });
+            child.on('close', (code) => {
+                clearTimeout(timer);
+                if (code !== 0 && !out.trim())
+                    return fail(new Error(`claude exited ${code}: ${err.slice(0, 500)}`));
+                done(parseStreamJson(out));
+            });
+        });
+    },
+};

package/dist/adapters/types.js

@@ -0,0 +1 @@
+export {};

package/dist/cache.js

@@ -0,0 +1,48 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+export function hashInputs(dirs, caseYaml) {
+    const h = createHash('sha1').update(caseYaml).update('\0');
+    const walk = (dir, prefix) => {
+        const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
+        for (const entry of entries) {
+            const rel = prefix + entry.name;
+            if (entry.isDirectory()) {
+                h.update('d\0' + rel + '\0');
+                walk(join(dir, entry.name), rel + '/');
+            }
+            else {
+                h.update('f\0' + rel + '\0').update(readFileSync(join(dir, entry.name))).update('\0');
+            }
+        }
+    };
+    const dirList = Array.isArray(dirs) ? dirs : [dirs];
+    dirList.forEach((dir, i) => {
+        h.update(`dir${i}\0`);
+        walk(dir, '');
+    });
+    return h.digest('hex');
+}
+export class ResultCache {
+    path;
+    data = {};
+    constructor(path) {
+        this.path = path;
+        if (existsSync(path)) {
+            try {
+                this.data = JSON.parse(readFileSync(path, 'utf8'));
+            }
+            catch {
+                this.data = {};
+            }
+        }
+    }
+    get(key) {
+        return this.data[key];
+    }
+    set(key, value) {
+        this.data[key] = value;
+        mkdirSync(dirname(this.path), { recursive: true });
+        writeFileSync(this.path, JSON.stringify(this.data, null, 2));
+    }
+}

package/dist/cli.js

@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { initTestFile } from './init.js';
+import { lintPath } from './lint/lint.js';
+import { printGithubAnnotations } from './reporters/github.js';
+import { printFindings, printReports } from './reporters/terminal.js';
+import { discoverTestFiles, reportsPass, runTests } from './runner.js';
+const program = new Command();
+program
+    .name('skillcheck')
+    .description('lint and behavioral tests for agent skills')
+    .version('0.1.0');
+program
+    .command('lint')
+    .argument('[path]', 'skill file or directory', '.')
+    .option('--json', 'machine-readable output')
+    .option('--github', 'github actions annotations')
+    .action((path, opts) => {
+    try {
+        const findings = lintPath(path);
+        if (opts.json)
+            console.log(JSON.stringify(findings, null, 2));
+        else if (opts.github)
+            printGithubAnnotations(findings);
+        else
+            printFindings(findings);
+        process.exitCode = findings.some((f) => f.severity === 'error') ? 1 : 0;
+    }
+    catch (e) {
+        console.error(e.message);
+        process.exitCode = 1;
+        return;
+    }
+});
+program
+    .command('run')
+    .argument('[path]', 'directory with *.test.yaml files', '.')
+    .option('--model <model>', 'model passed through to the agent')
+    .option('--budget <usd>', 'stop once this much has been spent')
+    .option('--timeout <seconds>', 'per-case timeout', '300')
+    .option('--no-cache', 'ignore and do not write cached results')
+    .option('--json', 'machine-readable output')
+    .action(async (path, opts) => {
+    try {
+        const files = discoverTestFiles(path);
+        if (!opts.json)
+            console.log(`${files.length} test file(s); live agent runs cost money — cap with --budget`);
+        const reports = await runTests(path, {
+            model: opts.model,
+            budgetUsd: opts.budget ? Number(opts.budget) : undefined,
+            timeoutMs: Number(opts.timeout) * 1000,
+            cache: opts.cache,
+        });
+        if (opts.json)
+            console.log(JSON.stringify(reports, null, 2));
+        else
+            printReports(reports);
+        process.exitCode = reportsPass(reports) ? 0 : 1;
+    }
+    catch (e) {
+        console.error(e.message);
+        process.exitCode = 1;
+    }
+});
+program
+    .command('init')
+    .argument('<skillDir>', 'skill directory to scaffold tests for')
+    .action((skillDir) => {
+    try {
+        console.log(`wrote ${initTestFile(skillDir)}`);
+    }
+    catch (e) {
+        console.error(e.message);
+        process.exitCode = 1;
+    }
+});
+program.parseAsync();

package/dist/expect.js

@@ -0,0 +1,47 @@
+export function checkExpectations(exp, transcript, diff) {
+    const out = [];
+    if (exp.files_changed !== undefined) {
+        const actual = [...diff.changed, ...diff.created, ...diff.deleted].sort();
+        const wanted = [...exp.files_changed].sort();
+        const pass = actual.length === wanted.length && actual.every((v, i) => v === wanted[i]);
+        out.push({
+            name: 'files_changed',
+            pass,
+            detail: pass ? 'ok' : `expected [${wanted.join(', ')}] got [${actual.join(', ')}]`,
+        });
+    }
+    if (exp.command_ran !== undefined) {
+        const pass = transcript.commands.some((c) => c.includes(exp.command_ran));
+        out.push({
+            name: 'command_ran',
+            pass,
+            detail: pass
+                ? 'ok'
+                : `no command containing "${exp.command_ran}" (ran: ${transcript.commands.join(' ; ') || 'none'})`,
+        });
+    }
+    if (exp.transcript_matches !== undefined) {
+        let pass = false;
+        let detail;
+        try {
+            const re = new RegExp(exp.transcript_matches, 'm');
+            const haystack = [...transcript.messages, ...transcript.commands].join('\n');
+            pass = re.test(haystack);
+            detail = pass ? 'ok' : `nothing in the transcript matches /${exp.transcript_matches}/`;
+        }
+        catch {
+            detail = `invalid regex: ${exp.transcript_matches}`;
+        }
+        out.push({ name: 'transcript_matches', pass, detail });
+    }
+    if (exp.skill_invoked !== undefined) {
+        const invoked = transcript.toolCalls.some((c) => c.name === 'Skill');
+        const pass = invoked === exp.skill_invoked;
+        out.push({
+            name: 'skill_invoked',
+            pass,
+            detail: pass ? 'ok' : invoked ? 'skill fired but was expected not to' : 'skill never fired',
+        });
+    }
+    return out;
+}

package/dist/init.js

@@ -0,0 +1,27 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { basename, join, resolve } from 'node:path';
+import { isParseError, loadSkill } from './skill.js';
+export function initTestFile(skillDir) {
+    const dir = resolve(skillDir);
+    const skill = loadSkill(join(dir, 'SKILL.md'));
+    if (isParseError(skill))
+        throw new Error(`${skill.path}: ${skill.error}`);
+    const name = basename(dir);
+    const out = join(dir, 'tests', `${name}.test.yaml`);
+    if (existsSync(out))
+        throw new Error(`${out} already exists`);
+    mkdirSync(join(dir, 'tests'), { recursive: true });
+    const firstLine = skill.description.split('\n')[0];
+    writeFileSync(out, [
+        `# behavioral tests for ${name}`,
+        `# description: ${firstLine}`,
+        'skill: ..',
+        'cases:',
+        '  - name: does its job',
+        '    prompt: "REPLACE: a prompt that should trigger this skill"',
+        '    expect:',
+        '      skill_invoked: true',
+        '',
+    ].join('\n'));
+    return out;
+}

package/dist/judge.js

@@ -0,0 +1,71 @@
+import { tmpdir } from 'node:os';
+import { claudeCode } from './adapters/claude-code.js';
+export function summarize(t) {
+    const parts = [];
+    if (t.messages.length)
+        parts.push('assistant said:\n' + t.messages.join('\n---\n'));
+    if (t.commands.length)
+        parts.push('commands run:\n' + t.commands.join('\n'));
+    const tools = t.toolCalls.map((c) => c.name).join(', ');
+    if (tools)
+        parts.push('tools used: ' + tools);
+    return parts.join('\n\n') || '(empty transcript)';
+}
+export function parseVerdict(text, passScore) {
+    const candidates = [...text.matchAll(/\{[\s\S]*?\}/g), ...(text.match(/\{[\s\S]*\}/) ? [[text.match(/\{[\s\S]*\}/)[0]]] : [])];
+    for (const m of candidates) {
+        let parsed;
+        try {
+            parsed = JSON.parse(m[0]);
+        }
+        catch {
+            continue;
+        }
+        const score = parsed.score;
+        if (typeof score !== 'number' || !Number.isFinite(score))
+            continue;
+        const clamped = Math.min(10, Math.max(0, score));
+        const reasoning = parsed.reasoning;
+        return {
+            score: clamped,
+            pass: clamped >= passScore,
+            reasoning: typeof reasoning === 'string' ? reasoning : '',
+        };
+    }
+    throw new Error('judge returned no JSON verdict');
+}
+export async function judgeTranscript(opts) {
+    const prompt = [
+        'You are grading an AI agent transcript against a rubric.',
+        'Rubric:',
+        opts.rubric,
+        '',
+        'Transcript:',
+        summarize(opts.transcript),
+        '',
+        'Reply with only a JSON object: {"score": <0-10>, "reasoning": "<one sentence>"}',
+    ].join('\n');
+    const passScore = opts.passScore ?? 7;
+    const adapter = opts.adapter ?? claudeCode;
+    let costUsd = 0;
+    const attempt = async () => {
+        const t = await adapter.run({
+            prompt,
+            cwd: tmpdir(),
+            bare: true,
+            model: opts.model ?? 'haiku',
+            maxTurns: 1,
+            timeoutMs: 120_000,
+        });
+        costUsd += t.result?.costUsd ?? 0;
+        return parseVerdict(t.messages.join('\n'), passScore);
+    };
+    let verdict;
+    try {
+        verdict = await attempt();
+    }
+    catch {
+        verdict = await attempt();
+    }
+    return { ...verdict, costUsd };
+}

package/dist/lint/lint.js

@@ -0,0 +1,35 @@
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { isParseError, loadSkill } from '../skill.js';
+import { allRules } from './rules.js';
+const SKIP_DIRS = new Set(['node_modules', '.git', 'dist']);
+export function discoverSkillFiles(path) {
+    if (!existsSync(path))
+        throw new Error(`${path}: no such file or directory`);
+    if (statSync(path).isFile())
+        return [path];
+    const found = [];
+    const walk = (dir) => {
+        for (const entry of readdirSync(dir, { withFileTypes: true })) {
+            if (entry.isDirectory() && !SKIP_DIRS.has(entry.name))
+                walk(join(dir, entry.name));
+            else if (entry.isFile() && entry.name === 'SKILL.md')
+                found.push(join(dir, entry.name));
+        }
+    };
+    walk(path);
+    return found.sort();
+}
+export function lintPath(path) {
+    const findings = [];
+    for (const file of discoverSkillFiles(path)) {
+        const skill = loadSkill(file);
+        if (isParseError(skill)) {
+            findings.push({ rule: 'parse', severity: 'error', message: skill.error, file });
+            continue;
+        }
+        for (const rule of allRules)
+            findings.push(...rule(skill));
+    }
+    return findings;
+}

package/dist/lint/rules.js

@@ -0,0 +1,57 @@
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+function finding(s, rule, severity, message) {
+    return { rule, severity, message, file: s.path };
+}
+export const frontmatterFields = (s) => {
+    const out = [];
+    if (typeof s.frontmatter.name !== 'string' || !s.frontmatter.name.trim())
+        out.push(finding(s, 'frontmatter-fields', 'error', 'frontmatter is missing a name'));
+    if (!s.description.trim())
+        out.push(finding(s, 'frontmatter-fields', 'error', 'frontmatter is missing a description'));
+    return out;
+};
+export const descriptionQuality = (s) => {
+    const d = s.description.trim();
+    if (!d)
+        return [];
+    const out = [];
+    if (d.length < 20)
+        out.push(finding(s, 'description-quality', 'warning', 'description under 20 chars; unlikely to trigger reliably'));
+    if (d.length > 1000)
+        out.push(finding(s, 'description-quality', 'warning', 'description over 1000 chars; gets truncated in listings'));
+    if (!/\b(use|when|for|trigger)\b/i.test(d))
+        out.push(finding(s, 'description-quality', 'warning', 'description has no trigger language ("use when ...") — the agent decides from this text alone'));
+    return out;
+};
+export const tokenBudget = (s) => {
+    const tokens = Math.round(s.raw.length / 4);
+    if (tokens > 10_000)
+        return [finding(s, 'token-budget', 'warning', `skill is roughly ${tokens} tokens; every invocation pays this cost`)];
+    return [];
+};
+export const relativeLinks = (s) => {
+    const out = [];
+    for (const m of s.body.matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) {
+        const target = m[1];
+        if (/^(https?:|mailto:|#)/.test(target))
+            continue;
+        if (!existsSync(resolve(s.dir, target.split('#')[0])))
+            out.push(finding(s, 'relative-links', 'error', `broken relative link: ${target}`));
+    }
+    return out;
+};
+const DANGEROUS = [
+    { re: /rm\s+-rf\s+[~/]/, why: 'destructive rm -rf on a home or root path' },
+    { re: /curl[^\n|]*\|\s*(ba)?sh/, why: 'pipes curl to a shell' },
+    { re: /--dangerously-skip-permissions/, why: 'tells the agent to skip permission prompts' },
+    { re: /\bsudo\b/, why: 'asks for sudo' },
+];
+export const dangerousPatterns = (s) => {
+    const out = [];
+    for (const { re, why } of DANGEROUS)
+        if (re.test(s.body))
+            out.push(finding(s, 'dangerous-patterns', 'warning', why));
+    return out;
+};
+export const allRules = [frontmatterFields, descriptionQuality, tokenBudget, relativeLinks, dangerousPatterns];

package/dist/reporters/github.js

@@ -0,0 +1,6 @@
+export function printGithubAnnotations(findings) {
+    for (const f of findings) {
+        const kind = f.severity === 'error' ? 'error' : 'warning';
+        console.log(`::${kind} file=${f.file}::[${f.rule}] ${f.message}`);
+    }
+}

package/dist/reporters/terminal.js

@@ -0,0 +1,27 @@
+import pc from 'picocolors';
+export function printFindings(findings) {
+    for (const f of findings) {
+        const tag = f.severity === 'error' ? pc.red('error') : pc.yellow('warn ');
+        console.log(`${tag} ${f.file} [${f.rule}] ${f.message}`);
+    }
+    const errors = findings.filter((f) => f.severity === 'error').length;
+    console.log(`\n${findings.length} finding(s), ${errors} error(s)`);
+}
+export function printReports(reports) {
+    let cost = 0;
+    for (const r of reports) {
+        cost += r.costUsd;
+        const failed = r.checks.filter((c) => !c.pass);
+        const ok = !r.error && failed.length === 0 && (r.verdict?.pass ?? true);
+        const mark = ok ? pc.green('pass') : pc.red('fail');
+        const cached = r.cached ? pc.dim(' (cached)') : '';
+        console.log(`${mark} ${r.case}${cached}`);
+        if (r.error)
+            console.log(pc.red(`     ${r.error}`));
+        for (const c of failed)
+            console.log(pc.red(`     ${c.name}: ${c.detail}`));
+        if (r.verdict && !r.verdict.pass)
+            console.log(pc.red(`     judge: ${r.verdict.score}/10 — ${r.verdict.reasoning}`));
+    }
+    console.log(`\n${reports.length} case(s), $${cost.toFixed(4)} spent`);
+}

package/dist/runner.js

@@ -0,0 +1,119 @@
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { rm } from 'node:fs/promises';
+import { basename, dirname, join, resolve } from 'node:path';
+import YAML from 'yaml';
+import { claudeCode } from './adapters/claude-code.js';
+import { hashInputs, ResultCache } from './cache.js';
+import { checkExpectations } from './expect.js';
+import { judgeTranscript } from './judge.js';
+import { diffWorkspace, makeWorkspace, snapshot } from './workspace.js';
+const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', '.skillcheck']);
+export function discoverTestFiles(path) {
+    if (statSync(path).isFile())
+        return [path];
+    const found = [];
+    const walk = (dir) => {
+        for (const entry of readdirSync(dir, { withFileTypes: true })) {
+            if (entry.isDirectory() && !SKIP_DIRS.has(entry.name))
+                walk(join(dir, entry.name));
+            else if (entry.isFile() && entry.name.endsWith('.test.yaml'))
+                found.push(join(dir, entry.name));
+        }
+    };
+    walk(path);
+    return found.sort();
+}
+export function loadTestFile(path) {
+    const doc = YAML.parse(readFileSync(path, 'utf8'));
+    if (!doc || typeof doc.skill !== 'string' || !Array.isArray(doc.cases))
+        throw new Error(`${path}: needs a "skill" path and a "cases" list`);
+    return { path, skill: doc.skill, cases: doc.cases };
+}
+export async function runTests(root, opts = {}) {
+    const adapter = opts.adapter ?? claudeCode;
+    const cache = new ResultCache(join(process.cwd(), '.skillcheck', 'cache.json'));
+    const reports = [];
+    let spent = 0;
+    for (const file of discoverTestFiles(root)) {
+        const testFile = loadTestFile(file);
+        const skillDir = resolve(dirname(file), testFile.skill);
+        for (const c of testFile.cases) {
+            if (opts.budgetUsd !== undefined && spent >= opts.budgetUsd) {
+                reports.push({
+                    file,
+                    case: c.name,
+                    checks: [],
+                    costUsd: 0,
+                    cached: false,
+                    error: `budget of $${opts.budgetUsd} reached before this case`,
+                });
+                continue;
+            }
+            const key = hashInputs(c.setup ? [skillDir, resolve(dirname(file), c.setup)] : skillDir, YAML.stringify(c) + '\0' + (opts.model ?? ''));
+            if (opts.cache !== false) {
+                const hit = cache.get(key);
+                if (hit) {
+                    reports.push({ ...hit, cached: true });
+                    continue;
+                }
+            }
+            let ws;
+            try {
+                ws = await makeWorkspace({
+                    fixture: c.setup ? resolve(dirname(file), c.setup) : undefined,
+                    skillDir,
+                    skillName: basename(skillDir),
+                });
+                const before = await snapshot(ws);
+                const transcript = await adapter.run({
+                    prompt: c.prompt,
+                    cwd: ws,
+                    model: opts.model,
+                    timeoutMs: opts.timeoutMs,
+                });
+                const diff = await diffWorkspace(ws, before);
+                const checks = c.expect ? checkExpectations(c.expect, transcript, diff) : [];
+                let verdict;
+                if (c.judge)
+                    verdict = await judgeTranscript({
+                        rubric: c.judge,
+                        transcript,
+                        model: opts.model,
+                        passScore: c.judge_pass,
+                        adapter,
+                    });
+                const judgeCost = verdict?.costUsd ?? 0;
+                const costUsd = (transcript.result?.costUsd ?? 0) + judgeCost;
+                spent += costUsd;
+                const report = { file, case: c.name, checks, verdict, costUsd, cached: false };
+                reports.push(report);
+                if (opts.cache !== false) {
+                    try {
+                        cache.set(key, report);
+                    }
+                    catch {
+                        // a failed cache write must not fail the case
+                    }
+                }
+            }
+            catch (e) {
+                reports.push({
+                    file,
+                    case: c.name,
+                    checks: [],
+                    costUsd: 0,
+                    cached: false,
+                    error: e.message,
+                });
+            }
+            finally {
+                if (ws)
+                    await rm(ws, { recursive: true, force: true });
+            }
+        }
+    }
+    return reports;
+}
+export function reportsPass(reports) {
+    return reports.every((r) => !r.error && r.checks.every((c) => c.pass) && (r.verdict?.pass ?? true));
+}

package/dist/skill.js

@@ -0,0 +1,37 @@
+import { readFileSync } from 'node:fs';
+import { basename, dirname } from 'node:path';
+import YAML from 'yaml';
+export function loadSkill(path) {
+    let raw;
+    try {
+        raw = readFileSync(path, 'utf8');
+    }
+    catch {
+        return { path, error: 'cannot read file' };
+    }
+    if (!raw.startsWith('---'))
+        return { path, error: 'missing frontmatter' };
+    const end = raw.indexOf('\n---', 3);
+    if (end === -1)
+        return { path, error: 'unterminated frontmatter' };
+    let fm;
+    try {
+        fm = YAML.parse(raw.slice(3, end)) ?? {};
+    }
+    catch (e) {
+        return { path, error: `bad yaml in frontmatter: ${e.message}` };
+    }
+    const body = raw.slice(raw.indexOf('\n', end + 1) + 1);
+    return {
+        path,
+        dir: dirname(path),
+        name: typeof fm.name === 'string' ? fm.name : basename(dirname(path)),
+        description: typeof fm.description === 'string' ? fm.description : '',
+        frontmatter: fm,
+        body,
+        raw,
+    };
+}
+export function isParseError(s) {
+    return 'error' in s;
+}

package/dist/transcript.js

@@ -0,0 +1,34 @@
+export function parseStreamJson(output) {
+    const t = { messages: [], toolCalls: [], commands: [] };
+    for (const line of output.split('\n')) {
+        if (!line.trim().startsWith('{'))
+            continue;
+        let event;
+        try {
+            event = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (event.type === 'assistant') {
+            for (const block of event.message?.content ?? []) {
+                if (block.type === 'text' && block.text)
+                    t.messages.push(block.text);
+                if (block.type === 'tool_use' && typeof block.name === 'string') {
+                    const call = { name: block.name, input: block.input ?? {} };
+                    t.toolCalls.push(call);
+                    if (call.name === 'Bash' && typeof call.input.command === 'string')
+                        t.commands.push(call.input.command);
+                }
+            }
+        }
+        if (event.type === 'result') {
+            t.result = {
+                costUsd: event.total_cost_usd ?? 0,
+                numTurns: event.num_turns ?? 0,
+                isError: !!event.is_error,
+            };
+        }
+    }
+    return t;
+}

package/dist/workspace.js

@@ -0,0 +1,49 @@
+import { createHash } from 'node:crypto';
+import { cp, mkdtemp, readdir, readFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join, relative, sep } from 'node:path';
+const SKIP = new Set(['.git', '.claude', 'node_modules']);
+async function walkFiles(root, dir = root) {
+    const out = [];
+    for (const entry of await readdir(dir, { withFileTypes: true })) {
+        if (SKIP.has(entry.name))
+            continue;
+        const p = join(dir, entry.name);
+        if (entry.isDirectory())
+            out.push(...(await walkFiles(root, p)));
+        else
+            out.push(p);
+    }
+    return out;
+}
+export async function snapshot(dir) {
+    const map = new Map();
+    for (const file of await walkFiles(dir)) {
+        const hash = createHash('sha1').update(await readFile(file)).digest('hex');
+        map.set(relative(dir, file).split(sep).join('/'), hash);
+    }
+    return map;
+}
+export async function makeWorkspace(opts) {
+    const dir = await mkdtemp(join(tmpdir(), 'skillcheck-'));
+    if (opts.fixture)
+        await cp(opts.fixture, dir, { recursive: true });
+    await cp(opts.skillDir, join(dir, '.claude', 'skills', opts.skillName), { recursive: true });
+    return dir;
+}
+export async function diffWorkspace(dir, before) {
+    const after = await snapshot(dir);
+    const changed = [];
+    const created = [];
+    const deleted = [];
+    for (const [path, hash] of after) {
+        if (!before.has(path))
+            created.push(path);
+        else if (before.get(path) !== hash)
+            changed.push(path);
+    }
+    for (const path of before.keys())
+        if (!after.has(path))
+            deleted.push(path);
+    return { changed: changed.sort(), created: created.sort(), deleted: deleted.sort() };
+}

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@yogiadhik/skillcheck",
+  "version": "0.1.0",
+  "description": "lint and behavioral tests for agent skills",
+  "license": "MIT",
+  "author": "Yogesh Adhikari",
+  "type": "module",
+  "bin": {
+    "skillcheck": "dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "prepublishOnly": "npm run build && npm test"
+  },
+  "dependencies": {
+    "commander": "^15.0.0",
+    "cross-spawn": "^7.0.6",
+    "picocolors": "^1.1.1",
+    "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "@types/cross-spawn": "^6.0.6",
+    "@types/node": "^26.1.1",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10"
+  }
+}