@alexanderzzlatkov/skilleval 0.1.0

package/dist/evaluator.js

@@ -0,0 +1,239 @@
+import { generateText } from 'ai';
+import { buildComplianceSystemPrompt, buildMockTools } from './context-builder.js';
+const RETRY_DELAY_MS = 2000;
+const MAX_RETRIES = 3;
+async function generateWithRetry(models, options) {
+    for (let modelIdx = 0; modelIdx < models.length; modelIdx++) {
+        for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+            try {
+                const { text } = await generateText({
+                    model: models[modelIdx],
+                    system: options.system,
+                    prompt: options.prompt,
+                    temperature: options.temperature,
+                });
+                return text;
+            }
+            catch {
+                if (attempt < MAX_RETRIES - 1) {
+                    await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS));
+                }
+            }
+        }
+    }
+    throw new Error('All judge models failed');
+}
+function stripCodeFences(text) {
+    return text.replace(/^```(?:json)?\s*\n?/gm, '').replace(/\n?```\s*$/gm, '').trim();
+}
+function safeParseTriggerEval(text) {
+    try {
+        const parsed = JSON.parse(stripCodeFences(text));
+        return {
+            triggered: Boolean(parsed.triggered),
+            correct: Boolean(parsed.correct),
+            reason: String(parsed.reason ?? ''),
+        };
+    }
+    catch {
+        return { triggered: false, correct: false, reason: 'Failed to parse judge response' };
+    }
+}
+function safeParseComplianceEval(text) {
+    try {
+        const parsed = JSON.parse(stripCodeFences(text));
+        return {
+            compliant: Boolean(parsed.compliant),
+            score: Math.max(0, Math.min(100, Number(parsed.score) || 0)),
+            reason: String(parsed.reason ?? ''),
+        };
+    }
+    catch {
+        return { compliant: false, score: 0, reason: 'Failed to parse judge response' };
+    }
+}
+const TRIGGER_JUDGE_SYSTEM = `You are an evaluation judge. You assess whether an AI model correctly identified when to use a specific skill.
+Respond ONLY with valid JSON, no other text.`;
+function triggerJudgePrompt(skill, promptText, promptType, response) {
+    return `## Skill Being Tested
+Name: ${skill.name}
+Description: ${skill.description}
+
+## User Prompt
+"${promptText}"
+
+## Expected Behavior
+This is a ${promptType} test case.
+${promptType === 'positive'
+        ? 'The model SHOULD have recognized this skill as relevant and indicated it would use it.'
+        : 'The model should NOT have mentioned or tried to use this skill.'}
+
+## Model Response
+${response}
+
+## Your Task
+Did the model behave correctly? Evaluate whether it correctly identified (or correctly ignored) the "${skill.name}" skill.
+
+Respond ONLY with JSON:
+{"triggered": true/false, "correct": true/false, "reason": "brief explanation"}`;
+}
+const COMPLIANCE_JUDGE_SYSTEM = `You are an evaluation judge. You assess whether an AI model correctly followed a skill's instructions by calling the right tools and following the right workflow.
+Respond ONLY with valid JSON, no other text.`;
+function complianceJudgePrompt(skill, promptText, response) {
+    return `## Skill Instructions
+${skill.body}
+
+## User Prompt
+"${promptText}"
+
+## Model Response and Tool Calls
+${response}
+
+## Your Task
+Did the model correctly follow the skill's instructions? Evaluate:
+1. Did it call the correct tools as described in the skill?
+2. Did it pass reasonable arguments to those tools?
+3. Did it follow the stated workflow/steps in the right order?
+4. Did it stay within the scope of the skill?
+
+Note: Tools returned mock results. Do not penalize for the quality of returned data — focus on whether the model called the right tools with the right parameters.
+
+Respond ONLY with JSON:
+{"compliant": true/false, "score": 0-100, "reason": "brief explanation"}`;
+}
+export async function evaluateResults(skill, testResults, judgeModels, models, verbose) {
+    const evalResults = [];
+    const total = testResults.length;
+    let completed = 0;
+    const mockTools = buildMockTools();
+    const toolNames = Object.keys(mockTools);
+    const complianceSystemPrompt = buildComplianceSystemPrompt(skill);
+    if (verbose) {
+        process.stderr.write(`\n  Compliance system prompt:\n  ---\n${complianceSystemPrompt}\n  ---\n`);
+        if (toolNames.length > 0) {
+            process.stderr.write(`  Mock tools provided: ${toolNames.join(', ')}\n`);
+        }
+        process.stderr.write('\n');
+    }
+    for (const result of testResults) {
+        completed++;
+        const prefix = `  [${completed}/${total}] ${result.modelId} — ${result.prompt.type}: "${result.prompt.text.slice(0, 40)}..."`;
+        if (result.error) {
+            process.stderr.write(`\n${prefix} — skipped (error)\n`);
+            evalResults.push({
+                modelId: result.modelId,
+                prompt: result.prompt,
+                response: result.response,
+                trigger: { triggered: false, correct: false, reason: `Error: ${result.error}` },
+            });
+            continue;
+        }
+        // Trigger evaluation
+        process.stderr.write(`\n${prefix}\n    Judging trigger...`);
+        let trigger;
+        try {
+            const text = await generateWithRetry(judgeModels, {
+                system: TRIGGER_JUDGE_SYSTEM,
+                prompt: triggerJudgePrompt(skill, result.prompt.text, result.prompt.type, result.response),
+                temperature: 0.1,
+            });
+            trigger = safeParseTriggerEval(text);
+            process.stderr.write(` ${trigger.correct ? 'PASS' : 'FAIL'} (triggered: ${trigger.triggered})\n`);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            trigger = { triggered: false, correct: false, reason: `Judge call failed: ${message}` };
+            process.stderr.write(` FAILED: ${message}\n`);
+        }
+        // Compliance evaluation (only for positive prompts where skill was triggered)
+        let compliance;
+        if (result.prompt.type === 'positive' && trigger.triggered) {
+            // Run compliance test: send the same prompt with full skill content and mock tools
+            const modelEntry = models.find(m => m.modelId === result.modelId);
+            if (modelEntry) {
+                try {
+                    process.stderr.write(`    Running compliance test...`);
+                    if (toolNames.length > 0) {
+                        process.stderr.write(` (mock tools: ${toolNames.join(', ')})`);
+                    }
+                    const { text: complianceResponse, toolCalls, steps } = await generateText({
+                        model: modelEntry.model,
+                        messages: [
+                            { role: 'system', content: complianceSystemPrompt },
+                            { role: 'user', content: result.prompt.text },
+                        ],
+                        tools: mockTools,
+                        maxSteps: 10,
+                        temperature: 0.3,
+                    });
+                    // Collect all tool calls across all steps
+                    const allToolCalls = steps.flatMap(step => step.toolCalls ?? []);
+                    const toolCallSummary = allToolCalls.length > 0
+                        ? `\n\nTool calls made:\n${allToolCalls.map(tc => `- ${tc.toolName}(${JSON.stringify(tc.args)})`).join('\n')}`
+                        : '\n\nNo tool calls were made.';
+                    const fullResponse = complianceResponse + toolCallSummary;
+                    process.stderr.write(` done (${allToolCalls.length} tool calls, ${steps.length} steps)\n`);
+                    if (verbose) {
+                        if (allToolCalls.length > 0) {
+                            process.stderr.write(`    Tool calls:\n`);
+                            for (const tc of allToolCalls) {
+                                process.stderr.write(`      - ${tc.toolName}(${JSON.stringify(tc.args).slice(0, 100)})\n`);
+                            }
+                        }
+                    }
+                    process.stderr.write(`    Judging compliance...`);
+                    const judgeText = await generateWithRetry(judgeModels, {
+                        system: COMPLIANCE_JUDGE_SYSTEM,
+                        prompt: complianceJudgePrompt(skill, result.prompt.text, fullResponse),
+                        temperature: 0.1,
+                    });
+                    compliance = safeParseComplianceEval(judgeText);
+                    process.stderr.write(` ${compliance.compliant ? 'PASS' : 'FAIL'} (${compliance.score}/100)\n`);
+                }
+                catch (err) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    compliance = { compliant: false, score: 0, reason: `Compliance evaluation failed: ${message}` };
+                    process.stderr.write(` FAILED: ${message}\n`);
+                }
+            }
+        }
+        evalResults.push({
+            modelId: result.modelId,
+            prompt: result.prompt,
+            response: result.response,
+            trigger,
+            compliance,
+        });
+        if (verbose) {
+            process.stderr.write(`    Reason: ${trigger.reason}\n`);
+            if (compliance) {
+                process.stderr.write(`    Compliance reason: ${compliance.reason}\n`);
+            }
+        }
+    }
+    process.stderr.write('\n');
+    return evalResults;
+}
+export function computeReport(evalResults, modelIds) {
+    return modelIds.map(modelId => {
+        const modelResults = evalResults.filter(r => r.modelId === modelId);
+        const triggerTotal = modelResults.length;
+        const triggerCorrect = modelResults.filter(r => r.trigger.correct).length;
+        const complianceResults = modelResults.filter(r => r.compliance != null);
+        const complianceCorrect = complianceResults.filter(r => r.compliance.compliant).length;
+        const complianceTotal = complianceResults.length;
+        const avgScore = complianceTotal > 0
+            ? complianceResults.reduce((sum, r) => sum + r.compliance.score, 0) / complianceTotal
+            : 0;
+        const triggerAcc = triggerTotal > 0 ? triggerCorrect / triggerTotal : 0;
+        const complianceAcc = complianceTotal > 0 ? complianceCorrect / complianceTotal : 0;
+        const overall = Math.round(triggerAcc * 50 + complianceAcc * 30 + (avgScore / 100) * 20);
+        return {
+            modelId,
+            triggerScore: { correct: triggerCorrect, total: triggerTotal },
+            complianceScore: { correct: complianceCorrect, total: complianceTotal, avgScore: Math.round(avgScore) },
+            overall,
+        };
+    }).sort((a, b) => b.overall - a.overall);
+}
+//# sourceMappingURL=evaluator.js.map

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import 'dotenv/config';

package/dist/index.js

@@ -0,0 +1,113 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { parseSkill } from './parser.js';
+import { createModel, resolveApiKey } from './providers.js';
+import { generateTestPrompts } from './test-generator.js';
+import { runTests } from './runner.js';
+import { evaluateResults, computeReport } from './evaluator.js';
+import { printReport } from './reporter.js';
+import { DEFAULT_FREE_MODELS, DEFAULT_GENERATOR_MODELS, DEFAULT_JUDGE_MODELS, PROVIDER_NAMES, } from './config.js';
+const program = new Command();
+program
+    .name('skilleval')
+    .description('Evaluate how well AI models understand Agent Skills (SKILL.md files)')
+    .version('0.1.0')
+    .argument('<skill>', 'Path, URL, or GitHub shorthand (owner/repo) to a SKILL.md file')
+    .option('-p, --provider <provider>', 'Provider: openrouter, anthropic, openai, google', 'openrouter')
+    .option('-m, --models <models>', 'Comma-separated model IDs to test')
+    .option('-k, --key <key>', 'API key (or use provider-specific env var)')
+    .option('--generator-model <model>', 'Model for test prompt generation (comma-separated for fallbacks)')
+    .option('--judge-model <model>', 'Model for evaluation judging (comma-separated for fallbacks)')
+    .option('--json', 'Output results as JSON', false)
+    .option('--verbose', 'Show detailed per-prompt results', false)
+    .option('--prompts <path>', 'Path to JSON file with custom test prompts')
+    .option('-s, --skill <name>', 'Skill name within the repo (looks for skills/<name>/SKILL.md)')
+    .option('-n, --count <number>', 'Number of positive+negative test prompts (default: 5+5)', '5')
+    .action(async (skillSource, opts) => {
+    try {
+        const provider = opts.provider;
+        if (!PROVIDER_NAMES.includes(provider)) {
+            console.error(chalk.red(`Invalid provider "${provider}". Must be one of: ${PROVIDER_NAMES.join(', ')}`));
+            process.exit(1);
+        }
+        // Resolve API key for the test models
+        let apiKey;
+        try {
+            apiKey = resolveApiKey(provider, opts.key);
+        }
+        catch (err) {
+            console.error(chalk.red(err.message));
+            process.exit(1);
+        }
+        // Resolve model IDs
+        const modelIds = opts.models
+            ? opts.models.split(',').map((m) => m.trim())
+            : (provider === 'openrouter' ? DEFAULT_FREE_MODELS : []);
+        if (modelIds.length === 0) {
+            console.error(chalk.red('No models specified. Use --models or default to openrouter provider for free models.'));
+            process.exit(1);
+        }
+        // Resolve generator/judge keys (always OpenRouter for internal models)
+        let internalApiKey;
+        try {
+            internalApiKey = resolveApiKey('openrouter', provider === 'openrouter' ? apiKey : undefined);
+        }
+        catch {
+            if (!opts.prompts) {
+                console.error(chalk.red('OPENROUTER_API_KEY is required for test generation and evaluation (uses free models).\n' +
+                    'Set OPENROUTER_API_KEY env var, or provide custom prompts with --prompts.'));
+                process.exit(1);
+            }
+            internalApiKey = '';
+        }
+        // Parse skill
+        process.stderr.write(chalk.cyan('Parsing skill...\n'));
+        const skill = await parseSkill(skillSource, opts.skill);
+        if (!opts.json) {
+            console.log(`\n${chalk.bold('skilleval')} v0.1.0`);
+            console.log(`${chalk.bold('Skill:')} ${skill.name}`);
+            console.log(`${chalk.bold('Description:')} ${skill.description}`);
+            console.log(`${chalk.bold('Provider:')} ${provider}`);
+            console.log(`${chalk.bold('Models:')} ${modelIds.length}\n`);
+        }
+        // Create model instances
+        const models = modelIds.map(id => ({
+            model: createModel(provider, id, apiKey),
+            modelId: id,
+        }));
+        // Generate test prompts
+        process.stderr.write(chalk.cyan('Generating test prompts...\n'));
+        const generatorModelIds = opts.generatorModel
+            ? opts.generatorModel.split(',').map((m) => m.trim())
+            : DEFAULT_GENERATOR_MODELS;
+        const generatorModels = generatorModelIds.map(id => createModel('openrouter', id, internalApiKey));
+        const count = parseInt(opts.count, 10);
+        const prompts = await generateTestPrompts(skill, generatorModels, count, opts.prompts, opts.verbose);
+        process.stderr.write(chalk.green(`  Generated ${prompts.length} test prompts\n\n`));
+        // Run trigger tests
+        process.stderr.write(chalk.cyan('Running trigger tests...\n'));
+        const testResults = await runTests(skill, prompts, models, opts.verbose);
+        // Evaluate results
+        process.stderr.write(chalk.cyan('Evaluating results...\n'));
+        const judgeModelIds = opts.judgeModel
+            ? opts.judgeModel.split(',').map((m) => m.trim())
+            : DEFAULT_JUDGE_MODELS;
+        const judgeModels = judgeModelIds.map(id => createModel('openrouter', id, internalApiKey));
+        const evalResults = await evaluateResults(skill, testResults, judgeModels, models, opts.verbose);
+        // Compute and print report
+        const reports = computeReport(evalResults, modelIds);
+        console.log('');
+        printReport(reports, evalResults, { json: opts.json, verbose: opts.verbose });
+        // Exit code based on scores
+        const allPassing = reports.every(r => r.overall >= 50);
+        process.exit(allPassing ? 0 : 1);
+    }
+    catch (err) {
+        console.error(chalk.red(`Error: ${err.message}`));
+        process.exit(1);
+    }
+});
+program.parse();
+//# sourceMappingURL=index.js.map

package/dist/parser.d.ts

@@ -0,0 +1,2 @@
+import type { SkillDefinition } from './config.js';
+export declare function parseSkill(source: string, skillName?: string): Promise<SkillDefinition>;

package/dist/parser.js

@@ -0,0 +1,152 @@
+import { readFile } from 'node:fs/promises';
+import { basename } from 'node:path';
+import matter from 'gray-matter';
+const GITHUB_BLOB_RE = /^https?:\/\/github\.com\/([^/]+)\/([^/]+)\/blob\/([^/]+)\/(.+)$/;
+const GITHUB_REPO_RE = /^https?:\/\/github\.com\/([^/]+)\/([^/]+)\/?$/;
+const GITHUB_SHORTHAND_RE = /^([^/\s]+)\/([^/\s]+)$/;
+const SKILL_SEARCH_PATHS = [
+    'SKILL.md',
+    '.claude/skills',
+    'skills',
+];
+async function fetchText(url) {
+    const res = await fetch(url);
+    if (!res.ok)
+        throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+    return res.text();
+}
+function toRawGitHubUrl(owner, repo, branch, path) {
+    return `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`;
+}
+async function discoverNamedSkill(owner, repo, skillName) {
+    const searchPaths = [
+        `skills/${skillName}/SKILL.md`,
+        `.claude/skills/${skillName}/SKILL.md`,
+        `${skillName}/SKILL.md`,
+    ];
+    for (const path of searchPaths) {
+        for (const branch of ['main', 'master']) {
+            const url = toRawGitHubUrl(owner, repo, branch, path);
+            try {
+                const res = await fetch(url, { method: 'HEAD' });
+                if (res.ok)
+                    return url;
+            }
+            catch { /* continue */ }
+        }
+    }
+    throw new Error(`Could not find skill "${skillName}" in ${owner}/${repo}. Searched ${searchPaths.join(', ')}.`);
+}
+async function discoverSkillInRepo(owner, repo) {
+    // Try root SKILL.md first
+    const rootUrl = toRawGitHubUrl(owner, repo, 'main', 'SKILL.md');
+    try {
+        const res = await fetch(rootUrl, { method: 'HEAD' });
+        if (res.ok)
+            return rootUrl;
+    }
+    catch { /* continue */ }
+    // Try 'master' branch
+    const masterUrl = toRawGitHubUrl(owner, repo, 'master', 'SKILL.md');
+    try {
+        const res = await fetch(masterUrl, { method: 'HEAD' });
+        if (res.ok)
+            return masterUrl;
+    }
+    catch { /* continue */ }
+    // Search via GitHub API for SKILL.md files in common directories
+    for (const searchPath of SKILL_SEARCH_PATHS.slice(1)) {
+        for (const branch of ['main', 'master']) {
+            const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${searchPath}?ref=${branch}`;
+            try {
+                const res = await fetch(apiUrl);
+                if (!res.ok)
+                    continue;
+                const items = await res.json();
+                if (!Array.isArray(items))
+                    continue;
+                for (const item of items) {
+                    if (item.name === 'SKILL.md') {
+                        return toRawGitHubUrl(owner, repo, branch, item.path);
+                    }
+                    // Check subdirectories (e.g., .claude/skills/my-skill/SKILL.md)
+                    const subApiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${item.path}?ref=${branch}`;
+                    try {
+                        const subRes = await fetch(subApiUrl);
+                        if (!subRes.ok)
+                            continue;
+                        const subItems = await subRes.json();
+                        if (!Array.isArray(subItems))
+                            continue;
+                        const skillFile = subItems.find(s => s.name === 'SKILL.md');
+                        if (skillFile) {
+                            return toRawGitHubUrl(owner, repo, branch, skillFile.path);
+                        }
+                    }
+                    catch { /* continue */ }
+                }
+            }
+            catch { /* continue */ }
+        }
+    }
+    throw new Error(`Could not find SKILL.md in ${owner}/${repo}. Searched root, .claude/skills/, and skills/ directories.`);
+}
+async function resolveSource(source, skillName) {
+    // Case 1: GitHub blob URL
+    const blobMatch = source.match(GITHUB_BLOB_RE);
+    if (blobMatch) {
+        const [, owner, repo, branch, path] = blobMatch;
+        const rawUrl = toRawGitHubUrl(owner, repo, branch, path);
+        return { content: await fetchText(rawUrl), sourceName: path };
+    }
+    // Case 2: GitHub repo URL
+    const repoMatch = source.match(GITHUB_REPO_RE);
+    if (repoMatch) {
+        const [, owner, repo] = repoMatch;
+        const rawUrl = skillName
+            ? await discoverNamedSkill(owner, repo, skillName)
+            : await discoverSkillInRepo(owner, repo);
+        return { content: await fetchText(rawUrl), sourceName: skillName ?? `${owner}/${repo}` };
+    }
+    // Case 3: GitHub shorthand (owner/repo)
+    const shorthandMatch = source.match(GITHUB_SHORTHAND_RE);
+    if (shorthandMatch && !source.includes('\\') && !source.includes(':')) {
+        const [, owner, repo] = shorthandMatch;
+        const rawUrl = skillName
+            ? await discoverNamedSkill(owner, repo, skillName)
+            : await discoverSkillInRepo(owner, repo);
+        return { content: await fetchText(rawUrl), sourceName: skillName ?? `${owner}/${repo}` };
+    }
+    // Case 4: URL
+    if (source.startsWith('http://') || source.startsWith('https://')) {
+        return { content: await fetchText(source), sourceName: source };
+    }
+    // Case 5: Local file path
+    const content = await readFile(source, 'utf-8');
+    return { content, sourceName: basename(source, '.md') };
+}
+export async function parseSkill(source, skillName) {
+    const { content: rawContent, sourceName } = await resolveSource(source, skillName);
+    const { data, content: body } = matter(rawContent);
+    if (!body || !body.trim()) {
+        throw new Error('SKILL.md has no content body');
+    }
+    // Resolve name
+    let name = data.name;
+    if (!name) {
+        const headingMatch = body.match(/^#\s+(.+)$/m);
+        name = headingMatch ? headingMatch[1].trim() : sourceName;
+    }
+    // Resolve description
+    let description = data.description;
+    if (!description) {
+        const firstParagraph = body
+            .split('\n\n')
+            .find(p => p.trim() && !p.trim().startsWith('#'));
+        description = firstParagraph
+            ? firstParagraph.trim().slice(0, 200)
+            : 'No description available';
+    }
+    return { name, description, body: body.trim(), rawContent };
+}
+//# sourceMappingURL=parser.js.map

package/dist/providers.d.ts

@@ -0,0 +1,4 @@
+import type { LanguageModel } from 'ai';
+import { type ProviderName } from './config.js';
+export declare function resolveApiKey(provider: ProviderName, cliKey?: string): string;
+export declare function createModel(provider: ProviderName, modelId: string, apiKey: string): LanguageModel;

package/dist/providers.js

@@ -0,0 +1,33 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import { createAnthropic } from '@ai-sdk/anthropic';
+import { createGoogleGenerativeAI } from '@ai-sdk/google';
+import { PROVIDER_ENV_VARS } from './config.js';
+export function resolveApiKey(provider, cliKey) {
+    if (cliKey)
+        return cliKey;
+    const envVar = PROVIDER_ENV_VARS[provider];
+    const envValue = process.env[envVar];
+    if (envValue)
+        return envValue;
+    throw new Error(`No API key found for ${provider}. Provide --key or set ${envVar} environment variable.`);
+}
+export function createModel(provider, modelId, apiKey) {
+    switch (provider) {
+        case 'openrouter':
+            return createOpenAI({
+                baseURL: 'https://openrouter.ai/api/v1',
+                apiKey,
+                headers: {
+                    'HTTP-Referer': 'https://github.com/zlatkov/skilleval',
+                    'X-Title': 'skilleval',
+                },
+            })(modelId);
+        case 'openai':
+            return createOpenAI({ apiKey })(modelId);
+        case 'anthropic':
+            return createAnthropic({ apiKey })(modelId);
+        case 'google':
+            return createGoogleGenerativeAI({ apiKey })(modelId);
+    }
+}
+//# sourceMappingURL=providers.js.map

package/dist/reporter.d.ts

@@ -0,0 +1,5 @@
+import type { EvalReport, EvalResult } from './config.js';
+export declare function printReport(reports: EvalReport[], evalResults: EvalResult[], options: {
+    json: boolean;
+    verbose: boolean;
+}): void;

package/dist/reporter.js

@@ -0,0 +1,80 @@
+import chalk from 'chalk';
+function padRight(str, len) {
+    return str.length >= len ? str.slice(0, len) : str + ' '.repeat(len - str.length);
+}
+function scoreColor(score) {
+    if (score >= 70)
+        return chalk.green;
+    if (score >= 40)
+        return chalk.yellow;
+    return chalk.red;
+}
+function printTable(reports) {
+    const modelWidth = Math.max(20, ...reports.map(r => r.modelId.length)) + 2;
+    const triggerWidth = 14;
+    const complianceWidth = 16;
+    const overallWidth = 9;
+    const line = (left, mid, right, fill) => left + fill.repeat(modelWidth) + mid + fill.repeat(triggerWidth) + mid + fill.repeat(complianceWidth) + mid + fill.repeat(overallWidth) + right;
+    console.log(line('┌', '┬', '┐', '─'));
+    console.log('│' + padRight(' Model', modelWidth) +
+        '│' + padRight(' Trigger', triggerWidth) +
+        '│' + padRight(' Compliance', complianceWidth) +
+        '│' + padRight(' Overall', overallWidth) + '│');
+    console.log(line('├', '┼', '┤', '─'));
+    for (const report of reports) {
+        const triggerStr = `${report.triggerScore.correct}/${report.triggerScore.total}`;
+        const complianceStr = report.complianceScore.total > 0
+            ? `${report.complianceScore.correct}/${report.complianceScore.total} (${report.complianceScore.avgScore})`
+            : 'N/A';
+        const overallStr = `${report.overall}%`;
+        const color = scoreColor(report.overall);
+        console.log('│' + padRight(` ${report.modelId}`, modelWidth) +
+            '│' + padRight(` ${triggerStr}`, triggerWidth) +
+            '│' + padRight(` ${complianceStr}`, complianceWidth) +
+            '│' + color(padRight(` ${overallStr}`, overallWidth)) + '│');
+    }
+    console.log(line('└', '┴', '┘', '─'));
+    if (reports.length > 0) {
+        const best = reports[0];
+        const worst = reports[reports.length - 1];
+        console.log(`\n${chalk.green('Best model:')} ${best.modelId} (${best.overall}%)`);
+        if (reports.length > 1) {
+            console.log(`${chalk.red('Worst model:')} ${worst.modelId} (${worst.overall}%)`);
+        }
+    }
+}
+function printVerbose(evalResults) {
+    const byModel = new Map();
+    for (const result of evalResults) {
+        const arr = byModel.get(result.modelId) ?? [];
+        arr.push(result);
+        byModel.set(result.modelId, arr);
+    }
+    for (const [modelId, results] of byModel) {
+        console.log(`\n${chalk.bold(`--- ${modelId} ---`)}`);
+        for (const result of results) {
+            const status = result.trigger.correct ? chalk.green('PASS') : chalk.red('FAIL');
+            console.log(`  [${status}] ${result.prompt.type}: "${result.prompt.text.slice(0, 60)}"`);
+            console.log(`         ${result.trigger.reason}`);
+            if (result.compliance) {
+                const compStatus = result.compliance.compliant ? chalk.green('PASS') : chalk.red('FAIL');
+                console.log(`    Compliance: [${compStatus}] ${result.compliance.score}/100 — ${result.compliance.reason}`);
+            }
+        }
+    }
+}
+export function printReport(reports, evalResults, options) {
+    if (options.json) {
+        console.log(JSON.stringify({ reports, evalResults }, null, 2));
+        return;
+    }
+    printTable(reports);
+    if (options.verbose) {
+        printVerbose(evalResults);
+    }
+    else {
+        console.log(`\nRun with ${chalk.cyan('--verbose')} to see individual test results.`);
+        console.log(`Run with ${chalk.cyan('--json')} to get machine-readable output.`);
+    }
+}
+//# sourceMappingURL=reporter.js.map

package/dist/runner.d.ts

@@ -0,0 +1,2 @@
+import type { ModelWithId, SkillDefinition, TestPrompt, TestResult } from './config.js';
+export declare function runTests(skill: SkillDefinition, prompts: TestPrompt[], models: ModelWithId[], verbose: boolean): Promise<TestResult[]>;