@alexanderzzlatkov/skilleval 0.1.0

package/dist/runner.js

@@ -0,0 +1,53 @@
+import { generateText } from 'ai';
+import { buildTriggerSystemPrompt } from './context-builder.js';
+export async function runTests(skill, prompts, models, verbose) {
+    const systemPrompt = buildTriggerSystemPrompt(skill);
+    const results = [];
+    const totalTests = models.length * prompts.length;
+    let completed = 0;
+    if (verbose) {
+        process.stderr.write(`\n  Trigger system prompt:\n  ---\n${systemPrompt}\n  ---\n\n`);
+    }
+    for (const { model, modelId } of models) {
+        for (const prompt of prompts) {
+            completed++;
+            process.stderr.write(`\r  [${completed}/${totalTests}] Testing ${modelId}...`);
+            const start = performance.now();
+            try {
+                const { text } = await generateText({
+                    model,
+                    messages: [
+                        { role: 'system', content: systemPrompt },
+                        { role: 'user', content: prompt.text },
+                    ],
+                    temperature: 0.3,
+                });
+                results.push({
+                    modelId,
+                    prompt,
+                    response: text,
+                    latencyMs: Math.round(performance.now() - start),
+                });
+                if (verbose) {
+                    process.stderr.write(`\n    ${prompt.type}: "${prompt.text.slice(0, 60)}..." → ${text.slice(0, 80)}...\n`);
+                }
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                results.push({
+                    modelId,
+                    prompt,
+                    response: '',
+                    latencyMs: Math.round(performance.now() - start),
+                    error: message,
+                });
+                if (verbose) {
+                    process.stderr.write(`\n    ERROR: ${message}\n`);
+                }
+            }
+        }
+    }
+    process.stderr.write('\n');
+    return results;
+}
+//# sourceMappingURL=runner.js.map

package/dist/test-generator.d.ts

@@ -0,0 +1,3 @@
+import { type LanguageModel } from 'ai';
+import type { SkillDefinition, TestPrompt } from './config.js';
+export declare function generateTestPrompts(skill: SkillDefinition, generatorModels: LanguageModel[], count: number, customPromptsPath?: string, verbose?: boolean): Promise<TestPrompt[]>;

package/dist/test-generator.js

@@ -0,0 +1,109 @@
+import { readFile } from 'node:fs/promises';
+import { generateText } from 'ai';
+const RETRY_DELAY_MS = 2000;
+function stripCodeFences(text) {
+    return text.replace(/^```(?:json)?\s*\n?/gm, '').replace(/\n?```\s*$/gm, '').trim();
+}
+function validatePrompts(parsed, count) {
+    if (!Array.isArray(parsed))
+        throw new Error('Expected JSON array');
+    const prompts = [];
+    for (const item of parsed) {
+        if (typeof item !== 'object' || item === null ||
+            typeof item.text !== 'string' || !item.text.trim() ||
+            (item.type !== 'positive' && item.type !== 'negative')) {
+            throw new Error('Invalid prompt format');
+        }
+        prompts.push({ text: item.text.trim(), type: item.type });
+    }
+    const positiveCount = prompts.filter(p => p.type === 'positive').length;
+    const negativeCount = prompts.filter(p => p.type === 'negative').length;
+    if (positiveCount !== count || negativeCount !== count) {
+        throw new Error(`Expected ${count} positive and ${count} negative prompts, got ${positiveCount} and ${negativeCount}`);
+    }
+    return prompts;
+}
+const ALL_FALLBACK_POSITIVE = [
+    s => `I need help with ${s.name}`,
+    s => `Can you use the ${s.name} skill for this task?`,
+    s => `Help me with something related to ${s.description.toLowerCase()}`,
+    s => `I'd like to ${s.description.toLowerCase().slice(0, 50)}`,
+    s => `Please assist me with ${s.name} functionality`,
+];
+const ALL_FALLBACK_NEGATIVE = [
+    `What's the weather like today?`,
+    `Tell me a joke about programming`,
+    `How do I make a cup of coffee?`,
+    `What is the capital of France?`,
+    `Help me write a haiku about nature`,
+];
+function fallbackPrompts(skill, count) {
+    const positive = ALL_FALLBACK_POSITIVE.slice(0, count).map(fn => ({ text: fn(skill), type: 'positive' }));
+    const negative = ALL_FALLBACK_NEGATIVE.slice(0, count).map(text => ({ text, type: 'negative' }));
+    return [...positive, ...negative];
+}
+const GENERATION_PROMPT = (skill, count) => `Generate test prompts for this AI skill:
+Name: ${skill.name}
+Description: ${skill.description}
+
+Content summary (first 500 chars): ${skill.body.slice(0, 500)}
+
+Generate exactly ${count * 2} prompts as a JSON array:
+- ${count} "positive" prompts that SHOULD trigger this skill (realistic user requests, varied wording)
+- ${count} "negative" prompts that should NOT trigger this skill (related but out of scope, or different topics)
+
+Format: [{"text": "...", "type": "positive"}, {"text": "...", "type": "negative"}]
+
+Respond ONLY with valid JSON, no other text.`;
+function logPrompts(prompts, source) {
+    process.stderr.write(`\n  Test prompts (${source}):\n`);
+    for (const p of prompts) {
+        const label = p.type === 'positive' ? '+' : '-';
+        process.stderr.write(`    [${label}] ${p.text}\n`);
+    }
+    process.stderr.write('\n');
+}
+export async function generateTestPrompts(skill, generatorModels, count, customPromptsPath, verbose) {
+    // Load custom prompts if provided
+    if (customPromptsPath) {
+        const raw = await readFile(customPromptsPath, 'utf-8');
+        const prompts = validatePrompts(JSON.parse(raw), count);
+        if (verbose)
+            logPrompts(prompts, 'custom');
+        return prompts;
+    }
+    // Try each model with retries
+    for (let modelIdx = 0; modelIdx < generatorModels.length; modelIdx++) {
+        const model = generatorModels[modelIdx];
+        for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+                const { text } = await generateText({
+                    model,
+                    system: 'You generate test prompts for evaluating AI skill detection. Respond ONLY with a JSON array. No markdown, no explanation.',
+                    prompt: attempt === 0
+                        ? GENERATION_PROMPT(skill, count)
+                        : `${GENERATION_PROMPT(skill, count)}\n\nYour previous response was invalid JSON. Return ONLY valid JSON.`,
+                    temperature: 0.8,
+                });
+                const cleaned = stripCodeFences(text);
+                const prompts = validatePrompts(JSON.parse(cleaned), count);
+                if (verbose)
+                    logPrompts(prompts, 'generated');
+                return prompts;
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                process.stderr.write(`Warning: Model ${modelIdx + 1}/${generatorModels.length}, attempt ${attempt + 1}/3 failed: ${message}\n`);
+                if (attempt < 2) {
+                    await new Promise(resolve => setTimeout(resolve, RETRY_DELAY_MS));
+                }
+            }
+        }
+    }
+    process.stderr.write('Warning: All generator models failed, using fallback prompts.\n');
+    const prompts = fallbackPrompts(skill, count);
+    if (verbose)
+        logPrompts(prompts, 'fallback');
+    return prompts;
+}
+//# sourceMappingURL=test-generator.js.map

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@alexanderzzlatkov/skilleval",
+  "version": "0.1.0",
+  "description": "Evaluate how well AI models understand Agent Skills (SKILL.md files)",
+  "keywords": [
+    "ai",
+    "agent",
+    "skills",
+    "eval",
+    "llm",
+    "claude-code",
+    "openrouter",
+    "skilleval"
+  ],
+  "homepage": "https://github.com/zlatkov/skilleval#readme",
+  "bugs": {
+    "url": "https://github.com/zlatkov/skilleval/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zlatkov/skilleval.git"
+  },
+  "license": "MIT",
+  "author": "Alexander Zlatkov (https://github.com/zlatkov)",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "skilleval": "dist/index.js"
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/index.ts",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {
+    "@ai-sdk/anthropic": "^1.0.0",
+    "@ai-sdk/google": "^1.0.0",
+    "@ai-sdk/openai": "^1.0.0",
+    "ai": "^4.0.0",
+    "chalk": "^5.4.0",
+    "commander": "^13.0.0",
+    "dotenv": "^17.3.1",
+    "gray-matter": "^4.0.3"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.7.0"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}