@fede0089/skill-eval 3.1.0 → 3.2.0

package/README.md

@@ -89,7 +89,8 @@ skill-eval functional --workspace <path> --skill <path> [options] [agent]
 | `--trials <number>` | no | `3` | Trials per task (for pass@k) |
 | `--timeout <seconds>` | no | none | Kill the agent after this many seconds |
 | `--eval-id <id>` | no | all | Run only the eval with this numeric ID |
-| `--compare-ref [refs...]` | no | — | Git references to compare against |
+| `--eval-file <name>` | no | all | Run only the evals from this file in `evals/` (e.g. `edge-cases.json`) |
+| `--compare-ref [refs...]` | no | — | Git references to compare against (variadic — put `[agent]` before it, not after) |
 | `--compare-baseline` | no | `false` | Also run the no-skill baseline alongside the skill |
 | `-v, --debug` | no | `false` | Enable verbose debug logging |
 | `[agent]` | no | `gemini-cli` | Agent backend to use |
@@ -118,6 +119,8 @@ my-skill/
 
 All `.json` files in `evals/` are loaded and merged into a single suite — you can split them by feature or regression category.
 
+Use `--eval-file <name>` to run just one of them while iterating (the `.json` extension is optional), and combine it with `--eval-id <id>` to narrow down to a single eval inside that file.
+
 **Trigger eval** — `id` must be a unique integer across all eval files:
 ```json
 {
@@ -250,6 +253,9 @@ npm run test:trigger -- codex            # run trigger evals with Codex
 npm run test:functional -- codex         # run functional evals with Codex
 npm run test:trigger -- claude-code      # run trigger evals with Claude Code
 npm run test:functional -- claude-code   # run functional evals with Claude Code
+
+npm run test:trigger -- --eval-file negative-triggers.json   # only the negative-trigger evals
+npm run test:trigger -- --eval-file edge-cases --eval-id 3   # a single eval inside one file
 ```
 
 ## Extending

package/dist/commands/functional.js

@@ -14,10 +14,10 @@ import { renderFunctionalTable, renderRunHeader } from '../utils/table-renderer.
 import { JsonReporter } from '../reporters/index.js';
 import chalk from 'chalk';
 import { git } from '../utils/git.js';
-export async function functionalCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = [], compareBaseline = false) {
+export async function functionalCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = [], compareBaseline = false, evalFile) {
     if (!injectedSuite)
         preflight(agent, workspace, skillPath);
-    const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath);
+    const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath, evalFile);
     // Negative evals (should_trigger: false) only make sense for the trigger command:
     // this pass instructs the agent that it MUST use the skill, which contradicts them.
     const triggerOnly = suite.tasks.filter(t => t.should_trigger === false);
@@ -26,6 +26,9 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
             throw new ConfigError(`Eval #${evalId} is trigger-only (should_trigger: false) and cannot run under 'functional'.`);
         }
         suite.tasks = suite.tasks.filter(t => t.should_trigger !== false);
+        if (suite.tasks.length === 0) {
+            throw new ConfigError(`No evals left to run: all ${triggerOnly.length} eval(s) in scope are trigger-only (should_trigger: false).`);
+        }
         Logger.write(chalk.dim(`   Skipping ${triggerOnly.length} trigger-only eval(s)\n`));
     }
     if (evalId !== undefined) {
@@ -94,7 +97,7 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
         ]
         : skillVersions.flatMap(v => Array.from({ length: numTrials }, (_, i) => `${v} ${i + 1}`));
     try {
-        renderRunHeader({ command: 'functional', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId });
+        renderRunHeader({ command: 'functional', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId, evalFile });
         Logger.write(`──────────────────────────────────────────────────\n`);
         for (let i = 0; i < tasks.length; i++) {
             const task = tasks[i];

package/dist/commands/trigger.js

@@ -13,10 +13,10 @@ import { renderTriggerTable, renderRunHeader } from '../utils/table-renderer.js'
 import { JsonReporter } from '../reporters/index.js';
 import chalk from 'chalk';
 import { git } from '../utils/git.js';
-export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = []) {
+export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = [], evalFile) {
     if (!injectedSuite)
         preflight(agent, workspace, skillPath);
-    const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath);
+    const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath, evalFile);
     if (evalId !== undefined) {
         suite.tasks = suite.tasks.filter(t => t.id === evalId);
         if (suite.tasks.length === 0) {
@@ -73,7 +73,7 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
     const skillVersions = Array.from(variantRunners.keys());
     const subtaskLabels = skillVersions.flatMap(v => Array.from({ length: numTrials }, (_, i) => `${v} ${i + 1}`));
     try {
-        renderRunHeader({ command: 'trigger', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId });
+        renderRunHeader({ command: 'trigger', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId, evalFile });
         Logger.write(`--- Trigger Pass ---\n`);
         Logger.write(`──────────────────────────────────────────────────\n`);
         for (let i = 0; i < tasks.length; i++) {

package/dist/index.js

@@ -3,7 +3,7 @@ import { Command } from 'commander';
 import { triggerCommand } from './commands/trigger.js';
 import { functionalCommand } from './commands/functional.js';
 import { Logger } from './utils/logger.js';
-import { AppError } from './core/errors.js';
+import { AppError, ConfigError } from './core/errors.js';
 import { HtmlReporter } from './reporters/index.js';
 import { DEFAULT_AGENT } from './runners/registry.js';
 import * as path from 'path';
@@ -25,6 +25,18 @@ const errorHandler = (err) => {
     }
     process.exit(1);
 };
+/**
+ * A variadic option used without values (`--compare-ref` on its own) is parsed as
+ * `true` by commander; reject it instead of iterating over a boolean.
+ */
+const parseCompareRefs = (value) => {
+    if (value === undefined)
+        return [];
+    if (Array.isArray(value))
+        return value;
+    errorHandler(new ConfigError('--compare-ref requires at least one git reference (e.g. --compare-ref HEAD~1).'));
+    return [];
+};
 program
     .name('skill-eval')
     .description('CLI to evaluate agent skills triggering and functionality')
@@ -42,6 +54,7 @@ program
     .option('--trials <number>', 'Number of trials per task for pass@k calculation', '3')
     .option('--timeout <seconds>', 'Agent timeout in seconds')
     .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
+    .option('--eval-file <name>', 'Run only the evals from this file in evals/ (e.g. edge-cases.json)')
     .option('--compare-ref [refs...]', 'Compare against historical git references')
     .action((agent, options) => {
     const workspace = path.resolve(options.workspace);
@@ -50,8 +63,8 @@ program
     const numTrials = parseInt(options.trials, 10);
     const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
     const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
-    const compareRefs = options.compareRef || [];
-    triggerCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId, compareRefs).catch(errorHandler);
+    const compareRefs = parseCompareRefs(options.compareRef);
+    triggerCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId, compareRefs, options.evalFile).catch(errorHandler);
 });
 program
     .command('functional [agent]')
@@ -62,6 +75,7 @@ program
     .option('--trials <number>', 'Number of trials per task for pass@k calculation', '3')
     .option('--timeout <seconds>', 'Agent timeout in seconds')
     .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
+    .option('--eval-file <name>', 'Run only the evals from this file in evals/ (e.g. edge-cases.json)')
     .option('--compare-ref [refs...]', 'Compare against historical git references')
     .option('--compare-baseline', 'Also run the no-skill baseline alongside the skill')
     .action((agent, options) => {
@@ -71,9 +85,9 @@ program
     const numTrials = parseInt(options.trials, 10);
     const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
     const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
-    const compareRefs = options.compareRef || [];
+    const compareRefs = parseCompareRefs(options.compareRef);
     const compareBaseline = !!options.compareBaseline;
-    functionalCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId, compareRefs, compareBaseline).catch(errorHandler);
+    functionalCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId, compareRefs, compareBaseline, options.evalFile).catch(errorHandler);
 });
 const isMain = process.argv[1] && (() => {
     try {

package/dist/utils/eval-loader.js

@@ -1,20 +1,36 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { ConfigError } from '../core/errors.js';
+/**
+ * Normalizes an --eval-file value to the bare file name used inside the evals directory.
+ * Accepts 'edge-cases', 'edge-cases.json' and 'path/to/evals/edge-cases.json' alike.
+ */
+function normalizeEvalFileName(evalFile) {
+    const base = path.basename(evalFile.trim());
+    return base.endsWith('.json') ? base : `${base}.json`;
+}
 /**
  * Loads and merges all JSON evaluation files from a skill's evals directory.
  * Aligned with Anthropic's recommendation to split evals by capability/regression.
  * Supports legacy 'tasks' and 'assertions' internally while maintaining 'evals' and 'expectations' in files.
+ * When 'evalFile' is given, only that file is loaded instead of the whole suite.
  */
-export function loadEvalSuite(skillPath) {
+export function loadEvalSuite(skillPath, evalFile) {
     const evalsDir = path.resolve(skillPath, 'evals');
     if (!fs.existsSync(evalsDir)) {
         throw new ConfigError(`Could not find evals directory at ${evalsDir}`);
     }
-    const files = fs.readdirSync(evalsDir).filter(file => file.endsWith('.json'));
+    let files = fs.readdirSync(evalsDir).filter(file => file.endsWith('.json'));
     if (files.length === 0) {
         throw new ConfigError(`No JSON evaluation files found in ${evalsDir}`);
     }
+    if (evalFile !== undefined) {
+        const wanted = normalizeEvalFileName(evalFile);
+        if (!files.includes(wanted)) {
+            throw new ConfigError(`Eval file '${wanted}' not found in ${evalsDir}. Available: ${[...files].sort().join(', ')}.`);
+        }
+        files = [wanted];
+    }
     let mergedSkillName = '';
     const mergedTasks = [];
     for (const file of files) {
@@ -59,7 +75,8 @@ export function loadEvalSuite(skillPath) {
         mergedTasks.push(...mappedTasks);
     }
     if (mergedTasks.length === 0) {
-        throw new ConfigError(`No evaluations found in any of the JSON files in ${evalsDir}`);
+        const scope = evalFile !== undefined ? `${files[0]} in ${evalsDir}` : `any of the JSON files in ${evalsDir}`;
+        throw new ConfigError(`No evaluations found in ${scope}`);
     }
     return {
         skill_name: mergedSkillName,

package/dist/utils/table-renderer.js

@@ -98,7 +98,7 @@ function boxLabel(key, value) {
  * Always shown regardless of debug mode.
  */
 export function renderRunHeader(config) {
-    const { command, skillName, agent, workspace, tasks, trials, maxAgents, timeoutMs, runDir, evalId } = config;
+    const { command, skillName, agent, workspace, tasks, trials, maxAgents, timeoutMs, runDir, evalId, evalFile } = config;
     let timeoutStr = 'None';
     if (timeoutMs && timeoutMs > 0) {
         const timeoutSec = timeoutMs / 1000;
@@ -111,7 +111,12 @@ export function renderRunHeader(config) {
     const dashes = '─'.repeat(BOX_INNER - titleLabel.length);
     const top = chalk.gray('┌─ ') + chalk.bold(titleLabel) + ' ' + chalk.gray(dashes + '┐');
     const bottom = chalk.gray('└' + '─'.repeat(BOX_INNER + 2) + '┘');
-    const commandPart = evalId !== undefined ? `${command}  ·  eval #${evalId}` : command;
+    const filterParts = [];
+    if (evalFile !== undefined)
+        filterParts.push(evalFile);
+    if (evalId !== undefined)
+        filterParts.push(`eval #${evalId}`);
+    const commandPart = [command, ...filterParts].join('  ·  ');
     const title = chalk.bold.cyan(skillName) + chalk.gray(`  ·  ${commandPart}`);
     const runLine = `${tasks} task${tasks !== 1 ? 's' : ''}  ·  ${trials} trial${trials !== 1 ? 's' : ''}  ·  agents ${maxAgents}`;
     process.stdout.write('\n');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fede0089/skill-eval",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "CLI to evaluate agent skills triggering and functionality",
   "main": "dist/index.js",
   "bin": {