@fede0089/skill-eval 1.4.0 → 1.4.1

package/README.md

@@ -20,7 +20,7 @@ For each eval prompt, skill-eval spins up parallel agent processes with the curr
                └───────┬───────┘
                        │
            ┌───────────┴───────────┐
-      ─ with skill ─          ─ baseline ─
+      ─ with skill ─        ─ baseline (opt) ─
       ┌──────┴──────┐         ┌─────┴──────┐
     agent 1      agent 2   agent 3      agent 4
       │              │         │             │
@@ -32,6 +32,8 @@ For each eval prompt, skill-eval spins up parallel agent processes with the curr
 ```
 
 > The `trigger` command only runs with-skill trials and checks whether the skill dispatch tool was actually invoked — no judge or baseline needed.
+>
+> The baseline branch is opt-in: enable it with `--compare-baseline` (no-skill control) or `--compare-ref <ref>` (historical skill versions).
 
 ## Installation
 
@@ -157,6 +159,7 @@ Refer to your runner's documentation for the full list of available settings and
 This repo includes a `mock-skill/` directory — a complete, working example of a license-generator skill with trigger and functional evals. Run it directly with:
 
 ```sh
+npm run test:unit        # run the unit test suite
 npm run test:trigger     # trigger evaluation against mock-skill
 npm run test:functional  # functional evaluation against mock-skill
 ```
@@ -181,17 +184,4 @@ The factory, preflight check, and CLI all pick it up automatically.
 ### Adding a new report format
 
 1. Create `src/reporters/<format>-reporter.ts` implementing `Reporter`.
-2. Export it and add a case in `createReporter()` in `src/reporters/index.ts`.
-3. Add the format string to `ReportFormat` in `src/types/index.ts`.
-er, binary: '<cli-binary-name>' },
-   ```
-
-The factory, preflight check, and CLI all pick it up automatically.
-
-> Implement `applyRunnerConfig(evalConfigBaseDir, worktreePath)` to copy `evalConfigBaseDir/<your-agent>/` into the appropriate config directory in the worktree (e.g. `.claude/` for a Claude runner). No-op silently if the directory doesn't exist.
-
-### Adding a new report format
-
-1. Create `src/reporters/<format>-reporter.ts` implementing `Reporter`.
-2. Export it and add a case in `createReporter()` in `src/reporters/index.ts`.
-3. Add the format string to `ReportFormat` in `src/types/index.ts`.
+2. Add a case for it in `createReporter()` in `src/reporters/index.ts`.

package/dist/index.js

@@ -8,7 +8,9 @@ import { HtmlReporter } from './reporters/index.js';
 import { DEFAULT_AGENT } from './runners/registry.js';
 import * as path from 'path';
 import * as fs from 'fs';
+import { createRequire } from 'module';
 import { fileURLToPath } from 'url';
+const pkg = createRequire(import.meta.url)('../package.json');
 export const program = new Command();
 const errorHandler = (err) => {
     if (err instanceof AppError) {
@@ -26,7 +28,7 @@ const errorHandler = (err) => {
 program
     .name('skill-eval')
     .description('CLI to evaluate agent skills triggering and functionality')
-    .version('1.0.0')
+    .version(pkg.version)
     .option('-v, --debug', 'Enable debug logging', false);
 program.on('option:debug', () => {
     process.env.DEBUG = 'true';
@@ -36,16 +38,16 @@ program
     .description('Evaluate triggering of an agent skill')
     .requiredOption('--workspace <path>', 'Path to the workspace/repo to evaluate against')
     .requiredOption('--skill <path>', 'Path to the skill directory')
-    .option('--agents <number>', 'Number of parallel agents')
-    .option('--trials <number>', 'Number of trials per task for pass@k calculation')
+    .option('--agents <number>', 'Number of parallel agents', '4')
+    .option('--trials <number>', 'Number of trials per task for pass@k calculation', '3')
     .option('--timeout <seconds>', 'Agent timeout in seconds')
     .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
     .option('--compare-ref [refs...]', 'Compare against historical git references')
     .action((agent, options) => {
     const workspace = path.resolve(options.workspace);
     const selectedAgent = agent || DEFAULT_AGENT;
-    const maxAgents = parseInt(options.agents, 10) || 4;
-    const numTrials = options.trials !== undefined ? (parseInt(options.trials, 10) || 3) : 3;
+    const maxAgents = parseInt(options.agents, 10);
+    const numTrials = parseInt(options.trials, 10);
     const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
     const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
     const compareRefs = options.compareRef || [];
@@ -53,11 +55,11 @@ program
 });
 program
     .command('functional [agent]')
-    .description('Evaluate functional correctness of an agent skill based on assertions')
+    .description('Evaluate functional correctness of an agent skill against expectations')
     .requiredOption('--workspace <path>', 'Path to the workspace/repo to evaluate against')
     .requiredOption('--skill <path>', 'Path to the skill directory')
-    .option('--agents <number>', 'Number of parallel agents')
-    .option('--trials <number>', 'Number of trials per task for pass@k calculation')
+    .option('--agents <number>', 'Number of parallel agents', '4')
+    .option('--trials <number>', 'Number of trials per task for pass@k calculation', '3')
     .option('--timeout <seconds>', 'Agent timeout in seconds')
     .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
     .option('--compare-ref [refs...]', 'Compare against historical git references')
@@ -65,8 +67,8 @@ program
     .action((agent, options) => {
     const workspace = path.resolve(options.workspace);
     const selectedAgent = agent || DEFAULT_AGENT;
-    const maxAgents = parseInt(options.agents, 10) || 4;
-    const numTrials = options.trials !== undefined ? (parseInt(options.trials, 10) || 3) : 3;
+    const maxAgents = parseInt(options.agents, 10);
+    const numTrials = parseInt(options.trials, 10);
     const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
     const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
     const compareRefs = options.compareRef || [];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fede0089/skill-eval",
-  "version": "1.4.0",
-  "description": "CLI to evaluate agent skills triggering",
+  "version": "1.4.1",
+  "description": "CLI to evaluate agent skills triggering and functionality",
   "main": "dist/index.js",
   "bin": {
     "skill-eval": "dist/index.js"