@fede0089/skill-eval 3.0.1 → 3.1.0

package/README.md

@@ -6,7 +6,7 @@ A CLI tool for evaluating Agent Skills locally. Tests whether your skill trigger
 
 skill-eval ships two commands, each targeting a different failure mode:
 
-- **Triggering** (`skill-eval trigger`) — checks whether the agent actually decides to invoke the skill in the right context. A skill that never gets triggered cannot help, no matter how good its instructions are.
+- **Triggering** (`skill-eval trigger`) — checks whether the agent actually decides to invoke the skill in the right context, and leaves it alone in the wrong one. A skill that never gets triggered cannot help, no matter how good its instructions are; one that triggers everywhere gets in the way.
 - **Functional correctness** (`skill-eval functional`) — checks whether the actions the agent takes while the skill is active match your expectations. An LLM judge grades each transcript against the expectation list you provide.
 
 ## Why run skill evals
@@ -38,7 +38,7 @@ For each eval prompt, skill-eval spins up parallel agent processes with the curr
                      pass@k
 ```
 
-> The `trigger` command only runs with-skill trials and checks whether the skill dispatch tool was actually invoked — no judge or baseline needed.
+> The `trigger` command only runs with-skill trials and checks whether the skill dispatch tool was actually invoked — no judge or baseline needed. Evals marked `should_trigger: false` assert the opposite: that it was *not* invoked.
 >
 > The baseline branch is opt-in: enable it with `--compare-baseline` (no-skill control) or `--compare-ref <ref>` (historical skill versions).
 
@@ -128,6 +128,22 @@ All `.json` files in `evals/` are loaded and merged into a single suite — you
 }
 ```
 
+**Negative trigger eval** — add `should_trigger: false` to assert the skill must *not* fire:
+```json
+{
+  "skill_name": "my-skill",
+  "evals": [
+    { "id": 2, "prompt": "Something my skill has no business handling", "should_trigger": false }
+  ]
+}
+```
+
+Positive evals measure **under-triggering** (the skill never fires when it should). Negative evals measure the opposite failure, **over-triggering** — the one you introduce when you widen a skill's `description` to catch more cases and silently lose precision. Both kinds live in the same suite and feed a single success rate, so a change that improves one at the expense of the other shows up immediately.
+
+A negative eval fails if the agent *attempts* to activate the skill at all, even if the activation itself errors out. These evals are trigger-only: `skill-eval functional` skips them, since that command instructs the agent to use the skill.
+
+> Trigger detection for Codex is heuristic (it infers activations from the event stream), so negative evals are most reliable on `claude-code` and `gemini-cli`.
+
 **Functional eval** — add `expectations` for the LLM judge to evaluate:
 ```json
 {
@@ -205,7 +221,7 @@ Refer to your runner's documentation for the full set of settings and policy key
 
 Each run writes to `.project-skill-evals/runs/<timestamp>/` and includes per-trial logs, the raw eval JSON, and a self-contained HTML report you can open in any browser. The report shows pass@k aggregates per eval, lets you expand each trial, and color-codes triggering vs. functional outcomes.
 
-A published sample report is available at [fede0089.github.io/skill-eval/sample-report.html](https://fede0089.github.io/skill-eval/sample-report.html),generated from this project root with:
+A published sample report is available at [fede0089.github.io/skill-eval/sample-report.html](https://fede0089.github.io/skill-eval/sample-report.html), generated from this project root with:
 
 ```sh
 skill-eval functional --workspace . --skill mock-skill --trials 2 --compare-baseline --debug claude-code
@@ -224,7 +240,7 @@ Without `--debug` these files are not written, so reach for the flag when you ne
 
 ## Try it out
 
-This repo includes a `mock-skill/` directory — a complete, working example of a license-generator skill with trigger and functional evals. Run it directly with:
+This repo includes a `mock-skill/` directory — a complete, working example of a license-generator skill with positive trigger, negative trigger, and functional evals. Run it directly with:
 
 ```sh
 npm run test:unit        # run the unit test suite

package/dist/commands/functional.js

@@ -8,6 +8,7 @@ import { EvalRunner } from '../core/eval-runner.js';
 import { AgentPool } from '../core/agent-pool.js';
 import { aggregatePassAtK, aggregateAssertionPassRate, aggregateTokenStats, aggregateDurationStats } from '../core/statistics.js';
 import { preflight } from '../core/preflight.js';
+import { ConfigError } from '../core/errors.js';
 import { withRetry } from '../core/trial-utils.js';
 import { renderFunctionalTable, renderRunHeader } from '../utils/table-renderer.js';
 import { JsonReporter } from '../reporters/index.js';
@@ -17,6 +18,16 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
     if (!injectedSuite)
         preflight(agent, workspace, skillPath);
     const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath);
+    // Negative evals (should_trigger: false) only make sense for the trigger command:
+    // this pass instructs the agent that it MUST use the skill, which contradicts them.
+    const triggerOnly = suite.tasks.filter(t => t.should_trigger === false);
+    if (triggerOnly.length > 0) {
+        if (evalId !== undefined && triggerOnly.some(t => t.id === evalId)) {
+            throw new ConfigError(`Eval #${evalId} is trigger-only (should_trigger: false) and cannot run under 'functional'.`);
+        }
+        suite.tasks = suite.tasks.filter(t => t.should_trigger !== false);
+        Logger.write(chalk.dim(`   Skipping ${triggerOnly.length} trigger-only eval(s)\n`));
+    }
     if (evalId !== undefined) {
         suite.tasks = suite.tasks.filter(t => t.id === evalId);
         if (suite.tasks.length === 0) {

package/dist/commands/trigger.js

@@ -149,7 +149,8 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
                         taskId: task.id,
                         prompt: task.prompt,
                         baselineTrials: [],
-                        skillTrials: taskSkillTrials
+                        skillTrials: taskSkillTrials,
+                        shouldTrigger: task.should_trigger !== false
                     };
                     taskResults.push(taskResult);
                     if (!localAllPassed && !multi) {

package/dist/core/eval-runner.js

@@ -65,22 +65,59 @@ export class EvalRunner {
         const tokenStats = transcript
             ? parseTokenStats(transcript.response || '') ?? undefined
             : undefined;
-        let triggered = false;
+        // Negative evals (should_trigger: false) assert the opposite: the skill must NOT activate.
+        const shouldTrigger = task.should_trigger !== false;
+        const assertionLabel = shouldTrigger ? 'Skill was triggered' : 'Skill was not triggered';
+        let trialPassed = false;
         const assertionResults = [];
         if (transcript && !transcript.error) {
             uiCtx.updateLog('Grading…');
-            triggered = this.triggerGrader.gradeTrigger(transcript);
-            assertionResults.push({
-                assertion: 'Skill was triggered',
-                passed: triggered,
-                reason: triggered ? 'Detected skill activation in transcript' : 'No skill activation detected in transcript',
-                graderType: 'programmatic'
-            });
+            if (shouldTrigger) {
+                const triggered = this.triggerGrader.gradeTrigger(transcript);
+                trialPassed = triggered;
+                assertionResults.push({
+                    assertion: assertionLabel,
+                    passed: triggered,
+                    reason: triggered ? 'Detected skill activation in transcript' : 'No skill activation detected in transcript',
+                    graderType: 'programmatic'
+                });
+            }
+            else if (!this.triggerGrader.hasParsableEvents(transcript)) {
+                // Without any parsable events the absence of an activation proves nothing —
+                // report it as an infrastructure error instead of a vacuous pass.
+                const reason = 'No parsable agent events — cannot verify that the skill did not trigger';
+                assertionResults.push({
+                    assertion: assertionLabel,
+                    passed: false,
+                    reason,
+                    graderType: 'programmatic'
+                });
+                return {
+                    id: trialId,
+                    transcript,
+                    assertionResults,
+                    trialPassed: false,
+                    isError: true,
+                    tokenStats,
+                    durationMs
+                };
+            }
+            else {
+                // Any activation attempt counts, even one that failed: the skill still fired.
+                const triggered = this.triggerGrader.detectSkillAttempt(transcript);
+                trialPassed = !triggered;
+                assertionResults.push({
+                    assertion: assertionLabel,
+                    passed: !triggered,
+                    reason: triggered ? 'Skill activation detected in transcript' : 'No skill activation detected in transcript',
+                    graderType: 'programmatic'
+                });
+            }
         }
         else {
             const errorMsg = transcript?.error || 'Error: No transcript was produced';
             assertionResults.push({
-                assertion: 'Skill was triggered',
+                assertion: assertionLabel,
                 passed: false,
                 reason: errorMsg,
                 graderType: 'programmatic'
@@ -99,7 +136,7 @@ export class EvalRunner {
             id: trialId,
             transcript: transcript || { error: 'No transcript produced' },
             assertionResults: assertionResults,
-            trialPassed: triggered,
+            trialPassed,
             tokenStats,
             durationMs
         };

package/dist/core/evaluator.js

@@ -89,10 +89,19 @@ export class TriggerGrader {
         }
         return false;
     }
+    /**
+     * Whether the transcript produced any parsable NDJSON events.
+     * Negative trigger evals rely on the absence of an activation event, so a transcript
+     * with no events at all cannot prove anything and must not be graded as a pass.
+     */
+    hasParsableEvents(transcript) {
+        return parseNdjsonEvents(transcript.raw_output || '').length > 0;
+    }
     /**
      * Checks whether the skill was attempted (any tool_use event for activate_skill matching the skill name).
      * Unlike gradeTrigger, does NOT require a successful tool_result — any attempt counts.
-     * Used to detect invalid baseline runs where the agent tried to invoke the restricted skill.
+     * Used to detect invalid baseline runs where the agent tried to invoke the restricted skill,
+     * and to grade negative trigger evals (an attempted activation still counts as a trigger).
      */
     detectSkillAttempt(transcript) {
         const rawOutput = transcript.raw_output || '';

package/dist/reporters/html-reporter.js

@@ -215,9 +215,11 @@ function renderTaskTable(report) {
     const headerRow = `<tr><th>#</th><th>Prompt</th><th>Details</th></tr>`;
     const rows = results.map(result => {
         const prompt = escapeHtml(result.prompt);
+        // Negative trigger evals assert the opposite outcome — flag them so the row is not misread.
+        const badge = result.shouldTrigger === false ? '<span class="badge-neg">no-trigger</span>' : '';
         const detailsBtn = `<button class="details-btn" data-target="details-${result.taskId}">▶</button>`;
         const detailsRow = `<tr class="details-row"><td colspan="3">${renderTaskDetails(result, functional)}</td></tr>`;
-        return `<tr><td>${result.taskId}</td><td class="prompt-cell">${prompt}</td><td>${detailsBtn}</td></tr>${detailsRow}`;
+        return `<tr><td>${result.taskId}</td><td class="prompt-cell">${badge}${prompt}</td><td>${detailsBtn}</td></tr>${detailsRow}`;
     }).join('');
     return `<div class="table-wrap"><table><thead>${headerRow}</thead><tbody>${rows}</tbody></table></div>`;
 }
@@ -292,6 +294,7 @@ th { background: #f8fafc; font-size: 11px; font-weight: 600; text-transform: upp
 td { padding: 10px 12px; border-bottom: 1px solid #f1f5f9; vertical-align: top; }
 tr:last-child td { border-bottom: none; }
 .prompt-cell { max-width: 480px; word-break: break-word; color: #334155; }
+.badge-neg { display: inline-block; margin-right: 8px; padding: 1px 7px; border-radius: 10px; background: #e0f2fe; color: #0369a1; font-size: 11px; font-weight: 600; vertical-align: 1px; }
 
 /* Details */
 .details-btn { background: none; border: 1px solid #e2e8f0; border-radius: 4px; cursor: pointer; padding: 2px 8px; font-size: 11px; color: #64748b; transition: background 0.15s; }

package/dist/runners/codex/runner.js

@@ -57,6 +57,10 @@ function eventSignalsSkill(event, skillName) {
     const eventType = String(event.type ?? '').toLowerCase();
     const item = getEventItem(event);
     const itemType = String(item?.type ?? '').toLowerCase();
+    // Assistant prose is not a tool call: an agent merely naming the skill (often to say it is
+    // not needed) must not be reported as an activation. Real activations arrive as skill items.
+    if (['agent_message', 'assistant_message', 'message'].includes(itemType))
+        return false;
     return (serialized.includes(normalizedSkill) &&
         (serialized.includes('skill') ||
             eventType.includes('skill') ||

package/dist/utils/eval-loader.js

@@ -44,12 +44,16 @@ export function loadEvalSuite(skillPath) {
             if (e.id === undefined || typeof e.id !== 'number') {
                 throw new ConfigError(`Invalid task ID in ${file}. ID must be a number.`);
             }
+            if (e.should_trigger !== undefined && typeof e.should_trigger !== 'boolean') {
+                throw new ConfigError(`Invalid 'should_trigger' for eval ${e.id} in ${file}. Must be a boolean.`);
+            }
             return {
                 id: e.id,
                 prompt: e.prompt,
                 expected_output: e.expected_output,
                 assertions: e.expectations || e.assertions,
-                files: e.files
+                files: e.files,
+                should_trigger: e.should_trigger
             };
         });
         mergedTasks.push(...mappedTasks);

package/dist/utils/table-renderer.js

@@ -134,7 +134,11 @@ export function renderTriggerTable(report) {
     const numTrials = metrics.numTrials || 1;
     // Identify all skill versions present in the results
     const skillVersions = results.length > 0 ? Object.keys(results[0].skillTrials) : ['local'];
+    // Only surface the polarity column when the suite actually contains negative evals.
+    const hasNegativeEvals = results.some(r => r.shouldTrigger === false);
     const header = ['ID', 'Prompt'];
+    if (hasNegativeEvals)
+        header.push('Expect');
     if (numTrials > 1) {
         for (const version of skillVersions) {
             header.push(`${version} Trials`, `${version} Rate`);
@@ -150,6 +154,9 @@ export function renderTriggerTable(report) {
     for (const result of results) {
         const promptSnippet = result.prompt.substring(0, 40) + (result.prompt.length > 40 ? '...' : '');
         const row = [result.taskId.toString(), promptSnippet];
+        if (hasNegativeEvals) {
+            row.push(result.shouldTrigger === false ? chalk.cyan('no-trigger') : chalk.dim('trigger'));
+        }
         for (const version of skillVersions) {
             const trials = result.skillTrials[version] || [];
             const p1Cell = formatPassAt1(trials);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fede0089/skill-eval",
-  "version": "3.0.1",
+  "version": "3.1.0",
   "description": "CLI to evaluate agent skills triggering and functionality",
   "main": "dist/index.js",
   "bin": {