@fede0089/skill-eval 3.1.0 → 3.3.0

package/dist/reporters/index.js

@@ -1,2 +1 @@
 export { HtmlReporter } from './html-reporter.js';
-export { JsonReporter } from './json-reporter.js';

package/dist/utils/eval-loader.js

@@ -1,20 +1,36 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { ConfigError } from '../core/errors.js';
+/**
+ * Normalizes an --eval-file value to the bare file name used inside the evals directory.
+ * Accepts 'edge-cases', 'edge-cases.json' and 'path/to/evals/edge-cases.json' alike.
+ */
+function normalizeEvalFileName(evalFile) {
+    const base = path.basename(evalFile.trim());
+    return base.endsWith('.json') ? base : `${base}.json`;
+}
 /**
  * Loads and merges all JSON evaluation files from a skill's evals directory.
  * Aligned with Anthropic's recommendation to split evals by capability/regression.
  * Supports legacy 'tasks' and 'assertions' internally while maintaining 'evals' and 'expectations' in files.
+ * When 'evalFile' is given, only that file is loaded instead of the whole suite.
  */
-export function loadEvalSuite(skillPath) {
+export function loadEvalSuite(skillPath, evalFile) {
     const evalsDir = path.resolve(skillPath, 'evals');
     if (!fs.existsSync(evalsDir)) {
         throw new ConfigError(`Could not find evals directory at ${evalsDir}`);
     }
-    const files = fs.readdirSync(evalsDir).filter(file => file.endsWith('.json'));
+    let files = fs.readdirSync(evalsDir).filter(file => file.endsWith('.json'));
     if (files.length === 0) {
         throw new ConfigError(`No JSON evaluation files found in ${evalsDir}`);
     }
+    if (evalFile !== undefined) {
+        const wanted = normalizeEvalFileName(evalFile);
+        if (!files.includes(wanted)) {
+            throw new ConfigError(`Eval file '${wanted}' not found in ${evalsDir}. Available: ${[...files].sort().join(', ')}.`);
+        }
+        files = [wanted];
+    }
     let mergedSkillName = '';
     const mergedTasks = [];
     for (const file of files) {
@@ -59,7 +75,8 @@ export function loadEvalSuite(skillPath) {
         mergedTasks.push(...mappedTasks);
     }
     if (mergedTasks.length === 0) {
-        throw new ConfigError(`No evaluations found in any of the JSON files in ${evalsDir}`);
+        const scope = evalFile !== undefined ? `${files[0]} in ${evalsDir}` : `any of the JSON files in ${evalsDir}`;
+        throw new ConfigError(`No evaluations found in ${scope}`);
     }
     return {
         skill_name: mergedSkillName,

package/dist/utils/ndjson.js

@@ -62,6 +62,25 @@ export function parseStreamResult(output) {
         (typeof resultEvent.response === 'string' ? resultEvent.response : '');
     return { response: text };
 }
+/**
+ * Counts tool invocations and reads the terminal status from a stream-json blob.
+ * Both are needed to tell a legitimate short answer apart from an agent that
+ * stopped early or degenerated: a trial with tool calls and a 'success' status
+ * that still produced two characters of text is a very different failure from
+ * one that never started.
+ * Returns toolCalls: 0 and an undefined status when the blob carries no events.
+ */
+export function parseStreamStats(output) {
+    let toolCalls = 0;
+    let status;
+    for (const event of parseNdjsonEvents(output)) {
+        if (event.type === 'tool_use')
+            toolCalls += 1;
+        else if (event.type === 'result')
+            status = event.status;
+    }
+    return { toolCalls, status };
+}
 /**
  * Extracts token consumption stats from a Gemini CLI stream-json stdout blob.
  * Looks for a result event with a stats.total_tokens field.

package/dist/utils/table-renderer.js

@@ -98,7 +98,7 @@ function boxLabel(key, value) {
  * Always shown regardless of debug mode.
  */
 export function renderRunHeader(config) {
-    const { command, skillName, agent, workspace, tasks, trials, maxAgents, timeoutMs, runDir, evalId } = config;
+    const { command, skillName, agent, workspace, tasks, trials, maxAgents, timeoutMs, runDir, evalId, evalFile } = config;
     let timeoutStr = 'None';
     if (timeoutMs && timeoutMs > 0) {
         const timeoutSec = timeoutMs / 1000;
@@ -111,7 +111,12 @@ export function renderRunHeader(config) {
     const dashes = '─'.repeat(BOX_INNER - titleLabel.length);
     const top = chalk.gray('┌─ ') + chalk.bold(titleLabel) + ' ' + chalk.gray(dashes + '┐');
     const bottom = chalk.gray('└' + '─'.repeat(BOX_INNER + 2) + '┘');
-    const commandPart = evalId !== undefined ? `${command}  ·  eval #${evalId}` : command;
+    const filterParts = [];
+    if (evalFile !== undefined)
+        filterParts.push(evalFile);
+    if (evalId !== undefined)
+        filterParts.push(`eval #${evalId}`);
+    const commandPart = [command, ...filterParts].join('  ·  ');
     const title = chalk.bold.cyan(skillName) + chalk.gray(`  ·  ${commandPart}`);
     const runLine = `${tasks} task${tasks !== 1 ? 's' : ''}  ·  ${trials} trial${trials !== 1 ? 's' : ''}  ·  agents ${maxAgents}`;
     process.stdout.write('\n');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fede0089/skill-eval",
-  "version": "3.1.0",
+  "version": "3.3.0",
   "description": "CLI to evaluate agent skills triggering and functionality",
   "main": "dist/index.js",
   "bin": {

package/dist/reporters/json-reporter.js

@@ -1,10 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { Logger } from '../utils/logger.js';
-export class JsonReporter {
-    generate(report, runDir) {
-        const jsonPath = path.join(runDir, 'summary.json');
-        fs.writeFileSync(jsonPath, JSON.stringify(report, null, 2), 'utf-8');
-        Logger.write(`\n   Report: file://${jsonPath}\n`);
-    }
-}