@fede0089/skill-eval 1.0.0 → 1.0.6

package/README.md

@@ -33,20 +33,33 @@ For each eval prompt, skill-eval spins up parallel agent processes — some with
 
 > The `trigger` command only runs with-skill trials and checks whether the skill dispatch tool was actually invoked — no judge or baseline needed.
 
-## Quick Start
+## Installation
+
+**Requirements:** Node.js, and the agent CLI you want to evaluate (e.g. `gemini`) installed and on `$PATH`.
+
+### Run without installing
+
+```sh
+npx @fede0089/skill-eval --help
+```
+
+### Install globally
 
 ```sh
-git clone <this-repo>
+npm install -g @fede0089/skill-eval
+skill-eval --help
+```
+
+### From source
+
+```sh
+git clone https://github.com/fede0089/skill-eval.git
 cd skill-eval
 npm install
 npm run build
-npm link
+npm link        # makes `skill-eval` available globally
 ```
 
-This makes `skill-eval` available globally in your terminal.
-
-**Requirements:** Node.js, and the agent CLI you want to evaluate (e.g. `gemini`) installed and on `$PATH`.
-
 ## Commands
 
 ```sh

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fede0089/skill-eval",
-  "version": "1.0.0",
+  "version": "1.0.6",
   "description": "CLI to evaluate agent skills triggering",
   "main": "dist/index.js",
   "bin": {
@@ -12,6 +12,9 @@
     "LICENSE"
   ],
   "scripts": {
+    "release:patch": "npm version patch && git push --follow-tags",
+    "release:minor": "npm version minor && git push --follow-tags",
+    "release:major": "npm version major && git push --follow-tags",
     "prepare": "npm run build",
     "build": "tsc",
     "start": "node dist/index.js",

package/dist/commands/rate.js

@@ -1,20 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.rateCommand = rateCommand;
-const logger_1 = require("../utils/logger");
-const errors_1 = require("../core/errors");
-async function rateCommand(score, options) {
-    const rating = parseInt(score, 10);
-    if (isNaN(rating) || rating < 1 || rating > 5) {
-        throw new errors_1.ValidationError('Rating must be a number between 1 and 5.');
-    }
-    logger_1.Logger.info('\n==========================================');
-    logger_1.Logger.info('       EXPERIENCE RATING VIEW           ');
-    logger_1.Logger.info('==========================================');
-    logger_1.Logger.info(`Rating: ${'★'.repeat(rating)}${'☆'.repeat(5 - rating)}`);
-    if (options.comment) {
-        logger_1.Logger.info(`Comment: ${options.comment}`);
-    }
-    logger_1.Logger.info('==========================================');
-    logger_1.Logger.info('Thank you for your feedback!\n');
-}

package/dist/commands/show.js

@@ -1,43 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import chalk from 'chalk';
-import { Logger } from '../utils/logger.js';
-import { AppError } from '../core/errors.js';
-import { renderTriggerTable, renderFunctionalTable } from '../utils/table-renderer.js';
-import { JsonReporter } from '../core/reporters/index.js';
-export async function showCommand(workspace, reporter = new JsonReporter()) {
-    const runsDir = path.resolve(workspace, '.project-skill-evals', 'runs');
-    if (!fs.existsSync(runsDir)) {
-        throw new AppError('No evaluation runs found. Run an evaluation first.');
-    }
-    const runs = fs.readdirSync(runsDir)
-        .filter(dir => fs.statSync(path.join(runsDir, dir)).isDirectory())
-        .sort((a, b) => b.localeCompare(a)); // Sort descending to get latest
-    if (runs.length === 0) {
-        throw new AppError('No evaluation runs found. Run an evaluation first.');
-    }
-    const latestRun = runs[0];
-    const summaryPath = path.join(runsDir, latestRun, 'summary.json');
-    if (!fs.existsSync(summaryPath)) {
-        throw new AppError(`Summary file not found for the latest run: ${latestRun}`);
-    }
-    const report = JSON.parse(fs.readFileSync(summaryPath, 'utf-8'));
-    const { skill_name, agent, metrics, results, timestamp } = report;
-    const numTrials = metrics.numTrials || 1;
-    const isFunctional = metrics.withoutSkillScore !== undefined;
-    Logger.write(`\n${chalk.bold('LATEST EVALUATION RESULTS')}\n`);
-    Logger.write(`Timestamp:  ${new Date(timestamp).toLocaleString()}\n`);
-    Logger.write(`Skill:      ${skill_name}\n`);
-    Logger.write(`Agent:      ${agent}\n`);
-    Logger.write(`Type:       ${isFunctional ? 'Functional' : 'Trigger'}\n`);
-    Logger.write(`──────────────────────────────────────────────────\n`);
-    if (isFunctional) {
-        renderFunctionalTable(report);
-    }
-    else {
-        renderTriggerTable(report);
-    }
-    Logger.write('\n');
-    const runDir = path.join(runsDir, latestRun);
-    reporter.generate(report, runDir);
-}

package/dist/commands/view.js

@@ -1,30 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.viewCommand = viewCommand;
-const logger_1 = require("../utils/logger");
-async function viewCommand(options) {
-    const ease = parseInt(options.ease || '0', 10);
-    const speed = parseInt(options.speed || '0', 10);
-    const accuracy = parseInt(options.accuracy || '0', 10);
-    const drawStars = (score) => {
-        const stars = '★'.repeat(score);
-        const empty = '☆'.repeat(5 - score);
-        return `[ ${stars}${empty} ]`;
-    };
-    logger_1.Logger.info('\n┌──────────────────────────────────────────┐');
-    logger_1.Logger.info('│          NUEVA VISTA DE CALIFICACIÓN     │');
-    logger_1.Logger.info('├──────────────────────────────────────────┤');
-    logger_1.Logger.info(`│ Facilidad de uso:   ${drawStars(ease)}     │`);
-    logger_1.Logger.info(`│ Velocidad:          ${drawStars(speed)}     │`);
-    logger_1.Logger.info(`│ Precisión:          ${drawStars(accuracy)}     │`);
-    logger_1.Logger.info('├──────────────────────────────────────────┤');
-    if (options.comment) {
-        logger_1.Logger.info(`│ Comentario:                              │`);
-        // Basic wrapping for the comment
-        const comment = options.comment.substring(0, 38);
-        logger_1.Logger.info(`│ ${comment.padEnd(40)} │`);
-    }
-    logger_1.Logger.info('├──────────────────────────────────────────┤');
-    logger_1.Logger.info('│ ¡Gracias por calificar tu experiencia!   │');
-    logger_1.Logger.info('└──────────────────────────────────────────┘\n');
-}

package/dist/core/config.js

@@ -1,58 +0,0 @@
-import * as fs from 'fs';
-import * as path from 'path';
-import { ConfigError } from './errors.js';
-const CONFIG_FILE = '.skill-eval.json';
-/**
- * Loads configuration from a `.skill-eval.json` file in the given directory.
- * Returns an empty object if the file does not exist (config is optional).
- *
- * CLI flags always take precedence over config file values — this function
- * only provides defaults for flags not explicitly passed on the command line.
- *
- * @throws ConfigError on malformed JSON or type mismatches.
- */
-export function loadConfig(cwd) {
-    const configPath = path.join(cwd, CONFIG_FILE);
-    if (!fs.existsSync(configPath)) {
-        return {};
-    }
-    let raw;
-    try {
-        raw = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
-    }
-    catch (err) {
-        throw new ConfigError(`Failed to parse ${CONFIG_FILE}: ${err instanceof Error ? err.message : String(err)}`);
-    }
-    if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
-        throw new ConfigError(`${CONFIG_FILE} must be a JSON object.`);
-    }
-    const config = raw;
-    const result = {};
-    if ('agent' in config) {
-        if (typeof config.agent !== 'string')
-            throw new ConfigError(`${CONFIG_FILE}: 'agent' must be a string.`);
-        result.agent = config.agent;
-    }
-    if ('concurrency' in config) {
-        if (typeof config.concurrency !== 'number')
-            throw new ConfigError(`${CONFIG_FILE}: 'concurrency' must be a number.`);
-        result.concurrency = config.concurrency;
-    }
-    if ('trials' in config) {
-        if (typeof config.trials !== 'number')
-            throw new ConfigError(`${CONFIG_FILE}: 'trials' must be a number.`);
-        result.trials = config.trials;
-    }
-    if ('report' in config) {
-        if (config.report !== 'html' && config.report !== 'json') {
-            throw new ConfigError(`${CONFIG_FILE}: 'report' must be 'html' or 'json'.`);
-        }
-        result.report = config.report;
-    }
-    if ('skill' in config) {
-        if (typeof config.skill !== 'string')
-            throw new ConfigError(`${CONFIG_FILE}: 'skill' must be a string.`);
-        result.skill = config.skill;
-    }
-    return result;
-}

package/dist/core/reporters/html-reporter.js

@@ -1,354 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { Logger } from '../../utils/logger.js';
-export class HtmlReporter {
-    generate(report, runDir) {
-        const html = generateHtml(report);
-        const htmlPath = path.join(runDir, 'report.html');
-        fs.writeFileSync(htmlPath, html, 'utf-8');
-        Logger.write(`\n   Report: file://${htmlPath}\n`);
-    }
-}
-// ---------------------------------------------------------------------------
-// HTML generation (module-private)
-// ---------------------------------------------------------------------------
-function escapeHtml(s) {
-    return s
-        .replace(/&/g, '&')
-        .replace(/</g, '&lt;')
-        .replace(/>/g, '&gt;')
-        .replace(/"/g, '&quot;')
-        .replace(/'/g, '&#39;');
-}
-function formatPercent(val) {
-    return `${Math.round(val * 100)}%`;
-}
-function passColorClass(val) {
-    if (val >= 0.8)
-        return 'green';
-    if (val >= 0.5)
-        return 'amber';
-    return 'red';
-}
-function isFunctional(report) {
-    return report.metrics.withoutSkillScore !== undefined;
-}
-// ---------------------------------------------------------------------------
-// Cards
-// ---------------------------------------------------------------------------
-function renderCard(label, value, colorClass) {
-    return `<div class="card"><div class="card-value ${colorClass}">${escapeHtml(value)}</div><div class="card-label">${escapeHtml(label)}</div></div>`;
-}
-function renderMetricsCards(report) {
-    const { metrics } = report;
-    const numTrials = metrics.numTrials ?? 1;
-    const cards = [];
-    if (isFunctional(report)) {
-        const bk = metrics.withoutSkillPassAtK ?? 0;
-        const tk = metrics.passAtK ?? 0;
-        const upliftRaw = parseInt(metrics.skillUplift ?? '0', 10);
-        const upliftClass = upliftRaw > 0 ? 'green' : upliftRaw < 0 ? 'red' : 'amber';
-        cards.push(renderCard('Without Skill p@1', formatPercent(bk), passColorClass(bk)));
-        if (numTrials > 1) {
-            const bn = metrics.withoutSkillPassAtN ?? 0;
-            cards.push(renderCard(`Without Skill p@${numTrials}`, formatPercent(bn), passColorClass(bn)));
-        }
-        cards.push(renderCard('With Skill p@1', formatPercent(tk), passColorClass(tk)));
-        if (numTrials > 1) {
-            const tn = metrics.passAtN ?? 0;
-            cards.push(renderCard(`With Skill p@${numTrials}`, formatPercent(tn), passColorClass(tn)));
-        }
-        cards.push(renderCard('Skill Uplift', escapeHtml(metrics.skillUplift ?? '0%'), upliftClass));
-    }
-    else {
-        const k = metrics.passAtK ?? 0;
-        cards.push(renderCard('pass@1', formatPercent(k), passColorClass(k)));
-        if (numTrials > 1) {
-            const n = metrics.passAtN ?? 0;
-            cards.push(renderCard(`pass@${numTrials}`, formatPercent(n), passColorClass(n)));
-        }
-        cards.push(renderCard('Tasks passed', `${metrics.passedCount}/${metrics.totalCount}`, passColorClass(metrics.passedCount / Math.max(metrics.totalCount, 1))));
-    }
-    return `<div class="cards">${cards.join('')}</div>`;
-}
-// ---------------------------------------------------------------------------
-// Chart
-// ---------------------------------------------------------------------------
-function renderChart(report) {
-    const { results, metrics } = report;
-    const numTrials = metrics.numTrials ?? 1;
-    const labels = results.map(r => `Task #${r.taskId}`);
-    let datasets;
-    if (isFunctional(report)) {
-        const withoutSkillData = results.map(r => {
-            const bt = r.withoutSkillTrials ?? [];
-            return bt.length === 0 ? 0 : Math.round((bt.filter(t => t.trialPassed).length / bt.length) * 100);
-        });
-        const withSkillData = results.map(r => Math.round((r.trials.filter(t => t.trialPassed).length / Math.max(r.trials.length, 1)) * 100));
-        datasets = [
-            { label: 'Without Skill p@1', data: withoutSkillData, backgroundColor: '#94a3b8', borderRadius: 4 },
-            { label: 'With Skill p@1', data: withSkillData, backgroundColor: '#3b82f6', borderRadius: 4 },
-        ];
-    }
-    else {
-        const passData = results.map(r => Math.round((r.trials.filter(t => t.trialPassed).length / Math.max(r.trials.length, 1)) * 100));
-        datasets = [{
-                label: numTrials > 1 ? 'pass@1' : 'Score',
-                data: passData,
-                backgroundColor: passData.map(v => v >= 80 ? '#22c55e' : v >= 50 ? '#f59e0b' : '#ef4444'),
-                borderRadius: 4,
-            }];
-    }
-    const chartData = JSON.stringify({ labels, datasets });
-    return `
-<div class="chart-wrap">
-  <script type="application/json" id="chart-data">${chartData}</script>
-  <canvas id="eval-chart"></canvas>
-</div>`;
-}
-// ---------------------------------------------------------------------------
-// Trial details
-// ---------------------------------------------------------------------------
-function renderAssertions(assertions) {
-    if (assertions.length === 0)
-        return '<p class="muted">No assertions recorded.</p>';
-    return assertions.map(a => {
-        const icon = a.passed ? '✓' : '✗';
-        const cls = a.passed ? 'assert-pass' : 'assert-fail';
-        const grader = a.graderType ? `<span class="badge">${escapeHtml(a.graderType)}</span>` : '';
-        return `<div class="assertion ${cls}">
-  <span class="assert-icon">${icon}</span>
-  <div class="assert-body">
-    <div class="assert-text">${escapeHtml(a.assertion)}${grader}</div>
-    ${a.reason ? `<div class="assert-reason">${escapeHtml(a.reason)}</div>` : ''}
-  </div>
-</div>`;
-    }).join('');
-}
-function renderTrial(trial, prefix) {
-    const cls = trial.trialPassed ? 'trial-pass' : 'trial-fail';
-    const badge = trial.trialPassed
-        ? '<span class="pill green">PASS</span>'
-        : '<span class="pill red">FAIL</span>';
-    return `<div class="trial ${cls}">
-  <div class="trial-header">${escapeHtml(prefix)} Trial ${trial.id} ${badge}</div>
-  <div class="trial-assertions">${renderAssertions(trial.assertionResults)}</div>
-</div>`;
-}
-function renderTaskDetails(result, isFunctionalEval) {
-    const sections = [];
-    if (isFunctionalEval && result.withoutSkillTrials && result.withoutSkillTrials.length > 0) {
-        sections.push('<div class="trial-group-label">Without Skill</div>');
-        sections.push(...result.withoutSkillTrials.map(t => renderTrial(t, 'Without Skill')));
-        sections.push('<div class="trial-group-label">With Skill</div>');
-    }
-    sections.push(...result.trials.map(t => renderTrial(t, 'With Skill')));
-    return `<div class="task-details" id="details-${result.taskId}">${sections.join('')}</div>`;
-}
-// ---------------------------------------------------------------------------
-// Task table
-// ---------------------------------------------------------------------------
-function renderTaskTable(report) {
-    const { results, metrics } = report;
-    const numTrials = metrics.numTrials ?? 1;
-    const functional = isFunctional(report);
-    const headerCells = functional
-        ? ['#', 'Prompt', 'W/o p@1', 'W/ p@1', 'Details']
-        : numTrials > 1
-            ? ['#', 'Prompt', 'pass@1', `pass@${numTrials}`, 'Details']
-            : ['#', 'Prompt', 'Status', 'Details'];
-    const headerRow = `<tr>${headerCells.map(h => `<th>${escapeHtml(h)}</th>`).join('')}</tr>`;
-    const rows = results.map(result => {
-        const prompt = escapeHtml(result.prompt);
-        const trials = result.trials;
-        const bt = result.withoutSkillTrials ?? [];
-        let statCells;
-        if (functional) {
-            const bp1 = bt.length ? Math.round((bt.filter(t => t.trialPassed).length / bt.length) * 100) : 0;
-            const tp1 = trials.length ? Math.round((trials.filter(t => t.trialPassed).length / trials.length) * 100) : 0;
-            statCells = `<td class="${passColorClass(bp1 / 100)}">${bp1}%</td><td class="${passColorClass(tp1 / 100)}">${tp1}%</td>`;
-        }
-        else if (numTrials > 1) {
-            const p1 = Math.round((trials.filter(t => t.trialPassed).length / Math.max(trials.length, 1)) * 100);
-            const passed = trials.filter(t => t.trialPassed).length;
-            const pn = trials.length > 0 ? Math.round((passed / trials.length) * 100) : 0;
-            statCells = `<td class="${passColorClass(p1 / 100)}">${p1}%</td><td class="${passColorClass(pn / 100)}">${pn}%</td>`;
-        }
-        else {
-            const passed = trials[0]?.trialPassed ?? false;
-            statCells = `<td class="${passed ? 'green' : 'red'}">${passed ? 'PASS' : 'FAIL'}</td>`;
-        }
-        const detailsBtn = `<button class="details-btn" data-target="details-${result.taskId}">▶</button>`;
-        const detailsRow = `<tr class="details-row"><td colspan="${headerCells.length}">${renderTaskDetails(result, functional)}</td></tr>`;
-        return `<tr><td>${result.taskId}</td><td class="prompt-cell">${prompt}</td>${statCells}<td>${detailsBtn}</td></tr>${detailsRow}`;
-    }).join('');
-    return `<div class="table-wrap"><table><thead>${headerRow}</thead><tbody>${rows}</tbody></table></div>`;
-}
-// ---------------------------------------------------------------------------
-// Full document
-// ---------------------------------------------------------------------------
-export function generateHtml(report) {
-    const { skill_name, agent, timestamp, metrics } = report;
-    const functional = isFunctional(report);
-    const evalType = functional ? 'Functional' : 'Trigger';
-    const overallScore = metrics.passAtK ?? 0;
-    const statusClass = passColorClass(overallScore);
-    const formattedDate = new Date(timestamp).toLocaleString();
-    return `<!DOCTYPE html>
-<html lang="en">
-<head>
-<meta charset="UTF-8">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>Skill Eval — ${escapeHtml(skill_name)}</title>
-<script src="https://cdn.jsdelivr.net/npm/chart.js@4/dist/chart.umd.min.js"></script>
-<style>
-*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
-body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #f8fafc; color: #1e293b; font-size: 14px; }
-a { color: #3b82f6; }
-
-/* Layout */
-.container { max-width: 960px; margin: 0 auto; padding: 24px 16px 48px; }
-
-/* Header */
-.header { background: #1e293b; color: #f1f5f9; padding: 24px 28px; border-radius: 10px; margin-bottom: 24px; }
-.header h1 { font-size: 22px; font-weight: 700; margin-bottom: 8px; }
-.header-meta { display: flex; gap: 24px; flex-wrap: wrap; font-size: 13px; color: #94a3b8; }
-.header-meta span b { color: #e2e8f0; }
-.status-bar { height: 4px; border-radius: 2px; margin-top: 16px; }
-.status-bar.green { background: #22c55e; }
-.status-bar.amber { background: #f59e0b; }
-.status-bar.red   { background: #ef4444; }
-
-/* Cards */
-.cards { display: flex; gap: 12px; flex-wrap: wrap; margin-bottom: 24px; }
-.card { background: #fff; border: 1px solid #e2e8f0; border-radius: 8px; padding: 16px 20px; min-width: 120px; flex: 1; }
-.card-value { font-size: 28px; font-weight: 700; line-height: 1; margin-bottom: 4px; }
-.card-label { font-size: 12px; color: #64748b; text-transform: uppercase; letter-spacing: 0.05em; }
-
-/* Color utilities */
-.green { color: #16a34a; }
-.amber { color: #d97706; }
-.red   { color: #dc2626; }
-
-/* Section */
-.section { background: #fff; border: 1px solid #e2e8f0; border-radius: 8px; margin-bottom: 20px; overflow: hidden; }
-.section-title { font-weight: 600; font-size: 13px; text-transform: uppercase; letter-spacing: 0.05em; color: #64748b; padding: 12px 16px; border-bottom: 1px solid #f1f5f9; }
-
-/* Chart */
-.chart-wrap { padding: 16px; }
-#eval-chart { max-height: 300px; }
-
-/* Table */
-.table-wrap { overflow-x: auto; }
-table { width: 100%; border-collapse: collapse; }
-th { background: #f8fafc; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; color: #64748b; padding: 10px 12px; text-align: left; border-bottom: 1px solid #e2e8f0; }
-td { padding: 10px 12px; border-bottom: 1px solid #f1f5f9; vertical-align: top; }
-tr:last-child td { border-bottom: none; }
-.prompt-cell { max-width: 360px; word-break: break-word; color: #334155; }
-
-/* Details */
-.details-btn { background: none; border: 1px solid #e2e8f0; border-radius: 4px; cursor: pointer; padding: 2px 8px; font-size: 11px; color: #64748b; transition: background 0.15s; }
-.details-btn:hover { background: #f1f5f9; }
-.details-btn.open { color: #3b82f6; border-color: #3b82f6; }
-.details-row > td { padding: 0; background: #f8fafc; }
-.task-details { display: none; padding: 12px 16px; }
-.task-details.visible { display: block; }
-.trial-group-label { font-size: 11px; font-weight: 600; text-transform: uppercase; color: #94a3b8; letter-spacing: 0.05em; margin: 8px 0 4px; }
-
-/* Trials */
-.trial { border: 1px solid #e2e8f0; border-radius: 6px; margin-bottom: 8px; overflow: hidden; }
-.trial-header { display: flex; align-items: center; gap: 8px; padding: 8px 12px; font-weight: 500; font-size: 13px; background: #f8fafc; }
-.trial-pass .trial-header { border-left: 3px solid #22c55e; }
-.trial-fail .trial-header { border-left: 3px solid #ef4444; }
-.trial-assertions { padding: 8px 12px; display: flex; flex-direction: column; gap: 6px; }
-
-/* Pills */
-.pill { display: inline-block; font-size: 10px; font-weight: 700; padding: 2px 7px; border-radius: 99px; letter-spacing: 0.05em; }
-.pill.green { background: #dcfce7; color: #15803d; }
-.pill.red   { background: #fee2e2; color: #b91c1c; }
-
-/* Badge */
-.badge { display: inline-block; font-size: 10px; font-weight: 500; padding: 1px 6px; border-radius: 4px; background: #e2e8f0; color: #475569; margin-left: 6px; vertical-align: middle; }
-
-/* Assertions */
-.assertion { display: flex; gap: 8px; }
-.assert-icon { flex-shrink: 0; font-size: 14px; margin-top: 1px; }
-.assert-pass .assert-icon { color: #16a34a; }
-.assert-fail .assert-icon { color: #dc2626; }
-.assert-body { flex: 1; min-width: 0; }
-.assert-text { font-size: 13px; color: #1e293b; word-break: break-word; }
-.assert-reason { font-size: 12px; color: #64748b; margin-top: 2px; word-break: break-word; }
-.muted { color: #94a3b8; font-size: 13px; }
-</style>
-</head>
-<body>
-<div class="container">
-
-  <!-- Header -->
-  <div class="header">
-    <h1>${escapeHtml(skill_name)}</h1>
-    <div class="header-meta">
-      <span><b>Agent</b> ${escapeHtml(agent)}</span>
-      <span><b>Type</b> ${evalType}</span>
-      <span><b>Date</b> ${escapeHtml(formattedDate)}</span>
-      <span><b>Score</b> ${escapeHtml(metrics.withSkillScore)}</span>
-    </div>
-    <div class="status-bar ${statusClass}"></div>
-  </div>
-
-  <!-- Metric Cards -->
-  ${renderMetricsCards(report)}
-
-  <!-- Chart -->
-  <div class="section">
-    <div class="section-title">Pass rate by task</div>
-    ${renderChart(report)}
-  </div>
-
-  <!-- Task Table -->
-  <div class="section">
-    <div class="section-title">Task results</div>
-    ${renderTaskTable(report)}
-  </div>
-
-</div>
-<script>
-(function () {
-  // Chart
-  const rawData = document.getElementById('chart-data');
-  if (rawData) {
-    const data = JSON.parse(rawData.textContent || '{}');
-    const ctx = document.getElementById('eval-chart');
-    if (ctx) {
-      new Chart(ctx, {
-        type: 'bar',
-        data: data,
-        options: {
-          indexAxis: 'y',
-          responsive: true,
-          plugins: { legend: { display: ${functional ? 'true' : 'false'} } },
-          scales: {
-            x: { min: 0, max: 100, ticks: { callback: v => v + '%' }, grid: { color: '#f1f5f9' } },
-            y: { grid: { display: false } }
-          }
-        }
-      });
-    }
-  }
-
-  // Accordion
-  document.querySelectorAll('.details-btn').forEach(function (btn) {
-    btn.addEventListener('click', function () {
-      const target = document.getElementById(btn.getAttribute('data-target'));
-      if (!target) return;
-      const isOpen = target.classList.contains('visible');
-      target.classList.toggle('visible', !isOpen);
-      btn.classList.toggle('open', !isOpen);
-      btn.textContent = isOpen ? '▶' : '▼';
-    });
-  });
-}());
-</script>
-</body>
-</html>`;
-}

package/dist/core/reporters/index.js

@@ -1,9 +0,0 @@
-import { HtmlReporter } from './html-reporter.js';
-import { JsonReporter } from './json-reporter.js';
-export { HtmlReporter } from './html-reporter.js';
-export { JsonReporter } from './json-reporter.js';
-export function createReporter(format) {
-    if (format === 'html')
-        return new HtmlReporter();
-    return new JsonReporter();
-}

package/dist/core/reporters/json-reporter.js

@@ -1,7 +0,0 @@
-import path from 'path';
-import { Logger } from '../../utils/logger.js';
-export class JsonReporter {
-    generate(_report, runDir) {
-        Logger.write(`\n   Report: file://${path.join(runDir, 'summary.json')}\n`);
-    }
-}

package/dist/core/reporters/reporter.js

@@ -1 +0,0 @@
-export {};

package/dist/core/runner.js

@@ -1,75 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HeadlessRunner = void 0;
-const child_process_1 = require("child_process");
-class HeadlessRunner {
-    agent;
-    constructor(agent) {
-        this.agent = agent;
-    }
-    /**
-     * Runs the prompt through a headless isolated gemini instance in auto_edit mode.
-     * Auto Edit mode automatically allows tools meant for modifying files, falling back interactively
-     * only for severe system-level actions (which we assume tests shouldn't invoke, or will fail if they do in non-interactive).
-     * @param prompt The evaluation prompt text
-     * @returns Parsed JSON output from Gemini
-     */
-    runPrompt(prompt) {
-        if (this.agent !== 'gemini-cli') {
-            console.error(`\n[Runner] Agent '${this.agent}' is not supported yet.`);
-            return null;
-        }
-        // Escaping the prompt just in case. spawnSync handles formatting but better safe.
-        try {
-            const child = (0, child_process_1.spawnSync)('gemini', [
-                '-p', prompt,
-                '-o', 'json',
-                '--approval-mode', 'auto_edit'
-            ], {
-                encoding: 'utf-8',
-                // Not passing default stdio because we need to parse stdout 
-                // silently without printing all the tool interactions to the user terminal.
-                stdio: ['ignore', 'pipe', 'pipe']
-            });
-            if (child.error) {
-                console.error(`\n[Runner] Failed to start gemini CLI. Error: ${child.error.message}`);
-                return null;
-            }
-            // If exit code is not 0, there was likely a crash or unhanded exception in gemini cli.
-            if (child.status !== 0) {
-                console.error(`\n[Runner] Gemini CLI exited with status ${child.status}`);
-                if (child.stderr) {
-                    console.error(`[Runner Stderr] ${child.stderr.trim()}`);
-                }
-            }
-            const rawOutput = child.stdout;
-            if (!rawOutput || rawOutput.trim() === '') {
-                console.error(`\n[Runner] Received empty output from Gemini CLI. Cannot evaluate.`);
-                return null;
-            }
-            // We need to safely extract the JSON part because Gemini CLI might emit 
-            // non-JSON warnings like "MCP issues detected. Run /mcp list for status." to stdout. 
-            // We'll search for the first '{' to begin our JSON payload.
-            let jsonPart = rawOutput.trim();
-            const firstBraceIndex = jsonPart.indexOf('{');
-            if (firstBraceIndex > 0) {
-                jsonPart = jsonPart.substring(firstBraceIndex);
-            }
-            try {
-                const parsed = JSON.parse(jsonPart);
-                return parsed;
-            }
-            catch (parseError) {
-                console.error(`\n[Runner] Failed to parse JSON output: ${parseError}`);
-                // Dump first 300 chars of raw to debug
-                console.error(`[Runner Output Preview] ${rawOutput.substring(0, 300)}`);
-                return null;
-            }
-        }
-        catch (e) {
-            console.error(`\n[Runner] Unexpected error running process: ${e}`);
-            return null;
-        }
-    }
-}
-exports.HeadlessRunner = HeadlessRunner;

package/dist/core/runners/factory.js

@@ -1,18 +0,0 @@
-import { GeminiCliRunner } from './gemini-cli.runner.js';
-export class RunnerFactory {
-    /**
-     * Factory method to create an agent runner based on the agent name.
-     * Add new agent implementations here.
-     * @param agent The name of the agent to run
-     * @returns An implementation of AgentRunner
-     * @throws Error if the agent is not supported
-     */
-    static create(agent) {
-        switch (agent) {
-            case 'gemini-cli':
-                return new GeminiCliRunner();
-            default:
-                throw new Error(`Agent '${agent}' is not supported yet.`);
-        }
-    }
-}

package/dist/core/runners/gemini-cli.runner.js

@@ -1,138 +0,0 @@
-import * as fs from 'fs';
-import child_process from 'child_process';
-import { Logger } from '../../utils/logger.js';
-export class GeminiCliRunner {
-    /**
-     * Runs the prompt through an isolated gemini instance.
-     * Default mode is headless using --approval-mode auto_edit.
-     *
-     * @param prompt The evaluation prompt text
-     * @param cwd Optional execution directory
-     * @param onLog Callback to receive real-time logs (from stderr)
-     * @param logPath Optional file path to save raw execution logs
-     * @returns Raw output from Gemini
-     */
-    async runPrompt(prompt, cwd, onLog, logPath, extraArgs = []) {
-        return new Promise((resolve) => {
-            let stdout = '';
-            let stderr = '';
-            let resolved = false;
-            // Use -p and --approval-mode auto_edit for headless mode.
-            const args = ['-p', prompt, '--approval-mode', 'auto_edit', ...extraArgs];
-            const spawnOptions = {
-                cwd: cwd,
-                env: { ...process.env, FORCE_COLOR: '1' }
-            };
-            const child = child_process.spawn('gemini', args, spawnOptions);
-            // Setup log stream if path is provided
-            let logStream = null;
-            let logStreamDone = true; // Default to true if no logPath
-            if (logPath) {
-                try {
-                    logStream = fs.createWriteStream(logPath, { flags: 'a' });
-                    logStreamDone = false;
-                    logStream.on('finish', () => {
-                        logStreamDone = true;
-                        checkAllDone();
-                    });
-                    logStream.write(`--- Gemini CLI Execution Start: ${new Date().toISOString()} ---\n`);
-                    logStream.write(`Command: gemini ${args.join(' ')}\n\n`);
-                }
-                catch (err) {
-                    Logger.warn(`Failed to create log file at ${logPath} — verbose output will not be saved. Continuing. Reason: ${err}`);
-                    logStreamDone = true;
-                }
-            }
-            // Safety timeout: 5 minutes (300,000 ms)
-            const timeout = setTimeout(() => {
-                if (!resolved) {
-                    resolved = true;
-                    child.kill('SIGKILL');
-                    if (logStream) {
-                        logStream.write('\n\n--- Gemini CLI process timed out ---\n');
-                        logStream.end();
-                    }
-                    Logger.error('\nGemini CLI process timed out after 5 minutes.');
-                    resolve({ error: 'Process timeout exceeded (5 minutes)', raw_output: stderr });
-                }
-            }, 300000);
-            // Track completion of streams
-            let stdoutDone = false;
-            let stderrDone = false;
-            let processDone = false;
-            function checkAllDone() {
-                if (stdoutDone && stderrDone && processDone && logStreamDone && !resolved) {
-                    resolved = true;
-                    clearTimeout(timeout);
-                    if (!stdout || stdout.trim() === '') {
-                        return resolve({ error: 'Empty output from Gemini CLI', raw_output: stderr });
-                    }
-                    return resolve({
-                        response: stdout.trim(),
-                        raw_output: `${stdout}\n--- STDERR ---\n${stderr}`
-                    });
-                }
-            }
-            if (child.stdout) {
-                child.stdout.on('data', (data) => {
-                    const chunk = data.toString();
-                    stdout += chunk;
-                    if (logStream)
-                        logStream.write(chunk);
-                });
-                child.stdout.on('end', () => {
-                    stdoutDone = true;
-                    checkAllDone();
-                });
-            }
-            else {
-                stdoutDone = true;
-            }
-            if (child.stderr) {
-                child.stderr.on('data', (data) => {
-                    const chunk = data.toString();
-                    stderr += chunk;
-                    if (onLog) {
-                        const lines = chunk.split('\n').filter((l) => l.trim() !== '');
-                        if (lines.length > 0) {
-                            onLog(lines[lines.length - 1]);
-                        }
-                    }
-                });
-                child.stderr.on('end', () => {
-                    stderrDone = true;
-                    checkAllDone();
-                });
-            }
-            else {
-                stderrDone = true;
-            }
-            child.on('error', (err) => {
-                if (!resolved) {
-                    resolved = true;
-                    clearTimeout(timeout);
-                    if (logStream) {
-                        logStream.write(`\n\n--- Error starting Gemini CLI: ${err.message} ---\n`);
-                        logStream.end();
-                    }
-                    Logger.error(`Failed to start gemini CLI. Error: ${err.message}`);
-                    resolve(null);
-                }
-            });
-            child.on('close', (code) => {
-                if (logStream) {
-                    logStream.write(`\n\n--- Gemini CLI exited with status ${code} ---\n`);
-                    logStream.end();
-                }
-                if (code !== 0 && !resolved) {
-                    Logger.error(`Gemini CLI exited with status ${code}`);
-                    if (stderr) {
-                        Logger.debug(`Gemini CLI Stderr: ${stderr.trim()}`);
-                    }
-                }
-                processDone = true;
-                checkAllDone();
-            });
-        });
-    }
-}

package/dist/core/runners/index.js

@@ -1,3 +0,0 @@
-export * from './runner.interface.js';
-export * from './gemini-cli.runner.js';
-export * from './factory.js';

package/dist/core/runners/runner.interface.js

@@ -1 +0,0 @@
-export {};