@fede0089/skill-eval 1.0.0

package/dist/core/runner.js

@@ -0,0 +1,75 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HeadlessRunner = void 0;
+const child_process_1 = require("child_process");
+class HeadlessRunner {
+    agent;
+    constructor(agent) {
+        this.agent = agent;
+    }
+    /**
+     * Runs the prompt through a headless isolated gemini instance in auto_edit mode.
+     * Auto Edit mode automatically allows tools meant for modifying files, falling back interactively
+     * only for severe system-level actions (which we assume tests shouldn't invoke, or will fail if they do in non-interactive).
+     * @param prompt The evaluation prompt text
+     * @returns Parsed JSON output from Gemini
+     */
+    runPrompt(prompt) {
+        if (this.agent !== 'gemini-cli') {
+            console.error(`\n[Runner] Agent '${this.agent}' is not supported yet.`);
+            return null;
+        }
+        // Escaping the prompt just in case. spawnSync handles formatting but better safe.
+        try {
+            const child = (0, child_process_1.spawnSync)('gemini', [
+                '-p', prompt,
+                '-o', 'json',
+                '--approval-mode', 'auto_edit'
+            ], {
+                encoding: 'utf-8',
+                // Not passing default stdio because we need to parse stdout 
+                // silently without printing all the tool interactions to the user terminal.
+                stdio: ['ignore', 'pipe', 'pipe']
+            });
+            if (child.error) {
+                console.error(`\n[Runner] Failed to start gemini CLI. Error: ${child.error.message}`);
+                return null;
+            }
+            // If exit code is not 0, there was likely a crash or unhanded exception in gemini cli.
+            if (child.status !== 0) {
+                console.error(`\n[Runner] Gemini CLI exited with status ${child.status}`);
+                if (child.stderr) {
+                    console.error(`[Runner Stderr] ${child.stderr.trim()}`);
+                }
+            }
+            const rawOutput = child.stdout;
+            if (!rawOutput || rawOutput.trim() === '') {
+                console.error(`\n[Runner] Received empty output from Gemini CLI. Cannot evaluate.`);
+                return null;
+            }
+            // We need to safely extract the JSON part because Gemini CLI might emit 
+            // non-JSON warnings like "MCP issues detected. Run /mcp list for status." to stdout. 
+            // We'll search for the first '{' to begin our JSON payload.
+            let jsonPart = rawOutput.trim();
+            const firstBraceIndex = jsonPart.indexOf('{');
+            if (firstBraceIndex > 0) {
+                jsonPart = jsonPart.substring(firstBraceIndex);
+            }
+            try {
+                const parsed = JSON.parse(jsonPart);
+                return parsed;
+            }
+            catch (parseError) {
+                console.error(`\n[Runner] Failed to parse JSON output: ${parseError}`);
+                // Dump first 300 chars of raw to debug
+                console.error(`[Runner Output Preview] ${rawOutput.substring(0, 300)}`);
+                return null;
+            }
+        }
+        catch (e) {
+            console.error(`\n[Runner] Unexpected error running process: ${e}`);
+            return null;
+        }
+    }
+}
+exports.HeadlessRunner = HeadlessRunner;

package/dist/core/runners/factory.js

@@ -0,0 +1,18 @@
+import { GeminiCliRunner } from './gemini-cli.runner.js';
+export class RunnerFactory {
+    /**
+     * Factory method to create an agent runner based on the agent name.
+     * Add new agent implementations here.
+     * @param agent The name of the agent to run
+     * @returns An implementation of AgentRunner
+     * @throws Error if the agent is not supported
+     */
+    static create(agent) {
+        switch (agent) {
+            case 'gemini-cli':
+                return new GeminiCliRunner();
+            default:
+                throw new Error(`Agent '${agent}' is not supported yet.`);
+        }
+    }
+}

package/dist/core/runners/gemini-cli.runner.js

@@ -0,0 +1,138 @@
+import * as fs from 'fs';
+import child_process from 'child_process';
+import { Logger } from '../../utils/logger.js';
+export class GeminiCliRunner {
+    /**
+     * Runs the prompt through an isolated gemini instance.
+     * Default mode is headless using --approval-mode auto_edit.
+     *
+     * @param prompt The evaluation prompt text
+     * @param cwd Optional execution directory
+     * @param onLog Callback to receive real-time logs (from stderr)
+     * @param logPath Optional file path to save raw execution logs
+     * @returns Raw output from Gemini
+     */
+    async runPrompt(prompt, cwd, onLog, logPath, extraArgs = []) {
+        return new Promise((resolve) => {
+            let stdout = '';
+            let stderr = '';
+            let resolved = false;
+            // Use -p and --approval-mode auto_edit for headless mode.
+            const args = ['-p', prompt, '--approval-mode', 'auto_edit', ...extraArgs];
+            const spawnOptions = {
+                cwd: cwd,
+                env: { ...process.env, FORCE_COLOR: '1' }
+            };
+            const child = child_process.spawn('gemini', args, spawnOptions);
+            // Setup log stream if path is provided
+            let logStream = null;
+            let logStreamDone = true; // Default to true if no logPath
+            if (logPath) {
+                try {
+                    logStream = fs.createWriteStream(logPath, { flags: 'a' });
+                    logStreamDone = false;
+                    logStream.on('finish', () => {
+                        logStreamDone = true;
+                        checkAllDone();
+                    });
+                    logStream.write(`--- Gemini CLI Execution Start: ${new Date().toISOString()} ---\n`);
+                    logStream.write(`Command: gemini ${args.join(' ')}\n\n`);
+                }
+                catch (err) {
+                    Logger.warn(`Failed to create log file at ${logPath} — verbose output will not be saved. Continuing. Reason: ${err}`);
+                    logStreamDone = true;
+                }
+            }
+            // Safety timeout: 5 minutes (300,000 ms)
+            const timeout = setTimeout(() => {
+                if (!resolved) {
+                    resolved = true;
+                    child.kill('SIGKILL');
+                    if (logStream) {
+                        logStream.write('\n\n--- Gemini CLI process timed out ---\n');
+                        logStream.end();
+                    }
+                    Logger.error('\nGemini CLI process timed out after 5 minutes.');
+                    resolve({ error: 'Process timeout exceeded (5 minutes)', raw_output: stderr });
+                }
+            }, 300000);
+            // Track completion of streams
+            let stdoutDone = false;
+            let stderrDone = false;
+            let processDone = false;
+            function checkAllDone() {
+                if (stdoutDone && stderrDone && processDone && logStreamDone && !resolved) {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    if (!stdout || stdout.trim() === '') {
+                        return resolve({ error: 'Empty output from Gemini CLI', raw_output: stderr });
+                    }
+                    return resolve({
+                        response: stdout.trim(),
+                        raw_output: `${stdout}\n--- STDERR ---\n${stderr}`
+                    });
+                }
+            }
+            if (child.stdout) {
+                child.stdout.on('data', (data) => {
+                    const chunk = data.toString();
+                    stdout += chunk;
+                    if (logStream)
+                        logStream.write(chunk);
+                });
+                child.stdout.on('end', () => {
+                    stdoutDone = true;
+                    checkAllDone();
+                });
+            }
+            else {
+                stdoutDone = true;
+            }
+            if (child.stderr) {
+                child.stderr.on('data', (data) => {
+                    const chunk = data.toString();
+                    stderr += chunk;
+                    if (onLog) {
+                        const lines = chunk.split('\n').filter((l) => l.trim() !== '');
+                        if (lines.length > 0) {
+                            onLog(lines[lines.length - 1]);
+                        }
+                    }
+                });
+                child.stderr.on('end', () => {
+                    stderrDone = true;
+                    checkAllDone();
+                });
+            }
+            else {
+                stderrDone = true;
+            }
+            child.on('error', (err) => {
+                if (!resolved) {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    if (logStream) {
+                        logStream.write(`\n\n--- Error starting Gemini CLI: ${err.message} ---\n`);
+                        logStream.end();
+                    }
+                    Logger.error(`Failed to start gemini CLI. Error: ${err.message}`);
+                    resolve(null);
+                }
+            });
+            child.on('close', (code) => {
+                if (logStream) {
+                    logStream.write(`\n\n--- Gemini CLI exited with status ${code} ---\n`);
+                    logStream.end();
+                }
+                if (code !== 0 && !resolved) {
+                    Logger.error(`Gemini CLI exited with status ${code}`);
+                    if (stderr) {
+                        Logger.debug(`Gemini CLI Stderr: ${stderr.trim()}`);
+                    }
+                }
+                processDone = true;
+                checkAllDone();
+            });
+        });
+    }
+}

package/dist/core/runners/index.js

@@ -0,0 +1,3 @@
+export * from './runner.interface.js';
+export * from './gemini-cli.runner.js';
+export * from './factory.js';

package/dist/core/runners/runner.interface.js

@@ -0,0 +1 @@
+export {};

package/dist/core/statistics.js

@@ -0,0 +1,79 @@
+/**
+ * Computes the fraction of individual assertions that passed across all non-error trials.
+ * Returns 0 if there are no non-error trials or no assertions.
+ */
+export function computeAssertionPassRate(trials) {
+    const relevant = trials.filter(t => !t.isError);
+    if (relevant.length === 0)
+        return 0;
+    const total = relevant.reduce((s, t) => s + t.assertionResults.length, 0);
+    if (total === 0)
+        return 0;
+    const passed = relevant.reduce((s, t) => s + t.assertionResults.filter(r => r.passed).length, 0);
+    return passed / total;
+}
+/**
+ * Averages computeAssertionPassRate across all task results.
+ */
+export function aggregateAssertionPassRate(results, trialSelector) {
+    if (results.length === 0)
+        return 0;
+    return results.reduce((sum, r) => sum + computeAssertionPassRate(trialSelector(r)), 0) / results.length;
+}
+/**
+ * Computes the pass rate (pass@1) for a set of trials.
+ * Returns the fraction of trials that passed: c / n.
+ */
+export function computePassAtK(trials, _k = 1) {
+    const n = trials.length;
+    if (n === 0)
+        return 0;
+    return trials.filter(t => t.trialPassed).length / n;
+}
+/**
+ * Aggregates pass@1 across all task results.
+ * Returns the average pass rate over all tasks.
+ *
+ * @param results       Array of task results to aggregate.
+ * @param _numTrials    Unused — kept for call-site compatibility.
+ * @param trialSelector Function that extracts the trial array from a TaskResult.
+ */
+export function aggregatePassAtK(results, _numTrials, trialSelector) {
+    if (results.length === 0)
+        return { passAtK: 0 };
+    return {
+        passAtK: results.reduce((sum, r) => sum + computePassAtK(trialSelector(r)), 0) / results.length
+    };
+}
+/**
+ * Computes average duration across trials that have durationMs set.
+ * Returns null if no trial has duration data.
+ */
+export function aggregateDurationStats(trials) {
+    const withDuration = trials.filter(t => t.durationMs != null);
+    if (withDuration.length === 0)
+        return null;
+    const n = withDuration.length;
+    return {
+        avgMs: Math.round(withDuration.reduce((s, t) => s + t.durationMs, 0) / n),
+        trialCount: n,
+    };
+}
+/**
+ * Computes average token consumption across trials that have token stats.
+ * Trials without tokenStats are excluded from the average.
+ * Returns null if no trial has token stats.
+ */
+export function aggregateTokenStats(trials) {
+    const withStats = trials.filter(t => t.tokenStats);
+    if (withStats.length === 0)
+        return null;
+    const n = withStats.length;
+    return {
+        avgTotal: Math.round(withStats.reduce((s, t) => s + t.tokenStats.totalTokens, 0) / n),
+        avgInput: Math.round(withStats.reduce((s, t) => s + t.tokenStats.inputTokens, 0) / n),
+        avgOutput: Math.round(withStats.reduce((s, t) => s + t.tokenStats.outputTokens, 0) / n),
+        avgCached: Math.round(withStats.reduce((s, t) => s + t.tokenStats.cachedTokens, 0) / n),
+        trialCount: n,
+    };
+}

package/dist/core/trial-utils.js

@@ -0,0 +1,58 @@
+/**
+ * Returns true when a trial represents an infrastructure failure (timeout, blocked
+ * interactive prompt, runner crash, etc.) rather than a legitimate judge verdict.
+ * Infrastructure-error trials are candidates for retry via withRetry().
+ */
+export function isTrialError(trial) {
+    return trial.isError === true;
+}
+/**
+ * Runs fn(), retrying up to maxRetries additional times with exponential backoff
+ * whenever the result is an infrastructure-error trial (isTrialError returns true).
+ * A successful judge verdict (pass OR fail) stops retrying immediately.
+ *
+ * Delays: attempt 1 → baseDelayMs, attempt 2 → baseDelayMs * 2
+ *
+ * @param onRetry Optional callback fired after an error result, before the next attempt.
+ *                Receives the upcoming attempt number (1-based) and the failed trial.
+ *                Use it to surface retry progress in the UI.
+ */
+export async function withRetry(fn, maxRetries = 2, baseDelayMs = 1000, onRetry) {
+    let last;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        if (attempt > 0) {
+            await new Promise(resolve => setTimeout(resolve, baseDelayMs * Math.pow(2, attempt - 1)));
+        }
+        last = await fn(attempt);
+        if (!isTrialError(last))
+            return last;
+        if (attempt < maxRetries)
+            onRetry?.(attempt + 1, last);
+    }
+    return last;
+}
+/**
+ * Pads the trials array up to targetCount when a trial loop aborts early.
+ * Ensures that pass@k calculations always reflect the full requested trial count.
+ *
+ * @param trials     Trials collected so far (may be shorter than targetCount).
+ * @param targetCount The requested number of trials (numTrials).
+ * @param assertionLabel The assertion label to use for the padded entries (e.g. 'Runner Execution').
+ */
+export function padAbortedTrials(trials, targetCount, assertionLabel) {
+    while (trials.length < targetCount) {
+        trials.push({
+            id: trials.length + 1,
+            transcript: { error: 'Trial not executed (previous trial aborted)' },
+            assertionResults: [{
+                    assertion: assertionLabel,
+                    passed: false,
+                    reason: 'Trial not executed (previous trial aborted)',
+                    graderType: 'programmatic'
+                }],
+            trialPassed: false,
+            isError: true
+        });
+    }
+    return trials;
+}

package/dist/index.js

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { triggerCommand } from './commands/trigger.js';
+import { functionalCommand } from './commands/functional.js';
+import { Logger } from './utils/logger.js';
+import { AppError } from './core/errors.js';
+import { HtmlReporter } from './reporters/index.js';
+import { DEFAULT_AGENT } from './runners/registry.js';
+import * as path from 'path';
+import * as fs from 'fs';
+import { fileURLToPath } from 'url';
+export const program = new Command();
+const errorHandler = (err) => {
+    if (err instanceof AppError) {
+        Logger.error(err.message);
+    }
+    else if (err instanceof Error) {
+        Logger.error(`An unexpected error occurred: ${err.message}`);
+        Logger.trace(err);
+    }
+    else {
+        Logger.error(`An unknown error occurred: ${String(err)}`);
+    }
+    process.exit(1);
+};
+program
+    .name('skill-eval')
+    .description('CLI to evaluate agent skills triggering and functionality')
+    .version('1.0.0')
+    .option('-v, --debug', 'Enable debug logging', false);
+program.on('option:debug', () => {
+    process.env.DEBUG = 'true';
+});
+program
+    .command('trigger [agent]')
+    .description('Evaluate triggering of an agent skill')
+    .requiredOption('--workspace <path>', 'Path to the workspace/repo to evaluate against')
+    .requiredOption('--skill <path>', 'Path to the skill directory')
+    .option('--agents <number>', 'Number of parallel agents')
+    .option('--trials <number>', 'Number of trials per task for pass@k calculation')
+    .option('--timeout <seconds>', 'Agent timeout in seconds')
+    .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
+    .action((agent, options) => {
+    const workspace = path.resolve(options.workspace);
+    const selectedAgent = agent || DEFAULT_AGENT;
+    const maxAgents = parseInt(options.agents, 10) || 4;
+    const numTrials = options.trials !== undefined ? (parseInt(options.trials, 10) || 3) : 3;
+    const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
+    const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
+    triggerCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId).catch(errorHandler);
+});
+program
+    .command('functional [agent]')
+    .description('Evaluate functional correctness of an agent skill based on assertions')
+    .requiredOption('--workspace <path>', 'Path to the workspace/repo to evaluate against')
+    .requiredOption('--skill <path>', 'Path to the skill directory')
+    .option('--agents <number>', 'Number of parallel agents')
+    .option('--trials <number>', 'Number of trials per task for pass@k calculation')
+    .option('--timeout <seconds>', 'Agent timeout in seconds')
+    .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
+    .action((agent, options) => {
+    const workspace = path.resolve(options.workspace);
+    const selectedAgent = agent || DEFAULT_AGENT;
+    const maxAgents = parseInt(options.agents, 10) || 4;
+    const numTrials = options.trials !== undefined ? (parseInt(options.trials, 10) || 3) : 3;
+    const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
+    const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
+    functionalCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId).catch(errorHandler);
+});
+const isMain = process.argv[1] && (() => {
+    try {
+        return fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
+    }
+    catch {
+        return false;
+    }
+})();
+if (isMain) {
+    program.parse(process.argv);
+}