@fede0089/skill-eval 1.0.0

package/dist/core/agent-pool.js

@@ -0,0 +1,40 @@
+/**
+ * A semaphore-based pool that limits the number of concurrently running agents.
+ * `acquire()` resolves to a `release` callback when a slot is available,
+ * blocking if the pool is at capacity. Releases are idempotent; queue is FIFO.
+ */
+export class AgentPool {
+    max;
+    active = 0;
+    queue = [];
+    constructor(max) {
+        this.max = max;
+    }
+    acquire() {
+        if (this.active < this.max) {
+            this.active++;
+            return Promise.resolve(this.makeRelease());
+        }
+        return new Promise(resolve => {
+            this.queue.push(() => {
+                this.active++;
+                resolve(this.makeRelease());
+            });
+        });
+    }
+    makeRelease() {
+        let released = false;
+        return () => {
+            if (released)
+                return;
+            released = true;
+            this.active--;
+            this.drain();
+        };
+    }
+    drain() {
+        while (this.active < this.max && this.queue.length > 0) {
+            this.queue.shift()();
+        }
+    }
+}

package/dist/core/config.js

@@ -0,0 +1,58 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { ConfigError } from './errors.js';
+const CONFIG_FILE = '.skill-eval.json';
+/**
+ * Loads configuration from a `.skill-eval.json` file in the given directory.
+ * Returns an empty object if the file does not exist (config is optional).
+ *
+ * CLI flags always take precedence over config file values — this function
+ * only provides defaults for flags not explicitly passed on the command line.
+ *
+ * @throws ConfigError on malformed JSON or type mismatches.
+ */
+export function loadConfig(cwd) {
+    const configPath = path.join(cwd, CONFIG_FILE);
+    if (!fs.existsSync(configPath)) {
+        return {};
+    }
+    let raw;
+    try {
+        raw = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+    }
+    catch (err) {
+        throw new ConfigError(`Failed to parse ${CONFIG_FILE}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
+        throw new ConfigError(`${CONFIG_FILE} must be a JSON object.`);
+    }
+    const config = raw;
+    const result = {};
+    if ('agent' in config) {
+        if (typeof config.agent !== 'string')
+            throw new ConfigError(`${CONFIG_FILE}: 'agent' must be a string.`);
+        result.agent = config.agent;
+    }
+    if ('concurrency' in config) {
+        if (typeof config.concurrency !== 'number')
+            throw new ConfigError(`${CONFIG_FILE}: 'concurrency' must be a number.`);
+        result.concurrency = config.concurrency;
+    }
+    if ('trials' in config) {
+        if (typeof config.trials !== 'number')
+            throw new ConfigError(`${CONFIG_FILE}: 'trials' must be a number.`);
+        result.trials = config.trials;
+    }
+    if ('report' in config) {
+        if (config.report !== 'html' && config.report !== 'json') {
+            throw new ConfigError(`${CONFIG_FILE}: 'report' must be 'html' or 'json'.`);
+        }
+        result.report = config.report;
+    }
+    if ('skill' in config) {
+        if (typeof config.skill !== 'string')
+            throw new ConfigError(`${CONFIG_FILE}: 'skill' must be a string.`);
+        result.skill = config.skill;
+    }
+    return result;
+}

package/dist/core/environment.js

@@ -0,0 +1,83 @@
+import { executor } from '../utils/exec.js';
+import * as path from 'path';
+import * as fs from 'fs';
+import { Logger } from '../utils/logger.js';
+import { ExecutionError } from './errors.js';
+export class EvalEnvironment {
+    workspace;
+    constructor(options) {
+        this.workspace = options.workspace;
+    }
+    async setup() {
+    }
+    async teardown() {
+        const worktreesDir = path.resolve(this.workspace, '.project-skill-evals', 'worktrees');
+        if (!fs.existsSync(worktreesDir))
+            return;
+        for (const entry of fs.readdirSync(worktreesDir)) {
+            this.removeWorktree(path.join(worktreesDir, entry));
+        }
+        executor.spawnSync('git', ['worktree', 'prune'], { stdio: 'ignore', cwd: this.workspace });
+    }
+    /**
+     * Creates a temporary git worktree for a specific evaluation.
+     * This provides isolation by ensuring each test runs in its own clean copy of the repo.
+     */
+    createWorktree(evalId) {
+        const worktreePath = path.resolve(this.workspace, '.project-skill-evals', 'worktrees', evalId);
+        // Ensure the path is clean before adding a worktree.
+        // We try to remove it first in case a previous run crashed.
+        executor.spawnSync('git', ['worktree', 'remove', '--force', worktreePath], { stdio: 'ignore', cwd: this.workspace });
+        // If git worktree remove failed (e.g. path was never registered, or git
+        // metadata is stale), fall back to a physical wipe and a metadata prune so
+        // that 'git worktree add' does not exit 128 on a pre-existing path.
+        if (fs.existsSync(worktreePath)) {
+            fs.rmSync(worktreePath, { recursive: true, force: true });
+        }
+        executor.spawnSync('git', ['worktree', 'prune'], { stdio: 'ignore', cwd: this.workspace });
+        const child = executor.spawnSync('git', ['worktree', 'add', worktreePath, '-f'], {
+            stdio: 'ignore',
+            encoding: 'utf-8',
+            cwd: this.workspace
+        });
+        if (child.status !== 0) {
+            throw new ExecutionError(`Failed to create git worktree at ${worktreePath}. Process exited with code ${child.status}`);
+        }
+        return worktreePath;
+    }
+    /**
+     * Removes a previously created git worktree and its associated branch.
+     */
+    removeWorktree(worktreePath) {
+        const branchName = path.basename(worktreePath);
+        const child = executor.spawnSync('git', ['worktree', 'remove', '--force', worktreePath], {
+            stdio: 'ignore',
+            encoding: 'utf-8',
+            cwd: this.workspace
+        });
+        if (child.status !== 0) {
+            // git worktree remove failed (e.g. path already deregistered by a previous prune).
+            // Fall back to physical removal and prune stale references.
+            try {
+                if (fs.existsSync(worktreePath)) {
+                    fs.rmSync(worktreePath, { recursive: true, force: true });
+                }
+                executor.spawnSync('git', ['worktree', 'prune'], { stdio: 'ignore', cwd: this.workspace });
+            }
+            catch (err) {
+                Logger.warn(`Failed to remove worktree at ${worktreePath}. Process exited with code ${child.status}. Manual cleanup may be required.`);
+            }
+        }
+        // Always try to delete the branch created for this worktree.
+        // Wrap in try-catch because the branch might already be gone.
+        try {
+            executor.spawnSync('git', ['branch', '-D', branchName], {
+                stdio: 'ignore',
+                cwd: this.workspace
+            });
+        }
+        catch (err) {
+            // Ignore errors deleting the temporary branch
+        }
+    }
+}

package/dist/core/errors.js

@@ -0,0 +1,29 @@
+export class AppError extends Error {
+    message;
+    code;
+    constructor(message, code = 'GENERIC_ERROR') {
+        super(message);
+        this.message = message;
+        this.code = code;
+        this.name = this.constructor.name;
+        Object.setPrototypeOf(this, AppError.prototype);
+    }
+}
+export class ConfigError extends AppError {
+    constructor(message) {
+        super(message, 'CONFIG_ERROR');
+        Object.setPrototypeOf(this, ConfigError.prototype);
+    }
+}
+export class ExecutionError extends AppError {
+    constructor(message) {
+        super(message, 'EXECUTION_ERROR');
+        Object.setPrototypeOf(this, ExecutionError.prototype);
+    }
+}
+export class ValidationError extends AppError {
+    constructor(message) {
+        super(message, 'VALIDATION_ERROR');
+        Object.setPrototypeOf(this, ValidationError.prototype);
+    }
+}

package/dist/core/eval-runner.js

@@ -0,0 +1,306 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { executor } from '../utils/exec.js';
+import { EvalEnvironment } from './environment.js';
+import { RunnerFactory } from '../runners/index.js';
+import { TriggerGrader, ModelBasedGrader } from './evaluator.js';
+import { parseStreamResult, parseTokenStats } from '../utils/ndjson.js';
+export class EvalRunner {
+    options;
+    env;
+    runner;
+    triggerGrader;
+    functionalGrader;
+    constructor(options) {
+        this.options = options;
+        this.env = new EvalEnvironment({ workspace: options.workspace });
+        this.runner = RunnerFactory.create(options.agent);
+        this.triggerGrader = new TriggerGrader(options.skillName, this.runner.skillDispatchToolName);
+        // Inject the same runner for judging so swapping the agent backend works end-to-end
+        this.functionalGrader = new ModelBasedGrader(options.skillName, this.runner);
+    }
+    async runTriggerTask(task, index, trialId, uiCtx, attempt = 0) {
+        const logFileName = `task_${task.id}_trial_${trialId}.log`;
+        const logPath = this.options.debug ? path.join(this.options.runDir, logFileName) : undefined;
+        let worktreePath;
+        let transcript = null;
+        let durationMs;
+        const worktreeId = attempt > 0 ? `task-${task.id}-trial-${trialId}-r${attempt}` : `task-${task.id}-trial-${trialId}`;
+        if (attempt > 0) {
+            const prevId = attempt === 1
+                ? `task-${task.id}-trial-${trialId}`
+                : `task-${task.id}-trial-${trialId}-r${attempt - 1}`;
+            this.env.removeWorktree(path.resolve(this.options.workspace, '.project-skill-evals', 'worktrees', prevId));
+        }
+        try {
+            uiCtx.updateLog('Setting up…');
+            worktreePath = this.env.createWorktree(worktreeId);
+            await this.runner.linkSkill(path.resolve(this.options.workspace, this.options.skillPath), worktreePath);
+            this.runner.applyRunnerConfig(path.resolve(this.options.workspace, this.options.skillPath, 'evals', 'config'), worktreePath);
+            uiCtx.updateLog('Executing prompt…');
+            const startMs = Date.now();
+            transcript = await this.runner.runPrompt(task.prompt, worktreePath, undefined, logPath, undefined, this.options.timeoutMs);
+            durationMs = Date.now() - startMs;
+        }
+        finally {
+            if (worktreePath) {
+                this.env.removeWorktree(worktreePath);
+            }
+        }
+        // Propagate stream-json errors: when the agent fails, Gemini CLI still
+        // writes a {"type":"result","status":"error",...} event to stdout so transcript.error
+        // is never set by the runner. Parse it here so the grading path is skipped correctly.
+        if (transcript && !transcript.error) {
+            const streamResult = parseStreamResult(transcript.response || '');
+            if (streamResult && 'error' in streamResult) {
+                transcript.error = streamResult.error;
+            }
+        }
+        // Extract token stats from the agent's NDJSON stream (never from the judge).
+        const tokenStats = transcript
+            ? parseTokenStats(transcript.response || '') ?? undefined
+            : undefined;
+        let triggered = false;
+        const assertionResults = [];
+        if (transcript && !transcript.error) {
+            uiCtx.updateLog('Grading…');
+            triggered = this.triggerGrader.gradeTrigger(transcript);
+            assertionResults.push({
+                assertion: 'Skill was triggered',
+                passed: triggered,
+                reason: triggered ? 'Detected skill activation in transcript' : 'No skill activation detected in transcript',
+                graderType: 'programmatic'
+            });
+        }
+        else {
+            const errorMsg = transcript?.error || 'Error: No transcript was produced';
+            assertionResults.push({
+                assertion: 'Skill was triggered',
+                passed: false,
+                reason: errorMsg,
+                graderType: 'programmatic'
+            });
+            return {
+                id: trialId,
+                transcript: transcript || { error: 'No transcript produced' },
+                assertionResults,
+                trialPassed: false,
+                isError: true,
+                tokenStats,
+                durationMs
+            };
+        }
+        return {
+            id: trialId,
+            transcript: transcript || { error: 'No transcript produced' },
+            assertionResults: assertionResults,
+            trialPassed: triggered,
+            tokenStats,
+            durationMs
+        };
+    }
+    async runFunctionalTask(task, index, trialId, uiCtx, attempt = 0) {
+        const skillDisabled = this.options.isBaseline;
+        const evalModeLabel = skillDisabled ? 'without-skill' : 'with-skill';
+        const promptToUse = skillDisabled
+            ? `${task.prompt}\n\nIMPORTANT: For this task, you MUST NOT use the '${this.options.skillName}' skill/tool, even if it appears available.`
+            : `${task.prompt}\n\nIMPORTANT: You must use the '${this.options.skillName}' skill/tool to solve this task.`;
+        const logFileName = `task_${task.id}_${evalModeLabel}_trial_${trialId}.log`;
+        const logPath = this.options.debug ? path.join(this.options.runDir, logFileName) : undefined;
+        let worktreePath;
+        let assertionResults = [];
+        let trialPassed = false;
+        let transcript = null;
+        let durationMs;
+        const worktreeId = attempt > 0 ? `task-${task.id}-${evalModeLabel}-trial-${trialId}-r${attempt}` : `task-${task.id}-${evalModeLabel}-trial-${trialId}`;
+        if (attempt > 0) {
+            const prevId = attempt === 1
+                ? `task-${task.id}-${evalModeLabel}-trial-${trialId}`
+                : `task-${task.id}-${evalModeLabel}-trial-${trialId}-r${attempt - 1}`;
+            this.env.removeWorktree(path.resolve(this.options.workspace, '.project-skill-evals', 'worktrees', prevId));
+        }
+        try {
+            uiCtx.updateLog('Setting up…');
+            worktreePath = this.env.createWorktree(worktreeId);
+            if (!skillDisabled) {
+                await this.runner.linkSkill(path.resolve(this.options.workspace, this.options.skillPath), worktreePath);
+            }
+            this.runner.applyRunnerConfig(path.resolve(this.options.workspace, this.options.skillPath, 'evals', 'config'), worktreePath);
+            uiCtx.updateLog('Executing prompt…');
+            if (logPath)
+                fs.appendFileSync(logPath, `\n# SECTION: ${evalModeLabel.toUpperCase()} AGENT RUN\n`);
+            const startMs = Date.now();
+            transcript = await this.runner.runPrompt(promptToUse, worktreePath, undefined, logPath, undefined, this.options.timeoutMs);
+            durationMs = Date.now() - startMs;
+            // Propagate stream-json errors: when the agent fails (e.g. quota), Gemini CLI still
+            // writes a {"type":"result","status":"error",...} event to stdout so transcript.error
+            // is never set by the runner. Parse it here so the grading path is skipped correctly.
+            if (transcript && !transcript.error) {
+                const streamResult = parseStreamResult(transcript.response || '');
+                if (streamResult && 'error' in streamResult) {
+                    transcript.error = streamResult.error;
+                }
+            }
+            // Extract token stats from the agent's NDJSON stream (never from the judge).
+            const tokenStats = transcript
+                ? parseTokenStats(transcript.response || '') ?? undefined
+                : undefined;
+            if (transcript && !transcript.error) {
+                if (skillDisabled && this.triggerGrader.detectSkillAttempt(transcript)) {
+                    return {
+                        id: trialId,
+                        transcript,
+                        assertionResults: [{
+                                assertion: 'Baseline must not invoke the restricted skill',
+                                passed: false,
+                                reason: `Invalid Without Skill: '${this.options.skillName}' activation detected during without-skill run`,
+                                graderType: 'programmatic'
+                            }],
+                        trialPassed: false,
+                        tokenStats,
+                        durationMs
+                    };
+                }
+                if (!skillDisabled && !this.triggerGrader.gradeTrigger(transcript)) {
+                    return {
+                        id: trialId,
+                        transcript,
+                        assertionResults: [{
+                                assertion: 'Target pass must invoke the skill',
+                                passed: false,
+                                reason: `Invalid With Skill: '${this.options.skillName}' was not successfully activated`,
+                                graderType: 'programmatic'
+                            }],
+                        trialPassed: false,
+                        tokenStats,
+                        durationMs
+                    };
+                }
+                let context = 'No changes detected or git not available.';
+                try {
+                    if (worktreePath) {
+                        const diff = executor.execSync('git diff HEAD', { encoding: 'utf-8', cwd: worktreePath });
+                        const untracked = executor.execSync('git ls-files --others --exclude-standard', { encoding: 'utf-8', cwd: worktreePath });
+                        if (diff || untracked) {
+                            context = `[DIFF]\n${diff}\n\n[UNTRACKED FILES]\n${untracked}`;
+                        }
+                        else {
+                            context = 'No changes detected (clean workspace).';
+                        }
+                    }
+                }
+                catch (e) { }
+                if (task.assertions && task.assertions.length > 0) {
+                    uiCtx.updateLog('Grading…');
+                    if (logPath)
+                        fs.appendFileSync(logPath, `\n# SECTION: ${evalModeLabel.toUpperCase()} JUDGE RUN\n`);
+                    // Use the already-parsed stream result to get clean text for the judge.
+                    const streamResult = parseStreamResult(transcript.response || '');
+                    const streamText = streamResult && 'response' in streamResult ? streamResult.response : '';
+                    const gradingTranscript = streamText ? { ...transcript, response: streamText } : transcript;
+                    // Retry only the judge on infrastructure failures (timeout, interactive prompt, etc.).
+                    // The agent has already run successfully — no need to re-run it.
+                    const MAX_JUDGE_RETRIES = 2;
+                    const judgeDelayMs = this.options.judgeRetryDelayMs ?? 1000;
+                    let judgeErrorAfterAllRetries = null;
+                    for (let judgeAttempt = 0; judgeAttempt <= MAX_JUDGE_RETRIES; judgeAttempt++) {
+                        if (judgeAttempt > 0) {
+                            uiCtx.updateLog(`Judge error, retrying (${judgeAttempt}/${MAX_JUDGE_RETRIES})…`);
+                            await new Promise(r => setTimeout(r, judgeDelayMs * Math.pow(2, judgeAttempt - 1)));
+                        }
+                        try {
+                            assertionResults = await this.functionalGrader.gradeModelBased(task.prompt, gradingTranscript, task.assertions, context, (log) => { uiCtx.updateLog(`Grading: ${log}`); }, logPath, worktreePath);
+                            judgeErrorAfterAllRetries = null;
+                            break;
+                        }
+                        catch (e) {
+                            judgeErrorAfterAllRetries = e instanceof Error ? e : new Error(String(e));
+                        }
+                    }
+                    if (judgeErrorAfterAllRetries) {
+                        // Judge exhausted all retries — return an inconclusive failure.
+                        // isError is intentionally NOT set so the outer withRetry does not
+                        // re-run the expensive agent for a judge infrastructure issue.
+                        const reason = judgeErrorAfterAllRetries.message;
+                        return {
+                            id: trialId,
+                            transcript: transcript || { error: 'No transcript produced' },
+                            assertionResults: task.assertions.map(a => ({
+                                assertion: a, passed: false, reason, graderType: 'model-based'
+                            })),
+                            trialPassed: false,
+                            tokenStats,
+                            durationMs
+                        };
+                    }
+                    trialPassed = assertionResults.every(r => r.passed);
+                }
+                else {
+                    trialPassed = true;
+                }
+                return {
+                    id: trialId,
+                    transcript: transcript || { error: 'No transcript produced' },
+                    assertionResults: assertionResults,
+                    trialPassed,
+                    tokenStats,
+                    durationMs
+                };
+            }
+            else {
+                const errorMsg = transcript?.error || 'Error: No transcript was produced';
+                if (task.assertions) {
+                    assertionResults = task.assertions.map(a => ({
+                        assertion: a,
+                        passed: false,
+                        reason: `Agent execution failed: ${errorMsg}`,
+                        graderType: 'model-based'
+                    }));
+                }
+                // isError return — finally still runs cleanup
+                return {
+                    id: trialId,
+                    transcript: { error: transcript?.error || 'No transcript produced' },
+                    assertionResults,
+                    trialPassed: false,
+                    isError: true,
+                    tokenStats,
+                    durationMs
+                };
+            }
+        }
+        catch (e) {
+            const errorMsg = e instanceof Error ? e.message : String(e);
+            if (task.assertions) {
+                assertionResults = task.assertions.map(a => ({
+                    assertion: a,
+                    passed: false,
+                    reason: `Execution failed: ${errorMsg}`,
+                    graderType: 'model-based'
+                }));
+            }
+            // isError return — finally still runs cleanup
+            return {
+                id: trialId,
+                transcript: { error: errorMsg },
+                assertionResults,
+                trialPassed: false,
+                isError: true
+            };
+        }
+        finally {
+            if (worktreePath) {
+                this.env.removeWorktree(worktreePath);
+            }
+        }
+        // Unreachable: all paths above return explicitly.
+        // This satisfies TypeScript's control-flow analysis.
+        return {
+            id: trialId,
+            transcript: { error: 'No transcript produced' },
+            assertionResults,
+            trialPassed: false,
+            isError: true
+        };
+    }
+}