@fede0089/skill-eval 1.0.0 → 1.1.0

package/README.md

@@ -33,20 +33,33 @@ For each eval prompt, skill-eval spins up parallel agent processes — some with
 
 > The `trigger` command only runs with-skill trials and checks whether the skill dispatch tool was actually invoked — no judge or baseline needed.
 
-## Quick Start
+## Installation
+
+**Requirements:** Node.js, and the agent CLI you want to evaluate (e.g. `gemini`) installed and on `$PATH`.
+
+### Run without installing
+
+```sh
+npx @fede0089/skill-eval --help
+```
+
+### Install globally
 
 ```sh
-git clone <this-repo>
+npm install -g @fede0089/skill-eval
+skill-eval --help
+```
+
+### From source
+
+```sh
+git clone https://github.com/fede0089/skill-eval.git
 cd skill-eval
 npm install
 npm run build
-npm link
+npm link        # makes `skill-eval` available globally
 ```
 
-This makes `skill-eval` available globally in your terminal.
-
-**Requirements:** Node.js, and the agent CLI you want to evaluate (e.g. `gemini`) installed and on `$PATH`.
-
 ## Commands
 
 ```sh
@@ -67,6 +80,7 @@ skill-eval functional --workspace <path> --skill <path> [options] [agent]
 | `--trials <number>` | no | `3` | Trials per task (for pass@k) |
 | `--timeout <seconds>` | no | none | Kill the agent after this many seconds |
 | `--eval-id <id>` | no | all | Run only the eval with this numeric ID |
+| `--compare-ref [refs...]` | no | — | Git references to compare against |
 | `-v, --debug` | no | `false` | Enable verbose debug logging |
 | `[agent]` | no | `gemini-cli` | Agent backend to use |
 
@@ -168,3 +182,15 @@ The factory, preflight check, and CLI all pick it up automatically.
 1. Create `src/reporters/<format>-reporter.ts` implementing `Reporter`.
 2. Export it and add a case in `createReporter()` in `src/reporters/index.ts`.
 3. Add the format string to `ReportFormat` in `src/types/index.ts`.
+er, binary: '<cli-binary-name>' },
+   ```
+
+The factory, preflight check, and CLI all pick it up automatically.
+
+> Implement `applyRunnerConfig(evalConfigBaseDir, worktreePath)` to copy `evalConfigBaseDir/<your-agent>/` into the appropriate config directory in the worktree (e.g. `.claude/` for a Claude runner). No-op silently if the directory doesn't exist.
+
+### Adding a new report format
+
+1. Create `src/reporters/<format>-reporter.ts` implementing `Reporter`.
+2. Export it and add a case in `createReporter()` in `src/reporters/index.ts`.
+3. Add the format string to `ReportFormat` in `src/types/index.ts`.

package/dist/commands/functional.js

@@ -11,7 +11,9 @@ import { preflight } from '../core/preflight.js';
 import { withRetry } from '../core/trial-utils.js';
 import { renderFunctionalTable, renderRunHeader } from '../utils/table-renderer.js';
 import { JsonReporter } from '../reporters/index.js';
-export async function functionalCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId) {
+import chalk from 'chalk';
+import { git } from '../utils/git.js';
+export async function functionalCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = []) {
     if (!injectedSuite)
         preflight(agent, workspace, skillPath);
     const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath);
@@ -35,39 +37,45 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
     const timestamp = startTime.toISOString().replace(/[:.]/g, '-');
     const runDir = path.resolve(workspace, '.project-skill-evals', 'runs', timestamp);
     fs.mkdirSync(runDir, { recursive: true });
+    const refPathBase = path.resolve(workspace, '.project-skill-evals', 'skill-refs');
+    const variantRunners = new Map();
+    // 1. Local Runner
+    variantRunners.set('local', new EvalRunner({
+        agent, workspace, skillPath, skillName: skill_name, runDir, isBaseline: false, debug, timeoutMs
+    }));
+    // 2. Historical Runners
+    for (const ref of compareRefs) {
+        const refDir = path.join(refPathBase, ref);
+        Logger.write(`   Extracting ref '${ref}'... `);
+        git.extractSkillRef(skillPath, ref, refDir);
+        Logger.write(chalk.green('Done\n'));
+        variantRunners.set(`ref:${ref}`, new EvalRunner({
+            agent,
+            workspace: refDir, // Run inside extracted repo
+            skillPath: path.join(refDir, path.relative(workspace, skillPath)), // Same relative path
+            skillName: skill_name,
+            runDir,
+            isBaseline: false,
+            debug,
+            timeoutMs
+        }));
+    }
+    // 3. Baseline Runner
+    const withoutSkillRunner = new EvalRunner({
+        agent, workspace, skillPath, skillName: skill_name, runDir, isBaseline: true, debug, timeoutMs
+    });
     const taskResults = [];
     let withSkillTasksAllPassedCount = 0;
     let baselineTasksAllPassedCount = 0;
-    // Per-task trial storage, indexed by task.id
-    const withoutSkillTrialsByTask = new Map();
-    const withoutSkillRunner = new EvalRunner({
-        agent,
-        workspace,
-        skillPath,
-        skillName: skill_name,
-        runDir,
-        isBaseline: true,
-        debug,
-        timeoutMs
-    });
-    const withSkillRunner = new EvalRunner({
-        agent,
-        workspace,
-        skillPath,
-        skillName: skill_name,
-        runDir,
-        isBaseline: false,
-        debug,
-        timeoutMs
-    });
     const pool = new AgentPool(maxAgents);
     const ui = new ListrEvalUI();
     // Barrier chain: each prompt waits until the previous prompt's trials have all acquired slots.
     let barrier = Promise.resolve();
-    // Subtask labels: Without Skill 1..N then With Skill 1..N for each prompt
+    // Subtask labels
+    const skillVersions = Array.from(variantRunners.keys());
     const subtaskLabels = [
         ...Array.from({ length: numTrials }, (_, i) => `Without Skill ${i + 1}`),
-        ...Array.from({ length: numTrials }, (_, i) => `With Skill ${i + 1}`)
+        ...skillVersions.flatMap(v => Array.from({ length: numTrials }, (_, i) => `${v} ${i + 1}`))
     ];
     try {
         renderRunHeader({ command: 'functional', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId });
@@ -85,7 +93,7 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
                 subtaskLabels,
                 task: async (uiCtx, multi) => {
                     await thisBarrier;
-                    // ── Without Skill trials (subtask IDs 1..numTrials) ──────────────
+                    // 1. Without Skill trials
                     const baselineTrialPromises = [];
                     for (let idx = 0; idx < numTrials; idx++) {
                         const trialId = idx + 1;
@@ -112,60 +120,74 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
                         }).finally(release);
                         baselineTrialPromises.push(p);
                     }
-                    // ── With Skill trials (subtask IDs numTrials+1..2*numTrials) ─────
-                    const withSkillTrialPromises = [];
-                    for (let idx = 0; idx < numTrials; idx++) {
-                        const subtaskId = numTrials + idx + 1; // Listr subtask position
-                        const runnerTrialId = idx + 1; // 1-based trial number within WI pass
-                        const trialCtx = multi?.getTrialCtx(subtaskId) ?? uiCtx;
-                        const release = await pool.acquire();
-                        const p = withRetry((attempt) => withSkillRunner.runFunctionalTask(task, i, runnerTrialId, trialCtx, attempt)
-                            .catch((error) => ({
-                            id: runnerTrialId,
-                            transcript: { error: error instanceof Error ? error.message : String(error) },
-                            assertionResults: [{ assertion: 'With Skill Execution', passed: false, reason: String(error) }],
-                            trialPassed: false,
-                            isError: true
-                        })), 2, 1000, (nextAttempt, lastTrial) => {
-                            const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
-                            trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
-                        }).then(trial => {
-                            if (multi) {
-                                const reason = trial.assertionResults.find(r => !r.passed)?.reason;
-                                const passedCount = trial.assertionResults.filter(r => r.passed).length;
-                                const totalCount = trial.assertionResults.length;
-                                multi.markTrialComplete(subtaskId, trial.trialPassed, reason, trial.isError, passedCount, totalCount);
-                            }
-                            return trial;
-                        }).finally(release);
-                        withSkillTrialPromises.push(p);
+                    // 2. Skill variants trials
+                    const variantTrialsPromises = {};
+                    let currentSubtaskIdx = numTrials;
+                    for (const [version, runner] of variantRunners.entries()) {
+                        variantTrialsPromises[version] = [];
+                        for (let idx = 0; idx < numTrials; idx++) {
+                            currentSubtaskIdx++;
+                            const subtaskId = currentSubtaskIdx;
+                            const runnerTrialId = idx + 1;
+                            const trialCtx = multi?.getTrialCtx(subtaskId) ?? uiCtx;
+                            const release = await pool.acquire();
+                            const p = withRetry((attempt) => runner.runFunctionalTask(task, i, runnerTrialId, trialCtx, attempt)
+                                .catch((error) => ({
+                                id: runnerTrialId,
+                                transcript: { error: error instanceof Error ? error.message : String(error) },
+                                assertionResults: [{ assertion: `${version} Execution`, passed: false, reason: String(error) }],
+                                trialPassed: false,
+                                isError: true
+                            })), 2, 1000, (nextAttempt, lastTrial) => {
+                                const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
+                                trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
+                            }).then(trial => {
+                                if (multi) {
+                                    const reason = trial.assertionResults.find(r => !r.passed)?.reason;
+                                    const passedCount = trial.assertionResults.filter(r => r.passed).length;
+                                    const totalCount = trial.assertionResults.length;
+                                    multi.markTrialComplete(subtaskId, trial.trialPassed, reason, trial.isError, passedCount, totalCount);
+                                }
+                                return trial;
+                            }).finally(release);
+                            variantTrialsPromises[version].push(p);
+                        }
                     }
-                    // All WO + WI slots for this prompt have been acquired — signal the next prompt.
+                    // All slots for this prompt have been acquired — signal the next prompt.
                     resolveBarrier();
-                    const [woTrials, wiTrials] = await Promise.all([
+                    const [woTrials, ...versionsTrials] = await Promise.all([
                         Promise.all(baselineTrialPromises),
-                        Promise.all(withSkillTrialPromises)
+                        ...Object.values(variantTrialsPromises).map(ps => Promise.all(ps))
                     ]);
-                    withoutSkillTrialsByTask.set(task.id, woTrials);
                     const woPassedCount = woTrials.filter(t => t.trialPassed).length;
                     if (woPassedCount === woTrials.length)
                         baselineTasksAllPassedCount++;
-                    const wiPassedCount = wiTrials.filter(t => t.trialPassed).length;
-                    const score = wiTrials.length > 0 ? wiPassedCount / wiTrials.length : 0;
-                    const withoutSkillTrials = withoutSkillTrialsByTask.get(task.id) ?? [];
+                    const taskSkillTrials = {};
+                    const variantNames = Object.keys(variantTrialsPromises);
+                    let localAllPassed = true;
+                    for (let vIdx = 0; vIdx < variantNames.length; vIdx++) {
+                        const vName = variantNames[vIdx];
+                        const vTrials = versionsTrials[vIdx];
+                        taskSkillTrials[vName] = vTrials.map(t => ({ ...t, transcript: undefined }));
+                        if (vName === 'local') {
+                            const passedCount = vTrials.filter(t => t.trialPassed).length;
+                            if (passedCount === vTrials.length) {
+                                withSkillTasksAllPassedCount++;
+                            }
+                            else {
+                                localAllPassed = false;
+                            }
+                        }
+                    }
                     const taskResult = {
                         taskId: task.id,
                         prompt: task.prompt,
-                        score,
-                        trials: wiTrials.map(t => ({ ...t, transcript: undefined })),
-                        withoutSkillTrials: withoutSkillTrials.map(t => ({ ...t, transcript: undefined }))
+                        baselineTrials: woTrials.map(t => ({ ...t, transcript: undefined })),
+                        skillTrials: taskSkillTrials
                     };
                     taskResults.push(taskResult);
-                    if (wiPassedCount === wiTrials.length) {
-                        withSkillTasksAllPassedCount++;
-                    }
-                    else if (!multi) {
-                        const failureReason = wiTrials.find(t => !t.trialPassed)?.assertionResults.find(r => !r.passed)?.reason || 'With Skill failed';
+                    if (!localAllPassed && !multi) {
+                        const failureReason = taskSkillTrials['local'].find(t => !t.trialPassed)?.assertionResults.find(r => !r.passed)?.reason || 'With Skill failed';
                         throw new Error(failureReason);
                     }
                 }
@@ -173,46 +195,60 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
         }
         await ui.run(tasks.length);
         // ==== REPORTING ====
-        const { passAtK } = aggregatePassAtK(taskResults, numTrials, r => r.trials);
-        const { passAtK: withoutSkillPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.withoutSkillTrials ?? []);
-        const assertionPassRate = aggregateAssertionPassRate(taskResults, r => r.trials);
-        const withoutSkillAssertionPassRate = aggregateAssertionPassRate(taskResults, r => r.withoutSkillTrials ?? []);
-        const withSkillPercentage = Math.round(assertionPassRate * 100);
-        const withoutSkillPercentage = Math.round(withoutSkillAssertionPassRate * 100);
-        const skillUplift = withSkillPercentage - withoutSkillPercentage;
-        const withSkillTokenStats = aggregateTokenStats(taskResults.flatMap(r => r.trials)) ?? undefined;
-        const withoutSkillTokenStats = aggregateTokenStats(taskResults.flatMap(r => r.withoutSkillTrials ?? [])) ?? undefined;
-        const withSkillDurationStats = aggregateDurationStats(taskResults.flatMap(r => r.trials)) ?? undefined;
-        const withoutSkillDurationStats = aggregateDurationStats(taskResults.flatMap(r => r.withoutSkillTrials ?? [])) ?? undefined;
+        const allSkillVersions = taskResults.length > 0 ? Object.keys(taskResults[0].skillTrials) : ['local'];
+        const allVersions = ['baseline', ...allSkillVersions];
+        const scores = {};
+        const passAtK = {};
+        const assertionPassRate = {};
+        const tokenStats = {};
+        const durationStats = {};
+        // 1. Baseline Metrics
+        const { passAtK: woPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.baselineTrials);
+        const woAssertionPassRate = aggregateAssertionPassRate(taskResults, r => r.baselineTrials);
+        const woPercentage = Math.round(woAssertionPassRate * 100);
+        scores['baseline'] = `${woPercentage}%`;
+        passAtK['baseline'] = Math.round(woPassAtK * 1000) / 1000;
+        assertionPassRate['baseline'] = Math.round(woAssertionPassRate * 1000) / 1000;
+        const woTokens = aggregateTokenStats(taskResults.flatMap(r => r.baselineTrials));
+        if (woTokens)
+            tokenStats['baseline'] = woTokens;
+        const woDuration = aggregateDurationStats(taskResults.flatMap(r => r.baselineTrials));
+        if (woDuration)
+            durationStats['baseline'] = woDuration;
+        // 2. Skill Variants Metrics
+        for (const version of allSkillVersions) {
+            const { passAtK: wiPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.skillTrials[version] || []);
+            const wiAssertionPassRate = aggregateAssertionPassRate(taskResults, r => r.skillTrials[version] || []);
+            const wiPercentage = Math.round(wiAssertionPassRate * 100);
+            scores[version] = `${wiPercentage}%`;
+            passAtK[version] = Math.round(wiPassAtK * 1000) / 1000;
+            assertionPassRate[version] = Math.round(wiAssertionPassRate * 1000) / 1000;
+            const wiTokens = aggregateTokenStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (wiTokens)
+                tokenStats[version] = wiTokens;
+            const wiDuration = aggregateDurationStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (wiDuration)
+                durationStats[version] = wiDuration;
+        }
         const report = {
             timestamp: startTime.toISOString(),
             skill_name,
             agent,
             metrics: {
-                withSkillScore: `${withSkillPercentage}%`,
-                withoutSkillScore: `${withoutSkillPercentage}%`,
-                skillUplift: `${skillUplift > 0 ? '+' : ''}${skillUplift}%`,
                 passedCount: withSkillTasksAllPassedCount,
                 totalCount: tasks.length,
                 numTrials,
-                passAtK: Math.round(passAtK * 1000) / 1000,
-                withoutSkillPassAtK: Math.round(withoutSkillPassAtK * 1000) / 1000,
-                assertionPassRate: Math.round(assertionPassRate * 1000) / 1000,
-                withoutSkillAssertionPassRate: Math.round(withoutSkillAssertionPassRate * 1000) / 1000,
-                tokenStats: (withSkillTokenStats || withoutSkillTokenStats)
-                    ? { withSkill: withSkillTokenStats, withoutSkill: withoutSkillTokenStats }
-                    : undefined,
-                durationStats: (withSkillDurationStats || withoutSkillDurationStats)
-                    ? { withSkill: withSkillDurationStats, withoutSkill: withoutSkillDurationStats }
-                    : undefined
+                scores,
+                passAtK,
+                assertionPassRate,
+                tokenStats,
+                durationStats
             },
             results: taskResults
         };
         Logger.write(`\nEVALUATION SUMMARY\n`);
         Logger.write(`──────────────────────────────────────────────────\n`);
         renderFunctionalTable(report);
-        const upliftSign = skillUplift > 0 ? '+' : '';
-        Logger.write(`\n   Skill Uplift:            ${upliftSign}${skillUplift}%\n`);
         Logger.write('\n');
         new JsonReporter().generate(report, runDir);
         reporter.generate(report, runDir);

package/dist/commands/trigger.js

@@ -6,12 +6,14 @@ import * as evalLoader from '../utils/eval-loader.js';
 import { ListrEvalUI } from '../utils/ui.js';
 import { EvalRunner } from '../core/eval-runner.js';
 import { AgentPool } from '../core/agent-pool.js';
-import { aggregatePassAtK, aggregateTokenStats, aggregateDurationStats } from '../core/statistics.js';
+import { aggregatePassAtK, aggregateAssertionPassRate, aggregateTokenStats, aggregateDurationStats } from '../core/statistics.js';
 import { preflight } from '../core/preflight.js';
 import { withRetry } from '../core/trial-utils.js';
 import { renderTriggerTable, renderRunHeader } from '../utils/table-renderer.js';
 import { JsonReporter } from '../reporters/index.js';
-export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId) {
+import chalk from 'chalk';
+import { git } from '../utils/git.js';
+export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = []) {
     if (!injectedSuite)
         preflight(agent, workspace, skillPath);
     const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath);
@@ -36,21 +38,38 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
     const timestamp = startTime.toISOString().replace(/[:.]/g, '-');
     const runDir = path.resolve(workspace, '.project-skill-evals', 'runs', timestamp);
     fs.mkdirSync(runDir, { recursive: true });
+    const refPathBase = path.resolve(workspace, '.project-skill-evals', 'skill-refs');
+    const variantRunners = new Map();
+    // 1. Local Runner
+    variantRunners.set('local', new EvalRunner({
+        agent, workspace, skillPath, skillName: skill_name, runDir, isBaseline: false, debug, timeoutMs
+    }));
+    // 2. Historical Runners
+    for (const ref of compareRefs) {
+        const refDir = path.join(refPathBase, ref);
+        Logger.write(`   Extracting ref '${ref}'... `);
+        git.extractSkillRef(skillPath, ref, refDir);
+        Logger.write(chalk.green('Done\n'));
+        variantRunners.set(`ref:${ref}`, new EvalRunner({
+            agent,
+            workspace: refDir,
+            skillPath: path.join(refDir, path.relative(workspace, skillPath)),
+            skillName: skill_name,
+            runDir,
+            isBaseline: false,
+            debug,
+            timeoutMs
+        }));
+    }
     const taskResults = [];
     let tasksPassedCount = 0;
-    const runner = new EvalRunner({
-        agent,
-        workspace,
-        skillPath,
-        skillName: skill_name,
-        runDir,
-        debug,
-        timeoutMs
-    });
     const pool = new AgentPool(maxAgents);
     const ui = new ListrEvalUI();
     // Barrier chain: each prompt waits until the previous prompt's trials have all acquired slots.
     let barrier = Promise.resolve();
+    // Subtask labels
+    const skillVersions = Array.from(variantRunners.keys());
+    const subtaskLabels = skillVersions.flatMap(v => Array.from({ length: numTrials }, (_, i) => `${v} ${i + 1}`));
     try {
         renderRunHeader({ command: 'trigger', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId });
         Logger.write(`--- Trigger Pass ---\n`);
@@ -65,54 +84,74 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
             ui.addTask({
                 id: task.id,
                 title: taskLabel,
-                numTrials,
+                subtaskLabels,
                 task: async (uiCtx, multi) => {
                     await thisBarrier;
-                    const trialPromises = [];
-                    for (let idx = 0; idx < numTrials; idx++) {
-                        const trialId = idx + 1;
-                        const trialCtx = multi?.getTrialCtx(trialId) ?? uiCtx;
-                        const release = await pool.acquire();
-                        const p = withRetry((attempt) => runner.runTriggerTask(task, i, trialId, trialCtx, attempt)
-                            .catch((error) => ({
-                            id: trialId,
-                            transcript: { error: error instanceof Error ? error.message : String(error) },
-                            assertionResults: [{
-                                    assertion: 'Runner Execution',
-                                    passed: false,
-                                    reason: error instanceof Error ? error.message : String(error)
-                                }],
-                            trialPassed: false,
-                            isError: true
-                        })), 2, 1000, (nextAttempt, lastTrial) => {
-                            const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
-                            trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
-                        }).then(trial => {
-                            if (multi) {
-                                const reason = trial.assertionResults.find(r => !r.passed)?.reason;
-                                multi.markTrialComplete(trialId, trial.trialPassed, reason, trial.isError);
-                            }
-                            return trial;
-                        }).finally(release);
-                        trialPromises.push(p);
+                    const variantTrialsPromises = {};
+                    let currentSubtaskIdx = 0;
+                    for (const [version, runner] of variantRunners.entries()) {
+                        variantTrialsPromises[version] = [];
+                        for (let idx = 0; idx < numTrials; idx++) {
+                            currentSubtaskIdx++;
+                            const subtaskId = currentSubtaskIdx;
+                            const runnerTrialId = idx + 1;
+                            const trialCtx = multi?.getTrialCtx(subtaskId) ?? uiCtx;
+                            const release = await pool.acquire();
+                            const p = withRetry((attempt) => runner.runTriggerTask(task, i, runnerTrialId, trialCtx, attempt)
+                                .catch((error) => ({
+                                id: runnerTrialId,
+                                transcript: { error: error instanceof Error ? error.message : String(error) },
+                                assertionResults: [{
+                                        assertion: `${version} Execution`,
+                                        passed: false,
+                                        reason: error instanceof Error ? error.message : String(error)
+                                    }],
+                                trialPassed: false,
+                                isError: true
+                            })), 2, 1000, (nextAttempt, lastTrial) => {
+                                const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
+                                trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
+                            }).then(trial => {
+                                if (multi) {
+                                    const reason = trial.assertionResults.find(r => !r.passed)?.reason;
+                                    const passedCount = trial.assertionResults.filter(r => r.passed).length;
+                                    const totalCount = trial.assertionResults.length;
+                                    multi.markTrialComplete(subtaskId, trial.trialPassed, reason, trial.isError, passedCount, totalCount);
+                                }
+                                return trial;
+                            }).finally(release);
+                            variantTrialsPromises[version].push(p);
+                        }
                     }
-                    // All trials for this prompt have acquired a slot — signal the next prompt.
+                    // All slots for this prompt have acquired a slot — signal the next prompt.
                     resolveBarrier();
-                    const trials = await Promise.all(trialPromises);
-                    const passedCount = trials.filter(t => t.trialPassed).length;
-                    const score = trials.length > 0 ? passedCount / trials.length : 0;
+                    const versionsTrials = await Promise.all(Object.values(variantTrialsPromises).map(ps => Promise.all(ps)));
+                    const taskSkillTrials = {};
+                    const variantNames = Object.keys(variantTrialsPromises);
+                    let localAllPassed = true;
+                    for (let vIdx = 0; vIdx < variantNames.length; vIdx++) {
+                        const vName = variantNames[vIdx];
+                        const vTrials = versionsTrials[vIdx];
+                        taskSkillTrials[vName] = vTrials.map(t => ({ ...t, transcript: undefined }));
+                        if (vName === 'local') {
+                            const passedCount = vTrials.filter(t => t.trialPassed).length;
+                            if (passedCount === vTrials.length) {
+                                tasksPassedCount++;
+                            }
+                            else {
+                                localAllPassed = false;
+                            }
+                        }
+                    }
                     const taskResult = {
                         taskId: task.id,
                         prompt: task.prompt,
-                        score,
-                        trials: trials.map(t => ({ ...t, transcript: undefined }))
+                        baselineTrials: [],
+                        skillTrials: taskSkillTrials
                     };
                     taskResults.push(taskResult);
-                    if (passedCount === trials.length) {
-                        tasksPassedCount++;
-                    }
-                    else if (!multi) {
-                        const failureReason = trials.find(t => !t.trialPassed)?.assertionResults.find(r => !r.passed)?.reason || 'Task failed evaluation';
+                    if (!localAllPassed && !multi) {
+                        const failureReason = taskSkillTrials['local'].find(t => t.trialPassed === false)?.assertionResults.find(r => !r.passed)?.reason || 'Task failed evaluation';
                         throw new Error(failureReason);
                     }
                 }
@@ -120,22 +159,39 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
         }
         await ui.run(tasks.length);
         // Compute aggregate metrics and build report
-        const { passAtK } = aggregatePassAtK(taskResults, numTrials, r => r.trials);
-        const percentage = Math.round(passAtK * 100);
-        const withSkillTokenStats = aggregateTokenStats(taskResults.flatMap(r => r.trials)) ?? undefined;
-        const withSkillDurationStats = aggregateDurationStats(taskResults.flatMap(r => r.trials)) ?? undefined;
+        const allSkillVersions = taskResults.length > 0 ? Object.keys(taskResults[0].skillTrials) : ['local'];
+        const scores = {};
+        const passAtK = {};
+        const assertionPassRate = {};
+        const tokenStats = {};
+        const durationStats = {};
+        for (const version of allSkillVersions) {
+            const { passAtK: vPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.skillTrials[version] || []);
+            const percentage = Math.round(vPassAtK * 100);
+            scores[version] = `${percentage}%`;
+            passAtK[version] = Math.round(vPassAtK * 1000) / 1000;
+            const vAssertionRate = aggregateAssertionPassRate(taskResults, r => r.skillTrials[version] || []);
+            assertionPassRate[version] = Math.round(vAssertionRate * 1000) / 1000;
+            const vTokens = aggregateTokenStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (vTokens)
+                tokenStats[version] = vTokens;
+            const vDuration = aggregateDurationStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (vDuration)
+                durationStats[version] = vDuration;
+        }
         const report = {
             timestamp: startTime.toISOString(),
             skill_name,
             agent,
             metrics: {
-                withSkillScore: `${percentage}%`,
                 passedCount: tasksPassedCount,
                 totalCount: tasks.length,
                 numTrials,
-                passAtK: Math.round(passAtK * 1000) / 1000,
-                tokenStats: withSkillTokenStats ? { withSkill: withSkillTokenStats } : undefined,
-                durationStats: withSkillDurationStats ? { withSkill: withSkillDurationStats } : undefined
+                scores,
+                passAtK,
+                assertionPassRate,
+                tokenStats,
+                durationStats
             },
             results: taskResults
         };

package/dist/core/environment.js

@@ -11,13 +11,25 @@ export class EvalEnvironment {
     async setup() {
     }
     async teardown() {
-        const worktreesDir = path.resolve(this.workspace, '.project-skill-evals', 'worktrees');
-        if (!fs.existsSync(worktreesDir))
-            return;
-        for (const entry of fs.readdirSync(worktreesDir)) {
-            this.removeWorktree(path.join(worktreesDir, entry));
+        const evalsDir = path.resolve(this.workspace, '.project-skill-evals');
+        // 1. Cleanup Worktrees
+        const worktreesDir = path.join(evalsDir, 'worktrees');
+        if (fs.existsSync(worktreesDir)) {
+            for (const entry of fs.readdirSync(worktreesDir)) {
+                this.removeWorktree(path.join(worktreesDir, entry));
+            }
+            executor.spawnSync('git', ['worktree', 'prune'], { stdio: 'ignore', cwd: this.workspace });
+        }
+        // 2. Cleanup Skill Refs
+        const skillRefsDir = path.join(evalsDir, 'skill-refs');
+        if (fs.existsSync(skillRefsDir)) {
+            try {
+                fs.rmSync(skillRefsDir, { recursive: true, force: true });
+            }
+            catch (err) {
+                Logger.warn(`Failed to remove skill-refs directory at ${skillRefsDir}. Manual cleanup may be required.`);
+            }
         }
-        executor.spawnSync('git', ['worktree', 'prune'], { stdio: 'ignore', cwd: this.workspace });
     }
     /**
      * Creates a temporary git worktree for a specific evaluation.