@fede0089/skill-eval 1.0.6 → 1.2.0

package/README.md

@@ -80,6 +80,7 @@ skill-eval functional --workspace <path> --skill <path> [options] [agent]
 | `--trials <number>` | no | `3` | Trials per task (for pass@k) |
 | `--timeout <seconds>` | no | none | Kill the agent after this many seconds |
 | `--eval-id <id>` | no | all | Run only the eval with this numeric ID |
+| `--compare-ref [refs...]` | no | — | Git references to compare against |
 | `-v, --debug` | no | `false` | Enable verbose debug logging |
 | `[agent]` | no | `gemini-cli` | Agent backend to use |
 
@@ -181,3 +182,15 @@ The factory, preflight check, and CLI all pick it up automatically.
 1. Create `src/reporters/<format>-reporter.ts` implementing `Reporter`.
 2. Export it and add a case in `createReporter()` in `src/reporters/index.ts`.
 3. Add the format string to `ReportFormat` in `src/types/index.ts`.
+er, binary: '<cli-binary-name>' },
+   ```
+
+The factory, preflight check, and CLI all pick it up automatically.
+
+> Implement `applyRunnerConfig(evalConfigBaseDir, worktreePath)` to copy `evalConfigBaseDir/<your-agent>/` into the appropriate config directory in the worktree (e.g. `.claude/` for a Claude runner). No-op silently if the directory doesn't exist.
+
+### Adding a new report format
+
+1. Create `src/reporters/<format>-reporter.ts` implementing `Reporter`.
+2. Export it and add a case in `createReporter()` in `src/reporters/index.ts`.
+3. Add the format string to `ReportFormat` in `src/types/index.ts`.

package/dist/commands/functional.js

@@ -11,7 +11,9 @@ import { preflight } from '../core/preflight.js';
 import { withRetry } from '../core/trial-utils.js';
 import { renderFunctionalTable, renderRunHeader } from '../utils/table-renderer.js';
 import { JsonReporter } from '../reporters/index.js';
-export async function functionalCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId) {
+import chalk from 'chalk';
+import { git } from '../utils/git.js';
+export async function functionalCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = []) {
     if (!injectedSuite)
         preflight(agent, workspace, skillPath);
     const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath);
@@ -35,39 +37,45 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
     const timestamp = startTime.toISOString().replace(/[:.]/g, '-');
     const runDir = path.resolve(workspace, '.project-skill-evals', 'runs', timestamp);
     fs.mkdirSync(runDir, { recursive: true });
+    const refPathBase = path.resolve(workspace, '.project-skill-evals', 'skill-refs');
+    const variantRunners = new Map();
+    // 1. Local Runner
+    variantRunners.set('local', new EvalRunner({
+        agent, workspace, skillPath, skillName: skill_name, runDir, isBaseline: false, debug, timeoutMs
+    }));
+    // 2. Historical Runners
+    for (const ref of compareRefs) {
+        const refDir = path.join(refPathBase, ref);
+        Logger.write(`   Extracting ref '${ref}'... `);
+        git.extractSkillRef(skillPath, ref, refDir);
+        Logger.write(chalk.green('Done\n'));
+        variantRunners.set(`ref:${ref}`, new EvalRunner({
+            agent,
+            workspace: refDir, // Run inside extracted repo
+            skillPath: path.join(refDir, path.relative(workspace, skillPath)), // Same relative path
+            skillName: skill_name,
+            runDir,
+            isBaseline: false,
+            debug,
+            timeoutMs
+        }));
+    }
+    // 3. Baseline Runner
+    const withoutSkillRunner = new EvalRunner({
+        agent, workspace, skillPath, skillName: skill_name, runDir, isBaseline: true, debug, timeoutMs
+    });
     const taskResults = [];
     let withSkillTasksAllPassedCount = 0;
     let baselineTasksAllPassedCount = 0;
-    // Per-task trial storage, indexed by task.id
-    const withoutSkillTrialsByTask = new Map();
-    const withoutSkillRunner = new EvalRunner({
-        agent,
-        workspace,
-        skillPath,
-        skillName: skill_name,
-        runDir,
-        isBaseline: true,
-        debug,
-        timeoutMs
-    });
-    const withSkillRunner = new EvalRunner({
-        agent,
-        workspace,
-        skillPath,
-        skillName: skill_name,
-        runDir,
-        isBaseline: false,
-        debug,
-        timeoutMs
-    });
     const pool = new AgentPool(maxAgents);
     const ui = new ListrEvalUI();
     // Barrier chain: each prompt waits until the previous prompt's trials have all acquired slots.
     let barrier = Promise.resolve();
-    // Subtask labels: Without Skill 1..N then With Skill 1..N for each prompt
+    // Subtask labels
+    const skillVersions = Array.from(variantRunners.keys());
     const subtaskLabels = [
         ...Array.from({ length: numTrials }, (_, i) => `Without Skill ${i + 1}`),
-        ...Array.from({ length: numTrials }, (_, i) => `With Skill ${i + 1}`)
+        ...skillVersions.flatMap(v => Array.from({ length: numTrials }, (_, i) => `${v} ${i + 1}`))
     ];
     try {
         renderRunHeader({ command: 'functional', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId });
@@ -85,7 +93,7 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
                 subtaskLabels,
                 task: async (uiCtx, multi) => {
                     await thisBarrier;
-                    // ── Without Skill trials (subtask IDs 1..numTrials) ──────────────
+                    // 1. Without Skill trials
                     const baselineTrialPromises = [];
                     for (let idx = 0; idx < numTrials; idx++) {
                         const trialId = idx + 1;
@@ -112,60 +120,74 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
                         }).finally(release);
                         baselineTrialPromises.push(p);
                     }
-                    // ── With Skill trials (subtask IDs numTrials+1..2*numTrials) ─────
-                    const withSkillTrialPromises = [];
-                    for (let idx = 0; idx < numTrials; idx++) {
-                        const subtaskId = numTrials + idx + 1; // Listr subtask position
-                        const runnerTrialId = idx + 1; // 1-based trial number within WI pass
-                        const trialCtx = multi?.getTrialCtx(subtaskId) ?? uiCtx;
-                        const release = await pool.acquire();
-                        const p = withRetry((attempt) => withSkillRunner.runFunctionalTask(task, i, runnerTrialId, trialCtx, attempt)
-                            .catch((error) => ({
-                            id: runnerTrialId,
-                            transcript: { error: error instanceof Error ? error.message : String(error) },
-                            assertionResults: [{ assertion: 'With Skill Execution', passed: false, reason: String(error) }],
-                            trialPassed: false,
-                            isError: true
-                        })), 2, 1000, (nextAttempt, lastTrial) => {
-                            const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
-                            trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
-                        }).then(trial => {
-                            if (multi) {
-                                const reason = trial.assertionResults.find(r => !r.passed)?.reason;
-                                const passedCount = trial.assertionResults.filter(r => r.passed).length;
-                                const totalCount = trial.assertionResults.length;
-                                multi.markTrialComplete(subtaskId, trial.trialPassed, reason, trial.isError, passedCount, totalCount);
-                            }
-                            return trial;
-                        }).finally(release);
-                        withSkillTrialPromises.push(p);
+                    // 2. Skill variants trials
+                    const variantTrialsPromises = {};
+                    let currentSubtaskIdx = numTrials;
+                    for (const [version, runner] of variantRunners.entries()) {
+                        variantTrialsPromises[version] = [];
+                        for (let idx = 0; idx < numTrials; idx++) {
+                            currentSubtaskIdx++;
+                            const subtaskId = currentSubtaskIdx;
+                            const runnerTrialId = idx + 1;
+                            const trialCtx = multi?.getTrialCtx(subtaskId) ?? uiCtx;
+                            const release = await pool.acquire();
+                            const p = withRetry((attempt) => runner.runFunctionalTask(task, i, runnerTrialId, trialCtx, attempt)
+                                .catch((error) => ({
+                                id: runnerTrialId,
+                                transcript: { error: error instanceof Error ? error.message : String(error) },
+                                assertionResults: [{ assertion: `${version} Execution`, passed: false, reason: String(error) }],
+                                trialPassed: false,
+                                isError: true
+                            })), 2, 1000, (nextAttempt, lastTrial) => {
+                                const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
+                                trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
+                            }).then(trial => {
+                                if (multi) {
+                                    const reason = trial.assertionResults.find(r => !r.passed)?.reason;
+                                    const passedCount = trial.assertionResults.filter(r => r.passed).length;
+                                    const totalCount = trial.assertionResults.length;
+                                    multi.markTrialComplete(subtaskId, trial.trialPassed, reason, trial.isError, passedCount, totalCount);
+                                }
+                                return trial;
+                            }).finally(release);
+                            variantTrialsPromises[version].push(p);
+                        }
                     }
-                    // All WO + WI slots for this prompt have been acquired — signal the next prompt.
+                    // All slots for this prompt have been acquired — signal the next prompt.
                     resolveBarrier();
-                    const [woTrials, wiTrials] = await Promise.all([
+                    const [woTrials, ...versionsTrials] = await Promise.all([
                         Promise.all(baselineTrialPromises),
-                        Promise.all(withSkillTrialPromises)
+                        ...Object.values(variantTrialsPromises).map(ps => Promise.all(ps))
                     ]);
-                    withoutSkillTrialsByTask.set(task.id, woTrials);
                     const woPassedCount = woTrials.filter(t => t.trialPassed).length;
                     if (woPassedCount === woTrials.length)
                         baselineTasksAllPassedCount++;
-                    const wiPassedCount = wiTrials.filter(t => t.trialPassed).length;
-                    const score = wiTrials.length > 0 ? wiPassedCount / wiTrials.length : 0;
-                    const withoutSkillTrials = withoutSkillTrialsByTask.get(task.id) ?? [];
+                    const taskSkillTrials = {};
+                    const variantNames = Object.keys(variantTrialsPromises);
+                    let localAllPassed = true;
+                    for (let vIdx = 0; vIdx < variantNames.length; vIdx++) {
+                        const vName = variantNames[vIdx];
+                        const vTrials = versionsTrials[vIdx];
+                        taskSkillTrials[vName] = vTrials.map(t => ({ ...t, transcript: undefined }));
+                        if (vName === 'local') {
+                            const passedCount = vTrials.filter(t => t.trialPassed).length;
+                            if (passedCount === vTrials.length) {
+                                withSkillTasksAllPassedCount++;
+                            }
+                            else {
+                                localAllPassed = false;
+                            }
+                        }
+                    }
                     const taskResult = {
                         taskId: task.id,
                         prompt: task.prompt,
-                        score,
-                        trials: wiTrials.map(t => ({ ...t, transcript: undefined })),
-                        withoutSkillTrials: withoutSkillTrials.map(t => ({ ...t, transcript: undefined }))
+                        baselineTrials: woTrials.map(t => ({ ...t, transcript: undefined })),
+                        skillTrials: taskSkillTrials
                     };
                     taskResults.push(taskResult);
-                    if (wiPassedCount === wiTrials.length) {
-                        withSkillTasksAllPassedCount++;
-                    }
-                    else if (!multi) {
-                        const failureReason = wiTrials.find(t => !t.trialPassed)?.assertionResults.find(r => !r.passed)?.reason || 'With Skill failed';
+                    if (!localAllPassed && !multi) {
+                        const failureReason = taskSkillTrials['local'].find(t => !t.trialPassed)?.assertionResults.find(r => !r.passed)?.reason || 'With Skill failed';
                         throw new Error(failureReason);
                     }
                 }
@@ -173,46 +195,60 @@ export async function functionalCommand(agent, workspace, skillPath, maxAgents =
         }
         await ui.run(tasks.length);
         // ==== REPORTING ====
-        const { passAtK } = aggregatePassAtK(taskResults, numTrials, r => r.trials);
-        const { passAtK: withoutSkillPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.withoutSkillTrials ?? []);
-        const assertionPassRate = aggregateAssertionPassRate(taskResults, r => r.trials);
-        const withoutSkillAssertionPassRate = aggregateAssertionPassRate(taskResults, r => r.withoutSkillTrials ?? []);
-        const withSkillPercentage = Math.round(assertionPassRate * 100);
-        const withoutSkillPercentage = Math.round(withoutSkillAssertionPassRate * 100);
-        const skillUplift = withSkillPercentage - withoutSkillPercentage;
-        const withSkillTokenStats = aggregateTokenStats(taskResults.flatMap(r => r.trials)) ?? undefined;
-        const withoutSkillTokenStats = aggregateTokenStats(taskResults.flatMap(r => r.withoutSkillTrials ?? [])) ?? undefined;
-        const withSkillDurationStats = aggregateDurationStats(taskResults.flatMap(r => r.trials)) ?? undefined;
-        const withoutSkillDurationStats = aggregateDurationStats(taskResults.flatMap(r => r.withoutSkillTrials ?? [])) ?? undefined;
+        const allSkillVersions = taskResults.length > 0 ? Object.keys(taskResults[0].skillTrials) : ['local'];
+        const allVersions = ['baseline', ...allSkillVersions];
+        const scores = {};
+        const passAtK = {};
+        const assertionPassRate = {};
+        const tokenStats = {};
+        const durationStats = {};
+        // 1. Baseline Metrics
+        const { passAtK: woPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.baselineTrials);
+        const woAssertionPassRate = aggregateAssertionPassRate(taskResults, r => r.baselineTrials);
+        const woPercentage = Math.round(woAssertionPassRate * 100);
+        scores['baseline'] = `${woPercentage}%`;
+        passAtK['baseline'] = Math.round(woPassAtK * 1000) / 1000;
+        assertionPassRate['baseline'] = Math.round(woAssertionPassRate * 1000) / 1000;
+        const woTokens = aggregateTokenStats(taskResults.flatMap(r => r.baselineTrials));
+        if (woTokens)
+            tokenStats['baseline'] = woTokens;
+        const woDuration = aggregateDurationStats(taskResults.flatMap(r => r.baselineTrials));
+        if (woDuration)
+            durationStats['baseline'] = woDuration;
+        // 2. Skill Variants Metrics
+        for (const version of allSkillVersions) {
+            const { passAtK: wiPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.skillTrials[version] || []);
+            const wiAssertionPassRate = aggregateAssertionPassRate(taskResults, r => r.skillTrials[version] || []);
+            const wiPercentage = Math.round(wiAssertionPassRate * 100);
+            scores[version] = `${wiPercentage}%`;
+            passAtK[version] = Math.round(wiPassAtK * 1000) / 1000;
+            assertionPassRate[version] = Math.round(wiAssertionPassRate * 1000) / 1000;
+            const wiTokens = aggregateTokenStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (wiTokens)
+                tokenStats[version] = wiTokens;
+            const wiDuration = aggregateDurationStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (wiDuration)
+                durationStats[version] = wiDuration;
+        }
         const report = {
             timestamp: startTime.toISOString(),
             skill_name,
             agent,
             metrics: {
-                withSkillScore: `${withSkillPercentage}%`,
-                withoutSkillScore: `${withoutSkillPercentage}%`,
-                skillUplift: `${skillUplift > 0 ? '+' : ''}${skillUplift}%`,
                 passedCount: withSkillTasksAllPassedCount,
                 totalCount: tasks.length,
                 numTrials,
-                passAtK: Math.round(passAtK * 1000) / 1000,
-                withoutSkillPassAtK: Math.round(withoutSkillPassAtK * 1000) / 1000,
-                assertionPassRate: Math.round(assertionPassRate * 1000) / 1000,
-                withoutSkillAssertionPassRate: Math.round(withoutSkillAssertionPassRate * 1000) / 1000,
-                tokenStats: (withSkillTokenStats || withoutSkillTokenStats)
-                    ? { withSkill: withSkillTokenStats, withoutSkill: withoutSkillTokenStats }
-                    : undefined,
-                durationStats: (withSkillDurationStats || withoutSkillDurationStats)
-                    ? { withSkill: withSkillDurationStats, withoutSkill: withoutSkillDurationStats }
-                    : undefined
+                scores,
+                passAtK,
+                assertionPassRate,
+                tokenStats,
+                durationStats
             },
             results: taskResults
         };
         Logger.write(`\nEVALUATION SUMMARY\n`);
         Logger.write(`──────────────────────────────────────────────────\n`);
         renderFunctionalTable(report);
-        const upliftSign = skillUplift > 0 ? '+' : '';
-        Logger.write(`\n   Skill Uplift:            ${upliftSign}${skillUplift}%\n`);
         Logger.write('\n');
         new JsonReporter().generate(report, runDir);
         reporter.generate(report, runDir);

package/dist/commands/trigger.js

@@ -6,12 +6,14 @@ import * as evalLoader from '../utils/eval-loader.js';
 import { ListrEvalUI } from '../utils/ui.js';
 import { EvalRunner } from '../core/eval-runner.js';
 import { AgentPool } from '../core/agent-pool.js';
-import { aggregatePassAtK, aggregateTokenStats, aggregateDurationStats } from '../core/statistics.js';
+import { aggregatePassAtK, aggregateAssertionPassRate, aggregateTokenStats, aggregateDurationStats } from '../core/statistics.js';
 import { preflight } from '../core/preflight.js';
 import { withRetry } from '../core/trial-utils.js';
 import { renderTriggerTable, renderRunHeader } from '../utils/table-renderer.js';
 import { JsonReporter } from '../reporters/index.js';
-export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId) {
+import chalk from 'chalk';
+import { git } from '../utils/git.js';
+export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4, injectedSuite, numTrials = 3, reporter = new JsonReporter(), timeoutMs, evalId, compareRefs = []) {
     if (!injectedSuite)
         preflight(agent, workspace, skillPath);
     const suite = injectedSuite || evalLoader.loadEvalSuite(skillPath);
@@ -36,21 +38,38 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
     const timestamp = startTime.toISOString().replace(/[:.]/g, '-');
     const runDir = path.resolve(workspace, '.project-skill-evals', 'runs', timestamp);
     fs.mkdirSync(runDir, { recursive: true });
+    const refPathBase = path.resolve(workspace, '.project-skill-evals', 'skill-refs');
+    const variantRunners = new Map();
+    // 1. Local Runner
+    variantRunners.set('local', new EvalRunner({
+        agent, workspace, skillPath, skillName: skill_name, runDir, isBaseline: false, debug, timeoutMs
+    }));
+    // 2. Historical Runners
+    for (const ref of compareRefs) {
+        const refDir = path.join(refPathBase, ref);
+        Logger.write(`   Extracting ref '${ref}'... `);
+        git.extractSkillRef(skillPath, ref, refDir);
+        Logger.write(chalk.green('Done\n'));
+        variantRunners.set(`ref:${ref}`, new EvalRunner({
+            agent,
+            workspace: refDir,
+            skillPath: path.join(refDir, path.relative(workspace, skillPath)),
+            skillName: skill_name,
+            runDir,
+            isBaseline: false,
+            debug,
+            timeoutMs
+        }));
+    }
     const taskResults = [];
     let tasksPassedCount = 0;
-    const runner = new EvalRunner({
-        agent,
-        workspace,
-        skillPath,
-        skillName: skill_name,
-        runDir,
-        debug,
-        timeoutMs
-    });
     const pool = new AgentPool(maxAgents);
     const ui = new ListrEvalUI();
     // Barrier chain: each prompt waits until the previous prompt's trials have all acquired slots.
     let barrier = Promise.resolve();
+    // Subtask labels
+    const skillVersions = Array.from(variantRunners.keys());
+    const subtaskLabels = skillVersions.flatMap(v => Array.from({ length: numTrials }, (_, i) => `${v} ${i + 1}`));
     try {
         renderRunHeader({ command: 'trigger', skillName: skill_name, agent, workspace, tasks: tasks.length, trials: numTrials, maxAgents, timeoutMs, runDir, evalId });
         Logger.write(`--- Trigger Pass ---\n`);
@@ -65,54 +84,74 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
             ui.addTask({
                 id: task.id,
                 title: taskLabel,
-                numTrials,
+                subtaskLabels,
                 task: async (uiCtx, multi) => {
                     await thisBarrier;
-                    const trialPromises = [];
-                    for (let idx = 0; idx < numTrials; idx++) {
-                        const trialId = idx + 1;
-                        const trialCtx = multi?.getTrialCtx(trialId) ?? uiCtx;
-                        const release = await pool.acquire();
-                        const p = withRetry((attempt) => runner.runTriggerTask(task, i, trialId, trialCtx, attempt)
-                            .catch((error) => ({
-                            id: trialId,
-                            transcript: { error: error instanceof Error ? error.message : String(error) },
-                            assertionResults: [{
-                                    assertion: 'Runner Execution',
-                                    passed: false,
-                                    reason: error instanceof Error ? error.message : String(error)
-                                }],
-                            trialPassed: false,
-                            isError: true
-                        })), 2, 1000, (nextAttempt, lastTrial) => {
-                            const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
-                            trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
-                        }).then(trial => {
-                            if (multi) {
-                                const reason = trial.assertionResults.find(r => !r.passed)?.reason;
-                                multi.markTrialComplete(trialId, trial.trialPassed, reason, trial.isError);
-                            }
-                            return trial;
-                        }).finally(release);
-                        trialPromises.push(p);
+                    const variantTrialsPromises = {};
+                    let currentSubtaskIdx = 0;
+                    for (const [version, runner] of variantRunners.entries()) {
+                        variantTrialsPromises[version] = [];
+                        for (let idx = 0; idx < numTrials; idx++) {
+                            currentSubtaskIdx++;
+                            const subtaskId = currentSubtaskIdx;
+                            const runnerTrialId = idx + 1;
+                            const trialCtx = multi?.getTrialCtx(subtaskId) ?? uiCtx;
+                            const release = await pool.acquire();
+                            const p = withRetry((attempt) => runner.runTriggerTask(task, i, runnerTrialId, trialCtx, attempt)
+                                .catch((error) => ({
+                                id: runnerTrialId,
+                                transcript: { error: error instanceof Error ? error.message : String(error) },
+                                assertionResults: [{
+                                        assertion: `${version} Execution`,
+                                        passed: false,
+                                        reason: error instanceof Error ? error.message : String(error)
+                                    }],
+                                trialPassed: false,
+                                isError: true
+                            })), 2, 1000, (nextAttempt, lastTrial) => {
+                                const reason = lastTrial.assertionResults[0]?.reason ?? 'infrastructure error';
+                                trialCtx.updateLog(`Retry ${nextAttempt}/2 — ${reason.substring(0, 50)}`);
+                            }).then(trial => {
+                                if (multi) {
+                                    const reason = trial.assertionResults.find(r => !r.passed)?.reason;
+                                    const passedCount = trial.assertionResults.filter(r => r.passed).length;
+                                    const totalCount = trial.assertionResults.length;
+                                    multi.markTrialComplete(subtaskId, trial.trialPassed, reason, trial.isError, passedCount, totalCount);
+                                }
+                                return trial;
+                            }).finally(release);
+                            variantTrialsPromises[version].push(p);
+                        }
                     }
-                    // All trials for this prompt have acquired a slot — signal the next prompt.
+                    // All slots for this prompt have acquired a slot — signal the next prompt.
                     resolveBarrier();
-                    const trials = await Promise.all(trialPromises);
-                    const passedCount = trials.filter(t => t.trialPassed).length;
-                    const score = trials.length > 0 ? passedCount / trials.length : 0;
+                    const versionsTrials = await Promise.all(Object.values(variantTrialsPromises).map(ps => Promise.all(ps)));
+                    const taskSkillTrials = {};
+                    const variantNames = Object.keys(variantTrialsPromises);
+                    let localAllPassed = true;
+                    for (let vIdx = 0; vIdx < variantNames.length; vIdx++) {
+                        const vName = variantNames[vIdx];
+                        const vTrials = versionsTrials[vIdx];
+                        taskSkillTrials[vName] = vTrials.map(t => ({ ...t, transcript: undefined }));
+                        if (vName === 'local') {
+                            const passedCount = vTrials.filter(t => t.trialPassed).length;
+                            if (passedCount === vTrials.length) {
+                                tasksPassedCount++;
+                            }
+                            else {
+                                localAllPassed = false;
+                            }
+                        }
+                    }
                     const taskResult = {
                         taskId: task.id,
                         prompt: task.prompt,
-                        score,
-                        trials: trials.map(t => ({ ...t, transcript: undefined }))
+                        baselineTrials: [],
+                        skillTrials: taskSkillTrials
                     };
                     taskResults.push(taskResult);
-                    if (passedCount === trials.length) {
-                        tasksPassedCount++;
-                    }
-                    else if (!multi) {
-                        const failureReason = trials.find(t => !t.trialPassed)?.assertionResults.find(r => !r.passed)?.reason || 'Task failed evaluation';
+                    if (!localAllPassed && !multi) {
+                        const failureReason = taskSkillTrials['local'].find(t => t.trialPassed === false)?.assertionResults.find(r => !r.passed)?.reason || 'Task failed evaluation';
                         throw new Error(failureReason);
                     }
                 }
@@ -120,22 +159,39 @@ export async function triggerCommand(agent, workspace, skillPath, maxAgents = 4,
         }
         await ui.run(tasks.length);
         // Compute aggregate metrics and build report
-        const { passAtK } = aggregatePassAtK(taskResults, numTrials, r => r.trials);
-        const percentage = Math.round(passAtK * 100);
-        const withSkillTokenStats = aggregateTokenStats(taskResults.flatMap(r => r.trials)) ?? undefined;
-        const withSkillDurationStats = aggregateDurationStats(taskResults.flatMap(r => r.trials)) ?? undefined;
+        const allSkillVersions = taskResults.length > 0 ? Object.keys(taskResults[0].skillTrials) : ['local'];
+        const scores = {};
+        const passAtK = {};
+        const assertionPassRate = {};
+        const tokenStats = {};
+        const durationStats = {};
+        for (const version of allSkillVersions) {
+            const { passAtK: vPassAtK } = aggregatePassAtK(taskResults, numTrials, r => r.skillTrials[version] || []);
+            const percentage = Math.round(vPassAtK * 100);
+            scores[version] = `${percentage}%`;
+            passAtK[version] = Math.round(vPassAtK * 1000) / 1000;
+            const vAssertionRate = aggregateAssertionPassRate(taskResults, r => r.skillTrials[version] || []);
+            assertionPassRate[version] = Math.round(vAssertionRate * 1000) / 1000;
+            const vTokens = aggregateTokenStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (vTokens)
+                tokenStats[version] = vTokens;
+            const vDuration = aggregateDurationStats(taskResults.flatMap(r => r.skillTrials[version] || []));
+            if (vDuration)
+                durationStats[version] = vDuration;
+        }
         const report = {
             timestamp: startTime.toISOString(),
             skill_name,
             agent,
             metrics: {
-                withSkillScore: `${percentage}%`,
                 passedCount: tasksPassedCount,
                 totalCount: tasks.length,
                 numTrials,
-                passAtK: Math.round(passAtK * 1000) / 1000,
-                tokenStats: withSkillTokenStats ? { withSkill: withSkillTokenStats } : undefined,
-                durationStats: withSkillDurationStats ? { withSkill: withSkillDurationStats } : undefined
+                scores,
+                passAtK,
+                assertionPassRate,
+                tokenStats,
+                durationStats
             },
             results: taskResults
         };

package/dist/core/environment.js

@@ -11,13 +11,25 @@ export class EvalEnvironment {
     async setup() {
     }
     async teardown() {
-        const worktreesDir = path.resolve(this.workspace, '.project-skill-evals', 'worktrees');
-        if (!fs.existsSync(worktreesDir))
-            return;
-        for (const entry of fs.readdirSync(worktreesDir)) {
-            this.removeWorktree(path.join(worktreesDir, entry));
+        const evalsDir = path.resolve(this.workspace, '.project-skill-evals');
+        // 1. Cleanup Worktrees
+        const worktreesDir = path.join(evalsDir, 'worktrees');
+        if (fs.existsSync(worktreesDir)) {
+            for (const entry of fs.readdirSync(worktreesDir)) {
+                this.removeWorktree(path.join(worktreesDir, entry));
+            }
+            executor.spawnSync('git', ['worktree', 'prune'], { stdio: 'ignore', cwd: this.workspace });
+        }
+        // 2. Cleanup Skill Refs
+        const skillRefsDir = path.join(evalsDir, 'skill-refs');
+        if (fs.existsSync(skillRefsDir)) {
+            try {
+                fs.rmSync(skillRefsDir, { recursive: true, force: true });
+            }
+            catch (err) {
+                Logger.warn(`Failed to remove skill-refs directory at ${skillRefsDir}. Manual cleanup may be required.`);
+            }
         }
-        executor.spawnSync('git', ['worktree', 'prune'], { stdio: 'ignore', cwd: this.workspace });
     }
     /**
      * Creates a temporary git worktree for a specific evaluation.

package/dist/index.js

@@ -40,6 +40,7 @@ program
     .option('--trials <number>', 'Number of trials per task for pass@k calculation')
     .option('--timeout <seconds>', 'Agent timeout in seconds')
     .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
+    .option('--compare-ref [refs...]', 'Compare against historical git references')
     .action((agent, options) => {
     const workspace = path.resolve(options.workspace);
     const selectedAgent = agent || DEFAULT_AGENT;
@@ -47,7 +48,8 @@ program
     const numTrials = options.trials !== undefined ? (parseInt(options.trials, 10) || 3) : 3;
     const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
     const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
-    triggerCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId).catch(errorHandler);
+    const compareRefs = options.compareRef || [];
+    triggerCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId, compareRefs).catch(errorHandler);
 });
 program
     .command('functional [agent]')
@@ -58,6 +60,7 @@ program
     .option('--trials <number>', 'Number of trials per task for pass@k calculation')
     .option('--timeout <seconds>', 'Agent timeout in seconds')
     .option('--eval-id <id>', 'Run only the eval with this ID (numeric)')
+    .option('--compare-ref [refs...]', 'Compare against historical git references')
     .action((agent, options) => {
     const workspace = path.resolve(options.workspace);
     const selectedAgent = agent || DEFAULT_AGENT;
@@ -65,7 +68,8 @@ program
     const numTrials = options.trials !== undefined ? (parseInt(options.trials, 10) || 3) : 3;
     const timeoutMs = options.timeout ? parseInt(options.timeout, 10) * 1000 : undefined;
     const evalId = options.evalId !== undefined ? parseInt(options.evalId, 10) : undefined;
-    functionalCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId).catch(errorHandler);
+    const compareRefs = options.compareRef || [];
+    functionalCommand(selectedAgent, workspace, options.skill, maxAgents, undefined, numTrials, new HtmlReporter(), timeoutMs, evalId, compareRefs).catch(errorHandler);
 });
 const isMain = process.argv[1] && (() => {
     try {

package/dist/reporters/html-reporter.js

@@ -32,7 +32,7 @@ function passColorClass(val) {
     return 'red';
 }
 function isFunctional(report) {
-    return report.metrics.withoutSkillScore !== undefined;
+    return report.metrics.assertionPassRate['baseline'] !== undefined;
 }
 // ---------------------------------------------------------------------------
 // Metrics Grid
@@ -47,71 +47,44 @@ function renderDeltaCell(base, target, format, cellClass) {
     return `<span class="${cellClass} ${cls}">${sign}${pct}%</span>`;
 }
 function renderMetricsGrid(report) {
-    const { metrics } = report;
+    const { metrics, results } = report;
     const functional = isFunctional(report);
-    if (functional) {
-        const bk = metrics.withoutSkillAssertionPassRate ?? metrics.withoutSkillPassAtK ?? 0;
-        const tk = metrics.assertionPassRate ?? metrics.passAtK ?? 0;
-        const upliftRaw = parseInt(metrics.skillUplift ?? '0', 10);
-        const upliftClass = upliftRaw > 0 ? 'green' : upliftRaw < 0 ? 'red' : '';
-        const wo = metrics.tokenStats?.withoutSkill;
-        const wi = metrics.tokenStats?.withSkill;
-        const wod = metrics.durationStats?.withoutSkill;
-        const wid = metrics.durationStats?.withSkill;
-        return `<div class="metrics-grid">
+    const skillVersions = results.length > 0 ? Object.keys(results[0].skillTrials) : ['local'];
+    const allVersions = functional ? ['baseline', ...skillVersions] : skillVersions;
+    const headerCells = allVersions.map(v => `<th>${v}</th>`).join('');
+    const successRows = `<tr>
+    <td>Success Rate</td>
+    ${allVersions.map(v => {
+        const val = metrics.assertionPassRate[v] ?? metrics.passAtK[v] ?? 0;
+        return `<td><span class="metric-val ${passColorClass(val)}">${formatPercent(val)}</span></td>`;
+    }).join('')}
+  </tr>`;
+    const tokenRows = `<tr>
+    <td>Tokens (avg)</td>
+    ${allVersions.map(v => {
+        const stats = metrics.tokenStats?.[v];
+        return `<td>${stats ? `<span class="metric-val">${formatTokens(stats.avgTotal)}</span><div class="metric-sub">avg total</div>` : '<span class="metric-val muted">—</span>'}</td>`;
+    }).join('')}
+  </tr>`;
+    const timeRows = `<tr>
+    <td>Time (avg)</td>
+    ${allVersions.map(v => {
+        const stats = metrics.durationStats?.[v];
+        return `<td>${stats ? `<span class="metric-val">${formatDuration(stats.avgMs)}</span>` : '<span class="metric-val muted">—</span>'}</td>`;
+    }).join('')}
+  </tr>`;
+    return `<div class="metrics-grid">
   <table>
     <thead>
-      <tr><th></th><th>Without Skill</th><th>With Skill</th><th>Delta</th></tr>
+      <tr><th></th>${headerCells}</tr>
     </thead>
     <tbody>
-      <tr>
-        <td>Success Rate</td>
-        <td><span class="metric-val ${passColorClass(bk)}">${formatPercent(bk)}</span></td>
-        <td><span class="metric-val ${passColorClass(tk)}">${formatPercent(tk)}</span></td>
-        <td><span class="metric-val ${upliftClass}">${escapeHtml(metrics.skillUplift ?? '0%')}</span></td>
-      </tr>
-      <tr>
-        <td>Tokens (avg)</td>
-        <td>${wo ? `<span class="metric-val">${formatTokens(wo.avgTotal)}</span><div class="metric-sub">avg total</div>` : '<span class="metric-val muted">—</span>'}</td>
-        <td>${wi ? `<span class="metric-val">${formatTokens(wi.avgTotal)}</span><div class="metric-sub">avg total</div>` : '<span class="metric-val muted">—</span>'}</td>
-        <td>${wo && wi ? renderDeltaCell(wo.avgTotal, wi.avgTotal, formatTokens, 'metric-val') : '<span class="metric-val muted">—</span>'}</td>
-      </tr>
-      <tr>
-        <td>Time (avg)</td>
-        <td>${wod ? `<span class="metric-val">${formatDuration(wod.avgMs)}</span>` : '<span class="metric-val muted">—</span>'}</td>
-        <td>${wid ? `<span class="metric-val">${formatDuration(wid.avgMs)}</span>` : '<span class="metric-val muted">—</span>'}</td>
-        <td>${wod && wid ? renderDeltaCell(wod.avgMs, wid.avgMs, formatDuration, 'metric-val') : '<span class="metric-val muted">—</span>'}</td>
-      </tr>
+      ${successRows}
+      ${tokenRows}
+      ${timeRows}
     </tbody>
   </table>
 </div>`;
-    }
-    else {
-        const k = metrics.passAtK ?? 0;
-        const wi = metrics.tokenStats?.withSkill;
-        const wid = metrics.durationStats?.withSkill;
-        return `<div class="metrics-grid">
-  <table>
-    <thead>
-      <tr><th></th><th>Score</th></tr>
-    </thead>
-    <tbody>
-      <tr>
-        <td>Success Rate</td>
-        <td><span class="metric-val ${passColorClass(k)}">${formatPercent(k)}</span></td>
-      </tr>
-      <tr>
-        <td>Tokens (avg)</td>
-        <td>${wi ? `<span class="metric-val">${formatTokens(wi.avgTotal)}</span><div class="metric-sub">avg total</div>` : '<span class="metric-val muted">—</span>'}</td>
-      </tr>
-      <tr>
-        <td>Time (avg)</td>
-        <td>${wid ? `<span class="metric-val">${formatDuration(wid.avgMs)}</span>` : '<span class="metric-val muted">—</span>'}</td>
-      </tr>
-    </tbody>
-  </table>
-</div>`;
-    }
 }
 // ---------------------------------------------------------------------------
 // Trial details
@@ -160,63 +133,58 @@ function avgTrialDuration(trials) {
         return null;
     return Math.round(withDuration.reduce((s, t) => s + t.durationMs, 0) / withDuration.length);
 }
-function renderTaskMiniGrid(result) {
-    const woTrials = result.withoutSkillTrials ?? [];
-    const wiTrials = result.trials;
-    const bk = woTrials.length ? Math.round(computeAssertionPassRate(woTrials) * 100) : 0;
-    const tk = wiTrials.length ? Math.round(computeAssertionPassRate(wiTrials) * 100) : 0;
-    const rateDelta = tk - bk;
-    const rateDeltaSign = rateDelta >= 0 ? '+' : '';
-    const rateDeltaClass = rateDelta > 0 ? 'green' : rateDelta < 0 ? 'red' : '';
-    const woTokens = avgTrialTokens(woTrials);
-    const wiTokens = avgTrialTokens(wiTrials);
-    const woMs = avgTrialDuration(woTrials);
-    const wiMs = avgTrialDuration(wiTrials);
-    const tokenDeltaCell = (woTokens && wiTokens)
-        ? renderDeltaCell(woTokens, wiTokens, formatTokens, 'metric-val-sm')
-        : '<span class="metric-val-sm muted">—</span>';
-    const durationDeltaCell = (woMs && wiMs)
-        ? renderDeltaCell(woMs, wiMs, formatDuration, 'metric-val-sm')
-        : '<span class="metric-val-sm muted">—</span>';
+function renderTaskMiniGrid(result, isFunctionalEval) {
+    const skillVersions = Object.keys(result.skillTrials);
+    const allVersions = isFunctionalEval ? ['baseline', ...skillVersions] : skillVersions;
+    const headerCells = allVersions.map(v => `<th>${v}</th>`).join('');
+    const successRows = `<tr>
+    <td>Success Rate</td>
+    ${allVersions.map(v => {
+        const trials = v === 'baseline' ? result.baselineTrials : result.skillTrials[v];
+        const rate = trials.length ? Math.round(computeAssertionPassRate(trials) * 100) : 0;
+        return `<td><span class="metric-val-sm ${passColorClass(rate / 100)}">${rate}%</span></td>`;
+    }).join('')}
+  </tr>`;
+    const tokenRows = `<tr>
+    <td>Tokens (avg)</td>
+    ${allVersions.map(v => {
+        const trials = v === 'baseline' ? result.baselineTrials : result.skillTrials[v];
+        const tokens = avgTrialTokens(trials);
+        return `<td>${tokens != null ? `<span class="metric-val-sm">${formatTokens(tokens)}</span>` : '<span class="metric-val-sm muted">—</span>'}</td>`;
+    }).join('')}
+  </tr>`;
+    const timeRows = `<tr>
+    <td>Time (avg)</td>
+    ${allVersions.map(v => {
+        const trials = v === 'baseline' ? result.baselineTrials : result.skillTrials[v];
+        const ms = avgTrialDuration(trials);
+        return `<td>${ms != null ? `<span class="metric-val-sm">${formatDuration(ms)}</span>` : '<span class="metric-val-sm muted">—</span>'}</td>`;
+    }).join('')}
+  </tr>`;
     return `<div class="metrics-grid-sm">
   <table>
     <thead>
-      <tr><th></th><th>Without Skill</th><th>With Skill</th><th>Delta</th></tr>
+      <tr><th></th>${headerCells}</tr>
     </thead>
     <tbody>
-      <tr>
-        <td>Success Rate</td>
-        <td><span class="metric-val-sm ${passColorClass(bk / 100)}">${bk}%</span></td>
-        <td><span class="metric-val-sm ${passColorClass(tk / 100)}">${tk}%</span></td>
-        <td><span class="metric-val-sm ${rateDeltaClass}">${rateDeltaSign}${rateDelta}%</span></td>
-      </tr>
-      <tr>
-        <td>Tokens (avg)</td>
-        <td>${woTokens != null ? `<span class="metric-val-sm">${formatTokens(woTokens)}</span>` : '<span class="metric-val-sm muted">—</span>'}</td>
-        <td>${wiTokens != null ? `<span class="metric-val-sm">${formatTokens(wiTokens)}</span>` : '<span class="metric-val-sm muted">—</span>'}</td>
-        <td>${tokenDeltaCell}</td>
-      </tr>
-      <tr>
-        <td>Time (avg)</td>
-        <td>${woMs != null ? `<span class="metric-val-sm">${formatDuration(woMs)}</span>` : '<span class="metric-val-sm muted">—</span>'}</td>
-        <td>${wiMs != null ? `<span class="metric-val-sm">${formatDuration(wiMs)}</span>` : '<span class="metric-val-sm muted">—</span>'}</td>
-        <td>${durationDeltaCell}</td>
-      </tr>
+      ${successRows}
+      ${tokenRows}
+      ${timeRows}
     </tbody>
   </table>
 </div>`;
 }
 function renderTaskDetails(result, isFunctionalEval) {
     const sections = [];
-    if (isFunctionalEval) {
-        sections.push(renderTaskMiniGrid(result));
+    sections.push(renderTaskMiniGrid(result, isFunctionalEval));
+    if (isFunctionalEval && result.baselineTrials && result.baselineTrials.length > 0) {
+        sections.push('<div class="trial-group-label">Baseline</div>');
+        sections.push(...result.baselineTrials.map(t => renderTrial(t)));
     }
-    if (isFunctionalEval && result.withoutSkillTrials && result.withoutSkillTrials.length > 0) {
-        sections.push('<div class="trial-group-label">Without Skill</div>');
-        sections.push(...result.withoutSkillTrials.map(t => renderTrial(t)));
-        sections.push('<div class="trial-group-label">With Skill</div>');
+    for (const version of Object.keys(result.skillTrials)) {
+        sections.push(`<div class="trial-group-label">${version}</div>`);
+        sections.push(...result.skillTrials[version].map(t => renderTrial(t)));
     }
-    sections.push(...result.trials.map(t => renderTrial(t)));
     return `<div class="task-details" id="details-${result.taskId}">${sections.join('')}</div>`;
 }
 // ---------------------------------------------------------------------------
@@ -241,7 +209,7 @@ export function generateHtml(report) {
     const { skill_name, agent, timestamp, metrics } = report;
     const functional = isFunctional(report);
     const evalType = functional ? 'Functional' : 'Trigger';
-    const overallScore = metrics.passAtK ?? 0;
+    const overallScore = metrics.passAtK['local'] ?? 0;
     const statusClass = passColorClass(overallScore);
     const formattedDate = new Date(timestamp).toLocaleString();
     return `<!DOCTYPE html>

package/dist/utils/exec.js

@@ -1,9 +1,16 @@
 import { execSync, spawnSync } from 'child_process';
 export const executor = {
     execSync(command, options) {
-        return execSync(command, options);
+        return execSync(command, {
+            ...options,
+            shell: process.env.SHELL || '/bin/sh',
+            env: { ...process.env, ...options?.env }
+        });
     },
     spawnSync(command, args, options) {
-        return spawnSync(command, args, options);
+        return spawnSync(command, args, {
+            ...options,
+            env: { ...process.env, ...options?.env }
+        });
     }
 };

package/dist/utils/git.js

@@ -0,0 +1,30 @@
+import path from 'path';
+import fs from 'fs';
+import { executor } from './exec.js';
+import { ExecutionError } from '../core/errors.js';
+export const git = {
+    extractSkillRef(skillPath, ref, targetDir) {
+        // 1. Identify repo root
+        let repoRoot;
+        try {
+            repoRoot = executor.execSync('git rev-parse --show-toplevel', { cwd: skillPath }).toString().trim();
+        }
+        catch (err) {
+            throw new ExecutionError(`Path is not inside a git repository: ${skillPath}`);
+        }
+        // 2. Resolve relative path of skill within repo
+        const relativeSkillPath = path.relative(repoRoot, path.resolve(skillPath));
+        // 3. Create target directory
+        fs.mkdirSync(targetDir, { recursive: true });
+        // 4. Extract via git archive
+        try {
+            const cmd = `git archive ${ref} | tar -x -C ${targetDir}`;
+            executor.execSync(cmd, { cwd: repoRoot });
+        }
+        catch (err) {
+            throw new ExecutionError(`Failed to extract git reference '${ref}': ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+};
+// Also keep named export for backward compatibility if needed
+export const extractSkillRef = git.extractSkillRef;

package/dist/utils/table-renderer.js

@@ -126,42 +126,60 @@ export function renderRunHeader(config) {
 export function renderTriggerTable(report) {
     const { results, metrics } = report;
     const numTrials = metrics.numTrials || 1;
-    const tableData = numTrials > 1
-        ? [['ID', 'Prompt', 'Trials', 'success rate']]
-        : [['ID', 'Prompt', 'success rate']];
+    // Identify all skill versions present in the results
+    const skillVersions = results.length > 0 ? Object.keys(results[0].skillTrials) : ['local'];
+    const header = ['ID', 'Prompt'];
+    if (numTrials > 1) {
+        for (const version of skillVersions) {
+            header.push(`${version} Trials`, `${version} Rate`);
+        }
+    }
+    else {
+        for (const version of skillVersions) {
+            header.push(`${version} Rate`);
+        }
+    }
+    const tableData = [header];
     let hasPartialErrors = false;
     for (const result of results) {
         const promptSnippet = result.prompt.substring(0, 40) + (result.prompt.length > 40 ? '...' : '');
-        const p1Cell = formatPassAt1(result.trials);
-        const someError = result.trials.some(t => t.isError);
-        const allError = result.trials.length > 0 && result.trials.every(t => t.isError);
-        if (someError && !allError)
-            hasPartialErrors = true;
-        if (numTrials > 1) {
-            const errorCount = result.trials.filter(t => t.isError).length;
-            const passedCount = result.trials.filter(t => t.trialPassed).length;
-            const trialsBase = `${passedCount}/${result.trials.length}`;
-            const trialsStr = errorCount > 0 ? `${trialsBase} (${errorCount}!)` : trialsBase;
-            const trials = result.score === 1.0 ? chalk.green(trialsStr) : errorCount > 0 ? chalk.yellow(trialsStr) : chalk.red(trialsStr);
-            tableData.push([result.taskId.toString(), promptSnippet, trials, p1Cell]);
-        }
-        else {
-            tableData.push([result.taskId.toString(), promptSnippet, p1Cell]);
+        const row = [result.taskId.toString(), promptSnippet];
+        for (const version of skillVersions) {
+            const trials = result.skillTrials[version] || [];
+            const p1Cell = formatPassAt1(trials);
+            const someError = trials.some(t => t.isError);
+            const allError = trials.length > 0 && trials.every(t => t.isError);
+            if (someError && !allError)
+                hasPartialErrors = true;
+            if (numTrials > 1) {
+                const errorCount = trials.filter(t => t.isError).length;
+                const passedCount = trials.filter(t => t.trialPassed).length;
+                const trialsBase = `${passedCount}/${trials.length}`;
+                const trialsStr = errorCount > 0 ? `${trialsBase} (${errorCount}!)` : trialsBase;
+                const trialsCell = passedCount === trials.length ? chalk.green(trialsStr) : errorCount > 0 ? chalk.yellow(trialsStr) : chalk.red(trialsStr);
+                row.push(trialsCell, p1Cell);
+            }
+            else {
+                row.push(p1Cell);
+            }
         }
+        tableData.push(row);
     }
     Logger.table(tableData);
     if (hasPartialErrors) {
         Logger.write(chalk.yellow('\n   * Some trials did not complete due to infrastructure errors. success rate is computed over the trials that ran.'));
     }
-    const percentage = Math.round((metrics.passAtK || 0) * 100);
-    Logger.write(`\n   Trigger Success Rate:   ${percentage}%`);
-    const withSkillTokenStats = metrics.tokenStats?.withSkill;
-    if (withSkillTokenStats) {
-        Logger.write(`\n   Avg Tokens:             ${formatTokenStatsLine(withSkillTokenStats)}`);
-    }
-    const withSkillDurationStats = metrics.durationStats?.withSkill;
-    if (withSkillDurationStats) {
-        Logger.write(`\n   Avg Time:               ${formatDuration(withSkillDurationStats.avgMs)}`);
+    for (const version of skillVersions) {
+        const percentage = Math.round((metrics.passAtK[version] || 0) * 100);
+        Logger.write(`\n   ${version} Success Rate:   ${percentage}%`);
+        const tokenStats = metrics.tokenStats?.[version];
+        if (tokenStats) {
+            Logger.write(`\n   Avg Tokens (${version}):   ${formatTokenStatsLine(tokenStats)}`);
+        }
+        const durationStats = metrics.durationStats?.[version];
+        if (durationStats) {
+            Logger.write(`\n   Avg Time (${version}):     ${formatDuration(durationStats.avgMs)}`);
+        }
     }
 }
 /**
@@ -169,61 +187,46 @@ export function renderTriggerTable(report) {
  */
 export function renderFunctionalTable(report) {
     const { results, metrics } = report;
-    const tableData = [['ID', 'Prompt', 'Without Skill', 'With Skill']];
+    // Identify all versions present (baseline + skill versions)
+    const skillVersions = results.length > 0 ? Object.keys(results[0].skillTrials) : ['local'];
+    const allVersions = ['baseline', ...skillVersions];
+    const header = ['ID', 'Prompt'];
+    for (const version of allVersions) {
+        header.push(version);
+    }
+    const tableData = [header];
     let hasPartialErrors = false;
     for (const result of results) {
         const promptSnippet = result.prompt.substring(0, 40) + (result.prompt.length > 40 ? '...' : '');
-        const withoutSkillTrials = result.withoutSkillTrials || [];
-        const withSkillTrials = result.trials;
-        const baselineSomeError = withoutSkillTrials.some(t => t.isError);
-        const baselineAllError = withoutSkillTrials.length > 0 && withoutSkillTrials.every(t => t.isError);
-        const withSkillSomeError = withSkillTrials.some(t => t.isError);
-        const withSkillAllError = withSkillTrials.length > 0 && withSkillTrials.every(t => t.isError);
-        if ((baselineSomeError && !baselineAllError) || (withSkillSomeError && !withSkillAllError))
+        const row = [result.taskId.toString(), promptSnippet];
+        // Baseline
+        const woTrials = result.baselineTrials || [];
+        if (woTrials.some(t => t.isError) && !woTrials.every(t => t.isError))
             hasPartialErrors = true;
-        tableData.push([
-            result.taskId.toString(),
-            promptSnippet,
-            formatAssertionRate(withoutSkillTrials),
-            formatAssertionRate(withSkillTrials),
-        ]);
+        row.push(formatAssertionRate(woTrials));
+        // Skills
+        for (const version of skillVersions) {
+            const wiTrials = result.skillTrials[version] || [];
+            if (wiTrials.some(t => t.isError) && !wiTrials.every(t => t.isError))
+                hasPartialErrors = true;
+            row.push(formatAssertionRate(wiTrials));
+        }
+        tableData.push(row);
     }
     Logger.table(tableData);
     if (hasPartialErrors) {
         Logger.write(chalk.yellow('\n   * Some trials did not complete due to infrastructure errors. success rate is computed over the trials that ran.'));
     }
-    const withoutSkillPercentage = Math.round(((metrics.withoutSkillAssertionPassRate ?? metrics.withoutSkillPassAtK) || 0) * 100);
-    const withSkillPercentage = Math.round(((metrics.assertionPassRate ?? metrics.passAtK) || 0) * 100);
-    Logger.write(`\n   Without Skill Rate:   ${withoutSkillPercentage}%`);
-    Logger.write(`\n   With Skill Rate:      ${withSkillPercentage}%`);
-    const baselineTokenStats = metrics.tokenStats?.withoutSkill;
-    const withSkillTokenStats = metrics.tokenStats?.withSkill;
-    if (baselineTokenStats || withSkillTokenStats) {
-        if (baselineTokenStats)
-            Logger.write(`\n   Tokens (w/o skill):   ${formatTokenStatsLine(baselineTokenStats)}`);
-        if (withSkillTokenStats)
-            Logger.write(`\n   Tokens (w/ skill):    ${formatTokenStatsLine(withSkillTokenStats)}`);
-        if (baselineTokenStats && withSkillTokenStats && baselineTokenStats.avgTotal > 0) {
-            const delta = withSkillTokenStats.avgTotal - baselineTokenStats.avgTotal;
-            const deltaSign = delta >= 0 ? '+' : '';
-            const deltaPct = Math.round((delta / baselineTokenStats.avgTotal) * 100);
-            const deltaStr = `${deltaSign}${formatTokens(Math.abs(delta))} (${deltaSign}${deltaPct}%)`;
-            Logger.write(`\n   Token Delta:          ${delta >= 0 ? chalk.yellow(deltaStr) : chalk.green(deltaStr)}`);
+    for (const version of allVersions) {
+        const rate = Math.round(((metrics.assertionPassRate[version] ?? metrics.passAtK[version]) || 0) * 100);
+        Logger.write(`\n   ${version} Rate:   ${rate}%`);
+        const stats = metrics.tokenStats?.[version];
+        if (stats) {
+            Logger.write(`\n   Tokens (${version}):   ${formatTokenStatsLine(stats)}`);
         }
-    }
-    const baselineDurationStats = metrics.durationStats?.withoutSkill;
-    const withSkillDurationStats = metrics.durationStats?.withSkill;
-    if (baselineDurationStats || withSkillDurationStats) {
-        if (baselineDurationStats)
-            Logger.write(`\n   Time (w/o skill):     ${formatDuration(baselineDurationStats.avgMs)} avg`);
-        if (withSkillDurationStats)
-            Logger.write(`\n   Time (w/ skill):      ${formatDuration(withSkillDurationStats.avgMs)} avg`);
-        if (baselineDurationStats && withSkillDurationStats && baselineDurationStats.avgMs > 0) {
-            const delta = withSkillDurationStats.avgMs - baselineDurationStats.avgMs;
-            const deltaSign = delta >= 0 ? '+' : '';
-            const deltaPct = Math.round((delta / baselineDurationStats.avgMs) * 100);
-            const deltaStr = `${deltaSign}${formatDuration(Math.abs(delta))} (${deltaSign}${deltaPct}%)`;
-            Logger.write(`\n   Time Delta:           ${delta >= 0 ? chalk.yellow(deltaStr) : chalk.green(deltaStr)}`);
+        const dStats = metrics.durationStats?.[version];
+        if (dStats) {
+            Logger.write(`\n   Time (${version}):     ${formatDuration(dStats.avgMs)} avg`);
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fede0089/skill-eval",
-  "version": "1.0.6",
+  "version": "1.2.0",
   "description": "CLI to evaluate agent skills triggering",
   "main": "dist/index.js",
   "bin": {