@su-record/vibe 2.7.9 → 2.7.11

package/dist/infra/lib/evolution/SkillBenchmark.js

@@ -0,0 +1,233 @@
+// Skill Benchmark - Phase 5: Aggregate eval results into benchmarks
+// Tracks pass rate, time, tokens across iterations
+//
+// Produces benchmark reports for comparing:
+// - with-skill vs baseline performance
+// - Current iteration vs previous iterations
+// - Per-eval and per-assertion breakdowns
+import { randomUUID } from 'crypto';
+import { SkillEvalRunner } from './SkillEvalRunner.js';
+export class SkillBenchmark {
+    db;
+    evalRunner;
+    constructor(storage) {
+        this.db = storage.getDatabase();
+        this.evalRunner = new SkillEvalRunner(storage);
+        this.initializeTables();
+    }
+    initializeTables() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS skill_benchmarks (
+        id TEXT PRIMARY KEY,
+        skillName TEXT NOT NULL,
+        iteration INTEGER NOT NULL,
+        summary TEXT NOT NULL,
+        evalBreakdowns TEXT NOT NULL,
+        createdAt TEXT NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_sb_skill ON skill_benchmarks(skillName);
+      CREATE INDEX IF NOT EXISTS idx_sb_iter ON skill_benchmarks(skillName, iteration);
+    `);
+    }
+    /**
+     * Aggregate latest eval runs into a benchmark
+     */
+    aggregate(skillName) {
+        const latestRuns = this.evalRunner.getLatestRuns(skillName);
+        const evalCases = this.evalRunner.getEvalCases(skillName);
+        const iteration = this.getNextIteration(skillName);
+        const withSkillRuns = [];
+        const baselineRuns = [];
+        const breakdowns = [];
+        for (const evalCase of evalCases) {
+            const runPair = latestRuns.get(evalCase.id);
+            const ws = runPair?.withSkill ?? null;
+            const bl = runPair?.baseline ?? null;
+            if (ws)
+                withSkillRuns.push(ws);
+            if (bl)
+                baselineRuns.push(bl);
+            breakdowns.push({
+                evalId: evalCase.id,
+                prompt: evalCase.prompt,
+                withSkillPassed: ws?.status === 'passed',
+                baselinePassed: bl?.status === 'passed',
+                withSkillDurationMs: ws?.durationMs ?? 0,
+                baselineDurationMs: bl?.durationMs ?? 0,
+                withSkillTokens: ws?.tokenCount ?? 0,
+                baselineTokens: bl?.tokenCount ?? 0,
+                assertionResults: this.mergeAssertionResults(ws, bl),
+            });
+        }
+        const withSkillStats = this.computeVariantStats(withSkillRuns);
+        const baselineStats = this.computeVariantStats(baselineRuns);
+        const summary = {
+            totalEvals: evalCases.length,
+            withSkill: withSkillStats,
+            baseline: baselineStats,
+            delta: {
+                passRateDelta: withSkillStats.passRate - baselineStats.passRate,
+                durationDeltaMs: withSkillStats.meanDurationMs - baselineStats.meanDurationMs,
+                tokenDelta: withSkillStats.meanTokens - baselineStats.meanTokens,
+            },
+        };
+        const id = `bench-${Date.now().toString(36)}-${randomUUID().replace(/-/g, '').slice(0, 8)}`;
+        const now = new Date().toISOString();
+        this.db.prepare(`
+      INSERT INTO skill_benchmarks (id, skillName, iteration, summary, evalBreakdowns, createdAt)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `).run(id, skillName, iteration, JSON.stringify(summary), JSON.stringify(breakdowns), now);
+        return { id, skillName, iteration, timestamp: now, summary, evalBreakdowns: breakdowns };
+    }
+    /**
+     * Get benchmark history for a skill
+     */
+    getHistory(skillName) {
+        const rows = this.db.prepare(`
+      SELECT * FROM skill_benchmarks WHERE skillName = ? ORDER BY iteration ASC
+    `).all(skillName);
+        return rows.map(this.rowToBenchmark);
+    }
+    /**
+     * Get the latest benchmark for a skill
+     */
+    getLatest(skillName) {
+        const row = this.db.prepare(`
+      SELECT * FROM skill_benchmarks WHERE skillName = ? ORDER BY iteration DESC LIMIT 1
+    `).get(skillName);
+        return row ? this.rowToBenchmark(row) : null;
+    }
+    /**
+     * Compare two benchmark iterations
+     */
+    compare(skillName, iterA, iterB) {
+        const a = this.getBenchmarkByIteration(skillName, iterA);
+        const b = this.getBenchmarkByIteration(skillName, iterB);
+        if (!a || !b) {
+            return { iterationA: a, iterationB: b, improvement: null };
+        }
+        return {
+            iterationA: a,
+            iterationB: b,
+            improvement: {
+                passRateDelta: b.summary.withSkill.passRate - a.summary.withSkill.passRate,
+                durationDeltaMs: b.summary.withSkill.meanDurationMs - a.summary.withSkill.meanDurationMs,
+                tokenDelta: b.summary.withSkill.meanTokens - a.summary.withSkill.meanTokens,
+            },
+        };
+    }
+    /**
+     * Format benchmark as markdown report
+     */
+    formatReport(benchmark) {
+        const { summary } = benchmark;
+        const lines = [
+            `# Benchmark: ${benchmark.skillName} (Iteration ${benchmark.iteration})`,
+            '',
+            `**Date**: ${benchmark.timestamp}`,
+            '',
+            '## Summary',
+            '',
+            '| Metric | With Skill | Baseline | Delta |',
+            '|--------|-----------|----------|-------|',
+            `| Pass Rate | ${this.pct(summary.withSkill.passRate)} | ${this.pct(summary.baseline.passRate)} | ${this.signedPct(summary.delta.passRateDelta)} |`,
+            `| Mean Duration | ${summary.withSkill.meanDurationMs.toFixed(0)}ms | ${summary.baseline.meanDurationMs.toFixed(0)}ms | ${summary.delta.durationDeltaMs > 0 ? '+' : ''}${summary.delta.durationDeltaMs.toFixed(0)}ms |`,
+            `| Mean Tokens | ${summary.withSkill.meanTokens.toFixed(0)} | ${summary.baseline.meanTokens.toFixed(0)} | ${summary.delta.tokenDelta > 0 ? '+' : ''}${summary.delta.tokenDelta.toFixed(0)} |`,
+            '',
+            '## Per-Eval Breakdown',
+            '',
+        ];
+        for (const bd of benchmark.evalBreakdowns) {
+            const wsIcon = bd.withSkillPassed ? 'PASS' : 'FAIL';
+            const blIcon = bd.baselinePassed ? 'PASS' : 'FAIL';
+            lines.push(`### ${bd.evalId}`);
+            lines.push(`- **Prompt**: ${bd.prompt.slice(0, 80)}${bd.prompt.length > 80 ? '...' : ''}`);
+            lines.push(`- **With Skill**: ${wsIcon} (${bd.withSkillDurationMs}ms, ${bd.withSkillTokens} tokens)`);
+            lines.push(`- **Baseline**: ${blIcon} (${bd.baselineDurationMs}ms, ${bd.baselineTokens} tokens)`);
+            if (bd.assertionResults.length > 0) {
+                lines.push('- **Assertions**:');
+                for (const ar of bd.assertionResults) {
+                    const wsA = ar.withSkillPassed ? 'PASS' : 'FAIL';
+                    const blA = ar.baselinePassed ? 'PASS' : 'FAIL';
+                    lines.push(`  - ${ar.description}: skill=${wsA}, baseline=${blA}`);
+                }
+            }
+            lines.push('');
+        }
+        return lines.join('\n');
+    }
+    getBenchmarkByIteration(skillName, iteration) {
+        const row = this.db.prepare(`
+      SELECT * FROM skill_benchmarks WHERE skillName = ? AND iteration = ?
+    `).get(skillName, iteration);
+        return row ? this.rowToBenchmark(row) : null;
+    }
+    getNextIteration(skillName) {
+        const row = this.db.prepare(`
+      SELECT MAX(iteration) as maxIter FROM skill_benchmarks WHERE skillName = ?
+    `).get(skillName);
+        return (row.maxIter ?? 0) + 1;
+    }
+    computeVariantStats(runs) {
+        if (runs.length === 0) {
+            return { passRate: 0, meanDurationMs: 0, stddevDurationMs: 0, meanTokens: 0, stddevTokens: 0, totalRuns: 0 };
+        }
+        const passed = runs.filter(r => r.status === 'passed').length;
+        const durations = runs.map(r => r.durationMs);
+        const tokens = runs.map(r => r.tokenCount);
+        return {
+            passRate: passed / runs.length,
+            meanDurationMs: mean(durations),
+            stddevDurationMs: stddev(durations),
+            meanTokens: mean(tokens),
+            stddevTokens: stddev(tokens),
+            totalRuns: runs.length,
+        };
+    }
+    mergeAssertionResults(ws, bl) {
+        const wsGrades = ws?.grades ?? [];
+        const blGrades = bl?.grades ?? [];
+        // Use ws assertions as base, merge baseline grades
+        const allDescriptions = new Set([
+            ...wsGrades.map(g => g.description),
+            ...blGrades.map(g => g.description),
+        ]);
+        return Array.from(allDescriptions).map(desc => ({
+            description: desc,
+            withSkillPassed: wsGrades.find(g => g.description === desc)?.passed ?? false,
+            baselinePassed: blGrades.find(g => g.description === desc)?.passed ?? false,
+        }));
+    }
+    pct(value) {
+        return `${(value * 100).toFixed(1)}%`;
+    }
+    signedPct(value) {
+        const sign = value > 0 ? '+' : '';
+        return `${sign}${(value * 100).toFixed(1)}%`;
+    }
+    rowToBenchmark(row) {
+        return {
+            id: row.id,
+            skillName: row.skillName,
+            iteration: row.iteration,
+            timestamp: row.createdAt,
+            summary: JSON.parse(row.summary),
+            evalBreakdowns: JSON.parse(row.evalBreakdowns),
+        };
+    }
+}
+// --- Utility functions ---
+function mean(values) {
+    if (values.length === 0)
+        return 0;
+    return values.reduce((a, b) => a + b, 0) / values.length;
+}
+function stddev(values) {
+    if (values.length < 2)
+        return 0;
+    const avg = mean(values);
+    const variance = values.reduce((sum, v) => sum + (v - avg) ** 2, 0) / (values.length - 1);
+    return Math.sqrt(variance);
+}
+//# sourceMappingURL=SkillBenchmark.js.map

package/dist/infra/lib/evolution/SkillBenchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SkillBenchmark.js","sourceRoot":"","sources":["../../../../src/infra/lib/evolution/SkillBenchmark.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,mDAAmD;AACnD,EAAE;AACF,4CAA4C;AAC5C,uCAAuC;AACvC,6CAA6C;AAC7C,0CAA0C;AAE1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,OAAO,EAAE,eAAe,EAAiB,MAAM,sBAAsB,CAAC;AA0DtE,MAAM,OAAO,cAAc;IACjB,EAAE,CAA2C;IAC7C,UAAU,CAAkB;IAEpC,YAAY,OAAsB;QAChC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC1B,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;KAYZ,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACI,SAAS,CAAC,SAAiB;QAChC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAEnD,MAAM,aAAa,GAAoB,EAAE,CAAC;QAC1C,MAAM,YAAY,GAAoB,EAAE,CAAC;QACzC,MAAM,UAAU,GAAoB,EAAE,CAAC;QAEvC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC5C,MAAM,EAAE,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC;YACtC,MAAM,EAAE,GAAG,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC;YAErC,IAAI,EAAE;gBAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,IAAI,EAAE;gBAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE9B,UAAU,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,eAAe,EAAE,EAAE,EAAE,MAAM,KAAK,QAAQ;gBACxC,cAAc,EAAE,EAAE,EAAE,MAAM,KAAK,QAAQ;gBACvC,mBAAmB,EAAE,EAAE,EAAE,UAAU,IAAI,CAAC;gBACxC,kBAAkB,EAAE,EAAE,EAAE,UAAU,IAAI,CAAC;gBACvC,eAAe,EAAE,EAAE,EAAE,UAAU,IAAI,CAAC;gBACpC,cAAc,EAAE,EAAE,EAAE,UAAU,IAAI,CAAC;gBACnC,gBAAgB,EAAE,IAAI,CAAC,qBAAqB,CAAC,EAAE,EAAE,EAAE,CAAC;aACrD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;QAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAqB;YAChC,UAAU,EAAE,SAAS,CAAC,MAAM;YAC5B,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,aAAa;YACvB,KAAK,EAAE;gBACL,aAAa,EAAE,cAAc,CAAC,QAAQ,GAAG,aAAa,CAAC,QAAQ;gBAC/D,eAAe,EAAE,cAAc,CAAC,cAAc,GAAG,aAAa,CAAC,cAAc;gBAC7E,UAAU,EAAE,cAAc,CAAC,UAAU,GAAG,aAAa,CAAC,UAAU;aACjE;SACF,CAAC;QAEF,MAAM,EAAE,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC5F,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAErC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAGf,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC;QAE3F,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;IAC3F,CAAC;IAED;;OAEG;IACI,UAAU,CAAC,SAAiB;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE5B,CAAC,CAAC,GAAG,CAAC,SAAS,CAAmB,CAAC;QACpC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACI,SAAS,CAAC,SAAiB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE3B,CAAC,CAAC,GAAG,CAAC,SAAS,CAA6B,CAAC;QAC9C,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/C,CAAC;IAED;;OAEG;IACI,OAAO,CAAC,SAAiB,EAAE,KAAa,EAAE,KAAa;QAK5D,MAAM,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACzD,MAAM,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAEzD,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACb,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC7D,CAAC;QAED,OAAO;YACL,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,CAAC;YACb,WAAW,EAAE;gBACX,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ;gBAC1E,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc;gBACxF,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU;aAC5E;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACI,YAAY,CAAC,SAA0B;QAC5C,MAAM,EAAE,OAAO,EAAE,GAAG,SAAS,CAAC;QAC9B,MAAM,KAAK,GAAa;YACtB,gBAAgB,SAAS,CAAC,SAAS,eAAe,SAAS,CAAC,SAAS,GAAG;YACxE,EAAE;YACF,aAAa,SAAS,CAAC,SAAS,EAAE;YAClC,EAAE;YACF,YAAY;YACZ,EAAE;YACF,4CAA4C;YAC5C,2CAA2C;YAC3C,iBAAiB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI;YACnJ,qBAAqB,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,OAAO,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YACvN,mBAAmB,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;YAC7L,EAAE;YACF,uBAAuB;YACvB,EAAE;SACH,CAAC;QAEF,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACpD,MAAM,MAAM,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACnD,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3F,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,KAAK,EAAE,CAAC,mBAAmB,OAAO,EAAE,CAAC,eAAe,UAAU,CAAC,CAAC;YACtG,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,KAAK,EAAE,CAAC,kBAAkB,OAAO,EAAE,CAAC,cAAc,UAAU,CAAC,CAAC;YAElG,IAAI,EAAE,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBAChC,KAAK,MAAM,EAAE,IAAI,EAAE,CAAC,gBAAgB,EAAE,CAAC;oBACrC,MAAM,GAAG,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;oBACjD,MAAM,GAAG,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,WAAW,WAAW,GAAG,cAAc,GAAG,EAAE,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAEO,uBAAuB,CAAC,SAAiB,EAAE,SAAiB;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE3B,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAA6B,CAAC;QACzD,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/C,CAAC;IAEO,gBAAgB,CAAC,SAAiB;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE3B,CAAC,CAAC,GAAG,CAAC,SAAS,CAA+B,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEO,mBAAmB,CAAC,IAAqB;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QAC/G,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAE3C,OAAO;YACL,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM;YAC9B,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/B,gBAAgB,EAAE,MAAM,CAAC,SAAS,CAAC;YACnC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC;YACxB,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC;YAC5B,SAAS,EAAE,IAAI,CAAC,MAAM;SACvB,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAC3B,EAAwB,EACxB,EAAwB;QAExB,MAAM,QAAQ,GAAG,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC;QAElC,mDAAmD;QACnD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;YAC9B,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;YACnC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;SACpC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9C,WAAW,EAAE,IAAI;YACjB,eAAe,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC,EAAE,MAAM,IAAI,KAAK;YAC5E,cAAc,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC,EAAE,MAAM,IAAI,KAAK;SAC5E,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,GAAG,CAAC,KAAa;QACvB,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACxC,CAAC;IAEO,SAAS,CAAC,KAAa;QAC7B,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC/C,CAAC;IAEO,cAAc,CAAC,GAAiB;QACtC,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;YAChC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC;SAC/C,CAAC;IACJ,CAAC;CACF;AAED,4BAA4B;AAE5B,SAAS,IAAI,CAAC,MAAgB;IAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC3D,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1F,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC"}

package/dist/infra/lib/evolution/SkillClassifier.d.ts

@@ -0,0 +1,35 @@
+import { MemoryStorage } from '../memory/MemoryStorage.js';
+export type SkillCategory = 'capability_uplift' | 'encoded_preference' | 'unknown';
+export interface ClassificationResult {
+    skillName: string;
+    category: SkillCategory;
+    confidence: number;
+    reasoning: string;
+    baselinePassRate: number;
+    withSkillPassRate: number;
+    trend: 'converging' | 'stable' | 'diverging' | 'insufficient_data';
+    recommendation: string;
+}
+export declare class SkillClassifier {
+    private benchmark;
+    constructor(storage: MemoryStorage);
+    /**
+     * Classify a skill based on benchmark history
+     */
+    classify(skillName: string): ClassificationResult;
+    /**
+     * Classify based on explicit pass rates (no DB lookup)
+     */
+    classifyFromRates(skillName: string, withSkillPassRate: number, baselinePassRate: number, trend?: ClassificationResult['trend']): ClassificationResult;
+    /**
+     * Check if a skill is becoming obsolete (capability uplift that model now handles)
+     */
+    isBecomingObsolete(skillName: string): {
+        obsolete: boolean;
+        reason: string;
+    };
+    private determineCategory;
+    private computeTrend;
+    private pct;
+}
+//# sourceMappingURL=SkillClassifier.d.ts.map

package/dist/infra/lib/evolution/SkillClassifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SkillClassifier.d.ts","sourceRoot":"","sources":["../../../../src/infra/lib/evolution/SkillClassifier.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAG3D,MAAM,MAAM,aAAa,GAAG,mBAAmB,GAAG,oBAAoB,GAAG,SAAS,CAAC;AAEnF,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,aAAa,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,mBAAmB,CAAC;IACnE,cAAc,EAAE,MAAM,CAAC;CACxB;AAUD,qBAAa,eAAe;IAC1B,OAAO,CAAC,SAAS,CAAiB;gBAEtB,OAAO,EAAE,aAAa;IAIlC;;OAEG;IACI,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,oBAAoB;IAuBxD;;OAEG;IACI,iBAAiB,CACtB,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,EACzB,gBAAgB,EAAE,MAAM,EACxB,KAAK,GAAE,oBAAoB,CAAC,OAAO,CAAuB,GACzD,oBAAoB;IAIvB;;OAEG;IACI,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;IAoBnF,OAAO,CAAC,iBAAiB;IAmFzB,OAAO,CAAC,YAAY;IA6BpB,OAAO,CAAC,GAAG;CAGZ"}

package/dist/infra/lib/evolution/SkillClassifier.js

@@ -0,0 +1,167 @@
+// Skill Classifier - Phase 5: Capability Uplift vs Encoded Preference
+//
+// Two types of skills (from Anthropic's taxonomy):
+//
+// 1. Capability Uplift: Compensates for what the model can't do well.
+//    - Becomes obsolete as models improve
+//    - Eval: if baseline (no skill) starts passing, the skill has served its purpose
+//
+// 2. Encoded Preference: Encodes team-specific workflow, style, or process.
+//    - Durable regardless of model improvements
+//    - Eval: baseline will never pass because the model can't know your preferences
+import { SkillBenchmark } from './SkillBenchmark.js';
+/**
+ * Thresholds for classification
+ */
+const BASELINE_HIGH_THRESHOLD = 0.7;
+const BASELINE_LOW_THRESHOLD = 0.3;
+const CONVERGENCE_THRESHOLD = 0.15;
+const MIN_BENCHMARKS_FOR_TREND = 2;
+export class SkillClassifier {
+    benchmark;
+    constructor(storage) {
+        this.benchmark = new SkillBenchmark(storage);
+    }
+    /**
+     * Classify a skill based on benchmark history
+     */
+    classify(skillName) {
+        const history = this.benchmark.getHistory(skillName);
+        if (history.length === 0) {
+            return {
+                skillName,
+                category: 'unknown',
+                confidence: 0,
+                reasoning: 'No benchmark data available. Run evals first.',
+                baselinePassRate: 0,
+                withSkillPassRate: 0,
+                trend: 'insufficient_data',
+                recommendation: 'Create eval cases and run benchmarks to classify this skill.',
+            };
+        }
+        const latest = history[history.length - 1];
+        const { withSkill, baseline } = latest.summary;
+        const trend = this.computeTrend(history);
+        return this.determineCategory(skillName, withSkill.passRate, baseline.passRate, trend, history.length);
+    }
+    /**
+     * Classify based on explicit pass rates (no DB lookup)
+     */
+    classifyFromRates(skillName, withSkillPassRate, baselinePassRate, trend = 'insufficient_data') {
+        return this.determineCategory(skillName, withSkillPassRate, baselinePassRate, trend, 1);
+    }
+    /**
+     * Check if a skill is becoming obsolete (capability uplift that model now handles)
+     */
+    isBecomingObsolete(skillName) {
+        const result = this.classify(skillName);
+        if (result.category === 'capability_uplift' && result.trend === 'converging') {
+            return {
+                obsolete: true,
+                reason: `Baseline pass rate (${this.pct(result.baselinePassRate)}) is converging with skill pass rate (${this.pct(result.withSkillPassRate)}). The model may now handle this without the skill.`,
+            };
+        }
+        if (result.baselinePassRate >= BASELINE_HIGH_THRESHOLD) {
+            return {
+                obsolete: true,
+                reason: `Baseline pass rate is ${this.pct(result.baselinePassRate)} — the model handles this well without the skill.`,
+            };
+        }
+        return { obsolete: false, reason: 'Skill still provides significant value.' };
+    }
+    determineCategory(skillName, wsRate, blRate, trend, benchmarkCount) {
+        const gap = wsRate - blRate;
+        // Case 1: Baseline already performs well → capability uplift (possibly obsolete)
+        if (blRate >= BASELINE_HIGH_THRESHOLD) {
+            return {
+                skillName,
+                category: 'capability_uplift',
+                confidence: Math.min(0.9, 0.5 + blRate * 0.4),
+                reasoning: `Baseline pass rate is high (${this.pct(blRate)}), indicating the model can handle this well without the skill. This is a capability uplift skill that may be nearing obsolescence.`,
+                baselinePassRate: blRate,
+                withSkillPassRate: wsRate,
+                trend,
+                recommendation: gap < CONVERGENCE_THRESHOLD
+                    ? 'Consider retiring this skill — the model handles this natively now.'
+                    : 'Monitor baseline trend. The skill still adds some value.',
+            };
+        }
+        // Case 2: Baseline performs poorly and skill helps a lot → likely encoded preference
+        if (blRate <= BASELINE_LOW_THRESHOLD && gap > CONVERGENCE_THRESHOLD) {
+            const isEncoded = trend === 'stable' || trend === 'diverging' || benchmarkCount < MIN_BENCHMARKS_FOR_TREND;
+            return {
+                skillName,
+                category: isEncoded ? 'encoded_preference' : 'capability_uplift',
+                confidence: isEncoded ? Math.min(0.85, 0.4 + gap * 0.5) : 0.5,
+                reasoning: isEncoded
+                    ? `Low baseline (${this.pct(blRate)}) with stable gap suggests team-specific preferences the model cannot infer.`
+                    : `Low baseline (${this.pct(blRate)}) but converging trend suggests model capability gap that is closing.`,
+                baselinePassRate: blRate,
+                withSkillPassRate: wsRate,
+                trend,
+                recommendation: isEncoded
+                    ? 'This skill encodes team preferences. Keep and maintain it.'
+                    : 'Capability uplift skill. Monitor baseline improvements across model updates.',
+            };
+        }
+        // Case 3: Middle ground — need more data or trend analysis
+        if (trend === 'converging') {
+            return {
+                skillName,
+                category: 'capability_uplift',
+                confidence: 0.6,
+                reasoning: `Baseline trend is converging toward skill performance, suggesting a capability gap that is closing.`,
+                baselinePassRate: blRate,
+                withSkillPassRate: wsRate,
+                trend,
+                recommendation: 'Likely capability uplift. Re-evaluate after model updates.',
+            };
+        }
+        if (trend === 'stable' || trend === 'diverging') {
+            return {
+                skillName,
+                category: 'encoded_preference',
+                confidence: 0.55,
+                reasoning: `Baseline-to-skill gap is stable/diverging, suggesting persistent team-specific knowledge.`,
+                baselinePassRate: blRate,
+                withSkillPassRate: wsRate,
+                trend,
+                recommendation: 'Likely encoded preference. Maintain and refine.',
+            };
+        }
+        return {
+            skillName,
+            category: 'unknown',
+            confidence: 0.3,
+            reasoning: `Not enough data to classify confidently. Baseline: ${this.pct(blRate)}, With-skill: ${this.pct(wsRate)}.`,
+            baselinePassRate: blRate,
+            withSkillPassRate: wsRate,
+            trend,
+            recommendation: 'Run more benchmark iterations to gather trend data.',
+        };
+    }
+    computeTrend(history) {
+        if (history.length < MIN_BENCHMARKS_FOR_TREND) {
+            return 'insufficient_data';
+        }
+        // Compare gap between first and last benchmarks
+        const first = history[0];
+        const last = history[history.length - 1];
+        const firstGap = first.summary.withSkill.passRate - first.summary.baseline.passRate;
+        const lastGap = last.summary.withSkill.passRate - last.summary.baseline.passRate;
+        const gapChange = lastGap - firstGap;
+        if (Math.abs(gapChange) < CONVERGENCE_THRESHOLD / 2) {
+            return 'stable';
+        }
+        // Gap is shrinking → baseline is catching up → converging
+        if (gapChange < 0) {
+            return 'converging';
+        }
+        // Gap is growing → skill is pulling ahead → diverging
+        return 'diverging';
+    }
+    pct(value) {
+        return `${(value * 100).toFixed(1)}%`;
+    }
+}
+//# sourceMappingURL=SkillClassifier.js.map

package/dist/infra/lib/evolution/SkillClassifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SkillClassifier.js","sourceRoot":"","sources":["../../../../src/infra/lib/evolution/SkillClassifier.ts"],"names":[],"mappings":"AAAA,sEAAsE;AACtE,EAAE;AACF,mDAAmD;AACnD,EAAE;AACF,sEAAsE;AACtE,0CAA0C;AAC1C,qFAAqF;AACrF,EAAE;AACF,4EAA4E;AAC5E,gDAAgD;AAChD,oFAAoF;AAGpF,OAAO,EAAE,cAAc,EAAmB,MAAM,qBAAqB,CAAC;AAetE;;GAEG;AACH,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,qBAAqB,GAAG,IAAI,CAAC;AACnC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAEnC,MAAM,OAAO,eAAe;IAClB,SAAS,CAAiB;IAElC,YAAY,OAAsB;QAChC,IAAI,CAAC,SAAS,GAAG,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACI,QAAQ,CAAC,SAAiB;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAErD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,SAAS;gBACT,QAAQ,EAAE,SAAS;gBACnB,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,+CAA+C;gBAC1D,gBAAgB,EAAE,CAAC;gBACnB,iBAAiB,EAAE,CAAC;gBACpB,KAAK,EAAE,mBAAmB;gBAC1B,cAAc,EAAE,8DAA8D;aAC/E,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAEzC,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACzG,CAAC;IAED;;OAEG;IACI,iBAAiB,CACtB,SAAiB,EACjB,iBAAyB,EACzB,gBAAwB,EACxB,QAAuC,mBAAmB;QAE1D,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED;;OAEG;IACI,kBAAkB,CAAC,SAAiB;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAExC,IAAI,MAAM,CAAC,QAAQ,KAAK,mBAAmB,IAAI,MAAM,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;YAC7E,OAAO;gBACL,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,uBAAuB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,yCAAyC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,qDAAqD;aACjM,CAAC;QACJ,CAAC;QAED,IAAI,MAAM,CAAC,gBAAgB,IAAI,uBAAuB,EAAE,CAAC;YACvD,OAAO;gBACL,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,yBAAyB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAC,mDAAmD;aACtH,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,yCAAyC,EAAE,CAAC;IAChF,CAAC;IAEO,iBAAiB,CACvB,SAAiB,EACjB,MAAc,EACd,MAAc,EACd,KAAoC,EACpC,cAAsB;QAEtB,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAC;QAE5B,iFAAiF;QACjF,IAAI,MAAM,IAAI,uBAAuB,EAAE,CAAC;YACtC,OAAO;gBACL,SAAS;gBACT,QAAQ,EAAE,mBAAmB;gBAC7B,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,GAAG,GAAG,CAAC;gBAC7C,SAAS,EAAE,+BAA+B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,qIAAqI;gBAC/L,gBAAgB,EAAE,MAAM;gBACxB,iBAAiB,EAAE,MAAM;gBACzB,KAAK;gBACL,cAAc,EAAE,GAAG,GAAG,qBAAqB;oBACzC,CAAC,CAAC,qEAAqE;oBACvE,CAAC,CAAC,0DAA0D;aAC/D,CAAC;QACJ,CAAC;QAED,qFAAqF;QACrF,IAAI,MAAM,IAAI,sBAAsB,IAAI,GAAG,GAAG,qBAAqB,EAAE,CAAC;YACpE,MAAM,SAAS,GAAG,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,WAAW,IAAI,cAAc,GAAG,wBAAwB,CAAC;YAC3G,OAAO;gBACL,SAAS;gBACT,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,mBAAmB;gBAChE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;gBAC7D,SAAS,EAAE,SAAS;oBAClB,CAAC,CAAC,iBAAiB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,8EAA8E;oBACjH,CAAC,CAAC,iBAAiB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,uEAAuE;gBAC5G,gBAAgB,EAAE,MAAM;gBACxB,iBAAiB,EAAE,MAAM;gBACzB,KAAK;gBACL,cAAc,EAAE,SAAS;oBACvB,CAAC,CAAC,4DAA4D;oBAC9D,CAAC,CAAC,8EAA8E;aACnF,CAAC;QACJ,CAAC;QAED,2DAA2D;QAC3D,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,OAAO;gBACL,SAAS;gBACT,QAAQ,EAAE,mBAAmB;gBAC7B,UAAU,EAAE,GAAG;gBACf,SAAS,EAAE,qGAAqG;gBAChH,gBAAgB,EAAE,MAAM;gBACxB,iBAAiB,EAAE,MAAM;gBACzB,KAAK;gBACL,cAAc,EAAE,4DAA4D;aAC7E,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;YAChD,OAAO;gBACL,SAAS;gBACT,QAAQ,EAAE,oBAAoB;gBAC9B,UAAU,EAAE,IAAI;gBAChB,SAAS,EAAE,2FAA2F;gBACtG,gBAAgB,EAAE,MAAM;gBACxB,iBAAiB,EAAE,MAAM;gBACzB,KAAK;gBACL,cAAc,EAAE,iDAAiD;aAClE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,SAAS;YACT,QAAQ,EAAE,SAAS;YACnB,UAAU,EAAE,GAAG;YACf,SAAS,EAAE,sDAAsD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG;YACrH,gBAAgB,EAAE,MAAM;YACxB,iBAAiB,EAAE,MAAM;YACzB,KAAK;YACL,cAAc,EAAE,qDAAqD;SACtE,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,OAA0B;QAE1B,IAAI,OAAO,CAAC,MAAM,GAAG,wBAAwB,EAAE,CAAC;YAC9C,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QAED,gDAAgD;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEzC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACpF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAEjF,MAAM,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;QAErC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;YACpD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,0DAA0D;QAC1D,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,sDAAsD;QACtD,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,GAAG,CAAC,KAAa;QACvB,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACxC,CAAC;CACF"}

package/dist/infra/lib/evolution/SkillEvalRunner.d.ts

@@ -0,0 +1,102 @@
+import { MemoryStorage } from '../memory/MemoryStorage.js';
+export type EvalStatus = 'pending' | 'running' | 'passed' | 'failed' | 'error';
+export interface SkillEvalCase {
+    id: string;
+    skillName: string;
+    prompt: string;
+    expectedOutput: string;
+    files: string[];
+    assertions: EvalAssertion[];
+}
+export interface EvalAssertion {
+    id: string;
+    description: string;
+    type: 'contains' | 'not_contains' | 'matches_regex' | 'custom';
+    value: string;
+}
+export interface EvalRunResult {
+    evalId: string;
+    runId: string;
+    skillName: string;
+    variant: 'with_skill' | 'baseline';
+    status: EvalStatus;
+    output: string;
+    grades: AssertionGrade[];
+    durationMs: number;
+    tokenCount: number;
+    createdAt: string;
+}
+export interface AssertionGrade {
+    assertionId: string;
+    description: string;
+    passed: boolean;
+    evidence: string;
+}
+export interface EvalSetInput {
+    skillName: string;
+    evals: Array<{
+        prompt: string;
+        expectedOutput: string;
+        files?: string[];
+        assertions?: Array<{
+            description: string;
+            type: 'contains' | 'not_contains' | 'matches_regex' | 'custom';
+            value: string;
+        }>;
+    }>;
+}
+export declare class SkillEvalRunner {
+    private db;
+    constructor(storage: MemoryStorage);
+    private initializeTables;
+    /**
+     * Create an eval set for a skill
+     */
+    createEvalSet(input: EvalSetInput): SkillEvalCase[];
+    /**
+     * Get all eval cases for a skill
+     */
+    getEvalCases(skillName: string): SkillEvalCase[];
+    /**
+     * Get a single eval case by ID
+     */
+    getEvalCase(evalId: string): SkillEvalCase | null;
+    /**
+     * Record the start of an eval run
+     */
+    startRun(evalId: string, skillName: string, variant: 'with_skill' | 'baseline'): string;
+    /**
+     * Complete an eval run with output and grades
+     */
+    completeRun(runId: string, output: string, grades: AssertionGrade[], durationMs: number, tokenCount: number): void;
+    /**
+     * Mark a run as errored
+     */
+    failRun(runId: string, errorMessage: string): void;
+    /**
+     * Grade output against assertions
+     */
+    gradeOutput(output: string, assertions: EvalAssertion[]): AssertionGrade[];
+    /**
+     * Get all runs for an eval case
+     */
+    getRunsForEval(evalId: string): EvalRunResult[];
+    /**
+     * Get all runs for a skill
+     */
+    getRunsForSkill(skillName: string): EvalRunResult[];
+    /**
+     * Get latest runs grouped by eval and variant
+     */
+    getLatestRuns(skillName: string): Map<string, {
+        withSkill: EvalRunResult | null;
+        baseline: EvalRunResult | null;
+    }>;
+    /**
+     * Delete all eval cases and runs for a skill
+     */
+    deleteEvalSet(skillName: string): number;
+    private rowToEvalCase;
+    private rowToRunResult;
+}
+//# sourceMappingURL=SkillEvalRunner.d.ts.map

package/dist/infra/lib/evolution/SkillEvalRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SkillEvalRunner.d.ts","sourceRoot":"","sources":["../../../../src/infra/lib/evolution/SkillEvalRunner.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAI3D,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/E,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,aAAa,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,UAAU,GAAG,cAAc,GAAG,eAAe,GAAG,QAAQ,CAAC;IAC/D,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,YAAY,GAAG,UAAU,CAAC;IACnC,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,cAAc,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;QACjB,UAAU,CAAC,EAAE,KAAK,CAAC;YACjB,WAAW,EAAE,MAAM,CAAC;YACpB,IAAI,EAAE,UAAU,GAAG,cAAc,GAAG,eAAe,GAAG,QAAQ,CAAC;YAC/D,KAAK,EAAE,MAAM,CAAC;SACf,CAAC,CAAC;KACJ,CAAC,CAAC;CACJ;AA4BD,qBAAa,eAAe;IAC1B,OAAO,CAAC,EAAE,CAA2C;gBAEzC,OAAO,EAAE,aAAa;IAKlC,OAAO,CAAC,gBAAgB;IAkCxB;;OAEG;IACI,aAAa,CAAC,KAAK,EAAE,YAAY,GAAG,aAAa,EAAE;IA6C1D;;OAEG;IACI,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,EAAE;IAOvD;;OAEG;IACI,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAOxD;;OAEG;IACI,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,MAAM;IAY9F;;OAEG;IACI,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,cAAc,EAAE,EACxB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,IAAI;IAWP;;OAEG;IACI,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI;IAMzD;;OAEG;IACI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG,cAAc,EAAE;IAkDjF;;OAEG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,EAAE;IAOtD;;OAEG;IACI,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,EAAE;IAO1D;;OAEG;IACI,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,aAAa,GAAG,IAAI,CAAC;QAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAA;KAAE,CAAC;IAmBzH;;OAEG;IACI,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM;IAa/C,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,cAAc;CAcvB"}