@gsep/core 0.8.0

package/dist/evaluation/BenchmarkSuites.js

@@ -0,0 +1,72 @@
+import coreGeneralV1Fixture from './fixtures/core-general-v1.json';
+import coreCodingV1Fixture from './fixtures/core-coding-v1.json';
+import proofOfValueV1Fixture from './fixtures/proof-of-value-v1.json';
+export const CORE_GENERAL_V1 = coreGeneralV1Fixture;
+export const CORE_CODING_V1 = coreCodingV1Fixture;
+export const PGA_SPECIFIC_V1 = {
+    metadata: {
+        id: 'pga-specific-v1',
+        version: '1.0.0',
+        owner: 'Luis Alfredo Velasquez Duran',
+        createdAt: '2026-02-27',
+        changedAt: '2026-02-27',
+        changelog: ['Initial release - GSEP-specific knowledge'],
+        frozen: true,
+    },
+    tasks: [
+        {
+            id: 'pga-architecture',
+            name: 'GSEP Architecture',
+            description: 'Test understanding of GSEP architecture',
+            userMessage: 'Explain the three-layer architecture of GSEP',
+            expectedOutcome: {
+                keywords: ['layer0', 'layer1', 'layer2', 'immutable', 'mutation'],
+                minLength: 100,
+            },
+            difficulty: 'easy',
+        },
+        {
+            id: 'pga-evolution',
+            name: 'Evolution Mechanism',
+            description: 'Test understanding of evolution process',
+            userMessage: 'How does GSEP evolve prompts?',
+            expectedOutcome: {
+                keywords: ['fitness', 'mutation', 'selection', 'genome'],
+                minLength: 100,
+            },
+            difficulty: 'medium',
+        },
+        {
+            id: 'pga-userdna',
+            name: 'User DNA Concept',
+            description: 'Test understanding of user DNA',
+            userMessage: 'What is User DNA in GSEP?',
+            expectedOutcome: {
+                keywords: ['user', 'preferences', 'traits', 'cognitive'],
+                minLength: 80,
+            },
+            difficulty: 'medium',
+        },
+    ],
+};
+export const PROOF_OF_VALUE_V1 = proofOfValueV1Fixture;
+export const BENCHMARK_SUITES = {
+    'core-general-v1': CORE_GENERAL_V1,
+    'core-coding-v1': CORE_CODING_V1,
+    'pga-specific-v1': PGA_SPECIFIC_V1,
+    'proof-of-value-v1': PROOF_OF_VALUE_V1,
+};
+export function getBenchmarkSuite(suiteId) {
+    const suite = BENCHMARK_SUITES[suiteId];
+    if (!suite) {
+        throw new Error(`Unknown benchmark suite: ${suiteId}`);
+    }
+    return suite;
+}
+export function getAvailableSuites() {
+    return Object.keys(BENCHMARK_SUITES);
+}
+export function validateSuiteFrozen(suiteId) {
+    return getBenchmarkSuite(suiteId).metadata.frozen;
+}
+//# sourceMappingURL=BenchmarkSuites.js.map

package/dist/evaluation/BenchmarkSuites.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BenchmarkSuites.js","sourceRoot":"","sources":["../../src/evaluation/BenchmarkSuites.ts"],"names":[],"mappings":"AAgBA,OAAO,oBAAoB,MAAM,iCAAiC,CAAC;AACnE,OAAO,mBAAmB,MAAM,gCAAgC,CAAC;AACjE,OAAO,qBAAqB,MAAM,mCAAmC,CAAC;AAsBtE,MAAM,CAAC,MAAM,eAAe,GAAmB,oBAAsC,CAAC;AAKtF,MAAM,CAAC,MAAM,cAAc,GAAmB,mBAAqC,CAAC;AAIpF,MAAM,CAAC,MAAM,eAAe,GAAmB;IAC3C,QAAQ,EAAE;QACN,EAAE,EAAE,iBAAiB;QACrB,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE,8BAA8B;QACrC,SAAS,EAAE,YAAY;QACvB,SAAS,EAAE,YAAY;QACvB,SAAS,EAAE,CAAC,2CAA2C,CAAC;QACxD,MAAM,EAAE,IAAI;KACf;IACD,KAAK,EAAE;QACH;YACI,EAAE,EAAE,kBAAkB;YACtB,IAAI,EAAE,mBAAmB;YACzB,WAAW,EAAE,yCAAyC;YACtD,WAAW,EAAE,8CAA8C;YAC3D,eAAe,EAAE;gBACb,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,CAAC;gBACjE,SAAS,EAAE,GAAG;aACjB;YACD,UAAU,EAAE,MAAM;SACrB;QACD;YACI,EAAE,EAAE,eAAe;YACnB,IAAI,EAAE,qBAAqB;YAC3B,WAAW,EAAE,yCAAyC;YACtD,WAAW,EAAE,+BAA+B;YAC5C,eAAe,EAAE;gBACb,QAAQ,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,CAAC;gBACxD,SAAS,EAAE,GAAG;aACjB;YACD,UAAU,EAAE,QAAQ;SACvB;QACD;YACI,EAAE,EAAE,aAAa;YACjB,IAAI,EAAE,kBAAkB;YACxB,WAAW,EAAE,gCAAgC;YAC7C,WAAW,EAAE,2BAA2B;YACxC,eAAe,EAAE;gBACb,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,CAAC;gBACxD,SAAS,EAAE,EAAE;aAChB;YACD,UAAU,EAAE,QAAQ;SACvB;KACJ;CACJ,CAAC;AAKF,MAAM,CAAC,MAAM,iBAAiB,GAAmB,qBAAuC,CAAC;AAIzF,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC5B,iBAAiB,EAAE,eAAe;IAClC,gBAAgB,EAAE,cAAc;IAChC,iBAAiB,EAAE,eAAe;IAClC,mBAAmB,EAAE,iBAAiB;CAChC,CAAC;AASX,MAAM,UAAU,iBAAiB,CAAC,OAAyB;IACvD,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAKD,MAAM,UAAU,kBAAkB;IAC9B,OAAO,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAuB,CAAC;AAC/D,CAAC;AAKD,MAAM,UAAU,mBAAmB,CAAC,OAAyB;IACzD,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;AACtD,CAAC"}

package/dist/evaluation/CalibrationManager.d.ts

@@ -0,0 +1,66 @@
+import type { StorageAdapter } from '../interfaces/StorageAdapter.js';
+export interface CalibrationPoint {
+    timestamp: Date;
+    context: {
+        layer?: 0 | 1 | 2;
+        operator?: string;
+        taskType?: string;
+    };
+    threshold: number;
+    metrics: {
+        totalCandidates: number;
+        passedSandbox: number;
+        deployedSuccessfully: number;
+        rolledBack: number;
+    };
+    performance: {
+        falsePositiveRate: number;
+        falseNegativeRate: number;
+        optimalThreshold: number;
+    };
+}
+export interface CalibrationHistory {
+    context: {
+        layer?: 0 | 1 | 2;
+        operator?: string;
+        taskType?: string;
+    };
+    points: CalibrationPoint[];
+    currentThreshold: number;
+    recommendedThreshold: number;
+    confidence: number;
+}
+export declare class CalibrationManager {
+    private storage;
+    private cache;
+    constructor(storage: StorageAdapter);
+    getCalibratedThreshold(context: {
+        layer?: 0 | 1 | 2;
+        operator?: string;
+        taskType?: string;
+    }): Promise<{
+        threshold: number;
+        confidence: number;
+        source: 'calibrated' | 'default' | 'cached';
+    }>;
+    recordCalibrationPoint(context: {
+        layer?: 0 | 1 | 2;
+        operator?: string;
+        taskType?: string;
+    }, metrics: {
+        totalCandidates: number;
+        passedSandbox: number;
+        deployedSuccessfully: number;
+        rolledBack: number;
+    }, currentThreshold: number): Promise<void>;
+    private calculateOptimalThreshold;
+    private loadCalibrationHistory;
+    private getContextKey;
+    private getDefaultThreshold;
+    getCalibrationReport(context?: {
+        layer?: 0 | 1 | 2;
+        operator?: string;
+        taskType?: string;
+    }): Promise<string>;
+}
+//# sourceMappingURL=CalibrationManager.d.ts.map

package/dist/evaluation/CalibrationManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CalibrationManager.d.ts","sourceRoot":"","sources":["../../src/evaluation/CalibrationManager.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAItE,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE;QACL,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACL,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,WAAW,EAAE;QACT,iBAAiB,EAAE,MAAM,CAAC;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,gBAAgB,EAAE,MAAM,CAAC;KAC5B,CAAC;CACL;AAED,MAAM,WAAW,kBAAkB;IAC/B,OAAO,EAAE;QACL,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;CACtB;AAID,qBAAa,kBAAkB;IAGf,OAAO,CAAC,OAAO;IAF3B,OAAO,CAAC,KAAK,CAAyC;gBAElC,OAAO,EAAE,cAAc;IASrC,sBAAsB,CAAC,OAAO,EAAE;QAClC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,QAAQ,CAAC;KAC/C,CAAC;IAwCI,sBAAsB,CACxB,OAAO,EAAE;QACL,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,EACD,OAAO,EAAE;QACL,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,UAAU,EAAE,MAAM,CAAC;KACtB,EACD,gBAAgB,EAAE,MAAM,GACzB,OAAO,CAAC,IAAI,CAAC;IAoDhB,OAAO,CAAC,yBAAyB;YAyBnB,sBAAsB;IA0DpC,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,mBAAmB;IA0BrB,oBAAoB,CAAC,OAAO,CAAC,EAAE;QACjC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,CAAC;CAsBtB"}

package/dist/evaluation/CalibrationManager.js

@@ -0,0 +1,157 @@
+export class CalibrationManager {
+    storage;
+    cache = new Map();
+    constructor(storage) {
+        this.storage = storage;
+    }
+    async getCalibratedThreshold(context) {
+        const cacheKey = this.getContextKey(context);
+        if (this.cache.has(cacheKey)) {
+            const history = this.cache.get(cacheKey);
+            return {
+                threshold: history.recommendedThreshold,
+                confidence: history.confidence,
+                source: 'cached',
+            };
+        }
+        const history = await this.loadCalibrationHistory(context);
+        if (history && history.points.length >= 10) {
+            this.cache.set(cacheKey, history);
+            return {
+                threshold: history.recommendedThreshold,
+                confidence: history.confidence,
+                source: 'calibrated',
+            };
+        }
+        const defaultThreshold = this.getDefaultThreshold(context);
+        return {
+            threshold: defaultThreshold,
+            confidence: 0.5,
+            source: 'default',
+        };
+    }
+    async recordCalibrationPoint(context, metrics, currentThreshold) {
+        const falsePositiveRate = metrics.passedSandbox > 0
+            ? metrics.rolledBack / metrics.passedSandbox
+            : 0;
+        const deploymentSuccessRate = metrics.deployedSuccessfully > 0
+            ? metrics.deployedSuccessfully / metrics.passedSandbox
+            : 0;
+        const falseNegativeRate = deploymentSuccessRate > 0.8
+            ? Math.max(0, 0.2 - (1 - currentThreshold))
+            : 0;
+        const optimalThreshold = this.calculateOptimalThreshold(falsePositiveRate, falseNegativeRate, currentThreshold);
+        if (this.storage.saveCalibrationPoint) {
+            await this.storage.saveCalibrationPoint({
+                contextKey: this.getContextKey(context),
+                layer: context.layer,
+                operator: context.operator,
+                taskType: context.taskType,
+                threshold: currentThreshold,
+                totalCandidates: metrics.totalCandidates,
+                passedSandbox: metrics.passedSandbox,
+                deployedSuccessfully: metrics.deployedSuccessfully,
+                rolledBack: metrics.rolledBack,
+                falsePositiveRate,
+                falseNegativeRate,
+                optimalThreshold,
+                timestamp: new Date(),
+            });
+        }
+        this.cache.delete(this.getContextKey(context));
+    }
+    calculateOptimalThreshold(falsePositiveRate, falseNegativeRate, currentThreshold) {
+        if (falsePositiveRate > 0.1) {
+            return Math.min(1.0, currentThreshold + 0.05);
+        }
+        if (falseNegativeRate > 0.2 && falsePositiveRate < 0.05) {
+            return Math.max(0.5, currentThreshold - 0.05);
+        }
+        return currentThreshold;
+    }
+    async loadCalibrationHistory(context) {
+        if (!this.storage.getCalibrationHistory) {
+            return null;
+        }
+        const contextKey = this.getContextKey(context);
+        const rows = await this.storage.getCalibrationHistory(contextKey, 100);
+        if (rows.length === 0) {
+            return null;
+        }
+        const points = rows.map(row => ({
+            timestamp: row.timestamp,
+            context: {
+                layer: row.layer,
+                operator: row.operator ?? undefined,
+                taskType: row.taskType ?? undefined,
+            },
+            threshold: row.threshold,
+            metrics: {
+                totalCandidates: row.totalCandidates,
+                passedSandbox: row.passedSandbox,
+                deployedSuccessfully: row.deployedSuccessfully,
+                rolledBack: row.rolledBack,
+            },
+            performance: {
+                falsePositiveRate: row.falsePositiveRate,
+                falseNegativeRate: row.falseNegativeRate,
+                optimalThreshold: row.optimalThreshold,
+            },
+        }));
+        const recentPoints = points.slice(0, 20);
+        const avgOptimal = recentPoints.reduce((sum, p) => sum + p.performance.optimalThreshold, 0) / recentPoints.length;
+        const confidence = Math.min(1, points.length / 50);
+        return {
+            context,
+            points,
+            currentThreshold: rows[0].threshold,
+            recommendedThreshold: avgOptimal,
+            confidence,
+        };
+    }
+    getContextKey(context) {
+        return `${context.layer ?? 'any'}_${context.operator ?? 'any'}_${context.taskType ?? 'any'}`;
+    }
+    getDefaultThreshold(context) {
+        if (context.layer === 0)
+            return 1.0;
+        if (context.layer === 1)
+            return 0.75;
+        if (context.layer === 2)
+            return 0.60;
+        if (context.operator === 'safety_reinforcement')
+            return 0.85;
+        if (context.operator === 'compress_instructions')
+            return 0.65;
+        if (context.operator === 'reorder_constraints')
+            return 0.70;
+        if (context.operator === 'tool_selection_bias')
+            return 0.70;
+        if (context.taskType === 'coding')
+            return 0.75;
+        if (context.taskType === 'general')
+            return 0.65;
+        return 0.65;
+    }
+    async getCalibrationReport(context) {
+        const lines = [];
+        lines.push('# 📊 Sandbox Calibration Report\n');
+        if (context) {
+            const result = await this.getCalibratedThreshold(context);
+            const key = this.getContextKey(context);
+            lines.push(`**Context**: ${key}`);
+            lines.push(`**Threshold**: ${(result.threshold * 100).toFixed(1)}%`);
+            lines.push(`**Confidence**: ${(result.confidence * 100).toFixed(1)}%`);
+            lines.push(`**Source**: ${result.source}`);
+            lines.push('');
+        }
+        else {
+            lines.push('**Status**: Calibration system active');
+            lines.push(`**Cached contexts**: ${this.cache.size}`);
+            lines.push('');
+            lines.push('Use specific context for detailed calibration data.');
+        }
+        return lines.join('\n');
+    }
+}
+//# sourceMappingURL=CalibrationManager.js.map

package/dist/evaluation/CalibrationManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CalibrationManager.js","sourceRoot":"","sources":["../../src/evaluation/CalibrationManager.ts"],"names":[],"mappings":"AAmDA,MAAM,OAAO,kBAAkB;IAGP;IAFZ,KAAK,GAAG,IAAI,GAAG,EAA8B,CAAC;IAEtD,YAAoB,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;IAE3C,CAAC;IAOD,KAAK,CAAC,sBAAsB,CAAC,OAI5B;QAKG,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAG7C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YAC1C,OAAO;gBACH,SAAS,EAAE,OAAO,CAAC,oBAAoB;gBACvC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,MAAM,EAAE,QAAQ;aACnB,CAAC;QACN,CAAC;QAGD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAE3D,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClC,OAAO;gBACH,SAAS,EAAE,OAAO,CAAC,oBAAoB;gBACvC,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,MAAM,EAAE,YAAY;aACvB,CAAC;QACN,CAAC;QAGD,MAAM,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC3D,OAAO;YACH,SAAS,EAAE,gBAAgB;YAC3B,UAAU,EAAE,GAAG;YACf,MAAM,EAAE,SAAS;SACpB,CAAC;IACN,CAAC;IAOD,KAAK,CAAC,sBAAsB,CACxB,OAIC,EACD,OAKC,EACD,gBAAwB;QAGxB,MAAM,iBAAiB,GAAG,OAAO,CAAC,aAAa,GAAG,CAAC;YAC/C,CAAC,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,aAAa;YAC5C,CAAC,CAAC,CAAC,CAAC;QAIR,MAAM,qBAAqB,GAAG,OAAO,CAAC,oBAAoB,GAAG,CAAC;YAC1D,CAAC,CAAC,OAAO,CAAC,oBAAoB,GAAG,OAAO,CAAC,aAAa;YACtD,CAAC,CAAC,CAAC,CAAC;QAER,MAAM,iBAAiB,GAAG,qBAAqB,GAAG,GAAG;YACjD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC,CAAC;QAGR,MAAM,gBAAgB,GAAG,IAAI,CAAC,yBAAyB,CACnD,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,CACnB,CAAC;QAGF,IAAI,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;gBACvC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,SAAS,EAAE,gBAAgB;gBAC3B,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;gBAClD,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,iBAAiB;gBACjB,iBAAiB;gBACjB,gBAAgB;gBAChB,SAAS,EAAE,IAAI,IAAI,EAAE;aACxB,CAAC,CAAC;QACP,CAAC;QAGD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,CAAC;IAQO,yBAAyB,CAC7B,iBAAyB,EACzB,iBAAyB,EACzB,gBAAwB;QAMxB,IAAI,iBAAiB,GAAG,GAAG,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAAC;QAClD,CAAC;QAGD,IAAI,iBAAiB,GAAG,GAAG,IAAI,iBAAiB,GAAG,IAAI,EAAE,CAAC;YACtD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,gBAAgB,GAAG,IAAI,CAAC,CAAC;QAClD,CAAC;QAGD,OAAO,gBAAgB,CAAC;IAC5B,CAAC;IAKO,KAAK,CAAC,sBAAsB,CAAC,OAIpC;QACG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;QAEvE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,MAAM,GAAuB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChD,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE;gBACL,KAAK,EAAE,GAAG,CAAC,KAA8B;gBACzC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;gBACnC,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,SAAS;aACtC;YACD,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE;gBACL,eAAe,EAAE,GAAG,CAAC,eAAe;gBACpC,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,oBAAoB,EAAE,GAAG,CAAC,oBAAoB;gBAC9C,UAAU,EAAE,GAAG,CAAC,UAAU;aAC7B;YACD,WAAW,EAAE;gBACT,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;gBACxC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;gBACxC,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;aACzC;SACJ,CAAC,CAAC,CAAC;QAGJ,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAClC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC,CACtD,GAAG,YAAY,CAAC,MAAM,CAAC;QAGxB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;QAEnD,OAAO;YACH,OAAO;YACP,MAAM;YACN,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACnC,oBAAoB,EAAE,UAAU;YAChC,UAAU;SACb,CAAC;IACN,CAAC;IAKO,aAAa,CAAC,OAIrB;QACG,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,IAAI,KAAK,EAAE,CAAC;IACjG,CAAC;IAKO,mBAAmB,CAAC,OAI3B;QAEG,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QACpC,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACrC,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAErC,IAAI,OAAO,CAAC,QAAQ,KAAK,sBAAsB;YAAE,OAAO,IAAI,CAAC;QAC7D,IAAI,OAAO,CAAC,QAAQ,KAAK,uBAAuB;YAAE,OAAO,IAAI,CAAC;QAC9D,IAAI,OAAO,CAAC,QAAQ,KAAK,qBAAqB;YAAE,OAAO,IAAI,CAAC;QAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,qBAAqB;YAAE,OAAO,IAAI,CAAC;QAE5D,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC/C,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QAEhD,OAAO,IAAI,CAAC;IAChB,CAAC;IAOD,KAAK,CAAC,oBAAoB,CAAC,OAI1B;QACG,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAEhD,IAAI,OAAO,EAAE,CAAC;YACV,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;YAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YAExC,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACrE,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACvE,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACtD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACtE,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;CACJ"}

package/dist/evaluation/Evaluator.d.ts

@@ -0,0 +1,79 @@
+import type { BenchmarkSuiteId } from './BenchmarkSuites.js';
+import type { LLMAdapter } from '../interfaces/LLMAdapter.js';
+export interface EvaluatableGenome {
+    chat(userMessage: string, context: {
+        userId?: string;
+    }): Promise<string>;
+    chatWithMetrics?(userMessage: string, context: {
+        userId?: string;
+    }): Promise<{
+        content: string;
+        metrics: {
+            tokensUsed: number;
+            responseTime: number;
+            fitnessScore: number;
+        };
+    }>;
+}
+export interface EvaluationTask {
+    id: string;
+    name: string;
+    description: string;
+    userMessage: string;
+    expectedOutcome: {
+        keywords?: string[];
+        minLength?: number;
+        maxLength?: number;
+        successCriteria?: (response: string) => boolean;
+    };
+    difficulty: 'easy' | 'medium' | 'hard';
+}
+export interface EvaluationResult {
+    taskId: string;
+    taskName: string;
+    success: boolean;
+    response: string;
+    tokensUsed: number;
+    responseTime: number;
+    qualityScore: number;
+    failureReason?: string;
+}
+export interface BenchmarkResult {
+    totalTasks: number;
+    successfulTasks: number;
+    successRate: number;
+    avgTokensPerTask: number;
+    avgResponseTime: number;
+    avgQualityScore: number;
+    results: EvaluationResult[];
+    timestamp: Date;
+}
+export interface ComparisonResult {
+    withPGA: BenchmarkResult;
+    withoutPGA: BenchmarkResult;
+    improvements: {
+        successRate: number;
+        tokenEfficiency: number;
+        responseTime: number;
+        qualityScore: number;
+    };
+    verdict: 'PGA_WINS' | 'BASELINE_WINS' | 'TIE';
+}
+export declare class Evaluator {
+    private semanticJudge?;
+    constructor(options?: {
+        llm?: LLMAdapter;
+        enableSemanticJudge?: boolean;
+    });
+    evaluate(genome: EvaluatableGenome, tasks: EvaluationTask[], userId: string): Promise<BenchmarkResult>;
+    private evaluateTask;
+    private checkSuccess;
+    private checkSemanticValidation;
+    private calculateQuality;
+    compareWithSuite(genome: EvaluatableGenome, suiteId: BenchmarkSuiteId, userId: string): Promise<BenchmarkResult>;
+    compare(genomeWithPGA: EvaluatableGenome, genomeBaseline: EvaluatableGenome, tasks: EvaluationTask[], userId: string): Promise<ComparisonResult>;
+    formatReport(benchmark: BenchmarkResult): string;
+    formatComparisonReport(comparison: ComparisonResult): string;
+}
+export declare const STANDARD_TASKS: EvaluationTask[];
+//# sourceMappingURL=Evaluator.d.ts.map

package/dist/evaluation/Evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluation/Evaluator.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAI7D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAS9D,MAAM,WAAW,iBAAiB;IAC9B,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzE,eAAe,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QACzE,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE;YAAE,UAAU,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC;KAC/E,CAAC,CAAC;CACN;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE;QACb,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;KACnD,CAAC;IACF,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,gBAAgB;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,SAAS,EAAE,IAAI,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC7B,OAAO,EAAE,eAAe,CAAC;IACzB,UAAU,EAAE,eAAe,CAAC;IAC5B,YAAY,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,OAAO,EAAE,UAAU,GAAG,eAAe,GAAG,KAAK,CAAC;CACjD;AAED,qBAAa,SAAS;IAClB,OAAO,CAAC,aAAa,CAAC,CAAgB;gBAE1B,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,UAAU,CAAC;QAAC,mBAAmB,CAAC,EAAE,OAAO,CAAA;KAAE;IAYnE,QAAQ,CACV,MAAM,EAAE,iBAAiB,EACzB,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC;YA+Bb,YAAY;IA2E1B,OAAO,CAAC,YAAY;IAwDpB,OAAO,CAAC,uBAAuB;IAyD/B,OAAO,CAAC,gBAAgB;IA4BlB,gBAAgB,CAClB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC;IAUrB,OAAO,CACT,aAAa,EAAE,iBAAiB,EAChC,cAAc,EAAE,iBAAiB,EACjC,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,GACf,OAAO,CAAC,gBAAgB,CAAC;IA4C5B,YAAY,CAAC,SAAS,EAAE,eAAe,GAAG,MAAM;IAqChD,sBAAsB,CAAC,UAAU,EAAE,gBAAgB,GAAG,MAAM;CAwE/D;AAKD,eAAO,MAAM,cAAc,EAAE,cAAc,EAwD1C,CAAC"}