@vfarcic/dot-ai 0.111.0 → 0.113.0

package/dist/evaluation/eval-runner.js

@@ -0,0 +1,399 @@
+#!/usr/bin/env npx tsx
+"use strict";
+/**
+ * Evaluation Runner for Multi-Model Comparative Analysis
+ *
+ * Runs comparative evaluation on available datasets from multiple models
+ * Automatically detects and evaluates both remediation and recommendation datasets
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const remediation_comparative_js_1 = require("./evaluators/remediation-comparative.js");
+const recommendation_comparative_js_1 = require("./evaluators/recommendation-comparative.js");
+const capability_comparative_js_1 = require("./evaluators/capability-comparative.js");
+const pattern_comparative_js_1 = require("./evaluators/pattern-comparative.js");
+const policy_comparative_js_1 = require("./evaluators/policy-comparative.js");
+const promises_1 = require("fs/promises");
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
+const EVALUATOR_CONFIG = {
+    remediation: {
+        evaluator: remediation_comparative_js_1.RemediationComparativeEvaluator,
+        prefix: 'remediate_',
+        title: 'Remediation AI Model Comparison Report'
+    },
+    recommendation: {
+        evaluator: recommendation_comparative_js_1.RecommendationComparativeEvaluator,
+        prefix: 'recommend_',
+        title: 'Recommendation AI Model Comparison Report'
+    },
+    capability: {
+        evaluator: capability_comparative_js_1.CapabilityComparativeEvaluator,
+        prefix: 'capability_',
+        title: 'Capability AI Model Comparison Report'
+    },
+    pattern: {
+        evaluator: pattern_comparative_js_1.PatternComparativeEvaluator,
+        prefix: 'pattern_',
+        title: 'Pattern AI Model Comparison Report'
+    },
+    policy: {
+        evaluator: policy_comparative_js_1.PolicyComparativeEvaluator,
+        prefix: 'policy_',
+        title: 'Policy AI Model Comparison Report'
+    }
+};
+function generateMarkdownReport(results, stats, evaluationType, finalAssessment) {
+    const timestamp = new Date().toISOString();
+    // Use final assessment if provided
+    const overallAssessment = finalAssessment?.overall_assessment || null;
+    // Calculate basic statistics for reference
+    const modelScores = new Map();
+    results.forEach(result => {
+        if (result.modelRankings) {
+            result.modelRankings.forEach((ranking) => {
+                if (!modelScores.has(ranking.model)) {
+                    modelScores.set(ranking.model, []);
+                }
+                modelScores.get(ranking.model).push(ranking.score);
+            });
+        }
+    });
+    // Calculate average scores for supplementary information
+    const modelAverages = new Map();
+    modelScores.forEach((scores, model) => {
+        const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
+        modelAverages.set(model, Math.round(avg * 1000) / 1000);
+    });
+    const reportTitle = EVALUATOR_CONFIG[evaluationType].title;
+    return `# ${reportTitle}
+
+**Generated**: ${timestamp}  
+**Scenarios Analyzed**: ${results.length}  
+**Models Evaluated**: ${stats.availableModels.length}  
+**Total Datasets**: ${stats.totalDatasets}
+
+## Executive Summary
+
+### 🏆 Overall Winner (AI Assessment)
+${overallAssessment ? `
+**${overallAssessment.winner}**
+
+${overallAssessment.rationale}
+` : 'Overall assessment not available'}
+
+### 📊 AI Reliability Rankings
+
+${overallAssessment ? overallAssessment.reliability_ranking
+        .map((ranking, index) => `${index + 1}. **${ranking.model}** (${Math.round(ranking.reliability_score * 100)}%) - ${ranking.reliability_notes}`)
+        .join('\n') : 'Reliability rankings not available'}
+
+### 📋 Production Recommendations
+
+${overallAssessment ? `
+- **Primary Choice**: ${overallAssessment.production_recommendations.primary}
+- **Secondary Option**: ${overallAssessment.production_recommendations.secondary}
+- **Avoid for Production**: ${overallAssessment.production_recommendations.avoid.length > 0 ? overallAssessment.production_recommendations.avoid.join(', ') : 'None'}
+${Object.keys(overallAssessment.production_recommendations.specialized_use).length > 0 ?
+        '\n**Specialized Use Cases:**\n' + Object.entries(overallAssessment.production_recommendations.specialized_use)
+            .map(([useCase, model]) => `- **${useCase}**: ${model}`)
+            .join('\n') : ''}
+` : 'Production recommendations not available'}
+
+### 📊 Supplementary Statistics (Reference Only)
+
+| Model | Avg Score | Notes |
+|-------|-----------|-------|
+${Array.from(modelAverages.entries())
+        .sort((a, b) => b[1] - a[1])
+        .map(([model, avgScore]) => `| ${model} | ${avgScore} | See AI assessment above |`)
+        .join('\n')}
+
+## Detailed Scenario Results
+
+${results.map((result, index) => {
+        const scenarioTitle = result.key.replace(/_/g, ' ').replace(/(remediation|recommendation) comparative /, '').toUpperCase();
+        return `### ${index + 1}. ${scenarioTitle}
+
+**Winner**: ${result.bestModel} (Score: ${result.score})  
+**Models Compared**: ${result.modelCount}  
+**Confidence**: ${result.confidence ? Math.round(result.confidence * 100) : 0}%
+
+#### Rankings
+${result.modelRankings ? result.modelRankings.map((rank) => `${rank.rank}. **${rank.model}** - ${rank.score}`).join('\n') : 'No detailed rankings available'}
+
+#### Analysis
+${result.comment}
+
+---`;
+    }).join('\n\n')}
+
+## AI Model Selection Guide
+
+${overallAssessment ? `
+### Key Insights
+${overallAssessment.key_insights}
+
+### Recommended Selection Strategy
+- **For Production Use**: Choose ${overallAssessment.production_recommendations.primary}
+- **For Secondary Option**: Consider ${overallAssessment.production_recommendations.secondary}
+${overallAssessment.production_recommendations.avoid.length > 0 ?
+        `- **Avoid**: ${overallAssessment.production_recommendations.avoid.join(', ')} (reliability concerns)` : ''}
+
+### Decision Framework
+The AI assessment prioritizes **reliability and consistency** over peak performance. Models that fail completely in any scenario are heavily penalized, ensuring production-ready recommendations.
+` : 'AI model selection guide not available'}
+
+---
+
+## Report Attribution
+
+Report generated by DevOps AI Toolkit Comparative Evaluation System
+`;
+}
+function loadModelMetadata() {
+    try {
+        const fs = require('fs');
+        const path = require('path');
+        const metadataPath = path.join(__dirname, 'model-metadata.json');
+        if (!fs.existsSync(metadataPath)) {
+            console.error('❌ Model metadata file not found');
+            console.error('📊 Pricing and capabilities data required for cost analysis');
+            console.error('');
+            console.error('🔄 To create model metadata, run:');
+            console.error('   /update-model-metadata');
+            console.error('');
+            process.exit(1);
+        }
+        const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
+        // Check if metadata is older than 30 days
+        const metadataAge = Date.now() - new Date(metadata.lastUpdated).getTime();
+        const thirtyDays = 30 * 24 * 60 * 60 * 1000;
+        if (metadataAge > thirtyDays) {
+            console.error('❌ Model metadata is over 30 days old (last updated: ' + metadata.lastUpdated + ')');
+            console.error('📊 Pricing and capabilities data may be outdated, affecting cost analysis accuracy');
+            console.error('');
+            console.error('🔄 To update model metadata, run:');
+            console.error('   /update-model-metadata');
+            console.error('');
+            process.exit(1);
+        }
+        console.log('✅ Model metadata loaded (updated: ' + metadata.lastUpdated + ')');
+        return metadata;
+    }
+    catch (error) {
+        console.error('❌ Failed to load model metadata:', error instanceof Error ? error.message : String(error));
+        console.error('🔄 To create model metadata, run: /update-model-metadata');
+        process.exit(1);
+    }
+}
+function generateJsonReport(results, stats, evaluationType, modelMetadata, finalAssessment) {
+    const timestamp = new Date().toISOString();
+    // Use final assessment if provided
+    const overallAssessment = finalAssessment || null;
+    return {
+        metadata: {
+            reportType: 'comparative-evaluation',
+            evaluationType: evaluationType,
+            generated: timestamp,
+            scenariosAnalyzed: results.length,
+            modelsEvaluated: stats.availableModels.length,
+            totalDatasets: stats.totalDatasets,
+            tool: EVALUATOR_CONFIG[evaluationType].title
+        },
+        modelMetadata: modelMetadata.models,
+        overallAssessment: overallAssessment,
+        results: results,
+        summary: stats
+    };
+}
+async function detectAvailableDatasets(datasetsDir, filterType) {
+    try {
+        const files = await (0, promises_1.readdir)(datasetsDir);
+        const result = {};
+        for (const [type, config] of Object.entries(EVALUATOR_CONFIG)) {
+            // If filter specified, only check for that type
+            if (filterType && type !== filterType) {
+                result[type] = false;
+            }
+            else {
+                result[type] = files.some(file => file.startsWith(config.prefix));
+            }
+        }
+        return result;
+    }
+    catch (error) {
+        console.warn('Could not read datasets directory, assuming no datasets available');
+        const result = {};
+        for (const type of Object.keys(EVALUATOR_CONFIG)) {
+            result[type] = false;
+        }
+        return result;
+    }
+}
+async function runEvaluation(evaluatorType, datasetsDir, modelMetadata) {
+    const EvaluatorClass = EVALUATOR_CONFIG[evaluatorType].evaluator;
+    const evaluator = new EvaluatorClass(datasetsDir);
+    console.log(`\n🔬 Starting ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} Evaluation\n`);
+    // Show dataset stats
+    console.log('📊 Dataset Analysis:');
+    const stats = evaluator.getDatasetStats();
+    console.log(`- Total datasets: ${stats.totalDatasets}`);
+    console.log(`- Available models: ${stats.availableModels.join(', ')}`);
+    console.log(`- Scenarios with multiple models: ${stats.scenariosWithMultipleModels}`);
+    console.log(`- Interaction types: ${stats.interactionTypes.join(', ')}`);
+    console.log();
+    // Show evaluation phases
+    console.log('🎯 Evaluation Phases:');
+    const phases = evaluator.getEvaluationPhases();
+    phases.forEach(phase => {
+        console.log(`- ${phase.phase}: ${phase.description}`);
+        console.log(`  Models: ${phase.availableModels.join(', ')}`);
+        console.log(`  Scenarios: ${phase.scenarioCount}`);
+        console.log();
+    });
+    // Run comparative evaluation on all scenarios
+    console.log('🚀 Running Comparative Evaluation...\n');
+    const results = await evaluator.evaluateAllScenarios();
+    console.log(`✅ ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} Evaluation Complete! Analyzed ${results.length} scenarios\n`);
+    // Conduct final assessment across all scenarios
+    const finalAssessment = await evaluator.conductFinalAssessment(results);
+    // Generate dual-format reports using final assessment
+    const reportContent = generateMarkdownReport(results, stats, evaluatorType, finalAssessment);
+    const jsonResults = generateJsonReport(results, stats, evaluatorType, modelMetadata, finalAssessment);
+    // Save reports to files
+    const dateStamp = new Date().toISOString().split('T')[0];
+    const markdownPath = `./eval/analysis/individual/${evaluatorType}-evaluation-${dateStamp}.md`;
+    const jsonPath = `./eval/analysis/individual/${evaluatorType}-results-${dateStamp}.json`;
+    const reportDir = './eval/analysis/individual';
+    // Ensure report directory exists
+    const fs = await Promise.resolve().then(() => __importStar(require('fs')));
+    if (!fs.existsSync(reportDir)) {
+        fs.mkdirSync(reportDir, { recursive: true });
+    }
+    fs.writeFileSync(markdownPath, reportContent);
+    fs.writeFileSync(jsonPath, JSON.stringify(jsonResults, null, 2));
+    console.log(`📊 ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} reports generated:`);
+    console.log(`   📝 Markdown: ${markdownPath}`);
+    console.log(`   📄 JSON: ${jsonPath}`);
+    // Brief console summary
+    console.log(`🏆 ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} Results:`);
+    results.forEach((result, index) => {
+        console.log(`   ${index + 1}. ${result.key}: ${result.bestModel} (${result.score})`);
+    });
+    return results;
+}
+async function main() {
+    console.log('🔬 Starting Multi-Model Comparative Evaluation\n');
+    // Clean old debug files but preserve evaluation datasets
+    console.log('🧹 Cleaning old debug files...');
+    try {
+        await execAsync('find ./tmp/debug-ai -type f ! -name \'*.jsonl\' -delete 2>/dev/null || true');
+        await execAsync('mkdir -p ./tmp/debug-ai');
+        console.log('✅ Debug files cleaned (datasets preserved)\n');
+    }
+    catch (error) {
+        console.warn('⚠️  Could not clean debug files:', error instanceof Error ? error.message : String(error));
+    }
+    // Check model metadata freshness before starting any evaluation work
+    const modelMetadata = loadModelMetadata();
+    const datasetsDir = './eval/datasets';
+    // Parse command line arguments for subset evaluation
+    const args = process.argv.slice(2);
+    let filterType = undefined;
+    if (args.length > 0) {
+        const requestedType = args[0];
+        if (requestedType in EVALUATOR_CONFIG) {
+            filterType = requestedType;
+        }
+        else {
+            console.error(`❌ Invalid evaluation type: "${requestedType}"`);
+            console.error(`✅ Available types: ${Object.keys(EVALUATOR_CONFIG).join(', ')}`);
+            process.exit(1);
+        }
+    }
+    const availableDatasets = await detectAvailableDatasets(datasetsDir, filterType);
+    console.log('🔍 Dataset Detection:');
+    for (const [type, available] of Object.entries(availableDatasets)) {
+        console.log(`- ${type.charAt(0).toUpperCase() + type.slice(1)} datasets: ${available ? '✅' : '❌'}`);
+    }
+    if (filterType) {
+        console.log(`\n🎯 Running evaluation for: ${filterType}`);
+    }
+    const hasAnyDatasets = Object.values(availableDatasets).some(Boolean);
+    if (!hasAnyDatasets) {
+        if (filterType) {
+            console.error(`❌ No datasets found for type: ${filterType}`);
+        }
+        else {
+            console.error('❌ No evaluation datasets found. Please run integration tests first to generate datasets.');
+        }
+        process.exit(1);
+    }
+    try {
+        const allResults = [];
+        // If filterType is specified, only run that evaluation type
+        if (filterType) {
+            if (availableDatasets[filterType]) {
+                const results = await runEvaluation(filterType, datasetsDir, modelMetadata);
+                allResults.push(...results);
+            }
+            else {
+                console.error(`❌ No datasets available for type: ${filterType}`);
+                process.exit(1);
+            }
+        }
+        else {
+            // Run all available evaluations
+            for (const [type, available] of Object.entries(availableDatasets)) {
+                if (available) {
+                    const results = await runEvaluation(type, datasetsDir, modelMetadata);
+                    allResults.push(...results);
+                }
+            }
+        }
+        console.log(`\n🎉 All Evaluations Complete! Total scenarios analyzed: ${allResults.length}`);
+        console.log(`📁 Check ./eval/reports/ for detailed analysis reports\n`);
+    }
+    catch (error) {
+        console.error('❌ Evaluation failed:', error);
+        process.exit(1);
+    }
+}
+// Run if this file is executed directly
+if (require.main === module) {
+    main().catch(console.error);
+}

package/dist/evaluation/evaluators/base-comparative.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Base Comparative Evaluator
+ *
+ * Shared functionality for comparing multiple AI models across scenarios
+ * Eliminates code duplication between remediation, recommendation, and capability evaluators
+ */
+import { EvaluationScore } from './base.js';
+import { VercelProvider } from '../../core/providers/vercel-provider';
+import { DatasetAnalyzer, ComparisonScenario } from '../dataset-analyzer.js';
+export interface ComparativeEvaluationResult {
+    scenario_summary: string;
+    models_compared: string[];
+    comparative_analysis: Record<string, {
+        quality_score?: number;
+        efficiency_score?: number;
+        performance_score?: number;
+        communication_score?: number;
+        accuracy_score?: number;
+        completeness_score?: number;
+        clarity_score?: number;
+        consistency_score?: number;
+        weighted_total: number;
+        strengths: string;
+        weaknesses: string;
+    }>;
+    ranking: Array<{
+        rank: number;
+        model: string;
+        score: number;
+        rationale?: string;
+        reasoning?: string;
+    }>;
+    overall_insights?: string;
+}
+export interface ComparativeEvaluationScore extends EvaluationScore {
+    modelRankings: Array<{
+        rank: number;
+        model: string;
+        score: number;
+    }>;
+    bestModel: string;
+    modelCount: number;
+}
+export declare abstract class BaseComparativeEvaluator {
+    abstract readonly name: string;
+    abstract readonly description: string;
+    protected abstract readonly promptFileName: string;
+    protected abstract readonly toolName: string;
+    protected evaluatorModel: VercelProvider;
+    protected datasetAnalyzer: DatasetAnalyzer;
+    protected promptTemplate: string;
+    constructor(datasetDir?: string);
+    /**
+     * Initialize the evaluator - must be called by subclass constructor
+     */
+    protected initializePrompt(): void;
+    /**
+     * Evaluate all available models for scenarios
+     * This method finds all scenarios with multiple model responses and evaluates them comparatively
+     */
+    evaluateAllScenarios(): Promise<ComparativeEvaluationScore[]>;
+    /**
+     * Conduct final assessment across all scenarios to determine overall winner
+     */
+    conductFinalAssessment(scenarioResults: ComparativeEvaluationScore[]): Promise<any>;
+    /**
+     * Evaluate a single scenario comparing all available models
+     */
+    evaluateScenario(scenario: ComparisonScenario): Promise<ComparativeEvaluationScore>;
+    /**
+     * Build the evaluation prompt - can be overridden by subclasses for custom behavior
+     */
+    protected buildEvaluationPrompt(scenario: ComparisonScenario, modelResponsesText: string, modelList: string): string;
+    /**
+     * Get statistics about available datasets
+     */
+    getDatasetStats(): {
+        totalDatasets: number;
+        availableModels: string[];
+        scenariosWithMultipleModels: number;
+        interactionTypes: string[];
+    };
+    /**
+     * Get detailed breakdown of evaluation phases available
+     * Must be implemented by subclasses to provide domain-specific phase descriptions
+     */
+    abstract getEvaluationPhases(): {
+        phase: string;
+        description: string;
+        availableModels: string[];
+        scenarioCount: number;
+    }[];
+}
+//# sourceMappingURL=base-comparative.d.ts.map

package/dist/evaluation/evaluators/base-comparative.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/base-comparative.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE7E,MAAM,WAAW,2BAA2B;IAC1C,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE;QACnC,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,0BAA2B,SAAQ,eAAe;IACjE,aAAa,EAAE,KAAK,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,8BAAsB,wBAAwB;IAC5C,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IACnD,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE7C,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,eAAe,EAAE,eAAe,CAAC;IAC3C,SAAS,CAAC,cAAc,EAAE,MAAM,CAAC;gBAErB,UAAU,CAAC,EAAE,MAAM;IAe/B;;OAEG;IACH,SAAS,CAAC,gBAAgB;IAK1B;;;OAGG;IACG,oBAAoB,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;IAkBnE;;OAEG;IACG,sBAAsB,CAAC,eAAe,EAAE,0BAA0B,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC;IA6CzF;;OAEG;IACG,gBAAgB,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAsFzF;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAQpH;;OAEG;IACH,eAAe;;;;;;IAIf;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,IAAI;QAC9B,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CACJ"}

package/dist/evaluation/evaluators/base-comparative.js

@@ -0,0 +1,187 @@
+"use strict";
+/**
+ * Base Comparative Evaluator
+ *
+ * Shared functionality for comparing multiple AI models across scenarios
+ * Eliminates code duplication between remediation, recommendation, and capability evaluators
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseComparativeEvaluator = void 0;
+const vercel_provider_1 = require("../../core/providers/vercel-provider");
+const model_config_1 = require("../../core/model-config");
+const platform_utils_1 = require("../../core/platform-utils");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const dataset_analyzer_js_1 = require("../dataset-analyzer.js");
+class BaseComparativeEvaluator {
+    evaluatorModel;
+    datasetAnalyzer;
+    promptTemplate;
+    constructor(datasetDir) {
+        // Use Claude via VercelProvider as the evaluator (most reliable for complex comparative evaluation)
+        this.evaluatorModel = new vercel_provider_1.VercelProvider({
+            provider: 'anthropic',
+            apiKey: process.env.ANTHROPIC_API_KEY,
+            model: (0, model_config_1.getCurrentModel)('anthropic'),
+            debugMode: process.env.DEBUG_DOT_AI === 'true'
+        });
+        this.datasetAnalyzer = new dataset_analyzer_js_1.DatasetAnalyzer(datasetDir || './eval/datasets');
+        // Prompt template will be loaded by subclass
+        this.promptTemplate = '';
+    }
+    /**
+     * Initialize the evaluator - must be called by subclass constructor
+     */
+    initializePrompt() {
+        const promptPath = (0, path_1.join)(process.cwd(), 'src', 'evaluation', 'prompts', this.promptFileName);
+        this.promptTemplate = (0, fs_1.readFileSync)(promptPath, 'utf8');
+    }
+    /**
+     * Evaluate all available models for scenarios
+     * This method finds all scenarios with multiple model responses and evaluates them comparatively
+     */
+    async evaluateAllScenarios() {
+        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+        const results = [];
+        console.log(`Found ${scenarios.length} scenarios with multiple models for comparative evaluation`);
+        for (const scenario of scenarios) {
+            try {
+                const result = await this.evaluateScenario(scenario);
+                results.push(result);
+            }
+            catch (error) {
+                console.error(`Failed to evaluate scenario ${scenario.interaction_id}:`, error);
+            }
+        }
+        return results;
+    }
+    /**
+     * Conduct final assessment across all scenarios to determine overall winner
+     */
+    async conductFinalAssessment(scenarioResults) {
+        if (scenarioResults.length === 0) {
+            throw new Error('No scenario results provided for final assessment');
+        }
+        // Load the overall winner assessment prompt
+        const promptPath = (0, path_1.join)(process.cwd(), 'src', 'evaluation', 'prompts', 'overall-winner-assessment.md');
+        const overallWinnerTemplate = (0, fs_1.readFileSync)(promptPath, 'utf8');
+        // Get all models that should have been tested (from first scenario)
+        const allModels = scenarioResults[0]?.modelRankings?.map(r => r.model) || [];
+        // Build the final assessment prompt with raw data
+        const finalPrompt = overallWinnerTemplate
+            .replace('{tool_type}', this.toolName)
+            .replace('{total_scenarios}', scenarioResults.length.toString())
+            .replace('{expected_models}', JSON.stringify(allModels))
+            .replace('{scenario_results}', JSON.stringify(scenarioResults, null, 2));
+        try {
+            console.log(`\n🔍 Conducting final assessment across ${scenarioResults.length} scenarios for ${this.toolName}\n`);
+            const response = await this.evaluatorModel.sendMessage(finalPrompt, `${this.name}-final-assessment`, {
+                user_intent: `Final cross-scenario assessment for ${this.toolName}`,
+                interaction_id: 'final-assessment'
+            });
+            // Extract JSON from AI response
+            const finalAssessment = (0, platform_utils_1.extractJsonFromAIResponse)(response.content);
+            console.log(`✅ Final Assessment Complete for ${this.toolName}`);
+            console.log(`🏆 Overall Winner: ${finalAssessment.overall_assessment?.winner || 'Unknown'}`);
+            return finalAssessment;
+        }
+        catch (error) {
+            console.error(`Final assessment failed for ${this.toolName}:`, error);
+            throw error;
+        }
+    }
+    /**
+     * Evaluate a single scenario comparing all available models
+     */
+    async evaluateScenario(scenario) {
+        // Build model responses section for the prompt
+        const modelResponsesText = scenario.models.map((modelResponse, index) => {
+            // Build failure analysis context
+            let reliabilityContext = '✅ Completed successfully';
+            if (modelResponse.metadata.failure_analysis) {
+                const failure = modelResponse.metadata.failure_analysis;
+                reliabilityContext = `⚠️  **${failure.failure_type.toUpperCase()} FAILURE**: ${failure.failure_reason}`;
+                if (failure.failure_type === 'timeout') {
+                    reliabilityContext += `\n- **Time to failure**: ${Math.round(failure.time_to_failure / 1000)}s (${Math.round(failure.time_to_failure / 60000)}min)`;
+                    reliabilityContext += `\n- **Impact**: Model could not complete full workflow within time limit`;
+                }
+            }
+            return `### Model ${index + 1}: ${modelResponse.model}
+
+**Performance Metrics:**
+- Duration: ${modelResponse.performance.duration_ms}ms
+- Input Tokens: ${modelResponse.performance.input_tokens}
+- Output Tokens: ${modelResponse.performance.output_tokens}
+- Total Tokens: ${modelResponse.performance.total_tokens}
+- Iterations: ${modelResponse.performance.iterations || 'N/A'}
+- Tool Calls: ${modelResponse.performance.tool_calls_executed || 'N/A'}
+- Cache Read: ${modelResponse.performance.cache_read_tokens || 0} tokens
+- Cache Creation: ${modelResponse.performance.cache_creation_tokens || 0} tokens
+
+**Reliability Status:**
+${reliabilityContext}
+
+**Response:**
+${modelResponse.response}
+
+---`;
+        }).join('\n\n');
+        const modelList = scenario.models.map(m => m.model).join('", "');
+        // Generate the comparative evaluation prompt
+        const evaluationPrompt = this.buildEvaluationPrompt(scenario, modelResponsesText, modelList);
+        try {
+            const response = await this.evaluatorModel.sendMessage(evaluationPrompt, `${this.name}-${scenario.interaction_id}`, {
+                user_intent: `Comparative ${this.name} evaluation for ${scenario.interaction_id}`,
+                interaction_id: scenario.interaction_id
+            });
+            // Extract JSON from AI response with robust parsing
+            const evaluation = (0, platform_utils_1.extractJsonFromAIResponse)(response.content);
+            // Convert to standard EvaluationScore format
+            const rankings = evaluation.ranking || [];
+            const bestModel = rankings.length > 0 ? rankings[0].model : scenario.models[0].model;
+            const bestScore = rankings.length > 0 ? rankings[0].score : 0;
+            return {
+                key: `${this.name}_${scenario.interaction_id}`,
+                score: bestScore,
+                comment: evaluation.overall_insights || 'Comparative evaluation completed',
+                confidence: 0.9, // High confidence for comparative evaluation
+                modelRankings: rankings.map(r => ({
+                    rank: r.rank,
+                    model: r.model,
+                    score: r.score
+                })),
+                bestModel,
+                modelCount: scenario.models.length
+            };
+        }
+        catch (error) {
+            console.error(`Comparative evaluation failed for ${scenario.interaction_id}:`, error);
+            return {
+                key: `${this.name}_${scenario.interaction_id}`,
+                score: 0,
+                comment: `Evaluation error: ${error}`,
+                confidence: 0,
+                modelRankings: [],
+                bestModel: 'unknown',
+                modelCount: scenario.models.length
+            };
+        }
+    }
+    /**
+     * Build the evaluation prompt - can be overridden by subclasses for custom behavior
+     */
+    buildEvaluationPrompt(scenario, modelResponsesText, modelList) {
+        return this.promptTemplate
+            .replace('{issue}', scenario.issue)
+            .replace('{model_responses}', modelResponsesText)
+            .replace('{model_list}', modelList)
+            .replace('{phase}', scenario.interaction_id);
+    }
+    /**
+     * Get statistics about available datasets
+     */
+    getDatasetStats() {
+        return this.datasetAnalyzer.getDatasetStats(this.toolName);
+    }
+}
+exports.BaseComparativeEvaluator = BaseComparativeEvaluator;