@vfarcic/dot-ai 0.112.0 → 0.114.0

package/dist/core/schema.js

@@ -352,10 +352,8 @@ class ResourceRecommender {
             const capabilityFilteredResources = relevantCapabilities.map(cap => ({
                 kind: this.extractKindFromResourceName(cap.data.resourceName),
                 group: this.extractGroupFromResourceName(cap.data.resourceName),
-                apiVersion: this.constructApiVersionFromResourceName(cap.data.resourceName),
                 resourceName: cap.data.resourceName,
-                namespaced: true, // Default assumption, could be enhanced
-                capabilities: cap.data // Include capability data for AI decision-making
+                capabilities: cap.data // Include capability data for AI decision-making (includes namespaced, etc.)
             }));
             // Phase 1: Add missing pattern-suggested resources to available resources list
             const enhancedResources = await this.addMissingPatternResources(capabilityFilteredResources, relevantPatterns);
@@ -433,9 +431,8 @@ class ResourceRecommender {
         const template = (0, shared_prompt_loader_1.loadPrompt)('resource-selection');
         // Format resources for the prompt with capability information
         const resourcesText = resources.map((resource, index) => {
-            return `${index}: ${resource.kind.toUpperCase()} (${resource.apiVersion})
+            return `${index}: ${resource.kind.toUpperCase()}
    Group: ${resource.group || 'core'}
-   Namespaced: ${resource.namespaced}
    Resource Name: ${resource.resourceName}
    Capabilities: ${Array.isArray(resource.capabilities.capabilities) ? resource.capabilities.capabilities.join(', ') : 'Not specified'}
    Providers: ${Array.isArray(resource.capabilities.providers) ? resource.capabilities.providers.join(', ') : resource.capabilities.providers || 'kubernetes'}
@@ -484,14 +481,10 @@ class ResourceRecommender {
                             const parts = suggestedResource.split('.');
                             const kind = parts[0]; // Use resource name as-is: resourcegroups, servicemonitors, etc.
                             const group = parts.length > 1 ? parts.slice(1).join('.') : '';
-                            const version = 'v1beta1'; // Default version for CRDs, could be enhanced
-                            const apiVersion = group ? `${group}/${version}` : version;
                             missingPatternResources.push({
                                 kind,
                                 group,
-                                apiVersion,
                                 resourceName,
-                                namespaced: true, // Default assumption for pattern resources
                                 capabilities: {
                                     resourceName,
                                     description: `Resource suggested by organizational pattern: ${pattern.description}`,
@@ -551,17 +544,8 @@ class ResourceRecommender {
         // Return everything after the first dot
         return resourceName.substring(resourceName.indexOf('.') + 1);
     }
-    /**
-     * Construct API version from resource name (simplified approach)
-     */
-    constructApiVersionFromResourceName(resourceName) {
-        if (!resourceName.includes('.')) {
-            return 'v1'; // Core resources typically use v1
-        }
-        // For CRDs, construct group/version format
-        const group = this.extractGroupFromResourceName(resourceName);
-        return `${group}/v1beta1`; // Default to v1beta1 for CRDs
-    }
+    // Note: constructApiVersionFromResourceName method removed - no longer needed
+    // API versions are extracted from kubectl explain schema content during manifest generation
     /**
      * Phase 0: Search for relevant organizational patterns using multi-concept approach
      * Returns empty array if Vector DB is not available - this is completely optional
@@ -585,98 +569,6 @@ class ResourceRecommender {
     }
     // REMOVED: selectResourceCandidates - replaced by single-phase assembleAndRankSolutions
     // REMOVED: fetchDetailedSchemas - no longer needed in single-phase architecture
-    /**
-        const basic = `${index}: ${resource.kind} (${resource.apiVersion})
-     Group: ${resource.group || 'core'}
-     Namespaced: ${resource.namespaced}`;
-        
-        // Include rich capability context if available (from capability-based pre-filtering)
-        if (resource.capabilities) {
-          const cap = resource.capabilities;
-          return `${basic}
-     Resource Name: ${resource.resourceName || 'Not specified'}
-     Capabilities: ${cap.capabilities?.join(', ') || 'Not specified'}
-     Providers: ${cap.providers?.join(', ') || 'Not specified'}
-     Complexity: ${cap.complexity || 'Not specified'}
-     Use Case: ${cap.useCase || 'Not specified'}
-     Description: ${cap.description || 'Not specified'}
-     Confidence: ${cap.confidence || 'N/A'}`;
-        }
-        
-        return basic;
-      }).join('\n\n');
-  
-      // Format organizational patterns for AI context
-      const patternsContext = patterns.length > 0
-        ? patterns.map(pattern =>
-            `- ID: ${pattern.id}
-              Description: ${pattern.description}
-              Suggested Resources: ${pattern.suggestedResources?.join(', ') || 'Not specified'}
-              Rationale: ${pattern.rationale}
-              Triggers: ${pattern.triggers?.join(', ') || 'None'}`
-          ).join('\n')
-        : 'No organizational patterns found for this request.';
-  
-  
-      const template = loadPrompt('resource-selection');
-      
-      const selectionPrompt = template
-        .replace('{intent}', intent)
-        .replace('{resources}', resourceSummary)
-        .replace('{patterns}', patternsContext);
-  
-  
-      const response = await this.aiProvider.sendMessage(selectionPrompt, 'resource-selection');
-      
-      try {
-        // Extract JSON from response with robust parsing
-        let jsonContent = response.content;
-        
-        // First try to find JSON array wrapped in code blocks
-        const codeBlockMatch = response.content.match(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/);
-        if (codeBlockMatch) {
-          jsonContent = codeBlockMatch[1];
-        } else {
-          // Try to find JSON array that starts with [ and find the matching closing ]
-          const startIndex = response.content.indexOf('[');
-          if (startIndex !== -1) {
-            let bracketCount = 0;
-            let endIndex = startIndex;
-            
-            for (let i = startIndex; i < response.content.length; i++) {
-              if (response.content[i] === '[') bracketCount++;
-              if (response.content[i] === ']') bracketCount--;
-              if (bracketCount === 0) {
-                endIndex = i;
-                break;
-              }
-            }
-            
-            if (bracketCount === 0) {
-              jsonContent = response.content.substring(startIndex, endIndex + 1);
-            }
-          }
-        }
-        
-        const selectedResources = JSON.parse(jsonContent.trim());
-        
-        if (!Array.isArray(selectedResources)) {
-          throw new Error('AI response is not an array');
-        }
-        
-        // Validate that each resource has required fields
-        for (const resource of selectedResources) {
-          if (!resource.kind || !resource.apiVersion) {
-            throw new Error(`AI selected invalid resource: ${JSON.stringify(resource)}`);
-          }
-        }
-        
-        return selectedResources;
-      } catch (error) {
-        throw new Error(`AI failed to select resources in valid JSON format. Error: ${(error as Error).message}. AI response: "${response.content.substring(0, 200)}..."`);
-      }
-    }
-  
     /**
      * Phase 2: Fetch detailed schemas for selected candidates
      */

package/dist/core/unified-creation-session.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"unified-creation-session.d.ts","sourceRoot":"","sources":["../../src/core/unified-creation-session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAQH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAIlD,OAAO,EACL,sBAAsB,EACtB,2BAA2B,EAC3B,iCAAiC,EACjC,UAAU,EAIX,MAAM,0BAA0B,CAAC;AAKlC,qBAAa,6BAA6B;IACxC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,SAAS,CAAsB;gBAE3B,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,mBAAmB;IAKnE;;OAEG;IACH,aAAa,CAAC,IAAI,EAAE,GAAG,GAAG,sBAAsB;IAqBhD;;OAEG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,sBAAsB,GAAG,IAAI;IAiBxE;;OAEG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,sBAAsB;IA2GvF;;OAEG;IACG,mBAAmB,CAAC,OAAO,EAAE,sBAAsB,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,2BAA2B,GAAG,iCAAiC,CAAC;IAoGhJ;;OAEG;YACW,4BAA4B;IA4C1C;;OAEG;YACW,gCAAgC;IAyC9C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA6E1B;;OAEG;YACW,gBAAgB;IAsF9B;;OAEG;YACW,4BAA4B;IAoH1C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAY1B;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;YACW,mBAAmB;IAwLjC;;OAEG;YACW,uBAAuB;IAmFrC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAU5B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBnB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAG1B"}
+{"version":3,"file":"unified-creation-session.d.ts","sourceRoot":"","sources":["../../src/core/unified-creation-session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAQH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKlD,OAAO,EACL,sBAAsB,EACtB,2BAA2B,EAC3B,iCAAiC,EACjC,UAAU,EAIX,MAAM,0BAA0B,CAAC;AAKlC,qBAAa,6BAA6B;IACxC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,SAAS,CAAsB;gBAE3B,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,mBAAmB;IAKnE;;OAEG;IACH,aAAa,CAAC,IAAI,EAAE,GAAG,GAAG,sBAAsB;IAqBhD;;OAEG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,sBAAsB,GAAG,IAAI;IAiBxE;;OAEG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,sBAAsB;IA2GvF;;OAEG;IACG,mBAAmB,CAAC,OAAO,EAAE,sBAAsB,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,2BAA2B,GAAG,iCAAiC,CAAC;IAoGhJ;;OAEG;YACW,4BAA4B;IA4C1C;;OAEG;YACW,gCAAgC;IAyC9C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA6E1B;;OAEG;YACW,gBAAgB;IAsF9B;;OAEG;YACW,4BAA4B;IAoH1C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAY1B;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;YACW,mBAAmB;IAyLjC;;OAEG;YACW,uBAAuB;IAmFrC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAU5B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBnB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAG1B"}

package/dist/core/unified-creation-session.js

@@ -50,6 +50,7 @@ const capability_vector_service_1 = require("./capability-vector-service");
 const discovery_1 = require("./discovery");
 const schema_1 = require("./schema");
 const version_1 = require("../tools/version");
+const platform_utils_1 = require("./platform-utils");
 const yaml = __importStar(require("js-yaml"));
 const unified_creation_types_1 = require("./unified-creation-types");
 const pattern_operations_1 = require("./pattern-operations");
@@ -743,7 +744,8 @@ The policy intent has been stored in the database. The Kyverno policy was not ap
                         interaction_id: args?.interaction_id || 'kyverno_generation'
                     });
                     // Response should be clean YAML with analysis comments
-                    const kyvernoPolicy = response.content.trim();
+                    // Extract YAML from code blocks if wrapped (using shared utility)
+                    const kyvernoPolicy = (0, platform_utils_1.extractContentFromMarkdownCodeBlocks)(response.content, 'yaml');
                     // Save policy to file immediately after generation
                     const yamlPath = path.join(policySessionDir, `${session.sessionId}-kyverno.yaml`);
                     fs.writeFileSync(yamlPath, kyvernoPolicy, 'utf8');

package/dist/evaluation/eval-runner.js

@@ -46,6 +46,9 @@ const capability_comparative_js_1 = require("./evaluators/capability-comparative
 const pattern_comparative_js_1 = require("./evaluators/pattern-comparative.js");
 const policy_comparative_js_1 = require("./evaluators/policy-comparative.js");
 const promises_1 = require("fs/promises");
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
 const EVALUATOR_CONFIG = {
     remediation: {
         evaluator: remediation_comparative_js_1.RemediationComparativeEvaluator,
@@ -73,15 +76,13 @@ const EVALUATOR_CONFIG = {
         title: 'Policy AI Model Comparison Report'
     }
 };
-function generateMarkdownReport(results, stats, evaluationType) {
+function generateMarkdownReport(results, stats, evaluationType, finalAssessment) {
     const timestamp = new Date().toISOString();
-    // Calculate overall statistics
-    const modelWins = new Map();
+    // Use final assessment if provided
+    const overallAssessment = finalAssessment?.overall_assessment || null;
+    // Calculate basic statistics for reference
     const modelScores = new Map();
     results.forEach(result => {
-        const winner = result.bestModel;
-        modelWins.set(winner, (modelWins.get(winner) || 0) + 1);
-        // Collect all scores for each model
         if (result.modelRankings) {
             result.modelRankings.forEach((ranking) => {
                 if (!modelScores.has(ranking.model)) {
@@ -91,8 +92,7 @@ function generateMarkdownReport(results, stats, evaluationType) {
             });
         }
     });
-    const sortedWins = Array.from(modelWins.entries()).sort((a, b) => b[1] - a[1]);
-    // Calculate average scores
+    // Calculate average scores for supplementary information
     const modelAverages = new Map();
     modelScores.forEach((scores, model) => {
         const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
@@ -108,20 +108,39 @@ function generateMarkdownReport(results, stats, evaluationType) {
 
 ## Executive Summary
 
-### 🏆 Overall Winners
-${sortedWins.map(([model, wins]) => `- **${model}**: ${wins} scenario${wins > 1 ? 's' : ''} won`).join('\n')}
+### 🏆 Overall Winner (AI Assessment)
+${overallAssessment ? `
+**${overallAssessment.winner}**
 
-### 📊 Model Performance Overview
+${overallAssessment.rationale}
+` : 'Overall assessment not available'}
 
-| Model | Avg Score | Scenarios Won | Performance Notes |
-|-------|-----------|---------------|-------------------|
+### 📊 AI Reliability Rankings
+
+${overallAssessment ? overallAssessment.reliability_ranking
+        .map((ranking, index) => `${index + 1}. **${ranking.model}** (${Math.round(ranking.reliability_score * 100)}%) - ${ranking.reliability_notes}`)
+        .join('\n') : 'Reliability rankings not available'}
+
+### 📋 Production Recommendations
+
+${overallAssessment ? `
+- **Primary Choice**: ${overallAssessment.production_recommendations.primary}
+- **Secondary Option**: ${overallAssessment.production_recommendations.secondary}
+- **Avoid for Production**: ${overallAssessment.production_recommendations.avoid.length > 0 ? overallAssessment.production_recommendations.avoid.join(', ') : 'None'}
+${Object.keys(overallAssessment.production_recommendations.specialized_use).length > 0 ?
+        '\n**Specialized Use Cases:**\n' + Object.entries(overallAssessment.production_recommendations.specialized_use)
+            .map(([useCase, model]) => `- **${useCase}**: ${model}`)
+            .join('\n') : ''}
+` : 'Production recommendations not available'}
+
+### 📊 Supplementary Statistics (Reference Only)
+
+| Model | Avg Score | Notes |
+|-------|-----------|-------|
 ${Array.from(modelAverages.entries())
         .sort((a, b) => b[1] - a[1])
-        .map(([model, avgScore]) => {
-        const wins = modelWins.get(model) || 0;
-        const performance = wins > 0 ? '🟢 Strong' : wins === 0 && avgScore > 0.8 ? '🟡 Good' : '🔴 Weak';
-        return `| ${model} | ${avgScore} | ${wins} | ${performance} |`;
-    }).join('\n')}
+        .map(([model, avgScore]) => `| ${model} | ${avgScore} | See AI assessment above |`)
+        .join('\n')}
 
 ## Detailed Scenario Results
 
@@ -142,14 +161,21 @@ ${result.comment}
 ---`;
     }).join('\n\n')}
 
-## Model Selection Guide
-${sortedWins.slice(0, 3).map(([model], index) => {
-        const score = modelAverages.get(model) || 0;
-        const useCase = index === 0 ? 'Primary choice for most scenarios' :
-            index === 1 ? 'Good alternative with different strengths' :
-                'Specialized use cases';
-        return `- **${model}** (Avg: ${score}): ${useCase}`;
-    }).join('\n')}
+## AI Model Selection Guide
+
+${overallAssessment ? `
+### Key Insights
+${overallAssessment.key_insights}
+
+### Recommended Selection Strategy
+- **For Production Use**: Choose ${overallAssessment.production_recommendations.primary}
+- **For Secondary Option**: Consider ${overallAssessment.production_recommendations.secondary}
+${overallAssessment.production_recommendations.avoid.length > 0 ?
+        `- **Avoid**: ${overallAssessment.production_recommendations.avoid.join(', ')} (reliability concerns)` : ''}
+
+### Decision Framework
+The AI assessment prioritizes **reliability and consistency** over peak performance. Models that fail completely in any scenario are heavily penalized, ensuring production-ready recommendations.
+` : 'AI model selection guide not available'}
 
 ---
 
@@ -158,12 +184,74 @@ ${sortedWins.slice(0, 3).map(([model], index) => {
 Report generated by DevOps AI Toolkit Comparative Evaluation System
 `;
 }
-async function detectAvailableDatasets(datasetsDir) {
+function loadModelMetadata() {
+    try {
+        const fs = require('fs');
+        const path = require('path');
+        const metadataPath = path.join(__dirname, 'model-metadata.json');
+        if (!fs.existsSync(metadataPath)) {
+            console.error('❌ Model metadata file not found');
+            console.error('📊 Pricing and capabilities data required for cost analysis');
+            console.error('');
+            console.error('🔄 To create model metadata, run:');
+            console.error('   /update-model-metadata');
+            console.error('');
+            process.exit(1);
+        }
+        const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
+        // Check if metadata is older than 30 days
+        const metadataAge = Date.now() - new Date(metadata.lastUpdated).getTime();
+        const thirtyDays = 30 * 24 * 60 * 60 * 1000;
+        if (metadataAge > thirtyDays) {
+            console.error('❌ Model metadata is over 30 days old (last updated: ' + metadata.lastUpdated + ')');
+            console.error('📊 Pricing and capabilities data may be outdated, affecting cost analysis accuracy');
+            console.error('');
+            console.error('🔄 To update model metadata, run:');
+            console.error('   /update-model-metadata');
+            console.error('');
+            process.exit(1);
+        }
+        console.log('✅ Model metadata loaded (updated: ' + metadata.lastUpdated + ')');
+        return metadata;
+    }
+    catch (error) {
+        console.error('❌ Failed to load model metadata:', error instanceof Error ? error.message : String(error));
+        console.error('🔄 To create model metadata, run: /update-model-metadata');
+        process.exit(1);
+    }
+}
+function generateJsonReport(results, stats, evaluationType, modelMetadata, finalAssessment) {
+    const timestamp = new Date().toISOString();
+    // Use final assessment if provided
+    const overallAssessment = finalAssessment || null;
+    return {
+        metadata: {
+            reportType: 'comparative-evaluation',
+            evaluationType: evaluationType,
+            generated: timestamp,
+            scenariosAnalyzed: results.length,
+            modelsEvaluated: stats.availableModels.length,
+            totalDatasets: stats.totalDatasets,
+            tool: EVALUATOR_CONFIG[evaluationType].title
+        },
+        modelMetadata: modelMetadata.models,
+        overallAssessment: overallAssessment,
+        results: results,
+        summary: stats
+    };
+}
+async function detectAvailableDatasets(datasetsDir, filterType) {
     try {
         const files = await (0, promises_1.readdir)(datasetsDir);
         const result = {};
         for (const [type, config] of Object.entries(EVALUATOR_CONFIG)) {
-            result[type] = files.some(file => file.startsWith(config.prefix));
+            // If filter specified, only check for that type
+            if (filterType && type !== filterType) {
+                result[type] = false;
+            }
+            else {
+                result[type] = files.some(file => file.startsWith(config.prefix));
+            }
         }
         return result;
     }
@@ -176,7 +264,7 @@ async function detectAvailableDatasets(datasetsDir) {
         return result;
     }
 }
-async function runEvaluation(evaluatorType, datasetsDir) {
+async function runEvaluation(evaluatorType, datasetsDir, modelMetadata) {
     const EvaluatorClass = EVALUATOR_CONFIG[evaluatorType].evaluator;
     const evaluator = new EvaluatorClass(datasetsDir);
     console.log(`\n🔬 Starting ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} Evaluation\n`);
@@ -201,18 +289,26 @@ async function runEvaluation(evaluatorType, datasetsDir) {
     console.log('🚀 Running Comparative Evaluation...\n');
     const results = await evaluator.evaluateAllScenarios();
     console.log(`✅ ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} Evaluation Complete! Analyzed ${results.length} scenarios\n`);
-    // Generate markdown report
-    const reportContent = generateMarkdownReport(results, stats, evaluatorType);
-    // Save report to file
-    const reportPath = `./eval/reports/${evaluatorType}-evaluation-${new Date().toISOString().split('T')[0]}.md`;
-    const reportDir = './eval/reports';
+    // Conduct final assessment across all scenarios
+    const finalAssessment = await evaluator.conductFinalAssessment(results);
+    // Generate dual-format reports using final assessment
+    const reportContent = generateMarkdownReport(results, stats, evaluatorType, finalAssessment);
+    const jsonResults = generateJsonReport(results, stats, evaluatorType, modelMetadata, finalAssessment);
+    // Save reports to files
+    const dateStamp = new Date().toISOString().split('T')[0];
+    const markdownPath = `./eval/analysis/individual/${evaluatorType}-evaluation-${dateStamp}.md`;
+    const jsonPath = `./eval/analysis/individual/${evaluatorType}-results-${dateStamp}.json`;
+    const reportDir = './eval/analysis/individual';
     // Ensure report directory exists
     const fs = await Promise.resolve().then(() => __importStar(require('fs')));
     if (!fs.existsSync(reportDir)) {
         fs.mkdirSync(reportDir, { recursive: true });
     }
-    fs.writeFileSync(reportPath, reportContent);
-    console.log(`📊 ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} report generated: ${reportPath}`);
+    fs.writeFileSync(markdownPath, reportContent);
+    fs.writeFileSync(jsonPath, JSON.stringify(jsonResults, null, 2));
+    console.log(`📊 ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} reports generated:`);
+    console.log(`   📝 Markdown: ${markdownPath}`);
+    console.log(`   📄 JSON: ${jsonPath}`);
     // Brief console summary
     console.log(`🏆 ${evaluatorType.charAt(0).toUpperCase() + evaluatorType.slice(1)} Results:`);
     results.forEach((result, index) => {
@@ -222,24 +318,72 @@ async function runEvaluation(evaluatorType, datasetsDir) {
 }
 async function main() {
     console.log('🔬 Starting Multi-Model Comparative Evaluation\n');
+    // Clean old debug files but preserve evaluation datasets
+    console.log('🧹 Cleaning old debug files...');
+    try {
+        await execAsync('find ./tmp/debug-ai -type f ! -name \'*.jsonl\' -delete 2>/dev/null || true');
+        await execAsync('mkdir -p ./tmp/debug-ai');
+        console.log('✅ Debug files cleaned (datasets preserved)\n');
+    }
+    catch (error) {
+        console.warn('⚠️  Could not clean debug files:', error instanceof Error ? error.message : String(error));
+    }
+    // Check model metadata freshness before starting any evaluation work
+    const modelMetadata = loadModelMetadata();
     const datasetsDir = './eval/datasets';
-    const availableDatasets = await detectAvailableDatasets(datasetsDir);
+    // Parse command line arguments for subset evaluation
+    const args = process.argv.slice(2);
+    let filterType = undefined;
+    if (args.length > 0) {
+        const requestedType = args[0];
+        if (requestedType in EVALUATOR_CONFIG) {
+            filterType = requestedType;
+        }
+        else {
+            console.error(`❌ Invalid evaluation type: "${requestedType}"`);
+            console.error(`✅ Available types: ${Object.keys(EVALUATOR_CONFIG).join(', ')}`);
+            process.exit(1);
+        }
+    }
+    const availableDatasets = await detectAvailableDatasets(datasetsDir, filterType);
     console.log('🔍 Dataset Detection:');
     for (const [type, available] of Object.entries(availableDatasets)) {
         console.log(`- ${type.charAt(0).toUpperCase() + type.slice(1)} datasets: ${available ? '✅' : '❌'}`);
     }
+    if (filterType) {
+        console.log(`\n🎯 Running evaluation for: ${filterType}`);
+    }
     const hasAnyDatasets = Object.values(availableDatasets).some(Boolean);
     if (!hasAnyDatasets) {
-        console.error('❌ No evaluation datasets found. Please run integration tests first to generate datasets.');
+        if (filterType) {
+            console.error(`❌ No datasets found for type: ${filterType}`);
+        }
+        else {
+            console.error('❌ No evaluation datasets found. Please run integration tests first to generate datasets.');
+        }
         process.exit(1);
     }
     try {
         const allResults = [];
-        for (const [type, available] of Object.entries(availableDatasets)) {
-            if (available) {
-                const results = await runEvaluation(type, datasetsDir);
+        // If filterType is specified, only run that evaluation type
+        if (filterType) {
+            if (availableDatasets[filterType]) {
+                const results = await runEvaluation(filterType, datasetsDir, modelMetadata);
                 allResults.push(...results);
             }
+            else {
+                console.error(`❌ No datasets available for type: ${filterType}`);
+                process.exit(1);
+            }
+        }
+        else {
+            // Run all available evaluations
+            for (const [type, available] of Object.entries(availableDatasets)) {
+                if (available) {
+                    const results = await runEvaluation(type, datasetsDir, modelMetadata);
+                    allResults.push(...results);
+                }
+            }
         }
         console.log(`\n🎉 All Evaluations Complete! Total scenarios analyzed: ${allResults.length}`);
         console.log(`📁 Check ./eval/reports/ for detailed analysis reports\n`);

package/dist/evaluation/evaluators/base-comparative.d.ts

@@ -31,7 +31,6 @@ export interface ComparativeEvaluationResult {
         reasoning?: string;
     }>;
     overall_insights?: string;
-    overall_assessment?: string;
 }
 export interface ComparativeEvaluationScore extends EvaluationScore {
     modelRankings: Array<{
@@ -60,6 +59,10 @@ export declare abstract class BaseComparativeEvaluator {
      * This method finds all scenarios with multiple model responses and evaluates them comparatively
      */
     evaluateAllScenarios(): Promise<ComparativeEvaluationScore[]>;
+    /**
+     * Conduct final assessment across all scenarios to determine overall winner
+     */
+    conductFinalAssessment(scenarioResults: ComparativeEvaluationScore[]): Promise<any>;
     /**
      * Evaluate a single scenario comparing all available models
      */

package/dist/evaluation/evaluators/base-comparative.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/base-comparative.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE7E,MAAM,WAAW,2BAA2B;IAC1C,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE;QACnC,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,0BAA2B,SAAQ,eAAe;IACjE,aAAa,EAAE,KAAK,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,8BAAsB,wBAAwB;IAC5C,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IACnD,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE7C,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,eAAe,EAAE,eAAe,CAAC;IAC3C,SAAS,CAAC,cAAc,EAAE,MAAM,CAAC;gBAErB,UAAU,CAAC,EAAE,MAAM;IAe/B;;OAEG;IACH,SAAS,CAAC,gBAAgB;IAK1B;;;OAGG;IACG,oBAAoB,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;IAkBnE;;OAEG;IACG,gBAAgB,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAsFzF;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAQpH;;OAEG;IACH,eAAe;;;;;;IAIf;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,IAAI;QAC9B,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CACJ"}
+{"version":3,"file":"base-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/base-comparative.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAKtE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE7E,MAAM,WAAW,2BAA2B;IAC1C,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE;QACnC,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,0BAA2B,SAAQ,eAAe;IACjE,aAAa,EAAE,KAAK,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,8BAAsB,wBAAwB;IAC5C,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IACtC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IACnD,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE7C,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,eAAe,EAAE,eAAe,CAAC;IAC3C,SAAS,CAAC,cAAc,EAAE,MAAM,CAAC;gBAErB,UAAU,CAAC,EAAE,MAAM;IAe/B;;OAEG;IACH,SAAS,CAAC,gBAAgB;IAK1B;;;OAGG;IACG,oBAAoB,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;IAkBnE;;OAEG;IACG,sBAAsB,CAAC,eAAe,EAAE,0BAA0B,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC;IA6CzF;;OAEG;IACG,gBAAgB,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAsFzF;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAQpH;;OAEG;IACH,eAAe;;;;;;IAIf;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,IAAI;QAC9B,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CACJ"}

package/dist/evaluation/evaluators/base-comparative.js

@@ -55,6 +55,41 @@ class BaseComparativeEvaluator {
         }
         return results;
     }
+    /**
+     * Conduct final assessment across all scenarios to determine overall winner
+     */
+    async conductFinalAssessment(scenarioResults) {
+        if (scenarioResults.length === 0) {
+            throw new Error('No scenario results provided for final assessment');
+        }
+        // Load the overall winner assessment prompt
+        const promptPath = (0, path_1.join)(process.cwd(), 'src', 'evaluation', 'prompts', 'overall-winner-assessment.md');
+        const overallWinnerTemplate = (0, fs_1.readFileSync)(promptPath, 'utf8');
+        // Get all models that should have been tested (from first scenario)
+        const allModels = scenarioResults[0]?.modelRankings?.map(r => r.model) || [];
+        // Build the final assessment prompt with raw data
+        const finalPrompt = overallWinnerTemplate
+            .replace('{tool_type}', this.toolName)
+            .replace('{total_scenarios}', scenarioResults.length.toString())
+            .replace('{expected_models}', JSON.stringify(allModels))
+            .replace('{scenario_results}', JSON.stringify(scenarioResults, null, 2));
+        try {
+            console.log(`\n🔍 Conducting final assessment across ${scenarioResults.length} scenarios for ${this.toolName}\n`);
+            const response = await this.evaluatorModel.sendMessage(finalPrompt, `${this.name}-final-assessment`, {
+                user_intent: `Final cross-scenario assessment for ${this.toolName}`,
+                interaction_id: 'final-assessment'
+            });
+            // Extract JSON from AI response
+            const finalAssessment = (0, platform_utils_1.extractJsonFromAIResponse)(response.content);
+            console.log(`✅ Final Assessment Complete for ${this.toolName}`);
+            console.log(`🏆 Overall Winner: ${finalAssessment.overall_assessment?.winner || 'Unknown'}`);
+            return finalAssessment;
+        }
+        catch (error) {
+            console.error(`Final assessment failed for ${this.toolName}:`, error);
+            throw error;
+        }
+    }
     /**
      * Evaluate a single scenario comparing all available models
      */
@@ -108,7 +143,7 @@ ${modelResponse.response}
             return {
                 key: `${this.name}_${scenario.interaction_id}`,
                 score: bestScore,
-                comment: evaluation.overall_insights || evaluation.overall_assessment || 'Comparative evaluation completed',
+                comment: evaluation.overall_insights || 'Comparative evaluation completed',
                 confidence: 0.9, // High confidence for comparative evaluation
                 modelRankings: rankings.map(r => ({
                     rank: r.rank,

package/dist/evaluation/platform-synthesizer.d.ts

@@ -0,0 +1,54 @@
+import { VercelProvider } from '../core/providers/vercel-provider.js';
+export interface ModelPerformance {
+    modelId: string;
+    provider: string;
+    toolScores: Record<string, number>;
+    averageScore: number;
+    participationRate: number;
+    reliabilityScore: number;
+    consistencyAcrossTools: number;
+    pricing: {
+        input_cost_per_million_tokens: number;
+        output_cost_per_million_tokens: number;
+    };
+    capabilities: {
+        context_window: number;
+        supports_function_calling: boolean;
+    };
+}
+export interface DecisionMatrix {
+    qualityLeaders: ModelPerformance[];
+    speedOptimized: ModelPerformance[];
+    costEffective: ModelPerformance[];
+    balanced: ModelPerformance[];
+    reliabilityFocused: ModelPerformance[];
+}
+export interface UsageRecommendation {
+    priority: 'quality-first' | 'speed-first' | 'cost-first' | 'balanced';
+    primaryModel: string;
+    fallbackModel: string;
+    reasoning: string;
+    costImplications: string;
+    useCases: string[];
+}
+export declare class PlatformSynthesizer {
+    private aiProvider;
+    private reportsDir;
+    constructor(aiProvider: VercelProvider, reportsDir?: string);
+    generatePlatformWideAnalysis(): Promise<string>;
+    private loadToolMetadata;
+    private loadAllReports;
+    private analyzeCrossToolPerformance;
+    private calculateModelPerformances;
+    private generateDecisionMatrices;
+    private generateUsageRecommendations;
+    private generatePlatformInsights;
+    private extractKeyFindings;
+    private categorizeModelTiers;
+    private identifyCrossToolPatterns;
+    private generateProductionRecommendations;
+    private calculateCostEstimate;
+    private extractBaseModelId;
+    saveSynthesisReport(markdownContent: string, outputPath?: string): Promise<void>;
+}
+//# sourceMappingURL=platform-synthesizer.d.ts.map

package/dist/evaluation/platform-synthesizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"platform-synthesizer.d.ts","sourceRoot":"","sources":["../../src/evaluation/platform-synthesizer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAEtE,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,OAAO,EAAE;QACP,6BAA6B,EAAE,MAAM,CAAC;QACtC,8BAA8B,EAAE,MAAM,CAAC;KACxC,CAAC;IACF,YAAY,EAAE;QACZ,cAAc,EAAE,MAAM,CAAC;QACvB,yBAAyB,EAAE,OAAO,CAAC;KACpC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,gBAAgB,EAAE,CAAC;IACnC,cAAc,EAAE,gBAAgB,EAAE,CAAC;IACnC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,kBAAkB,EAAE,gBAAgB,EAAE,CAAC;CACxC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,YAAY,GAAG,UAAU,CAAC;IACtE,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,UAAU,CAAiB;IACnC,OAAO,CAAC,UAAU,CAAS;gBAEf,UAAU,EAAE,cAAc,EAAE,UAAU,SAA+B;IAK3E,4BAA4B,IAAI,OAAO,CAAC,MAAM,CAAC;IA8BrD,OAAO,CAAC,gBAAgB;YAaV,cAAc;YA0Bd,2BAA2B;IA8DzC,OAAO,CAAC,0BAA0B;IA2ElC,OAAO,CAAC,wBAAwB;IAiDhC,OAAO,CAAC,4BAA4B;YA0CtB,wBAAwB;IAoBtC,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,oBAAoB;IAsB5B,OAAO,CAAC,yBAAyB;IAWjC,OAAO,CAAC,iCAAiC;IASzC,OAAO,CAAC,qBAAqB;IAQ7B,OAAO,CAAC,kBAAkB;IASpB,mBAAmB,CACvB,eAAe,EAAE,MAAM,EACvB,UAAU,SAAiD,GAC1D,OAAO,CAAC,IAAI,CAAC;CAWjB"}