@vfarcic/dot-ai 0.111.0 → 0.113.0

package/dist/evaluation/platform-synthesizer.d.ts

@@ -0,0 +1,54 @@
+import { VercelProvider } from '../core/providers/vercel-provider.js';
+export interface ModelPerformance {
+    modelId: string;
+    provider: string;
+    toolScores: Record<string, number>;
+    averageScore: number;
+    participationRate: number;
+    reliabilityScore: number;
+    consistencyAcrossTools: number;
+    pricing: {
+        input_cost_per_million_tokens: number;
+        output_cost_per_million_tokens: number;
+    };
+    capabilities: {
+        context_window: number;
+        supports_function_calling: boolean;
+    };
+}
+export interface DecisionMatrix {
+    qualityLeaders: ModelPerformance[];
+    speedOptimized: ModelPerformance[];
+    costEffective: ModelPerformance[];
+    balanced: ModelPerformance[];
+    reliabilityFocused: ModelPerformance[];
+}
+export interface UsageRecommendation {
+    priority: 'quality-first' | 'speed-first' | 'cost-first' | 'balanced';
+    primaryModel: string;
+    fallbackModel: string;
+    reasoning: string;
+    costImplications: string;
+    useCases: string[];
+}
+export declare class PlatformSynthesizer {
+    private aiProvider;
+    private reportsDir;
+    constructor(aiProvider: VercelProvider, reportsDir?: string);
+    generatePlatformWideAnalysis(): Promise<string>;
+    private loadToolMetadata;
+    private loadAllReports;
+    private analyzeCrossToolPerformance;
+    private calculateModelPerformances;
+    private generateDecisionMatrices;
+    private generateUsageRecommendations;
+    private generatePlatformInsights;
+    private extractKeyFindings;
+    private categorizeModelTiers;
+    private identifyCrossToolPatterns;
+    private generateProductionRecommendations;
+    private calculateCostEstimate;
+    private extractBaseModelId;
+    saveSynthesisReport(markdownContent: string, outputPath?: string): Promise<void>;
+}
+//# sourceMappingURL=platform-synthesizer.d.ts.map

package/dist/evaluation/platform-synthesizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"platform-synthesizer.d.ts","sourceRoot":"","sources":["../../src/evaluation/platform-synthesizer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAEtE,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,OAAO,EAAE;QACP,6BAA6B,EAAE,MAAM,CAAC;QACtC,8BAA8B,EAAE,MAAM,CAAC;KACxC,CAAC;IACF,YAAY,EAAE;QACZ,cAAc,EAAE,MAAM,CAAC;QACvB,yBAAyB,EAAE,OAAO,CAAC;KACpC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,gBAAgB,EAAE,CAAC;IACnC,cAAc,EAAE,gBAAgB,EAAE,CAAC;IACnC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,kBAAkB,EAAE,gBAAgB,EAAE,CAAC;CACxC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,eAAe,GAAG,aAAa,GAAG,YAAY,GAAG,UAAU,CAAC;IACtE,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,UAAU,CAAiB;IACnC,OAAO,CAAC,UAAU,CAAS;gBAEf,UAAU,EAAE,cAAc,EAAE,UAAU,SAA+B;IAK3E,4BAA4B,IAAI,OAAO,CAAC,MAAM,CAAC;IA8BrD,OAAO,CAAC,gBAAgB;YAaV,cAAc;YA0Bd,2BAA2B;IA8DzC,OAAO,CAAC,0BAA0B;IA2ElC,OAAO,CAAC,wBAAwB;IAiDhC,OAAO,CAAC,4BAA4B;YA0CtB,wBAAwB;IAoBtC,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,oBAAoB;IAsB5B,OAAO,CAAC,yBAAyB;IAWjC,OAAO,CAAC,iCAAiC;IASzC,OAAO,CAAC,qBAAqB;IAQ7B,OAAO,CAAC,kBAAkB;IASpB,mBAAmB,CACvB,eAAe,EAAE,MAAM,EACvB,UAAU,SAAiD,GAC1D,OAAO,CAAC,IAAI,CAAC;CAWjB"}

package/dist/evaluation/platform-synthesizer.js

@@ -0,0 +1,368 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PlatformSynthesizer = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+class PlatformSynthesizer {
+    aiProvider;
+    reportsDir;
+    constructor(aiProvider, reportsDir = './eval/analysis/individual') {
+        this.aiProvider = aiProvider;
+        this.reportsDir = reportsDir;
+    }
+    async generatePlatformWideAnalysis() {
+        console.log('🔍 Loading all evaluation reports...');
+        const allReports = await this.loadAllReports();
+        console.log('🔧 Loading tool metadata...');
+        const toolMetadata = this.loadToolMetadata();
+        console.log('📊 Analyzing cross-tool performance patterns...');
+        const crossToolAnalysis = await this.analyzeCrossToolPerformance(allReports);
+        console.log('🎯 Generating decision matrices...');
+        const decisionMatrices = this.generateDecisionMatrices(crossToolAnalysis.modelPerformances);
+        console.log('💡 Creating usage recommendations...');
+        const usageRecommendations = this.generateUsageRecommendations(crossToolAnalysis, decisionMatrices);
+        console.log('🚀 Generating comprehensive AI-powered report...');
+        const markdownReport = await this.generatePlatformInsights(crossToolAnalysis, decisionMatrices, usageRecommendations, toolMetadata);
+        return markdownReport;
+    }
+    loadToolMetadata() {
+        try {
+            const metadataPath = path.join(process.cwd(), 'src', 'evaluation', 'model-metadata.json');
+            const metadataContent = fs.readFileSync(metadataPath, 'utf8');
+            const metadata = JSON.parse(metadataContent);
+            console.log(`✅ Loaded tool metadata with ${Object.keys(metadata.tools || {}).length} tools`);
+            return metadata.tools || {};
+        }
+        catch (error) {
+            console.warn('⚠️  Failed to load tool metadata, proceeding without it:', error);
+            return {};
+        }
+    }
+    async loadAllReports() {
+        const reports = {};
+        // Load all JSON result files from the directory
+        const reportFiles = fs.readdirSync(this.reportsDir)
+            .filter(file => file.endsWith('-results-*.json') || file.includes('-results-'))
+            .filter(file => file.endsWith('.json'));
+        if (reportFiles.length === 0) {
+            throw new Error(`No evaluation result files found in ${this.reportsDir}`);
+        }
+        for (const fileName of reportFiles) {
+            const reportPath = path.join(this.reportsDir, fileName);
+            const reportContent = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+            // Extract tool type from filename (e.g., "capability-results-2025-10-15.json" -> "capability")
+            const toolType = fileName.split('-results-')[0];
+            reports[toolType] = reportContent;
+            console.log(`✅ Loaded ${toolType} report: ${fileName}`);
+        }
+        console.log(`📊 Total reports loaded: ${Object.keys(reports).length}`);
+        return reports;
+    }
+    async analyzeCrossToolPerformance(allReports) {
+        const modelPerformances = this.calculateModelPerformances(allReports);
+        // Calculate cross-tool consistency scores
+        const crossToolConsistency = {};
+        for (const model of modelPerformances) {
+            const scores = Object.values(model.toolScores);
+            if (scores.length > 1) {
+                const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
+                const variance = scores.reduce((sum, score) => sum + Math.pow(score - mean, 2), 0) / scores.length;
+                const standardDeviation = Math.sqrt(variance);
+                // Lower standard deviation = higher consistency (invert for consistency score)
+                crossToolConsistency[model.modelId] = Math.max(0, 1 - (standardDeviation / mean));
+            }
+        }
+        // Identify tool-specific leaders and universal performers
+        const toolSpecificLeaders = {};
+        const toolTypes = Object.keys(allReports);
+        for (const toolType of toolTypes) {
+            const bestModel = modelPerformances
+                .filter(m => m.toolScores[toolType] !== undefined)
+                .sort((a, b) => b.toolScores[toolType] - a.toolScores[toolType])[0];
+            if (bestModel) {
+                toolSpecificLeaders[toolType] = bestModel.modelId;
+            }
+        }
+        // Universal performers = models that rank in top 3 across all tools they participate in
+        const universalPerformers = [];
+        for (const model of modelPerformances) {
+            const participatingTools = Object.keys(model.toolScores);
+            let topThreeCount = 0;
+            for (const toolType of participatingTools) {
+                const ranking = modelPerformances
+                    .filter(m => m.toolScores[toolType] !== undefined)
+                    .sort((a, b) => b.toolScores[toolType] - a.toolScores[toolType])
+                    .findIndex(m => m.modelId === model.modelId);
+                if (ranking < 3)
+                    topThreeCount++;
+            }
+            if (participatingTools.length >= 3 && topThreeCount >= participatingTools.length * 0.75) {
+                universalPerformers.push(model.modelId);
+            }
+        }
+        return {
+            modelPerformances,
+            crossToolConsistency,
+            toolSpecificLeaders,
+            universalPerformers
+        };
+    }
+    calculateModelPerformances(allReports) {
+        const modelMap = new Map();
+        // Process each tool's evaluation results
+        for (const [toolType, report] of Object.entries(allReports)) {
+            if (!report.overallAssessment?.detailed_analysis)
+                continue;
+            for (const [modelKey, assessment] of Object.entries(report.overallAssessment.detailed_analysis)) {
+                const modelId = modelKey;
+                if (!modelMap.has(modelId)) {
+                    const metadata = report.modelMetadata?.[this.extractBaseModelId(modelId)] || {};
+                    modelMap.set(modelId, {
+                        modelId,
+                        provider: metadata.provider || 'Unknown',
+                        toolScores: {},
+                        pricing: metadata.pricing || { input_cost_per_million_tokens: 0, output_cost_per_million_tokens: 0 },
+                        capabilities: {
+                            context_window: metadata.context_window || 0,
+                            supports_function_calling: metadata.supports_function_calling || false
+                        }
+                    });
+                }
+                const modelData = modelMap.get(modelId);
+                const assessmentData = assessment;
+                // Extract average score for this tool
+                if (typeof assessmentData.average_score === 'number') {
+                    modelData.toolScores[toolType] = assessmentData.average_score;
+                }
+                // Update participation and reliability metrics
+                if (typeof assessmentData.participation_rate === 'number') {
+                    modelData.participationRate = (modelData.participationRate || 0) + assessmentData.participation_rate;
+                }
+                if (typeof assessmentData.reliability_score === 'number') {
+                    modelData.reliabilityScore = (modelData.reliabilityScore || 0) + assessmentData.reliability_score;
+                }
+            }
+        }
+        // Calculate final metrics
+        const modelPerformances = [];
+        for (const [modelId, data] of modelMap.entries()) {
+            const toolCount = Object.keys(data.toolScores).length;
+            if (toolCount === 0)
+                continue;
+            const averageScore = Object.values(data.toolScores).reduce((a, b) => a + b, 0) / toolCount;
+            const participationRate = (data.participationRate || 0) / toolCount;
+            const reliabilityScore = (data.reliabilityScore || 0) / toolCount;
+            // Calculate consistency across tools
+            const scores = Object.values(data.toolScores);
+            const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
+            const variance = scores.reduce((sum, score) => sum + Math.pow(score - mean, 2), 0) / scores.length;
+            const consistencyAcrossTools = Math.max(0, 1 - (Math.sqrt(variance) / mean));
+            modelPerformances.push({
+                modelId,
+                provider: data.provider,
+                toolScores: data.toolScores,
+                averageScore,
+                participationRate,
+                reliabilityScore,
+                consistencyAcrossTools,
+                pricing: data.pricing,
+                capabilities: data.capabilities
+            });
+        }
+        return modelPerformances.sort((a, b) => b.averageScore - a.averageScore);
+    }
+    generateDecisionMatrices(modelPerformances) {
+        // Sort models by different criteria
+        const qualityLeaders = [...modelPerformances]
+            .sort((a, b) => b.averageScore - a.averageScore)
+            .slice(0, 5);
+        const speedOptimized = [...modelPerformances]
+            .filter(m => m.pricing.input_cost_per_million_tokens > 0) // Filter out models with no pricing data
+            .sort((a, b) => a.pricing.input_cost_per_million_tokens - b.pricing.input_cost_per_million_tokens)
+            .slice(0, 5);
+        const costEffective = [...modelPerformances]
+            .filter(m => m.pricing.input_cost_per_million_tokens > 0 && m.pricing.output_cost_per_million_tokens > 0)
+            .map(model => ({
+            ...model,
+            valueScore: model.averageScore / ((model.pricing.input_cost_per_million_tokens + model.pricing.output_cost_per_million_tokens) / 2)
+        }))
+            .sort((a, b) => b.valueScore - a.valueScore)
+            .slice(0, 5);
+        const balanced = [...modelPerformances]
+            .filter(m => m.pricing.input_cost_per_million_tokens > 0)
+            .map(model => ({
+            ...model,
+            balancedScore: (model.averageScore * 0.4) + (model.consistencyAcrossTools * 0.3) +
+                (model.reliabilityScore * 0.3) -
+                ((model.pricing.input_cost_per_million_tokens + model.pricing.output_cost_per_million_tokens) / 100)
+        }))
+            .sort((a, b) => b.balancedScore - a.balancedScore)
+            .slice(0, 5);
+        const reliabilityFocused = [...modelPerformances]
+            .sort((a, b) => {
+            if (b.reliabilityScore !== a.reliabilityScore) {
+                return b.reliabilityScore - a.reliabilityScore;
+            }
+            return b.consistencyAcrossTools - a.consistencyAcrossTools;
+        })
+            .slice(0, 5);
+        return {
+            qualityLeaders,
+            speedOptimized,
+            costEffective,
+            balanced,
+            reliabilityFocused
+        };
+    }
+    generateUsageRecommendations(crossToolAnalysis, decisionMatrices) {
+        const recommendations = [
+            {
+                priority: 'quality-first',
+                primaryModel: decisionMatrices.qualityLeaders[0]?.modelId || '',
+                fallbackModel: decisionMatrices.qualityLeaders[1]?.modelId || '',
+                reasoning: 'Optimized for maximum accuracy and completeness across all MCP tools',
+                costImplications: `Estimated cost: $${this.calculateCostEstimate(decisionMatrices.qualityLeaders[0])}/1M tokens`,
+                useCases: ['Production deployments', 'Critical troubleshooting', 'Complex recommendations']
+            },
+            {
+                priority: 'cost-first',
+                primaryModel: decisionMatrices.costEffective[0]?.modelId || '',
+                fallbackModel: decisionMatrices.costEffective[1]?.modelId || '',
+                reasoning: 'Best value ratio of performance per dollar spent',
+                costImplications: `Estimated cost: $${this.calculateCostEstimate(decisionMatrices.costEffective[0])}/1M tokens`,
+                useCases: ['Budget-conscious deployments', 'Frequent operations', 'Cost-sensitive workflows']
+            },
+            {
+                priority: 'speed-first',
+                primaryModel: decisionMatrices.speedOptimized[0]?.modelId || '',
+                fallbackModel: decisionMatrices.speedOptimized[1]?.modelId || '',
+                reasoning: 'Optimized for fastest response times and lowest latency',
+                costImplications: `Estimated cost: $${this.calculateCostEstimate(decisionMatrices.speedOptimized[0])}/1M tokens`,
+                useCases: ['Time-sensitive troubleshooting', 'Interactive debugging', 'Rapid prototyping']
+            },
+            {
+                priority: 'balanced',
+                primaryModel: decisionMatrices.balanced[0]?.modelId || '',
+                fallbackModel: decisionMatrices.balanced[1]?.modelId || '',
+                reasoning: 'Optimal balance of quality, reliability, and cost considerations',
+                costImplications: `Estimated cost: $${this.calculateCostEstimate(decisionMatrices.balanced[0])}/1M tokens`,
+                useCases: ['General purpose usage', 'Mixed workloads', 'Default recommendation']
+            }
+        ];
+        return recommendations;
+    }
+    async generatePlatformInsights(crossToolAnalysis, decisionMatrices, usageRecommendations, toolMetadata) {
+        // Load prompt template from evaluation prompts directory
+        const promptPath = path.join(process.cwd(), 'src', 'evaluation', 'prompts', 'platform-synthesis.md');
+        const promptTemplate = fs.readFileSync(promptPath, 'utf8');
+        const promptWithData = promptTemplate
+            .replace('{crossToolAnalysisJson}', JSON.stringify(crossToolAnalysis, null, 2))
+            .replace('{decisionMatricesJson}', JSON.stringify(decisionMatrices, null, 2))
+            .replace('{usageRecommendationsJson}', JSON.stringify(usageRecommendations, null, 2))
+            .replace('{toolMetadataJson}', JSON.stringify(toolMetadata, null, 2));
+        const aiResponse = await this.aiProvider.sendMessage(promptWithData);
+        return aiResponse.content; // Return the AI-generated markdown directly
+    }
+    extractKeyFindings(crossToolAnalysis) {
+        const findings = [];
+        findings.push(`${crossToolAnalysis.modelPerformances.length} models evaluated across ${Object.keys(crossToolAnalysis.toolSpecificLeaders).length} tool types`);
+        findings.push(`${crossToolAnalysis.universalPerformers.length} models demonstrated consistent cross-tool performance`);
+        // Add performance spread analysis
+        const scores = crossToolAnalysis.modelPerformances.map((m) => m.averageScore);
+        const maxScore = Math.max(...scores);
+        const minScore = Math.min(...scores);
+        findings.push(`Performance spread: ${(maxScore - minScore).toFixed(3)} (${maxScore.toFixed(3)} - ${minScore.toFixed(3)})`);
+        return findings;
+    }
+    categorizeModelTiers(modelPerformances) {
+        const sorted = [...modelPerformances].sort((a, b) => b.averageScore - a.averageScore);
+        // Use reliability score and consistency to determine production readiness
+        const productionReady = sorted.filter(m => m.reliabilityScore >= 0.8 && m.consistencyAcrossTools >= 0.7);
+        const costOptimized = sorted.filter(m => m.reliabilityScore >= 0.7 &&
+            m.consistencyAcrossTools >= 0.6 &&
+            !productionReady.includes(m) &&
+            (m.pricing.input_cost_per_million_tokens + m.pricing.output_cost_per_million_tokens) < 10);
+        const avoidForProduction = sorted.filter(m => !productionReady.includes(m) && !costOptimized.includes(m));
+        return {
+            'Production Ready': productionReady.map(m => m.modelId),
+            'Cost-Optimized': costOptimized.map(m => m.modelId),
+            'Avoid for Production': avoidForProduction.map(m => m.modelId)
+        };
+    }
+    identifyCrossToolPatterns(crossToolAnalysis) {
+        return {
+            consistencyLeaders: Object.entries(crossToolAnalysis.crossToolConsistency)
+                .sort(([, a], [, b]) => b - a)
+                .slice(0, 3)
+                .map(([model]) => model),
+            toolSpecificLeaders: crossToolAnalysis.toolSpecificLeaders,
+            universalPerformers: crossToolAnalysis.universalPerformers
+        };
+    }
+    generateProductionRecommendations(decisionMatrices) {
+        return {
+            'Primary Production Model': decisionMatrices.qualityLeaders[0]?.modelId || 'None',
+            'Cost-Optimized Alternative': decisionMatrices.costEffective[0]?.modelId || 'None',
+            'High-Reliability Option': decisionMatrices.reliabilityFocused[0]?.modelId || 'None',
+            'Balanced General Use': decisionMatrices.balanced[0]?.modelId || 'None'
+        };
+    }
+    calculateCostEstimate(model) {
+        if (!model || !model.pricing.input_cost_per_million_tokens)
+            return '0.00';
+        // Estimate average cost per 1M tokens (assuming 50% input, 50% output)
+        const avgCost = (model.pricing.input_cost_per_million_tokens + model.pricing.output_cost_per_million_tokens) / 2;
+        return avgCost.toFixed(2);
+    }
+    extractBaseModelId(fullModelId) {
+        // Extract base model from full ID like "vercel_claude-sonnet-4-5-20250929_2025-10-15"
+        const parts = fullModelId.split('_');
+        if (parts.length >= 2) {
+            return parts[1]; // Return the middle part (actual model name)
+        }
+        return fullModelId;
+    }
+    async saveSynthesisReport(markdownContent, outputPath = './eval/analysis/platform/synthesis-report.md') {
+        const dir = path.dirname(outputPath);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        // Save the AI-generated markdown directly
+        fs.writeFileSync(outputPath, markdownContent);
+        console.log(`✅ Platform synthesis report saved: ${outputPath}`);
+    }
+}
+exports.PlatformSynthesizer = PlatformSynthesizer;

package/dist/evaluation/run-platform-synthesis.d.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+/**
+ * Platform-Wide AI Model Synthesis Runner
+ *
+ * Executes comprehensive cross-tool analysis using all individual evaluation reports
+ * Generates platform-wide insights, decision matrices, and usage recommendations
+ */
+export {};
+//# sourceMappingURL=run-platform-synthesis.d.ts.map

package/dist/evaluation/run-platform-synthesis.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-platform-synthesis.d.ts","sourceRoot":"","sources":["../../src/evaluation/run-platform-synthesis.ts"],"names":[],"mappings":";AAEA;;;;;GAKG"}

package/dist/evaluation/run-platform-synthesis.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * Platform-Wide AI Model Synthesis Runner
+ *
+ * Executes comprehensive cross-tool analysis using all individual evaluation reports
+ * Generates platform-wide insights, decision matrices, and usage recommendations
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const platform_synthesizer_js_1 = require("./platform-synthesizer.js");
+const vercel_provider_js_1 = require("../core/providers/vercel-provider.js");
+const model_config_js_1 = require("../core/model-config.js");
+async function runPlatformSynthesis() {
+    console.log('🚀 Starting Platform-Wide AI Model Synthesis...\n');
+    try {
+        // Initialize AI provider for synthesis analysis (use Claude for comprehensive analysis)
+        const aiProvider = new vercel_provider_js_1.VercelProvider({
+            provider: 'anthropic',
+            apiKey: process.env.ANTHROPIC_API_KEY,
+            model: (0, model_config_js_1.getCurrentModel)('anthropic'),
+            debugMode: process.env.DEBUG_DOT_AI === 'true'
+        });
+        // Initialize synthesizer
+        const synthesizer = new platform_synthesizer_js_1.PlatformSynthesizer(aiProvider);
+        // Generate comprehensive platform-wide analysis
+        console.log('📊 Generating platform-wide analysis...');
+        const markdownReport = await synthesizer.generatePlatformWideAnalysis();
+        // Save synthesis report
+        console.log('\n💾 Saving synthesis report...');
+        await synthesizer.saveSynthesisReport(markdownReport);
+        console.log('\n✅ Platform-wide synthesis complete!');
+        console.log('📄 Report saved: ./eval/analysis/platform/synthesis-report.md');
+        console.log('\n✨ AI-generated comprehensive report includes:');
+        console.log('   • Detailed model profiles with strengths/weaknesses');
+        console.log('   • Production recommendations by priority');
+        console.log('   • Cross-tool performance insights');
+        console.log('   • Critical warnings and actionable guidance');
+    }
+    catch (error) {
+        console.error('❌ Platform synthesis failed:', error);
+        process.exit(1);
+    }
+}
+// Run synthesis
+runPlatformSynthesis();

package/dist/interfaces/mcp.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../../src/interfaces/mcp.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAiDtC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC;CACxC;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAa;IACrC,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,UAAU,CAAC,CAAkC;IACrD,OAAO,CAAC,aAAa,CAAC,CAAgC;IACtD,OAAO,CAAC,YAAY,CAAmB;IACvC,OAAO,CAAC,aAAa,CAAgB;gBAEzB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe;IAsCjD;;OAEG;IACH,OAAO,CAAC,YAAY;IAsBpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA+HrB;;OAEG;IACH,OAAO,CAAC,eAAe;IAqCvB,OAAO,CAAC,iBAAiB;IAInB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBd,mBAAmB;YAMnB,kBAAkB;YAiFlB,gBAAgB;IAexB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB3B,OAAO,IAAI,OAAO;CAGnB"}
+{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../../src/interfaces/mcp.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AA8CtC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC;CACxC;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAa;IACrC,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,UAAU,CAAC,CAAkC;IACrD,OAAO,CAAC,aAAa,CAAC,CAAgC;IACtD,OAAO,CAAC,YAAY,CAAmB;IACvC,OAAO,CAAC,aAAa,CAAgB;gBAEzB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe;IAsCjD;;OAEG;IACH,OAAO,CAAC,YAAY;IAsBpB;;OAEG;IACH,OAAO,CAAC,aAAa;IA4HrB;;OAEG;IACH,OAAO,CAAC,eAAe;IAqCvB,OAAO,CAAC,iBAAiB;IAInB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAkBd,mBAAmB;YAMnB,kBAAkB;YAiFlB,gBAAgB;IAexB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB3B,OAAO,IAAI,OAAO;CAGnB"}

package/dist/interfaces/mcp.js

@@ -19,15 +19,12 @@ const version_1 = require("../tools/version");
 const test_docs_1 = require("../tools/test-docs");
 const organizational_data_1 = require("../tools/organizational-data");
 const remediate_1 = require("../tools/remediate");
-/*
-// DEVELOPER NOTE: Platform Building tool disabled - imports commented out
-import {
-  BUILD_PLATFORM_TOOL_NAME,
-  BUILD_PLATFORM_TOOL_DESCRIPTION,
-  BUILD_PLATFORM_TOOL_INPUT_SCHEMA,
-  handleBuildPlatformTool,
-} from '../tools/build-platform';
-*/
+// import {
+//   BUILD_PLATFORM_TOOL_NAME,
+//   BUILD_PLATFORM_TOOL_DESCRIPTION,
+//   BUILD_PLATFORM_TOOL_INPUT_SCHEMA,
+//   handleBuildPlatformTool,
+// } from '../tools/build-platform';
 const prompts_1 = require("../tools/prompts");
 const rest_registry_1 = require("./rest-registry");
 const rest_api_1 = require("./rest-api");
@@ -125,33 +122,30 @@ class MCPServer {
             this.logger.info(`Processing ${remediate_1.REMEDIATE_TOOL_NAME} tool request`, { requestId });
             return await (0, remediate_1.handleRemediateTool)(args);
         }, 'Troubleshooting', ['remediation', 'troubleshooting', 'kubernetes', 'analysis']);
-        /*
-        // DEVELOPER NOTE: buildPlatform tool disabled - under active development with incomplete functionality
         // Register buildPlatform tool
-        this.registerTool(
-          BUILD_PLATFORM_TOOL_NAME,
-          BUILD_PLATFORM_TOOL_DESCRIPTION,
-          BUILD_PLATFORM_TOOL_INPUT_SCHEMA,
-          async (args: any) => {
-            const requestId = this.generateRequestId();
-            this.logger.info(
-              `Processing ${BUILD_PLATFORM_TOOL_NAME} tool request`,
-              { requestId }
-            );
-            return await handleBuildPlatformTool(args, this.dotAI, this.logger, requestId);
-          },
-          'Platform',
-          ['platform', 'kubernetes', 'installation', 'infrastructure']
-        );
-        */
+        // this.registerTool(
+        //   BUILD_PLATFORM_TOOL_NAME,
+        //   BUILD_PLATFORM_TOOL_DESCRIPTION,
+        //   BUILD_PLATFORM_TOOL_INPUT_SCHEMA,
+        //   async (args: any) => {
+        //     const requestId = this.generateRequestId();
+        //     this.logger.info(
+        //       `Processing ${BUILD_PLATFORM_TOOL_NAME} tool request`,
+        //       { requestId }
+        //     );
+        //     return await handleBuildPlatformTool(args, this.dotAI, this.logger, requestId);
+        //   },
+        //   'Platform',
+        //   ['platform', 'kubernetes', 'installation', 'infrastructure']
+        // );
         this.logger.info('Registered all tools with McpServer', {
             tools: [
                 recommend_1.RECOMMEND_TOOL_NAME,
                 version_1.VERSION_TOOL_NAME,
                 test_docs_1.TESTDOCS_TOOL_NAME,
                 organizational_data_1.ORGANIZATIONAL_DATA_TOOL_NAME,
-                remediate_1.REMEDIATE_TOOL_NAME,
-                // BUILD_PLATFORM_TOOL_NAME, // Disabled - under development
+                remediate_1.REMEDIATE_TOOL_NAME
+                // BUILD_PLATFORM_TOOL_NAME,
             ],
             totalTools: 5,
         });

package/dist/interfaces/rest-api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rest-api.d.ts","sourceRoot":"","sources":["../../src/interfaces/rest-api.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE5D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC;;GAEG;AACH,oBAAY,UAAU;IACpB,EAAE,MAAM;IACR,WAAW,MAAM;IACjB,SAAS,MAAM;IACf,kBAAkB,MAAM;IACxB,qBAAqB,MAAM;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,GAAG,CAAC;KACf,CAAC;IACF,IAAI,CAAC,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,IAAI,CAAC,EAAE;QACL,MAAM,EAAE,GAAG,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,IAAI,CAAC,EAAE;QACL,KAAK,EAAE,QAAQ,EAAE,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,cAAc,CAAa;gBAGjC,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,OAAO,CAAC,aAAa,CAAM;IAoBrC;;OAEG;IACG,aAAa,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IA+EzF;;OAEG;IACH,OAAO,CAAC,YAAY;IAmCpB;;OAEG;YACW,mBAAmB;IA2CjC;;OAEG;YACW,mBAAmB;IA8FjC;;OAEG;YACW,iBAAiB;IA8B/B;;OAEG;IACH,OAAO,CAAC,cAAc;IAOtB;;OAEG;YACW,gBAAgB;IAK9B;;OAEG;YACW,iBAAiB;IAyB/B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIvC;;OAEG;IACH,SAAS,IAAI,aAAa;CAG3B"}
+{"version":3,"file":"rest-api.d.ts","sourceRoot":"","sources":["../../src/interfaces/rest-api.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE5D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC;;GAEG;AACH,oBAAY,UAAU;IACpB,EAAE,MAAM;IACR,WAAW,MAAM;IACjB,SAAS,MAAM;IACf,kBAAkB,MAAM;IACxB,qBAAqB,MAAM;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,GAAG,CAAC;KACf,CAAC;IACF,IAAI,CAAC,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,IAAI,CAAC,EAAE;QACL,MAAM,EAAE,GAAG,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC5D,IAAI,CAAC,EAAE;QACL,KAAK,EAAE,QAAQ,EAAE,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,cAAc,CAAa;gBAGjC,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,OAAO,CAAC,aAAa,CAAM;IAoBrC;;OAEG;IACG,aAAa,CAAC,GAAG,EAAE,eAAe,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IA+EzF;;OAEG;IACH,OAAO,CAAC,YAAY;IAmCpB;;OAEG;YACW,mBAAmB;IA2CjC;;OAEG;YACW,mBAAmB;IA+FjC;;OAEG;YACW,iBAAiB;IA8B/B;;OAEG;IACH,OAAO,CAAC,cAAc;IAOtB;;OAEG;YACW,gBAAgB;IAK9B;;OAEG;YACW,iBAAiB;IAyB/B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIvC;;OAEG;IACH,SAAS,IAAI,aAAa;CAG3B"}

package/dist/tools/answer-question.d.ts

@@ -10,6 +10,7 @@ export declare const ANSWERQUESTION_TOOL_INPUT_SCHEMA: {
     solutionId: z.ZodString;
     stage: z.ZodEnum<["required", "basic", "advanced", "open"]>;
     answers: z.ZodRecord<z.ZodString, z.ZodAny>;
+    interaction_id: z.ZodOptional<z.ZodString>;
 };
 /**
  * Direct MCP tool handler for answerQuestion functionality
@@ -18,6 +19,7 @@ export declare function handleAnswerQuestionTool(args: {
     solutionId: string;
     stage: 'required' | 'basic' | 'advanced' | 'open';
     answers: Record<string, any>;
+    interaction_id?: string;
 }, dotAI: DotAI, logger: Logger, requestId: string): Promise<{
     content: {
         type: 'text';

package/dist/tools/answer-question.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"answer-question.d.ts","sourceRoot":"","sources":["../../src/tools/answer-question.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAQhD,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,8HAA4H,CAAC;AAGzK,eAAO,MAAM,gCAAgC;;;;CAI5C,CAAC;AAgiBF;;GAEG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAAE,EAC7G,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAAE,CAAC,CAqUxD"}
+{"version":3,"file":"answer-question.d.ts","sourceRoot":"","sources":["../../src/tools/answer-question.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAQhD,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AACzD,eAAO,MAAM,+BAA+B,8HAA4H,CAAC;AAGzK,eAAO,MAAM,gCAAgC;;;;;CAK5C,CAAC;AA0iBF;;GAEG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,EACtI,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAAE,CAAC,CAqUxD"}