@vfarcic/dot-ai 0.115.0 → 0.117.0

package/dist/evaluation/evaluators/base-comparative.js

@@ -13,10 +13,12 @@ const platform_utils_1 = require("../../core/platform-utils");
 const fs_1 = require("fs");
 const path_1 = require("path");
 const dataset_analyzer_js_1 = require("../dataset-analyzer.js");
+const metadata_loader_js_1 = require("../metadata-loader.js");
 class BaseComparativeEvaluator {
     evaluatorModel;
     datasetAnalyzer;
     promptTemplate;
+    metadata;
     constructor(datasetDir) {
         // Use Claude via VercelProvider as the evaluator (most reliable for complex comparative evaluation)
         this.evaluatorModel = new vercel_provider_1.VercelProvider({
@@ -28,6 +30,8 @@ class BaseComparativeEvaluator {
         this.datasetAnalyzer = new dataset_analyzer_js_1.DatasetAnalyzer(datasetDir || './eval/datasets');
         // Prompt template will be loaded by subclass
         this.promptTemplate = '';
+        // Load metadata
+        this.metadata = (0, metadata_loader_js_1.loadEvaluationMetadata)();
     }
     /**
      * Initialize the evaluator - must be called by subclass constructor
@@ -122,6 +126,7 @@ class BaseComparativeEvaluator {
 ${reliabilityContext}
 
 **Response:**
+
 ${modelResponse.response}
 
 ---`;
@@ -171,11 +176,18 @@ ${modelResponse.response}
      * Build the evaluation prompt - can be overridden by subclasses for custom behavior
      */
     buildEvaluationPrompt(scenario, modelResponsesText, modelList) {
+        // Build metadata context sections
+        const pricingContext = (0, metadata_loader_js_1.buildModelPricingContext)(this.metadata.models);
+        const toolContext = (0, metadata_loader_js_1.buildToolContext)(this.toolName, this.metadata.tools);
+        // Inject all data into prompt template via placeholders
         return this.promptTemplate
+            .replace('{pricing_context}', pricingContext)
+            .replace('{tool_context}', toolContext)
             .replace('{issue}', scenario.issue)
             .replace('{model_responses}', modelResponsesText)
             .replace('{model_list}', modelList)
-            .replace('{phase}', scenario.interaction_id);
+            .replace('{phase}', scenario.interaction_id)
+            .replace('{scenario_name}', scenario.interaction_id);
     }
     /**
      * Get statistics about available datasets

package/dist/evaluation/graph-generator.d.ts

@@ -0,0 +1,56 @@
+import type { ModelPerformance } from './platform-synthesizer.js';
+export interface GraphGenerationResult {
+    success: boolean;
+    graphPath?: string;
+    error?: string;
+}
+/**
+ * GraphGenerator creates data visualizations for platform synthesis reports.
+ * Uses QuickChart.io API to generate chart images without requiring native dependencies.
+ */
+export declare class GraphGenerator {
+    private outputDir;
+    private quickchartBaseUrl;
+    constructor(outputDir?: string);
+    /**
+     * Generates all or specific graphs for the platform report
+     * @param modelPerformances Model performance data
+     * @param graphNames Optional array of specific graph names to generate. If not provided, generates all graphs.
+     *                   Valid names: 'performance-tiers', 'cost-vs-quality', 'reliability-comparison',
+     *                   'tool-performance-heatmap', 'context-window-correlation'
+     */
+    generateAllGraphs(modelPerformances: ModelPerformance[], graphNames?: string[]): Promise<Record<string, GraphGenerationResult>>;
+    /**
+     * Graph 1: Performance Tiers - Grouped bar chart showing score, reliability, and consistency
+     */
+    private generatePerformanceTiersGraph;
+    /**
+     * Graph 2: Cost vs Quality - Line chart showing input/output cost range per model
+     */
+    private generateCostVsQualityGraph;
+    /**
+     * Graph 3: Reliability Comparison - Bar chart with reliability scores
+     */
+    private generateReliabilityComparisonGraph;
+    /**
+     * Graph 4: Tool Performance Heatmap - Shows model scores per tool
+     */
+    private generateToolPerformanceHeatmap;
+    /**
+     * Graph 5: Context Window Correlation - Scatter plot showing context window vs performance
+     */
+    private generateContextWindowCorrelationGraph;
+    /**
+     * Downloads a chart from QuickChart.io API and saves it as PNG
+     */
+    private downloadChart;
+    /**
+     * Cleans model names by removing provider prefixes
+     */
+    private cleanModelName;
+    /**
+     * Returns a consistent color for each tool index (supports up to 10 tools)
+     */
+    private getToolColor;
+}
+//# sourceMappingURL=graph-generator.d.ts.map

package/dist/evaluation/graph-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-generator.d.ts","sourceRoot":"","sources":["../../src/evaluation/graph-generator.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAElE,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,iBAAiB,CAAiC;gBAE9C,SAAS,SAAoC;IAIzD;;;;;;OAMG;IACG,iBAAiB,CACrB,iBAAiB,EAAE,gBAAgB,EAAE,EACrC,UAAU,CAAC,EAAE,MAAM,EAAE,GACpB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAkCjD;;OAEG;YACW,6BAA6B;IAiH3C;;OAEG;YACW,0BAA0B;IAsHxC;;OAEG;YACW,kCAAkC;IA4GhD;;OAEG;YACW,8BAA8B;IAqG5C;;OAEG;YACW,qCAAqC;IAyHnD;;OAEG;YACW,aAAa;IA8B3B;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,YAAY;CAerB"}