@vfarcic/dot-ai 0.111.0 → 0.113.0

package/dist/evaluation/dataset-analyzer.js

@@ -0,0 +1,234 @@
+"use strict";
+/**
+ * Dataset Analyzer for Multi-Model Comparison
+ *
+ * Analyzes evaluation datasets to group them by scenario and extract
+ * model responses for comparative evaluation.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatasetAnalyzer = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+class DatasetAnalyzer {
+    datasetDir;
+    constructor(datasetDir = './eval/datasets') {
+        this.datasetDir = datasetDir;
+    }
+    /**
+     * Find all available datasets for a specific tool
+     */
+    findDatasets(tool) {
+        const files = (0, fs_1.readdirSync)(this.datasetDir);
+        return files
+            .filter(file => file.startsWith(`${tool}_`) && file.endsWith('.jsonl'))
+            .map(file => (0, path_1.join)(this.datasetDir, file));
+    }
+    /**
+     * Parse dataset filename to extract components
+     * Format: {tool}_{interaction_id}_{sdk}_{model}_{timestamp}.jsonl
+     */
+    parseDatasetFilename(filename) {
+        const basename = filename.replace(/^.*\//, '').replace(/\.jsonl$/, '');
+        const parts = basename.split('_');
+        if (parts.length < 5)
+            return null;
+        // For remediate datasets: remediate_{phase}_{action}_vercel_{model}_{timestamp}
+        // e.g., remediate_manual_analyze_vercel_gpt_timestamp
+        const tool = parts[0];
+        const timestamp = parts[parts.length - 1];
+        // Find 'vercel' SDK position to split correctly
+        const sdkIndex = parts.indexOf('vercel');
+        if (sdkIndex === -1)
+            return null;
+        // interaction_id is everything between tool and sdk
+        const interaction_id = parts.slice(1, sdkIndex).join('_');
+        const sdk = parts[sdkIndex];
+        const model = parts.slice(sdkIndex + 1, -1).join('_');
+        return { tool, interaction_id, sdk, model, timestamp };
+    }
+    /**
+     * Load and parse a dataset file
+     */
+    loadDataset(filepath) {
+        try {
+            const content = (0, fs_1.readFileSync)(filepath, 'utf8').trim();
+            if (!content)
+                return null;
+            return JSON.parse(content);
+        }
+        catch (error) {
+            console.warn(`Failed to load dataset ${filepath}:`, error);
+            return null;
+        }
+    }
+    /**
+     * Group datasets by scenario for comparative evaluation
+     * Returns scenarios that have data from multiple models
+     * Groups by both tool and interaction_id to create separate evaluations for each phase
+     */
+    groupByScenario(tool) {
+        const datasets = this.findDatasets(tool);
+        const scenarioGroups = new Map();
+        // Group datasets by filename pattern up to provider, then by model
+        for (const filepath of datasets) {
+            const sample = this.loadDataset(filepath);
+            if (!sample)
+                continue;
+            // Extract scenario key from filename pattern (up to provider)
+            const filename = filepath.replace(/^.*\//, ''); // Remove directory path
+            const filenameParts = filename.split('_');
+            const beforeProvider = [];
+            for (const part of filenameParts) {
+                if (part === 'vercel')
+                    break; // Stop at SDK name
+                beforeProvider.push(part);
+            }
+            const scenarioKey = beforeProvider.join('_');
+            // Group by model within each scenario
+            const modelKey = `${sample.performance.sdk}_${sample.performance.model_version}`;
+            if (!scenarioGroups.has(scenarioKey)) {
+                scenarioGroups.set(scenarioKey, new Map());
+            }
+            const modelGroups = scenarioGroups.get(scenarioKey);
+            if (!modelGroups.has(modelKey)) {
+                modelGroups.set(modelKey, []);
+            }
+            // Parse failure_analysis if it exists
+            let failure_analysis = undefined;
+            if (sample.metadata.failure_analysis && sample.metadata.failure_analysis !== "") {
+                try {
+                    if (typeof sample.metadata.failure_analysis === 'string') {
+                        failure_analysis = JSON.parse(sample.metadata.failure_analysis);
+                    }
+                    else {
+                        failure_analysis = sample.metadata.failure_analysis;
+                    }
+                }
+                catch (error) {
+                    // If parsing fails, treat as no failure analysis
+                    failure_analysis = undefined;
+                }
+            }
+            modelGroups.get(modelKey).push({
+                model: modelKey,
+                response: sample.output,
+                performance: sample.performance,
+                metadata: {
+                    timestamp: sample.metadata.timestamp,
+                    complexity: sample.metadata.complexity,
+                    test_scenario: sample.metadata.test_scenario,
+                    issue: sample.input.issue,
+                    failure_analysis
+                }
+            });
+        }
+        // Convert to comparison scenarios - include ALL scenarios (remove multi-model filter)
+        const scenarios = [];
+        for (const [scenarioKey, modelGroups] of scenarioGroups) {
+            // Flatten model groups: each model may have multiple interactions
+            const allModelResponses = [];
+            for (const [modelKey, interactions] of modelGroups) {
+                // Combine multiple interactions per model into a single response
+                if (interactions.length === 1) {
+                    allModelResponses.push(interactions[0]);
+                }
+                else {
+                    // Multiple interactions per model - combine them
+                    const combinedResponse = this.combineModelInteractions(modelKey, interactions);
+                    allModelResponses.push(combinedResponse);
+                }
+            }
+            // Get representative issue from first model's first interaction
+            const firstModel = Array.from(modelGroups.values())[0]?.[0];
+            const issue = firstModel?.metadata?.issue || scenarioKey;
+            scenarios.push({
+                issue,
+                interaction_id: scenarioKey,
+                tool,
+                models: allModelResponses
+            });
+        }
+        return scenarios;
+    }
+    /**
+     * Combine multiple interactions per model into a single response for evaluation
+     */
+    combineModelInteractions(modelKey, interactions) {
+        // Sort interactions by timestamp
+        const sorted = interactions.sort((a, b) => new Date(a.metadata.timestamp).getTime() - new Date(b.metadata.timestamp).getTime());
+        // Create combined response showing all interactions
+        const combinedResponse = sorted.map((interaction, index) => `**Interaction ${index + 1}:**\n` +
+            `Issue: ${interaction.metadata.issue}\n` +
+            `Response: ${interaction.response}\n`).join('\n---\n');
+        // Aggregate performance metrics
+        const totalDuration = sorted.reduce((sum, i) => sum + i.performance.duration_ms, 0);
+        const totalInputTokens = sorted.reduce((sum, i) => sum + i.performance.input_tokens, 0);
+        const totalOutputTokens = sorted.reduce((sum, i) => sum + i.performance.output_tokens, 0);
+        // Collect all failure analyses from all interactions that have them
+        const allFailures = [];
+        sorted.forEach((interaction, index) => {
+            if (interaction.metadata.failure_analysis) {
+                allFailures.push({
+                    interaction_number: index + 1,
+                    issue: interaction.metadata.issue,
+                    ...interaction.metadata.failure_analysis
+                });
+            }
+        });
+        // Use the first failure as the primary failure_analysis, but preserve all failures
+        const primaryFailureAnalysis = allFailures.length > 0 ? allFailures[0] : undefined;
+        return {
+            model: modelKey,
+            response: combinedResponse,
+            performance: {
+                ...sorted[0].performance,
+                duration_ms: totalDuration,
+                input_tokens: totalInputTokens,
+                output_tokens: totalOutputTokens,
+                total_tokens: totalInputTokens + totalOutputTokens
+            },
+            metadata: {
+                ...sorted[0].metadata,
+                issue: sorted[0].metadata.issue, // Use first interaction's issue as primary
+                interaction_count: interactions.length,
+                failure_analysis: primaryFailureAnalysis,
+                all_failures: allFailures.length > 0 ? allFailures : undefined
+            }
+        };
+    }
+    /**
+     * Get summary of available models across all scenarios for a tool
+     */
+    getAvailableModels(tool) {
+        const datasets = this.findDatasets(tool);
+        const models = new Set();
+        for (const filepath of datasets) {
+            const parsed = this.parseDatasetFilename(filepath);
+            if (parsed) {
+                models.add(`${parsed.sdk}_${parsed.model}`);
+            }
+        }
+        return Array.from(models).sort();
+    }
+    /**
+     * Get statistics about dataset availability
+     */
+    getDatasetStats(tool) {
+        const scenarios = this.groupByScenario(tool);
+        const datasets = this.findDatasets(tool);
+        const interactionTypes = new Set();
+        for (const filepath of datasets) {
+            const parsed = this.parseDatasetFilename(filepath);
+            if (parsed) {
+                interactionTypes.add(parsed.interaction_id);
+            }
+        }
+        return {
+            totalDatasets: datasets.length,
+            availableModels: this.getAvailableModels(tool),
+            scenariosWithMultipleModels: scenarios.length,
+            interactionTypes: Array.from(interactionTypes).sort()
+        };
+    }
+}
+exports.DatasetAnalyzer = DatasetAnalyzer;

package/dist/evaluation/datasets/loader.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Dataset Loader for Standard OpenAI Evals Format
+ *
+ * Loads JSONL evaluation datasets following OpenAI Evals standard:
+ * - Each line contains: {input, ideal, metadata}
+ * - Supports filtering by category, complexity, tags
+ * - Used by both integration tests and evaluation framework
+ */
+export interface StandardEvalSample {
+    input: Record<string, any>;
+    ideal: any;
+    metadata: {
+        category: string;
+        complexity: 'low' | 'medium' | 'high';
+        tags: string[];
+        source: string;
+        phase?: string;
+        tool?: string;
+    };
+}
+export interface DatasetFilter {
+    category?: string;
+    complexity?: 'low' | 'medium' | 'high';
+    tags?: string[];
+    phase?: string;
+    tool?: string;
+}
+/**
+ * Load evaluation dataset from JSONL file
+ * @param datasetName - Name of the dataset file (without .jsonl extension)
+ * @param filter - Optional filter criteria
+ * @returns Array of evaluation samples
+ */
+export declare function loadEvalDataset(datasetName: string, filter?: DatasetFilter): StandardEvalSample[];
+/**
+ * Load samples for a specific test phase
+ * @param datasetName - Dataset name
+ * @param phase - Test phase to load
+ * @returns Array of samples for that phase
+ */
+export declare function loadTestPhase(datasetName: string, phase: string): StandardEvalSample[];
+//# sourceMappingURL=loader.d.ts.map

package/dist/evaluation/datasets/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/evaluation/datasets/loader.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,KAAK,EAAE,GAAG,CAAC;IACX,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;QACtC,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,aAAa,GACrB,kBAAkB,EAAE,CA+CtB;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAEtF"}

package/dist/evaluation/datasets/loader.js

@@ -0,0 +1,104 @@
+"use strict";
+/**
+ * Dataset Loader for Standard OpenAI Evals Format
+ *
+ * Loads JSONL evaluation datasets following OpenAI Evals standard:
+ * - Each line contains: {input, ideal, metadata}
+ * - Supports filtering by category, complexity, tags
+ * - Used by both integration tests and evaluation framework
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadEvalDataset = loadEvalDataset;
+exports.loadTestPhase = loadTestPhase;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/**
+ * Load evaluation dataset from JSONL file
+ * @param datasetName - Name of the dataset file (without .jsonl extension)
+ * @param filter - Optional filter criteria
+ * @returns Array of evaluation samples
+ */
+function loadEvalDataset(datasetName, filter) {
+    const datasetsDir = path.join(process.cwd(), 'eval', 'datasets');
+    const datasetPath = path.join(datasetsDir, `${datasetName}.jsonl`);
+    if (!fs.existsSync(datasetPath)) {
+        throw new Error(`Dataset not found: ${datasetPath}`);
+    }
+    const fileContent = fs.readFileSync(datasetPath, 'utf8');
+    const lines = fileContent.trim().split('\n').filter(line => line.trim());
+    const samples = lines.map((line, index) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error(`Invalid JSON at line ${index + 1} in ${datasetName}.jsonl: ${error}`);
+        }
+    });
+    // Apply filters if provided
+    if (filter) {
+        return samples.filter(sample => {
+            if (filter.category && sample.metadata.category !== filter.category) {
+                return false;
+            }
+            if (filter.complexity && sample.metadata.complexity !== filter.complexity) {
+                return false;
+            }
+            if (filter.phase && sample.metadata.phase !== filter.phase) {
+                return false;
+            }
+            if (filter.tool && sample.metadata.tool !== filter.tool) {
+                return false;
+            }
+            if (filter.tags) {
+                const hasAllTags = filter.tags.every(tag => sample.metadata.tags.includes(tag));
+                if (!hasAllTags) {
+                    return false;
+                }
+            }
+            return true;
+        });
+    }
+    return samples;
+}
+/**
+ * Load samples for a specific test phase
+ * @param datasetName - Dataset name
+ * @param phase - Test phase to load
+ * @returns Array of samples for that phase
+ */
+function loadTestPhase(datasetName, phase) {
+    return loadEvalDataset(datasetName, { phase });
+}

package/dist/evaluation/eval-runner.d.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env npx tsx
+/**
+ * Evaluation Runner for Multi-Model Comparative Analysis
+ *
+ * Runs comparative evaluation on available datasets from multiple models
+ * Automatically detects and evaluates both remediation and recommendation datasets
+ */
+export {};
+//# sourceMappingURL=eval-runner.d.ts.map

package/dist/evaluation/eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../src/evaluation/eval-runner.ts"],"names":[],"mappings":";AAEA;;;;;GAKG"}