@vfarcic/dot-ai 0.111.0 → 0.113.0

package/dist/core/providers/provider-debug-utils.js

@@ -41,7 +41,8 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ensureDebugDirectory = ensureDebugDirectory;
 exports.generateDebugId = generateDebugId;
-exports.logMetrics = logMetrics;
+exports.shouldSkipDatasetGeneration = shouldSkipDatasetGeneration;
+exports.logEvaluationDataset = logEvaluationDataset;
 exports.createAndLogAgenticResult = createAndLogAgenticResult;
 exports.debugLogInteraction = debugLogInteraction;
 const fs = __importStar(require("fs"));
@@ -67,70 +68,92 @@ function generateDebugId(operation) {
     return `${dateTime}_${randomHex}_${operation}`;
 }
 /**
- * Log metrics for token usage and execution time when DEBUG_DOT_AI=true
- *
- * PRD #143 Decision 5: Extended metrics for model comparison analysis
+ * Determine if dataset generation should be skipped for specific operations
  */
-function logMetrics(operation, sdk, result, durationMs, debugMode) {
+function shouldSkipDatasetGeneration(operation) {
+    const skipDatasetOperations = ['version-connectivity-check', 'generic'];
+    return skipDatasetOperations.includes(operation);
+}
+/**
+ * Log unified evaluation metrics when DEBUG_DOT_AI=true
+ * Single function for all metrics and evaluation data capture
+ */
+/**
+ * Generate eval dataset entry in standard OpenAI Evals format
+ * Logs evaluation metrics to JSONL dataset files for AI quality assessment
+ */
+function logEvaluationDataset(metrics, debugMode = false) {
     if (!debugMode)
         return;
+    // Skip dataset generation for non-evaluable operations
+    if (shouldSkipDatasetGeneration(metrics.test_scenario))
+        return;
     try {
-        const debugDir = ensureDebugDirectory();
-        const metricsFile = path.join(debugDir, 'metrics.jsonl');
-        const entry = {
-            timestamp: new Date().toISOString(),
-            sdk,
-            operation,
-            inputTokens: result.totalTokens.input,
-            outputTokens: result.totalTokens.output,
-            durationMs
-        };
-        // Add cache metrics if present
-        if (result.totalTokens.cacheCreation !== undefined) {
-            entry.cacheCreationTokens = result.totalTokens.cacheCreation;
+        // Parse operation for tool name
+        const operationParts = metrics.operation.split('-');
+        const toolName = operationParts[0]; // e.g., "remediate"
+        // Check if this is a comparative evaluation
+        const isComparativeEvaluation = metrics.operation.includes('-comparative-');
+        // Use different directories for comparative evaluations vs raw test datasets
+        const baseDir = isComparativeEvaluation ?
+            path.join(process.cwd(), 'eval', 'results') : // Comparative evaluation results go here
+            path.join(process.cwd(), 'eval', 'datasets'); // Raw test datasets go here
+        // Ensure directory exists
+        if (!fs.existsSync(baseDir)) {
+            fs.mkdirSync(baseDir, { recursive: true });
         }
-        if (result.totalTokens.cacheRead !== undefined) {
-            entry.cacheReadTokens = result.totalTokens.cacheRead;
+        let datasetFile;
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '').split('T').join('_');
+        if (isComparativeEvaluation) {
+            // For comparative evaluations, save to results directory
+            datasetFile = path.join(baseDir, `${toolName}_comparative_evaluation_${timestamp}.jsonl`);
         }
-        // Calculate cache hit rate (percentage)
-        if (result.totalTokens.cacheRead !== undefined && result.totalTokens.input > 0) {
-            entry.cacheHitRate = Math.round((result.totalTokens.cacheRead / result.totalTokens.input) * 100);
+        else {
+            // Use modelVersion directly for accurate model identification
+            const modelName = metrics.modelVersion || 'unknown';
+            // Create filename with interaction ID, SDK, model, and timestamp for single-model datasets
+            datasetFile = path.join(baseDir, `${toolName}_${metrics.interaction_id}_${metrics.sdk}_${modelName}_${timestamp}.jsonl`);
         }
-        // Add extended metrics (PRD #143 Decision 5)
-        if (result.iterations !== undefined) {
-            entry.iterationCount = result.iterations;
-        }
-        if (result.toolCallsExecuted) {
-            entry.toolCallCount = result.toolCallsExecuted.length;
-            // Extract unique tool names
-            const uniqueTools = [...new Set(result.toolCallsExecuted.map(tc => tc.tool))];
-            entry.uniqueToolsUsed = uniqueTools;
-        }
-        if (result.status) {
-            entry.status = result.status;
-        }
-        if (result.completionReason) {
-            entry.completionReason = result.completionReason;
-        }
-        if (result.modelVersion) {
-            entry.modelVersion = result.modelVersion;
-        }
-        // Manual annotation placeholders (populate after test analysis)
-        entry.manualNotes = '';
-        entry.failureReason = '';
-        entry.qualityIssues = [];
-        entry.comparisonNotes = '';
-        fs.appendFileSync(metricsFile, JSON.stringify(entry) + '\n');
+        // Transform metrics into OpenAI Evals format (no ideal field - using model-graded evaluation)
+        const evalEntry = {
+            input: {
+                issue: metrics.user_intent || "Tool execution scenario"
+            },
+            output: metrics.ai_response_summary || "",
+            performance: {
+                duration_ms: metrics.durationMs,
+                input_tokens: metrics.inputTokens,
+                output_tokens: metrics.outputTokens,
+                total_tokens: metrics.inputTokens + metrics.outputTokens,
+                sdk: metrics.sdk,
+                model_version: metrics.modelVersion,
+                iterations: metrics.iterationCount,
+                tool_calls_executed: metrics.toolCallCount,
+                cache_read_tokens: metrics.cacheReadTokens || 0,
+                cache_creation_tokens: metrics.cacheCreationTokens || 0
+            },
+            metadata: {
+                timestamp: new Date().toISOString(),
+                complexity: "medium",
+                tags: ["troubleshooting"],
+                source: "integration_test",
+                tool: toolName,
+                test_scenario: metrics.test_scenario || `${toolName}_test`,
+                failure_analysis: metrics.failure_analysis || ""
+            }
+        };
+        fs.writeFileSync(datasetFile, JSON.stringify(evalEntry) + '\n');
+        console.log(`📊 Generated eval dataset: ${path.basename(datasetFile)} (${metrics.interaction_id}, ${metrics.durationMs}ms, ${metrics.inputTokens}+${metrics.outputTokens} tokens)`);
     }
     catch (error) {
-        console.warn('Failed to log metrics:', error);
+        console.error(`❌ Failed to generate eval dataset for ${metrics.interaction_id} (${metrics.test_scenario}):`, error);
     }
 }
 /**
  * Create AgenticResult and log metrics in one step
  * Reduces code duplication across providers
  *
- * PRD #143 Decision 5: Standardized metrics logging
+ * PRD #154: Updated to use unified evaluation metrics
  */
 function createAndLogAgenticResult(config) {
     const result = {
@@ -144,7 +167,39 @@ function createAndLogAgenticResult(config) {
     };
     const durationMs = Date.now() - config.startTime;
     if (config.debugMode) {
-        logMetrics(config.operation, config.sdk, result, durationMs, config.debugMode);
+        // PRD #154: Use unified evaluation metrics system
+        const evaluationMetrics = {
+            // Core execution data
+            operation: config.operation,
+            sdk: config.sdk,
+            inputTokens: config.totalTokens.input,
+            outputTokens: config.totalTokens.output,
+            durationMs,
+            // Required fields
+            iterationCount: config.iterations,
+            toolCallCount: config.toolCallsExecuted.length,
+            status: config.status,
+            completionReason: config.completionReason,
+            modelVersion: config.modelVersion,
+            // Required evaluation context - NO DEFAULTS, must be provided
+            test_scenario: config.operation,
+            ai_response_summary: config.finalMessage,
+            user_intent: config.evaluationContext?.user_intent || '', // Will be enhanced later by EvalDatasetEnhancer
+            interaction_id: config.interaction_id || '', // Will be enhanced later if missing
+            // Optional performance data
+            ...(config.totalTokens.cacheCreation !== undefined && { cacheCreationTokens: config.totalTokens.cacheCreation }),
+            ...(config.totalTokens.cacheRead !== undefined && { cacheReadTokens: config.totalTokens.cacheRead }),
+            ...(config.toolCallsExecuted.length > 0 && {
+                uniqueToolsUsed: [...new Set(config.toolCallsExecuted.map(tc => tc.tool))]
+            }),
+            ...(config.debugFiles && { debug_files: { full_prompt: config.debugFiles.promptFile, full_response: config.debugFiles.responseFile } }),
+            ...(config.evaluationContext?.failure_analysis && { failure_analysis: config.evaluationContext.failure_analysis })
+        };
+        // Calculate cache hit rate if applicable
+        if (config.totalTokens.cacheRead !== undefined && config.totalTokens.input > 0) {
+            evaluationMetrics.cacheHitRate = Math.round((config.totalTokens.cacheRead / config.totalTokens.input) * 100);
+        }
+        logEvaluationDataset(evaluationMetrics, config.debugMode);
     }
     return result;
 }

package/dist/core/providers/vercel-provider.d.ts

@@ -16,9 +16,14 @@ export declare class VercelProvider implements AIProvider {
     private initializeModel;
     getProviderType(): string;
     getDefaultModel(): string;
+    getModelName(): string;
+    getSDKProvider(): string;
     isInitialized(): boolean;
     private logDebugIfEnabled;
-    sendMessage(message: string, operation?: string): Promise<AIResponse>;
+    sendMessage(message: string, operation?: string, evaluationContext?: {
+        user_intent?: string;
+        interaction_id?: string;
+    }): Promise<AIResponse>;
     /**
      * Agentic tool loop using Vercel AI SDK
      *

package/dist/core/providers/vercel-provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vercel-provider.d.ts","sourceRoot":"","sources":["../../../src/core/providers/vercel-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,aAAa,EACd,MAAM,0BAA0B,CAAC;AAclC,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,aAAa,CAAM;gBAEf,MAAM,EAAE,gBAAgB;IAUpC,OAAO,CAAC,qBAAqB;IAU7B,OAAO,CAAC,eAAe;IAgCvB,eAAe,IAAI,MAAM;IAIzB,eAAe,IAAI,MAAM;IAIzB,aAAa,IAAI,OAAO;IAIxB,OAAO,CAAC,iBAAiB;IAqBnB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,GAAE,MAAkB,GAAG,OAAO,CAAC,UAAU,CAAC;IA8CtF;;;;;;;;;;;;OAYG;IACG,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CA0P/D"}
+{"version":3,"file":"vercel-provider.d.ts","sourceRoot":"","sources":["../../../src/core/providers/vercel-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AASH,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,aAAa,EACd,MAAM,0BAA0B,CAAC;AASlC,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,aAAa,CAAM;gBAEf,MAAM,EAAE,gBAAgB;IAUpC,OAAO,CAAC,qBAAqB;IAU7B,OAAO,CAAC,eAAe;IA4CvB,eAAe,IAAI,MAAM;IAIzB,eAAe,IAAI,MAAM;IAIzB,YAAY,IAAI,MAAM;IAItB,cAAc,IAAI,MAAM;IAIxB,aAAa,IAAI,OAAO;IAIxB,OAAO,CAAC,iBAAiB;IAiBnB,WAAW,CACf,OAAO,EAAE,MAAM,EACf,SAAS,GAAE,MAAkB,EAC7B,iBAAiB,CAAC,EAAE;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GACA,OAAO,CAAC,UAAU,CAAC;IAoGtB;;;;;;;;;;;;OAYG;IACG,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CA0T/D"}