@vfarcic/dot-ai 0.111.0 → 0.112.0

package/dist/core/providers/provider-debug-utils.d.ts

@@ -14,32 +14,52 @@ export declare function ensureDebugDirectory(): string;
  */
 export declare function generateDebugId(operation: string): string;
 /**
- * Log metrics for token usage and execution time when DEBUG_DOT_AI=true
- *
- * PRD #143 Decision 5: Extended metrics for model comparison analysis
+ * Unified evaluation metrics entry for AI quality assessment and performance tracking
+ * PRD #154: Single interface for all metrics and evaluation data
  */
-export declare function logMetrics(operation: string, sdk: string, result: {
-    totalTokens: {
-        input: number;
-        output: number;
-        cacheCreation?: number;
-        cacheRead?: number;
+export interface EvaluationMetrics {
+    operation: string;
+    sdk: string;
+    inputTokens: number;
+    outputTokens: number;
+    durationMs: number;
+    iterationCount: number;
+    toolCallCount: number;
+    status: string;
+    completionReason: string;
+    modelVersion: string;
+    cacheCreationTokens?: number;
+    cacheReadTokens?: number;
+    cacheHitRate?: number;
+    uniqueToolsUsed?: string[];
+    test_scenario: string;
+    ai_response_summary: string;
+    debug_files?: {
+        full_prompt: string;
+        full_response: string;
     };
-    iterations?: number;
-    toolCallsExecuted?: Array<{
-        tool: string;
-        input: any;
-        output: any;
-    }>;
-    status?: string;
-    completionReason?: string;
-    modelVersion?: string;
-}, durationMs: number, debugMode: boolean): void;
+    user_intent: string;
+    interaction_id: string;
+    failure_analysis?: string;
+}
+/**
+ * Determine if dataset generation should be skipped for specific operations
+ */
+export declare function shouldSkipDatasetGeneration(operation: string): boolean;
+/**
+ * Log unified evaluation metrics when DEBUG_DOT_AI=true
+ * Single function for all metrics and evaluation data capture
+ */
+/**
+ * Generate eval dataset entry in standard OpenAI Evals format
+ * Logs evaluation metrics to JSONL dataset files for AI quality assessment
+ */
+export declare function logEvaluationDataset(metrics: EvaluationMetrics, debugMode?: boolean): void;
 /**
  * Create AgenticResult and log metrics in one step
  * Reduces code duplication across providers
  *
- * PRD #143 Decision 5: Standardized metrics logging
+ * PRD #154: Updated to use unified evaluation metrics
  */
 export declare function createAndLogAgenticResult(config: {
     finalMessage: string;
@@ -62,6 +82,15 @@ export declare function createAndLogAgenticResult(config: {
     sdk: string;
     startTime: number;
     debugMode: boolean;
+    debugFiles?: {
+        promptFile: string;
+        responseFile: string;
+    } | null;
+    evaluationContext?: {
+        user_intent?: string;
+        failure_analysis?: string;
+    };
+    interaction_id?: string;
 }): AgenticResult;
 /**
  * Save AI interaction for debugging when DEBUG_DOT_AI=true

package/dist/core/providers/provider-debug-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provider-debug-utils.d.ts","sourceRoot":"","sources":["../../../src/core/providers/provider-debug-utils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAErE;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAM7C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAKzD;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE;IACN,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAC;QAAC,MAAM,EAAE,GAAG,CAAA;KAAE,CAAC,CAAC;IACrE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,OAAO,GACjB,IAAI,CA2DN;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAC;QAAC,MAAM,EAAE,GAAG,CAAA;KAAE,CAAC,CAAC;IACpE,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,aAAa,CAAC;IACzD,gBAAgB,EAAE,wBAAwB,GAAG,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,OAAO,CAAC;IAC5G,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACpB,GAAG,aAAa,CAiBhB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,UAAU,EACpB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,OAAO,GACjB,IAAI,CAkCN"}
+{"version":3,"file":"provider-debug-utils.d.ts","sourceRoot":"","sources":["../../../src/core/providers/provider-debug-utils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAErE;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAM7C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAKzD;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAEhC,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IAGnB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IAGrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAG3B,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE;QACZ,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAGF,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IAGvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAGtE;AAED;;;GAGG;AACH;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,iBAAiB,EAC1B,SAAS,GAAE,OAAe,GACzB,IAAI,CAiFN;AAGD;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAC;QAAC,MAAM,EAAE,GAAG,CAAA;KAAE,CAAC,CAAC;IACpE,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,aAAa,CAAC;IACzD,gBAAgB,EAAE,wBAAwB,GAAG,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,OAAO,CAAC;IAC5G,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAGjE,iBAAiB,CAAC,EAAE;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC;IAGF,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GAAG,aAAa,CAsDhB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,UAAU,EACpB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,OAAO,GACjB,IAAI,CAkCN"}

package/dist/core/providers/provider-debug-utils.js

@@ -41,7 +41,8 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ensureDebugDirectory = ensureDebugDirectory;
 exports.generateDebugId = generateDebugId;
-exports.logMetrics = logMetrics;
+exports.shouldSkipDatasetGeneration = shouldSkipDatasetGeneration;
+exports.logEvaluationDataset = logEvaluationDataset;
 exports.createAndLogAgenticResult = createAndLogAgenticResult;
 exports.debugLogInteraction = debugLogInteraction;
 const fs = __importStar(require("fs"));
@@ -67,70 +68,103 @@ function generateDebugId(operation) {
     return `${dateTime}_${randomHex}_${operation}`;
 }
 /**
- * Log metrics for token usage and execution time when DEBUG_DOT_AI=true
- *
- * PRD #143 Decision 5: Extended metrics for model comparison analysis
+ * Determine if dataset generation should be skipped for specific operations
  */
-function logMetrics(operation, sdk, result, durationMs, debugMode) {
+function shouldSkipDatasetGeneration(operation) {
+    const skipDatasetOperations = ['version-connectivity-check', 'generic'];
+    return skipDatasetOperations.includes(operation);
+}
+/**
+ * Log unified evaluation metrics when DEBUG_DOT_AI=true
+ * Single function for all metrics and evaluation data capture
+ */
+/**
+ * Generate eval dataset entry in standard OpenAI Evals format
+ * Logs evaluation metrics to JSONL dataset files for AI quality assessment
+ */
+function logEvaluationDataset(metrics, debugMode = false) {
     if (!debugMode)
         return;
+    // Skip dataset generation for non-evaluable operations
+    if (shouldSkipDatasetGeneration(metrics.test_scenario))
+        return;
     try {
-        const debugDir = ensureDebugDirectory();
-        const metricsFile = path.join(debugDir, 'metrics.jsonl');
-        const entry = {
-            timestamp: new Date().toISOString(),
-            sdk,
-            operation,
-            inputTokens: result.totalTokens.input,
-            outputTokens: result.totalTokens.output,
-            durationMs
-        };
-        // Add cache metrics if present
-        if (result.totalTokens.cacheCreation !== undefined) {
-            entry.cacheCreationTokens = result.totalTokens.cacheCreation;
+        const evalDir = path.join(process.cwd(), 'eval', 'datasets');
+        // Ensure eval datasets directory exists
+        if (!fs.existsSync(evalDir)) {
+            fs.mkdirSync(evalDir, { recursive: true });
         }
-        if (result.totalTokens.cacheRead !== undefined) {
-            entry.cacheReadTokens = result.totalTokens.cacheRead;
+        // Parse operation for tool name
+        const operationParts = metrics.operation.split('-');
+        const toolName = operationParts[0]; // e.g., "remediate"
+        // Check if this is a comparative evaluation
+        const isComparativeEvaluation = metrics.operation.includes('-comparative-');
+        let datasetFile;
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '').split('T').join('_');
+        if (isComparativeEvaluation) {
+            // For comparative evaluations, don't include single model name since it compares multiple models
+            datasetFile = path.join(evalDir, `${toolName}_comparative_evaluation_${timestamp}.jsonl`);
         }
-        // Calculate cache hit rate (percentage)
-        if (result.totalTokens.cacheRead !== undefined && result.totalTokens.input > 0) {
-            entry.cacheHitRate = Math.round((result.totalTokens.cacheRead / result.totalTokens.input) * 100);
+        else {
+            // Extract model name from modelVersion or sdk for single-model datasets
+            let modelName = 'unknown';
+            if (metrics.modelVersion) {
+                if (metrics.modelVersion.includes('sonnet')) {
+                    modelName = 'sonnet';
+                }
+                else if (metrics.modelVersion.includes('gpt-5-pro')) {
+                    modelName = 'gpt-pro';
+                }
+                else if (metrics.modelVersion.includes('gpt')) {
+                    modelName = 'gpt';
+                }
+                else if (metrics.modelVersion.includes('gemini')) {
+                    modelName = 'gemini';
+                }
+            }
+            // Create filename with interaction ID, SDK, model, and timestamp for single-model datasets
+            datasetFile = path.join(evalDir, `${toolName}_${metrics.interaction_id}_${metrics.sdk}_${modelName}_${timestamp}.jsonl`);
         }
-        // Add extended metrics (PRD #143 Decision 5)
-        if (result.iterations !== undefined) {
-            entry.iterationCount = result.iterations;
-        }
-        if (result.toolCallsExecuted) {
-            entry.toolCallCount = result.toolCallsExecuted.length;
-            // Extract unique tool names
-            const uniqueTools = [...new Set(result.toolCallsExecuted.map(tc => tc.tool))];
-            entry.uniqueToolsUsed = uniqueTools;
-        }
-        if (result.status) {
-            entry.status = result.status;
-        }
-        if (result.completionReason) {
-            entry.completionReason = result.completionReason;
-        }
-        if (result.modelVersion) {
-            entry.modelVersion = result.modelVersion;
-        }
-        // Manual annotation placeholders (populate after test analysis)
-        entry.manualNotes = '';
-        entry.failureReason = '';
-        entry.qualityIssues = [];
-        entry.comparisonNotes = '';
-        fs.appendFileSync(metricsFile, JSON.stringify(entry) + '\n');
+        // Transform metrics into OpenAI Evals format (no ideal field - using model-graded evaluation)
+        const evalEntry = {
+            input: {
+                issue: metrics.user_intent || "Tool execution scenario"
+            },
+            output: metrics.ai_response_summary || "",
+            performance: {
+                duration_ms: metrics.durationMs,
+                input_tokens: metrics.inputTokens,
+                output_tokens: metrics.outputTokens,
+                total_tokens: metrics.inputTokens + metrics.outputTokens,
+                sdk: metrics.sdk,
+                model_version: metrics.modelVersion,
+                iterations: metrics.iterationCount,
+                tool_calls_executed: metrics.toolCallCount,
+                cache_read_tokens: metrics.cacheReadTokens || 0,
+                cache_creation_tokens: metrics.cacheCreationTokens || 0
+            },
+            metadata: {
+                timestamp: new Date().toISOString(),
+                complexity: "medium",
+                tags: ["troubleshooting"],
+                source: "integration_test",
+                tool: toolName,
+                test_scenario: metrics.test_scenario || `${toolName}_test`,
+                failure_analysis: metrics.failure_analysis || ""
+            }
+        };
+        fs.writeFileSync(datasetFile, JSON.stringify(evalEntry) + '\n');
+        console.log(`📊 Generated eval dataset: ${path.basename(datasetFile)} (${metrics.interaction_id}, ${metrics.durationMs}ms, ${metrics.inputTokens}+${metrics.outputTokens} tokens)`);
     }
     catch (error) {
-        console.warn('Failed to log metrics:', error);
+        console.error(`❌ Failed to generate eval dataset for ${metrics.interaction_id} (${metrics.test_scenario}):`, error);
     }
 }
 /**
  * Create AgenticResult and log metrics in one step
  * Reduces code duplication across providers
  *
- * PRD #143 Decision 5: Standardized metrics logging
+ * PRD #154: Updated to use unified evaluation metrics
  */
 function createAndLogAgenticResult(config) {
     const result = {
@@ -144,7 +178,39 @@ function createAndLogAgenticResult(config) {
     };
     const durationMs = Date.now() - config.startTime;
     if (config.debugMode) {
-        logMetrics(config.operation, config.sdk, result, durationMs, config.debugMode);
+        // PRD #154: Use unified evaluation metrics system
+        const evaluationMetrics = {
+            // Core execution data
+            operation: config.operation,
+            sdk: config.sdk,
+            inputTokens: config.totalTokens.input,
+            outputTokens: config.totalTokens.output,
+            durationMs,
+            // Required fields
+            iterationCount: config.iterations,
+            toolCallCount: config.toolCallsExecuted.length,
+            status: config.status,
+            completionReason: config.completionReason,
+            modelVersion: config.modelVersion,
+            // Required evaluation context - NO DEFAULTS, must be provided
+            test_scenario: config.operation,
+            ai_response_summary: config.finalMessage,
+            user_intent: config.evaluationContext?.user_intent || '', // Will be enhanced later by EvalDatasetEnhancer
+            interaction_id: config.interaction_id || '', // Will be enhanced later if missing
+            // Optional performance data
+            ...(config.totalTokens.cacheCreation !== undefined && { cacheCreationTokens: config.totalTokens.cacheCreation }),
+            ...(config.totalTokens.cacheRead !== undefined && { cacheReadTokens: config.totalTokens.cacheRead }),
+            ...(config.toolCallsExecuted.length > 0 && {
+                uniqueToolsUsed: [...new Set(config.toolCallsExecuted.map(tc => tc.tool))]
+            }),
+            ...(config.debugFiles && { debug_files: { full_prompt: config.debugFiles.promptFile, full_response: config.debugFiles.responseFile } }),
+            ...(config.evaluationContext?.failure_analysis && { failure_analysis: config.evaluationContext.failure_analysis })
+        };
+        // Calculate cache hit rate if applicable
+        if (config.totalTokens.cacheRead !== undefined && config.totalTokens.input > 0) {
+            evaluationMetrics.cacheHitRate = Math.round((config.totalTokens.cacheRead / config.totalTokens.input) * 100);
+        }
+        logEvaluationDataset(evaluationMetrics, config.debugMode);
     }
     return result;
 }

package/dist/core/providers/vercel-provider.d.ts

@@ -18,7 +18,10 @@ export declare class VercelProvider implements AIProvider {
     getDefaultModel(): string;
     isInitialized(): boolean;
     private logDebugIfEnabled;
-    sendMessage(message: string, operation?: string): Promise<AIResponse>;
+    sendMessage(message: string, operation?: string, evaluationContext?: {
+        user_intent?: string;
+        interaction_id?: string;
+    }): Promise<AIResponse>;
     /**
      * Agentic tool loop using Vercel AI SDK
      *

package/dist/core/providers/vercel-provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vercel-provider.d.ts","sourceRoot":"","sources":["../../../src/core/providers/vercel-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,aAAa,EACd,MAAM,0BAA0B,CAAC;AAclC,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,aAAa,CAAM;gBAEf,MAAM,EAAE,gBAAgB;IAUpC,OAAO,CAAC,qBAAqB;IAU7B,OAAO,CAAC,eAAe;IAgCvB,eAAe,IAAI,MAAM;IAIzB,eAAe,IAAI,MAAM;IAIzB,aAAa,IAAI,OAAO;IAIxB,OAAO,CAAC,iBAAiB;IAqBnB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,GAAE,MAAkB,GAAG,OAAO,CAAC,UAAU,CAAC;IA8CtF;;;;;;;;;;;;OAYG;IACG,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CA0P/D"}
+{"version":3,"file":"vercel-provider.d.ts","sourceRoot":"","sources":["../../../src/core/providers/vercel-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,aAAa,EACd,MAAM,0BAA0B,CAAC;AAMlC,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,YAAY,CAAoB;IACxC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,aAAa,CAAM;gBAEf,MAAM,EAAE,gBAAgB;IAUpC,OAAO,CAAC,qBAAqB;IAU7B,OAAO,CAAC,eAAe;IAiCvB,eAAe,IAAI,MAAM;IAIzB,eAAe,IAAI,MAAM;IAIzB,aAAa,IAAI,OAAO;IAIxB,OAAO,CAAC,iBAAiB;IAiBnB,WAAW,CACf,OAAO,EAAE,MAAM,EACf,SAAS,GAAE,MAAkB,EAC7B,iBAAiB,CAAC,EAAE;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GACA,OAAO,CAAC,UAAU,CAAC;IAwEtB;;;;;;;;;;;;OAYG;IACG,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CAwQ/D"}

package/dist/core/providers/vercel-provider.js

@@ -5,39 +5,6 @@
  * Implements AIProvider interface using Vercel AI SDK.
  * Supports OpenAI and Google Gemini providers through unified interface.
  */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VercelProvider = void 0;
 const ai_1 = require("ai");
@@ -45,14 +12,7 @@ const openai_1 = require("@ai-sdk/openai");
 const google_1 = require("@ai-sdk/google");
 const anthropic_1 = require("@ai-sdk/anthropic");
 const provider_debug_utils_1 = require("./provider-debug-utils");
-/**
- * Provider-specific default models
- */
-const PROVIDER_MODELS = {
-    openai: 'gpt-5',
-    google: 'gemini-2.5-pro',
-    anthropic: 'claude-sonnet-4-5-20250929'
-};
+const model_config_1 = require("../model-config");
 class VercelProvider {
     providerType;
     model;
@@ -71,14 +31,15 @@ class VercelProvider {
         if (!this.apiKey) {
             throw new Error(`API key is required for ${this.providerType} provider`);
         }
-        if (!['openai', 'google', 'anthropic'].includes(this.providerType)) {
-            throw new Error(`Unsupported provider: ${this.providerType}. Must be 'openai', 'google', or 'anthropic'`);
+        if (!['openai', 'openai_pro', 'google', 'anthropic'].includes(this.providerType)) {
+            throw new Error(`Unsupported provider: ${this.providerType}. Must be 'openai', 'openai_pro', 'google', or 'anthropic'`);
         }
     }
     initializeModel() {
         try {
             switch (this.providerType) {
-                case 'openai': {
+                case 'openai':
+                case 'openai_pro': {
                     const provider = (0, openai_1.createOpenAI)({
                         apiKey: this.apiKey
                     });
@@ -111,27 +72,23 @@ class VercelProvider {
         return 'vercel';
     }
     getDefaultModel() {
-        return PROVIDER_MODELS[this.providerType];
+        return model_config_1.CURRENT_MODELS[this.providerType];
     }
     isInitialized() {
         return this.modelInstance !== undefined;
     }
-    logDebugIfEnabled(operation, prompt, response, durationMs) {
+    logDebugIfEnabled(operation, prompt, response) {
         if (!this.debugMode)
-            return;
+            return null;
         const debugId = (0, provider_debug_utils_1.generateDebugId)(operation);
         (0, provider_debug_utils_1.debugLogInteraction)(debugId, prompt, response, operation, this.getProviderType(), this.model, this.debugMode);
-        // Use logMetrics for sendMessage calls (simple token structure, no extended metrics)
-        (0, provider_debug_utils_1.logMetrics)(operation, this.getProviderType(), {
-            totalTokens: {
-                input: response.usage.input_tokens,
-                output: response.usage.output_tokens,
-                cacheCreation: response.usage.cache_creation_input_tokens,
-                cacheRead: response.usage.cache_read_input_tokens
-            }
-        }, durationMs, this.debugMode);
+        // Return the actual debug file names created
+        return {
+            promptFile: `${debugId}_prompt.md`,
+            responseFile: `${debugId}_response.md`
+        };
     }
-    async sendMessage(message, operation = 'generic') {
+    async sendMessage(message, operation = 'generic', evaluationContext) {
         if (!this.isInitialized()) {
             throw new Error(`${this.providerType} provider not initialized`);
         }
@@ -146,8 +103,8 @@ class VercelProvider {
             const response = {
                 content: result.text,
                 usage: {
-                    input_tokens: result.usage.inputTokens || 0,
-                    output_tokens: result.usage.outputTokens || 0
+                    input_tokens: (result.totalUsage || result.usage).inputTokens || 0,
+                    output_tokens: (result.totalUsage || result.usage).outputTokens || 0
                 }
             };
             const durationMs = Date.now() - startTime;
@@ -155,14 +112,34 @@ class VercelProvider {
             if (this.debugMode) {
                 const debugId = (0, provider_debug_utils_1.generateDebugId)(operation);
                 (0, provider_debug_utils_1.debugLogInteraction)(debugId, message, response, operation, this.getProviderType(), this.model, this.debugMode);
-                (0, provider_debug_utils_1.logMetrics)(operation, this.getProviderType(), {
-                    totalTokens: {
-                        input: response.usage.input_tokens,
-                        output: response.usage.output_tokens,
-                        cacheCreation: response.usage.cache_creation_input_tokens,
-                        cacheRead: response.usage.cache_read_input_tokens
-                    }
-                }, durationMs, this.debugMode);
+                // PRD #154: Always use new evaluation dataset system
+                const evaluationMetrics = {
+                    // Core execution data
+                    operation,
+                    sdk: this.getProviderType(),
+                    inputTokens: response.usage.input_tokens,
+                    outputTokens: response.usage.output_tokens,
+                    durationMs,
+                    // Required fields
+                    iterationCount: 1,
+                    toolCallCount: 0,
+                    status: 'completed',
+                    completionReason: 'stop',
+                    modelVersion: this.model,
+                    // Required evaluation context - NO DEFAULTS, must be provided
+                    test_scenario: operation,
+                    ai_response_summary: response.content,
+                    user_intent: evaluationContext?.user_intent || '',
+                    interaction_id: evaluationContext?.interaction_id || '',
+                    // Optional performance data
+                    ...(response.usage.cache_creation_input_tokens && { cacheCreationTokens: response.usage.cache_creation_input_tokens }),
+                    ...(response.usage.cache_read_input_tokens && { cacheReadTokens: response.usage.cache_read_input_tokens })
+                };
+                // Calculate cache hit rate if applicable
+                if (response.usage.cache_read_input_tokens && response.usage.input_tokens > 0) {
+                    evaluationMetrics.cacheHitRate = Math.round((response.usage.cache_read_input_tokens / response.usage.input_tokens) * 100);
+                }
+                (0, provider_debug_utils_1.logEvaluationDataset)(evaluationMetrics, this.debugMode);
             }
             return response;
         }
@@ -272,48 +249,6 @@ class VercelProvider {
                 generateConfig.system = systemParam;
             }
             const result = await (0, ai_1.generateText)(generateConfig);
-            // Debug: Log the full cumulative context that was actually sent to the AI
-            if (this.debugMode && result.response?.messages) {
-                const path = await Promise.resolve().then(() => __importStar(require('path')));
-                const debugId = (0, provider_debug_utils_1.generateDebugId)(`${operation}-final-context`);
-                const debugDir = path.join(process.cwd(), 'tmp', 'debug-ai');
-                const contextFile = path.join(debugDir, `${debugId}_full-context.md`);
-                // Build full conversation history representation
-                const messages = result.response.messages;
-                const contextParts = [`# Full Conversation Context - ${operation}\n`];
-                contextParts.push(`\nTimestamp: ${new Date().toISOString()}`);
-                contextParts.push(`Provider: ${this.getProviderType()}`);
-                contextParts.push(`Model: ${this.model}`);
-                contextParts.push(`Total Messages: ${messages.length}`);
-                contextParts.push(`Total Steps: ${result.steps?.length || 0}`);
-                contextParts.push('\n---\n');
-                for (let i = 0; i < messages.length; i++) {
-                    const msg = messages[i];
-                    contextParts.push(`\n## Message ${i + 1} - Role: ${msg.role}\n`);
-                    if (typeof msg.content === 'string') {
-                        contextParts.push(msg.content);
-                    }
-                    else if (Array.isArray(msg.content)) {
-                        for (const part of msg.content) {
-                            if (part.type === 'text') {
-                                contextParts.push(part.text || '');
-                            }
-                            else if (part.type === 'tool-call') {
-                                contextParts.push(`\n[TOOL CALL: ${part.toolName}]`);
-                                contextParts.push(JSON.stringify(part.args, null, 2));
-                            }
-                            else if (part.type === 'tool-result') {
-                                contextParts.push(`\n[TOOL RESULT: ${part.toolName}]`);
-                                const resultData = part.output || part.result || part.content || part;
-                                contextParts.push(JSON.stringify(resultData, null, 2));
-                            }
-                        }
-                    }
-                }
-                const fs = await Promise.resolve().then(() => __importStar(require('fs')));
-                fs.writeFileSync(contextFile, contextParts.join('\n'));
-                console.log(`🐛 DEBUG: Full conversation context logged to ${contextFile}`);
-            }
             // Extract tool call history from steps
             const toolCallsExecuted = [];
             for (const step of result.steps || []) {
@@ -332,11 +267,11 @@ class VercelProvider {
             // - GitHub Issue #8795: Token reporting issues with Anthropic provider (streaming)
             // Our version (5.0.60, released Oct 2, 2025) includes these fixes.
             // However, testing still shows ~70% fewer tokens reported vs Anthropic native SDK.
-            // Root cause unknown - may be additional unreported bugs or different calculation methods.
-            const usage = result.usage;
+            // Root cause: We were using result.usage (final step only) instead of result.totalUsage (sum of all steps)!
+            const usage = result.totalUsage || result.usage;
             let cacheReadTokens = 0;
             let cacheCreationTokens = 0;
-            // Anthropic via Vercel uses cachedInputTokens
+            // Anthropic via Vercel uses cachedInputTokens (confirmed in AI SDK 5+)
             if (usage.cachedInputTokens) {
                 cacheReadTokens = usage.cachedInputTokens;
             }
@@ -367,6 +302,57 @@ class VercelProvider {
                     }
                 }
             }
+            // Log debug for summary operations to capture complete prompts/responses for evaluation
+            let debugFiles = null;
+            if (this.debugMode) {
+                // Build the full conversation context like Anthropic provider does
+                let finalPrompt = `System: ${config.systemPrompt}\n\n`;
+                // Always include the original user intent first
+                finalPrompt += `user: ${config.userMessage}\n\n`;
+                // Then add the conversation history if available
+                if (result.response?.messages) {
+                    finalPrompt += result.response.messages
+                        .map(msg => {
+                        if (typeof msg.content === 'string') {
+                            return `${msg.role}: ${msg.content}`;
+                        }
+                        else if (Array.isArray(msg.content)) {
+                            const contentParts = msg.content.map(part => {
+                                if (part.type === 'text') {
+                                    return part.text;
+                                }
+                                else if (part.type === 'tool-call') {
+                                    return `[TOOL_USE: ${part.toolName}]`;
+                                }
+                                else if (part.type === 'tool-result') {
+                                    const resultData = part.output || part.result || part.content;
+                                    if (typeof resultData === 'string') {
+                                        return `[TOOL_RESULT: ${part.toolName}]\n${resultData}`;
+                                    }
+                                    else if (resultData) {
+                                        return `[TOOL_RESULT: ${part.toolName}]\n${JSON.stringify(resultData, null, 2)}`;
+                                    }
+                                    return `[TOOL_RESULT: ${part.toolName}]`;
+                                }
+                                return `[${part.type}]`;
+                            }).join(' ');
+                            return `${msg.role}: ${contentParts}`;
+                        }
+                        return `${msg.role}: [complex_content]`;
+                    })
+                        .join('\n\n');
+                }
+                const aiResponse = {
+                    content: finalText || '',
+                    usage: {
+                        input_tokens: usage.inputTokens || 0,
+                        output_tokens: usage.outputTokens || 0,
+                        cache_creation_input_tokens: cacheCreationTokens,
+                        cache_read_input_tokens: cacheReadTokens
+                    }
+                };
+                debugFiles = this.logDebugIfEnabled(`${operation}-summary`, finalPrompt, aiResponse);
+            }
             return (0, provider_debug_utils_1.createAndLogAgenticResult)({
                 finalMessage: finalText || '',
                 iterations: result.steps?.length || 1,
@@ -383,7 +369,10 @@ class VercelProvider {
                 operation: `${operation}-summary`,
                 sdk: this.getProviderType(),
                 startTime,
-                debugMode: this.debugMode
+                debugMode: this.debugMode,
+                debugFiles,
+                evaluationContext: config.evaluationContext,
+                interaction_id: config.interaction_id
             });
         }
         catch (error) {
@@ -404,7 +393,9 @@ class VercelProvider {
                 operation: `${operation}-error`,
                 sdk: this.getProviderType(),
                 startTime,
-                debugMode: this.debugMode
+                debugMode: this.debugMode,
+                evaluationContext: config.evaluationContext,
+                interaction_id: config.interaction_id
             });
         }
     }