@vfarcic/dot-ai 0.111.0 → 0.113.0

package/dist/core/platform-operations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"platform-operations.d.ts","sourceRoot":"","sources":["../../src/core/platform-operations.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAM1C,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,4CAA4C,CAAC;AAGzF,OAAO,EAAE,SAAS,EAAE,MAAM,4CAA4C,CAAC;AAEvE,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC;IACjD,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,WAAW,EAAE,mBAAmB,GAAG,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;IACtE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,SAAS,EAAE,CAAC,CAmCtB;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,SAAS,EAAE,EACvB,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAuE9B;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,EAAE,iBAAiB,EAAE,EAC/B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,eAAe,CAAC,CA8B1B;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GACb,eAAe,GAAG,IAAI,CAmBxB;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,eAAe,EACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAkE/F"}
+{"version":3,"file":"platform-operations.d.ts","sourceRoot":"","sources":["../../src/core/platform-operations.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAM1C,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,4CAA4C,CAAC;AAGzF,OAAO,EAAE,SAAS,EAAE,MAAM,4CAA4C,CAAC;AAEvE,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC;IACjD,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,GAAG,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7B,WAAW,EAAE,mBAAmB,GAAG,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;IACtE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,SAAS,EAAE,CAAC,CAkCtB;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,SAAS,EAAE,EACvB,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,aAAa,CAAC,CAoCxB;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAuE9B;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,EAAE,iBAAiB,EAAE,EAC/B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,eAAe,CAAC,CA8B1B;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GACb,eAAe,GAAG,IAAI,CAmBxB;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,eAAe,EACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAkE/F"}

package/dist/core/platform-operations.js

@@ -69,8 +69,7 @@ async function discoverOperations(aiProvider, logger) {
         // Single AI call with injected data
         const response = await aiProvider.sendMessage(prompt, 'platform-discover-operations');
         // Parse operations from AI response
-        const jsonContent = (0, platform_utils_1.stripMarkdownCodeBlocks)(response.content);
-        const operations = JSON.parse(jsonContent);
+        const operations = (0, platform_utils_1.extractJsonFromAIResponse)(response.content);
         logger.info?.('Discovered operations from Nu scripts', {
             count: operations.length
         });
@@ -95,9 +94,8 @@ async function mapIntentToOperation(intent, operations, aiProvider, logger) {
             .replace('{operations}', JSON.stringify(operations, null, 2));
         // Send to AI provider for AI-powered intent matching
         const response = await aiProvider.sendMessage(prompt, 'platform-map-intent');
-        // Strip markdown code blocks and parse JSON
-        const jsonContent = (0, platform_utils_1.stripMarkdownCodeBlocks)(response.content);
-        const mapping = JSON.parse(jsonContent);
+        // Extract JSON from AI response with robust parsing
+        const mapping = (0, platform_utils_1.extractJsonFromAIResponse)(response.content);
         // Validate that AI returned required fields
         if (mapping.matched && mapping.operation) {
             if (!mapping.operation.command || !Array.isArray(mapping.operation.command)) {

package/dist/core/platform-utils.d.ts

@@ -10,7 +10,18 @@ export declare const execAsync: typeof exec.__promisify__;
  */
 export declare function getScriptsDir(): string;
 /**
- * Strip markdown code blocks from AI response
+ * Extract JSON object from AI response with robust parsing
+ * Handles markdown code blocks and finds proper JSON boundaries
  */
-export declare function stripMarkdownCodeBlocks(content: string): string;
+export declare function extractJsonFromAIResponse(aiResponse: string): any;
+/**
+ * Extract content from markdown code blocks in AI responses
+ * Handles various code block formats: ```yaml, ```yml, ```json, or plain ```
+ */
+export declare function extractContentFromMarkdownCodeBlocks(content: string, language?: string): string;
+/**
+ * Extract JSON array from AI response with robust parsing
+ * Handles markdown code blocks and finds proper array boundaries
+ */
+export declare function extractJsonArrayFromAIResponse(aiResponse: string): any[];
 //# sourceMappingURL=platform-utils.d.ts.map

package/dist/core/platform-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"platform-utils.d.ts","sourceRoot":"","sources":["../../src/core/platform-utils.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAIrC,eAAO,MAAM,SAAS,2BAAkB,CAAC;AAEzC;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,CAItC;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAQ/D"}
+{"version":3,"file":"platform-utils.d.ts","sourceRoot":"","sources":["../../src/core/platform-utils.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAIrC,eAAO,MAAM,SAAS,2BAAkB,CAAC;AAEzC;;GAEG;AACH,wBAAgB,aAAa,IAAI,MAAM,CAItC;AAGD;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAkCjE;AAED;;;GAGG;AACH,wBAAgB,oCAAoC,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAY/F;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,EAAE,CAkCxE"}

package/dist/core/platform-utils.js

@@ -40,7 +40,9 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.execAsync = void 0;
 exports.getScriptsDir = getScriptsDir;
-exports.stripMarkdownCodeBlocks = stripMarkdownCodeBlocks;
+exports.extractJsonFromAIResponse = extractJsonFromAIResponse;
+exports.extractContentFromMarkdownCodeBlocks = extractContentFromMarkdownCodeBlocks;
+exports.extractJsonArrayFromAIResponse = extractJsonArrayFromAIResponse;
 const child_process_1 = require("child_process");
 const util_1 = require("util");
 const path = __importStar(require("path"));
@@ -54,15 +56,95 @@ function getScriptsDir() {
     return path.join(__dirname, '..', '..', 'scripts');
 }
 /**
- * Strip markdown code blocks from AI response
+ * Extract JSON object from AI response with robust parsing
+ * Handles markdown code blocks and finds proper JSON boundaries
  */
-function stripMarkdownCodeBlocks(content) {
-    let jsonContent = content.trim();
-    if (jsonContent.startsWith('```json')) {
-        jsonContent = jsonContent.replace(/^```json\s*/, '').replace(/\s*```$/, '');
+function extractJsonFromAIResponse(aiResponse) {
+    let jsonContent = aiResponse;
+    // First try to find JSON wrapped in code blocks
+    const codeBlockMatch = aiResponse.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
+    if (codeBlockMatch) {
+        jsonContent = codeBlockMatch[1];
     }
-    else if (jsonContent.startsWith('```')) {
-        jsonContent = jsonContent.replace(/^```\s*/, '').replace(/\s*```$/, '');
+    else {
+        // Try to find JSON that starts with { and find the matching closing }
+        const startIndex = aiResponse.indexOf('{');
+        if (startIndex !== -1) {
+            let braceCount = 0;
+            let endIndex = startIndex;
+            for (let i = startIndex; i < aiResponse.length; i++) {
+                if (aiResponse[i] === '{')
+                    braceCount++;
+                if (aiResponse[i] === '}')
+                    braceCount--;
+                if (braceCount === 0) {
+                    endIndex = i;
+                    break;
+                }
+            }
+            if (endIndex > startIndex) {
+                jsonContent = aiResponse.substring(startIndex, endIndex + 1);
+            }
+        }
+    }
+    try {
+        return JSON.parse(jsonContent.trim());
+    }
+    catch (error) {
+        throw new Error(`Failed to parse JSON from AI response: ${error}`);
+    }
+}
+/**
+ * Extract content from markdown code blocks in AI responses
+ * Handles various code block formats: ```yaml, ```yml, ```json, or plain ```
+ */
+function extractContentFromMarkdownCodeBlocks(content, language) {
+    // Create regex pattern for the specified language or any language
+    const languagePattern = language ? `(?:${language})` : '(?:yaml|yml|json)?';
+    const regex = new RegExp(`\`\`\`${languagePattern}\\s*([\\s\\S]*?)\\s*\`\`\``, 'g');
+    const match = regex.exec(content);
+    if (match && match[1]) {
+        return match[1].trim();
+    }
+    // Return original content if no code blocks found
+    return content.trim();
+}
+/**
+ * Extract JSON array from AI response with robust parsing
+ * Handles markdown code blocks and finds proper array boundaries
+ */
+function extractJsonArrayFromAIResponse(aiResponse) {
+    let jsonContent = aiResponse;
+    // First try to find JSON array wrapped in code blocks
+    const codeBlockMatch = aiResponse.match(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/);
+    if (codeBlockMatch) {
+        jsonContent = codeBlockMatch[1];
+    }
+    else {
+        // Try to find JSON array that starts with [ and find the matching closing ]
+        const startIndex = aiResponse.indexOf('[');
+        if (startIndex !== -1) {
+            let bracketCount = 0;
+            let endIndex = startIndex;
+            for (let i = startIndex; i < aiResponse.length; i++) {
+                if (aiResponse[i] === '[')
+                    bracketCount++;
+                if (aiResponse[i] === ']')
+                    bracketCount--;
+                if (bracketCount === 0) {
+                    endIndex = i;
+                    break;
+                }
+            }
+            if (bracketCount === 0) {
+                jsonContent = aiResponse.substring(startIndex, endIndex + 1);
+            }
+        }
+    }
+    try {
+        return JSON.parse(jsonContent.trim());
+    }
+    catch (error) {
+        throw new Error(`Failed to parse JSON array from AI response: ${error}`);
     }
-    return jsonContent;
 }

package/dist/core/providers/anthropic-provider.d.ts

@@ -14,12 +14,17 @@ export declare class AnthropicProvider implements AIProvider {
     private validateApiKey;
     getProviderType(): string;
     getDefaultModel(): string;
+    getModelName(): string;
+    getSDKProvider(): string;
     isInitialized(): boolean;
     /**
      * Helper method to log debug information if debug mode is enabled
      */
     private logDebugIfEnabled;
-    sendMessage(message: string, operation?: string): Promise<AIResponse>;
+    sendMessage(message: string, operation?: string, evaluationContext?: {
+        user_intent?: string;
+        interaction_id?: string;
+    }): Promise<AIResponse>;
     /**
      * Agentic tool loop implementation
      *

package/dist/core/providers/anthropic-provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"anthropic-provider.d.ts","sourceRoot":"","sources":["../../../src/core/providers/anthropic-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,aAAa,EACd,MAAM,0BAA0B,CAAC;AAGlC,qBAAa,iBAAkB,YAAW,UAAU;IAClD,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAU;gBAEf,MAAM,EAAE,gBAAgB;IAWpC,OAAO,CAAC,cAAc;IAStB,eAAe,IAAI,MAAM;IAIzB,eAAe,IAAI,MAAM;IAIzB,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAqBnB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,GAAE,MAAkB,GAAG,OAAO,CAAC,UAAU,CAAC;IAoDtF;;;;;;;;;;;;;;;;;;;OAmBG;IACG,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CAmO/D"}
+{"version":3,"file":"anthropic-provider.d.ts","sourceRoot":"","sources":["../../../src/core/providers/anthropic-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,aAAa,EACd,MAAM,0BAA0B,CAAC;AAIlC,qBAAa,iBAAkB,YAAW,UAAU;IAClD,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,SAAS,CAAU;gBAEf,MAAM,EAAE,gBAAgB;IAepC,OAAO,CAAC,cAAc;IAStB,eAAe,IAAI,MAAM;IAIzB,eAAe,IAAI,MAAM;IAIzB,YAAY,IAAI,MAAM;IAItB,cAAc,IAAI,MAAM;IAIxB,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAiBnB,WAAW,CACf,OAAO,EAAE,MAAM,EACf,SAAS,GAAE,MAAkB,EAC7B,iBAAiB,CAAC,EAAE;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GACA,OAAO,CAAC,UAAU,CAAC;IA+EtB;;;;;;;;;;;;;;;;;;;OAmBG;IACG,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CAsQ/D"}

package/dist/core/providers/anthropic-provider.js

@@ -12,6 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.AnthropicProvider = void 0;
 const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
 const provider_debug_utils_1 = require("./provider-debug-utils");
+const model_config_1 = require("../model-config");
 class AnthropicProvider {
     client;
     apiKey;
@@ -24,6 +25,10 @@ class AnthropicProvider {
         this.validateApiKey();
         this.client = new sdk_1.default({
             apiKey: this.apiKey,
+            // Enable 1M token context window for Claude Sonnet 4 (5x increase from 200K)
+            defaultHeaders: {
+                'anthropic-beta': 'context-1m-2025-08-07'
+            }
         });
     }
     validateApiKey() {
@@ -38,7 +43,13 @@ class AnthropicProvider {
         return 'anthropic';
     }
     getDefaultModel() {
-        return 'claude-sonnet-4-5-20250929';
+        return (0, model_config_1.getCurrentModel)('anthropic');
+    }
+    getModelName() {
+        return this.model;
+    }
+    getSDKProvider() {
+        return 'anthropic';
     }
     isInitialized() {
         return this.client !== undefined;
@@ -46,22 +57,18 @@ class AnthropicProvider {
     /**
      * Helper method to log debug information if debug mode is enabled
      */
-    logDebugIfEnabled(operation, prompt, response, durationMs) {
+    logDebugIfEnabled(operation, prompt, response) {
         if (!this.debugMode)
-            return;
+            return null;
         const debugId = (0, provider_debug_utils_1.generateDebugId)(operation);
         (0, provider_debug_utils_1.debugLogInteraction)(debugId, prompt, response, operation, this.getProviderType(), this.model, this.debugMode);
-        // Use logMetrics for sendMessage calls (simple token structure, no extended metrics)
-        (0, provider_debug_utils_1.logMetrics)(operation, this.getProviderType(), {
-            totalTokens: {
-                input: response.usage.input_tokens,
-                output: response.usage.output_tokens,
-                cacheCreation: response.usage.cache_creation_input_tokens,
-                cacheRead: response.usage.cache_read_input_tokens
-            }
-        }, durationMs, this.debugMode);
+        // Return the actual debug file names created
+        return {
+            promptFile: `${debugId}_prompt.md`,
+            responseFile: `${debugId}_response.md`
+        };
     }
-    async sendMessage(message, operation = 'generic') {
+    async sendMessage(message, operation = 'generic', evaluationContext) {
         if (!this.client) {
             throw new Error('Anthropic client not initialized');
         }
@@ -97,9 +104,32 @@ class AnthropicProvider {
                     output_tokens
                 }
             };
-            const durationMs = Date.now() - startTime;
             // Debug log the interaction if enabled
-            this.logDebugIfEnabled(operation, message, response, durationMs);
+            this.logDebugIfEnabled(operation, message, response);
+            // PRD #154: Log evaluation dataset if evaluation context is provided
+            if (this.debugMode && evaluationContext?.interaction_id) {
+                const durationMs = Date.now() - startTime;
+                const evaluationMetrics = {
+                    // Core execution data
+                    operation,
+                    sdk: this.getProviderType(),
+                    inputTokens: input_tokens,
+                    outputTokens: output_tokens,
+                    durationMs,
+                    // Required fields
+                    iterationCount: 1,
+                    toolCallCount: 0,
+                    status: 'completed',
+                    completionReason: 'stop',
+                    modelVersion: this.model,
+                    // Required evaluation context - NO DEFAULTS, must be provided
+                    test_scenario: operation,
+                    ai_response_summary: content,
+                    user_intent: evaluationContext?.user_intent || '',
+                    interaction_id: evaluationContext?.interaction_id || '',
+                };
+                (0, provider_debug_utils_1.logEvaluationDataset)(evaluationMetrics, this.debugMode);
+            }
             return response;
         }
         catch (error) {
@@ -167,7 +197,40 @@ class AnthropicProvider {
         try {
             while (iterations < maxIterations) {
                 iterations++;
-                const iterationStartTime = Date.now();
+                // Build current prompt for debug logging
+                const currentPrompt = `System: ${config.systemPrompt}\n\n${conversationHistory.map(msg => {
+                    let content = '';
+                    if (typeof msg.content === 'string') {
+                        content = msg.content;
+                    }
+                    else if (Array.isArray(msg.content)) {
+                        // Extract text from content blocks
+                        content = msg.content.map(block => {
+                            if (block.type === 'text') {
+                                return block.text;
+                            }
+                            else if (block.type === 'tool_use') {
+                                return `[TOOL_USE: ${block.name}]`;
+                            }
+                            else if (block.type === 'tool_result') {
+                                const content = block.content;
+                                if (typeof content === 'string') {
+                                    return `[TOOL_RESULT: ${block.tool_use_id}]\n${content}`;
+                                }
+                                else if (Array.isArray(content)) {
+                                    const textContent = content.map(c => c.type === 'text' ? c.text : `[${c.type}]`).join(' ');
+                                    return `[TOOL_RESULT: ${block.tool_use_id}]\n${textContent}`;
+                                }
+                                return `[TOOL_RESULT: ${block.tool_use_id}]`;
+                            }
+                            return `[${block.type}]`;
+                        }).join(' ');
+                    }
+                    else {
+                        content = '[complex_content]';
+                    }
+                    return `${msg.role}: ${content}`;
+                }).join('\n\n')}`;
                 // Call Anthropic API with tools and cached system prompt
                 const response = await this.client.messages.create({
                     model: this.model,
@@ -186,11 +249,16 @@ class AnthropicProvider {
                 if ('cache_read_input_tokens' in response.usage) {
                     totalTokens.cacheRead += response.usage.cache_read_input_tokens || 0;
                 }
-                // Debug log this iteration if enabled
-                if (this.debugMode) {
-                    const currentPrompt = conversationHistory.map(m => typeof m.content === 'string' ? m.content : JSON.stringify(m.content, null, 2)).join('\n\n---\n\n');
+                // Check if AI wants to use tools
+                const toolUses = response.content.filter((c) => c.type === 'tool_use');
+                // Log debug for final iteration to capture complete prompts/responses for evaluation
+                let debugFiles = null;
+                if (toolUses.length === 0) {
                     const aiResponse = {
-                        content: response.content.map(c => c.type === 'text' ? c.text : `[${c.type}]`).join('\n'),
+                        content: response.content
+                            .filter((c) => c.type === 'text')
+                            .map(c => c.text)
+                            .join('\n\n'),
                         usage: {
                             input_tokens: response.usage.input_tokens,
                             output_tokens: response.usage.output_tokens,
@@ -198,11 +266,8 @@ class AnthropicProvider {
                             cache_read_input_tokens: response.usage.cache_read_input_tokens
                         }
                     };
-                    const iterationDurationMs = Date.now() - iterationStartTime;
-                    this.logDebugIfEnabled(`${operation}-iter${iterations}`, currentPrompt, aiResponse, iterationDurationMs);
+                    debugFiles = this.logDebugIfEnabled(`${config.operation}-summary`, currentPrompt, aiResponse);
                 }
-                // Check if AI wants to use tools
-                const toolUses = response.content.filter((c) => c.type === 'tool_use');
                 if (toolUses.length === 0) {
                     // AI is done - extract final text message
                     const textContent = response.content.find((c) => c.type === 'text');
@@ -222,7 +287,10 @@ class AnthropicProvider {
                         operation: `${operation}-summary`,
                         sdk: this.getProviderType(),
                         startTime,
-                        debugMode: this.debugMode
+                        debugMode: this.debugMode,
+                        debugFiles,
+                        evaluationContext: config.evaluationContext,
+                        interaction_id: config.interaction_id
                     });
                 }
                 // Execute all requested tools in parallel
@@ -295,7 +363,9 @@ class AnthropicProvider {
                 operation: `${operation}-max-iterations`,
                 sdk: this.getProviderType(),
                 startTime,
-                debugMode: this.debugMode
+                debugMode: this.debugMode,
+                evaluationContext: config.evaluationContext,
+                interaction_id: config.interaction_id
             });
         }
         catch (error) {
@@ -316,7 +386,9 @@ class AnthropicProvider {
                 operation: `${operation}-error`,
                 sdk: this.getProviderType(),
                 startTime,
-                debugMode: this.debugMode
+                debugMode: this.debugMode,
+                evaluationContext: config.evaluationContext,
+                interaction_id: config.interaction_id
             });
         }
     }

package/dist/core/providers/provider-debug-utils.d.ts

@@ -14,32 +14,56 @@ export declare function ensureDebugDirectory(): string;
  */
 export declare function generateDebugId(operation: string): string;
 /**
- * Log metrics for token usage and execution time when DEBUG_DOT_AI=true
- *
- * PRD #143 Decision 5: Extended metrics for model comparison analysis
+ * Unified evaluation metrics entry for AI quality assessment and performance tracking
+ * PRD #154: Single interface for all metrics and evaluation data
  */
-export declare function logMetrics(operation: string, sdk: string, result: {
-    totalTokens: {
-        input: number;
-        output: number;
-        cacheCreation?: number;
-        cacheRead?: number;
+export interface EvaluationMetrics {
+    operation: string;
+    sdk: string;
+    inputTokens: number;
+    outputTokens: number;
+    durationMs: number;
+    iterationCount: number;
+    toolCallCount: number;
+    status: string;
+    completionReason: string;
+    modelVersion: string;
+    cacheCreationTokens?: number;
+    cacheReadTokens?: number;
+    cacheHitRate?: number;
+    uniqueToolsUsed?: string[];
+    test_scenario: string;
+    ai_response_summary: string;
+    debug_files?: {
+        full_prompt: string;
+        full_response: string;
     };
-    iterations?: number;
-    toolCallsExecuted?: Array<{
-        tool: string;
-        input: any;
-        output: any;
-    }>;
-    status?: string;
-    completionReason?: string;
-    modelVersion?: string;
-}, durationMs: number, debugMode: boolean): void;
+    user_intent: string;
+    interaction_id: string;
+    failure_analysis?: string | {
+        failure_type: "timeout" | "error" | "infrastructure";
+        failure_reason: string;
+        time_to_failure: number;
+    };
+}
+/**
+ * Determine if dataset generation should be skipped for specific operations
+ */
+export declare function shouldSkipDatasetGeneration(operation: string): boolean;
+/**
+ * Log unified evaluation metrics when DEBUG_DOT_AI=true
+ * Single function for all metrics and evaluation data capture
+ */
+/**
+ * Generate eval dataset entry in standard OpenAI Evals format
+ * Logs evaluation metrics to JSONL dataset files for AI quality assessment
+ */
+export declare function logEvaluationDataset(metrics: EvaluationMetrics, debugMode?: boolean): void;
 /**
  * Create AgenticResult and log metrics in one step
  * Reduces code duplication across providers
  *
- * PRD #143 Decision 5: Standardized metrics logging
+ * PRD #154: Updated to use unified evaluation metrics
  */
 export declare function createAndLogAgenticResult(config: {
     finalMessage: string;
@@ -62,6 +86,15 @@ export declare function createAndLogAgenticResult(config: {
     sdk: string;
     startTime: number;
     debugMode: boolean;
+    debugFiles?: {
+        promptFile: string;
+        responseFile: string;
+    } | null;
+    evaluationContext?: {
+        user_intent?: string;
+        failure_analysis?: string;
+    };
+    interaction_id?: string;
 }): AgenticResult;
 /**
  * Save AI interaction for debugging when DEBUG_DOT_AI=true

package/dist/core/providers/provider-debug-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provider-debug-utils.d.ts","sourceRoot":"","sources":["../../../src/core/providers/provider-debug-utils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAErE;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAM7C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAKzD;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE;IACN,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAC;QAAC,MAAM,EAAE,GAAG,CAAA;KAAE,CAAC,CAAC;IACrE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,OAAO,GACjB,IAAI,CA2DN;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAC;QAAC,MAAM,EAAE,GAAG,CAAA;KAAE,CAAC,CAAC;IACpE,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,aAAa,CAAC;IACzD,gBAAgB,EAAE,wBAAwB,GAAG,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,OAAO,CAAC;IAC5G,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACpB,GAAG,aAAa,CAiBhB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,UAAU,EACpB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,OAAO,GACjB,IAAI,CAkCN"}
+{"version":3,"file":"provider-debug-utils.d.ts","sourceRoot":"","sources":["../../../src/core/providers/provider-debug-utils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAErE;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAM7C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAKzD;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAEhC,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IAGnB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IAGrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAG3B,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE;QACZ,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAGF,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IAGvB,gBAAgB,CAAC,EAAE,MAAM,GAAG;QAC1B,YAAY,EAAE,SAAS,GAAG,OAAO,GAAG,gBAAgB,CAAC;QACrD,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAGtE;AAED;;;GAGG;AACH;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,iBAAiB,EAC1B,SAAS,GAAE,OAAe,GACzB,IAAI,CAyEN;AAGD;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAC;QAAC,MAAM,EAAE,GAAG,CAAA;KAAE,CAAC,CAAC;IACpE,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,aAAa,CAAC;IACzD,gBAAgB,EAAE,wBAAwB,GAAG,gBAAgB,GAAG,eAAe,GAAG,eAAe,GAAG,OAAO,CAAC;IAC5G,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAGjE,iBAAiB,CAAC,EAAE;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC;IAGF,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GAAG,aAAa,CAsDhB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,UAAU,EACpB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,OAAO,GACjB,IAAI,CAkCN"}