@vfarcic/dot-ai 0.111.0 → 0.112.0

package/dist/tools/answer-question.js

@@ -52,7 +52,8 @@ exports.ANSWERQUESTION_TOOL_DESCRIPTION = 'Process user answers and return remai
 exports.ANSWERQUESTION_TOOL_INPUT_SCHEMA = {
     solutionId: zod_1.z.string().regex(/^sol_[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}_[a-f0-9]+$/).describe('The solution ID to update (e.g., sol_2025-07-01T154349_1e1e242592ff)'),
     stage: zod_1.z.enum(['required', 'basic', 'advanced', 'open']).describe('The configuration stage being addressed'),
-    answers: zod_1.z.record(zod_1.z.any()).describe('User answers to configuration questions for the specified stage. For required/basic/advanced stages, use questionId as key. For open stage, use "open" as key (e.g., {"open": "add persistent storage"})')
+    answers: zod_1.z.record(zod_1.z.any()).describe('User answers to configuration questions for the specified stage. For required/basic/advanced stages, use questionId as key. For open stage, use "open" as key (e.g., {"open": "add persistent storage"})'),
+    interaction_id: zod_1.z.string().optional().describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.')
 };
 /**
  * Load solution file by ID
@@ -353,7 +354,7 @@ function getStageGuidance(stage) {
 /**
  * Phase 1: Analyze what resources are needed for the user request
  */
-async function analyzeResourceNeeds(currentSolution, openResponse, context) {
+async function analyzeResourceNeeds(currentSolution, openResponse, context, interaction_id) {
     const template = (0, shared_prompt_loader_1.loadPrompt)('resource-analysis');
     // Get available resources from solution or use defaults
     const availableResources = currentSolution.availableResources || {
@@ -376,7 +377,10 @@ async function analyzeResourceNeeds(currentSolution, openResponse, context) {
         availableResourceCount: availableResourceTypes.length
     });
     try {
-        const response = await aiProvider.sendMessage(analysisPrompt);
+        const response = await aiProvider.sendMessage(analysisPrompt, 'answer-question-resource-analysis', {
+            user_intent: openResponse,
+            interaction_id: interaction_id
+        });
         const analysisResult = parseEnhancementResponse(response.content);
         // Check for capability gap and throw specific error
         if (analysisResult.approach === 'capability_gap') {
@@ -399,7 +403,7 @@ async function analyzeResourceNeeds(currentSolution, openResponse, context) {
 /**
  * Phase 2: Apply enhancements based on analysis result
  */
-async function applySolutionEnhancement(solution, openResponse, analysisResult, context) {
+async function applySolutionEnhancement(solution, openResponse, analysisResult, context, interaction_id) {
     if (analysisResult.approach === 'capability_gap') {
         throw new Error(`Enhancement capability gap: ${analysisResult.reasoning}. ${analysisResult.suggestedAction}`);
     }
@@ -409,7 +413,7 @@ async function applySolutionEnhancement(solution, openResponse, analysisResult,
             approach: analysisResult.approach,
             reasoning: analysisResult.reasoning
         });
-        return autoPopulateQuestions(solution, openResponse, analysisResult, context);
+        return autoPopulateQuestions(solution, openResponse, analysisResult, context, interaction_id);
     }
     if (analysisResult.approach === 'add_resources') {
         // Add new resources and their questions
@@ -425,7 +429,7 @@ async function applySolutionEnhancement(solution, openResponse, analysisResult,
 /**
  * Auto-populate existing questions based on user requirements
  */
-async function autoPopulateQuestions(solution, openResponse, analysisResult, context) {
+async function autoPopulateQuestions(solution, openResponse, analysisResult, context, interaction_id) {
     const template = (0, shared_prompt_loader_1.loadPrompt)('solution-enhancement');
     const enhancementPrompt = template
         .replace('{current_solution}', JSON.stringify(solution, null, 2))
@@ -434,7 +438,10 @@ async function autoPopulateQuestions(solution, openResponse, analysisResult, con
         .replace('{open_response}', openResponse);
     // Get AI provider from context
     const aiProvider = context.dotAI.ai;
-    const response = await aiProvider.sendMessage(enhancementPrompt);
+    const response = await aiProvider.sendMessage(enhancementPrompt, 'answer-question-solution-enhancement', {
+        user_intent: openResponse,
+        interaction_id: interaction_id
+    });
     const enhancementData = parseEnhancementResponse(response.content);
     if (enhancementData.enhancedSolution) {
         return enhancementData.enhancedSolution;
@@ -473,16 +480,16 @@ function parseEnhancementResponse(content) {
 /**
  * Enhance solution with AI analysis of open question
  */
-async function enhanceSolutionWithOpenAnswer(solution, openAnswer, context) {
+async function enhanceSolutionWithOpenAnswer(solution, openAnswer, context, interaction_id) {
     try {
         context.logger.info('Starting AI enhancement of solution', {
             solutionId: solution.solutionId,
             openAnswer
         });
         // Phase 1: Analyze what resources are needed
-        const analysisResult = await analyzeResourceNeeds(solution, openAnswer, context);
+        const analysisResult = await analyzeResourceNeeds(solution, openAnswer, context, interaction_id);
         // Phase 2: Apply enhancements based on analysis
-        const enhancedSolution = await applySolutionEnhancement(solution, openAnswer, analysisResult, context);
+        const enhancedSolution = await applySolutionEnhancement(solution, openAnswer, analysisResult, context, interaction_id);
         context.logger.info('AI enhancement completed', {
             approach: analysisResult.approach,
             changed: enhancedSolution !== solution
@@ -671,7 +678,7 @@ async function handleAnswerQuestionTool(args, dotAI, logger, requestId) {
                         solutionId: args.solutionId,
                         openAnswer
                     });
-                    solution = await enhanceSolutionWithOpenAnswer(solution, openAnswer, { requestId, logger, dotAI });
+                    solution = await enhanceSolutionWithOpenAnswer(solution, openAnswer, { requestId, logger, dotAI }, args.interaction_id);
                     // Save enhanced solution
                     saveSolutionFile(solution, args.solutionId, sessionDir);
                     logger.info('Enhanced solution saved', {

package/dist/tools/generate-manifests.d.ts

@@ -8,12 +8,14 @@ export declare const GENERATEMANIFESTS_TOOL_NAME = "generateManifests";
 export declare const GENERATEMANIFESTS_TOOL_DESCRIPTION = "Generate final Kubernetes manifests from fully configured solution (ONLY after completing ALL stages: required, basic, advanced, and open)";
 export declare const GENERATEMANIFESTS_TOOL_INPUT_SCHEMA: {
     solutionId: z.ZodString;
+    interaction_id: z.ZodOptional<z.ZodString>;
 };
 /**
  * Direct MCP tool handler for generateManifests functionality
  */
 export declare function handleGenerateManifestsTool(args: {
     solutionId: string;
+    interaction_id?: string;
 }, dotAI: DotAI, logger: Logger, requestId: string): Promise<{
     content: {
         type: 'text';

package/dist/tools/generate-manifests.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate-manifests.d.ts","sourceRoot":"","sources":["../../src/tools/generate-manifests.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAWhD,eAAO,MAAM,2BAA2B,sBAAsB,CAAC;AAC/D,eAAO,MAAM,kCAAkC,+IAA+I,CAAC;AAG/L,eAAO,MAAM,mCAAmC;;CAE/C,CAAC;AA+RF;;GAEG;AACH,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,EAC5B,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAAE,CAAC,CAqMxD"}
+{"version":3,"file":"generate-manifests.d.ts","sourceRoot":"","sources":["../../src/tools/generate-manifests.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAWhD,eAAO,MAAM,2BAA2B,sBAAsB,CAAC;AAC/D,eAAO,MAAM,kCAAkC,+IAA+I,CAAC;AAG/L,eAAO,MAAM,mCAAmC;;;CAG/C,CAAC;AAmSF;;GAEG;AACH,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,EACrD,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAAE,CAAC,CAsMxD"}

package/dist/tools/generate-manifests.js

@@ -53,7 +53,8 @@ exports.GENERATEMANIFESTS_TOOL_NAME = 'generateManifests';
 exports.GENERATEMANIFESTS_TOOL_DESCRIPTION = 'Generate final Kubernetes manifests from fully configured solution (ONLY after completing ALL stages: required, basic, advanced, and open)';
 // Zod schema for MCP registration
 exports.GENERATEMANIFESTS_TOOL_INPUT_SCHEMA = {
-    solutionId: zod_1.z.string().regex(/^sol_[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}_[a-f0-9]+$/).describe('The solution ID to generate manifests for (e.g., sol_2025-07-01T154349_1e1e242592ff)')
+    solutionId: zod_1.z.string().regex(/^sol_[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}_[a-f0-9]+$/).describe('The solution ID to generate manifests for (e.g., sol_2025-07-01T154349_1e1e242592ff)'),
+    interaction_id: zod_1.z.string().optional().describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.')
 };
 /**
  * Load solution file and validate structure
@@ -179,7 +180,7 @@ async function validateManifests(yamlPath) {
 /**
  * Generate manifests using AI provider
  */
-async function generateManifestsWithAI(solution, dotAI, logger, errorContext, dotAiLabels) {
+async function generateManifestsWithAI(solution, dotAI, logger, errorContext, dotAiLabels, interaction_id) {
     // Load prompt template
     const template = (0, shared_prompt_loader_1.loadPrompt)('manifest-generation');
     // Retrieve schemas for solution resources
@@ -216,7 +217,10 @@ ${errorContext.previousManifests}
     // Get AI provider from dotAI
     const aiProvider = dotAI.ai;
     // Send prompt to AI
-    const response = await aiProvider.sendMessage(aiPrompt);
+    const response = await aiProvider.sendMessage(aiPrompt, 'recommend-manifests-generation', {
+        user_intent: solution.initialIntent || 'Kubernetes manifest generation',
+        interaction_id: interaction_id
+    });
     // Extract YAML content from response
     let manifestContent = response.content;
     // Try to extract YAML from code blocks if wrapped
@@ -357,7 +361,7 @@ async function handleGenerateManifestsTool(args, dotAI, logger, requestId) {
                 const userAnswers = (0, solution_utils_1.extractUserAnswers)(solution);
                 const dotAiLabels = (0, solution_utils_1.addDotAiLabels)(undefined, userAnswers, solution);
                 // Generate manifests with AI (including labels)
-                const aiManifests = await generateManifestsWithAI(solution, dotAI, logger, lastError, dotAiLabels);
+                const aiManifests = await generateManifestsWithAI(solution, dotAI, logger, lastError, dotAiLabels, args.interaction_id);
                 // Generate metadata ConfigMap
                 const metadataConfigMap = generateMetadataConfigMap(solution, userAnswers, logger);
                 // Combine ConfigMap with AI-generated manifests

package/dist/tools/organizational-data.d.ts

@@ -35,6 +35,7 @@ export declare const ORGANIZATIONAL_DATA_TOOL_INPUT_SCHEMA: {
     }>>;
     resourceList: z.ZodOptional<z.ZodString>;
     collection: z.ZodOptional<z.ZodString>;
+    interaction_id: z.ZodOptional<z.ZodString>;
 };
 /**
  * Main tool handler - routes to appropriate data type handler

package/dist/tools/organizational-data.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"organizational-data.d.ts","sourceRoot":"","sources":["../../src/tools/organizational-data.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAehD,eAAO,MAAM,6BAA6B,kBAAkB,CAAC;AAC7D,eAAO,MAAM,oCAAoC,+jBAAyjB,CAAC;AAG3mB,eAAO,MAAM,qCAAqC;;;;;;;;;;;;;;;;;;;;;;;CA2BjD,CAAC;AAujBF;;GAEG;AACH,wBAAsB,4BAA4B,CAChD,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,KAAK,GAAG,IAAI,EACpB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,GAAG,CAAC,CA0Hd"}
+{"version":3,"file":"organizational-data.d.ts","sourceRoot":"","sources":["../../src/tools/organizational-data.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAehD,eAAO,MAAM,6BAA6B,kBAAkB,CAAC;AAC7D,eAAO,MAAM,oCAAoC,+jBAAyjB,CAAC;AAG3mB,eAAO,MAAM,qCAAqC;;;;;;;;;;;;;;;;;;;;;;;;CA4BjD,CAAC;AAujBF;;GAEG;AACH,wBAAsB,4BAA4B,CAChD,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,KAAK,GAAG,IAAI,EACpB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,GAAG,CAAC,CA0Hd"}

package/dist/tools/organizational-data.js

@@ -81,7 +81,8 @@ exports.ORGANIZATIONAL_DATA_TOOL_INPUT_SCHEMA = {
     // Resource list for specific resource scanning
     resourceList: zod_1.z.string().optional().describe('Comma-separated list of resources to scan (format: Kind.group or Kind for core resources)'),
     // Collection name for capabilities (allows using different collections for different purposes)
-    collection: zod_1.z.string().optional().describe('Collection name for capabilities operations (default: "capabilities", use "capabilities-policies" for pre-populated test data)')
+    collection: zod_1.z.string().optional().describe('Collection name for capabilities operations (default: "capabilities", use "capabilities-policies" for pre-populated test data)'),
+    interaction_id: zod_1.z.string().optional().describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.')
 };
 /**
  * Validate Vector DB connection and return helpful error if unavailable

package/dist/tools/recommend.d.ts

@@ -13,6 +13,7 @@ export declare const RECOMMEND_TOOL_INPUT_SCHEMA: {
     solutionId: z.ZodOptional<z.ZodString>;
     answers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
     timeout: z.ZodOptional<z.ZodNumber>;
+    interaction_id: z.ZodOptional<z.ZodString>;
 };
 /**
  * Direct MCP tool handler for recommend functionality (unified with stage routing)

package/dist/tools/recommend.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recommend.d.ts","sourceRoot":"","sources":["../../src/tools/recommend.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAahD,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,iRAAiR,CAAC;AAGzT,eAAO,MAAM,2BAA2B;;;;;;;CAUvC,CAAC;AAqHF;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,GAAG,EACT,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAAE,CAAC,CA4QxD"}
+{"version":3,"file":"recommend.d.ts","sourceRoot":"","sources":["../../src/tools/recommend.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAahD,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,iRAAiR,CAAC;AAGzT,eAAO,MAAM,2BAA2B;;;;;;;;CAWvC,CAAC;AA2HF;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,GAAG,EACT,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAAE,CAAC,CAsRxD"}

package/dist/tools/recommend.js

@@ -64,7 +64,8 @@ exports.RECOMMEND_TOOL_INPUT_SCHEMA = {
     // Parameters for answerQuestion stage (stage parameter contains the config stage like "answerQuestion:required")
     answers: zod_1.z.record(zod_1.z.any()).optional().describe('User answers for answerQuestion stage'),
     // Parameters for deployManifests stage
-    timeout: zod_1.z.number().optional().describe('Deployment timeout in seconds for deployManifests stage')
+    timeout: zod_1.z.number().optional().describe('Deployment timeout in seconds for deployManifests stage'),
+    interaction_id: zod_1.z.string().optional().describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.')
 };
 /**
  * Analyze intent for clarification opportunities using AI
@@ -75,7 +76,7 @@ exports.RECOMMEND_TOOL_INPUT_SCHEMA = {
  * @param organizationalPatterns Optional organizational patterns context
  * @returns Analysis result with clarification opportunities
  */
-async function analyzeIntentForClarification(intent, aiProvider, logger, organizationalPatterns = '') {
+async function analyzeIntentForClarification(intent, aiProvider, logger, organizationalPatterns = '', evaluationContext) {
     try {
         // Load intent analysis prompt template
         const analysisPrompt = (0, shared_prompt_loader_1.loadPrompt)('intent-analysis', {
@@ -83,7 +84,7 @@ async function analyzeIntentForClarification(intent, aiProvider, logger, organiz
             organizational_patterns: organizationalPatterns || 'No specific organizational patterns available'
         });
         // Send to AI for analysis
-        const response = await aiProvider.sendMessage(analysisPrompt, 'intent-analysis');
+        const response = await aiProvider.sendMessage(analysisPrompt, 'recommend-intent-analysis', evaluationContext);
         // Parse JSON response with robust error handling
         let jsonContent = response.content;
         // Try to find JSON object wrapped in code blocks
@@ -224,7 +225,11 @@ async function handleRecommendTool(args, dotAI, logger, requestId) {
         // Check if intent clarification is needed (unless final=true)
         if (!args.final) {
             logger.debug('Analyzing intent for clarification opportunities', { requestId, intent: args.intent });
-            const analysisResult = await analyzeIntentForClarification(args.intent, aiProvider, logger);
+            const analysisResult = await analyzeIntentForClarification(args.intent, aiProvider, logger, '', // organizationalPatterns - empty for now
+            {
+                user_intent: args.intent,
+                interaction_id: args.interaction_id
+            });
             // If clarification opportunities exist, return them to the client agent
             if (analysisResult.clarificationOpportunities &&
                 analysisResult.clarificationOpportunities.length > 0 &&
@@ -280,7 +285,7 @@ async function handleRecommendTool(args, dotAI, logger, requestId) {
         };
         // Find best solutions for the user intent
         logger.debug('Generating recommendations with AI', { requestId });
-        const solutions = await recommender.findBestSolutions(args.intent, explainResourceFn);
+        const solutions = await recommender.findBestSolutions(args.intent, explainResourceFn, args.interaction_id);
         logger.info('Recommendation process completed', {
             requestId,
             solutionCount: solutions.length,

package/dist/tools/remediate.d.ts

@@ -12,6 +12,7 @@ export declare const REMEDIATE_TOOL_INPUT_SCHEMA: {
     executeChoice: z.ZodOptional<z.ZodNumber>;
     sessionId: z.ZodOptional<z.ZodString>;
     executedCommands: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    interaction_id: z.ZodOptional<z.ZodString>;
 };
 export interface RemediateInput {
     issue?: string;
@@ -21,11 +22,13 @@ export interface RemediateInput {
     executeChoice?: number;
     sessionId?: string;
     executedCommands?: string[];
+    interaction_id?: string;
 }
 export interface RemediateSession {
     sessionId: string;
     issue: string;
     mode: 'manual' | 'automatic';
+    interaction_id?: string;
     finalAnalysis?: RemediateOutput;
     created: Date;
     updated: Date;

package/dist/tools/remediate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"remediate.d.ts","sourceRoot":"","sources":["../../src/tools/remediate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,yfAAwf,CAAC;AAIhiB,eAAO,MAAM,2BAA2B;;;;;;;;CAQvC,CAAC;AAGF,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACzC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC7B,aAAa,CAAC,EAAE,eAAe,CAAC;IAChC,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,IAAI,CAAC;IACd,MAAM,EAAE,eAAe,GAAG,mBAAmB,GAAG,QAAQ,GAAG,uBAAuB,GAAG,sBAAsB,GAAG,WAAW,CAAC;IAC1H,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,wBAAwB,CAAC;IACxD,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC;IACF,QAAQ,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,EAAE,CAAC;KACnB,CAAC;IACF,WAAW,EAAE;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,iBAAiB,EAAE,CAAC;QAC7B,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;KACjC,CAAC;IAEF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC;IACrC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;CAC/B;AAwLD;;GAEG;AACH,UAAU,uBAAuB;IAC/B,WAAW,EAAE,QAAQ,GAAG,UAAU,GAAG,cAAc,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,iBAAiB,EAAE,CAAC;QAC7B,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;KACjC,CAAC;IACF,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,uBAAuB,CA0FhF;AAwVD;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAqLjE"}
+{"version":3,"file":"remediate.d.ts","sourceRoot":"","sources":["../../src/tools/remediate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,0BAA0B,yfAAwf,CAAC;AAIhiB,eAAO,MAAM,2BAA2B;;;;;;;;;CASvC,CAAC;AAGF,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC9B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACzC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,eAAe,CAAC;IAChC,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,IAAI,CAAC;IACd,MAAM,EAAE,eAAe,GAAG,mBAAmB,GAAG,QAAQ,GAAG,uBAAuB,GAAG,sBAAsB,GAAG,WAAW,CAAC;IAC1H,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,wBAAwB,CAAC;IACxD,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC;IACF,QAAQ,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,EAAE,CAAC;KACnB,CAAC;IACF,WAAW,EAAE;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,iBAAiB,EAAE,CAAC;QAC7B,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;KACjC,CAAC;IAEF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC;IACrC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;CAC/B;AAgMD;;GAEG;AACH,UAAU,uBAAuB;IAC/B,WAAW,EAAE,QAAQ,GAAG,UAAU,GAAG,cAAc,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE;QACX,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,iBAAiB,EAAE,CAAC;QAC7B,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;KACjC,CAAC;IACF,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,uBAAuB,CA0FhF;AA2VD;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CA2LjE"}

package/dist/tools/remediate.js

@@ -59,7 +59,8 @@ exports.REMEDIATE_TOOL_INPUT_SCHEMA = {
     maxRiskLevel: zod_1.z.enum(['low', 'medium', 'high']).optional().default('low').describe('For automatic mode: maximum risk level allowed for execution (default: low)'),
     executeChoice: zod_1.z.number().min(1).max(2).optional().describe('Execute a previously generated choice (1=Execute via MCP, 2=Execute via agent)'),
     sessionId: zod_1.z.string().optional().describe('Session ID from previous remediate call when executing a choice'),
-    executedCommands: zod_1.z.array(zod_1.z.string()).optional().describe('Commands that were executed to remediate the issue')
+    executedCommands: zod_1.z.array(zod_1.z.string()).optional().describe('Commands that were executed to remediate the issue'),
+    interaction_id: zod_1.z.string().optional().describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.')
 };
 /**
  * Generate unique session ID for investigation tracking
@@ -103,7 +104,7 @@ function updateSessionFile(sessionDir, sessionId, updates) {
 /**
  * AI-driven investigation - uses toolLoop for single-phase investigation and analysis
  */
-async function conductInvestigation(session, sessionDir, aiProvider, logger, requestId) {
+async function conductInvestigation(session, sessionDir, aiProvider, logger, requestId, isValidation = false, interactionId) {
     const maxIterations = 20;
     logger.info('Starting AI investigation with toolLoop', {
         requestId,
@@ -121,13 +122,18 @@ async function conductInvestigation(session, sessionDir, aiProvider, logger, req
         });
         // Use toolLoop for AI-driven investigation with kubectl tools
         // System prompt is static (cached), issue description is dynamic (userMessage)
+        const operationName = isValidation ? 'remediate-validation' : 'remediate-investigation';
         const result = await aiProvider.toolLoop({
             systemPrompt: systemPrompt,
             userMessage: `Investigate this Kubernetes issue: ${session.issue}`,
             tools: kubectl_tools_1.KUBECTL_INVESTIGATION_TOOLS,
             toolExecutor: kubectl_tools_1.executeKubectlTools,
             maxIterations: maxIterations,
-            operation: 'remediate-investigation'
+            operation: operationName,
+            evaluationContext: {
+                user_intent: session.issue
+            },
+            interaction_id: interactionId
         });
         logger.info('Investigation completed by toolLoop', {
             requestId,
@@ -293,7 +299,7 @@ function parseAIFinalAnalysis(aiResponse) {
 /**
  * Execute user choice from previous session
  */
-async function executeUserChoice(sessionDir, sessionId, choice, logger, requestId) {
+async function executeUserChoice(sessionDir, sessionId, choice, logger, requestId, currentInteractionId) {
     try {
         // Load previous session
         const session = readSessionFile(sessionDir, sessionId);
@@ -309,7 +315,7 @@ async function executeUserChoice(sessionDir, sessionId, choice, logger, requestI
         // Handle different choices
         switch (choice) {
             case 1: // Execute automatically via MCP
-                return await executeRemediationCommands(session, sessionDir, logger, requestId);
+                return await executeRemediationCommands(session, sessionDir, logger, requestId, currentInteractionId);
             case 2: { // Execute via agent
                 // Use validation intent directly from final analysis
                 const validationIntent = session.finalAnalysis.validationIntent || 'Check the status of the affected resources to verify the issue has been resolved';
@@ -351,7 +357,7 @@ async function executeUserChoice(sessionDir, sessionId, choice, logger, requestI
 /**
  * Execute remediation commands via kubectl
  */
-async function executeRemediationCommands(session, sessionDir, logger, requestId) {
+async function executeRemediationCommands(session, sessionDir, logger, requestId, currentInteractionId) {
     const results = [];
     const finalAnalysis = session.finalAnalysis;
     let overallSuccess = true;
@@ -424,7 +430,8 @@ async function executeRemediationCommands(session, sessionDir, logger, requestId
             const validationInput = {
                 issue: validationIntent,
                 sessionDir: sessionDir,
-                executedCommands: executedCommands
+                executedCommands: executedCommands,
+                interaction_id: currentInteractionId || session.interaction_id // Use current interaction_id for validation
             };
             // Recursive call to main function for validation
             const validationResponse = await handleRemediateTool(validationInput);
@@ -584,7 +591,7 @@ async function handleRemediateTool(args) {
                 choice: validatedInput.executeChoice,
                 sessionId: validatedInput.sessionId
             });
-            return await executeUserChoice(sessionDir, validatedInput.sessionId, validatedInput.executeChoice, logger, requestId);
+            return await executeUserChoice(sessionDir, validatedInput.sessionId, validatedInput.executeChoice, logger, requestId, validatedInput.interaction_id);
         }
         // Validate that we have an issue for new investigations
         if (!validatedInput.issue) {
@@ -596,6 +603,7 @@ async function handleRemediateTool(args) {
             sessionId,
             issue: validatedInput.issue,
             mode: validatedInput.mode || 'manual',
+            interaction_id: validatedInput.interaction_id,
             created: new Date(),
             updated: new Date(),
             status: 'investigating'
@@ -605,8 +613,9 @@ async function handleRemediateTool(args) {
         logger.info('Investigation session created', { requestId, sessionId });
         // Initialize AI provider (will validate API key automatically)
         const aiProvider = (0, ai_provider_factory_1.createAIProvider)();
-        // Conduct AI-driven investigation
-        const finalAnalysis = await conductInvestigation(session, sessionDir, aiProvider, logger, requestId);
+        // Conduct AI-driven investigation (detect if this is post-execution validation)
+        const isValidation = validatedInput.executedCommands && validatedInput.executedCommands.length > 0;
+        const finalAnalysis = await conductInvestigation(session, sessionDir, aiProvider, logger, requestId, isValidation, validatedInput.interaction_id);
         logger.info('Remediation analysis completed', {
             requestId,
             sessionId,
@@ -670,7 +679,7 @@ async function handleRemediateTool(args) {
             // Update session object with final analysis for execution
             session.finalAnalysis = finalAnalysis;
             // Execute commands and return the complete result (includes post-execution validation)
-            return await executeRemediationCommands(session, sessionDir, logger, requestId);
+            return await executeRemediationCommands(session, sessionDir, logger, requestId, validatedInput.interaction_id);
         }
         // Return MCP-compliant response
         return {
@@ -752,7 +761,11 @@ function validateRemediateInput(args) {
                 exports.REMEDIATE_TOOL_INPUT_SCHEMA.maxRiskLevel.parse(args.maxRiskLevel) : 'low',
             executeChoice: args.executeChoice !== undefined ?
                 exports.REMEDIATE_TOOL_INPUT_SCHEMA.executeChoice.parse(args.executeChoice) : undefined,
-            sessionId: args.sessionId ? exports.REMEDIATE_TOOL_INPUT_SCHEMA.sessionId.parse(args.sessionId) : undefined
+            sessionId: args.sessionId ? exports.REMEDIATE_TOOL_INPUT_SCHEMA.sessionId.parse(args.sessionId) : undefined,
+            executedCommands: args.executedCommands
+                ? exports.REMEDIATE_TOOL_INPUT_SCHEMA.executedCommands.parse(args.executedCommands)
+                : undefined,
+            interaction_id: args.interaction_id ? exports.REMEDIATE_TOOL_INPUT_SCHEMA.interaction_id.parse(args.interaction_id) : undefined
         };
         return validated;
     }

package/dist/tools/test-docs.d.ts

@@ -13,6 +13,7 @@ export declare const TESTDOCS_TOOL_INPUT_SCHEMA: {
     sectionId: z.ZodOptional<z.ZodString>;
     results: z.ZodOptional<z.ZodString>;
     filePattern: z.ZodOptional<z.ZodString>;
+    interaction_id: z.ZodOptional<z.ZodString>;
 };
 /**
  * Handle test-docs tool request

package/dist/tools/test-docs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"test-docs.d.ts","sourceRoot":"","sources":["../../src/tools/test-docs.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAOhD,eAAO,MAAM,kBAAkB,aAAa,CAAC;AAC7C,eAAO,MAAM,yBAAyB,+gBAA6gB,CAAC;AAGpjB,eAAO,MAAM,0BAA0B;;;;;;;CAOtC,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,KAAK,GAAG,IAAI,EACpB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,GAAG,CAAC,CAiWd"}
+{"version":3,"file":"test-docs.d.ts","sourceRoot":"","sources":["../../src/tools/test-docs.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAOhD,eAAO,MAAM,kBAAkB,aAAa,CAAC;AAC7C,eAAO,MAAM,yBAAyB,+gBAA6gB,CAAC;AAGpjB,eAAO,MAAM,0BAA0B;;;;;;;;CAQtC,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,KAAK,GAAG,IAAI,EACpB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,GAAG,CAAC,CAkWd"}

package/dist/tools/test-docs.js

@@ -53,7 +53,8 @@ exports.TESTDOCS_TOOL_INPUT_SCHEMA = {
     phase: zod_1.z.enum(['scan', 'test', 'analyze', 'fix', 'done']).optional().describe('Specific phase to run (defaults to scan)'),
     sectionId: zod_1.z.string().optional().describe('Section ID when submitting test results'),
     results: zod_1.z.string().optional().describe('Test results to store (for client agent reporting back)'),
-    filePattern: zod_1.z.string().optional().describe('File pattern for discovery (e.g., "**/*.md", "*.rst")')
+    filePattern: zod_1.z.string().optional().describe('File pattern for discovery (e.g., "**/*.md", "*.rst")'),
+    interaction_id: zod_1.z.string().optional().describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.')
 };
 /**
  * Handle test-docs tool request
@@ -64,7 +65,8 @@ async function handleTestDocsTool(args, _dotAI, logger, requestId) {
             requestId,
             filePath: args.filePath,
             sessionId: args.sessionId,
-            phase: args.phase
+            phase: args.phase,
+            interaction_id: args.interaction_id
         });
         // Check if we're in discovery mode (no filePath and no sessionId provided)
         if (!args.filePath && !args.sessionId) {

package/dist/tools/version.d.ts

@@ -4,10 +4,13 @@
  * Provides comprehensive system status including version information,
  * Vector DB connection status, and embedding service capabilities
  */
+import { z } from 'zod';
 import { Logger } from '../core/error-handling';
 export declare const VERSION_TOOL_NAME = "version";
 export declare const VERSION_TOOL_DESCRIPTION = "Get comprehensive system status including version information, Vector DB connection status, embedding service capabilities, AI provider connectivity, Kubernetes cluster connectivity, Kyverno policy engine status, and pattern management health check";
-export declare const VERSION_TOOL_INPUT_SCHEMA: {};
+export declare const VERSION_TOOL_INPUT_SCHEMA: {
+    interaction_id: z.ZodOptional<z.ZodString>;
+};
 export interface VersionInfo {
     version: string;
     nodeVersion: string;

package/dist/tools/version.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../../src/tools/version.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAMhD,eAAO,MAAM,iBAAiB,YAAY,CAAC;AAC3C,eAAO,MAAM,wBAAwB,6PAA6P,CAAC;AACnS,eAAO,MAAM,yBAAyB,IAAK,CAAC;AAE5C,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE;QACR,SAAS,EAAE,OAAO,CAAC;QACnB,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,EAAE;YACX,QAAQ,EAAE;gBACR,MAAM,EAAE,OAAO,CAAC;gBAChB,cAAc,CAAC,EAAE,MAAM,CAAC;gBACxB,KAAK,CAAC,EAAE,MAAM,CAAC;aAChB,CAAC;YACF,QAAQ,EAAE;gBACR,MAAM,EAAE,OAAO,CAAC;gBAChB,cAAc,CAAC,EAAE,MAAM,CAAC;gBACxB,KAAK,CAAC,EAAE,MAAM,CAAC;aAChB,CAAC;YACF,YAAY,EAAE;gBACZ,MAAM,EAAE,OAAO,CAAC;gBAChB,cAAc,CAAC,EAAE,MAAM,CAAC;gBACxB,KAAK,CAAC,EAAE,MAAM,CAAC;aAChB,CAAC;SACH,CAAC;KACH,CAAC;IACF,SAAS,EAAE;QACT,SAAS,EAAE,OAAO,CAAC;QACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,UAAU,EAAE;QACV,SAAS,EAAE,OAAO,CAAC;QACnB,aAAa,EAAE,OAAO,CAAC;QACvB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,UAAU,EAAE;QACV,SAAS,EAAE,OAAO,CAAC;QACnB,WAAW,CAAC,EAAE;YACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,OAAO,CAAC,EAAE,MAAM,CAAC;SAClB,CAAC;QACF,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,YAAY,EAAE;QACZ,WAAW,EAAE,OAAO,CAAC;QACrB,eAAe,EAAE,OAAO,CAAC;QACzB,oBAAoB,EAAE,OAAO,CAAC;QAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,qBAAqB,EAAE,OAAO,CAAC;QAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,KAAK,EAAE,OAAO,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAgOD;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CA8HzE;AAsHD;;GAEG;AACH,wBAAgB,cAAc,IAAI,WAAW,CAqB5C;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,GAAG,CAAC,CAqFd"}
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../../src/tools/version.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAMhD,eAAO,MAAM,iBAAiB,YAAY,CAAC;AAC3C,eAAO,MAAM,wBAAwB,6PAA6P,CAAC;AACnS,eAAO,MAAM,yBAAyB;;CAErC,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE;QACR,SAAS,EAAE,OAAO,CAAC;QACnB,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,EAAE;YACX,QAAQ,EAAE;gBACR,MAAM,EAAE,OAAO,CAAC;gBAChB,cAAc,CAAC,EAAE,MAAM,CAAC;gBACxB,KAAK,CAAC,EAAE,MAAM,CAAC;aAChB,CAAC;YACF,QAAQ,EAAE;gBACR,MAAM,EAAE,OAAO,CAAC;gBAChB,cAAc,CAAC,EAAE,MAAM,CAAC;gBACxB,KAAK,CAAC,EAAE,MAAM,CAAC;aAChB,CAAC;YACF,YAAY,EAAE;gBACZ,MAAM,EAAE,OAAO,CAAC;gBAChB,cAAc,CAAC,EAAE,MAAM,CAAC;gBACxB,KAAK,CAAC,EAAE,MAAM,CAAC;aAChB,CAAC;SACH,CAAC;KACH,CAAC;IACF,SAAS,EAAE;QACT,SAAS,EAAE,OAAO,CAAC;QACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,UAAU,EAAE;QACV,SAAS,EAAE,OAAO,CAAC;QACnB,aAAa,EAAE,OAAO,CAAC;QACvB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,UAAU,EAAE;QACV,SAAS,EAAE,OAAO,CAAC;QACnB,WAAW,CAAC,EAAE;YACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,OAAO,CAAC,EAAE,MAAM,CAAC;SAClB,CAAC;QACF,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,YAAY,EAAE;QACZ,WAAW,EAAE,OAAO,CAAC;QACrB,eAAe,EAAE,OAAO,CAAC;QACzB,oBAAoB,EAAE,OAAO,CAAC;QAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,qBAAqB,EAAE,OAAO,CAAC;QAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,OAAO,EAAE;QACP,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,KAAK,EAAE,OAAO,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAgOD;;GAEG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CA8HzE;AAyHD;;GAEG;AACH,wBAAgB,cAAc,IAAI,WAAW,CAqB5C;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,GAAG,CAAC,CAwFd"}

package/dist/tools/version.js

@@ -45,6 +45,7 @@ exports.getVersionInfo = getVersionInfo;
 exports.handleVersionTool = handleVersionTool;
 const fs_1 = require("fs");
 const path_1 = require("path");
+const zod_1 = require("zod");
 const k8s = __importStar(require("@kubernetes/client-node"));
 const index_1 = require("../core/index");
 const discovery_1 = require("../core/discovery");
@@ -52,7 +53,9 @@ const kubernetes_utils_1 = require("../core/kubernetes-utils");
 const nushell_runtime_1 = require("../core/nushell-runtime");
 exports.VERSION_TOOL_NAME = 'version';
 exports.VERSION_TOOL_DESCRIPTION = 'Get comprehensive system status including version information, Vector DB connection status, embedding service capabilities, AI provider connectivity, Kubernetes cluster connectivity, Kyverno policy engine status, and pattern management health check';
-exports.VERSION_TOOL_INPUT_SCHEMA = {};
+exports.VERSION_TOOL_INPUT_SCHEMA = {
+    interaction_id: zod_1.z.string().optional().describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.')
+};
 /**
  * Test Vector DB connectivity and get status for all collections
  */
@@ -423,7 +426,7 @@ async function getNushellStatus() {
 /**
  * Test AI provider connectivity
  */
-async function getAIProviderStatus() {
+async function getAIProviderStatus(interaction_id) {
     try {
         // Import AI provider factory and test connectivity
         const { createAIProvider } = await Promise.resolve().then(() => __importStar(require('../core/ai-provider-factory')));
@@ -437,7 +440,10 @@ async function getAIProviderStatus() {
             };
         }
         // Test with a minimal request to check connectivity
-        await aiProvider.sendMessage('test');
+        await aiProvider.sendMessage('test', 'version-connectivity-check', {
+            user_intent: 'Test AI provider connectivity and system version check',
+            interaction_id: interaction_id
+        });
         return {
             connected: true,
             keyConfigured: true,
@@ -492,6 +498,8 @@ function getVersionInfo() {
  */
 async function handleVersionTool(args, logger, requestId) {
     try {
+        // Extract interaction_id for evaluation dataset generation
+        const interaction_id = args.interaction_id ? exports.VERSION_TOOL_INPUT_SCHEMA.interaction_id.parse(args.interaction_id) : undefined;
         logger.info('Processing version tool request with system diagnostics', { requestId });
         // Get version info
         const version = getVersionInfo();
@@ -500,7 +508,7 @@ async function handleVersionTool(args, logger, requestId) {
         const [vectorDBStatus, embeddingStatus, aiProviderStatus, kubernetesStatus, capabilityStatus, kyvernoStatus, nushellStatus] = await Promise.all([
             getVectorDBStatus(),
             getEmbeddingStatus(),
-            getAIProviderStatus(),
+            getAIProviderStatus(interaction_id),
             getKubernetesStatus(),
             getCapabilityStatus(),
             getKyvernoStatus(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vfarcic/dot-ai",
-  "version": "0.111.0",
+  "version": "0.112.0",
   "description": "AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance",
   "mcpName": "io.github.vfarcic/dot-ai",
   "main": "dist/index.js",
@@ -18,6 +18,10 @@
     "test:integration:server": "KUBECONFIG=./kubeconfig-test.yaml PORT=3456 DOT_AI_SESSION_DIR=./tmp/sessions TRANSPORT_TYPE=http QDRANT_URL=http://localhost:6335 QDRANT_CAPABILITIES_COLLECTION=capabilities-policies ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY OPENAI_API_KEY=$OPENAI_API_KEY node dist/mcp/server.js",
     "test:integration": "./tests/integration/infrastructure/run-integration-tests.sh",
     "test:integration:watch": "vitest --config=vitest.integration.config.ts --test-timeout=1200000",
+    "test:integration:sonnet": "AI_PROVIDER=anthropic AI_PROVIDER_SDK=vercel DEBUG_DOT_AI=true ./tests/integration/infrastructure/run-integration-tests.sh",
+    "test:integration:gpt": "AI_PROVIDER=openai AI_PROVIDER_SDK=vercel DEBUG_DOT_AI=true ./tests/integration/infrastructure/run-integration-tests.sh",
+    "test:integration:gpt-pro": "AI_PROVIDER=openai_pro AI_PROVIDER_SDK=vercel DEBUG_DOT_AI=true ./tests/integration/infrastructure/run-integration-tests.sh",
+    "eval:comparative": "DEBUG_DOT_AI=true npx tsx src/evaluation/eval-runner.ts",
     "clean": "rm -rf dist",
     "prebuild": "npm run clean && npm run lint",
     "build": "tsc --sourceMap false",