@vfarcic/dot-ai 0.111.0 → 0.113.0

package/dist/evaluation/evaluators/base.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Standard Evaluator Interface Following OpenAI Evals Pattern
+ *
+ * Based on OpenAI Evals framework standards:
+ * - Each evaluator has a name and description
+ * - evaluate() method takes input, output, and optional ideal
+ * - Returns standardized EvaluationScore
+ */
+export interface EvaluationScore {
+    key: string;
+    score: number;
+    comment?: string;
+    confidence?: number;
+}
+export interface EvaluationSample {
+    input: Record<string, any>;
+    output: string;
+    ideal?: any;
+    metadata?: Record<string, any>;
+}
+export interface PerformanceMetrics {
+    duration_ms: number;
+    input_tokens: number;
+    output_tokens: number;
+    total_tokens: number;
+    cost_usd?: number;
+    iterations?: number;
+    tool_calls_executed?: number;
+    cache_hit_rate?: number;
+    model_version: string;
+}
+export interface EvaluationResult {
+    sample_id: string;
+    model: string;
+    timestamp: string;
+    quality_scores: Record<string, EvaluationScore>;
+    performance: PerformanceMetrics;
+    efficiency: {
+        quality_per_second: number;
+        quality_per_dollar?: number;
+        quality_per_token: number;
+    };
+    input: Record<string, any>;
+    output: string;
+    ideal?: any;
+}
+//# sourceMappingURL=base.d.ts.map

package/dist/evaluation/evaluators/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/base.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAGlB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAGhD,WAAW,EAAE,kBAAkB,CAAC;IAGhC,UAAU,EAAE;QACV,kBAAkB,EAAE,MAAM,CAAC;QAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;IAGF,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,GAAG,CAAC;CACb"}

package/dist/evaluation/evaluators/base.js

@@ -0,0 +1,10 @@
+"use strict";
+/**
+ * Standard Evaluator Interface Following OpenAI Evals Pattern
+ *
+ * Based on OpenAI Evals framework standards:
+ * - Each evaluator has a name and description
+ * - evaluate() method takes input, output, and optional ideal
+ * - Returns standardized EvaluationScore
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/evaluation/evaluators/capability-comparative.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Capability Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes capability inference scenarios
+ * Groups by interaction_id (e.g., auto_scan, crud_auto_scan) and evaluates
+ * quality of capability analyses across different models
+ */
+import { BaseComparativeEvaluator, ComparativeEvaluationScore } from './base-comparative.js';
+import { ComparisonScenario } from '../dataset-analyzer.js';
+export declare class CapabilityComparativeEvaluator extends BaseComparativeEvaluator {
+    readonly name = "capability-comparative";
+    readonly description = "Compares AI models on Kubernetes capability inference quality";
+    protected readonly promptFileName = "capability-comparative.md";
+    protected readonly toolName = "capability";
+    constructor(datasetDir?: string);
+    evaluateAllScenarios(): Promise<ComparativeEvaluationScore[]>;
+    /**
+     * Build the evaluation prompt - uses base class reliability context with capability-specific template
+     */
+    protected buildEvaluationPrompt(scenario: ComparisonScenario, modelResponsesText: string, modelList: string): string;
+    private extractResourceName;
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases(): {
+        phase: string;
+        description: string;
+        availableModels: string[];
+        scenarioCount: number;
+    }[];
+}
+//# sourceMappingURL=capability-comparative.d.ts.map

package/dist/evaluation/evaluators/capability-comparative.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"capability-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/capability-comparative.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAC7F,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,qBAAa,8BAA+B,SAAQ,wBAAwB;IAC1E,QAAQ,CAAC,IAAI,4BAA4B;IACzC,QAAQ,CAAC,WAAW,mEAAmE;IACvF,SAAS,CAAC,QAAQ,CAAC,cAAc,+BAA+B;IAChE,SAAS,CAAC,QAAQ,CAAC,QAAQ,gBAAgB;gBAE/B,UAAU,CAAC,EAAE,MAAM;IAKzB,oBAAoB,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;IA+BnE;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAYpH,OAAO,CAAC,mBAAmB;IAS3B;;OAEG;IACH,mBAAmB,IAAI;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CAsCJ"}

package/dist/evaluation/evaluators/capability-comparative.js

@@ -0,0 +1,104 @@
+"use strict";
+/**
+ * Capability Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes capability inference scenarios
+ * Groups by interaction_id (e.g., auto_scan, crud_auto_scan) and evaluates
+ * quality of capability analyses across different models
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CapabilityComparativeEvaluator = void 0;
+const base_comparative_js_1 = require("./base-comparative.js");
+class CapabilityComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
+    name = 'capability-comparative';
+    description = 'Compares AI models on Kubernetes capability inference quality';
+    promptFileName = 'capability-comparative.md';
+    toolName = 'capability';
+    constructor(datasetDir) {
+        super(datasetDir);
+        this.initializePrompt();
+    }
+    async evaluateAllScenarios() {
+        try {
+            const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+            const results = [];
+            console.log(`Found ${scenarios.length} capability scenarios with multiple models for comparative evaluation`);
+            for (const scenario of scenarios) {
+                try {
+                    const result = await this.evaluateScenario(scenario);
+                    results.push(result);
+                }
+                catch (error) {
+                    console.error(`Failed to evaluate scenario ${scenario.interaction_id}:`, error);
+                }
+            }
+            return results;
+        }
+        catch (error) {
+            console.error(`Capability comparative evaluation failed:`, error);
+            return [{
+                    key: `${this.name}_error`,
+                    score: 0,
+                    comment: `Evaluation error: ${error instanceof Error ? error.message : String(error)}`,
+                    confidence: 0,
+                    modelRankings: [],
+                    bestModel: 'unknown',
+                    modelCount: 0
+                }];
+        }
+    }
+    /**
+     * Build the evaluation prompt - uses base class reliability context with capability-specific template
+     */
+    buildEvaluationPrompt(scenario, modelResponsesText, modelList) {
+        // Use the base class's properly formatted model responses which include:
+        // - Reliability Status (✅ Completed successfully OR ⚠️ TIMEOUT FAILURE)
+        // - Performance metrics
+        // - All model responses
+        return this.promptTemplate
+            .replace('{scenario_name}', scenario.interaction_id)
+            .replace('{model_responses}', modelResponsesText)
+            .replace('{models}', modelList);
+    }
+    extractResourceName(input) {
+        if (input?.issue) {
+            const match = input.issue.match(/resource: (.+)/);
+            return match ? match[1] : 'unknown';
+        }
+        return 'unknown';
+    }
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases() {
+        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+        const phaseGroups = new Map();
+        // Group scenarios by phase type
+        for (const scenario of scenarios) {
+            const phase = scenario.interaction_id;
+            if (!phaseGroups.has(phase)) {
+                phaseGroups.set(phase, {
+                    models: new Set(),
+                    count: 0
+                });
+            }
+            const group = phaseGroups.get(phase);
+            scenario.models.forEach(model => group.models.add(model.model));
+            group.count++;
+        }
+        // Convert to structured output with descriptions
+        const phaseDescriptions = {
+            'auto_scan': 'Auto Scan Phase - How well each model analyzes cluster resource capabilities automatically',
+            'crud_auto_scan': 'CRUD Auto Scan Phase - How well each model handles capability analysis with CRUD operations',
+            'list_auto_scan': 'List Auto Scan Phase - How well each model handles capability listing and organization',
+            'search_auto_scan': 'Search Auto Scan Phase - How well each model handles capability search and filtering'
+        };
+        return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
+            phase,
+            description: phaseDescriptions[phase] || `${phase} phase evaluation`,
+            availableModels: Array.from(data.models).sort(),
+            scenarioCount: data.count
+        }));
+    }
+}
+exports.CapabilityComparativeEvaluator = CapabilityComparativeEvaluator;

package/dist/evaluation/evaluators/pattern-comparative.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Pattern Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes organizational pattern management scenarios
+ * Groups by interaction_id (e.g., pattern_create_workflow) and evaluates
+ * quality of pattern creation, identification, and management across different models
+ */
+import { BaseComparativeEvaluator, ComparativeEvaluationScore } from './base-comparative.js';
+import { ComparisonScenario } from '../dataset-analyzer.js';
+export declare class PatternComparativeEvaluator extends BaseComparativeEvaluator {
+    readonly name = "pattern-comparative";
+    readonly description = "Compares AI models on Kubernetes organizational pattern management quality";
+    protected readonly promptFileName = "pattern-comparative.md";
+    protected readonly toolName = "pattern";
+    constructor(datasetDir?: string);
+    evaluateAllScenarios(): Promise<ComparativeEvaluationScore[]>;
+    /**
+     * Build the evaluation prompt - uses base class reliability context with pattern-specific template
+     */
+    protected buildEvaluationPrompt(scenario: ComparisonScenario, modelResponsesText: string, modelList: string): string;
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases(): {
+        phase: string;
+        description: string;
+        availableModels: string[];
+        scenarioCount: number;
+    }[];
+}
+//# sourceMappingURL=pattern-comparative.d.ts.map

package/dist/evaluation/evaluators/pattern-comparative.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pattern-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/pattern-comparative.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAC7F,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,qBAAa,2BAA4B,SAAQ,wBAAwB;IACvE,QAAQ,CAAC,IAAI,yBAAyB;IACtC,QAAQ,CAAC,WAAW,gFAAgF;IACpG,SAAS,CAAC,QAAQ,CAAC,cAAc,4BAA4B;IAC7D,SAAS,CAAC,QAAQ,CAAC,QAAQ,aAAa;gBAE5B,UAAU,CAAC,EAAE,MAAM;IAKzB,oBAAoB,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;IA+BnE;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAYpH;;OAEG;IACH,mBAAmB,IAAI;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CAsCJ"}

package/dist/evaluation/evaluators/pattern-comparative.js

@@ -0,0 +1,97 @@
+"use strict";
+/**
+ * Pattern Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes organizational pattern management scenarios
+ * Groups by interaction_id (e.g., pattern_create_workflow) and evaluates
+ * quality of pattern creation, identification, and management across different models
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PatternComparativeEvaluator = void 0;
+const base_comparative_js_1 = require("./base-comparative.js");
+class PatternComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
+    name = 'pattern-comparative';
+    description = 'Compares AI models on Kubernetes organizational pattern management quality';
+    promptFileName = 'pattern-comparative.md';
+    toolName = 'pattern';
+    constructor(datasetDir) {
+        super(datasetDir);
+        this.initializePrompt();
+    }
+    async evaluateAllScenarios() {
+        try {
+            const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+            const results = [];
+            console.log(`Found ${scenarios.length} pattern scenarios with multiple models for comparative evaluation`);
+            for (const scenario of scenarios) {
+                try {
+                    const result = await this.evaluateScenario(scenario);
+                    results.push(result);
+                }
+                catch (error) {
+                    console.error(`Failed to evaluate scenario ${scenario.interaction_id}:`, error);
+                }
+            }
+            return results;
+        }
+        catch (error) {
+            console.error(`Pattern comparative evaluation failed:`, error);
+            return [{
+                    key: `${this.name}_error`,
+                    score: 0,
+                    comment: `Evaluation error: ${error instanceof Error ? error.message : String(error)}`,
+                    confidence: 0,
+                    modelRankings: [],
+                    bestModel: 'unknown',
+                    modelCount: 0
+                }];
+        }
+    }
+    /**
+     * Build the evaluation prompt - uses base class reliability context with pattern-specific template
+     */
+    buildEvaluationPrompt(scenario, modelResponsesText, modelList) {
+        // Use the base class's properly formatted model responses which include:
+        // - Reliability Status (✅ Completed successfully OR ⚠️ TIMEOUT FAILURE)
+        // - Performance metrics
+        // - All model responses
+        return this.promptTemplate
+            .replace('{scenario_name}', scenario.interaction_id)
+            .replace('{model_responses}', modelResponsesText)
+            .replace('{models}', modelList);
+    }
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases() {
+        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+        const phaseGroups = new Map();
+        // Group scenarios by phase type
+        for (const scenario of scenarios) {
+            const phase = scenario.interaction_id;
+            if (!phaseGroups.has(phase)) {
+                phaseGroups.set(phase, {
+                    models: new Set(),
+                    count: 0
+                });
+            }
+            const group = phaseGroups.get(phase);
+            scenario.models.forEach(model => group.models.add(model.model));
+            group.count++;
+        }
+        // Convert to structured output with descriptions
+        const phaseDescriptions = {
+            'pattern_create_workflow': 'Pattern Creation Workflow - How well each model guides users through creating organizational patterns',
+            'trigger_expansion': 'Trigger Expansion Phase - How well each model expands infrastructure triggers for patterns',
+            'pattern_validation': 'Pattern Validation Phase - How well each model validates organizational patterns',
+            'pattern_matching': 'Pattern Matching Phase - How well each model matches user requirements to existing patterns'
+        };
+        return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
+            phase,
+            description: phaseDescriptions[phase] || `${phase} phase evaluation`,
+            availableModels: Array.from(data.models).sort(),
+            scenarioCount: data.count
+        }));
+    }
+}
+exports.PatternComparativeEvaluator = PatternComparativeEvaluator;

package/dist/evaluation/evaluators/policy-comparative.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Policy Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes organizational policy intent management scenarios
+ * Groups by interaction_id (e.g., policy_create_workflow) and evaluates
+ * quality of policy creation, validation, and enforcement recommendations across different models
+ */
+import { BaseComparativeEvaluator, ComparativeEvaluationScore } from './base-comparative.js';
+import { ComparisonScenario } from '../dataset-analyzer.js';
+export declare class PolicyComparativeEvaluator extends BaseComparativeEvaluator {
+    readonly name = "policy-comparative";
+    readonly description = "Compares AI models on Kubernetes organizational policy intent management quality";
+    protected readonly promptFileName = "policy-comparative.md";
+    protected readonly toolName = "policy";
+    constructor(datasetDir?: string);
+    evaluateAllScenarios(): Promise<ComparativeEvaluationScore[]>;
+    /**
+     * Build the evaluation prompt - uses base class reliability context with policy-specific template
+     */
+    protected buildEvaluationPrompt(scenario: ComparisonScenario, modelResponsesText: string, modelList: string): string;
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases(): {
+        phase: string;
+        description: string;
+        availableModels: string[];
+        scenarioCount: number;
+    }[];
+}
+//# sourceMappingURL=policy-comparative.d.ts.map

package/dist/evaluation/evaluators/policy-comparative.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"policy-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/policy-comparative.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAC;AAC7F,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,qBAAa,0BAA2B,SAAQ,wBAAwB;IACtE,QAAQ,CAAC,IAAI,wBAAwB;IACrC,QAAQ,CAAC,WAAW,sFAAsF;IAC1G,SAAS,CAAC,QAAQ,CAAC,cAAc,2BAA2B;IAC5D,SAAS,CAAC,QAAQ,CAAC,QAAQ,YAAY;gBAE3B,UAAU,CAAC,EAAE,MAAM;IAKzB,oBAAoB,IAAI,OAAO,CAAC,0BAA0B,EAAE,CAAC;IA+BnE;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAYpH;;OAEG;IACH,mBAAmB,IAAI;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CAsCJ"}

package/dist/evaluation/evaluators/policy-comparative.js

@@ -0,0 +1,97 @@
+"use strict";
+/**
+ * Policy Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes organizational policy intent management scenarios
+ * Groups by interaction_id (e.g., policy_create_workflow) and evaluates
+ * quality of policy creation, validation, and enforcement recommendations across different models
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PolicyComparativeEvaluator = void 0;
+const base_comparative_js_1 = require("./base-comparative.js");
+class PolicyComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
+    name = 'policy-comparative';
+    description = 'Compares AI models on Kubernetes organizational policy intent management quality';
+    promptFileName = 'policy-comparative.md';
+    toolName = 'policy';
+    constructor(datasetDir) {
+        super(datasetDir);
+        this.initializePrompt();
+    }
+    async evaluateAllScenarios() {
+        try {
+            const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+            const results = [];
+            console.log(`Found ${scenarios.length} policy scenarios with multiple models for comparative evaluation`);
+            for (const scenario of scenarios) {
+                try {
+                    const result = await this.evaluateScenario(scenario);
+                    results.push(result);
+                }
+                catch (error) {
+                    console.error(`Failed to evaluate scenario ${scenario.interaction_id}:`, error);
+                }
+            }
+            return results;
+        }
+        catch (error) {
+            console.error(`Policy comparative evaluation failed:`, error);
+            return [{
+                    key: `${this.name}_error`,
+                    score: 0,
+                    comment: `Evaluation error: ${error instanceof Error ? error.message : String(error)}`,
+                    confidence: 0,
+                    modelRankings: [],
+                    bestModel: 'unknown',
+                    modelCount: 0
+                }];
+        }
+    }
+    /**
+     * Build the evaluation prompt - uses base class reliability context with policy-specific template
+     */
+    buildEvaluationPrompt(scenario, modelResponsesText, modelList) {
+        // Use the base class's properly formatted model responses which include:
+        // - Reliability Status (✅ Completed successfully OR ⚠️ TIMEOUT FAILURE)
+        // - Performance metrics
+        // - All model responses
+        return this.promptTemplate
+            .replace('{scenario_name}', scenario.interaction_id)
+            .replace('{model_responses}', modelResponsesText)
+            .replace('{models}', modelList);
+    }
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases() {
+        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+        const phaseGroups = new Map();
+        // Group scenarios by phase type
+        for (const scenario of scenarios) {
+            const phase = scenario.interaction_id;
+            if (!phaseGroups.has(phase)) {
+                phaseGroups.set(phase, {
+                    models: new Set(),
+                    count: 0
+                });
+            }
+            const group = phaseGroups.get(phase);
+            scenario.models.forEach(model => group.models.add(model.model));
+            group.count++;
+        }
+        // Convert to structured output with descriptions
+        const phaseDescriptions = {
+            'policy_create_workflow': 'Policy Creation Workflow - How well each model guides users through creating organizational policies',
+            'policy_validation': 'Policy Validation Phase - How well each model validates policy intent correctness and enforceability',
+            'policy_enforcement_recommendations': 'Policy Enforcement Recommendations - How well each model provides enforcement strategies',
+            'policy_compliance_analysis': 'Policy Compliance Analysis - How well each model analyzes compliance with existing policies'
+        };
+        return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
+            phase,
+            description: phaseDescriptions[phase] || `${phase} phase evaluation`,
+            availableModels: Array.from(data.models).sort(),
+            scenarioCount: data.count
+        }));
+    }
+}
+exports.PolicyComparativeEvaluator = PolicyComparativeEvaluator;

package/dist/evaluation/evaluators/recommendation-comparative.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Recommendation Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes recommendation scenarios
+ * Uses dynamic model inclusion based on available datasets
+ * Follows reference-free comparative evaluation methodology
+ */
+import { BaseComparativeEvaluator } from './base-comparative.js';
+export declare class RecommendationComparativeEvaluator extends BaseComparativeEvaluator {
+    readonly name = "recommendation_comparative";
+    readonly description = "Compares multiple AI models on Kubernetes deployment recommendation scenarios";
+    protected readonly promptFileName = "recommendation-comparative.md";
+    protected readonly toolName = "recommend";
+    constructor(datasetDir?: string);
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases(): {
+        phase: string;
+        description: string;
+        availableModels: string[];
+        scenarioCount: number;
+    }[];
+}
+//# sourceMappingURL=recommendation-comparative.d.ts.map

package/dist/evaluation/evaluators/recommendation-comparative.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recommendation-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/recommendation-comparative.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,qBAAa,kCAAmC,SAAQ,wBAAwB;IAC9E,QAAQ,CAAC,IAAI,gCAAgC;IAC7C,QAAQ,CAAC,WAAW,mFAAmF;IACvG,SAAS,CAAC,QAAQ,CAAC,cAAc,mCAAmC;IACpE,SAAS,CAAC,QAAQ,CAAC,QAAQ,eAAe;gBAE9B,UAAU,CAAC,EAAE,MAAM;IAK/B;;OAEG;IACH,mBAAmB,IAAI;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CAsCJ"}

package/dist/evaluation/evaluators/recommendation-comparative.js

@@ -0,0 +1,55 @@
+"use strict";
+/**
+ * Recommendation Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes recommendation scenarios
+ * Uses dynamic model inclusion based on available datasets
+ * Follows reference-free comparative evaluation methodology
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecommendationComparativeEvaluator = void 0;
+const base_comparative_js_1 = require("./base-comparative.js");
+class RecommendationComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
+    name = 'recommendation_comparative';
+    description = 'Compares multiple AI models on Kubernetes deployment recommendation scenarios';
+    promptFileName = 'recommendation-comparative.md';
+    toolName = 'recommend';
+    constructor(datasetDir) {
+        super(datasetDir);
+        this.initializePrompt();
+    }
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases() {
+        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+        const phaseGroups = new Map();
+        // Group scenarios by phase type
+        for (const scenario of scenarios) {
+            const phase = scenario.interaction_id;
+            if (!phaseGroups.has(phase)) {
+                phaseGroups.set(phase, {
+                    models: new Set(),
+                    count: 0
+                });
+            }
+            const group = phaseGroups.get(phase);
+            scenario.models.forEach(model => group.models.add(model.model));
+            group.count++;
+        }
+        // Convert to structured output with descriptions
+        const phaseDescriptions = {
+            'clarification_phase': 'Intent Analysis Phase - How well each model analyzes user intents and identifies missing context',
+            'question_generation': 'Question Generation Phase - How well each model generates clarifying questions to enhance requirements',
+            'solution_assembly': 'Solution Assembly Phase - How well each model selects appropriate Kubernetes resources and deployment patterns',
+            'generate_manifests_phase': 'Manifest Generation Phase - How well each model generates production-ready Kubernetes manifests'
+        };
+        return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
+            phase,
+            description: phaseDescriptions[phase] || `${phase} phase evaluation`,
+            availableModels: Array.from(data.models).sort(),
+            scenarioCount: data.count
+        }));
+    }
+}
+exports.RecommendationComparativeEvaluator = RecommendationComparativeEvaluator;

package/dist/evaluation/evaluators/remediation-comparative.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Remediation Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes troubleshooting scenarios
+ * Uses dynamic model inclusion based on available datasets
+ * Follows reference-free comparative evaluation methodology
+ */
+import { BaseComparativeEvaluator } from './base-comparative.js';
+export declare class RemediationComparativeEvaluator extends BaseComparativeEvaluator {
+    readonly name = "remediation_comparative";
+    readonly description = "Compares multiple AI models on Kubernetes troubleshooting scenarios";
+    protected readonly promptFileName = "remediation-comparative.md";
+    protected readonly toolName = "remediate";
+    constructor(datasetDir?: string);
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases(): {
+        phase: string;
+        description: string;
+        availableModels: string[];
+        scenarioCount: number;
+    }[];
+}
+//# sourceMappingURL=remediation-comparative.d.ts.map

package/dist/evaluation/evaluators/remediation-comparative.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remediation-comparative.d.ts","sourceRoot":"","sources":["../../../src/evaluation/evaluators/remediation-comparative.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,qBAAa,+BAAgC,SAAQ,wBAAwB;IAC3E,QAAQ,CAAC,IAAI,6BAA6B;IAC1C,QAAQ,CAAC,WAAW,yEAAyE;IAC7F,SAAS,CAAC,QAAQ,CAAC,cAAc,gCAAgC;IACjE,SAAS,CAAC,QAAQ,CAAC,QAAQ,eAAe;gBAE9B,UAAU,CAAC,EAAE,MAAM;IAK/B;;OAEG;IACH,mBAAmB,IAAI;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,EAAE;CAqCJ"}

package/dist/evaluation/evaluators/remediation-comparative.js

@@ -0,0 +1,54 @@
+"use strict";
+/**
+ * Remediation Comparative Evaluator
+ *
+ * Compares multiple AI models on Kubernetes troubleshooting scenarios
+ * Uses dynamic model inclusion based on available datasets
+ * Follows reference-free comparative evaluation methodology
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemediationComparativeEvaluator = void 0;
+const base_comparative_js_1 = require("./base-comparative.js");
+class RemediationComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
+    name = 'remediation_comparative';
+    description = 'Compares multiple AI models on Kubernetes troubleshooting scenarios';
+    promptFileName = 'remediation-comparative.md';
+    toolName = 'remediate';
+    constructor(datasetDir) {
+        super(datasetDir);
+        this.initializePrompt();
+    }
+    /**
+     * Get detailed breakdown of evaluation phases available
+     */
+    getEvaluationPhases() {
+        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
+        const phaseGroups = new Map();
+        // Group scenarios by phase type
+        for (const scenario of scenarios) {
+            const phase = scenario.interaction_id;
+            if (!phaseGroups.has(phase)) {
+                phaseGroups.set(phase, {
+                    models: new Set(),
+                    count: 0
+                });
+            }
+            const group = phaseGroups.get(phase);
+            scenario.models.forEach(model => group.models.add(model.model));
+            group.count++;
+        }
+        // Convert to structured output with descriptions
+        const phaseDescriptions = {
+            'manual_analyze': 'Manual Investigation Phase - How well each model investigates and diagnoses issues',
+            'manual_execute': 'Manual Execution Phase - How well each model validates and confirms fixes worked',
+            'automatic_analyze_execute': 'Automatic Full Workflow - End-to-end troubleshooting in single automated workflow'
+        };
+        return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
+            phase,
+            description: phaseDescriptions[phase] || `${phase} phase evaluation`,
+            availableModels: Array.from(data.models).sort(),
+            scenarioCount: data.count
+        }));
+    }
+}
+exports.RemediationComparativeEvaluator = RemediationComparativeEvaluator;