coding-agent-benchmarks 0.1.0

package/dist/types.d.ts

@@ -0,0 +1,354 @@
+/**
+ * Core type definitions for coding-agent-benchmarks
+ */
+/**
+ * Generic categories for test scenarios
+ */
+export type TestCategory = 'typescript' | 'react' | 'testing' | 'architecture' | 'performance' | 'general';
+/**
+ * Severity levels for test scenarios
+ */
+export type Severity = 'critical' | 'major' | 'minor';
+/**
+ * Supported adapter types
+ */
+export type AdapterType = 'copilot' | 'claude-code';
+/**
+ * Pattern matching configuration for code validation
+ */
+export interface PatternValidation {
+    /**
+     * Regex patterns that should NOT appear in generated code
+     */
+    forbiddenPatterns?: readonly RegExp[];
+    /**
+     * Regex patterns that MUST appear in generated code
+     */
+    requiredPatterns?: readonly RegExp[];
+    /**
+     * Import statements that should NOT be present
+     * Example: ['from "lodash"', 'import * as']
+     */
+    forbiddenImports?: readonly string[];
+    /**
+     * Import statements that MUST be present
+     */
+    requiredImports?: readonly string[];
+    /**
+     * File name patterns that should NOT be created
+     */
+    forbiddenFileNamePatterns?: readonly RegExp[];
+    /**
+     * File name patterns that MUST be created
+     */
+    requiredFileNamePatterns?: readonly RegExp[];
+}
+/**
+ * LLM-as-judge validation configuration
+ */
+export interface LLMJudgeValidation {
+    /**
+     * Enable LLM-based semantic validation
+     */
+    enabled: boolean;
+    /**
+     * Custom prompt for the LLM judge
+     * If not provided, a default judgment prompt will be used
+     */
+    judgmentPrompt?: string;
+    /**
+     * Model to use for judgment (default: openai/gpt-4.1)
+     */
+    model?: string;
+}
+/**
+ * ESLint validation configuration
+ */
+export interface ESLintValidation {
+    /**
+     * Enable ESLint validation on generated code
+     */
+    enabled: boolean;
+    /**
+     * Custom ESLint config path (optional)
+     */
+    configPath?: string;
+}
+/**
+ * Validation strategy combining multiple validation methods
+ */
+export interface ValidationStrategy {
+    /**
+     * Pattern-based validation
+     */
+    patterns?: PatternValidation;
+    /**
+     * LLM-as-judge validation
+     */
+    llmJudge?: LLMJudgeValidation;
+    /**
+     * ESLint validation
+     */
+    eslint?: ESLintValidation;
+}
+/**
+ * A test scenario defining a coding task and validation criteria
+ */
+export interface TestScenario {
+    /**
+     * Unique identifier for the scenario
+     */
+    id: string;
+    /**
+     * Category of the test
+     */
+    category: TestCategory;
+    /**
+     * Severity level of violations
+     */
+    severity: Severity;
+    /**
+     * Tags for filtering and organization
+     */
+    tags: readonly string[];
+    /**
+     * Human-readable description of what this scenario tests
+     */
+    description: string;
+    /**
+     * The prompt/instruction given to the coding agent
+     */
+    prompt: string;
+    /**
+     * Optional inline context to include with the prompt
+     */
+    context?: string;
+    /**
+     * Optional context files to provide to the agent
+     * Paths are resolved relative to the workspace root
+     */
+    contextFiles?: readonly string[];
+    /**
+     * Validation strategy to apply to generated code
+     */
+    validationStrategy: ValidationStrategy;
+    /**
+     * Optional timeout in milliseconds
+     * - number: Specific timeout (e.g., 120000 = 2 minutes)
+     * - null: No timeout (wait indefinitely)
+     * - undefined: Use defaultTimeout from config, or 120000ms default
+     */
+    timeout?: number | null;
+}
+/**
+ * A violation found during validation
+ */
+export interface Violation {
+    /**
+     * Type of validation that found this violation
+     */
+    type: 'pattern' | 'llm-judge' | 'eslint';
+    /**
+     * Description of the violation
+     */
+    message: string;
+    /**
+     * File where the violation was found (if applicable)
+     */
+    file?: string;
+    /**
+     * Line number where the violation was found (if applicable)
+     */
+    line?: number;
+    /**
+     * Severity of the violation
+     */
+    severity: Severity;
+    /**
+     * Additional details about the violation
+     */
+    details?: string;
+}
+/**
+ * Result of a single validation
+ */
+export interface ValidationResult {
+    /**
+     * Whether the validation passed
+     */
+    passed: boolean;
+    /**
+     * Score from 0.0 to 1.0 (1.0 = perfect, 0.0 = failed)
+     * -1 = validation was skipped
+     */
+    score: number;
+    /**
+     * List of violations found
+     */
+    violations: Violation[];
+    /**
+     * Type of validator that produced this result
+     */
+    validatorType: 'pattern' | 'llm-judge' | 'eslint';
+    /**
+     * Error message if the validator itself failed
+     */
+    error?: string;
+}
+/**
+ * Result of evaluating a single scenario
+ */
+export interface EvaluationResult {
+    /**
+     * The scenario that was evaluated
+     */
+    scenario: TestScenario;
+    /**
+     * Whether the scenario passed overall
+     */
+    passed: boolean;
+    /**
+     * Overall score (average of all validator scores)
+     */
+    score: number;
+    /**
+     * Results from each validator
+     */
+    validationResults: ValidationResult[];
+    /**
+     * All violations found across all validators
+     */
+    violations: Violation[];
+    /**
+     * Generated code (if available)
+     */
+    generatedCode?: {
+        files: {
+            path: string;
+            content: string;
+        }[];
+    };
+    /**
+     * Time taken to evaluate (milliseconds)
+     */
+    duration: number;
+    /**
+     * Error message if evaluation failed
+     */
+    error?: string;
+    /**
+     * Baseline comparison (if available)
+     */
+    baselineComparison?: {
+        baselineScore: number;
+        delta: number;
+        isImprovement: boolean;
+    };
+}
+/**
+ * Full evaluation report for multiple scenarios
+ */
+export interface EvaluationReport {
+    /**
+     * Adapter used for code generation
+     */
+    adapter: AdapterType;
+    /**
+     * Model used (if applicable)
+     */
+    model?: string;
+    /**
+     * Timestamp when evaluation started
+     */
+    timestamp: string;
+    /**
+     * Results for each scenario
+     */
+    results: EvaluationResult[];
+    /**
+     * Summary statistics
+     */
+    summary: {
+        total: number;
+        passed: number;
+        failed: number;
+        skipped: number;
+        averageScore: number;
+        totalViolations: number;
+    };
+    /**
+     * Total duration of all evaluations (milliseconds)
+     */
+    totalDuration: number;
+}
+/**
+ * Interface for code generation adapters
+ */
+export interface CodeGenerationAdapter {
+    /**
+     * Type identifier for this adapter
+     */
+    type: AdapterType;
+    /**
+     * Check if the adapter's CLI tool is available
+     */
+    checkAvailability(): Promise<boolean>;
+    /**
+     * Generate code based on a prompt
+     * @param prompt The instruction/prompt for the coding agent
+     * @param contextFiles Optional context files to provide
+     * @param timeout Timeout in milliseconds, or null for no timeout (default: null = no timeout)
+     * @returns List of files that were created or modified
+     */
+    generate(prompt: string, contextFiles?: readonly string[], timeout?: number | null): Promise<string[]>;
+}
+/**
+ * Interface for code validators
+ */
+export interface CodeValidator {
+    /**
+     * Type identifier for this validator
+     */
+    type: 'pattern' | 'llm-judge' | 'eslint';
+    /**
+     * Validate generated code
+     * @param files List of file paths that were generated
+     * @param scenario The test scenario being evaluated
+     * @returns Validation result
+     */
+    validate(files: readonly string[], scenario: TestScenario): Promise<ValidationResult>;
+}
+/**
+ * Configuration file structure
+ */
+export interface BenchmarkConfig {
+    /**
+     * Custom test scenarios
+     */
+    scenarios?: TestScenario[];
+    /**
+     * Default adapter to use
+     */
+    defaultAdapter?: AdapterType;
+    /**
+     * Default LLM model for judge
+     */
+    defaultModel?: string;
+    /**
+     * Workspace root directory (auto-detected if not specified)
+     */
+    workspaceRoot?: string;
+    /**
+     * Default timeout for code generation in milliseconds
+     * Applies to all scenarios unless overridden per-scenario
+     * - number: Specific timeout (e.g., 120000 = 2 minutes)
+     * - null: No timeout (wait indefinitely)
+     * - undefined: Use built-in default of 120000ms
+     */
+    defaultTimeout?: number | null;
+    /**
+     * Output directory for reports
+     */
+    outputDir?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,cAAc,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3G;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,OAAO,GAAG,OAAO,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,aAAa,CAAC;AAEpD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,iBAAiB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEtC;;OAEG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAErC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAErC;;OAEG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEpC;;OAEG;IACH,yBAAyB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE9C;;OAEG;IACH,wBAAwB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAE9B;;OAEG;IACH,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IAEvB;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAExB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEjC;;OAEG;IACH,kBAAkB,EAAE,kBAAkB,CAAC;IAEvC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;OAEG;IACH,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEzC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,UAAU,EAAE,SAAS,EAAE,CAAC;IAExB;;OAEG;IACH,aAAa,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAElD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IAEvB;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAEhB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IAEtC;;OAEG;IACH,UAAU,EAAE,SAAS,EAAE,CAAC;IAExB;;OAEG;IACH,aAAa,CAAC,EAAE;QACd,KAAK,EAAE;YACL,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,MAAM,CAAC;SACjB,EAAE,CAAC;KACL,CAAC;IAEF;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,OAAO,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,OAAO,EAAE,WAAW,CAAC;IAErB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAE5B;;OAEG;IACH,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IAEF;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,IAAI,EAAE,WAAW,CAAC;IAElB;;OAEG;IACH,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtC;;;;;;OAMG;IACH,QAAQ,CACN,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;OAEG;IACH,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CACN,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;IAE3B;;OAEG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC;IAE7B;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}

package/dist/types.js

@@ -0,0 +1,6 @@
+"use strict";
+/**
+ * Core type definitions for coding-agent-benchmarks
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA;;GAEG"}

package/dist/utils/baselineManager.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Baseline management for tracking evaluation results over time
+ */
+import { AdapterType, EvaluationResult, Violation } from '../types';
+export interface BaselineData {
+    scenarioId: string;
+    score: number;
+    violations: Violation[];
+    timestamp: string;
+    adapter: AdapterType;
+    model?: string;
+}
+export declare class BaselineManager {
+    private baselineDir;
+    constructor(workspaceRoot: string);
+    /**
+     * Get the baseline file path for a scenario
+     */
+    private getBaselinePath;
+    /**
+     * Save a baseline result
+     */
+    saveBaseline(result: EvaluationResult, adapter: AdapterType, model?: string): void;
+    /**
+     * Load a baseline result
+     */
+    loadBaseline(adapter: AdapterType, model: string, scenarioId: string): BaselineData | null;
+    /**
+     * Compare current result with baseline
+     */
+    compareWithBaseline(result: EvaluationResult, adapter: AdapterType, model?: string): {
+        baselineScore: number;
+        delta: number;
+        isImprovement: boolean;
+    } | null;
+    /**
+     * List all baselines for an adapter
+     */
+    listBaselines(adapter: AdapterType, model?: string): BaselineData[];
+    /**
+     * Collect baselines from a directory
+     */
+    private collectBaselinesFromDir;
+    /**
+     * Delete a baseline
+     */
+    deleteBaseline(adapter: AdapterType, model: string, scenarioId: string): boolean;
+    /**
+     * Delete all baselines for an adapter
+     */
+    deleteAllBaselines(adapter: AdapterType, model?: string): number;
+}
+//# sourceMappingURL=baselineManager.d.ts.map

package/dist/utils/baselineManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"baselineManager.d.ts","sourceRoot":"","sources":["../../src/utils/baselineManager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAEpE,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,WAAW,CAAS;gBAEhB,aAAa,EAAE,MAAM;IAIjC;;OAEG;IACH,OAAO,CAAC,eAAe;IAQvB;;OAEG;IACH,YAAY,CACV,MAAM,EAAE,gBAAgB,EACxB,OAAO,EAAE,WAAW,EACpB,KAAK,GAAE,MAAkB,GACxB,IAAI;IAsBP;;OAEG;IACH,YAAY,CACV,OAAO,EAAE,WAAW,EACpB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,GACjB,YAAY,GAAG,IAAI;IAgBtB;;OAEG;IACH,mBAAmB,CACjB,MAAM,EAAE,gBAAgB,EACxB,OAAO,EAAE,WAAW,EACpB,KAAK,GAAE,MAAkB,GACxB;QACD,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,OAAO,CAAC;KACxB,GAAG,IAAI;IAiBR;;OAEG;IACH,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE;IA4BnE;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAe/B;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO;IAgBhF;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;CA8CjE"}

package/dist/utils/baselineManager.js

@@ -0,0 +1,220 @@
+"use strict";
+/**
+ * Baseline management for tracking evaluation results over time
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaselineManager = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+class BaselineManager {
+    constructor(workspaceRoot) {
+        this.baselineDir = path.join(workspaceRoot, '.benchmarks', 'baselines');
+    }
+    /**
+     * Get the baseline file path for a scenario
+     */
+    getBaselinePath(adapter, model, scenarioId) {
+        return path.join(this.baselineDir, adapter, model, `${scenarioId}.json`);
+    }
+    /**
+     * Save a baseline result
+     */
+    saveBaseline(result, adapter, model = 'default') {
+        const baseline = {
+            scenarioId: result.scenario.id,
+            score: result.score,
+            violations: result.violations,
+            timestamp: new Date().toISOString(),
+            adapter,
+            model,
+        };
+        const baselinePath = this.getBaselinePath(adapter, model, result.scenario.id);
+        const dir = path.dirname(baselinePath);
+        // Ensure directory exists
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        // Write baseline file
+        fs.writeFileSync(baselinePath, JSON.stringify(baseline, null, 2), 'utf-8');
+    }
+    /**
+     * Load a baseline result
+     */
+    loadBaseline(adapter, model, scenarioId) {
+        const baselinePath = this.getBaselinePath(adapter, model, scenarioId);
+        if (!fs.existsSync(baselinePath)) {
+            return null;
+        }
+        try {
+            const content = fs.readFileSync(baselinePath, 'utf-8');
+            return JSON.parse(content);
+        }
+        catch (error) {
+            console.warn(`Failed to load baseline for ${scenarioId}:`, error);
+            return null;
+        }
+    }
+    /**
+     * Compare current result with baseline
+     */
+    compareWithBaseline(result, adapter, model = 'default') {
+        const baseline = this.loadBaseline(adapter, model, result.scenario.id);
+        if (!baseline) {
+            return null;
+        }
+        const delta = result.score - baseline.score;
+        const isImprovement = delta > 0;
+        return {
+            baselineScore: baseline.score,
+            delta,
+            isImprovement,
+        };
+    }
+    /**
+     * List all baselines for an adapter
+     */
+    listBaselines(adapter, model) {
+        const baselines = [];
+        const adapterDir = path.join(this.baselineDir, adapter);
+        if (!fs.existsSync(adapterDir)) {
+            return baselines;
+        }
+        // If model specified, only search that model directory
+        if (model) {
+            const modelDir = path.join(adapterDir, model);
+            if (fs.existsSync(modelDir)) {
+                this.collectBaselinesFromDir(modelDir, baselines);
+            }
+        }
+        else {
+            // Search all model directories
+            const modelDirs = fs.readdirSync(adapterDir, { withFileTypes: true });
+            for (const dir of modelDirs) {
+                if (dir.isDirectory()) {
+                    const modelDir = path.join(adapterDir, dir.name);
+                    this.collectBaselinesFromDir(modelDir, baselines);
+                }
+            }
+        }
+        return baselines;
+    }
+    /**
+     * Collect baselines from a directory
+     */
+    collectBaselinesFromDir(dir, baselines) {
+        const files = fs.readdirSync(dir);
+        for (const file of files) {
+            if (file.endsWith('.json')) {
+                try {
+                    const content = fs.readFileSync(path.join(dir, file), 'utf-8');
+                    const baseline = JSON.parse(content);
+                    baselines.push(baseline);
+                }
+                catch (error) {
+                    console.warn(`Failed to load baseline ${file}:`, error);
+                }
+            }
+        }
+    }
+    /**
+     * Delete a baseline
+     */
+    deleteBaseline(adapter, model, scenarioId) {
+        const baselinePath = this.getBaselinePath(adapter, model, scenarioId);
+        if (!fs.existsSync(baselinePath)) {
+            return false;
+        }
+        try {
+            fs.unlinkSync(baselinePath);
+            return true;
+        }
+        catch (error) {
+            console.warn(`Failed to delete baseline for ${scenarioId}:`, error);
+            return false;
+        }
+    }
+    /**
+     * Delete all baselines for an adapter
+     */
+    deleteAllBaselines(adapter, model) {
+        let count = 0;
+        const adapterDir = path.join(this.baselineDir, adapter);
+        if (!fs.existsSync(adapterDir)) {
+            return 0;
+        }
+        if (model) {
+            const modelDir = path.join(adapterDir, model);
+            if (fs.existsSync(modelDir)) {
+                const files = fs.readdirSync(modelDir);
+                for (const file of files) {
+                    if (file.endsWith('.json')) {
+                        try {
+                            fs.unlinkSync(path.join(modelDir, file));
+                            count++;
+                        }
+                        catch (error) {
+                            console.warn(`Failed to delete ${file}:`, error);
+                        }
+                    }
+                }
+            }
+        }
+        else {
+            // Delete all models
+            const modelDirs = fs.readdirSync(adapterDir, { withFileTypes: true });
+            for (const dir of modelDirs) {
+                if (dir.isDirectory()) {
+                    const modelDir = path.join(adapterDir, dir.name);
+                    const files = fs.readdirSync(modelDir);
+                    for (const file of files) {
+                        if (file.endsWith('.json')) {
+                            try {
+                                fs.unlinkSync(path.join(modelDir, file));
+                                count++;
+                            }
+                            catch (error) {
+                                console.warn(`Failed to delete ${file}:`, error);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return count;
+    }
+}
+exports.BaselineManager = BaselineManager;
+//# sourceMappingURL=baselineManager.js.map

package/dist/utils/baselineManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"baselineManager.js","sourceRoot":"","sources":["../../src/utils/baselineManager.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAY7B,MAAa,eAAe;IAG1B,YAAY,aAAqB;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACK,eAAe,CACrB,OAAoB,EACpB,KAAa,EACb,UAAkB;QAElB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACH,YAAY,CACV,MAAwB,EACxB,OAAoB,EACpB,QAAgB,SAAS;QAEzB,MAAM,QAAQ,GAAiB;YAC7B,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE;YAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO;YACP,KAAK;SACN,CAAC;QAEF,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC9E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAEvC,0BAA0B;QAC1B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,sBAAsB;QACtB,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACH,YAAY,CACV,OAAoB,EACpB,KAAa,EACb,UAAkB;QAElB,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;QAEtE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAiB,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,+BAA+B,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;YAClE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,mBAAmB,CACjB,MAAwB,EACxB,OAAoB,EACpB,QAAgB,SAAS;QAMzB,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEvE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QAC5C,MAAM,aAAa,GAAG,KAAK,GAAG,CAAC,CAAC;QAEhC,OAAO;YACL,aAAa,EAAE,QAAQ,CAAC,KAAK;YAC7B,KAAK;YACL,aAAa;SACd,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,OAAoB,EAAE,KAAc;QAChD,MAAM,SAAS,GAAmB,EAAE,CAAC;QAErC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACxD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,uDAAuD;QACvD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,uBAAuB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,+BAA+B;YAC/B,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjD,IAAI,CAAC,uBAAuB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBACpD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,GAAW,EAAE,SAAyB;QACpE,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;oBAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAiB,CAAC;oBACrD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,IAAI,CAAC,2BAA2B,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,OAAoB,EAAE,KAAa,EAAE,UAAkB;QACpE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;QAEtE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,iCAAiC,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;YACpE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,OAAoB,EAAE,KAAc;QACrD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAExD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9C,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC3B,IAAI,CAAC;4BACH,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;4BACzC,KAAK,EAAE,CAAC;wBACV,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,OAAO,CAAC,IAAI,CAAC,oBAAoB,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;wBACnD,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,oBAAoB;YACpB,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;oBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjD,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;wBACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,IAAI,CAAC;gCACH,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;gCACzC,KAAK,EAAE,CAAC;4BACV,CAAC;4BAAC,OAAO,KAAK,EAAE,CAAC;gCACf,OAAO,CAAC,IAAI,CAAC,oBAAoB,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;4BACnD,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAvND,0CAuNC"}