coding-agent-benchmarks 0.1.2 → 0.1.4

package/README.md

@@ -93,7 +93,7 @@ module.exports = {
       validationStrategy: {
         patterns: {
           forbiddenPatterns: [/:\s*any\b/],
-          requiredPatterns: [/interface\s+User/],
+          requiredPatterns: [/interface\s+User\b/],
         },
       },
       timeout: 120000,
@@ -298,6 +298,43 @@ Test LLM judge with a custom prompt (for debugging).
 **Options:**
 - `--model <model>`: LLM model to use
 
+## Understanding Output
+
+When running evaluations, each scenario displays a live status that updates as it progresses:
+
+```
+Evaluating 3 scenario(s)...
+
+✓ [1/3] typescript-no-any PASS (score: 1.00) 14.8s
+✗ [2/3] react-inline-styles FAIL (score: 0.60) 22.1s
+○ [3/3] async-error-handling SKIP (error) 2m 5s
+
+============================================================
+EVALUATION SUMMARY
+============================================================
+Total scenarios: 3
+Passed: 1
+Failed: 1
+Skipped: 1
+...
+```
+
+### Status Indicators
+
+| Status | Symbol | Meaning |
+|--------|--------|---------|
+| **PASS** | ✓ | Scenario passed validation (score ≥ 0.8, no violations) |
+| **FAIL** | ✗ | Scenario was evaluated but didn't pass (low score or violations) |
+| **SKIP** | ○ | Scenario couldn't be evaluated due to an error |
+
+### Common Causes for SKIP
+
+- **Timeout**: Code generation exceeded the configured timeout
+- **Adapter failure**: The CLI (Copilot or Claude Code) crashed or returned an error
+- **File system errors**: Couldn't read context files or write generated code
+
+> **Note**: In interactive terminals, you'll see a spinner animation (⠋) while scenarios are running. In CI/non-TTY environments, output falls back to simple line-by-line logging.
+
 ## Programmatic Usage
 
 You can also use the framework programmatically:

package/dist/User.d.ts

@@ -0,0 +1,10 @@
+/**
+ * User interface representing a user entity
+ */
+export interface User {
+    id: number;
+    name: string;
+    email: string;
+    metadata: Record<string, unknown>;
+}
+//# sourceMappingURL=User.d.ts.map

package/dist/User.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"User.d.ts","sourceRoot":"","sources":["../src/User.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC"}

package/dist/User.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=User.js.map

package/dist/User.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"User.js","sourceRoot":"","sources":["../src/User.ts"],"names":[],"mappings":""}

package/dist/adapters/claudeCodeCLI.d.ts

@@ -2,14 +2,23 @@
  * Claude Code CLI Adapter
  */
 import { CodeGenerationAdapter } from '../types';
+export interface ClaudeCodeCLIAdapterOptions {
+    workspaceRoot?: string;
+    model?: string;
+}
 export declare class ClaudeCodeCLIAdapter implements CodeGenerationAdapter {
     readonly type: "claude-code";
     private workspaceRoot;
-    constructor(workspaceRoot?: string);
+    private model;
+    constructor(options?: ClaudeCodeCLIAdapterOptions);
     /**
      * Check if Claude Code CLI is available
      */
     checkAvailability(): Promise<boolean>;
+    /**
+     * Build the full prompt with context files
+     */
+    private buildPrompt;
     /**
      * Generate code using Claude Code CLI
      * @param timeout Timeout in milliseconds, or null for no timeout

package/dist/adapters/claudeCodeCLI.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"claudeCodeCLI.d.ts","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,qBAAa,oBAAqB,YAAW,qBAAqB;IAChE,SAAgB,IAAI,EAAG,aAAa,CAAU;IAC9C,OAAO,CAAC,aAAa,CAAS;gBAElB,aAAa,CAAC,EAAE,MAAM;IAIlC;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB3C;;;OAGG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;CA8ErB"}
+{"version":3,"file":"claudeCodeCLI.d.ts","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,MAAM,WAAW,2BAA2B;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,qBAAa,oBAAqB,YAAW,qBAAqB;IAChE,SAAgB,IAAI,EAAG,aAAa,CAAU;IAC9C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,KAAK,CAAS;gBAEV,OAAO,CAAC,EAAE,2BAA2B;IAKjD;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB3C;;OAEG;IACH,OAAO,CAAC,WAAW;IA8BnB;;;OAGG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;CA8ErB"}

package/dist/adapters/claudeCodeCLI.js

@@ -2,15 +2,52 @@
 /**
  * Claude Code CLI Adapter
  */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ClaudeCodeCLIAdapter = void 0;
 const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
 const gitUtils_1 = require("../utils/gitUtils");
 const workspaceUtils_1 = require("../utils/workspaceUtils");
+const DEFAULT_MODEL = 'sonnet';
 class ClaudeCodeCLIAdapter {
-    constructor(workspaceRoot) {
+    constructor(options) {
         this.type = 'claude-code';
-        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(workspaceRoot);
+        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(options?.workspaceRoot);
+        this.model = options?.model || DEFAULT_MODEL;
     }
     /**
      * Check if Claude Code CLI is available
@@ -29,36 +66,45 @@ class ClaudeCodeCLIAdapter {
         });
     }
     /**
-     * Generate code using Claude Code CLI
-     * @param timeout Timeout in milliseconds, or null for no timeout
+     * Build the full prompt with context files
      */
-    async generate(prompt, contextFiles, timeout) {
-        // Reset workspace to clean state before generation
-        try {
-            (0, gitUtils_1.resetGitWorkingDirectory)(this.workspaceRoot);
-        }
-        catch (error) {
-            console.warn('Warning: Could not reset git working directory:', error);
-        }
-        // Build the full prompt with context
-        let fullPrompt = prompt;
+    buildPrompt(prompt, contextFiles) {
+        const parts = [];
         if (contextFiles && contextFiles.length > 0) {
             const contexts = (0, workspaceUtils_1.readContextFiles)(this.workspaceRoot, contextFiles);
             if (contexts.length > 0) {
-                const contextSection = contexts
-                    .map(ctx => `\n\n### Context from ${ctx.path}:\n\`\`\`\n${ctx.content}\n\`\`\``)
-                    .join('\n');
-                fullPrompt = `${prompt}${contextSection}`;
+                parts.push("# Reference Files\n");
+                const contextContent = contexts
+                    .map((ctx) => {
+                    const ext = path.extname(ctx.path).slice(1) || "typescript";
+                    return `### ${ctx.path}\n\`\`\`${ext}\n${ctx.content}\n\`\`\``;
+                })
+                    .join("\n\n");
+                parts.push(contextContent);
+                parts.push("\n---\n");
             }
         }
-        // Spawn the claude CLI process
-        // Note: Claude Code CLI may require different flags or approach
-        // This is a basic implementation that may need adjustment
+        parts.push("# Task\n");
+        parts.push(prompt);
+        parts.push("\n\nCreate/update the necessary file(s). Do not output code to the terminal - write it to files instead.");
+        return parts.join("\n");
+    }
+    /**
+     * Generate code using Claude Code CLI
+     * @param timeout Timeout in milliseconds, or null for no timeout
+     */
+    async generate(prompt, contextFiles, timeout) {
+        const fullPrompt = this.buildPrompt(prompt, contextFiles);
+        // Capture git status before generation
+        const statusBefore = (0, gitUtils_1.getGitStatusPorcelain)(this.workspaceRoot);
+        // Write prompt to temp file and pipe via stdin (matches @copilot-evals pattern)
         return new Promise((resolve, reject) => {
-            const proc = (0, child_process_1.spawn)('claude', ['--non-interactive', fullPrompt], {
+            const tempFile = path.join(this.workspaceRoot, '.claude-eval-prompt.txt');
+            fs.writeFileSync(tempFile, fullPrompt, 'utf8');
+            const command = `cat "${tempFile}" | claude --model ${this.model} --dangerously-skip-permissions --disallowed-tools 'Bash(rm)' --disallowed-tools 'Bash(git push)' --disallowed-tools 'Bash(git commit)'`;
+            const proc = (0, child_process_1.spawn)('sh', ['-c', command], {
                 cwd: this.workspaceRoot,
                 stdio: ['pipe', 'pipe', 'pipe'],
-                shell: true,
             });
             let stdout = '';
             let stderr = '';
@@ -80,13 +126,21 @@ class ClaudeCodeCLIAdapter {
                 if (timeoutHandle) {
                     clearTimeout(timeoutHandle);
                 }
+                // Clean up temp file
+                try {
+                    fs.unlinkSync(tempFile);
+                }
+                catch {
+                    // Ignore cleanup errors
+                }
                 if (code !== 0) {
                     reject(new Error(`Claude Code CLI exited with code ${code}\nStderr: ${stderr}`));
                     return;
                 }
-                // Get the list of changed files
+                // Get files changed during generation (diff before/after)
                 try {
-                    const changedFiles = (0, gitUtils_1.getChangedFiles)(this.workspaceRoot);
+                    const statusAfter = (0, gitUtils_1.getGitStatusPorcelain)(this.workspaceRoot);
+                    const changedFiles = (0, gitUtils_1.getChangedFilesDiff)(statusBefore, statusAfter);
                     resolve(changedFiles);
                 }
                 catch (error) {
@@ -97,6 +151,13 @@ class ClaudeCodeCLIAdapter {
                 if (timeoutHandle) {
                     clearTimeout(timeoutHandle);
                 }
+                // Clean up temp file
+                try {
+                    fs.unlinkSync(tempFile);
+                }
+                catch {
+                    // Ignore cleanup errors
+                }
                 reject(new Error(`Failed to spawn Claude Code CLI: ${error}`));
             });
         });

package/dist/adapters/claudeCodeCLI.js.map

@@ -1 +1 @@
-{"version":3,"file":"claudeCodeCLI.js","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iDAAsC;AAEtC,gDAA8E;AAC9E,4DAAiF;AAEjF,MAAa,oBAAoB;IAI/B,YAAY,aAAsB;QAHlB,SAAI,GAAG,aAAsB,CAAC;QAI5C,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE;gBACtC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,MAAc,EACd,YAAgC,EAChC,OAAuB;QAEvB,mDAAmD;QACnD,IAAI,CAAC;YACH,IAAA,mCAAwB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,qCAAqC;QACrC,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,cAAc,GAAG,QAAQ;qBAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,wBAAwB,GAAG,CAAC,IAAI,cAAc,GAAG,CAAC,OAAO,UAAU,CAAC;qBAC/E,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,UAAU,GAAG,GAAG,MAAM,GAAG,cAAc,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,gEAAgE;QAChE,0DAA0D;QAC1D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,QAAQ,EAAE,CAAC,mBAAmB,EAAE,UAAU,CAAC,EAAE;gBAC9D,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,aAAa,GAA0B,IAAI,CAAC;YAChD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,OAAO,IAAI,CAAC,CAAC,CAAC;gBACpE,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBAED,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,IAAI,aAAa,MAAM,EAAE,CAAC,CAAC,CAAC;oBACjF,OAAO;gBACT,CAAC;gBAED,gCAAgC;gBAChC,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,IAAA,0BAAe,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBACzD,OAAO,CAAC,YAAY,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBACD,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAjHD,oDAiHC"}
+{"version":3,"file":"claudeCodeCLI.js","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,iDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,gDAA+E;AAC/E,4DAAiF;AAOjF,MAAM,aAAa,GAAG,QAAQ,CAAC;AAE/B,MAAa,oBAAoB;IAK/B,YAAY,OAAqC;QAJjC,SAAI,GAAG,aAAsB,CAAC;QAK5C,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAClE,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE;gBACtC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,MAAc,EACd,YAAgC;QAEhC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAClC,MAAM,cAAc,GAAG,QAAQ;qBAC5B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC;oBAC5D,OAAO,OAAO,GAAG,CAAC,IAAI,WAAW,GAAG,KAAK,GAAG,CAAC,OAAO,UAAU,CAAC;gBACjE,CAAC,CAAC;qBACD,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,KAAK,CAAC,IAAI,CACR,0GAA0G,CAC3G,CAAC;QAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,MAAc,EACd,YAAgC,EAChC,OAAuB;QAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAE1D,uCAAuC;QACvC,MAAM,YAAY,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE/D,gFAAgF;QAChF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,yBAAyB,CAAC,CAAC;YAC1E,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAE/C,MAAM,OAAO,GAAG,QAAQ,QAAQ,sBAAsB,IAAI,CAAC,KAAK,yIAAyI,CAAC;YAC1M,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE;gBACxC,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;aAChC,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,aAAa,GAA0B,IAAI,CAAC;YAChD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,OAAO,IAAI,CAAC,CAAC,CAAC;gBACpE,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBAED,qBAAqB;gBACrB,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;gBAED,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,IAAI,aAAa,MAAM,EAAE,CAAC,CAAC,CAAC;oBACjF,OAAO;gBACT,CAAC;gBAED,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC9D,MAAM,YAAY,GAAG,IAAA,8BAAmB,EAAC,YAAY,EAAE,WAAW,CAAC,CAAC;oBACpE,OAAO,CAAC,YAAY,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBACD,qBAAqB;gBACrB,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;gBACD,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AApJD,oDAoJC"}

package/dist/adapters/copilotCLI.d.ts

@@ -1,15 +1,24 @@
 /**
  * GitHub Copilot CLI Adapter
  */
-import { CodeGenerationAdapter } from '../types';
+import { CodeGenerationAdapter } from "../types";
+export interface CopilotCLIAdapterOptions {
+    workspaceRoot?: string;
+    model?: string;
+}
 export declare class CopilotCLIAdapter implements CodeGenerationAdapter {
     readonly type: "copilot";
     private workspaceRoot;
-    constructor(workspaceRoot?: string);
+    private model;
+    constructor(options?: CopilotCLIAdapterOptions);
     /**
      * Check if GitHub Copilot CLI is available
      */
     checkAvailability(): Promise<boolean>;
+    /**
+     * Build the full prompt with context files
+     */
+    private buildPrompt;
     /**
      * Generate code using GitHub Copilot CLI
      * @param timeout Timeout in milliseconds, or null for no timeout

package/dist/adapters/copilotCLI.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"copilotCLI.d.ts","sourceRoot":"","sources":["../../src/adapters/copilotCLI.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAIjD,qBAAa,iBAAkB,YAAW,qBAAqB;IAC7D,SAAgB,IAAI,EAAG,SAAS,CAAU;IAC1C,OAAO,CAAC,aAAa,CAAS;gBAElB,aAAa,CAAC,EAAE,MAAM;IAIlC;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB3C;;;OAGG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;CA4ErB"}
+{"version":3,"file":"copilotCLI.d.ts","sourceRoot":"","sources":["../../src/adapters/copilotCLI.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAOjD,MAAM,WAAW,wBAAwB;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,qBAAa,iBAAkB,YAAW,qBAAqB;IAC7D,SAAgB,IAAI,EAAG,SAAS,CAAU;IAC1C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,KAAK,CAAS;gBAEV,OAAO,CAAC,EAAE,wBAAwB;IAK9C;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB3C;;OAEG;IACH,OAAO,CAAC,WAAW;IA8BnB;;;OAGG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;CAkFrB"}

package/dist/adapters/copilotCLI.js

@@ -2,99 +2,162 @@
 /**
  * GitHub Copilot CLI Adapter
  */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CopilotCLIAdapter = void 0;
 const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
 const gitUtils_1 = require("../utils/gitUtils");
 const workspaceUtils_1 = require("../utils/workspaceUtils");
+const DEFAULT_MODEL = "claude-sonnet-4.5";
 class CopilotCLIAdapter {
-    constructor(workspaceRoot) {
-        this.type = 'copilot';
-        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(workspaceRoot);
+    constructor(options) {
+        this.type = "copilot";
+        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(options?.workspaceRoot);
+        this.model = options?.model || DEFAULT_MODEL;
     }
     /**
      * Check if GitHub Copilot CLI is available
      */
     async checkAvailability() {
         return new Promise((resolve) => {
-            const proc = (0, child_process_1.spawn)('which', ['copilot'], {
-                stdio: 'pipe',
+            const proc = (0, child_process_1.spawn)("which", ["copilot"], {
+                stdio: "pipe",
             });
-            proc.on('close', (code) => {
+            proc.on("close", (code) => {
                 resolve(code === 0);
             });
-            proc.on('error', () => {
+            proc.on("error", () => {
                 resolve(false);
             });
         });
     }
     /**
-     * Generate code using GitHub Copilot CLI
-     * @param timeout Timeout in milliseconds, or null for no timeout
+     * Build the full prompt with context files
      */
-    async generate(prompt, contextFiles, timeout) {
-        // Reset workspace to clean state before generation
-        try {
-            (0, gitUtils_1.resetGitWorkingDirectory)(this.workspaceRoot);
-        }
-        catch (error) {
-            console.warn('Warning: Could not reset git working directory:', error);
-        }
-        // Build the full prompt with context
-        let fullPrompt = prompt;
+    buildPrompt(prompt, contextFiles) {
+        const parts = [];
         if (contextFiles && contextFiles.length > 0) {
             const contexts = (0, workspaceUtils_1.readContextFiles)(this.workspaceRoot, contextFiles);
             if (contexts.length > 0) {
-                const contextSection = contexts
-                    .map(ctx => `\n\n### Context from ${ctx.path}:\n\`\`\`\n${ctx.content}\n\`\`\``)
-                    .join('\n');
-                fullPrompt = `${prompt}${contextSection}`;
+                parts.push("# Reference Files\n");
+                const contextContent = contexts
+                    .map((ctx) => {
+                    const ext = path.extname(ctx.path).slice(1) || "typescript";
+                    return `### ${ctx.path}\n\`\`\`${ext}\n${ctx.content}\n\`\`\``;
+                })
+                    .join("\n\n");
+                parts.push(contextContent);
+                parts.push("\n---\n");
             }
         }
-        // Spawn the copilot CLI process
+        parts.push("# Task\n");
+        parts.push(prompt);
+        parts.push("\n\nCreate/update the necessary file(s). Do not output code to the terminal - write it to files instead.");
+        return parts.join("\n");
+    }
+    /**
+     * Generate code using GitHub Copilot CLI
+     * @param timeout Timeout in milliseconds, or null for no timeout
+     */
+    async generate(prompt, contextFiles, timeout) {
+        const fullPrompt = this.buildPrompt(prompt, contextFiles);
+        // Capture git status before generation
+        const statusBefore = (0, gitUtils_1.getGitStatusPorcelain)(this.workspaceRoot);
+        // Write prompt to temp file and pipe via stdin (matches @copilot-evals pattern)
         return new Promise((resolve, reject) => {
-            const proc = (0, child_process_1.spawn)('copilot', [fullPrompt], {
+            const tempFile = path.join(this.workspaceRoot, '.copilot-eval-prompt.txt');
+            fs.writeFileSync(tempFile, fullPrompt, "utf8");
+            const command = `cat "${tempFile}" | copilot --model ${this.model} --allow-all-tools --deny-tool 'shell(rm)' --deny-tool 'shell(git push)' --deny-tool 'shell(git commit)'`;
+            const proc = (0, child_process_1.spawn)("sh", ["-c", command], {
                 cwd: this.workspaceRoot,
-                stdio: ['pipe', 'pipe', 'pipe'],
-                shell: true,
+                stdio: ["pipe", "pipe", "pipe"],
             });
-            let stdout = '';
-            let stderr = '';
-            proc.stdout?.on('data', (data) => {
+            let stdout = "";
+            let stderr = "";
+            proc.stdout?.on("data", (data) => {
                 stdout += data.toString();
             });
-            proc.stderr?.on('data', (data) => {
+            proc.stderr?.on("data", (data) => {
                 stderr += data.toString();
             });
             // Set timeout only if specified (null/undefined = no timeout)
             let timeoutHandle = null;
             if (timeout !== null && timeout !== undefined) {
                 timeoutHandle = setTimeout(() => {
-                    proc.kill('SIGTERM');
+                    proc.kill("SIGTERM");
                     reject(new Error(`Copilot CLI timed out after ${timeout}ms`));
                 }, timeout);
             }
-            proc.on('close', (code) => {
+            proc.on("close", (code) => {
                 if (timeoutHandle) {
                     clearTimeout(timeoutHandle);
                 }
+                // Clean up temp file
+                try {
+                    fs.unlinkSync(tempFile);
+                }
+                catch {
+                    // Ignore cleanup errors
+                }
                 if (code !== 0) {
                     reject(new Error(`Copilot CLI exited with code ${code}\nStderr: ${stderr}`));
                     return;
                 }
-                // Get the list of changed files
+                // Get files changed during generation (diff before/after)
                 try {
-                    const changedFiles = (0, gitUtils_1.getChangedFiles)(this.workspaceRoot);
+                    const statusAfter = (0, gitUtils_1.getGitStatusPorcelain)(this.workspaceRoot);
+                    const changedFiles = (0, gitUtils_1.getChangedFilesDiff)(statusBefore, statusAfter);
                     resolve(changedFiles);
                 }
                 catch (error) {
                     reject(new Error(`Failed to get changed files: ${error}`));
                 }
             });
-            proc.on('error', (error) => {
+            proc.on("error", (error) => {
                 if (timeoutHandle) {
                     clearTimeout(timeoutHandle);
                 }
+                // Clean up temp file
+                try {
+                    fs.unlinkSync(tempFile);
+                }
+                catch {
+                    // Ignore cleanup errors
+                }
                 reject(new Error(`Failed to spawn Copilot CLI: ${error}`));
             });
         });

package/dist/adapters/copilotCLI.js.map

@@ -1 +1 @@
-{"version":3,"file":"copilotCLI.js","sourceRoot":"","sources":["../../src/adapters/copilotCLI.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iDAAsC;AAEtC,gDAA8E;AAC9E,4DAAiF;AAEjF,MAAa,iBAAiB;IAI5B,YAAY,aAAsB;QAHlB,SAAI,GAAG,SAAkB,CAAC;QAIxC,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE;gBACvC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,MAAc,EACd,YAAgC,EAChC,OAAuB;QAEvB,mDAAmD;QACnD,IAAI,CAAC;YACH,IAAA,mCAAwB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,qCAAqC;QACrC,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,cAAc,GAAG,QAAQ;qBAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,wBAAwB,GAAG,CAAC,IAAI,cAAc,GAAG,CAAC,OAAO,UAAU,CAAC;qBAC/E,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,UAAU,GAAG,GAAG,MAAM,GAAG,cAAc,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,SAAS,EAAE,CAAC,UAAU,CAAC,EAAE;gBAC1C,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,aAAa,GAA0B,IAAI,CAAC;YAChD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,OAAO,IAAI,CAAC,CAAC,CAAC;gBAChE,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBAED,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,IAAI,aAAa,MAAM,EAAE,CAAC,CAAC,CAAC;oBAC7E,OAAO;gBACT,CAAC;gBAED,gCAAgC;gBAChC,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,IAAA,0BAAe,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBACzD,OAAO,CAAC,YAAY,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBACD,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AA/GD,8CA+GC"}
+{"version":3,"file":"copilotCLI.js","sourceRoot":"","sources":["../../src/adapters/copilotCLI.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,iDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,gDAA+E;AAC/E,4DAGiC;AAOjC,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAE1C,MAAa,iBAAiB;IAK5B,YAAY,OAAkC;QAJ9B,SAAI,GAAG,SAAkB,CAAC;QAKxC,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAClE,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE;gBACvC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,MAAc,EACd,YAAgC;QAEhC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAClC,MAAM,cAAc,GAAG,QAAQ;qBAC5B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC;oBAC5D,OAAO,OAAO,GAAG,CAAC,IAAI,WAAW,GAAG,KAAK,GAAG,CAAC,OAAO,UAAU,CAAC;gBACjE,CAAC,CAAC;qBACD,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,KAAK,CAAC,IAAI,CACR,0GAA0G,CAC3G,CAAC;QAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,MAAc,EACd,YAAgC,EAChC,OAAuB;QAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAE1D,uCAAuC;QACvC,MAAM,YAAY,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE/D,gFAAgF;QAChF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;YAC3E,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAE/C,MAAM,OAAO,GAAG,QAAQ,QAAQ,uBAAuB,IAAI,CAAC,KAAK,0GAA0G,CAAC;YAC5K,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE;gBACxC,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;aAChC,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,aAAa,GAA0B,IAAI,CAAC;YAChD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,OAAO,IAAI,CAAC,CAAC,CAAC;gBAChE,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBAED,qBAAqB;gBACrB,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;gBAED,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CACJ,IAAI,KAAK,CACP,gCAAgC,IAAI,aAAa,MAAM,EAAE,CAC1D,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBAED,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC9D,MAAM,YAAY,GAAG,IAAA,8BAAmB,EAAC,YAAY,EAAE,WAAW,CAAC,CAAC;oBACpE,OAAO,CAAC,YAAY,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBACD,qBAAqB;gBACrB,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAC1B,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;gBACD,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAxJD,8CAwJC"}

package/dist/evaluator.d.ts

@@ -1,7 +1,20 @@
 /**
  * Main evaluation engine
  */
-import { AdapterType, TestScenario, EvaluationResult, EvaluationReport } from './types';
+import { EventEmitter } from "events";
+import { AdapterType, TestScenario, EvaluationResult, EvaluationReport } from "./types";
+/**
+ * Events emitted by the Evaluator during execution
+ */
+export interface EvaluatorEvents {
+    "evaluation:start": (scenarios: TestScenario[]) => void;
+    "scenario:start": (scenarioId: string, scenario: TestScenario) => void;
+    "scenario:generating": (scenarioId: string) => void;
+    "scenario:validating": (scenarioId: string) => void;
+    "scenario:complete": (scenarioId: string, result: EvaluationResult) => void;
+    "evaluation:complete": (report: EvaluationReport) => void;
+    log: (message: string) => void;
+}
 export interface EvaluatorOptions {
     adapter: AdapterType;
     model?: string;
@@ -11,12 +24,17 @@ export interface EvaluatorOptions {
     saveBaseline?: boolean;
     compareBaseline?: boolean;
 }
-export declare class Evaluator {
+export declare class Evaluator extends EventEmitter {
     private adapter;
     private workspaceRoot;
     private baselineManager;
     private options;
     constructor(options: EvaluatorOptions);
+    /**
+     * Type-safe event emitter methods
+     */
+    on<K extends keyof EvaluatorEvents>(event: K, listener: EvaluatorEvents[K]): this;
+    emit<K extends keyof EvaluatorEvents>(event: K, ...args: Parameters<EvaluatorEvents[K]>): boolean;
     /**
      * Create adapter instance based on type
      */

package/dist/evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,WAAW,EAEX,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAEjB,MAAM,SAAS,CAAC;AASjB,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,OAAO,CAAmB;gBAEtB,OAAO,EAAE,gBAAgB;IASrC;;OAEG;IACH,OAAO,CAAC,aAAa;IAWrB;;OAEG;IACG,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC;IAIlD;;OAEG;IACH,eAAe,CACb,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,EAAE;QACP,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KACjB,GACA,YAAY,EAAE;IA0BjB;;OAEG;IACG,gBAAgB,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA8IzE;;OAEG;IACG,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAuDrE"}
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EACL,WAAW,EAEX,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAEjB,MAAM,SAAS,CAAC;AASjB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,kBAAkB,EAAE,CAAC,SAAS,EAAE,YAAY,EAAE,KAAK,IAAI,CAAC;IACxD,gBAAgB,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;IACvE,qBAAqB,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,qBAAqB,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,IAAI,CAAC;IACpD,mBAAmB,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5E,qBAAqB,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC1D,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,SAAU,SAAQ,YAAY;IACzC,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,OAAO,CAAmB;gBAEtB,OAAO,EAAE,gBAAgB;IAUrC;;OAEG;IACM,EAAE,CAAC,CAAC,SAAS,MAAM,eAAe,EACzC,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC,GAC3B,IAAI;IAIE,IAAI,CAAC,CAAC,SAAS,MAAM,eAAe,EAC3C,KAAK,EAAE,CAAC,EACR,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,GACtC,OAAO;IAIV;;OAEG;IACH,OAAO,CAAC,aAAa;IAiBrB;;OAEG;IACG,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC;IAIlD;;OAEG;IACH,eAAe,CACb,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,EAAE;QACP,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KACjB,GACA,YAAY,EAAE;IA0BjB;;OAEG;IACG,gBAAgB,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6JzE;;OAEG;IACG,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAsDrE"}