coding-agent-benchmarks 0.1.0

package/dist/adapters/copilotCLI.js

@@ -0,0 +1,104 @@
+"use strict";
+/**
+ * GitHub Copilot CLI Adapter
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CopilotCLIAdapter = void 0;
+const child_process_1 = require("child_process");
+const gitUtils_1 = require("../utils/gitUtils");
+const workspaceUtils_1 = require("../utils/workspaceUtils");
+class CopilotCLIAdapter {
+    constructor(workspaceRoot) {
+        this.type = 'copilot';
+        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(workspaceRoot);
+    }
+    /**
+     * Check if GitHub Copilot CLI is available
+     */
+    async checkAvailability() {
+        return new Promise((resolve) => {
+            const proc = (0, child_process_1.spawn)('which', ['copilot'], {
+                stdio: 'pipe',
+            });
+            proc.on('close', (code) => {
+                resolve(code === 0);
+            });
+            proc.on('error', () => {
+                resolve(false);
+            });
+        });
+    }
+    /**
+     * Generate code using GitHub Copilot CLI
+     * @param timeout Timeout in milliseconds, or null for no timeout
+     */
+    async generate(prompt, contextFiles, timeout) {
+        // Reset workspace to clean state before generation
+        try {
+            (0, gitUtils_1.resetGitWorkingDirectory)(this.workspaceRoot);
+        }
+        catch (error) {
+            console.warn('Warning: Could not reset git working directory:', error);
+        }
+        // Build the full prompt with context
+        let fullPrompt = prompt;
+        if (contextFiles && contextFiles.length > 0) {
+            const contexts = (0, workspaceUtils_1.readContextFiles)(this.workspaceRoot, contextFiles);
+            if (contexts.length > 0) {
+                const contextSection = contexts
+                    .map(ctx => `\n\n### Context from ${ctx.path}:\n\`\`\`\n${ctx.content}\n\`\`\``)
+                    .join('\n');
+                fullPrompt = `${prompt}${contextSection}`;
+            }
+        }
+        // Spawn the copilot CLI process
+        return new Promise((resolve, reject) => {
+            const proc = (0, child_process_1.spawn)('copilot', [fullPrompt], {
+                cwd: this.workspaceRoot,
+                stdio: ['pipe', 'pipe', 'pipe'],
+                shell: true,
+            });
+            let stdout = '';
+            let stderr = '';
+            proc.stdout?.on('data', (data) => {
+                stdout += data.toString();
+            });
+            proc.stderr?.on('data', (data) => {
+                stderr += data.toString();
+            });
+            // Set timeout only if specified (null/undefined = no timeout)
+            let timeoutHandle = null;
+            if (timeout !== null && timeout !== undefined) {
+                timeoutHandle = setTimeout(() => {
+                    proc.kill('SIGTERM');
+                    reject(new Error(`Copilot CLI timed out after ${timeout}ms`));
+                }, timeout);
+            }
+            proc.on('close', (code) => {
+                if (timeoutHandle) {
+                    clearTimeout(timeoutHandle);
+                }
+                if (code !== 0) {
+                    reject(new Error(`Copilot CLI exited with code ${code}\nStderr: ${stderr}`));
+                    return;
+                }
+                // Get the list of changed files
+                try {
+                    const changedFiles = (0, gitUtils_1.getChangedFiles)(this.workspaceRoot);
+                    resolve(changedFiles);
+                }
+                catch (error) {
+                    reject(new Error(`Failed to get changed files: ${error}`));
+                }
+            });
+            proc.on('error', (error) => {
+                if (timeoutHandle) {
+                    clearTimeout(timeoutHandle);
+                }
+                reject(new Error(`Failed to spawn Copilot CLI: ${error}`));
+            });
+        });
+    }
+}
+exports.CopilotCLIAdapter = CopilotCLIAdapter;
+//# sourceMappingURL=copilotCLI.js.map

package/dist/adapters/copilotCLI.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"copilotCLI.js","sourceRoot":"","sources":["../../src/adapters/copilotCLI.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH,iDAAsC;AAEtC,gDAA8E;AAC9E,4DAAiF;AAEjF,MAAa,iBAAiB;IAI5B,YAAY,aAAsB;QAHlB,SAAI,GAAG,SAAkB,CAAC;QAIxC,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE;gBACvC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,MAAc,EACd,YAAgC,EAChC,OAAuB;QAEvB,mDAAmD;QACnD,IAAI,CAAC;YACH,IAAA,mCAAwB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,qCAAqC;QACrC,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,cAAc,GAAG,QAAQ;qBAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,wBAAwB,GAAG,CAAC,IAAI,cAAc,GAAG,CAAC,OAAO,UAAU,CAAC;qBAC/E,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,UAAU,GAAG,GAAG,MAAM,GAAG,cAAc,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,SAAS,EAAE,CAAC,UAAU,CAAC,EAAE;gBAC1C,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,aAAa,GAA0B,IAAI,CAAC;YAChD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,OAAO,IAAI,CAAC,CAAC,CAAC;gBAChE,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBAED,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,IAAI,aAAa,MAAM,EAAE,CAAC,CAAC,CAAC;oBAC7E,OAAO;gBACT,CAAC;gBAED,gCAAgC;gBAChC,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,IAAA,0BAAe,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBACzD,OAAO,CAAC,YAAY,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBACD,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AA/GD,8CA+GC"}

package/dist/config/defaultScenarios.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Default test scenarios
+ */
+import { TestScenario } from '../types';
+export declare const getDefaultScenarios: () => TestScenario[];
+//# sourceMappingURL=defaultScenarios.d.ts.map

package/dist/config/defaultScenarios.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"defaultScenarios.d.ts","sourceRoot":"","sources":["../../src/config/defaultScenarios.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,eAAO,MAAM,mBAAmB,QAAO,YAAY,EAyNhD,CAAC"}

package/dist/config/defaultScenarios.js

@@ -0,0 +1,209 @@
+"use strict";
+/**
+ * Default test scenarios
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDefaultScenarios = void 0;
+const getDefaultScenarios = () => [
+    {
+        id: 'typescript-no-any',
+        category: 'typescript',
+        severity: 'critical',
+        tags: ['typescript', 'types', 'safety'],
+        description: 'Ensure TypeScript interfaces use explicit types instead of "any"',
+        prompt: 'Create a TypeScript interface called User with fields: id (number), name (string), email (string), and metadata (object with key-value pairs)',
+        validationStrategy: {
+            patterns: {
+                forbiddenPatterns: [/:\s*any\b/, /:\s*any\s*[,;]/],
+                requiredPatterns: [/interface\s+User/],
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'typescript-readonly-props',
+        category: 'typescript',
+        severity: 'major',
+        tags: ['typescript', 'immutability', 'best-practices'],
+        description: 'Enforce readonly properties in TypeScript interfaces',
+        prompt: 'Create a TypeScript interface called AppConfig with readonly properties: apiUrl (string), timeout (number), and features (array of strings)',
+        validationStrategy: {
+            patterns: {
+                requiredPatterns: [
+                    /interface\s+AppConfig/,
+                    /readonly\s+apiUrl/,
+                    /readonly\s+timeout/,
+                    /readonly\s+features/,
+                ],
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'react-no-inline-styles',
+        category: 'react',
+        severity: 'major',
+        tags: ['react', 'styling', 'best-practices'],
+        description: 'Forbid inline style objects in React components',
+        prompt: 'Create a React functional component called Button that accepts a "label" prop and renders a styled button. Use CSS classes instead of inline styles.',
+        validationStrategy: {
+            patterns: {
+                forbiddenPatterns: [
+                    /style\s*=\s*\{\{/,
+                    /style\s*=\s*\{[^}]*\}/,
+                ],
+                requiredPatterns: [
+                    /function\s+Button|const\s+Button.*=|export\s+(default\s+)?function\s+Button/,
+                    /className/,
+                ],
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'react-proper-hooks',
+        category: 'react',
+        severity: 'critical',
+        tags: ['react', 'hooks', 'correctness'],
+        description: 'Ensure React hooks are not called conditionally',
+        prompt: 'Create a React component called UserProfile that uses useState to manage a "loading" state and useEffect to fetch user data. Make sure hooks are used correctly.',
+        validationStrategy: {
+            patterns: {
+                // Forbid hooks inside if statements or loops
+                forbiddenPatterns: [
+                    /if\s*\([^)]*\)\s*\{[^}]*use(State|Effect|Context|Reducer|Callback|Memo|Ref)/,
+                ],
+                requiredPatterns: [
+                    /useState/,
+                    /useEffect/,
+                ],
+            },
+            llmJudge: {
+                enabled: true,
+                judgmentPrompt: `Evaluate whether this React component follows the Rules of Hooks:
+1. Hooks must be called at the top level (not inside conditions, loops, or nested functions)
+2. Hooks must be called in the same order every render
+3. useState and useEffect should be used correctly
+
+Return JSON: {"passed": boolean, "score": 0-1, "reasoning": "explanation", "violations": []}`,
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'async-error-handling',
+        category: 'general',
+        severity: 'critical',
+        tags: ['async', 'error-handling', 'robustness'],
+        description: 'Ensure async functions have proper error handling',
+        prompt: 'Create an async function called fetchUserData that takes a userId parameter, makes an HTTP request to fetch user data, and returns the user object. Handle errors appropriately.',
+        validationStrategy: {
+            patterns: {
+                requiredPatterns: [
+                    /async\s+function\s+fetchUserData|const\s+fetchUserData.*async/,
+                    /try|catch|\.catch\(/,
+                ],
+            },
+            llmJudge: {
+                enabled: true,
+                judgmentPrompt: `Evaluate the error handling in this async function:
+1. Does it use try/catch or .catch()?
+2. Are errors logged or re-thrown appropriately?
+3. Does it prevent unhandled promise rejections?
+
+Return JSON: {"passed": boolean, "score": 0-1, "reasoning": "explanation", "violations": []}`,
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'typescript-explicit-return-types',
+        category: 'typescript',
+        severity: 'major',
+        tags: ['typescript', 'types', 'documentation'],
+        description: 'Functions should have explicit return type annotations',
+        prompt: 'Create a TypeScript function called calculateTotal that takes an array of numbers and returns their sum. Include explicit type annotations for parameters and return type.',
+        validationStrategy: {
+            patterns: {
+                requiredPatterns: [
+                    /function\s+calculateTotal.*:\s*number|const\s+calculateTotal.*:\s*\([^)]*\)\s*=>\s*number/,
+                ],
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'typescript-strict-null-safety',
+        category: 'typescript',
+        severity: 'critical',
+        tags: ['typescript', 'null-safety', 'types', 'best-practices'],
+        description: 'Proper null/undefined handling with optional chaining and type guards',
+        prompt: 'Create a TypeScript function called getUserEmail that takes a user object (which might be null or undefined) and returns their email address. The user object has an optional email field. Handle all null/undefined cases safely without using type assertions (as or !).',
+        validationStrategy: {
+            patterns: {
+                forbiddenPatterns: [
+                    /as\s+\w+/, // No type assertions
+                    /!\s*\./, // No non-null assertions
+                    /!\s*\[/, // No non-null assertions on arrays
+                ],
+                requiredPatterns: [
+                    /function\s+getUserEmail|const\s+getUserEmail/,
+                    /\?\.|if\s*\(.*\)/, // Optional chaining or null checks
+                ],
+            },
+            llmJudge: {
+                enabled: true,
+                judgmentPrompt: `Evaluate the null safety of this function:
+1. Does it handle null/undefined user objects safely?
+2. Does it handle undefined email fields?
+3. Does it avoid type assertions (as, !) and use proper type guards or optional chaining?
+4. Does it have a clear return type that reflects possible null/undefined?
+
+Return JSON: {"passed": boolean, "score": 0-1, "reasoning": "explanation", "violations": []}`,
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'no-console-logs-production',
+        category: 'general',
+        severity: 'minor',
+        tags: ['logging', 'production', 'cleanup'],
+        description: 'Production code should not contain console.log statements',
+        prompt: 'Create a utility function called processData that processes an array of items and returns the results. Include proper error handling but avoid console.log statements.',
+        validationStrategy: {
+            patterns: {
+                forbiddenPatterns: [/console\.log/],
+            },
+        },
+        timeout: 120000,
+    },
+    {
+        id: 'react-key-prop',
+        category: 'react',
+        severity: 'critical',
+        tags: ['react', 'lists', 'performance'],
+        description: 'React lists must include unique key props',
+        prompt: 'Create a React component called TodoList that renders a list of todo items from an array. Each todo has an id and text. Render them as list items.',
+        validationStrategy: {
+            patterns: {
+                requiredPatterns: [
+                    /key\s*=\s*\{/,
+                    /\.map\(/,
+                ],
+            },
+            llmJudge: {
+                enabled: true,
+                judgmentPrompt: `Evaluate if the React component properly uses key props:
+1. Are key props used when rendering lists?
+2. Are the keys unique and stable?
+3. Are keys based on item IDs rather than indexes?
+
+Return JSON: {"passed": boolean, "score": 0-1, "reasoning": "explanation", "violations": []}`,
+            },
+        },
+        timeout: 120000,
+    },
+];
+exports.getDefaultScenarios = getDefaultScenarios;
+//# sourceMappingURL=defaultScenarios.js.map

package/dist/config/defaultScenarios.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"defaultScenarios.js","sourceRoot":"","sources":["../../src/config/defaultScenarios.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAII,MAAM,mBAAmB,GAAG,GAAmB,EAAE,CAAC;IACrD;QACE,EAAE,EAAE,mBAAmB;QACvB,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,CAAC,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC;QACvC,WAAW,EAAE,kEAAkE;QAC/E,MAAM,EACJ,+IAA+I;QACjJ,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,iBAAiB,EAAE,CAAC,WAAW,EAAE,gBAAgB,CAAC;gBAClD,gBAAgB,EAAE,CAAC,kBAAkB,CAAC;aACvC;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,2BAA2B;QAC/B,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,OAAO;QACjB,IAAI,EAAE,CAAC,YAAY,EAAE,cAAc,EAAE,gBAAgB,CAAC;QACtD,WAAW,EAAE,sDAAsD;QACnE,MAAM,EACJ,6IAA6I;QAC/I,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,gBAAgB,EAAE;oBAChB,uBAAuB;oBACvB,mBAAmB;oBACnB,oBAAoB;oBACpB,qBAAqB;iBACtB;aACF;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,wBAAwB;QAC5B,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,OAAO;QACjB,IAAI,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,gBAAgB,CAAC;QAC5C,WAAW,EAAE,iDAAiD;QAC9D,MAAM,EACJ,sJAAsJ;QACxJ,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,iBAAiB,EAAE;oBACjB,kBAAkB;oBAClB,uBAAuB;iBACxB;gBACD,gBAAgB,EAAE;oBAChB,6EAA6E;oBAC7E,WAAW;iBACZ;aACF;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,oBAAoB;QACxB,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC;QACvC,WAAW,EAAE,iDAAiD;QAC9D,MAAM,EACJ,kKAAkK;QACpK,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,6CAA6C;gBAC7C,iBAAiB,EAAE;oBACjB,6EAA6E;iBAC9E;gBACD,gBAAgB,EAAE;oBAChB,UAAU;oBACV,WAAW;iBACZ;aACF;YACD,QAAQ,EAAE;gBACR,OAAO,EAAE,IAAI;gBACb,cAAc,EAAE;;;;;6FAKmE;aACpF;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,sBAAsB;QAC1B,QAAQ,EAAE,SAAS;QACnB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,YAAY,CAAC;QAC/C,WAAW,EAAE,mDAAmD;QAChE,MAAM,EACJ,kLAAkL;QACpL,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,gBAAgB,EAAE;oBAChB,+DAA+D;oBAC/D,qBAAqB;iBACtB;aACF;YACD,QAAQ,EAAE;gBACR,OAAO,EAAE,IAAI;gBACb,cAAc,EAAE;;;;;6FAKmE;aACpF;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,kCAAkC;QACtC,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,OAAO;QACjB,IAAI,EAAE,CAAC,YAAY,EAAE,OAAO,EAAE,eAAe,CAAC;QAC9C,WAAW,EAAE,wDAAwD;QACrE,MAAM,EACJ,4KAA4K;QAC9K,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,gBAAgB,EAAE;oBAChB,2FAA2F;iBAC5F;aACF;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,+BAA+B;QACnC,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,CAAC,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,gBAAgB,CAAC;QAC9D,WAAW,EAAE,uEAAuE;QACpF,MAAM,EACJ,4QAA4Q;QAC9Q,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,iBAAiB,EAAE;oBACjB,UAAU,EAAqB,qBAAqB;oBACpD,QAAQ,EAAuB,yBAAyB;oBACxD,QAAQ,EAAuB,mCAAmC;iBACnE;gBACD,gBAAgB,EAAE;oBAChB,8CAA8C;oBAC9C,kBAAkB,EAAY,mCAAmC;iBAClE;aACF;YACD,QAAQ,EAAE;gBACR,OAAO,EAAE,IAAI;gBACb,cAAc,EAAE;;;;;;6FAMmE;aACpF;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,4BAA4B;QAChC,QAAQ,EAAE,SAAS;QACnB,QAAQ,EAAE,OAAO;QACjB,IAAI,EAAE,CAAC,SAAS,EAAE,YAAY,EAAE,SAAS,CAAC;QAC1C,WAAW,EAAE,2DAA2D;QACxE,MAAM,EACJ,wKAAwK;QAC1K,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,iBAAiB,EAAE,CAAC,cAAc,CAAC;aACpC;SACF;QACD,OAAO,EAAE,MAAM;KAChB;IAED;QACE,EAAE,EAAE,gBAAgB;QACpB,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC;QACvC,WAAW,EAAE,2CAA2C;QACxD,MAAM,EACJ,oJAAoJ;QACtJ,kBAAkB,EAAE;YAClB,QAAQ,EAAE;gBACR,gBAAgB,EAAE;oBAChB,cAAc;oBACd,SAAS;iBACV;aACF;YACD,QAAQ,EAAE;gBACR,OAAO,EAAE,IAAI;gBACb,cAAc,EAAE;;;;;6FAKmE;aACpF;SACF;QACD,OAAO,EAAE,MAAM;KAChB;CACF,CAAC;AAzNS,QAAA,mBAAmB,uBAyN5B"}

package/dist/config/loader.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Configuration file loader
+ */
+import { BenchmarkConfig, TestScenario } from '../types';
+/**
+ * Load benchmark configuration
+ */
+export declare const loadConfig: (cwd?: string) => Promise<{
+    config: BenchmarkConfig;
+    scenarios: TestScenario[];
+    configPath: string | null;
+}>;
+//# sourceMappingURL=loader.d.ts.map

package/dist/config/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AA+FzD;;GAEG;AACH,eAAO,MAAM,UAAU,GAAU,MAAK,MAAsB,KAAG,OAAO,CAAC;IACrE,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAiCA,CAAC"}

package/dist/config/loader.js

@@ -0,0 +1,153 @@
+"use strict";
+/**
+ * Configuration file loader
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadConfig = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/**
+ * Find configuration file in project directory
+ */
+const findConfigFile = (cwd) => {
+    const possiblePaths = [
+        path.join(cwd, 'benchmarks.config.js'),
+        path.join(cwd, 'benchmarks.config.ts'),
+        path.join(cwd, 'benchmarks.config.mjs'),
+        path.join(cwd, 'benchmarks.config.cjs'),
+    ];
+    for (const configPath of possiblePaths) {
+        if (fs.existsSync(configPath)) {
+            return configPath;
+        }
+    }
+    const packageJsonPath = path.join(cwd, 'package.json');
+    if (fs.existsSync(packageJsonPath)) {
+        try {
+            const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
+            if (packageJson.benchmarks) {
+                return packageJsonPath;
+            }
+        }
+        catch {
+            // Ignore parse errors
+        }
+    }
+    return null;
+};
+/**
+ * Load configuration from a file
+ */
+const loadConfigFromFile = async (configPath) => {
+    const ext = path.extname(configPath);
+    // Load from package.json
+    if (configPath.endsWith('package.json')) {
+        const packageJson = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+        return packageJson.benchmarks;
+    }
+    // Load CommonJS or ESM
+    try {
+        // For .ts files, we'd need tsx or ts-node
+        if (ext === '.ts') {
+            console.warn('TypeScript config files require tsx to be installed. Falling back to default config.');
+            return {};
+        }
+        // Use dynamic import for .js, .mjs, .cjs
+        const configModule = await Promise.resolve(`${configPath}`).then(s => __importStar(require(s)));
+        return configModule.default || configModule;
+    }
+    catch (error) {
+        throw new Error(`Failed to load config from ${configPath}: ${error}`);
+    }
+};
+/**
+ * Validate configuration
+ */
+const validateConfig = (config) => {
+    if (config.scenarios) {
+        for (const scenario of config.scenarios) {
+            if (!scenario.id) {
+                throw new Error('Scenario missing required field: id');
+            }
+            if (!scenario.category) {
+                throw new Error(`Scenario ${scenario.id} missing required field: category`);
+            }
+            if (!scenario.severity) {
+                throw new Error(`Scenario ${scenario.id} missing required field: severity`);
+            }
+            if (!scenario.description) {
+                throw new Error(`Scenario ${scenario.id} missing required field: description`);
+            }
+            if (!scenario.prompt) {
+                throw new Error(`Scenario ${scenario.id} missing required field: prompt`);
+            }
+            if (!scenario.validationStrategy) {
+                throw new Error(`Scenario ${scenario.id} missing required field: validationStrategy`);
+            }
+        }
+    }
+};
+/**
+ * Load benchmark configuration
+ */
+const loadConfig = async (cwd = process.cwd()) => {
+    // Find config file
+    const configPath = findConfigFile(cwd);
+    if (!configPath) {
+        throw new Error('No config file found. Please create a benchmarks.config.js file with scenarios defined.');
+    }
+    let config = {};
+    // Load user config
+    try {
+        config = await loadConfigFromFile(configPath);
+        validateConfig(config);
+        console.log(`Loaded config from: ${configPath}`);
+    }
+    catch (error) {
+        throw new Error(`Failed to load config: ${error}`);
+    }
+    // Ensure scenarios are defined
+    if (!config.scenarios || config.scenarios.length === 0) {
+        throw new Error('No scenarios defined in config. Please add scenarios to your config file.');
+    }
+    return {
+        config,
+        scenarios: config.scenarios,
+        configPath,
+    };
+};
+exports.loadConfig = loadConfig;
+//# sourceMappingURL=loader.js.map

package/dist/config/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAG7B;;GAEG;AACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAiB,EAAE;IACpD,MAAM,aAAa,GAAG;QACpB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,uBAAuB,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,uBAAuB,CAAC;KACxC,CAAC;IAEF,KAAK,MAAM,UAAU,IAAI,aAAa,EAAE,CAAC;QACvC,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;IACvD,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC;YAC1E,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;gBAC3B,OAAO,eAAe,CAAC;YACzB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,kBAAkB,GAAG,KAAK,EAAE,UAAkB,EAA4B,EAAE;IAChF,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAErC,yBAAyB;IACzB,IAAI,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACrE,OAAO,WAAW,CAAC,UAA6B,CAAC;IACnD,CAAC;IAED,uBAAuB;IACvB,IAAI,CAAC;QACH,0CAA0C;QAC1C,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CACV,sFAAsF,CACvF,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,yCAAyC;QACzC,MAAM,YAAY,GAAG,yBAAa,UAAU,uCAAC,CAAC;QAC9C,OAAO,YAAY,CAAC,OAAO,IAAI,YAAY,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,KAAK,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,cAAc,GAAG,CAAC,MAAuB,EAAQ,EAAE;IACvD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACxC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACzD,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,CAAC,EAAE,mCAAmC,CAAC,CAAC;YAC9E,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,CAAC,EAAE,mCAAmC,CAAC,CAAC;YAC9E,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,CAAC,EAAE,sCAAsC,CAAC,CAAC;YACjF,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,CAAC,EAAE,iCAAiC,CAAC,CAAC;YAC5E,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CACb,YAAY,QAAQ,CAAC,EAAE,6CAA6C,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAGF;;GAEG;AACI,MAAM,UAAU,GAAG,KAAK,EAAE,MAAc,OAAO,CAAC,GAAG,EAAE,EAIzD,EAAE;IACH,mBAAmB;IACnB,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAEvC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,GAAoB,EAAE,CAAC;IAEjC,mBAAmB;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC9C,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,uBAAuB,UAAU,EAAE,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,+BAA+B;IAC/B,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM;QACN,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU;KACX,CAAC;AACJ,CAAC,CAAC;AArCW,QAAA,UAAU,cAqCrB"}

package/dist/evaluator.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Main evaluation engine
+ */
+import { AdapterType, TestScenario, EvaluationResult, EvaluationReport } from './types';
+export interface EvaluatorOptions {
+    adapter: AdapterType;
+    model?: string;
+    workspaceRoot?: string;
+    defaultTimeout?: number | null;
+    verbose?: boolean;
+    saveBaseline?: boolean;
+    compareBaseline?: boolean;
+}
+export declare class Evaluator {
+    private adapter;
+    private workspaceRoot;
+    private baselineManager;
+    private options;
+    constructor(options: EvaluatorOptions);
+    /**
+     * Create adapter instance based on type
+     */
+    private createAdapter;
+    /**
+     * Check if adapter is available
+     */
+    checkAdapterAvailability(): Promise<boolean>;
+    /**
+     * Filter scenarios based on criteria
+     */
+    filterScenarios(scenarios: TestScenario[], filters: {
+        scenarioPattern?: string;
+        category?: string;
+        tags?: string[];
+    }): TestScenario[];
+    /**
+     * Evaluate a single scenario
+     */
+    evaluateScenario(scenario: TestScenario): Promise<EvaluationResult>;
+    /**
+     * Evaluate multiple scenarios
+     */
+    evaluate(scenarios: TestScenario[]): Promise<EvaluationReport>;
+}
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,WAAW,EAEX,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAEjB,MAAM,SAAS,CAAC;AASjB,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,SAAS;IACpB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,OAAO,CAAmB;gBAEtB,OAAO,EAAE,gBAAgB;IASrC;;OAEG;IACH,OAAO,CAAC,aAAa;IAWrB;;OAEG;IACG,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC;IAIlD;;OAEG;IACH,eAAe,CACb,SAAS,EAAE,YAAY,EAAE,EACzB,OAAO,EAAE;QACP,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KACjB,GACA,YAAY,EAAE;IA0BjB;;OAEG;IACG,gBAAgB,CAAC,QAAQ,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA8IzE;;OAEG;IACG,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAuDrE"}