coding-agent-benchmarks 0.1.0

package/dist/validators/patternValidator.js

@@ -0,0 +1,233 @@
+"use strict";
+/**
+ * Pattern-based code validator
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PatternValidator = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const workspaceUtils_1 = require("../utils/workspaceUtils");
+class PatternValidator {
+    constructor(workspaceRoot) {
+        this.type = 'pattern';
+        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(workspaceRoot);
+    }
+    /**
+     * Validate generated code against pattern rules
+     */
+    async validate(files, scenario) {
+        const patterns = scenario.validationStrategy.patterns;
+        // If no pattern validation configured, skip
+        if (!patterns) {
+            return {
+                passed: true,
+                score: -1,
+                violations: [],
+                validatorType: 'pattern',
+            };
+        }
+        const violations = [];
+        const absolutePaths = (0, workspaceUtils_1.resolveFilePaths)(this.workspaceRoot, files);
+        // Validate each file
+        for (const filePath of absolutePaths) {
+            if (!fs.existsSync(filePath)) {
+                continue;
+            }
+            try {
+                const content = fs.readFileSync(filePath, 'utf-8');
+                const relativePath = path.relative(this.workspaceRoot, filePath);
+                const fileName = path.basename(filePath);
+                // Check forbidden patterns
+                if (patterns.forbiddenPatterns) {
+                    for (const pattern of patterns.forbiddenPatterns) {
+                        const matches = this.findPatternMatches(content, pattern);
+                        for (const match of matches) {
+                            violations.push({
+                                type: 'pattern',
+                                message: `Forbidden pattern found: ${pattern.source}`,
+                                file: relativePath,
+                                line: match.line,
+                                severity: scenario.severity,
+                                details: `Matched: "${match.text}"`,
+                            });
+                        }
+                    }
+                }
+                // Check required patterns
+                if (patterns.requiredPatterns) {
+                    for (const pattern of patterns.requiredPatterns) {
+                        if (!pattern.test(content)) {
+                            violations.push({
+                                type: 'pattern',
+                                message: `Required pattern not found: ${pattern.source}`,
+                                file: relativePath,
+                                severity: scenario.severity,
+                            });
+                        }
+                    }
+                }
+                // Check forbidden imports
+                if (patterns.forbiddenImports) {
+                    for (const importPattern of patterns.forbiddenImports) {
+                        if (content.includes(importPattern)) {
+                            const lineInfo = this.findLineWithText(content, importPattern);
+                            violations.push({
+                                type: 'pattern',
+                                message: `Forbidden import found: ${importPattern}`,
+                                file: relativePath,
+                                line: lineInfo?.line,
+                                severity: scenario.severity,
+                                details: lineInfo ? `Line: "${lineInfo.text}"` : undefined,
+                            });
+                        }
+                    }
+                }
+                // Check required imports
+                if (patterns.requiredImports) {
+                    for (const importPattern of patterns.requiredImports) {
+                        if (!content.includes(importPattern)) {
+                            violations.push({
+                                type: 'pattern',
+                                message: `Required import not found: ${importPattern}`,
+                                file: relativePath,
+                                severity: scenario.severity,
+                            });
+                        }
+                    }
+                }
+                // Check forbidden file name patterns
+                if (patterns.forbiddenFileNamePatterns) {
+                    for (const pattern of patterns.forbiddenFileNamePatterns) {
+                        if (pattern.test(fileName)) {
+                            violations.push({
+                                type: 'pattern',
+                                message: `Forbidden file name pattern: ${pattern.source}`,
+                                file: relativePath,
+                                severity: scenario.severity,
+                                details: `File name: "${fileName}"`,
+                            });
+                        }
+                    }
+                }
+                // Check required file name patterns
+                if (patterns.requiredFileNamePatterns) {
+                    const hasRequiredFile = absolutePaths.some(p => patterns.requiredFileNamePatterns.some(pattern => pattern.test(path.basename(p))));
+                    if (!hasRequiredFile) {
+                        violations.push({
+                            type: 'pattern',
+                            message: `Required file name pattern not found: ${patterns.requiredFileNamePatterns.map(p => p.source).join(', ')}`,
+                            severity: scenario.severity,
+                        });
+                    }
+                }
+            }
+            catch (error) {
+                return {
+                    passed: false,
+                    score: 0,
+                    violations: [],
+                    validatorType: 'pattern',
+                    error: `Failed to read file ${filePath}: ${error}`,
+                };
+            }
+        }
+        // Calculate score based on violations
+        const passed = violations.length === 0;
+        const score = this.calculateScore(violations);
+        return {
+            passed,
+            score,
+            violations,
+            validatorType: 'pattern',
+        };
+    }
+    /**
+     * Find all matches of a pattern in text with line numbers
+     */
+    findPatternMatches(text, pattern) {
+        const lines = text.split('\n');
+        const matches = [];
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            if (pattern.test(line)) {
+                matches.push({
+                    line: i + 1,
+                    text: line.trim(),
+                });
+            }
+        }
+        return matches;
+    }
+    /**
+     * Find the line number containing specific text
+     */
+    findLineWithText(text, searchText) {
+        const lines = text.split('\n');
+        for (let i = 0; i < lines.length; i++) {
+            if (lines[i].includes(searchText)) {
+                return {
+                    line: i + 1,
+                    text: lines[i].trim(),
+                };
+            }
+        }
+        return null;
+    }
+    /**
+     * Calculate score based on number and severity of violations
+     * Critical violations have more weight than minor ones
+     */
+    calculateScore(violations) {
+        if (violations.length === 0) {
+            return 1.0;
+        }
+        // Weight violations by severity
+        const weights = {
+            critical: 1.0,
+            major: 0.7,
+            minor: 0.3,
+        };
+        const totalWeight = violations.reduce((sum, v) => {
+            return sum + weights[v.severity];
+        }, 0);
+        // Score decreases with more weighted violations
+        // Using exponential decay: score = e^(-totalWeight)
+        const score = Math.exp(-totalWeight);
+        return Math.max(0, Math.min(1, score));
+    }
+}
+exports.PatternValidator = PatternValidator;
+//# sourceMappingURL=patternValidator.js.map

package/dist/validators/patternValidator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"patternValidator.js","sourceRoot":"","sources":["../../src/validators/patternValidator.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAE7B,4DAAiF;AAEjF,MAAa,gBAAgB;IAI3B,YAAY,aAAsB;QAHlB,SAAI,GAAG,SAAkB,CAAC;QAIxC,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,KAAwB,EACxB,QAAsB;QAEtB,MAAM,QAAQ,GAAG,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC;QAEtD,4CAA4C;QAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC;gBACT,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,SAAS;aACzB,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAgB,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAElE,qBAAqB;QACrB,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACnD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAEzC,2BAA2B;gBAC3B,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;oBAC/B,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;wBACjD,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;wBAC1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;4BAC5B,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,SAAS;gCACf,OAAO,EAAE,4BAA4B,OAAO,CAAC,MAAM,EAAE;gCACrD,IAAI,EAAE,YAAY;gCAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gCAChB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gCAC3B,OAAO,EAAE,aAAa,KAAK,CAAC,IAAI,GAAG;6BACpC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,0BAA0B;gBAC1B,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;oBAC9B,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;wBAChD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,SAAS;gCACf,OAAO,EAAE,+BAA+B,OAAO,CAAC,MAAM,EAAE;gCACxD,IAAI,EAAE,YAAY;gCAClB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;6BAC5B,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,0BAA0B;gBAC1B,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;oBAC9B,KAAK,MAAM,aAAa,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;wBACtD,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;4BACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;4BAC/D,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,SAAS;gCACf,OAAO,EAAE,2BAA2B,aAAa,EAAE;gCACnD,IAAI,EAAE,YAAY;gCAClB,IAAI,EAAE,QAAQ,EAAE,IAAI;gCACpB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gCAC3B,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS;6BAC3D,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,yBAAyB;gBACzB,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;oBAC7B,KAAK,MAAM,aAAa,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC;wBACrD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;4BACrC,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,SAAS;gCACf,OAAO,EAAE,8BAA8B,aAAa,EAAE;gCACtD,IAAI,EAAE,YAAY;gCAClB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;6BAC5B,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,CAAC,yBAAyB,EAAE,CAAC;oBACvC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,yBAAyB,EAAE,CAAC;wBACzD,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;4BAC3B,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,SAAS;gCACf,OAAO,EAAE,gCAAgC,OAAO,CAAC,MAAM,EAAE;gCACzD,IAAI,EAAE,YAAY;gCAClB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gCAC3B,OAAO,EAAE,eAAe,QAAQ,GAAG;6BACpC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,oCAAoC;gBACpC,IAAI,QAAQ,CAAC,wBAAwB,EAAE,CAAC;oBACtC,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAC7C,QAAQ,CAAC,wBAAyB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAChD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAC/B,CACF,CAAC;oBACF,IAAI,CAAC,eAAe,EAAE,CAAC;wBACrB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,SAAS;4BACf,OAAO,EAAE,yCAAyC,QAAQ,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;4BACnH,QAAQ,EAAE,QAAQ,CAAC,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,EAAE;oBACd,aAAa,EAAE,SAAS;oBACxB,KAAK,EAAE,uBAAuB,QAAQ,KAAK,KAAK,EAAE;iBACnD,CAAC;YACJ,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAE9C,OAAO;YACL,MAAM;YACN,KAAK;YACL,UAAU;YACV,aAAa,EAAE,SAAS;SACzB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,kBAAkB,CACxB,IAAY,EACZ,OAAe;QAEf,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,OAAO,GAA0C,EAAE,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,CAAC,GAAG,CAAC;oBACX,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;iBAClB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,gBAAgB,CACtB,IAAY,EACZ,UAAkB;QAElB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClC,OAAO;oBACL,IAAI,EAAE,CAAC,GAAG,CAAC;oBACX,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;iBACtB,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,cAAc,CAAC,UAAuB;QAC5C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,CAAC;QACb,CAAC;QAED,gCAAgC;QAChC,MAAM,OAAO,GAAG;YACd,QAAQ,EAAE,GAAG;YACb,KAAK,EAAE,GAAG;YACV,KAAK,EAAE,GAAG;SACX,CAAC;QAEF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YAC/C,OAAO,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,gDAAgD;QAChD,oDAAoD;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;QAErC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;CACF;AAhOD,4CAgOC"}

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "coding-agent-benchmarks",
+  "version": "0.1.0",
+  "description": "Open-source framework for evaluating coding agents (GitHub Copilot CLI, Claude Code, etc.) against coding standards and best practices",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "coding-agent-benchmarks": "dist/runner.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/runner.ts",
+    "prepublishOnly": "npm run build",
+    "test": "echo \"Tests not implemented yet\" && exit 0"
+  },
+  "keywords": [
+    "coding-agent",
+    "benchmark",
+    "evaluation",
+    "copilot",
+    "claude-code",
+    "ai",
+    "testing",
+    "code-quality"
+  ],
+  "author": "chiltepin",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/chiltepin/coding-agent-benchmarks"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "commander": "^13.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.9",
+    "eslint": "^9.39.2",
+    "prettier": "^3.8.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  }
+}