coding-agent-benchmarks 0.1.0

package/dist/validators/eslintValidator.js

@@ -0,0 +1,217 @@
+"use strict";
+/**
+ * ESLint validator for generated code
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ESLintValidator = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const workspaceUtils_1 = require("../utils/workspaceUtils");
+const child_process_1 = require("child_process");
+class ESLintValidator {
+    constructor(workspaceRoot) {
+        this.type = 'eslint';
+        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(workspaceRoot);
+    }
+    /**
+     * Check if ESLint is available in the project
+     */
+    async checkESLintAvailability() {
+        try {
+            // Check if eslint is installed
+            (0, child_process_1.execSync)('npx eslint --version', {
+                cwd: this.workspaceRoot,
+                stdio: 'pipe',
+            });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Validate generated code using ESLint
+     */
+    async validate(files, scenario) {
+        const eslintConfig = scenario.validationStrategy.eslint;
+        // If ESLint not enabled, skip
+        if (!eslintConfig?.enabled) {
+            return {
+                passed: true,
+                score: -1,
+                violations: [],
+                validatorType: 'eslint',
+            };
+        }
+        // Check if ESLint is available
+        const isAvailable = await this.checkESLintAvailability();
+        if (!isAvailable) {
+            console.warn('ESLint not found in project, skipping ESLint validation');
+            return {
+                passed: true,
+                score: -1,
+                violations: [],
+                validatorType: 'eslint',
+                error: 'ESLint not found',
+            };
+        }
+        try {
+            const absolutePaths = (0, workspaceUtils_1.resolveFilePaths)(this.workspaceRoot, files);
+            const violations = [];
+            // Run ESLint on each file
+            for (const filePath of absolutePaths) {
+                if (!fs.existsSync(filePath)) {
+                    continue;
+                }
+                // Only lint JS/TS files
+                const ext = path.extname(filePath).toLowerCase();
+                if (!['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs'].includes(ext)) {
+                    continue;
+                }
+                try {
+                    // Run ESLint with JSON output
+                    const configArg = eslintConfig.configPath
+                        ? `--config ${eslintConfig.configPath}`
+                        : '';
+                    const output = (0, child_process_1.execSync)(`npx eslint ${configArg} --format json "${filePath}"`, {
+                        cwd: this.workspaceRoot,
+                        encoding: 'utf-8',
+                        stdio: 'pipe',
+                    });
+                    // Parse ESLint output
+                    const results = JSON.parse(output);
+                    const relativePath = path.relative(this.workspaceRoot, filePath);
+                    // Extract violations from ESLint results
+                    for (const result of results) {
+                        for (const message of result.messages || []) {
+                            const severity = message.severity === 2
+                                ? 'major'
+                                : message.severity === 1
+                                    ? 'minor'
+                                    : 'minor';
+                            violations.push({
+                                type: 'eslint',
+                                message: `${message.ruleId}: ${message.message}`,
+                                file: relativePath,
+                                line: message.line,
+                                severity: severity,
+                                details: `Column ${message.column}`,
+                            });
+                        }
+                    }
+                }
+                catch (error) {
+                    // ESLint exits with non-zero code if there are errors
+                    // Try to parse the error output
+                    if (error.stdout) {
+                        try {
+                            const results = JSON.parse(error.stdout);
+                            const relativePath = path.relative(this.workspaceRoot, filePath);
+                            for (const result of results) {
+                                for (const message of result.messages || []) {
+                                    const severity = message.severity === 2
+                                        ? 'major'
+                                        : message.severity === 1
+                                            ? 'minor'
+                                            : 'minor';
+                                    violations.push({
+                                        type: 'eslint',
+                                        message: `${message.ruleId}: ${message.message}`,
+                                        file: relativePath,
+                                        line: message.line,
+                                        severity: severity,
+                                        details: `Column ${message.column}`,
+                                    });
+                                }
+                            }
+                        }
+                        catch {
+                            // If we can't parse the output, treat it as a general error
+                            violations.push({
+                                type: 'eslint',
+                                message: `ESLint failed for ${path.basename(filePath)}`,
+                                file: path.relative(this.workspaceRoot, filePath),
+                                severity: scenario.severity,
+                                details: error.message,
+                            });
+                        }
+                    }
+                }
+            }
+            // Calculate score based on violations
+            const passed = violations.length === 0;
+            const score = this.calculateScore(violations);
+            return {
+                passed,
+                score,
+                violations,
+                validatorType: 'eslint',
+            };
+        }
+        catch (error) {
+            return {
+                passed: false,
+                score: 0,
+                violations: [],
+                validatorType: 'eslint',
+                error: `ESLint validation failed: ${error}`,
+            };
+        }
+    }
+    /**
+     * Calculate score based on number and severity of violations
+     */
+    calculateScore(violations) {
+        if (violations.length === 0) {
+            return 1.0;
+        }
+        // Weight violations by severity
+        const weights = {
+            critical: 1.0,
+            major: 0.7,
+            minor: 0.3,
+        };
+        const totalWeight = violations.reduce((sum, v) => {
+            return sum + weights[v.severity];
+        }, 0);
+        // Score decreases with more weighted violations
+        // Using exponential decay: score = e^(-totalWeight/2)
+        const score = Math.exp(-totalWeight / 2);
+        return Math.max(0, Math.min(1, score));
+    }
+}
+exports.ESLintValidator = ESLintValidator;
+//# sourceMappingURL=eslintValidator.js.map

package/dist/validators/eslintValidator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eslintValidator.js","sourceRoot":"","sources":["../../src/validators/eslintValidator.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAE7B,4DAAiF;AACjF,iDAAyC;AAEzC,MAAa,eAAe;IAI1B,YAAY,aAAsB;QAHlB,SAAI,GAAG,QAAiB,CAAC;QAIvC,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB;QACnC,IAAI,CAAC;YACH,+BAA+B;YAC/B,IAAA,wBAAQ,EAAC,sBAAsB,EAAE;gBAC/B,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,KAAwB,EACxB,QAAsB;QAEtB,MAAM,YAAY,GAAG,QAAQ,CAAC,kBAAkB,CAAC,MAAM,CAAC;QAExD,8BAA8B;QAC9B,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;YAC3B,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC;gBACT,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,QAAQ;aACxB,CAAC;QACJ,CAAC;QAED,+BAA+B;QAC/B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACzD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;YACxE,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC;gBACT,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,QAAQ;gBACvB,KAAK,EAAE,kBAAkB;aAC1B,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAClE,MAAM,UAAU,GAAgB,EAAE,CAAC;YAEnC,0BAA0B;YAC1B,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,wBAAwB;gBACxB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjD,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClE,SAAS;gBACX,CAAC;gBAED,IAAI,CAAC;oBACH,8BAA8B;oBAC9B,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU;wBACvC,CAAC,CAAC,YAAY,YAAY,CAAC,UAAU,EAAE;wBACvC,CAAC,CAAC,EAAE,CAAC;oBAEP,MAAM,MAAM,GAAG,IAAA,wBAAQ,EACrB,cAAc,SAAS,mBAAmB,QAAQ,GAAG,EACrD;wBACE,GAAG,EAAE,IAAI,CAAC,aAAa;wBACvB,QAAQ,EAAE,OAAO;wBACjB,KAAK,EAAE,MAAM;qBACd,CACF,CAAC;oBAEF,sBAAsB;oBACtB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;oBAEjE,yCAAyC;oBACzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;4BAC5C,MAAM,QAAQ,GACZ,OAAO,CAAC,QAAQ,KAAK,CAAC;gCACpB,CAAC,CAAC,OAAO;gCACT,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC;oCACtB,CAAC,CAAC,OAAO;oCACT,CAAC,CAAC,OAAO,CAAC;4BAEhB,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,QAAQ;gCACd,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE;gCAChD,IAAI,EAAE,YAAY;gCAClB,IAAI,EAAE,OAAO,CAAC,IAAI;gCAClB,QAAQ,EAAE,QAA6B;gCACvC,OAAO,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;6BACpC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,sDAAsD;oBACtD,gCAAgC;oBAChC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;wBACjB,IAAI,CAAC;4BACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;4BACzC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;4BAEjE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gCAC7B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oCAC5C,MAAM,QAAQ,GACZ,OAAO,CAAC,QAAQ,KAAK,CAAC;wCACpB,CAAC,CAAC,OAAO;wCACT,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC;4CACtB,CAAC,CAAC,OAAO;4CACT,CAAC,CAAC,OAAO,CAAC;oCAEhB,UAAU,CAAC,IAAI,CAAC;wCACd,IAAI,EAAE,QAAQ;wCACd,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE;wCAChD,IAAI,EAAE,YAAY;wCAClB,IAAI,EAAE,OAAO,CAAC,IAAI;wCAClB,QAAQ,EAAE,QAA6B;wCACvC,OAAO,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;qCACpC,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;wBACH,CAAC;wBAAC,MAAM,CAAC;4BACP,4DAA4D;4BAC5D,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,QAAQ;gCACd,OAAO,EAAE,qBAAqB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gCACvD,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;gCACjD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gCAC3B,OAAO,EAAE,KAAK,CAAC,OAAO;6BACvB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,sCAAsC;YACtC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YAE9C,OAAO;gBACL,MAAM;gBACN,KAAK;gBACL,UAAU;gBACV,aAAa,EAAE,QAAQ;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,QAAQ;gBACvB,KAAK,EAAE,6BAA6B,KAAK,EAAE;aAC5C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,UAAuB;QAC5C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,CAAC;QACb,CAAC;QAED,gCAAgC;QAChC,MAAM,OAAO,GAAG;YACd,QAAQ,EAAE,GAAG;YACb,KAAK,EAAE,GAAG;YACV,KAAK,EAAE,GAAG;SACX,CAAC;QAEF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YAC/C,OAAO,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,gDAAgD;QAChD,sDAAsD;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;QAEzC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;CACF;AAtMD,0CAsMC"}

package/dist/validators/llmJudge.d.ts

@@ -0,0 +1,28 @@
+/**
+ * LLM-as-Judge validator using GitHub Models API
+ */
+import { CodeValidator, TestScenario, ValidationResult } from '../types';
+export declare class LLMJudgeValidator implements CodeValidator {
+    readonly type: "llm-judge";
+    private workspaceRoot;
+    private apiToken;
+    private defaultModel;
+    constructor(workspaceRoot?: string, model?: string);
+    /**
+     * Validate generated code using LLM judgment
+     */
+    validate(files: readonly string[], scenario: TestScenario): Promise<ValidationResult>;
+    /**
+     * Build the judgment prompt for the LLM
+     */
+    private buildJudgmentPrompt;
+    /**
+     * Call the GitHub Models API (or other LLM API)
+     */
+    private callLLMAPI;
+    /**
+     * Test the LLM judge with a sample prompt (for debugging)
+     */
+    testJudge(prompt: string, model?: string): Promise<string>;
+}
+//# sourceMappingURL=llmJudge.d.ts.map

package/dist/validators/llmJudge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmJudge.d.ts","sourceRoot":"","sources":["../../src/validators/llmJudge.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAa,MAAM,UAAU,CAAC;AAkDpF,qBAAa,iBAAkB,YAAW,aAAa;IACrD,SAAgB,IAAI,EAAG,WAAW,CAAU;IAC5C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,YAAY,CAAS;gBAEjB,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,MAAyB;IAMpE;;OAEG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA+B3B;;OAEG;YACW,UAAU;IAqExB;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAYjE"}

package/dist/validators/llmJudge.js

@@ -0,0 +1,241 @@
+"use strict";
+/**
+ * LLM-as-Judge validator using GitHub Models API
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LLMJudgeValidator = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const workspaceUtils_1 = require("../utils/workspaceUtils");
+const githubAuth_1 = require("../utils/githubAuth");
+const judgeSystemPrompt = `You are a code review judge evaluating whether generated code follows specific coding guidelines.
+
+Your task is to evaluate the provided code against a set of criteria and return a JSON assessment.
+
+Be strict but fair. Only mark criteria as FAIL if there is a clear violation.
+
+Respond ONLY with valid JSON in this exact format:
+{
+  "evaluations": [
+    {
+      "criterion": "criterion text",
+      "result": "PASS" | "FAIL" | "N/A",
+      "explanation": "brief explanation"
+    }
+  ],
+  "overallScore": 0.0 to 1.0,
+  "summary": "one sentence summary"
+}`;
+class LLMJudgeValidator {
+    constructor(workspaceRoot, model = 'openai/gpt-4.1') {
+        this.type = 'llm-judge';
+        this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(workspaceRoot);
+        this.apiToken = (0, githubAuth_1.getGitHubToken)(); // Auto-detect from env or GitHub CLI
+        this.defaultModel = model;
+    }
+    /**
+     * Validate generated code using LLM judgment
+     */
+    async validate(files, scenario) {
+        const llmConfig = scenario.validationStrategy.llmJudge;
+        // If LLM judge not enabled, skip
+        if (!llmConfig?.enabled) {
+            return {
+                passed: true,
+                score: -1,
+                violations: [],
+                validatorType: 'llm-judge',
+            };
+        }
+        // If no API token, skip
+        if (!this.apiToken) {
+            console.warn('GITHUB_TOKEN not found, skipping LLM judge validation');
+            return {
+                passed: true,
+                score: -1,
+                violations: [],
+                validatorType: 'llm-judge',
+                error: 'GITHUB_TOKEN not found',
+            };
+        }
+        try {
+            // Read all generated files
+            const absolutePaths = (0, workspaceUtils_1.resolveFilePaths)(this.workspaceRoot, files);
+            const fileContents = [];
+            for (const filePath of absolutePaths) {
+                if (!fs.existsSync(filePath)) {
+                    continue;
+                }
+                const content = fs.readFileSync(filePath, 'utf-8');
+                const relativePath = path.relative(this.workspaceRoot, filePath);
+                fileContents.push({ path: relativePath, content });
+            }
+            // Build judgment prompt
+            const judgmentPrompt = this.buildJudgmentPrompt(scenario, fileContents, llmConfig.judgmentPrompt);
+            // Call LLM API
+            const model = llmConfig.model || this.defaultModel;
+            const judgment = await this.callLLMAPI(judgmentPrompt, model);
+            // Convert judgment to violations
+            const violations = (judgment.violations ?? []).map(v => ({
+                type: 'llm-judge',
+                message: v.message,
+                file: v.file,
+                line: v.line,
+                severity: scenario.severity,
+                details: judgment.reasoning,
+            }));
+            return {
+                passed: judgment.passed,
+                score: judgment.score,
+                violations,
+                validatorType: 'llm-judge',
+            };
+        }
+        catch (error) {
+            return {
+                passed: false,
+                score: 0,
+                violations: [],
+                validatorType: 'llm-judge',
+                error: `LLM judge failed: ${error}`,
+            };
+        }
+    }
+    /**
+     * Build the judgment prompt for the LLM
+     */
+    buildJudgmentPrompt(scenario, fileContents, customPrompt) {
+        if (customPrompt) {
+            return customPrompt;
+        }
+        const filesSection = fileContents
+            .map(f => `### ${f.path}\n\`\`\`\n${f.content}\n\`\`\``)
+            .join('\n\n');
+        return `# Task Description
+${scenario.description}
+
+# Original Prompt Given to AI
+${scenario.prompt}
+
+# Generated Code
+${filesSection}
+
+# Evaluation Criteria
+Evaluate whether the generated code:
+1. Correctly implements the requirements from the prompt
+2. Follows best practices for ${scenario.category}
+3. Meets the quality standards for a ${scenario.severity} severity scenario
+
+Be strict but fair in your evaluation.`;
+    }
+    /**
+     * Call the GitHub Models API (or other LLM API)
+     */
+    async callLLMAPI(prompt, model) {
+        const apiUrl = 'https://models.github.ai/inference/chat/completions';
+        const response = await fetch(apiUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+            body: JSON.stringify({
+                model,
+                messages: [
+                    { role: 'system', content: judgeSystemPrompt },
+                    { role: 'user', content: prompt },
+                ],
+                temperature: 0,
+                response_format: { type: 'json_object' },
+            }),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`GitHub Models API error: ${response.status} ${errorText}`);
+        }
+        const data = (await response.json());
+        const content = data.choices[0]?.message?.content;
+        if (!content) {
+            throw new Error('No content in LLM response');
+        }
+        // Parse JSON response
+        try {
+            const apiResponse = JSON.parse(content);
+            // Validate API response structure
+            if (!Array.isArray(apiResponse.evaluations) ||
+                apiResponse.overallScore == null ||
+                apiResponse.summary == null) {
+                throw new Error('Invalid judgment structure');
+            }
+            // Transform API response to internal judgment format
+            // Extract violations from FAIL evaluations
+            const violations = apiResponse.evaluations
+                .filter(e => e.result === 'FAIL')
+                .map(e => ({
+                message: `${e.criterion}: ${e.explanation}`,
+            }));
+            // Determine if passed based on violations and score threshold
+            const passed = violations.length === 0 && apiResponse.overallScore >= 0.7;
+            const judgment = {
+                passed,
+                score: Math.max(0, Math.min(1, apiResponse.overallScore)), // Ensure score is in valid range
+                reasoning: apiResponse.summary,
+                violations,
+            };
+            return judgment;
+        }
+        catch (error) {
+            throw new Error(`Failed to parse LLM response: ${error}\nContent: ${content}`);
+        }
+    }
+    /**
+     * Test the LLM judge with a sample prompt (for debugging)
+     */
+    async testJudge(prompt, model) {
+        if (!this.apiToken) {
+            return 'Error: GITHUB_TOKEN not found';
+        }
+        try {
+            const result = await this.callLLMAPI(prompt, model || this.defaultModel);
+            return JSON.stringify(result, null, 2);
+        }
+        catch (error) {
+            return `Error: ${error}`;
+        }
+    }
+}
+exports.LLMJudgeValidator = LLMJudgeValidator;
+//# sourceMappingURL=llmJudge.js.map

package/dist/validators/llmJudge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmJudge.js","sourceRoot":"","sources":["../../src/validators/llmJudge.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAE7B,4DAAiF;AACjF,oDAAqD;AA6BrD,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;EAiBxB,CAAC;AAEH,MAAa,iBAAiB;IAM5B,YAAY,aAAsB,EAAE,QAAgB,gBAAgB;QALpD,SAAI,GAAG,WAAoB,CAAC;QAM1C,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAA,2BAAc,GAAE,CAAC,CAAC,qCAAqC;QACvE,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,KAAwB,EACxB,QAAsB;QAEtB,MAAM,SAAS,GAAG,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC;QAEvD,iCAAiC;QACjC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC;gBACT,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;aAC3B,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACtE,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC;gBACT,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,wBAAwB;aAChC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,2BAA2B;YAC3B,MAAM,aAAa,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAClE,MAAM,YAAY,GAA6C,EAAE,CAAC;YAElE,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACnD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBACjE,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,wBAAwB;YACxB,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAC7C,QAAQ,EACR,YAAY,EACZ,SAAS,CAAC,cAAc,CACzB,CAAC;YAEF,eAAe;YACf,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;YACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAE9D,iCAAiC;YACjC,MAAM,UAAU,GAAgB,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpE,IAAI,EAAE,WAAoB;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,OAAO,EAAE,QAAQ,CAAC,SAAS;aAC5B,CAAC,CAAC,CAAC;YAEJ,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,UAAU;gBACV,aAAa,EAAE,WAAW;aAC3B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,qBAAqB,KAAK,EAAE;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,QAAsB,EACtB,YAAsD,EACtD,YAAqB;QAErB,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,MAAM,YAAY,GAAG,YAAY;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,OAAO,UAAU,CAAC;aACvD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,OAAO;EACT,QAAQ,CAAC,WAAW;;;EAGpB,QAAQ,CAAC,MAAM;;;EAGf,YAAY;;;;;gCAKkB,QAAQ,CAAC,QAAQ;uCACV,QAAQ,CAAC,QAAQ;;uCAEjB,CAAC;IACtC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,MAAc,EAAE,KAAa;QACpD,MAAM,MAAM,GAAG,qDAAqD,CAAC;QAErE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YACnC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,QAAQ,EAAE;aACzC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE;oBAC9C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;iBAClC;gBACD,WAAW,EAAE,CAAC;gBACd,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;aACzC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAQ,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QAElD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;YAE1D,kCAAkC;YAClC,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC;gBACvC,WAAW,CAAC,YAAY,IAAI,IAAI;gBAChC,WAAW,CAAC,OAAO,IAAI,IAAI,EAC3B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAChD,CAAC;YAED,qDAAqD;YACrD,2CAA2C;YAC3C,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW;iBACvC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACT,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,WAAW,EAAE;aAC5C,CAAC,CAAC,CAAC;YAEN,8DAA8D;YAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,YAAY,IAAI,GAAG,CAAC;YAE1E,MAAM,QAAQ,GAAgB;gBAC5B,MAAM;gBACN,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC,EAAE,iCAAiC;gBAC5F,SAAS,EAAE,WAAW,CAAC,OAAO;gBAC9B,UAAU;aACX,CAAC;YAEF,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,cAAc,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,MAAc,EAAE,KAAc;QAC5C,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,+BAA+B,CAAC;QACzC,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,UAAU,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;CACF;AAzND,8CAyNC"}

package/dist/validators/patternValidator.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Pattern-based code validator
+ */
+import { CodeValidator, TestScenario, ValidationResult } from '../types';
+export declare class PatternValidator implements CodeValidator {
+    readonly type: "pattern";
+    private workspaceRoot;
+    constructor(workspaceRoot?: string);
+    /**
+     * Validate generated code against pattern rules
+     */
+    validate(files: readonly string[], scenario: TestScenario): Promise<ValidationResult>;
+    /**
+     * Find all matches of a pattern in text with line numbers
+     */
+    private findPatternMatches;
+    /**
+     * Find the line number containing specific text
+     */
+    private findLineWithText;
+    /**
+     * Calculate score based on number and severity of violations
+     * Critical violations have more weight than minor ones
+     */
+    private calculateScore;
+}
+//# sourceMappingURL=patternValidator.d.ts.map

package/dist/validators/patternValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"patternValidator.d.ts","sourceRoot":"","sources":["../../src/validators/patternValidator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAa,MAAM,UAAU,CAAC;AAGpF,qBAAa,gBAAiB,YAAW,aAAa;IACpD,SAAgB,IAAI,EAAG,SAAS,CAAU;IAC1C,OAAO,CAAC,aAAa,CAAS;gBAElB,aAAa,CAAC,EAAE,MAAM;IAIlC;;OAEG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC;IA8I5B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAoB1B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAgBxB;;;OAGG;IACH,OAAO,CAAC,cAAc;CAsBvB"}