coding-agent-benchmarks 0.5.0 → 0.5.2

package/dist/utils/timeUtils.d.ts

@@ -1,3 +1,6 @@
+/**
+ * Time difference units
+ */
 export interface TimeDifference {
     milliseconds: number;
     seconds: number;
@@ -5,5 +8,17 @@ export interface TimeDifference {
     hours: number;
     days: number;
 }
-export declare const calculateTimeDifference: (startDate: Date, endDate: Date) => TimeDifference;
+/**
+ * Calculate the absolute time difference between two dates
+ */
+export declare const getTimeDifference: (date1: Date, date2: Date) => TimeDifference;
+/**
+ * Calculate the signed time difference between two dates (date2 - date1)
+ * Positive if date2 is after date1, negative if before
+ */
+export declare const getSignedTimeDifference: (date1: Date, date2: Date) => TimeDifference;
+/**
+ * Format time difference as human-readable string
+ */
+export declare const formatTimeDifference: (diff: TimeDifference) => string;
 //# sourceMappingURL=timeUtils.d.ts.map

package/dist/utils/timeUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"timeUtils.d.ts","sourceRoot":"","sources":["../../src/utils/timeUtils.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAkBD,eAAO,MAAM,uBAAuB,GAAI,WAAW,IAAI,EAAE,SAAS,IAAI,KAAG,cAUxE,CAAC"}
+{"version":3,"file":"timeUtils.d.ts","sourceRoot":"","sources":["../../src/utils/timeUtils.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd;AAoBD;;GAEG;AACH,eAAO,MAAM,iBAAiB,GAC5B,OAAO,IAAI,EACX,OAAO,IAAI,KACV,cAGF,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,uBAAuB,GAClC,OAAO,IAAI,EACX,OAAO,IAAI,KACV,cAGF,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,oBAAoB,GAAI,MAAM,cAAc,KAAG,MAmB3D,CAAC"}

package/dist/utils/timeUtils.js

@@ -1,23 +1,60 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.calculateTimeDifference = void 0;
-const MILLISECONDS_PER_SECOND = 1000;
-const SECONDS_PER_MINUTE = 60;
-const MINUTES_PER_HOUR = 60;
-const HOURS_PER_DAY = 24;
-const millisecondsToSeconds = (ms) => ms / MILLISECONDS_PER_SECOND;
-const millisecondsToMinutes = (ms) => ms / (MILLISECONDS_PER_SECOND * SECONDS_PER_MINUTE);
-const millisecondsToHours = (ms) => ms / (MILLISECONDS_PER_SECOND * SECONDS_PER_MINUTE * MINUTES_PER_HOUR);
-const millisecondsToDays = (ms) => ms / (MILLISECONDS_PER_SECOND * SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY);
-const calculateTimeDifference = (startDate, endDate) => {
-    const milliseconds = Math.abs(endDate.getTime() - startDate.getTime());
+exports.formatTimeDifference = exports.getSignedTimeDifference = exports.getTimeDifference = void 0;
+/**
+ * Convert milliseconds to other time units
+ */
+const convertMilliseconds = (ms) => {
+    const seconds = ms / 1000;
+    const minutes = seconds / 60;
+    const hours = minutes / 60;
+    const days = hours / 24;
     return {
-        milliseconds,
-        seconds: millisecondsToSeconds(milliseconds),
-        minutes: millisecondsToMinutes(milliseconds),
-        hours: millisecondsToHours(milliseconds),
-        days: millisecondsToDays(milliseconds),
+        milliseconds: ms,
+        seconds,
+        minutes,
+        hours,
+        days,
     };
 };
-exports.calculateTimeDifference = calculateTimeDifference;
+/**
+ * Calculate the absolute time difference between two dates
+ */
+const getTimeDifference = (date1, date2) => {
+    const ms = Math.abs(date1.getTime() - date2.getTime());
+    return convertMilliseconds(ms);
+};
+exports.getTimeDifference = getTimeDifference;
+/**
+ * Calculate the signed time difference between two dates (date2 - date1)
+ * Positive if date2 is after date1, negative if before
+ */
+const getSignedTimeDifference = (date1, date2) => {
+    const ms = date2.getTime() - date1.getTime();
+    return convertMilliseconds(ms);
+};
+exports.getSignedTimeDifference = getSignedTimeDifference;
+/**
+ * Format time difference as human-readable string
+ */
+const formatTimeDifference = (diff) => {
+    const absDays = Math.abs(diff.days);
+    const absHours = Math.abs(diff.hours);
+    const absMinutes = Math.abs(diff.minutes);
+    const absSeconds = Math.abs(diff.seconds);
+    if (absDays >= 1) {
+        return `${diff.days.toFixed(2)} days`;
+    }
+    if (absHours >= 1) {
+        return `${diff.hours.toFixed(2)} hours`;
+    }
+    if (absMinutes >= 1) {
+        return `${diff.minutes.toFixed(2)} minutes`;
+    }
+    if (absSeconds >= 1) {
+        return `${diff.seconds.toFixed(2)} seconds`;
+    }
+    return `${diff.milliseconds} milliseconds`;
+};
+exports.formatTimeDifference = formatTimeDifference;
 //# sourceMappingURL=timeUtils.js.map

package/dist/utils/timeUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"timeUtils.js","sourceRoot":"","sources":["../../src/utils/timeUtils.ts"],"names":[],"mappings":";;;AAQA,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAC5B,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,qBAAqB,GAAG,CAAC,EAAU,EAAU,EAAE,CAAC,EAAE,GAAG,uBAAuB,CAAC;AAEnF,MAAM,qBAAqB,GAAG,CAAC,EAAU,EAAU,EAAE,CACnD,EAAE,GAAG,CAAC,uBAAuB,GAAG,kBAAkB,CAAC,CAAC;AAEtD,MAAM,mBAAmB,GAAG,CAAC,EAAU,EAAU,EAAE,CACjD,EAAE,GAAG,CAAC,uBAAuB,GAAG,kBAAkB,GAAG,gBAAgB,CAAC,CAAC;AAEzE,MAAM,kBAAkB,GAAG,CAAC,EAAU,EAAU,EAAE,CAChD,EAAE,GAAG,CAAC,uBAAuB,GAAG,kBAAkB,GAAG,gBAAgB,GAAG,aAAa,CAAC,CAAC;AAElF,MAAM,uBAAuB,GAAG,CAAC,SAAe,EAAE,OAAa,EAAkB,EAAE;IACxF,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;IAEvE,OAAO;QACL,YAAY;QACZ,OAAO,EAAE,qBAAqB,CAAC,YAAY,CAAC;QAC5C,OAAO,EAAE,qBAAqB,CAAC,YAAY,CAAC;QAC5C,KAAK,EAAE,mBAAmB,CAAC,YAAY,CAAC;QACxC,IAAI,EAAE,kBAAkB,CAAC,YAAY,CAAC;KACvC,CAAC;AACJ,CAAC,CAAC;AAVW,QAAA,uBAAuB,2BAUlC"}
+{"version":3,"file":"timeUtils.js","sourceRoot":"","sources":["../../src/utils/timeUtils.ts"],"names":[],"mappings":";;;AAWA;;GAEG;AACH,MAAM,mBAAmB,GAAG,CAAC,EAAU,EAAkB,EAAE;IACzD,MAAM,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC;IAC1B,MAAM,OAAO,GAAG,OAAO,GAAG,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,OAAO,GAAG,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;IAExB,OAAO;QACL,YAAY,EAAE,EAAE;QAChB,OAAO;QACP,OAAO;QACP,KAAK;QACL,IAAI;KACL,CAAC;AACJ,CAAC,CAAC;AAEF;;GAEG;AACI,MAAM,iBAAiB,GAAG,CAC/B,KAAW,EACX,KAAW,EACK,EAAE;IAClB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACvD,OAAO,mBAAmB,CAAC,EAAE,CAAC,CAAC;AACjC,CAAC,CAAC;AANW,QAAA,iBAAiB,qBAM5B;AAEF;;;GAGG;AACI,MAAM,uBAAuB,GAAG,CACrC,KAAW,EACX,KAAW,EACK,EAAE;IAClB,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;IAC7C,OAAO,mBAAmB,CAAC,EAAE,CAAC,CAAC;AACjC,CAAC,CAAC;AANW,QAAA,uBAAuB,2BAMlC;AAEF;;GAEG;AACI,MAAM,oBAAoB,GAAG,CAAC,IAAoB,EAAU,EAAE;IACnE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAE1C,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACjB,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACxC,CAAC;IACD,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC1C,CAAC;IACD,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;IAC9C,CAAC;IACD,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,YAAY,eAAe,CAAC;AAC7C,CAAC,CAAC;AAnBW,QAAA,oBAAoB,wBAmB/B"}

package/dist/utils/userReport.d.ts

@@ -0,0 +1,10 @@
+/**
+ * User report printing utilities
+ */
+export interface UserWithStatus {
+    name: string;
+    email: string;
+    accountStatus: string;
+}
+export declare const printUserReport: (users: readonly UserWithStatus[]) => void;
+//# sourceMappingURL=userReport.d.ts.map

package/dist/utils/userReport.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"userReport.d.ts","sourceRoot":"","sources":["../../src/utils/userReport.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,eAAe,GAAI,OAAO,SAAS,cAAc,EAAE,KAAG,IAYlE,CAAC"}

package/dist/utils/userReport.js

@@ -0,0 +1,19 @@
+"use strict";
+/**
+ * User report printing utilities
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.printUserReport = void 0;
+const printUserReport = (users) => {
+    console.log('\n=== User Report ===\n');
+    users.forEach((user, index) => {
+        console.log(`User #${index + 1}:`);
+        console.log(`  Name:           ${user.name}`);
+        console.log(`  Email:          ${user.email}`);
+        console.log(`  Account Status: ${user.accountStatus}`);
+        console.log('');
+    });
+    console.log(`Total Users: ${users.length}\n`);
+};
+exports.printUserReport = printUserReport;
+//# sourceMappingURL=userReport.js.map

package/dist/utils/userReport.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"userReport.js","sourceRoot":"","sources":["../../src/utils/userReport.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAQI,MAAM,eAAe,GAAG,CAAC,KAAgC,EAAQ,EAAE;IACxE,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAEvC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;AAChD,CAAC,CAAC;AAZW,QAAA,eAAe,mBAY1B"}

package/dist/validators/llmJudge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llmJudge.d.ts","sourceRoot":"","sources":["../../src/validators/llmJudge.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAEjB,MAAM,UAAU,CAAC;AA6DlB,qBAAa,iBAAkB,YAAW,aAAa;IACrD,SAAgB,IAAI,EAAG,WAAW,CAAU;IAC5C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,YAAY,CAAS;gBAEjB,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,MAAuB;IAMlE;;OAEG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA+B3B;;OAEG;YACW,UAAU;IA4ExB;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAYjE"}
+{"version":3,"file":"llmJudge.d.ts","sourceRoot":"","sources":["../../src/validators/llmJudge.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAEjB,MAAM,UAAU,CAAC;AA6DlB,qBAAa,iBAAkB,YAAW,aAAa;IACrD,SAAgB,IAAI,EAAG,WAAW,CAAU;IAC5C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,YAAY,CAAS;gBAEjB,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,MAAuB;IAMlE;;OAEG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC;IA4E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA+B3B;;OAEG;YACW,UAAU;IAsFxB;;OAEG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAYjE"}

package/dist/validators/llmJudge.js

@@ -103,9 +103,7 @@ class LLMJudgeValidator {
                 const relativePath = path.relative(this.workspaceRoot, filePath);
                 fileContents.push({ path: relativePath, content });
             }
-            // Build judgment prompt
             const judgmentPrompt = this.buildJudgmentPrompt(scenario, fileContents, llmConfig.judgmentPrompt);
-            // Calling LLM API
             const model = llmConfig.model || this.defaultModel;
             const judgment = await this.callLLMAPI(judgmentPrompt, model);
             // Convert judgment to violations
@@ -126,8 +124,8 @@ class LLMJudgeValidator {
         }
         catch (error) {
             return {
-                passed: false,
-                score: 0,
+                passed: true,
+                score: -1,
                 violations: [],
                 validatorType: "llm-judge",
                 error: `LLM judge failed: ${error}`,
@@ -177,10 +175,15 @@ Be strict but fair in your evaluation.`;
                     { role: "system", content: judgeSystemPrompt },
                     { role: "user", content: prompt },
                 ],
-                max_completion_tokens: 1000,
                 response_format: { type: "json_object" },
             }),
         });
+        if (response.status === 401) {
+            throw new Error("Unauthorized: Invalid GITHUB_TOKEN");
+        }
+        if (response.status === 429) {
+            throw new Error("Rate limit exceeded: Too many requests to LLM API");
+        }
         if (!response.ok) {
             const errorText = await response.text();
             throw new Error(`GitHub Models API error: ${response.status} ${errorText}`);
@@ -188,7 +191,8 @@ Be strict but fair in your evaluation.`;
         const data = (await response.json());
         const content = data.choices[0]?.message?.content;
         if (!content) {
-            throw new Error("No content in LLM response");
+            const finishReason = data.choices[0]?.finish_reason ?? "unknown";
+            throw new Error(`No content in LLM response (finish_reason: ${finishReason})`);
         }
         // Parse JSON response
         try {

package/dist/validators/llmJudge.js.map

@@ -1 +1 @@
-{"version":3,"file":"llmJudge.js","sourceRoot":"","sources":["../../src/validators/llmJudge.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAO7B,4DAGiC;AACjC,oDAAqD;AAqCrD,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;EAiBxB,CAAC;AAEH,MAAa,iBAAiB;IAM5B,YAAY,aAAsB,EAAE,QAAgB,cAAc;QALlD,SAAI,GAAG,WAAoB,CAAC;QAM1C,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAA,2BAAc,GAAE,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,KAAwB,EACxB,QAAsB;QAEtB,MAAM,SAAS,GAAG,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC;QAEvD,iCAAiC;QACjC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,EAAE,mBAAmB;gBAC9B,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;aAC3B,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACtE,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,EAAE,mBAAmB;gBAC9B,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,wBAAwB;aAChC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,2BAA2B;YAC3B,MAAM,aAAa,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAClE,MAAM,YAAY,GAAsB,EAAE,CAAC;YAE3C,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACnD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBACjE,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,wBAAwB;YACxB,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAC7C,QAAQ,EACR,YAAY,EACZ,SAAS,CAAC,cAAc,CACzB,CAAC;YAEF,kBAAkB;YAClB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;YACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAE9D,iCAAiC;YACjC,MAAM,UAAU,GAAgB,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtE,IAAI,EAAE,WAAoB;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,OAAO,EAAE,QAAQ,CAAC,SAAS;aAC5B,CAAC,CAAC,CAAC;YAEJ,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,UAAU;gBACV,aAAa,EAAE,WAAW;aAC3B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,qBAAqB,KAAK,EAAE;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,QAAsB,EACtB,YAA+B,EAC/B,YAAqB;QAErB,MAAM,YAAY,GAAG,YAAY;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,OAAO,UAAU,CAAC;aACzD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,kBAAkB,GACtB,YAAY;YACZ;;gCAE0B,QAAQ,CAAC,QAAQ;uCACV,QAAQ,CAAC,QAAQ;;uCAEjB,CAAC;QAEpC,OAAO;UACD,QAAQ,CAAC,WAAW;;;UAGpB,QAAQ,CAAC,MAAM;;;UAGf,YAAY;;;QAGd,kBAAkB,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CACtB,MAAc,EACd,KAAa;QAEb,MAAM,MAAM,GAAG,qDAAqD,CAAC;QAErE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YACnC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,QAAQ,EAAE;aACzC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE;oBAC9C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;iBAClC;gBACD,qBAAqB,EAAE,IAAI;gBAC3B,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;aACzC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CACb,4BAA4B,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE,CAC3D,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAQ,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QAElD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;YAE1D,kCAAkC;YAClC,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC;gBACvC,WAAW,CAAC,YAAY,IAAI,IAAI;gBAChC,WAAW,CAAC,OAAO,IAAI,IAAI,EAC3B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAChD,CAAC;YAED,qDAAqD;YACrD,2CAA2C;YAC3C,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW;iBACvC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;iBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,WAAW,EAAE;aAC5C,CAAC,CAAC,CAAC;YAEN,8DAA8D;YAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,YAAY,IAAI,GAAG,CAAC;YAE1E,MAAM,QAAQ,GAAgB;gBAC5B,MAAM;gBACN,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC,EAAE,iCAAiC;gBAC5F,SAAS,EAAE,WAAW,CAAC,OAAO;gBAC9B,UAAU;aACX,CAAC;YAEF,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,iCAAiC,KAAK,cAAc,OAAO,EAAE,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,MAAc,EAAE,KAAc;QAC5C,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,+BAA+B,CAAC;QACzC,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,UAAU,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;CACF;AAhOD,8CAgOC"}
+{"version":3,"file":"llmJudge.js","sourceRoot":"","sources":["../../src/validators/llmJudge.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAO7B,4DAGiC;AACjC,oDAAqD;AAqCrD,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;EAiBxB,CAAC;AAEH,MAAa,iBAAiB;IAM5B,YAAY,aAAsB,EAAE,QAAgB,cAAc;QALlD,SAAI,GAAG,WAAoB,CAAC;QAM1C,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,IAAA,2BAAc,GAAE,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,KAAwB,EACxB,QAAsB;QAEtB,MAAM,SAAS,GAAG,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC;QAEvD,iCAAiC;QACjC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,EAAE,mBAAmB;gBAC9B,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;aAC3B,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACtE,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,EAAE,mBAAmB;gBAC9B,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,wBAAwB;aAChC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,2BAA2B;YAC3B,MAAM,aAAa,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAClE,MAAM,YAAY,GAAsB,EAAE,CAAC;YAE3C,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACnD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBACjE,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAC7C,QAAQ,EACR,YAAY,EACZ,SAAS,CAAC,cAAc,CACzB,CAAC;YAEF,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC;YACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAE9D,iCAAiC;YACjC,MAAM,UAAU,GAAgB,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtE,IAAI,EAAE,WAAoB;gBAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,OAAO,EAAE,QAAQ,CAAC,SAAS;aAC5B,CAAC,CAAC,CAAC;YAEJ,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,UAAU;gBACV,aAAa,EAAE,WAAW;aAC3B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC;gBACT,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,qBAAqB,KAAK,EAAE;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,QAAsB,EACtB,YAA+B,EAC/B,YAAqB;QAErB,MAAM,YAAY,GAAG,YAAY;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,OAAO,UAAU,CAAC;aACzD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,kBAAkB,GACtB,YAAY;YACZ;;gCAE0B,QAAQ,CAAC,QAAQ;uCACV,QAAQ,CAAC,QAAQ;;uCAEjB,CAAC;QAEpC,OAAO;UACD,QAAQ,CAAC,WAAW;;;UAGpB,QAAQ,CAAC,MAAM;;;UAGf,YAAY;;;QAGd,kBAAkB,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CACtB,MAAc,EACd,KAAa;QAEb,MAAM,MAAM,GAAG,qDAAqD,CAAC;QAErE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YACnC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,QAAQ,EAAE;aACzC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE;oBAC9C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;iBAClC;gBACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;aACzC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CACb,4BAA4B,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE,CAC3D,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAQ,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QAElD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,aAAa,IAAI,SAAS,CAAC;YACjE,MAAM,IAAI,KAAK,CACb,8CAA8C,YAAY,GAAG,CAC9D,CAAC;QACJ,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;YAE1D,kCAAkC;YAClC,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC;gBACvC,WAAW,CAAC,YAAY,IAAI,IAAI;gBAChC,WAAW,CAAC,OAAO,IAAI,IAAI,EAC3B,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAChD,CAAC;YAED,qDAAqD;YACrD,2CAA2C;YAC3C,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW;iBACvC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;iBAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,OAAO,EAAE,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,WAAW,EAAE;aAC5C,CAAC,CAAC,CAAC;YAEN,8DAA8D;YAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,YAAY,IAAI,GAAG,CAAC;YAE1E,MAAM,QAAQ,GAAgB;gBAC5B,MAAM;gBACN,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC,EAAE,iCAAiC;gBAC5F,SAAS,EAAE,WAAW,CAAC,OAAO;gBAC9B,UAAU;aACX,CAAC;YAEF,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,iCAAiC,KAAK,cAAc,OAAO,EAAE,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,MAAc,EAAE,KAAc;QAC5C,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,+BAA+B,CAAC;QACzC,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,UAAU,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;CACF;AAxOD,8CAwOC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coding-agent-benchmarks",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Testing coding agents (GitHub Copilot CLI, Claude Code, etc.) with your repo's context to evaluate their code generation quality.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",