coding-agent-benchmarks 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/adapters/claudeCodeCLI.d.ts.map +1 -1
- package/dist/adapters/claudeCodeCLI.js +1 -0
- package/dist/adapters/claudeCodeCLI.js.map +1 -1
- package/dist/runner.d.ts +0 -3
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +1 -4
- package/dist/runner.js.map +1 -1
- package/dist/validators/eslintValidator.d.ts +1 -1
- package/dist/validators/eslintValidator.d.ts.map +1 -1
- package/dist/validators/eslintValidator.js +23 -23
- package/dist/validators/eslintValidator.js.map +1 -1
- package/package.json +7 -1
package/README.md
CHANGED
|
@@ -320,6 +320,8 @@ The LLM judge provides semantic understanding beyond pattern matching, evaluatin
|
|
|
320
320
|
|
|
321
321
|
#### ESLint Validator
|
|
322
322
|
|
|
323
|
+
This validator runs ESLint on the generated code and scores based on the number and severity of linting violations. Note that ESLint must be installed and configured in your project for this validator to work. If you don't have ESLint set up globally, disable this validator or provide a custom validator.
|
|
324
|
+
|
|
323
325
|
Uses exponential decay with a dampening factor:
|
|
324
326
|
|
|
325
327
|
```
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claudeCodeCLI.d.ts","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAOjD,MAAM,WAAW,2BAA2B;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,qBAAa,oBAAqB,YAAW,qBAAqB;IAChE,SAAgB,IAAI,EAAG,aAAa,CAAU;IAC9C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,KAAK,CAAS;gBAEV,OAAO,CAAC,EAAE,2BAA2B;IAKjD;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB3C;;OAEG;IACH,QAAQ,IAAI,MAAM;IAIlB;;OAEG;IACH,OAAO,CAAC,WAAW;IA8BnB;;;OAGG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"claudeCodeCLI.d.ts","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAOjD,MAAM,WAAW,2BAA2B;IAC1C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,qBAAa,oBAAqB,YAAW,qBAAqB;IAChE,SAAgB,IAAI,EAAG,aAAa,CAAU;IAC9C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,KAAK,CAAS;gBAEV,OAAO,CAAC,EAAE,2BAA2B;IAKjD;;OAEG;IACG,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB3C;;OAEG;IACH,QAAQ,IAAI,MAAM;IAIlB;;OAEG;IACH,OAAO,CAAC,WAAW;IA8BnB;;;OAGG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,EAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;CAsGrB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claudeCodeCLI.js","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,iDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,gDAA+E;AAC/E,4DAGiC;AAOjC,MAAM,aAAa,GAAG,QAAQ,CAAC;AAE/B,MAAa,oBAAoB;IAK/B,YAAY,OAAqC;QAJjC,SAAI,GAAG,aAAsB,CAAC;QAK5C,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAClE,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE;gBACtC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,MAAc,EACd,YAAgC;QAEhC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAClC,MAAM,cAAc,GAAG,QAAQ;qBAC5B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC;oBAC5D,OAAO,OAAO,GAAG,CAAC,IAAI,WAAW,GAAG,KAAK,GAAG,CAAC,OAAO,UAAU,CAAC;gBACjE,CAAC,CAAC;qBACD,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,KAAK,CAAC,IAAI,CACR,0GAA0G,CAC3G,CAAC;QAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,MAAc,EACd,YAAgC,EAChC,OAAuB;QAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAE1D,uCAAuC;QACvC,MAAM,YAAY,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE/D,gFAAgF;QAChF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,yBAAyB,CAAC,CAAC;YAC1E,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAE/C,mBAAmB;YACnB,MAAM,OAAO,GAAG,GAAS,EAAE;gBACzB,IAAI,CAAC;oBACH,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5B,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;YACH,CAAC,CAAC;YAEF,0CAA0C;YAC1C,MAAM,aAAa,GAAG,GAAS,EAAE;gBAC/B,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAG,QAAQ,QAAQ,sBAAsB,IAAI,CAAC,KAAK,yIAAyI,CAAC;YAC1M,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE;gBACxC,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;aAChC,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,aAAa,GAA0B,IAAI,CAAC;YAChD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,OAAO,EAAE,CAAC;oBACV,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;oBAChD,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;oBACjD,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,OAAO,IAAI,CAAC,CAAC,CAAC;gBACpE,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBAED,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAChD,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;gBAEjD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CACJ,IAAI,KAAK,CACP,oCAAoC,IAAI,aAAa,MAAM,EAAE,CAC9D,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBAED,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC9D,MAAM,YAAY,GAAG,IAAA,8BAAmB,EACtC,YAAY,EACZ,WAAW,EACX,IAAI,CAAC,aAAa,CACnB,CAAC;oBACF,OAAO,CAAC,YAAY,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBACD,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAChD,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;gBACjD,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;
|
|
1
|
+
{"version":3,"file":"claudeCodeCLI.js","sourceRoot":"","sources":["../../src/adapters/claudeCodeCLI.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,iDAAsC;AACtC,uCAAyB;AACzB,2CAA6B;AAE7B,gDAA+E;AAC/E,4DAGiC;AAOjC,MAAM,aAAa,GAAG,QAAQ,CAAC;AAE/B,MAAa,oBAAoB;IAK/B,YAAY,OAAqC;QAJjC,SAAI,GAAG,aAAsB,CAAC;QAK5C,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAClE,IAAI,CAAC,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE;gBACtC,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACpB,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,MAAc,EACd,YAAgC;QAEhC,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAClC,MAAM,cAAc,GAAG,QAAQ;qBAC5B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC;oBAC5D,OAAO,OAAO,GAAG,CAAC,IAAI,WAAW,GAAG,KAAK,GAAG,CAAC,OAAO,UAAU,CAAC;gBACjE,CAAC,CAAC;qBACD,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,KAAK,CAAC,IAAI,CACR,0GAA0G,CAC3G,CAAC;QAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CACZ,MAAc,EACd,YAAgC,EAChC,OAAuB;QAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAE1D,uCAAuC;QACvC,MAAM,YAAY,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE/D,gFAAgF;QAChF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,yBAAyB,CAAC,CAAC;YAC1E,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAE/C,mBAAmB;YACnB,MAAM,OAAO,GAAG,GAAS,EAAE;gBACzB,IAAI,CAAC;oBACH,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5B,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;YACH,CAAC,CAAC;YAEF,0CAA0C;YAC1C,MAAM,aAAa,GAAG,GAAS,EAAE;gBAC/B,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAG,QAAQ,QAAQ,sBAAsB,IAAI,CAAC,KAAK,yIAAyI,CAAC;YAC1M,MAAM,IAAI,GAAG,IAAA,qBAAK,EAAC,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE;gBACxC,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;aAChC,CAAC,CAAC;YAEH,6DAA6D;YAC7D,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC/B,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,aAAa,GAA0B,IAAI,CAAC;YAChD,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACrB,OAAO,EAAE,CAAC;oBACV,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;oBAChD,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;oBACjD,MAAM,CAAC,IAAI,KAAK,CAAC,mCAAmC,OAAO,IAAI,CAAC,CAAC,CAAC;gBACpE,CAAC,EAAE,OAAO,CAAC,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBAED,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAChD,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;gBAEjD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACf,MAAM,CACJ,IAAI,KAAK,CACP,oCAAoC,IAAI,aAAa,MAAM,EAAE,CAC9D,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBAED,0DAA0D;gBAC1D,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,IAAA,gCAAqB,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC9D,MAAM,YAAY,GAAG,IAAA,8BAAmB,EACtC,YAAY,EACZ,WAAW,EACX,IAAI,CAAC,aAAa,CACnB,CAAC;oBACF,OAAO,CAAC,YAAY,CAAC,CAAC;gBACxB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACzB,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC9B,CAAC;gBACD,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;gBAChD,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;gBACjD,MAAM,CAAC,IAAI,KAAK,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAnLD,oDAmLC"}
|
package/dist/runner.d.ts
CHANGED
package/dist/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":""}
|
package/dist/runner.js
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
"use strict";
|
|
3
|
-
/**
|
|
4
|
-
* CLI interface for coding-agent-benchmarks
|
|
5
|
-
*/
|
|
6
3
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
4
|
if (k2 === undefined) k2 = k;
|
|
8
5
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
@@ -91,7 +88,7 @@ program
|
|
|
91
88
|
evaluator.on("scenario:start", (scenarioId) => {
|
|
92
89
|
reporter.onScenarioStart(scenarioId);
|
|
93
90
|
});
|
|
94
|
-
evaluator.on("scenario:generating", (
|
|
91
|
+
evaluator.on("scenario:generating", (_scenarioId) => {
|
|
95
92
|
// Phase already set in onScenarioStart, but could be used for more granular updates
|
|
96
93
|
});
|
|
97
94
|
evaluator.on("scenario:validating", (scenarioId) => {
|
package/dist/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,yCAAoC;AACpC,uCAAyB;AACzB,4CAA6C;AAC7C,2CAAwC;AAExC,sDAA0D;AAC1D,4DAAgE;AAChE,oDAA0D;AAC1D,mDAAqD;AACrD,uDAAuD;AACvD,yCAA8C;AAE9C,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,yBAAyB,CAAC;KAC/B,WAAW,CACV,oEAAoE,CACrE;KACA,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,2BAA2B,CAAC;KACxC,MAAM,CACL,sBAAsB,EACtB,qDAAqD,CACtD;KACA,MAAM,CAAC,yBAAyB,EAAE,sCAAsC,CAAC;KACzE,MAAM,CAAC,cAAc,EAAE,kCAAkC,CAAC;KAC1D,MAAM,CACL,kBAAkB,EAClB,kDAAkD,EAClD,SAAS,CACV;KACA,MAAM,CAAC,iBAAiB,EAAE,+CAA+C,CAAC;KAC1E,MAAM,CAAC,sBAAsB,EAAE,uBAAuB,EAAE,KAAK,CAAC;KAC9D,MAAM,CAAC,WAAW,EAAE,sBAAsB,CAAC;KAC3C,MAAM,CAAC,iBAAiB,EAAE,4BAA4B,CAAC;KACvD,MAAM,CAAC,yBAAyB,EAAE,0BAA0B,CAAC;KAC7D,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAA,mBAAU,EAC5C,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG,EAAE,CACvC,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,qBAAS,CAAC;YAC9B,OAAO,EAAE,OAAO,CAAC,OAAsB;YACvC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,eAAe,EAAE,MAAM,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,2BAAgB,CAAC;YACpC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,kBAAkB,EAAE,CAAC,YAAY,EAAE,EAAE;YAChD,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,UAAU,EAAE,EAAE;YAC5C,QAAQ,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,WAAW,EAAE,EAAE;YAClD,oFAAoF;QACtF,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,UAAU,EAAE,EAAE;YACjD,QAAQ,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE;YAC9D,QAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,6BAA6B;QAC7B,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,wBAAwB,EAAE,CAAC;QAC/D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,UAAU,OAAO,CAAC,OAAO,gBAAgB,CAAC,CAAC;YACzD,OAAO,CAAC,KAAK,CACX,kBAAkB,OAAO,CAAC,OAAO,0BAA0B,CAC5D,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,mBAAmB;QACnB,MAAM,iBAAiB,GAAG,SAAS,CAAC,eAAe,CAAC,SAAS,EAAE;YAC7D,eAAe,EAAE,OAAO,CAAC,QAAQ;YACjC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,GAAG;gBACf,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACrD,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;YACxD,OAAO;QACT,CAAC;QAED,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAE3D,sCAAsC;QACtC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAExB,kCAAkC;QAClC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,EAAE,CAAC,aAAa,CACd,OAAO,CAAC,MAAM,EACd,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAC/B,OAAO,CACR,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,yBAAyB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAA,8BAAe,EAAC,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,+BAA+B,CAAC;KAC5C,MAAM,CAAC,yBAAyB,EAAE,oBAAoB,CAAC;KACvD,MAAM,CAAC,cAAc,EAAE,kCAAkC,CAAC;KAC1D,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAA,mBAAU,GAAE,CAAC;QAEzC,IAAI,QAAQ,GAAG,SAAS,CAAC;QAEzB,qBAAqB;QACrB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ;iBAChC,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,iBAAiB;QACjB,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACjE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,IAAI,CAAC,IAAI,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CACjD,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,0BAA0B,QAAQ,CAAC,MAAM,MAAM,CAAC,CAAC;QAE7D,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAoB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,0DAA0D,CAAC;KACvE,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAElD,MAAM,QAAQ,GAA+C;QAC3D,EAAE,IAAI,EAAE,oBAAoB,EAAE,IAAI,EAAE,SAAS,EAAE;QAC/C,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,aAAa,EAAE;KACjD,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,OAAO,CAAC;QACZ,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,GAAG,IAAI,8BAAiB,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,IAAI,oCAAoB,EAAE,CAAC;QACvC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,IAAA,4BAAe,GAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClD,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;IAEnD,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CACT,6DAA6D,CAC9D,CAAC;QACF,OAAO,CAAC,GAAG,CACT,kEAAkE,CACnE,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;IAC3E,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,OAAO;KACJ,OAAO,CAAC,UAAU,CAAC;KACnB,WAAW,CAAC,qCAAqC,CAAC;KAClD,MAAM,CAAC,iBAAiB,EAAE,4CAA4C,CAAC;KACvE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QAEvD,yBAAyB;QACzB,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;YACjC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE/B,MAAM,SAAS,GAAG,IAAI,4BAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAClE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAEhE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* ESLint validator for generated code
|
|
3
3
|
*/
|
|
4
|
-
import { CodeValidator, TestScenario, ValidationResult } from
|
|
4
|
+
import { CodeValidator, TestScenario, ValidationResult } from "../types";
|
|
5
5
|
export declare class ESLintValidator implements CodeValidator {
|
|
6
6
|
readonly type: "eslint";
|
|
7
7
|
private workspaceRoot;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eslintValidator.d.ts","sourceRoot":"","sources":["../../src/validators/eslintValidator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,
|
|
1
|
+
{"version":3,"file":"eslintValidator.d.ts","sourceRoot":"","sources":["../../src/validators/eslintValidator.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EACL,aAAa,EACb,YAAY,EACZ,gBAAgB,EAEjB,MAAM,UAAU,CAAC;AAOlB,qBAAa,eAAgB,YAAW,aAAa;IACnD,SAAgB,IAAI,EAAG,QAAQ,CAAU;IACzC,OAAO,CAAC,aAAa,CAAS;gBAElB,aAAa,CAAC,EAAE,MAAM;IAIlC;;OAEG;YACW,uBAAuB;IAarC;;OAEG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,EAAE,YAAY,GACrB,OAAO,CAAC,gBAAgB,CAAC;IA+I5B;;OAEG;IACH,OAAO,CAAC,cAAc;CAsBvB"}
|
|
@@ -43,7 +43,7 @@ const workspaceUtils_1 = require("../utils/workspaceUtils");
|
|
|
43
43
|
const child_process_1 = require("child_process");
|
|
44
44
|
class ESLintValidator {
|
|
45
45
|
constructor(workspaceRoot) {
|
|
46
|
-
this.type =
|
|
46
|
+
this.type = "eslint";
|
|
47
47
|
this.workspaceRoot = (0, workspaceUtils_1.resolveWorkspaceRoot)(workspaceRoot);
|
|
48
48
|
}
|
|
49
49
|
/**
|
|
@@ -52,9 +52,9 @@ class ESLintValidator {
|
|
|
52
52
|
async checkESLintAvailability() {
|
|
53
53
|
try {
|
|
54
54
|
// Check if eslint is installed
|
|
55
|
-
(0, child_process_1.execSync)(
|
|
55
|
+
(0, child_process_1.execSync)("npx eslint --version", {
|
|
56
56
|
cwd: this.workspaceRoot,
|
|
57
|
-
stdio:
|
|
57
|
+
stdio: "pipe",
|
|
58
58
|
});
|
|
59
59
|
return true;
|
|
60
60
|
}
|
|
@@ -71,21 +71,21 @@ class ESLintValidator {
|
|
|
71
71
|
if (!eslintConfig?.enabled) {
|
|
72
72
|
return {
|
|
73
73
|
passed: true,
|
|
74
|
-
score: -1,
|
|
74
|
+
score: -1, // skip
|
|
75
75
|
violations: [],
|
|
76
|
-
validatorType:
|
|
76
|
+
validatorType: "eslint",
|
|
77
77
|
};
|
|
78
78
|
}
|
|
79
79
|
// Check if ESLint is available
|
|
80
80
|
const isAvailable = await this.checkESLintAvailability();
|
|
81
81
|
if (!isAvailable) {
|
|
82
|
-
console.warn(
|
|
82
|
+
console.warn("ESLint not found in project, skipping ESLint validation");
|
|
83
83
|
return {
|
|
84
84
|
passed: true,
|
|
85
85
|
score: -1,
|
|
86
86
|
violations: [],
|
|
87
|
-
validatorType:
|
|
88
|
-
error:
|
|
87
|
+
validatorType: "eslint",
|
|
88
|
+
error: "ESLint not found",
|
|
89
89
|
};
|
|
90
90
|
}
|
|
91
91
|
try {
|
|
@@ -98,18 +98,18 @@ class ESLintValidator {
|
|
|
98
98
|
}
|
|
99
99
|
// Only lint JS/TS files
|
|
100
100
|
const ext = path.extname(filePath).toLowerCase();
|
|
101
|
-
if (![
|
|
101
|
+
if (![".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs"].includes(ext)) {
|
|
102
102
|
continue;
|
|
103
103
|
}
|
|
104
104
|
try {
|
|
105
105
|
// Run ESLint with JSON output
|
|
106
106
|
const configArg = eslintConfig.configPath
|
|
107
107
|
? `--config ${eslintConfig.configPath}`
|
|
108
|
-
:
|
|
108
|
+
: "";
|
|
109
109
|
const output = (0, child_process_1.execSync)(`npx eslint ${configArg} --format json "${filePath}"`, {
|
|
110
110
|
cwd: this.workspaceRoot,
|
|
111
|
-
encoding:
|
|
112
|
-
stdio:
|
|
111
|
+
encoding: "utf-8",
|
|
112
|
+
stdio: "pipe",
|
|
113
113
|
});
|
|
114
114
|
// Parse ESLint output
|
|
115
115
|
const results = JSON.parse(output);
|
|
@@ -118,12 +118,12 @@ class ESLintValidator {
|
|
|
118
118
|
for (const result of results) {
|
|
119
119
|
for (const message of result.messages || []) {
|
|
120
120
|
const severity = message.severity === 2
|
|
121
|
-
?
|
|
121
|
+
? "major"
|
|
122
122
|
: message.severity === 1
|
|
123
|
-
?
|
|
124
|
-
:
|
|
123
|
+
? "minor"
|
|
124
|
+
: "minor";
|
|
125
125
|
violations.push({
|
|
126
|
-
type:
|
|
126
|
+
type: "eslint",
|
|
127
127
|
message: `${message.ruleId}: ${message.message}`,
|
|
128
128
|
file: relativePath,
|
|
129
129
|
line: message.line,
|
|
@@ -143,12 +143,12 @@ class ESLintValidator {
|
|
|
143
143
|
for (const result of results) {
|
|
144
144
|
for (const message of result.messages || []) {
|
|
145
145
|
const severity = message.severity === 2
|
|
146
|
-
?
|
|
146
|
+
? "major"
|
|
147
147
|
: message.severity === 1
|
|
148
|
-
?
|
|
149
|
-
:
|
|
148
|
+
? "minor"
|
|
149
|
+
: "minor";
|
|
150
150
|
violations.push({
|
|
151
|
-
type:
|
|
151
|
+
type: "eslint",
|
|
152
152
|
message: `${message.ruleId}: ${message.message}`,
|
|
153
153
|
file: relativePath,
|
|
154
154
|
line: message.line,
|
|
@@ -161,7 +161,7 @@ class ESLintValidator {
|
|
|
161
161
|
catch {
|
|
162
162
|
// If we can't parse the output, treat it as a general error
|
|
163
163
|
violations.push({
|
|
164
|
-
type:
|
|
164
|
+
type: "eslint",
|
|
165
165
|
message: `ESLint failed for ${path.basename(filePath)}`,
|
|
166
166
|
file: path.relative(this.workspaceRoot, filePath),
|
|
167
167
|
severity: scenario.severity,
|
|
@@ -178,7 +178,7 @@ class ESLintValidator {
|
|
|
178
178
|
passed,
|
|
179
179
|
score,
|
|
180
180
|
violations,
|
|
181
|
-
validatorType:
|
|
181
|
+
validatorType: "eslint",
|
|
182
182
|
};
|
|
183
183
|
}
|
|
184
184
|
catch (error) {
|
|
@@ -186,7 +186,7 @@ class ESLintValidator {
|
|
|
186
186
|
passed: false,
|
|
187
187
|
score: 0,
|
|
188
188
|
violations: [],
|
|
189
|
-
validatorType:
|
|
189
|
+
validatorType: "eslint",
|
|
190
190
|
error: `ESLint validation failed: ${error}`,
|
|
191
191
|
};
|
|
192
192
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eslintValidator.js","sourceRoot":"","sources":["../../src/validators/eslintValidator.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;
|
|
1
|
+
{"version":3,"file":"eslintValidator.js","sourceRoot":"","sources":["../../src/validators/eslintValidator.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,2CAA6B;AAO7B,4DAGiC;AACjC,iDAAyC;AAEzC,MAAa,eAAe;IAI1B,YAAY,aAAsB;QAHlB,SAAI,GAAG,QAAiB,CAAC;QAIvC,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAoB,EAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB;QACnC,IAAI,CAAC;YACH,+BAA+B;YAC/B,IAAA,wBAAQ,EAAC,sBAAsB,EAAE;gBAC/B,GAAG,EAAE,IAAI,CAAC,aAAa;gBACvB,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,KAAwB,EACxB,QAAsB;QAEtB,MAAM,YAAY,GAAG,QAAQ,CAAC,kBAAkB,CAAC,MAAM,CAAC;QAExD,8BAA8B;QAC9B,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;YAC3B,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,EAAE,OAAO;gBAClB,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,QAAQ;aACxB,CAAC;QACJ,CAAC;QAED,+BAA+B;QAC/B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACzD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;YACxE,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC;gBACT,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,QAAQ;gBACvB,KAAK,EAAE,kBAAkB;aAC1B,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAA,iCAAgB,EAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YAClE,MAAM,UAAU,GAAgB,EAAE,CAAC;YAEnC,0BAA0B;YAC1B,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,wBAAwB;gBACxB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjD,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClE,SAAS;gBACX,CAAC;gBAED,IAAI,CAAC;oBACH,8BAA8B;oBAC9B,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU;wBACvC,CAAC,CAAC,YAAY,YAAY,CAAC,UAAU,EAAE;wBACvC,CAAC,CAAC,EAAE,CAAC;oBAEP,MAAM,MAAM,GAAG,IAAA,wBAAQ,EACrB,cAAc,SAAS,mBAAmB,QAAQ,GAAG,EACrD;wBACE,GAAG,EAAE,IAAI,CAAC,aAAa;wBACvB,QAAQ,EAAE,OAAO;wBACjB,KAAK,EAAE,MAAM;qBACd,CACF,CAAC;oBAEF,sBAAsB;oBACtB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;oBAEjE,yCAAyC;oBACzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC7B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;4BAC5C,MAAM,QAAQ,GACZ,OAAO,CAAC,QAAQ,KAAK,CAAC;gCACpB,CAAC,CAAC,OAAO;gCACT,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC;oCACtB,CAAC,CAAC,OAAO;oCACT,CAAC,CAAC,OAAO,CAAC;4BAEhB,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,QAAQ;gCACd,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE;gCAChD,IAAI,EAAE,YAAY;gCAClB,IAAI,EAAE,OAAO,CAAC,IAAI;gCAClB,QAAQ,EAAE,QAA6B;gCACvC,OAAO,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;6BACpC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,sDAAsD;oBACtD,gCAAgC;oBAChC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;wBACjB,IAAI,CAAC;4BACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;4BACzC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;4BAEjE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gCAC7B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oCAC5C,MAAM,QAAQ,GACZ,OAAO,CAAC,QAAQ,KAAK,CAAC;wCACpB,CAAC,CAAC,OAAO;wCACT,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC;4CACtB,CAAC,CAAC,OAAO;4CACT,CAAC,CAAC,OAAO,CAAC;oCAEhB,UAAU,CAAC,IAAI,CAAC;wCACd,IAAI,EAAE,QAAQ;wCACd,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE;wCAChD,IAAI,EAAE,YAAY;wCAClB,IAAI,EAAE,OAAO,CAAC,IAAI;wCAClB,QAAQ,EAAE,QAA6B;wCACvC,OAAO,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;qCACpC,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;wBACH,CAAC;wBAAC,MAAM,CAAC;4BACP,4DAA4D;4BAC5D,UAAU,CAAC,IAAI,CAAC;gCACd,IAAI,EAAE,QAAQ;gCACd,OAAO,EAAE,qBAAqB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gCACvD,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC;gCACjD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gCAC3B,OAAO,EAAE,KAAK,CAAC,OAAO;6BACvB,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,sCAAsC;YACtC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YAE9C,OAAO;gBACL,MAAM;gBACN,KAAK;gBACL,UAAU;gBACV,aAAa,EAAE,QAAQ;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,EAAE;gBACd,aAAa,EAAE,QAAQ;gBACvB,KAAK,EAAE,6BAA6B,KAAK,EAAE;aAC5C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,UAAuB;QAC5C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,CAAC;QACb,CAAC;QAED,gCAAgC;QAChC,MAAM,OAAO,GAAG;YACd,QAAQ,EAAE,GAAG;YACb,KAAK,EAAE,GAAG;YACV,KAAK,EAAE,GAAG;SACX,CAAC;QAEF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YAC/C,OAAO,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,gDAAgD;QAChD,sDAAsD;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;QAEzC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;CACF;AAtMD,0CAsMC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "coding-agent-benchmarks",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.2",
|
|
4
4
|
"description": "Testing coding agents (GitHub Copilot CLI, Claude Code, etc.) with your repo's context to evaluate their code generation quality.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
"scripts": {
|
|
11
11
|
"build": "tsc",
|
|
12
12
|
"dev": "tsx src/runner.ts",
|
|
13
|
+
"lint": "eslint .",
|
|
14
|
+
"lint:fix": "eslint . --fix",
|
|
13
15
|
"prepublishOnly": "npm run build",
|
|
14
16
|
"test": "echo \"Tests not implemented yet\" && exit 0"
|
|
15
17
|
},
|
|
@@ -57,8 +59,12 @@
|
|
|
57
59
|
"log-update": "^4.0.0"
|
|
58
60
|
},
|
|
59
61
|
"devDependencies": {
|
|
62
|
+
"@eslint/js": "^9.39.2",
|
|
60
63
|
"@types/node": "^25.0.9",
|
|
64
|
+
"@typescript-eslint/eslint-plugin": "^8.20.0",
|
|
65
|
+
"@typescript-eslint/parser": "^8.20.0",
|
|
61
66
|
"eslint": "^9.39.2",
|
|
67
|
+
"globals": "^17.1.0",
|
|
62
68
|
"prettier": "^3.8.0",
|
|
63
69
|
"tsup": "^8.5.1",
|
|
64
70
|
"tsx": "^4.21.0",
|