@kevinrabun/judges 3.28.0 → 3.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +125 -0
- package/dist/cli.js.map +1 -1
- package/dist/commands/calibration-dashboard.d.ts +2 -0
- package/dist/commands/calibration-dashboard.d.ts.map +1 -0
- package/dist/commands/calibration-dashboard.js +97 -0
- package/dist/commands/calibration-dashboard.js.map +1 -0
- package/dist/commands/community-patterns.d.ts +2 -0
- package/dist/commands/community-patterns.d.ts.map +1 -0
- package/dist/commands/community-patterns.js +132 -0
- package/dist/commands/community-patterns.js.map +1 -0
- package/dist/commands/diff.d.ts.map +1 -1
- package/dist/commands/diff.js +91 -0
- package/dist/commands/diff.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +26 -0
- package/dist/config.js.map +1 -1
- package/dist/evaluators/api-contract.d.ts +10 -0
- package/dist/evaluators/api-contract.d.ts.map +1 -0
- package/dist/evaluators/api-contract.js +181 -0
- package/dist/evaluators/api-contract.js.map +1 -0
- package/dist/evaluators/index.d.ts.map +1 -1
- package/dist/evaluators/index.js +33 -1
- package/dist/evaluators/index.js.map +1 -1
- package/dist/evaluators/intent-alignment.d.ts +15 -0
- package/dist/evaluators/intent-alignment.d.ts.map +1 -0
- package/dist/evaluators/intent-alignment.js +233 -0
- package/dist/evaluators/intent-alignment.js.map +1 -0
- package/dist/evaluators/model-fingerprint.d.ts +3 -0
- package/dist/evaluators/model-fingerprint.d.ts.map +1 -0
- package/dist/evaluators/model-fingerprint.js +152 -0
- package/dist/evaluators/model-fingerprint.js.map +1 -0
- package/dist/evaluators/multi-turn-coherence.d.ts +14 -0
- package/dist/evaluators/multi-turn-coherence.d.ts.map +1 -0
- package/dist/evaluators/multi-turn-coherence.js +171 -0
- package/dist/evaluators/multi-turn-coherence.js.map +1 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/judges/api-contract.d.ts +3 -0
- package/dist/judges/api-contract.d.ts.map +1 -0
- package/dist/judges/api-contract.js +28 -0
- package/dist/judges/api-contract.js.map +1 -0
- package/dist/judges/index.d.ts.map +1 -1
- package/dist/judges/index.js +16 -0
- package/dist/judges/index.js.map +1 -1
- package/dist/judges/intent-alignment.d.ts +3 -0
- package/dist/judges/intent-alignment.d.ts.map +1 -0
- package/dist/judges/intent-alignment.js +28 -0
- package/dist/judges/intent-alignment.js.map +1 -0
- package/dist/judges/model-fingerprint.d.ts +3 -0
- package/dist/judges/model-fingerprint.d.ts.map +1 -0
- package/dist/judges/model-fingerprint.js +30 -0
- package/dist/judges/model-fingerprint.js.map +1 -0
- package/dist/judges/multi-turn-coherence.d.ts +3 -0
- package/dist/judges/multi-turn-coherence.d.ts.map +1 -0
- package/dist/judges/multi-turn-coherence.js +27 -0
- package/dist/judges/multi-turn-coherence.js.map +1 -0
- package/dist/patches/index.d.ts.map +1 -1
- package/dist/patches/index.js +372 -0
- package/dist/patches/index.js.map +1 -1
- package/dist/presets.d.ts.map +1 -1
- package/dist/presets.js +76 -0
- package/dist/presets.js.map +1 -1
- package/dist/types.d.ts +44 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
- package/server.json +3 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"intent-alignment.js","sourceRoot":"","sources":["../../src/evaluators/intent-alignment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY,EAAE,SAAiB;IACpE,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC;IAExB,4EAA4E;IAC5E,8EAA8E;IAC9E,MAAM,WAAW,GACf,4PAA4P,CAAC;IAC/P,MAAM,YAAY,GAChB,oPAAoP,CAAC;IACvP,MAAM,aAAa,GACjB,gLAAgL,CAAC;IAEnL,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC5C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAChG,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,yDAAyD;QACzD,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAChC,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,EAAE;gBAAE,SAAS;YAChD,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI;gBAAE,MAAM;YACnE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEtC,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,GAAG,MAAM,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE;gBACjD,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ;gBAC5C,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,8BAA8B,MAAM,MAAM,CAAC,CAAC,CAAC,oBAAoB,MAAM,MAAM;gBACjG,WAAW,EAAE,UAAU;oBACrB,CAAC,CAAC,cAAc,MAAM,0EAA0E;wBAC9F,0FAA0F;oBAC5F,CAAC,CAAC,cAAc,MAAM,iEAAiE;wBACrF,kEAAkE;gBACtE,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBACpB,cAAc,EAAE,UAAU;oBACxB,CAAC,CAAC,iBAAiB,MAAM,yGAAyG;oBAClI,CAAC,CAAC,oIAAoI;gBACxI,SAAS,EAAE,4CAA4C;gBACvD,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;gBACpC,UAAU,EAAE,kBAAkB;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,MAAM,cAAc,GAClB,0JAA0J,CAAC;IAE7J,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,6CAA6C;QAC7C,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3D,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAExC,sBAAsB;QACtB,MAAM,OAAO,GACX,iGAAiG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/G,IAAI,OAAO,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,yCAAyC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5F,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,GAAG,MAAM,MAAM;gBACvB,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,0BAA0B,MAAM,MAAM;gBAC7C,WAAW,EACT,cAAc,MAAM,kFAAkF;oBACtG,sFAAsF;gBACxF,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBACpB,cAAc,EACZ,4JAA4J;gBAC9J,SAAS,EAAE,8CAA8C;gBACzD,UAAU,EAAE,IAAI;gBAChB,UAAU,EAAE,kBAAkB;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,MAAM,YAAY,GAChB,6HAA6H,CAAC;IAEhI,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAS;QAEpD,6DAA6D;QAC7D,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAChC,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,EAAE;gBAAE,SAAS;YAChD,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI;gBAAE,MAAM;YACnE,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;QAED,IACE,SAAS,CAAC,MAAM,KAAK,CAAC;YACtB,8FAA8F,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EACjH,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,GAAG,MAAM,MAAM;gBACvB,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,2BAA2B,MAAM,MAAM;gBAC9C,WAAW,EACT,cAAc,MAAM,8DAA8D;oBAClF,oDAAoD,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK;gBAC9E,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBACpB,cAAc,EACZ,uFAAuF;oBACvF,yCAAyC;gBAC3C,SAAS,EAAE,qCAAqC;gBAChD,UAAU,EAAE,GAAG;gBACf,UAAU,EAAE,kBAAkB;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,uEAAuE;IACvE,MAAM,UAAU,GAAG,qBAAqB,CAAC;IACzC,IAAI,UAAU,CAAC;IACf,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACtE,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE,CAAC;YAC3E,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAErC,qDAAqD;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QACpH,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;QAChH,IAAI,CAAC,QAAQ;YAAE,SAAS;QAExB,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC;aAC7B,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACT,CAAC;aACE,IAAI,EAAE;aACN,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;aACtB,IAAI,EAAE,CACV;aACA,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACrE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM,EAAE,GAAG,MAAM,MAAM;gBACvB,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,mDAAmD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9E,WAAW,EACT,sCAAsC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;oBAClF,wCAAwC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,GAAG;gBACvG,WAAW,EAAE,CAAC,UAAU,CAAC;gBACzB,cAAc,EAAE,yFAAyF;gBACzG,SAAS,EAAE,mCAAmC;gBAC9C,UAAU,EAAE,GAAG;gBACf,UAAU,EAAE,kBAAkB;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,MAAM,iBAAiB,GACrB,0PAA0P,CAAC;IAE7P,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ;YAAE,SAAS;QAExB,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,gCAAgC;QAChC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvB,SAAS,GAAG,IAAI,CAAC;gBACjB,UAAU,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACjD,CAAC;YACD,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvB,UAAU,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;YACjD,CAAC;YACD,IAAI,SAAS;gBAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,SAAS,IAAI,UAAU,IAAI,CAAC;gBAAE,MAAM;QAC1C,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElC,6DAA6D;QAC7D,MAAM,gBAAgB,GACpB,yLAAyL,CAAC,IAAI,CAC5L,IAAI,CACL,IAAI,iFAAiF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpG,iFAAiF;QACjF,IAAI,CAAC,gBAAgB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,wEAAwE;YACxE,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClD,QAAQ,CAAC,IAAI,CAAC;oBACZ,MAAM,EAAE,GAAG,MAAM,MAAM;oBACvB,QAAQ,EAAE,MAAM;oBAChB,KAAK,EAAE,sBAAsB,MAAM,2BAA2B;oBAC9D,WAAW,EACT,cAAc,MAAM,oDAAoD;wBACxE,8GAA8G;oBAChH,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;oBACpB,cAAc,EACZ,2BAA2B,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU;wBAClG,4DAA4D;oBAC9D,SAAS,EAAE,yCAAyC;oBACpD,UAAU,EAAE,IAAI;oBAChB,UAAU,EAAE,kBAAkB;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-fingerprint.d.ts","sourceRoot":"","sources":["../../src/evaluators/model-fingerprint.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AA6G3C,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,EAAE,CAmElF"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// ─── Model-Specific AI Fingerprint Detection ─────────────────────────────────
|
|
2
|
+
// Detects stylistic fingerprints and patterns characteristic of specific AI
|
|
3
|
+
// code generators (ChatGPT/GPT-4, Claude, Copilot, Gemini, etc.). These are
|
|
4
|
+
// not security vulnerabilities — they are signals that code was generated by a
|
|
5
|
+
// particular model and may carry that model's known biases or blind spots.
|
|
6
|
+
// ──────────────────────────────────────────────────────────────────────────────
|
|
7
|
+
import { isCommentLine } from "./shared.js";
|
|
8
|
+
const FINGERPRINT_SIGNALS = [
|
|
9
|
+
// ── ChatGPT / GPT-4 signatures ──────────────────────────────────────────
|
|
10
|
+
{
|
|
11
|
+
pattern: /\/\/\s*(?:Example|Note|Remember|Important):\s/,
|
|
12
|
+
model: "ChatGPT/GPT-4",
|
|
13
|
+
description: 'Imperative tutorial-style comments ("Note:", "Remember:", "Important:")',
|
|
14
|
+
weight: 0.3,
|
|
15
|
+
checkComments: true,
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
pattern: /\/\/\s*This\s+(?:function|method|class|variable|constant|module)\s+(?:is|will|does|handles|represents)/,
|
|
19
|
+
model: "ChatGPT/GPT-4",
|
|
20
|
+
description: 'Verbose "This function does X" documentation style in inline comments',
|
|
21
|
+
weight: 0.3,
|
|
22
|
+
checkComments: true,
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
pattern: /\/\/ (?:Step \d|First,|Second,|Third,|Next,|Finally,|Then,)/,
|
|
26
|
+
model: "ChatGPT/GPT-4",
|
|
27
|
+
description: 'Sequential step-numbered comments ("Step 1:", "First,", "Then,")',
|
|
28
|
+
weight: 0.4,
|
|
29
|
+
checkComments: true,
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
pattern: /console\.log\s*\(\s*['"`](?:Result|Output|Data|Response|Error|Success|Value|Hello):/,
|
|
33
|
+
model: "ChatGPT/GPT-4",
|
|
34
|
+
description: "Demonstration console.log with label-colon pattern",
|
|
35
|
+
weight: 0.2,
|
|
36
|
+
checkComments: true,
|
|
37
|
+
},
|
|
38
|
+
// ── Copilot signatures ───────────────────────────────────────────────────
|
|
39
|
+
{
|
|
40
|
+
pattern: /\/\/\s*TODO:\s*(?:implement|add|handle|fix|replace)\s/i,
|
|
41
|
+
model: "Copilot",
|
|
42
|
+
description: "Placeholder TODO stubs with generic verbs",
|
|
43
|
+
weight: 0.2,
|
|
44
|
+
checkComments: true,
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
pattern: /(?:\/\/|#)\s*(?:Copilot|GitHub Copilot|Generated by Copilot)/i,
|
|
48
|
+
model: "Copilot",
|
|
49
|
+
description: "Explicit Copilot attribution comment",
|
|
50
|
+
weight: 0.9,
|
|
51
|
+
checkComments: true,
|
|
52
|
+
},
|
|
53
|
+
// ── Claude / Anthropic signatures ────────────────────────────────────────
|
|
54
|
+
{
|
|
55
|
+
pattern: /\/\/\s*(?:Let me|I'll|Here's|Here is|Let's)\s/,
|
|
56
|
+
model: "Claude",
|
|
57
|
+
description: 'Conversational first-person comments ("Let me", "I\'ll", "Here\'s")',
|
|
58
|
+
weight: 0.4,
|
|
59
|
+
checkComments: true,
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
pattern: /\/\*\*\s*\n\s*\*\s*@(?:description|summary)\s+[A-Z][^.]+\.\s*\n\s*\*\s*\n\s*\*\s*@(?:param|returns)/,
|
|
63
|
+
model: "Claude",
|
|
64
|
+
description: "Dense multi-paragraph JSDoc with blank-line separators",
|
|
65
|
+
weight: 0.3,
|
|
66
|
+
checkComments: true,
|
|
67
|
+
},
|
|
68
|
+
// ── Gemini / Google signatures ───────────────────────────────────────────
|
|
69
|
+
{
|
|
70
|
+
pattern: /\/\/\s*(?:See|Refer to|For more information|Documentation):\s*https?:\/\//,
|
|
71
|
+
model: "Gemini",
|
|
72
|
+
description: "Inline URL documentation references",
|
|
73
|
+
weight: 0.3,
|
|
74
|
+
checkComments: true,
|
|
75
|
+
},
|
|
76
|
+
// ── Generic AI code signals (model-agnostic) ────────────────────────────
|
|
77
|
+
{
|
|
78
|
+
pattern: /(?:\/\/|#)\s*(?:Generated by|Auto-generated|AI[ -]generated|Created by AI|Written by AI)/i,
|
|
79
|
+
model: "AI-generated",
|
|
80
|
+
description: "Explicit AI-generation attribution comment",
|
|
81
|
+
weight: 0.9,
|
|
82
|
+
checkComments: true,
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
pattern: /\/\/\s*[-─]{3,}\s*\w+.*[-─]{3,}/,
|
|
86
|
+
model: "AI-generated",
|
|
87
|
+
description: "Decorative section-divider comments with dashes",
|
|
88
|
+
weight: 0.2,
|
|
89
|
+
checkComments: true,
|
|
90
|
+
},
|
|
91
|
+
];
|
|
92
|
+
// ─── Main Analyzer ──────────────────────────────────────────────────────────
|
|
93
|
+
export function analyzeModelFingerprint(code, _language) {
|
|
94
|
+
const findings = [];
|
|
95
|
+
const lines = code.split("\n");
|
|
96
|
+
const prefix = "MFPR";
|
|
97
|
+
// Accumulate signals per model
|
|
98
|
+
const modelSignals = new Map();
|
|
99
|
+
for (const signal of FINGERPRINT_SIGNALS) {
|
|
100
|
+
const matchedLines = [];
|
|
101
|
+
for (let i = 0; i < lines.length; i++) {
|
|
102
|
+
const line = lines[i];
|
|
103
|
+
const isComment = isCommentLine(line);
|
|
104
|
+
// Only check comments for comment-only signals; check non-comments for code signals
|
|
105
|
+
if (signal.checkComments && !isComment)
|
|
106
|
+
continue;
|
|
107
|
+
if (!signal.checkComments && isComment)
|
|
108
|
+
continue;
|
|
109
|
+
signal.pattern.lastIndex = 0;
|
|
110
|
+
if (signal.pattern.test(line)) {
|
|
111
|
+
matchedLines.push(i + 1);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (matchedLines.length > 0) {
|
|
115
|
+
const existing = modelSignals.get(signal.model) ?? { lines: [], totalWeight: 0, descriptions: [] };
|
|
116
|
+
existing.lines.push(...matchedLines);
|
|
117
|
+
existing.totalWeight += signal.weight * Math.min(matchedLines.length, 3);
|
|
118
|
+
if (!existing.descriptions.includes(signal.description)) {
|
|
119
|
+
existing.descriptions.push(signal.description);
|
|
120
|
+
}
|
|
121
|
+
modelSignals.set(signal.model, existing);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
// Emit findings for models with sufficient signal accumulation
|
|
125
|
+
let ruleNum = 1;
|
|
126
|
+
for (const [model, data] of modelSignals) {
|
|
127
|
+
// Require at least 2 distinct signal types OR one high-weight signal
|
|
128
|
+
if (data.descriptions.length < 2 && data.totalWeight < 0.8)
|
|
129
|
+
continue;
|
|
130
|
+
const uniqueLines = [...new Set(data.lines)].sort((a, b) => a - b);
|
|
131
|
+
const confidence = Math.min(0.9, 0.4 + data.totalWeight * 0.15);
|
|
132
|
+
findings.push({
|
|
133
|
+
ruleId: `${prefix}-${String(ruleNum++).padStart(3, "0")}`,
|
|
134
|
+
severity: "info",
|
|
135
|
+
title: `Code fingerprint matches ${model} generation patterns`,
|
|
136
|
+
description: `Detected ${data.descriptions.length} stylistic signal(s) characteristic of ${model}-generated code: ` +
|
|
137
|
+
data.descriptions.join("; ") +
|
|
138
|
+
". This is informational — AI-generated code should receive extra review for hallucinated APIs, " +
|
|
139
|
+
"placeholder security comments, and model-specific blind spots.",
|
|
140
|
+
lineNumbers: uniqueLines.slice(0, 10),
|
|
141
|
+
recommendation: `This code shows patterns typical of ${model}. Review carefully for: ` +
|
|
142
|
+
"(1) hallucinated imports/APIs, (2) placeholder security comments, " +
|
|
143
|
+
"(3) incomplete error handling, (4) overly verbose or missing documentation. " +
|
|
144
|
+
"Consider running the hallucination-detection and ai-code-safety judges specifically.",
|
|
145
|
+
reference: "AI Code Safety — Model Fingerprint Detection",
|
|
146
|
+
confidence,
|
|
147
|
+
provenance: "regex-pattern-match",
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
return findings;
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=model-fingerprint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-fingerprint.js","sourceRoot":"","sources":["../../src/evaluators/model-fingerprint.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,4EAA4E;AAC5E,6EAA6E;AAC7E,+EAA+E;AAC/E,2EAA2E;AAC3E,iFAAiF;AAGjF,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAiB5C,MAAM,mBAAmB,GAAwB;IAC/C,2EAA2E;IAC3E;QACE,OAAO,EAAE,+CAA+C;QACxD,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,yEAAyE;QACtF,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IACD;QACE,OAAO,EAAE,wGAAwG;QACjH,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,uEAAuE;QACpF,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IACD;QACE,OAAO,EAAE,6DAA6D;QACtE,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,kEAAkE;QAC/E,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IACD;QACE,OAAO,EAAE,qFAAqF;QAC9F,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,oDAAoD;QACjE,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IAED,4EAA4E;IAC5E;QACE,OAAO,EAAE,wDAAwD;QACjE,KAAK,EAAE,SAAS;QAChB,WAAW,EAAE,2CAA2C;QACxD,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IACD;QACE,OAAO,EAAE,+DAA+D;QACxE,KAAK,EAAE,SAAS;QAChB,WAAW,EAAE,sCAAsC;QACnD,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IAED,4EAA4E;IAC5E;QACE,OAAO,EAAE,+CAA+C;QACxD,KAAK,EAAE,QAAQ;QACf,WAAW,EAAE,qEAAqE;QAClF,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IACD;QACE,OAAO,EAAE,qGAAqG;QAC9G,KAAK,EAAE,QAAQ;QACf,WAAW,EAAE,wDAAwD;QACrE,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IAED,4EAA4E;IAC5E;QACE,OAAO,EAAE,2EAA2E;QACpF,KAAK,EAAE,QAAQ;QACf,WAAW,EAAE,qCAAqC;QAClD,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IAED,2EAA2E;IAC3E;QACE,OAAO,EAAE,2FAA2F;QACpG,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,4CAA4C;QACzD,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;IACD;QACE,OAAO,EAAE,iCAAiC;QAC1C,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,iDAAiD;QAC9D,MAAM,EAAE,GAAG;QACX,aAAa,EAAE,IAAI;KACpB;CACF,CAAC;AAEF,+EAA+E;AAE/E,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,SAAiB;IACrE,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,MAAM,CAAC;IAEtB,+BAA+B;IAC/B,MAAM,YAAY,GAAkF,IAAI,GAAG,EAAE,CAAC;IAE9G,KAAK,MAAM,MAAM,IAAI,mBAAmB,EAAE,CAAC;QACzC,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;YAEtC,oFAAoF;YACpF,IAAI,MAAM,CAAC,aAAa,IAAI,CAAC,SAAS;gBAAE,SAAS;YACjD,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,SAAS;gBAAE,SAAS;YAEjD,MAAM,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;YAC7B,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9B,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;YACnG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YACrC,QAAQ,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACzE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxD,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACjD,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,YAAY,EAAE,CAAC;QACzC,qEAAqE;QACrE,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,GAAG,GAAG;YAAE,SAAS;QAErE,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC;QAEhE,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzD,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,4BAA4B,KAAK,sBAAsB;YAC9D,WAAW,EACT,YAAY,IAAI,CAAC,YAAY,CAAC,MAAM,0CAA0C,KAAK,mBAAmB;gBACtG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,iGAAiG;gBACjG,gEAAgE;YAClE,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACrC,cAAc,EACZ,uCAAuC,KAAK,0BAA0B;gBACtE,oEAAoE;gBACpE,8EAA8E;gBAC9E,sFAAsF;YACxF,SAAS,EAAE,8CAA8C;YACzD,UAAU;YACV,UAAU,EAAE,qBAAqB;SAClC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Finding } from "../types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Multi-turn coherence evaluator.
|
|
4
|
+
*
|
|
5
|
+
* Detects self-contradicting patterns within a single code file that suggest
|
|
6
|
+
* incomplete refactoring, copy-paste inconsistencies, or conflicting logic:
|
|
7
|
+
* - Duplicate function definitions with different implementations
|
|
8
|
+
* - Contradictory boolean assignments to the same variable
|
|
9
|
+
* - Dead code after unconditional return/throw
|
|
10
|
+
* - Import statements for unused modules alongside re-implementations
|
|
11
|
+
* - Conflicting configuration values
|
|
12
|
+
*/
|
|
13
|
+
export declare function analyzeMultiTurnCoherence(code: string, _language: string): Finding[];
|
|
14
|
+
//# sourceMappingURL=multi-turn-coherence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multi-turn-coherence.d.ts","sourceRoot":"","sources":["../../src/evaluators/multi-turn-coherence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAG3C;;;;;;;;;;GAUG;AACH,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,EAAE,CAyKpF"}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { getLineNumbers } from "./shared.js";
|
|
2
|
+
/**
|
|
3
|
+
* Multi-turn coherence evaluator.
|
|
4
|
+
*
|
|
5
|
+
* Detects self-contradicting patterns within a single code file that suggest
|
|
6
|
+
* incomplete refactoring, copy-paste inconsistencies, or conflicting logic:
|
|
7
|
+
* - Duplicate function definitions with different implementations
|
|
8
|
+
* - Contradictory boolean assignments to the same variable
|
|
9
|
+
* - Dead code after unconditional return/throw
|
|
10
|
+
* - Import statements for unused modules alongside re-implementations
|
|
11
|
+
* - Conflicting configuration values
|
|
12
|
+
*/
|
|
13
|
+
export function analyzeMultiTurnCoherence(code, _language) {
|
|
14
|
+
const findings = [];
|
|
15
|
+
const prefix = "COH";
|
|
16
|
+
let ruleNum = 1;
|
|
17
|
+
const lines = code.split("\n");
|
|
18
|
+
// ── COH-001: Contradictory boolean assignments ────────────────────────
|
|
19
|
+
const boolAssignPattern = /\b(\w+)\s*=\s*(true|false)\s*;/g;
|
|
20
|
+
const boolAssignments = new Map();
|
|
21
|
+
let match;
|
|
22
|
+
while ((match = boolAssignPattern.exec(code)) !== null) {
|
|
23
|
+
const varName = match[1];
|
|
24
|
+
const value = match[2];
|
|
25
|
+
const lineNum = code.slice(0, match.index).split("\n").length;
|
|
26
|
+
if (!boolAssignments.has(varName))
|
|
27
|
+
boolAssignments.set(varName, []);
|
|
28
|
+
boolAssignments.get(varName).push({ value, line: lineNum });
|
|
29
|
+
}
|
|
30
|
+
const contradictoryVars = [];
|
|
31
|
+
for (const [, assignments] of boolAssignments) {
|
|
32
|
+
if (assignments.length >= 2) {
|
|
33
|
+
const values = new Set(assignments.map((a) => a.value));
|
|
34
|
+
if (values.size > 1) {
|
|
35
|
+
// Same variable assigned both true and false — check they're at the same scope level
|
|
36
|
+
// (simple heuristic: within 5 lines of each other with no control flow between)
|
|
37
|
+
for (let i = 0; i < assignments.length - 1; i++) {
|
|
38
|
+
const a = assignments[i];
|
|
39
|
+
const b = assignments[i + 1];
|
|
40
|
+
if (a.value !== b.value && b.line - a.line <= 5) {
|
|
41
|
+
const between = lines.slice(a.line - 1, b.line).join("\n");
|
|
42
|
+
if (!/\b(if|else|switch|case|for|while|catch|try)\b/.test(between)) {
|
|
43
|
+
contradictoryVars.push(a.line, b.line);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (contradictoryVars.length > 0) {
|
|
51
|
+
findings.push({
|
|
52
|
+
ruleId: `${prefix}-${String(ruleNum++).padStart(3, "0")}`,
|
|
53
|
+
severity: "medium",
|
|
54
|
+
title: "Contradictory boolean assignments",
|
|
55
|
+
description: "The same variable is assigned both true and false in close proximity without " +
|
|
56
|
+
"intervening control flow, suggesting incomplete refactoring or a logic error.",
|
|
57
|
+
lineNumbers: [...new Set(contradictoryVars)],
|
|
58
|
+
recommendation: "Review the variable assignments and remove the stale one, or add proper " +
|
|
59
|
+
"conditional logic if both assignments are intentional.",
|
|
60
|
+
confidence: 0.7,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
// ── COH-002: Dead code after unconditional return/throw ───────────────
|
|
64
|
+
ruleNum = 2;
|
|
65
|
+
const deadCodeLines = [];
|
|
66
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
67
|
+
const line = lines[i].trim();
|
|
68
|
+
const nextLine = lines[i + 1]?.trim() || "";
|
|
69
|
+
// Unconditional return/throw not inside a ternary or short-circuit
|
|
70
|
+
if (/^(return\b|throw\b)/.test(line) && !line.includes("?") && !line.includes("&&")) {
|
|
71
|
+
// Next line is code (not a closing brace, comment, or blank)
|
|
72
|
+
if (nextLine && !/^[}\])]/.test(nextLine) && !/^\/[/*]/.test(nextLine) && !/^$/.test(nextLine)) {
|
|
73
|
+
deadCodeLines.push(i + 2); // +2 because 0-indexed +1 for line number +1 for next line
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
if (deadCodeLines.length > 0) {
|
|
78
|
+
findings.push({
|
|
79
|
+
ruleId: `${prefix}-${String(ruleNum++).padStart(3, "0")}`,
|
|
80
|
+
severity: "medium",
|
|
81
|
+
title: "Unreachable code after return/throw",
|
|
82
|
+
description: "Code exists after an unconditional return or throw statement. " +
|
|
83
|
+
"This code will never execute and may indicate incomplete refactoring.",
|
|
84
|
+
lineNumbers: deadCodeLines.slice(0, 5),
|
|
85
|
+
recommendation: "Remove the unreachable code or restructure the logic so the code " +
|
|
86
|
+
"is reached under the intended conditions.",
|
|
87
|
+
confidence: 0.75,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
// ── COH-003: Duplicate function definitions ───────────────────────────
|
|
91
|
+
ruleNum = 3;
|
|
92
|
+
const funcDefPattern = /^(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(/gm;
|
|
93
|
+
const funcDefs = new Map();
|
|
94
|
+
while ((match = funcDefPattern.exec(code)) !== null) {
|
|
95
|
+
const name = match[1];
|
|
96
|
+
const lineNum = code.slice(0, match.index).split("\n").length;
|
|
97
|
+
if (!funcDefs.has(name))
|
|
98
|
+
funcDefs.set(name, []);
|
|
99
|
+
funcDefs.get(name).push(lineNum);
|
|
100
|
+
}
|
|
101
|
+
const duplicateFuncLines = [];
|
|
102
|
+
for (const [, defLines] of funcDefs) {
|
|
103
|
+
if (defLines.length >= 2) {
|
|
104
|
+
duplicateFuncLines.push(...defLines);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (duplicateFuncLines.length > 0) {
|
|
108
|
+
findings.push({
|
|
109
|
+
ruleId: `${prefix}-${String(ruleNum++).padStart(3, "0")}`,
|
|
110
|
+
severity: "high",
|
|
111
|
+
title: "Duplicate function definitions",
|
|
112
|
+
description: "Multiple functions with the same name are defined in this file. " +
|
|
113
|
+
"Only the last definition will be effective, which may cause unexpected behaviour.",
|
|
114
|
+
lineNumbers: duplicateFuncLines,
|
|
115
|
+
recommendation: "Remove or rename the duplicate function definitions. If they serve " +
|
|
116
|
+
"different purposes, give them distinct names.",
|
|
117
|
+
confidence: 0.85,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
// ── COH-004: Conflicting config values ────────────────────────────────
|
|
121
|
+
ruleNum = 4;
|
|
122
|
+
const configPattern = /(['"])([\w.]+)\1\s*:\s*(['"]?)([^'",}\]\n]+)\3/g;
|
|
123
|
+
const configValues = new Map();
|
|
124
|
+
while ((match = configPattern.exec(code)) !== null) {
|
|
125
|
+
const key = match[2];
|
|
126
|
+
const value = match[4].trim();
|
|
127
|
+
const lineNum = code.slice(0, match.index).split("\n").length;
|
|
128
|
+
if (!configValues.has(key))
|
|
129
|
+
configValues.set(key, []);
|
|
130
|
+
configValues.get(key).push({ value, line: lineNum });
|
|
131
|
+
}
|
|
132
|
+
const conflictLines = [];
|
|
133
|
+
for (const [, vals] of configValues) {
|
|
134
|
+
if (vals.length >= 2) {
|
|
135
|
+
const uniqueValues = new Set(vals.map((v) => v.value));
|
|
136
|
+
if (uniqueValues.size > 1) {
|
|
137
|
+
conflictLines.push(...vals.map((v) => v.line));
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (conflictLines.length > 0 && conflictLines.length <= 10) {
|
|
142
|
+
findings.push({
|
|
143
|
+
ruleId: `${prefix}-${String(ruleNum++).padStart(3, "0")}`,
|
|
144
|
+
severity: "low",
|
|
145
|
+
title: "Potentially conflicting configuration values",
|
|
146
|
+
description: "The same configuration key appears multiple times with different values. " +
|
|
147
|
+
"This may indicate a copy-paste error or conflicting settings.",
|
|
148
|
+
lineNumbers: [...new Set(conflictLines)].slice(0, 5),
|
|
149
|
+
recommendation: "Verify that each configuration key has the intended value and remove duplicates.",
|
|
150
|
+
confidence: 0.5,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
// ── COH-005: TODO/FIXME/HACK with contradicting code ─────────────────
|
|
154
|
+
ruleNum = 5;
|
|
155
|
+
const todoLines = getLineNumbers(code, /\/\/\s*(TODO|FIXME|HACK|XXX)\b/i);
|
|
156
|
+
if (todoLines.length >= 5) {
|
|
157
|
+
findings.push({
|
|
158
|
+
ruleId: `${prefix}-${String(ruleNum++).padStart(3, "0")}`,
|
|
159
|
+
severity: "info",
|
|
160
|
+
title: "High density of TODO/FIXME markers",
|
|
161
|
+
description: `${todoLines.length} TODO/FIXME/HACK markers found. High density of unresolved ` +
|
|
162
|
+
"markers suggests incomplete implementation or deferred work that may affect reliability.",
|
|
163
|
+
lineNumbers: todoLines.slice(0, 5),
|
|
164
|
+
recommendation: "Prioritize and address the TODO/FIXME items, or create tracked issues " +
|
|
165
|
+
"for each one and remove stale markers.",
|
|
166
|
+
confidence: 0.8,
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
return findings;
|
|
170
|
+
}
|
|
171
|
+
//# sourceMappingURL=multi-turn-coherence.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multi-turn-coherence.js","sourceRoot":"","sources":["../../src/evaluators/multi-turn-coherence.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C;;;;;;;;;;GAUG;AACH,MAAM,UAAU,yBAAyB,CAAC,IAAY,EAAE,SAAiB;IACvE,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAE/B,yEAAyE;IACzE,MAAM,iBAAiB,GAAG,iCAAiC,CAAC;IAC5D,MAAM,eAAe,GAAG,IAAI,GAAG,EAA6C,CAAC;IAC7E,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC9D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACpE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,EAAE,WAAW,CAAC,IAAI,eAAe,EAAE,CAAC;QAC9C,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACpB,qFAAqF;gBACrF,gFAAgF;gBAChF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAChD,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;oBACzB,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC7B,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;wBAChD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAC3D,IAAI,CAAC,+CAA+C,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;4BACnE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;wBACzC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzD,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,mCAAmC;YAC1C,WAAW,EACT,+EAA+E;gBAC/E,+EAA+E;YACjF,WAAW,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC;YAC5C,cAAc,EACZ,0EAA0E;gBAC1E,wDAAwD;YAC1D,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,OAAO,GAAG,CAAC,CAAC;IACZ,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC5C,mEAAmE;QACnE,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpF,6DAA6D;YAC7D,IAAI,QAAQ,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/F,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,2DAA2D;YACxF,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzD,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,qCAAqC;YAC5C,WAAW,EACT,gEAAgE;gBAChE,uEAAuE;YACzE,WAAW,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACtC,cAAc,EACZ,mEAAmE;gBACnE,2CAA2C;YAC7C,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,OAAO,GAAG,CAAC,CAAC;IACZ,MAAM,cAAc,GAAG,qDAAqD,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC9D,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,kBAAkB,GAAa,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,EAAE,QAAQ,CAAC,IAAI,QAAQ,EAAE,CAAC;QACpC,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACzB,kBAAkB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IACD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzD,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,gCAAgC;YACvC,WAAW,EACT,kEAAkE;gBAClE,mFAAmF;YACrF,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EACZ,qEAAqE;gBACrE,+CAA+C;YACjD,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,OAAO,GAAG,CAAC,CAAC;IACZ,MAAM,aAAa,GAAG,iDAAiD,CAAC;IACxE,MAAM,YAAY,GAAG,IAAI,GAAG,EAA6C,CAAC;IAC1E,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC9D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACtD,YAAY,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,YAAY,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACvD,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1B,aAAa,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QAC3D,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzD,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,8CAA8C;YACrD,WAAW,EACT,2EAA2E;gBAC3E,+DAA+D;YACjE,WAAW,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACpD,cAAc,EAAE,kFAAkF;YAClG,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,wEAAwE;IACxE,OAAO,GAAG,CAAC,CAAC;IACZ,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,iCAAiC,CAAC,CAAC;IAC1E,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1B,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzD,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,oCAAoC;YAC3C,WAAW,EACT,GAAG,SAAS,CAAC,MAAM,6DAA6D;gBAChF,0FAA0F;YAC5F,WAAW,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAClC,cAAc,EACZ,wEAAwE;gBACxE,wCAAwC;YAC1C,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;;;;GAgBG;AAEH,+EAA+E;AAC/E,6EAA6E;AAE7E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC;IAC1B,MAAM;IACN,MAAM;IACN,UAAU;IACV,MAAM;IACN,KAAK;IACL,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,UAAU;IACV,cAAc;IACd,aAAa;IACb,MAAM;IACN,UAAU;IACV,QAAQ;IACR,WAAW;IACX,MAAM;IACN,MAAM;IACN,QAAQ;IACR,SAAS;IACT,QAAQ;IACR,MAAM;IACN,KAAK;IACL,OAAO;IACP,QAAQ;IACR,iBAAiB;IACjB,KAAK;IACL,SAAS;CACV,CAAC,CAAC;AACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;AAC9E,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAEjC,IAAI,QAAQ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;IACtE,8DAA8D;IAC9D,MAAM,CAAC,UAAU,CAAC;SACf,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC1C,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;KAAM,CAAC;IACN,2EAA2E;IAE3E,MAAM,CAAC,yCAAyC,CAAC;SAC9C,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;QAC5B,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,2CAA2C,CAAC,CAAC;QAC3F,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAC9D,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;QAE9C,mEAAmE;QACnE,oDAAoD;QACpD,IAAI,OAAO,GAAG,OAAO,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YACvD,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;YAC3B,IAAI,EAAE,QAAQ;YACd,OAAO;SACR,CAAC,CAAC;QAEH,aAAa,CAAC,MAAM,CAAC,CAAC;QACtB,eAAe,CAAC,MAAM,CAAC,CAAC;QAExB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;QAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC5D,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;;;;GAgBG;AAEH,+EAA+E;AAC/E,6EAA6E;AAE7E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC;IAC1B,MAAM;IACN,MAAM;IACN,UAAU;IACV,MAAM;IACN,KAAK;IACL,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,UAAU;IACV,cAAc;IACd,aAAa;IACb,MAAM;IACN,UAAU;IACV,QAAQ;IACR,WAAW;IACX,MAAM;IACN,MAAM;IACN,QAAQ;IACR,SAAS;IACT,QAAQ;IACR,MAAM;IACN,KAAK;IACL,OAAO;IACP,QAAQ;IACR,iBAAiB;IACjB,uBAAuB;IACvB,oBAAoB;IACpB,KAAK;IACL,SAAS;CACV,CAAC,CAAC;AACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;AAC9E,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAEjC,IAAI,QAAQ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;IACtE,8DAA8D;IAC9D,MAAM,CAAC,UAAU,CAAC;SACf,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC1C,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;KAAM,CAAC;IACN,2EAA2E;IAE3E,MAAM,CAAC,yCAAyC,CAAC;SAC9C,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;QAC5B,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,2CAA2C,CAAC,CAAC;QAC3F,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAC9D,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;QAE9C,mEAAmE;QACnE,oDAAoD;QACpD,IAAI,OAAO,GAAG,OAAO,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YACtD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YACvD,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;YAC3B,IAAI,EAAE,QAAQ;YACd,OAAO;SACR,CAAC,CAAC;QAEH,aAAa,CAAC,MAAM,CAAC,CAAC;QACtB,eAAe,CAAC,MAAM,CAAC,CAAC;QAExB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;QAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC5D,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api-contract.d.ts","sourceRoot":"","sources":["../../src/judges/api-contract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,gBAAgB,EAAE,eA2B9B,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export const apiContractJudge = {
|
|
2
|
+
id: "api-contract",
|
|
3
|
+
name: "API Contract Conformance",
|
|
4
|
+
description: "Evaluates API endpoint implementations for contract conformance: input validation, " +
|
|
5
|
+
"proper status codes, error handling, rate limiting, versioning, and content-type management.",
|
|
6
|
+
domain: "API Design & REST Best Practices",
|
|
7
|
+
rulePrefix: "API",
|
|
8
|
+
systemPrompt: `You are Judge API Contract Conformance — an expert in REST API design, HTTP semantics, and contract-first development.
|
|
9
|
+
|
|
10
|
+
YOUR EVALUATION CRITERIA:
|
|
11
|
+
1. **Input Validation**: Every endpoint must validate and sanitize all user-supplied input (query params, body, headers) before use.
|
|
12
|
+
2. **Status Codes**: Responses must use semantically correct HTTP status codes (e.g., 201 for creation, 404 for missing, 422 for validation errors).
|
|
13
|
+
3. **Error Handling**: Errors must return structured JSON bodies with a consistent schema; stack traces must never leak to clients.
|
|
14
|
+
4. **Rate Limiting**: Public-facing endpoints should implement or reference rate-limiting middleware.
|
|
15
|
+
5. **Versioning**: API routes should include a version segment (e.g., /v1/) or accept a version header.
|
|
16
|
+
6. **Content-Type**: Endpoints must set and validate Content-Type / Accept headers appropriately.
|
|
17
|
+
|
|
18
|
+
SEVERITY MAPPING:
|
|
19
|
+
- **critical**: Missing input validation on security-sensitive endpoints, leaked stack traces
|
|
20
|
+
- **high**: Wrong status codes that break client contracts, missing error bodies
|
|
21
|
+
- **medium**: Missing rate limiting, absent versioning
|
|
22
|
+
- **low**: Minor Content-Type mismatches, inconsistent error schemas
|
|
23
|
+
|
|
24
|
+
ADVERSARIAL MANDATE:
|
|
25
|
+
- Flag every deviation from RESTful best practices.
|
|
26
|
+
- Do NOT assume middleware handles validation unless explicitly imported and applied.`,
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=api-contract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api-contract.js","sourceRoot":"","sources":["../../src/judges/api-contract.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,gBAAgB,GAAoB;IAC/C,EAAE,EAAE,cAAc;IAClB,IAAI,EAAE,0BAA0B;IAChC,WAAW,EACT,qFAAqF;QACrF,8FAA8F;IAChG,MAAM,EAAE,kCAAkC;IAC1C,UAAU,EAAE,KAAK;IACjB,YAAY,EAAE;;;;;;;;;;;;;;;;;;sFAkBsE;CACrF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/judges/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/judges/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAuInD;;;;;;GAMG;AACH,eAAO,MAAM,MAAM,EAAE,eAAe,EA4CnC,CAAC;AAEF;;GAEG;AACH,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAEhE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,KAAK,CAAC;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC,CAOD"}
|
package/dist/judges/index.js
CHANGED
|
@@ -37,6 +37,10 @@ import { iacSecurityJudge } from "./iac-security.js";
|
|
|
37
37
|
import { securityJudge } from "./security.js";
|
|
38
38
|
import { falsePositiveReviewJudge } from "./false-positive-review.js";
|
|
39
39
|
import { hallucinationDetectionJudge } from "./hallucination-detection.js";
|
|
40
|
+
import { intentAlignmentJudge } from "./intent-alignment.js";
|
|
41
|
+
import { apiContractJudge } from "./api-contract.js";
|
|
42
|
+
import { multiTurnCoherenceJudge } from "./multi-turn-coherence.js";
|
|
43
|
+
import { modelFingerprintJudge } from "./model-fingerprint.js";
|
|
40
44
|
// ─── Analyzer Imports ────────────────────────────────────────────────────────
|
|
41
45
|
import { analyzeDataSecurity } from "../evaluators/data-security.js";
|
|
42
46
|
import { analyzeCybersecurity } from "../evaluators/cybersecurity.js";
|
|
@@ -76,6 +80,10 @@ import { analyzeFrameworkSafety } from "../evaluators/framework-safety.js";
|
|
|
76
80
|
import { analyzeIacSecurity } from "../evaluators/iac-security.js";
|
|
77
81
|
import { analyzeSecurity } from "../evaluators/security.js";
|
|
78
82
|
import { analyzeHallucinationDetection } from "../evaluators/hallucination-detection.js";
|
|
83
|
+
import { analyzeIntentAlignment } from "../evaluators/intent-alignment.js";
|
|
84
|
+
import { analyzeApiContract } from "../evaluators/api-contract.js";
|
|
85
|
+
import { analyzeMultiTurnCoherence } from "../evaluators/multi-turn-coherence.js";
|
|
86
|
+
import { analyzeModelFingerprint } from "../evaluators/model-fingerprint.js";
|
|
79
87
|
// ─── Wire each judge to its analyzer ─────────────────────────────────────────
|
|
80
88
|
dataSecurityJudge.analyze = analyzeDataSecurity;
|
|
81
89
|
cybersecurityJudge.analyze = analyzeCybersecurity;
|
|
@@ -115,6 +123,10 @@ frameworkSafetyJudge.analyze = analyzeFrameworkSafety;
|
|
|
115
123
|
iacSecurityJudge.analyze = analyzeIacSecurity;
|
|
116
124
|
securityJudge.analyze = analyzeSecurity;
|
|
117
125
|
hallucinationDetectionJudge.analyze = analyzeHallucinationDetection;
|
|
126
|
+
intentAlignmentJudge.analyze = analyzeIntentAlignment;
|
|
127
|
+
apiContractJudge.analyze = analyzeApiContract;
|
|
128
|
+
multiTurnCoherenceJudge.analyze = analyzeMultiTurnCoherence;
|
|
129
|
+
modelFingerprintJudge.analyze = analyzeModelFingerprint;
|
|
118
130
|
/**
|
|
119
131
|
* The panel of judges that comprise the Judges Panel.
|
|
120
132
|
*
|
|
@@ -161,6 +173,10 @@ export const JUDGES = [
|
|
|
161
173
|
iacSecurityJudge,
|
|
162
174
|
securityJudge,
|
|
163
175
|
hallucinationDetectionJudge,
|
|
176
|
+
intentAlignmentJudge,
|
|
177
|
+
apiContractJudge,
|
|
178
|
+
multiTurnCoherenceJudge,
|
|
179
|
+
modelFingerprintJudge,
|
|
164
180
|
falsePositiveReviewJudge,
|
|
165
181
|
];
|
|
166
182
|
/**
|
package/dist/judges/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/judges/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/judges/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAE/D,gFAAgF;AAChF,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,2BAA2B,EAAE,MAAM,uCAAuC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAC;AAC1E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,iCAAiC,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,8BAA8B,EAAE,MAAM,2CAA2C,CAAC;AAC3F,OAAO,EAAE,6BAA6B,EAAE,MAAM,0CAA0C,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,6BAA6B,EAAE,MAAM,0CAA0C,CAAC;AACzF,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,yBAAyB,EAAE,MAAM,uCAAuC,CAAC;AAClF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAE7E,gFAAgF;AAEhF,iBAAiB,CAAC,OAAO,GAAG,mBAAmB,CAAC;AAChD,kBAAkB,CAAC,OAAO,GAAG,oBAAoB,CAAC;AAClD,sBAAsB,CAAC,OAAO,GAAG,wBAAwB,CAAC;AAC1D,gBAAgB,CAAC,OAAO,GAAG,kBAAkB,CAAC;AAC9C,mBAAmB,CAAC,OAAO,GAAG,qBAAqB,CAAC;AACpD,sBAAsB,CAAC,OAAO,GAAG,wBAAwB,CAAC;AAC1D,kBAAkB,CAAC,OAAO,GAAG,oBAAoB,CAAC;AAClD,cAAc,CAAC,OAAO,GAAG,gBAAgB,CAAC;AAC1C,gBAAgB,CAAC,OAAO,GAAG,kBAAkB,CAAC;AAC9C,kBAAkB,CAAC,OAAO,GAAG,oBAAoB,CAAC;AAClD,gBAAgB,CAAC,OAAO,GAAG,kBAAkB,CAAC;AAC9C,eAAe,CAAC,OAAO,GAAG,iBAAiB,CAAC;AAC5C,oBAAoB,CAAC,OAAO,GAAG,sBAAsB,CAAC;AACtD,YAAY,CAAC,OAAO,GAAG,cAAc,CAAC;AACtC,kBAAkB,CAAC,OAAO,GAAG,oBAAoB,CAAC;AAClD,yBAAyB,CAAC,OAAO,GAAG,2BAA2B,CAAC;AAChE,qBAAqB,CAAC,OAAO,GAAG,uBAAuB,CAAC;AACxD,gBAAgB,CAAC,OAAO,GAAG,kBAAkB,CAAC;AAC9C,eAAe,CAAC,OAAO,GAAG,iBAAiB,CAAC;AAC5C,oBAAoB,CAAC,OAAO,GAAG,sBAAsB,CAAC;AACtD,kBAAkB,CAAC,OAAO,GAAG,oBAAoB,CAAC;AAClD,mBAAmB,CAAC,OAAO,GAAG,qBAAqB,CAAC;AACpD,aAAa,CAAC,OAAO,GAAG,eAAe,CAAC;AACxC,YAAY,CAAC,OAAO,GAAG,cAAc,CAAC;AACtC,4BAA4B,CAAC,OAAO,GAAG,8BAA8B,CAAC;AACtE,2BAA2B,CAAC,OAAO,GAAG,6BAA6B,CAAC;AACpE,gBAAgB,CAAC,OAAO,GAAG,kBAAkB,CAAC;AAC9C,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;AAC5B,mBAAmB,CAAC,OAAO,GAAG,qBAAqB,CAAC;AACpD,iBAAiB,CAAC,OAAO,GAAG,mBAAmB,CAAC;AAChD,SAAS,CAAC,OAAO,GAAG,WAAW,CAAC;AAChC,kBAAkB,CAAC,OAAO,GAAG,oBAAoB,CAAC;AAClD,sBAAsB,CAAC,OAAO,GAAG,wBAAwB,CAAC;AAC1D,iBAAiB,CAAC,OAAO,GAAG,mBAAmB,CAAC;AAChD,oBAAoB,CAAC,OAAO,GAAG,sBAAsB,CAAC;AACtD,gBAAgB,CAAC,OAAO,GAAG,kBAAkB,CAAC;AAC9C,aAAa,CAAC,OAAO,GAAG,eAAe,CAAC;AACxC,2BAA2B,CAAC,OAAO,GAAG,6BAA6B,CAAC;AACpE,oBAAoB,CAAC,OAAO,GAAG,sBAAsB,CAAC;AACtD,gBAAgB,CAAC,OAAO,GAAG,kBAAkB,CAAC;AAC9C,uBAAuB,CAAC,OAAO,GAAG,yBAAyB,CAAC;AAC5D,qBAAqB,CAAC,OAAO,GAAG,uBAAuB,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,MAAM,GAAsB;IACvC,iBAAiB;IACjB,kBAAkB;IAClB,sBAAsB;IACtB,gBAAgB;IAChB,mBAAmB;IACnB,sBAAsB;IACtB,kBAAkB;IAClB,cAAc;IACd,gBAAgB;IAChB,kBAAkB;IAClB,gBAAgB;IAChB,eAAe;IACf,oBAAoB;IACpB,YAAY;IACZ,kBAAkB;IAClB,yBAAyB;IACzB,qBAAqB;IACrB,gBAAgB;IAChB,eAAe;IACf,oBAAoB;IACpB,kBAAkB;IAClB,mBAAmB;IACnB,aAAa;IACb,YAAY;IACZ,4BAA4B;IAC5B,2BAA2B;IAC3B,gBAAgB;IAChB,OAAO;IACP,mBAAmB;IACnB,iBAAiB;IACjB,SAAS;IACT,kBAAkB;IAClB,sBAAsB;IACtB,iBAAiB;IACjB,oBAAoB;IACpB,gBAAgB;IAChB,aAAa;IACb,2BAA2B;IAC3B,oBAAoB;IACpB,gBAAgB;IAChB,uBAAuB;IACvB,qBAAqB;IACrB,wBAAwB;CACzB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAM/B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;QACxD,EAAE;QACF,IAAI;QACJ,MAAM;QACN,WAAW;KACZ,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"intent-alignment.d.ts","sourceRoot":"","sources":["../../src/judges/intent-alignment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,oBAAoB,EAAE,eA2BlC,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export const intentAlignmentJudge = {
|
|
2
|
+
id: "intent-alignment",
|
|
3
|
+
name: "Judge Intent Alignment",
|
|
4
|
+
domain: "Code–Comment Alignment & Stub Detection",
|
|
5
|
+
description: "Detects mismatches between stated intent (comments, docstrings, function names) and actual implementation — stubs, TODO-only bodies, misleading names, and empty implementations that AI code generators commonly produce.",
|
|
6
|
+
rulePrefix: "INTENT",
|
|
7
|
+
systemPrompt: `You are Judge Intent Alignment — your role is to verify that code does what its documentation, names, and comments claim.
|
|
8
|
+
|
|
9
|
+
YOUR EVALUATION CRITERIA:
|
|
10
|
+
1. **Stub Functions**: Functions with TODO/FIXME bodies or that throw "not implemented" without real logic.
|
|
11
|
+
2. **Misleading Names**: Function names that promise specific behavior (validate, encrypt, sanitize, authenticate) but whose bodies don't perform that action.
|
|
12
|
+
3. **Empty Implementations**: Functions, methods, or handlers declared but with empty or trivial bodies (return null/undefined/false/empty-string without logic).
|
|
13
|
+
4. **Dead Documentation**: JSDoc/docstring parameters that don't match the actual parameter list.
|
|
14
|
+
5. **Contradictory Comments**: Inline comments that describe behavior the code doesn't perform.
|
|
15
|
+
6. **Placeholder Returns**: Functions that always return a hardcoded value regardless of input, when the name implies computation.
|
|
16
|
+
|
|
17
|
+
SEVERITY MAPPING:
|
|
18
|
+
- **critical**: Security-sensitive stubs (validate, authenticate, authorize, encrypt, sanitize)
|
|
19
|
+
- **high**: Functions with misleading names that could cause logical errors
|
|
20
|
+
- **medium**: TODO stubs, placeholder implementations, dead documentation
|
|
21
|
+
- **low**: Minor name mismatches, outdated comments
|
|
22
|
+
|
|
23
|
+
Each finding must include:
|
|
24
|
+
- The specific function/method name and its declared intent
|
|
25
|
+
- What the implementation actually does (or doesn't do)
|
|
26
|
+
- A concrete recommendation for fixing the gap`,
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=intent-alignment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"intent-alignment.js","sourceRoot":"","sources":["../../src/judges/intent-alignment.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,oBAAoB,GAAoB;IACnD,EAAE,EAAE,kBAAkB;IACtB,IAAI,EAAE,wBAAwB;IAC9B,MAAM,EAAE,yCAAyC;IACjD,WAAW,EACT,4NAA4N;IAC9N,UAAU,EAAE,QAAQ;IACpB,YAAY,EAAE;;;;;;;;;;;;;;;;;;;+CAmB+B;CAC9C,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-fingerprint.d.ts","sourceRoot":"","sources":["../../src/judges/model-fingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,eAAO,MAAM,qBAAqB,EAAE,eA6BnC,CAAC"}
|