crewly 1.5.17 → 1.5.19

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-scorer.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/eval-scorer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACL,iBAAiB,EACjB,YAAY,EACZ,aAAa,GAWd,MAAM,iBAAiB,CAAC;AAEzB,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,WAAwB,EAAE,GAAW;IACtE,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAC9C,OAAO;QACL,WAAW;QACX,GAAG,EAAE,OAAO;QACZ,MAAM;QACN,QAAQ,EAAE,OAAO,GAAG,MAAM;KAC3B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,WAAmB,EAAE,YAAoB;IACpF,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC;IACtE,MAAM,UAAU,GAAG,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC;IACzE,OAAO,SAAS,GAAG,UAAU,CAAC;AAChC,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,YAA8B;IAC9D,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,OAAO,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IACpD,OAAO,KAAK,CAAC,SAAS,GAAG,OAAO,CAAC,CAAC;AACpC,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,kBAAkB,CAAC,YAA8B;IAC/D,MAAM,EAAE,GAAG,YAAY,CAAC,WAAW,CAAC;IACpC,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,EAAE,CAAC,eAAe;QAAE,KAAK,IAAI,EAAE,CAAC;IACpC,IAAI,EAAE,CAAC,UAAU;QAAE,KAAK,IAAI,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,EAAE,CAAC,eAAe,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;IACzC,IAAI,EAAE,CAAC,cAAc;QAAE,KAAK,IAAI,EAAE,CAAC;IACnC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC;AACtB,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAoB;IACtD,IAAI,OAAO,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC;IACtE,MAAM,aAAa,GAAG,OAAO,CAAC,oBAAoB,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAEnC,OAAO,KAAK,CAAC,CAAC,YAAY,GAAG,EAAE,GAAG,aAAa,GAAG,EAAE,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,WAAmB;IACjD,OAAO,KAAK,CAAC,GAAG,GAAG,WAAW,GAAG,EAAE,CAAC,CAAC;AACvC,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAgB,EAAE,YAA8B;IACnF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,CAAC;IAChD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC1D,OAAO,KAAK,CAAC,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;AACpD,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAoB,EACpB,WAAgC;IAEhC,IAAI,KAAK,GAAG,GAAG,CAAC;IAChB,IAAI,OAAO,CAAC,YAAY;QAAE,KAAK,IAAI,EAAE,CAAC;IACtC,KAAK,IAAI,OAAO,CAAC,UAAU,GAAG,EAAE,CAAC;IACjC,IAAI,OAAO,CAAC,QAAQ;QAAE,KAAK,IAAI,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,WAAW,CAAC,cAAc;YAAE,YAAY,IAAI,EAAE,CAAC;QACnD,IAAI,WAAW,CAAC,gBAAgB;YAAE,YAAY,IAAI,EAAE,CAAC;QACrD,IAAI,WAAW,CAAC,YAAY;YAAE,YAAY,IAAI,EAAE,CAAC;QACjD,IAAI,WAAW,CAAC,oBAAoB;YAAE,YAAY,IAAI,EAAE,CAAC;QACzD,gEAAgE;QAChE,KAAK,IAAI,YAAY,CAAC;IACxB,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC;AACtB,CAAC;AAmBD;;;;;;;;;GASG;AACH,MAAM,UAAU,yBAAyB,CACvC,MAAc,EACd,YAA8B;IAE9B,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,IAAI,EAAE,CAAC;IACtD,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CACvC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC;IAElD,gEAAgE;IAChE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO;YACV,OAAO;gBACL,cAAc,EAAE,MAAM,CAAC,kCAAkC,CAAC;gBAC1D,gBAAgB,EAAE,MAAM,CAAC,6BAA6B,CAAC;gBACvD,YAAY,EAAE,MAAM,CAAC,6BAA6B,CAAC;gBACnD,oBAAoB,EAAE,KAAK,EAAE,mCAAmC;aACjE,CAAC;QACJ,KAAK,OAAO;YACV,OAAO;gBACL,cAAc,EAAE,MAAM,CAAC,4BAA4B,CAAC;gBACpD,gBAAgB,EAAE,MAAM,CAAC,sBAAsB,CAAC;gBAChD,YAAY,EAAE,MAAM,CAAC,uBAAuB,CAAC;gBAC7C,oBAAoB,EAAE,KAAK;aAC5B,CAAC;QACJ,KAAK,OAAO;YACV,OAAO;gBACL,cAAc,EAAE,MAAM,CAAC,2BAA2B,CAAC;gBACnD,gBAAgB,EAAE,MAAM,CAAC,wBAAwB,CAAC;gBAClD,YAAY,EAAE,MAAM,CAAC,0BAA0B,CAAC;gBAChD,oBAAoB,EAAE,KAAK;aAC5B,CAAC;QACJ;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,qBAAqB,CAAC,aAAqB,EAAE,eAAuB;IAClF,IAAI,eAAe,IAAI,CAAC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,eAAe,GAAG,aAAa,CAAC;IAC9C,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACf,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAAC,MAAqB,EAAE,IAAc,EAAE,KAAa;IACjF,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IAEnC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,gBAAgB,EAAE,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,oBAAoB,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAK,IAAI,CAAC,MAAkC,EAAE,CAAC;YAC1G,UAAU,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IAED,qEAAqE;IACrE,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9F,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE;QACjE,OAAO;QACP,aAAa,EAAE,SAAS,CAAC,MAAM;QAC/B,gBAAgB;QAChB,oBAAoB;QACpB,WAAW;QACX,UAAU;QACV,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,KAAK;QAC1C,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;KACnC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CACvB,IAAc,EACd,MAAqB,EACrB,YAA8B,EAC9B,OAAkB,EAClB,KAAa,EACb,eAAwB;IAExB,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;IAEpD,mDAAmD;IACnD,MAAM,WAAW,GAAG,yBAAyB,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;IAErE,MAAM,UAAU,GAAqB;QACnC,kBAAkB,CAAC,eAAe,EAAE,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACpE,kBAAkB,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACtE,kBAAkB,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACnE,kBAAkB,CAAC,aAAa,EAAE,eAAe,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvE,kBAAkB,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACtF,kBAAkB,CAAC,cAAc,EAAE,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC1E,kBAAkB,CAAC,mBAAmB,EAAE,qBAAqB,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,IAAI,CAAC,CAAC,CAAC;KACtG,CAAC;IAEF,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAErE,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU;QACV,SAAS;QACT,OAAO;QACP,YAAY;QACZ,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAC9B,WAA6B,EAC7B,OAAkB,EAClB,KAAa;IAEb,MAAM,YAAY,GAAkB;QAClC,eAAe;QACf,gBAAgB;QAChB,iBAAiB;QACjB,aAAa;QACb,kBAAkB;QAClB,cAAc;QACd,mBAAmB;KACpB,CAAC;IAEF,MAAM,mBAAmB,GAAqB,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACpE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,kBAAkB,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,MAAM,GACV,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE;YAC7B,MAAM,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;YAC5D,OAAO,GAAG,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/B,CAAC,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC;QAC7B,OAAO,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IACxF,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAE9E,OAAO;QACL,OAAO,EAAE,YAAY;QACrB,SAAS,EAAE,OAAO;QAClB,KAAK;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,WAAW;QACX,mBAAmB;QACnB,gBAAgB;QAChB,eAAe;QACf,YAAY;QACZ,WAAW;QACX,UAAU,EAAE,WAAW,CAAC,MAAM;KAC/B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAqB,EACrB,cAA6B;IAE7B,MAAM,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACjE,MAAM,WAAW,GAAG,cAAc,CAAC,mBAAmB,CAAC,IAAI,CACzD,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,CAC3C,CAAC;QACF,MAAM,WAAW,GAAG,WAAW,EAAE,GAAG,IAAI,GAAG,CAAC;QAC5C,MAAM,aAAa,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;QAChF,OAAO,kBAAkB,CAAC,GAAG,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAErF,OAAO;QACL,GAAG,MAAM;QACT,mBAAmB,EAAE,mBAAmB;QACxC,gBAAgB;KACjB,CAAC;AACJ,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-types.d.ts

@@ -0,0 +1,313 @@
+/**
+ * Crewly Agent Evaluation Framework - Type Definitions
+ *
+ * Defines all types, interfaces, and constants used by the eval framework
+ * for benchmarking agent runtimes across multiple dimensions.
+ *
+ * @module eval-types
+ */
+import type { ToolCallRecord } from '../types.js';
+/** Current evaluation framework version */
+export declare const EVAL_VERSION = "1.0.0";
+/** Default timeout per task in milliseconds (5 minutes) */
+export declare const DEFAULT_TASK_TIMEOUT_MS = 300000;
+/** Maximum number of steps a task may take */
+export declare const MAX_TASK_STEPS = 50;
+/**
+ * Complexity level of an evaluation task.
+ *
+ * - L1 -- basic single-tool operations
+ * - L2 -- multi-step workflows
+ * - L3 -- complex reasoning and planning
+ * - L4 -- multi-agent collaboration
+ */
+export type EvalLevel = 'L1' | 'L2' | 'L3' | 'L4';
+/**
+ * Setup / verify / teardown lifecycle hooks for a task.
+ *
+ * - `setup`    -- prepares the work directory before the agent runs
+ * - `verify`   -- inspects the work directory after the agent finishes
+ * - `teardown` -- cleans up resources regardless of outcome
+ */
+export interface EvalTask {
+    /** Unique task identifier, e.g. "L1-01" */
+    id: string;
+    /** Complexity level */
+    level: EvalLevel;
+    /** Human-readable title */
+    title: string;
+    /** The prompt sent to the agent runtime */
+    prompt: string;
+    /** Maximum steps the agent is allowed to take */
+    maxSteps: number;
+    /** Per-task timeout in milliseconds */
+    timeoutMs: number;
+    /** Tool names the task is expected to exercise */
+    expectedTools: string[];
+    /**
+     * Prepare the work directory before the task runs.
+     *
+     * @param workDir - absolute path to the temporary work directory
+     */
+    setup: (workDir: string) => Promise<void>;
+    /**
+     * Verify the agent's output after it finishes.
+     *
+     * @param workDir    - absolute path to the work directory
+     * @param toolCalls  - tool call records from the runtime
+     * @returns verification result with pass/fail and details
+     */
+    verify: (workDir: string, toolCalls: ToolCallRecord[]) => Promise<EvalVerification>;
+    /**
+     * Optional cleanup hook that runs after verify, even on failure.
+     *
+     * @param workDir - absolute path to the work directory
+     */
+    teardown?: (workDir: string) => Promise<void>;
+    /** Freeform tags for filtering (e.g. "file-ops", "git") */
+    tags?: string[];
+    /** Whether this task requires multi-agent collaboration (L4) */
+    multiAgent?: boolean;
+}
+/** Supported runtime identifiers */
+export type RuntimeId = 'crewly-agent' | 'claude-code' | 'gemini-cli' | 'codex-cli';
+/**
+ * Configuration passed to a RuntimeAdapter during initialization.
+ */
+export interface AdapterConfig {
+    /** Runtime identifier */
+    runtimeId: RuntimeId;
+    /** Model identifier (e.g. "claude-opus-4") */
+    model: string;
+    /** Working directory root for all tasks */
+    workDir: string;
+    /** Whether to emit verbose logs */
+    verbose?: boolean;
+    /** Additional adapter-specific options */
+    options?: Record<string, unknown>;
+}
+/**
+ * Adapter interface that each supported runtime must implement.
+ */
+export interface RuntimeAdapter {
+    /**
+     * One-time initialization (start processes, authenticate, etc.).
+     *
+     * @param config - adapter configuration
+     */
+    initialize(config: AdapterConfig): Promise<void>;
+    /**
+     * Execute a single eval task and return the raw result.
+     *
+     * @param task    - the task definition
+     * @param workDir - absolute path to the task's work directory
+     * @returns raw runtime result with output, timing, and tool calls
+     */
+    runTask(task: EvalTask, workDir: string): Promise<RuntimeResult>;
+    /**
+     * Graceful shutdown -- release resources, kill child processes.
+     */
+    shutdown(): Promise<void>;
+}
+/**
+ * Raw result returned by a runtime adapter after executing a task.
+ */
+export interface RuntimeResult {
+    /** Whether the task completed without crashing */
+    success: boolean;
+    /** Final text output from the runtime */
+    output: string;
+    /** Wall-clock duration in milliseconds */
+    durationMs: number;
+    /** Number of reasoning / action steps taken */
+    steps: number;
+    /** Token usage */
+    usage: {
+        input: number;
+        output: number;
+    };
+    /** Ordered list of tool calls */
+    toolCalls: ToolCallRecord[];
+    /** Number of times the agent had to be nudged / re-prompted */
+    humanNudges: number;
+    /** Error message if the task failed */
+    error?: string;
+    /** Whether the task timed out */
+    timedOut?: boolean;
+    /** Whether a loop was detected */
+    loopDetected?: boolean;
+}
+/**
+ * Result of verifying a completed task.
+ */
+export interface EvalVerification {
+    /** Overall pass / fail */
+    passed: boolean;
+    /** Detailed check results */
+    checks: EvalCheck[];
+    /** Optional code quality analysis */
+    codeQuality?: CodeQualityResult;
+    /** Collaboration-specific checks for L4 tasks */
+    collaborationChecks?: EvalCheck[];
+}
+/**
+ * A single boolean check inside a verification.
+ */
+export interface EvalCheck {
+    /** Short machine-readable name */
+    name: string;
+    /** Whether this check passed */
+    passed: boolean;
+    /** Human-readable explanation */
+    message: string;
+}
+/**
+ * Code quality metrics produced by static analysis of agent output.
+ */
+export interface CodeQualityResult {
+    /** TypeScript type-check passes */
+    typecheckPassed: boolean;
+    /** ESLint passes with zero errors */
+    lintPassed: boolean;
+    /** Test coverage percentage (0-100) */
+    coveragePercent: number;
+    /** Whether prohibited patterns (e.g. `any`, `console.log`) are absent */
+    noAntiPatterns: boolean;
+}
+/**
+ * The seven evaluation dimensions.
+ *
+ * Dimension weights must sum to 1.0.
+ */
+export type DimensionId = 'D1_completion' | 'D2_codeQuality' | 'D3_toolAccuracy' | 'D4_autonomy' | 'D5_collaboration' | 'D6_stability' | 'D7_costEfficiency';
+/**
+ * Weights applied to each dimension when computing the composite score.
+ * All values sum to 1.0.
+ */
+export declare const DIMENSION_WEIGHTS: Readonly<Record<DimensionId, number>>;
+/**
+ * Score for a single dimension, 0-100.
+ */
+export interface DimensionScore {
+    /** Dimension identifier */
+    dimensionId: DimensionId;
+    /** Raw score before weighting (0-100) */
+    raw: number;
+    /** Weight applied to this dimension */
+    weight: number;
+    /** Weighted score contribution (raw * weight) */
+    weighted: number;
+}
+/**
+ * Extracted metrics from a single task run, used as input to the scorer.
+ */
+export interface EvalMetrics {
+    /** Wall-clock duration in ms */
+    durationMs: number;
+    /** Number of steps taken */
+    steps: number;
+    /** Token usage */
+    usage: {
+        input: number;
+        output: number;
+    };
+    /** Estimated cost in USD */
+    costUSD: number;
+    /** Total number of tool calls */
+    toolCallCount: number;
+    /** Number of tool calls that matched expected tools */
+    correctToolCalls: number;
+    /** Number of unnecessary / unexpected tool calls */
+    unnecessaryToolCalls: number;
+    /** Ratio of correct argument usage (0-1) */
+    argAccuracy: number;
+    /** Number of tool errors encountered */
+    toolErrors: number;
+    /** Number of times the agent was nudged */
+    humanNudges: number;
+    /** Whether a loop was detected */
+    loopDetected: boolean;
+    /** Whether the task timed out */
+    timedOut: boolean;
+}
+/**
+ * Complete result for a single evaluated task.
+ */
+export interface EvalTaskResult {
+    /** Task identifier */
+    taskId: string;
+    /** Task level */
+    level: EvalLevel;
+    /** Task title */
+    title: string;
+    /** Whether the runtime produced a result (did not crash) */
+    success: boolean;
+    /** Per-dimension scores */
+    dimensions: DimensionScore[];
+    /** Composite weighted score (0-100) */
+    composite: number;
+    /** Extracted metrics */
+    metrics: EvalMetrics;
+    /** Verification result */
+    verification: EvalVerification;
+    /** Error message if the task failed to run */
+    error?: string;
+}
+/**
+ * Aggregated report for a full evaluation run.
+ */
+export interface EvalRunReport {
+    /** Framework version */
+    version: string;
+    /** Runtime that was evaluated */
+    runtimeId: RuntimeId;
+    /** Model used */
+    model: string;
+    /** ISO 8601 timestamp of when the run started */
+    timestamp: string;
+    /** Individual task results */
+    taskResults: EvalTaskResult[];
+    /** Aggregate dimension scores (averaged across tasks) */
+    aggregateDimensions: DimensionScore[];
+    /** Overall composite score (0-100) */
+    overallComposite: number;
+    /** Total duration in ms */
+    totalDurationMs: number;
+    /** Total estimated cost in USD */
+    totalCostUSD: number;
+    /** Number of tasks that passed */
+    passedCount: number;
+    /** Total number of tasks evaluated */
+    totalCount: number;
+}
+/**
+ * Side-by-side comparison between two eval run reports.
+ */
+export interface EvalComparison {
+    /** Baseline report (reference) */
+    baseline: EvalRunReport;
+    /** Candidate report (being compared) */
+    candidate: EvalRunReport;
+    /** Per-dimension deltas (candidate minus baseline) */
+    dimensionDeltas: Record<DimensionId, number>;
+    /** Overall composite delta */
+    compositeScoreDelta: number;
+    /** Cost delta in USD */
+    costDelta: number;
+}
+/**
+ * Pricing information for a single model, per million tokens.
+ */
+export interface ModelPricing {
+    /** Cost per 1 M input tokens in USD */
+    inputPerMillion: number;
+    /** Cost per 1 M output tokens in USD */
+    outputPerMillion: number;
+}
+/**
+ * Known model pricing table.
+ *
+ * Prices are in USD per 1 million tokens.
+ */
+export declare const MODEL_PRICING: Readonly<Record<string, ModelPricing>>;
+//# sourceMappingURL=eval-types.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-types.d.ts","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/eval-types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAMlD,2CAA2C;AAC3C,eAAO,MAAM,YAAY,UAAU,CAAC;AAEpC,2DAA2D;AAC3D,eAAO,MAAM,uBAAuB,SAAU,CAAC;AAE/C,8CAA8C;AAC9C,eAAO,MAAM,cAAc,KAAK,CAAC;AAMjC;;;;;;;GAOG;AACH,MAAM,MAAM,SAAS,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAElD;;;;;;GAMG;AACH,MAAM,WAAW,QAAQ;IACvB,2CAA2C;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,uBAAuB;IACvB,KAAK,EAAE,SAAS,CAAC;IACjB,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB;;;;OAIG;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACpF;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,2DAA2D;IAC3D,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD,oCAAoC;AACpC,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,aAAa,GAAG,YAAY,GAAG,WAAW,CAAC;AAEpF;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,yBAAyB;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,mCAAmC;IACnC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0CAA0C;IAC1C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,UAAU,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD;;;;;;OAMG;IACH,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAEjE;;OAEG;IACH,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kDAAkD;IAClD,OAAO,EAAE,OAAO,CAAC;IACjB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB;IAClB,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,iCAAiC;IACjC,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,kCAAkC;IAClC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,6BAA6B;IAC7B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,qCAAqC;IACrC,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,iDAAiD;IACjD,mBAAmB,CAAC,EAAE,SAAS,EAAE,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,kCAAkC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,gCAAgC;IAChC,MAAM,EAAE,OAAO,CAAC;IAChB,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,mCAAmC;IACnC,eAAe,EAAE,OAAO,CAAC;IACzB,qCAAqC;IACrC,UAAU,EAAE,OAAO,CAAC;IACpB,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB,yEAAyE;IACzE,cAAc,EAAE,OAAO,CAAC;CACzB;AAMD;;;;GAIG;AACH,MAAM,MAAM,WAAW,GACnB,eAAe,GACf,gBAAgB,GAChB,iBAAiB,GACjB,aAAa,GACb,kBAAkB,GAClB,cAAc,GACd,mBAAmB,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAQ1D,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,2BAA2B;IAC3B,WAAW,EAAE,WAAW,CAAC;IACzB,yCAAyC;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAMD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB;IAClB,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,iCAAiC;IACjC,aAAa,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,gBAAgB,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,4CAA4C;IAC5C,WAAW,EAAE,MAAM,CAAC;IACpB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,YAAY,EAAE,OAAO,CAAC;IACtB,iCAAiC;IACjC,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,2BAA2B;IAC3B,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,OAAO,EAAE,WAAW,CAAC;IACrB,0BAA0B;IAC1B,YAAY,EAAE,gBAAgB,CAAC;IAC/B,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,iCAAiC;IACjC,SAAS,EAAE,SAAS,CAAC;IACrB,iBAAiB;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,yDAAyD;IACzD,mBAAmB,EAAE,cAAc,EAAE,CAAC;IACtC,sCAAsC;IACtC,gBAAgB,EAAE,MAAM,CAAC;IACzB,2BAA2B;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,kCAAkC;IAClC,QAAQ,EAAE,aAAa,CAAC;IACxB,wCAAwC;IACxC,SAAS,EAAE,aAAa,CAAC;IACzB,sDAAsD;IACtD,eAAe,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC7C,8BAA8B;IAC9B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAQvD,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-types.js

@@ -0,0 +1,45 @@
+/**
+ * Crewly Agent Evaluation Framework - Type Definitions
+ *
+ * Defines all types, interfaces, and constants used by the eval framework
+ * for benchmarking agent runtimes across multiple dimensions.
+ *
+ * @module eval-types
+ */
+// ---------------------------------------------------------------------------
+// Version & defaults
+// ---------------------------------------------------------------------------
+/** Current evaluation framework version */
+export const EVAL_VERSION = '1.0.0';
+/** Default timeout per task in milliseconds (5 minutes) */
+export const DEFAULT_TASK_TIMEOUT_MS = 300_000;
+/** Maximum number of steps a task may take */
+export const MAX_TASK_STEPS = 50;
+/**
+ * Weights applied to each dimension when computing the composite score.
+ * All values sum to 1.0.
+ */
+export const DIMENSION_WEIGHTS = {
+    D1_completion: 0.25,
+    D2_codeQuality: 0.15,
+    D3_toolAccuracy: 0.15,
+    D4_autonomy: 0.15,
+    D5_collaboration: 0.10,
+    D6_stability: 0.10,
+    D7_costEfficiency: 0.10,
+};
+/**
+ * Known model pricing table.
+ *
+ * Prices are in USD per 1 million tokens.
+ */
+export const MODEL_PRICING = {
+    'claude-opus-4': { inputPerMillion: 15, outputPerMillion: 75 },
+    'claude-sonnet-4': { inputPerMillion: 3, outputPerMillion: 15 },
+    'claude-3-5-haiku': { inputPerMillion: 1, outputPerMillion: 5 },
+    'gpt-4o': { inputPerMillion: 2.5, outputPerMillion: 10 },
+    'o3-mini': { inputPerMillion: 1.1, outputPerMillion: 4.4 },
+    'gemini-2.0-flash': { inputPerMillion: 0.1, outputPerMillion: 0.4 },
+    'gemini-2.5-pro': { inputPerMillion: 1.25, outputPerMillion: 10 },
+};
+//# sourceMappingURL=eval-types.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/eval-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-types.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/eval-types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,2CAA2C;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,OAAO,CAAC;AAEpC,2DAA2D;AAC3D,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,CAAC;AAE/C,8CAA8C;AAC9C,MAAM,CAAC,MAAM,cAAc,GAAG,EAAE,CAAC;AAyMjC;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA0C;IACtE,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,eAAe,EAAE,IAAI;IACrB,WAAW,EAAE,IAAI;IACjB,gBAAgB,EAAE,IAAI;IACtB,YAAY,EAAE,IAAI;IAClB,iBAAiB,EAAE,IAAI;CACf,CAAC;AAoIX;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAA2C;IACnE,eAAe,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE;IAC9D,iBAAiB,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,gBAAgB,EAAE,EAAE,EAAE;IAC/D,kBAAkB,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IAC/D,QAAQ,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE;IACxD,SAAS,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;IAC1D,kBAAkB,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;IACnE,gBAAgB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE;CACzD,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/index.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Crewly Agent Evaluation Framework
+ *
+ * Barrel export for types, constants, scoring functions, runner, tasks,
+ * and adapters.
+ *
+ * @module eval
+ */
+export type { EvalLevel, EvalTask, RuntimeId, AdapterConfig, RuntimeAdapter, RuntimeResult, EvalVerification, EvalCheck, CodeQualityResult, DimensionId, DimensionScore, EvalMetrics, EvalTaskResult, EvalRunReport, EvalComparison, ModelPricing, } from './eval-types.js';
+export { EVAL_VERSION, DEFAULT_TASK_TIMEOUT_MS, MAX_TASK_STEPS, DIMENSION_WEIGHTS, MODEL_PRICING, } from './eval-types.js';
+export { calculateCost, scoreD1Completion, scoreD2CodeQuality, scoreD3ToolAccuracy, scoreD4Autonomy, scoreD5Collaboration, scoreD6Stability, scoreD7CostEfficiency, extractMetrics, scoreTask, aggregateResults, normalizeToBaseline, makeDimensionScore, extractSelfHealingSignals, } from './eval-scorer.js';
+export type { SelfHealingSignals } from './eval-scorer.js';
+export { runEvaluation } from './eval-runner.js';
+export type { RunnerConfig } from './eval-runner.js';
+export { ALL_TASKS, getTasksByLevel, getTaskById, getTaskSummary, L1_TASKS, L2_TASKS, L3_TASKS, L4_TASKS, } from './tasks/index.js';
+export { CrewlyAgentAdapter, CLIAdapter, createAdapter } from './adapters/index.js';
+export type { CLIAdapterConfig } from './adapters/index.js';
+export { generateContextMarkdown, generateTeamConfig, setupEvalWorkspace, DEFAULT_EVAL_TEAM, } from './context-generator.js';
+export type { EvalContextConfig, EvalTeamMember, SetupOptions } from './context-generator.js';
+export { parseToolLog, parseToolLogContent, skillWasUsed, getSkillCalls, getDelegationTargets, getMessageRecipients, countSkillCalls, mergeToolCallRecords, } from './tool-log-parser.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,YAAY,EACV,SAAS,EACT,QAAQ,EACR,SAAS,EACT,aAAa,EACb,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,SAAS,EACT,iBAAiB,EACjB,WAAW,EACX,cAAc,EACd,WAAW,EACX,cAAc,EACd,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,YAAY,EACZ,uBAAuB,EACvB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAG3D,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAGrD,OAAO,EACL,SAAS,EACT,eAAe,EACf,WAAW,EACX,cAAc,EACd,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,GACT,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpF,YAAY,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAG5D,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,wBAAwB,CAAC;AAChC,YAAY,EAAE,iBAAiB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,oBAAoB,GACrB,MAAM,sBAAsB,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/index.js

@@ -0,0 +1,21 @@
+/**
+ * Crewly Agent Evaluation Framework
+ *
+ * Barrel export for types, constants, scoring functions, runner, tasks,
+ * and adapters.
+ *
+ * @module eval
+ */
+export { EVAL_VERSION, DEFAULT_TASK_TIMEOUT_MS, MAX_TASK_STEPS, DIMENSION_WEIGHTS, MODEL_PRICING, } from './eval-types.js';
+// Scorer
+export { calculateCost, scoreD1Completion, scoreD2CodeQuality, scoreD3ToolAccuracy, scoreD4Autonomy, scoreD5Collaboration, scoreD6Stability, scoreD7CostEfficiency, extractMetrics, scoreTask, aggregateResults, normalizeToBaseline, makeDimensionScore, extractSelfHealingSignals, } from './eval-scorer.js';
+// Runner
+export { runEvaluation } from './eval-runner.js';
+// Tasks
+export { ALL_TASKS, getTasksByLevel, getTaskById, getTaskSummary, L1_TASKS, L2_TASKS, L3_TASKS, L4_TASKS, } from './tasks/index.js';
+// Adapters
+export { CrewlyAgentAdapter, CLIAdapter, createAdapter } from './adapters/index.js';
+// Context & tool log
+export { generateContextMarkdown, generateTeamConfig, setupEvalWorkspace, DEFAULT_EVAL_TEAM, } from './context-generator.js';
+export { parseToolLog, parseToolLogContent, skillWasUsed, getSkillCalls, getDelegationTargets, getMessageRecipients, countSkillCalls, mergeToolCallRecords, } from './tool-log-parser.js';
+//# sourceMappingURL=index.js.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAsBH,OAAO,EACL,YAAY,EACZ,uBAAuB,EACvB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,iBAAiB,CAAC;AAEzB,SAAS;AACT,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAClB,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,kBAAkB,CAAC;AAG1B,SAAS;AACT,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGjD,QAAQ;AACR,OAAO,EACL,SAAS,EACT,eAAe,EACf,WAAW,EACX,cAAc,EACd,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,GACT,MAAM,kBAAkB,CAAC;AAE1B,WAAW;AACX,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpF,qBAAqB;AACrB,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,oBAAoB,GACrB,MAAM,sBAAsB,CAAC"}

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-cli.d.ts

@@ -0,0 +1,15 @@
+#!/usr/bin/env npx tsx
+/**
+ * Crewly Eval CLI — Quick Runner
+ *
+ * Runs L1 eval tasks against the Crewly Agent runtime via the running backend.
+ * Uses the backend's chat API to send prompts and collect results.
+ *
+ * Usage:
+ *   npx tsx backend/src/services/agent/crewly-agent/eval/run-eval-cli.ts
+ *   npx tsx backend/src/services/agent/crewly-agent/eval/run-eval-cli.ts --tasks L1-01,L1-02
+ *
+ * @module eval/run-eval-cli
+ */
+export {};
+//# sourceMappingURL=run-eval-cli.d.ts.map

package/dist/backend/backend/src/services/agent/crewly-agent/eval/run-eval-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-eval-cli.d.ts","sourceRoot":"","sources":["../../../../../../../../backend/src/services/agent/crewly-agent/eval/run-eval-cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;GAWG"}