@gaunt-sloth/batch 2.0.0-alpha.19 → 2.0.0-alpha.21

package/dist/evalSuite.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalSuite.js","sourceRoot":"","sources":["../src/evalSuite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAGhE;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAC5C,KAAK,CACJ,WAAW,EACX,wFAAwF;QACtF,0FAA0F,CAC7F;IACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wCAAwC,CAAC;IACnE,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC5C,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CACrD,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC/B,CAAC;IACF,QAAQ,EAAE,CAAC;SACR,MAAM,CAAC;QACN,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;KACrD,CAAC;SACD,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,sCAAsC,CAAC;CAC7E,CAAC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,UAAmB;IAClE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,kCAAkC,MAAM,IAAI;YAC1C,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aACvE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IAEzB,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8BAA8B,IAAI,CAAC,MAAM,CAAC,IAAI,mBAAmB;YAC1F,wFAAwF,CAC3F,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,iCAAiC,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM;YACnF,0EAA0E;YAC1E,8EAA8E,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,IAAI,2BAA2B,CAAC;IAE3F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,wBAAwB,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAExB,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;QAC/C,MAAM,cAAc,GAAG,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC;QAC1D,MAAM,SAAS,GACb,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;QACrF,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,CAAC;QAE/B,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,yBAAyB;gBACxF,kFAAkF;gBAClF,2DAA2D,CAC9D,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW;YACX,cAAc;YACd,gBAAgB;YAChB,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YAC/C,aAAa,EAAE,OAAO,CAAC,cAAc,IAAI,qBAAqB;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAC3D,KAAK;KACN,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evalSuite.js","sourceRoot":"","sources":["../src/evalSuite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4CAA4C,CAAC;IACrE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAEH;sGACsG;AACtG,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC5C,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAClD,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACzC,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC7C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9C,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IACrD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,MAAM,CAAC;IACtD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAEH;;;mEAGmE;AACnE,MAAM,aAAa,GAAG,mBAAmB,CAAC,MAAM,CAAC;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,mBAAmB,CAAC,MAAM,CAAC;IAC/C,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAC5C,KAAK,CACJ,WAAW,EACX,wFAAwF;QACtF,0FAA0F,CAC7F;IACH,uFAAuF;IACvF,8EAA8E;IAC9E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,iGAAiG;IACjG,mEAAmE;IACnE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,QAAQ,EAAE;IAChD,mGAAmG;IACnG,+FAA+F;IAC/F,sDAAsD;IACtD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IACxC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CACrD,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC9B,0FAA0F;QAC1F,2FAA2F;QAC3F,mEAAmE;QACnE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC1B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAChC,CAAC;IACF,kGAAkG;IAClG,gGAAgG;IAChG,gGAAgG;IAChG,iGAAiG;IACjG,8FAA8F;IAC9F,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,kGAAkG;IAClG,2FAA2F;IAC3F,8BAA8B;IAC9B,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,CAAC;SACR,MAAM,CAAC;QACN,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;KACrD,CAAC;SACD,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,sCAAsC,CAAC;CAC7E,CAAC,CAAC;AAEH;;2CAE2C;AAC3C,MAAM,mBAAmB,GAAG;IAC1B,cAAc;IACd,kBAAkB;IAClB,oBAAoB;IACpB,WAAW;IACX,eAAe;IACf,YAAY;IACZ,gBAAgB;IAChB,WAAW;IACX,OAAO;CACC,CAAC;AAEX;;2CAE2C;AAC3C,MAAM,gBAAgB,GAAG,WAAW,CAAC;AAIrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,UAAmB;IAClE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,UAAU,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,kCAAkC,MAAM,IAAI;YAC1C,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aACvE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IAEzB,qGAAqG;IACrG,oGAAoG;IACpG,IAAI,MAAkB,CAAC;IACvB,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC3E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,iCAAiC,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM;gBACnF,mFAAmF;gBACnF,wCAAwC,CAC3C,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC/D,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC5C,mFAAmF;QACnF,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,+DAA+D;gBACxF,4EAA4E;gBAC5E,4BAA4B,CAC/B,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,gEAAgE;gBACzF,uEAAuE,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,WAAW,EAAE,CAAC;IAC5F,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QACxC,kGAAkG;QAClG,4FAA4F;QAC5F,2FAA2F;QAC3F,+EAA+E;QAC/E,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,0FAA0F;gBAC1F,gDAAgD,CACnD,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,mEAAmE;gBAC5F,6EAA6E;gBAC7E,2CAA2C,CAC9C,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,uEAAuE,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8BAA8B,IAAI,CAAC,MAAM,CAAC,IAAI,mBAAmB;YAC1F,uFAAuF;YACvF,4FAA4F;YAC5F,oCAAoC,CACvC,CAAC;IACJ,CAAC;IAED,gGAAgG;IAChG,0FAA0F;IAC1F,yCAAyC;IACzC,IAAI,UAAgC,CAAC;IACrC,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;gBACvF,4CAA4C,CAC/C,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACnC,4FAA4F;YAC5F,yFAAyF;YACzF,sEAAsE;YACtE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,eAAe,IAAI,iCAAiC;oBAC7E,sFAAsF;oBACtF,uEAAuE,CAC1E,CAAC;YACJ,CAAC;YACD,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,yBAAyB,IAAI,IAAI,CAAC,CAAC;YAChF,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IAC/B,CAAC;IACD,MAAM,kBAAkB,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAExE,oGAAoG;IACpG,kGAAkG;IAClG,kGAAkG;IAClG,uEAAuE;IACvE,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,kEAAkE;YAC3F,uFAAuF;YACvF,uEAAuE,CAC1E,CAAC;IACJ,CAAC;IAED,kGAAkG;IAClG,oGAAoG;IACpG,+FAA+F;IAC/F,8FAA8F;IAC9F,qFAAqF;IACrF,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,8DAA8D;YACvF,uFAAuF;YACvF,yEAAyE,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,qBAAqB,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,IAAI,2BAA2B,CAAC;IAE3F,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,wBAAwB,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAExB,4FAA4F;QAC5F,iGAAiG;QACjG,iGAAiG;QACjG,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,6BAA6B;gBAC5F,qFAAqF,CACxF,CAAC;QACJ,CAAC;QAED,IAAI,KAAiB,CAAC;QACtB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,4FAA4F;YAC5F,6FAA6F;YAC7F,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC;YACxD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,IAAI,CAC/C,CAAC,GAAG,EAAE,EAAE,CAAE,OAAmC,CAAC,GAAG,CAAC,KAAK,SAAS,CACjE,CAAC;YACF,IAAI,kBAAkB,IAAI,gBAAgB,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,sBAAsB,OAAO,CAAC,EAAE,YAAY,KAAK,aAAa;oBACvF,qFAAqF;oBACrF,mDAAmD,CACtD,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;YAC/B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,iBAAiB;oBAChF,wEAAwE,CAC3E,CAAC;YACJ,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE;gBAC1C,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACnE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,UAAU,SAAS,GAAG;wBACrF,0CAA0C,CAC7C,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,qBAAqB,CAAC,OAAO,EAAE;wBAC3C,MAAM;wBACN,MAAM,EAAE,OAAO,CAAC,EAAE;wBAClB,SAAS,EAAE,KAAK;wBAChB,SAAS;wBACT,kBAAkB;wBAClB,UAAU;qBACX,CAAC;iBACH,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,8FAA8F;YAC9F,kDAAkD;YAClD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,OAAO,CAAC,EAAE,YAAY,KAAK,mBAAmB;oBAClF,oEAAoE,CACvE,CAAC;YACJ,CAAC;YACD,KAAK,GAAG;gBACN;oBACE,IAAI,EAAE,OAAO,CAAC,MAAM;oBACpB,YAAY,EAAE,qBAAqB,CAAC,OAAO,EAAE;wBAC3C,MAAM;wBACN,MAAM,EAAE,OAAO,CAAC,EAAE;wBAClB,SAAS,EAAE,KAAK;wBAChB,SAAS,EAAE,SAAS;wBACpB,kBAAkB;wBAClB,UAAU;qBACX,CAAC;iBACH;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK;YACL,aAAa,EAAE,OAAO,CAAC,cAAc,IAAI,qBAAqB;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,mGAAmG;IACnG,yFAAyF;IACzF,8FAA8F;IAC9F,6FAA6F;IAC7F,mGAAmG;IACnG,6FAA6F;IAC7F,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAChC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAClC,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC5C,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1E,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,WAAW,QAAQ,CAAC,EAAE,0CAA0C;4BACzF,sFAAsF;4BACtF,sFAAsF;4BACtF,kFAAkF;4BAClF,6DAA6D,CAChE,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,kGAAkG;IAClG,2CAA2C;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;IAC7D,mGAAmG;IACnG,6FAA6F;IAC7F,oGAAoG;IACpG,IAAI,YAAY,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,KAAK,CACb,qBAAqB,MAAM,oBAAoB,YAAY,iCAAiC;YAC1F,+BAA+B,CAClC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM;QACN,YAAY;QACZ,UAAU;QACV,KAAK;KACN,CAAC;AACJ,CAAC;AAsBD;;;;;;;GAOG;AACH,SAAS,qBAAqB,CAAC,GAAsB,EAAE,GAAgB;IACrE,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,SAAS,EAAE,CAAC;IAC7E,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,EAAE,CAAC;IACzE,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC;IAE5E,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC;IAC3C,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,IAAI,CAChD,CAAC,GAAG,EAAE,EAAE,CAAE,GAA+B,CAAC,GAAG,CAAC,KAAK,SAAS,CAC7D,CAAC;IACF,IAAI,SAAS,IAAI,iBAAiB,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,kBAAkB,SAAS,qBAAqB;YACvF,sFAAsF;YACtF,8CAA8C,CACjD,CAAC;IACJ,CAAC;IAED,IAAI,YAA+B,CAAC;IACpC,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,SAAS,GAAG,GAAG,CAAC,MAAO,CAAC;QAC9B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,+CAA+C;gBACtF,8BAA8B,CACjC,CAAC;QACJ,CAAC;QACD,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CACjD,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,EAAE;YACxC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,UAAU;YACV,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gGAAgG;QAChG,YAAY,GAAG;YACb,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE;gBAC/B,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,UAAU,EAAE,SAAS;gBACrB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;aAC3C,CAAC;SACH,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,mGAAmG;IACnG,0FAA0F;IAC1F,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACrF,CAAC;YACF,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,8CAA8C;oBACrF,IAAI,QAAQ,2EAA2E;oBACvF,qBAAqB,QAAQ,sDAAsD;oBACnF,cAAc,CACjB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAeD;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,GAAkB,EAClB,eAAqC,EACrC,GAAuB;IAEvB,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,SAAS,EAAE,CAAC;IAC7E,MAAM,KAAK,GACT,GAAG,CAAC,UAAU,KAAK,SAAS;QAC1B,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,EAAE;QAC5D,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,IAAI,QAAQ,iBAAiB,GAAG,CAAC,UAAU,EAAE,CAAC;IAEhG,+FAA+F;IAC/F,6FAA6F;IAC7F,2BAA2B;IAC3B,IAAI,UAAgC,CAAC;IACrC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,iDAAiD;gBACxF,gDAAgD,CACnD,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjE,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,yBAAyB,IAAI,oBAAoB;oBACxF,iEAAiE,CACpE,CAAC;YACJ,CAAC;QACH,CAAC;QACD,UAAU,GAAG,eAAe,CAAC;IAC/B,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC;IAC3C,MAAM,cAAc,GAAG,GAAG,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAClD,MAAM,gBAAgB,GAAG,GAAG,CAAC,kBAAkB,IAAI,EAAE,CAAC;IACtD,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;IAE5C,iGAAiG;IACjG,4FAA4F;IAC5F,MAAM,cAAc,GAAG,CAAC,QAA8B,EAAE,KAAa,EAAY,EAAE,CACjF,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,mBAAmB,KAAK,WAAW;gBAC1E,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI;gBAC9B,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IACL,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IAE1E,iGAAiG;IACjG,yFAAyF;IACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;QAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC;QACjD,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,yBAAyB,KAAK,CAAC,IAAI,aAAa;gBACvF,wCAAwC,CAC3C,CAAC;QACJ,CAAC;QACD,OAAO,WAAW;YAChB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE;YAChD,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GACb,WAAW,CAAC,MAAM,GAAG,CAAC;QACtB,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC3B,QAAQ,CAAC,MAAM,GAAG,CAAC;QACnB,WAAW,CAAC,MAAM,GAAG,CAAC;QACtB,SAAS,CAAC,MAAM,GAAG,CAAC;QACpB,YAAY,CAAC,MAAM,GAAG,CAAC;QACvB,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACtB,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,CAAC;IAE/B,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,MAAM,KAAK,KAAK,+CAA+C;YACtF,2FAA2F;YAC3F,+EAA+E,CAClF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU;QACV,WAAW;QACX,cAAc;QACd,gBAAgB;QAChB,QAAQ;QACR,WAAW;QACX,SAAS;QACT,YAAY;QACZ,QAAQ;QACR,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;KAChD,CAAC;AACJ,CAAC"}

package/dist/evalTypes.d.ts

@@ -11,32 +11,168 @@
  * Kept as our own constant (not imported) since review's is private to that module and coupled to
  * middleware/tool-call/artifact-store plumbing that doesn't fit a plain structured-output call. */
 export declare const DEFAULT_EVAL_PASS_THRESHOLD = 6;
-/** One case's target — this task only supports `gth-agent` against the run's own resolved config
- * (no `--identities`, no pluggable CLI/HTTP targets — see the BATCH-2 Task 2 brief). */
-export interface EvalTarget {
+/** The in-process SUT target: an agent built from the run's own resolved config (the original and
+ * default target). */
+export interface GthAgentTarget {
     type: 'gth-agent';
-    /** Suite-level profile hint. Only `undefined`/`'default'` is accepted this task (see
+    /** Suite-level profile hint. Only `undefined`/`'default'` is accepted (see
      * {@link ../evalSuite.js}'s `parseEvalSuite`) — per-case/per-identity profile switching is
-     * `--identities` scope, not this task's. */
+     * `identities` scope, not this. */
     profile?: string;
 }
-/** One case parsed and normalized from suite YAML — snake_case YAML keys become camelCase here,
- * arrays default to `[]` (not `undefined`) so callers never need an existence check, and
- * `passThreshold` is pre-resolved (case override ?? suite `defaults.pass_threshold` ??
- * {@link DEFAULT_EVAL_PASS_THRESHOLD}). */
-export interface EvalCase {
-    id: string;
-    prompt: string;
+/**
+ * BATCH-14 — an EXTERNAL Google ADK agent, graded over the A2A (Agent-to-Agent) protocol. The eval
+ * drives a *running* ADK agent by A2A text send and grades its answers with the same content
+ * assertions as {@link GthAgentTarget}. The agent runs out-of-process (its own model/tools/auth), so
+ * this target only needs the A2A connection, not a gth config.
+ *
+ * Honest-boundary note (BATCH-14 design point 4): A2A's wire content is only text/file/data parts
+ * plus task status/artifact events — it does NOT expose the agent's intermediate tool/function calls
+ * in any standardized form. So `must_call`/`must_not_call` cannot be graded against this target and
+ * are rejected at parse time (never a silent pass); the `identities` matrix (per-identity gth
+ * configs) is likewise meaningless for an external agent and rejected.
+ */
+export interface AdkAgentTarget {
+    type: 'adk-agent';
+    /** The ADK agent's A2A endpoint / agent-card base URL (mirrors `A2AClientConfig.agentUrl`; the
+     * SDK fetches `/.well-known/agent-card.json` from it). Required — a suite without it is a parse
+     * error. */
+    url: string;
+    /** Optional label for the agent (mirrors `A2AClientConfig.agentId`; used only for debug logging).
+     * Defaults to `'adk-agent'` when the suite omits it. */
+    agentId?: string;
+}
+/**
+ * BATCH-15 — an EXTERNAL agent exposed over the AG-UI protocol, graded by driving its HTTP/SSE run
+ * endpoint (`POST {url}/agents/{agentId}/run`). The eval sends the conversation as a `RunAgentInput`
+ * and decodes the streamed AG-UI events, grading the assembled answer with the SAME content
+ * assertions as {@link GthAgentTarget}. Like {@link AdkAgentTarget}, the agent runs out-of-process
+ * (its own model/tools/auth), so this target only needs the AG-UI connection, not a gth config.
+ *
+ * KEY DIFFERENCE from {@link AdkAgentTarget}: the AG-UI wire DOES stream the agent's tool calls
+ * (`TOOL_CALL_START` events). So — unlike A2A, where the tool trace is invisible and
+ * `must_call`/`must_not_call` are rejected at parse time — the ag-ui runner captures each
+ * `TOOL_CALL_START`'s tool name into the outcome's `tools`, and `must_call`/`must_not_call` grade
+ * normally against this target.
+ */
+export interface AgUiAgentTarget {
+    type: 'ag-ui';
+    /** The AG-UI server's base URL — the origin of `POST {url}/agents/{agentId}/run`. Required; a
+     * suite without it is a parse error. */
+    url: string;
+    /** The `{agentId}` path segment of `/agents/{agentId}/run` (per the AG-UI protocol). Required; a
+     * suite without it is a parse error. */
+    agentId: string;
+}
+/** One suite's target — the in-process gth agent (default), an external ADK agent over A2A
+ * (BATCH-14), or an external AG-UI agent over HTTP/SSE (BATCH-15). Discriminated by `type`; the
+ * runner is target-agnostic (it consumes an injected `RunCellFn`/`RunConversationFn`), so the target
+ * only changes which runner the command builds. */
+export type EvalTarget = GthAgentTarget | AdkAgentTarget | AgUiAgentTarget;
+/** One `json_path` assertion (BATCH-10): resolve `path` against the answer-parsed-as-JSON and check
+ * it. Exactly one of `equals`/`contains` is set (enforced in {@link ../evalSuite.js}'s parse):
+ * - `equals` — the resolved value must deep-equal this (any JSON value, incl. `null`).
+ * - `contains` — the resolved value must be a string containing this substring. */
+export interface JsonPathCheck {
+    path: string;
+    equals?: unknown;
+    contains?: string;
+}
+/**
+ * One expectation block (BATCH-12) — the BATCH-10 assertion bundle PLUS an optional `identities`
+ * scope. This is the atom the whole eval layer normalizes to: a flat case is sugar for ONE
+ * unscoped expectation (applies to every identity), and a matrix case's `expect:` array is a list
+ * of these. snake_case YAML keys become camelCase here; arrays default to `[]` (not `undefined`)
+ * so callers never need an existence check.
+ */
+export interface EvalExpectation {
+    /**
+     * The identity profiles this block grades (BATCH-12). Absent/empty = applies to EVERY applicable
+     * identity — this is what a flat case's single block carries (the sugar), and what an unscoped
+     * `expect:` block carries. A block naming `[admin]` grades only the `admin` run. Every name here
+     * is validated at parse time to be one the suite declares in its top-level `identities` list.
+     */
+    identities?: string[];
     mustContain: string[];
     mustNotContain: string[];
     shouldContainAny: string[];
-    /** The judge rubric, when present and non-blank. `undefined` = no judge for this case. */
+    /** BATCH-10 tool-trace assertions, matched against the case's captured tool names with
+     * glob support (see `@gaunt-sloth/core/utils/toolMatching.js`, shared with `allowedTools`). */
+    mustCall: string[];
+    mustNotCall: string[];
+    /** BATCH-10 regex assertions over the raw answer — compiled at parse time (bad patterns are a
+     * suite error, never a run-time crash) and stored so the compiled `RegExp` is reused, not
+     * rebuilt. No implicit case-folding: authors control flags in the pattern themselves. */
+    mustMatch: RegExp[];
+    mustNotMatch: RegExp[];
+    /** BATCH-10 minimal JSON-path assertions over the answer parsed as JSON — see {@link JsonPathCheck}. */
+    jsonPath: JsonPathCheck[];
+    /** The judge rubric, when present and non-blank. `undefined` = no judge for this block. */
     judgeRubric?: string;
+}
+/**
+ * One conversational turn (BATCH-12): the `user` message to send, and the {@link EvalExpectation}
+ * blocks that grade its answer. **Task 1 keeps every case at exactly one turn** (single-turn); the
+ * general `turns[]` type exists NOW so Task 2 adds multi-turn *execution*, not a schema migration.
+ */
+export interface EvalTurn {
+    user: string;
+    expectations: EvalExpectation[];
+}
+/**
+ * One turn's raw run outcome inside a multi-turn conversation (BATCH-12 Task 2). Structurally the
+ * per-turn analogue of BATCH-1's {@link ../types.js CellRunOutcome}: the turn's answer plus the
+ * tools/tokens captured FOR THAT TURN — a per-invoke GS2-16 delta (each `processMessages` call
+ * resets the tally), NOT the cumulative conversation total. `ok:false` with `error` set means that
+ * turn's SUT invocation failed (no answer to grade). The conversational runner returns one of these
+ * per turn it attempted, in turn order (a short array = the conversation aborted mid-way).
+ */
+export interface TurnRunOutcome {
+    ok: boolean;
+    answer?: string;
+    tokensInput?: number;
+    tokensOutput?: number;
+    tools?: string[];
+    error?: string;
+}
+/**
+ * Injectable "run one whole scripted conversation" function (BATCH-12 Task 2) — the multi-turn
+ * analogue of BATCH-1's {@link ../types.js RunCellFn}, and the seam that lets
+ * {@link ../evalRunner.js}'s multi-turn path be unit tested without any real LLM/MCP. Given the
+ * ordered user messages of ONE (case × identity) conversation, it builds the agent + resolves tools
+ * ONCE, runs each turn against the accumulated message history, and returns one {@link TurnRunOutcome}
+ * per turn (per-turn answer + per-turn tool delta), cleaning up once. The production wiring
+ * (`batchCommand.ts`'s `buildProductionRunConversation`) adapts core's `runConversation`; tests
+ * inject a fake returning scripted per-turn answers + traces.
+ */
+export type RunConversationFn = (userMessages: string[]) => Promise<TurnRunOutcome[]>;
+/**
+ * One case parsed and normalized from suite YAML. Every case — flat or matrix — reduces to this
+ * ONE shape: a list of {@link EvalTurn}s (Task 1: length exactly 1, its `user` = the case's
+ * `prompt`) whose expectations carry the assertions. `passThreshold` is per-case and pre-resolved
+ * (case override ?? suite `defaults.pass_threshold` ?? {@link DEFAULT_EVAL_PASS_THRESHOLD}).
+ */
+export interface EvalCase {
+    id: string;
+    turns: EvalTurn[];
     passThreshold: number;
 }
 /** A fully parsed and validated suite — see {@link ../evalSuite.js}'s `parseEvalSuite`. */
 export interface EvalSuite {
     target: EvalTarget;
+    /**
+     * Suite-level identity profiles (BATCH-12 / #405 his #1): run every case once per identity, each
+     * under `initConfig({ …, identityProfile })`, to test per-identity authorization / data-isolation.
+     * Absent = no matrix — a single run under the invoked profile (today's behavior, unchanged). Each
+     * name is a plain profile that must resolve to its own `.gsloth-settings/<name>/` config.
+     */
+    identities?: string[];
+    /** Optional suite-level judge identity profile (BATCH-10 Task 2): the profile whose model grades
+     * the cases, when the suite wants a judge distinct from the SUT. A top-level sibling of
+     * `target`/`cases` — distinct from `target.profile` (which selects the SUT and is still rejected
+     * unless `default`). Resolved by the CLI as `--judge <profile>` > this > none (none = judge uses
+     * the SUT's `config.llm`, the pre-Task-2 behavior). Absent/blank = no separate judge. */
+    judgeProfile?: string;
     cases: EvalCase[];
 }
 /** The result of running one case's answer through its deterministic checks. */
@@ -70,9 +206,36 @@ export interface JudgeOutcome {
  * of BATCH-1's `RunCellFn`. The production wiring (`evalCommand.ts`) adapts `judgeEvalCase` to
  * this shape; tests inject a fake that resolves/fails as needed. */
 export type JudgeFn = (answer: string, rubric: string) => Promise<JudgeOutcome>;
-/** One case's full graded outcome, as written to `<id>.json` and summarized in `results.json`. */
+/**
+ * One graded turn inside a multi-turn cell's result (BATCH-12 Task 2). Present only on a multi-turn
+ * {@link EvalCaseResult} (via its `turns` array); a single-turn cell keeps its flat top-level shape
+ * unchanged. Carries the turn's user message, its answer/tools/tokens, and its own verdict + reasons
+ * so the output pinpoints exactly which turn failed and why.
+ */
+export interface EvalTurnResult {
+    user: string;
+    answer?: string;
+    tokensInput?: number;
+    tokensOutput?: number;
+    tools?: string[];
+    /** Whether this turn's SUT invocation ran (mirrors {@link TurnRunOutcome.ok}); `false` = the turn
+     * produced no answer (transport/agent error, or the conversation aborted before reaching it). */
+    ok: boolean;
+    verdict: 'PASS' | 'FAIL';
+    checks?: DeterministicCheckResult;
+    judge?: JudgeOutcome;
+    /** Every reason this turn FAILed; empty when `verdict` is `PASS`. */
+    reasons: string[];
+}
+/** One graded cell's full outcome, as written to `<id>.json` (matrix: `<id>__<identity>.json`) and
+ * summarized in `results.json`. A cell is one (case × identity) pair: without a suite `identities`
+ * list there is exactly one cell per case (`identity` omitted, byte-for-byte as before BATCH-12);
+ * with identities there is one cell per (case, identity). */
 export interface EvalCaseResult {
     id: string;
+    /** The identity profile this cell ran under (BATCH-12). Omitted entirely (not `undefined`-valued)
+     * for a no-identities suite, so its `<id>.json` output stays byte-for-byte identical to before. */
+    identity?: string;
     verdict: 'PASS' | 'FAIL';
     passThreshold: number;
     /** Whether the SUT run itself completed without error (mirrors `CellRunOutcome.ok`). `false`
@@ -86,8 +249,17 @@ export interface EvalCaseResult {
     checks?: DeterministicCheckResult;
     judge?: JudgeOutcome;
     /** Every reason the case FAILed (deterministic check failures, judge-below-threshold, judge
-     * error, SUT failure). Empty when `verdict` is `PASS`. */
+     * error, SUT failure). Empty when `verdict` is `PASS`. For a multi-turn cell each reason is
+     * prefixed with the failing turn (`turn N: …`), and the full per-turn breakdown is in {@link turns}. */
     reasons: string[];
+    /**
+     * BATCH-12 Task 2 — the per-turn breakdown of a MULTI-TURN cell (one entry per conversational
+     * turn, in order). Present only for a multi-turn case; **omitted entirely for a single-turn
+     * cell**, so an existing single-`prompt` case's `<id>.json` stays byte-for-byte identical. The
+     * cell PASSES iff every turn here PASSes; the top-level `answer`/`checks`/`judge`/`tools` fields
+     * are left unset for a multi-turn cell (there is no single answer) — read {@link turns} instead.
+     */
+    turns?: EvalTurnResult[];
 }
 /** The suite-level aggregate written to `results.json` — `gth eval` exits 0 iff `failed === 0`. */
 export interface EvalSuiteSummary {

package/dist/index.d.ts

@@ -11,8 +11,12 @@ export { judgeEvalCase, buildJudgeMessages, EvalVerdictSchema, EVAL_JUDGE_DEFAUL
 export type { EvalJudgeVerdict } from '#src/judge.js';
 export { runEvalSuite } from '#src/evalRunner.js';
 export { writeEvalOutput } from '#src/evalOutput.js';
-export type { DeterministicCheckResult, EvalCase, EvalCaseResult, EvalSuite, EvalSuiteSummary, EvalTarget, JudgeFn, JudgeOutcome, JudgeVerdict, } from '#src/evalTypes.js';
+export type { AdkAgentTarget, AgUiAgentTarget, DeterministicCheckResult, EvalCase, EvalCaseResult, EvalExpectation, EvalSuite, EvalSuiteSummary, EvalTarget, GthAgentTarget, EvalTurn, EvalTurnResult, JudgeFn, JudgeOutcome, JudgeVerdict, RunConversationFn, TurnRunOutcome, } from '#src/evalTypes.js';
 export type { RunEvalSuiteOptions } from '#src/evalRunner.js';
 export { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
+export { resolveReporters, availableReporterNames } from '#src/reporters/registry.js';
+export { driveReporters } from '#src/reporters/drive.js';
+export { createTextReporter } from '#src/reporters/textReporter.js';
+export type { EvalReporter, EvalReporterFactory, EvalRunContext, NamedReporter, } from '#src/reporters/reporterTypes.js';
 export { runWorkflow } from '#src/workflow/runWorkflow.js';
 export type { WorkflowAgentOptions, WorkflowContext, RunWorkflowOptions, } from '#src/workflow/runWorkflow.js';

package/dist/index.js

@@ -11,6 +11,12 @@ export { judgeEvalCase, buildJudgeMessages, EvalVerdictSchema, EVAL_JUDGE_DEFAUL
 export { runEvalSuite } from '#src/evalRunner.js';
 export { writeEvalOutput } from '#src/evalOutput.js';
 export { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
+// BATCH-19 — the `gth eval` reporter facility (A1 seam). These are the public plugin contract an
+// out-of-core `@gaunt-sloth/eval-reporter-*` package implements, exported from the package root so a
+// reporter package can type its factory against ONE import.
+export { resolveReporters, availableReporterNames } from '#src/reporters/registry.js';
+export { driveReporters } from '#src/reporters/drive.js';
+export { createTextReporter } from '#src/reporters/textReporter.js';
 // BATCH-3 — the `gth workflow` host: runs a local JS orchestration script that drives one or more
 // LLM calls through a small WorkflowContext.
 export { runWorkflow } from '#src/workflow/runWorkflow.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAUlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAarD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAUlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqBrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,iGAAiG;AACjG,qGAAqG;AACrG,4DAA4D;AAC5D,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAQpE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC"}

package/dist/pipelineCli.d.ts

@@ -0,0 +1,102 @@
+/**
+ * @module pipelineCli
+ *
+ * BATCH-9 — the standalone `gth-batch` pipeline runner. A thin entry point that runs the BATCH-1
+ * matrix runtime ({@link buildMatrix} + {@link runBatchMatrix}) directly from a shell pipeline,
+ * without pulling in the whole `gaunt-sloth` app. It takes a prompt-executable script + `--over`
+ * data (inline JSON/YAML, or piped on stdin), runs the matrix, and streams the **same structured
+ * per-cell `CellResult`** that `gth batch` writes — one JSON object per line (JSONL) on stdout.
+ *
+ * Relationship to `gth batch` (packages/app/src/commands/batchCommand.ts): the two share the exact
+ * matrix runtime and the exact per-cell run wiring. The production per-cell adapter
+ * (`buildProductionRunCell`) lives in the app; this file mirrors it — its own `createResolvers()`,
+ * a lean agent factory, `runSingleShot` in `exec` mode, and `cleanupTools()` on every path — so the
+ * standalone bin needs only `@gaunt-sloth/core` + `@gaunt-sloth/agent` (both already dependencies),
+ * never the app. The only intentional differences: over-data is inline/stdin (not a file path, which
+ * the pipeline shell already handles) and output is JSONL on stdout (not a directory of files).
+ *
+ * stdout discipline: the run itself is noisy (the runtime's `display()`/`ProgressIndicator`/token
+ * streaming all target `process.stdout`). The bin entry ({@link file://./bin.ts}) redirects
+ * `process.stdout.write` to stderr for the duration and this module writes the machine JSONL
+ * straight to fd 1 (`fs.writeSync`), so stdout stays a clean data channel — the same "protocol
+ * channel" discipline `packages/app/cli.js` uses for ACP.
+ */
+import type { CommandLineConfigOverrides, GthConfig } from '@gaunt-sloth/core/config.js';
+import type { BatchSummary, MatrixRow, RunCellFn } from '#src/types.js';
+/** Parsed `gth-batch` invocation. */
+export interface BatchCliArgs {
+    /** Path to the `.md` prompt-executable script to run over the matrix. */
+    script: string;
+    /** Inline `--over` data (JSON/YAML array of row objects). `undefined` = fall back to stdin. */
+    over?: string;
+    /** `--models a,b,c` parsed into a trimmed, non-empty list; `undefined` when the flag is absent. */
+    models?: string[];
+    /** `-j/--concurrency <n>` — max in-flight cells. */
+    concurrency?: number;
+    /** `--retry <n>` — retries on a failed cell. */
+    retry?: number;
+}
+/** Injectable seams — all default to the real implementations; overridden in unit tests. */
+export interface BatchCliDeps {
+    /** Read the script file; default reads from disk. */
+    readScript?: (scriptPath: string) => string;
+    /** Read all of stdin to a string; default reads `process.stdin` to EOF. */
+    readStdin?: () => Promise<string>;
+    /** Machine-output sink for the JSONL cell records; default writes straight to fd 1. */
+    write?: (chunk: string) => void;
+    /** Human/error/summary sink; default writes to stderr. */
+    logError?: (chunk: string) => void;
+    /** The per-cell run function; default is the real `runSingleShot`-backed adapter. Injecting a
+     * fake here is how unit tests drive the runtime with no live LLM. */
+    runCell?: RunCellFn;
+    /** Config loader; default is the real `initConfig`. Injectable so tests skip provider resolution. */
+    initConfig?: (overrides: CommandLineConfigOverrides) => Promise<GthConfig>;
+}
+/** Parse `--models a,b,c` into a trimmed, non-empty list; `undefined` when empty/absent. */
+export declare function parseModels(models: string): string[] | undefined;
+/**
+ * Parse `gth-batch`'s argv (already sliced past `node <script>`). Pure and dependency-free so the
+ * arg surface is trivially unit-testable. Supports `--flag value` and `--flag=value`; throws a
+ * clear `Error` (with usage) on an unknown option, a missing value, or a missing `<script>`.
+ */
+export declare function parseArgs(argv: string[]): BatchCliArgs;
+/**
+ * Parse inline/stdin `--over` data — a JSON or YAML **array of row objects** — into
+ * {@link MatrixRow}s. (The app's `parseOverFile` picks CSV vs JSONL by file extension; a shell
+ * pipeline has no filename, so this bin standardizes on one self-describing format: JSON, which is
+ * also valid YAML.) Throws a descriptive `Error` on malformed input — the one thing that should
+ * make the bin exit non-zero.
+ */
+export declare function parseOverData(text: string): MatrixRow[];
+/**
+ * Resolve the input axis: inline `--over` if present, otherwise piped stdin, otherwise `undefined`
+ * (no input axis → a single cell over the script's own content, exactly like `exec`).
+ */
+export declare function resolveRows(args: BatchCliArgs, readStdin: () => Promise<string>): Promise<MatrixRow[] | undefined>;
+/** Build the exec-mode system preamble the same way `gth batch` does (app's `getExecSystemPrompt`),
+ * flattened to a string, using only `@gaunt-sloth/core` prompt builders. */
+export declare function getExecPreamble(config: GthConfig): string;
+/**
+ * Run the matrix and stream one JSON `CellResult` per line (JSONL) to `write`. Returns the
+ * aggregate {@link BatchSummary}. Each emitted line is the exact object `gth batch` writes to
+ * `<id>.json` (compact rather than pretty-printed) — the "same structured per-cell output".
+ */
+export declare function runMatrixToStream(scriptContent: string, models: string[] | undefined, rows: MatrixRow[] | undefined, options: {
+    concurrency?: number;
+    retry?: number;
+    runCell: RunCellFn;
+    write: (chunk: string) => void;
+}): Promise<BatchSummary>;
+/**
+ * The standalone `gth-batch` entry. Parses argv, resolves the script + input axis, wires the
+ * per-cell runtime, runs the matrix, and streams JSONL cell records via `deps.write`.
+ *
+ * Exit-code contract (identical to `gth batch`): resolves to `0` iff the cells *ran* — a per-cell
+ * failure is recorded in that cell's JSON, never reflected in the exit code. A harness-level error
+ * (bad args, unreadable script, malformed `--over`, config failure) resolves to `1`. A downstream
+ * reader closing the pipe early (`EPIPE` on the stdout channel — e.g. `| head`) is normal pipeline
+ * usage and also resolves to `0`: the partial output already written is correct.
+ *
+ * @returns the process exit code (0 or 1).
+ */
+export declare function runBatchCli(argv: string[], deps?: BatchCliDeps): Promise<number>;