@uns-kit/assistant-workflow 2.0.72

package/dist/eval-case.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-case.js","sourceRoot":"","sources":["../src/eval-case.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,gCAAgC,IAAI,eAAe,EACnD,iCAAiC,IAAI,sBAAsB,EAC3D,2BAA2B,IAAI,gBAAgB,EAC/C,gCAAgC,IAAI,eAAe,GACpD,MAAM,oBAAoB,CAAC;AAE5B,MAAM,CAAC,MAAM,2CAA2C,GAAG,CAAC,CAAC;AA8F7D,MAAM,UAAU,8BAA8B,CAAC,KAAqC;IAClF,MAAM,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,6BAA6B,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACvE,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,qBAAqB,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;IAEtC,OAAO;QACL,aAAa,EAAE,2CAA2C;QAC1D,EAAE,EAAE,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,eAAe,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC;QAClG,MAAM;QACN,QAAQ,EAAE,KAAK,CAAC,QAAQ,KAAK,IAAI;QACjC,IAAI;QACJ,MAAM;QACN,YAAY;QACZ,KAAK;QACL,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gDAAgD,CAC9D,SAA8C,EAC9C,UAAiD,EAAE;IAEnD,OAAO,8BAA8B,CAAC;QACpC,MAAM,EAAE,SAAS,CAAC,aAAa;QAC/B,MAAM,EAAE;YACN,GAAG,OAAO,CAAC,MAAM;YACjB,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,cAAc;SAC7C;QACD,YAAY,EAAE;YACZ,MAAM,EAAE,SAAS,CAAC,cAAc;YAChC,YAAY,EAAE,SAAS,CAAC,oBAAoB;YAC5C,WAAW,EAAE,SAAS,CAAC,mBAAmB;YAC1C,4BAA4B,EAAE,SAAS,CAAC,oCAAoC;YAC5E,cAAc,EAAE,SAAS,CAAC,sBAAsB;YAChD,KAAK,EAAE,SAAS,CAAC,aAAa;YAC9B,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;SACjD;QACD,KAAK,EAAE,CAAC,GAAG,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACrD,GAAG,CAAC,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1E,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,qDAAqD,CACnE,MAA4C,EAC5C,OAAkD;IAElD,OAAO,8BAA8B,CAAC;QACpC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE;YACN,GAAG,OAAO,CAAC,MAAM;YACjB,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,YAAY;YAC1C,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,IAAI,MAAM,CAAC,UAAU;YAC3D,eAAe,EAAE,OAAO,CAAC,MAAM,EAAE,eAAe,IAAI,MAAM,CAAC,eAAe;SAC3E;QACD,YAAY,EAAE;YACZ,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YACnH,YAAY,EAAE,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;YACvF,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC3G,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC;YACpC,4BAA4B,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAChG,cAAc,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;YACpE,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;YACtC,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC;SACrC;QACD,GAAG,CAAC,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1E,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iEAAiE,CAC/E,QAA0D,EAC1D,OAAsD;IAEtD,OAAO,8BAA8B,CAAC;QACpC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE;YACN,GAAG,OAAO,CAAC,MAAM;YACjB,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,gBAAgB;YAC9C,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,IAAI,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;YAC3H,eAAe,EAAE,OAAO,CAAC,MAAM,EAAE,eAAe,IAAI,mBAAmB,CAAC,QAAQ,CAAC;SAClF;QACD,YAAY,EAAE;YACZ,MAAM,EAAE,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,QAAQ,CAAC,CAAC;mBACjE,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,aAAa,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;YAC9F,YAAY,EAAE,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,uBAAuB,CAAC,CAAC;mBACtF,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,aAAa,EAAE,UAAU,EAAE,uBAAuB,CAAC,CAAC;YAC7G,WAAW,EAAE,wBAAwB,CAAC,QAAQ,CAAC;YAC/C,4BAA4B,EAAE,yCAAyC,CAAC,QAAQ,CAAC;YACjF,cAAc,EAAE,2BAA2B,CAAC,QAAQ,CAAC;YACrD,KAAK,EAAE,QAAQ,CAAC,kBAAkB;YAClC,WAAW,EAAE,qBAAqB,CAAC;gBACjC,QAAQ,CAAC,SAAS,CAAC,MAAM;gBACzB,QAAQ,CAAC,SAAS,CAAC,MAAM;aAC1B,CAAC;SACH;QACD,KAAK,EAAE;YACL,6BAA6B,QAAQ,CAAC,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE;YACrF,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;SACzB;QACD,QAAQ,EAAE;YACR,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC3B,eAAe,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM;YAC1C,eAAe,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM;SAC3C;QACD,GAAG,CAAC,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,QAAmC,EACnC,KAAc;IAEd,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,uCAAuC,CACrD,KAA2C,EAC3C,UAAyC,EAAE;IAE3C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,OAAO,OAAO,CAAC,eAAe,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACvF,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,KAAc;IAC3D,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,KAAK,CAAC,eAAe,CAAC,KAAK,2CAA2C;QAAE,OAAO,IAAI,CAAC;IACxF,IAAI,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/E,IAAI,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACvF,IAAI,OAAO,KAAK,CAAC,UAAU,CAAC,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACxD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpD,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAChE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAElD,OAAO;QACL,aAAa,EAAE,2CAA2C;QAC1D,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC;QACf,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC;QACvB,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;QAC3B,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;QACnB,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC;QACvB,YAAY,EAAE,6BAA6B,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAClE,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC;QACrB,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,KAAa;IAC/D,MAAM,KAAK,GAAgC,EAAE,CAAC;IAC9C,MAAM,MAAM,GAAyC,EAAE,CAAC;IACxD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,SAAS;QAC9B,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7B,SAAS,IAAI,CAAC,CAAC;QAEf,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC;gBACV,UAAU;gBACV,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;aAC9B,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC;gBACV,UAAU;gBACV,MAAM,EAAE,mBAAmB;gBAC3B,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;aAC9B,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,OAAO;QACL,SAAS;QACT,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,KAAK;QACL,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,MAAmE,EACnE,YAAiD;IAEjD,OAAO;QACL,IAAI,EAAE,oBAAoB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY;QACrE,SAAS,EAAE,uBAAuB,CAAC,MAAM,EAAE,SAAS,CAAC;QACrD,OAAO,EAAE,uBAAuB,CAAC,MAAM,EAAE,OAAO,CAAC;QACjD,UAAU,EAAE,uBAAuB,CAAC,MAAM,EAAE,UAAU,CAAC;QACvD,eAAe,EAAE,OAAO,MAAM,EAAE,eAAe,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,eAAe,CAAC;YACrG,CAAC,CAAC,MAAM,CAAC,eAAe;YACxB,CAAC,CAAC,IAAI;QACR,SAAS,EAAE,uBAAuB,CAAC,MAAM,EAAE,SAAS,CAAC;KACtD,CAAC;AACJ,CAAC;AAED,SAAS,6BAA6B,CACpC,YAA+E;IAE/E,OAAO;QACL,MAAM,EAAE,uBAAuB,CAAC,YAAY,EAAE,MAAM,CAAC;QACrD,YAAY,EAAE,uBAAuB,CAAC,YAAY,EAAE,YAAY,CAAC;QACjE,MAAM,EAAE,iBAAiB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QAC5E,WAAW,EAAE,uBAAuB,CAAC,YAAY,EAAE,WAAW,CAAC;QAC/D,WAAW,EAAE,qBAAqB,CAAC,YAAY,EAAE,WAAW,IAAI,EAAE,CAAC;QACnE,4BAA4B,EAAE,qBAAqB,CAAC,YAAY,EAAE,4BAA4B,IAAI,EAAE,CAAC;QACrG,cAAc,EAAE,qBAAqB,CAAC,YAAY,EAAE,cAAc,IAAI,EAAE,CAAC;QACzE,KAAK,EAAE,qBAAqB,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC;QACvD,WAAW,EAAE,qBAAqB,CAAC,YAAY,EAAE,WAAW,IAAI,EAAE,CAAC;QACnE,kBAAkB,EAAE,qBAAqB,CAAC,YAAY,EAAE,kBAAkB,IAAI,EAAE,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,MAA4C;IACnE,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC;IACtE,OAAO,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,uBAAuB,CAAC,MAA4C;IAC3E,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC,CAAC;IACxF,OAAO,qBAAqB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAC9D,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CACrD,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,MAA4C;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IAChE,OAAO,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtG,CAAC;AAED,SAAS,wBAAwB,CAAC,QAA0D;IAC1F,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAClF,IAAI,QAAQ,CAAC,MAAM;QAAE,OAAO,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC5D,MAAM,iBAAiB,GAAG,eAAe,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC;IAClG,OAAO,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxG,CAAC;AAED,SAAS,yCAAyC,CAAC,QAA0D;IAC3G,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,MAAM,EAAE,8BAA8B,CAAC,CAAC,CAAC;IACvG,IAAI,QAAQ,CAAC,MAAM;QAAE,OAAO,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC5D,OAAO,eAAe,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,aAAa,EAAE,UAAU,EAAE,MAAM,EAAE,8BAA8B,CAAC,CAAC;SACpH,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,2BAA2B,CAAC,QAA0D;IAC7F,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACzF,IAAI,QAAQ,CAAC,MAAM;QAAE,OAAO,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC5D,OAAO,eAAe,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC,aAAa,EAAE,UAAU,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;SACtG,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,mBAAmB,CAAC,QAA0D;IACrF,MAAM,OAAO,GAAG,QAAQ,CAAC,iBAAiB,CAAC;IAC3C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC3C,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,OAAO,WAAW,CAAC,iBAAiB,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;QAC1G,CAAC,CAAC,WAAW,CAAC,iBAAiB,CAAC;QAChC,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,eAAe,CACtB,MAAc,EACd,MAAqB,EACrB,UAA+C;IAE/C,MAAM,MAAM,GAAG,CAAC,UAAU,EAAE,MAAM,IAAI,SAAS,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5E,OAAO,GAAG,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAClG,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AACpD,CAAC;AAED,SAAS,qBAAqB,CAAC,MAA0B;IACvD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,SAAS;QACxC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAgC;IAC/D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAa,EAAE,SAAiB;IAC/D,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,gCAAgC,SAAS,eAAe,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,CACL,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnC,gBAAgB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACpC,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAClC,gBAAgB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACrC,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,iBAAiB,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;QAClI,gBAAgB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CACrC,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC5C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,CACL,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjC,gBAAgB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACvC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChE,gBAAgB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACtC,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACnC,CAAC,KAAK,CAAC,8BAA8B,CAAC,KAAK,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;QAC7G,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACjF,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACnC,aAAa,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAC3C,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,cAAc,IAAI,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,gBAAgB,CAAC;AAChH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,CACL,KAAK,KAAK,WAAW;QACrB,KAAK,KAAK,eAAe;QACzB,KAAK,KAAK,UAAU;QACpB,KAAK,KAAK,QAAQ;QAClB,KAAK,KAAK,aAAa,CACxB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACzE,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC7D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC1D,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC;AAChE,CAAC","sourcesContent":["import type { AssistantWorkflowRunReportStatus } from \"./run-report.js\";\nimport type {\n  AssistantWorkflowJsonValue,\n  AssistantWorkflowSerializedRunReport,\n} from \"./run-report-json.js\";\nimport type { AssistantWorkflowSerializedToolSelectionDecision } from \"./tool-selection-json.js\";\nimport type { AssistantWorkflowTraceEvalCandidate } from \"./trace-replay.js\";\nimport {\n  readAssistantWorkflowNestedArray as readNestedArray,\n  readAssistantWorkflowNestedString as readNestedStringOrNull,\n  readAssistantWorkflowString as readStringOrNull,\n  readAssistantWorkflowStringArray as readStringArray,\n} from \"./value-readers.js\";\n\nexport const ASSISTANT_WORKFLOW_EVAL_CASE_SCHEMA_VERSION = 1;\n\nexport type AssistantWorkflowEvalCaseSourceKind =\n  | \"manual\"\n  | \"trace-replay\"\n  | \"run-report\"\n  | \"tool-selection\";\n\nexport type AssistantWorkflowEvalCaseSource = {\n  kind: AssistantWorkflowEvalCaseSourceKind;\n  requestId: string | null;\n  debugId: string | null;\n  workflowId: string | null;\n  workflowVersion: number | null;\n  createdAt: string | null;\n};\n\nexport type AssistantWorkflowEvalCaseExpectations = {\n  intent: string | null;\n  presentation: string | null;\n  status: AssistantWorkflowRunReportStatus | null;\n  outcomeKind: string | null;\n  planStepIds: string[];\n  activePlanningStepProfileIds: string[];\n  profileStepIds: string[];\n  tools: string[];\n  signalNames: string[];\n  qualitySignalNames: string[];\n};\n\ntype AssistantWorkflowEvalCaseExpectationRecord =\n  Omit<AssistantWorkflowEvalCaseExpectations, \"activePlanningStepProfileIds\" | \"profileStepIds\"> &\n    Partial<Pick<AssistantWorkflowEvalCaseExpectations, \"activePlanningStepProfileIds\" | \"profileStepIds\">>;\n\nexport type AssistantWorkflowEvalCase = {\n  schemaVersion: typeof ASSISTANT_WORKFLOW_EVAL_CASE_SCHEMA_VERSION;\n  id: string;\n  prompt: string;\n  required: boolean;\n  tags: string[];\n  source: AssistantWorkflowEvalCaseSource;\n  expectations: AssistantWorkflowEvalCaseExpectations;\n  notes: string[];\n  metadata: Record<string, AssistantWorkflowJsonValue>;\n};\n\nexport type AssistantWorkflowEvalCaseInput = {\n  id?: string | null;\n  prompt: string;\n  required?: boolean | null;\n  tags?: readonly string[] | null;\n  source?: Partial<AssistantWorkflowEvalCaseSource> | null;\n  expectations?: Partial<AssistantWorkflowEvalCaseExpectations> | null;\n  notes?: readonly string[] | null;\n  metadata?: Record<string, AssistantWorkflowJsonValue> | null;\n};\n\nexport type AssistantWorkflowTraceEvalCaseOptions = {\n  id?: string | null;\n  required?: boolean | null;\n  tags?: readonly string[] | null;\n  source?: Partial<AssistantWorkflowEvalCaseSource> | null;\n  notes?: readonly string[] | null;\n  metadata?: Record<string, AssistantWorkflowJsonValue> | null;\n};\n\nexport type AssistantWorkflowRunReportEvalCaseOptions =\n  AssistantWorkflowTraceEvalCaseOptions & {\n    prompt: string;\n  };\n\nexport type AssistantWorkflowToolSelectionEvalCaseOptions =\n  AssistantWorkflowTraceEvalCaseOptions & {\n    prompt: string;\n  };\n\nexport type AssistantWorkflowEvalCaseLineErrorReason =\n  | \"invalid_json\"\n  | \"invalid_eval_case\";\n\nexport type AssistantWorkflowEvalCaseLineError = {\n  lineNumber: number;\n  reason: AssistantWorkflowEvalCaseLineErrorReason;\n  preview: string;\n};\n\nexport type AssistantWorkflowEvalCaseLinesParseResult = {\n  lineCount: number;\n  caseCount: number;\n  errorCount: number;\n  cases: AssistantWorkflowEvalCase[];\n  errors: AssistantWorkflowEvalCaseLineError[];\n};\n\nexport function buildAssistantWorkflowEvalCase(input: AssistantWorkflowEvalCaseInput): AssistantWorkflowEvalCase {\n  const prompt = normalizeRequiredString(input.prompt, \"prompt\");\n  const source = normalizeEvalCaseSource(input.source, \"manual\");\n  const expectations = normalizeEvalCaseExpectations(input.expectations);\n  const tags = uniqueNonEmptyStrings(input.tags ?? []);\n  const notes = uniqueNonEmptyStrings(input.notes ?? []);\n  const metadata = input.metadata ?? {};\n\n  return {\n    schemaVersion: ASSISTANT_WORKFLOW_EVAL_CASE_SCHEMA_VERSION,\n    id: normalizeOptionalString(input.id) ?? buildEvalCaseId(prompt, expectations.intent, source.kind),\n    prompt,\n    required: input.required === true,\n    tags,\n    source,\n    expectations,\n    notes,\n    metadata,\n  };\n}\n\nexport function buildAssistantWorkflowEvalCaseFromTraceCandidate(\n  candidate: AssistantWorkflowTraceEvalCandidate,\n  options: AssistantWorkflowTraceEvalCaseOptions = {},\n): AssistantWorkflowEvalCase {\n  return buildAssistantWorkflowEvalCase({\n    prompt: candidate.promptPreview,\n    source: {\n      ...options.source,\n      kind: options.source?.kind ?? \"trace-replay\",\n    },\n    expectations: {\n      intent: candidate.expectedIntent,\n      presentation: candidate.expectedPresentation,\n      planStepIds: candidate.expectedPlanStepIds,\n      activePlanningStepProfileIds: candidate.expectedActivePlanningStepProfileIds,\n      profileStepIds: candidate.expectedProfileStepIds,\n      tools: candidate.expectedTools,\n      qualitySignalNames: candidate.qualitySignalNames,\n    },\n    notes: [...candidate.notes, ...(options.notes ?? [])],\n    ...(options.id !== undefined ? { id: options.id } : {}),\n    ...(options.required !== undefined ? { required: options.required } : {}),\n    ...(options.tags !== undefined ? { tags: options.tags } : {}),\n    ...(options.metadata !== undefined ? { metadata: options.metadata } : {}),\n  });\n}\n\nexport function buildAssistantWorkflowEvalCaseFromSerializedRunReport(\n  report: AssistantWorkflowSerializedRunReport,\n  options: AssistantWorkflowRunReportEvalCaseOptions,\n): AssistantWorkflowEvalCase {\n  return buildAssistantWorkflowEvalCase({\n    prompt: options.prompt,\n    source: {\n      ...options.source,\n      kind: options.source?.kind ?? \"run-report\",\n      workflowId: options.source?.workflowId ?? report.workflowId,\n      workflowVersion: options.source?.workflowVersion ?? report.workflowVersion,\n    },\n    expectations: {\n      intent: readStringOrNull(report.evaluation[\"intent\"]) ?? readNestedStringOrNull(report.run, [\"decision\", \"intent\"]),\n      presentation: readNestedStringOrNull(report.run, [\"decision\", \"effectivePresentation\"]),\n      status: report.status,\n      outcomeKind: readStringOrNull(report.evaluation[\"outcomeKind\"]) ?? readStringOrNull(report.outcome[\"kind\"]),\n      planStepIds: readPlanStepIds(report),\n      activePlanningStepProfileIds: readStringArray(report.evaluation[\"activePlanningStepProfileIds\"]),\n      profileStepIds: readStringArray(report.evaluation[\"profileStepIds\"]),\n      tools: readInvocationToolNames(report),\n      signalNames: readSignalNames(report),\n    },\n    ...(options.id !== undefined ? { id: options.id } : {}),\n    ...(options.required !== undefined ? { required: options.required } : {}),\n    ...(options.tags !== undefined ? { tags: options.tags } : {}),\n    ...(options.notes !== undefined ? { notes: options.notes } : {}),\n    ...(options.metadata !== undefined ? { metadata: options.metadata } : {}),\n  });\n}\n\nexport function buildAssistantWorkflowEvalCaseFromSerializedToolSelectionDecision(\n  decision: AssistantWorkflowSerializedToolSelectionDecision,\n  options: AssistantWorkflowToolSelectionEvalCaseOptions,\n): AssistantWorkflowEvalCase {\n  return buildAssistantWorkflowEvalCase({\n    prompt: options.prompt,\n    source: {\n      ...options.source,\n      kind: options.source?.kind ?? \"tool-selection\",\n      workflowId: options.source?.workflowId ?? readNestedStringOrNull(decision.comparisonPayload, [\"workflowRun\", \"workflowId\"]),\n      workflowVersion: options.source?.workflowVersion ?? readWorkflowVersion(decision),\n    },\n    expectations: {\n      intent: readNestedStringOrNull(decision.comparisonPayload, [\"intent\"])\n        ?? readNestedStringOrNull(decision.comparisonPayload, [\"workflowRun\", \"decision\", \"intent\"]),\n      presentation: readNestedStringOrNull(decision.comparisonPayload, [\"effectivePresentation\"])\n        ?? readNestedStringOrNull(decision.comparisonPayload, [\"workflowRun\", \"decision\", \"effectivePresentation\"]),\n      planStepIds: readSelectionPlanStepIds(decision),\n      activePlanningStepProfileIds: readSelectionActivePlanningStepProfileIds(decision),\n      profileStepIds: readSelectionProfileStepIds(decision),\n      tools: decision.effectiveToolNames,\n      signalNames: uniqueNonEmptyStrings([\n        decision.authority.source,\n        decision.authority.reason,\n      ]),\n    },\n    notes: [\n      `tool_selection_authority: ${decision.authority.source}/${decision.authority.reason}`,\n      ...(options.notes ?? []),\n    ],\n    metadata: {\n      ...(options.metadata ?? {}),\n      authoritySource: decision.authority.source,\n      authorityReason: decision.authority.reason,\n    },\n    ...(options.id !== undefined ? { id: options.id } : {}),\n    ...(options.required !== undefined ? { required: options.required } : {}),\n    ...(options.tags !== undefined ? { tags: options.tags } : {}),\n  });\n}\n\nexport function stringifyAssistantWorkflowEvalCase(\n  evalCase: AssistantWorkflowEvalCase,\n  space?: number,\n): string {\n  return JSON.stringify(evalCase, null, space);\n}\n\nexport function stringifyAssistantWorkflowEvalCaseLines(\n  cases: readonly AssistantWorkflowEvalCase[],\n  options: { trailingNewline?: boolean } = {},\n): string {\n  const lines = cases.map((evalCase) => JSON.stringify(evalCase));\n  if (!lines.length) return \"\";\n  return options.trailingNewline === true ? `${lines.join(\"\\n\")}\\n` : lines.join(\"\\n\");\n}\n\nexport function parseAssistantWorkflowEvalCase(value: unknown): AssistantWorkflowEvalCase | null {\n  if (!isRecord(value)) return null;\n  if (value[\"schemaVersion\"] !== ASSISTANT_WORKFLOW_EVAL_CASE_SCHEMA_VERSION) return null;\n  if (typeof value[\"id\"] !== \"string\" || !value[\"id\"].trim().length) return null;\n  if (typeof value[\"prompt\"] !== \"string\" || !value[\"prompt\"].trim().length) return null;\n  if (typeof value[\"required\"] !== \"boolean\") return null;\n  if (!isStringArray(value[\"tags\"])) return null;\n  if (!isEvalCaseSource(value[\"source\"])) return null;\n  if (!isEvalCaseExpectations(value[\"expectations\"])) return null;\n  if (!isStringArray(value[\"notes\"])) return null;\n  if (!isJsonRecord(value[\"metadata\"])) return null;\n\n  return {\n    schemaVersion: ASSISTANT_WORKFLOW_EVAL_CASE_SCHEMA_VERSION,\n    id: value[\"id\"],\n    prompt: value[\"prompt\"],\n    required: value[\"required\"],\n    tags: value[\"tags\"],\n    source: value[\"source\"],\n    expectations: normalizeEvalCaseExpectations(value[\"expectations\"]),\n    notes: value[\"notes\"],\n    metadata: value[\"metadata\"],\n  };\n}\n\nexport function parseAssistantWorkflowEvalCaseLines(input: string): AssistantWorkflowEvalCaseLinesParseResult {\n  const cases: AssistantWorkflowEvalCase[] = [];\n  const errors: AssistantWorkflowEvalCaseLineError[] = [];\n  let lineCount = 0;\n\n  for (const [index, line] of input.split(/\\r?\\n/).entries()) {\n    const trimmed = line.trim();\n    if (!trimmed.length) continue;\n    const lineNumber = index + 1;\n    lineCount += 1;\n\n    let parsed: unknown;\n    try {\n      parsed = JSON.parse(trimmed);\n    } catch {\n      errors.push({\n        lineNumber,\n        reason: \"invalid_json\",\n        preview: previewLine(trimmed),\n      });\n      continue;\n    }\n\n    const evalCase = parseAssistantWorkflowEvalCase(parsed);\n    if (!evalCase) {\n      errors.push({\n        lineNumber,\n        reason: \"invalid_eval_case\",\n        preview: previewLine(trimmed),\n      });\n      continue;\n    }\n    cases.push(evalCase);\n  }\n\n  return {\n    lineCount,\n    caseCount: cases.length,\n    errorCount: errors.length,\n    cases,\n    errors,\n  };\n}\n\nfunction normalizeEvalCaseSource(\n  source: Partial<AssistantWorkflowEvalCaseSource> | null | undefined,\n  fallbackKind: AssistantWorkflowEvalCaseSourceKind,\n): AssistantWorkflowEvalCaseSource {\n  return {\n    kind: isEvalCaseSourceKind(source?.kind) ? source.kind : fallbackKind,\n    requestId: normalizeOptionalString(source?.requestId),\n    debugId: normalizeOptionalString(source?.debugId),\n    workflowId: normalizeOptionalString(source?.workflowId),\n    workflowVersion: typeof source?.workflowVersion === \"number\" && Number.isFinite(source.workflowVersion)\n      ? source.workflowVersion\n      : null,\n    createdAt: normalizeOptionalString(source?.createdAt),\n  };\n}\n\nfunction normalizeEvalCaseExpectations(\n  expectations: Partial<AssistantWorkflowEvalCaseExpectations> | null | undefined,\n): AssistantWorkflowEvalCaseExpectations {\n  return {\n    intent: normalizeOptionalString(expectations?.intent),\n    presentation: normalizeOptionalString(expectations?.presentation),\n    status: isRunReportStatus(expectations?.status) ? expectations.status : null,\n    outcomeKind: normalizeOptionalString(expectations?.outcomeKind),\n    planStepIds: uniqueNonEmptyStrings(expectations?.planStepIds ?? []),\n    activePlanningStepProfileIds: uniqueNonEmptyStrings(expectations?.activePlanningStepProfileIds ?? []),\n    profileStepIds: uniqueNonEmptyStrings(expectations?.profileStepIds ?? []),\n    tools: uniqueNonEmptyStrings(expectations?.tools ?? []),\n    signalNames: uniqueNonEmptyStrings(expectations?.signalNames ?? []),\n    qualitySignalNames: uniqueNonEmptyStrings(expectations?.qualitySignalNames ?? []),\n  };\n}\n\nfunction readPlanStepIds(report: AssistantWorkflowSerializedRunReport): string[] {\n  const steps = readNestedArray(report.run, [\"executionPlan\", \"steps\"]);\n  return uniqueNonEmptyStrings(steps.flatMap((step) => isRecord(step) ? [step[\"id\"]] : []));\n}\n\nfunction readInvocationToolNames(report: AssistantWorkflowSerializedRunReport): string[] {\n  const invocations = readNestedArray(report.run, [\"toolInvocationQueue\", \"invocations\"]);\n  return uniqueNonEmptyStrings(invocations.flatMap((invocation) =>\n    isRecord(invocation) ? [invocation[\"toolName\"]] : []\n  ));\n}\n\nfunction readSignalNames(report: AssistantWorkflowSerializedRunReport): string[] {\n  const signals = readNestedArray(report.evaluation, [\"signals\"]);\n  return uniqueNonEmptyStrings(signals.flatMap((signal) => isRecord(signal) ? [signal[\"name\"]] : []));\n}\n\nfunction readSelectionPlanStepIds(decision: AssistantWorkflowSerializedToolSelectionDecision): string[] {\n  const fromPlan = readNestedArray(decision.comparisonPayload, [\"plan\", \"stepIds\"]);\n  if (fromPlan.length) return uniqueNonEmptyStrings(fromPlan);\n  const fromExecutionPlan = readNestedArray(decision.comparisonPayload, [\"executionPlan\", \"steps\"]);\n  return uniqueNonEmptyStrings(fromExecutionPlan.flatMap((step) => isRecord(step) ? [step[\"id\"]] : []));\n}\n\nfunction readSelectionActivePlanningStepProfileIds(decision: AssistantWorkflowSerializedToolSelectionDecision): string[] {\n  const fromPlan = readNestedArray(decision.comparisonPayload, [\"plan\", \"activePlanningStepProfileIds\"]);\n  if (fromPlan.length) return uniqueNonEmptyStrings(fromPlan);\n  return readNestedArray(decision.comparisonPayload, [\"workflowRun\", \"decision\", \"plan\", \"activePlanningStepProfileIds\"])\n    .filter((value): value is string => typeof value === \"string\");\n}\n\nfunction readSelectionProfileStepIds(decision: AssistantWorkflowSerializedToolSelectionDecision): string[] {\n  const fromPlan = readNestedArray(decision.comparisonPayload, [\"plan\", \"profileStepIds\"]);\n  if (fromPlan.length) return uniqueNonEmptyStrings(fromPlan);\n  return readNestedArray(decision.comparisonPayload, [\"workflowRun\", \"decision\", \"plan\", \"profileStepIds\"])\n    .filter((value): value is string => typeof value === \"string\");\n}\n\nfunction readWorkflowVersion(decision: AssistantWorkflowSerializedToolSelectionDecision): number | null {\n  const payload = decision.comparisonPayload;\n  if (!isRecord(payload)) return null;\n  const workflowRun = payload[\"workflowRun\"];\n  if (!isRecord(workflowRun)) return null;\n  return typeof workflowRun[\"workflowVersion\"] === \"number\" && Number.isFinite(workflowRun[\"workflowVersion\"])\n    ? workflowRun[\"workflowVersion\"]\n    : null;\n}\n\nfunction buildEvalCaseId(\n  prompt: string,\n  intent: string | null,\n  sourceKind: AssistantWorkflowEvalCaseSourceKind,\n): string {\n  const prefix = [sourceKind, intent ?? \"unknown\"].map(toIdSegment).join(\"-\");\n  return `${prefix}-${hashString(prompt)}`;\n}\n\nfunction hashString(value: string): string {\n  let hash = 5381;\n  for (let index = 0; index < value.length; index += 1) {\n    hash = ((hash << 5) + hash) ^ value.charCodeAt(index);\n  }\n  return (hash >>> 0).toString(36);\n}\n\nfunction toIdSegment(value: string): string {\n  const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, \"-\").replace(/^-+|-+$/g, \"\");\n  return normalized.length ? normalized : \"unknown\";\n}\n\nfunction uniqueNonEmptyStrings(values: readonly unknown[]): string[] {\n  const out: string[] = [];\n  const seen = new Set<string>();\n  for (const value of values) {\n    if (typeof value !== \"string\") continue;\n    const trimmed = value.trim();\n    if (!trimmed.length || seen.has(trimmed)) continue;\n    seen.add(trimmed);\n    out.push(trimmed);\n  }\n  return out;\n}\n\nfunction normalizeOptionalString(value: string | null | undefined): string | null {\n  if (typeof value !== \"string\") return null;\n  const trimmed = value.trim();\n  return trimmed.length ? trimmed : null;\n}\n\nfunction normalizeRequiredString(value: string, fieldName: string): string {\n  const trimmed = value.trim();\n  if (!trimmed.length) {\n    throw new Error(`Assistant workflow eval case ${fieldName} is required.`);\n  }\n  return trimmed;\n}\n\nfunction isEvalCaseSource(value: unknown): value is AssistantWorkflowEvalCaseSource {\n  if (!isRecord(value)) return false;\n  return (\n    isEvalCaseSourceKind(value[\"kind\"]) &&\n    isNullableString(value[\"requestId\"]) &&\n    isNullableString(value[\"debugId\"]) &&\n    isNullableString(value[\"workflowId\"]) &&\n    (value[\"workflowVersion\"] === null || (typeof value[\"workflowVersion\"] === \"number\" && Number.isFinite(value[\"workflowVersion\"]))) &&\n    isNullableString(value[\"createdAt\"])\n  );\n}\n\nfunction isEvalCaseExpectations(value: unknown): value is AssistantWorkflowEvalCaseExpectationRecord {\n  if (!isRecord(value)) return false;\n  return (\n    isNullableString(value[\"intent\"]) &&\n    isNullableString(value[\"presentation\"]) &&\n    (value[\"status\"] === null || isRunReportStatus(value[\"status\"])) &&\n    isNullableString(value[\"outcomeKind\"]) &&\n    isStringArray(value[\"planStepIds\"]) &&\n    (value[\"activePlanningStepProfileIds\"] === undefined || isStringArray(value[\"activePlanningStepProfileIds\"])) &&\n    (value[\"profileStepIds\"] === undefined || isStringArray(value[\"profileStepIds\"])) &&\n    isStringArray(value[\"tools\"]) &&\n    isStringArray(value[\"signalNames\"]) &&\n    isStringArray(value[\"qualitySignalNames\"])\n  );\n}\n\nfunction isEvalCaseSourceKind(value: unknown): value is AssistantWorkflowEvalCaseSourceKind {\n  return value === \"manual\" || value === \"trace-replay\" || value === \"run-report\" || value === \"tool-selection\";\n}\n\nfunction isRunReportStatus(value: unknown): value is AssistantWorkflowRunReportStatus {\n  return (\n    value === \"completed\" ||\n    value === \"clarification\" ||\n    value === \"degraded\" ||\n    value === \"failed\" ||\n    value === \"not_handled\"\n  );\n}\n\nfunction isNullableString(value: unknown): value is string | null {\n  return value === null || typeof value === \"string\";\n}\n\nfunction isStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.every((entry) => typeof entry === \"string\");\n}\n\nfunction isJsonRecord(value: unknown): value is Record<string, AssistantWorkflowJsonValue> {\n  if (!isRecord(value)) return false;\n  return Object.values(value).every(isJsonValue);\n}\n\nfunction isJsonValue(value: unknown): value is AssistantWorkflowJsonValue {\n  if (value === null) return true;\n  if (typeof value === \"string\" || typeof value === \"boolean\") return true;\n  if (typeof value === \"number\") return Number.isFinite(value);\n  if (Array.isArray(value)) return value.every(isJsonValue);\n  return isJsonRecord(value);\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction previewLine(line: string): string {\n  return line.length <= 160 ? line : `${line.slice(0, 157)}...`;\n}\n"]}

package/dist/eval-replay.d.ts

@@ -0,0 +1,18 @@
+import { type AssistantWorkflowEvalCaseLinesParseResult } from "./eval-case.js";
+import { type AssistantWorkflowEvalRun, type AssistantWorkflowEvalRunReportsOptions } from "./eval-runner.js";
+import type { AssistantWorkflowJsonValue } from "./run-report-json.js";
+import { type AssistantWorkflowSerializedRunReportLinesParseResult } from "./run-report-json.js";
+export type AssistantWorkflowEvalReplayLinesOptions = AssistantWorkflowEvalRunReportsOptions;
+export type AssistantWorkflowEvalReplayLinesInput = {
+    evalCaseLines: string;
+    reportLines: string;
+};
+export type AssistantWorkflowEvalReplayLinesResult = {
+    caseParse: AssistantWorkflowEvalCaseLinesParseResult;
+    reportParse: AssistantWorkflowSerializedRunReportLinesParseResult;
+    parseErrorCount: number;
+    run: AssistantWorkflowEvalRun;
+};
+export declare function runAssistantWorkflowEvalReplayLines(input: AssistantWorkflowEvalReplayLinesInput, options?: AssistantWorkflowEvalReplayLinesOptions): AssistantWorkflowEvalReplayLinesResult;
+export declare function buildAssistantWorkflowEvalReplayLinesTracePayload(result: AssistantWorkflowEvalReplayLinesResult): Record<string, AssistantWorkflowJsonValue>;
+//# sourceMappingURL=eval-replay.d.ts.map

package/dist/eval-replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-replay.d.ts","sourceRoot":"","sources":["../src/eval-replay.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,yCAAyC,EAC/C,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAGL,KAAK,wBAAwB,EAC7B,KAAK,sCAAsC,EAC5C,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAEL,KAAK,oDAAoD,EAC1D,MAAM,sBAAsB,CAAC;AAE9B,MAAM,MAAM,uCAAuC,GAAG,sCAAsC,CAAC;AAE7F,MAAM,MAAM,qCAAqC,GAAG;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,sCAAsC,GAAG;IACnD,SAAS,EAAE,yCAAyC,CAAC;IACrD,WAAW,EAAE,oDAAoD,CAAC;IAClE,eAAe,EAAE,MAAM,CAAC;IACxB,GAAG,EAAE,wBAAwB,CAAC;CAC/B,CAAC;AAEF,wBAAgB,mCAAmC,CACjD,KAAK,EAAE,qCAAqC,EAC5C,OAAO,GAAE,uCAA4C,GACpD,sCAAsC,CAexC;AAED,wBAAgB,iDAAiD,CAC/D,MAAM,EAAE,sCAAsC,GAC7C,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAyB5C"}

package/dist/eval-replay.js

@@ -0,0 +1,41 @@
+import { parseAssistantWorkflowEvalCaseLines, } from "./eval-case.js";
+import { buildAssistantWorkflowEvalRunTracePayload, runAssistantWorkflowEvalCasesAgainstSerializedRunReports, } from "./eval-runner.js";
+import { parseAssistantWorkflowSerializedRunReportLines, } from "./run-report-json.js";
+export function runAssistantWorkflowEvalReplayLines(input, options = {}) {
+    const caseParse = parseAssistantWorkflowEvalCaseLines(input.evalCaseLines);
+    const reportParse = parseAssistantWorkflowSerializedRunReportLines(input.reportLines);
+    const run = runAssistantWorkflowEvalCasesAgainstSerializedRunReports(caseParse.cases, reportParse.reports, options);
+    return {
+        caseParse,
+        reportParse,
+        parseErrorCount: caseParse.errorCount + reportParse.errorCount,
+        run,
+    };
+}
+export function buildAssistantWorkflowEvalReplayLinesTracePayload(result) {
+    return {
+        caseParse: {
+            lineCount: result.caseParse.lineCount,
+            caseCount: result.caseParse.caseCount,
+            errorCount: result.caseParse.errorCount,
+            errors: result.caseParse.errors.map((error) => ({
+                lineNumber: error.lineNumber,
+                reason: error.reason,
+                preview: error.preview,
+            })),
+        },
+        reportParse: {
+            lineCount: result.reportParse.lineCount,
+            reportCount: result.reportParse.reportCount,
+            errorCount: result.reportParse.errorCount,
+            errors: result.reportParse.errors.map((error) => ({
+                lineNumber: error.lineNumber,
+                reason: error.reason,
+                preview: error.preview,
+            })),
+        },
+        parseErrorCount: result.parseErrorCount,
+        run: buildAssistantWorkflowEvalRunTracePayload(result.run),
+    };
+}
+//# sourceMappingURL=eval-replay.js.map

package/dist/eval-replay.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-replay.js","sourceRoot":"","sources":["../src/eval-replay.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mCAAmC,GAEpC,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,yCAAyC,EACzC,wDAAwD,GAGzD,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,8CAA8C,GAE/C,MAAM,sBAAsB,CAAC;AAgB9B,MAAM,UAAU,mCAAmC,CACjD,KAA4C,EAC5C,UAAmD,EAAE;IAErD,MAAM,SAAS,GAAG,mCAAmC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3E,MAAM,WAAW,GAAG,8CAA8C,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACtF,MAAM,GAAG,GAAG,wDAAwD,CAClE,SAAS,CAAC,KAAK,EACf,WAAW,CAAC,OAAO,EACnB,OAAO,CACR,CAAC;IAEF,OAAO;QACL,SAAS;QACT,WAAW;QACX,eAAe,EAAE,SAAS,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU;QAC9D,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iDAAiD,CAC/D,MAA8C;IAE9C,OAAO;QACL,SAAS,EAAE;YACT,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS;YACrC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS;YACrC,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU;YACvC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC9C,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC,CAAC;SACJ;QACD,WAAW,EAAE;YACX,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,SAAS;YACvC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,WAAW;YAC3C,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU;YACzC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAChD,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC,CAAC;SACJ;QACD,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,GAAG,EAAE,yCAAyC,CAAC,MAAM,CAAC,GAAG,CAAC;KAC3D,CAAC;AACJ,CAAC","sourcesContent":["import {\n  parseAssistantWorkflowEvalCaseLines,\n  type AssistantWorkflowEvalCaseLinesParseResult,\n} from \"./eval-case.js\";\nimport {\n  buildAssistantWorkflowEvalRunTracePayload,\n  runAssistantWorkflowEvalCasesAgainstSerializedRunReports,\n  type AssistantWorkflowEvalRun,\n  type AssistantWorkflowEvalRunReportsOptions,\n} from \"./eval-runner.js\";\nimport type { AssistantWorkflowJsonValue } from \"./run-report-json.js\";\nimport {\n  parseAssistantWorkflowSerializedRunReportLines,\n  type AssistantWorkflowSerializedRunReportLinesParseResult,\n} from \"./run-report-json.js\";\n\nexport type AssistantWorkflowEvalReplayLinesOptions = AssistantWorkflowEvalRunReportsOptions;\n\nexport type AssistantWorkflowEvalReplayLinesInput = {\n  evalCaseLines: string;\n  reportLines: string;\n};\n\nexport type AssistantWorkflowEvalReplayLinesResult = {\n  caseParse: AssistantWorkflowEvalCaseLinesParseResult;\n  reportParse: AssistantWorkflowSerializedRunReportLinesParseResult;\n  parseErrorCount: number;\n  run: AssistantWorkflowEvalRun;\n};\n\nexport function runAssistantWorkflowEvalReplayLines(\n  input: AssistantWorkflowEvalReplayLinesInput,\n  options: AssistantWorkflowEvalReplayLinesOptions = {},\n): AssistantWorkflowEvalReplayLinesResult {\n  const caseParse = parseAssistantWorkflowEvalCaseLines(input.evalCaseLines);\n  const reportParse = parseAssistantWorkflowSerializedRunReportLines(input.reportLines);\n  const run = runAssistantWorkflowEvalCasesAgainstSerializedRunReports(\n    caseParse.cases,\n    reportParse.reports,\n    options,\n  );\n\n  return {\n    caseParse,\n    reportParse,\n    parseErrorCount: caseParse.errorCount + reportParse.errorCount,\n    run,\n  };\n}\n\nexport function buildAssistantWorkflowEvalReplayLinesTracePayload(\n  result: AssistantWorkflowEvalReplayLinesResult,\n): Record<string, AssistantWorkflowJsonValue> {\n  return {\n    caseParse: {\n      lineCount: result.caseParse.lineCount,\n      caseCount: result.caseParse.caseCount,\n      errorCount: result.caseParse.errorCount,\n      errors: result.caseParse.errors.map((error) => ({\n        lineNumber: error.lineNumber,\n        reason: error.reason,\n        preview: error.preview,\n      })),\n    },\n    reportParse: {\n      lineCount: result.reportParse.lineCount,\n      reportCount: result.reportParse.reportCount,\n      errorCount: result.reportParse.errorCount,\n      errors: result.reportParse.errors.map((error) => ({\n        lineNumber: error.lineNumber,\n        reason: error.reason,\n        preview: error.preview,\n      })),\n    },\n    parseErrorCount: result.parseErrorCount,\n    run: buildAssistantWorkflowEvalRunTracePayload(result.run),\n  };\n}\n"]}

package/dist/eval-result.d.ts

@@ -0,0 +1,46 @@
+import type { AssistantWorkflowEvalCase } from "./eval-case.js";
+import type { AssistantWorkflowRunReportStatus } from "./run-report.js";
+import type { AssistantWorkflowJsonValue, AssistantWorkflowSerializedRunReport } from "./run-report-json.js";
+export type AssistantWorkflowEvalActual = {
+    workflowId: string | null;
+    workflowVersion: number | null;
+    intent: string | null;
+    presentation: string | null;
+    status: AssistantWorkflowRunReportStatus | null;
+    outcomeKind: string | null;
+    planStepIds: string[];
+    activePlanningStepProfileIds: string[];
+    profileStepIds: string[];
+    tools: string[];
+    signalNames: string[];
+    qualitySignalNames: string[];
+};
+export type AssistantWorkflowEvalCheckName = "intent" | "presentation" | "status" | "outcome_kind" | "plan_steps" | "planning_profiles" | "profile_steps" | "tools" | "signals" | "quality_signals";
+export type AssistantWorkflowEvalCheckStatus = "pass" | "fail" | "skipped";
+export type AssistantWorkflowEvalCheck = {
+    name: AssistantWorkflowEvalCheckName;
+    status: AssistantWorkflowEvalCheckStatus;
+    expected: string | string[] | null;
+    actual: string | string[] | null;
+    detail: string | null;
+};
+export type AssistantWorkflowEvalResultStatus = "pass" | "fail" | "skipped";
+export type AssistantWorkflowEvalResult = {
+    caseId: string;
+    required: boolean;
+    status: AssistantWorkflowEvalResultStatus;
+    passedCount: number;
+    failedCount: number;
+    skippedCount: number;
+    checks: AssistantWorkflowEvalCheck[];
+    actual: AssistantWorkflowEvalActual;
+};
+export type AssistantWorkflowEvalActualOptions = {
+    qualitySignalNames?: readonly string[] | null;
+};
+export declare function buildMissingAssistantWorkflowEvalActual(): AssistantWorkflowEvalActual;
+export declare function buildAssistantWorkflowEvalActualFromSerializedRunReport(report: AssistantWorkflowSerializedRunReport, options?: AssistantWorkflowEvalActualOptions): AssistantWorkflowEvalActual;
+export declare function evaluateAssistantWorkflowEvalCase(evalCase: AssistantWorkflowEvalCase, actual: AssistantWorkflowEvalActual): AssistantWorkflowEvalResult;
+export declare function evaluateAssistantWorkflowEvalCaseAgainstSerializedRunReport(evalCase: AssistantWorkflowEvalCase, report: AssistantWorkflowSerializedRunReport, options?: AssistantWorkflowEvalActualOptions): AssistantWorkflowEvalResult;
+export declare function buildAssistantWorkflowEvalResultTracePayload(result: AssistantWorkflowEvalResult): Record<string, AssistantWorkflowJsonValue>;
+//# sourceMappingURL=eval-result.d.ts.map

package/dist/eval-result.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-result.d.ts","sourceRoot":"","sources":["../src/eval-result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,yBAAyB,EAE1B,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,KAAK,EACV,0BAA0B,EAC1B,oCAAoC,EACrC,MAAM,sBAAsB,CAAC;AAQ9B,MAAM,MAAM,2BAA2B,GAAG;IACxC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,gCAAgC,GAAG,IAAI,CAAC;IAChD,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4BAA4B,EAAE,MAAM,EAAE,CAAC;IACvC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,8BAA8B,GACtC,QAAQ,GACR,cAAc,GACd,QAAQ,GACR,cAAc,GACd,YAAY,GACZ,mBAAmB,GACnB,eAAe,GACf,OAAO,GACP,SAAS,GACT,iBAAiB,CAAC;AAEtB,MAAM,MAAM,gCAAgC,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAE3E,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,8BAA8B,CAAC;IACrC,MAAM,EAAE,gCAAgC,CAAC;IACzC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,CAAC;IACnC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,iCAAiC,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAE5E,MAAM,MAAM,2BAA2B,GAAG;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,iCAAiC,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACrC,MAAM,EAAE,2BAA2B,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG;IAC/C,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;CAC/C,CAAC;AAEF,wBAAgB,uCAAuC,IAAI,2BAA2B,CAerF;AAED,wBAAgB,uDAAuD,CACrE,MAAM,EAAE,oCAAoC,EAC5C,OAAO,GAAE,kCAAuC,GAC/C,2BAA2B,CAe7B;AAED,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,yBAAyB,EACnC,MAAM,EAAE,2BAA2B,GAClC,2BAA2B,CA4B7B;AAED,wBAAgB,2DAA2D,CACzE,QAAQ,EAAE,yBAAyB,EACnC,MAAM,EAAE,oCAAoC,EAC5C,OAAO,GAAE,kCAAuC,GAC/C,2BAA2B,CAK7B;AAED,wBAAgB,4CAA4C,CAC1D,MAAM,EAAE,2BAA2B,GAClC,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CA8B5C"}

package/dist/eval-result.js

@@ -0,0 +1,183 @@
+import { readAssistantWorkflowNestedArray as readNestedArray, readAssistantWorkflowNestedString as readNestedStringOrNull, readAssistantWorkflowString as readStringOrNull, readAssistantWorkflowStringArray as readStringArray, } from "./value-readers.js";
+export function buildMissingAssistantWorkflowEvalActual() {
+    return {
+        workflowId: null,
+        workflowVersion: null,
+        intent: null,
+        presentation: null,
+        status: null,
+        outcomeKind: null,
+        planStepIds: [],
+        activePlanningStepProfileIds: [],
+        profileStepIds: [],
+        tools: [],
+        signalNames: [],
+        qualitySignalNames: [],
+    };
+}
+export function buildAssistantWorkflowEvalActualFromSerializedRunReport(report, options = {}) {
+    return {
+        workflowId: report.workflowId,
+        workflowVersion: report.workflowVersion,
+        intent: readStringOrNull(report.evaluation["intent"]) ?? readNestedStringOrNull(report.run, ["decision", "intent"]),
+        presentation: readNestedStringOrNull(report.run, ["decision", "effectivePresentation"]),
+        status: report.status,
+        outcomeKind: readStringOrNull(report.evaluation["outcomeKind"]) ?? readStringOrNull(report.outcome["kind"]),
+        planStepIds: readPlanStepIds(report),
+        activePlanningStepProfileIds: readStringArray(report.evaluation["activePlanningStepProfileIds"]),
+        profileStepIds: readStringArray(report.evaluation["profileStepIds"]),
+        tools: readInvocationToolNames(report),
+        signalNames: readSignalNames(report),
+        qualitySignalNames: uniqueNonEmptyStrings(options.qualitySignalNames ?? []),
+    };
+}
+export function evaluateAssistantWorkflowEvalCase(evalCase, actual) {
+    const expectations = evalCase.expectations;
+    const checks = [
+        compareScalar("intent", expectations.intent, actual.intent),
+        compareScalar("presentation", expectations.presentation, actual.presentation),
+        compareScalar("status", expectations.status, actual.status),
+        compareScalar("outcome_kind", expectations.outcomeKind, actual.outcomeKind),
+        compareStringSet("plan_steps", expectations.planStepIds, actual.planStepIds),
+        compareStringSet("planning_profiles", expectations.activePlanningStepProfileIds, actual.activePlanningStepProfileIds),
+        compareStringSet("profile_steps", expectations.profileStepIds, actual.profileStepIds),
+        compareStringSet("tools", expectations.tools, actual.tools),
+        compareStringSet("signals", expectations.signalNames, actual.signalNames),
+        compareStringSet("quality_signals", expectations.qualitySignalNames, actual.qualitySignalNames),
+    ];
+    const passedCount = checks.filter((check) => check.status === "pass").length;
+    const failedCount = checks.filter((check) => check.status === "fail").length;
+    const skippedCount = checks.filter((check) => check.status === "skipped").length;
+    return {
+        caseId: evalCase.id,
+        required: evalCase.required,
+        status: failedCount > 0 ? "fail" : passedCount > 0 ? "pass" : "skipped",
+        passedCount,
+        failedCount,
+        skippedCount,
+        checks,
+        actual,
+    };
+}
+export function evaluateAssistantWorkflowEvalCaseAgainstSerializedRunReport(evalCase, report, options = {}) {
+    return evaluateAssistantWorkflowEvalCase(evalCase, buildAssistantWorkflowEvalActualFromSerializedRunReport(report, options));
+}
+export function buildAssistantWorkflowEvalResultTracePayload(result) {
+    return {
+        caseId: result.caseId,
+        required: result.required,
+        status: result.status,
+        passedCount: result.passedCount,
+        failedCount: result.failedCount,
+        skippedCount: result.skippedCount,
+        checks: result.checks.map((check) => ({
+            name: check.name,
+            status: check.status,
+            expected: check.expected,
+            actual: check.actual,
+            detail: check.detail,
+        })),
+        actual: {
+            workflowId: result.actual.workflowId,
+            workflowVersion: result.actual.workflowVersion,
+            intent: result.actual.intent,
+            presentation: result.actual.presentation,
+            status: result.actual.status,
+            outcomeKind: result.actual.outcomeKind,
+            planStepIds: result.actual.planStepIds,
+            activePlanningStepProfileIds: result.actual.activePlanningStepProfileIds,
+            profileStepIds: result.actual.profileStepIds,
+            tools: result.actual.tools,
+            signalNames: result.actual.signalNames,
+            qualitySignalNames: result.actual.qualitySignalNames,
+        },
+    };
+}
+function compareScalar(name, expected, actual) {
+    if (expected === null) {
+        return {
+            name,
+            status: "skipped",
+            expected: null,
+            actual,
+            detail: "no expectation",
+        };
+    }
+    if (actual === expected) {
+        return {
+            name,
+            status: "pass",
+            expected,
+            actual,
+            detail: null,
+        };
+    }
+    return {
+        name,
+        status: "fail",
+        expected,
+        actual,
+        detail: actual === null ? "actual value is missing" : `expected ${expected}, got ${actual}`,
+    };
+}
+function compareStringSet(name, expected, actual) {
+    const normalizedExpected = uniqueNonEmptyStrings(expected);
+    const normalizedActual = uniqueNonEmptyStrings(actual);
+    if (!normalizedExpected.length) {
+        return {
+            name,
+            status: "skipped",
+            expected: [],
+            actual: normalizedActual,
+            detail: "no expectation",
+        };
+    }
+    const actualSet = new Set(normalizedActual);
+    const missing = normalizedExpected.filter((value) => !actualSet.has(value));
+    if (!missing.length) {
+        return {
+            name,
+            status: "pass",
+            expected: normalizedExpected,
+            actual: normalizedActual,
+            detail: null,
+        };
+    }
+    return {
+        name,
+        status: "fail",
+        expected: normalizedExpected,
+        actual: normalizedActual,
+        detail: `missing: ${missing.join(", ")}`,
+    };
+}
+function readPlanStepIds(report) {
+    const steps = readNestedArray(report.run, ["executionPlan", "steps"]);
+    return uniqueNonEmptyStrings(steps.flatMap((step) => isRecord(step) ? [step["id"]] : []));
+}
+function readInvocationToolNames(report) {
+    const invocations = readNestedArray(report.run, ["toolInvocationQueue", "invocations"]);
+    return uniqueNonEmptyStrings(invocations.flatMap((invocation) => isRecord(invocation) ? [invocation["toolName"]] : []));
+}
+function readSignalNames(report) {
+    const signals = readNestedArray(report.evaluation, ["signals"]);
+    return uniqueNonEmptyStrings(signals.flatMap((signal) => isRecord(signal) ? [signal["name"]] : []));
+}
+function uniqueNonEmptyStrings(values) {
+    const out = [];
+    const seen = new Set();
+    for (const value of values) {
+        if (typeof value !== "string")
+            continue;
+        const trimmed = value.trim();
+        if (!trimmed.length || seen.has(trimmed))
+            continue;
+        seen.add(trimmed);
+        out.push(trimmed);
+    }
+    return out;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//# sourceMappingURL=eval-result.js.map

package/dist/eval-result.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-result.js","sourceRoot":"","sources":["../src/eval-result.ts"],"names":[],"mappings":"AASA,OAAO,EACL,gCAAgC,IAAI,eAAe,EACnD,iCAAiC,IAAI,sBAAsB,EAC3D,2BAA2B,IAAI,gBAAgB,EAC/C,gCAAgC,IAAI,eAAe,GACpD,MAAM,oBAAoB,CAAC;AAwD5B,MAAM,UAAU,uCAAuC;IACrD,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,eAAe,EAAE,IAAI;QACrB,MAAM,EAAE,IAAI;QACZ,YAAY,EAAE,IAAI;QAClB,MAAM,EAAE,IAAI;QACZ,WAAW,EAAE,IAAI;QACjB,WAAW,EAAE,EAAE;QACf,4BAA4B,EAAE,EAAE;QAChC,cAAc,EAAE,EAAE;QAClB,KAAK,EAAE,EAAE;QACT,WAAW,EAAE,EAAE;QACf,kBAAkB,EAAE,EAAE;KACvB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uDAAuD,CACrE,MAA4C,EAC5C,UAA8C,EAAE;IAEhD,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACnH,YAAY,EAAE,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;QACvF,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,IAAI,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3G,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC;QACpC,4BAA4B,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;QAChG,cAAc,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACpE,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;QACtC,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC;QACpC,kBAAkB,EAAE,qBAAqB,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC;KAC5E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,QAAmC,EACnC,MAAmC;IAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;IAC3C,MAAM,MAAM,GAAiC;QAC3C,aAAa,CAAC,QAAQ,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;QAC3D,aAAa,CAAC,cAAc,EAAE,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAC7E,aAAa,CAAC,QAAQ,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;QAC3D,aAAa,CAAC,cAAc,EAAE,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAC3E,gBAAgB,CAAC,YAAY,EAAE,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAC5E,gBAAgB,CAAC,mBAAmB,EAAE,YAAY,CAAC,4BAA4B,EAAE,MAAM,CAAC,4BAA4B,CAAC;QACrH,gBAAgB,CAAC,eAAe,EAAE,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,cAAc,CAAC;QACrF,gBAAgB,CAAC,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;QAC3D,gBAAgB,CAAC,SAAS,EAAE,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QACzE,gBAAgB,CAAC,iBAAiB,EAAE,YAAY,CAAC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,CAAC;KAChG,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC7E,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAC7E,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAEjF,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,EAAE;QACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvE,WAAW;QACX,WAAW;QACX,YAAY;QACZ,MAAM;QACN,MAAM;KACP,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,2DAA2D,CACzE,QAAmC,EACnC,MAA4C,EAC5C,UAA8C,EAAE;IAEhD,OAAO,iCAAiC,CACtC,QAAQ,EACR,uDAAuD,CAAC,MAAM,EAAE,OAAO,CAAC,CACzE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4CAA4C,CAC1D,MAAmC;IAEnC,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;QACH,MAAM,EAAE;YACN,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU;YACpC,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,eAAe;YAC9C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY;YACxC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAC5B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;YACtC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;YACtC,4BAA4B,EAAE,MAAM,CAAC,MAAM,CAAC,4BAA4B;YACxE,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc;YAC5C,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;YAC1B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;YACtC,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,kBAAkB;SACrD;KACF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,IAAoC,EACpC,QAAuB,EACvB,MAAqB;IAErB,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,IAAI;YACd,MAAM;YACN,MAAM,EAAE,gBAAgB;SACzB,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,MAAM;YACd,QAAQ;YACR,MAAM;YACN,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,MAAM;QACd,QAAQ;QACR,MAAM;QACN,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,YAAY,QAAQ,SAAS,MAAM,EAAE;KAC5F,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,IAAoC,EACpC,QAA2B,EAC3B,MAAyB;IAEzB,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACvD,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;QAC/B,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,gBAAgB;YACxB,MAAM,EAAE,gBAAgB;SACzB,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE,kBAAkB;YAC5B,MAAM,EAAE,gBAAgB;YACxB,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,MAAM;QACd,QAAQ,EAAE,kBAAkB;QAC5B,MAAM,EAAE,gBAAgB;QACxB,MAAM,EAAE,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,MAA4C;IACnE,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC;IACtE,OAAO,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,uBAAuB,CAAC,MAA4C;IAC3E,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC,CAAC;IACxF,OAAO,qBAAqB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAC9D,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CACrD,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,MAA4C;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IAChE,OAAO,qBAAqB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtG,CAAC;AAED,SAAS,qBAAqB,CAAC,MAA0B;IACvD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,SAAS;QACxC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,SAAS;QACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC","sourcesContent":["import type {\n  AssistantWorkflowEvalCase,\n  AssistantWorkflowEvalCaseExpectations,\n} from \"./eval-case.js\";\nimport type { AssistantWorkflowRunReportStatus } from \"./run-report.js\";\nimport type {\n  AssistantWorkflowJsonValue,\n  AssistantWorkflowSerializedRunReport,\n} from \"./run-report-json.js\";\nimport {\n  readAssistantWorkflowNestedArray as readNestedArray,\n  readAssistantWorkflowNestedString as readNestedStringOrNull,\n  readAssistantWorkflowString as readStringOrNull,\n  readAssistantWorkflowStringArray as readStringArray,\n} from \"./value-readers.js\";\n\nexport type AssistantWorkflowEvalActual = {\n  workflowId: string | null;\n  workflowVersion: number | null;\n  intent: string | null;\n  presentation: string | null;\n  status: AssistantWorkflowRunReportStatus | null;\n  outcomeKind: string | null;\n  planStepIds: string[];\n  activePlanningStepProfileIds: string[];\n  profileStepIds: string[];\n  tools: string[];\n  signalNames: string[];\n  qualitySignalNames: string[];\n};\n\nexport type AssistantWorkflowEvalCheckName =\n  | \"intent\"\n  | \"presentation\"\n  | \"status\"\n  | \"outcome_kind\"\n  | \"plan_steps\"\n  | \"planning_profiles\"\n  | \"profile_steps\"\n  | \"tools\"\n  | \"signals\"\n  | \"quality_signals\";\n\nexport type AssistantWorkflowEvalCheckStatus = \"pass\" | \"fail\" | \"skipped\";\n\nexport type AssistantWorkflowEvalCheck = {\n  name: AssistantWorkflowEvalCheckName;\n  status: AssistantWorkflowEvalCheckStatus;\n  expected: string | string[] | null;\n  actual: string | string[] | null;\n  detail: string | null;\n};\n\nexport type AssistantWorkflowEvalResultStatus = \"pass\" | \"fail\" | \"skipped\";\n\nexport type AssistantWorkflowEvalResult = {\n  caseId: string;\n  required: boolean;\n  status: AssistantWorkflowEvalResultStatus;\n  passedCount: number;\n  failedCount: number;\n  skippedCount: number;\n  checks: AssistantWorkflowEvalCheck[];\n  actual: AssistantWorkflowEvalActual;\n};\n\nexport type AssistantWorkflowEvalActualOptions = {\n  qualitySignalNames?: readonly string[] | null;\n};\n\nexport function buildMissingAssistantWorkflowEvalActual(): AssistantWorkflowEvalActual {\n  return {\n    workflowId: null,\n    workflowVersion: null,\n    intent: null,\n    presentation: null,\n    status: null,\n    outcomeKind: null,\n    planStepIds: [],\n    activePlanningStepProfileIds: [],\n    profileStepIds: [],\n    tools: [],\n    signalNames: [],\n    qualitySignalNames: [],\n  };\n}\n\nexport function buildAssistantWorkflowEvalActualFromSerializedRunReport(\n  report: AssistantWorkflowSerializedRunReport,\n  options: AssistantWorkflowEvalActualOptions = {},\n): AssistantWorkflowEvalActual {\n  return {\n    workflowId: report.workflowId,\n    workflowVersion: report.workflowVersion,\n    intent: readStringOrNull(report.evaluation[\"intent\"]) ?? readNestedStringOrNull(report.run, [\"decision\", \"intent\"]),\n    presentation: readNestedStringOrNull(report.run, [\"decision\", \"effectivePresentation\"]),\n    status: report.status,\n    outcomeKind: readStringOrNull(report.evaluation[\"outcomeKind\"]) ?? readStringOrNull(report.outcome[\"kind\"]),\n    planStepIds: readPlanStepIds(report),\n    activePlanningStepProfileIds: readStringArray(report.evaluation[\"activePlanningStepProfileIds\"]),\n    profileStepIds: readStringArray(report.evaluation[\"profileStepIds\"]),\n    tools: readInvocationToolNames(report),\n    signalNames: readSignalNames(report),\n    qualitySignalNames: uniqueNonEmptyStrings(options.qualitySignalNames ?? []),\n  };\n}\n\nexport function evaluateAssistantWorkflowEvalCase(\n  evalCase: AssistantWorkflowEvalCase,\n  actual: AssistantWorkflowEvalActual,\n): AssistantWorkflowEvalResult {\n  const expectations = evalCase.expectations;\n  const checks: AssistantWorkflowEvalCheck[] = [\n    compareScalar(\"intent\", expectations.intent, actual.intent),\n    compareScalar(\"presentation\", expectations.presentation, actual.presentation),\n    compareScalar(\"status\", expectations.status, actual.status),\n    compareScalar(\"outcome_kind\", expectations.outcomeKind, actual.outcomeKind),\n    compareStringSet(\"plan_steps\", expectations.planStepIds, actual.planStepIds),\n    compareStringSet(\"planning_profiles\", expectations.activePlanningStepProfileIds, actual.activePlanningStepProfileIds),\n    compareStringSet(\"profile_steps\", expectations.profileStepIds, actual.profileStepIds),\n    compareStringSet(\"tools\", expectations.tools, actual.tools),\n    compareStringSet(\"signals\", expectations.signalNames, actual.signalNames),\n    compareStringSet(\"quality_signals\", expectations.qualitySignalNames, actual.qualitySignalNames),\n  ];\n  const passedCount = checks.filter((check) => check.status === \"pass\").length;\n  const failedCount = checks.filter((check) => check.status === \"fail\").length;\n  const skippedCount = checks.filter((check) => check.status === \"skipped\").length;\n\n  return {\n    caseId: evalCase.id,\n    required: evalCase.required,\n    status: failedCount > 0 ? \"fail\" : passedCount > 0 ? \"pass\" : \"skipped\",\n    passedCount,\n    failedCount,\n    skippedCount,\n    checks,\n    actual,\n  };\n}\n\nexport function evaluateAssistantWorkflowEvalCaseAgainstSerializedRunReport(\n  evalCase: AssistantWorkflowEvalCase,\n  report: AssistantWorkflowSerializedRunReport,\n  options: AssistantWorkflowEvalActualOptions = {},\n): AssistantWorkflowEvalResult {\n  return evaluateAssistantWorkflowEvalCase(\n    evalCase,\n    buildAssistantWorkflowEvalActualFromSerializedRunReport(report, options),\n  );\n}\n\nexport function buildAssistantWorkflowEvalResultTracePayload(\n  result: AssistantWorkflowEvalResult,\n): Record<string, AssistantWorkflowJsonValue> {\n  return {\n    caseId: result.caseId,\n    required: result.required,\n    status: result.status,\n    passedCount: result.passedCount,\n    failedCount: result.failedCount,\n    skippedCount: result.skippedCount,\n    checks: result.checks.map((check) => ({\n      name: check.name,\n      status: check.status,\n      expected: check.expected,\n      actual: check.actual,\n      detail: check.detail,\n    })),\n    actual: {\n      workflowId: result.actual.workflowId,\n      workflowVersion: result.actual.workflowVersion,\n      intent: result.actual.intent,\n      presentation: result.actual.presentation,\n      status: result.actual.status,\n      outcomeKind: result.actual.outcomeKind,\n      planStepIds: result.actual.planStepIds,\n      activePlanningStepProfileIds: result.actual.activePlanningStepProfileIds,\n      profileStepIds: result.actual.profileStepIds,\n      tools: result.actual.tools,\n      signalNames: result.actual.signalNames,\n      qualitySignalNames: result.actual.qualitySignalNames,\n    },\n  };\n}\n\nfunction compareScalar(\n  name: AssistantWorkflowEvalCheckName,\n  expected: string | null,\n  actual: string | null,\n): AssistantWorkflowEvalCheck {\n  if (expected === null) {\n    return {\n      name,\n      status: \"skipped\",\n      expected: null,\n      actual,\n      detail: \"no expectation\",\n    };\n  }\n  if (actual === expected) {\n    return {\n      name,\n      status: \"pass\",\n      expected,\n      actual,\n      detail: null,\n    };\n  }\n  return {\n    name,\n    status: \"fail\",\n    expected,\n    actual,\n    detail: actual === null ? \"actual value is missing\" : `expected ${expected}, got ${actual}`,\n  };\n}\n\nfunction compareStringSet(\n  name: AssistantWorkflowEvalCheckName,\n  expected: readonly string[],\n  actual: readonly string[],\n): AssistantWorkflowEvalCheck {\n  const normalizedExpected = uniqueNonEmptyStrings(expected);\n  const normalizedActual = uniqueNonEmptyStrings(actual);\n  if (!normalizedExpected.length) {\n    return {\n      name,\n      status: \"skipped\",\n      expected: [],\n      actual: normalizedActual,\n      detail: \"no expectation\",\n    };\n  }\n\n  const actualSet = new Set(normalizedActual);\n  const missing = normalizedExpected.filter((value) => !actualSet.has(value));\n  if (!missing.length) {\n    return {\n      name,\n      status: \"pass\",\n      expected: normalizedExpected,\n      actual: normalizedActual,\n      detail: null,\n    };\n  }\n  return {\n    name,\n    status: \"fail\",\n    expected: normalizedExpected,\n    actual: normalizedActual,\n    detail: `missing: ${missing.join(\", \")}`,\n  };\n}\n\nfunction readPlanStepIds(report: AssistantWorkflowSerializedRunReport): string[] {\n  const steps = readNestedArray(report.run, [\"executionPlan\", \"steps\"]);\n  return uniqueNonEmptyStrings(steps.flatMap((step) => isRecord(step) ? [step[\"id\"]] : []));\n}\n\nfunction readInvocationToolNames(report: AssistantWorkflowSerializedRunReport): string[] {\n  const invocations = readNestedArray(report.run, [\"toolInvocationQueue\", \"invocations\"]);\n  return uniqueNonEmptyStrings(invocations.flatMap((invocation) =>\n    isRecord(invocation) ? [invocation[\"toolName\"]] : []\n  ));\n}\n\nfunction readSignalNames(report: AssistantWorkflowSerializedRunReport): string[] {\n  const signals = readNestedArray(report.evaluation, [\"signals\"]);\n  return uniqueNonEmptyStrings(signals.flatMap((signal) => isRecord(signal) ? [signal[\"name\"]] : []));\n}\n\nfunction uniqueNonEmptyStrings(values: readonly unknown[]): string[] {\n  const out: string[] = [];\n  const seen = new Set<string>();\n  for (const value of values) {\n    if (typeof value !== \"string\") continue;\n    const trimmed = value.trim();\n    if (!trimmed.length || seen.has(trimmed)) continue;\n    seen.add(trimmed);\n    out.push(trimmed);\n  }\n  return out;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n  return typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n"]}

package/dist/eval-runner.d.ts

@@ -0,0 +1,30 @@
+import type { AssistantWorkflowEvalCase } from "./eval-case.js";
+import { type AssistantWorkflowEvalResult } from "./eval-result.js";
+import { type AssistantWorkflowEvalSuite } from "./eval-suite.js";
+import type { AssistantWorkflowJsonValue, AssistantWorkflowSerializedRunReport } from "./run-report-json.js";
+export type AssistantWorkflowEvalRunCaseInput = {
+    evalCase: AssistantWorkflowEvalCase;
+    report?: AssistantWorkflowSerializedRunReport | null;
+    qualitySignalNames?: readonly string[] | null;
+};
+export type AssistantWorkflowEvalRunOptions = {
+    onlyFailures?: boolean;
+};
+export type AssistantWorkflowEvalRun = {
+    generatedAt: string;
+    caseCount: number;
+    reportCount: number;
+    resultCount: number;
+    missingReportCount: number;
+    unmatchedReportCount: number;
+    missingReportCaseIds: string[];
+    suite: AssistantWorkflowEvalSuite;
+    results: AssistantWorkflowEvalResult[];
+};
+export type AssistantWorkflowEvalRunReportsOptions = AssistantWorkflowEvalRunOptions & {
+    qualitySignalNamesByCaseId?: Record<string, readonly string[]> | null;
+};
+export declare function runAssistantWorkflowEvalCases(inputs: readonly AssistantWorkflowEvalRunCaseInput[], options?: AssistantWorkflowEvalRunOptions): AssistantWorkflowEvalRun;
+export declare function runAssistantWorkflowEvalCasesAgainstSerializedRunReports(evalCases: readonly AssistantWorkflowEvalCase[], reports: readonly AssistantWorkflowSerializedRunReport[], options?: AssistantWorkflowEvalRunReportsOptions): AssistantWorkflowEvalRun;
+export declare function buildAssistantWorkflowEvalRunTracePayload(run: AssistantWorkflowEvalRun): Record<string, AssistantWorkflowJsonValue>;
+//# sourceMappingURL=eval-runner.d.ts.map

package/dist/eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../src/eval-runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAOL,KAAK,2BAA2B,EACjC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EACV,0BAA0B,EAC1B,oCAAoC,EACrC,MAAM,sBAAsB,CAAC;AAE9B,MAAM,MAAM,iCAAiC,GAAG;IAC9C,QAAQ,EAAE,yBAAyB,CAAC;IACpC,MAAM,CAAC,EAAE,oCAAoC,GAAG,IAAI,CAAC;IACrD,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;CAC/C,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,KAAK,EAAE,0BAA0B,CAAC;IAClC,OAAO,EAAE,2BAA2B,EAAE,CAAC;CACxC,CAAC;AAEF,MAAM,MAAM,sCAAsC,GAChD,+BAA+B,GAAG;IAChC,0BAA0B,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC;CACvE,CAAC;AAEJ,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,SAAS,iCAAiC,EAAE,EACpD,OAAO,GAAE,+BAAoC,GAC5C,wBAAwB,CAgC1B;AAED,wBAAgB,wDAAwD,CACtE,SAAS,EAAE,SAAS,yBAAyB,EAAE,EAC/C,OAAO,EAAE,SAAS,oCAAoC,EAAE,EACxD,OAAO,GAAE,sCAA2C,GACnD,wBAAwB,CAY1B;AAED,wBAAgB,yCAAyC,CACvD,GAAG,EAAE,wBAAwB,GAC5B,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAY5C"}

package/dist/eval-runner.js

@@ -0,0 +1,57 @@
+import { buildAssistantWorkflowEvalResultTracePayload, buildMissingAssistantWorkflowEvalActual, evaluateAssistantWorkflowEvalCase, evaluateAssistantWorkflowEvalCaseAgainstSerializedRunReport, } from "./eval-result.js";
+import { buildAssistantWorkflowEvalSuite, buildAssistantWorkflowEvalSuiteTracePayload, } from "./eval-suite.js";
+export function runAssistantWorkflowEvalCases(inputs, options = {}) {
+    const results = inputs.map((input) => {
+        if (!input.report) {
+            return evaluateAssistantWorkflowEvalCase(input.evalCase, buildMissingAssistantWorkflowEvalActual());
+        }
+        return evaluateAssistantWorkflowEvalCaseAgainstSerializedRunReport(input.evalCase, input.report, buildActualOptions(input.qualitySignalNames));
+    });
+    const suite = buildAssistantWorkflowEvalSuite(results, {
+        ...(options.onlyFailures !== undefined ? { onlyFailures: options.onlyFailures } : {}),
+    });
+    const missingReportCaseIds = inputs
+        .filter((input) => !input.report)
+        .map((input) => input.evalCase.id);
+    return {
+        generatedAt: suite.generatedAt,
+        caseCount: inputs.length,
+        reportCount: inputs.filter((input) => input.report).length,
+        resultCount: results.length,
+        missingReportCount: missingReportCaseIds.length,
+        unmatchedReportCount: 0,
+        missingReportCaseIds,
+        suite,
+        results,
+    };
+}
+export function runAssistantWorkflowEvalCasesAgainstSerializedRunReports(evalCases, reports, options = {}) {
+    const inputs = evalCases.map((evalCase, index) => ({
+        evalCase,
+        report: reports[index] ?? null,
+        qualitySignalNames: options.qualitySignalNamesByCaseId?.[evalCase.id] ?? null,
+    }));
+    const run = runAssistantWorkflowEvalCases(inputs, options);
+    return {
+        ...run,
+        reportCount: reports.length,
+        unmatchedReportCount: Math.max(0, reports.length - evalCases.length),
+    };
+}
+export function buildAssistantWorkflowEvalRunTracePayload(run) {
+    return {
+        generatedAt: run.generatedAt,
+        caseCount: run.caseCount,
+        reportCount: run.reportCount,
+        resultCount: run.resultCount,
+        missingReportCount: run.missingReportCount,
+        unmatchedReportCount: run.unmatchedReportCount,
+        missingReportCaseIds: run.missingReportCaseIds,
+        suite: buildAssistantWorkflowEvalSuiteTracePayload(run.suite),
+        results: run.results.map(buildAssistantWorkflowEvalResultTracePayload),
+    };
+}
+function buildActualOptions(qualitySignalNames) {
+    return qualitySignalNames ? { qualitySignalNames } : {};
+}
+//# sourceMappingURL=eval-runner.js.map

package/dist/eval-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.js","sourceRoot":"","sources":["../src/eval-runner.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,4CAA4C,EAC5C,uCAAuC,EACvC,iCAAiC,EACjC,2DAA2D,GAG5D,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,+BAA+B,EAC/B,2CAA2C,GAE5C,MAAM,iBAAiB,CAAC;AAiCzB,MAAM,UAAU,6BAA6B,CAC3C,MAAoD,EACpD,UAA2C,EAAE;IAE7C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACnC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO,iCAAiC,CACtC,KAAK,CAAC,QAAQ,EACd,uCAAuC,EAAE,CAC1C,CAAC;QACJ,CAAC;QACD,OAAO,2DAA2D,CAChE,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,MAAM,EACZ,kBAAkB,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,+BAA+B,CAAC,OAAO,EAAE;QACrD,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtF,CAAC,CAAC;IACH,MAAM,oBAAoB,GAAG,MAAM;SAChC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;SAChC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAErC,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS,EAAE,MAAM,CAAC,MAAM;QACxB,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM;QAC1D,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,kBAAkB,EAAE,oBAAoB,CAAC,MAAM;QAC/C,oBAAoB,EAAE,CAAC;QACvB,oBAAoB;QACpB,KAAK;QACL,OAAO;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wDAAwD,CACtE,SAA+C,EAC/C,OAAwD,EACxD,UAAkD,EAAE;IAEpD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACjD,QAAQ;QACR,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI;QAC9B,kBAAkB,EAAE,OAAO,CAAC,0BAA0B,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI;KAC9E,CAAC,CAAC,CAAC;IACJ,MAAM,GAAG,GAAG,6BAA6B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3D,OAAO;QACL,GAAG,GAAG;QACN,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,oBAAoB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yCAAyC,CACvD,GAA6B;IAE7B,OAAO;QACL,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;QAC1C,oBAAoB,EAAE,GAAG,CAAC,oBAAoB;QAC9C,oBAAoB,EAAE,GAAG,CAAC,oBAAoB;QAC9C,KAAK,EAAE,2CAA2C,CAAC,GAAG,CAAC,KAAK,CAAC;QAC7D,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC;KACvE,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,kBAAwD;IAExD,OAAO,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC1D,CAAC","sourcesContent":["import type { AssistantWorkflowEvalCase } from \"./eval-case.js\";\nimport {\n  buildAssistantWorkflowEvalActualFromSerializedRunReport,\n  buildAssistantWorkflowEvalResultTracePayload,\n  buildMissingAssistantWorkflowEvalActual,\n  evaluateAssistantWorkflowEvalCase,\n  evaluateAssistantWorkflowEvalCaseAgainstSerializedRunReport,\n  type AssistantWorkflowEvalActualOptions,\n  type AssistantWorkflowEvalResult,\n} from \"./eval-result.js\";\nimport {\n  buildAssistantWorkflowEvalSuite,\n  buildAssistantWorkflowEvalSuiteTracePayload,\n  type AssistantWorkflowEvalSuite,\n} from \"./eval-suite.js\";\nimport type {\n  AssistantWorkflowJsonValue,\n  AssistantWorkflowSerializedRunReport,\n} from \"./run-report-json.js\";\n\nexport type AssistantWorkflowEvalRunCaseInput = {\n  evalCase: AssistantWorkflowEvalCase;\n  report?: AssistantWorkflowSerializedRunReport | null;\n  qualitySignalNames?: readonly string[] | null;\n};\n\nexport type AssistantWorkflowEvalRunOptions = {\n  onlyFailures?: boolean;\n};\n\nexport type AssistantWorkflowEvalRun = {\n  generatedAt: string;\n  caseCount: number;\n  reportCount: number;\n  resultCount: number;\n  missingReportCount: number;\n  unmatchedReportCount: number;\n  missingReportCaseIds: string[];\n  suite: AssistantWorkflowEvalSuite;\n  results: AssistantWorkflowEvalResult[];\n};\n\nexport type AssistantWorkflowEvalRunReportsOptions =\n  AssistantWorkflowEvalRunOptions & {\n    qualitySignalNamesByCaseId?: Record<string, readonly string[]> | null;\n  };\n\nexport function runAssistantWorkflowEvalCases(\n  inputs: readonly AssistantWorkflowEvalRunCaseInput[],\n  options: AssistantWorkflowEvalRunOptions = {},\n): AssistantWorkflowEvalRun {\n  const results = inputs.map((input) => {\n    if (!input.report) {\n      return evaluateAssistantWorkflowEvalCase(\n        input.evalCase,\n        buildMissingAssistantWorkflowEvalActual(),\n      );\n    }\n    return evaluateAssistantWorkflowEvalCaseAgainstSerializedRunReport(\n      input.evalCase,\n      input.report,\n      buildActualOptions(input.qualitySignalNames),\n    );\n  });\n  const suite = buildAssistantWorkflowEvalSuite(results, {\n    ...(options.onlyFailures !== undefined ? { onlyFailures: options.onlyFailures } : {}),\n  });\n  const missingReportCaseIds = inputs\n    .filter((input) => !input.report)\n    .map((input) => input.evalCase.id);\n\n  return {\n    generatedAt: suite.generatedAt,\n    caseCount: inputs.length,\n    reportCount: inputs.filter((input) => input.report).length,\n    resultCount: results.length,\n    missingReportCount: missingReportCaseIds.length,\n    unmatchedReportCount: 0,\n    missingReportCaseIds,\n    suite,\n    results,\n  };\n}\n\nexport function runAssistantWorkflowEvalCasesAgainstSerializedRunReports(\n  evalCases: readonly AssistantWorkflowEvalCase[],\n  reports: readonly AssistantWorkflowSerializedRunReport[],\n  options: AssistantWorkflowEvalRunReportsOptions = {},\n): AssistantWorkflowEvalRun {\n  const inputs = evalCases.map((evalCase, index) => ({\n    evalCase,\n    report: reports[index] ?? null,\n    qualitySignalNames: options.qualitySignalNamesByCaseId?.[evalCase.id] ?? null,\n  }));\n  const run = runAssistantWorkflowEvalCases(inputs, options);\n  return {\n    ...run,\n    reportCount: reports.length,\n    unmatchedReportCount: Math.max(0, reports.length - evalCases.length),\n  };\n}\n\nexport function buildAssistantWorkflowEvalRunTracePayload(\n  run: AssistantWorkflowEvalRun,\n): Record<string, AssistantWorkflowJsonValue> {\n  return {\n    generatedAt: run.generatedAt,\n    caseCount: run.caseCount,\n    reportCount: run.reportCount,\n    resultCount: run.resultCount,\n    missingReportCount: run.missingReportCount,\n    unmatchedReportCount: run.unmatchedReportCount,\n    missingReportCaseIds: run.missingReportCaseIds,\n    suite: buildAssistantWorkflowEvalSuiteTracePayload(run.suite),\n    results: run.results.map(buildAssistantWorkflowEvalResultTracePayload),\n  };\n}\n\nfunction buildActualOptions(\n  qualitySignalNames: readonly string[] | null | undefined,\n): AssistantWorkflowEvalActualOptions {\n  return qualitySignalNames ? { qualitySignalNames } : {};\n}\n"]}

package/dist/eval-suite.d.ts

@@ -0,0 +1,43 @@
+import type { AssistantWorkflowEvalCheckName, AssistantWorkflowEvalCheckStatus, AssistantWorkflowEvalResult, AssistantWorkflowEvalResultStatus } from "./eval-result.js";
+import type { AssistantWorkflowJsonValue } from "./run-report-json.js";
+export type AssistantWorkflowEvalSuiteOptions = {
+    onlyFailures?: boolean;
+};
+export type AssistantWorkflowEvalSuiteCount = {
+    key: string;
+    count: number;
+};
+export type AssistantWorkflowEvalSuiteCheckCount = {
+    name: AssistantWorkflowEvalCheckName;
+    status: AssistantWorkflowEvalCheckStatus;
+    count: number;
+};
+export type AssistantWorkflowEvalSuiteRow = {
+    caseId: string;
+    required: boolean;
+    status: AssistantWorkflowEvalResultStatus;
+    failedCheckNames: AssistantWorkflowEvalCheckName[];
+    passedCount: number;
+    failedCount: number;
+    skippedCount: number;
+};
+export type AssistantWorkflowEvalSuite = {
+    generatedAt: string;
+    sourceResultCount: number;
+    resultCount: number;
+    passCount: number;
+    failCount: number;
+    skippedCount: number;
+    requiredCount: number;
+    requiredFailedCount: number;
+    optionalFailedCount: number;
+    requiredFailedCaseIds: string[];
+    optionalFailedCaseIds: string[];
+    statusCounts: AssistantWorkflowEvalSuiteCount[];
+    checkCounts: AssistantWorkflowEvalSuiteCheckCount[];
+    failedCheckCounts: AssistantWorkflowEvalSuiteCount[];
+    rows: AssistantWorkflowEvalSuiteRow[];
+};
+export declare function buildAssistantWorkflowEvalSuite(results: readonly AssistantWorkflowEvalResult[], options?: AssistantWorkflowEvalSuiteOptions): AssistantWorkflowEvalSuite;
+export declare function buildAssistantWorkflowEvalSuiteTracePayload(suite: AssistantWorkflowEvalSuite): Record<string, AssistantWorkflowJsonValue>;
+//# sourceMappingURL=eval-suite.d.ts.map

package/dist/eval-suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,8BAA8B,EAC9B,gCAAgC,EAChC,2BAA2B,EAC3B,iCAAiC,EAClC,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAEvE,MAAM,MAAM,iCAAiC,GAAG;IAC9C,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,oCAAoC,GAAG;IACjD,IAAI,EAAE,8BAA8B,CAAC;IACrC,MAAM,EAAE,gCAAgC,CAAC;IACzC,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,iCAAiC,CAAC;IAC1C,gBAAgB,EAAE,8BAA8B,EAAE,CAAC;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,YAAY,EAAE,+BAA+B,EAAE,CAAC;IAChD,WAAW,EAAE,oCAAoC,EAAE,CAAC;IACpD,iBAAiB,EAAE,+BAA+B,EAAE,CAAC;IACrD,IAAI,EAAE,6BAA6B,EAAE,CAAC;CACvC,CAAC;AAEF,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,SAAS,2BAA2B,EAAE,EAC/C,OAAO,GAAE,iCAAsC,GAC9C,0BAA0B,CA0B5B;AAED,wBAAgB,2CAA2C,CACzD,KAAK,EAAE,0BAA0B,GAChC,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAiB5C"}