@appliqation/autotest 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +79 -0
- package/dist/cli/audit.js +24 -0
- package/dist/cli/audit.js.map +1 -0
- package/dist/cli/index.js +496 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/output.js +33 -0
- package/dist/cli/output.js.map +1 -0
- package/dist/config/env.js +72 -0
- package/dist/config/env.js.map +1 -0
- package/dist/orchestrator/coveragePolicy.js +59 -0
- package/dist/orchestrator/coveragePolicy.js.map +1 -0
- package/dist/orchestrator/judgeTc.js +121 -0
- package/dist/orchestrator/judgeTc.js.map +1 -0
- package/dist/orchestrator/pollResults.js +37 -0
- package/dist/orchestrator/pollResults.js.map +1 -0
- package/dist/tools/browserLabel.js +19 -0
- package/dist/tools/browserLabel.js.map +1 -0
- package/dist/tools/dryRun.js +23 -0
- package/dist/tools/dryRun.js.map +1 -0
- package/dist/tools/safety.js +61 -0
- package/dist/tools/safety.js.map +1 -0
- package/dist/tools/screenshotViewer.js +112 -0
- package/dist/tools/screenshotViewer.js.map +1 -0
- package/package.json +39 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AACA,4EAA4E;AAC5E,2EAA2E;AAC3E,4EAA4E;AAC5E,kEAAkE;AAClE,yEAAyE;AACzE,gEAAgE;AAChE,oDAAoD;AAEpD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EAItB,mBAAmB,EACnB,cAAc,EACd,oBAAoB,EACpB,SAAS,EACT,SAAS,EACT,UAAU,EACV,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,UAAU,GAIX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEzE,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/E,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;AAE3F,sGAAsG;AACtG,SAAS,YAAY,CAAC,IAA8B;IAClD,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,QAAQ,KAAK,WAAW;QAC7B,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,eAAgB,EAAE,KAAK,EAAE,MAAM,CAAC,kBAAkB,CAAC;QACnF,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,YAAa,EAAE,KAAK,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,QAAQ,CAAC,MAAc,EAAE,OAAyH;IACzJ,OAAO,CAAC,CAAqC,EAAE,EAAE;QAC/C,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,CAAE,CAAC,CAAC,MAAiB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACjD,yEAAyE;YACzE,IAAI,IAAI;gBAAE,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,cAAc,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,MAA0C,CAAC;YACvD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC5B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,CAAC,GAAG,CAAC,CAAC,MAA4G,CAAC;YACzH,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YACb,MAAM,SAAS,GAAG,CAAC,CAAC,eAAe;gBACjC,CAAC,CAAC,KAAK,CAAC,CAAC,eAAe,cAAc;gBACtC,CAAC,CAAC,CAAC,CAAC,gBAAgB;oBAClB,CAAC,CAAC,KAAK,CAAC,CAAC,gBAAgB,oBAAoB;oBAC7C,CAAC,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC,WAAW,QAAQ,CAAC,CAAC,YAAY,GAAG,SAAS,EAAE,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,MAAkB;IACpD,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3B,OAAO,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,KAAK,4BAA4B,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;AACxF,CAAC;AAED,MAAM,sBAAsB,GAAG;IAC7B,yBAAyB;IACzB,0GAA0G;QACxG,mGAAmG;QACnG,6GAA6G;CACvG,CAAC;AAEX,MAAM,cAAc,GAAG;IACrB,WAAW;IACX,6GAA6G;QAC3G,0GAA0G;QAC1G,8GAA8G;CACxG,CAAC;AAEX,MAAM,WAAW,GAAG;IAClB,QAAQ;IACR,4GAA4G;QAC1G,0EAA0E;CACpE,CAAC;AAEX,MAAM,SAAS,GAAG;IAChB,MAAM;IACN,uGAAuG;QACrG,wGAAwG;QACxG,oDAAoD;CAC9C,CAAC;AAEX,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAC9B,OAAO;KACJ,IAAI,CAAC,sBAAsB,CAAC;KAC5B,WAAW,CAAC,8EAA8E,CAAC,CAAC;AAE/F,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CACV,4GAA4G;IAC1G,iGAAiG;IACjG,yGAAyG;IACzG,uGAAuG;IACvG,2GAA2G;IAC3G,0GAA0G;IAC1G,4GAA4G;IAC5G,kHAAkH;IAClH,oGAAoG;IACpG,yGAAyG;IACzG,yGAAyG;IACzG,sEAAsE,CACzE;KACA,MAAM,CACL,yBAAyB,EACzB,yGAAyG;IACvG,wGAAwG;IACxG,4BAA4B,CAC/B;KACA,MAAM,CAAC,oBAAoB,EAAE,2EAA2E,CAAC;KACzG,MAAM,CACL,oBAAoB,EACpB,4GAA4G;IAC1G,yGAAyG;IACzG,2GAA2G;IAC3G,+FAA+F,CAClG;KACA,cAAc,CACb,sBAAsB,EACtB,0GAA0G;IACxG,kEAAkE,CACrE;KACA,MAAM,CAAC,eAAe,EAAE,+CAA+C,CAAC;KACxE,MAAM,CACL,eAAe,EACf,0GAA0G;IACxG,6GAA6G;IAC7G,0GAA0G;IAC1G,0GAA0G;IAC1G,0FAA0F,CAC7F;KACA,MAAM,CACL,qBAAqB,EACrB,2GAA2G;IACzG,4FAA4F,EAC9F,mBAAmB,CACpB;KACA,MAAM,CACL,sBAAsB,EACtB,0GAA0G;IACxG,sGAAsG;IACtG,yGAAyG;IACzG,iBAAiB,CACpB;KACA,MAAM,CACL,wBAAwB,EACxB,2HAA2H,CAC5H;KACA,MAAM,CAAC,GAAG,sBAAsB,CAAC;KACjC,MAAM,CAAC,GAAG,cAAc,CAAC;KACzB,MAAM,CAAC,GAAG,WAAW,CAAC;KACtB,MAAM,CAAC,GAAG,SAAS,CAAC;KACpB,MAAM,CACL,KAAK,EAAE,IAcN,EAAE,EAAE;IACH,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,CAAC;IACxD,MAAM,eAAe,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,gBAAgB,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACnD,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,IAAI,MAAM,CAAC,mBAAmB,CAAC;IACnF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;IAEpC,oEAAoE;IACpE,kEAAkE;IAClE,oEAAoE;IACpE,qEAAqE;IACrE,uEAAuE;IACvE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,sBAAsB,EAAE,CAAC;IACvC,IAAI,OAA+B,CAAC;IAEpC,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAoC,CAAC;QAEnE,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrB,oEAAoE;YACpE,gEAAgE;YAChE,8DAA8D;YAC9D,+DAA+D;YAC/D,sDAAsD;YACtD,gEAAgE;YAChE,kEAAkE;YAClE,gEAAgE;YAChE,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACrE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAClE,MAAM,UAAU,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAC;YACnD,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/F,IAAI,IAAI,CAAC,IAAI;gBAAE,OAAO,CAAC,KAAK,CAAC,kCAAkC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;YAE7E,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAE7F,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;gBACrD,OAAO;YACT,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAsB,CAAC;YACjD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,MAAM,eAAe,GAAG,oBAAoB,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;gBAC9D,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC;oBAAE,UAAU,CAAC,GAAG,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;gBAC1E,UAAU,CAAC,GAAG,CAAC,eAAe,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,OAAO,CAAC,KAAK,CACX,qBAAqB,GAAG,CAAC,MAAM,sBAAsB,UAAU,CAAC,IAAI,2BAA2B,IAAI,CAAC,QAAQ,EAAE,CAC/G,CAAC;YAEF,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;YAClD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAmB,CAAC;YACnD,KAAK,MAAM,kBAAkB,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnD,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,0BAA0B,EAAE;oBACxE,WAAW,EAAE,kBAAkB;oBAC/B,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAC;gBACH,IAAI,eAAe,CAAC,EAAE,EAAE,CAAC;oBACvB,MAAM,SAAS,GACb,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAGhC,CAAC,SAAS,CAAC;oBACZ,KAAK,MAAM,CAAC,IAAI,SAAS;wBAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC;gBAC3F,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,KAAK,CAAC,wDAAwD,kBAAkB,KAAK,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvH,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE;oBACrC,UAAU,EAAE,MAAM,CAAC,kBAAkB,CAAC;oBACtC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;oBAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;iBAC9B,CAAC,CAAC;gBACH,eAAe,CAAC,GAAG,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC;YACjD,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,wBAAwB,mBAAmB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,cAAc,MAAM,EAAE,CAAC,CAAC;YAE7G,MAAM,OAAO,GAAe,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;YAC/B,KAAK,MAAM,WAAW,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC9C,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;oBAC5B,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC;oBACnE,MAAM,UAAU,GAAG,wBAAwB,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,kBAAkB,EAAE,YAAY,EAAE,CAAC,CAAC;oBACtG,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC5C,CAAC,CAAC,CAAC;YACL,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,MAAM,gBAAgB,OAAO,CAAC,MAAM,0BAA0B,OAAO,CAAC,MAAM,qBAAqB,CAAC,CAAC;YAEhI,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkD,CAAC;YAC5F,KAAK,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;gBACzB,MAAM,eAAe,GAAG,oBAAoB,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;gBAC9D,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,eAAe,CAAE,CAAC;gBACpD,OAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,YAAY,cAAc,eAAe,OAAO,CAAC,CAAC;gBACpF,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,gBAAgB,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBACpE,IAAI,YAAY,GAAG,oBAAoB,CAAC;oBACxC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oBACrB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;wBACf,MAAM,YAAY,GAAG,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;wBAC/C,IAAI,YAAY,EAAE,CAAC;4BACjB,IAAI,GAAG,YAAY,CAAC;4BACpB,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;gCACjD,yBAAyB,CAAC,GAAG,CAAC,YAAY,EAAE,mBAAmB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;4BAC5F,CAAC;4BACD,YAAY,GAAG,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;4BAC3D,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,YAAY,4BAA4B,YAAY,cAAc,CAAC,CAAC;wBAC3F,CAAC;oBACH,CAAC;oBACD,MAAM,aAAa,GAAG,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;oBAC/F,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC;wBACxC,MAAM;wBACN,KAAK;wBACL,YAAY,EAAE,EAAE,CAAC,YAAY;wBAC7B,GAAG;wBACH,QAAQ;wBACR,YAAY;wBACZ,aAAa;wBACb,eAAe;wBACf,gBAAgB;wBAChB,MAAM,EAAE,MAAM,CAAC,MAAM;wBACrB,mBAAmB;wBACnB,MAAM;wBACN,aAAa,EAAE,MAAM,CAAC,qBAAqB;wBAC3C,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,YAAY,IAAI,KAAK,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;qBACpF,CAAC,CAAC;oBACH,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,YAAY,yBAAyB,eAAe,CAAC,KAAK,SAAS,CAAC,CAAC;gBAC5F,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,YAAY,mBAAoB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,2CAA2C,UAAU,CAAC,IAAI,kBAAkB,aAAa,aAAa,CAAC,CAAC;YACtH,MAAM,aAAa,GAAG,IAAI,GAAG,EAA4D,CAAC;YAC1F,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,KAAK,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAAC,IAAI,UAAU,EAAE,CAAC;oBAC3D,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,kBAAkB,CAAE,CAAC;oBACvD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE;wBAC3C,KAAK;wBACL,UAAU,EAAE,kBAAkB;wBAC9B,SAAS,EAAE,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;wBAC1D,SAAS,EAAE,aAAa;wBACxB,UAAU,EAAE,MAAM,CAAC,cAAc;qBAClC,CAAC,CAAC;oBACH,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM;wBAAE,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;YAED,MAAM,QAAQ,GAAgB,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;gBAC3C,MAAM,eAAe,GAAG,oBAAoB,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;gBAC9D,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC;gBACnE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,EAAE,CAAC,YAAY,CAAC,CAAC;gBAC1E,MAAM,IAAI,GAAsB,SAAS,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC;gBAC3H,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;gBAClD,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,SAAS,CAAC,CAAC;gBAClE,OAAO;oBACL,YAAY,EAAE,EAAE,CAAC,YAAY;oBAC7B,IAAI;oBACJ,MAAM;oBACN,YAAY,EAAE,MAAM,EAAE,YAAY;oBAClC,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,eAAe,CAAC;oBAC3C,UAAU,EAAE,eAAe;iBAC5B,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,OAAO,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;YACnD,IAAI,IAAI;gBAAE,gBAAgB,CAAC,OAAO,CAAC,CAAC;;gBAC/B,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAEhC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;YACtE,IAAI,CAAC,MAAM,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,KAAK,CAAC,KAAK,OAAO,iGAAiG,CAAC,CAAC;YAC/H,CAAC;YAED,OAAO,CAAC,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YACxC,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACvE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAClE,qEAAqE;QACrE,oEAAoE;QACpE,kEAAkE;QAClE,qEAAqE;QACrE,yDAAyD;QACzD,MAAM,UAAU,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/F,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,KAAK,CAAC,kCAAkC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE;YACrC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;YAC9B,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;YAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;QACH,OAAO,CAAC,KAAK,CAAC,wBAAwB,mBAAmB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,cAAc,MAAM,EAAE,CAAC,CAAC;QAE7G,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,+EAA+E;YAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;YACvC,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,YAAY,CAAC,CAAC;YAChE,MAAM,QAAQ,GAAG,gBAAgB,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAClF,IAAI,YAAY,GAAG,oBAAoB,CAAC;YACxC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACnE,IAAI,YAAY,EAAE,CAAC;oBACjB,IAAI,GAAG,YAAY,CAAC;oBACpB,YAAY,GAAG,mBAAmB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;oBAC5D,OAAO,CAAC,KAAK,CAAC,kCAAkC,YAAY,cAAc,CAAC,CAAC;gBAC9E,CAAC;YACH,CAAC;YACD,MAAM,aAAa,GAAG,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/F,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC;gBACxD,MAAM;gBACN,KAAK;gBACL,YAAY;gBACZ,GAAG;gBACH,QAAQ;gBACR,YAAY;gBACZ,aAAa;gBACb,eAAe;gBACf,gBAAgB;gBAChB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,mBAAmB;gBACnB,MAAM;gBACN,aAAa,EAAE,MAAM,CAAC,qBAAqB;gBAC3C,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;aACjE,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,WAAW,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;gBAC/C,WAAW,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;YACnD,CAAC;YAED,qEAAqE;YACrE,gEAAgE;YAChE,sEAAsE;YACtE,IAAI,MAAM,GAAG,SAAS,CAAC;YACvB,IAAI,YAAgC,CAAC;YACrC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE;oBAC3C,KAAK;oBACL,UAAU;oBACV,SAAS,EAAE,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;oBAClC,SAAS,EAAE,MAAM,CAAC,aAAa;oBAC/B,UAAU,EAAE,MAAM,CAAC,cAAc;iBAClC,CAAC,CAAC;gBACH,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBACpC,MAAM,GAAG,EAAE,EAAE,MAAM,IAAI,SAAS,CAAC;gBACjC,YAAY,GAAG,EAAE,EAAE,YAAY,CAAC;YAClC,CAAC;YAED,MAAM,OAAO,GAAc,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;YACnF,OAAO,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI;gBAAE,gBAAgB,CAAC,OAAO,CAAC,CAAC;;gBAC/B,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAChC,OAAO,CAAC,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YACxC,OAAO;QACT,CAAC;QAED,uBAAuB;QACvB,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QAC7F,OAAO,CAAC,KAAK,CAAC,qBAAqB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEpD,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,0BAA0B,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9H,IAAI,CAAC,eAAe,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;QACrG,MAAM,SAAS,GACb,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAGhC,CAAC,SAAS,CAAC;QAEZ,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,UAAU,GAAG,wBAAwB,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,kBAAkB,EAAE,EAAE,CAAC,oBAAoB,EAAE,CAAC,CAAC;YACjH,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,CAAC,KAAK,CACX,WAAW,SAAS,CAAC,MAAM,gBAAgB,OAAO,CAAC,MAAM,0BAA0B,OAAO,CAAC,MAAM,qBAAqB,CACvH,CAAC;QAEF,sEAAsE;QACtE,iEAAiE;QACjE,kEAAkE;QAClE,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkD,CAAC;QAC5F,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,iBAAiB,MAAM,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,MAAM,CAAC,CAAC;gBAC1D,MAAM,QAAQ,GAAG,gBAAgB,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAClF,IAAI,YAAY,GAAG,oBAAoB,CAAC;gBACxC,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBACnE,IAAI,YAAY,EAAE,CAAC;wBACjB,IAAI,GAAG,YAAY,CAAC;wBACpB,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;4BACjD,yBAAyB,CAAC,GAAG,CAAC,YAAY,EAAE,mBAAmB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;wBAC5F,CAAC;wBACD,YAAY,GAAG,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;wBAC3D,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,4BAA4B,YAAY,cAAc,CAAC,CAAC;oBAClF,CAAC;gBACH,CAAC;gBACD,MAAM,aAAa,GAAG,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC/F,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,OAAO,CAAC;oBACxC,MAAM;oBACN,KAAK;oBACL,YAAY,EAAE,MAAM;oBACpB,GAAG;oBACH,QAAQ;oBACR,YAAY;oBACZ,aAAa;oBACb,eAAe;oBACf,gBAAgB;oBAChB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,mBAAmB;oBACnB,MAAM;oBACN,aAAa,EAAE,MAAM,CAAC,qBAAqB;oBAC3C,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,MAAM,IAAI,KAAK,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;iBAC3E,CAAC,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,yBAAyB,eAAe,CAAC,KAAK,SAAS,CAAC,CAAC;YACnF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,mBAAoB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,0DAA0D;QAC1D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,OAAO,CAAC,KAAK,CAAC,8CAA8C,aAAa,QAAQ,CAAC,CAAC;QACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,MAAM;YACpB,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,qEAAqE;YACjF,CAAC,CAAC,MAAM,eAAe,CAAC,MAAM,EAAE;gBAC5B,KAAK;gBACL,UAAU;gBACV,SAAS,EAAE,QAAQ;gBACnB,SAAS,EAAE,aAAa;gBACxB,UAAU,EAAE,MAAM,CAAC,cAAc;aAClC,CAAC,CAAC;QAEP,MAAM,QAAQ,GAAgB,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACjD,MAAM,IAAI,GAAsB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,cAAc,CAAC;gBACjE,CAAC,CAAC,EAAE,CAAC,oBAAoB;oBACvB,CAAC,CAAC,4BAA4B;oBAC9B,CAAC,CAAC,SAAS;gBACb,CAAC,CAAC,kBAAkB,CAAC;YACvB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,SAAS,CAAC,CAAC;YAClE,OAAO,EAAE,YAAY,EAAE,EAAE,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QAC/F,CAAC,CAAC,CAAC;QACH,OAAO,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QAC3D,IAAI,IAAI;YAAE,gBAAgB,CAAC,OAAO,CAAC,CAAC;;YAC/B,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAEhC,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;QACtE,IAAI,CAAC,MAAM,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACpC,OAAO,CAAC,KAAK,CAAC,KAAK,OAAO,gGAAgG,CAAC,CAAC;QAC9H,CAAC;QAED,OAAO,CAAC,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;YAAS,CAAC;QACT,kEAAkE;QAClE,iEAAiE;QACjE,4DAA4D;QAC5D,yDAAyD;QACzD,oEAAoE;QACpE,MAAM,cAAc,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC,SAAS;YACtB,SAAS;YACT,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;YACnB,aAAa,EAAE,YAAY,CAAC,UAAU,CAAC;YACvC,cAAc,EAAE,YAAY,CAAC,WAAW,CAAC;YACzC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE;YACrB,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;YACvC,OAAO;SACR,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CACF,CAAC;AAEJ,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Machine-readable / CI-friendly final summary for `judge` and `run`. Progress
|
|
2
|
+
// logs (onEvent -> console.error) are untouched by this — this only changes
|
|
3
|
+
// how the final outcome is rendered on stdout and what exit code the process
|
|
4
|
+
// uses, so piping to `jq` or wiring a CI status check doesn't require
|
|
5
|
+
// scraping human-readable prose.
|
|
6
|
+
// Statuses that mean "a CI check consuming this should fail the build" —
|
|
7
|
+
// includes 'pending' deliberately: a poll timeout means this app never
|
|
8
|
+
// confirmed the outcome, and treating that as green would defeat the point
|
|
9
|
+
// of gating on it.
|
|
10
|
+
const FAILING_STATUSES = new Set(['failed', 'blocked', 'pending']);
|
|
11
|
+
export function printJsonSummary(summary) {
|
|
12
|
+
console.log(JSON.stringify(summary, null, 2));
|
|
13
|
+
}
|
|
14
|
+
export function printHumanSummary(summary) {
|
|
15
|
+
const header = summary.testSetId
|
|
16
|
+
? `Test set ${summary.testSetId}`
|
|
17
|
+
: summary.scenarioId
|
|
18
|
+
? `Run ${summary.runId} — scenario ${summary.scenarioId}`
|
|
19
|
+
: `Run ${summary.runId}`;
|
|
20
|
+
console.log(`\n=== ${header} ===\n`);
|
|
21
|
+
for (const r of summary.results) {
|
|
22
|
+
console.log(` ${r.testCaseUuid} [${r.path}] ${r.status}`);
|
|
23
|
+
if (r.errorMessage)
|
|
24
|
+
console.log(` ${r.errorMessage.slice(0, 300)}`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/** 0 unless a real (non-dry-run) result actually failed, was blocked, or never settled. */
|
|
28
|
+
export function exitCodeFor(summary) {
|
|
29
|
+
if (summary.dryRun)
|
|
30
|
+
return 0;
|
|
31
|
+
return summary.results.some((r) => FAILING_STATUSES.has(r.status)) ? 1 : 0;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=output.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/cli/output.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,sEAAsE;AACtE,iCAAiC;AA4BjC,yEAAyE;AACzE,uEAAuE;AACvE,2EAA2E;AAC3E,mBAAmB;AACnB,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IAClD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAmB;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS;QAC9B,CAAC,CAAC,YAAY,OAAO,CAAC,SAAS,EAAE;QACjC,CAAC,CAAC,OAAO,CAAC,UAAU;YAClB,CAAC,CAAC,OAAO,OAAO,CAAC,KAAK,eAAe,OAAO,CAAC,UAAU,EAAE;YACzD,CAAC,CAAC,OAAO,OAAO,CAAC,KAAK,EAAE,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,QAAQ,CAAC,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,YAAY,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,YAAY;YAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,WAAW,CAAC,OAAmB;IAC7C,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import 'dotenv/config';
|
|
2
|
+
import { DEFAULT_ANTHROPIC_MODEL, DEFAULT_OPENAI_MODEL } from '@appliqation/agent-core/providers';
|
|
3
|
+
import { required, optional } from '@appliqation/agent-core/config';
|
|
4
|
+
import { resolveAuditSink } from '@appliqation/agent-core/audit';
|
|
5
|
+
export const config = {
|
|
6
|
+
appqOrigin: optional('APPQ_ORIGIN') ?? 'https://appq.appliqation.io',
|
|
7
|
+
appqApiKey: () => required('APPQ_API_KEY'),
|
|
8
|
+
anthropicApiKey: optional('ANTHROPIC_API_KEY'),
|
|
9
|
+
openaiApiKey: optional('OPENAI_API_KEY'),
|
|
10
|
+
// Default for the validator's image-viewing mode. Deliberately a
|
|
11
|
+
// deployment-level setting (env var here, overridable per-run via
|
|
12
|
+
// --mandatory-image-check), not something the workflow prompt decides —
|
|
13
|
+
// it's orchestration policy (customer's token-cost appetite), not
|
|
14
|
+
// testing methodology. false = on-demand (model requests view_screenshot
|
|
15
|
+
// only when text evidence isn't enough); true = every step's screenshot
|
|
16
|
+
// is fetched and attached unconditionally, enforced in code.
|
|
17
|
+
mandatoryImageCheck: (optional('MANDATORY_IMAGE_CHECK') ?? 'false') === 'true',
|
|
18
|
+
budget: {
|
|
19
|
+
maxCalls: Number(optional('BUDGET_MAX_CALLS') ?? 50),
|
|
20
|
+
maxPages: Number(optional('BUDGET_MAX_PAGES') ?? 12),
|
|
21
|
+
maxMillis: Number(optional('BUDGET_MAX_MILLIS') ?? 15 * 60 * 1000),
|
|
22
|
+
maxTurns: Number(optional('BUDGET_MAX_TURNS') ?? 80),
|
|
23
|
+
},
|
|
24
|
+
// Response length cap, provider-specific param names (Anthropic:
|
|
25
|
+
// max_tokens, OpenAI: max_output_tokens) — same reasoning as splitting
|
|
26
|
+
// models by role/provider: don't collapse genuinely different knobs into
|
|
27
|
+
// one setting just because they're conceptually similar.
|
|
28
|
+
anthropicMaxTokens: Number(optional('ANTHROPIC_MAX_TOKENS') ?? 4096),
|
|
29
|
+
openaiMaxOutputTokens: Number(optional('OPENAI_MAX_OUTPUT_TOKENS') ?? 4096),
|
|
30
|
+
evidenceRingBufferCap: Number(optional('EVIDENCE_RING_BUFFER_CAP') ?? 500),
|
|
31
|
+
pollIntervalMs: Number(optional('POLL_INTERVAL_MS') ?? 5000),
|
|
32
|
+
pollTimeoutMs: Number(optional('POLL_TIMEOUT_MS') ?? 120000),
|
|
33
|
+
// Observability, entirely opt-in — see @appliqation/agent-core's audit/sink.ts.
|
|
34
|
+
auditSink: resolveAuditSink({
|
|
35
|
+
auditMongoUri: optional('AUDIT_MONGO_URI'),
|
|
36
|
+
auditMongoDb: optional('AUDIT_MONGO_DB'),
|
|
37
|
+
auditMongoCollection: optional('AUDIT_MONGO_COLLECTION'),
|
|
38
|
+
auditJsonlPath: optional('AUDIT_JSONL_PATH'),
|
|
39
|
+
}),
|
|
40
|
+
};
|
|
41
|
+
export function resolveProvider() {
|
|
42
|
+
if (config.anthropicApiKey)
|
|
43
|
+
return 'anthropic';
|
|
44
|
+
if (config.openaiApiKey)
|
|
45
|
+
return 'openai';
|
|
46
|
+
throw new Error('Set ANTHROPIC_API_KEY or OPENAI_API_KEY');
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Resolves which model to use for a given role. Role-specific overrides
|
|
50
|
+
* (*_EXECUTOR_MODEL / *_VALIDATOR_MODEL) take precedence over the blanket
|
|
51
|
+
* *_MODEL override, which takes precedence over the provider's own default.
|
|
52
|
+
*
|
|
53
|
+
* Split by role deliberately, not just a single global override: judging
|
|
54
|
+
* captured evidence against a known expected_result (the validator) is
|
|
55
|
+
* closer to bounded classification than the open-ended planning driving a
|
|
56
|
+
* browser needs (the executor) — a cheaper/faster model is a reasonable
|
|
57
|
+
* fit for the former specifically, and a genuinely different model is one
|
|
58
|
+
* more decorrelation lever against same-model self-grading risk. See the
|
|
59
|
+
* plan/session notes on this. A blanket *_MODEL override (e.g. for cheap
|
|
60
|
+
* end-to-end testing before trusting this against a real project) still
|
|
61
|
+
* works — it's just the fallback both roles share unless overridden.
|
|
62
|
+
*/
|
|
63
|
+
export function resolveModel(role) {
|
|
64
|
+
const provider = resolveProvider();
|
|
65
|
+
if (provider === 'anthropic') {
|
|
66
|
+
const roleOverride = optional(role === 'executor' ? 'ANTHROPIC_EXECUTOR_MODEL' : 'ANTHROPIC_VALIDATOR_MODEL');
|
|
67
|
+
return roleOverride ?? optional('ANTHROPIC_MODEL') ?? DEFAULT_ANTHROPIC_MODEL;
|
|
68
|
+
}
|
|
69
|
+
const roleOverride = optional(role === 'executor' ? 'OPENAI_EXECUTOR_MODEL' : 'OPENAI_VALIDATOR_MODEL');
|
|
70
|
+
return roleOverride ?? optional('OPENAI_MODEL') ?? DEFAULT_OPENAI_MODEL;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=env.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../src/config/env.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEjE,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,UAAU,EAAE,QAAQ,CAAC,aAAa,CAAC,IAAI,6BAA6B;IACpE,UAAU,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IAC1C,eAAe,EAAE,QAAQ,CAAC,mBAAmB,CAAC;IAC9C,YAAY,EAAE,QAAQ,CAAC,gBAAgB,CAAC;IACxC,iEAAiE;IACjE,kEAAkE;IAClE,wEAAwE;IACxE,kEAAkE;IAClE,yEAAyE;IACzE,wEAAwE;IACxE,6DAA6D;IAC7D,mBAAmB,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,OAAO,CAAC,KAAK,MAAM;IAC9E,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QACpD,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QACpD,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAClE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;KACrD;IACD,iEAAiE;IACjE,uEAAuE;IACvE,yEAAyE;IACzE,yDAAyD;IACzD,kBAAkB,EAAE,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAC,IAAI,IAAI,CAAC;IACpE,qBAAqB,EAAE,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,IAAI,IAAI,CAAC;IAC3E,qBAAqB,EAAE,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,IAAI,GAAG,CAAC;IAC1E,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC;IAC5D,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,MAAM,CAAC;IAE5D,gFAAgF;IAChF,SAAS,EAAE,gBAAgB,CAAC;QAC1B,aAAa,EAAE,QAAQ,CAAC,iBAAiB,CAAC;QAC1C,YAAY,EAAE,QAAQ,CAAC,gBAAgB,CAAC;QACxC,oBAAoB,EAAE,QAAQ,CAAC,wBAAwB,CAAC;QACxD,cAAc,EAAE,QAAQ,CAAC,kBAAkB,CAAC;KAC7C,CAAC;CACH,CAAC;AAEF,MAAM,UAAU,eAAe;IAC7B,IAAI,MAAM,CAAC,eAAe;QAAE,OAAO,WAAW,CAAC;IAC/C,IAAI,MAAM,CAAC,YAAY;QAAE,OAAO,QAAQ,CAAC;IACzC,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,YAAY,CAAC,IAA8B;IACzD,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC;QAC9G,OAAO,YAAY,IAAI,QAAQ,CAAC,iBAAiB,CAAC,IAAI,uBAAuB,CAAC;IAChF,CAAC;IACD,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC;IACxG,OAAO,YAAY,IAAI,QAAQ,CAAC,cAAc,CAAC,IAAI,oBAAoB,CAAC;AAC1E,CAAC"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Whether the agentic executor/validator pair also runs for a TC that
|
|
2
|
+
// already has a canonical script — deliberately never hardcoded (see the
|
|
3
|
+
// plan doc: "the coverage decision"). Config-driven for now (CLI flag /
|
|
4
|
+
// env var), not read from appq project settings — that field doesn't exist
|
|
5
|
+
// there yet. When it does, this is the one place to wire that read in;
|
|
6
|
+
// nothing else in the orchestrator should need to change.
|
|
7
|
+
//
|
|
8
|
+
// The default ('on-script-absence') is a conservative bootstrap choice for
|
|
9
|
+
// early/dev use, not a product stance — "always" is the philosophically
|
|
10
|
+
// preferred position from the complementary-signals design (see the plan),
|
|
11
|
+
// but costs real tokens per TC per run. Pick deliberately once there's
|
|
12
|
+
// enough live usage to reason about that tradeoff.
|
|
13
|
+
export function parseCoveragePolicy(raw) {
|
|
14
|
+
if (raw === 'always')
|
|
15
|
+
return { kind: 'always' };
|
|
16
|
+
if (raw === 'on-script-absence')
|
|
17
|
+
return { kind: 'on-script-absence' };
|
|
18
|
+
if (raw === 'external')
|
|
19
|
+
return { kind: 'external' };
|
|
20
|
+
const sampledMatch = /^sampled:(\d+)$/.exec(raw);
|
|
21
|
+
if (sampledMatch) {
|
|
22
|
+
const n = Number(sampledMatch[1]);
|
|
23
|
+
if (n < 1)
|
|
24
|
+
throw new Error(`Invalid coverage policy "${raw}": sampled:N requires N >= 1`);
|
|
25
|
+
return { kind: 'sampled', n };
|
|
26
|
+
}
|
|
27
|
+
throw new Error(`Invalid coverage policy "${raw}". Expected: always | on-script-absence | sampled:N | external`);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Decides whether to run the agentic executor/validator pair for one TC.
|
|
31
|
+
* The deterministic canonical-script path (if one exists) always runs
|
|
32
|
+
* regardless of this decision — it fires automatically on run creation,
|
|
33
|
+
* outside this app's control. This only decides whether agentic coverage
|
|
34
|
+
* *also* runs alongside it.
|
|
35
|
+
*
|
|
36
|
+
* @param externalDecider Required when policy.kind === 'external' — the
|
|
37
|
+
* pluggable hook for a future orchestrating agent to make this call
|
|
38
|
+
* dynamically. Not implemented today; passing 'external' without one
|
|
39
|
+
* throws, deliberately, rather than silently falling back to a
|
|
40
|
+
* different policy.
|
|
41
|
+
*/
|
|
42
|
+
export function shouldRunAgenticCoverage(policy, input, externalDecider) {
|
|
43
|
+
switch (policy.kind) {
|
|
44
|
+
case 'always':
|
|
45
|
+
return true;
|
|
46
|
+
case 'on-script-absence':
|
|
47
|
+
return !input.hasCanonicalScript;
|
|
48
|
+
case 'sampled':
|
|
49
|
+
return input.tcIndex % policy.n === 0;
|
|
50
|
+
case 'external':
|
|
51
|
+
if (!externalDecider) {
|
|
52
|
+
throw new Error('Coverage policy "external" has no decision-maker wired up — this is a hook for a future ' +
|
|
53
|
+
'orchestrating agent, not implemented today. Pass a different policy (always | on-script-absence | ' +
|
|
54
|
+
'sampled:N) until one exists.');
|
|
55
|
+
}
|
|
56
|
+
return externalDecider(input);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=coveragePolicy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coveragePolicy.js","sourceRoot":"","sources":["../../src/orchestrator/coveragePolicy.ts"],"names":[],"mappings":"AAAA,sEAAsE;AACtE,yEAAyE;AACzE,wEAAwE;AACxE,2EAA2E;AAC3E,uEAAuE;AACvE,0DAA0D;AAC1D,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,2EAA2E;AAC3E,uEAAuE;AACvE,mDAAmD;AAQnD,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAChD,IAAI,GAAG,KAAK,mBAAmB;QAAE,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC;IACtE,IAAI,GAAG,KAAK,UAAU;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IACpD,MAAM,YAAY,GAAG,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjD,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,8BAA8B,CAAC,CAAC;QAC1F,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,gEAAgE,CAAC,CAAC;AACnH,CAAC;AAOD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,wBAAwB,CACtC,MAAsB,EACtB,KAA4B,EAC5B,eAA2D;IAE3D,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ;YACX,OAAO,IAAI,CAAC;QACd,KAAK,mBAAmB;YACtB,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC;QACnC,KAAK,SAAS;YACZ,OAAO,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;QACxC,KAAK,UAAU;YACb,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CACb,0FAA0F;oBACxF,oGAAoG;oBACpG,8BAA8B,CACjC,CAAC;YACJ,CAAC;YACD,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// The executor -> validator two-stage pattern for one test case, factored
|
|
2
|
+
// out of cli/index.ts so both `judge` (single TC) and `run` (whole
|
|
3
|
+
// scenario) share exactly one implementation rather than two copies that
|
|
4
|
+
// can drift.
|
|
5
|
+
import { chromium, request } from 'playwright';
|
|
6
|
+
import { PlaywrightBrowserTools, BROWSER_TOOL_DEFS, ApiRequestTools, API_TOOL_DEFS, fetchAppqToolDefs, createGatedAppqDispatcher, runWorkflow, } from '@appliqation/agent-core';
|
|
7
|
+
import { executorAllowedAppqTools, validatorAllowedAppqTools } from '../tools/safety.js';
|
|
8
|
+
import { ScreenshotViewer } from '../tools/screenshotViewer.js';
|
|
9
|
+
import { createDryRunDispatcher } from '../tools/dryRun.js';
|
|
10
|
+
import { formatBrowserLabel, createBrowserLabelDispatcher } from '../tools/browserLabel.js';
|
|
11
|
+
export async function judgeTc(opts) {
|
|
12
|
+
const { client, runId, testCaseUuid, url, storageState, apiAuthHeader, executorAdapter, validatorAdapter, budget, mandatoryImageCheck, dryRun, ringBufferCap, onEvent } = opts;
|
|
13
|
+
const testType = opts.testType ?? 'ui';
|
|
14
|
+
const executorToolDefs = await fetchAppqToolDefs(client, executorAllowedAppqTools());
|
|
15
|
+
const gatedExecutorAppq = createGatedAppqDispatcher(client, executorAllowedAppqTools());
|
|
16
|
+
let executorResult;
|
|
17
|
+
let browserLabel;
|
|
18
|
+
if (testType === 'api') {
|
|
19
|
+
// Stage 1 (API path): a real HTTP client, no browser launched at all.
|
|
20
|
+
// Captured before the context disposes below — Stage 2 never makes its
|
|
21
|
+
// own request, so this is the only point that needs to know the mode.
|
|
22
|
+
browserLabel = 'API';
|
|
23
|
+
const apiContext = await request.newContext({
|
|
24
|
+
baseURL: url,
|
|
25
|
+
extraHTTPHeaders: apiAuthHeader ? { [apiAuthHeader.name]: apiAuthHeader.value } : undefined,
|
|
26
|
+
});
|
|
27
|
+
try {
|
|
28
|
+
const apiTools = new ApiRequestTools(apiContext, dryRun);
|
|
29
|
+
const apiDispatch = async (name, args) => {
|
|
30
|
+
if (name === 'http_request')
|
|
31
|
+
return apiTools.dispatch(name, args);
|
|
32
|
+
return gatedExecutorAppq(name, args);
|
|
33
|
+
};
|
|
34
|
+
executorResult = await runWorkflow({
|
|
35
|
+
source: { kind: 'appq', name: 'appq:autotest-executor', args: { run_id: runId, test_case_uuid: testCaseUuid, url, test_type: 'api' } },
|
|
36
|
+
fetchPrompt: client.fetchPrompt,
|
|
37
|
+
seedMessage: `Run: ${runId}\nTest case: ${testCaseUuid}\nBase URL: ${url}\nBegin now — start with get_scenario.`,
|
|
38
|
+
tools: [...API_TOOL_DEFS, ...executorToolDefs],
|
|
39
|
+
dispatch: apiDispatch,
|
|
40
|
+
adapter: executorAdapter,
|
|
41
|
+
budget,
|
|
42
|
+
onEvent: (e) => onEvent?.('executor', e),
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
finally {
|
|
46
|
+
await apiContext.dispose();
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
// Stage 1 (UI path): drives a real browser; may write only evidence.
|
|
51
|
+
const browser = await chromium.launch();
|
|
52
|
+
// Captured before the browser closes below — Stage 2 never launches its
|
|
53
|
+
// own browser, so this is the only point with access to the real version.
|
|
54
|
+
browserLabel = formatBrowserLabel(browser.version());
|
|
55
|
+
// browser.newPage() is Playwright's single-context convenience shortcut —
|
|
56
|
+
// switch to an explicit context so storageState has something to attach
|
|
57
|
+
// to. No behavior change when storageState is undefined (today's path).
|
|
58
|
+
const context = await browser.newContext(storageState ? { storageState } : {});
|
|
59
|
+
const page = await context.newPage();
|
|
60
|
+
try {
|
|
61
|
+
const browserTools = new PlaywrightBrowserTools(page, ringBufferCap, {
|
|
62
|
+
screenshotSink: async (png, label) => {
|
|
63
|
+
try {
|
|
64
|
+
return { ok: true, ref: await client.uploadScreenshot(png, label) };
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
return { ok: false, note: err.message };
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
const executorDispatch = async (name, args) => {
|
|
72
|
+
if (name.startsWith('browser_'))
|
|
73
|
+
return browserTools.dispatch(name, args);
|
|
74
|
+
return gatedExecutorAppq(name, args);
|
|
75
|
+
};
|
|
76
|
+
executorResult = await runWorkflow({
|
|
77
|
+
source: { kind: 'appq', name: 'appq:autotest-executor', args: { run_id: runId, test_case_uuid: testCaseUuid, url } },
|
|
78
|
+
fetchPrompt: client.fetchPrompt,
|
|
79
|
+
seedMessage: `Run: ${runId}\nTest case: ${testCaseUuid}\nURL under test: ${url}\nBegin now — start with get_scenario.`,
|
|
80
|
+
tools: [...BROWSER_TOOL_DEFS, ...executorToolDefs],
|
|
81
|
+
dispatch: executorDispatch,
|
|
82
|
+
adapter: executorAdapter,
|
|
83
|
+
budget,
|
|
84
|
+
onEvent: (e) => onEvent?.('executor', e),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
await browser.close();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// Stage 2: validator. A genuinely fresh runWorkflow() call — no browser
|
|
92
|
+
// tools, no messages carried over from stage 1, nothing but run_id and
|
|
93
|
+
// test_case_uuid in common.
|
|
94
|
+
const mandatoryNote = mandatoryImageCheck ? ' (mandatory image check)' : '';
|
|
95
|
+
const dryRunNote = dryRun ? ' (dry run — writeback suppressed)' : '';
|
|
96
|
+
if (mandatoryNote || dryRunNote)
|
|
97
|
+
onEvent?.('validator', { type: 'log', detail: `mode:${mandatoryNote}${dryRunNote}` });
|
|
98
|
+
const screenshotViewer = new ScreenshotViewer(mandatoryImageCheck);
|
|
99
|
+
const validatorToolDefs = await fetchAppqToolDefs(client, validatorAllowedAppqTools());
|
|
100
|
+
const gatedValidatorAppq = createGatedAppqDispatcher(client, validatorAllowedAppqTools());
|
|
101
|
+
// Browser-label correction must be OUTERMOST, applied before dry-run's
|
|
102
|
+
// interception decides what to log — otherwise a dry-run's "would call
|
|
103
|
+
// create_defect with..." preview shows the model's own raw (possibly
|
|
104
|
+
// bare/wrong) browser value instead of what a real call would actually
|
|
105
|
+
// send. createDryRunDispatcher only inspects args for VERDICT_WRITE_TOOLS
|
|
106
|
+
// (create_defect/update_run_results); wrapping it inside the label
|
|
107
|
+
// dispatcher doesn't change behavior for any other tool.
|
|
108
|
+
const dispatch = createBrowserLabelDispatcher(createDryRunDispatcher(screenshotViewer.wrapDispatch(gatedValidatorAppq), dryRun), browserLabel);
|
|
109
|
+
const validatorResult = await runWorkflow({
|
|
110
|
+
source: { kind: 'appq', name: 'appq:autotest-validator', args: { run_id: runId, test_case_uuid: testCaseUuid, test_type: testType } },
|
|
111
|
+
fetchPrompt: client.fetchPrompt,
|
|
112
|
+
seedMessage: `Run: ${runId}\nTest case: ${testCaseUuid}\nBegin now — start with get_scenario.`,
|
|
113
|
+
tools: [...validatorToolDefs, ...screenshotViewer.toolDefs()],
|
|
114
|
+
dispatch,
|
|
115
|
+
adapter: validatorAdapter,
|
|
116
|
+
budget,
|
|
117
|
+
onEvent: (e) => onEvent?.('validator', e),
|
|
118
|
+
});
|
|
119
|
+
return { executorResult, validatorResult };
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=judgeTc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judgeTc.js","sourceRoot":"","sources":["../../src/orchestrator/judgeTc.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,mEAAmE;AACnE,yEAAyE;AACzE,aAAa;AAEb,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE/C,OAAO,EACL,sBAAsB,EACtB,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,yBAAyB,EACzB,WAAW,GAOZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,0BAA0B,CAAC;AAoD5F,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAoB;IAChD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAC/K,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;IAEvC,MAAM,gBAAgB,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,wBAAwB,EAAE,CAAC,CAAC;IACrF,MAAM,iBAAiB,GAAG,yBAAyB,CAAC,MAAM,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAExF,IAAI,cAA0B,CAAC;IAC/B,IAAI,YAAoB,CAAC;IAEzB,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;QACvB,sEAAsE;QACtE,uEAAuE;QACvE,sEAAsE;QACtE,YAAY,GAAG,KAAK,CAAC;QACrB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YAC1C,OAAO,EAAE,GAAG;YACZ,gBAAgB,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS;SAC5F,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,WAAW,GAAmB,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;gBACvD,IAAI,IAAI,KAAK,cAAc;oBAAE,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAClE,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACvC,CAAC,CAAC;YAEF,cAAc,GAAG,MAAM,WAAW,CAAC;gBACjC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE;gBACtI,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,WAAW,EAAE,QAAQ,KAAK,gBAAgB,YAAY,eAAe,GAAG,wCAAwC;gBAChH,KAAK,EAAE,CAAC,GAAG,aAAa,EAAE,GAAG,gBAAgB,CAAC;gBAC9C,QAAQ,EAAE,WAAW;gBACrB,OAAO,EAAE,eAAe;gBACxB,MAAM;gBACN,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;aACzC,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;SACI,CAAC;QACJ,qEAAqE;QACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,CAAC;QACxC,wEAAwE;QACxE,0EAA0E;QAC1E,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,0EAA0E;QAC1E,wEAAwE;QACxE,wEAAwE;QACxE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAI,sBAAsB,CAAC,IAAI,EAAE,aAAa,EAAE;gBACnE,cAAc,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;oBACnC,IAAI,CAAC;wBACH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;oBACtE,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC;oBACrD,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,MAAM,gBAAgB,GAAmB,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;gBAC5D,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;oBAAE,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC1E,OAAO,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACvC,CAAC,CAAC;YAEF,cAAc,GAAG,MAAM,WAAW,CAAC;gBACjC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE;gBACpH,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,WAAW,EAAE,QAAQ,KAAK,gBAAgB,YAAY,qBAAqB,GAAG,wCAAwC;gBACtH,KAAK,EAAE,CAAC,GAAG,iBAAiB,EAAE,GAAG,gBAAgB,CAAC;gBAClD,QAAQ,EAAE,gBAAgB;gBAC1B,OAAO,EAAE,eAAe;gBACxB,MAAM;gBACN,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;aACzC,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,uEAAuE;IACvE,4BAA4B;IAC5B,MAAM,aAAa,GAAG,mBAAmB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5E,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,mCAAmC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,IAAI,aAAa,IAAI,UAAU;QAAE,OAAO,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,aAAa,GAAG,UAAU,EAAE,EAAE,CAAC,CAAC;IAEvH,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,yBAAyB,EAAE,CAAC,CAAC;IACvF,MAAM,kBAAkB,GAAG,yBAAyB,CAAC,MAAM,EAAE,yBAAyB,EAAE,CAAC,CAAC;IAC1F,uEAAuE;IACvE,uEAAuE;IACvE,qEAAqE;IACrE,uEAAuE;IACvE,0EAA0E;IAC1E,mEAAmE;IACnE,yDAAyD;IACzD,MAAM,QAAQ,GAAG,4BAA4B,CAC3C,sBAAsB,CAAC,gBAAgB,CAAC,YAAY,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC,EACjF,YAAY,CACb,CAAC;IAEF,MAAM,eAAe,GAAG,MAAM,WAAW,CAAC;QACxC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,yBAAyB,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE;QACrI,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,WAAW,EAAE,QAAQ,KAAK,gBAAgB,YAAY,wCAAwC;QAC9F,KAAK,EAAE,CAAC,GAAG,iBAAiB,EAAE,GAAG,gBAAgB,CAAC,QAAQ,EAAE,CAAC;QAC7D,QAAQ;QACR,OAAO,EAAE,gBAAgB;QACzB,MAAM;QACN,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;KAC1C,CAAC,CAAC;IAEH,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;AAC7C,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// get_test_results reads the run matrix appq's own automatic pipeline
|
|
2
|
+
// writes to (fired by create_run, fanned out over SQS by workers/) --
|
|
3
|
+
// there's no blocking/webhook API for it, so this is poll-until-settled,
|
|
4
|
+
// not a single call. Only used for TCs this app didn't itself judge (their
|
|
5
|
+
// deterministic-only result still needs to show up in the consolidated
|
|
6
|
+
// report) -- agentically-covered TCs already got their verdict from the
|
|
7
|
+
// validator directly.
|
|
8
|
+
export const DEFAULT_POLL_INTERVAL_MS = 5000;
|
|
9
|
+
export async function pollTestResults(client, args) {
|
|
10
|
+
const interval = args.intervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
11
|
+
const deadline = Date.now() + args.timeoutMs;
|
|
12
|
+
const found = new Map();
|
|
13
|
+
while (Date.now() < deadline) {
|
|
14
|
+
const result = await client.callTool('get_test_results', {
|
|
15
|
+
run_id: args.runId,
|
|
16
|
+
...(args.scenarioId ? { scenario_id: args.scenarioId } : {}),
|
|
17
|
+
});
|
|
18
|
+
if (result.ok) {
|
|
19
|
+
try {
|
|
20
|
+
const parsed = JSON.parse(result.text);
|
|
21
|
+
for (const r of parsed.results ?? []) {
|
|
22
|
+
if (args.wantUuids.has(r.uuid) && r.status) {
|
|
23
|
+
found.set(r.uuid, { uuid: r.uuid, status: r.status, errorMessage: r.error_message });
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
// Non-JSON or unexpected shape — treat as "not settled yet" and keep polling.
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (found.size >= args.wantUuids.size)
|
|
32
|
+
return found;
|
|
33
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
34
|
+
}
|
|
35
|
+
return found; // whatever settled before the timeout — caller reports the rest as "pending"
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=pollResults.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pollResults.js","sourceRoot":"","sources":["../../src/orchestrator/pollResults.ts"],"names":[],"mappings":"AAAA,sEAAsE;AACtE,sEAAsE;AACtE,yEAAyE;AACzE,2EAA2E;AAC3E,uEAAuE;AACvE,wEAAwE;AACxE,sBAAsB;AAItB,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAQ7C,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAiB,EACjB,IAQC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,wBAAwB,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE1C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,kBAAkB,EAAE;YACvD,MAAM,EAAE,IAAI,CAAC,KAAK;YAClB,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7D,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YACd,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAkF,CAAC;gBACxH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;oBACrC,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;wBAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC;oBACvF,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8EAA8E;YAChF,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACpD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,6EAA6E;AAC7F,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// create_defect's `browser` field is LLM-composed prose — the validator has
|
|
2
|
+
// no access to the actual runtime, so it can only ever guess a bare engine
|
|
3
|
+
// name ("Chromium") with no version, which is a real fact this app already
|
|
4
|
+
// knows and shouldn't leave to a guess. Same reasoning as the dry-run/
|
|
5
|
+
// destructive-action gates: enforce facts in code, not prompts.
|
|
6
|
+
/** "133.0.6943.16" (Playwright's Browser.version()) -> "Chromium 133". */
|
|
7
|
+
export function formatBrowserLabel(versionString) {
|
|
8
|
+
const major = versionString.split('.')[0];
|
|
9
|
+
return `Chromium ${major}`;
|
|
10
|
+
}
|
|
11
|
+
/** Overrides whatever `browser` value the model passed to create_defect with the real one. */
|
|
12
|
+
export function createBrowserLabelDispatcher(inner, browserLabel) {
|
|
13
|
+
return async (name, args) => {
|
|
14
|
+
if (name === 'create_defect')
|
|
15
|
+
return inner(name, { ...args, browser: browserLabel });
|
|
16
|
+
return inner(name, args);
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=browserLabel.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browserLabel.js","sourceRoot":"","sources":["../../src/tools/browserLabel.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,2EAA2E;AAC3E,2EAA2E;AAC3E,uEAAuE;AACvE,gEAAgE;AAIhE,0EAA0E;AAC1E,MAAM,UAAU,kBAAkB,CAAC,aAAqB;IACtD,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,YAAY,KAAK,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,4BAA4B,CAAC,KAAqB,EAAE,YAAoB;IACtF,OAAO,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAC1B,IAAI,IAAI,KAAK,eAAe;YAAE,OAAO,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QACrF,OAAO,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// --dry-run support: computes verdicts normally but suppresses the actual
|
|
2
|
+
// writeback. Implemented as a dispatch-level intercept, not a prompt
|
|
3
|
+
// instruction — the validator's own workflow prose is what decides to call
|
|
4
|
+
// update_run_results/create_defect, so "please don't write" can't be a
|
|
5
|
+
// prompt-level ask without also changing what the workflow is allowed to
|
|
6
|
+
// do. Intercepting the call itself is the same enforcement pattern as the
|
|
7
|
+
// destructive-action gate and mandatory image check: code-level, not
|
|
8
|
+
// model-compliance-dependent.
|
|
9
|
+
const VERDICT_WRITE_TOOLS = new Set(['update_run_results', 'create_defect']);
|
|
10
|
+
export function createDryRunDispatcher(inner, dryRun) {
|
|
11
|
+
if (!dryRun)
|
|
12
|
+
return inner;
|
|
13
|
+
return async (name, args) => {
|
|
14
|
+
if (!VERDICT_WRITE_TOOLS.has(name))
|
|
15
|
+
return inner(name, args);
|
|
16
|
+
console.error(`[dry-run] would call ${name} with: ${JSON.stringify(args, null, 2)}`);
|
|
17
|
+
return {
|
|
18
|
+
ok: true,
|
|
19
|
+
text: `[dry-run] ${name} suppressed — no write happened. Args were logged for review, not sent to appq.`,
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=dryRun.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dryRun.js","sourceRoot":"","sources":["../../src/tools/dryRun.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,qEAAqE;AACrE,2EAA2E;AAC3E,uEAAuE;AACvE,yEAAyE;AACzE,0EAA0E;AAC1E,qEAAqE;AACrE,8BAA8B;AAI9B,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,oBAAoB,EAAE,eAAe,CAAC,CAAC,CAAC;AAE7E,MAAM,UAAU,sBAAsB,CACpC,KAA2E,EAC3E,MAAe;IAEf,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAE1B,OAAO,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAC1B,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAE7D,OAAO,CAAC,KAAK,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QACrF,OAAO;YACL,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,aAAa,IAAI,iFAAiF;SACzG,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// The one hardcoded client-side invariant (see plan doc, "The one hardcoded
|
|
2
|
+
// client-side invariant: write-tool reachability"). This app's own domain
|
|
3
|
+
// knowledge of which appq tools each stage may touch — the enforcement
|
|
4
|
+
// mechanism itself (assertToolAllowed / the gated dispatcher) lives in
|
|
5
|
+
// @appliqation/agent-core, shared with every sibling agent; only the
|
|
6
|
+
// allowlist *content* is local, since it's specific to this app's
|
|
7
|
+
// executor/validator split. A served prompt can never widen these sets.
|
|
8
|
+
// Read-only appq MCP tools — safe for both the executor and validator stages.
|
|
9
|
+
export const READONLY_APPQ_TOOLS = new Set([
|
|
10
|
+
'get_accessibility_report',
|
|
11
|
+
'get_analytics',
|
|
12
|
+
'get_coverage_analysis',
|
|
13
|
+
'get_defect_context',
|
|
14
|
+
'get_defects',
|
|
15
|
+
'get_evidence_summary',
|
|
16
|
+
'get_failure_patterns',
|
|
17
|
+
'get_project_settings',
|
|
18
|
+
'get_project_test_data',
|
|
19
|
+
'get_quality_context',
|
|
20
|
+
'get_run_evidence',
|
|
21
|
+
'get_execution_evidence',
|
|
22
|
+
'get_scenario',
|
|
23
|
+
'get_test_results',
|
|
24
|
+
'get_test_set',
|
|
25
|
+
'get_validation_targets',
|
|
26
|
+
'get_automation_readiness',
|
|
27
|
+
'list_projects',
|
|
28
|
+
'list_scenarios',
|
|
29
|
+
'list_test_sets',
|
|
30
|
+
'search_tests',
|
|
31
|
+
'start_workflow',
|
|
32
|
+
]);
|
|
33
|
+
// The one write tool the executor stage may call: observational only, never
|
|
34
|
+
// a verdict.
|
|
35
|
+
export const EXECUTOR_WRITE_TOOL = 'submit_execution_evidence';
|
|
36
|
+
// Verdict-bearing / mutating tools. Reachable only from the validator stage's
|
|
37
|
+
// tool palette, and only after its judgment phases — never from the executor.
|
|
38
|
+
export const VALIDATOR_ONLY_APPQ_TOOLS = new Set([
|
|
39
|
+
'update_run_results',
|
|
40
|
+
'create_defect',
|
|
41
|
+
'commit_validated_script',
|
|
42
|
+
'export_to_automation',
|
|
43
|
+
'create_scenario',
|
|
44
|
+
'add_test_cases',
|
|
45
|
+
'update_test_cases',
|
|
46
|
+
'update_scenario',
|
|
47
|
+
'run_tests',
|
|
48
|
+
'create_test_set',
|
|
49
|
+
'add_to_test_set',
|
|
50
|
+
'remove_from_test_set',
|
|
51
|
+
'explore',
|
|
52
|
+
'generate_tests_from_code',
|
|
53
|
+
'enrich_project_context', // has a write action variant; treat the whole tool as gated
|
|
54
|
+
]);
|
|
55
|
+
export function executorAllowedAppqTools() {
|
|
56
|
+
return new Set([...READONLY_APPQ_TOOLS, EXECUTOR_WRITE_TOOL]);
|
|
57
|
+
}
|
|
58
|
+
export function validatorAllowedAppqTools() {
|
|
59
|
+
return new Set([...READONLY_APPQ_TOOLS, ...VALIDATOR_ONLY_APPQ_TOOLS]);
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=safety.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safety.js","sourceRoot":"","sources":["../../src/tools/safety.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,0EAA0E;AAC1E,uEAAuE;AACvE,uEAAuE;AACvE,qEAAqE;AACrE,kEAAkE;AAClE,wEAAwE;AAExE,8EAA8E;AAC9E,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IACzC,0BAA0B;IAC1B,eAAe;IACf,uBAAuB;IACvB,oBAAoB;IACpB,aAAa;IACb,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,uBAAuB;IACvB,qBAAqB;IACrB,kBAAkB;IAClB,wBAAwB;IACxB,cAAc;IACd,kBAAkB;IAClB,cAAc;IACd,wBAAwB;IACxB,0BAA0B;IAC1B,eAAe;IACf,gBAAgB;IAChB,gBAAgB;IAChB,cAAc;IACd,gBAAgB;CACjB,CAAC,CAAC;AAEH,4EAA4E;AAC5E,aAAa;AACb,MAAM,CAAC,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;AAE/D,8EAA8E;AAC9E,8EAA8E;AAC9E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC;IAC/C,oBAAoB;IACpB,eAAe;IACf,yBAAyB;IACzB,sBAAsB;IACtB,iBAAiB;IACjB,gBAAgB;IAChB,mBAAmB;IACnB,iBAAiB;IACjB,WAAW;IACX,iBAAiB;IACjB,iBAAiB;IACjB,sBAAsB;IACtB,SAAS;IACT,0BAA0B;IAC1B,wBAAwB,EAAE,4DAA4D;CACvF,CAAC,CAAC;AAEH,MAAM,UAAU,wBAAwB;IACtC,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,mBAAmB,EAAE,mBAAmB,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,yBAAyB,CAAC,CAAC,CAAC;AACzE,CAAC"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// Turns get_execution_evidence's screenshot_url (a string sitting in text,
|
|
2
|
+
// never actually seen by the model) into a real vision input. Two modes,
|
|
3
|
+
// both driven by orchestration config (see cli/index.ts's
|
|
4
|
+
// --mandatory-image-check), never by the workflow prompt itself:
|
|
5
|
+
//
|
|
6
|
+
// - on-demand (default): the model gets a `view_screenshot(step_index)`
|
|
7
|
+
// tool and decides for itself when text evidence (accessibility
|
|
8
|
+
// snapshot, console/network deltas) isn't enough to judge a step —
|
|
9
|
+
// most functional assertions don't need pixels at all.
|
|
10
|
+
// - mandatory: every step's screenshot is fetched and attached inline the
|
|
11
|
+
// moment get_execution_evidence returns, unconditionally. This is
|
|
12
|
+
// enforced in code, not requested via the prompt — same reasoning as
|
|
13
|
+
// the destructive-action gate: a "please look at every screenshot"
|
|
14
|
+
// instruction depends on model compliance, an eager fetch doesn't.
|
|
15
|
+
export const VIEW_SCREENSHOT_TOOL = {
|
|
16
|
+
name: 'view_screenshot',
|
|
17
|
+
description: "Load a specific step's screenshot as an actual image so you can visually inspect it. Use this when a " +
|
|
18
|
+
"step's expected_result depends on something the accessibility snapshot and console/network deltas can't " +
|
|
19
|
+
'establish on their own — colour, layout, visual rendering. Most functional assertions (element present, ' +
|
|
20
|
+
'correct text, field has a value) are already verifiable from that text evidence and do not need this.',
|
|
21
|
+
inputSchema: {
|
|
22
|
+
type: 'object',
|
|
23
|
+
properties: {
|
|
24
|
+
step_index: { type: 'number', description: "The step to view, from get_execution_evidence's response" },
|
|
25
|
+
},
|
|
26
|
+
required: ['step_index'],
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
async function fetchImageAsBase64(url) {
|
|
30
|
+
const res = await fetch(url);
|
|
31
|
+
if (!res.ok)
|
|
32
|
+
throw new Error(`Failed to fetch screenshot: HTTP ${res.status}`);
|
|
33
|
+
const mimeType = res.headers.get('content-type') || 'image/png';
|
|
34
|
+
const buffer = Buffer.from(await res.arrayBuffer());
|
|
35
|
+
return { data: buffer.toString('base64'), mimeType };
|
|
36
|
+
}
|
|
37
|
+
export class ScreenshotViewer {
|
|
38
|
+
mandatory;
|
|
39
|
+
stepUrls = new Map();
|
|
40
|
+
constructor(mandatory) {
|
|
41
|
+
this.mandatory = mandatory;
|
|
42
|
+
}
|
|
43
|
+
/** Tool defs to add to the validator's palette — empty in mandatory mode, since there's nothing to ask for. */
|
|
44
|
+
toolDefs() {
|
|
45
|
+
return this.mandatory ? [] : [VIEW_SCREENSHOT_TOOL];
|
|
46
|
+
}
|
|
47
|
+
/** Wraps a dispatcher: handles view_screenshot itself, and intercepts get_execution_evidence to cache/attach. */
|
|
48
|
+
wrapDispatch(inner) {
|
|
49
|
+
return async (name, args) => {
|
|
50
|
+
if (name === 'view_screenshot') {
|
|
51
|
+
return this.viewScreenshot(Number(args.step_index));
|
|
52
|
+
}
|
|
53
|
+
const result = await inner(name, args);
|
|
54
|
+
if (name !== 'get_execution_evidence' || !result.ok)
|
|
55
|
+
return result;
|
|
56
|
+
return this.interceptEvidence(result);
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
async interceptEvidence(result) {
|
|
60
|
+
let parsed;
|
|
61
|
+
try {
|
|
62
|
+
parsed = JSON.parse(result.text);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return result;
|
|
66
|
+
}
|
|
67
|
+
const steps = parsed.steps ?? [];
|
|
68
|
+
for (const step of steps) {
|
|
69
|
+
if (typeof step.step_index === 'number' && step.screenshot_url) {
|
|
70
|
+
this.stepUrls.set(step.step_index, step.screenshot_url);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (!this.mandatory)
|
|
74
|
+
return result;
|
|
75
|
+
const images = [];
|
|
76
|
+
for (const step of steps) {
|
|
77
|
+
if (typeof step.step_index !== 'number' || !step.screenshot_url)
|
|
78
|
+
continue;
|
|
79
|
+
try {
|
|
80
|
+
const img = await fetchImageAsBase64(step.screenshot_url);
|
|
81
|
+
images.push({ ...img, label: `step ${step.step_index} screenshot` });
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// Non-fatal — the step's text evidence (accessibility snapshot,
|
|
85
|
+
// console/network) still stands on its own; one screenshot failing
|
|
86
|
+
// to load shouldn't fail the whole evidence fetch.
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return { ...result, images };
|
|
90
|
+
}
|
|
91
|
+
async viewScreenshot(stepIndex) {
|
|
92
|
+
const url = this.stepUrls.get(stepIndex);
|
|
93
|
+
if (!url) {
|
|
94
|
+
return {
|
|
95
|
+
ok: false,
|
|
96
|
+
text: `No screenshot URL known for step ${stepIndex} — call get_execution_evidence first.`,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
const img = await fetchImageAsBase64(url);
|
|
101
|
+
return {
|
|
102
|
+
ok: true,
|
|
103
|
+
text: `Screenshot for step ${stepIndex} attached below.`,
|
|
104
|
+
images: [{ ...img, label: `step ${stepIndex} screenshot` }],
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
return { ok: false, text: `Failed to load screenshot for step ${stepIndex}: ${err.message}` };
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
//# sourceMappingURL=screenshotViewer.js.map
|