@post-print/agent-test 2.3.8 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/sdk/baselines.d.ts +7 -0
- package/dist/sdk/baselines.d.ts.map +1 -0
- package/dist/sdk/baselines.js +40 -0
- package/dist/sdk/baselines.js.map +1 -0
- package/dist/sdk/claims.d.ts +65 -0
- package/dist/sdk/claims.d.ts.map +1 -0
- package/dist/sdk/claims.js +156 -0
- package/dist/sdk/claims.js.map +1 -0
- package/dist/sdk/cli.d.ts +2 -0
- package/dist/sdk/cli.d.ts.map +1 -1
- package/dist/sdk/cli.js +26 -1
- package/dist/sdk/cli.js.map +1 -1
- package/dist/sdk/definitions.d.ts.map +1 -1
- package/dist/sdk/definitions.js +1 -1
- package/dist/sdk/definitions.js.map +1 -1
- package/dist/sdk/environment-observation.d.ts +38 -0
- package/dist/sdk/environment-observation.d.ts.map +1 -0
- package/dist/sdk/environment-observation.js +108 -0
- package/dist/sdk/environment-observation.js.map +1 -0
- package/dist/sdk/execution-reporter.d.ts.map +1 -1
- package/dist/sdk/execution-reporter.js +6 -0
- package/dist/sdk/execution-reporter.js.map +1 -1
- package/dist/sdk/execution-runner.d.ts +5 -0
- package/dist/sdk/execution-runner.d.ts.map +1 -1
- package/dist/sdk/execution-runner.js +43 -7
- package/dist/sdk/execution-runner.js.map +1 -1
- package/dist/sdk/execution-store.d.ts +12 -0
- package/dist/sdk/execution-store.d.ts.map +1 -1
- package/dist/sdk/execution-store.js +23 -1
- package/dist/sdk/execution-store.js.map +1 -1
- package/dist/sdk/expect.d.ts +15 -1
- package/dist/sdk/expect.d.ts.map +1 -1
- package/dist/sdk/expect.js +38 -3
- package/dist/sdk/expect.js.map +1 -1
- package/dist/sdk/failure.d.ts +1 -1
- package/dist/sdk/failure.d.ts.map +1 -1
- package/dist/sdk/failure.js +4 -4
- package/dist/sdk/failure.js.map +1 -1
- package/dist/sdk/judge.d.ts.map +1 -1
- package/dist/sdk/judge.js +11 -7
- package/dist/sdk/judge.js.map +1 -1
- package/dist/sdk/operation-ledger.d.ts +25 -0
- package/dist/sdk/operation-ledger.d.ts.map +1 -0
- package/dist/sdk/operation-ledger.js +72 -0
- package/dist/sdk/operation-ledger.js.map +1 -0
- package/dist/sdk/pass-rate.d.ts +57 -0
- package/dist/sdk/pass-rate.d.ts.map +1 -0
- package/dist/sdk/pass-rate.js +113 -0
- package/dist/sdk/pass-rate.js.map +1 -0
- package/dist/sdk/run-record.d.ts +76 -0
- package/dist/sdk/run-record.d.ts.map +1 -0
- package/dist/sdk/run-record.js +152 -0
- package/dist/sdk/run-record.js.map +1 -0
- package/dist/sdk/runtime.d.ts +11 -0
- package/dist/sdk/runtime.d.ts.map +1 -1
- package/dist/sdk/runtime.js +56 -14
- package/dist/sdk/runtime.js.map +1 -1
- package/dist/sdk/sampling.d.ts +23 -0
- package/dist/sdk/sampling.d.ts.map +1 -0
- package/dist/sdk/sampling.js +55 -0
- package/dist/sdk/sampling.js.map +1 -0
- package/dist/sdk/test.d.ts +7 -2
- package/dist/sdk/test.d.ts.map +1 -1
- package/dist/sdk/test.js +32 -3
- package/dist/sdk/test.js.map +1 -1
- package/dist/sdk/types.d.ts +14 -0
- package/dist/sdk/types.d.ts.map +1 -1
- package/dist/sdk/verdict.d.ts +43 -0
- package/dist/sdk/verdict.d.ts.map +1 -0
- package/dist/sdk/verdict.js +195 -0
- package/dist/sdk/verdict.js.map +1 -0
- package/dist/viewer/client-entry.js +24 -23
- package/dist/viewer/presentation.d.ts +4 -0
- package/dist/viewer/presentation.d.ts.map +1 -1
- package/dist/viewer/presentation.js +17 -0
- package/dist/viewer/presentation.js.map +1 -1
- package/dist/viewer/run-inbox.d.ts.map +1 -1
- package/dist/viewer/run-inbox.js +7 -0
- package/dist/viewer/run-inbox.js.map +1 -1
- package/dist/viewer/v2-client.d.ts.map +1 -1
- package/dist/viewer/v2-client.js +13 -2
- package/dist/viewer/v2-client.js.map +1 -1
- package/dist/viewer/viewer-queries.d.ts +2 -0
- package/dist/viewer/viewer-queries.d.ts.map +1 -1
- package/dist/viewer/viewer-queries.js.map +1 -1
- package/package.json +6 -6
package/dist/sdk/test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test.js","sourceRoot":"","sources":["../../src/sdk/test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EACN,IAAI,IAAI,IAAI,EAEZ,YAAY,IAAI,gBAAgB,GAEhC,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAsB,SAAS,EAAqC,MAAM,kBAAkB,CAAC;AACpG,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"test.js","sourceRoot":"","sources":["../../src/sdk/test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EACN,IAAI,IAAI,IAAI,EAEZ,YAAY,IAAI,gBAAgB,GAEhC,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAsB,SAAS,EAAqC,MAAM,kBAAkB,CAAC;AACpG,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EACN,cAAc,EACd,YAAY,EACZ,iBAAiB,EAEjB,gBAAgB,GAChB,MAAM,eAAe,CAAC;AAQvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAsC;IAC/D,eAAe,EAAE,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC9C,eAAe,EAAE,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC9C,eAAe,EAAE,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACxC,OAAO,EAAE,KAAK,EAAE,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACnF,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9F,IAAI,CAAC;YACJ,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;gBAAS,CAAC;YACV,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACF,CAAC;CACD,CAAC,CAAC;AACH,KAAK,UAAU,YAAY,CAAC,QAAkB,EAAE,IAAc;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAC9C,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IACzC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC3B,IAAI,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC;QAC/B,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;QACvF,SAAS;QACT,KAAK,EAAE,QAAQ,CAAC,eAAe;QAC/B,KAAK,EAAE,QAAQ,CAAC,eAAe;QAC/B,SAAS,EAAE,QAAQ,CAAC,eAAe;QACnC,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC,MAAM;QACpC,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/F,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAClB,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3E,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,GAAG;gBAC1F,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC;KACD,CAAC,CAAC;IACH,OAAO;QACN,OAAO;QACP,KAAK,EAAE,KAAK,IAAI,EAAE;YACjB,IAAI,CAAC;gBACJ,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;oBAAS,CAAC;gBACV,MAAM,MAAM,CAAC;gBACb,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,sBAAsB,EAAE,CAAC,CAAC;YAClF,CAAC;QACF,CAAC;KACD,CAAC;AACH,CAAC;AA0BD,SAAS,IAAI,CAAsB,SAAY,EAAE,OAAoB;IACpE,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;QACnD,IAAI;QACJ,QAAQ,CAAC,IAAI,KAAK,OAAO;YACxB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC;YAC9D,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;KACzC,CAAC,CACa,CAAC;AAClB,CAAC;AAaD,MAAM,UAAU,QAAQ,CACvB,IAAY,EACZ,eAAkD,EAClD,WAAgC;IAEhC,MAAM,MAAM,GAAG,OAAO,eAAe,KAAK,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,WAAW,CAAC;IACrF,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,OAAO,eAAe,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC;IAClF,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IACpC,IACC,CAAC,SAAS;QACV,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAC5B,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CACtE;QAED,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC7E,OAAO,CAAC,KAAK,EAAE,aAA2C,EAAE,IAAkB,EAAE,EAAE;QACjF,MAAM,QAAQ,GAAG,OAAO,aAAa,KAAK,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5E,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,OAAO,aAAa,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC;QAChF,MAAM,UAAU,GAAG;YAClB,GAAG,eAAe,CAAC,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC;YAChD,GAAG,gBAAgB,CAAC,KAAK,CAAC;SAC1B,CAAC;QACF,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;QAC1C,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,KAAK,SAAS,IAAI,OAAO,EAAE,QAAQ,KAAK,SAAS,CAAC;QAClF,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE;YAC1B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE;oBACzD,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;oBACnD,OAAO,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzC,CAAC,CAAC,CAAC;gBACH,OAAO;YACR,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC5D,MAAM,WAAW,GAAG;oBACnB,GAAG,UAAU;oBACb,EAAE,IAAI,EAAE,iBAAiB,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;iBAC5E,CAAC;gBACF,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE;oBAC9E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;oBAClE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,gDAAgD,CAAC,CAAC;oBACtE,IAAI,MAAM,GAAG,IAAI,CAAC;oBAClB,IAAI,CAAC;wBACJ,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;wBACjF,OAAO,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;wBACxC,MAAM,GAAG,KAAK,CAAC;oBAChB,CAAC;4BAAS,CAAC;wBACV,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;oBACtD,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAoB;IAC7C,OAAO,KAAK,EAAE,WAAW;QACxB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,8BAA8B,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QAC5E,CAAC,CAAC,EAAE,CAAC;AACP,CAAC;AAED,SAAS,eAAe,CACvB,SAAY,EACZ,IAAiB,EACjB,OAAwB;IAExB,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,EAAE,MAAM;QACzC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC;QACvB,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1C,OAAO;QACN;YACC,IAAI,EAAE,sBAAsB;YAC5B,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;SAC5E;QACD,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,CAAC,QAAQ,CAAC,MAAM;YAClB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1E,CAAC,CAAC,EAAE,CAAC;KACN,CAAC;AACH,CAAC;AAWD,SAAS,SAAS,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,EAAiB;IACrE,OAAO;QACN,GAAG,IAAI;QACP,GAAG,EAAE;YACJ,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD;KACD,CAAC;AACH,CAAC;AACD,MAAM,UAAU,YAAY,CAAC,MAAuB;IACnD,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IACrC,OAAO,gBAAgB,CAAC;QACvB,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,CAAC;QACV,OAAO,EAAE,CAAC;QACV,aAAa,EAAE,IAAI;QACnB,GAAG,SAAS,CAAC,IAAI,CAAC;QAClB,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC;KAClC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACxB,SAAY,EACZ,QAA8C;IAE9C,MAAM,KAAK,GAAG,QAAQ,IAAK,MAAM,CAAC,IAAI,CAAC,SAAS,CAAgC,CAAC;IACjF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;QACjE,OAAO,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACJ,CAAC;AACD,SAAS,qBAAqB,CAAC,IAAY,EAAE,QAAkB;IAC9D,MAAM,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC;IAC9B,OAAO;QACN,IAAI;QACJ,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,IAAI,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9E,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK;QACtD,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC/E,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC1F,SAAS,EAAE,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC9E,MAAM,EAAE,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;KACxE,CAAC;AACH,CAAC"}
|
package/dist/sdk/types.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import type { AgentCapabilities, AgentDefinition, AgentOptions, AgentTrace } from "@post-print/agent-harness";
|
|
2
2
|
import type { z } from "zod";
|
|
3
|
+
import type { EnvironmentObservation, ObserveEnvironment } from "./environment-observation.js";
|
|
4
|
+
import type { RunExposure, RunRecord } from "./run-record.js";
|
|
3
5
|
import type { StartingContext, Workspace, WorkspaceSnapshot } from "./workspace.js";
|
|
4
6
|
export interface AgentSettings extends AgentOptions {
|
|
5
7
|
agent?: AgentDefinition;
|
|
6
8
|
description?: string;
|
|
7
9
|
workspace?: string;
|
|
10
|
+
/** Record processes, listening ports, and named paths left behind after each run. */
|
|
11
|
+
observeEnvironment?: ObserveEnvironment;
|
|
8
12
|
}
|
|
9
13
|
export interface RunOptions extends Omit<AgentSettings, "agent" | "description"> {
|
|
10
14
|
prompt: string;
|
|
@@ -38,6 +42,11 @@ export interface Run {
|
|
|
38
42
|
usage: RunUsage;
|
|
39
43
|
capabilities: AgentCapabilities;
|
|
40
44
|
startingContext: StartingContext;
|
|
45
|
+
/** The conditions this run happened under. */
|
|
46
|
+
record: RunRecord;
|
|
47
|
+
/** What the agent could see it was offered. */
|
|
48
|
+
exposure: RunExposure;
|
|
49
|
+
environment?: EnvironmentObservation;
|
|
41
50
|
artifact: string;
|
|
42
51
|
workspace: {
|
|
43
52
|
root: string;
|
|
@@ -55,6 +64,11 @@ export interface Evaluation<T = unknown> {
|
|
|
55
64
|
output: T;
|
|
56
65
|
usage: RunUsage;
|
|
57
66
|
artifact: string;
|
|
67
|
+
/** Which model judged, because a verdict is only as stable as its model. */
|
|
68
|
+
model: {
|
|
69
|
+
requested?: string;
|
|
70
|
+
observed?: string;
|
|
71
|
+
};
|
|
58
72
|
}
|
|
59
73
|
export type WorkspaceSetup = (workspace: Workspace) => void | Promise<void>;
|
|
60
74
|
export interface AgentFixture {
|
package/dist/sdk/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/sdk/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,UAAU,EACV,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACpF,MAAM,WAAW,aAAc,SAAQ,YAAY;IAClD,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/sdk/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,UAAU,EACV,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,KAAK,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAC/F,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACpF,MAAM,WAAW,aAAc,SAAQ,YAAY;IAClD,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACxC;AACD,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,aAAa,EAAE,OAAO,GAAG,aAAa,CAAC;IAC/E,MAAM,EAAE,MAAM,CAAC;CACf;AACD,MAAM,WAAW,aAAa,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAC7D,SAAQ,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,CAAC,CAAC;CACV;AACD,MAAM,MAAM,SAAS,GAClB,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAChC,MAAM,WAAW,QAAQ;IACxB,MAAM,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACF;AACD,MAAM,WAAW,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,UAAU,CAAC;IAClB,YAAY,EAAE,UAAU,CAAC;IACzB,SAAS,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,QAAQ,CAAC;IAChB,YAAY,EAAE,iBAAiB,CAAC;IAChC,eAAe,EAAE,eAAe,CAAC;IACjC,8CAA8C;IAC9C,MAAM,EAAE,SAAS,CAAC;IAClB,+CAA+C;IAC/C,QAAQ,EAAE,WAAW,CAAC;IACtB,WAAW,CAAC,EAAE,sBAAsB,CAAC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,iBAAiB,CAAC;QAC3B,KAAK,EAAE,iBAAiB,CAAC;QACzB,YAAY,EAAE,MAAM,EAAE,CAAC;KACvB,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;CACpD;AACD,MAAM,WAAW,UAAU,CAAC,CAAC,GAAG,OAAO;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,CAAC,CAAC;IACV,KAAK,EAAE,QAAQ,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,4EAA4E;IAC5E,KAAK,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACjD;AACD,MAAM,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,SAAS,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAC5E,MAAM,WAAW,YAAY;IAC5B,KAAK,CAAC,OAAO,EAAE,cAAc,GAAG,YAAY,CAAC;IAC7C,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;CACvC;AACD,MAAM,WAAW,YAAY,CAAC,CAAC;IAC9B,GAAG,CAAC,OAAO,EAAE;QAAE,KAAK,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;CAC3D;AACD,MAAM,WAAW,UAAU;IAC1B,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACZ"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type BaselineRecord, type ClaimResult, type SampleAnnotation } from "./claims.js";
|
|
2
|
+
export type RunVerdict = "passed" | "measured" | "agent_failed" | "control_survived" | "infra_invalid" | "conditions_unknown" | "vacuous" | "flaky" | "skipped_only" | "interrupted" | "listed" | "instrument_error";
|
|
3
|
+
export interface VerdictAttempt {
|
|
4
|
+
testId?: string;
|
|
5
|
+
/** Project and title path, the identity shared by every run of one test. */
|
|
6
|
+
claim?: string;
|
|
7
|
+
sample?: SampleAnnotation;
|
|
8
|
+
retry: number;
|
|
9
|
+
status: "running" | "passed" | "failed" | "skipped" | "interrupted";
|
|
10
|
+
expectedFailure?: boolean;
|
|
11
|
+
/** A control that must fail passed, so its test cannot tell good from bad. */
|
|
12
|
+
survivedControl?: boolean;
|
|
13
|
+
errors: unknown[];
|
|
14
|
+
operations: {
|
|
15
|
+
data: unknown[];
|
|
16
|
+
}[];
|
|
17
|
+
}
|
|
18
|
+
export interface VerdictInput {
|
|
19
|
+
attempts: VerdictAttempt[];
|
|
20
|
+
interrupted: boolean;
|
|
21
|
+
listOnly: boolean;
|
|
22
|
+
/** The test runner's own exit code, checked against what the attempts show. */
|
|
23
|
+
runnerExitCode: number;
|
|
24
|
+
baselines?: Record<string, BaselineRecord>;
|
|
25
|
+
/** Report repeats from the command line as rates. Otherwise a failed repeat fails the run. */
|
|
26
|
+
measureRepeats?: boolean;
|
|
27
|
+
}
|
|
28
|
+
export interface VerdictResult {
|
|
29
|
+
verdict: RunVerdict;
|
|
30
|
+
exitCode: number;
|
|
31
|
+
tests: Record<string, RunVerdict>;
|
|
32
|
+
counts: {
|
|
33
|
+
passed: number;
|
|
34
|
+
total: number;
|
|
35
|
+
};
|
|
36
|
+
/** Tests run more than once, reported as rates. */
|
|
37
|
+
claims: ClaimResult[];
|
|
38
|
+
}
|
|
39
|
+
/** The one place a run's meaning is decided. The CLI exit code and the viewer both read this. */
|
|
40
|
+
export declare function reduceVerdict(input: VerdictInput): VerdictResult;
|
|
41
|
+
/** Plain words for a person. A count is a count of runs, never a claim of reliability. */
|
|
42
|
+
export declare function verdictSentence({ verdict, counts }: VerdictResult): string;
|
|
43
|
+
//# sourceMappingURL=verdict.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/sdk/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,KAAK,cAAc,EAEnB,KAAK,WAAW,EAEhB,KAAK,gBAAgB,EACrB,MAAM,aAAa,CAAC;AAQrB,MAAM,MAAM,UAAU,GACnB,QAAQ,GACR,UAAU,GACV,cAAc,GACd,kBAAkB,GAClB,eAAe,GACf,oBAAoB,GACpB,SAAS,GACT,OAAO,GACP,cAAc,GACd,aAAa,GACb,QAAQ,GACR,kBAAkB,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,aAAa,CAAC;IACpE,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,8EAA8E;IAC9E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,UAAU,EAAE;QAAE,IAAI,EAAE,OAAO,EAAE,CAAA;KAAE,EAAE,CAAC;CAClC;AACD,MAAM,WAAW,YAAY;IAC5B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,WAAW,EAAE,OAAO,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;IAClB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC3C,8FAA8F;IAC9F,cAAc,CAAC,EAAE,OAAO,CAAC;CACzB;AACD,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE,UAAU,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,mDAAmD;IACnD,MAAM,EAAE,WAAW,EAAE,CAAC;CACtB;AA6HD,iGAAiG;AACjG,wBAAgB,aAAa,CAAC,KAAK,EAAE,YAAY,GAAG,aAAa,CAiChE;AAgBD,0FAA0F;AAC1F,wBAAgB,eAAe,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,aAAa,GAAG,MAAM,CAM1E"}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { buildClaims, } from "./claims.js";
|
|
2
|
+
/** Ledger messages a test fails with when it did not earn a result. See operation-ledger.ts. */
|
|
3
|
+
const NOT_EARNED = /without a completed agent operation|were not awaited and did not complete/;
|
|
4
|
+
const INFRASTRUCTURE = /AgentInfrastructureError|Agent worker exited|Test timeout of \d+ms exceeded|Agent session cancelled/;
|
|
5
|
+
const CONDITIONS_UNKNOWN = /ConditionsUnknownError/;
|
|
6
|
+
const EXIT_CODES = {
|
|
7
|
+
passed: 0,
|
|
8
|
+
// A rate with no threshold to meet: nothing was asked of it, so nothing failed.
|
|
9
|
+
measured: 0,
|
|
10
|
+
// Listing tests is not a test run, so nothing failed.
|
|
11
|
+
listed: 0,
|
|
12
|
+
agent_failed: 1,
|
|
13
|
+
control_survived: 1,
|
|
14
|
+
infra_invalid: 2,
|
|
15
|
+
conditions_unknown: 2,
|
|
16
|
+
vacuous: 2,
|
|
17
|
+
flaky: 2,
|
|
18
|
+
skipped_only: 2,
|
|
19
|
+
interrupted: 2,
|
|
20
|
+
instrument_error: 2,
|
|
21
|
+
};
|
|
22
|
+
function errorText(attempt) {
|
|
23
|
+
return attempt.errors
|
|
24
|
+
.map((error) => error?.message ?? "")
|
|
25
|
+
.join("\n");
|
|
26
|
+
}
|
|
27
|
+
function operationReportedInfrastructure(attempt) {
|
|
28
|
+
return attempt.operations.some(({ data }) => data.some((item) => item?.kind === "infrastructure"));
|
|
29
|
+
}
|
|
30
|
+
function failedAttemptVerdict(attempt) {
|
|
31
|
+
const text = errorText(attempt);
|
|
32
|
+
if (CONDITIONS_UNKNOWN.test(text))
|
|
33
|
+
return "conditions_unknown";
|
|
34
|
+
if (NOT_EARNED.test(text))
|
|
35
|
+
return "vacuous";
|
|
36
|
+
if (INFRASTRUCTURE.test(text) || operationReportedInfrastructure(attempt))
|
|
37
|
+
return "infra_invalid";
|
|
38
|
+
return "agent_failed";
|
|
39
|
+
}
|
|
40
|
+
/** An expected failure only counts when a check failed, not when the host or runner did. */
|
|
41
|
+
function passedAttempt(attempt) {
|
|
42
|
+
if (attempt.status === "passed")
|
|
43
|
+
return true;
|
|
44
|
+
return (attempt.status === "failed" &&
|
|
45
|
+
attempt.expectedFailure === true &&
|
|
46
|
+
failedAttemptVerdict(attempt) === "agent_failed");
|
|
47
|
+
}
|
|
48
|
+
/** One test across all its retries. A pass after a failure stays visible as flaky. */
|
|
49
|
+
function testVerdict(attempts) {
|
|
50
|
+
const last = attempts.reduce((latest, attempt) => attempt.retry >= latest.retry ? attempt : latest);
|
|
51
|
+
if (last.status === "skipped")
|
|
52
|
+
return "skipped_only";
|
|
53
|
+
if (last.status === "interrupted" || last.status === "running")
|
|
54
|
+
return "interrupted";
|
|
55
|
+
if (last.survivedControl)
|
|
56
|
+
return "control_survived";
|
|
57
|
+
if (!passedAttempt(last))
|
|
58
|
+
return failedAttemptVerdict(last);
|
|
59
|
+
return attempts.every(passedAttempt) ? "passed" : "flaky";
|
|
60
|
+
}
|
|
61
|
+
/** The most serious first: a real agent failure outranks tests that have no verdict. */
|
|
62
|
+
const PRECEDENCE = [
|
|
63
|
+
"agent_failed",
|
|
64
|
+
"control_survived",
|
|
65
|
+
"interrupted",
|
|
66
|
+
"infra_invalid",
|
|
67
|
+
"conditions_unknown",
|
|
68
|
+
"vacuous",
|
|
69
|
+
"flaky",
|
|
70
|
+
"measured",
|
|
71
|
+
"passed",
|
|
72
|
+
"skipped_only",
|
|
73
|
+
];
|
|
74
|
+
function attemptsByTest(attempts) {
|
|
75
|
+
const grouped = new Map();
|
|
76
|
+
for (const attempt of attempts) {
|
|
77
|
+
const key = attempt.testId ?? "unknown";
|
|
78
|
+
grouped.set(key, [...(grouped.get(key) ?? []), attempt]);
|
|
79
|
+
}
|
|
80
|
+
return grouped;
|
|
81
|
+
}
|
|
82
|
+
/** The attempts and the runner must tell the same story, or neither can be believed. */
|
|
83
|
+
function runnerDisagrees(verdict, input, tolerated) {
|
|
84
|
+
if (EXIT_CODES[verdict] === 0 && input.runnerExitCode !== 0)
|
|
85
|
+
return !tolerated;
|
|
86
|
+
return EXIT_CODES[verdict] === 1 && input.runnerExitCode === 0;
|
|
87
|
+
}
|
|
88
|
+
function runVerdict(present, input, tolerated) {
|
|
89
|
+
if (input.listOnly)
|
|
90
|
+
return "listed";
|
|
91
|
+
if (input.interrupted)
|
|
92
|
+
return "interrupted";
|
|
93
|
+
const verdict = PRECEDENCE.find((candidate) => present.has(candidate)) ?? "skipped_only";
|
|
94
|
+
return runnerDisagrees(verdict, input, tolerated) ? "instrument_error" : verdict;
|
|
95
|
+
}
|
|
96
|
+
function sampleOutcome(attempt) {
|
|
97
|
+
if (attempt.status === "skipped")
|
|
98
|
+
return "not_run";
|
|
99
|
+
if (passedAttempt(attempt))
|
|
100
|
+
return "passed";
|
|
101
|
+
if (attempt.status !== "failed")
|
|
102
|
+
return "no_verdict";
|
|
103
|
+
return failedAttemptVerdict(attempt) === "agent_failed" ? "failed" : "no_verdict";
|
|
104
|
+
}
|
|
105
|
+
function attemptRecord(attempt) {
|
|
106
|
+
for (const { data } of attempt.operations)
|
|
107
|
+
for (const item of data) {
|
|
108
|
+
const record = item?.record;
|
|
109
|
+
if (record)
|
|
110
|
+
return record;
|
|
111
|
+
}
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
/** Sampled when declared, or when one title ran as several tests through a repeat. */
|
|
115
|
+
function sampledClaims(attempts) {
|
|
116
|
+
const testIds = new Map();
|
|
117
|
+
const sampled = new Set();
|
|
118
|
+
for (const { claim, sample, testId } of attempts) {
|
|
119
|
+
if (!claim)
|
|
120
|
+
continue;
|
|
121
|
+
if (sample)
|
|
122
|
+
sampled.add(claim);
|
|
123
|
+
testIds.set(claim, (testIds.get(claim) ?? new Set()).add(testId));
|
|
124
|
+
if ((testIds.get(claim)?.size ?? 0) > 1)
|
|
125
|
+
sampled.add(claim);
|
|
126
|
+
}
|
|
127
|
+
return sampled;
|
|
128
|
+
}
|
|
129
|
+
/** Asking for repeats is not asking for a rate: a gate using --repeat-each must still go red. */
|
|
130
|
+
function claimVerdict(claim, measureRepeats) {
|
|
131
|
+
if (claim.comparisonRefused)
|
|
132
|
+
return "conditions_unknown";
|
|
133
|
+
if (claim.decision === "no_verdict")
|
|
134
|
+
return "infra_invalid";
|
|
135
|
+
if (!claim.declared && !measureRepeats && claim.arms.some(({ failed }) => failed > 0))
|
|
136
|
+
return "agent_failed";
|
|
137
|
+
if (claim.decision === "flagged")
|
|
138
|
+
return "agent_failed";
|
|
139
|
+
if (claim.baseline.status === "same_conditions" && claim.baseline.regressed)
|
|
140
|
+
return "agent_failed";
|
|
141
|
+
return claim.decision === "clear" ? "passed" : "measured";
|
|
142
|
+
}
|
|
143
|
+
/** The one place a run's meaning is decided. The CLI exit code and the viewer both read this. */
|
|
144
|
+
export function reduceVerdict(input) {
|
|
145
|
+
const sampled = sampledClaims(input.attempts);
|
|
146
|
+
const isSampled = ({ claim }) => claim !== undefined && sampled.has(claim);
|
|
147
|
+
const samples = input.attempts.filter(isSampled);
|
|
148
|
+
const grouped = [...attemptsByTest(input.attempts.filter((attempt) => !isSampled(attempt)))];
|
|
149
|
+
const tests = Object.fromEntries(grouped.map(([testId, attempts]) => [testId, testVerdict(attempts)]));
|
|
150
|
+
const claims = buildClaims(samples.map((attempt) => ({
|
|
151
|
+
claim: attempt.claim ?? "",
|
|
152
|
+
sample: attempt.sample,
|
|
153
|
+
outcome: sampleOutcome(attempt),
|
|
154
|
+
record: attemptRecord(attempt),
|
|
155
|
+
})), input.baselines);
|
|
156
|
+
const ran = Object.values(tests).filter((value) => value !== "skipped_only");
|
|
157
|
+
const present = new Set([
|
|
158
|
+
...ran,
|
|
159
|
+
...Object.values(tests),
|
|
160
|
+
...claims.map((claim) => claimVerdict(claim, input.measureRepeats === true)),
|
|
161
|
+
]);
|
|
162
|
+
// Failed samples make the runner exit non-zero; the claim, not one sample, is the verdict.
|
|
163
|
+
const tolerated = samples.some((attempt) => attempt.status === "failed");
|
|
164
|
+
const verdict = runVerdict(present, input, tolerated);
|
|
165
|
+
return {
|
|
166
|
+
verdict,
|
|
167
|
+
exitCode: EXIT_CODES[verdict],
|
|
168
|
+
tests,
|
|
169
|
+
claims,
|
|
170
|
+
counts: { passed: ran.filter((value) => value === "passed").length, total: ran.length },
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
const SENTENCES = {
|
|
174
|
+
passed: "Passed",
|
|
175
|
+
measured: "Measured: rates are reported below, with nothing required of them",
|
|
176
|
+
agent_failed: "Failed: the agent did not meet a check",
|
|
177
|
+
control_survived: "Failed: a control that must fail passed, so its test cannot tell good from bad",
|
|
178
|
+
infra_invalid: "No verdict: the host or runner failed, so nothing is known about the agent",
|
|
179
|
+
conditions_unknown: "No verdict: this run cannot show what a test asked about",
|
|
180
|
+
vacuous: "No verdict: a test finished without earning its result",
|
|
181
|
+
flaky: "No single verdict: a test failed and then passed on retry",
|
|
182
|
+
skipped_only: "No verdict: no test ran",
|
|
183
|
+
interrupted: "No verdict: the run was interrupted",
|
|
184
|
+
listed: "Listed tests only",
|
|
185
|
+
instrument_error: "No verdict: the runner and the recorded attempts disagree",
|
|
186
|
+
};
|
|
187
|
+
/** Plain words for a person. A count is a count of runs, never a claim of reliability. */
|
|
188
|
+
export function verdictSentence({ verdict, counts }) {
|
|
189
|
+
if (verdict === "passed")
|
|
190
|
+
return `Passed, ${counts.passed} of ${counts.total} tests, one run each`;
|
|
191
|
+
if (verdict === "listed" || verdict === "skipped_only" || verdict === "measured")
|
|
192
|
+
return SENTENCES[verdict];
|
|
193
|
+
return `${SENTENCES[verdict]} (${counts.passed} of ${counts.total} tests passed)`;
|
|
194
|
+
}
|
|
195
|
+
//# sourceMappingURL=verdict.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/sdk/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,WAAW,GAIX,MAAM,aAAa,CAAC;AAErB,gGAAgG;AAChG,MAAM,UAAU,GAAG,2EAA2E,CAAC;AAC/F,MAAM,cAAc,GACnB,qGAAqG,CAAC;AACvG,MAAM,kBAAkB,GAAG,wBAAwB,CAAC;AAgDpD,MAAM,UAAU,GAA+B;IAC9C,MAAM,EAAE,CAAC;IACT,gFAAgF;IAChF,QAAQ,EAAE,CAAC;IACX,sDAAsD;IACtD,MAAM,EAAE,CAAC;IACT,YAAY,EAAE,CAAC;IACf,gBAAgB,EAAE,CAAC;IACnB,aAAa,EAAE,CAAC;IAChB,kBAAkB,EAAE,CAAC;IACrB,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,CAAC;IACR,YAAY,EAAE,CAAC;IACf,WAAW,EAAE,CAAC;IACd,gBAAgB,EAAE,CAAC;CACnB,CAAC;AAEF,SAAS,SAAS,CAAC,OAAuB;IACzC,OAAO,OAAO,CAAC,MAAM;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAE,KAAqC,EAAE,OAAO,IAAI,EAAE,CAAC;SACrE,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC;AACD,SAAS,+BAA+B,CAAC,OAAuB;IAC/D,OAAO,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAC3C,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAE,IAAiC,EAAE,IAAI,KAAK,gBAAgB,CAAC,CAClF,CAAC;AACH,CAAC;AACD,SAAS,oBAAoB,CAAC,OAAuB;IACpD,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,oBAAoB,CAAC;IAC/D,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,+BAA+B,CAAC,OAAO,CAAC;QAAE,OAAO,eAAe,CAAC;IAClG,OAAO,cAAc,CAAC;AACvB,CAAC;AACD,4FAA4F;AAC5F,SAAS,aAAa,CAAC,OAAuB;IAC7C,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7C,OAAO,CACN,OAAO,CAAC,MAAM,KAAK,QAAQ;QAC3B,OAAO,CAAC,eAAe,KAAK,IAAI;QAChC,oBAAoB,CAAC,OAAO,CAAC,KAAK,cAAc,CAChD,CAAC;AACH,CAAC;AACD,sFAAsF;AACtF,SAAS,WAAW,CAAC,QAA0B;IAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAChD,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAChD,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,cAAc,CAAC;IACrD,IAAI,IAAI,CAAC,MAAM,KAAK,aAAa,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,aAAa,CAAC;IACrF,IAAI,IAAI,CAAC,eAAe;QAAE,OAAO,kBAAkB,CAAC;IACpD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC5D,OAAO,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;AAC3D,CAAC;AACD,wFAAwF;AACxF,MAAM,UAAU,GAAiB;IAChC,cAAc;IACd,kBAAkB;IAClB,aAAa;IACb,eAAe;IACf,oBAAoB;IACpB,SAAS;IACT,OAAO;IACP,UAAU;IACV,QAAQ;IACR,cAAc;CACd,CAAC;AACF,SAAS,cAAc,CAAC,QAA0B;IACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AACD,wFAAwF;AACxF,SAAS,eAAe,CAAC,OAAmB,EAAE,KAAmB,EAAE,SAAkB;IACpF,IAAI,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC;QAAE,OAAO,CAAC,SAAS,CAAC;IAC/E,OAAO,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,CAAC;AAChE,CAAC;AACD,SAAS,UAAU,CAAC,OAAwB,EAAE,KAAmB,EAAE,SAAkB;IACpF,IAAI,KAAK,CAAC,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,aAAa,CAAC;IAC5C,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,cAAc,CAAC;IACzF,OAAO,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC;AAClF,CAAC;AACD,SAAS,aAAa,CAAC,OAAuB;IAC7C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACnD,IAAI,aAAa,CAAC,OAAO,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC5C,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,YAAY,CAAC;IACrD,OAAO,oBAAoB,CAAC,OAAO,CAAC,KAAK,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC;AACnF,CAAC;AACD,SAAS,aAAa,CAAC,OAAuB;IAC7C,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,OAAO,CAAC,UAAU;QACxC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,MAAM,GAAI,IAAoD,EAAE,MAAM,CAAC;YAC7E,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC3B,CAAC;IACF,OAAO,SAAS,CAAC;AAClB,CAAC;AACD,sFAAsF;AACtF,SAAS,aAAa,CAAC,QAA0B;IAChD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC3D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAClD,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,IAAI,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AACD,iGAAiG;AACjG,SAAS,YAAY,CAAC,KAAkB,EAAE,cAAuB;IAChE,IAAI,KAAK,CAAC,iBAAiB;QAAE,OAAO,oBAAoB,CAAC;IACzD,IAAI,KAAK,CAAC,QAAQ,KAAK,YAAY;QAAE,OAAO,eAAe,CAAC;IAC5D,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,cAAc,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACpF,OAAO,cAAc,CAAC;IACvB,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,cAAc,CAAC;IACxD,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,iBAAiB,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS;QAC1E,OAAO,cAAc,CAAC;IACvB,OAAO,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC;AAC3D,CAAC;AACD,iGAAiG;AACjG,MAAM,UAAU,aAAa,CAAC,KAAmB;IAChD,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,CAAC,EAAE,KAAK,EAAkB,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3F,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CACpE,CAAC;IACF,MAAM,MAAM,GAAG,WAAW,CACzB,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC;QAC/B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC,EACH,KAAK,CAAC,SAAS,CACf,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,cAAc,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC;QACvB,GAAG,GAAG;QACN,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;QACvB,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC;KAC5E,CAAC,CAAC;IACH,2FAA2F;IAC3F,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IACtD,OAAO;QACN,OAAO;QACP,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC;QAC7B,KAAK;QACL,MAAM;QACN,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;KACvF,CAAC;AACH,CAAC;AACD,MAAM,SAAS,GAA+B;IAC7C,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,mEAAmE;IAC7E,YAAY,EAAE,wCAAwC;IACtD,gBAAgB,EACf,gFAAgF;IACjF,aAAa,EAAE,4EAA4E;IAC3F,kBAAkB,EAAE,0DAA0D;IAC9E,OAAO,EAAE,wDAAwD;IACjE,KAAK,EAAE,2DAA2D;IAClE,YAAY,EAAE,yBAAyB;IACvC,WAAW,EAAE,qCAAqC;IAClD,MAAM,EAAE,mBAAmB;IAC3B,gBAAgB,EAAE,2DAA2D;CAC7E,CAAC;AACF,0FAA0F;AAC1F,MAAM,UAAU,eAAe,CAAC,EAAE,OAAO,EAAE,MAAM,EAAiB;IACjE,IAAI,OAAO,KAAK,QAAQ;QACvB,OAAO,WAAW,MAAM,CAAC,MAAM,OAAO,MAAM,CAAC,KAAK,sBAAsB,CAAC;IAC1E,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,cAAc,IAAI,OAAO,KAAK,UAAU;QAC/E,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,OAAO,MAAM,CAAC,KAAK,gBAAgB,CAAC;AACnF,CAAC"}
|