@hona/openeval 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/JUDGING.md +7 -5
- package/VERIFICATION.md +8 -1
- package/package.json +1 -1
- package/src/app/input-fingerprints.ts +9 -1
- package/src/app/load-benchmark.ts +29 -8
- package/src/app/run-eval.ts +4 -3
- package/src/index.ts +2 -1
- package/src/infra/judging/code-source.ts +45 -37
- package/src/infra/verification/image.ts +9 -1
- package/src/infra/verification/session.ts +1 -1
- package/src/judge-context.ts +5 -0
- package/src/types.ts +10 -3
- package/src/view.ts +29 -6
- package/viewer/assets/{abnfDiagram-VCTEODGH-DheibVLk.js → abnfDiagram-VCTEODGH-ug3iRKuR.js} +1 -1
- package/viewer/assets/{angular-html-4AdCdqA_.js → angular-html-BrDmDg4M.js} +1 -1
- package/viewer/assets/{angular-ts-D-G3eLWz.js → angular-ts-Cog8g0dp.js} +1 -1
- package/viewer/assets/{apl-Ckl95Nnu.js → apl-u0m73XJK.js} +1 -1
- package/viewer/assets/{arc-DBaUWWiS.js → arc-DwFAF4oI.js} +1 -1
- package/viewer/assets/architecture-7GRP2DOG-BE4yjtoN.js +1 -0
- package/viewer/assets/{architectureDiagram-5GKGNRK7-BSEYyvUG.js → architectureDiagram-5GKGNRK7-DQCcEnAE.js} +1 -1
- package/viewer/assets/{astro-BpiishHg.js → astro-Bh_DypAJ.js} +1 -1
- package/viewer/assets/{blade-DvRbLCOy.js → blade-BKpbbd0G.js} +1 -1
- package/viewer/assets/{blockDiagram-I7D4REHJ-DLxyKv07.js → blockDiagram-I7D4REHJ-CMWWJZCD.js} +1 -1
- package/viewer/assets/{c-BZYE5LIm.js → c-3D1Q4Rnv.js} +1 -1
- package/viewer/assets/{c4Diagram-7LVT6UL2--I50ySVl.js → c4Diagram-7LVT6UL2-T2XwgnDq.js} +1 -1
- package/viewer/assets/channel-LYhuLo7L.js +1 -0
- package/viewer/assets/{chapel-T-o4rT-z.js → chapel-C8bO2CrC.js} +1 -1
- package/viewer/assets/{chunk-4HAMMTFA-B9xd7TwX.js → chunk-4HAMMTFA-DP15Gd37.js} +1 -1
- package/viewer/assets/{chunk-75Z2AOVW-B4XIg4PJ.js → chunk-75Z2AOVW-BH0f-Wnb.js} +1 -1
- package/viewer/assets/{chunk-DU6HZSFF-Cty-qidK.js → chunk-DU6HZSFF-BI4SVpji.js} +1 -1
- package/viewer/assets/{chunk-F27PBJKO-8WDDKCG7.js → chunk-F27PBJKO-BSNNTPfe.js} +1 -1
- package/viewer/assets/{chunk-GMAD6QVW-G948aN1X.js → chunk-GMAD6QVW-WnVsWLvz.js} +1 -1
- package/viewer/assets/{chunk-GVQU2GXP-CVFpDQVF.js → chunk-GVQU2GXP-CdFmNEz4.js} +1 -1
- package/viewer/assets/{chunk-IMKFNOWR-BUQviXHJ.js → chunk-IMKFNOWR-Dvd33dQw.js} +1 -1
- package/viewer/assets/{chunk-L3NEJ4N5-BXvzxNPv.js → chunk-L3NEJ4N5-CL6o4wty.js} +1 -1
- package/viewer/assets/{chunk-OSK3NFVY-wLdUx9yD.js → chunk-OSK3NFVY-ITNjoAvY.js} +1 -1
- package/viewer/assets/{chunk-P2QGCYS3-hDwRspSg.js → chunk-P2QGCYS3-BSCEl9ld.js} +1 -1
- package/viewer/assets/{chunk-POPQ4Y6H-LbeiA8tL.js → chunk-POPQ4Y6H-cdNgvxKY.js} +1 -1
- package/viewer/assets/{chunk-PWAF6VOD-BcGZUKY4.js → chunk-PWAF6VOD-BMdmv8wx.js} +1 -1
- package/viewer/assets/{chunk-SHT3W25Y-BqKh1v-r.js → chunk-SHT3W25Y-CjnAASPR.js} +1 -1
- package/viewer/assets/{chunk-SVP7TREG-BEMJeRHW.js → chunk-SVP7TREG-CGuEYo-A.js} +1 -1
- package/viewer/assets/{chunk-TICWLB2K-BiHATE6U.js → chunk-TICWLB2K-Ctd8ESX_.js} +1 -1
- package/viewer/assets/{chunk-XXDRQBXY-B0X2aTwI.js → chunk-XXDRQBXY-CeNgfrFR.js} +1 -1
- package/viewer/assets/classDiagram-ZZMXUADV-BgxrVQRR.js +1 -0
- package/viewer/assets/classDiagram-v2-VYDZK3BY-BgxrVQRR.js +1 -0
- package/viewer/assets/{cobol-DS-OUv7W.js → cobol-osZcAF7j.js} +1 -1
- package/viewer/assets/{coffee-UJZBXrQS.js → coffee-3IUVSrSB.js} +1 -1
- package/viewer/assets/{cose-bilkent-JH36ORCC-h7Gx4dzS.js → cose-bilkent-JH36ORCC-CyD9he_v.js} +1 -1
- package/viewer/assets/{cpp-BuO_io74.js → cpp-CN2PLd-1.js} +1 -1
- package/viewer/assets/{crystal-D_Q6HPhm.js → crystal-DkrkhqdA.js} +1 -1
- package/viewer/assets/{css-B9bdOUvs.js → css-Y7OW5uKN.js} +1 -1
- package/viewer/assets/{cynefin-OW5HDTMX-BONm5P8S.js → cynefin-OW5HDTMX-H8lBmHMH.js} +1 -1
- package/viewer/assets/{cynefinDiagram-5FMLGOSQ-CWsyzZBO.js → cynefinDiagram-5FMLGOSQ-CJIYAkvT.js} +1 -1
- package/viewer/assets/{dagre-GXQ25YYZ-CJG_-oax.js → dagre-GXQ25YYZ-pxdbKP2Y.js} +1 -1
- package/viewer/assets/{diagram-S7CK7UJ4-uOGlO2Fe.js → diagram-S7CK7UJ4-VtFOpSmi.js} +1 -1
- package/viewer/assets/{diagram-UQ7AKVKN-B2mgymOu.js → diagram-UQ7AKVKN-D_P4AgV5.js} +1 -1
- package/viewer/assets/{diagram-VSXAHHWV-UkXvypTd.js → diagram-VSXAHHWV-DAGjorXD.js} +1 -1
- package/viewer/assets/{diagram-VX7I27RA-Bzge-YWf.js → diagram-VX7I27RA-2zCtxu12.js} +1 -1
- package/viewer/assets/{diagram-Z3DM3KII-BAqqEKF_.js → diagram-Z3DM3KII-EYZlAOgk.js} +1 -1
- package/viewer/assets/{dist-CACpyRbj.js → dist-GOYxPZ4F.js} +1 -1
- package/viewer/assets/{ebnfDiagram-PWID7BFC-dsRp8_0H.js → ebnfDiagram-PWID7BFC-DjPdLCtD.js} +1 -1
- package/viewer/assets/{edge-DUnvwjmM.js → edge-BXhOSzIo.js} +1 -1
- package/viewer/assets/{elixir-QfE-NI0D.js → elixir-7AwElWAC.js} +1 -1
- package/viewer/assets/{elm-D4J4FYyf.js → elm-Rl2m4djM.js} +1 -1
- package/viewer/assets/{erDiagram-RLTQ6QDP-Dmfq32IQ.js → erDiagram-RLTQ6QDP-SmyuIGbH.js} +1 -1
- package/viewer/assets/{erb-DfUVZzqX.js → erb-CaYBWLa-.js} +1 -1
- package/viewer/assets/eventmodeling-NTZA5JFV-mpkZTTpn.js +1 -0
- package/viewer/assets/flowDiagram-HODETNUW-BCQvK0Ku.js +1 -0
- package/viewer/assets/{ganttDiagram-EL5Y4UJY-DVTv_n_T.js → ganttDiagram-EL5Y4UJY-DfSG_mot.js} +1 -1
- package/viewer/assets/{git-rebase-B22-kUR2.js → git-rebase-Y999REzC.js} +1 -1
- package/viewer/assets/{gitGraph-4MIJSDKK-C3ZU1Sml.js → gitGraph-4MIJSDKK-BXiKSzAB.js} +1 -1
- package/viewer/assets/{gitGraphDiagram-WWUBYQGX-BJqeOFYd.js → gitGraphDiagram-WWUBYQGX-BouZhMNn.js} +1 -1
- package/viewer/assets/{glimmer-js-Cm5f8kCy.js → glimmer-js-DTFvFoOE.js} +1 -1
- package/viewer/assets/{glimmer-ts-CQ-nNZCl.js → glimmer-ts-RbXN3mhk.js} +1 -1
- package/viewer/assets/{glsl-XxfyrGb3.js → glsl-BhmvaPUj.js} +1 -1
- package/viewer/assets/{graphql-DBlG1DNQ.js → graphql-CohJYf5Y.js} +1 -1
- package/viewer/assets/{hack-Dkc1g494.js → hack-xLFm4gc4.js} +1 -1
- package/viewer/assets/{haml-D9E0-Hw0.js → haml-B433XQTv.js} +1 -1
- package/viewer/assets/{handlebars-Cb17q7C4.js → handlebars-BDpXFZR1.js} +1 -1
- package/viewer/assets/{html-5E3Hc3ha.js → html-BvwGTXL2.js} +1 -1
- package/viewer/assets/{html-derivative-DOZTqg-2.js → html-derivative-DSCM_5Uj.js} +1 -1
- package/viewer/assets/{http-DJH-LMLV.js → http-ZyduKjGN.js} +1 -1
- package/viewer/assets/{hurl-CEYg3T6-.js → hurl-Dtl73Mol.js} +1 -1
- package/viewer/assets/{index-CkY7ZfKs.css → index-BmSDOoQ0.css} +1 -1
- package/viewer/assets/{index-Dfdu9MKA.js → index-DOIiHLHw.js} +76 -76
- package/viewer/assets/{info-A6RAGUB7-B-4F1obM.js → info-A6RAGUB7-DuWuidi4.js} +1 -1
- package/viewer/assets/{infoDiagram-27XIBGKW-B8RIO086.js → infoDiagram-27XIBGKW-DczGtN5R.js} +1 -1
- package/viewer/assets/{ishikawaDiagram-5VMMS53U-DIGOldDh.js → ishikawaDiagram-5VMMS53U-DidQ3_7N.js} +1 -1
- package/viewer/assets/{java-e3vi9Uzs.js → java-mouNBEOT.js} +1 -1
- package/viewer/assets/{javascript-DXnYMYO7.js → javascript-U-JBzSDk.js} +1 -1
- package/viewer/assets/{jinja-C-Qo2832.js → jinja-DBlWVTqh.js} +1 -1
- package/viewer/assets/{jison-CfzTRAm0.js → jison-ELt1NYXo.js} +1 -1
- package/viewer/assets/{journeyDiagram-3NMN7TZE-DbQ5PKPQ.js → journeyDiagram-3NMN7TZE-0fr5OEH9.js} +1 -1
- package/viewer/assets/{json-BaqYW62L.js → json-B_LODAI7.js} +1 -1
- package/viewer/assets/{jsx-DTafsOf-.js → jsx-BohFPbbA.js} +1 -1
- package/viewer/assets/{julia-Djdi6Tch.js → julia-Dco09L8E.js} +1 -1
- package/viewer/assets/{just-gZxWcw47.js → just-Dpx2bwga.js} +1 -1
- package/viewer/assets/{kanban-definition-UXKFOSKX-CwEeXphJ.js → kanban-definition-UXKFOSKX-3FngVt7d.js} +1 -1
- package/viewer/assets/{latex-CIO1OdmU.js → latex-Dccz5tQA.js} +1 -1
- package/viewer/assets/{line-Basn7TUv.js → line-Dhwi7DC5.js} +1 -1
- package/viewer/assets/{linear-Bu88wzXh.js → linear-gHMHD-Wa.js} +1 -1
- package/viewer/assets/{liquid-mYuCSXLi.js → liquid-D8w3l38v.js} +1 -1
- package/viewer/assets/{lua-lXhbu8ma.js → lua-tuIjH_94.js} +1 -1
- package/viewer/assets/{marko-BH-anPb4.js → marko-W3rkETdp.js} +1 -1
- package/viewer/assets/{mdc-DOuUtwPr.js → mdc-OJ1zd1BA.js} +1 -1
- package/viewer/assets/{mermaid-parser.core-CiGT3tRM.js → mermaid-parser.core-VTfqiyyK.js} +3 -3
- package/viewer/assets/{mermaid.core-BfFqBx2V.js → mermaid.core-eoOgKmWC.js} +4 -4
- package/viewer/assets/{mindmap-definition-YA3MSWOX-d_TnVAJG.js → mindmap-definition-YA3MSWOX-DeA-tJsi.js} +1 -1
- package/viewer/assets/{nginx-BB1dPcHJ.js → nginx-lXyaNnvL.js} +1 -1
- package/viewer/assets/{nim-BSJ4UdAS.js → nim-SOay_4P1.js} +1 -1
- package/viewer/assets/{org-JRn8h-K_.js → org-CRmJq-uC.js} +1 -1
- package/viewer/assets/{packet-AYTQ26CC-BA7VIQ3l.js → packet-AYTQ26CC-QgHmAlwp.js} +1 -1
- package/viewer/assets/{pegDiagram-XKGWAZYB-pfsoYtx9.js → pegDiagram-XKGWAZYB-BP0VAXd_.js} +1 -1
- package/viewer/assets/{perl-GzZ7epP9.js → perl-CHndoS_5.js} +1 -1
- package/viewer/assets/{php-Crolu7lU.js → php-DZCOF2Hm.js} +1 -1
- package/viewer/assets/{pie-WAS4IAKB-B9gCbpVP.js → pie-WAS4IAKB-Lghc8UPt.js} +1 -1
- package/viewer/assets/{pieDiagram-E7YTZNPT-D578iJLj.js → pieDiagram-E7YTZNPT-_ZEa88u4.js} +1 -1
- package/viewer/assets/{pug-BRaBHOlA.js → pug-Bj-uu214.js} +1 -1
- package/viewer/assets/{qml-BBlfnxMr.js → qml-DLfThugB.js} +1 -1
- package/viewer/assets/{quadrantDiagram-AXDQQJYC-DoKeLBeZ.js → quadrantDiagram-AXDQQJYC-BLeHQnim.js} +1 -1
- package/viewer/assets/{r-DG1YsJtP.js → r-Cnkx2_Um.js} +1 -1
- package/viewer/assets/{radar-RG4KPBEZ-Dr6Zcwqy.js → radar-RG4KPBEZ-CLW3C4xB.js} +1 -1
- package/viewer/assets/{railroad-74A4TZTK-CEncWApU.js → railroad-74A4TZTK-DLtGvURT.js} +1 -1
- package/viewer/assets/railroad-abnf-HS5TGJTU-CMoBRU-d.js +1 -0
- package/viewer/assets/railroad-ebnf-LZEXJU2U-Cs-h2401.js +1 -0
- package/viewer/assets/railroad-peg-WCYAUIDC-Q-VU3IQG.js +1 -0
- package/viewer/assets/{railroadDiagram-O6MQD6OU-7j0Th0r6.js → railroadDiagram-O6MQD6OU-DocKmoio.js} +1 -1
- package/viewer/assets/{razor-CY_sYdwH.js → razor-OW6qQyun.js} +1 -1
- package/viewer/assets/{regexp-DZiZuF5z.js → regexp-6RZ3RRCf.js} +1 -1
- package/viewer/assets/{requirementDiagram-BXWQKSXE-Dk1vCdh-.js → requirementDiagram-BXWQKSXE-cEjmGoYY.js} +1 -1
- package/viewer/assets/{rst-Dn_AtwYd.js → rst-CpEVnW9I.js} +1 -1
- package/viewer/assets/{ruby-BIB6MqpW.js → ruby-B2fIwwQt.js} +1 -1
- package/viewer/assets/{sankeyDiagram-P5KCCOFB-DaB06Zgj.js → sankeyDiagram-P5KCCOFB-BUdtY7Wt.js} +1 -1
- package/viewer/assets/{sas-Ccxd8Air.js → sas-DCHEi6wl.js} +1 -1
- package/viewer/assets/{scss-Cr7j0d4P.js → scss-D2bGvizF.js} +1 -1
- package/viewer/assets/{sequenceDiagram-WJ2MYXX4-BGEb3NdT.js → sequenceDiagram-WJ2MYXX4-DmsKCV1w.js} +1 -1
- package/viewer/assets/{shellscript-CHN-dwU6.js → shellscript-dnMGBhYw.js} +1 -1
- package/viewer/assets/{shellsession-uVGeQNm3.js → shellsession-EXXCHS9B.js} +1 -1
- package/viewer/assets/{soy-D7NWwJ5L.js → soy-BhJTFcrF.js} +1 -1
- package/viewer/assets/{sql-DRImQ2Oi.js → sql-Bg9JbbKb.js} +1 -1
- package/viewer/assets/{src-BnSwVg9U.js → src-C171e0nu.js} +1 -1
- package/viewer/assets/{stata-BIe0AUpg.js → stata-CAxSMxW7.js} +1 -1
- package/viewer/assets/{stateDiagram-D77RDMKH-DsRu-B2h.js → stateDiagram-D77RDMKH-OFqeW2Tm.js} +1 -1
- package/viewer/assets/stateDiagram-v2-MP3YSRHH-BeZW_eXx.js +1 -0
- package/viewer/assets/{surrealql-LEEn9cBl.js → surrealql-ChhGZa01.js} +1 -1
- package/viewer/assets/{svelte-CAMB-f9H.js → svelte-nQiI4XLd.js} +1 -1
- package/viewer/assets/{swimlanes-42K2YHIH-DqXHORT2.js → swimlanes-42K2YHIH-DHYfKuk0.js} +1 -1
- package/viewer/assets/swimlanesDiagram-VR7AAH4N-BYZ-vs6I.js +8 -0
- package/viewer/assets/{templ-zrRSW6Nf.js → templ-QcYlpMb-.js} +1 -1
- package/viewer/assets/{tex-D4XEXDuZ.js → tex-DkJXeRDT.js} +1 -1
- package/viewer/assets/{timeline-definition-24CTP7MA-DSpHKRpt.js → timeline-definition-24CTP7MA-CyRzlxCL.js} +1 -1
- package/viewer/assets/{treeView-Q6P3EWNA-pE7RJ0LU.js → treeView-Q6P3EWNA-dNcA3Vfo.js} +1 -1
- package/viewer/assets/{treemap-WGGIJYW6-CBfU2tIc.js → treemap-WGGIJYW6-BZWPBgpp.js} +1 -1
- package/viewer/assets/{ts-tags-DqFdF1-k.js → ts-tags-Fro3huiC.js} +1 -1
- package/viewer/assets/{tsx-Dz5osFq9.js → tsx-Dpl8Dyu3.js} +1 -1
- package/viewer/assets/{twig-CQ3wxbq3.js → twig-V8RRy40u.js} +1 -1
- package/viewer/assets/{typescript-Cu1dPDHw.js → typescript-DkL9-LA1.js} +1 -1
- package/viewer/assets/{typst-DHrnWrov.js → typst-DlTx93FO.js} +1 -1
- package/viewer/assets/{vennDiagram-4TSXK5OY-BPPSR6nr.js → vennDiagram-4TSXK5OY-CzTHbS5I.js} +1 -1
- package/viewer/assets/{vue-BaoJW0em.js → vue-B3Y_KP0j.js} +1 -1
- package/viewer/assets/{vue-html-DB9LdXl9.js → vue-html-CU1oynDW.js} +1 -1
- package/viewer/assets/{vue-vine-CqtlBPYW.js → vue-vine-zaYTs9kI.js} +1 -1
- package/viewer/assets/{wardley-WFR3VGLG-DfCbsSvk.js → wardley-WFR3VGLG-BHfRoe2C.js} +1 -1
- package/viewer/assets/{wardleyDiagram-VM6X3IG4-DeiKcJbE.js → wardleyDiagram-VM6X3IG4-YhghL-ul.js} +1 -1
- package/viewer/assets/{xml-CQ5T6aNZ.js → xml-CnVdnzon.js} +1 -1
- package/viewer/assets/{xsl-DVQ2v5YX.js → xsl-B9rDiHW0.js} +1 -1
- package/viewer/assets/{xychartDiagram-S5SC5T6Z-BsvlKd7r.js → xychartDiagram-S5SC5T6Z-BwmCelEN.js} +1 -1
- package/viewer/assets/{yaml-DXrThoWS.js → yaml-CbhKf1fN.js} +1 -1
- package/viewer/index.html +2 -2
- package/viewer/assets/architecture-7GRP2DOG-DtdqvSTn.js +0 -1
- package/viewer/assets/channel-BksTOWku.js +0 -1
- package/viewer/assets/classDiagram-ZZMXUADV-Cbfu0NLs.js +0 -1
- package/viewer/assets/classDiagram-v2-VYDZK3BY-Cbfu0NLs.js +0 -1
- package/viewer/assets/eventmodeling-NTZA5JFV-BsZg3yrD.js +0 -1
- package/viewer/assets/flowDiagram-HODETNUW-55jB5pet.js +0 -1
- package/viewer/assets/railroad-abnf-HS5TGJTU-BUgwRsng.js +0 -1
- package/viewer/assets/railroad-ebnf-LZEXJU2U-C22SR2MX.js +0 -1
- package/viewer/assets/railroad-peg-WCYAUIDC-DjI1jJwc.js +0 -1
- package/viewer/assets/stateDiagram-v2-MP3YSRHH-DtGwflXN.js +0 -1
- package/viewer/assets/swimlanesDiagram-VR7AAH4N-DCpSoqRu.js +0 -8
package/JUDGING.md
CHANGED
|
@@ -47,11 +47,13 @@ cannot silently disappear from the benchmark denominator. Use consistent score
|
|
|
47
47
|
IDs across models and repetitions; a missing required score stays unresolved.
|
|
48
48
|
There are no built-in task-specific scorers, registration steps, or builder APIs.
|
|
49
49
|
|
|
50
|
-
The host bundles local imports before execution, captures source maps and
|
|
51
|
-
|
|
52
|
-
references, or
|
|
53
|
-
|
|
54
|
-
|
|
50
|
+
The host bundles local imports before execution, captures source maps, and
|
|
51
|
+
fingerprints the bundled code with the name and version of each imported
|
|
52
|
+
package. Code, imported references, or package version changes schedule
|
|
53
|
+
rejudging rather than candidate execution. The checkout location, working
|
|
54
|
+
directory, install layout, lockfiles, and SDK version do not. Use static imports
|
|
55
|
+
for reference data; make runtime network and file inputs reproducible when an
|
|
56
|
+
author-owned judge uses them.
|
|
55
57
|
|
|
56
58
|
The viewer shows normalized criterion scores, original returned JSON, frozen
|
|
57
59
|
source, process logs, and recorded candidate metrics. A synchronous loop can be
|
package/VERIFICATION.md
CHANGED
|
@@ -8,7 +8,7 @@ run it in a disposable OCI container rather than on the runner host.
|
|
|
8
8
|
import { VERIFICATION_IMAGE, type Benchmark } from "@hona/openeval";
|
|
9
9
|
export default {
|
|
10
10
|
models: ["example/model"],
|
|
11
|
-
judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096 } },
|
|
11
|
+
judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096, workspaceMiB: 2048 } },
|
|
12
12
|
} satisfies Benchmark;
|
|
13
13
|
```
|
|
14
14
|
|
|
@@ -61,6 +61,13 @@ prefix does not necessarily establish final-task failure.
|
|
|
61
61
|
- Containers self-expire. The parent also removes containers bearing only its
|
|
62
62
|
unique execution label after worker interruption. No shared/broad cleanup.
|
|
63
63
|
|
|
64
|
+
The workspace tmpfs defaults to 512 MiB. Set `judge.verification.workspaceMiB`
|
|
65
|
+
for a larger restored project or prepared dependency tree. The value must be a
|
|
66
|
+
positive integer no larger than `memoryMiB` (which defaults to 4096). It is
|
|
67
|
+
retained in verification receipts and code-judge identity. Omitting it preserves
|
|
68
|
+
existing identities and limits. Network, credentials, privileges, transfer caps,
|
|
69
|
+
and the read-only image filesystem are unchanged.
|
|
70
|
+
|
|
64
71
|
The standard image includes Bun 1.4.2, Python, Git, Node, and Playwright 1.63.0
|
|
65
72
|
with Chromium. Import Playwright from
|
|
66
73
|
`/opt/verify/node_modules/playwright/index.mjs`. Dependencies needed by a task
|
package/package.json
CHANGED
|
@@ -43,6 +43,14 @@ export function candidateProviders(
|
|
|
43
43
|
: undefined;
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
+
/** The candidate session limit for one eval: its eval.ts override, else the benchmark value. */
|
|
47
|
+
export const candidateTimeout = (
|
|
48
|
+
definition: BenchmarkDefinition,
|
|
49
|
+
evalId: string,
|
|
50
|
+
) =>
|
|
51
|
+
definition.evals.find((item) => item.id === evalId)?.settings.candidate
|
|
52
|
+
?.timeoutMs ?? definition.candidate.timeoutMs;
|
|
53
|
+
|
|
46
54
|
/** Declared, candidate-visible inputs. Harness implementation hashes are provenance.
|
|
47
55
|
* Early stopping is a judge-side policy; the planner re-collects sessions it cut short. */
|
|
48
56
|
export function candidateFingerprint(
|
|
@@ -59,7 +67,7 @@ export function candidateFingerprint(
|
|
|
59
67
|
model,
|
|
60
68
|
image: runtime.imageId,
|
|
61
69
|
candidate: {
|
|
62
|
-
timeoutMs: definition
|
|
70
|
+
timeoutMs: candidateTimeout(definition, evalId),
|
|
63
71
|
websearch: definition.candidate.websearch,
|
|
64
72
|
provider: scope && { ...scope.settings, model: scope.override },
|
|
65
73
|
},
|
|
@@ -8,7 +8,7 @@ import type {
|
|
|
8
8
|
EvalDefinition,
|
|
9
9
|
ModelRef,
|
|
10
10
|
} from "../types";
|
|
11
|
-
import { CANDIDATE_TIMEOUT_MS } from "../types";
|
|
11
|
+
import { CANDIDATE_TIMEOUT_MS, MAX_CANDIDATE_TIMEOUT_MS } from "../types";
|
|
12
12
|
import { rubricCriteria } from "../judgment";
|
|
13
13
|
import {
|
|
14
14
|
categoryKey,
|
|
@@ -32,6 +32,12 @@ const positive = (value: unknown, fallback: number, label: string) => {
|
|
|
32
32
|
throw new Error(`${label} must be a positive integer`);
|
|
33
33
|
return result;
|
|
34
34
|
};
|
|
35
|
+
const timeLimit = (value: unknown, label: string) => {
|
|
36
|
+
const timeoutMs = positive(value, CANDIDATE_TIMEOUT_MS, label);
|
|
37
|
+
if (timeoutMs > MAX_CANDIDATE_TIMEOUT_MS)
|
|
38
|
+
throw new Error(`${label} cannot exceed 12 hours`);
|
|
39
|
+
return timeoutMs;
|
|
40
|
+
};
|
|
35
41
|
export const modelRef = (value: unknown): ModelRef => {
|
|
36
42
|
if (
|
|
37
43
|
typeof value !== "string" ||
|
|
@@ -66,10 +72,23 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
|
|
|
66
72
|
throw new Error(`${id}/eval.ts must export a declaration`);
|
|
67
73
|
if (
|
|
68
74
|
Object.keys(settings).some(
|
|
69
|
-
(key) =>
|
|
75
|
+
(key) =>
|
|
76
|
+
!["workspace", "prepare", "earlyStop", "candidate"].includes(key),
|
|
70
77
|
)
|
|
71
78
|
)
|
|
72
79
|
throw new Error(`${id}/eval.ts has unsupported settings`);
|
|
80
|
+
if (settings.candidate !== undefined) {
|
|
81
|
+
const candidate: unknown = settings.candidate;
|
|
82
|
+
if (
|
|
83
|
+
!candidate ||
|
|
84
|
+
typeof candidate !== "object" ||
|
|
85
|
+
Array.isArray(candidate) ||
|
|
86
|
+
Object.keys(candidate).some((key) => key !== "timeoutMs")
|
|
87
|
+
)
|
|
88
|
+
throw new Error(`${id}: candidate contains unsupported settings`);
|
|
89
|
+
if (settings.candidate.timeoutMs !== undefined)
|
|
90
|
+
timeLimit(settings.candidate.timeoutMs, `${id}: candidate timeout`);
|
|
91
|
+
}
|
|
73
92
|
monitorPolicy(settings.earlyStop);
|
|
74
93
|
if (hasCode && settings.earlyStop)
|
|
75
94
|
throw new Error(
|
|
@@ -308,13 +327,10 @@ export async function loadBenchmark(
|
|
|
308
327
|
);
|
|
309
328
|
if (new Set(models).size !== models.length)
|
|
310
329
|
throw new Error("Benchmark contains duplicate models");
|
|
311
|
-
const timeoutMs =
|
|
330
|
+
const timeoutMs = timeLimit(
|
|
312
331
|
definition.candidate?.timeoutMs,
|
|
313
|
-
CANDIDATE_TIMEOUT_MS,
|
|
314
332
|
"Candidate timeout",
|
|
315
333
|
);
|
|
316
|
-
if (timeoutMs > CANDIDATE_TIMEOUT_MS)
|
|
317
|
-
throw new Error("Candidate timeout cannot exceed 45 minutes");
|
|
318
334
|
const evalRoot = resolve(directory, "evals");
|
|
319
335
|
const directories = (await readdir(evalRoot, { withFileTypes: true }))
|
|
320
336
|
.filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
|
|
@@ -348,10 +364,14 @@ export async function loadBenchmark(
|
|
|
348
364
|
const verification = definition.judge?.verification;
|
|
349
365
|
if (verification && (
|
|
350
366
|
typeof verification !== "object" || Array.isArray(verification) ||
|
|
351
|
-
Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB"].includes(key)) ||
|
|
367
|
+
Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB", "workspaceMiB"].includes(key)) ||
|
|
352
368
|
typeof verification.image !== "string" || !verification.image.trim() || /\s/.test(verification.image) ||
|
|
353
369
|
!["docker", "podman"].includes(verification.engine ?? engine)
|
|
354
370
|
)) throw new Error("judge.verification requires an image and valid container limits");
|
|
371
|
+
const verificationMemory = positive(verification?.memoryMiB, 4096, "Verification memory");
|
|
372
|
+
if (verification?.workspaceMiB !== undefined &&
|
|
373
|
+
(!Number.isSafeInteger(verification.workspaceMiB) || verification.workspaceMiB < 1 || verification.workspaceMiB > verificationMemory))
|
|
374
|
+
throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
|
|
355
375
|
for (const search of [
|
|
356
376
|
definition.candidate?.websearch,
|
|
357
377
|
definition.judge?.websearch,
|
|
@@ -387,7 +407,8 @@ export async function loadBenchmark(
|
|
|
387
407
|
image: verification.image,
|
|
388
408
|
engine: verification.engine ?? engine,
|
|
389
409
|
cpus: positive(verification.cpus, 2, "Verification CPU count"),
|
|
390
|
-
memoryMiB:
|
|
410
|
+
memoryMiB: verificationMemory,
|
|
411
|
+
...(verification.workspaceMiB === undefined ? {} : { workspaceMiB: verification.workspaceMiB }),
|
|
391
412
|
} } : {}),
|
|
392
413
|
},
|
|
393
414
|
container: {
|
package/src/app/run-eval.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { executeCandidate } from "../infra/containers/candidate";
|
|
|
5
5
|
import type { EvidenceFeed } from "../evidence";
|
|
6
6
|
import { CandidateEvidence } from "../infra/evidence";
|
|
7
7
|
import { measureRecording } from "../infra/recording/metrics";
|
|
8
|
-
import { candidateProviders } from "./input-fingerprints";
|
|
8
|
+
import { candidateProviders, candidateTimeout } from "./input-fingerprints";
|
|
9
9
|
|
|
10
10
|
/** The normal candidate use case: record its inputs, execute, finalize its evidence. */
|
|
11
11
|
export async function runEval(
|
|
@@ -20,6 +20,7 @@ export async function runEval(
|
|
|
20
20
|
const definition = context.definition.evals.find(
|
|
21
21
|
(item) => item.id === slot.evalId,
|
|
22
22
|
)!;
|
|
23
|
+
const timeoutMs = candidateTimeout(context.definition, slot.evalId);
|
|
23
24
|
const run = context.results.startEval(slot, {
|
|
24
25
|
evalId: slot.evalId,
|
|
25
26
|
model: slot.model,
|
|
@@ -28,7 +29,7 @@ export async function runEval(
|
|
|
28
29
|
candidateHash: slot.candidateHash,
|
|
29
30
|
sourceHash: definition.sourceHash,
|
|
30
31
|
imageId: context.runtime.imageId,
|
|
31
|
-
timeoutMs
|
|
32
|
+
timeoutMs,
|
|
32
33
|
earlyStop: !!controls.onEvidence,
|
|
33
34
|
runtime: context.runtime,
|
|
34
35
|
});
|
|
@@ -39,7 +40,7 @@ export async function runEval(
|
|
|
39
40
|
prompt: definition.prompt,
|
|
40
41
|
workspace,
|
|
41
42
|
prepare: definition.settings.prepare ?? [],
|
|
42
|
-
timeoutMs
|
|
43
|
+
timeoutMs,
|
|
43
44
|
websearch: context.definition.candidate.websearch,
|
|
44
45
|
container: context.definition.container,
|
|
45
46
|
imageId: context.runtime.imageId,
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export type {
|
|
2
2
|
Benchmark,
|
|
3
|
+
CandidateLimits,
|
|
3
4
|
Eval,
|
|
4
5
|
ModelRef,
|
|
5
6
|
ProviderDefinitions,
|
|
@@ -30,7 +31,7 @@ export type {
|
|
|
30
31
|
EvidenceQuery,
|
|
31
32
|
JudgeSession,
|
|
32
33
|
} from "./evidence";
|
|
33
|
-
export { CANDIDATE_TIMEOUT_MS } from "./types";
|
|
34
|
+
export { CANDIDATE_TIMEOUT_MS, MAX_CANDIDATE_TIMEOUT_MS } from "./types";
|
|
34
35
|
export { rubricCriteria, criterionMean, isScored } from "./judgment";
|
|
35
36
|
export {
|
|
36
37
|
categoryKey,
|
|
@@ -1,18 +1,34 @@
|
|
|
1
|
-
import { dirname, isAbsolute, resolve } from "node:path";
|
|
2
|
-
import { fileURLToPath } from "node:url";
|
|
1
|
+
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
|
|
3
2
|
import type { CodeJudgeDefinition } from "../../judge-context";
|
|
4
3
|
import { fingerprint } from "../files";
|
|
5
4
|
|
|
5
|
+
/** Raise when a release changes what judge.ts receives or how it runs; every code judge then runs again. */
|
|
6
|
+
const CODE_JUDGE_PROTOCOL = 1;
|
|
7
|
+
const printer = new Bun.Transpiler({
|
|
8
|
+
loader: "js",
|
|
9
|
+
target: "bun",
|
|
10
|
+
deadCodeElimination: false,
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
/** The package that provides an external import, as `name@version/path`. */
|
|
14
|
+
async function packageSpecifier(path: string, from: string) {
|
|
15
|
+
for (let directory = dirname(path); ; directory = dirname(directory)) {
|
|
16
|
+
const manifest = Bun.file(resolve(directory, "package.json"));
|
|
17
|
+
const { name, version } = (await manifest.exists())
|
|
18
|
+
? await manifest.json().catch(() => ({}))
|
|
19
|
+
: {};
|
|
20
|
+
if (typeof name === "string" && name)
|
|
21
|
+
return `${name}@${version ?? ""}/${relative(directory, path).split(sep).join("/")}`;
|
|
22
|
+
if (dirname(directory) === directory)
|
|
23
|
+
return relative(from, path).split(sep).join("/");
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
6
27
|
/** Bundle local imports without executing author code during planning. */
|
|
7
28
|
export async function compileCodeJudge(
|
|
8
29
|
file: string,
|
|
9
30
|
): Promise<CodeJudgeDefinition> {
|
|
10
31
|
file = resolve(file);
|
|
11
|
-
const dependencies: Record<string, string> = {};
|
|
12
|
-
const sdkManifest = fileURLToPath(
|
|
13
|
-
new URL("../../../package.json", import.meta.url),
|
|
14
|
-
);
|
|
15
|
-
dependencies[sdkManifest] = await Bun.file(sdkManifest).text();
|
|
16
32
|
const external = new Set<string>();
|
|
17
33
|
const built = await Bun.build({
|
|
18
34
|
// Import only the judge function: unused reporting metadata is tree-shaken.
|
|
@@ -65,40 +81,32 @@ export async function compileCodeJudge(
|
|
|
65
81
|
(await built.outputs
|
|
66
82
|
.find((output) => output.kind === "sourcemap")
|
|
67
83
|
?.text()) ?? "";
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
: await Bun.file(path).text();
|
|
85
|
-
found ||= name === "package.json";
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
if (found && stopAtPackage) return;
|
|
89
|
-
const parent = dirname(directory);
|
|
90
|
-
if (parent === directory) return;
|
|
91
|
-
directory = parent;
|
|
92
|
-
}
|
|
93
|
-
};
|
|
94
|
-
await collect(dirname(file), false);
|
|
95
|
-
for (const path of external) await collect(dirname(path), true);
|
|
84
|
+
// The executable keeps absolute import paths; identity uses the package name and version.
|
|
85
|
+
const dependencies: Record<string, string> = {};
|
|
86
|
+
let portable = source;
|
|
87
|
+
for (const path of external) {
|
|
88
|
+
const specifier = await packageSpecifier(path, dirname(file));
|
|
89
|
+
dependencies[specifier] = path;
|
|
90
|
+
portable = portable.replaceAll(
|
|
91
|
+
JSON.stringify(path),
|
|
92
|
+
JSON.stringify(specifier),
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
// Bun writes __dirname and __filename as absolute paths. Like other runtime file inputs, they are not identity.
|
|
96
|
+
portable = portable.replace(
|
|
97
|
+
/^\s*var __(?:dirname|filename) = .*$/gm,
|
|
98
|
+
(line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
|
|
99
|
+
);
|
|
96
100
|
return {
|
|
97
101
|
file,
|
|
98
102
|
source,
|
|
99
103
|
sourceMap,
|
|
100
104
|
dependencies,
|
|
101
|
-
//
|
|
102
|
-
|
|
105
|
+
// Reprinting drops comments: module comments name files relative to the working
|
|
106
|
+
// directory, and the debug ID depends on them. Source maps are retained, not hashed.
|
|
107
|
+
hash: fingerprint({
|
|
108
|
+
protocol: CODE_JUDGE_PROTOCOL,
|
|
109
|
+
source: printer.transformSync(portable),
|
|
110
|
+
}),
|
|
103
111
|
};
|
|
104
112
|
}
|
|
@@ -21,6 +21,10 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
|
|
|
21
21
|
if (!["docker", "podman"].includes(engine) || !environment.image || /\s/.test(environment.image) ||
|
|
22
22
|
[environment.cpus ?? 2, environment.memoryMiB ?? 4096].some(value => !Number.isSafeInteger(value) || value < 1))
|
|
23
23
|
throw new Error("Verification requires a valid image, engine, and positive integer resource limits");
|
|
24
|
+
if (environment.workspaceMiB !== undefined &&
|
|
25
|
+
(!Number.isSafeInteger(environment.workspaceMiB) || environment.workspaceMiB < 1 ||
|
|
26
|
+
environment.workspaceMiB > (environment.memoryMiB ?? 4096)))
|
|
27
|
+
throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
|
|
24
28
|
const output = await engineCommand(engine, [
|
|
25
29
|
"image", "inspect", environment.image, "--format",
|
|
26
30
|
'{{.Id}}|{{index .Config.Labels "openeval.verification"}}',
|
|
@@ -29,5 +33,9 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
|
|
|
29
33
|
if (!/^sha256:[a-f0-9]{64}$/.test(imageId)) throw new Error("Verification image has no immutable image ID");
|
|
30
34
|
if (environment.image === VERIFICATION_IMAGE && label !== await treeHash(directory))
|
|
31
35
|
throw new Error("Verification runtime changed. Run image --verification first.");
|
|
32
|
-
return {
|
|
36
|
+
return {
|
|
37
|
+
engine, image: environment.image, imageId,
|
|
38
|
+
cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096,
|
|
39
|
+
...(environment.workspaceMiB === undefined ? {} : { workspaceMiB: environment.workspaceMiB }),
|
|
40
|
+
};
|
|
33
41
|
}
|
|
@@ -26,7 +26,7 @@ export function verificationArgs(runtime: VerificationRuntime, owner: string, na
|
|
|
26
26
|
"--network=none", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--read-only",
|
|
27
27
|
"--pids-limit", "128", "--cpus", String(runtime.cpus), "--memory", `${runtime.memoryMiB}m`,
|
|
28
28
|
"--memory-swap", `${runtime.memoryMiB}m`, "--user", "10001:10001",
|
|
29
|
-
"--tmpfs",
|
|
29
|
+
"--tmpfs", `/workspace:rw,exec,nosuid,nodev,mode=1777,size=${runtime.workspaceMiB ?? 512}m`,
|
|
30
30
|
"--tmpfs", "/verification:rw,exec,nosuid,nodev,mode=0755,size=64m",
|
|
31
31
|
"--tmpfs", "/tmp:rw,exec,nosuid,nodev,mode=1777,size=256m",
|
|
32
32
|
"--env", "HOME=/tmp/home", "--env", "CI=1",
|
package/src/judge-context.ts
CHANGED
|
@@ -39,6 +39,8 @@ export type VerificationEnvironment = {
|
|
|
39
39
|
image: string;
|
|
40
40
|
cpus?: number;
|
|
41
41
|
memoryMiB?: number;
|
|
42
|
+
/** Workspace tmpfs capacity in MiB; default 512. A custom value must fit within memoryMiB. */
|
|
43
|
+
workspaceMiB?: number;
|
|
42
44
|
};
|
|
43
45
|
export type VerificationRuntime = {
|
|
44
46
|
engine: Engine;
|
|
@@ -46,6 +48,7 @@ export type VerificationRuntime = {
|
|
|
46
48
|
imageId: string;
|
|
47
49
|
cpus: number;
|
|
48
50
|
memoryMiB: number;
|
|
51
|
+
workspaceMiB?: number;
|
|
49
52
|
};
|
|
50
53
|
export type VerificationRequest = {
|
|
51
54
|
revision?: Revision;
|
|
@@ -185,9 +188,11 @@ export interface JudgeContext {
|
|
|
185
188
|
/** Frozen judging source and dependency identity, kept in host-side run records. */
|
|
186
189
|
export type CodeJudgeDefinition = {
|
|
187
190
|
file: string;
|
|
191
|
+
/** Executable identity: bundled code and imported package versions, without host paths or comments. */
|
|
188
192
|
hash: string;
|
|
189
193
|
source: string;
|
|
190
194
|
sourceMap: string;
|
|
195
|
+
/** Packages imported at run time, as `name@version/path`, with the file each resolved to. */
|
|
191
196
|
dependencies: Record<string, string>;
|
|
192
197
|
};
|
|
193
198
|
export type CodeJudgeExecution = {
|
package/src/types.ts
CHANGED
|
@@ -15,6 +15,7 @@ import type {
|
|
|
15
15
|
export type OpenCodeStreamEvent = OpenCodeEvent | SessionLogItem;
|
|
16
16
|
|
|
17
17
|
export const CANDIDATE_TIMEOUT_MS = 45 * 60 * 1000;
|
|
18
|
+
export const MAX_CANDIDATE_TIMEOUT_MS = 12 * 60 * 60 * 1000;
|
|
18
19
|
export type ModelRef = `${string}/${string}`;
|
|
19
20
|
export type Engine = "docker" | "podman";
|
|
20
21
|
export type ProviderDefinitions = NonNullable<
|
|
@@ -34,6 +35,11 @@ export type Judge = {
|
|
|
34
35
|
websearch?: "exa" | false;
|
|
35
36
|
verification?: VerificationEnvironment;
|
|
36
37
|
};
|
|
38
|
+
/** Execution controls for one candidate attempt. Eval values override benchmark values. */
|
|
39
|
+
export type CandidateLimits = {
|
|
40
|
+
/** Agent session limit, after preparation; default 45 minutes, at most 12 hours. */
|
|
41
|
+
timeoutMs?: number;
|
|
42
|
+
};
|
|
37
43
|
export type Benchmark = {
|
|
38
44
|
name?: string;
|
|
39
45
|
/** Display label for one benchmark.ts among several that share `name`, such as "Frontier". */
|
|
@@ -42,8 +48,7 @@ export type Benchmark = {
|
|
|
42
48
|
judge?: Judge;
|
|
43
49
|
repetitions?: number;
|
|
44
50
|
concurrency?: number;
|
|
45
|
-
candidate?: {
|
|
46
|
-
timeoutMs?: number;
|
|
51
|
+
candidate?: CandidateLimits & {
|
|
47
52
|
websearch?: "exa" | false;
|
|
48
53
|
providers?: ProviderDefinitions;
|
|
49
54
|
};
|
|
@@ -82,6 +87,8 @@ export type Eval = {
|
|
|
82
87
|
overlay?: string;
|
|
83
88
|
};
|
|
84
89
|
prepare?: readonly PreparationStep[];
|
|
90
|
+
/** Overrides benchmark.candidate for this eval. */
|
|
91
|
+
candidate?: CandidateLimits;
|
|
85
92
|
};
|
|
86
93
|
export type EvalDefinition = {
|
|
87
94
|
id: string;
|
|
@@ -106,7 +113,7 @@ export type BenchmarkDefinition = {
|
|
|
106
113
|
suite?: string;
|
|
107
114
|
directory: string;
|
|
108
115
|
models: ModelRef[];
|
|
109
|
-
judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false; verification?: VerificationRuntime |
|
|
116
|
+
judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false; verification?: VerificationRuntime | Omit<VerificationRuntime, "imageId"> };
|
|
110
117
|
repetitions: number;
|
|
111
118
|
concurrency: number;
|
|
112
119
|
candidate: {
|
package/src/view.ts
CHANGED
|
@@ -171,13 +171,33 @@ function componentBounds(
|
|
|
171
171
|
coverage: coverage * scale,
|
|
172
172
|
};
|
|
173
173
|
}
|
|
174
|
-
|
|
174
|
+
/** A cost summed over executions. `unaccounted` counts executions without a final cost. */
|
|
175
|
+
export type CostTotal = Cost & { unaccounted: number };
|
|
176
|
+
/** Sum execution costs or totals; a single execution without a final cost counts once. */
|
|
177
|
+
export const sumCosts = (
|
|
178
|
+
costs: readonly (Cost | CostTotal | undefined)[],
|
|
179
|
+
): CostTotal => {
|
|
175
180
|
const reportedUSD = costs.reduce(
|
|
176
181
|
(sum, cost) => sum + (cost?.reportedUSD ?? 0),
|
|
177
182
|
0,
|
|
178
183
|
);
|
|
179
184
|
const complete = costs.every((cost) => cost?.complete === true);
|
|
180
|
-
|
|
185
|
+
const unaccounted = costs.reduce(
|
|
186
|
+
(sum, cost) =>
|
|
187
|
+
sum +
|
|
188
|
+
(cost && "unaccounted" in cost
|
|
189
|
+
? cost.unaccounted
|
|
190
|
+
: cost?.complete
|
|
191
|
+
? 0
|
|
192
|
+
: 1),
|
|
193
|
+
0,
|
|
194
|
+
);
|
|
195
|
+
return {
|
|
196
|
+
reportedUSD,
|
|
197
|
+
complete,
|
|
198
|
+
usd: complete ? reportedUSD : null,
|
|
199
|
+
unaccounted,
|
|
200
|
+
};
|
|
181
201
|
};
|
|
182
202
|
export type ResultEntry = {
|
|
183
203
|
id: string;
|
|
@@ -209,8 +229,8 @@ export type Overview = {
|
|
|
209
229
|
evalNames: Record<string, string>;
|
|
210
230
|
modelNames: ModelNames;
|
|
211
231
|
categories: CategorySummary[];
|
|
212
|
-
cost:
|
|
213
|
-
evalCosts: Record<string,
|
|
232
|
+
cost: CostTotal;
|
|
233
|
+
evalCosts: Record<string, CostTotal>;
|
|
214
234
|
runtime: RuntimeInterval[];
|
|
215
235
|
};
|
|
216
236
|
export type ResultSummary = {
|
|
@@ -305,7 +325,10 @@ export type JudgeAudit = {
|
|
|
305
325
|
Pick<JudgeRun, "id" | "state" | "startedAt"> & { mode: "monitor" | "final" }
|
|
306
326
|
>;
|
|
307
327
|
};
|
|
308
|
-
export type EvalRunSummary = LiveEvalRun & {
|
|
328
|
+
export type EvalRunSummary = LiveEvalRun & {
|
|
329
|
+
cost: CostTotal;
|
|
330
|
+
elapsedMs: number;
|
|
331
|
+
};
|
|
309
332
|
export type EvalRunIndex = {
|
|
310
333
|
entry: ResultEntry;
|
|
311
334
|
eval: string;
|
|
@@ -316,7 +339,7 @@ export type ActivityRun = {
|
|
|
316
339
|
id: string;
|
|
317
340
|
benchmarkId: string;
|
|
318
341
|
resultId: string;
|
|
319
|
-
cost:
|
|
342
|
+
cost: CostTotal;
|
|
320
343
|
state: {
|
|
321
344
|
id: string;
|
|
322
345
|
eval: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{m as e,t}from"./mermaid-parser.core-
|
|
1
|
+
import{m as e,t}from"./mermaid-parser.core-VTfqiyyK.js";import{n}from"./chunk-Y2CYZVJY-DsF7k-Jl.js";import{m as r}from"./src-C171e0nu.js";import"./chunk-DU6HZSFF-BI4SVpji.js";import{n as i,r as a,t as o}from"./chunk-SVP7TREG-CGuEYo-A.js";import{t as s}from"./chunk-JWPE2WC7-DVXcaiue.js";import"./mermaid.core-eoOgKmWC.js";var c=e().RailroadAbnf.parser.LangiumParser,l=n(e=>{let t=e.alternatives.map(u);return t.length===1?t[0]:{type:`choice`,alternatives:t}},`transformAlternation`),u=n(e=>{let t=e.elements.map(f);return t.length===1?t[0]:{type:`sequence`,elements:t}},`transformConcatenation`),d=n(e=>{if(e.includes(`*`)){let[t,n]=e.split(`*`);return{min:t?parseInt(t,10):0,max:n?parseInt(n,10):1/0}}let t=parseInt(e,10);return{min:t,max:t}},`parseRepeat`),f=n(e=>{let t=p(e.primary);if(!e.repeat)return t;let{min:n,max:r}=d(e.repeat);return n===0&&r===1?{type:`optional`,element:t}:{type:`repetition`,element:t,min:n,max:r}},`transformElement`),p=n(e=>{switch(e.$type){case`AbnfStringLiteral`:return{type:`terminal`,value:e.value};case`AbnfNumVal`:return{type:`terminal`,value:e.value};case`AbnfRuleName`:return{type:`nonterminal`,name:e.name};case`AbnfGroup`:return l(e.element);case`AbnfOptionalGroup`:return{type:`optional`,element:l(e.element)};default:throw Error(`Unsupported ABNF primary node: ${e.$type}`)}},`transformPrimary`),m=n(e=>({name:e.name,definition:l(e.definition)}),`transformRule`),h=n(e=>{s(e,o),e.title&&o.setTitle(e.title),e.rules.map(e=>o.addRule(m(e)))},`populateDb`),g={parser:{parse:n(e=>{o.clear(),r.debug(`[ABNF Parser] Starting Langium parse`);let n=c.parse(e);if(n.lexerErrors.length>0||n.parserErrors.length>0)throw new t(n);let i=n.value;r.debug(`[ABNF Parser] Parsed rules:`,i.rules.length),h(i),r.debug(`[ABNF Parser] Parse complete`)},`parse`),parser:{yy:o}},db:o,renderer:a,styles:i};export{g as diagram};
|