chainlesschain 0.166.21 → 0.166.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -9
- package/package.json +1 -1
- package/src/assets/web-panel/assets/{AIOps-BjMnsUci.js → AIOps-oJ7USzMC.js} +1 -1
- package/src/assets/web-panel/assets/{ActionButton-DdYFP03A.js → ActionButton-BlgnZe_C.js} +1 -1
- package/src/assets/web-panel/assets/{Analytics-Bb8TuuK_.js → Analytics-D2rRbTLu.js} +2 -2
- package/src/assets/web-panel/assets/{AppLayout-Kn3TIOtk.js → AppLayout-FwS3f0B2.js} +5 -5
- package/src/assets/web-panel/assets/{Artifacts-4IxpqI_b.js → Artifacts-iF-_08Hi.js} +1 -1
- package/src/assets/web-panel/assets/{Audit-BOV2caqU.js → Audit-D24p0zQw.js} +1 -1
- package/src/assets/web-panel/assets/{BackgroundAgents-B86k-ay4.js → BackgroundAgents-Cg3mRVi9.js} +1 -1
- package/src/assets/web-panel/assets/{Backup-C5_ie1e3.js → Backup-EOqht3H-.js} +1 -1
- package/src/assets/web-panel/assets/{BaseInput-xpwg6QCW.js → BaseInput-9XWYq2hj.js} +1 -1
- package/src/assets/web-panel/assets/{Chat-CVZOcZau.js → Chat-Dq5yardA.js} +4 -4
- package/src/assets/web-panel/assets/{ChatBubbleRenderer-D8fPYhUu.js → ChatBubbleRenderer-Ds3pA-ET.js} +1 -1
- package/src/assets/web-panel/assets/{Checkbox-MKTq3zdP.js → Checkbox-CuyCBoBf.js} +1 -1
- package/src/assets/web-panel/assets/{Codegen-JUk3wzep.js → Codegen-DjWtYHNe.js} +1 -1
- package/src/assets/web-panel/assets/{Col-JPvTA0kC.js → Col-CrVr_iuG.js} +1 -1
- package/src/assets/web-panel/assets/{Community-Tqgl87NQ.js → Community-C6QxpfGc.js} +1 -1
- package/src/assets/web-panel/assets/{Compact-BuAFYBmN.js → Compact-B-v5r6Pa.js} +1 -1
- package/src/assets/web-panel/assets/{Compliance-CXS9WIPZ.js → Compliance-zNiLiFgD.js} +1 -1
- package/src/assets/web-panel/assets/{Cowork-Ds2bFHrr.js → Cowork-D9K7lMOu.js} +2 -2
- package/src/assets/web-panel/assets/{Cron-C_dV_OeW.js → Cron-DAjO36z4.js} +2 -2
- package/src/assets/web-panel/assets/{Crosschain-BsO6tIRo.js → Crosschain-Bxqn0aqB.js} +1 -1
- package/src/assets/web-panel/assets/{DID-rNOFqBac.js → DID-DfRmLUe-.js} +2 -2
- package/src/assets/web-panel/assets/{Dashboard-C2fcKKVY.js → Dashboard-Cz8KF3fr.js} +2 -2
- package/src/assets/web-panel/assets/{Dropdown-Y7Ndr0X_.js → Dropdown-C4oVfkP0.js} +1 -1
- package/src/assets/web-panel/assets/{EmailListRenderer-DcP_uPmL.js → EmailListRenderer-ivyZKN8k.js} +1 -1
- package/src/assets/web-panel/assets/{FamilyGuardDashboard-Cwn3gDzk.js → FamilyGuardDashboard-DoxY1CPW.js} +1 -1
- package/src/assets/web-panel/assets/{Federation-ClQuOsqb.js → Federation-C93gZxnb.js} +1 -1
- package/src/assets/web-panel/assets/{FormItemContext-DfwtmlcK.js → FormItemContext-D_lEjYJE.js} +1 -1
- package/src/assets/web-panel/assets/{GenericCardRenderer-zNZhtbC5.js → GenericCardRenderer-BNFN9YjN.js} +1 -1
- package/src/assets/web-panel/assets/{Git-BZmdW-G5.js → Git-mUu2HcLI.js} +2 -2
- package/src/assets/web-panel/assets/{Governance-CqBk4lEn.js → Governance-CNuosrge.js} +1 -1
- package/src/assets/web-panel/assets/{Inference-Dbmt3ZXk.js → Inference-DwY5pbCB.js} +1 -1
- package/src/assets/web-panel/assets/{KnowledgeGraph-hMTH6qf3.js → KnowledgeGraph-aRUxhACD.js} +1 -1
- package/src/assets/web-panel/assets/{Logs-CkiYgiW2.js → Logs-DFfsWYy_.js} +2 -2
- package/src/assets/web-panel/assets/{MarkdownRenderer-BL1FElXg.js → MarkdownRenderer-DqTx7_u1.js} +1 -1
- package/src/assets/web-panel/assets/{Marketplace-CbB6wfin.js → Marketplace-COI-ciM-.js} +1 -1
- package/src/assets/web-panel/assets/{McpTools-BSwiFtte.js → McpTools-BJJwaIpe.js} +2 -2
- package/src/assets/web-panel/assets/{Memory-B03PeHrz.js → Memory-CDO6SGDZ.js} +2 -2
- package/src/assets/web-panel/assets/{MobileBridge-CRGrnd-X.js → MobileBridge-BlZ5tFf4.js} +1 -1
- package/src/assets/web-panel/assets/{MobileProjects-CE_5v2kO.js → MobileProjects-DuoxbgWz.js} +1 -1
- package/src/assets/web-panel/assets/{Mtc-Dp11QVx1.js → Mtc-6ec9j70b.js} +2 -2
- package/src/assets/web-panel/assets/{MtcAudit-CYu-3JBu.js → MtcAudit-BuL7HkC7.js} +2 -2
- package/src/assets/web-panel/assets/{Multisig-CGyJ0Xl3.js → Multisig-CJxR85lk.js} +3 -3
- package/src/assets/web-panel/assets/{NLProgramming-CoWi_ov1.js → NLProgramming-B71O1S4l.js} +1 -1
- package/src/assets/web-panel/assets/Notes-Bsw5zbpV.js +7 -0
- package/src/assets/web-panel/assets/{NotificationSettings-DX4I2b4W.js → NotificationSettings-BnAhXuK_.js} +1 -1
- package/src/assets/web-panel/assets/OrderTableRenderer-BJ0R5GGz.js +1 -0
- package/src/assets/web-panel/assets/{Organization-oopLO_tk.js → Organization-B7wwNxoP.js} +4 -4
- package/src/assets/web-panel/assets/{Overflow-BFVvjMyf.js → Overflow-CfAGr0gM.js} +1 -1
- package/src/assets/web-panel/assets/{P2P-CBy5t-Tk.js → P2P-KhMg22Fz.js} +2 -2
- package/src/assets/web-panel/assets/{PdhVaultBrowser--YWJNsu6.js → PdhVaultBrowser-NdAXI9ft.js} +3 -3
- package/src/assets/web-panel/assets/{Permissions-C2dZ8scb.js → Permissions-CIYjqyT7.js} +4 -4
- package/src/assets/web-panel/assets/{PersonalDataHub-Bs3l7Hlx.js → PersonalDataHub-D2ht3P_R.js} +4 -4
- package/src/assets/web-panel/assets/{Pipeline-DEBWbZuD.js → Pipeline-i6flZW1m.js} +1 -1
- package/src/assets/web-panel/assets/{Privacy-CjHBb6NU.js → Privacy-DE0lhYVc.js} +1 -1
- package/src/assets/web-panel/assets/{ProjectInit-D2WipUz5.js → ProjectInit-BL3YbyxS.js} +2 -2
- package/src/assets/web-panel/assets/{ProjectSettings-B6WfnrGB.js → ProjectSettings-CEtdg7ni.js} +2 -2
- package/src/assets/web-panel/assets/{Projects-FF_thRi6.js → Projects-CvtU5peJ.js} +1 -1
- package/src/assets/web-panel/assets/{Providers-CCv5M_Dv.js → Providers-DKt_Cg-j.js} +1 -1
- package/src/assets/web-panel/assets/{QrScannerModal-CGthxU54.js → QrScannerModal-DA61jNSx.js} +1 -1
- package/src/assets/web-panel/assets/{QuickAsk-CnPmXPFb.js → QuickAsk-fS9ryUmq.js} +1 -1
- package/src/assets/web-panel/assets/{Recommend-Df_tGEa9.js → Recommend-CR9dZU5Y.js} +1 -1
- package/src/assets/web-panel/assets/{RemoteSession-Z58N0Loe.js → RemoteSession-BV0BR2Dz.js} +2 -2
- package/src/assets/web-panel/assets/{Reputation-CZTBQ2Vt.js → Reputation-BIeGUvYD.js} +1 -1
- package/src/assets/web-panel/assets/{Row-CdK3YpCQ.js → Row-DHDT1aro.js} +1 -1
- package/src/assets/web-panel/assets/{RssFeed-CFMVT5sb.js → RssFeed-Cq6YKb1h.js} +3 -3
- package/src/assets/web-panel/assets/{Search-DrF8RIvD.js → Search-DEF6vsy6.js} +1 -1
- package/src/assets/web-panel/assets/{Security-Ddk2f2DM.js → Security-CHtYiKY9.js} +4 -4
- package/src/assets/web-panel/assets/{Services-CCuK-VOo.js → Services-u16lvBJN.js} +2 -2
- package/src/assets/web-panel/assets/{Skeleton-DxCpHzeM.js → Skeleton-CGbizyDj.js} +1 -1
- package/src/assets/web-panel/assets/{Skills-t2oTDgcv.js → Skills-BZMQcAa7.js} +1 -1
- package/src/assets/web-panel/assets/{Sla-Df63M9_R.js → Sla-Bm-CycVi.js} +1 -1
- package/src/assets/web-panel/assets/{SpeechSettings-l7ox6KuX.js → SpeechSettings-BudpA5kr.js} +1 -1
- package/src/assets/web-panel/assets/{SyncSettings-COd_tgPp.js → SyncSettings-B8q1C1eC.js} +2 -2
- package/src/assets/web-panel/assets/{Tasks-CzDCcrik.js → Tasks-SU1pBWc-.js} +1 -1
- package/src/assets/web-panel/assets/{Templates-D-vfxTGA.js → Templates-CyOfJzmt.js} +1 -1
- package/src/assets/web-panel/assets/{Tenant-BlpT_Gxc.js → Tenant-xqWSm2GP.js} +1 -1
- package/src/assets/web-panel/assets/{Terminal-D5rq52Hg.js → Terminal-D1Ttgjsi.js} +2 -2
- package/src/assets/web-panel/assets/{TimelineRenderer-DdoMndkK.js → TimelineRenderer-Da2lYs5D.js} +1 -1
- package/src/assets/web-panel/assets/{Tokens-BhgBsI6g.js → Tokens-B34-SduO.js} +1 -1
- package/src/assets/web-panel/assets/{Trigger-DwwzoikZ.js → Trigger-C8zyYm4R.js} +1 -1
- package/src/assets/web-panel/assets/{Trust-HkTpzXFb.js → Trust-D1Pvqems.js} +1 -1
- package/src/assets/web-panel/assets/{UkeySign-D9nk6Ehe.js → UkeySign-Do3VVnAy.js} +1 -1
- package/src/assets/web-panel/assets/{VideoEditing-BKk0rfoy.js → VideoEditing-Dz7egVIA.js} +1 -1
- package/src/assets/web-panel/assets/{Wallet-IVWIU_Lq.js → Wallet-Dcuw-OxR.js} +4 -4
- package/src/assets/web-panel/assets/{WebAuthn-CFTCpfsG.js → WebAuthn-DPlw-QsB.js} +4 -4
- package/src/assets/web-panel/assets/{WorkflowEditor-Dnf4Obif.js → WorkflowEditor-h4OsnU5q.js} +1 -1
- package/src/assets/web-panel/assets/{chat-D2snERdj.js → chat-eO9wEwrW.js} +1 -1
- package/src/assets/web-panel/assets/{colors-DKEjK1N1.js → colors-34lSmOXw.js} +1 -1
- package/src/assets/web-panel/assets/{compact-item-BEDLwQdt.js → compact-item-CK4kEWJ4.js} +1 -1
- package/src/assets/web-panel/assets/{createContext-DA9iW1a0.js → createContext-E0A57xed.js} +1 -1
- package/src/assets/web-panel/assets/devWarning-CF3xafX2.js +1 -0
- package/src/assets/web-panel/assets/{hasIn-5l2xEcY9.js → hasIn-BD9ZMGAl.js} +1 -1
- package/src/assets/web-panel/assets/{index-C7sKbM5s.js → index--N0u6kTG.js} +1 -1
- package/src/assets/web-panel/assets/{index-DU2TXJu-.js → index-8qMA1DWQ.js} +1 -1
- package/src/assets/web-panel/assets/{index-CxEtQYdi.js → index-B0rkHxKK.js} +1 -1
- package/src/assets/web-panel/assets/{index-B7_W_rYd.js → index-B_G_ElKO.js} +1 -1
- package/src/assets/web-panel/assets/{index-B4QYBliR.js → index-BbX_Dted.js} +1 -1
- package/src/assets/web-panel/assets/{index-DLoalqRX.js → index-Bjs90Eh9.js} +1 -1
- package/src/assets/web-panel/assets/{index-Dbkghqn_.js → index-BmNbU2Z6.js} +1 -1
- package/src/assets/web-panel/assets/{index-BgfImyZI.js → index-Brqc9Emx.js} +1 -1
- package/src/assets/web-panel/assets/{index-5pW4t1iD.js → index-BsGmo6v1.js} +1 -1
- package/src/assets/web-panel/assets/index-C907rPwd.js +1 -0
- package/src/assets/web-panel/assets/{index-CRPrHE7h.js → index-CT1_eRL-.js} +1 -1
- package/src/assets/web-panel/assets/{index-DvV03Ayk.js → index-CZBaqe3d.js} +1 -1
- package/src/assets/web-panel/assets/{index-D7z4zB_a.js → index-CZkFMeWL.js} +1 -1
- package/src/assets/web-panel/assets/{index-CmruhzFi.js → index-Ch175NUw.js} +1 -1
- package/src/assets/web-panel/assets/{index-e-e40id6.js → index-CsrKJJcX.js} +1 -1
- package/src/assets/web-panel/assets/{index-CGlotREA.js → index-CuSau5ya.js} +1 -1
- package/src/assets/web-panel/assets/{index-CjWltbi_.js → index-Cx3Jn1r0.js} +1 -1
- package/src/assets/web-panel/assets/{index-fccZmC_h.js → index-Cz3unNfL.js} +1 -1
- package/src/assets/web-panel/assets/{index-NJieh3T1.js → index-D4bB_x_v.js} +1 -1
- package/src/assets/web-panel/assets/{index-CkMKuqUs.js → index-DCn-ECvK.js} +1 -1
- package/src/assets/web-panel/assets/{index-BZ5lY2zx.js → index-DFLE2qO9.js} +1 -1
- package/src/assets/web-panel/assets/{index-B_eCoZgW.js → index-DGRxt5wj.js} +1 -1
- package/src/assets/web-panel/assets/{index-7P1b8HZo.js → index-DONOUtlz.js} +1 -1
- package/src/assets/web-panel/assets/{index-yYEQXYaU.js → index-DP-UZAQN.js} +1 -1
- package/src/assets/web-panel/assets/{index-Dbd7eEex.js → index-Da5fmOHB.js} +1 -1
- package/src/assets/web-panel/assets/{index-Bqc8tdIF.js → index-DcE1BspQ.js} +1 -1
- package/src/assets/web-panel/assets/{index-DXet1xol.js → index-DcbgVLOH.js} +1 -1
- package/src/assets/web-panel/assets/{index-BO9dQjpt.js → index-DeXNCyQJ.js} +1 -1
- package/src/assets/web-panel/assets/{index-CX1x4AFn.js → index-DiT_REby.js} +1 -1
- package/src/assets/web-panel/assets/{index-Bq5P3piS.js → index-DpfisY1g.js} +1 -1
- package/src/assets/web-panel/assets/{index-Csfb-7hw.js → index-DrOalGut.js} +1 -1
- package/src/assets/web-panel/assets/{index-DpqtbXdE.js → index-DrSAAuJj.js} +1 -1
- package/src/assets/web-panel/assets/{index-B00GeekA.js → index-DrsDA_md.js} +1 -1
- package/src/assets/web-panel/assets/{index-Dg2a9owQ.js → index-DvKcroix.js} +1 -1
- package/src/assets/web-panel/assets/{index-fgYk0QXO.js → index-DzXjP44T.js} +1 -1
- package/src/assets/web-panel/assets/{index-Td8ZwyoC.js → index-MIg5Z8qe.js} +1 -1
- package/src/assets/web-panel/assets/{index-DqwPFbOb.js → index-o8pYehRq.js} +3 -3
- package/src/assets/web-panel/assets/index-wueRSK2d.js +1 -0
- package/src/assets/web-panel/assets/{index-CbTPvp2_.js → index-yAGBu2E0.js} +1 -1
- package/src/assets/web-panel/assets/{initDefaultProps-CPrcB1TC.js → initDefaultProps-DMTkAomF.js} +1 -1
- package/src/assets/web-panel/assets/{motion-BQ7mWRh_.js → motion-Cnti1F5g.js} +1 -1
- package/src/assets/web-panel/assets/{move-CzOEq99p.js → move-CGxmHUga.js} +1 -1
- package/src/assets/web-panel/assets/{mtc-parser-BlQI5jrw.js → mtc-parser-CuwwLoq7.js} +1 -1
- package/src/assets/web-panel/assets/{omit-DlW4gxmv.js → omit-BpdlTkPM.js} +1 -1
- package/src/assets/web-panel/assets/{pickAttrs-BBYms5nP.js → pickAttrs-D7nWrThp.js} +1 -1
- package/src/assets/web-panel/assets/{placementArrow-az2MPJ_7.js → placementArrow-Bhx0TV8F.js} +1 -1
- package/src/assets/web-panel/assets/{responsiveObserve-D1-ibEol.js → responsiveObserve-CZ1KesZh.js} +1 -1
- package/src/assets/web-panel/assets/{slide-DHGqa4GA.js → slide-CnldF8J6.js} +1 -1
- package/src/assets/web-panel/assets/{statusUtils-CYGDo6lD.js → statusUtils-CZfo_GOA.js} +1 -1
- package/src/assets/web-panel/assets/{styleChecker-Cl2EM9K6.js → styleChecker-B2QCcAv5.js} +1 -1
- package/src/assets/web-panel/assets/{useFlexGapSupport-DdYhzuv1.js → useFlexGapSupport-DzzjIS4J.js} +1 -1
- package/src/assets/web-panel/assets/{useFs-DWumPuBW.js → useFs-B_atg9M0.js} +1 -1
- package/src/assets/web-panel/assets/{usePersonalDataHub-DxAO-Aqi.js → usePersonalDataHub-DhZs3QYN.js} +1 -1
- package/src/assets/web-panel/assets/{vnode-CplsQf_N.js → vnode-DcePTYAm.js} +1 -1
- package/src/assets/web-panel/assets/{zoom-Cw5HXNSG.js → zoom-BYmlnjRk.js} +1 -1
- package/src/assets/web-panel/index.html +1 -1
- package/src/commands/serve.js +31 -0
- package/src/data/changelog.json +13 -0
- package/src/index.js +11 -3
- package/src/lazy-dispatch.js +9 -1
- package/src/lib/app-server/websocket-transport.js +21 -0
- package/src/lib/evolution/evolution-artifact-ports.js +9 -0
- package/src/lib/evolution/evolution-deployment-loader.js +208 -0
- package/src/lib/evolution/evolution-release-train-ledger-adapter.js +335 -0
- package/src/lib/evolution/evolution-release-train.js +538 -0
- package/src/lib/evolution/evolution-workbench-projection.js +20 -1
- package/src/lib/evolution/evolvable-artifact-ledger-adapter.js +688 -0
- package/src/lib/evolution/governed-knowledge-artifact-lifecycle.js +364 -0
- package/src/lib/evolution/governed-knowledge-merge-ledger-executor.js +29 -2
- package/src/lib/evolution/governed-knowledge-merge-publisher-authority.js +31 -3
- package/src/lib/evolution/governed-knowledge-sync-ledger-adapter.js +92 -6
- package/src/lib/evolution/governed-knowledge-sync-merge-publisher.js +57 -4
- package/src/lib/evolution/governed-knowledge-sync.js +317 -7
- package/src/lib/evolution/skill-writer-inventory-manifest.js +7 -6
- package/src/lib/evolution/statistical-progressive-canary.js +683 -0
- package/src/lib/evolution/wikiskill-benchmark.js +510 -0
- package/src/lib/session-host-lease.js +11 -16
- package/src/runtime/agent-core.js +61 -7
- package/src/runtime/headless-stream.js +3 -0
- package/src/assets/web-panel/assets/Notes-CgU9DDo9.js +0 -7
- package/src/assets/web-panel/assets/OrderTableRenderer-CwHc15Zs.js +0 -1
- package/src/assets/web-panel/assets/devWarning-DyhgpXPf.js +0 -1
- package/src/assets/web-panel/assets/index-Ck1jhFE0.js +0 -1
- package/src/assets/web-panel/assets/index-c8jJEHVs.js +0 -1
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
export const WIKISKILL_BENCHMARK_PLAN_SCHEMA =
|
|
4
|
+
"chainlesschain.wikiskill-benchmark-plan/v1";
|
|
5
|
+
export const WIKISKILL_BENCHMARK_REPORT_SCHEMA =
|
|
6
|
+
"chainlesschain.wikiskill-benchmark-report/v1";
|
|
7
|
+
export const WIKISKILL_BENCHMARK_ENVELOPE_SCHEMA =
|
|
8
|
+
"chainlesschain.wikiskill-benchmark-envelope/v1";
|
|
9
|
+
|
|
10
|
+
const DIGEST = /^sha256:[a-f0-9]{64}$/u;
|
|
11
|
+
const COMMIT = /^[a-f0-9]{40}$/u;
|
|
12
|
+
const FAILURE_CLASSES = new Set([
|
|
13
|
+
"none",
|
|
14
|
+
"model",
|
|
15
|
+
"tool",
|
|
16
|
+
"provider",
|
|
17
|
+
"sandbox",
|
|
18
|
+
"permission",
|
|
19
|
+
"infrastructure",
|
|
20
|
+
"grader",
|
|
21
|
+
"unknown",
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
function canonical(value) {
|
|
25
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
26
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
|
27
|
+
return `{${Object.keys(value)
|
|
28
|
+
.sort()
|
|
29
|
+
.map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`)
|
|
30
|
+
.join(",")}}`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function digest(domain, value) {
|
|
34
|
+
return `sha256:${createHash("sha256")
|
|
35
|
+
.update(domain)
|
|
36
|
+
.update("\0")
|
|
37
|
+
.update(canonical(value))
|
|
38
|
+
.digest("hex")}`;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function exactKeys(value, keys, name) {
|
|
42
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
43
|
+
throw new TypeError(`${name} must be an object`);
|
|
44
|
+
const actual = Object.keys(value).sort();
|
|
45
|
+
const expected = [...keys].sort();
|
|
46
|
+
if (canonical(actual) !== canonical(expected))
|
|
47
|
+
throw new TypeError(`${name} has unexpected or missing fields`);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function string(value, name) {
|
|
51
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
52
|
+
throw new TypeError(`${name} must be a non-empty string`);
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function sha(value, name) {
|
|
57
|
+
if (!DIGEST.test(value ?? ""))
|
|
58
|
+
throw new TypeError(`${name} must be a sha256 digest`);
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function finite(value, name, { min = 0, max = Number.MAX_VALUE } = {}) {
|
|
63
|
+
const number = Number(value);
|
|
64
|
+
if (!Number.isFinite(number) || number < min || number > max)
|
|
65
|
+
throw new TypeError(`${name} is outside its allowed range`);
|
|
66
|
+
return number;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function normalizeDatasets(datasets) {
|
|
70
|
+
if (!Array.isArray(datasets) || datasets.length !== 5)
|
|
71
|
+
throw new TypeError("datasets must contain exactly five entries");
|
|
72
|
+
const ids = new Set();
|
|
73
|
+
return datasets.map((dataset, index) => {
|
|
74
|
+
exactKeys(
|
|
75
|
+
dataset,
|
|
76
|
+
["id", "version", "digest", "splitIds"],
|
|
77
|
+
`dataset ${index}`,
|
|
78
|
+
);
|
|
79
|
+
const id = string(dataset.id, `dataset ${index} id`);
|
|
80
|
+
if (ids.has(id)) throw new TypeError(`duplicate dataset id: ${id}`);
|
|
81
|
+
ids.add(id);
|
|
82
|
+
if (!Array.isArray(dataset.splitIds) || dataset.splitIds.length === 0)
|
|
83
|
+
throw new TypeError(`dataset ${id} must contain splitIds`);
|
|
84
|
+
const splitIds = dataset.splitIds.map((item, splitIndex) =>
|
|
85
|
+
string(item, `dataset ${id} split ${splitIndex}`),
|
|
86
|
+
);
|
|
87
|
+
if (new Set(splitIds).size !== splitIds.length)
|
|
88
|
+
throw new TypeError(`dataset ${id} contains duplicate splitIds`);
|
|
89
|
+
return Object.freeze({
|
|
90
|
+
id,
|
|
91
|
+
version: string(dataset.version, `dataset ${id} version`),
|
|
92
|
+
digest: sha(dataset.digest, `dataset ${id} digest`),
|
|
93
|
+
splitIds: Object.freeze(splitIds),
|
|
94
|
+
splitDigest: digest(WIKISKILL_BENCHMARK_PLAN_SCHEMA, splitIds),
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function createWikiSkillBenchmarkPlan(input = {}) {
|
|
100
|
+
exactKeys(
|
|
101
|
+
input,
|
|
102
|
+
[
|
|
103
|
+
"gitCommit",
|
|
104
|
+
"runnerDigest",
|
|
105
|
+
"model",
|
|
106
|
+
"inference",
|
|
107
|
+
"environment",
|
|
108
|
+
"datasets",
|
|
109
|
+
"toolDigest",
|
|
110
|
+
"apiDigest",
|
|
111
|
+
"promptDigest",
|
|
112
|
+
"skillDigest",
|
|
113
|
+
"wikiDigest",
|
|
114
|
+
"seedSchedule",
|
|
115
|
+
"bootstrapSamples",
|
|
116
|
+
],
|
|
117
|
+
"benchmark plan",
|
|
118
|
+
);
|
|
119
|
+
exactKeys(input.model, ["checkpoint", "digest"], "model");
|
|
120
|
+
exactKeys(
|
|
121
|
+
input.environment,
|
|
122
|
+
["containerDigest", "vllmVersion", "hardware"],
|
|
123
|
+
"environment",
|
|
124
|
+
);
|
|
125
|
+
if (
|
|
126
|
+
!input.inference ||
|
|
127
|
+
typeof input.inference !== "object" ||
|
|
128
|
+
Array.isArray(input.inference)
|
|
129
|
+
)
|
|
130
|
+
throw new TypeError("inference must be an object");
|
|
131
|
+
if (!Array.isArray(input.seedSchedule) || input.seedSchedule.length < 3)
|
|
132
|
+
throw new TypeError(
|
|
133
|
+
"seedSchedule must contain at least three independent runs",
|
|
134
|
+
);
|
|
135
|
+
const seedSchedule = input.seedSchedule.map((seed, index) => {
|
|
136
|
+
if (!Number.isSafeInteger(seed) || seed < 0)
|
|
137
|
+
throw new TypeError(
|
|
138
|
+
`seedSchedule ${index} must be a non-negative safe integer`,
|
|
139
|
+
);
|
|
140
|
+
return seed;
|
|
141
|
+
});
|
|
142
|
+
if (new Set(seedSchedule).size !== seedSchedule.length)
|
|
143
|
+
throw new TypeError("seedSchedule must not contain duplicates");
|
|
144
|
+
if (input.bootstrapSamples !== 1_000)
|
|
145
|
+
throw new TypeError("bootstrapSamples must equal 1000");
|
|
146
|
+
const gitCommit = string(input.gitCommit, "gitCommit");
|
|
147
|
+
if (!COMMIT.test(gitCommit))
|
|
148
|
+
throw new TypeError("gitCommit must be a full commit SHA");
|
|
149
|
+
const core = {
|
|
150
|
+
schema: WIKISKILL_BENCHMARK_PLAN_SCHEMA,
|
|
151
|
+
gitCommit,
|
|
152
|
+
runnerDigest: sha(input.runnerDigest, "runnerDigest"),
|
|
153
|
+
model: Object.freeze({
|
|
154
|
+
checkpoint: string(input.model.checkpoint, "model checkpoint"),
|
|
155
|
+
digest: sha(input.model.digest, "model digest"),
|
|
156
|
+
}),
|
|
157
|
+
inference: Object.freeze(JSON.parse(canonical(input.inference))),
|
|
158
|
+
environment: Object.freeze({
|
|
159
|
+
containerDigest: sha(
|
|
160
|
+
input.environment.containerDigest,
|
|
161
|
+
"container digest",
|
|
162
|
+
),
|
|
163
|
+
vllmVersion: string(input.environment.vllmVersion, "vLLM version"),
|
|
164
|
+
hardware: string(input.environment.hardware, "hardware"),
|
|
165
|
+
}),
|
|
166
|
+
datasets: Object.freeze(normalizeDatasets(input.datasets)),
|
|
167
|
+
toolDigest: sha(input.toolDigest, "toolDigest"),
|
|
168
|
+
apiDigest: sha(input.apiDigest, "apiDigest"),
|
|
169
|
+
promptDigest: sha(input.promptDigest, "promptDigest"),
|
|
170
|
+
skillDigest: sha(input.skillDigest, "skillDigest"),
|
|
171
|
+
wikiDigest: sha(input.wikiDigest, "wikiDigest"),
|
|
172
|
+
seedSchedule: Object.freeze(seedSchedule),
|
|
173
|
+
bootstrapSamples: 1_000,
|
|
174
|
+
};
|
|
175
|
+
return Object.freeze({
|
|
176
|
+
...core,
|
|
177
|
+
planDigest: digest(WIKISKILL_BENCHMARK_PLAN_SCHEMA, core),
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function normalizeArm(arm, name) {
|
|
182
|
+
exactKeys(
|
|
183
|
+
arm,
|
|
184
|
+
[
|
|
185
|
+
"score",
|
|
186
|
+
"traceDigest",
|
|
187
|
+
"graderReceiptDigest",
|
|
188
|
+
"failureClass",
|
|
189
|
+
"tokens",
|
|
190
|
+
"cost",
|
|
191
|
+
"latencyMs",
|
|
192
|
+
],
|
|
193
|
+
name,
|
|
194
|
+
);
|
|
195
|
+
if (!FAILURE_CLASSES.has(arm.failureClass))
|
|
196
|
+
throw new TypeError(`${name} failureClass is invalid`);
|
|
197
|
+
return Object.freeze({
|
|
198
|
+
score: finite(arm.score, `${name} score`, { max: 1 }),
|
|
199
|
+
traceDigest: sha(arm.traceDigest, `${name} traceDigest`),
|
|
200
|
+
graderReceiptDigest: sha(
|
|
201
|
+
arm.graderReceiptDigest,
|
|
202
|
+
`${name} graderReceiptDigest`,
|
|
203
|
+
),
|
|
204
|
+
failureClass: arm.failureClass,
|
|
205
|
+
tokens: finite(arm.tokens, `${name} tokens`),
|
|
206
|
+
cost: finite(arm.cost, `${name} cost`),
|
|
207
|
+
latencyMs: finite(arm.latencyMs, `${name} latencyMs`),
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function percentile(sorted, probability) {
|
|
212
|
+
if (sorted.length === 1) return sorted[0];
|
|
213
|
+
const position = (sorted.length - 1) * probability;
|
|
214
|
+
const lower = Math.floor(position);
|
|
215
|
+
const upper = Math.ceil(position);
|
|
216
|
+
const weight = position - lower;
|
|
217
|
+
return sorted[lower] * (1 - weight) + sorted[upper] * weight;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function prngFromDigest(value) {
|
|
221
|
+
let state = Number.parseInt(value.slice(7, 15), 16) || 0x9e3779b9;
|
|
222
|
+
return () => {
|
|
223
|
+
state ^= state << 13;
|
|
224
|
+
state ^= state >>> 17;
|
|
225
|
+
state ^= state << 5;
|
|
226
|
+
return (state >>> 0) / 0x1_0000_0000;
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function mean(values) {
|
|
231
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
export function buildWikiSkillBenchmarkReport({ plan, runs } = {}) {
|
|
235
|
+
if (
|
|
236
|
+
!plan ||
|
|
237
|
+
plan.schema !== WIKISKILL_BENCHMARK_PLAN_SCHEMA ||
|
|
238
|
+
!DIGEST.test(plan.planDigest ?? "")
|
|
239
|
+
)
|
|
240
|
+
throw new TypeError("a canonical benchmark plan is required");
|
|
241
|
+
const expectedPlanDigest = digest(
|
|
242
|
+
WIKISKILL_BENCHMARK_PLAN_SCHEMA,
|
|
243
|
+
Object.fromEntries(
|
|
244
|
+
Object.entries(plan).filter(([key]) => key !== "planDigest"),
|
|
245
|
+
),
|
|
246
|
+
);
|
|
247
|
+
if (expectedPlanDigest !== plan.planDigest)
|
|
248
|
+
throw new Error("benchmark plan digest mismatch");
|
|
249
|
+
if (!Array.isArray(runs) || runs.length !== plan.seedSchedule.length)
|
|
250
|
+
throw new TypeError("runs must exactly cover the seed schedule");
|
|
251
|
+
const expectedCases = new Map(
|
|
252
|
+
plan.datasets.flatMap((dataset) =>
|
|
253
|
+
dataset.splitIds.map((splitId) => [`${dataset.id}\0${splitId}`, dataset]),
|
|
254
|
+
),
|
|
255
|
+
);
|
|
256
|
+
const seenSeeds = new Set();
|
|
257
|
+
const normalizedRuns = runs.map((run, runIndex) => {
|
|
258
|
+
exactKeys(run, ["runId", "seed", "cases"], `run ${runIndex}`);
|
|
259
|
+
if (!plan.seedSchedule.includes(run.seed) || seenSeeds.has(run.seed))
|
|
260
|
+
throw new TypeError(`run ${runIndex} seed is absent or duplicated`);
|
|
261
|
+
seenSeeds.add(run.seed);
|
|
262
|
+
if (!Array.isArray(run.cases) || run.cases.length !== expectedCases.size)
|
|
263
|
+
throw new TypeError(`run ${runIndex} does not cover every split`);
|
|
264
|
+
const seenCases = new Set();
|
|
265
|
+
const cases = run.cases.map((item, caseIndex) => {
|
|
266
|
+
exactKeys(
|
|
267
|
+
item,
|
|
268
|
+
["datasetId", "splitId", "baseline", "skill"],
|
|
269
|
+
`run ${runIndex} case ${caseIndex}`,
|
|
270
|
+
);
|
|
271
|
+
const key = `${item.datasetId}\0${item.splitId}`;
|
|
272
|
+
if (!expectedCases.has(key) || seenCases.has(key))
|
|
273
|
+
throw new TypeError(`run ${runIndex} has an absent or duplicate split`);
|
|
274
|
+
seenCases.add(key);
|
|
275
|
+
return Object.freeze({
|
|
276
|
+
datasetId: item.datasetId,
|
|
277
|
+
splitId: item.splitId,
|
|
278
|
+
baseline: normalizeArm(item.baseline, `run ${runIndex} baseline`),
|
|
279
|
+
skill: normalizeArm(item.skill, `run ${runIndex} skill`),
|
|
280
|
+
});
|
|
281
|
+
});
|
|
282
|
+
cases.sort((left, right) =>
|
|
283
|
+
`${left.datasetId}\0${left.splitId}`.localeCompare(
|
|
284
|
+
`${right.datasetId}\0${right.splitId}`,
|
|
285
|
+
),
|
|
286
|
+
);
|
|
287
|
+
return Object.freeze({
|
|
288
|
+
runId: string(run.runId, `run ${runIndex} runId`),
|
|
289
|
+
seed: run.seed,
|
|
290
|
+
cases: Object.freeze(cases),
|
|
291
|
+
});
|
|
292
|
+
});
|
|
293
|
+
normalizedRuns.sort(
|
|
294
|
+
(left, right) =>
|
|
295
|
+
plan.seedSchedule.indexOf(left.seed) -
|
|
296
|
+
plan.seedSchedule.indexOf(right.seed),
|
|
297
|
+
);
|
|
298
|
+
|
|
299
|
+
const perDataset = plan.datasets.map((dataset) => {
|
|
300
|
+
const observations = normalizedRuns.flatMap((run) =>
|
|
301
|
+
run.cases.filter((item) => item.datasetId === dataset.id),
|
|
302
|
+
);
|
|
303
|
+
const baseline = mean(observations.map((item) => item.baseline.score));
|
|
304
|
+
const skill = mean(observations.map((item) => item.skill.score));
|
|
305
|
+
return Object.freeze({
|
|
306
|
+
datasetId: dataset.id,
|
|
307
|
+
count: observations.length,
|
|
308
|
+
baseline,
|
|
309
|
+
skill,
|
|
310
|
+
delta: skill - baseline,
|
|
311
|
+
});
|
|
312
|
+
});
|
|
313
|
+
const equalWeightBaseline = mean(perDataset.map((item) => item.baseline));
|
|
314
|
+
const equalWeightSkill = mean(perDataset.map((item) => item.skill));
|
|
315
|
+
const deltas = normalizedRuns.flatMap((run) =>
|
|
316
|
+
run.cases.map((item) => item.skill.score - item.baseline.score),
|
|
317
|
+
);
|
|
318
|
+
const random = prngFromDigest(plan.planDigest);
|
|
319
|
+
const bootstrap = Array.from({ length: plan.bootstrapSamples }, () => {
|
|
320
|
+
const sample = Array.from(
|
|
321
|
+
{ length: deltas.length },
|
|
322
|
+
() => deltas[Math.floor(random() * deltas.length)],
|
|
323
|
+
);
|
|
324
|
+
return mean(sample);
|
|
325
|
+
}).sort((left, right) => left - right);
|
|
326
|
+
const measurements = normalizedRuns.flatMap((run) =>
|
|
327
|
+
run.cases.flatMap((item) => [item.baseline, item.skill]),
|
|
328
|
+
);
|
|
329
|
+
const core = {
|
|
330
|
+
schema: WIKISKILL_BENCHMARK_REPORT_SCHEMA,
|
|
331
|
+
planDigest: plan.planDigest,
|
|
332
|
+
gitCommit: plan.gitCommit,
|
|
333
|
+
runnerDigest: plan.runnerDigest,
|
|
334
|
+
datasetDigests: Object.freeze(
|
|
335
|
+
plan.datasets.map(({ id, digest: datasetDigest, splitDigest }) => ({
|
|
336
|
+
id,
|
|
337
|
+
digest: datasetDigest,
|
|
338
|
+
splitDigest,
|
|
339
|
+
})),
|
|
340
|
+
),
|
|
341
|
+
runCount: normalizedRuns.length,
|
|
342
|
+
pairedObservationCount: deltas.length,
|
|
343
|
+
perDataset: Object.freeze(perDataset),
|
|
344
|
+
metrics: Object.freeze({
|
|
345
|
+
equalWeightBaseline,
|
|
346
|
+
equalWeightSkill,
|
|
347
|
+
delta: equalWeightSkill - equalWeightBaseline,
|
|
348
|
+
pairedBootstrap95Ci: Object.freeze([
|
|
349
|
+
percentile(bootstrap, 0.025),
|
|
350
|
+
percentile(bootstrap, 0.975),
|
|
351
|
+
]),
|
|
352
|
+
bootstrapSamples: plan.bootstrapSamples,
|
|
353
|
+
tokens: measurements.reduce((sum, item) => sum + item.tokens, 0),
|
|
354
|
+
cost: measurements.reduce((sum, item) => sum + item.cost, 0),
|
|
355
|
+
latencyMs: Object.freeze({
|
|
356
|
+
p50: percentile(
|
|
357
|
+
measurements.map((item) => item.latencyMs).sort((a, b) => a - b),
|
|
358
|
+
0.5,
|
|
359
|
+
),
|
|
360
|
+
p95: percentile(
|
|
361
|
+
measurements.map((item) => item.latencyMs).sort((a, b) => a - b),
|
|
362
|
+
0.95,
|
|
363
|
+
),
|
|
364
|
+
p99: percentile(
|
|
365
|
+
measurements.map((item) => item.latencyMs).sort((a, b) => a - b),
|
|
366
|
+
0.99,
|
|
367
|
+
),
|
|
368
|
+
}),
|
|
369
|
+
}),
|
|
370
|
+
failureCounts: Object.freeze(
|
|
371
|
+
[...FAILURE_CLASSES].reduce((result, classification) => {
|
|
372
|
+
result[classification] = measurements.filter(
|
|
373
|
+
(item) => item.failureClass === classification,
|
|
374
|
+
).length;
|
|
375
|
+
return result;
|
|
376
|
+
}, {}),
|
|
377
|
+
),
|
|
378
|
+
runs: Object.freeze(normalizedRuns),
|
|
379
|
+
};
|
|
380
|
+
return Object.freeze({
|
|
381
|
+
...core,
|
|
382
|
+
reportDigest: digest(WIKISKILL_BENCHMARK_REPORT_SCHEMA, core),
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export async function executeWikiSkillBenchmark({ plan, runner } = {}) {
|
|
387
|
+
if (
|
|
388
|
+
!plan ||
|
|
389
|
+
plan.schema !== WIKISKILL_BENCHMARK_PLAN_SCHEMA ||
|
|
390
|
+
!DIGEST.test(plan.planDigest ?? "")
|
|
391
|
+
)
|
|
392
|
+
throw new TypeError("a canonical benchmark plan is required");
|
|
393
|
+
if (typeof runner !== "function")
|
|
394
|
+
throw new TypeError("benchmark runner is required");
|
|
395
|
+
const runs = [];
|
|
396
|
+
for (const seed of plan.seedSchedule) {
|
|
397
|
+
const cases = [];
|
|
398
|
+
for (const dataset of plan.datasets) {
|
|
399
|
+
for (const splitId of dataset.splitIds) {
|
|
400
|
+
const common = Object.freeze({
|
|
401
|
+
planDigest: plan.planDigest,
|
|
402
|
+
seed,
|
|
403
|
+
dataset: Object.freeze({
|
|
404
|
+
id: dataset.id,
|
|
405
|
+
version: dataset.version,
|
|
406
|
+
digest: dataset.digest,
|
|
407
|
+
splitId,
|
|
408
|
+
splitDigest: dataset.splitDigest,
|
|
409
|
+
}),
|
|
410
|
+
model: plan.model,
|
|
411
|
+
inference: plan.inference,
|
|
412
|
+
environment: plan.environment,
|
|
413
|
+
toolDigest: plan.toolDigest,
|
|
414
|
+
apiDigest: plan.apiDigest,
|
|
415
|
+
promptDigest: plan.promptDigest,
|
|
416
|
+
runnerDigest: plan.runnerDigest,
|
|
417
|
+
});
|
|
418
|
+
const baseline = await runner(
|
|
419
|
+
Object.freeze({
|
|
420
|
+
...common,
|
|
421
|
+
arm: "no-skill",
|
|
422
|
+
skillDigest: null,
|
|
423
|
+
wikiDigest: null,
|
|
424
|
+
}),
|
|
425
|
+
);
|
|
426
|
+
const skill = await runner(
|
|
427
|
+
Object.freeze({
|
|
428
|
+
...common,
|
|
429
|
+
arm: "skill",
|
|
430
|
+
skillDigest: plan.skillDigest,
|
|
431
|
+
wikiDigest: plan.wikiDigest,
|
|
432
|
+
}),
|
|
433
|
+
);
|
|
434
|
+
cases.push({
|
|
435
|
+
datasetId: dataset.id,
|
|
436
|
+
splitId,
|
|
437
|
+
baseline,
|
|
438
|
+
skill,
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
runs.push({
|
|
443
|
+
runId: digest(WIKISKILL_BENCHMARK_REPORT_SCHEMA, {
|
|
444
|
+
planDigest: plan.planDigest,
|
|
445
|
+
seed,
|
|
446
|
+
}),
|
|
447
|
+
seed,
|
|
448
|
+
cases,
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
return buildWikiSkillBenchmarkReport({ plan, runs });
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
export async function signWikiSkillBenchmarkReport({ report, attestor } = {}) {
|
|
455
|
+
if (
|
|
456
|
+
!report ||
|
|
457
|
+
report.schema !== WIKISKILL_BENCHMARK_REPORT_SCHEMA ||
|
|
458
|
+
!DIGEST.test(report.reportDigest ?? "")
|
|
459
|
+
)
|
|
460
|
+
throw new TypeError("a canonical benchmark report is required");
|
|
461
|
+
if (typeof attestor !== "function")
|
|
462
|
+
throw new TypeError("attestor is required");
|
|
463
|
+
const attestation = await attestor(report.reportDigest);
|
|
464
|
+
if (!attestation || typeof attestation !== "object")
|
|
465
|
+
throw new Error("benchmark attestor returned no attestation");
|
|
466
|
+
return Object.freeze({
|
|
467
|
+
schema: WIKISKILL_BENCHMARK_ENVELOPE_SCHEMA,
|
|
468
|
+
report,
|
|
469
|
+
attestation: Object.freeze({ ...attestation }),
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
export async function projectWikiSkillBenchmarkClaim({
|
|
474
|
+
envelope,
|
|
475
|
+
verifyAttestation,
|
|
476
|
+
} = {}) {
|
|
477
|
+
const hold = Object.freeze({
|
|
478
|
+
provenance: "external-paper-only",
|
|
479
|
+
status: "HOLD",
|
|
480
|
+
reportDigest: null,
|
|
481
|
+
metrics: null,
|
|
482
|
+
});
|
|
483
|
+
if (!envelope) return hold;
|
|
484
|
+
if (
|
|
485
|
+
envelope.schema !== WIKISKILL_BENCHMARK_ENVELOPE_SCHEMA ||
|
|
486
|
+
typeof verifyAttestation !== "function"
|
|
487
|
+
)
|
|
488
|
+
throw new TypeError("signed benchmark envelope and verifier are required");
|
|
489
|
+
const { report } = envelope;
|
|
490
|
+
if (!report || report.schema !== WIKISKILL_BENCHMARK_REPORT_SCHEMA)
|
|
491
|
+
throw new Error("benchmark report is invalid");
|
|
492
|
+
const core = Object.fromEntries(
|
|
493
|
+
Object.entries(report).filter(([key]) => key !== "reportDigest"),
|
|
494
|
+
);
|
|
495
|
+
if (digest(WIKISKILL_BENCHMARK_REPORT_SCHEMA, core) !== report.reportDigest)
|
|
496
|
+
throw new Error("benchmark report digest mismatch");
|
|
497
|
+
if (
|
|
498
|
+
!(await verifyAttestation({
|
|
499
|
+
digest: report.reportDigest,
|
|
500
|
+
attestation: envelope.attestation,
|
|
501
|
+
}))
|
|
502
|
+
)
|
|
503
|
+
throw new Error("benchmark report attestation rejected");
|
|
504
|
+
return Object.freeze({
|
|
505
|
+
provenance: "chainlesschain-measured",
|
|
506
|
+
status: "VERIFIED",
|
|
507
|
+
reportDigest: report.reportDigest,
|
|
508
|
+
metrics: report.metrics,
|
|
509
|
+
});
|
|
510
|
+
}
|
|
@@ -3,10 +3,13 @@
|
|
|
3
3
|
*
|
|
4
4
|
* A lease is cooperative authority for production hosts, not a same-UID
|
|
5
5
|
* sandbox. Every process sharing CHAINLESSCHAIN_HOME observes the same strict
|
|
6
|
-
* state file. A monotonically increasing fencing token prevents
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
6
|
+
* state file. A monotonically increasing fencing token prevents a crashed host
|
|
7
|
+
* from publishing through guarded session-store write paths after a successor
|
|
8
|
+
* takes over. A separate monotonic revocation epoch lets an operator fence a
|
|
9
|
+
* still-live host; every guarded transition re-reads it. An exact local owner
|
|
10
|
+
* may renew after its TTL when the event loop was blocked: no successor can
|
|
11
|
+
* have been admitted while that owner PID remained live, and any revocation or
|
|
12
|
+
* takeover still changes the durable authority tuple and fences the old host.
|
|
10
13
|
*/
|
|
11
14
|
|
|
12
15
|
import fs from "node:fs";
|
|
@@ -440,9 +443,9 @@ export class SessionHostLeaseAuthority {
|
|
|
440
443
|
const incumbent = snapshot.active;
|
|
441
444
|
// Never take authority from a process that the OS still reports as
|
|
442
445
|
// alive, even after its heartbeat TTL. It may be blocked inside an
|
|
443
|
-
// external side effect and resume later.
|
|
444
|
-
//
|
|
445
|
-
//
|
|
446
|
+
// external side effect and resume later. The exact local owner may
|
|
447
|
+
// renew late; takeover additionally requires proven process death so
|
|
448
|
+
// two executors cannot overlap.
|
|
446
449
|
if (incumbent && this._isProcessAlive(incumbent.ownerPid)) {
|
|
447
450
|
throw leaseError(
|
|
448
451
|
SESSION_HOST_LEASE_HELD_CODE,
|
|
@@ -522,13 +525,6 @@ export class SessionHostLeaseAuthority {
|
|
|
522
525
|
);
|
|
523
526
|
}
|
|
524
527
|
const now = Math.trunc(this._now());
|
|
525
|
-
if (snapshot.active.expiresAtMs <= now) {
|
|
526
|
-
throw leaseError(
|
|
527
|
-
SESSION_HOST_LEASE_FENCED_CODE,
|
|
528
|
-
`Session host lease expired for ${id}`,
|
|
529
|
-
{ sessionId: id, fencingToken: record.active.fencingToken },
|
|
530
|
-
);
|
|
531
|
-
}
|
|
532
528
|
const next = {
|
|
533
529
|
...snapshot.active,
|
|
534
530
|
renewedAtMs: now,
|
|
@@ -898,8 +894,7 @@ export class SessionHostLeaseAuthority {
|
|
|
898
894
|
record &&
|
|
899
895
|
(record.released ||
|
|
900
896
|
record.active.revocationEpoch !== snapshot.revocationEpoch ||
|
|
901
|
-
!sameLease(snapshot.active, record.active)
|
|
902
|
-
snapshot.active.expiresAtMs <= now)
|
|
897
|
+
!sameLease(snapshot.active, record.active))
|
|
903
898
|
) {
|
|
904
899
|
const error = leaseError(
|
|
905
900
|
SESSION_HOST_LEASE_FENCED_CODE,
|
|
@@ -368,12 +368,12 @@ function resolveBackgroundShellWorkspacePaths(
|
|
|
368
368
|
|
|
369
369
|
// ─── Background shell tasks ────────────────────────────────────────────────
|
|
370
370
|
//
|
|
371
|
-
// run_shell
|
|
372
|
-
//
|
|
373
|
-
//
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
371
|
+
// run_shell remains synchronous by default for compatibility. Persistent
|
|
372
|
+
// stream hosts opt into an async foreground path so commands cannot block the
|
|
373
|
+
// session heartbeat or protocol pump. When the model passes
|
|
374
|
+
// run_in_background:true the command is spawned instead, returns a task_id
|
|
375
|
+
// immediately, and streams its output into this registry. The agent then polls
|
|
376
|
+
// completion + incremental output via the check_shell tool — the
|
|
377
377
|
// run_in_background + BashOutput pattern from Claude Code.
|
|
378
378
|
//
|
|
379
379
|
// In-memory, process-lifetime: a task_id is only valid within the agent process
|
|
@@ -384,6 +384,22 @@ const MAX_BG_BUFFER = 1024 * 1024; // 1 MB retained tail per stream
|
|
|
384
384
|
const _backgroundShellTasks = new Map();
|
|
385
385
|
let _backgroundTaskSeq = 0;
|
|
386
386
|
|
|
387
|
+
function _runForegroundProcessAsync(file, args, options) {
|
|
388
|
+
return new Promise((resolve, reject) => {
|
|
389
|
+
try {
|
|
390
|
+
broker.execFile(file, args, options, (error, stdout, stderr) => {
|
|
391
|
+
if (error) {
|
|
392
|
+
reject(error);
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
resolve({ stdout: stdout ?? "", stderr: stderr ?? "" });
|
|
396
|
+
});
|
|
397
|
+
} catch (error) {
|
|
398
|
+
reject(error);
|
|
399
|
+
}
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
|
|
387
403
|
function _newBgStream() {
|
|
388
404
|
return { buf: "", total: 0, dropped: 0, cursor: 0 };
|
|
389
405
|
}
|
|
@@ -3600,6 +3616,7 @@ export async function executeTool(name, args, context = {}) {
|
|
|
3600
3616
|
context.subtreeInstructionScope || context.sessionId || "__legacy__",
|
|
3601
3617
|
shellPolicyOverrides: context.shellPolicyOverrides || null,
|
|
3602
3618
|
classifyAllShell: context.classifyAllShell === true,
|
|
3619
|
+
nonBlockingShell: context.nonBlockingShell === true,
|
|
3603
3620
|
approvalGate: context.approvalGate || null,
|
|
3604
3621
|
shellConfirm: context.shellConfirm || null,
|
|
3605
3622
|
additionalDirectories: context.additionalDirectories || null,
|
|
@@ -5168,6 +5185,7 @@ async function executeToolInner(
|
|
|
5168
5185
|
// (spawn_sub_agent / isolated run_skill) as their parent_id.
|
|
5169
5186
|
hookTraceId = null,
|
|
5170
5187
|
skillLifecycleMode = "active",
|
|
5188
|
+
nonBlockingShell = false,
|
|
5171
5189
|
},
|
|
5172
5190
|
) {
|
|
5173
5191
|
const localToolDescriptor =
|
|
@@ -6637,7 +6655,23 @@ async function executeToolInner(
|
|
|
6637
6655
|
auditContext: createShellProcessAuditContext(),
|
|
6638
6656
|
...processProvenance,
|
|
6639
6657
|
};
|
|
6640
|
-
if (pluginBinInvocation) {
|
|
6658
|
+
if (pluginBinInvocation && nonBlockingShell) {
|
|
6659
|
+
const res = await _runForegroundProcessAsync(
|
|
6660
|
+
executionPlugin.sandboxExecution?.kind === "strict-plugin-node-bin"
|
|
6661
|
+
? executionPlugin.sandboxExecution.runtimePath
|
|
6662
|
+
: executionPlugin.command,
|
|
6663
|
+
executionPlugin.argv,
|
|
6664
|
+
{
|
|
6665
|
+
...brokerExecOpts,
|
|
6666
|
+
cwd:
|
|
6667
|
+
executionPlugin.sandboxExecution?.workingDirectory ||
|
|
6668
|
+
brokerExecOpts.cwd,
|
|
6669
|
+
shell: false,
|
|
6670
|
+
windowsHide: true,
|
|
6671
|
+
},
|
|
6672
|
+
);
|
|
6673
|
+
output = res.stdout;
|
|
6674
|
+
} else if (pluginBinInvocation) {
|
|
6641
6675
|
const res = broker.spawnSync(
|
|
6642
6676
|
executionPlugin.sandboxExecution?.kind === "strict-plugin-node-bin"
|
|
6643
6677
|
? executionPlugin.sandboxExecution.runtimePath
|
|
@@ -6663,8 +6697,25 @@ async function executeToolInner(
|
|
|
6663
6697
|
throw e;
|
|
6664
6698
|
}
|
|
6665
6699
|
output = res.stdout ?? "";
|
|
6700
|
+
} else if (shellInv.useDefaultShell && nonBlockingShell) {
|
|
6701
|
+
const res = await _runForegroundProcessAsync(args.command, [], {
|
|
6702
|
+
...brokerExecOpts,
|
|
6703
|
+
shell: true,
|
|
6704
|
+
windowsHide: true,
|
|
6705
|
+
});
|
|
6706
|
+
output = res.stdout;
|
|
6666
6707
|
} else if (shellInv.useDefaultShell) {
|
|
6667
6708
|
output = broker.execSync(args.command, brokerExecOpts);
|
|
6709
|
+
} else if (nonBlockingShell) {
|
|
6710
|
+
const res = await _runForegroundProcessAsync(
|
|
6711
|
+
shellInv.file,
|
|
6712
|
+
shellInv.argv,
|
|
6713
|
+
{
|
|
6714
|
+
...brokerExecOpts,
|
|
6715
|
+
windowsHide: true,
|
|
6716
|
+
},
|
|
6717
|
+
);
|
|
6718
|
+
output = res.stdout;
|
|
6668
6719
|
} else {
|
|
6669
6720
|
// PowerShell route: explicit argv, no intermediate default shell.
|
|
6670
6721
|
// Reproduce execSync's contract so the shared catch shapes errors
|
|
@@ -13334,6 +13385,9 @@ export async function* agentLoop(messages, options) {
|
|
|
13334
13385
|
// verification allowlist (npm test / rg / …) is classified through the
|
|
13335
13386
|
// ApprovalGate instead of fast-pathed, so no shell command auto-runs.
|
|
13336
13387
|
classifyAllShell: options.classifyAllShell || false,
|
|
13388
|
+
// Persistent stream hosts must not block their own session-lease heartbeat
|
|
13389
|
+
// while a foreground shell command (notably a networked git push) runs.
|
|
13390
|
+
nonBlockingShell: options.nonBlockingShell === true,
|
|
13337
13391
|
approvalGate: hermeticExecution ? null : options.approvalGate || null,
|
|
13338
13392
|
shellConfirm: hermeticExecution ? null : options.shellConfirm || null,
|
|
13339
13393
|
// Interactive sessions (the REPL) set this so run_code is gated through the
|
|
@@ -4580,6 +4580,9 @@ async function runAgentHeadlessStreamInWorkspace(
|
|
|
4580
4580
|
onUsageSettlement: persistUsageEvent,
|
|
4581
4581
|
onToolCallBoundary: persistChildToolBoundary,
|
|
4582
4582
|
onToolCallSettlement: persistChildToolSettlement,
|
|
4583
|
+
// Keep the session heartbeat and protocol pumps live while a
|
|
4584
|
+
// foreground command such as a networked git push is still running.
|
|
4585
|
+
nonBlockingShell: true,
|
|
4583
4586
|
// Resume-degenerate role merge for the first live model call only.
|
|
4584
4587
|
mergeRoles: mergeRolesThisTurn,
|
|
4585
4588
|
},
|