@iowarp/clio-coder 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +4 -4
- package/CHANGELOG.md +35 -0
- package/CONTRIBUTING.md +9 -0
- package/README.md +12 -9
- package/ROADMAP.md +30 -5
- package/dist/{acp-KS7ARGV6.js → acp-F6PGOFZ4.js} +2 -2
- package/dist/{agents-YOGJ2YEJ.js → agents-ZHP4R6EX.js} +22 -21
- package/dist/assets/codewiki.json +1 -1
- package/dist/{auth-CEDCO6O6.js → auth-N75U2H3Z.js} +4 -4
- package/dist/{chunk-D3KWFOUT.js → chunk-23UIXH6Z.js} +3 -3
- package/dist/{chunk-ZR5YA6ZF.js → chunk-36XVK5P5.js} +5 -5
- package/dist/{chunk-AVSR2HKI.js → chunk-3DZ4LPLS.js} +6 -6
- package/dist/{chunk-LYJJ6546.js → chunk-3GSPCWFX.js} +12 -12
- package/dist/{chunk-CRHPCPZP.js → chunk-4KC4CHKJ.js} +2 -2
- package/dist/{chunk-4VDHLCF5.js → chunk-4WJBWKMI.js} +2 -2
- package/dist/{chunk-KBUTSJSR.js → chunk-4XBYHXGK.js} +2 -2
- package/dist/{chunk-6W75D37F.js → chunk-53P775GE.js} +6 -3
- package/dist/{chunk-XZWSP67B.js → chunk-5APYAXCB.js} +2 -2
- package/dist/{chunk-D65E25JG.js → chunk-67HPYNTJ.js} +2 -2
- package/dist/{chunk-5JQR5B4D.js → chunk-6KV4KCQZ.js} +4 -4
- package/dist/{chunk-V4XJWDHR.js → chunk-6UKFRTXI.js} +2 -1
- package/dist/{chunk-NDPF2N2L.js → chunk-7DW7CJW2.js} +2 -2
- package/dist/{chunk-5KQEUIWM.js → chunk-7LUEQODV.js} +4 -4
- package/dist/{chunk-LNOXCGN2.js → chunk-7UFDIOBT.js} +3 -3
- package/dist/{chunk-DXSETN72.js → chunk-7WUGNN43.js} +2 -2
- package/dist/{chunk-V35KZDUZ.js → chunk-AA3GRZWS.js} +2 -2
- package/dist/{chunk-X4CEFOSG.js → chunk-ADO3OQIC.js} +4 -4
- package/dist/{chunk-UX2L5CUD.js → chunk-AI36V3ON.js} +3 -3
- package/dist/{chunk-B7GY6QER.js → chunk-B26OUMJW.js} +2 -2
- package/dist/{chunk-LAR7DX5E.js → chunk-B5TASTCK.js} +153 -62
- package/dist/{chunk-TGWZXJ7L.js → chunk-BEMQK5Q7.js} +92 -84
- package/dist/{chunk-RNDCROSG.js → chunk-CNXU7TR2.js} +3 -3
- package/dist/{chunk-3JLTRPN7.js → chunk-CVMUXT64.js} +35 -66
- package/dist/{chunk-733ZBT5P.js → chunk-E74MJF6G.js} +4 -4
- package/dist/{chunk-T2MDZES3.js → chunk-EBMTBE76.js} +3 -3
- package/dist/{chunk-OZ76TLGU.js → chunk-EEITU7S6.js} +2 -2
- package/dist/{chunk-GRHOXDKB.js → chunk-ELDJWJKM.js} +97 -14
- package/dist/{chunk-GCQFN7YI.js → chunk-EMYEQLWO.js} +2 -2
- package/dist/{chunk-LTWH2ACR.js → chunk-EQPFEDFV.js} +222 -127
- package/dist/{chunk-TDGYTWM6.js → chunk-F4I3H56H.js} +2 -2
- package/dist/chunk-F6KCJO3U.js +523 -0
- package/dist/{chunk-FGSQHUCK.js → chunk-F6VKWTTY.js} +5 -5
- package/dist/{chunk-UJD5MPP6.js → chunk-FLNXQQ5B.js} +2 -2
- package/dist/{chunk-EX57VVWW.js → chunk-FOSRJ2VZ.js} +4 -4
- package/dist/{chunk-ZARZ4POR.js → chunk-G5ZDQQE3.js} +3 -3
- package/dist/{chunk-HQVN7G4F.js → chunk-G62JBPXN.js} +7 -7
- package/dist/{chunk-ZJPORZTC.js → chunk-GIAI7A2K.js} +2 -2
- package/dist/chunk-GPTR6OTU.js +2551 -0
- package/dist/{chunk-EZD2BXGP.js → chunk-GUFQPTQK.js} +2 -2
- package/dist/{chunk-OPD7GL6F.js → chunk-H6FQVNZC.js} +8 -7
- package/dist/chunk-H7REJ7L5.js +80 -0
- package/dist/{chunk-EYTLV3W3.js → chunk-HTWTORF7.js} +2 -2
- package/dist/{chunk-DF6PI6GN.js → chunk-I6UKQPGS.js} +3 -3
- package/dist/{chunk-ISHUS7HC.js → chunk-JRW236JT.js} +3 -3
- package/dist/{chunk-DE2LU267.js → chunk-KKAJGT3B.js} +3 -2
- package/dist/{chunk-OV62D6KO.js → chunk-KQMYL5GR.js} +3 -3
- package/dist/{chunk-QDUUOZ3S.js → chunk-KWMFC7FN.js} +3 -3
- package/dist/{chunk-R7FGRZNF.js → chunk-LXYHCCIZ.js} +9 -9
- package/dist/{chunk-NFSK2VNU.js → chunk-LZMTVVN4.js} +2 -2
- package/dist/{chunk-62UK7AGW.js → chunk-M5PMRZSR.js} +397 -44
- package/dist/chunk-MMZMM6SW.js +131 -0
- package/dist/{chunk-FFYTC2KQ.js → chunk-MNABNEBX.js} +2 -2
- package/dist/{chunk-XMDBTYCH.js → chunk-N3CWOGVF.js} +3 -3
- package/dist/{chunk-VYRQHORQ.js → chunk-NMQSO6Z6.js} +4 -3
- package/dist/{chunk-R2UUNSNQ.js → chunk-O3ZAF7UY.js} +18 -5
- package/dist/{chunk-4SYHGVDE.js → chunk-OQNCLZI4.js} +4 -4
- package/dist/{chunk-EG4ARXPC.js → chunk-OX6QTA4N.js} +3 -3
- package/dist/{chunk-P4GNZPXO.js → chunk-OYUS3UZP.js} +2 -2
- package/dist/{chunk-OQ2EOAHA.js → chunk-PF36KOGI.js} +291 -112
- package/dist/{chunk-55RMDDFM.js → chunk-PGQG7ZMW.js} +2 -2
- package/dist/{chunk-CBAQPTDU.js → chunk-PIU6BXXW.js} +2 -2
- package/dist/{chunk-EKV4UBCM.js → chunk-PZBQUJ2F.js} +2 -2
- package/dist/{chunk-7RET77VB.js → chunk-Q2HKY32Y.js} +2 -2
- package/dist/{chunk-U6YTUVOX.js → chunk-Q4RMNWMZ.js} +4 -4
- package/dist/chunk-Q6FRR3AQ.js +48 -0
- package/dist/{chunk-6RJMYYIE.js → chunk-QJSIAJBY.js} +3 -3
- package/dist/{chunk-DOUISWEE.js → chunk-QOJ4ERMU.js} +5 -5
- package/dist/{chunk-IFPDAD7R.js → chunk-RWSRKXLT.js} +3 -3
- package/dist/{chunk-T33IYTZM.js → chunk-SAASVMIY.js} +3 -3
- package/dist/{chunk-2DTIWSAW.js → chunk-SEQDH6JC.js} +3 -3
- package/dist/{chunk-BRCOSE7O.js → chunk-SZUO5BSE.js} +2 -2
- package/dist/{chunk-HOEDA42N.js → chunk-T7HZ7KJF.js} +2 -2
- package/dist/{chunk-OJXOA6YU.js → chunk-TL6LNERH.js} +2 -2
- package/dist/{chunk-UCINIRHW.js → chunk-TPCZXWLT.js} +3 -3
- package/dist/{chunk-ZUKUCZYZ.js → chunk-TQ2KTH4A.js} +2 -2
- package/dist/{chunk-4WOOTKFD.js → chunk-U3FETNQB.js} +341 -20
- package/dist/{chunk-5B3RTYAK.js → chunk-UGDSB4AN.js} +2 -2
- package/dist/{chunk-VKHLUZNO.js → chunk-UZ7YBL43.js} +3 -3
- package/dist/{chunk-QNUQ7K7D.js → chunk-VKMSEO7Y.js} +4 -4
- package/dist/{chunk-IN7DGBVS.js → chunk-VLX5VZ35.js} +5 -5
- package/dist/{chunk-SJGS3GDI.js → chunk-WAGBMMNX.js} +5 -5
- package/dist/{chunk-BFOSV5EZ.js → chunk-WX2YCH7F.js} +2 -2
- package/dist/{chunk-3LT34CAM.js → chunk-XBIGUILU.js} +32 -17
- package/dist/{chunk-XDHUDE5K.js → chunk-XKYBFRWR.js} +5 -5
- package/dist/{chunk-U5QU5ZOD.js → chunk-XYPWFSU5.js} +3 -3
- package/dist/{chunk-YBUECLAF.js → chunk-YEJQDODA.js} +2 -2
- package/dist/{chunk-YQ6XEFVK.js → chunk-Z74OGONW.js} +39 -7
- package/dist/{chunk-I4ELN5BX.js → chunk-ZTNOEJRI.js} +2 -2
- package/dist/cli/index.js +26 -26
- package/dist/{clio-K2PBVCEM.js → clio-4IKFE25F.js} +2 -2
- package/dist/{clio-context-tools-GR4OHFSO.js → clio-context-tools-OHJXYX4R.js} +25 -23
- package/dist/{code-nav-4X3OI6DK.js → code-nav-XPM3MY5I.js} +7 -7
- package/dist/{config-Q5J7UBAB.js → config-343YS25T.js} +43 -41
- package/dist/{config-graph-P3555574.js → config-graph-WI263KDX.js} +43 -41
- package/dist/{configure-KIJ32ZO6.js → configure-GZDG4HER.js} +21 -21
- package/dist/{context-LFBQOSXT.js → context-4OFO7N7Y.js} +12 -12
- package/dist/{context-WBH3KSVR.js → context-JLK2RLOG.js} +42 -40
- package/dist/{context-NNLBES2Y.js → context-O6DIWCF6.js} +23 -21
- package/dist/{context-clear-YOHMQ6SQ.js → context-clear-MR7KOGQI.js} +42 -40
- package/dist/{context-working-set-MRAEVDNV.js → context-working-set-YZAD3KWO.js} +14 -13
- package/dist/{data-tool-JJPBGMQJ.js → data-tool-EKD7DFHG.js} +7 -7
- package/dist/{detail-RHJOTGN4.js → detail-YZPHFEVJ.js} +43 -41
- package/dist/{dispatch-runner-NC7PYNIO.js → dispatch-runner-VKVXVTWJ.js} +44 -42
- package/dist/{doctor-S5KZ2GTN.js → doctor-53AVQXAO.js} +17 -16
- package/dist/{doctor-deep-2EJQSHIZ.js → doctor-deep-BLRB2NCG.js} +5 -5
- package/dist/{eval-3D6M36N7.js → eval-P3BQSLQE.js} +21 -20
- package/dist/{evidence-XNVPPLVK.js → evidence-6LLI6SZH.js} +44 -42
- package/dist/{evidence-6IYN52ET.js → evidence-HTN3JLUO.js} +42 -40
- package/dist/{evidence-JVECPI3F.js → evidence-NJRE3R5H.js} +42 -40
- package/dist/{evolve-TLPUDJDM.js → evolve-WWZHJVJP.js} +42 -40
- package/dist/{fleet-WKVBF5SZ.js → fleet-GGZ4BVJE.js} +64 -62
- package/dist/{fleet-6RJWIOEO.js → fleet-QG3IIWWM.js} +44 -42
- package/dist/{fleet-commands-L6WFT7KZ.js → fleet-commands-J2ZIKISQ.js} +7 -7
- package/dist/{fleet-decisions-BTQMWU5A.js → fleet-decisions-Y2Q7375Q.js} +8 -8
- package/dist/{fleet-graph-553DZ67Q.js → fleet-graph-TCBCCCBP.js} +11 -11
- package/dist/{fleet-inspect-236FZPRB.js → fleet-inspect-KOLKBC56.js} +44 -42
- package/dist/{fleet-preflight-NZQHABWO.js → fleet-preflight-44UNCC4Y.js} +27 -25
- package/dist/{fleet-validate-7NLAE26R.js → fleet-validate-K2FXRZVA.js} +13 -13
- package/dist/{fleet-verify-E3RO6A5W.js → fleet-verify-AS444WDW.js} +42 -40
- package/dist/{fleet-view-KMZQY7YJ.js → fleet-view-X4JU4J6O.js} +43 -41
- package/dist/gui/ops-worker.js +9 -9
- package/dist/gui/reads-worker.js +9 -8
- package/dist/{init-DV5B6HQG.js → init-VSOWFHVL.js} +54 -52
- package/dist/{interactive-OXGYZYAS.js → interactive-JRCBENLZ.js} +1733 -577
- package/dist/{interop-J7UOBOSF.js → interop-JR6BCY7R.js} +10 -10
- package/dist/{inventory-OEGIL6PY.js → inventory-3JUGLVJI.js} +43 -41
- package/dist/{library-ROZAMUIL.js → library-7BOH3Y7N.js} +15 -15
- package/dist/{library-B3UPGY4V.js → library-7PFONXUI.js} +9 -9
- package/dist/{library-MXQA5DLT.js → library-TUYZMFIA.js} +7 -7
- package/dist/{library-import-2LTWOVBD.js → library-import-TJVXKLDK.js} +10 -10
- package/dist/{library-inventory-LJ3XHLWO.js → library-inventory-MD3AYTP4.js} +9 -9
- package/dist/{library-validation-DARHHUNW.js → library-validation-ZDLITEA3.js} +7 -7
- package/dist/{mcp-2GSPYQ54.js → mcp-EFCWUPUR.js} +3 -3
- package/dist/{memory-J3STUUIK.js → memory-Q2YVOKK5.js} +42 -40
- package/dist/{models-Y3GMFJ2V.js → models-M7BOCW7F.js} +17 -16
- package/dist/{monitor-LHNH577J.js → monitor-2HD3MC67.js} +47 -45
- package/dist/{orchestrator-DTAPXULA.js → orchestrator-OUW3ZN6Y.js} +1894 -226
- package/dist/{panes-A743ONZ5.js → panes-TTPPSYCL.js} +3 -3
- package/dist/{preload-OJBMAIXF.js → preload-A47C2NUF.js} +42 -40
- package/dist/{providers-CMETPZNS.js → providers-T43W7EL4.js} +4 -4
- package/dist/{resources-FY6XYFN2.js → resources-TP6X2V6W.js} +20 -11
- package/dist/{run-H64MEVTA.js → run-LDSBPMCS.js} +61 -59
- package/dist/{share-CAFMMQLM.js → share-URHKKMHR.js} +9 -9
- package/dist/{skills-ERKQUC7L.js → skills-E477MOOO.js} +13 -11
- package/dist/{skills-eval-CQ23MYQK.js → skills-eval-AIBR2L7H.js} +583 -93
- package/dist/{skills-inventory-TVUYJDVN.js → skills-inventory-HLMKY3R5.js} +13 -11
- package/dist/{slash-commands-ROG3KIFJ.js → slash-commands-LIIZKA5Y.js} +28 -26
- package/dist/{startup-background-KDWCQMWT.js → startup-background-JR2KDK6R.js} +43 -41
- package/dist/{steer-KKYXJSH4.js → steer-D7RTAPCM.js} +3 -3
- package/dist/{system-ZDW44ZGX.js → system-X2P5N4IY.js} +14 -13
- package/dist/{targets-AJWA6E7Z.js → targets-VFFAKUFM.js} +25 -24
- package/dist/{tasks-6L2JJRC5.js → tasks-ERAWZA22.js} +6 -6
- package/dist/{terminal-lease-YFUSDPAD.js → terminal-lease-V2N2BIKQ.js} +2 -2
- package/dist/{trace-LA5MVR4Y.js → trace-MA5BT7XY.js} +3 -3
- package/dist/{usage-XSILDHKF.js → usage-WYFY2L6M.js} +50 -48
- package/dist/{verifiers-ECA5GIHL.js → verifiers-TYIW2XKF.js} +7 -7
- package/dist/{verify-R6QQCCEB.js → verify-242UFYTP.js} +6 -6
- package/dist/{web-fetch-PVZNMBOB.js → web-fetch-LRAGYU6Y.js} +3 -3
- package/dist/{wiki-generate-RP22WP5K.js → wiki-generate-OFKOX2T6.js} +53 -51
- package/dist/worker/entry.js +31 -29
- package/docs/architecture/architecture.md +1 -0
- package/docs/architecture/context-engine.md +4 -2
- package/docs/architecture/observability.md +6 -4
- package/docs/architecture/prompt-envelope-and-tools.md +1 -1
- package/docs/architecture/session-lifecycle.md +1 -1
- package/docs/architecture/tui-design.md +12 -9
- package/docs/gui/parity/02-slash-and-surfaces.md +1 -1
- package/docs/guide/commands-and-modes.md +91 -5
- package/docs/guide/configuration-and-targets.md +2 -2
- package/docs/guide/context-continuity.md +49 -0
- package/docs/guide/proactive-memory.md +6 -3
- package/library/registry.yaml +292 -32
- package/library/skills/README.md +27 -2
- package/library/skills/context/context-handoff/SKILL.md +10 -4
- package/library/skills/meta/clio-coder-dev/SKILL.md +92 -88
- package/library/skills/meta/clio-coder-dev/evals.md +51 -46
- package/library/skills/meta/clio-coder-dev/plugin.json +2 -2
- package/library/skills/meta/clio-coder-dev/references/change-map.md +49 -0
- package/library/skills/meta/clio-coder-test/SKILL.md +89 -142
- package/library/skills/meta/clio-coder-test/evals.md +58 -62
- package/library/skills/meta/clio-coder-test/plugin.json +2 -2
- package/library/skills/meta/clio-coder-test/references/harness.md +3 -3
- package/library/skills/meta/clio-coder-test/references/lifecycle-validation.md +30 -0
- package/library/skills/meta/clio-coder-test/references/test-map.md +60 -89
- package/library/skills/registry.yaml +5 -5
- package/library/skills/skill-marketplace.json +12 -14
- package/package.json +1 -1
- package/src/cli/skills-eval.ts +1064 -54
- package/src/cli/usage.ts +2 -2
- package/src/core/clio-repo.ts +3 -0
- package/src/core/tool-names.ts +1 -0
- package/src/domains/context/budget/inspection.ts +9 -0
- package/src/domains/context/budget/live-view.ts +418 -0
- package/src/domains/context/budget/pressure.ts +393 -0
- package/src/domains/context/budget/request-fit.ts +15 -0
- package/src/domains/evidence/build.ts +24 -0
- package/src/domains/gateway/mcp/client.ts +1 -1
- package/src/domains/memory/commit-state.ts +160 -0
- package/src/domains/memory/operations.ts +18 -10
- package/src/domains/memory/prompt-cache.ts +95 -0
- package/src/domains/memory/prompt-section.ts +50 -7
- package/src/domains/memory/relevance.ts +93 -0
- package/src/domains/memory/restoration.ts +78 -0
- package/src/domains/memory/store.ts +39 -1
- package/src/domains/middleware/memory-intervention.ts +116 -24
- package/src/domains/observability/background-memory-usage.ts +1 -1
- package/src/domains/observability/cost.ts +3 -3
- package/src/domains/observability/extension.ts +1 -1
- package/src/domains/observability/metrics.ts +1 -1
- package/src/domains/prompts/compiler.ts +1 -1
- package/src/domains/prompts/extension.ts +38 -1
- package/src/domains/quota/anthropic-max-provider.ts +134 -0
- package/src/domains/quota/anthropic-usage.ts +220 -0
- package/src/domains/quota/antigravity-provider.ts +339 -0
- package/src/domains/quota/cache.ts +87 -0
- package/src/domains/quota/claude-code-provider.ts +169 -0
- package/src/domains/quota/codex-provider.ts +237 -0
- package/src/domains/quota/presentation.ts +240 -0
- package/src/domains/quota/registry.ts +23 -0
- package/src/domains/quota/service.ts +93 -0
- package/src/domains/quota/summary-feed.ts +86 -0
- package/src/domains/quota/types.ts +84 -0
- package/src/domains/resources/index.ts +11 -0
- package/src/domains/resources/skills/catalog-view.ts +571 -0
- package/src/domains/resources/skills/lexical-match.ts +136 -0
- package/src/domains/resources/skills/loader.ts +38 -0
- package/src/domains/resources/skills/promotion.ts +1 -55
- package/src/domains/resources/skills/provenance-pin.ts +50 -20
- package/src/domains/safety/action-classifier.ts +1 -0
- package/src/domains/session/compaction/branch-summary.ts +4 -1
- package/src/domains/session/compaction/compact.ts +18 -2
- package/src/domains/session/compaction/cut-point.ts +20 -1
- package/src/domains/session/compaction/tokens.ts +48 -3
- package/src/domains/session/context-accounting.ts +8 -1
- package/src/domains/session/continuity/carry.ts +59 -0
- package/src/domains/session/continuity/contract.ts +592 -0
- package/src/domains/session/continuity/evidence.ts +290 -0
- package/src/domains/session/continuity/fold.ts +1075 -0
- package/src/domains/session/continuity/note.ts +79 -0
- package/src/domains/session/continuity/operator-request.ts +104 -0
- package/src/domains/session/continuity/persistence.ts +408 -0
- package/src/domains/session/continuity/ports.ts +231 -0
- package/src/domains/session/continuity/projection.ts +538 -0
- package/src/domains/session/continuity/validate.ts +352 -0
- package/src/domains/session/entries.ts +50 -3
- package/src/domains/session/index.ts +48 -0
- package/src/domains/session/migrations/index.ts +7 -3
- package/src/domains/session/tree/fork.ts +15 -1
- package/src/domains/session/usage.ts +2 -2
- package/src/engine/acp/commands.ts +1 -1
- package/src/engine/agent.ts +133 -9
- package/src/engine/session.ts +13 -6
- package/src/entry/orchestrator.ts +116 -33
- package/src/interactive/chat-loop-messages.ts +2 -2
- package/src/interactive/chat-loop.ts +219 -63
- package/src/interactive/chat-renderer.ts +99 -8
- package/src/interactive/context-overlay.ts +24 -3
- package/src/interactive/continuity-controller.ts +526 -0
- package/src/interactive/dispatch-board.ts +21 -3
- package/src/interactive/footer/dashboard.ts +25 -1
- package/src/interactive/footer/key-hints.ts +2 -2
- package/src/interactive/footer/pages.ts +70 -27
- package/src/interactive/footer/widgets.ts +32 -7
- package/src/interactive/footer-panel.ts +1 -1
- package/src/interactive/interactive-application.ts +7 -2
- package/src/interactive/interactive-input-runtime.ts +2 -2
- package/src/interactive/interactive-presentation.ts +15 -0
- package/src/interactive/interactive-slash-runtime.ts +45 -8
- package/src/interactive/interactive-tickers.ts +7 -1
- package/src/interactive/model-session-replay.ts +130 -3
- package/src/interactive/output-reserve.ts +35 -0
- package/src/interactive/overlay-general-openers.ts +12 -8
- package/src/interactive/overlay-key-routing.ts +6 -9
- package/src/interactive/overlay-lifecycle.ts +10 -6
- package/src/interactive/overlay-session-lifecycle.ts +21 -14
- package/src/interactive/quota-view.ts +229 -0
- package/src/interactive/session-last-turn.ts +1 -1
- package/src/interactive/session-transcript.ts +2 -1
- package/src/interactive/session-usage-reseed.ts +2 -2
- package/src/interactive/side-question.ts +2 -2
- package/src/interactive/slash-commands.ts +28 -9
- package/src/interactive/turn-context.ts +686 -63
- package/src/interactive/turn-middleware.ts +39 -6
- package/src/interactive/turn-persistence.ts +15 -2
- package/src/interactive/turn-prewarm.ts +1 -1
- package/src/interactive/turn-runtime.ts +48 -5
- package/src/interactive/{cost-overlay.ts → usage-overlay.ts} +121 -33
- package/src/interactive/welcome-dashboard.ts +26 -3
- package/src/tools/agent-tools.ts +3 -1
- package/src/tools/bootstrap.ts +6 -0
- package/src/tools/builtin-tool-catalog.ts +9 -0
- package/src/tools/context/index.ts +148 -113
- package/src/tools/context/surface.ts +11 -7
- package/src/tools/core-bootstrap.ts +4 -0
- package/src/tools/observation.ts +8 -2
- package/src/tools/policy.ts +3 -0
- package/src/tools/self-compact.ts +31 -0
- package/src/tools/surface.ts +1 -0
- package/src/tools/tasks.ts +1 -1
- package/dist/chunk-CTFPFW3H.js +0 -44
- package/dist/chunk-EOJPDNUP.js +0 -1325
- package/dist/chunk-I4Y4WKZR.js +0 -386
|
@@ -1,39 +1,40 @@
|
|
|
1
1
|
import { createRequire as __clioCreateRequire } from "node:module"; const require = __clioCreateRequire(import.meta.url);
|
|
2
2
|
import "./chunk-GGPF6LLE.js";
|
|
3
3
|
import {
|
|
4
|
+
checkSkillDrift,
|
|
4
5
|
parseSkillEvals
|
|
5
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-U3FETNQB.js";
|
|
6
7
|
import {
|
|
7
8
|
buildEvalEvidence,
|
|
8
9
|
summarizeEvalResults
|
|
9
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-EQPFEDFV.js";
|
|
10
11
|
import "./chunk-PECBKCDK.js";
|
|
11
|
-
import "./chunk-
|
|
12
|
-
import "./chunk-
|
|
13
|
-
import "./chunk-
|
|
14
|
-
import "./chunk-
|
|
12
|
+
import "./chunk-KQMYL5GR.js";
|
|
13
|
+
import "./chunk-ZTNOEJRI.js";
|
|
14
|
+
import "./chunk-7WUGNN43.js";
|
|
15
|
+
import "./chunk-LZMTVVN4.js";
|
|
15
16
|
import {
|
|
16
17
|
evalClioProvenance,
|
|
17
18
|
evalEnvironmentProvenance
|
|
18
|
-
} from "./chunk-
|
|
19
|
-
import "./chunk-
|
|
19
|
+
} from "./chunk-EEITU7S6.js";
|
|
20
|
+
import "./chunk-GPTR6OTU.js";
|
|
20
21
|
import {
|
|
21
22
|
ZERO_EVAL_HARNESS_METRICS,
|
|
22
23
|
writeEvalArtifact
|
|
23
24
|
} from "./chunk-BOBCCDJ6.js";
|
|
24
25
|
import "./chunk-E6IPKW2E.js";
|
|
25
|
-
import "./chunk-
|
|
26
|
+
import "./chunk-N3CWOGVF.js";
|
|
26
27
|
import "./chunk-UQIVHSE4.js";
|
|
27
28
|
import "./chunk-BYFHHCFV.js";
|
|
28
29
|
import "./chunk-TKDRT2DF.js";
|
|
29
30
|
import "./chunk-LQTKDAUO.js";
|
|
30
31
|
import "./chunk-ZU4JPCPR.js";
|
|
31
|
-
import "./chunk-
|
|
32
|
+
import "./chunk-XYPWFSU5.js";
|
|
32
33
|
import "./chunk-DNHVRBH4.js";
|
|
33
34
|
import "./chunk-O5F25NIE.js";
|
|
34
|
-
import "./chunk-
|
|
35
|
+
import "./chunk-Q4RMNWMZ.js";
|
|
35
36
|
import "./chunk-SD7SHNDW.js";
|
|
36
|
-
import "./chunk-
|
|
37
|
+
import "./chunk-Q2HKY32Y.js";
|
|
37
38
|
import "./chunk-YQ3GFEUY.js";
|
|
38
39
|
import "./chunk-WTQGRBHV.js";
|
|
39
40
|
import "./chunk-GLIQT7SU.js";
|
|
@@ -44,55 +45,56 @@ import {
|
|
|
44
45
|
} from "./chunk-NBXFPI7U.js";
|
|
45
46
|
import "./chunk-SQZHXUB3.js";
|
|
46
47
|
import "./chunk-SUTLW6ND.js";
|
|
47
|
-
import "./chunk-
|
|
48
|
-
import "./chunk-
|
|
49
|
-
import "./chunk-
|
|
50
|
-
import "./chunk-
|
|
51
|
-
import "./chunk-
|
|
48
|
+
import "./chunk-CNXU7TR2.js";
|
|
49
|
+
import "./chunk-7LUEQODV.js";
|
|
50
|
+
import "./chunk-6KV4KCQZ.js";
|
|
51
|
+
import "./chunk-G5ZDQQE3.js";
|
|
52
|
+
import "./chunk-AI36V3ON.js";
|
|
52
53
|
import {
|
|
53
54
|
discoverMarketplaceSkills,
|
|
54
55
|
loadSkills
|
|
55
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-B5TASTCK.js";
|
|
56
57
|
import "./chunk-DNCE23JJ.js";
|
|
57
58
|
import "./chunk-CDQ2UC6W.js";
|
|
58
59
|
import "./chunk-5LP5FNNR.js";
|
|
59
60
|
import "./chunk-NVPXXW2H.js";
|
|
60
|
-
import "./chunk-
|
|
61
|
-
import "./chunk-
|
|
62
|
-
import "./chunk-
|
|
61
|
+
import "./chunk-LXYHCCIZ.js";
|
|
62
|
+
import "./chunk-QOJ4ERMU.js";
|
|
63
|
+
import "./chunk-UZ7YBL43.js";
|
|
63
64
|
import {
|
|
64
65
|
HEADLESS_PERMISSION_DENIED_MARKER,
|
|
65
66
|
combineBashOutput,
|
|
66
67
|
runBashCommand
|
|
67
|
-
} from "./chunk-
|
|
68
|
-
import "./chunk-
|
|
68
|
+
} from "./chunk-PF36KOGI.js";
|
|
69
|
+
import "./chunk-7DW7CJW2.js";
|
|
69
70
|
import "./chunk-7QXCO3AF.js";
|
|
70
|
-
import "./chunk-
|
|
71
|
+
import "./chunk-SAASVMIY.js";
|
|
71
72
|
import "./chunk-OH3YYA3B.js";
|
|
72
73
|
import "./chunk-VESL6DQY.js";
|
|
73
74
|
import "./chunk-FD3IAHQM.js";
|
|
74
75
|
import "./chunk-XNIZBOFV.js";
|
|
75
|
-
import "./chunk-
|
|
76
|
-
import "./chunk-
|
|
77
|
-
import "./chunk-
|
|
76
|
+
import "./chunk-5APYAXCB.js";
|
|
77
|
+
import "./chunk-SZUO5BSE.js";
|
|
78
|
+
import "./chunk-PGQG7ZMW.js";
|
|
78
79
|
import "./chunk-L4N3DWNW.js";
|
|
79
|
-
import "./chunk-
|
|
80
|
-
import "./chunk-
|
|
80
|
+
import "./chunk-F4I3H56H.js";
|
|
81
|
+
import "./chunk-Q6FRR3AQ.js";
|
|
81
82
|
import {
|
|
82
83
|
effectiveToolCall
|
|
83
|
-
} from "./chunk-
|
|
84
|
+
} from "./chunk-KKAJGT3B.js";
|
|
85
|
+
import "./chunk-H7REJ7L5.js";
|
|
84
86
|
import "./chunk-YYLCPDCC.js";
|
|
85
87
|
import "./chunk-EFSEIZE6.js";
|
|
86
88
|
import {
|
|
87
89
|
NO_NETWORK_TOOLS_ENV
|
|
88
|
-
} from "./chunk-
|
|
89
|
-
import "./chunk-
|
|
90
|
-
import "./chunk-
|
|
91
|
-
import "./chunk-
|
|
92
|
-
import "./chunk-
|
|
90
|
+
} from "./chunk-4XBYHXGK.js";
|
|
91
|
+
import "./chunk-YEJQDODA.js";
|
|
92
|
+
import "./chunk-4KC4CHKJ.js";
|
|
93
|
+
import "./chunk-MNABNEBX.js";
|
|
94
|
+
import "./chunk-FLNXQQ5B.js";
|
|
93
95
|
import "./chunk-DZLR4UEH.js";
|
|
94
|
-
import "./chunk-
|
|
95
|
-
import "./chunk-
|
|
96
|
+
import "./chunk-H6FQVNZC.js";
|
|
97
|
+
import "./chunk-E74MJF6G.js";
|
|
96
98
|
import "./chunk-GHXGMFQ5.js";
|
|
97
99
|
import "./chunk-KECNXMDE.js";
|
|
98
100
|
import "./chunk-WXA2CFJO.js";
|
|
@@ -100,12 +102,12 @@ import "./chunk-6CTHLZFB.js";
|
|
|
100
102
|
import "./chunk-2KUBKL63.js";
|
|
101
103
|
import "./chunk-MYI32N4R.js";
|
|
102
104
|
import "./chunk-X6FLVUM5.js";
|
|
103
|
-
import "./chunk-
|
|
105
|
+
import "./chunk-HTWTORF7.js";
|
|
104
106
|
import "./chunk-MCJKEGHO.js";
|
|
105
107
|
import "./chunk-DVTBERFY.js";
|
|
106
108
|
import "./chunk-6EJYVCTN.js";
|
|
107
109
|
import "./chunk-W53XSRQE.js";
|
|
108
|
-
import "./chunk-
|
|
110
|
+
import "./chunk-53P775GE.js";
|
|
109
111
|
import "./chunk-77DCC6JZ.js";
|
|
110
112
|
import "./chunk-SYF7FVBL.js";
|
|
111
113
|
import "./chunk-Z4GLJ6J4.js";
|
|
@@ -116,22 +118,23 @@ import "./chunk-L7G7K4I2.js";
|
|
|
116
118
|
import "./chunk-ALJWT6O6.js";
|
|
117
119
|
import "./chunk-4WV4PI66.js";
|
|
118
120
|
import "./chunk-HMN2C6YL.js";
|
|
119
|
-
import "./chunk-
|
|
121
|
+
import "./chunk-F6KCJO3U.js";
|
|
120
122
|
import "./chunk-UZVYEF57.js";
|
|
123
|
+
import "./chunk-MMZMM6SW.js";
|
|
121
124
|
import "./chunk-O3VV6IVK.js";
|
|
122
|
-
import "./chunk-
|
|
125
|
+
import "./chunk-PZBQUJ2F.js";
|
|
123
126
|
import "./chunk-UHW5V7HC.js";
|
|
124
|
-
import "./chunk-
|
|
127
|
+
import "./chunk-7UFDIOBT.js";
|
|
125
128
|
import "./chunk-BJH6WFIB.js";
|
|
126
129
|
import "./chunk-5EQO5NWP.js";
|
|
127
|
-
import "./chunk-
|
|
128
|
-
import "./chunk-
|
|
130
|
+
import "./chunk-NMQSO6Z6.js";
|
|
131
|
+
import "./chunk-OYUS3UZP.js";
|
|
129
132
|
import "./chunk-R2P2U3ZS.js";
|
|
130
133
|
import "./chunk-NTTS3O53.js";
|
|
131
134
|
import "./chunk-C2CPOIH2.js";
|
|
132
135
|
import "./chunk-MGLRTOCK.js";
|
|
133
136
|
import "./chunk-M4KFUXS7.js";
|
|
134
|
-
import "./chunk-
|
|
137
|
+
import "./chunk-6UKFRTXI.js";
|
|
135
138
|
import "./chunk-56XP4FJU.js";
|
|
136
139
|
import "./chunk-QZTTRSJE.js";
|
|
137
140
|
import "./chunk-U2MSJ57E.js";
|
|
@@ -174,10 +177,10 @@ import {
|
|
|
174
177
|
init_esm_shims();
|
|
175
178
|
import { spawn } from "node:child_process";
|
|
176
179
|
import { createHash, randomBytes } from "node:crypto";
|
|
177
|
-
import { existsSync } from "node:fs";
|
|
178
|
-
import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
180
|
+
import { existsSync, realpathSync } from "node:fs";
|
|
181
|
+
import { cp, mkdir, mkdtemp, readdir, readFile, readlink, realpath, rm, stat, writeFile } from "node:fs/promises";
|
|
179
182
|
import { tmpdir } from "node:os";
|
|
180
|
-
import { join, resolve } from "node:path";
|
|
183
|
+
import { isAbsolute, join, relative as relativePath, resolve } from "node:path";
|
|
181
184
|
import { performance } from "node:perf_hooks";
|
|
182
185
|
var DEFAULT_RUN_TIMEOUT_MS = 6e5;
|
|
183
186
|
var ARM_AUTONOMY = "full-auto";
|
|
@@ -186,6 +189,115 @@ var PREVIEW_MAX_CHARS = 300;
|
|
|
186
189
|
var TRANSCRIPT_HEAD_CHARS = 9e3;
|
|
187
190
|
var TRANSCRIPT_TAIL_CHARS = 4e3;
|
|
188
191
|
var SKILL_EVAL_SIDECAR = "skill-eval.json";
|
|
192
|
+
function deriveBulletAttribution(baseline, treatment) {
|
|
193
|
+
if (baseline === "unmeasured" || baseline === "error") return "unmeasured";
|
|
194
|
+
if (treatment === "unmeasured" || treatment === "error") return "unmeasured";
|
|
195
|
+
if (baseline === "fail" && treatment === "pass") return "helped";
|
|
196
|
+
if (baseline === "pass" && treatment === "fail") return "regressed";
|
|
197
|
+
return baseline === "pass" ? "no-change-pass" : "no-change-fail";
|
|
198
|
+
}
|
|
199
|
+
function deriveScenarioAttributionVerdict(bullets) {
|
|
200
|
+
if (bullets.length === 0) return "unmeasured";
|
|
201
|
+
if (bullets.some((bullet) => bullet.attribution === "unmeasured")) return "unmeasured";
|
|
202
|
+
const helped = bullets.some((bullet) => bullet.attribution === "helped");
|
|
203
|
+
const regressed = bullets.some((bullet) => bullet.attribution === "regressed");
|
|
204
|
+
if (helped && regressed) return "mixed";
|
|
205
|
+
if (regressed) return "regressed";
|
|
206
|
+
if (helped) return "helped";
|
|
207
|
+
return "no-change";
|
|
208
|
+
}
|
|
209
|
+
function attributionCounts(bullets) {
|
|
210
|
+
const counts = { helped: 0, regressed: 0, noChangePass: 0, noChangeFail: 0, unmeasured: 0 };
|
|
211
|
+
for (const bullet of bullets) {
|
|
212
|
+
if (bullet.attribution === "helped") counts.helped += 1;
|
|
213
|
+
else if (bullet.attribution === "regressed") counts.regressed += 1;
|
|
214
|
+
else if (bullet.attribution === "no-change-pass") counts.noChangePass += 1;
|
|
215
|
+
else if (bullet.attribution === "no-change-fail") counts.noChangeFail += 1;
|
|
216
|
+
else counts.unmeasured += 1;
|
|
217
|
+
}
|
|
218
|
+
return counts;
|
|
219
|
+
}
|
|
220
|
+
function attributionFromStrict(scenario, baseline, treatment) {
|
|
221
|
+
const verdictOf = (pass) => pass === void 0 ? "unmeasured" : pass ? "pass" : "fail";
|
|
222
|
+
const refusal = (side, strict, index) => {
|
|
223
|
+
if (strict.verdicts.has(index)) return null;
|
|
224
|
+
if (strict.absent !== null) return `${side} judge: ${strict.absent}`;
|
|
225
|
+
return `${side} judge: ${strict.rejected.get(index) ?? `no verdict for bullet ${index}`}`;
|
|
226
|
+
};
|
|
227
|
+
const bullets = scenario.expected.map((text, i) => {
|
|
228
|
+
const index = i + 1;
|
|
229
|
+
const baselinePass = baseline.verdicts.get(index);
|
|
230
|
+
const treatmentPass = treatment.verdicts.get(index);
|
|
231
|
+
const baselineVerdict = verdictOf(baselinePass);
|
|
232
|
+
const treatmentVerdict = verdictOf(treatmentPass);
|
|
233
|
+
const reason = refusal("baseline", baseline, index) ?? refusal("treatment", treatment, index);
|
|
234
|
+
return {
|
|
235
|
+
index,
|
|
236
|
+
text,
|
|
237
|
+
baseline: baselineVerdict,
|
|
238
|
+
treatment: treatmentVerdict,
|
|
239
|
+
attribution: deriveBulletAttribution(baselineVerdict, treatmentVerdict),
|
|
240
|
+
...reason !== null ? { reason } : {}
|
|
241
|
+
};
|
|
242
|
+
});
|
|
243
|
+
const verdict = deriveScenarioAttributionVerdict(bullets);
|
|
244
|
+
const firstReason = bullets.find((bullet) => bullet.reason !== void 0)?.reason ?? null;
|
|
245
|
+
return {
|
|
246
|
+
verdict,
|
|
247
|
+
reason: verdict === "unmeasured" ? firstReason : null,
|
|
248
|
+
bullets,
|
|
249
|
+
counts: attributionCounts(bullets)
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
async function skillTreeDigest(baseDir) {
|
|
253
|
+
const root = resolve(baseDir);
|
|
254
|
+
const entries = [];
|
|
255
|
+
const seenDirs = /* @__PURE__ */ new Set();
|
|
256
|
+
const walk = async (dir) => {
|
|
257
|
+
const real = await realpath(dir);
|
|
258
|
+
if (seenDirs.has(real)) throw new Error("cycle");
|
|
259
|
+
seenDirs.add(real);
|
|
260
|
+
const found = await readdir(dir, { withFileTypes: true });
|
|
261
|
+
for (const entry of found.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
262
|
+
const full = join(dir, entry.name);
|
|
263
|
+
if (entry.isSymbolicLink()) {
|
|
264
|
+
const target = await readlink(full);
|
|
265
|
+
const resolved = resolve(dir, target);
|
|
266
|
+
const relative = relativePath(root, resolved);
|
|
267
|
+
if (relative.startsWith("..") || isAbsolute(relative)) throw new Error("escaping symlink");
|
|
268
|
+
entries.push({ relative: full.slice(root.length), kind: "symlink", target });
|
|
269
|
+
continue;
|
|
270
|
+
}
|
|
271
|
+
if (entry.isDirectory()) await walk(full);
|
|
272
|
+
else if (entry.isFile()) entries.push({ relative: full.slice(root.length), kind: "file" });
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
try {
|
|
276
|
+
await walk(root);
|
|
277
|
+
const hash = createHash("sha256");
|
|
278
|
+
for (const entry of entries.sort((a, b) => a.relative.localeCompare(b.relative))) {
|
|
279
|
+
hash.update(entry.relative, "utf8");
|
|
280
|
+
hash.update("\0");
|
|
281
|
+
hash.update(entry.kind, "utf8");
|
|
282
|
+
hash.update("\0");
|
|
283
|
+
if (entry.kind === "symlink") hash.update(entry.target ?? "", "utf8");
|
|
284
|
+
else hash.update(await readFile(join(root, entry.relative)));
|
|
285
|
+
hash.update("\0");
|
|
286
|
+
}
|
|
287
|
+
return hash.digest("hex");
|
|
288
|
+
} catch {
|
|
289
|
+
return null;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
function artifactIsPinnable(skillBody) {
|
|
293
|
+
return !skillBody.includes("${pluginRoot}") && !skillBody.includes("${component:");
|
|
294
|
+
}
|
|
295
|
+
function notAttemptedAttribution(reason) {
|
|
296
|
+
return { verdict: "not-attempted", reason, bullets: [], counts: attributionCounts([]) };
|
|
297
|
+
}
|
|
298
|
+
function unmeasurableAttribution(reason) {
|
|
299
|
+
return { verdict: "unmeasured", reason, bullets: [], counts: attributionCounts([]) };
|
|
300
|
+
}
|
|
189
301
|
async function runSkillsEvalCommand(nameOrPath, options) {
|
|
190
302
|
const resolved = resolveSkillBaseDir(nameOrPath, process.cwd());
|
|
191
303
|
if (resolved.baseDir === null) {
|
|
@@ -215,6 +327,29 @@ async function runSkillsEvalCommand(nameOrPath, options) {
|
|
|
215
327
|
process.stderr.write(`clio-coder eval skill: ${diagnostic}
|
|
216
328
|
`);
|
|
217
329
|
}
|
|
330
|
+
const subjectTree = await skillTreeDigest(resolved.baseDir);
|
|
331
|
+
const reread = await readFile(join(resolved.baseDir, "SKILL.md"), "utf8").catch(() => null);
|
|
332
|
+
const coherent = reread !== null && createHash("sha256").update(reread, "utf8").digest("hex") === skill.hash;
|
|
333
|
+
const pinnable = reread !== null && artifactIsPinnable(reread);
|
|
334
|
+
const driftReport = checkSkillDrift(skill, process.cwd());
|
|
335
|
+
const subject = {
|
|
336
|
+
name: skill.name,
|
|
337
|
+
baseDir: resolved.baseDir,
|
|
338
|
+
origin: resolved.origin ?? "unknown",
|
|
339
|
+
sha256: skill.hash,
|
|
340
|
+
normalizedHash: skill.normalizedHash,
|
|
341
|
+
treeSha256: coherent ? subjectTree : null,
|
|
342
|
+
pinnable,
|
|
343
|
+
evalsPath,
|
|
344
|
+
evalsSha256: createHash("sha256").update(evalsRaw, "utf8").digest("hex"),
|
|
345
|
+
drift: driftReport === null ? null : { verdict: driftReport.verdict, authority: driftReport.authority, expected: driftReport.expected }
|
|
346
|
+
};
|
|
347
|
+
if (subject.drift?.verdict === "mismatch") {
|
|
348
|
+
process.stderr.write(
|
|
349
|
+
`clio-coder eval skill: WARNING skill_drift: ${skill.name} content (sha256 ${skill.normalizedHash.slice(0, 12)}\u2026) does not match the hash recorded for it by the ${subject.drift.authority} (expected ${subject.drift.expected.slice(0, 12)}\u2026); this run measures the copy on disk, not the recorded one
|
|
350
|
+
`
|
|
351
|
+
);
|
|
352
|
+
}
|
|
218
353
|
const matcher = options.scenario === void 0 ? null : scenarioMatcher(options.scenario);
|
|
219
354
|
if (options.scenario !== void 0 && matcher === null) {
|
|
220
355
|
printError(`invalid --scenario "${options.scenario}": use a scenario id like S1 or a bare number`);
|
|
@@ -238,24 +373,38 @@ async function runSkillsEvalCommand(nameOrPath, options) {
|
|
|
238
373
|
}
|
|
239
374
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
240
375
|
const outcomes = [];
|
|
376
|
+
const attributionEnabled = !options.noAttribution;
|
|
241
377
|
for (const scenario of scenarios) {
|
|
242
|
-
process.stderr.write(
|
|
243
|
-
`
|
|
378
|
+
process.stderr.write(
|
|
379
|
+
`clio-coder eval skill: ${skill.name} ${scenario.id} baseline/treatment/judge${attributionEnabled ? "/baseline-judge" : ""}...
|
|
380
|
+
`
|
|
381
|
+
);
|
|
244
382
|
outcomes.push(
|
|
245
|
-
await runScenario(
|
|
246
|
-
skill.name,
|
|
247
|
-
resolved.baseDir,
|
|
383
|
+
await runScenario({
|
|
384
|
+
skillName: skill.name,
|
|
385
|
+
skillBaseDir: resolved.baseDir,
|
|
248
386
|
scenario,
|
|
249
|
-
options.target,
|
|
387
|
+
target: options.target,
|
|
250
388
|
timeoutMs,
|
|
251
389
|
workspaceOverride,
|
|
252
|
-
options.trustFixtures,
|
|
253
|
-
childEnv
|
|
254
|
-
|
|
390
|
+
trustFixtures: options.trustFixtures,
|
|
391
|
+
childEnv,
|
|
392
|
+
attributionEnabled,
|
|
393
|
+
subjectTreeSha256: subject.treeSha256,
|
|
394
|
+
subjectSha256: subject.sha256,
|
|
395
|
+
pinnable: subject.pinnable
|
|
396
|
+
})
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
const unverified = outcomes.filter((outcome) => outcome.subjectVerification === "mismatch");
|
|
400
|
+
if (unverified.length > 0) {
|
|
401
|
+
process.stderr.write(
|
|
402
|
+
`clio-coder eval skill: WARNING subject_changed: the skill tree at ${resolved.baseDir} changed during ${unverified.map((outcome) => outcome.scenario.id).join(", ")}; those scenarios' rubric results stand but their baseline comparison is unmeasured
|
|
403
|
+
`
|
|
255
404
|
);
|
|
256
405
|
}
|
|
257
406
|
const endedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
258
|
-
const artifact = synthesizeArtifact(
|
|
407
|
+
const artifact = synthesizeArtifact(subject, evalsPath, evalsRaw, startedAt, endedAt, outcomes, options.target);
|
|
259
408
|
let evidenceId = null;
|
|
260
409
|
let evidenceDirectory = null;
|
|
261
410
|
const evidenceErrors = [];
|
|
@@ -271,7 +420,7 @@ async function runSkillsEvalCommand(nameOrPath, options) {
|
|
|
271
420
|
evidenceDirectory = built.directory;
|
|
272
421
|
await writeFile(
|
|
273
422
|
join(built.directory, SKILL_EVAL_SIDECAR),
|
|
274
|
-
`${JSON.stringify(sidecar(
|
|
423
|
+
`${JSON.stringify(sidecar(subject, artifact.evalId, outcomes, options.allowNetwork, attributionEnabled), null, 2)}
|
|
275
424
|
`,
|
|
276
425
|
"utf8"
|
|
277
426
|
);
|
|
@@ -284,23 +433,35 @@ async function runSkillsEvalCommand(nameOrPath, options) {
|
|
|
284
433
|
}
|
|
285
434
|
if (options.json) {
|
|
286
435
|
for (const outcome of outcomes) {
|
|
436
|
+
const attributed = new Map(outcome.attribution.bullets.map((item) => [item.index, item]));
|
|
287
437
|
for (const bullet of outcome.bullets) {
|
|
438
|
+
const pair = attributed.get(bullet.index);
|
|
288
439
|
process.stdout.write(
|
|
289
440
|
`${JSON.stringify({
|
|
290
441
|
schema: "experimental",
|
|
291
442
|
kind: "skill-eval-bullet",
|
|
292
443
|
skill: skill.name,
|
|
444
|
+
skillSha256: subject.normalizedHash,
|
|
445
|
+
skillOrigin: subject.origin,
|
|
446
|
+
skillDrift: subject.drift?.verdict ?? null,
|
|
293
447
|
scenario: outcome.scenario.id,
|
|
294
448
|
title: outcome.scenario.title,
|
|
295
449
|
bullet: bullet.index,
|
|
296
450
|
expected: bullet.text,
|
|
297
451
|
verdict: bullet.verdict,
|
|
298
452
|
reason: bullet.reason,
|
|
453
|
+
// Advisory, never a gate: `verdict` above is the rubric result and
|
|
454
|
+
// is unchanged by anything here.
|
|
455
|
+
baselineVerdict: pair?.baseline ?? null,
|
|
456
|
+
attribution: pair?.attribution ?? null,
|
|
457
|
+
scenarioAttribution: outcome.attribution.verdict,
|
|
458
|
+
attributionReason: outcome.attribution.reason,
|
|
299
459
|
network: networkPolicyLabel(options.allowNetwork),
|
|
300
460
|
autonomy: ARM_AUTONOMY,
|
|
301
461
|
baselineSessionId: outcome.baseline?.sessionId ?? null,
|
|
302
462
|
treatmentSessionId: outcome.treatment?.sessionId ?? null,
|
|
303
463
|
judgeSessionId: outcome.judge?.sessionId ?? null,
|
|
464
|
+
baselineJudgeSessionId: outcome.baselineJudge?.sessionId ?? null,
|
|
304
465
|
evalId: artifact.evalId,
|
|
305
466
|
evidenceId
|
|
306
467
|
})}
|
|
@@ -309,7 +470,7 @@ async function runSkillsEvalCommand(nameOrPath, options) {
|
|
|
309
470
|
}
|
|
310
471
|
}
|
|
311
472
|
} else {
|
|
312
|
-
printHumanReport(
|
|
473
|
+
printHumanReport(subject, outcomes, artifact.evalId, evidenceId, evidenceDirectory, options.allowNetwork);
|
|
313
474
|
}
|
|
314
475
|
const anyFailure = outcomes.some(
|
|
315
476
|
(outcome) => outcome.bullets.some((bullet) => bullet.verdict === "fail" || bullet.verdict === "error")
|
|
@@ -329,9 +490,12 @@ function describeArmPolicyOutcome(allowNetwork) {
|
|
|
329
490
|
const network = allowNetwork ? "network: allowed (--allow-network); arm runs kept the web tools" : "retrieval: disabled; bash networking was not constrained";
|
|
330
491
|
return `policy: the baseline and treatment arms ran at autonomy ${ARM_AUTONOMY}; ${network}`;
|
|
331
492
|
}
|
|
493
|
+
function isJudgeArm(arm) {
|
|
494
|
+
return arm === "judge" || arm === "baseline-judge";
|
|
495
|
+
}
|
|
332
496
|
function armRunArgs(arm, prompt, options = {}) {
|
|
333
497
|
const args = ["run", "--json", "--json-events", "full", "--no-skills"];
|
|
334
|
-
if (arm
|
|
498
|
+
if (!isJudgeArm(arm)) args.push("--autonomy", ARM_AUTONOMY);
|
|
335
499
|
if (options.skillBaseDir !== void 0) args.push("--skill", options.skillBaseDir);
|
|
336
500
|
if (options.target !== void 0) args.push("--target", options.target);
|
|
337
501
|
args.push(prompt);
|
|
@@ -401,11 +565,35 @@ async function resolveWorkspaceOverride(workspace) {
|
|
|
401
565
|
return { error: `--workspace is not readable: ${resolved}: ${detail}` };
|
|
402
566
|
}
|
|
403
567
|
}
|
|
404
|
-
async function runScenario(
|
|
568
|
+
async function runScenario(input) {
|
|
569
|
+
const {
|
|
570
|
+
skillName,
|
|
571
|
+
skillBaseDir,
|
|
572
|
+
scenario,
|
|
573
|
+
target,
|
|
574
|
+
timeoutMs,
|
|
575
|
+
workspaceOverride,
|
|
576
|
+
trustFixtures,
|
|
577
|
+
childEnv,
|
|
578
|
+
attributionEnabled,
|
|
579
|
+
subjectTreeSha256,
|
|
580
|
+
subjectSha256,
|
|
581
|
+
pinnable
|
|
582
|
+
} = input;
|
|
583
|
+
const runArm = input.runner ?? captureHeadlessRun;
|
|
405
584
|
const scenarioStart = performance.now();
|
|
406
585
|
const workspace = await mkdtemp(join(tmpdir(), "clio-coder-skill-eval-seed-"));
|
|
407
586
|
let runWorkspaces = null;
|
|
587
|
+
let subjectVerification = "not-checked";
|
|
588
|
+
let observedTreeSha256 = null;
|
|
408
589
|
try {
|
|
590
|
+
const skipAttribution = (reason) => attributionEnabled ? unmeasurableAttribution(reason) : notAttemptedAttribution(ATTRIBUTION_DISABLED_REASON);
|
|
591
|
+
const outcomeBase = () => ({
|
|
592
|
+
workspace,
|
|
593
|
+
subjectVerification,
|
|
594
|
+
observedTreeSha256,
|
|
595
|
+
wallTimeMs: Math.round(performance.now() - scenarioStart)
|
|
596
|
+
});
|
|
409
597
|
if (workspaceOverride !== null) await copyWorkspace(workspaceOverride, workspace);
|
|
410
598
|
const fixtureError = await runFixtureCommands(scenario, workspace, timeoutMs, trustFixtures);
|
|
411
599
|
if (fixtureError !== null) {
|
|
@@ -415,24 +603,31 @@ async function runScenario(skillName, skillBaseDir, scenario, target, timeoutMs,
|
|
|
415
603
|
baseline: null,
|
|
416
604
|
treatment: null,
|
|
417
605
|
judge: null,
|
|
418
|
-
|
|
419
|
-
|
|
606
|
+
baselineJudge: null,
|
|
607
|
+
attribution: skipAttribution(fixtureError),
|
|
608
|
+
...outcomeBase(),
|
|
420
609
|
infraError: fixtureError
|
|
421
610
|
});
|
|
422
611
|
}
|
|
423
|
-
runWorkspaces = await materializeSkillEvalWorkspaces(workspace);
|
|
424
|
-
const baseline = await
|
|
612
|
+
runWorkspaces = await materializeSkillEvalWorkspaces(workspace, pinnable ? skillBaseDir : null);
|
|
613
|
+
const baseline = await runArm(
|
|
425
614
|
armRunArgs("baseline", scenario.setup, { target }),
|
|
426
615
|
runWorkspaces.baseline,
|
|
427
616
|
timeoutMs,
|
|
428
617
|
childEnv
|
|
429
618
|
);
|
|
430
|
-
const
|
|
431
|
-
|
|
619
|
+
const pinned = pinnable ? runWorkspaces.pin : null;
|
|
620
|
+
const armSkillDir = pinned ?? skillBaseDir;
|
|
621
|
+
const before = await skillTreeDigest(armSkillDir);
|
|
622
|
+
const treatment = await runArm(
|
|
623
|
+
armRunArgs("treatment", `/skill ${skillName} ${scenario.setup}`, { target, skillBaseDir: armSkillDir }),
|
|
432
624
|
runWorkspaces.treatment,
|
|
433
625
|
timeoutMs,
|
|
434
626
|
childEnv
|
|
435
627
|
);
|
|
628
|
+
const after = await skillTreeDigest(armSkillDir);
|
|
629
|
+
observedTreeSha256 = after;
|
|
630
|
+
subjectVerification = !pinnable ? "not-pinned" : verifySubjectTree(subjectTreeSha256, before, after) === "verified" ? verifyObservedActivation(skillName, subjectSha256, join(armSkillDir, "SKILL.md"), treatment.activations) : verifySubjectTree(subjectTreeSha256, before, after);
|
|
436
631
|
const infra = runInfraError("baseline", baseline) ?? runInfraError("treatment", treatment);
|
|
437
632
|
if (infra !== null) {
|
|
438
633
|
return await completeScenarioOutcome({
|
|
@@ -441,8 +636,9 @@ async function runScenario(skillName, skillBaseDir, scenario, target, timeoutMs,
|
|
|
441
636
|
baseline,
|
|
442
637
|
treatment,
|
|
443
638
|
judge: null,
|
|
444
|
-
|
|
445
|
-
|
|
639
|
+
baselineJudge: null,
|
|
640
|
+
attribution: skipAttribution(infra),
|
|
641
|
+
...outcomeBase(),
|
|
446
642
|
infraError: infra
|
|
447
643
|
});
|
|
448
644
|
}
|
|
@@ -454,12 +650,13 @@ async function runScenario(skillName, skillBaseDir, scenario, target, timeoutMs,
|
|
|
454
650
|
baseline,
|
|
455
651
|
treatment,
|
|
456
652
|
judge: null,
|
|
457
|
-
|
|
458
|
-
|
|
653
|
+
baselineJudge: null,
|
|
654
|
+
attribution: skipAttribution(wall),
|
|
655
|
+
...outcomeBase(),
|
|
459
656
|
infraError: wall
|
|
460
657
|
});
|
|
461
658
|
}
|
|
462
|
-
const judge = await
|
|
659
|
+
const judge = await runArm(
|
|
463
660
|
armRunArgs("judge", judgePrompt(scenario, baseline.transcript, treatment.transcript), { target }),
|
|
464
661
|
runWorkspaces.judge,
|
|
465
662
|
timeoutMs,
|
|
@@ -473,20 +670,35 @@ async function runScenario(skillName, skillBaseDir, scenario, target, timeoutMs,
|
|
|
473
670
|
baseline,
|
|
474
671
|
treatment,
|
|
475
672
|
judge,
|
|
476
|
-
|
|
477
|
-
|
|
673
|
+
baselineJudge: null,
|
|
674
|
+
attribution: skipAttribution(judgeInfra),
|
|
675
|
+
...outcomeBase(),
|
|
478
676
|
infraError: judgeInfra
|
|
479
677
|
});
|
|
480
678
|
}
|
|
481
679
|
const bullets = parseJudgeVerdicts(scenario, judge);
|
|
680
|
+
const attributed = await attributeScenario({
|
|
681
|
+
scenario,
|
|
682
|
+
bullets,
|
|
683
|
+
baseline,
|
|
684
|
+
judge,
|
|
685
|
+
target,
|
|
686
|
+
timeoutMs,
|
|
687
|
+
childEnv,
|
|
688
|
+
attributionEnabled,
|
|
689
|
+
subjectVerification,
|
|
690
|
+
workspace: runWorkspaces.baselineJudge,
|
|
691
|
+
runner: runArm
|
|
692
|
+
});
|
|
482
693
|
return await completeScenarioOutcome({
|
|
483
694
|
scenario,
|
|
484
695
|
bullets,
|
|
485
696
|
baseline,
|
|
486
697
|
treatment,
|
|
487
698
|
judge,
|
|
488
|
-
|
|
489
|
-
|
|
699
|
+
baselineJudge: attributed.baselineJudge,
|
|
700
|
+
attribution: attributed.attribution,
|
|
701
|
+
...outcomeBase(),
|
|
490
702
|
infraError: null
|
|
491
703
|
});
|
|
492
704
|
} finally {
|
|
@@ -497,6 +709,77 @@ async function runScenario(skillName, skillBaseDir, scenario, target, timeoutMs,
|
|
|
497
709
|
}
|
|
498
710
|
}
|
|
499
711
|
}
|
|
712
|
+
var ATTRIBUTION_DISABLED_REASON = "disabled by --no-attribution";
|
|
713
|
+
function verifySubjectTree(subject, before, after) {
|
|
714
|
+
if (subject === null || before === null || after === null) return "unreadable";
|
|
715
|
+
return subject === before && before === after ? "verified" : "mismatch";
|
|
716
|
+
}
|
|
717
|
+
function verifyObservedActivation(skillName, expectedSha256, expectedPath, activations) {
|
|
718
|
+
const matching = activations.filter((entry) => entry.name === skillName);
|
|
719
|
+
if (matching.length === 0) return "not-activated";
|
|
720
|
+
if (matching.length > 1 && new Set(matching.map((entry) => `${entry.hash}|${entry.path}`)).size > 1) {
|
|
721
|
+
return "activation-mismatch";
|
|
722
|
+
}
|
|
723
|
+
const observed = matching[0];
|
|
724
|
+
if (observed === void 0 || observed.hash !== expectedSha256) return "activation-mismatch";
|
|
725
|
+
return samePath(observed.path, expectedPath) ? "verified" : "activation-mismatch";
|
|
726
|
+
}
|
|
727
|
+
function samePath(left, right) {
|
|
728
|
+
if (left.length === 0 || right.length === 0) return false;
|
|
729
|
+
try {
|
|
730
|
+
return realpathSync.native(left) === realpathSync.native(right);
|
|
731
|
+
} catch {
|
|
732
|
+
return resolve(left) === resolve(right);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
var SUBJECT_REFUSAL = {
|
|
736
|
+
mismatch: "the measured skill tree changed on disk during this scenario, so its two arms did not run against one artifact",
|
|
737
|
+
unreadable: "the measured skill tree could not be re-read, so the artifact under test is unverified",
|
|
738
|
+
"not-pinned": "the skill body carries package references, which resolve against an owning package root outside the skill directory, so the artifact could not be copied faithfully for this run",
|
|
739
|
+
"not-activated": "the treatment arm recorded no successful activation of this skill, so nothing establishes that the measured artifact ran",
|
|
740
|
+
"activation-mismatch": "the treatment arm activated content whose hash is not the pinned artifact's, so the comparison would name the wrong revision",
|
|
741
|
+
"not-checked": "the artifact identity was never checked for this scenario"
|
|
742
|
+
};
|
|
743
|
+
async function attributeScenario(input) {
|
|
744
|
+
if (!input.attributionEnabled) {
|
|
745
|
+
return { attribution: notAttemptedAttribution(ATTRIBUTION_DISABLED_REASON), baselineJudge: null };
|
|
746
|
+
}
|
|
747
|
+
if (input.subjectVerification !== "verified") {
|
|
748
|
+
return { attribution: unmeasurableAttribution(SUBJECT_REFUSAL[input.subjectVerification]), baselineJudge: null };
|
|
749
|
+
}
|
|
750
|
+
const treatmentWall = permissionWallReason("judge", input.judge);
|
|
751
|
+
if (treatmentWall !== null) return { attribution: unmeasurableAttribution(treatmentWall), baselineJudge: null };
|
|
752
|
+
if (!input.bullets.some((bullet) => bullet.verdict === "pass" || bullet.verdict === "fail")) {
|
|
753
|
+
return {
|
|
754
|
+
attribution: unmeasurableAttribution(
|
|
755
|
+
"the treatment arm produced no scored bullet, so there is nothing to compare a baseline against"
|
|
756
|
+
),
|
|
757
|
+
baselineJudge: null
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
const baselineJudge = await input.runner(
|
|
761
|
+
armRunArgs("baseline-judge", baselineJudgePrompt(input.scenario, input.baseline.transcript), {
|
|
762
|
+
target: input.target
|
|
763
|
+
}),
|
|
764
|
+
input.workspace,
|
|
765
|
+
input.timeoutMs,
|
|
766
|
+
input.childEnv
|
|
767
|
+
);
|
|
768
|
+
const infra = runInfraError("baseline-judge", baselineJudge);
|
|
769
|
+
if (infra !== null) {
|
|
770
|
+
return { attribution: unmeasurableAttribution(infra), baselineJudge };
|
|
771
|
+
}
|
|
772
|
+
const wall = permissionWallReason("baseline-judge", baselineJudge);
|
|
773
|
+
if (wall !== null) return { attribution: unmeasurableAttribution(wall), baselineJudge };
|
|
774
|
+
return {
|
|
775
|
+
attribution: attributionFromStrict(
|
|
776
|
+
input.scenario,
|
|
777
|
+
strictJudgeVerdicts(input.scenario, baselineJudge),
|
|
778
|
+
strictJudgeVerdicts(input.scenario, input.judge)
|
|
779
|
+
),
|
|
780
|
+
baselineJudge
|
|
781
|
+
};
|
|
782
|
+
}
|
|
500
783
|
async function armWorkspace(created) {
|
|
501
784
|
const root = await mkdtemp(join(tmpdir(), "clio-coder-skill-eval-"));
|
|
502
785
|
created.push(root);
|
|
@@ -504,17 +787,27 @@ async function armWorkspace(created) {
|
|
|
504
787
|
await mkdir(workspace, { recursive: true });
|
|
505
788
|
return workspace;
|
|
506
789
|
}
|
|
507
|
-
async function materializeSkillEvalWorkspaces(seedWorkspace) {
|
|
790
|
+
async function materializeSkillEvalWorkspaces(seedWorkspace, pinSource) {
|
|
508
791
|
const created = [];
|
|
509
792
|
try {
|
|
510
793
|
const baseline = await armWorkspace(created);
|
|
511
794
|
const treatment = await armWorkspace(created);
|
|
512
795
|
const judge = await armWorkspace(created);
|
|
796
|
+
const baselineJudge = await armWorkspace(created);
|
|
797
|
+
let pin = null;
|
|
798
|
+
if (pinSource !== null) {
|
|
799
|
+
const root = await mkdtemp(join(tmpdir(), "clio-coder-skill-eval-pin-"));
|
|
800
|
+
created.push(root);
|
|
801
|
+
pin = join(root, "skill");
|
|
802
|
+
await cp(pinSource, pin, { recursive: true, preserveTimestamps: true, verbatimSymlinks: true });
|
|
803
|
+
}
|
|
513
804
|
await Promise.all([copyWorkspace(seedWorkspace, baseline), copyWorkspace(seedWorkspace, treatment)]);
|
|
514
805
|
return {
|
|
515
806
|
baseline,
|
|
516
807
|
treatment,
|
|
517
808
|
judge,
|
|
809
|
+
baselineJudge,
|
|
810
|
+
pin,
|
|
518
811
|
cleanup: async () => {
|
|
519
812
|
await Promise.all(created.map((path) => rm(path, { recursive: true, force: true })));
|
|
520
813
|
}
|
|
@@ -534,7 +827,7 @@ async function copyWorkspace(source, destination) {
|
|
|
534
827
|
async function completeScenarioOutcome(outcome) {
|
|
535
828
|
return {
|
|
536
829
|
...outcome,
|
|
537
|
-
usage: await usageForCapturedRuns([outcome.baseline, outcome.treatment, outcome.judge])
|
|
830
|
+
usage: await usageForCapturedRuns([outcome.baseline, outcome.treatment, outcome.judge, outcome.baselineJudge])
|
|
538
831
|
};
|
|
539
832
|
}
|
|
540
833
|
async function runFixtureCommands(scenario, workspace, timeoutMs, trustFixtures) {
|
|
@@ -703,11 +996,21 @@ function loadsSkillBody(tool, args) {
|
|
|
703
996
|
if (tool !== "context" || !isRecord(args)) return false;
|
|
704
997
|
return args.scope === "skills" && typeof args.name === "string" && args.name.trim().length > 0;
|
|
705
998
|
}
|
|
999
|
+
function activationFromResult(result) {
|
|
1000
|
+
if (!isRecord(result)) return null;
|
|
1001
|
+
const details = isRecord(result.details) ? result.details : null;
|
|
1002
|
+
if (details === null) return null;
|
|
1003
|
+
const name = readString(details.name);
|
|
1004
|
+
const hash = readString(details.hash);
|
|
1005
|
+
if (name === null || hash === null) return null;
|
|
1006
|
+
return { name, hash, path: readString(details.filePath) ?? readString(details.path) ?? "" };
|
|
1007
|
+
}
|
|
706
1008
|
function parseRunStdout(stdout) {
|
|
707
1009
|
let sessionId = null;
|
|
708
1010
|
const lines = [];
|
|
709
1011
|
let finalText = "";
|
|
710
1012
|
let sawJson = false;
|
|
1013
|
+
const activations = [];
|
|
711
1014
|
const streamedText = /* @__PURE__ */ new Map();
|
|
712
1015
|
const skillBodyCallIds = /* @__PURE__ */ new Set();
|
|
713
1016
|
for (const line of stdout.split("\n")) {
|
|
@@ -748,6 +1051,10 @@ ${args.content}`);
|
|
|
748
1051
|
const tool = readString(event.toolName) ?? readString(event.tool) ?? "tool";
|
|
749
1052
|
const status = event.isError === true ? "error" : "ok";
|
|
750
1053
|
const callId = readString(event.toolCallId);
|
|
1054
|
+
if (event.isError !== true && callId !== null && skillBodyCallIds.has(callId)) {
|
|
1055
|
+
const activation = activationFromResult(event.result);
|
|
1056
|
+
if (activation !== null) activations.push(activation);
|
|
1057
|
+
}
|
|
751
1058
|
if (callId !== null && skillBodyCallIds.has(callId)) {
|
|
752
1059
|
lines.push(`RESULT ${tool} ${status}: <skill body withheld from the judge>`);
|
|
753
1060
|
continue;
|
|
@@ -771,9 +1078,9 @@ ${args.content}`);
|
|
|
771
1078
|
}
|
|
772
1079
|
if (!sawJson) {
|
|
773
1080
|
const text = stdout.trim();
|
|
774
|
-
return { sessionId: null, transcript: text, finalText: text };
|
|
1081
|
+
return { sessionId: null, transcript: text, finalText: text, activations: [] };
|
|
775
1082
|
}
|
|
776
|
-
return { sessionId, transcript: elide(lines.join("\n")), finalText };
|
|
1083
|
+
return { sessionId, transcript: elide(lines.join("\n")), finalText, activations };
|
|
777
1084
|
}
|
|
778
1085
|
function judgePrompt(scenario, baselineTranscript, treatmentTranscript) {
|
|
779
1086
|
const bullets = scenario.expected.map((text, index) => `${index + 1}. ${text}`).join("\n");
|
|
@@ -798,6 +1105,26 @@ function judgePrompt(scenario, baselineTranscript, treatmentTranscript) {
|
|
|
798
1105
|
treatmentTranscript.length > 0 ? treatmentTranscript : "(empty)"
|
|
799
1106
|
].join("\n");
|
|
800
1107
|
}
|
|
1108
|
+
function baselineJudgePrompt(scenario, baselineTranscript) {
|
|
1109
|
+
const bullets = scenario.expected.map((text, index) => `${index + 1}. ${text}`).join("\n");
|
|
1110
|
+
return [
|
|
1111
|
+
"You are scoring an agent run. One transcript follows: it ran WITHOUT any skill loaded.",
|
|
1112
|
+
"Score each EXPECTED bullet strictly against the TRANSCRIPT.",
|
|
1113
|
+
"A bullet passes only if the transcript observably satisfies it; anything unverifiable from the transcript fails.",
|
|
1114
|
+
'Reply with STRICT JSON only, no prose and no code fences, exactly: {"bullets":[{"index":1,"pass":true,"reason":"<= 25 words"}]}',
|
|
1115
|
+
`Include one entry per bullet, indexes 1 through ${scenario.expected.length} in order.`,
|
|
1116
|
+
"Do not use any tools. Respond with the JSON verdict directly.",
|
|
1117
|
+
"",
|
|
1118
|
+
`SCENARIO ${scenario.id} - ${scenario.title}`,
|
|
1119
|
+
`SETUP: ${scenario.setup}`,
|
|
1120
|
+
"",
|
|
1121
|
+
"EXPECTED BULLETS:",
|
|
1122
|
+
bullets,
|
|
1123
|
+
"",
|
|
1124
|
+
"TRANSCRIPT:",
|
|
1125
|
+
baselineTranscript.length > 0 ? baselineTranscript : "(empty)"
|
|
1126
|
+
].join("\n");
|
|
1127
|
+
}
|
|
801
1128
|
function judgeVerdictAbsenceReason(judge) {
|
|
802
1129
|
const sawBulletsKey = `${judge.finalText}
|
|
803
1130
|
${judge.transcript}`.includes('"bullets"');
|
|
@@ -835,6 +1162,47 @@ function parseJudgeVerdicts(scenario, judge) {
|
|
|
835
1162
|
return { index, text, verdict: entry.pass ? "pass" : "fail", reason: entry.reason };
|
|
836
1163
|
});
|
|
837
1164
|
}
|
|
1165
|
+
function strictJudgeVerdicts(scenario, judge) {
|
|
1166
|
+
const verdicts = /* @__PURE__ */ new Map();
|
|
1167
|
+
const rejected = /* @__PURE__ */ new Map();
|
|
1168
|
+
const parsed = extractBulletsObject(judge.finalText) ?? extractBulletsObject(judge.transcript);
|
|
1169
|
+
if (parsed === null) return { verdicts, rejected, absent: judgeVerdictAbsenceReason(judge) };
|
|
1170
|
+
if (!Array.isArray(parsed.bullets)) {
|
|
1171
|
+
return { verdicts, rejected, absent: "judge verdict object carried no bullets array" };
|
|
1172
|
+
}
|
|
1173
|
+
const count = scenario.expected.length;
|
|
1174
|
+
const seen = /* @__PURE__ */ new Map();
|
|
1175
|
+
for (const item of parsed.bullets) {
|
|
1176
|
+
if (!isRecord(item)) continue;
|
|
1177
|
+
const index = item.index;
|
|
1178
|
+
if (typeof index !== "number" || !Number.isInteger(index) || index < 1 || index > count) {
|
|
1179
|
+
continue;
|
|
1180
|
+
}
|
|
1181
|
+
if (typeof item.pass !== "boolean") {
|
|
1182
|
+
rejected.set(index, `judge gave no boolean pass for bullet ${index}: comparison evidence refused`);
|
|
1183
|
+
verdicts.delete(index);
|
|
1184
|
+
continue;
|
|
1185
|
+
}
|
|
1186
|
+
if (rejected.has(index)) {
|
|
1187
|
+
verdicts.delete(index);
|
|
1188
|
+
continue;
|
|
1189
|
+
}
|
|
1190
|
+
const previous = seen.get(index);
|
|
1191
|
+
if (previous !== void 0 && previous !== item.pass) {
|
|
1192
|
+
rejected.set(index, `judge gave contradictory verdicts for bullet ${index}: comparison evidence refused`);
|
|
1193
|
+
verdicts.delete(index);
|
|
1194
|
+
continue;
|
|
1195
|
+
}
|
|
1196
|
+
seen.set(index, item.pass);
|
|
1197
|
+
verdicts.set(index, item.pass);
|
|
1198
|
+
}
|
|
1199
|
+
for (let index = 1; index <= count; index += 1) {
|
|
1200
|
+
if (!verdicts.has(index) && !rejected.has(index)) {
|
|
1201
|
+
rejected.set(index, `judge omitted bullet ${index}: comparison evidence absent`);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
return { verdicts, rejected, absent: null };
|
|
1205
|
+
}
|
|
838
1206
|
function extractBulletsObject(text) {
|
|
839
1207
|
const stripped = text.replace(/```(?:json)?/g, "");
|
|
840
1208
|
const bulletsAt = stripped.indexOf('"bullets"');
|
|
@@ -881,7 +1249,8 @@ function balancedJsonSlice(text, start) {
|
|
|
881
1249
|
}
|
|
882
1250
|
return null;
|
|
883
1251
|
}
|
|
884
|
-
function synthesizeArtifact(
|
|
1252
|
+
function synthesizeArtifact(subject, evalsPath, evalsRaw, startedAt, endedAt, outcomes, target) {
|
|
1253
|
+
const skillName = subject.name;
|
|
885
1254
|
const contentHash = createHash("sha256").update(evalsRaw, "utf8").digest("hex");
|
|
886
1255
|
const stamp = startedAt.replace(/[-:.]/g, "");
|
|
887
1256
|
const evalId = `skill-${skillName}-${stamp}-${contentHash.slice(0, 8)}-${randomBytes(6).toString("hex")}`;
|
|
@@ -898,6 +1267,14 @@ function synthesizeArtifact(skillName, evalsPath, evalsRaw, startedAt, endedAt,
|
|
|
898
1267
|
tags: [
|
|
899
1268
|
"skill-eval",
|
|
900
1269
|
`skill:${skillName}`,
|
|
1270
|
+
// The artifact identity, on every record. Two bundles for one skill
|
|
1271
|
+
// name are only comparable when both say which content they ran.
|
|
1272
|
+
`skill-sha:${subject.normalizedHash.slice(0, 12)}`,
|
|
1273
|
+
`skill-origin:${subject.origin}`,
|
|
1274
|
+
...subject.drift !== null ? [`skill-drift:${subject.drift.verdict}`] : [],
|
|
1275
|
+
// Advisory. `pass` and `exitCode` below keep their treatment-only
|
|
1276
|
+
// meaning; this tag is read by nothing that gates.
|
|
1277
|
+
`attribution:${outcome.attribution.verdict}`,
|
|
901
1278
|
...unmeasured ? ["scenario:unmeasured"] : [],
|
|
902
1279
|
...outcome.bullets.map((bullet) => `bullet-${bullet.index}:${bullet.verdict}`)
|
|
903
1280
|
],
|
|
@@ -934,20 +1311,39 @@ function synthesizeArtifact(skillName, evalsPath, evalsRaw, startedAt, endedAt,
|
|
|
934
1311
|
results
|
|
935
1312
|
};
|
|
936
1313
|
}
|
|
937
|
-
function sidecar(
|
|
1314
|
+
function sidecar(subject, evalId, outcomes, allowNetwork, attributionEnabled) {
|
|
938
1315
|
return {
|
|
939
|
-
version:
|
|
1316
|
+
version: 2,
|
|
940
1317
|
schema: "experimental",
|
|
941
1318
|
kind: "skill-eval",
|
|
942
|
-
skill:
|
|
1319
|
+
skill: subject.name,
|
|
1320
|
+
// What this run measured, by identity rather than by name. `version` moved
|
|
1321
|
+
// to 2 for this block; readers of version 1 keep working, they just never
|
|
1322
|
+
// learn which copy produced their numbers.
|
|
1323
|
+
subject,
|
|
943
1324
|
evalId,
|
|
944
1325
|
network: networkPolicyLabel(allowNetwork),
|
|
945
1326
|
autonomy: ARM_AUTONOMY,
|
|
1327
|
+
attributionEnabled,
|
|
1328
|
+
attributionSummary: attributionSummary(outcomes),
|
|
946
1329
|
deltas: [
|
|
947
1330
|
"bullet verdicts are judge-scored from run transcripts, not command exit codes; the evals-domain artifact carries scenario-level records with empty command lists",
|
|
948
1331
|
"a bullet the judge never scored is recorded unmeasured, not failed: its scenario record is pass:false with exitCode 3 and no failureClass",
|
|
949
1332
|
"an arm whose transcript carries the headless permission wall is recorded unmeasured with an infraError: the harness's own gate is not a verdict about the skill",
|
|
950
1333
|
"tokens and cost are rolled up from headless main-agent receipts when those receipts are present",
|
|
1334
|
+
"attribution is advisory and gates nothing: pass, exitCode and failureClass keep their treatment-only meaning",
|
|
1335
|
+
"attribution unmeasured means the comparison was impossible; not-attempted means it was never run. Neither is no-change",
|
|
1336
|
+
"the baseline judge scores the baseline transcript alone; the treatment judge prompt is unchanged from version 1 bundles",
|
|
1337
|
+
"comparison reads both judges' raw output under strict rules (boolean pass, integer in-range index, no contradictory duplicate); refused evidence stays unmeasured and never becomes a verdict",
|
|
1338
|
+
"the rubric bullets above keep the legacy parser's forgiving coercions: tightening comparison eligibility does not move the pass/exitCode gate",
|
|
1339
|
+
"the treatment arm runs against a private per-run copy of the skill directory, so an edit to the source during the arm cannot reach the child; subject.pinnable is false when the body carries package references, which resolve above the skill directory and cannot be snapshotted faithfully",
|
|
1340
|
+
"that copy is an isolated snapshot, not an immutable pin: it lives in a writable temp directory and the arm runs at full-auto, so a model that edited the copy, read it and restored it would not be detected. These are results about a cooperative model, the same caveat the arm workspaces already carry",
|
|
1341
|
+
"subjectVerification is verified only when the snapshot held (tree digest equal before and after the arm) AND the treatment arm reported a successful activation whose sha256 equals subject.sha256 and whose file is the snapshot's own SKILL.md by canonical path. not-activated, activation-mismatch, not-pinned, mismatch and unreadable each name what was not established",
|
|
1342
|
+
"a scenario whose subjectVerification is not verified keeps its rubric result and records its comparison unmeasured",
|
|
1343
|
+
"treeSha256 covers regular files and in-tree symlink targets; a symlink leaving the directory makes the tree unverifiable rather than partially hashed",
|
|
1344
|
+
"both judges are checked for the headless permission wall before their verdicts are eligible for comparison; the legacy rubric result is unaffected either way",
|
|
1345
|
+
"subject.drift records whether the measured content still matches its recorded hash; a mismatch never blocks the run",
|
|
1346
|
+
"subject hashes cover SKILL.md (sha256, normalizedHash) and the whole skill directory (treeSha256); neither covers resources the skill reads from outside its own directory",
|
|
951
1347
|
"this sidecar is additive and is registered in overview.json files[]"
|
|
952
1348
|
],
|
|
953
1349
|
scenarios: outcomes.map((outcome) => ({
|
|
@@ -958,12 +1354,28 @@ function sidecar(skillName, evalId, outcomes, allowNetwork) {
|
|
|
958
1354
|
infraError: outcome.infraError,
|
|
959
1355
|
usage: outcome.usage,
|
|
960
1356
|
bullets: outcome.bullets,
|
|
1357
|
+
subjectVerification: outcome.subjectVerification,
|
|
1358
|
+
observedTreeSha256: outcome.observedTreeSha256,
|
|
1359
|
+
attribution: outcome.attribution,
|
|
961
1360
|
baseline: sidecarRun(outcome.baseline),
|
|
962
1361
|
treatment: sidecarRun(outcome.treatment),
|
|
963
|
-
judge: sidecarRun(outcome.judge)
|
|
1362
|
+
judge: sidecarRun(outcome.judge),
|
|
1363
|
+
baselineJudge: sidecarRun(outcome.baselineJudge)
|
|
964
1364
|
}))
|
|
965
1365
|
};
|
|
966
1366
|
}
|
|
1367
|
+
function attributionSummary(outcomes) {
|
|
1368
|
+
const summary = {
|
|
1369
|
+
helped: 0,
|
|
1370
|
+
regressed: 0,
|
|
1371
|
+
mixed: 0,
|
|
1372
|
+
"no-change": 0,
|
|
1373
|
+
unmeasured: 0,
|
|
1374
|
+
"not-attempted": 0
|
|
1375
|
+
};
|
|
1376
|
+
for (const outcome of outcomes) summary[outcome.attribution.verdict] += 1;
|
|
1377
|
+
return summary;
|
|
1378
|
+
}
|
|
967
1379
|
function sidecarRun(run) {
|
|
968
1380
|
if (run === null) return null;
|
|
969
1381
|
return {
|
|
@@ -975,11 +1387,19 @@ function sidecarRun(run) {
|
|
|
975
1387
|
stderrTail: run.stderr.slice(-500)
|
|
976
1388
|
};
|
|
977
1389
|
}
|
|
978
|
-
function printHumanReport(
|
|
979
|
-
const
|
|
1390
|
+
function printHumanReport(subject, outcomes, evalId, evidenceId, evidenceDirectory, allowNetwork) {
|
|
1391
|
+
const skillName = subject.name;
|
|
1392
|
+
const rows = [["scenario", "bullet", "verdict", "vs base", "expected"]];
|
|
980
1393
|
for (const outcome of outcomes) {
|
|
1394
|
+
const attributed = new Map(outcome.attribution.bullets.map((item) => [item.index, item]));
|
|
981
1395
|
for (const bullet of outcome.bullets) {
|
|
982
|
-
rows.push([
|
|
1396
|
+
rows.push([
|
|
1397
|
+
outcome.scenario.id,
|
|
1398
|
+
String(bullet.index),
|
|
1399
|
+
bullet.verdict,
|
|
1400
|
+
attributed.get(bullet.index)?.attribution ?? "-",
|
|
1401
|
+
truncate(bullet.text, 64)
|
|
1402
|
+
]);
|
|
983
1403
|
}
|
|
984
1404
|
}
|
|
985
1405
|
process.stdout.write(formatColumns(rows));
|
|
@@ -1017,8 +1437,27 @@ ${skillName}: ${passed}/${total} bullets passed${unmeasuredNote} (judge-scored,
|
|
|
1017
1437
|
);
|
|
1018
1438
|
}
|
|
1019
1439
|
}
|
|
1440
|
+
printAttributionReport(outcomes);
|
|
1020
1441
|
process.stdout.write(`${describeArmPolicyOutcome(allowNetwork)}
|
|
1021
1442
|
`);
|
|
1443
|
+
process.stdout.write(
|
|
1444
|
+
`subject: ${skillName} from ${subject.origin} at ${subject.baseDir} (sha256 ${subject.normalizedHash.slice(0, 12)}\u2026)
|
|
1445
|
+
`
|
|
1446
|
+
);
|
|
1447
|
+
if (subject.drift !== null) {
|
|
1448
|
+
process.stdout.write(
|
|
1449
|
+
subject.drift.verdict === "mismatch" ? `subject drift: MISMATCH against the ${subject.drift.authority} (expected ${subject.drift.expected.slice(0, 12)}\u2026); this run measured the copy on disk
|
|
1450
|
+
` : `subject drift: matches the ${subject.drift.authority}
|
|
1451
|
+
`
|
|
1452
|
+
);
|
|
1453
|
+
}
|
|
1454
|
+
const unverified = outcomes.filter((outcome) => outcome.subjectVerification !== "verified");
|
|
1455
|
+
if (unverified.length > 0) {
|
|
1456
|
+
process.stdout.write(
|
|
1457
|
+
`subject verification: ${unverified.map((o) => `${o.scenario.id}=${o.subjectVerification}`).join(", ")}; those scenarios kept their rubric result and recorded no baseline comparison
|
|
1458
|
+
`
|
|
1459
|
+
);
|
|
1460
|
+
}
|
|
1022
1461
|
process.stdout.write(`eval artifact: ${evalId}
|
|
1023
1462
|
`);
|
|
1024
1463
|
if (evidenceId !== null && evidenceDirectory !== null) {
|
|
@@ -1026,6 +1465,45 @@ ${skillName}: ${passed}/${total} bullets passed${unmeasuredNote} (judge-scored,
|
|
|
1026
1465
|
`);
|
|
1027
1466
|
}
|
|
1028
1467
|
}
|
|
1468
|
+
function printAttributionReport(outcomes) {
|
|
1469
|
+
if (outcomes.length === 0) return;
|
|
1470
|
+
const summary = attributionSummary(outcomes);
|
|
1471
|
+
const compared = summary.helped + summary.regressed + summary.mixed + summary["no-change"];
|
|
1472
|
+
if (compared === 0) {
|
|
1473
|
+
const reason = outcomes.find((outcome) => outcome.attribution.reason !== null)?.attribution.reason ?? null;
|
|
1474
|
+
process.stdout.write(
|
|
1475
|
+
`attribution: no scenario was compared against its baseline${reason !== null ? ` (${reason})` : ""}
|
|
1476
|
+
`
|
|
1477
|
+
);
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
const parts = [
|
|
1481
|
+
`${summary.helped} helped`,
|
|
1482
|
+
`${summary.regressed} regressed`,
|
|
1483
|
+
`${summary.mixed} mixed`,
|
|
1484
|
+
`${summary["no-change"]} no-change`
|
|
1485
|
+
];
|
|
1486
|
+
if (summary.unmeasured > 0) parts.push(`${summary.unmeasured} unmeasured`);
|
|
1487
|
+
if (summary["not-attempted"] > 0) parts.push(`${summary["not-attempted"]} not-attempted`);
|
|
1488
|
+
process.stdout.write(
|
|
1489
|
+
`attribution (advisory, vs the no-skill baseline; gates nothing): ${parts.join(", ")} of ${outcomes.length} scenario${outcomes.length === 1 ? "" : "s"}
|
|
1490
|
+
`
|
|
1491
|
+
);
|
|
1492
|
+
for (const outcome of outcomes) {
|
|
1493
|
+
if (outcome.attribution.verdict === "regressed" || outcome.attribution.verdict === "mixed") {
|
|
1494
|
+
const regressed = outcome.attribution.bullets.filter((bullet) => bullet.attribution === "regressed").map((bullet) => String(bullet.index));
|
|
1495
|
+
process.stdout.write(
|
|
1496
|
+
`${outcome.scenario.id}: ${outcome.attribution.verdict}; the baseline passed bullet${regressed.length === 1 ? "" : "s"} ${regressed.join(", ")} and the treatment did not
|
|
1497
|
+
`
|
|
1498
|
+
);
|
|
1499
|
+
} else if (outcome.attribution.reason !== null) {
|
|
1500
|
+
process.stdout.write(
|
|
1501
|
+
`${outcome.scenario.id}: attribution ${outcome.attribution.verdict}: ${outcome.attribution.reason}
|
|
1502
|
+
`
|
|
1503
|
+
);
|
|
1504
|
+
}
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1029
1507
|
function preview(value) {
|
|
1030
1508
|
if (value === void 0) return "";
|
|
1031
1509
|
const text = typeof value === "string" ? value : safeStringify(value);
|
|
@@ -1058,7 +1536,7 @@ function isRecord(value) {
|
|
|
1058
1536
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1059
1537
|
}
|
|
1060
1538
|
async function runSkillEvalCli(args) {
|
|
1061
|
-
const options = { json: false, trustFixtures: false, allowNetwork: false };
|
|
1539
|
+
const options = { json: false, trustFixtures: false, allowNetwork: false, noAttribution: false };
|
|
1062
1540
|
let source;
|
|
1063
1541
|
try {
|
|
1064
1542
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -1067,6 +1545,7 @@ async function runSkillEvalCli(args) {
|
|
|
1067
1545
|
if (arg === "--json") options.json = true;
|
|
1068
1546
|
else if (arg === "--trust-fixtures") options.trustFixtures = true;
|
|
1069
1547
|
else if (arg === "--allow-network") options.allowNetwork = true;
|
|
1548
|
+
else if (arg === "--no-attribution") options.noAttribution = true;
|
|
1070
1549
|
else if (["--scenario", "--target", "--workspace", "--timeout"].includes(arg)) {
|
|
1071
1550
|
const value = args[++i];
|
|
1072
1551
|
if (!value || value.startsWith("-")) throw new Error(`${arg} requires a value`);
|
|
@@ -1079,7 +1558,7 @@ async function runSkillEvalCli(args) {
|
|
|
1079
1558
|
else options.workspace = value;
|
|
1080
1559
|
} else if (arg === "--help" || arg === "-h") {
|
|
1081
1560
|
process.stdout.write(
|
|
1082
|
-
"clio-coder eval skill <name|path> [--scenario <id>] [--target <id>] [--workspace <path>] [--timeout <seconds>] [--trust-fixtures] [--allow-network] [--json]\n"
|
|
1561
|
+
"clio-coder eval skill <name|path> [--scenario <id>] [--target <id>] [--workspace <path>] [--timeout <seconds>] [--trust-fixtures] [--allow-network] [--no-attribution] [--json]\n --no-attribution skip the baseline judge and the paired comparison; saves one judge run per scenario\n"
|
|
1083
1562
|
);
|
|
1084
1563
|
return 0;
|
|
1085
1564
|
} else if (arg.startsWith("-") || source) throw new Error(`unexpected skill eval argument: ${arg}`);
|
|
@@ -1093,7 +1572,18 @@ async function runSkillEvalCli(args) {
|
|
|
1093
1572
|
}
|
|
1094
1573
|
}
|
|
1095
1574
|
export {
|
|
1575
|
+
artifactIsPinnable,
|
|
1576
|
+
attributionFromStrict,
|
|
1577
|
+
deriveBulletAttribution,
|
|
1578
|
+
deriveScenarioAttributionVerdict,
|
|
1096
1579
|
evalChildEnv,
|
|
1097
|
-
|
|
1580
|
+
parseRunStdout,
|
|
1581
|
+
runScenario,
|
|
1582
|
+
runSkillEvalCli,
|
|
1583
|
+
sidecar,
|
|
1584
|
+
skillTreeDigest,
|
|
1585
|
+
strictJudgeVerdicts,
|
|
1586
|
+
verifyObservedActivation,
|
|
1587
|
+
verifySubjectTree
|
|
1098
1588
|
};
|
|
1099
|
-
//# sourceMappingURL=skills-eval-
|
|
1589
|
+
//# sourceMappingURL=skills-eval-AIBR2L7H.js.map
|