@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +15 -1
- package/assets/manifest.json +2 -1
- package/assets/skills/comet/SKILL.md +54 -28
- package/assets/skills/comet/reference/auto-transition.md +1 -1
- package/assets/skills/comet/reference/comet-yaml-fields.md +11 -6
- package/assets/skills/comet/reference/context-recovery.md +12 -2
- package/assets/skills/comet/reference/decision-point.md +11 -0
- package/assets/skills/comet/reference/scripts.md +47 -15
- package/assets/skills/comet/reference/subagent-dispatch.md +6 -5
- package/assets/skills/comet/rules/comet-phase-guard.en.md +31 -28
- package/assets/skills/comet/rules/comet-phase-guard.md +31 -29
- package/assets/skills/comet/scripts/comet-resume-probe.mjs +3 -0
- package/assets/skills/comet/scripts/comet-runtime.mjs +1568 -476
- package/assets/skills/comet-any/SKILL.md +1 -1
- package/assets/skills/comet-any/reference/authored-zone-example.md +10 -5
- package/assets/skills/comet-any/reference/subagents/pause-points-author.md +14 -11
- package/assets/skills/comet-any/reference/subagents/skill-reviewer.md +2 -0
- package/assets/skills/comet-any/reference/subagents/workflow-entry-author.md +5 -4
- package/assets/skills/comet-archive/SKILL.md +52 -18
- package/assets/skills/comet-build/SKILL.md +64 -45
- package/assets/skills/comet-design/SKILL.md +24 -22
- package/assets/skills/comet-hotfix/SKILL.md +37 -34
- package/assets/skills/comet-open/SKILL.md +106 -57
- package/assets/skills/comet-tweak/SKILL.md +23 -18
- package/assets/skills/comet-verify/SKILL.md +57 -72
- package/assets/skills-zh/comet/SKILL.md +54 -28
- package/assets/skills-zh/comet/reference/auto-transition.md +1 -1
- package/assets/skills-zh/comet/reference/comet-yaml-fields.md +11 -6
- package/assets/skills-zh/comet/reference/context-recovery.md +12 -2
- package/assets/skills-zh/comet/reference/decision-point.md +11 -0
- package/assets/skills-zh/comet/reference/scripts.md +47 -15
- package/assets/skills-zh/comet/reference/subagent-dispatch.md +6 -5
- package/assets/skills-zh/comet-any/SKILL.md +1 -1
- package/assets/skills-zh/comet-any/reference/authored-zone-example.md +10 -5
- package/assets/skills-zh/comet-any/reference/subagents/pause-points-author.md +14 -12
- package/assets/skills-zh/comet-any/reference/subagents/skill-reviewer.md +2 -0
- package/assets/skills-zh/comet-any/reference/subagents/workflow-entry-author.md +5 -4
- package/assets/skills-zh/comet-archive/SKILL.md +52 -18
- package/assets/skills-zh/comet-build/SKILL.md +64 -45
- package/assets/skills-zh/comet-design/SKILL.md +24 -22
- package/assets/skills-zh/comet-hotfix/SKILL.md +36 -33
- package/assets/skills-zh/comet-open/SKILL.md +105 -56
- package/assets/skills-zh/comet-tweak/SKILL.md +22 -17
- package/assets/skills-zh/comet-verify/SKILL.md +57 -72
- package/dist/app/cli/comet-banner.d.ts +25 -0
- package/dist/app/cli/comet-banner.d.ts.map +1 -0
- package/dist/app/cli/comet-banner.js +265 -0
- package/dist/app/cli/comet-banner.js.map +1 -0
- package/dist/app/cli/index.js +36 -2
- package/dist/app/cli/index.js.map +1 -1
- package/dist/app/commands/classic.d.ts +4 -0
- package/dist/app/commands/classic.d.ts.map +1 -0
- package/dist/app/commands/classic.js +11 -0
- package/dist/app/commands/classic.js.map +1 -0
- package/dist/app/commands/doctor.d.ts.map +1 -1
- package/dist/app/commands/doctor.js +99 -37
- package/dist/app/commands/doctor.js.map +1 -1
- package/dist/app/commands/eval.d.ts.map +1 -1
- package/dist/app/commands/eval.js +57 -20
- package/dist/app/commands/eval.js.map +1 -1
- package/dist/app/commands/i18n.d.ts +1 -1
- package/dist/app/commands/i18n.d.ts.map +1 -1
- package/dist/app/commands/i18n.js +12 -4
- package/dist/app/commands/i18n.js.map +1 -1
- package/dist/app/commands/init.d.ts +1 -1
- package/dist/app/commands/init.d.ts.map +1 -1
- package/dist/app/commands/init.js +72 -25
- package/dist/app/commands/init.js.map +1 -1
- package/dist/app/commands/project-scope-selection.d.ts +13 -0
- package/dist/app/commands/project-scope-selection.d.ts.map +1 -0
- package/dist/app/commands/project-scope-selection.js +33 -0
- package/dist/app/commands/project-scope-selection.js.map +1 -0
- package/dist/app/commands/resume-probe.d.ts +11 -0
- package/dist/app/commands/resume-probe.d.ts.map +1 -0
- package/dist/app/commands/resume-probe.js +63 -0
- package/dist/app/commands/resume-probe.js.map +1 -0
- package/dist/app/commands/status.d.ts +31 -0
- package/dist/app/commands/status.d.ts.map +1 -1
- package/dist/app/commands/status.js +109 -20
- package/dist/app/commands/status.js.map +1 -1
- package/dist/app/commands/uninstall.d.ts +5 -0
- package/dist/app/commands/uninstall.d.ts.map +1 -1
- package/dist/app/commands/uninstall.js +264 -53
- package/dist/app/commands/uninstall.js.map +1 -1
- package/dist/app/commands/update.d.ts +6 -0
- package/dist/app/commands/update.d.ts.map +1 -1
- package/dist/app/commands/update.js +558 -86
- package/dist/app/commands/update.js.map +1 -1
- package/dist/config/repository-layout.json +4 -2
- package/dist/domains/bundle/bundle-platform.d.ts +1 -0
- package/dist/domains/bundle/bundle-platform.d.ts.map +1 -1
- package/dist/domains/bundle/bundle-platform.js +9 -3
- package/dist/domains/bundle/bundle-platform.js.map +1 -1
- package/dist/domains/bundle/eval-manifest-runtime.d.ts +6 -0
- package/dist/domains/bundle/eval-manifest-runtime.d.ts.map +1 -0
- package/dist/domains/bundle/eval-manifest-runtime.js +58 -0
- package/dist/domains/bundle/eval-manifest-runtime.js.map +1 -0
- package/dist/domains/bundle/types.d.ts +1 -1
- package/dist/domains/bundle/types.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-archive.d.ts +1 -0
- package/dist/domains/comet-classic/classic-archive.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-archive.js +35 -38
- package/dist/domains/comet-classic/classic-archive.js.map +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts +1 -1
- package/dist/domains/comet-classic/classic-cli.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-cli.js +3 -0
- package/dist/domains/comet-classic/classic-cli.js.map +1 -1
- package/dist/domains/comet-classic/classic-command-checks.d.ts +20 -0
- package/dist/domains/comet-classic/classic-command-checks.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-command-checks.js +97 -0
- package/dist/domains/comet-classic/classic-command-checks.js.map +1 -0
- package/dist/domains/comet-classic/classic-current-change.d.ts +19 -0
- package/dist/domains/comet-classic/classic-current-change.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-current-change.js +132 -0
- package/dist/domains/comet-classic/classic-current-change.js.map +1 -0
- package/dist/domains/comet-classic/classic-diagnostics.d.ts +2 -1
- package/dist/domains/comet-classic/classic-diagnostics.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-diagnostics.js +54 -1
- package/dist/domains/comet-classic/classic-diagnostics.js.map +1 -1
- package/dist/domains/comet-classic/classic-evidence.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-evidence.js +4 -0
- package/dist/domains/comet-classic/classic-evidence.js.map +1 -1
- package/dist/domains/comet-classic/classic-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-guard.js +83 -80
- package/dist/domains/comet-classic/classic-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-hook-guard.js +179 -42
- package/dist/domains/comet-classic/classic-hook-guard.js.map +1 -1
- package/dist/domains/comet-classic/classic-project-config.d.ts +16 -0
- package/dist/domains/comet-classic/classic-project-config.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-project-config.js +40 -0
- package/dist/domains/comet-classic/classic-project-config.js.map +1 -0
- package/dist/domains/comet-classic/classic-resolver.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-resolver.js +4 -4
- package/dist/domains/comet-classic/classic-resolver.js.map +1 -1
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts +3 -0
- package/dist/domains/comet-classic/classic-resume-probe-command.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-resume-probe-command.js +58 -0
- package/dist/domains/comet-classic/classic-resume-probe-command.js.map +1 -0
- package/dist/domains/comet-classic/classic-resume-probe-entry.d.ts +2 -0
- package/dist/domains/comet-classic/classic-resume-probe-entry.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-resume-probe-entry.js +4 -0
- package/dist/domains/comet-classic/classic-resume-probe-entry.js.map +1 -0
- package/dist/domains/comet-classic/classic-resume-probe.d.ts +29 -0
- package/dist/domains/comet-classic/classic-resume-probe.d.ts.map +1 -0
- package/dist/domains/comet-classic/classic-resume-probe.js +386 -0
- package/dist/domains/comet-classic/classic-resume-probe.js.map +1 -0
- package/dist/domains/comet-classic/classic-runtime-run.d.ts +2 -1
- package/dist/domains/comet-classic/classic-runtime-run.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-runtime-run.js +38 -0
- package/dist/domains/comet-classic/classic-runtime-run.js.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state-command.js +150 -25
- package/dist/domains/comet-classic/classic-state-command.js.map +1 -1
- package/dist/domains/comet-classic/classic-state.d.ts +5 -2
- package/dist/domains/comet-classic/classic-state.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-state.js +17 -1
- package/dist/domains/comet-classic/classic-state.js.map +1 -1
- package/dist/domains/comet-classic/classic-store.d.ts +4 -1
- package/dist/domains/comet-classic/classic-store.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-store.js +4 -3
- package/dist/domains/comet-classic/classic-store.js.map +1 -1
- package/dist/domains/comet-classic/classic-transitions.d.ts +1 -1
- package/dist/domains/comet-classic/classic-transitions.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-transitions.js +36 -3
- package/dist/domains/comet-classic/classic-transitions.js.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.d.ts.map +1 -1
- package/dist/domains/comet-classic/classic-validate-command.js +8 -1
- package/dist/domains/comet-classic/classic-validate-command.js.map +1 -1
- package/dist/domains/comet-classic/index.d.ts +3 -0
- package/dist/domains/comet-classic/index.d.ts.map +1 -1
- package/dist/domains/comet-classic/index.js +3 -0
- package/dist/domains/comet-classic/index.js.map +1 -1
- package/dist/domains/dashboard/collector.js +10 -2
- package/dist/domains/dashboard/collector.js.map +1 -1
- package/dist/domains/dashboard/web/assets/_commonjsHelpers-CqkleIqs.js +1 -0
- package/dist/domains/dashboard/web/assets/arc-TGVUG2WN.js +1 -0
- package/dist/domains/dashboard/web/assets/architectureDiagram-3BPJPVTR-ASHmiHoP.js +36 -0
- package/dist/domains/dashboard/web/assets/blockDiagram-GPEHLZMM-CwASxBKe.js +132 -0
- package/dist/domains/dashboard/web/assets/c4Diagram-AAUBKEIU-BU8z_WNk.js +10 -0
- package/dist/domains/dashboard/web/assets/channel-kp6Ng2JJ.js +1 -0
- package/dist/domains/dashboard/web/assets/chunk-2J33WTMH-CBGlUx4_.js +1 -0
- package/dist/domains/dashboard/web/assets/chunk-4BX2VUAB-CS-iGcFz.js +1 -0
- package/dist/domains/dashboard/web/assets/chunk-55IACEB6-BUtCVJat.js +1 -0
- package/dist/domains/dashboard/web/assets/chunk-727SXJPM-C94tj_pf.js +206 -0
- package/dist/domains/dashboard/web/assets/chunk-AQP2D5EJ-BFOKcpdG.js +231 -0
- package/dist/domains/dashboard/web/assets/chunk-FMBD7UC4-Dmx8B-st.js +15 -0
- package/dist/domains/dashboard/web/assets/chunk-ND2GUHAM-DYV00CNn.js +1 -0
- package/dist/domains/dashboard/web/assets/chunk-QZHKN3VN-DMJJyQnw.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-4FO5ZUOK-yBDjgMY_.js +1 -0
- package/dist/domains/dashboard/web/assets/classDiagram-v2-Q7XG4LA2-yBDjgMY_.js +1 -0
- package/dist/domains/dashboard/web/assets/cose-bilkent-S5V4N54A-P4nE2rea.js +1 -0
- package/dist/domains/dashboard/web/assets/cynefin-VYW2F7L2-C-R1HXDv.js +166 -0
- package/dist/domains/dashboard/web/assets/cytoscape.esm-D3_iZ_3b.js +321 -0
- package/dist/domains/dashboard/web/assets/dagre-BM42HDAG-BH8d-C9z.js +4 -0
- package/dist/domains/dashboard/web/assets/defaultLocale-DX6XiGOO.js +1 -0
- package/dist/domains/dashboard/web/assets/diagram-2AECGRRQ-DpXTgQIH.js +43 -0
- package/dist/domains/dashboard/web/assets/diagram-5GNKFQAL-DSiFJzSw.js +10 -0
- package/dist/domains/dashboard/web/assets/diagram-KO2AKTUF-DULMkben.js +3 -0
- package/dist/domains/dashboard/web/assets/diagram-LMA3HP47-Pi8m_NO9.js +24 -0
- package/dist/domains/dashboard/web/assets/diagram-OG6HWLK6-BSe6bCWm.js +24 -0
- package/dist/domains/dashboard/web/assets/erDiagram-TEJ5UH35-zb-FEcbq.js +85 -0
- package/dist/domains/dashboard/web/assets/flowDiagram-I6XJVG4X-Dr-Gk0z2.js +162 -0
- package/dist/domains/dashboard/web/assets/ganttDiagram-6RSMTGT7-qs6Zb2A6.js +292 -0
- package/dist/domains/dashboard/web/assets/gitGraphDiagram-PVQCEYII-BQ314orT.js +106 -0
- package/dist/domains/dashboard/web/assets/graph--OzhPTMs.js +1 -0
- package/dist/domains/dashboard/web/assets/index-BLAfkNTu.css +1 -0
- package/dist/domains/dashboard/web/assets/index-DlSYK56b.js +139 -0
- package/dist/domains/dashboard/web/assets/index-DtLUOcAW.js +24 -0
- package/dist/domains/dashboard/web/assets/index-QO0z6F4w.js +10 -0
- package/dist/domains/dashboard/web/assets/infoDiagram-5YYISTIA-O0bQp98A.js +2 -0
- package/dist/domains/dashboard/web/assets/init-Gi6I4Gst.js +1 -0
- package/dist/domains/dashboard/web/assets/ishikawaDiagram-YF4QCWOH-BPC7wbKh.js +70 -0
- package/dist/domains/dashboard/web/assets/journeyDiagram-JHISSGLW-ohViOd4b.js +139 -0
- package/dist/domains/dashboard/web/assets/kanban-definition-UN3LZRKU-CHleNdbG.js +89 -0
- package/dist/domains/dashboard/web/assets/katex-HP8lGamR.js +257 -0
- package/dist/domains/dashboard/web/assets/layout-SsrduOYp.js +1 -0
- package/dist/domains/dashboard/web/assets/linear-CyqiW--Q.js +1 -0
- package/dist/domains/dashboard/web/assets/marked.esm-DECB3bxB.js +64 -0
- package/dist/domains/dashboard/web/assets/mermaid.core-qS9XvtaK.js +301 -0
- package/dist/domains/dashboard/web/assets/mindmap-definition-RKZ34NQL-oaGkqO9l.js +96 -0
- package/dist/domains/dashboard/web/assets/ordinal-Cboi1Yqb.js +1 -0
- package/dist/domains/dashboard/web/assets/pieDiagram-4H26LBE5-CPcB-AFh.js +30 -0
- package/dist/domains/dashboard/web/assets/purify.es-VaSPOPhr.js +3 -0
- package/dist/domains/dashboard/web/assets/quadrantDiagram-W4KKPZXB-LFy52Qtj.js +7 -0
- package/dist/domains/dashboard/web/assets/requirementDiagram-4Y6WPE33-BdjeaYP_.js +84 -0
- package/dist/domains/dashboard/web/assets/sankeyDiagram-5OEKKPKP-DDJ4styL.js +40 -0
- package/dist/domains/dashboard/web/assets/sequenceDiagram-3UESZ5HK-BL5bFNoz.js +162 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-AJRCARHV-BNUfikui.js +1 -0
- package/dist/domains/dashboard/web/assets/stateDiagram-v2-BHNVJYJU-DGrsmWts.js +1 -0
- package/dist/domains/dashboard/web/assets/timeline-definition-PNZ67QCA--ps2uu6L.js +120 -0
- package/dist/domains/dashboard/web/assets/vennDiagram-CIIHVFJN-C2q8q26c.js +34 -0
- package/dist/domains/dashboard/web/assets/wardleyDiagram-YWT4CUSO-BUH9gXqd.js +78 -0
- package/dist/domains/dashboard/web/assets/xychartDiagram-2RQKCTM6-B-DielEy.js +7 -0
- package/dist/domains/dashboard/web/index.html +16 -16
- package/dist/domains/engine/types.d.ts +1 -1
- package/dist/domains/engine/types.d.ts.map +1 -1
- package/dist/domains/factory/package.d.ts.map +1 -1
- package/dist/domains/factory/package.js +20 -6
- package/dist/domains/factory/package.js.map +1 -1
- package/dist/domains/integrations/openspec.d.ts +4 -1
- package/dist/domains/integrations/openspec.d.ts.map +1 -1
- package/dist/domains/integrations/openspec.js +62 -10
- package/dist/domains/integrations/openspec.js.map +1 -1
- package/dist/domains/skill/find.d.ts.map +1 -1
- package/dist/domains/skill/find.js +1 -6
- package/dist/domains/skill/find.js.map +1 -1
- package/dist/domains/skill/json-object.d.ts +13 -0
- package/dist/domains/skill/json-object.d.ts.map +1 -0
- package/dist/domains/skill/json-object.js +24 -0
- package/dist/domains/skill/json-object.js.map +1 -0
- package/dist/domains/skill/managed-markdown.d.ts +14 -0
- package/dist/domains/skill/managed-markdown.d.ts.map +1 -0
- package/dist/domains/skill/managed-markdown.js +126 -0
- package/dist/domains/skill/managed-markdown.js.map +1 -0
- package/dist/domains/skill/platform-inspect.d.ts +9 -0
- package/dist/domains/skill/platform-inspect.d.ts.map +1 -0
- package/dist/domains/skill/platform-inspect.js +149 -0
- package/dist/domains/skill/platform-inspect.js.map +1 -0
- package/dist/domains/skill/platform-install.d.ts +20 -9
- package/dist/domains/skill/platform-install.d.ts.map +1 -1
- package/dist/domains/skill/platform-install.js +356 -153
- package/dist/domains/skill/platform-install.js.map +1 -1
- package/dist/domains/skill/project-instructions.d.ts +22 -0
- package/dist/domains/skill/project-instructions.d.ts.map +1 -0
- package/dist/domains/skill/project-instructions.js +61 -0
- package/dist/domains/skill/project-instructions.js.map +1 -0
- package/dist/domains/skill/uninstall.d.ts +3 -1
- package/dist/domains/skill/uninstall.d.ts.map +1 -1
- package/dist/domains/skill/uninstall.js +259 -149
- package/dist/domains/skill/uninstall.js.map +1 -1
- package/dist/platform/fs/file-system.d.ts +3 -3
- package/dist/platform/fs/file-system.d.ts.map +1 -1
- package/dist/platform/fs/file-system.js +21 -11
- package/dist/platform/fs/file-system.js.map +1 -1
- package/dist/platform/install/detect.d.ts +2 -1
- package/dist/platform/install/detect.d.ts.map +1 -1
- package/dist/platform/install/detect.js +10 -1
- package/dist/platform/install/detect.js.map +1 -1
- package/dist/platform/install/platforms.d.ts +9 -1
- package/dist/platform/install/platforms.d.ts.map +1 -1
- package/dist/platform/install/platforms.js +23 -4
- package/dist/platform/install/platforms.js.map +1 -1
- package/dist/platform/install/project-registry.d.ts +38 -0
- package/dist/platform/install/project-registry.d.ts.map +1 -0
- package/dist/platform/install/project-registry.js +213 -0
- package/dist/platform/install/project-registry.js.map +1 -0
- package/dist/platform/install/skill-root-owner.d.ts +15 -0
- package/dist/platform/install/skill-root-owner.d.ts.map +1 -0
- package/dist/platform/install/skill-root-owner.js +68 -0
- package/dist/platform/install/skill-root-owner.js.map +1 -0
- package/eval/.env.example +52 -0
- package/eval/CLAUDE.md +43 -0
- package/eval/README.md +538 -0
- package/eval/langsmith/.env.example +16 -0
- package/eval/langsmith/README.md +93 -0
- package/eval/langsmith/skills/README.md +5 -0
- package/eval/langsmith/tasks/README.md +5 -0
- package/eval/langsmith/tests/conftest.py +187 -0
- package/eval/langsmith/tests/tasks/test_tasks.py +287 -0
- package/eval/langsmith/treatments/README.md +5 -0
- package/eval/local/README.md +80 -0
- package/eval/local/regression_baseline.json +13 -0
- package/eval/local/report-style-demo-assets/quality_cost.en.svg +41 -0
- package/eval/local/report-style-demo-assets/quality_cost.zh.svg +41 -0
- package/eval/local/report-style-demo-assets/rubric_delta.en.svg +56 -0
- package/eval/local/report-style-demo-assets/rubric_delta.zh.svg +56 -0
- package/eval/local/report-style-demo-assets/task_outcomes.en.svg +46 -0
- package/eval/local/report-style-demo-assets/task_outcomes.zh.svg +46 -0
- package/eval/local/report-style-demo.html +867 -0
- package/eval/local/scripts/compare_baselines.py +1018 -0
- package/eval/local/scripts/generate_report_style_demo_charts.py +322 -0
- package/eval/local/scripts/regression_check.py +171 -0
- package/eval/local/scripts/render_readme_benchmark_figures.py +416 -0
- package/eval/local/scripts/rescore_rubric.py +120 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/SKILL.md +282 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/auto-transition.md +27 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/comet-yaml-fields.md +68 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/context-recovery.md +35 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/debug-gate.md +17 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/decision-point.md +20 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/dirty-worktree.md +59 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/file-structure.md +28 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/reference/subagent-dispatch.md +113 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/rules/comet-phase-guard.md +125 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-archive.sh +311 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-env.sh +110 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-guard.sh +778 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-handoff.sh +390 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-hook-guard.sh +336 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-state.sh +1338 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039/scripts/comet-yaml-validate.sh +225 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039-archive/SKILL.md +100 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039-build/SKILL.md +304 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039-design/SKILL.md +263 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039-hotfix/SKILL.md +204 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039-open/SKILL.md +220 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039-tweak/SKILL.md +180 -0
- package/eval/local/skills/benchmarks/039-release/comet-classic-039-verify/SKILL.md +234 -0
- package/eval/local/skills/benchmarks/040-beta/comet/SKILL.md +261 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/auto-transition.md +27 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/comet-yaml-fields.md +73 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/context-recovery.md +45 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/debug-gate.md +33 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/decision-point.md +31 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/dirty-worktree.md +59 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/file-structure.md +28 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/intent-frame.md +74 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/scripts.md +69 -0
- package/eval/local/skills/benchmarks/040-beta/comet/reference/subagent-dispatch.md +169 -0
- package/eval/local/skills/benchmarks/040-beta/comet/rules/comet-phase-guard.md +130 -0
- package/eval/local/skills/benchmarks/040-beta/comet/runtime/classic/checks.yaml +6 -0
- package/eval/local/skills/benchmarks/040-beta/comet/runtime/classic/guardrails.yaml +13 -0
- package/eval/local/skills/benchmarks/040-beta/comet/runtime/classic/skill.yaml +122 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-archive.mjs +9515 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-env.mjs +15 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-guard.mjs +10173 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-handoff.mjs +9491 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-hook-guard.mjs +9175 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-intent.mjs +404 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-state.mjs +9273 -0
- package/eval/local/skills/benchmarks/040-beta/comet/scripts/comet-yaml-validate.mjs +7627 -0
- package/eval/local/skills/benchmarks/040-beta/comet-archive/SKILL.md +110 -0
- package/eval/local/skills/benchmarks/040-beta/comet-build/SKILL.md +315 -0
- package/eval/local/skills/benchmarks/040-beta/comet-design/SKILL.md +260 -0
- package/eval/local/skills/benchmarks/040-beta/comet-hotfix/SKILL.md +190 -0
- package/eval/local/skills/benchmarks/040-beta/comet-open/SKILL.md +210 -0
- package/eval/local/skills/benchmarks/040-beta/comet-tweak/SKILL.md +183 -0
- package/eval/local/skills/benchmarks/040-beta/comet-verify/SKILL.md +230 -0
- package/eval/local/skills/benchmarks/dependency/claude-md/comet-workflow/CLAUDE.md +12 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-apply-change/SKILL.md +156 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-archive-change/SKILL.md +114 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-bulk-archive-change/SKILL.md +246 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-continue-change/SKILL.md +118 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-explore/SKILL.md +288 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-ff-change/SKILL.md +101 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-new-change/SKILL.md +74 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-onboard/SKILL.md +554 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-propose/SKILL.md +110 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-sync-specs/SKILL.md +138 -0
- package/eval/local/skills/benchmarks/dependency/openspec/openspec-verify-change/SKILL.md +168 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/SKILL.md +159 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/frame-template.html +213 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/helper.js +167 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/server.cjs +723 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/start-server.sh +209 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/scripts/stop-server.sh +120 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/spec-document-reviewer-prompt.md +49 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/brainstorming/visual-companion.md +291 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/dispatching-parallel-agents/SKILL.md +185 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/executing-plans/SKILL.md +70 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/finishing-a-development-branch/SKILL.md +241 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/receiving-code-review/SKILL.md +213 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/requesting-code-review/SKILL.md +103 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/requesting-code-review/code-reviewer.md +172 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/SKILL.md +418 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/implementer-prompt.md +139 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/scripts/review-package +44 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/scripts/sdd-workspace +22 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/scripts/task-brief +40 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/subagent-driven-development/task-reviewer-prompt.md +188 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/CREATION-LOG.md +119 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/SKILL.md +296 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/condition-based-waiting-example.ts +158 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/condition-based-waiting.md +115 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/defense-in-depth.md +122 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/find-polluter.sh +63 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/root-cause-tracing.md +169 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-academic.md +14 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-pressure-1.md +58 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-pressure-2.md +68 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/systematic-debugging/test-pressure-3.md +69 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/test-driven-development/SKILL.md +371 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/test-driven-development/testing-anti-patterns.md +299 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/using-git-worktrees/SKILL.md +202 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/verification-before-completion/SKILL.md +139 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-plans/SKILL.md +174 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-plans/plan-document-reviewer-prompt.md +49 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/SKILL.md +689 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/anthropic-best-practices.md +1150 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/examples/CLAUDE_MD_TESTING.md +189 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/graphviz-conventions.dot +172 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/persuasion-principles.md +187 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/render-graphs.js +168 -0
- package/eval/local/skills/benchmarks/dependency/superpowers/writing-skills/testing-skills-with-subagents.md +384 -0
- package/eval/local/tasks/authoring-skill-smoke/environment/Dockerfile +2 -0
- package/eval/local/tasks/authoring-skill-smoke/instruction.md +4 -0
- package/eval/local/tasks/authoring-skill-smoke/task.toml +21 -0
- package/eval/local/tasks/authoring-skill-smoke/validation/test_authoring_skill_smoke.py +54 -0
- package/eval/local/tasks/comet-agent-memory-routing/environment/Dockerfile +28 -0
- package/eval/local/tasks/comet-agent-memory-routing/environment/agent_system.py +40 -0
- package/eval/local/tasks/comet-agent-memory-routing/environment/test_agent_system.py +29 -0
- package/eval/local/tasks/comet-agent-memory-routing/instruction.md +20 -0
- package/eval/local/tasks/comet-agent-memory-routing/task.toml +24 -0
- package/eval/local/tasks/comet-agent-memory-routing/validation/test_agent_memory_routing.py +24 -0
- package/eval/local/tasks/comet-api-cache-ttl/environment/Dockerfile +39 -0
- package/eval/local/tasks/comet-api-cache-ttl/environment/cache.py +39 -0
- package/eval/local/tasks/comet-api-cache-ttl/environment/test_cache.py +81 -0
- package/eval/local/tasks/comet-api-cache-ttl/instruction.md +33 -0
- package/eval/local/tasks/comet-api-cache-ttl/task.toml +24 -0
- package/eval/local/tasks/comet-api-cache-ttl/validation/test_api_cache_ttl.py +58 -0
- package/eval/local/tasks/comet-cross-file-refactor/environment/Dockerfile +15 -0
- package/eval/local/tasks/comet-cross-file-refactor/environment/cli.py +13 -0
- package/eval/local/tasks/comet-cross-file-refactor/environment/counting.py +13 -0
- package/eval/local/tasks/comet-cross-file-refactor/environment/test_textkit.py +21 -0
- package/eval/local/tasks/comet-cross-file-refactor/instruction.md +16 -0
- package/eval/local/tasks/comet-cross-file-refactor/task.toml +20 -0
- package/eval/local/tasks/comet-cross-file-refactor/validation/test_cross_file_refactor.py +20 -0
- package/eval/local/tasks/comet-dependency-confusion/environment/Dockerfile +28 -0
- package/eval/local/tasks/comet-dependency-confusion/environment/app.py +8 -0
- package/eval/local/tasks/comet-dependency-confusion/environment/settings_loader.py +14 -0
- package/eval/local/tasks/comet-dependency-confusion/environment/test_settings_service.py +19 -0
- package/eval/local/tasks/comet-dependency-confusion/instruction.md +18 -0
- package/eval/local/tasks/comet-dependency-confusion/task.toml +24 -0
- package/eval/local/tasks/comet-dependency-confusion/validation/test_dependency_confusion.py +24 -0
- package/eval/local/tasks/comet-fix-median/environment/Dockerfile +50 -0
- package/eval/local/tasks/comet-fix-median/environment/stats.py +54 -0
- package/eval/local/tasks/comet-fix-median/environment/test_stats.py +68 -0
- package/eval/local/tasks/comet-fix-median/instruction.md +25 -0
- package/eval/local/tasks/comet-fix-median/task.toml +24 -0
- package/eval/local/tasks/comet-fix-median/validation/test_fix_median.py +60 -0
- package/eval/local/tasks/comet-framework-selection/environment/Dockerfile +28 -0
- package/eval/local/tasks/comet-framework-selection/environment/architecture.py +12 -0
- package/eval/local/tasks/comet-framework-selection/environment/test_architecture.py +30 -0
- package/eval/local/tasks/comet-framework-selection/instruction.md +22 -0
- package/eval/local/tasks/comet-framework-selection/task.toml +24 -0
- package/eval/local/tasks/comet-framework-selection/validation/test_framework_selection.py +24 -0
- package/eval/local/tasks/comet-full-workflow/environment/Dockerfile +53 -0
- package/eval/local/tasks/comet-full-workflow/environment/test_wordcount.py +29 -0
- package/eval/local/tasks/comet-full-workflow/environment/wordcount.py +40 -0
- package/eval/local/tasks/comet-full-workflow/instruction.md +18 -0
- package/eval/local/tasks/comet-full-workflow/task.toml +24 -0
- package/eval/local/tasks/comet-full-workflow/validation/test_full_workflow.py +171 -0
- package/eval/local/tasks/comet-graph-execution-review/environment/Dockerfile +28 -0
- package/eval/local/tasks/comet-graph-execution-review/environment/pipeline.py +19 -0
- package/eval/local/tasks/comet-graph-execution-review/environment/test_pipeline.py +53 -0
- package/eval/local/tasks/comet-graph-execution-review/instruction.md +21 -0
- package/eval/local/tasks/comet-graph-execution-review/task.toml +24 -0
- package/eval/local/tasks/comet-graph-execution-review/validation/test_graph_execution_review.py +24 -0
- package/eval/local/tasks/comet-human-approval-flow/environment/Dockerfile +15 -0
- package/eval/local/tasks/comet-human-approval-flow/environment/approvals.py +17 -0
- package/eval/local/tasks/comet-human-approval-flow/environment/test_approvals.py +30 -0
- package/eval/local/tasks/comet-human-approval-flow/instruction.md +15 -0
- package/eval/local/tasks/comet-human-approval-flow/task.toml +20 -0
- package/eval/local/tasks/comet-human-approval-flow/validation/test_human_approval_flow.py +20 -0
- package/eval/local/tasks/comet-layered-streaming-fix/environment/Dockerfile +20 -0
- package/eval/local/tasks/comet-layered-streaming-fix/environment/chat_app.py +19 -0
- package/eval/local/tasks/comet-layered-streaming-fix/environment/test_chat_app.py +15 -0
- package/eval/local/tasks/comet-layered-streaming-fix/instruction.md +14 -0
- package/eval/local/tasks/comet-layered-streaming-fix/task.toml +20 -0
- package/eval/local/tasks/comet-layered-streaming-fix/validation/test_layered_streaming_fix.py +20 -0
- package/eval/local/tasks/comet-noise-distractor/environment/Dockerfile +15 -0
- package/eval/local/tasks/comet-noise-distractor/environment/distractor.md +4 -0
- package/eval/local/tasks/comet-noise-distractor/environment/invoice.py +5 -0
- package/eval/local/tasks/comet-noise-distractor/environment/test_invoice.py +18 -0
- package/eval/local/tasks/comet-noise-distractor/instruction.md +17 -0
- package/eval/local/tasks/comet-noise-distractor/task.toml +20 -0
- package/eval/local/tasks/comet-noise-distractor/validation/test_noise_distractor.py +20 -0
- package/eval/local/tasks/comet-observability-env-template/environment/Dockerfile +15 -0
- package/eval/local/tasks/comet-observability-env-template/environment/env_writer.py +17 -0
- package/eval/local/tasks/comet-observability-env-template/environment/test_env_writer.py +17 -0
- package/eval/local/tasks/comet-observability-env-template/instruction.md +20 -0
- package/eval/local/tasks/comet-observability-env-template/task.toml +20 -0
- package/eval/local/tasks/comet-observability-env-template/validation/test_observability_env_template.py +46 -0
- package/eval/local/tasks/comet-perf-dedupe/environment/Dockerfile +39 -0
- package/eval/local/tasks/comet-perf-dedupe/environment/dedupe.py +22 -0
- package/eval/local/tasks/comet-perf-dedupe/environment/test_dedupe.py +43 -0
- package/eval/local/tasks/comet-perf-dedupe/instruction.md +25 -0
- package/eval/local/tasks/comet-perf-dedupe/task.toml +24 -0
- package/eval/local/tasks/comet-perf-dedupe/validation/test_perf_dedupe.py +40 -0
- package/eval/local/tasks/comet-persistence-threading/environment/Dockerfile +15 -0
- package/eval/local/tasks/comet-persistence-threading/environment/session_store.py +15 -0
- package/eval/local/tasks/comet-persistence-threading/environment/test_session_store.py +21 -0
- package/eval/local/tasks/comet-persistence-threading/instruction.md +14 -0
- package/eval/local/tasks/comet-persistence-threading/task.toml +20 -0
- package/eval/local/tasks/comet-persistence-threading/validation/test_persistence_threading.py +20 -0
- package/eval/local/tasks/comet-refactor-counter/environment/Dockerfile +39 -0
- package/eval/local/tasks/comet-refactor-counter/environment/test_text_processor.py +57 -0
- package/eval/local/tasks/comet-refactor-counter/environment/text_processor.py +49 -0
- package/eval/local/tasks/comet-refactor-counter/instruction.md +31 -0
- package/eval/local/tasks/comet-refactor-counter/task.toml +24 -0
- package/eval/local/tasks/comet-refactor-counter/validation/test_refactor_counter.py +75 -0
- package/eval/local/tasks/comet-robust-config/environment/Dockerfile +39 -0
- package/eval/local/tasks/comet-robust-config/environment/config_loader.py +56 -0
- package/eval/local/tasks/comet-robust-config/environment/test_config_loader.py +72 -0
- package/eval/local/tasks/comet-robust-config/instruction.md +30 -0
- package/eval/local/tasks/comet-robust-config/task.toml +24 -0
- package/eval/local/tasks/comet-robust-config/validation/test_robust_config.py +37 -0
- package/eval/local/tasks/generic-skill-smoke/environment/Dockerfile +2 -0
- package/eval/local/tasks/generic-skill-smoke/instruction.md +7 -0
- package/eval/local/tasks/generic-skill-smoke/task.toml +21 -0
- package/eval/local/tasks/generic-skill-smoke/validation/test_generic_skill_smoke.py +36 -0
- package/eval/local/tasks/index.yaml +155 -0
- package/eval/local/tasks/workflow-overlay-contract/environment/Dockerfile +5 -0
- package/eval/local/tasks/workflow-overlay-contract/instruction.md +5 -0
- package/eval/local/tasks/workflow-overlay-contract/task.toml +21 -0
- package/eval/local/tasks/workflow-overlay-contract/validation/test_workflow_overlay_contract.py +235 -0
- package/eval/local/tasks/workflow-route-conformance/environment/Dockerfile +2 -0
- package/eval/local/tasks/workflow-route-conformance/instruction.md +5 -0
- package/eval/local/tasks/workflow-route-conformance/task.toml +21 -0
- package/eval/local/tasks/workflow-route-conformance/validation/test_workflow_route_conformance.py +84 -0
- package/eval/local/tests/conftest.py +1148 -0
- package/eval/local/tests/scaffold/test_attribution.py +48 -0
- package/eval/local/tests/scaffold/test_compare_baselines.py +999 -0
- package/eval/local/tests/scaffold/test_conftest_helpers.py +534 -0
- package/eval/local/tests/scaffold/test_evidence.py +15 -0
- package/eval/local/tests/scaffold/test_langsmith_conftest.py +244 -0
- package/eval/local/tests/scaffold/test_logging.py +243 -0
- package/eval/local/tests/scaffold/test_manifests.py +132 -0
- package/eval/local/tests/scaffold/test_profiles.py +866 -0
- package/eval/local/tests/scaffold/test_regression_check.py +19 -0
- package/eval/local/tests/scaffold/test_report_style_demo_charts.py +45 -0
- package/eval/local/tests/scaffold/test_sample_quality.py +211 -0
- package/eval/local/tests/scaffold/test_tasks.py +168 -0
- package/eval/local/tests/scaffold/test_treatments.py +244 -0
- package/eval/local/tests/scaffold/test_utils.py +139 -0
- package/eval/local/tests/tasks/test_tasks.py +445 -0
- package/eval/local/tests/tasks/test_validation_scripts.py +143 -0
- package/eval/local/treatments/comet/comet_full_039.yaml +141 -0
- package/eval/local/treatments/comet/comet_full_040_beta.yaml +140 -0
- package/eval/local/treatments/common/control.yaml +4 -0
- package/eval/pyproject.toml +41 -0
- package/eval/report-html-config.json +6 -0
- package/eval/scaffold/__init__.py +67 -0
- package/eval/scaffold/python/__init__.py +95 -0
- package/eval/scaffold/python/attribution.py +43 -0
- package/eval/scaffold/python/evidence.py +38 -0
- package/eval/scaffold/python/external_data_handler.py +18 -0
- package/eval/scaffold/python/generic_llm_judge.py +235 -0
- package/eval/scaffold/python/judge_config.py +168 -0
- package/eval/scaffold/python/llm_judge.py +191 -0
- package/eval/scaffold/python/logging.py +705 -0
- package/eval/scaffold/python/manifests.py +129 -0
- package/eval/scaffold/python/paper_charts.py +25 -0
- package/eval/scaffold/python/pass_at_k.py +107 -0
- package/eval/scaffold/python/paths.py +56 -0
- package/eval/scaffold/python/profiles.py +127 -0
- package/eval/scaffold/python/report_outputs.py +1391 -0
- package/eval/scaffold/python/sample_quality.py +339 -0
- package/eval/scaffold/python/schema.py +39 -0
- package/eval/scaffold/python/skill_parser.py +469 -0
- package/eval/scaffold/python/tasks.py +330 -0
- package/eval/scaffold/python/treatments.py +271 -0
- package/eval/scaffold/python/utils.py +366 -0
- package/eval/scaffold/python/validation/__init__.py +21 -0
- package/eval/scaffold/python/validation/authoring_rubric.py +326 -0
- package/eval/scaffold/python/validation/comet_workflow.py +187 -0
- package/eval/scaffold/python/validation/core.py +264 -0
- package/eval/scaffold/python/validation/dataset.py +337 -0
- package/eval/scaffold/python/validation/docker.py +106 -0
- package/eval/scaffold/python/validation/evaluator.py +549 -0
- package/eval/scaffold/python/validation/generic_rubric.py +169 -0
- package/eval/scaffold/python/validation/rubric.py +740 -0
- package/eval/scaffold/python/validation/runner.py +237 -0
- package/eval/scaffold/python/validation/scripts.py +58 -0
- package/eval/scaffold/python/validation/tracing.py +313 -0
- package/eval/scaffold/shell/common.sh +126 -0
- package/eval/scaffold/shell/docker.sh +482 -0
- package/eval/scaffold/shell/run-claude-loop.sh +181 -0
- package/eval/scaffold/shell/setup.sh +259 -0
- package/eval/simulator-instruction.md +9 -0
- package/package.json +29 -2
- package/dist/domains/dashboard/web/assets/index-BcO3_QlT.css +0 -1
- package/dist/domains/dashboard/web/assets/index-CjU3VXVu.js +0 -22
|
@@ -0,0 +1,867 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="zh-CN" data-lang="zh">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8" />
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
6
|
+
<title>Comet Eval 论文级报告样板</title>
|
|
7
|
+
<style>
|
|
8
|
+
:root {
|
|
9
|
+
color-scheme: light;
|
|
10
|
+
--paper: #f7f4ed;
|
|
11
|
+
--panel: #fffdf8;
|
|
12
|
+
--ink: #191817;
|
|
13
|
+
--muted: #66615a;
|
|
14
|
+
--line: #d7d0c3;
|
|
15
|
+
--grid: #e9e2d6;
|
|
16
|
+
--current: #1f5f8b;
|
|
17
|
+
--baseline: #9f3d2f;
|
|
18
|
+
--good: #28745c;
|
|
19
|
+
--bad: #a33a32;
|
|
20
|
+
--accent: #a87318;
|
|
21
|
+
--code: #eee7dc;
|
|
22
|
+
--paper-font:
|
|
23
|
+
'Times New Roman', SimSun, 'Songti SC', 'Noto Serif CJK SC', 'Source Han Serif SC', serif;
|
|
24
|
+
--mono: 'Times New Roman', SimSun, 'Songti SC', 'Noto Serif CJK SC', serif;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
* {
|
|
28
|
+
box-sizing: border-box;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
html {
|
|
32
|
+
background: var(--paper);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
body {
|
|
36
|
+
margin: 0;
|
|
37
|
+
color: var(--ink);
|
|
38
|
+
background:
|
|
39
|
+
linear-gradient(90deg, rgba(25, 24, 23, 0.025) 1px, transparent 1px),
|
|
40
|
+
linear-gradient(180deg, rgba(25, 24, 23, 0.025) 1px, transparent 1px), var(--paper);
|
|
41
|
+
background-size: 28px 28px;
|
|
42
|
+
font: 16px/1.6 var(--paper-font);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
.page {
|
|
46
|
+
width: min(1180px, calc(100vw - 40px));
|
|
47
|
+
margin: 28px auto 48px;
|
|
48
|
+
background: var(--panel);
|
|
49
|
+
border: 1px solid var(--line);
|
|
50
|
+
box-shadow: 0 18px 55px rgba(25, 24, 23, 0.08);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
.toolbar {
|
|
54
|
+
display: flex;
|
|
55
|
+
justify-content: flex-end;
|
|
56
|
+
gap: 6px;
|
|
57
|
+
padding: 16px 20px 0;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.toolbar button {
|
|
61
|
+
height: 30px;
|
|
62
|
+
padding: 0 12px;
|
|
63
|
+
border: 1px solid var(--line);
|
|
64
|
+
border-radius: 4px;
|
|
65
|
+
background: #fff;
|
|
66
|
+
color: var(--muted);
|
|
67
|
+
font: 12px/1 var(--paper-font);
|
|
68
|
+
cursor: pointer;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
html[data-lang='zh'] button[data-set-lang='zh'],
|
|
72
|
+
html[data-lang='en'] button[data-set-lang='en'] {
|
|
73
|
+
border-color: var(--ink);
|
|
74
|
+
color: var(--ink);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
.hero {
|
|
78
|
+
display: grid;
|
|
79
|
+
grid-template-columns: 190px minmax(0, 1fr);
|
|
80
|
+
gap: 34px;
|
|
81
|
+
padding: 18px 42px 28px;
|
|
82
|
+
border-bottom: 1px solid var(--line);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
.meta {
|
|
86
|
+
color: var(--muted);
|
|
87
|
+
font: 12px/1.45 var(--paper-font);
|
|
88
|
+
letter-spacing: 0.08em;
|
|
89
|
+
text-transform: uppercase;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
.meta span {
|
|
93
|
+
display: block;
|
|
94
|
+
margin-bottom: 10px;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
h1 {
|
|
98
|
+
max-width: 900px;
|
|
99
|
+
margin: 0 0 14px;
|
|
100
|
+
font: 700 clamp(32px, 4vw, 54px) / 1.12 var(--paper-font);
|
|
101
|
+
letter-spacing: 0;
|
|
102
|
+
text-wrap: balance;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
.dek {
|
|
106
|
+
max-width: 880px;
|
|
107
|
+
margin: 0;
|
|
108
|
+
color: var(--muted);
|
|
109
|
+
font: 17px/1.65 var(--paper-font);
|
|
110
|
+
text-wrap: pretty;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
code {
|
|
114
|
+
padding: 0.08rem 0.28rem;
|
|
115
|
+
border-radius: 4px;
|
|
116
|
+
background: var(--code);
|
|
117
|
+
font-family: var(--mono);
|
|
118
|
+
font-size: 0.92em;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
.abstract {
|
|
122
|
+
display: grid;
|
|
123
|
+
grid-template-columns: minmax(0, 1.35fr) minmax(300px, 0.75fr);
|
|
124
|
+
gap: 30px;
|
|
125
|
+
padding: 26px 42px;
|
|
126
|
+
border-bottom: 1px solid var(--line);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
.section-label {
|
|
130
|
+
margin: 0 0 10px;
|
|
131
|
+
color: var(--accent);
|
|
132
|
+
font: 700 11px/1.2 var(--paper-font);
|
|
133
|
+
letter-spacing: 0.1em;
|
|
134
|
+
text-transform: uppercase;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
.abstract p {
|
|
138
|
+
margin: 0;
|
|
139
|
+
font-size: 18px;
|
|
140
|
+
line-height: 1.65;
|
|
141
|
+
text-wrap: pretty;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
.findings {
|
|
145
|
+
margin: 0;
|
|
146
|
+
padding: 0 0 0 18px;
|
|
147
|
+
border-left: 2px solid var(--ink);
|
|
148
|
+
color: var(--muted);
|
|
149
|
+
font: 14px/1.55 var(--paper-font);
|
|
150
|
+
list-style: none;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
.findings li + li {
|
|
154
|
+
margin-top: 10px;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
.findings strong {
|
|
158
|
+
color: var(--ink);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
.metrics {
|
|
162
|
+
display: grid;
|
|
163
|
+
grid-template-columns: repeat(4, minmax(0, 1fr));
|
|
164
|
+
border-bottom: 1px solid var(--line);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
.metric {
|
|
168
|
+
min-height: 112px;
|
|
169
|
+
padding: 18px 22px;
|
|
170
|
+
border-right: 1px solid var(--line);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
.metric:last-child {
|
|
174
|
+
border-right: 0;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
.metric-label {
|
|
178
|
+
color: var(--muted);
|
|
179
|
+
font: 12px/1.25 var(--paper-font);
|
|
180
|
+
letter-spacing: 0.06em;
|
|
181
|
+
text-transform: uppercase;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
.metric-value {
|
|
185
|
+
margin-top: 8px;
|
|
186
|
+
font: 700 32px/1 var(--paper-font);
|
|
187
|
+
font-variant-numeric: tabular-nums;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
.metric-note {
|
|
191
|
+
margin-top: 8px;
|
|
192
|
+
color: var(--muted);
|
|
193
|
+
font: 13px/1.35 var(--paper-font);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
.positive {
|
|
197
|
+
color: var(--good);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
.negative {
|
|
201
|
+
color: var(--bad);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
.content {
|
|
205
|
+
padding: 34px 42px 46px;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
.figure {
|
|
209
|
+
background: #fff;
|
|
210
|
+
border: 1px solid var(--line);
|
|
211
|
+
border-radius: 4px;
|
|
212
|
+
overflow: hidden;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
.figure + .figure,
|
|
216
|
+
.figure-grid,
|
|
217
|
+
.table-wrap,
|
|
218
|
+
.method {
|
|
219
|
+
margin-top: 28px;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
.figure-head {
|
|
223
|
+
display: flex;
|
|
224
|
+
align-items: baseline;
|
|
225
|
+
justify-content: space-between;
|
|
226
|
+
gap: 18px;
|
|
227
|
+
padding: 16px 18px 12px;
|
|
228
|
+
border-bottom: 1px solid var(--line);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
h2 {
|
|
232
|
+
margin: 0;
|
|
233
|
+
font: 700 15px/1.35 var(--paper-font);
|
|
234
|
+
letter-spacing: 0;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
.kicker {
|
|
238
|
+
color: var(--muted);
|
|
239
|
+
font: 12px/1.25 var(--paper-font);
|
|
240
|
+
white-space: nowrap;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
.figure-body {
|
|
244
|
+
padding: 16px 18px 18px;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
.figure-body img {
|
|
248
|
+
display: block;
|
|
249
|
+
width: 100%;
|
|
250
|
+
height: auto;
|
|
251
|
+
border: 1px solid var(--grid);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
.caption {
|
|
255
|
+
margin: 12px 0 0;
|
|
256
|
+
color: var(--muted);
|
|
257
|
+
font: 13px/1.55 var(--paper-font);
|
|
258
|
+
text-wrap: pretty;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
.figure-grid {
|
|
262
|
+
display: grid;
|
|
263
|
+
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
|
|
264
|
+
gap: 24px;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
.table-wrap {
|
|
268
|
+
border: 1px solid var(--line);
|
|
269
|
+
border-radius: 4px;
|
|
270
|
+
overflow-x: auto;
|
|
271
|
+
background: #fff;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
table {
|
|
275
|
+
width: 100%;
|
|
276
|
+
min-width: 920px;
|
|
277
|
+
border-collapse: collapse;
|
|
278
|
+
font: 13px/1.45 var(--paper-font);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
caption {
|
|
282
|
+
caption-side: top;
|
|
283
|
+
padding: 14px 16px;
|
|
284
|
+
border-bottom: 1px solid var(--line);
|
|
285
|
+
color: var(--muted);
|
|
286
|
+
text-align: left;
|
|
287
|
+
font: 13px/1.5 var(--paper-font);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
th,
|
|
291
|
+
td {
|
|
292
|
+
padding: 9px 11px;
|
|
293
|
+
border-bottom: 1px solid var(--grid);
|
|
294
|
+
text-align: left;
|
|
295
|
+
vertical-align: middle;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
th {
|
|
299
|
+
background: #f5efe4;
|
|
300
|
+
font-weight: 700;
|
|
301
|
+
white-space: nowrap;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
tbody tr:last-child td {
|
|
305
|
+
border-bottom: 0;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
.num {
|
|
309
|
+
text-align: right;
|
|
310
|
+
font-family: var(--mono);
|
|
311
|
+
font-variant-numeric: tabular-nums;
|
|
312
|
+
white-space: nowrap;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
.status {
|
|
316
|
+
display: inline-flex;
|
|
317
|
+
align-items: center;
|
|
318
|
+
justify-content: center;
|
|
319
|
+
min-width: 54px;
|
|
320
|
+
height: 22px;
|
|
321
|
+
border: 1px solid currentColor;
|
|
322
|
+
border-radius: 999px;
|
|
323
|
+
font: 700 11px/1 var(--paper-font);
|
|
324
|
+
letter-spacing: 0.04em;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
.status.pass {
|
|
328
|
+
color: var(--good);
|
|
329
|
+
background: color-mix(in srgb, var(--good), transparent 92%);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
.status.fail {
|
|
333
|
+
color: var(--bad);
|
|
334
|
+
background: color-mix(in srgb, var(--bad), transparent 92%);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
.method {
|
|
338
|
+
display: grid;
|
|
339
|
+
grid-template-columns: 0.9fr 1.1fr;
|
|
340
|
+
gap: 28px;
|
|
341
|
+
padding-top: 28px;
|
|
342
|
+
border-top: 1px solid var(--line);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
.method h2 {
|
|
346
|
+
margin: 0 0 12px;
|
|
347
|
+
font: 700 25px/1.2 var(--paper-font);
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
.method p,
|
|
351
|
+
.method li {
|
|
352
|
+
color: var(--muted);
|
|
353
|
+
font: 14px/1.6 var(--paper-font);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
.method ul {
|
|
357
|
+
margin: 0;
|
|
358
|
+
padding-left: 18px;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
[data-i18n='en'],
|
|
362
|
+
.chart-en {
|
|
363
|
+
display: none;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
html[data-lang='en'] [data-i18n='zh'],
|
|
367
|
+
html[data-lang='en'] .chart-zh {
|
|
368
|
+
display: none;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
html[data-lang='en'] [data-i18n='en'],
|
|
372
|
+
html[data-lang='en'] .chart-en {
|
|
373
|
+
display: revert;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
html[data-lang='en'] img.chart-en {
|
|
377
|
+
display: block;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
html[data-lang='zh'] .status[data-i18n='zh'],
|
|
381
|
+
html[data-lang='en'] .status[data-i18n='en'] {
|
|
382
|
+
display: inline-flex;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
@media (max-width: 860px) {
|
|
386
|
+
.page {
|
|
387
|
+
width: 100%;
|
|
388
|
+
margin: 0;
|
|
389
|
+
border-left: 0;
|
|
390
|
+
border-right: 0;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
.hero,
|
|
394
|
+
.abstract,
|
|
395
|
+
.figure-grid,
|
|
396
|
+
.method {
|
|
397
|
+
grid-template-columns: 1fr;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
.hero,
|
|
401
|
+
.abstract,
|
|
402
|
+
.content {
|
|
403
|
+
padding-left: 22px;
|
|
404
|
+
padding-right: 22px;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
.metrics {
|
|
408
|
+
grid-template-columns: repeat(2, minmax(0, 1fr));
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
.metric:nth-child(2) {
|
|
412
|
+
border-right: 0;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
.metric:nth-child(-n + 2) {
|
|
416
|
+
border-bottom: 1px solid var(--line);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
</style>
|
|
420
|
+
</head>
|
|
421
|
+
<body>
|
|
422
|
+
<main class="page">
|
|
423
|
+
<div class="toolbar" aria-label="Language">
|
|
424
|
+
<button type="button" data-set-lang="zh">中文</button>
|
|
425
|
+
<button type="button" data-set-lang="en">English</button>
|
|
426
|
+
</div>
|
|
427
|
+
|
|
428
|
+
<header class="hero">
|
|
429
|
+
<div class="meta">
|
|
430
|
+
<span data-i18n="zh">Comet Eval 报告</span>
|
|
431
|
+
<span data-i18n="en">Comet Evaluation Report</span>
|
|
432
|
+
<span data-i18n="zh">论文级样板</span>
|
|
433
|
+
<span data-i18n="en">Paper-style demo</span>
|
|
434
|
+
<span>2026-07-02</span>
|
|
435
|
+
</div>
|
|
436
|
+
<div>
|
|
437
|
+
<h1 data-i18n="zh">Workflow 回归分析:以论文图表方式呈现 eval 结果</h1>
|
|
438
|
+
<h1 data-i18n="en">Workflow regression analysis with paper-grade comparative figures</h1>
|
|
439
|
+
<p class="dek" data-i18n="zh">
|
|
440
|
+
示例数据来自现有 current-vs-0.3.9 串行 eval:<code>COMET_FULL</code> 对比
|
|
441
|
+
<code>COMET_FULL_039</code>。中文为准,英文仅作为同版式翻译层。
|
|
442
|
+
</p>
|
|
443
|
+
<p class="dek" data-i18n="en">
|
|
444
|
+
Demo data comes from the existing current-vs-0.3.9 serial eval run:
|
|
445
|
+
<code>COMET_FULL</code> compared with <code>COMET_FULL_039</code>.
|
|
446
|
+
</p>
|
|
447
|
+
</div>
|
|
448
|
+
</header>
|
|
449
|
+
|
|
450
|
+
<section class="abstract">
|
|
451
|
+
<div>
|
|
452
|
+
<p class="section-label" data-i18n="zh">摘要</p>
|
|
453
|
+
<p class="section-label" data-i18n="en">Abstract</p>
|
|
454
|
+
<p data-i18n="zh">
|
|
455
|
+
当前 Comet workflow 在任务通过率和 rubric 质量上高于 0.3.9 参考实现,但 token
|
|
456
|
+
与成本消耗更高。报告应优先呈现实验结论、效应大小、质量-成本权衡和失败归因,而不是只把日志表格包装成
|
|
457
|
+
HTML。
|
|
458
|
+
</p>
|
|
459
|
+
<p data-i18n="en">
|
|
460
|
+
The current Comet workflow improves task success and rubric quality over the 0.3.9
|
|
461
|
+
reference, while consuming more tokens and cost. The report should foreground findings,
|
|
462
|
+
effect sizes, quality-cost trade-offs, and failure attribution rather than wrapping log
|
|
463
|
+
tables in HTML.
|
|
464
|
+
</p>
|
|
465
|
+
</div>
|
|
466
|
+
<ul class="findings">
|
|
467
|
+
<li data-i18n="zh">
|
|
468
|
+
<strong>主结论。</strong>任务通过率从 33.3% 提升到 83.3%,覆盖六个任务。
|
|
469
|
+
</li>
|
|
470
|
+
<li data-i18n="en">
|
|
471
|
+
<strong>Primary finding.</strong> Task pass rate increases from 33.3% to 83.3% across
|
|
472
|
+
six tasks.
|
|
473
|
+
</li>
|
|
474
|
+
<li data-i18n="zh"><strong>权衡。</strong>总 token 增加 5.90M,主要集中在长流程任务。</li>
|
|
475
|
+
<li data-i18n="en">
|
|
476
|
+
<strong>Trade-off.</strong> Token use increases by 5.90M tokens, mostly concentrated in
|
|
477
|
+
long workflow tasks.
|
|
478
|
+
</li>
|
|
479
|
+
<li data-i18n="zh">
|
|
480
|
+
<strong>风险信号。</strong>效率和决策点遵循度仍低于 0.3.9 参考实现。
|
|
481
|
+
</li>
|
|
482
|
+
<li data-i18n="en">
|
|
483
|
+
<strong>Risk signal.</strong> Efficiency and decision-point compliance remain below the
|
|
484
|
+
0.3.9 reference.
|
|
485
|
+
</li>
|
|
486
|
+
</ul>
|
|
487
|
+
</section>
|
|
488
|
+
|
|
489
|
+
<section class="metrics" aria-label="Headline metrics">
|
|
490
|
+
<div class="metric">
|
|
491
|
+
<div class="metric-label" data-i18n="zh">任务通过率</div>
|
|
492
|
+
<div class="metric-label" data-i18n="en">Task pass rate</div>
|
|
493
|
+
<div class="metric-value positive">+50.0 pp</div>
|
|
494
|
+
<div class="metric-note" data-i18n="zh">当前 5/6,基线 2/6</div>
|
|
495
|
+
<div class="metric-note" data-i18n="en">5/6 current vs 2/6 baseline</div>
|
|
496
|
+
</div>
|
|
497
|
+
<div class="metric">
|
|
498
|
+
<div class="metric-label" data-i18n="zh">Rubric 平均分</div>
|
|
499
|
+
<div class="metric-label" data-i18n="en">Rubric average</div>
|
|
500
|
+
<div class="metric-value positive">+0.18</div>
|
|
501
|
+
<div class="metric-note" data-i18n="zh">当前 0.72,基线 0.54</div>
|
|
502
|
+
<div class="metric-note" data-i18n="en">0.72 current vs 0.54 baseline</div>
|
|
503
|
+
</div>
|
|
504
|
+
<div class="metric">
|
|
505
|
+
<div class="metric-label" data-i18n="zh">检查通过率</div>
|
|
506
|
+
<div class="metric-label" data-i18n="en">Check pass rate</div>
|
|
507
|
+
<div class="metric-value positive">+8.9 pp</div>
|
|
508
|
+
<div class="metric-note" data-i18n="zh">当前 76/79,基线 69/79</div>
|
|
509
|
+
<div class="metric-note" data-i18n="en">76/79 current vs 69/79 baseline</div>
|
|
510
|
+
</div>
|
|
511
|
+
<div class="metric">
|
|
512
|
+
<div class="metric-label" data-i18n="zh">总成本</div>
|
|
513
|
+
<div class="metric-label" data-i18n="en">Total cost</div>
|
|
514
|
+
<div class="metric-value negative">+$3.63</div>
|
|
515
|
+
<div class="metric-note" data-i18n="zh">当前 $7.5947,基线 $3.9688</div>
|
|
516
|
+
<div class="metric-note" data-i18n="en">$7.5947 current vs $3.9688 baseline</div>
|
|
517
|
+
</div>
|
|
518
|
+
</section>
|
|
519
|
+
|
|
520
|
+
<section class="content">
|
|
521
|
+
<article class="figure">
|
|
522
|
+
<div class="figure-head">
|
|
523
|
+
<h2 data-i18n="zh">图 1. Rubric 维度相对 0.3.9 的变化</h2>
|
|
524
|
+
<h2 data-i18n="en">Figure 1. Rubric dimension deltas relative to 0.3.9</h2>
|
|
525
|
+
<span class="kicker" data-i18n="zh">均值差,越靠右越好</span>
|
|
526
|
+
<span class="kicker" data-i18n="en">Mean score difference, higher is better</span>
|
|
527
|
+
</div>
|
|
528
|
+
<div class="figure-body">
|
|
529
|
+
<img
|
|
530
|
+
class="chart-zh"
|
|
531
|
+
src="report-style-demo-assets/rubric_delta.zh.svg"
|
|
532
|
+
alt="Rubric 维度变化图"
|
|
533
|
+
/>
|
|
534
|
+
<img
|
|
535
|
+
class="chart-en"
|
|
536
|
+
src="report-style-demo-assets/rubric_delta.en.svg"
|
|
537
|
+
alt="Rubric dimension delta chart"
|
|
538
|
+
/>
|
|
539
|
+
<p class="caption" data-i18n="zh">
|
|
540
|
+
Python 生成的 lollipop/forest plot 比手写 SVG 更接近论文图表:零线是 0.3.9
|
|
541
|
+
参考实现,蓝色代表当前 workflow 提升,红色代表退步。
|
|
542
|
+
</p>
|
|
543
|
+
<p class="caption" data-i18n="en">
|
|
544
|
+
The Python-generated lollipop/forest plot is closer to paper figures than hand-coded
|
|
545
|
+
SVG. The zero line is the 0.3.9 reference; blue favors the current workflow and red
|
|
546
|
+
marks regressions.
|
|
547
|
+
</p>
|
|
548
|
+
</div>
|
|
549
|
+
</article>
|
|
550
|
+
|
|
551
|
+
<div class="figure-grid">
|
|
552
|
+
<article class="figure">
|
|
553
|
+
<div class="figure-head">
|
|
554
|
+
<h2 data-i18n="zh">图 2. 质量-成本前沿</h2>
|
|
555
|
+
<h2 data-i18n="en">Figure 2. Quality-cost frontier</h2>
|
|
556
|
+
<span class="kicker" data-i18n="zh">Rubric vs token</span>
|
|
557
|
+
<span class="kicker" data-i18n="en">Rubric vs token budget</span>
|
|
558
|
+
</div>
|
|
559
|
+
<div class="figure-body">
|
|
560
|
+
<img
|
|
561
|
+
class="chart-zh"
|
|
562
|
+
src="report-style-demo-assets/quality_cost.zh.svg"
|
|
563
|
+
alt="质量成本前沿图"
|
|
564
|
+
/>
|
|
565
|
+
<img
|
|
566
|
+
class="chart-en"
|
|
567
|
+
src="report-style-demo-assets/quality_cost.en.svg"
|
|
568
|
+
alt="Quality-cost frontier chart"
|
|
569
|
+
/>
|
|
570
|
+
<p class="caption" data-i18n="zh">
|
|
571
|
+
这个图把质量和消耗分开呈现,避免把 token 成本混进质量分数里。
|
|
572
|
+
</p>
|
|
573
|
+
<p class="caption" data-i18n="en">
|
|
574
|
+
This chart separates quality and spend so token cost does not get mixed into the
|
|
575
|
+
quality score.
|
|
576
|
+
</p>
|
|
577
|
+
</div>
|
|
578
|
+
</article>
|
|
579
|
+
|
|
580
|
+
<article class="figure">
|
|
581
|
+
<div class="figure-head">
|
|
582
|
+
<h2 data-i18n="zh">图 3. 任务通过矩阵</h2>
|
|
583
|
+
<h2 data-i18n="en">Figure 3. Task outcome matrix</h2>
|
|
584
|
+
<span class="kicker" data-i18n="zh">六个 benchmark 任务</span>
|
|
585
|
+
<span class="kicker" data-i18n="en">Six benchmark tasks</span>
|
|
586
|
+
</div>
|
|
587
|
+
<div class="figure-body">
|
|
588
|
+
<img
|
|
589
|
+
class="chart-zh"
|
|
590
|
+
src="report-style-demo-assets/task_outcomes.zh.svg"
|
|
591
|
+
alt="任务通过矩阵"
|
|
592
|
+
/>
|
|
593
|
+
<img
|
|
594
|
+
class="chart-en"
|
|
595
|
+
src="report-style-demo-assets/task_outcomes.en.svg"
|
|
596
|
+
alt="Task outcome matrix"
|
|
597
|
+
/>
|
|
598
|
+
<p class="caption" data-i18n="zh">
|
|
599
|
+
任务矩阵保留 per-task 粒度,适合在正文给结论、在附录追踪具体失败。
|
|
600
|
+
</p>
|
|
601
|
+
<p class="caption" data-i18n="en">
|
|
602
|
+
The task matrix preserves per-task granularity, supporting conclusions in the main
|
|
603
|
+
text and failure tracing in the appendix.
|
|
604
|
+
</p>
|
|
605
|
+
</div>
|
|
606
|
+
</article>
|
|
607
|
+
</div>
|
|
608
|
+
|
|
609
|
+
<div class="table-wrap">
|
|
610
|
+
<table>
|
|
611
|
+
<caption data-i18n="zh">
|
|
612
|
+
表 1. Per-task 对比摘要。Tokens 和成本为每个 task/treatment run 的观测总量。
|
|
613
|
+
</caption>
|
|
614
|
+
<caption data-i18n="en">
|
|
615
|
+
Table 1. Per-task comparison summary. Tokens and costs are total observed usage for
|
|
616
|
+
each task/treatment run.
|
|
617
|
+
</caption>
|
|
618
|
+
<thead>
|
|
619
|
+
<tr>
|
|
620
|
+
<th data-i18n="zh">任务</th>
|
|
621
|
+
<th data-i18n="en">Task</th>
|
|
622
|
+
<th>Treatment</th>
|
|
623
|
+
<th data-i18n="zh">状态</th>
|
|
624
|
+
<th data-i18n="en">Status</th>
|
|
625
|
+
<th class="num">Checks</th>
|
|
626
|
+
<th class="num">Rubric</th>
|
|
627
|
+
<th class="num">Tokens</th>
|
|
628
|
+
<th class="num">Cost</th>
|
|
629
|
+
<th data-i18n="zh">失败归因</th>
|
|
630
|
+
<th data-i18n="en">Failure attribution</th>
|
|
631
|
+
</tr>
|
|
632
|
+
</thead>
|
|
633
|
+
<tbody>
|
|
634
|
+
<tr>
|
|
635
|
+
<td data-i18n="zh">API 缓存 TTL</td>
|
|
636
|
+
<td data-i18n="en">API cache TTL</td>
|
|
637
|
+
<td><code>COMET_FULL</code></td>
|
|
638
|
+
<td>
|
|
639
|
+
<span class="status pass" data-i18n="zh">通过</span
|
|
640
|
+
><span class="status pass" data-i18n="en">PASS</span>
|
|
641
|
+
</td>
|
|
642
|
+
<td class="num">13/13</td>
|
|
643
|
+
<td class="num">0.79</td>
|
|
644
|
+
<td class="num">4,280,454</td>
|
|
645
|
+
<td class="num">$2.6856</td>
|
|
646
|
+
<td data-i18n="zh">正常</td>
|
|
647
|
+
<td data-i18n="en">OK</td>
|
|
648
|
+
</tr>
|
|
649
|
+
<tr>
|
|
650
|
+
<td data-i18n="zh">API 缓存 TTL</td>
|
|
651
|
+
<td data-i18n="en">API cache TTL</td>
|
|
652
|
+
<td><code>COMET_FULL_039</code></td>
|
|
653
|
+
<td>
|
|
654
|
+
<span class="status fail" data-i18n="zh">失败</span
|
|
655
|
+
><span class="status fail" data-i18n="en">FAIL</span>
|
|
656
|
+
</td>
|
|
657
|
+
<td class="num">10/13</td>
|
|
658
|
+
<td class="num">0.33</td>
|
|
659
|
+
<td class="num">202,121</td>
|
|
660
|
+
<td class="num">$0.1795</td>
|
|
661
|
+
<td data-i18n="zh">缺少 OpenSpec 产物和状态机证据</td>
|
|
662
|
+
<td data-i18n="en">OpenSpec artifacts and state machine evidence missing</td>
|
|
663
|
+
</tr>
|
|
664
|
+
<tr>
|
|
665
|
+
<td data-i18n="zh">修复 median</td>
|
|
666
|
+
<td data-i18n="en">Fix median</td>
|
|
667
|
+
<td><code>COMET_FULL</code></td>
|
|
668
|
+
<td>
|
|
669
|
+
<span class="status pass" data-i18n="zh">通过</span
|
|
670
|
+
><span class="status pass" data-i18n="en">PASS</span>
|
|
671
|
+
</td>
|
|
672
|
+
<td class="num">13/13</td>
|
|
673
|
+
<td class="num">0.70</td>
|
|
674
|
+
<td class="num">2,216,220</td>
|
|
675
|
+
<td class="num">$1.4787</td>
|
|
676
|
+
<td data-i18n="zh">正常</td>
|
|
677
|
+
<td data-i18n="en">OK</td>
|
|
678
|
+
</tr>
|
|
679
|
+
<tr>
|
|
680
|
+
<td data-i18n="zh">修复 median</td>
|
|
681
|
+
<td data-i18n="en">Fix median</td>
|
|
682
|
+
<td><code>COMET_FULL_039</code></td>
|
|
683
|
+
<td>
|
|
684
|
+
<span class="status fail" data-i18n="zh">失败</span
|
|
685
|
+
><span class="status fail" data-i18n="en">FAIL</span>
|
|
686
|
+
</td>
|
|
687
|
+
<td class="num">10/13</td>
|
|
688
|
+
<td class="num">0.33</td>
|
|
689
|
+
<td class="num">148,330</td>
|
|
690
|
+
<td class="num">$0.1217</td>
|
|
691
|
+
<td data-i18n="zh">缺少 OpenSpec 产物和状态机证据</td>
|
|
692
|
+
<td data-i18n="en">OpenSpec artifacts and state machine evidence missing</td>
|
|
693
|
+
</tr>
|
|
694
|
+
<tr>
|
|
695
|
+
<td data-i18n="zh">完整流程</td>
|
|
696
|
+
<td data-i18n="en">Full workflow</td>
|
|
697
|
+
<td><code>COMET_FULL</code></td>
|
|
698
|
+
<td>
|
|
699
|
+
<span class="status pass" data-i18n="zh">通过</span
|
|
700
|
+
><span class="status pass" data-i18n="en">PASS</span>
|
|
701
|
+
</td>
|
|
702
|
+
<td class="num">13/13</td>
|
|
703
|
+
<td class="num">0.84</td>
|
|
704
|
+
<td class="num">161,476</td>
|
|
705
|
+
<td class="num">$0.0942</td>
|
|
706
|
+
<td data-i18n="zh">正常</td>
|
|
707
|
+
<td data-i18n="en">OK</td>
|
|
708
|
+
</tr>
|
|
709
|
+
<tr>
|
|
710
|
+
<td data-i18n="zh">完整流程</td>
|
|
711
|
+
<td data-i18n="en">Full workflow</td>
|
|
712
|
+
<td><code>COMET_FULL_039</code></td>
|
|
713
|
+
<td>
|
|
714
|
+
<span class="status pass" data-i18n="zh">通过</span
|
|
715
|
+
><span class="status pass" data-i18n="en">PASS</span>
|
|
716
|
+
</td>
|
|
717
|
+
<td class="num">13/13</td>
|
|
718
|
+
<td class="num">0.88</td>
|
|
719
|
+
<td class="num">224,163</td>
|
|
720
|
+
<td class="num">$0.1309</td>
|
|
721
|
+
<td data-i18n="zh">正常</td>
|
|
722
|
+
<td data-i18n="en">OK</td>
|
|
723
|
+
</tr>
|
|
724
|
+
<tr>
|
|
725
|
+
<td data-i18n="zh">性能去重</td>
|
|
726
|
+
<td data-i18n="en">Perf dedupe</td>
|
|
727
|
+
<td><code>COMET_FULL</code></td>
|
|
728
|
+
<td>
|
|
729
|
+
<span class="status pass" data-i18n="zh">通过</span
|
|
730
|
+
><span class="status pass" data-i18n="en">PASS</span>
|
|
731
|
+
</td>
|
|
732
|
+
<td class="num">13/13</td>
|
|
733
|
+
<td class="num">0.79</td>
|
|
734
|
+
<td class="num">3,042,862</td>
|
|
735
|
+
<td class="num">$2.0644</td>
|
|
736
|
+
<td data-i18n="zh">正常</td>
|
|
737
|
+
<td data-i18n="en">OK</td>
|
|
738
|
+
</tr>
|
|
739
|
+
<tr>
|
|
740
|
+
<td data-i18n="zh">性能去重</td>
|
|
741
|
+
<td data-i18n="en">Perf dedupe</td>
|
|
742
|
+
<td><code>COMET_FULL_039</code></td>
|
|
743
|
+
<td>
|
|
744
|
+
<span class="status fail" data-i18n="zh">失败</span
|
|
745
|
+
><span class="status fail" data-i18n="en">FAIL</span>
|
|
746
|
+
</td>
|
|
747
|
+
<td class="num">10/13</td>
|
|
748
|
+
<td class="num">0.34</td>
|
|
749
|
+
<td class="num">49,682</td>
|
|
750
|
+
<td class="num">$0.0280</td>
|
|
751
|
+
<td data-i18n="zh">缺少 OpenSpec 产物和状态机证据</td>
|
|
752
|
+
<td data-i18n="en">OpenSpec artifacts and state machine evidence missing</td>
|
|
753
|
+
</tr>
|
|
754
|
+
<tr>
|
|
755
|
+
<td data-i18n="zh">重构 counter</td>
|
|
756
|
+
<td data-i18n="en">Refactor counter</td>
|
|
757
|
+
<td><code>COMET_FULL</code></td>
|
|
758
|
+
<td>
|
|
759
|
+
<span class="status pass" data-i18n="zh">通过</span
|
|
760
|
+
><span class="status pass" data-i18n="en">PASS</span>
|
|
761
|
+
</td>
|
|
762
|
+
<td class="num">14/14</td>
|
|
763
|
+
<td class="num">0.84</td>
|
|
764
|
+
<td class="num">1,536,724</td>
|
|
765
|
+
<td class="num">$1.0945</td>
|
|
766
|
+
<td data-i18n="zh">正常</td>
|
|
767
|
+
<td data-i18n="en">OK</td>
|
|
768
|
+
</tr>
|
|
769
|
+
<tr>
|
|
770
|
+
<td data-i18n="zh">重构 counter</td>
|
|
771
|
+
<td data-i18n="en">Refactor counter</td>
|
|
772
|
+
<td><code>COMET_FULL_039</code></td>
|
|
773
|
+
<td>
|
|
774
|
+
<span class="status pass" data-i18n="zh">通过</span
|
|
775
|
+
><span class="status pass" data-i18n="en">PASS</span>
|
|
776
|
+
</td>
|
|
777
|
+
<td class="num">14/14</td>
|
|
778
|
+
<td class="num">0.80</td>
|
|
779
|
+
<td class="num">4,304,050</td>
|
|
780
|
+
<td class="num">$3.0287</td>
|
|
781
|
+
<td data-i18n="zh">正常</td>
|
|
782
|
+
<td data-i18n="en">OK</td>
|
|
783
|
+
</tr>
|
|
784
|
+
<tr>
|
|
785
|
+
<td data-i18n="zh">健壮配置</td>
|
|
786
|
+
<td data-i18n="en">Robust config</td>
|
|
787
|
+
<td><code>COMET_FULL</code></td>
|
|
788
|
+
<td>
|
|
789
|
+
<span class="status fail" data-i18n="zh">失败</span
|
|
790
|
+
><span class="status fail" data-i18n="en">FAIL</span>
|
|
791
|
+
</td>
|
|
792
|
+
<td class="num">10/13</td>
|
|
793
|
+
<td class="num">0.33</td>
|
|
794
|
+
<td class="num">208,867</td>
|
|
795
|
+
<td class="num">$0.1774</td>
|
|
796
|
+
<td data-i18n="zh">缺少 OpenSpec 产物和状态机证据</td>
|
|
797
|
+
<td data-i18n="en">OpenSpec artifacts and state machine evidence missing</td>
|
|
798
|
+
</tr>
|
|
799
|
+
<tr>
|
|
800
|
+
<td data-i18n="zh">健壮配置</td>
|
|
801
|
+
<td data-i18n="en">Robust config</td>
|
|
802
|
+
<td><code>COMET_FULL_039</code></td>
|
|
803
|
+
<td>
|
|
804
|
+
<span class="status fail" data-i18n="zh">失败</span
|
|
805
|
+
><span class="status fail" data-i18n="en">FAIL</span>
|
|
806
|
+
</td>
|
|
807
|
+
<td class="num">12/13</td>
|
|
808
|
+
<td class="num">0.56</td>
|
|
809
|
+
<td class="num">621,935</td>
|
|
810
|
+
<td class="num">$0.4801</td>
|
|
811
|
+
<td data-i18n="zh">workflow 阶段证据不完整</td>
|
|
812
|
+
<td data-i18n="en">Workflow phase evidence incomplete</td>
|
|
813
|
+
</tr>
|
|
814
|
+
</tbody>
|
|
815
|
+
</table>
|
|
816
|
+
</div>
|
|
817
|
+
|
|
818
|
+
<section class="method">
|
|
819
|
+
<div>
|
|
820
|
+
<p class="section-label" data-i18n="zh">方法</p>
|
|
821
|
+
<p class="section-label" data-i18n="en">Methods</p>
|
|
822
|
+
<h2 data-i18n="zh">图表由 Python 生成,HTML 只负责报告排版</h2>
|
|
823
|
+
<h2 data-i18n="en">Charts are generated by Python; HTML owns report layout only</h2>
|
|
824
|
+
<p data-i18n="zh">
|
|
825
|
+
这个样板把数据可视化从 HTML 中拆出来:Python 脚本生成可复现 SVG,HTML
|
|
826
|
+
引用图表资产并提供中英文切换。后续真实 report renderer 可以继续复用这个边界。
|
|
827
|
+
</p>
|
|
828
|
+
<p data-i18n="en">
|
|
829
|
+
This sample separates visualization from report layout: the Python script generates
|
|
830
|
+
reproducible SVG assets, while HTML references those figures and provides bilingual
|
|
831
|
+
switching.
|
|
832
|
+
</p>
|
|
833
|
+
</div>
|
|
834
|
+
<ul>
|
|
835
|
+
<li data-i18n="zh">中文内容是主版本;英文只作为同结构翻译。</li>
|
|
836
|
+
<li data-i18n="en">
|
|
837
|
+
Chinese is the source version; English is the same-structure translation.
|
|
838
|
+
</li>
|
|
839
|
+
<li data-i18n="zh">图表文件位于 <code>eval/local/report-style-demo-assets/</code>。</li>
|
|
840
|
+
<li data-i18n="en">
|
|
841
|
+
Chart files live under <code>eval/local/report-style-demo-assets/</code>.
|
|
842
|
+
</li>
|
|
843
|
+
<li data-i18n="zh">当前没有多次 run 的方差,因此不画置信区间。</li>
|
|
844
|
+
<li data-i18n="en">
|
|
845
|
+
The current demo has no multi-run variance, so confidence intervals are not drawn.
|
|
846
|
+
</li>
|
|
847
|
+
</ul>
|
|
848
|
+
</section>
|
|
849
|
+
</section>
|
|
850
|
+
</main>
|
|
851
|
+
|
|
852
|
+
<script>
|
|
853
|
+
const applyLanguage = (lang) => {
|
|
854
|
+
const next = lang === 'en' ? 'en' : 'zh';
|
|
855
|
+
document.documentElement.dataset.lang = next;
|
|
856
|
+
document.documentElement.lang = next === 'zh' ? 'zh-CN' : 'en';
|
|
857
|
+
localStorage.setItem('comet-report-demo-lang', next);
|
|
858
|
+
};
|
|
859
|
+
|
|
860
|
+
document.querySelectorAll('[data-set-lang]').forEach((button) => {
|
|
861
|
+
button.addEventListener('click', () => applyLanguage(button.dataset.setLang));
|
|
862
|
+
});
|
|
863
|
+
|
|
864
|
+
applyLanguage(localStorage.getItem('comet-report-demo-lang') || 'zh');
|
|
865
|
+
</script>
|
|
866
|
+
</body>
|
|
867
|
+
</html>
|