@rulemetric/cli 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/base-command.js +1 -1
- package/dist/{chunk-UE76F2NQ.js → chunk-2WQUOC6S.js} +20 -20
- package/dist/{chunk-DUBBL57L.js → chunk-5XEGRU2R.js} +1 -1
- package/dist/chunk-74QIUVWY.js +1 -0
- package/dist/chunk-7ML2UCI7.js +1 -0
- package/dist/chunk-AHLVY52I.js +1 -0
- package/dist/{chunk-O2HCD6XL.js → chunk-AT2MMCOJ.js} +1 -1
- package/dist/{chunk-GJBS6QYC.js → chunk-AXJAGWZB.js} +1 -1
- package/dist/{chunk-UYPBIXF2.js → chunk-BVOXWAJO.js} +1 -1
- package/dist/chunk-CT7FWSIU.js +3 -0
- package/dist/{chunk-KV5A77XW.js → chunk-DDYGA2DS.js} +1 -1
- package/dist/{chunk-5XK6LB2T.js → chunk-E5YE25QG.js} +1 -1
- package/dist/chunk-E7GQJML3.js +1 -0
- package/dist/{chunk-RK663O4M.js → chunk-E7RH43OS.js} +6 -6
- package/dist/{chunk-2EX27VJ3.js → chunk-ET7753IY.js} +1 -1
- package/dist/{chunk-25Q3Y7TC.js → chunk-EWGE7YTQ.js} +1 -1
- package/dist/{chunk-WOJ733IB.js → chunk-EXVXKJ7Y.js} +1 -1
- package/dist/chunk-FCTFHY3X.js +13 -0
- package/dist/{chunk-RJX75Y7J.js → chunk-FJX7WDPG.js} +1 -1
- package/dist/{chunk-3UPHCQWI.js → chunk-FK55FXKQ.js} +1 -1
- package/dist/chunk-GH4EZK2O.js +1 -0
- package/dist/chunk-GN46S7T4.js +9 -0
- package/dist/{chunk-BG55WDDX.js → chunk-GQGGFHX7.js} +1 -1
- package/dist/chunk-GYNZ3OZF.js +1 -0
- package/dist/{chunk-WJZY5Y22.js → chunk-H4LUPIHJ.js} +1 -1
- package/dist/{chunk-25RDT32M.js → chunk-HAYO6JYI.js} +1 -1
- package/dist/{chunk-YI2OWO6R.js → chunk-HFSISGCJ.js} +1 -1
- package/dist/chunk-HRJDKAJ4.js +1 -0
- package/dist/chunk-IC6VZHE6.js +1 -0
- package/dist/chunk-INAZLRY5.js +1 -0
- package/dist/{chunk-VXUKXGRX.js → chunk-IOX4VJPR.js} +2 -2
- package/dist/chunk-J4W6GNCF.js +2 -0
- package/dist/{chunk-VH2EAP4J.js → chunk-J7UZXRTH.js} +1 -1
- package/dist/{chunk-6GAIYJ6K.js → chunk-JKWB6UXA.js} +1 -1
- package/dist/{chunk-JWFRI32Y.js → chunk-JLBQKV4V.js} +1 -1
- package/dist/{chunk-VIQ25ORK.js → chunk-JRAXWASH.js} +1 -1
- package/dist/{chunk-PVGUDJD7.js → chunk-JRWQJZYQ.js} +1 -1
- package/dist/{chunk-6ZPSVZNH.js → chunk-LDUJWJ6A.js} +1 -1
- package/dist/chunk-LMCICVE7.js +12 -0
- package/dist/chunk-MHVHGKDK.js +3 -0
- package/dist/chunk-MMM4B3KH.js +1 -0
- package/dist/{chunk-CYYDQFRY.js → chunk-MX34N7Y5.js} +1 -1
- package/dist/{chunk-7AWFSFR7.js → chunk-N26UEHI3.js} +1 -1
- package/dist/{chunk-ZGW3N3A6.js → chunk-N4RJ6CN7.js} +1 -1
- package/dist/{chunk-J6QMBMC2.js → chunk-N7SZWKCU.js} +1 -1
- package/dist/{chunk-QFKUSNUB.js → chunk-NRYVMGJN.js} +1 -1
- package/dist/{chunk-4COLIGD7.js → chunk-NSPDDSRR.js} +1 -1
- package/dist/{chunk-GBWQENAH.js → chunk-O62OLKQL.js} +1 -1
- package/dist/{chunk-WY3VVX3R.js → chunk-OLJF4K7L.js} +1 -1
- package/dist/{chunk-6W2GSV4O.js → chunk-OMRH4JXS.js} +1 -1
- package/dist/{chunk-WH3IPXXQ.js → chunk-OO2ZRKZY.js} +5 -5
- package/dist/chunk-OUK2AWEI.js +1 -0
- package/dist/chunk-OYAOLJUH.js +1 -0
- package/dist/chunk-PPAHIVG5.js +1 -0
- package/dist/{chunk-3MDEGOUG.js → chunk-PPIZMPNI.js} +1 -1
- package/dist/chunk-PUX6P5KC.js +2 -0
- package/dist/chunk-QI3FJKGK.js +1 -0
- package/dist/chunk-RRM4HQG4.js +1 -0
- package/dist/{chunk-QTCWOIT4.js → chunk-S5CQSP35.js} +1 -1
- package/dist/chunk-S5U76YUI.js +1 -0
- package/dist/chunk-SE3UKA3M.js +25 -0
- package/dist/{chunk-BZS5QRWB.js → chunk-SG4UDDHB.js} +1 -1
- package/dist/chunk-SPJ73X7A.js +4 -0
- package/dist/chunk-SV73ASP2.js +1 -0
- package/dist/chunk-SWSOEYC7.js +1 -0
- package/dist/{chunk-I3M5VBLT.js → chunk-TTSDPDRH.js} +1 -1
- package/dist/chunk-TWPG3TEQ.js +48 -0
- package/dist/{chunk-WQX4RRBQ.js → chunk-U7BPWFOP.js} +1 -1
- package/dist/{chunk-NBEZ5FAU.js → chunk-UFQ43KXE.js} +1 -1
- package/dist/chunk-UJ7EPLKC.js +1 -0
- package/dist/{chunk-V2USQH27.js → chunk-UQDOOQCM.js} +1 -1
- package/dist/{chunk-M2VVRQKA.js → chunk-V4PUYPTU.js} +1 -1
- package/dist/{chunk-SHYP27B7.js → chunk-VALRPZXH.js} +1 -1
- package/dist/{chunk-LTQ5BJYL.js → chunk-VLWNSTYW.js} +1 -1
- package/dist/chunk-WSWZPNLG.js +1 -0
- package/dist/{chunk-MNQPCNGQ.js → chunk-XAOLOR4Q.js} +1 -1
- package/dist/{chunk-6QLZT7EP.js → chunk-XAXEZI6Z.js} +1 -1
- package/dist/{chunk-Y7QIOPN5.js → chunk-XFLKW6SJ.js} +1 -1
- package/dist/chunk-YNLHJHUP.js +1 -0
- package/dist/{chunk-U7TBEC3I.js → chunk-YP2CBUXH.js} +1 -1
- package/dist/{chunk-P2WSDHIA.js → chunk-YUIRDBQQ.js} +1 -1
- package/dist/{chunk-4N43IBXH.js → chunk-ZA6NXNPR.js} +1 -1
- package/dist/{chunk-APDRR6FS.js → chunk-ZZHIBOMT.js} +1 -1
- package/dist/commands/antigravity/backfill.js +1 -1
- package/dist/commands/auth/create-key.js +1 -1
- package/dist/commands/auth/login.js +1 -1
- package/dist/commands/auth/logout.js +1 -1
- package/dist/commands/auth/revoke-key.js +1 -1
- package/dist/commands/auth/signup.js +1 -1
- package/dist/commands/auth/status.js +1 -1
- package/dist/commands/capture/scope.js +1 -1
- package/dist/commands/catalog/index.js +1 -1
- package/dist/commands/catalog/recommend.js +1 -1
- package/dist/commands/crons/list.js +1 -1
- package/dist/commands/crons/run.js +2 -2
- package/dist/commands/dashboard.js +1 -1
- package/dist/commands/diagnostics.js +1 -1
- package/dist/commands/eval-targets/create.js +1 -1
- package/dist/commands/eval-targets/list.js +1 -1
- package/dist/commands/evals/agent.js +3 -3
- package/dist/commands/evals/auto-run.js +1 -1
- package/dist/commands/evals/benchmark.js +1 -1
- package/dist/commands/evals/compare.js +1 -1
- package/dist/commands/evals/create.js +1 -1
- package/dist/commands/evals/optimize-description.js +1 -1
- package/dist/commands/evals/optimize.js +1 -1
- package/dist/commands/evals/refine-cases.js +1 -1
- package/dist/commands/evals/run.js +1 -1
- package/dist/commands/evals/validate-judge.js +1 -1
- package/dist/commands/experiments/list.js +1 -1
- package/dist/commands/gateway/ensure.js +1 -1
- package/dist/commands/hooks/emit-instructions.js +1 -1
- package/dist/commands/hooks/install.js +1 -1
- package/dist/commands/hooks/run.js +1 -1
- package/dist/commands/hooks/uninstall.js +1 -1
- package/dist/commands/instructions/create.js +1 -1
- package/dist/commands/instructions/dedupe.js +1 -1
- package/dist/commands/instructions/delete.js +1 -1
- package/dist/commands/instructions/fork.js +1 -1
- package/dist/commands/instructions/get.js +1 -1
- package/dist/commands/instructions/list.js +1 -1
- package/dist/commands/instructions/promote.js +1 -1
- package/dist/commands/instructions/pull.js +1 -1
- package/dist/commands/instructions/restore.js +1 -1
- package/dist/commands/instructions/retract.js +1 -1
- package/dist/commands/instructions/upstream.js +1 -1
- package/dist/commands/instructions/versions.js +1 -1
- package/dist/commands/launcher.js +1 -1
- package/dist/commands/local/down.js +1 -1
- package/dist/commands/local/up.js +1 -1
- package/dist/commands/org/current.js +1 -1
- package/dist/commands/org/list.js +1 -1
- package/dist/commands/org/switch.js +1 -1
- package/dist/commands/packs/add-instruction.js +1 -1
- package/dist/commands/packs/apply.js +1 -1
- package/dist/commands/packs/create.js +1 -1
- package/dist/commands/packs/fork.js +1 -1
- package/dist/commands/packs/get.js +1 -1
- package/dist/commands/packs/list.js +1 -1
- package/dist/commands/packs/remove-instruction.js +1 -1
- package/dist/commands/proxy/env.js +1 -1
- package/dist/commands/proxy/logs.js +1 -1
- package/dist/commands/proxy/restart.js +1 -1
- package/dist/commands/proxy/setup.js +1 -1
- package/dist/commands/proxy/start.js +1 -1
- package/dist/commands/proxy/status.js +1 -1
- package/dist/commands/proxy/stop.js +1 -1
- package/dist/commands/research/backfill.js +1 -1
- package/dist/commands/research/tail-transcript.js +1 -1
- package/dist/commands/service/install.js +18 -18
- package/dist/commands/service/status.js +1 -1
- package/dist/commands/service/uninstall.js +1 -1
- package/dist/commands/sessions/analyze.js +1 -1
- package/dist/commands/sessions/delete.js +1 -1
- package/dist/commands/sessions/end.js +1 -1
- package/dist/commands/sessions/import.js +1 -1
- package/dist/commands/sessions/list.js +1 -1
- package/dist/commands/sessions/start.js +1 -1
- package/dist/commands/setup.js +1 -1
- package/dist/commands/skills/export-approved.js +1 -1
- package/dist/commands/skills/install.js +1 -1
- package/dist/commands/skills/list.js +1 -1
- package/dist/commands/skills/publish.js +1 -1
- package/dist/commands/skills/search.js +1 -1
- package/dist/commands/skills/update.js +1 -1
- package/dist/commands/suggestions/accept.js +1 -1
- package/dist/commands/suggestions/auto.js +1 -1
- package/dist/commands/suggestions/dismiss.js +1 -1
- package/dist/commands/suggestions/list.js +1 -1
- package/dist/commands/suggestions/refresh.js +1 -1
- package/dist/commands/suggestions/undo.js +1 -1
- package/dist/dashboard/Dashboard.js +1 -1
- package/dist/dashboard/data.js +1 -1
- package/dist/dashboard/launcher/LauncherOverlay.js +1 -1
- package/dist/dashboard/launcher/api.js +1 -1
- package/dist/dashboard/launcher/useLaunchJobs.js +1 -1
- package/dist/dist-NHS7HY5Q.js +1 -0
- package/dist/harbor/cost_delta.py +377 -0
- package/dist/harbor/mcp_container_shell.py +156 -0
- package/dist/harbor/provenance.py +361 -0
- package/dist/harbor/rm-batch-task/environment/Dockerfile +6 -0
- package/dist/harbor/rm-batch-task/environment/items.jsonl +60 -0
- package/dist/harbor/rm-batch-task/inject.md +8 -0
- package/dist/harbor/rm-batch-task/instruction.md +8 -0
- package/dist/harbor/rm-batch-task/solution/solve.sh +68 -0
- package/dist/harbor/rm-batch-task/task.toml +30 -0
- package/dist/harbor/rm-batch-task/tests/test.sh +23 -0
- package/dist/harbor/rm-batch-task/tests/test_outputs.py +32 -0
- package/dist/harbor/rm-locality-task/environment/Dockerfile +8 -0
- package/dist/harbor/rm-locality-task/environment/workspace/deploy/prod/ingest.yaml +3 -0
- package/dist/harbor/rm-locality-task/environment/workspace/deploy/staging/ingest.yaml +3 -0
- package/dist/harbor/rm-locality-task/environment/workspace/docs/architecture/ingest.md +6 -0
- package/dist/harbor/rm-locality-task/environment/workspace/legacy/ingest_v1/config.yaml +2 -0
- package/dist/harbor/rm-locality-task/environment/workspace/packages/shared/defaults.py +4 -0
- package/dist/harbor/rm-locality-task/environment/workspace/services/ingest/config/limits.defaults.yaml +4 -0
- package/dist/harbor/rm-locality-task/environment/workspace/services/ingest/config/limits.yaml +3 -0
- package/dist/harbor/rm-locality-task/environment/workspace/services/ingest/loader.py +15 -0
- package/dist/harbor/rm-locality-task/environment/workspace/tests/fixtures/ingest_config.yaml +3 -0
- package/dist/harbor/rm-locality-task/inject.md +12 -0
- package/dist/harbor/rm-locality-task/instruction.md +4 -0
- package/dist/harbor/rm-locality-task/solution/solve.sh +14 -0
- package/dist/harbor/rm-locality-task/task.toml +30 -0
- package/dist/harbor/rm-locality-task/tests/test.sh +23 -0
- package/dist/harbor/rm-locality-task/tests/test_outputs.py +19 -0
- package/dist/harbor/rm-migrations-task/environment/Dockerfile +7 -0
- package/dist/harbor/rm-migrations-task/environment/applied.txt +2 -0
- package/dist/harbor/rm-migrations-task/environment/migrations/00001_init.sql +1 -0
- package/dist/harbor/rm-migrations-task/environment/migrations/00002_add_email.sql +1 -0
- package/dist/harbor/rm-migrations-task/inject.md +7 -0
- package/dist/harbor/rm-migrations-task/instruction.md +8 -0
- package/dist/harbor/rm-migrations-task/solution/solve.sh +8 -0
- package/dist/harbor/rm-migrations-task/task.toml +30 -0
- package/dist/harbor/rm-migrations-task/tests/test.sh +23 -0
- package/dist/harbor/rm-migrations-task/tests/test_outputs.py +36 -0
- package/dist/harbor/rm-pagination-task/environment/Dockerfile +6 -0
- package/dist/harbor/rm-pagination-task/environment/store.py +29 -0
- package/dist/harbor/rm-pagination-task/inject.md +8 -0
- package/dist/harbor/rm-pagination-task/instruction.md +5 -0
- package/dist/harbor/rm-pagination-task/solution/solve.sh +7 -0
- package/dist/harbor/rm-pagination-task/task.toml +30 -0
- package/dist/harbor/rm-pagination-task/tests/test.sh +23 -0
- package/dist/harbor/rm-pagination-task/tests/test_outputs.py +14 -0
- package/dist/harbor/rm_host_claude.py +264 -0
- package/dist/harbor/rm_host_codex.py +208 -0
- package/dist/harbor/run.sh +268 -0
- package/dist/harbor/summarize.py +248 -0
- package/dist/lib/active-org-refresh.js +1 -1
- package/dist/lib/agent-loop.js +1 -1
- package/dist/lib/antigravity-transcript.js +1 -1
- package/dist/lib/api-client.js +1 -1
- package/dist/lib/capture-scope-config.js +1 -1
- package/dist/lib/codex-capture-scope.js +1 -1
- package/dist/lib/codex-exec.js +1 -1
- package/dist/lib/ensure-api-key.js +1 -1
- package/dist/lib/ensure-fresh-token.js +1 -1
- package/dist/lib/eval-accumulation.js +1 -0
- package/dist/lib/eval-analyzer.js +1 -1
- package/dist/lib/eval-benchmark.js +1 -1
- package/dist/lib/eval-candidate-run.js +1 -1
- package/dist/lib/eval-case-generator.js +1 -1
- package/dist/lib/eval-conversation-handler.js +1 -1
- package/dist/lib/eval-executor.js +1 -1
- package/dist/lib/eval-grader.js +1 -1
- package/dist/lib/eval-meta-judge.js +1 -1
- package/dist/lib/eval-resume.js +1 -1
- package/dist/lib/eval-run-batch.js +1 -1
- package/dist/lib/eval-trigger-tester.js +1 -1
- package/dist/lib/evolution-enrolment.js +1 -1
- package/dist/lib/harbor-readiness.js +1 -1
- package/dist/lib/hooks-capture-gate.js +1 -1
- package/dist/lib/insights/generation/installed-asks.js +1 -1
- package/dist/lib/insights/nomination/nominate.js +1 -1
- package/dist/lib/instruction-proposer.js +1 -1
- package/dist/lib/instruction-snapshot.js +1 -1
- package/dist/lib/instructions/acceptance/accept-events.js +1 -1
- package/dist/lib/instructions/acceptance/accept-hook.js +1 -1
- package/dist/lib/instructions/acceptance/agent-instruction-files.js +1 -1
- package/dist/lib/instructions/acceptance/hook-install.js +1 -1
- package/dist/lib/instructions/acceptance/reverse-applications.js +1 -1
- package/dist/lib/instructions/acceptance/suggestion-accept.js +1 -1
- package/dist/lib/jobs/handlers/cron-auto-accept-suggestions.js +1 -1
- package/dist/lib/jobs/handlers/cron-eval-autorun.js +1 -1
- package/dist/lib/jobs/handlers/cron-instruction-evolution.js +1 -1
- package/dist/lib/jobs/handlers/cron-instruction-training.js +1 -1
- package/dist/lib/jobs/handlers/cron-reconcile-attribution.js +1 -1
- package/dist/lib/jobs/handlers/cron-refine-cases.js +1 -1
- package/dist/lib/jobs/handlers/cron-refresh-pricing.js +1 -1
- package/dist/lib/jobs/handlers/cron-retire-unused-skills.js +1 -1
- package/dist/lib/jobs/handlers/cron-suggest-instructions.js +1 -1
- package/dist/lib/jobs/handlers/distill-memories.js +1 -1
- package/dist/lib/jobs/handlers/harbor-ab-verdict.js +1 -1
- package/dist/lib/jobs/handlers/harbor-ab.js +1 -1
- package/dist/lib/jobs/handlers/process-changelog.js +1 -1
- package/dist/lib/jobs/handlers/process-conversation.js +1 -1
- package/dist/lib/jobs/handlers/process-eval.js +1 -1
- package/dist/lib/jobs/handlers/process-insights.js +1 -1
- package/dist/lib/jobs/handlers/process-judge-review.js +1 -1
- package/dist/lib/jobs/handlers/process-launch.js +1 -1
- package/dist/lib/jobs/handlers/process-memory-ab.js +1 -1
- package/dist/lib/jobs/handlers/process-run-cleanup.js +1 -1
- package/dist/lib/jobs/handlers/process-session-goal.js +1 -1
- package/dist/lib/jobs/shed-retry.js +1 -1
- package/dist/lib/judge-review-runner.js +1 -1
- package/dist/lib/llm-client.js +1 -1
- package/dist/lib/llm-reply-parsers.js +1 -0
- package/dist/lib/manual-tasks.js +1 -1
- package/dist/lib/measurement-llm.js +1 -0
- package/dist/lib/pi-exec.js +1 -0
- package/dist/lib/projects/workspace/eval-workspace.js +1 -1
- package/dist/lib/projects/workspace/worktree.js +1 -1
- package/dist/lib/refine-cases.js +1 -1
- package/dist/lib/research-client.js +1 -1
- package/dist/lib/retraction.js +1 -1
- package/dist/lib/service-environment.js +1 -0
- package/dist/lib/setup-steps.js +1 -1
- package/dist/lib/telemetry.js +1 -1
- package/dist/lib/terminal-target-heartbeat.js +1 -1
- package/dist/lib/tool-cli-map.js +1 -0
- package/dist/lib/transcript-instruction-linking.js +1 -1
- package/dist/lib/transcript-sync.js +1 -1
- package/dist/lib/work-report.js +1 -1
- package/oclif.manifest.json +7 -1
- package/package.json +5 -4
- package/dist/chunk-4FSQSETR.js +0 -12
- package/dist/chunk-6WQG6C7F.js +0 -1
- package/dist/chunk-C42NTI2V.js +0 -3
- package/dist/chunk-F6DAHCJC.js +0 -20
- package/dist/chunk-GN3ZHQ46.js +0 -1
- package/dist/chunk-H3BCWSYO.js +0 -1
- package/dist/chunk-HLNHBZLI.js +0 -1
- package/dist/chunk-HLQZR4XJ.js +0 -13
- package/dist/chunk-J5HW45PZ.js +0 -48
- package/dist/chunk-JNFB22FD.js +0 -1
- package/dist/chunk-KCOJLTVX.js +0 -1
- package/dist/chunk-MLV2RHRE.js +0 -1
- package/dist/chunk-N7GK33PU.js +0 -1
- package/dist/chunk-NRYLYMDM.js +0 -1
- package/dist/chunk-OZNKMJ5B.js +0 -1
- package/dist/chunk-PPXOC7IV.js +0 -1
- package/dist/chunk-QXPYGN7I.js +0 -1
- package/dist/chunk-R4QIPG4V.js +0 -2
- package/dist/chunk-RGUCR7T6.js +0 -1
- package/dist/chunk-RO3ZZVCG.js +0 -1
- package/dist/chunk-SN3CHV66.js +0 -1
- package/dist/chunk-U3XKXJ3A.js +0 -5
- package/dist/chunk-UM4CBCXI.js +0 -1
- package/dist/chunk-WHQ4BDSM.js +0 -4
- package/dist/dist-45KJH5M7.js +0 -1
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{a as n}from"../../chunk-CDI5CCHU.js";import{a}from"../../chunk-
|
|
1
|
+
import{a as n}from"../../chunk-CDI5CCHU.js";import{a}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{Args as m,Flags as o}from"@oclif/core";var l=class r extends a{static{e(this,"SkillsSearch")}static description="Search the community skills registry";static examples=['<%= config.bin %> skills search "tdd"',"<%= config.bin %> skills search --tool cursor --category testing","<%= config.bin %> skills search --language python"];static args={query:m.string({description:"Search term",required:!1})};static flags={tool:o.string({description:"Filter by tool",options:["claude_code","cursor","copilot","windsurf","cline","universal"]}),category:o.string({description:"Filter by category",options:["workflow","language","framework","tool","security","testing","architecture","devops","documentation","performance"]}),language:o.string({description:"Filter by programming language"}),framework:o.string({description:"Filter by framework"}),limit:o.integer({description:"Max results",default:20})};async run(){let{args:g,flags:i}=await this.parse(r),s=(await n(this)).search({query:g.query,tool:i.tool,category:i.category,language:i.language,framework:i.framework,limit:i.limit});if(s.length===0){this.log("No skills found matching your criteria."),this.log(`
|
|
2
2
|
Try a broader search or run: rulemetric skills list`);return}this.log(`Found ${s.length} skill${s.length===1?"":"s"}:
|
|
3
3
|
`);for(let t of s){let c=t.popularity?.stars?` (${p(t.popularity.stars)} stars)`:"",u=t.kind==="supporting"?" [supporting file]":"",d=t.tools.join(", ");this.log(` ${t.id}`),this.log(` ${t.name}${u} \u2014 ${t.description}`),this.log(` tools: ${d} category: ${t.category}${c}`),t.tags.length>0&&this.log(` tags: ${t.tags.join(", ")}`),this.log("")}this.log("Run: rulemetric skills install <id> to install a skill")}};function p(r){return r>=1e3?`${(r/1e3).toFixed(1)}k`:String(r)}e(p,"formatStars");export{l as default};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{a as n}from"../../chunk-
|
|
1
|
+
import{a as n}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a as e}from"../../chunk-ZLLER3HP.js";import*as s from"node:fs";import*as r from"node:path";import*as m from"node:os";import{Flags as h}from"@oclif/core";import{fetchAllSources as y,TIER_1_SOURCES as c,TIER_2_SOURCES as S}from"@rulemetric/skills-registry";var i=r.join(m.homedir(),".config","rulemetric"),f=r.join(i,"skills-registry.json"),t=class g extends n{static{e(this,"SkillsUpdate")}static description="Fetch latest skills from GitHub sources and update local registry";static examples=["<%= config.bin %> skills update","<%= config.bin %> skills update --all"];static flags={all:h.boolean({description:"Include Tier 2 (niche) sources",default:!1})};async run(){let{flags:o}=await this.parse(g),l=o.all?[...c,...S]:c;this.log(`Fetching skills from ${l.length} sources${o.all?" (Tier 1 + 2)":""}...
|
|
2
2
|
`);let a=await y(l,{onProgress:e((p,u)=>{this.log(` ${p}: ${u} skills`)},"onProgress")}),d={version:"1.0.0",generatedAt:new Date().toISOString(),skills:a.entries};s.existsSync(i)||s.mkdirSync(i,{recursive:!0}),s.writeFileSync(f,JSON.stringify(d)),this.log(`
|
|
3
3
|
Updated registry: ${a.entries.length} skills cached at ${f}`)}};export{t as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a}from"../../chunk-WLVQJH4E.js";import{h as r}from"../../chunk-
|
|
1
|
+
import{a}from"../../chunk-WLVQJH4E.js";import{h as r}from"../../chunk-LMCICVE7.js";import"../../chunk-J7UZXRTH.js";import"../../chunk-N7SZWKCU.js";import"../../chunk-BVOXWAJO.js";import"../../chunk-LJRE3LZ2.js";import{a as o}from"../../chunk-MODXEGCU.js";import"../../chunk-OZQNSELV.js";import"../../chunk-HFGO5762.js";import"../../chunk-VS5ROGDP.js";import"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import{a as i}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a as s}from"../../chunk-ZLLER3HP.js";import{Args as m,Flags as p}from"@oclif/core";var e=class c extends i{static{s(this,"SuggestionsAccept")}static description="Accept a suggestion and append the instruction to CLAUDE.md";static examples=["<%= config.bin %> suggestions accept <id>"];static args={id:m.string({description:"Suggestion UUID from `suggestions list`",required:!0})};static flags={force:p.boolean({description:"Add the rule even if this project already says the same thing in other words",default:!1})};async run(){let{args:t,flags:d}=await this.parse(c);this.requireAuth(),a(t.id,"suggestion ID");let n=await r({suggestionId:t.id,dir:process.cwd(),projectPath:o(process.cwd()),actor:"human",force:d.force});for(let g of n.messages)this.log(g)}};export{e as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as p}from"../../chunk-
|
|
1
|
+
import{a as p}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import{e as u,h as l}from"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a as c}from"../../chunk-ZLLER3HP.js";import{Flags as t}from"@oclif/core";var d=class r extends p{static{c(this,"SuggestionsAuto")}static description="Show or change automatic adoption of high-confidence suggestions into CLAUDE.md";static examples=["<%= config.bin %> suggestions auto","<%= config.bin %> suggestions auto --off","<%= config.bin %> suggestions auto --on","<%= config.bin %> suggestions auto --off --project <project-id>"];static flags={on:t.boolean({description:"Enable automatic adoption",exclusive:["off"]}),off:t.boolean({description:"Disable automatic adoption",exclusive:["on"]}),project:t.string({description:"Apply to one project only (overrides the account default)"}),"max-per-window":t.integer({description:"How many rules the loop may adopt per project per window (0 stops it adopting)"}),"max-per-run":t.integer({description:"How many rules one nightly run may adopt"}),"window-days":t.integer({description:"Length of the budget window, in days"}),"min-score":t.string({description:"Minimum suggestion score the loop may adopt (0\u20131)"}),reset:t.boolean({description:"Clear the budget overrides at this level and fall back to the shipped defaults"})};async run(){let{flags:e}=await this.parse(r);this.requireAuth();let i=e.reset?{minScore:null,maxPerRun:null,maxPerWindow:null,windowDays:null}:{...e["min-score"]!==void 0?{minScore:Number(e["min-score"])}:{},...e["max-per-run"]!==void 0?{maxPerRun:e["max-per-run"]}:{},...e["max-per-window"]!==void 0?{maxPerWindow:e["max-per-window"]}:{},...e["window-days"]!==void 0?{windowDays:e["window-days"]}:{}},n=Object.keys(i).length>0;if(e.on||e.off||n){let s=e.on||e.off?!!e.on:void 0,g=await l("/api/settings/auto-accept",{...s!==void 0?{enabled:s}:{},...n?{limits:i}:{},...e.project?{projectId:e.project}:{}});if(n){let a=g.limits;this.log(`Budget for ${e.project?`project ${e.project}`:"your account"}: ${a.maxPerRun} per night, ${a.maxPerWindow} per ${a.windowDays} days, score >= ${a.minScore}.`)}if(s===void 0)return;let m=e.project?`project ${e.project}`:"your account";this.log(`Automatic adoption is now ${s?"ON":"OFF"} for ${m}.`),s||(this.log("Already-adopted rules stay in CLAUDE.md \u2014 remove one with:"),this.log(" rulemetric suggestions undo <instruction-id>"));return}let o=await u("/api/settings/auto-accept"+(e.project?`?projectId=${encodeURIComponent(e.project)}`:""));this.log(`Automatic adoption: ${o.effective?"ON":"OFF"} (from ${o.source})`),this.log(` account default: ${o.accountDefault?"on":"off"}`),e.project&&this.log(` project override: ${o.projectOverride===null?"none":o.projectOverride?"on":"off"}`),this.log(""),this.log("When on, the nightly loop may add \u2014 and only add \u2014 suggestions that:"),this.log(` \u2022 score >= ${o.limits.minScore} (already injected into every session)`),this.log(` \u2022 are '${o.limits.kind}' proposals; removals and trims always wait for you`),this.log(" \u2022 contain no unfilled placeholders"),this.log(` \u2022 at most ${o.limits.maxPerRun} per night, ${o.limits.maxPerWindow} per ${o.limits.windowDays} days \u2014 per PROJECT, counting every checkout of it (${f(o.limits.source)})`),this.log(""),this.log("Change the budget: rulemetric suggestions auto --max-per-window 20"),this.log("Back to defaults: rulemetric suggestions auto --reset"),this.log(""),this.log("Every adopted rule is a tagged block in CLAUDE.md; undo one with:"),this.log(" rulemetric suggestions undo <instruction-id>"),this.log("Turn it off entirely with: rulemetric suggestions auto --off")}};function f(r){let e=new Set(Object.values(r));if(e.size===1){let i=[...e][0];return i==="default"?"shipped defaults":`set on your ${i}`}return Object.entries(r).map(([i,n])=>`${i}: ${n}`).join(", ")}c(f,"describeSource");export{d as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a}from"../../chunk-WLVQJH4E.js";import{a as r}from"../../chunk-MODXEGCU.js";import{a as o}from"../../chunk-
|
|
1
|
+
import{a}from"../../chunk-WLVQJH4E.js";import{a as r}from"../../chunk-MODXEGCU.js";import{a as o}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import{f as t}from"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{Args as m}from"@oclif/core";var s=class n extends o{static{e(this,"SuggestionsDismiss")}static description="Dismiss a suggestion so it will not re-surface for 30 days";static examples=["<%= config.bin %> suggestions dismiss <id>"];static args={id:m.string({description:"Suggestion UUID from `suggestions list`",required:!0})};async run(){let{args:i}=await this.parse(n);this.requireAuth(),a(i.id,"suggestion ID");let d=r(process.cwd());await t(`/api/instruction-suggestions/${encodeURIComponent(i.id)}/events`,{eventType:"dismissed",projectPath:d}),this.log("Dismissed. Won't re-surface for 30 days.")}};export{s as default};
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{a as u}from"../../chunk-MODXEGCU.js";import{a as g}from"../../chunk-
|
|
1
|
+
import{a as u}from"../../chunk-MODXEGCU.js";import{a as g}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import{e as d}from"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a}from"../../chunk-ZLLER3HP.js";import{Flags as j}from"@oclif/core";import{printTable as v}from"@oclif/table";var r=class p extends g{static{a(this,"SuggestionsList")}static description="List instruction suggestions for a project";static examples=["<%= config.bin %> suggestions list","<%= config.bin %> suggestions list --project /path/to/project"];static flags={project:j.string({char:"p",description:"Project path (defaults to current directory)"})};async run(){let{flags:m}=await this.parse(p);this.requireAuth();let l=u(m.project??process.cwd()),f=encodeURIComponent(l),s=await d(`/api/instruction-suggestions?projectPath=${f}&limit=10&groupByKind=true`);if(s.suggestions.length===0){this.log("No suggestions for this project. They are recomputed daily at 04:30 UTC by a server-side cron and run on your background worker \u2014 make sure one is running (`rulemetric evals agent`, or `rulemetric service install --worker-only`).");return}v({data:s.suggestions.map(e=>({ID:e.id,Action:(e.kind??"add")==="add"?"add":(e.kind??"add").toUpperCase(),Scope:e.instructionScope,Name:e.name,Tokens:e.estimatedTokens?String(e.estimatedTokens):"-",Score:e.score.toFixed(2),Reason:e.reason.length>60?e.reason.slice(0,59)+"\u2026":e.reason})),columns:["ID","Action","Scope","Name","Tokens","Score","Reason"]});let o=s.suggestions.filter(e=>e.kind==="remove");if(o.length>0){let e=o.reduce((h,k)=>h+(k.estimatedTokens??0),0);this.log(`
|
|
2
2
|
${o.length} retirement proposal(s) \u2014 accepting one archives the instruction (reversible: rulemetric instructions restore <instruction-id>).`+(e>0?` Together they cost roughly ${e} tokens of context per session.`:""))}let t=s.stats??{},i=(t.surfaced??0)+(t.surfaced_with_content??0)+(t.proxy_injected??0),n=(t.accepted??0)+(t.promoted??0),c=t.dismissed??0;i+n+c>0&&this.log(`
|
|
3
3
|
Lifetime for this project: ${i} surfaced \xB7 ${n} accepted \xB7 ${c} dismissed`)}};export{r as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as i}from"../../chunk-
|
|
1
|
+
import{a as i}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a as r}from"../../chunk-ZLLER3HP.js";var e=class extends i{static{r(this,"SuggestionsRefresh")}static description="Information about refreshing instruction suggestions";static examples=["<%= config.bin %> suggestions refresh"];async run(){this.log("Suggestions are recomputed automatically: a server-side cron enqueues a job daily at 04:30 UTC, and your background worker picks it up \u2014 there is no manual trigger."),this.log("Just make sure a worker is running: `rulemetric evals agent` (foreground), or install it persistently with `rulemetric service install --worker-only`.")}};export{e as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as f,c as g}from"../../chunk-
|
|
1
|
+
import{a as f,c as g}from"../../chunk-XAXEZI6Z.js";import{a as l}from"../../chunk-WLVQJH4E.js";import{d as p}from"../../chunk-LMCICVE7.js";import"../../chunk-J7UZXRTH.js";import"../../chunk-N7SZWKCU.js";import"../../chunk-BVOXWAJO.js";import"../../chunk-LJRE3LZ2.js";import{a as m}from"../../chunk-MODXEGCU.js";import"../../chunk-OZQNSELV.js";import"../../chunk-HFGO5762.js";import"../../chunk-VS5ROGDP.js";import{d as u}from"../../chunk-GYNZ3OZF.js";import"../../chunk-UPFH72PF.js";import{a as d}from"../../chunk-N4RJ6CN7.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-E5YE25QG.js";import"../../chunk-YUIRDBQQ.js";import"../../chunk-HAYO6JYI.js";import"../../chunk-3AV3GTMP.js";import{f as a}from"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import{a as c}from"../../chunk-ZLLER3HP.js";import{relative as h}from"node:path";import{Args as P}from"@oclif/core";var o=class I extends d{static{c(this,"SuggestionsUndo")}static description="Remove an adopted instruction from this project's agent instruction files and stop it being re-proposed";static examples=["<%= config.bin %> suggestions undo <instruction-id>"];static args={instructionId:P.string({description:"Instruction ID from the rulemetric:start marker in the instruction file",required:!0})};async run(){let{args:r}=await this.parse(I);this.requireAuth(),l(r.instructionId,"instruction ID");let e=m(process.cwd()),i=process.cwd(),v=u(i).map(t=>h(i,t.path)),w=p(i,r.instructionId).map(t=>h(i,t.path)),A=await f({dir:process.cwd(),instructionId:r.instructionId,projectPath:e}),n=null,s=null;try{n=(await a("/api/instruction-suggestions/undo",{projectPath:e,instructionId:r.instructionId})).name}catch(t){s=t instanceof Error?t.message:String(t)}for(let t of g({stripped:w,searched:v,reversals:A,name:n,instructionId:r.instructionId,dismissalError:s}))this.log(t);s&&this.exit(1)}};export{o as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a as F}from"../chunk-DVN7GFJG.js";import{a as M}from"../chunk-O2SDD2WW.js";import{a as E}from"../chunk-GBMF3TP6.js";import{a as O}from"../chunk-WSCK4WNK.js";import{a as D}from"../chunk-4Z5ECCJE.js";import"../chunk-ARNZQBMX.js";import{a as B,b as S,c as R}from"../chunk-
|
|
1
|
+
import{a as F}from"../chunk-DVN7GFJG.js";import{a as M}from"../chunk-O2SDD2WW.js";import{a as E}from"../chunk-GBMF3TP6.js";import{a as O}from"../chunk-WSCK4WNK.js";import{a as D}from"../chunk-4Z5ECCJE.js";import"../chunk-ARNZQBMX.js";import{a as B,b as S,c as R}from"../chunk-FK55FXKQ.js";import{a as $}from"../chunk-H4LUPIHJ.js";import"../chunk-N26UEHI3.js";import"../chunk-NBTC5FNV.js";import"../chunk-TU2AR6XO.js";import{a as u,b as T,d as a,e as v}from"../chunk-NJ5UED3H.js";import"../chunk-MX34N7Y5.js";import"../chunk-ZXF5NE54.js";import"../chunk-SE3UKA3M.js";import{a as X}from"../chunk-3AV3GTMP.js";import"../chunk-ZZHIBOMT.js";import"../chunk-TWPG3TEQ.js";import"../chunk-74QIUVWY.js";import"../chunk-PZNWH4VQ.js";import{a as m}from"../chunk-ZLLER3HP.js";import{useState as w,useEffect as nt,useCallback as G,useRef as st}from"react";import{render as U,Box as c,Text as o,useInput as at,useApp as lt}from"ink";import{Fragment as ht,jsx as t,jsxs as h}from"react/jsx-runtime";var dt=90;function J(){let s=process.stdout.columns||80,n=Math.min(160,s),i=n>=dt,d=i?Math.floor(n/2):n,b=Math.max(6,Math.min(10,d-34));return{dashWidth:n,wide:i,halfWidth:d,barWidth:b}}m(J,"getLayout");function z({active:s,providerName:n}){return h(c,{justifyContent:"space-between",paddingX:1,children:[t(c,{gap:1,children:u.map(i=>t(o,{bold:s===i,color:s===i?a:v,children:s===i?`[ ${T[i]} ]`:` ${T[i]} `},i))}),n&&t(o,{color:a,bold:!0,children:n})]})}m(z,"PeriodTabs");function H({width:s,insightsFocused:n=!1}){return t(c,{borderStyle:"round",borderColor:v,width:s,justifyContent:"center",paddingX:1,children:h(o,{children:[t(o,{color:a,bold:!0,children:"<"}),t(o,{color:a,children:">"}),t(o,{dimColor:!0,children:" switch "}),t(o,{color:a,bold:!0,children:"q"}),t(o,{dimColor:!0,children:" quit "}),t(o,{color:a,bold:!0,children:"s"}),h(o,{dimColor:!0,children:[" ",n?"back":"suggestions"," "]}),t(o,{color:a,bold:!0,children:"1"}),t(o,{dimColor:!0,children:"-"}),t(o,{color:a,bold:!0,children:"5"}),t(o,{dimColor:!0,children:" period "}),t(o,{color:a,bold:!0,children:"r"}),t(o,{dimColor:!0,children:" refresh "}),t(o,{color:a,bold:!0,children:"n"}),t(o,{dimColor:!0,children:" new session "}),t(o,{color:a,bold:!0,children:"l"}),t(o,{dimColor:!0,children:" launcher"})]})})}m(H,"StatusBar");function ct({wide:s,width:n,children:i}){return s?t(c,{width:n,children:i}):t(ht,{children:i})}m(ct,"Row");function ut({initialLocal:s,initialApi:n,initialInstructions:i,initialPeriod:d,initialLauncherFocused:b=!1}){let{exit:j}=lt(),[e,L]=w(d),[K,Q]=w(s),[W,V]=w(n),[Z,tt]=w(i),[ot,k]=w(!1),[p,I]=w(!1),[Y,_]=w(b),et=X()?.orgSlug??null,y=st(null),it=J(),g=G(async()=>{k(!0);let[r,x,P]=await Promise.all([B(),S(),R()]);Q(r),V(x),tt(P),k(!1)},[]);nt(()=>{let r=setInterval(g,3e4);return()=>clearInterval(r)},[g]);let f=G(r=>{r!==e&&(L(r),y.current&&clearTimeout(y.current),y.current=setTimeout(g,300))},[e,g]);at((r,x)=>{if(Y)return;if(r==="q"){if(p){I(!1);return}j();return}if(r==="s"){I(A=>!A);return}if(x.escape&&p){I(!1);return}if(r==="l"&&!p){_(!0);return}if(p)return;if(r==="r"){g();return}if(r==="n"){import("../dist-NHS7HY5Q.js").then(({getLauncher:A})=>{A("claude_code").launch({projectPath:process.cwd(),mode:"interactive"})});return}let P=u.indexOf(e);x.leftArrow?f(u[(P-1+u.length)%u.length]):x.rightArrow||x.tab?f(u[(P+1)%u.length]):r==="1"?f("today"):r==="2"?f("week"):r==="3"?f("30days"):r==="4"?f("month"):r==="5"&&f("all")});let{dashWidth:l,wide:q,halfWidth:rt,barWidth:C}=it,N=q?rt:l;return ot?h(c,{flexDirection:"column",width:l,children:[t(z,{active:e}),t(c,{borderStyle:"round",borderColor:a,paddingX:1,width:l,children:h(o,{dimColor:!0,children:["Loading ",T[e],"..."]})}),t(H,{width:l})]}):Y?t(c,{flexDirection:"column",width:l,children:t($,{onClose:()=>_(!1),activeOrg:et,defaultProjectPath:process.cwd(),width:l})}):h(c,{flexDirection:"column",width:l,children:[t(z,{active:e}),t(D,{local:K,api:W,label:T[e],width:l}),h(ct,{wide:q,width:l,children:[t(O,{api:W,width:N,barWidth:C}),t(F,{api:W,width:N,barWidth:C})]}),t(E,{instructions:Z,width:l,barWidth:C}),t(M,{api:W,width:l,focused:p}),t(H,{width:l,insightsFocused:p})]})}m(ut,"InteractiveDashboard");async function yt(s="week",n={}){let[i,d,b]=await Promise.all([B(),S(),R()]);if(process.stdin.isTTY&&process.stdout.isTTY){let{waitUntilExit:e}=U(t(ut,{initialLocal:i,initialApi:d,initialInstructions:b,initialPeriod:s,initialLauncherFocused:n.openLauncher??!1}));await e()}else{let e=J(),{unmount:L}=U(h(c,{flexDirection:"column",width:e.dashWidth,children:[t(D,{local:i,api:d,label:T[s],width:e.dashWidth}),t(O,{api:d,width:e.dashWidth,barWidth:e.barWidth}),t(F,{api:d,width:e.dashWidth,barWidth:e.barWidth}),t(E,{instructions:b,width:e.dashWidth,barWidth:e.barWidth}),t(M,{api:d,width:e.dashWidth})]}),{patchConsole:!1});L()}}m(yt,"renderDashboard");export{yt as renderDashboard};
|
package/dist/dashboard/data.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c}from"../chunk-
|
|
1
|
+
import{a,b,c}from"../chunk-FK55FXKQ.js";import"../chunk-SE3UKA3M.js";import"../chunk-ZZHIBOMT.js";import"../chunk-TWPG3TEQ.js";import"../chunk-74QIUVWY.js";import"../chunk-PZNWH4VQ.js";import"../chunk-ZLLER3HP.js";export{b as loadApiInsights,c as loadLinkedInstructions,a as loadLocalStats};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a}from"../../chunk-
|
|
1
|
+
import{a}from"../../chunk-H4LUPIHJ.js";import"../../chunk-N26UEHI3.js";import"../../chunk-NBTC5FNV.js";import"../../chunk-TU2AR6XO.js";import"../../chunk-NJ5UED3H.js";import"../../chunk-MX34N7Y5.js";import"../../chunk-ZXF5NE54.js";import"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import"../../chunk-ZLLER3HP.js";export{a as LauncherOverlay};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a,b,c,d}from"../../chunk-
|
|
1
|
+
import{a,b,c,d}from"../../chunk-MX34N7Y5.js";import"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import"../../chunk-ZLLER3HP.js";export{c as cancelLaunchJob,b as createLaunchJob,a as fetchLaunchJobs,d as listLaunchers};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{a}from"../../chunk-
|
|
1
|
+
import{a}from"../../chunk-N26UEHI3.js";import"../../chunk-MX34N7Y5.js";import"../../chunk-ZZHIBOMT.js";import"../../chunk-TWPG3TEQ.js";import"../../chunk-74QIUVWY.js";import"../../chunk-PZNWH4VQ.js";import"../../chunk-ZLLER3HP.js";export{a as useLaunchJobs};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x}from"./chunk-SE3UKA3M.js";import"./chunk-ZLLER3HP.js";export{c as ClaudeCodeAdapter,i as CodexAdapter,f as CopilotAgentAdapter,d as CursorAdapter,j as PiAdapter,g as RuleMetricMcpAdapter,b as SESSION_EVENT_TYPES,a as SESSION_TOOLS,e as VSCodeCopilotAdapter,t as buildTmuxAttachCommand,s as buildTmuxHasSessionArgs,q as buildTmuxLaunchArgs,r as buildTmuxSendKeysArgs,l as detectTool,n as discoverAllSessions,h as freshCodexState,k as getAdapter,m as getAllProviders,v as getAvailableLaunchers,u as getLauncher,o as getProvider,w as sanitizeBranch,p as sanitizeTmuxName,x as worktreePathFor};
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""The COST half of a Harbor A/B: a per-trial cost delta with an interval.
|
|
3
|
+
|
|
4
|
+
python3 cost_delta.py <baseline-job-dir> <with-job-dir> [--json]
|
|
5
|
+
|
|
6
|
+
`verdict_bridge.py` gives the REWARD half of an A/B a Newcombe CI, a Fisher p
|
|
7
|
+
and a graduation gate that refuses rather than publish an undecided result.
|
|
8
|
+
Until 2026-09-02 the cost half had none of that: `summarize.py` summed cost over
|
|
9
|
+
trials and `run.sh` printed the difference of two sums. That is wrong twice.
|
|
10
|
+
|
|
11
|
+
1. A sum has no variance, so "the with-arm was $0.09 cheaper" could never be
|
|
12
|
+
told apart from noise. Cost is right-skewed -- one flailing trial costs
|
|
13
|
+
several times a clean one -- so a difference of totals is exactly the shape
|
|
14
|
+
that reads as a finding and is a coin flip.
|
|
15
|
+
2. A sum is confounded by TRIAL COUNT. Unequal `n_trials` between arms (a
|
|
16
|
+
crashed trial, a different `-l` cap) puts "this arm ran more trials" into
|
|
17
|
+
the number and calls it cost. The per-trial MEAN is the comparable unit.
|
|
18
|
+
|
|
19
|
+
Estimator: percentile bootstrap on the difference of per-trial means. Not a
|
|
20
|
+
t-interval -- that assumes a symmetry cost does not have. Seeded, so a published
|
|
21
|
+
verdict is reproducible from the same job dirs.
|
|
22
|
+
|
|
23
|
+
The gate mirrors verdict_bridge.py on purpose: below COST_MIN_N per arm, or a CI
|
|
24
|
+
spanning zero, the decision is `undecided`. "We could not tell" must never be
|
|
25
|
+
published as "it is cheaper".
|
|
26
|
+
|
|
27
|
+
Two honesty rules are load-bearing:
|
|
28
|
+
* An arm with no per-trial cost output (the remote agent bills the key
|
|
29
|
+
directly) is UNKNOWN, and this tool exits non-zero. Reading it as $0 would
|
|
30
|
+
print the largest saving the tool is capable of printing.
|
|
31
|
+
* The REWARD delta travels with the cost number. Cheaper because the with-arm
|
|
32
|
+
gave up early is not a saving, and the person writing the case study must
|
|
33
|
+
not have to look that up separately.
|
|
34
|
+
|
|
35
|
+
Dependency-free and importable, so the contract test can drive it on synthetic
|
|
36
|
+
job dirs with no harbor install.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import json
|
|
42
|
+
import random
|
|
43
|
+
import sys
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
from typing import Any
|
|
46
|
+
|
|
47
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
48
|
+
from summarize import summarize # noqa: E402
|
|
49
|
+
|
|
50
|
+
# Same floor as verdict_bridge.py's GRADUATION_MIN_N. A cost saving and a reward
|
|
51
|
+
# gain are published side by side; letting cost graduate on a smaller n would
|
|
52
|
+
# make the cheaper number the easy one to quote.
|
|
53
|
+
import provenance
|
|
54
|
+
|
|
55
|
+
COST_MIN_N = 25
|
|
56
|
+
# Harbor rewards on these tasks are pass/fail. A partial reward is not a success:
|
|
57
|
+
# you cannot redo 60% of a task, so the redo it would price is a whole trial.
|
|
58
|
+
SUCCESS_THRESHOLD = 1.0
|
|
59
|
+
# Above this share of bootstrap resamples with no successes in an arm, the ratio
|
|
60
|
+
# has no sampling distribution worth reporting and the verdict is refused by name.
|
|
61
|
+
MAX_UNDEFINED_FRACTION = 0.05
|
|
62
|
+
RESAMPLES = 10000
|
|
63
|
+
SEED = 20260902
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def arm(job_dir: Path) -> dict[str, Any]:
|
|
67
|
+
"""Per-trial cost series for one arm, plus its reward."""
|
|
68
|
+
s = summarize(job_dir)
|
|
69
|
+
trials = s.get("trials") or []
|
|
70
|
+
costs = [float(t["cost_usd"]) for t in trials]
|
|
71
|
+
# Paired (cost, reward) for the per-success ratio. A trial whose reward is
|
|
72
|
+
# UNKNOWN (harbor wrote no verdict) is dropped from this pairing rather than
|
|
73
|
+
# scored as a failure -- a crashed trial must not read as an instruction that
|
|
74
|
+
# did not work. It still counts in the per-trial cost above: it was paid for.
|
|
75
|
+
paired = [(float(t["cost_usd"]), float(t["reward"])) for t in trials if t.get("reward") is not None]
|
|
76
|
+
successes = sum(1 for _, r in paired if r >= SUCCESS_THRESHOLD)
|
|
77
|
+
return {
|
|
78
|
+
"job": s["job"],
|
|
79
|
+
"n": len(costs),
|
|
80
|
+
"n_trials_reported": s.get("n_trials"),
|
|
81
|
+
"costs": costs,
|
|
82
|
+
"cost_sum_usd": round(sum(costs), 6),
|
|
83
|
+
"cost_mean_usd": round(sum(costs) / len(costs), 6) if costs else None,
|
|
84
|
+
"duration_mean_s": round(sum(t["duration_ms"] for t in trials) / len(trials) / 1000, 3)
|
|
85
|
+
if trials
|
|
86
|
+
else None,
|
|
87
|
+
"turns_mean": round(sum(t["turns"] for t in trials) / len(trials), 3) if trials else None,
|
|
88
|
+
"agent_errors": sum(1 for t in trials if t["is_error"]) if trials else 0,
|
|
89
|
+
"reward_mean": s.get("reward_mean"),
|
|
90
|
+
"paired": paired,
|
|
91
|
+
"n_scored": len(paired),
|
|
92
|
+
"successes": successes,
|
|
93
|
+
# None, not infinity: an arm that never passed has no cost PER SUCCESS.
|
|
94
|
+
# Every trial was paid for and none of them produced one.
|
|
95
|
+
"cost_per_success_usd": round(sum(c for c, _ in paired) / successes, 6) if successes else None,
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def bootstrap_ci(a: list[float], b: list[float], resamples: int = RESAMPLES, seed: int = SEED):
|
|
100
|
+
"""Percentile 95% CI for mean(b) - mean(a). Seeded: same input, same interval."""
|
|
101
|
+
rng = random.Random(seed)
|
|
102
|
+
na, nb = len(a), len(b)
|
|
103
|
+
deltas = []
|
|
104
|
+
for _ in range(resamples):
|
|
105
|
+
ma = sum(a[rng.randrange(na)] for _ in range(na)) / na
|
|
106
|
+
mb = sum(b[rng.randrange(nb)] for _ in range(nb)) / nb
|
|
107
|
+
deltas.append(mb - ma)
|
|
108
|
+
deltas.sort()
|
|
109
|
+
lo = deltas[int(0.025 * (resamples - 1))]
|
|
110
|
+
hi = deltas[int(0.975 * (resamples - 1))]
|
|
111
|
+
return round(lo, 6), round(hi, 6)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def per_success_ci(
|
|
116
|
+
a: list[tuple[float, float]],
|
|
117
|
+
b: list[tuple[float, float]],
|
|
118
|
+
resamples: int = RESAMPLES,
|
|
119
|
+
seed: int = SEED,
|
|
120
|
+
):
|
|
121
|
+
"""95% CI for cost-per-success(b) - cost-per-success(a), and the share of
|
|
122
|
+
resamples where the ratio was undefined.
|
|
123
|
+
|
|
124
|
+
Resampling is PAIRED within an arm -- the trial that cost this much is the
|
|
125
|
+
trial that did or did not pass -- so the ratio's numerator and denominator
|
|
126
|
+
move together the way they do in a real run. Resamples where either arm drew
|
|
127
|
+
no successes have no ratio; they are counted, not quietly dropped, because
|
|
128
|
+
"the interval came from the 40% of resamples that happened to work" is not an
|
|
129
|
+
interval anyone should publish.
|
|
130
|
+
"""
|
|
131
|
+
rng = random.Random(seed + 1)
|
|
132
|
+
na, nb = len(a), len(b)
|
|
133
|
+
deltas: list[float] = []
|
|
134
|
+
undefined = 0
|
|
135
|
+
for _ in range(resamples):
|
|
136
|
+
sa = [a[rng.randrange(na)] for _ in range(na)]
|
|
137
|
+
sb = [b[rng.randrange(nb)] for _ in range(nb)]
|
|
138
|
+
ka = sum(1 for _, r in sa if r >= SUCCESS_THRESHOLD)
|
|
139
|
+
kb = sum(1 for _, r in sb if r >= SUCCESS_THRESHOLD)
|
|
140
|
+
if ka == 0 or kb == 0:
|
|
141
|
+
undefined += 1
|
|
142
|
+
continue
|
|
143
|
+
deltas.append(sum(c for c, _ in sb) / kb - sum(c for c, _ in sa) / ka)
|
|
144
|
+
fraction = undefined / resamples
|
|
145
|
+
if not deltas:
|
|
146
|
+
return None, None, fraction
|
|
147
|
+
deltas.sort()
|
|
148
|
+
n = len(deltas)
|
|
149
|
+
return round(deltas[int(0.025 * (n - 1))], 6), round(deltas[int(0.975 * (n - 1))], 6), fraction
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def per_success_block(b: dict[str, Any], w: dict[str, Any], min_n: int, resamples: int):
|
|
153
|
+
"""The cost-per-success half of the report, with its own gate.
|
|
154
|
+
|
|
155
|
+
Separate from the per-trial gate on purpose: an instruction routinely makes a
|
|
156
|
+
trial cost MORE and a success cost LESS, and one decision cannot carry both
|
|
157
|
+
without one of them being buried.
|
|
158
|
+
"""
|
|
159
|
+
reasons: list[str] = []
|
|
160
|
+
bs, ws = b["cost_per_success_usd"], w["cost_per_success_usd"]
|
|
161
|
+
|
|
162
|
+
if b["n_scored"] < min_n or w["n_scored"] < min_n:
|
|
163
|
+
reasons.append(
|
|
164
|
+
f"below COST_MIN_N={min_n} scored trial(s) per arm "
|
|
165
|
+
f"(baseline {b['n_scored']}, with {w['n_scored']})"
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
delta = ci = pct = None
|
|
169
|
+
ci_pct = [None, None]
|
|
170
|
+
fraction = None
|
|
171
|
+
if b["successes"] == 0 or w["successes"] == 0:
|
|
172
|
+
which = "baseline" if b["successes"] == 0 else "with-arm"
|
|
173
|
+
reasons.append(
|
|
174
|
+
f"{which} never succeeded — its cost per success is unbounded, not a ratio "
|
|
175
|
+
f"(every trial was paid for and none passed); compare the pass rates instead"
|
|
176
|
+
)
|
|
177
|
+
else:
|
|
178
|
+
delta = round(ws - bs, 6)
|
|
179
|
+
pct = round(delta / bs * 100, 4) if bs else None
|
|
180
|
+
lo, hi, fraction = per_success_ci(b["paired"], w["paired"], resamples=resamples)
|
|
181
|
+
if lo is None:
|
|
182
|
+
reasons.append("cost per success was undefined in every resample")
|
|
183
|
+
else:
|
|
184
|
+
ci = [lo, hi]
|
|
185
|
+
ci_pct = [round(lo / bs * 100, 4), round(hi / bs * 100, 4)] if bs else [None, None]
|
|
186
|
+
if fraction > MAX_UNDEFINED_FRACTION:
|
|
187
|
+
reasons.append(
|
|
188
|
+
f"cost per success undefined in {fraction:.1%} of resamples "
|
|
189
|
+
f"(> {MAX_UNDEFINED_FRACTION:.0%}) — too few successes to price one"
|
|
190
|
+
)
|
|
191
|
+
elif lo <= 0 <= hi:
|
|
192
|
+
reasons.append("CI spans zero — undecided, refusing rather than claiming a saving")
|
|
193
|
+
|
|
194
|
+
if reasons:
|
|
195
|
+
decision = "undecided"
|
|
196
|
+
else:
|
|
197
|
+
decision = "cheaper" if delta < 0 else "costlier"
|
|
198
|
+
|
|
199
|
+
return {
|
|
200
|
+
"baseline_usd": bs,
|
|
201
|
+
"with_usd": ws,
|
|
202
|
+
"baseline_successes": f"{b['successes']}/{b['n_scored']}",
|
|
203
|
+
"with_successes": f"{w['successes']}/{w['n_scored']}",
|
|
204
|
+
"delta_usd": delta,
|
|
205
|
+
"delta_pct": pct,
|
|
206
|
+
"ci_usd": ci,
|
|
207
|
+
"ci_pct": ci_pct,
|
|
208
|
+
"undefined_fraction": round(fraction, 4) if fraction is not None else None,
|
|
209
|
+
"decision": decision,
|
|
210
|
+
"refused_reasons": reasons,
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def compare(baseline_dir: Path, with_dir: Path, min_n: int = COST_MIN_N, resamples: int = RESAMPLES,
|
|
215
|
+
allow_cross=(), allow_unstamped: bool = False):
|
|
216
|
+
# Provenance FIRST: if the two arms were not the same harness, the
|
|
217
|
+
# numbers below are a comparison of harnesses, and no interval around
|
|
218
|
+
# them means what it appears to mean (#415, 2026-08-31 -> 2026-09-02).
|
|
219
|
+
prov = provenance.compare(
|
|
220
|
+
baseline_dir, with_dir, allow_cross=allow_cross, allow_unstamped=allow_unstamped
|
|
221
|
+
)
|
|
222
|
+
b = arm(baseline_dir)
|
|
223
|
+
w = arm(with_dir)
|
|
224
|
+
|
|
225
|
+
for a in (b, w):
|
|
226
|
+
if a["n"] == 0:
|
|
227
|
+
raise ValueError(
|
|
228
|
+
f"no per-trial cost measured in {a['job']} — an unmeasured arm is UNKNOWN, "
|
|
229
|
+
"not $0.00 (the remote agent bills the key directly and writes no "
|
|
230
|
+
"claude-output.json; a host claude arm would have written one per trial)"
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
delta = w["cost_mean_usd"] - b["cost_mean_usd"]
|
|
234
|
+
lo, hi = bootstrap_ci(b["costs"], w["costs"], resamples=resamples)
|
|
235
|
+
|
|
236
|
+
def pct(x):
|
|
237
|
+
return round(x / b["cost_mean_usd"] * 100, 4) if b["cost_mean_usd"] else None
|
|
238
|
+
|
|
239
|
+
reasons = []
|
|
240
|
+
if b["n"] < min_n or w["n"] < min_n:
|
|
241
|
+
reasons.append(f"below COST_MIN_N={min_n} per arm (baseline {b['n']}, with {w['n']})")
|
|
242
|
+
if lo <= 0 <= hi:
|
|
243
|
+
reasons.append("CI spans zero — undecided, refusing rather than claiming a saving")
|
|
244
|
+
if not prov["comparable"]:
|
|
245
|
+
reasons.extend(prov["reasons"])
|
|
246
|
+
|
|
247
|
+
if reasons:
|
|
248
|
+
decision = "undecided"
|
|
249
|
+
else:
|
|
250
|
+
decision = "cheaper" if delta < 0 else "costlier"
|
|
251
|
+
|
|
252
|
+
# per_success_block builds its OWN decision, so the provenance refusal has to
|
|
253
|
+
# reach it too. Otherwise an incomparable pair could refuse per-trial and
|
|
254
|
+
# still carry a confident "cheaper per success" onto the job row.
|
|
255
|
+
per_success = per_success_block(b, w, min_n=min_n, resamples=resamples)
|
|
256
|
+
if per_success is not None and not prov["comparable"]:
|
|
257
|
+
per_success = dict(
|
|
258
|
+
per_success,
|
|
259
|
+
decision="undecided",
|
|
260
|
+
refused_reasons=list(per_success.get("refused_reasons", [])) + prov["reasons"],
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
rb, rw = b["reward_mean"], w["reward_mean"]
|
|
264
|
+
regressed = rb is not None and rw is not None and rw < rb - 1e-9
|
|
265
|
+
|
|
266
|
+
drop = ("costs", "paired")
|
|
267
|
+
return {
|
|
268
|
+
"baseline": {k: v for k, v in b.items() if k not in drop},
|
|
269
|
+
"with": {k: v for k, v in w.items() if k not in drop},
|
|
270
|
+
# The metric that can show an instruction making work CHEAPER. Per-trial
|
|
271
|
+
# cost prices a failed trial and a passing one alike; a baseline that
|
|
272
|
+
# fails is not cheap, it is a trial you pay for and then pay again.
|
|
273
|
+
"per_success": per_success,
|
|
274
|
+
"delta_usd": round(delta, 6),
|
|
275
|
+
"delta_pct": pct(delta),
|
|
276
|
+
"ci_usd": [lo, hi],
|
|
277
|
+
"ci_pct": [pct(lo), pct(hi)],
|
|
278
|
+
"decision": decision,
|
|
279
|
+
"refused_reasons": reasons,
|
|
280
|
+
# Carried whether or not it refused: a verdict that pooled across a key
|
|
281
|
+
# by --allow-cross, or could not verify at all, must say so wherever it
|
|
282
|
+
# is read -- including the scheduled handler, which reads this JSON.
|
|
283
|
+
"provenance": prov,
|
|
284
|
+
"reward": {
|
|
285
|
+
"baseline": rb,
|
|
286
|
+
"with": rw,
|
|
287
|
+
"delta": round(rw - rb, 6) if rb is not None and rw is not None else None,
|
|
288
|
+
# A cost win under a reward regression is not a win. Stated here AND
|
|
289
|
+
# in the headline so it cannot be dropped on the way to a slide.
|
|
290
|
+
"regressed": regressed,
|
|
291
|
+
},
|
|
292
|
+
"min_n": min_n,
|
|
293
|
+
"resamples": resamples,
|
|
294
|
+
"seed": SEED,
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def pretty(r: dict[str, Any], brief: bool = False) -> str:
|
|
299
|
+
"""Human form. `brief` drops the per-arm lines for run.sh, which already
|
|
300
|
+
prints its own arm summaries — the cost verdict is what it lacks."""
|
|
301
|
+
b, w = r["baseline"], r["with"]
|
|
302
|
+
lines = [] if brief else [
|
|
303
|
+
f"baseline : {b['n']} trial(s) · ${b['cost_mean_usd']:.4f}/trial "
|
|
304
|
+
f"(${b['cost_sum_usd']:.4f} total) · {b['duration_mean_s']:.0f}s · {b['turns_mean']:.1f} turns "
|
|
305
|
+
f"· reward {b['reward_mean']}",
|
|
306
|
+
f"with-arm : {w['n']} trial(s) · ${w['cost_mean_usd']:.4f}/trial "
|
|
307
|
+
f"(${w['cost_sum_usd']:.4f} total) · {w['duration_mean_s']:.0f}s · {w['turns_mean']:.1f} turns "
|
|
308
|
+
f"· reward {w['reward_mean']}",
|
|
309
|
+
]
|
|
310
|
+
lines += [
|
|
311
|
+
f"cost : {r['delta_usd']:+.4f} USD/trial ({r['delta_pct']:+.1f}%) "
|
|
312
|
+
f"95% CI [{r['ci_usd'][0]:+.4f}, {r['ci_usd'][1]:+.4f}]",
|
|
313
|
+
f"DECISION : {r['decision']} (per trial)",
|
|
314
|
+
]
|
|
315
|
+
for reason in r["refused_reasons"]:
|
|
316
|
+
lines.append(f" refused: {reason}")
|
|
317
|
+
|
|
318
|
+
ps = r["per_success"]
|
|
319
|
+
bs = "n/a" if ps["baseline_usd"] is None else f"${ps['baseline_usd']:.4f}"
|
|
320
|
+
ws = "n/a" if ps["with_usd"] is None else f"${ps['with_usd']:.4f}"
|
|
321
|
+
lines.append(
|
|
322
|
+
f"per-succ : {bs} → {ws} per success "
|
|
323
|
+
f"(passed {ps['baseline_successes']} → {ps['with_successes']})"
|
|
324
|
+
)
|
|
325
|
+
if ps["delta_usd"] is not None and ps["ci_usd"]:
|
|
326
|
+
lines.append(
|
|
327
|
+
f" {ps['delta_usd']:+.4f} USD/success ({ps['delta_pct']:+.1f}%) "
|
|
328
|
+
f"95% CI [{ps['ci_usd'][0]:+.4f}, {ps['ci_usd'][1]:+.4f}]"
|
|
329
|
+
)
|
|
330
|
+
lines.append(f"DECISION : {ps['decision']} (per success)")
|
|
331
|
+
for reason in ps["refused_reasons"]:
|
|
332
|
+
lines.append(f" refused: {reason}")
|
|
333
|
+
if r["reward"]["regressed"]:
|
|
334
|
+
lines.append(
|
|
335
|
+
f" WARNING: reward regressed ({r['reward']['baseline']} → {r['reward']['with']}) — "
|
|
336
|
+
"a cheaper arm that solves less is not a saving; do not quote the cost delta alone"
|
|
337
|
+
)
|
|
338
|
+
return "\n".join(lines)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def main(argv: list[str]) -> int:
|
|
342
|
+
args = [a for a in argv[1:] if not a.startswith("--")]
|
|
343
|
+
if len(args) < 2:
|
|
344
|
+
print("usage: cost_delta.py <baseline-job-dir> <with-job-dir> [--json]", file=sys.stderr)
|
|
345
|
+
return 2
|
|
346
|
+
|
|
347
|
+
def opt(name, default):
|
|
348
|
+
for i, a in enumerate(argv):
|
|
349
|
+
if a == name and i + 1 < len(argv):
|
|
350
|
+
return int(argv[i + 1])
|
|
351
|
+
return default
|
|
352
|
+
|
|
353
|
+
try:
|
|
354
|
+
def strval(name, default=""):
|
|
355
|
+
for i, a in enumerate(argv):
|
|
356
|
+
if a == name and i + 1 < len(argv):
|
|
357
|
+
return argv[i + 1]
|
|
358
|
+
return default
|
|
359
|
+
|
|
360
|
+
r = compare(
|
|
361
|
+
Path(args[0]),
|
|
362
|
+
Path(args[1]),
|
|
363
|
+
min_n=opt("--min-n", COST_MIN_N),
|
|
364
|
+
resamples=opt("--resamples", RESAMPLES),
|
|
365
|
+
allow_cross=[k.strip() for k in strval("--allow-cross").split(",") if k.strip()],
|
|
366
|
+
allow_unstamped="--allow-unstamped" in argv,
|
|
367
|
+
)
|
|
368
|
+
except ValueError as e:
|
|
369
|
+
print(str(e), file=sys.stderr)
|
|
370
|
+
return 3
|
|
371
|
+
|
|
372
|
+
print(json.dumps(r) if "--json" in argv[1:] else pretty(r, brief="--brief" in argv[1:]))
|
|
373
|
+
return 0
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
if __name__ == "__main__":
|
|
377
|
+
sys.exit(main(sys.argv))
|