@event4u/agent-config 14.12.0 → 14.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.augment-plugin/marketplace.json +3 -3
- package/.augment-plugin/plugin.json +1 -1
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +118 -155
- package/CONTRIBUTING.md +38 -0
- package/CREDITS.md +135 -0
- package/NOTICE +13 -0
- package/README.md +8 -6
- package/dist/agent-src/commands/analyze/inbox.md +145 -2
- package/dist/agent-src/commands/feature/dev.md +7 -2
- package/dist/agent-src/commands/ghostwriter/write.md +5 -1
- package/dist/agent-src/commands/implement-ticket.md +1 -1
- package/dist/agent-src/commands/mission/upgrade.md +1 -1
- package/dist/agent-src/commands/orchestrate.md +2 -2
- package/dist/agent-src/commands/post-as/me.md +5 -1
- package/dist/agent-src/commands/pr/merge.md +19 -6
- package/dist/agent-src/commands/roadmap/process-full.md +19 -5
- package/dist/agent-src/commands/tdd/red.md +12 -2
- package/dist/agent-src/commands/work.md +1 -1
- package/dist/agent-src/contexts/communication/rules-auto/token-efficiency-mechanics.md +40 -0
- package/dist/agent-src/contexts/execution/auto-dispatch-classification.md +5 -2
- package/dist/agent-src/contexts/execution/autonomy-mechanics.md +21 -0
- package/dist/agent-src/contexts/execution/orchestration-telemetry.md +1 -0
- package/dist/agent-src/contexts/execution/subagent-response-contract.md +40 -0
- package/dist/agent-src/contexts/execution/subagent-steering.md +13 -1
- package/dist/agent-src/contexts/execution/terminal-states.md +96 -0
- package/dist/agent-src/rules/artifact-drafting-protocol.md +1 -1
- package/dist/agent-src/rules/broken-access-control.md +4 -0
- package/dist/agent-src/rules/code-comment-discipline.md +6 -0
- package/dist/agent-src/rules/code-provenance.md +4 -0
- package/dist/agent-src/rules/design-fidelity.md +1 -1
- package/dist/agent-src/rules/domain-safety-pii.md +4 -0
- package/dist/agent-src/rules/domain-safety-retention.md +4 -0
- package/dist/agent-src/rules/engineering-safety-floor.md +2 -0
- package/dist/agent-src/rules/external-code-graph-interop.md +14 -12
- package/dist/agent-src/rules/improve-before-implement.md +6 -8
- package/dist/agent-src/rules/laravel-routing.md +1 -1
- package/dist/agent-src/rules/lethal-trifecta-guard.md +4 -0
- package/dist/agent-src/rules/secret-vcs-guard.md +2 -0
- package/dist/agent-src/rules/security-sensitive-stop.md +2 -0
- package/dist/agent-src/rules/source-confidentiality.md +12 -34
- package/dist/agent-src/rules/symfony-routing.md +1 -1
- package/dist/agent-src/rules/token-efficiency.md +14 -0
- package/dist/agent-src/rules/tool-safety.md +4 -0
- package/dist/agent-src/rules/untrusted-input-defense.md +4 -0
- package/dist/agent-src/scripts/archive_completed_roadmaps.ts +252 -8
- package/dist/agent-src/scripts/guarded_baseline.ts +343 -0
- package/dist/agent-src/scripts/roadmap_set_step.ts +253 -0
- package/dist/agent-src/scripts/stub_queue.ts +82 -0
- package/dist/agent-src/scripts/stubs_due.ts +516 -0
- package/dist/agent-src/scripts/ticket_bundles.ts +84 -0
- package/dist/agent-src/scripts/update_roadmap_progress.ts +16 -12
- package/dist/agent-src/skills/brand-asset-generation/SKILL.md +17 -4
- package/dist/agent-src/skills/brand-audit/SKILL.md +33 -0
- package/dist/agent-src/skills/code-intelligence/SKILL.md +57 -0
- package/dist/agent-src/skills/database/SKILL.md +8 -0
- package/dist/agent-src/skills/dependency-upgrade/SKILL.md +27 -0
- package/dist/agent-src/skills/design-review/SKILL.md +1 -1
- package/dist/agent-src/skills/doc-coauthoring/SKILL.md +1 -1
- package/dist/agent-src/skills/existing-ui-audit/SKILL.md +17 -1
- package/dist/agent-src/skills/feature-planning/SKILL.md +9 -2
- package/dist/agent-src/skills/frontend-render-security/SKILL.md +44 -0
- package/dist/agent-src/skills/git-workflow/SKILL.md +1 -1
- package/dist/agent-src/skills/js-library-packaging/scripts/check_package_surface.ts +38 -4
- package/dist/agent-src/skills/judge-artifact-completeness/rubrics/ticket-quality-score.json +2 -2
- package/dist/agent-src/skills/laravel/SKILL.md +35 -0
- package/dist/agent-src/skills/laravel-migration/SKILL.md +57 -3
- package/dist/agent-src/skills/monorepo-workspace/SKILL.md +26 -0
- package/dist/agent-src/skills/playbook-authoring/SKILL.md +28 -1
- package/dist/agent-src/skills/react-shadcn-ui/SKILL.md +1 -1
- package/dist/agent-src/skills/skill-writing/references/procedure.md +5 -5
- package/dist/agent-src/skills/sql-writing/SKILL.md +7 -1
- package/dist/agent-src/skills/standards-from-config/SKILL.md +27 -4
- package/dist/agent-src/skills/supply-chain-intake/SKILL.md +11 -4
- package/dist/agent-src/skills/symfony-workflow/SKILL.md +35 -0
- package/dist/agent-src/skills/test-case-discovery/SKILL.md +85 -0
- package/dist/agent-src/skills/test-driven-development/SKILL.md +33 -9
- package/dist/agent-src/skills/testing-anti-patterns/SKILL.md +1 -1
- package/dist/agent-src/skills/testing-anti-patterns/process-anti-patterns.md +37 -0
- package/dist/agent-src/skills/ui-component-architect/SKILL.md +45 -2
- package/dist/agent-src/skills/upstream-contribute/SKILL.md +38 -1
- package/dist/agent-src/skills/verify-repair-loop/SKILL.md +9 -4
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/roadmaps.md +97 -3
- package/dist/agent-src/templates/scripts/check_memory.ts +258 -17
- package/dist/agent-src/templates/scripts/check_memory_proposal.ts +9 -3
- package/dist/agent-src/templates/scripts/memory_lookup.ts +86 -0
- package/dist/agent-src/templates/scripts/memory_signal.ts +44 -0
- package/dist/agent-src/templates/scripts/work_engine/directives/backend/implement.ts +90 -0
- package/dist/agent-src/templates/scripts/work_engine/directives/backend/index.ts +11 -0
- package/dist/agent-src/templates/scripts/work_engine/directives/backend/test.ts +23 -1
- package/dist/agent-src/templates/scripts/work_engine/directives/backend/verify.ts +10 -1
- package/dist/agent-src/templates/scripts/work_engine/directives/mixed/stitch.ts +5 -1
- package/dist/agent-src/templates/scripts/work_engine/stack/runner.ts +102 -14
- package/dist/cli/commands/uiAudit.js +124 -5
- package/dist/cli/registry.js +2 -0
- package/dist/cli-delegate/{chunk-HYWSRNUX.js → chunk-2DK2VJWG.js} +3 -3
- package/dist/cli-delegate/{chunk-JCW7ZLHK.js → chunk-4FS4WRXP.js} +5 -0
- package/dist/cli-delegate/{chunk-OMIVVCG2.js → chunk-A4QJ6ENT.js} +24 -0
- package/dist/cli-delegate/{chunk-JJA2QEWA.js → chunk-AKJGUQNO.js} +466 -285
- package/dist/cli-delegate/{chunk-E46KPN6K.js → chunk-BTWUYDGN.js} +424 -363
- package/dist/cli-delegate/{chunk-K4FFODTS.js → chunk-ILMUPLGW.js} +1227 -316
- package/dist/cli-delegate/{chunk-FF7WAFMR.js → chunk-KPBWFOPZ.js} +5 -1
- package/dist/cli-delegate/cmd_adr_effective.js +102 -0
- package/dist/cli-delegate/cmd_conformance.js +80 -11
- package/dist/cli-delegate/cmd_doctor.js +3 -3
- package/dist/cli-delegate/cmd_export.js +3 -2
- package/dist/cli-delegate/cmd_fleet.js +2 -2
- package/dist/cli-delegate/cmd_handoff.js +1 -1
- package/dist/cli-delegate/cmd_preflight.js +2 -2
- package/dist/cli-delegate/cmd_prune.js +3 -2
- package/dist/cli-delegate/cmd_settings_get.js +1 -1
- package/dist/cli-delegate/cmd_settings_migrate.js +3 -2
- package/dist/cli-delegate/cmd_settings_set.js +2 -2
- package/dist/cli-delegate/cmd_sync.js +3 -2
- package/dist/cli-delegate/cmd_uninstall.js +3 -2
- package/dist/cli-delegate/cmd_validate.js +3 -2
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +58 -58
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +2900 -1049
- package/dist/install/claudeRuleRewrite.js +254 -0
- package/dist/install/detect.js +29 -16
- package/dist/install/detect_php_shape.js +159 -0
- package/dist/install/install.mjs +634 -428
- package/dist/mcp/awesome-mcp-servers.row.md +1 -1
- package/dist/mcp/mcp-cloudflare-catalogue.json +1 -1
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/mcp/server.json +3 -3
- package/dist/mcp/server.mjs +4895 -2627
- package/dist/server/schemas/settings.js +3 -0
- package/dist/ui/assets/index-D0jaCRPM.js +29 -0
- package/dist/ui/index.html +1 -1
- package/docs/MIGRATION.md +1 -1
- package/docs/THIRD-PARTY-NOTICES.md +25 -0
- package/docs/contracts/provider-lifecycle.md +1 -1
- package/docs/contracts/settings-classes.md +10 -4
- package/docs/guidelines/agent-infra/5w2h-analysis.md +3 -3
- package/docs/guidelines/agent-infra/agent-interaction-and-decision-quality.md +1 -0
- package/docs/guidelines/agent-infra/artifact-drafting-protocol-mechanics.md +58 -3
- package/docs/guidelines/agent-infra/ask-when-uncertain-demos.md +2 -2
- package/docs/guidelines/agent-infra/asking-and-brevity-examples.md +3 -3
- package/docs/guidelines/agent-infra/break-glass-usage.md +7 -7
- package/docs/guidelines/agent-infra/carve-out-predicates.md +3 -3
- package/docs/guidelines/agent-infra/comparison-matrix.md +1 -1
- package/docs/guidelines/agent-infra/critical-thinking.md +4 -4
- package/docs/guidelines/agent-infra/direct-answers-demos.md +2 -2
- package/docs/guidelines/agent-infra/domain-eval-anti-pattern.md +1 -1
- package/docs/guidelines/agent-infra/engineering-memory-data-format.md +3 -3
- package/docs/guidelines/agent-infra/existence-question-verdicts.md +80 -0
- package/docs/guidelines/agent-infra/first-principles.md +2 -2
- package/docs/guidelines/agent-infra/framework-neutrality-patterns.md +2 -2
- package/docs/guidelines/agent-infra/guarded-baseline.md +39 -0
- package/docs/guidelines/agent-infra/inversion-thinking.md +5 -5
- package/docs/guidelines/agent-infra/language-and-tone-examples.md +1 -1
- package/docs/guidelines/agent-infra/layered-settings.md +10 -10
- package/docs/guidelines/agent-infra/memory-access.md +1 -1
- package/docs/guidelines/agent-infra/mental-models.md +3 -3
- package/docs/guidelines/agent-infra/reuse-verdict-mechanics.md +71 -0
- package/docs/guidelines/agent-infra/review-routing-data-format.md +7 -7
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +13 -7
- package/docs/guidelines/agent-infra/role-contracts.md +2 -2
- package/docs/guidelines/agent-infra/role-mode-router.md +3 -3
- package/docs/guidelines/agent-infra/rule-type-governance.md +2 -2
- package/docs/guidelines/agent-infra/scqa-framework.md +5 -5
- package/docs/guidelines/agent-infra/self-improvement-pipeline.md +6 -6
- package/docs/guidelines/agent-infra/simplicity-and-goal-demos.md +2 -2
- package/docs/guidelines/agent-infra/six-hats.md +3 -3
- package/docs/guidelines/agent-infra/skill-quality-checklist.md +6 -6
- package/docs/guidelines/agent-infra/source-confidentiality-mechanics.md +109 -0
- package/docs/guidelines/agent-infra/systems-thinking.md +1 -1
- package/docs/guidelines/agent-infra/verify-before-complete-demos.md +2 -2
- package/docs/guidelines/augment-portability-patterns.md +6 -4
- package/docs/guidelines/code-clarity.md +18 -0
- package/docs/guidelines/component-oriented-and-oop-development.md +3 -2
- package/docs/guidelines/cross-role-handoff.md +2 -2
- package/docs/guidelines/design-fidelity-mechanics.md +1 -1
- package/docs/guidelines/docs/readme-size-and-splitting.md +3 -3
- package/docs/guidelines/php/database.md +90 -7
- package/docs/guidelines/prompt-templates.md +7 -7
- package/llms.txt +2 -2
- package/package.json +5 -2
- package/src/agent-src/scripts/archive_completed_roadmaps.ts +252 -8
- package/src/agent-src/scripts/guarded_baseline.ts +343 -0
- package/src/agent-src/scripts/roadmap_set_step.ts +253 -0
- package/src/agent-src/scripts/stub_queue.ts +82 -0
- package/src/agent-src/scripts/stubs_due.ts +516 -0
- package/src/agent-src/scripts/ticket_bundles.ts +84 -0
- package/src/agent-src/scripts/update_roadmap_progress.ts +16 -12
- package/src/agent-src/templates/scripts/check_memory.ts +258 -17
- package/src/agent-src/templates/scripts/check_memory_proposal.ts +9 -3
- package/src/agent-src/templates/scripts/memory_lookup.ts +86 -0
- package/src/agent-src/templates/scripts/memory_signal.ts +44 -0
- package/src/agent-src/templates/scripts/work_engine/directives/backend/implement.ts +90 -0
- package/src/agent-src/templates/scripts/work_engine/directives/backend/index.ts +11 -0
- package/src/agent-src/templates/scripts/work_engine/directives/backend/test.ts +23 -1
- package/src/agent-src/templates/scripts/work_engine/directives/backend/verify.ts +10 -1
- package/src/agent-src/templates/scripts/work_engine/directives/mixed/stitch.ts +5 -1
- package/src/agent-src/templates/scripts/work_engine/stack/runner.ts +102 -14
- package/src/config/agent-settings.template.yml +32 -3
- package/src/config/assurance-capability-registry.json +21 -9
- package/src/config/canonical-terms.yml +119 -0
- package/src/config/capability-native-outcome-bars.json +81 -0
- package/src/config/ci-local-parity.yml +80 -0
- package/src/config/drafting-channels.yml +43 -0
- package/src/config/estate-count-budget.json +57 -36
- package/src/config/evaluator-budgets.json +14 -14
- package/src/config/gate-coverage.yml +729 -63
- package/src/config/gate-reachability-exemptions.json +21 -0
- package/src/config/gate-violation-baselines.json +42 -19
- package/src/config/harness-evolution-budget.json +25 -0
- package/src/config/lapsed-beta-baseline.json +96 -0
- package/src/config/memory-twin-verdicts.yml +129 -0
- package/src/config/metric-registry.yml +124 -0
- package/src/config/pack-size-budget.json +16 -15
- package/src/config/preamble-payload-budget.json +12 -4
- package/src/config/profiles/balanced.ini +27 -0
- package/src/config/profiles/full.ini +27 -0
- package/src/config/profiles/minimal.ini +27 -0
- package/src/config/publish-surface.json +9 -2
- package/src/config/routing-coverage-seed.json +16 -2
- package/src/config/rule-stub-ceilings.json +133 -112
- package/src/config/turnaround-budget.json +66 -0
- package/src/install/claudeRuleRewrite.ts +288 -0
- package/src/install/detect.ts +60 -17
- package/src/install/detect_php_shape.ts +191 -0
- package/src/scripts/_cli/cmd_doctor.ts +27 -112
- package/src/scripts/_dispatch.bash +23 -0
- package/src/scripts/_lib/activation_ladder.ts +255 -0
- package/src/scripts/_lib/activation_receipt_producer.ts +341 -0
- package/src/scripts/_lib/activation_states.ts +88 -0
- package/src/scripts/_lib/adherence_detectors.ts +143 -0
- package/src/scripts/_lib/adr_frontmatter.ts +26 -0
- package/src/scripts/_lib/arith_claims.ts +181 -0
- package/src/scripts/_lib/arm_ranking.ts +62 -0
- package/src/scripts/_lib/asset_delivery_ledger.ts +303 -0
- package/src/scripts/_lib/audit_field_provenance.ts +54 -0
- package/src/scripts/_lib/auto_dispatch.ts +32 -3
- package/src/scripts/_lib/bench_ab_size_claim.ts +98 -4
- package/src/scripts/_lib/benchmark_cache_fields.ts +158 -0
- package/src/scripts/_lib/candidate_pair_delta.ts +349 -0
- package/src/scripts/_lib/candidate_proposer.ts +402 -0
- package/src/scripts/_lib/candidate_record.ts +601 -0
- package/src/scripts/_lib/capture_rate.ts +319 -0
- package/src/scripts/_lib/carrier_status.ts +118 -0
- package/src/scripts/_lib/cascade_stage_enumeration.ts +92 -0
- package/src/scripts/_lib/collector_denominator.ts +771 -0
- package/src/scripts/_lib/collector_record.ts +340 -0
- package/src/scripts/_lib/collector_store.ts +664 -0
- package/src/scripts/_lib/collector_supervision.ts +827 -0
- package/src/scripts/_lib/concern_estate.ts +77 -0
- package/src/scripts/_lib/config_chain.ts +477 -0
- package/src/scripts/_lib/config_cost.ts +209 -0
- package/src/scripts/_lib/corpus_manifest.ts +726 -0
- package/src/scripts/_lib/council_settings_block.ts +6 -0
- package/src/scripts/_lib/curator_ops.ts +172 -0
- package/src/scripts/_lib/delivery_arm_experiment.ts +282 -0
- package/src/scripts/_lib/delivery_criticality.ts +126 -0
- package/src/scripts/_lib/doctor_runtime_checks.ts +70 -0
- package/src/scripts/_lib/empty_cycles.ts +88 -0
- package/src/scripts/_lib/estate_carrier_cases.ts +72 -0
- package/src/scripts/_lib/eval_publication.ts +354 -0
- package/src/scripts/_lib/evaluation_cascade.ts +452 -0
- package/src/scripts/_lib/evaluation_vector.ts +283 -0
- package/src/scripts/_lib/evaluator_promotion.ts +225 -0
- package/src/scripts/_lib/evidence_basis.ts +118 -0
- package/src/scripts/_lib/evolution_roi.ts +562 -0
- package/src/scripts/_lib/exec_evidence.ts +8 -0
- package/src/scripts/_lib/experience_card.ts +272 -0
- package/src/scripts/_lib/experience_report.ts +151 -0
- package/src/scripts/_lib/experiment_freeze.ts +112 -0
- package/src/scripts/_lib/generated_by.ts +66 -0
- package/src/scripts/_lib/harness_evolution_guards.ts +239 -0
- package/src/scripts/_lib/hook_effect_probe.ts +141 -0
- package/src/scripts/_lib/host_denominator.ts +687 -0
- package/src/scripts/_lib/host_projection_reach.ts +157 -0
- package/src/scripts/_lib/ignored_blocker.ts +148 -0
- package/src/scripts/_lib/install_reach_checks.ts +495 -0
- package/src/scripts/_lib/json_duplicate_keys.ts +150 -0
- package/src/scripts/_lib/judge_hygiene.ts +228 -0
- package/src/scripts/_lib/legacy_boundary_map.ts +228 -0
- package/src/scripts/_lib/lexical_shortlist.ts +215 -0
- package/src/scripts/_lib/llm_candidate_proposer.ts +461 -0
- package/src/scripts/_lib/llm_proposer_transport.ts +153 -0
- package/src/scripts/_lib/loop_guards.ts +299 -0
- package/src/scripts/_lib/loss_class.ts +129 -0
- package/src/scripts/_lib/machine_wake.ts +51 -0
- package/src/scripts/_lib/md_prose_lines.ts +177 -0
- package/src/scripts/_lib/minimality_tiebreak.ts +168 -0
- package/src/scripts/_lib/one_resolver_invariant.ts +449 -0
- package/src/scripts/_lib/orchestration_record.ts +253 -5
- package/src/scripts/_lib/outcome_envelope.ts +227 -0
- package/src/scripts/_lib/outcome_vocabularies.ts +125 -0
- package/src/scripts/_lib/paired_stats.ts +245 -0
- package/src/scripts/_lib/paired_verdict.ts +226 -0
- package/src/scripts/_lib/pathology_archive.ts +393 -0
- package/src/scripts/_lib/prefix_stable_surfaces.ts +123 -0
- package/src/scripts/_lib/privacy_class.ts +47 -0
- package/src/scripts/_lib/promotion_capability.ts +315 -0
- package/src/scripts/_lib/promotion_evidence.ts +414 -0
- package/src/scripts/_lib/promotion_review.ts +294 -0
- package/src/scripts/_lib/prompt_shape.ts +92 -0
- package/src/scripts/_lib/regression_neighbourhood.ts +259 -0
- package/src/scripts/_lib/release_highlights.ts +70 -1
- package/src/scripts/_lib/release_material.ts +125 -2
- package/src/scripts/_lib/repeated_failure.ts +105 -0
- package/src/scripts/_lib/repo_root.ts +111 -0
- package/src/scripts/_lib/review_skipped_record.ts +37 -0
- package/src/scripts/_lib/role_split.ts +338 -0
- package/src/scripts/_lib/routing_corpus.ts +307 -0
- package/src/scripts/_lib/routing_index_input.ts +104 -0
- package/src/scripts/_lib/rule_injection.ts +22 -1
- package/src/scripts/_lib/runtime_journal.ts +1499 -0
- package/src/scripts/_lib/runtime_wiring_checks.ts +430 -0
- package/src/scripts/_lib/semantic_noop.ts +167 -0
- package/src/scripts/_lib/skill_catalogue.ts +11 -1
- package/src/scripts/_lib/source_digest.ts +200 -0
- package/src/scripts/_lib/source_redact.ts +260 -0
- package/src/scripts/_lib/source_shape.ts +502 -0
- package/src/scripts/_lib/source_snapshot_dedup.ts +199 -0
- package/src/scripts/_lib/strip_comments.d.mts +8 -0
- package/src/scripts/_lib/strip_comments.mjs +65 -0
- package/src/scripts/_lib/test_red_state.ts +154 -0
- package/src/scripts/_lib/trigger_shift.ts +144 -0
- package/src/scripts/_lib/untrusted_content.ts +82 -7
- package/src/scripts/activation_receipt.ts +258 -0
- package/src/scripts/adoption_report.ts +1 -1
- package/src/scripts/adr/evidence_census.ts +1 -1
- package/src/scripts/adr/regenerate_index.ts +26 -2
- package/src/scripts/adr_cite_check.ts +164 -1
- package/src/scripts/ai_council/blind_review.ts +11 -1
- package/src/scripts/ai_council/clients.ts +4 -0
- package/src/scripts/ai_council/config.ts +3 -0
- package/src/scripts/ai_council/consensus.ts +28 -8
- package/src/scripts/ai_council/consensus_round.ts +226 -0
- package/src/scripts/ai_council/cross_exam.ts +128 -0
- package/src/scripts/ai_council/disagreement_signal.ts +338 -0
- package/src/scripts/ai_council/explain_route.ts +157 -0
- package/src/scripts/ai_council/information_gain.ts +293 -0
- package/src/scripts/ai_council/inline_findings.ts +340 -0
- package/src/scripts/ai_council/judge_position_bias.ts +197 -0
- package/src/scripts/ai_council/leakage_corpus.ts +385 -0
- package/src/scripts/ai_council/leakage_patterns.ts +453 -0
- package/src/scripts/ai_council/minority_retention.ts +171 -0
- package/src/scripts/ai_council/orchestrator.ts +16 -203
- package/src/scripts/ai_council/orchestrator_results.ts +23 -3
- package/src/scripts/ai_council/pricing.ts +106 -0
- package/src/scripts/ai_council/prompts.ts +74 -7
- package/src/scripts/ai_council/provider_leakage_bench.ts +327 -0
- package/src/scripts/ai_council/py_format.ts +14 -3
- package/src/scripts/ai_council/recouncil_guard.ts +293 -0
- package/src/scripts/ai_council/recouncil_savings.ts +299 -0
- package/src/scripts/ai_council/replay_route.ts +223 -0
- package/src/scripts/ai_council/response_record.ts +50 -0
- package/src/scripts/ai_council/reviewer_assignment.ts +279 -0
- package/src/scripts/ai_council/routing_training_row.ts +197 -0
- package/src/scripts/ai_council/synthesis_strategy.ts +159 -0
- package/src/scripts/ai_council/topology_bench_manifest.ts +860 -0
- package/src/scripts/ai_council/topology_vocabulary.ts +175 -0
- package/src/scripts/asset_delivery_ledger.ts +157 -0
- package/src/scripts/attest_artifact.ts +193 -0
- package/src/scripts/audit_auto_rules.ts +2 -1
- package/src/scripts/audit_command_surface.ts +1 -1
- package/src/scripts/audit_mcp_tools.ts +2 -2
- package/src/scripts/audit_skill_overlap_allowlist.json +6 -1
- package/src/scripts/audit_user_type_axis.ts +2 -1
- package/src/scripts/bench_ab_clone.ts +215 -10
- package/src/scripts/bench_ab_integrity.ts +119 -5
- package/src/scripts/bench_ab_v2_stats.ts +40 -213
- package/src/scripts/build_archive_index.ts +1 -1
- package/src/scripts/build_linear_digest.ts +1 -1
- package/src/scripts/build_proof.ts +11 -4
- package/src/scripts/build_rule_trigger_matrix.ts +1 -1
- package/src/scripts/build_source_digests.ts +157 -0
- package/src/scripts/cache_realization_report.ts +103 -0
- package/src/scripts/check_adr_frontmatter.ts +131 -10
- package/src/scripts/check_benchmark_report_fields.ts +207 -0
- package/src/scripts/check_beta_review_markers.ts +235 -6
- package/src/scripts/check_bundle_path_leakage.ts +34 -1
- package/src/scripts/check_claims.ts +576 -6
- package/src/scripts/check_composite_arming.ts +35 -0
- package/src/scripts/check_concern_admissions.ts +365 -0
- package/src/scripts/check_enforcement_coverage.ts +28 -0
- package/src/scripts/check_estate_count.ts +347 -25
- package/src/scripts/check_gate_reachability.ts +383 -0
- package/src/scripts/check_host_loadability.ts +104 -0
- package/src/scripts/check_loss_class_declared.ts +308 -0
- package/src/scripts/check_md_language.ts +15 -98
- package/src/scripts/check_memory_proposal.ts +18 -11
- package/src/scripts/check_no_currency_in_cost_surfaces.ts +225 -0
- package/src/scripts/check_no_external_sources.ts +281 -6
- package/src/scripts/check_pack_conformance_fixture.ts +11 -1
- package/src/scripts/check_pack_size.ts +59 -2
- package/src/scripts/check_preamble_payload_budget.ts +114 -3
- package/src/scripts/check_prefix_stable_mutation.ts +372 -0
- package/src/scripts/check_publish_surface.ts +10 -1
- package/src/scripts/check_references.ts +24 -1
- package/src/scripts/check_release_highlights.ts +96 -13
- package/src/scripts/check_roadmap_trackable.ts +17 -2
- package/src/scripts/check_rule_evidence_declaration.ts +316 -0
- package/src/scripts/check_rule_projection_integrity.ts +14 -4
- package/src/scripts/check_score_contract.ts +26 -3
- package/src/scripts/check_static_parity.ts +285 -0
- package/src/scripts/check_supervision_claim_atomicity.ts +498 -0
- package/src/scripts/check_suppression_hygiene.ts +15 -0
- package/src/scripts/ci_settle.ts +39 -2
- package/src/scripts/code_graph/build.ts +60 -3
- package/src/scripts/code_graph/extract.ts +44 -0
- package/src/scripts/code_graph/types.ts +24 -0
- package/src/scripts/collector_daemon.ts +714 -0
- package/src/scripts/config_cost_report.ts +302 -0
- package/src/scripts/consumer_matrix.ts +74 -1
- package/src/scripts/corpus_manifest.ts +235 -0
- package/src/scripts/cost/budget.mjs +10 -3
- package/src/scripts/cost/track.mjs +10 -3
- package/src/scripts/council_cli.ts +37 -54
- package/src/scripts/dispatch_r2_reviewer.ts +46 -5
- package/src/scripts/evolution_lab.ts +1153 -0
- package/src/scripts/external_sources_denylist.json +5 -18
- package/src/scripts/extract_audit_patterns.ts +21 -0
- package/src/scripts/fold_intake.ts +12 -1
- package/src/scripts/gen_discovery_baseline.ts +1 -1
- package/src/scripts/generate_capabilities_index.ts +12 -5
- package/src/scripts/generate_capability_matrix.ts +3 -3
- package/src/scripts/generate_command_flows.ts +2 -2
- package/src/scripts/generate_cookbook.ts +2 -2
- package/src/scripts/generate_index.ts +8 -1
- package/src/scripts/generate_knowledge_index.ts +2 -1
- package/src/scripts/generate_ownership_matrix.ts +2 -1
- package/src/scripts/generate_pack_manifests.ts +10 -2
- package/src/scripts/generate_role_experiences_catalog.ts +2 -2
- package/src/scripts/generate_settings_reference.ts +1 -1
- package/src/scripts/git_authorization_hook.ts +22 -1
- package/src/scripts/grade_target_readiness.ts +48 -5
- package/src/scripts/hook_effect_doctor.ts +261 -0
- package/src/scripts/hook_manifest.json +1 -1
- package/src/scripts/hook_manifest.yaml +125 -16
- package/src/scripts/hooks/block_speaking_inbox_dir.ts +271 -0
- package/src/scripts/hooks/block_unauthorized_git.ts +18 -0
- package/src/scripts/hooks/comment_discipline_hook.ts +146 -0
- package/src/scripts/hooks/concern_registry.ts +6 -0
- package/src/scripts/hooks/dispatch_hook.ts +32 -0
- package/src/scripts/hooks/journal_record_hook.ts +345 -0
- package/src/scripts/hooks/prepush_metadata_sources.sh +110 -0
- package/src/scripts/hooks/run_continuation_hook.ts +44 -9
- package/src/scripts/hooks/shims/install.sh +61 -0
- package/src/scripts/hooks/shims/php +101 -0
- package/src/scripts/hooks/suggestion_capture_hook.ts +10 -0
- package/src/scripts/hooks/turn_end_gate_hook.ts +9 -1
- package/src/scripts/hot_context_hook.ts +16 -0
- package/src/scripts/install-hooks.sh +20 -5
- package/src/scripts/install.ts +17 -35
- package/src/scripts/inventory_frontmatter.ts +2 -1
- package/src/scripts/language_mirror_hook.ts +14 -2
- package/src/scripts/lint_adr_sweep_routing.ts +12 -0
- package/src/scripts/lint_canonical_terms.ts +604 -0
- package/src/scripts/lint_carrier_integrity.ts +820 -0
- package/src/scripts/lint_code_comments.ts +643 -0
- package/src/scripts/lint_composition_review.ts +619 -0
- package/src/scripts/lint_consolidation_lineage.ts +666 -0
- package/src/scripts/lint_eval_specs.ts +593 -0
- package/src/scripts/lint_hook_manifest.ts +19 -0
- package/src/scripts/lint_host_portability.ts +373 -0
- package/src/scripts/lint_judge_prompt_expectation.ts +27 -1
- package/src/scripts/lint_memory_twin_parity.ts +335 -0
- package/src/scripts/lint_metric_consumers.ts +355 -0
- package/src/scripts/lint_plan_risk_register.ts +39 -19
- package/src/scripts/lint_promotion_paths.ts +779 -0
- package/src/scripts/lint_skill_descriptions.ts +18 -1
- package/src/scripts/lint_skill_frontmatter_safety.ts +58 -0
- package/src/scripts/lint_skill_link_reach.ts +252 -0
- package/src/scripts/lint_skill_trigger_corpus.ts +426 -23
- package/src/scripts/lint_ticket_buildable.ts +17 -1
- package/src/scripts/llm_propose.ts +256 -0
- package/src/scripts/mcp_server/consumer_tool_catalog.json +1 -1
- package/src/scripts/mcp_server/tool_catalog_source.ts +1 -1
- package/src/scripts/measure_delivery_sets.ts +361 -0
- package/src/scripts/measure_docs_dead_links.ts +348 -0
- package/src/scripts/measure_host_capture.ts +461 -0
- package/src/scripts/measure_routing_signal.ts +307 -0
- package/src/scripts/migration_status.ts +3 -3
- package/src/scripts/model_rule_injection.ts +76 -0
- package/src/scripts/orchestration_record.ts +14 -0
- package/src/scripts/preamble_byte_census.ts +139 -10
- package/src/scripts/prepack-check.mjs +4 -1
- package/src/scripts/probe_council_retention.ts +495 -0
- package/src/scripts/probe_turnaround.ts +384 -0
- package/src/scripts/probe_unblocked_ask.ts +279 -0
- package/src/scripts/release.ts +21 -37
- package/src/scripts/release_drill.ts +143 -0
- package/src/scripts/release_env.ts +35 -0
- package/src/scripts/release_publication.ts +84 -0
- package/src/scripts/render_benchmark_md.ts +1 -1
- package/src/scripts/rule_activation_census.ts +70 -9
- package/src/scripts/rule_backlinks.ts +1 -1
- package/src/scripts/run_lifecycle_suite.ts +208 -0
- package/src/scripts/run_skill_evals.ts +40 -0
- package/src/scripts/schemas/delivery-manifest.schema.json +43 -0
- package/src/scripts/schemas/drafting-channels.schema.json +43 -0
- package/src/scripts/schemas/evals.schema.json +24 -1
- package/src/scripts/schemas/rule.schema.json +52 -1
- package/src/scripts/schemas/skill.schema.json +17 -2
- package/src/scripts/skill_overlap.ts +7 -1
- package/src/scripts/skill_tools/run_block_d_eval.ts +8 -1
- package/src/scripts/skill_tools/score_skill_relevance.ts +84 -3
- package/src/scripts/skill_tools/suggest_skill_for_task.ts +8 -1
- package/src/scripts/skill_usage_report.ts +2 -1
- package/src/scripts/stubs_due.ts +211 -0
- package/src/scripts/sweep_source_surfaces.ts +678 -0
- package/src/scripts/sync_pr_branch.ts +3 -3
- package/src/scripts/trigger_eval_grandfather.json +202 -222
- package/src/scripts/utilization_report.ts +22 -2
- package/src/server/schemas/settings.ts +5 -0
- package/src/templates/marketing-copy.yml +1 -1
- package/dist/ui/assets/index-Cxgy63kz.js +0 -26
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The run report, and the two things step 5.6 asks it to carry.
|
|
3
|
+
*
|
|
4
|
+
* `road-to-governed-harness-evolution` Phase 5, step 5.6.
|
|
5
|
+
*
|
|
6
|
+
* > *5.6 Cheap proposer models first, and track evolution ROI. […] its own
|
|
7
|
+
* > cross-critique faults both parents as cost-blind and answers with a hard
|
|
8
|
+
* > budget cap, while dropping the only cost-*reduction* mechanism both parents
|
|
9
|
+
* > proposed. Improvement per evolution dollar is a reported figure.*
|
|
10
|
+
* > verify: **the ROI figure appears in every run report, and a cheaper model
|
|
11
|
+
* > is tried before an expensive one on each defect class.**
|
|
12
|
+
*
|
|
13
|
+
* ## Why the ROI row is a refusal and not a field somebody remembers
|
|
14
|
+
*
|
|
15
|
+
* A budget cap answers "how much may this cost". It says nothing about whether
|
|
16
|
+
* the spend bought anything, so a programme can spend its whole ceiling on
|
|
17
|
+
* candidates that improved nothing and read as compliant the entire way. The
|
|
18
|
+
* figure that would have caught that is improvement per dollar, and a figure
|
|
19
|
+
* that a report MAY omit is a figure that goes missing on exactly the run whose
|
|
20
|
+
* ratio is embarrassing.
|
|
21
|
+
*
|
|
22
|
+
* So {@link buildRunReport} REFUSES a report without the ROI figure, in the
|
|
23
|
+
* same spirit as `_lib/evaluation_vector.ts`'s {@link buildVector} refusing a
|
|
24
|
+
* vector that omits its artifact-count row: the sprawl measurement belongs
|
|
25
|
+
* inside the gate, and the cost measurement belongs inside the report.
|
|
26
|
+
* {@link RunReport} carries no optional `roi` field, and because types vanish
|
|
27
|
+
* at runtime the refusal is checked against an object cast past the compiler.
|
|
28
|
+
*
|
|
29
|
+
* ## Three ROI kinds, because a ratio is not always defined
|
|
30
|
+
*
|
|
31
|
+
* `improved_rows / dollars` is `Infinity` at zero spend and `NaN` at zero of
|
|
32
|
+
* both. Printing either would be a number where the honest answer is "there is
|
|
33
|
+
* no ratio here". So the figure is a union: `ratio` when spend is positive and
|
|
34
|
+
* at least one candidate was evaluated, `no-spend` when the run cost nothing
|
|
35
|
+
* (the counts are still real and still printed), and `unmeasured` when no
|
|
36
|
+
* candidate carried an evaluation at all.
|
|
37
|
+
*
|
|
38
|
+
* `unmeasured` is the honest state of this programme today and is meant to be
|
|
39
|
+
* read as a finding rather than as a placeholder: `run` clones candidates, and
|
|
40
|
+
* nothing in Phase 5 evaluates them, because step 5.2 keeps the live-floors
|
|
41
|
+
* park intact and there is no live harness to evaluate against. A report that
|
|
42
|
+
* says `unmeasured` is telling the truth about a run that measured nothing.
|
|
43
|
+
*
|
|
44
|
+
* ## The ladder is an ORDERING POLICY, and its subject is HALF live
|
|
45
|
+
*
|
|
46
|
+
* "A cheaper model is tried before an expensive one" needs an attempt sequence
|
|
47
|
+
* to police. **CORRECTED 2026-09-01 — one exists now.** This section used to
|
|
48
|
+
* say there was none and that {@link assertCheapestFirst} stood "over a
|
|
49
|
+
* mechanism that does not yet exist"; both became false when the metered arm
|
|
50
|
+
* landed. Two production callers in `_lib/llm_candidate_proposer.ts` produce
|
|
51
|
+
* {@link LadderAttempt}s: `proposeCandidatesWithModel` calls the guard at
|
|
52
|
+
* `:417` over the attempts a walk actually made, and `plannedAttempts` at
|
|
53
|
+
* `:446` over the dry-run plan. Both are reached from `llm_propose.ts` — `:212`
|
|
54
|
+
* on `--confirm`, `:137` on the dry path, which spends nothing.
|
|
55
|
+
*
|
|
56
|
+
* What has NOT happened is a LIVE run. Every population produced so far is
|
|
57
|
+
* either the dry plan — one `lite` rung per observation on `reason_unknown`, in
|
|
58
|
+
* which no ordering decision arises at all — or a test population under a
|
|
59
|
+
* stubbed generator. So the guard stands over a real, non-empty population and
|
|
60
|
+
* has still never governed a SPENT one, which is what
|
|
61
|
+
* `road-to-governed-evidence-production` AC-3 stays open on.
|
|
62
|
+
*
|
|
63
|
+
* Both halves are stated deliberately. A guard described as if it were policing
|
|
64
|
+
* real traffic is the coverage inflation this tree's own records name
|
|
65
|
+
* repeatedly — and a guard still described as policing nothing, after it has
|
|
66
|
+
* callers, is that same inflation inverted.
|
|
67
|
+
*
|
|
68
|
+
* The live-floors park is untouched by any of this: it forbids a live
|
|
69
|
+
* **routing** harness (5.2), `tests/scripts/governed_harness_no_live_harness.test.ts`
|
|
70
|
+
* still holds that, and a metered proposer is not one.
|
|
71
|
+
*
|
|
72
|
+
* What IS live is the plan: {@link buildRunReport} puts the per-class ladder
|
|
73
|
+
* into every report, so the cheapest untried tier for each defect class is on
|
|
74
|
+
* the operator's screen before the first metered call is licensed.
|
|
75
|
+
*
|
|
76
|
+
* ## Tiers are vendor-neutral, and three classes carry an EMPTY ladder
|
|
77
|
+
*
|
|
78
|
+
* `lite | medium | high` is a capability band, not a vendor. Naming a model
|
|
79
|
+
* would tie a cost policy to a price list that changes without this file.
|
|
80
|
+
*
|
|
81
|
+
* `policy_blocked`, `dependency_unavailable` and `human_rejected` get an empty
|
|
82
|
+
* ladder, and that is the strongest cost reduction available rather than an
|
|
83
|
+
* omission: a candidate that a policy refused, that could not find its
|
|
84
|
+
* dependency, or that a human turned down is not made acceptable by a larger
|
|
85
|
+
* model, so no tier is licensed for it at all. {@link nextTier} returns `null`
|
|
86
|
+
* there, which a caller must read as "spend nothing", never as "start at the
|
|
87
|
+
* top".
|
|
88
|
+
*/
|
|
89
|
+
import type { PairedVerdict, PairedVerdictKind } from './paired_verdict.js';
|
|
90
|
+
import { PATHOLOGY_WHY, type PathologyWhy } from './pathology_archive.js';
|
|
91
|
+
import {
|
|
92
|
+
ARTIFACT_COUNT_METRIC,
|
|
93
|
+
type MetricRow,
|
|
94
|
+
type MetricVector,
|
|
95
|
+
buildVector,
|
|
96
|
+
} from './evaluation_vector.js';
|
|
97
|
+
|
|
98
|
+
/** The defect-class axis. REUSED from the pathology archive, never re-invented. */
|
|
99
|
+
export type DefectClass = PathologyWhy;
|
|
100
|
+
export const DEFECT_CLASSES = PATHOLOGY_WHY;
|
|
101
|
+
|
|
102
|
+
/** Capability bands, CHEAPEST FIRST. The order is the policy. */
|
|
103
|
+
export const MODEL_TIERS = ['lite', 'medium', 'high'] as const;
|
|
104
|
+
export type ModelTier = (typeof MODEL_TIERS)[number];
|
|
105
|
+
|
|
106
|
+
/** Position in {@link MODEL_TIERS}. Lower is cheaper. */
|
|
107
|
+
export function tierRank(tier: ModelTier): number {
|
|
108
|
+
return MODEL_TIERS.indexOf(tier);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* The ladder per defect class, cheapest first.
|
|
113
|
+
*
|
|
114
|
+
* Every list is a prefix of {@link MODEL_TIERS} or empty — a ladder that skipped
|
|
115
|
+
* a rung would be an expensive tier tried before a cheaper one by construction,
|
|
116
|
+
* which is the thing the step forbids. {@link assertLadderWellFormed} checks it.
|
|
117
|
+
*/
|
|
118
|
+
export const LADDER: Readonly<Record<DefectClass, readonly ModelTier[]>> = {
|
|
119
|
+
// A missing precondition is usually a prompt/context defect: cheap first,
|
|
120
|
+
// and there is a real ceiling because a large model still cannot invent a
|
|
121
|
+
// precondition that is absent.
|
|
122
|
+
precondition_unsatisfied: ['lite', 'medium'],
|
|
123
|
+
// A policy refusal is not a capability problem.
|
|
124
|
+
policy_blocked: [],
|
|
125
|
+
// Neither is a missing dependency.
|
|
126
|
+
dependency_unavailable: [],
|
|
127
|
+
// Execution failures are where capability plausibly helps, so the full ladder.
|
|
128
|
+
execution_failed: ['lite', 'medium', 'high'],
|
|
129
|
+
// A violated output contract is the classic cheap-model-first case: most are
|
|
130
|
+
// format, and format is the cheapest thing a small model gets right.
|
|
131
|
+
output_contract_violated: ['lite', 'medium'],
|
|
132
|
+
// Missing evidence needs more trials, not a bigger proposer.
|
|
133
|
+
evidence_missing: ['lite'],
|
|
134
|
+
// A human said no. No tier changes that.
|
|
135
|
+
human_rejected: [],
|
|
136
|
+
// Unclassified: one cheap attempt is licensed, and no more, because
|
|
137
|
+
// escalating on a reason nobody established is spending on a guess.
|
|
138
|
+
reason_unknown: ['lite'],
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
export class LadderOrderError extends Error {
|
|
142
|
+
constructor(message: string) {
|
|
143
|
+
super(message);
|
|
144
|
+
this.name = 'LadderOrderError';
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Every ladder is a prefix of {@link MODEL_TIERS}, or the empty ladder. */
|
|
149
|
+
export function assertLadderWellFormed(
|
|
150
|
+
ladder: Readonly<Record<DefectClass, readonly ModelTier[]>> = LADDER,
|
|
151
|
+
): void {
|
|
152
|
+
for (const cls of DEFECT_CLASSES) {
|
|
153
|
+
const rungs = ladder[cls];
|
|
154
|
+
for (let i = 0; i < rungs.length; i += 1) {
|
|
155
|
+
const expected = MODEL_TIERS[i];
|
|
156
|
+
if (rungs[i] !== expected) {
|
|
157
|
+
throw new LadderOrderError(
|
|
158
|
+
`${cls}: rung ${String(i)} is '${String(rungs[i])}' where the cheapest-first ` +
|
|
159
|
+
`order requires '${String(expected)}' — a ladder that skips a rung tries ` +
|
|
160
|
+
'an expensive tier before a cheaper one by construction',
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** The ladder for one class. Empty means no metered attempt is licensed. */
|
|
168
|
+
export function ladderFor(cls: DefectClass): readonly ModelTier[] {
|
|
169
|
+
return LADDER[cls];
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* The cheapest tier not yet tried for this class, or `null`.
|
|
174
|
+
*
|
|
175
|
+
* `null` means STOP, in both of its two causes: the ladder is empty for this
|
|
176
|
+
* class, or every licensed rung has been spent. A caller that reads `null` as
|
|
177
|
+
* "escalate" has inverted the policy.
|
|
178
|
+
*/
|
|
179
|
+
export function nextTier(cls: DefectClass, tried: readonly ModelTier[]): ModelTier | null {
|
|
180
|
+
const spent = new Set(tried);
|
|
181
|
+
for (const rung of ladderFor(cls)) {
|
|
182
|
+
if (!spent.has(rung)) return rung;
|
|
183
|
+
}
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** One metered attempt, as it would be recorded by a harness that made one. */
|
|
188
|
+
export interface LadderAttempt {
|
|
189
|
+
readonly defect_class: DefectClass;
|
|
190
|
+
readonly tier: ModelTier;
|
|
191
|
+
/** Monotonic within one class. The ONLY ordering key. */
|
|
192
|
+
readonly sequence: number;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Refuse an attempt sequence in which a costlier tier ran before a cheaper one
|
|
197
|
+
* for the same defect class, or in which a tier outside the class's ladder ran
|
|
198
|
+
* at all.
|
|
199
|
+
*
|
|
200
|
+
* Per class, in sequence order: each attempt's tier must be the cheapest rung
|
|
201
|
+
* not already spent. A repeat of an already-spent rung is allowed — retrying
|
|
202
|
+
* `lite` is not an escalation — but a jump is not.
|
|
203
|
+
*
|
|
204
|
+
* SUBJECT EXISTS; A SPENT POPULATION DOES NOT (corrected 2026-09-01). This
|
|
205
|
+
* paragraph used to read "NO LIVE SUBJECT. Nothing in this programme produces
|
|
206
|
+
* {@link LadderAttempt}s today", and that stopped being true when the metered
|
|
207
|
+
* arm landed: `_lib/llm_candidate_proposer.ts` produces them from two
|
|
208
|
+
* production callers — `proposeCandidatesWithModel` (`:417`) and
|
|
209
|
+
* `plannedAttempts` (`:446`), both reachable from `llm_propose.ts`. No LIVE run
|
|
210
|
+
* has produced attempts yet, so the ordering this polices has not governed a
|
|
211
|
+
* spent population; see the header, and AC-3 of
|
|
212
|
+
* `road-to-governed-evidence-production`. The unit test still proves the guard
|
|
213
|
+
* FIRES rather than proving anything ran.
|
|
214
|
+
*/
|
|
215
|
+
export function assertCheapestFirst(attempts: readonly LadderAttempt[]): void {
|
|
216
|
+
const byClass = new Map<DefectClass, LadderAttempt[]>();
|
|
217
|
+
for (const a of attempts) {
|
|
218
|
+
const list = byClass.get(a.defect_class) ?? [];
|
|
219
|
+
list.push(a);
|
|
220
|
+
byClass.set(a.defect_class, list);
|
|
221
|
+
}
|
|
222
|
+
for (const [cls, list] of byClass) {
|
|
223
|
+
const rungs = ladderFor(cls);
|
|
224
|
+
if (rungs.length === 0) {
|
|
225
|
+
const first = [...list].sort((x, y) => x.sequence - y.sequence)[0] as LadderAttempt;
|
|
226
|
+
throw new LadderOrderError(
|
|
227
|
+
`${cls}: tier '${first.tier}' was tried on a class whose ladder is empty — ` +
|
|
228
|
+
'no metered attempt is licensed here, so the cheapest thing to try is nothing',
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
const spent: ModelTier[] = [];
|
|
232
|
+
for (const a of [...list].sort((x, y) => x.sequence - y.sequence)) {
|
|
233
|
+
if (!rungs.includes(a.tier)) {
|
|
234
|
+
throw new LadderOrderError(
|
|
235
|
+
`${cls}: tier '${a.tier}' is not on this class's ladder ` +
|
|
236
|
+
`(${rungs.join(' < ') || 'empty'})`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
if (spent.includes(a.tier)) continue;
|
|
240
|
+
const cheapest = nextTier(cls, spent);
|
|
241
|
+
if (cheapest !== null && a.tier !== cheapest) {
|
|
242
|
+
throw new LadderOrderError(
|
|
243
|
+
`${cls}: tier '${a.tier}' was tried at sequence ${String(a.sequence)} while ` +
|
|
244
|
+
`'${cheapest}' was still untried — cheaper models go first`,
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
spent.push(a.tier);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** One class's ladder as it goes into a report. */
|
|
253
|
+
export interface LadderPlanRow {
|
|
254
|
+
readonly defect_class: DefectClass;
|
|
255
|
+
readonly ladder: readonly ModelTier[];
|
|
256
|
+
/** The cheapest untried rung given `tried`, or `null` for "spend nothing". */
|
|
257
|
+
readonly next: ModelTier | null;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** The whole ladder plan, one row per defect class, in vocabulary order. */
|
|
261
|
+
export function ladderPlan(
|
|
262
|
+
tried: Readonly<Partial<Record<DefectClass, readonly ModelTier[]>>> = {},
|
|
263
|
+
): LadderPlanRow[] {
|
|
264
|
+
return DEFECT_CLASSES.map((cls) => ({
|
|
265
|
+
defect_class: cls,
|
|
266
|
+
ladder: ladderFor(cls),
|
|
267
|
+
next: nextTier(cls, tried[cls] ?? []),
|
|
268
|
+
}));
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// --- the ROI figure ---------------------------------------------------------
|
|
272
|
+
|
|
273
|
+
/** Row counts read off the evaluated vectors. Never inferred. */
|
|
274
|
+
export interface RoiCounts {
|
|
275
|
+
readonly evaluated_candidates: number;
|
|
276
|
+
readonly improved_rows: number;
|
|
277
|
+
readonly regressed_rows: number;
|
|
278
|
+
readonly underpowered_rows: number;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Improvement per evolution dollar.
|
|
283
|
+
*
|
|
284
|
+
* `regressed_rows` and `underpowered_rows` ride alongside the ratio rather than
|
|
285
|
+
* being netted into it. Netting would let a candidate that improved two rows
|
|
286
|
+
* and regressed two report as neutral, which is the collapse
|
|
287
|
+
* `_lib/evaluation_vector.ts` refuses for the same reason.
|
|
288
|
+
*/
|
|
289
|
+
export type RoiFigure =
|
|
290
|
+
| ({ readonly kind: 'ratio'; readonly spend_cents: number; readonly improvement_per_dollar: number } & RoiCounts)
|
|
291
|
+
| ({ readonly kind: 'no-spend'; readonly spend_cents: 0 } & RoiCounts)
|
|
292
|
+
| ({ readonly kind: 'unmeasured'; readonly spend_cents: number } & RoiCounts);
|
|
293
|
+
|
|
294
|
+
/** Count the paired rows by verdict kind across the evaluated vectors. */
|
|
295
|
+
export function roiCounts(vectors: readonly MetricVector[]): RoiCounts {
|
|
296
|
+
let improved = 0;
|
|
297
|
+
let regressed = 0;
|
|
298
|
+
let underpowered = 0;
|
|
299
|
+
for (const v of vectors) {
|
|
300
|
+
for (const r of v.rows) {
|
|
301
|
+
if (r.kind !== 'paired') continue;
|
|
302
|
+
if (r.verdict.kind === 'pass') improved += 1;
|
|
303
|
+
else if (r.verdict.kind === 'regression') regressed += 1;
|
|
304
|
+
else if (r.verdict.kind === 'underpowered') underpowered += 1;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return {
|
|
308
|
+
evaluated_candidates: vectors.length,
|
|
309
|
+
improved_rows: improved,
|
|
310
|
+
regressed_rows: regressed,
|
|
311
|
+
underpowered_rows: underpowered,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export class RoiShapeError extends Error {
|
|
316
|
+
constructor(message: string) {
|
|
317
|
+
super(message);
|
|
318
|
+
this.name = 'RoiShapeError';
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* The figure. `spendCents` is the run's declared spend in WHOLE CENTS, matching
|
|
324
|
+
* `RunBudget.maxSpendCents` — no float ever decides a kind.
|
|
325
|
+
*/
|
|
326
|
+
export function roiFigure(vectors: readonly MetricVector[], spendCents: number): RoiFigure {
|
|
327
|
+
if (!Number.isInteger(spendCents) || spendCents < 0) {
|
|
328
|
+
throw new RoiShapeError(
|
|
329
|
+
`spend must be a non-negative whole number of cents (got ${String(spendCents)})`,
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
const counts = roiCounts(vectors);
|
|
333
|
+
if (counts.evaluated_candidates === 0) {
|
|
334
|
+
return { kind: 'unmeasured', spend_cents: spendCents, ...counts };
|
|
335
|
+
}
|
|
336
|
+
if (spendCents === 0) {
|
|
337
|
+
return { kind: 'no-spend', spend_cents: 0, ...counts };
|
|
338
|
+
}
|
|
339
|
+
return {
|
|
340
|
+
kind: 'ratio',
|
|
341
|
+
spend_cents: spendCents,
|
|
342
|
+
improvement_per_dollar: counts.improved_rows / (spendCents / 100),
|
|
343
|
+
...counts,
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// --- the run report ---------------------------------------------------------
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* A run report. There is no optional field here, and `roi` is the reason:
|
|
351
|
+
* an optional ROI figure is one a report omits on the run that needed it.
|
|
352
|
+
*/
|
|
353
|
+
export interface RunReport {
|
|
354
|
+
readonly run_id: string;
|
|
355
|
+
readonly candidates: number;
|
|
356
|
+
readonly trials_per_candidate: number;
|
|
357
|
+
readonly roi: RoiFigure;
|
|
358
|
+
readonly ladder: readonly LadderPlanRow[];
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
export interface RunReportInput {
|
|
362
|
+
readonly run_id: string;
|
|
363
|
+
readonly candidates: number;
|
|
364
|
+
readonly trials_per_candidate: number;
|
|
365
|
+
readonly roi: RoiFigure;
|
|
366
|
+
readonly ladder?: readonly LadderPlanRow[];
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
export class RunReportShapeError extends Error {
|
|
370
|
+
constructor(message: string) {
|
|
371
|
+
super(message);
|
|
372
|
+
this.name = 'RunReportShapeError';
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Kinds a `roi` value may carry. Checked at runtime, where types are gone. */
|
|
377
|
+
const ROI_KINDS = new Set(['ratio', 'no-spend', 'unmeasured']);
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Build a report, REFUSING one without an ROI figure.
|
|
381
|
+
*
|
|
382
|
+
* The check is deliberately a runtime one over a value the compiler already
|
|
383
|
+
* types: a caller reaching this function through `as RunReportInput`, a JSON
|
|
384
|
+
* parse, or a `delete` is exactly the caller who would drop the row, and the
|
|
385
|
+
* type system cannot see any of them.
|
|
386
|
+
*/
|
|
387
|
+
export function buildRunReport(input: RunReportInput): RunReport {
|
|
388
|
+
if (input.run_id.trim() === '') {
|
|
389
|
+
throw new RunReportShapeError('a report with no run id names no run');
|
|
390
|
+
}
|
|
391
|
+
const roi = (input as { roi?: unknown }).roi;
|
|
392
|
+
if (roi === undefined || roi === null || typeof roi !== 'object') {
|
|
393
|
+
throw new RunReportShapeError(
|
|
394
|
+
'missing the ROI figure — improvement per evolution dollar is what a budget cap ' +
|
|
395
|
+
'does not report, so a run report without it is the cost-blind report step 5.6 ' +
|
|
396
|
+
'exists to prevent',
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
const kind = (roi as { kind?: unknown }).kind;
|
|
400
|
+
if (typeof kind !== 'string' || !ROI_KINDS.has(kind)) {
|
|
401
|
+
throw new RunReportShapeError(
|
|
402
|
+
`ROI figure carries an unknown kind ${JSON.stringify(kind)} — expected one of ` +
|
|
403
|
+
`${[...ROI_KINDS].join(', ')}`,
|
|
404
|
+
);
|
|
405
|
+
}
|
|
406
|
+
assertLadderWellFormed();
|
|
407
|
+
return {
|
|
408
|
+
run_id: input.run_id,
|
|
409
|
+
candidates: input.candidates,
|
|
410
|
+
trials_per_candidate: input.trials_per_candidate,
|
|
411
|
+
roi: input.roi,
|
|
412
|
+
ladder: input.ladder ?? ladderPlan(),
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/** The ROI line, verbatim. One line, always present, never conditional. */
|
|
417
|
+
export function renderRoi(roi: RoiFigure): string {
|
|
418
|
+
const head =
|
|
419
|
+
`roi: improved=${String(roi.improved_rows)} regressed=${String(roi.regressed_rows)} ` +
|
|
420
|
+
`underpowered=${String(roi.underpowered_rows)} over ` +
|
|
421
|
+
`${String(roi.evaluated_candidates)} evaluated candidate(s), spend=${String(roi.spend_cents)}c`;
|
|
422
|
+
if (roi.kind === 'ratio') {
|
|
423
|
+
return `${head} -> ${roi.improvement_per_dollar.toFixed(3)} improved rows per dollar`;
|
|
424
|
+
}
|
|
425
|
+
if (roi.kind === 'no-spend') {
|
|
426
|
+
return `${head} -> no ratio: the run spent nothing, so improvement per dollar is undefined`;
|
|
427
|
+
}
|
|
428
|
+
return `${head} -> no ratio: no candidate in this run carried an evaluation`;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** The whole report as lines a caller writes to stdout unchanged. */
|
|
432
|
+
export function renderRunReport(r: RunReport): string[] {
|
|
433
|
+
const lines = [
|
|
434
|
+
`run-report: ${r.run_id}`,
|
|
435
|
+
`run-report: candidates=${String(r.candidates)} trials-per-candidate=${String(r.trials_per_candidate)}`,
|
|
436
|
+
`run-report: ${renderRoi(r.roi)}`,
|
|
437
|
+
];
|
|
438
|
+
for (const row of r.ladder) {
|
|
439
|
+
const rungs = row.ladder.length === 0 ? '(none licensed)' : row.ladder.join(' < ');
|
|
440
|
+
const next = row.next === null ? 'spend nothing' : row.next;
|
|
441
|
+
lines.push(`run-report: ladder ${row.defect_class}: ${rungs} | next: ${next}`);
|
|
442
|
+
}
|
|
443
|
+
return lines;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// --- reading an evaluation off disk -----------------------------------------
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* Parse a `MetricVector` from JSON, reusing {@link buildVector} so the
|
|
450
|
+
* artifact-count refusal is inherited rather than re-implemented.
|
|
451
|
+
*/
|
|
452
|
+
export function parseMetricVectorJson(text: string, source: string): MetricVector {
|
|
453
|
+
let raw: unknown;
|
|
454
|
+
try {
|
|
455
|
+
raw = JSON.parse(text);
|
|
456
|
+
} catch (e) {
|
|
457
|
+
throw new RoiShapeError(`${source}: not JSON — ${(e as Error).message}`);
|
|
458
|
+
}
|
|
459
|
+
// An array IS an object in JS, so the array check is not pedantry: without
|
|
460
|
+
// it a `[]` payload falls through to the field checks and reports a missing
|
|
461
|
+
// `candidate_id`, which names the wrong defect.
|
|
462
|
+
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
|
|
463
|
+
throw new RoiShapeError(`${source}: expected a JSON object`);
|
|
464
|
+
}
|
|
465
|
+
const obj = raw as { candidate_id?: unknown; rows?: unknown };
|
|
466
|
+
if (typeof obj.candidate_id !== 'string' || obj.candidate_id.trim() === '') {
|
|
467
|
+
throw new RoiShapeError(`${source}: 'candidate_id' must be a non-empty string`);
|
|
468
|
+
}
|
|
469
|
+
if (!Array.isArray(obj.rows)) {
|
|
470
|
+
throw new RoiShapeError(`${source}: 'rows' must be an array`);
|
|
471
|
+
}
|
|
472
|
+
const rows: MetricRow[] = [];
|
|
473
|
+
for (const [i, r] of (obj.rows as unknown[]).entries()) {
|
|
474
|
+
rows.push(parseRow(r, `${source}: rows[${String(i)}]`));
|
|
475
|
+
}
|
|
476
|
+
return buildVector(obj.candidate_id, rows);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const DIRECTIONS = new Set(['higher-better', 'lower-better']);
|
|
480
|
+
const VERDICT_KINDS = new Set(['pass', 'no-change', 'regression', 'underpowered']);
|
|
481
|
+
|
|
482
|
+
function parseRow(raw: unknown, where: string): MetricRow {
|
|
483
|
+
if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
|
|
484
|
+
throw new RoiShapeError(`${where}: expected an object`);
|
|
485
|
+
}
|
|
486
|
+
const r = raw as Record<string, unknown>;
|
|
487
|
+
const metric = r.metric;
|
|
488
|
+
const direction = r.direction;
|
|
489
|
+
if (typeof metric !== 'string' || metric.trim() === '') {
|
|
490
|
+
throw new RoiShapeError(`${where}: 'metric' must be a non-empty string`);
|
|
491
|
+
}
|
|
492
|
+
if (typeof direction !== 'string' || !DIRECTIONS.has(direction)) {
|
|
493
|
+
throw new RoiShapeError(`${where}: 'direction' must be higher-better or lower-better`);
|
|
494
|
+
}
|
|
495
|
+
if (r.kind === 'counted') {
|
|
496
|
+
if (!Number.isInteger(r.delta)) {
|
|
497
|
+
throw new RoiShapeError(`${where}: a counted row needs an integer 'delta'`);
|
|
498
|
+
}
|
|
499
|
+
return {
|
|
500
|
+
kind: 'counted',
|
|
501
|
+
metric,
|
|
502
|
+
direction: direction as 'higher-better' | 'lower-better',
|
|
503
|
+
delta: r.delta as number,
|
|
504
|
+
};
|
|
505
|
+
}
|
|
506
|
+
if (r.kind === 'paired') {
|
|
507
|
+
const v = r.verdict;
|
|
508
|
+
if (typeof v !== 'object' || v === null) {
|
|
509
|
+
throw new RoiShapeError(`${where}: a paired row needs a 'verdict' object`);
|
|
510
|
+
}
|
|
511
|
+
const raw_v = v as Record<string, unknown>;
|
|
512
|
+
const kind = raw_v.kind;
|
|
513
|
+
if (typeof kind !== 'string' || !VERDICT_KINDS.has(kind)) {
|
|
514
|
+
throw new RoiShapeError(
|
|
515
|
+
`${where}: verdict kind ${JSON.stringify(kind)} is not one of ` +
|
|
516
|
+
`${[...VERDICT_KINDS].join(', ')}`,
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
// Every counted field is validated rather than spread through: a
|
|
520
|
+
// verdict read off disk is untrusted input, and a string where an
|
|
521
|
+
// integer belongs is how a report ends up printing `NaN` per dollar.
|
|
522
|
+
const verdict: PairedVerdict = {
|
|
523
|
+
kind: kind as PairedVerdictKind,
|
|
524
|
+
discordant: intField(raw_v, 'discordant', where),
|
|
525
|
+
wins: intField(raw_v, 'wins', where),
|
|
526
|
+
losses: intField(raw_v, 'losses', where),
|
|
527
|
+
p: numField(raw_v, 'p', where),
|
|
528
|
+
magnitude_mean:
|
|
529
|
+
raw_v.magnitude_mean === null || raw_v.magnitude_mean === undefined
|
|
530
|
+
? null
|
|
531
|
+
: numField(raw_v, 'magnitude_mean', where),
|
|
532
|
+
at_floor: raw_v.at_floor === true,
|
|
533
|
+
reason: typeof raw_v.reason === 'string' ? raw_v.reason : '',
|
|
534
|
+
};
|
|
535
|
+
return {
|
|
536
|
+
kind: 'paired',
|
|
537
|
+
metric,
|
|
538
|
+
direction: direction as 'higher-better' | 'lower-better',
|
|
539
|
+
verdict,
|
|
540
|
+
};
|
|
541
|
+
}
|
|
542
|
+
throw new RoiShapeError(
|
|
543
|
+
`${where}: 'kind' must be 'paired' or 'counted' (got ${JSON.stringify(r.kind)}) — ` +
|
|
544
|
+
`a metric row without a kind cannot be counted toward ${ARTIFACT_COUNT_METRIC} or ROI`,
|
|
545
|
+
);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function intField(o: Record<string, unknown>, key: string, where: string): number {
|
|
549
|
+
const v = o[key];
|
|
550
|
+
if (!Number.isInteger(v)) {
|
|
551
|
+
throw new RoiShapeError(`${where}: verdict '${key}' must be an integer`);
|
|
552
|
+
}
|
|
553
|
+
return v as number;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function numField(o: Record<string, unknown>, key: string, where: string): number {
|
|
557
|
+
const v = o[key];
|
|
558
|
+
if (typeof v !== 'number' || !Number.isFinite(v)) {
|
|
559
|
+
throw new RoiShapeError(`${where}: verdict '${key}' must be a finite number`);
|
|
560
|
+
}
|
|
561
|
+
return v;
|
|
562
|
+
}
|
|
@@ -60,6 +60,14 @@ export const ALLOWLIST_PREFIXES: readonly (readonly string[])[] = [
|
|
|
60
60
|
// report exists. Read-only, deterministic, model-free, and already run in
|
|
61
61
|
// CI (`rule-backstops.yml`) for its own reasons.
|
|
62
62
|
['check_enforcement_denominator'],
|
|
63
|
+
// road-to-source-silence Phase 5.3. The source-confidentiality gate's exit
|
|
64
|
+
// code IS the verdict for `claim:plaintext-source-attribution`: 0 means zero
|
|
65
|
+
// deny-pattern matches in tracked content AND in every tracked path, plus the
|
|
66
|
+
// attribution-shape block count at or below its ratchet baseline. An
|
|
67
|
+
// existence-check pointer would have gone stale the first time somebody added
|
|
68
|
+
// a name. Read-only, deterministic, model-free, and already run in CI for its
|
|
69
|
+
// own reasons — the same class as its neighbours here.
|
|
70
|
+
['check_no_external_sources'],
|
|
63
71
|
['check_token_regression'],
|
|
64
72
|
['lint_agent_security'],
|
|
65
73
|
['measure_lexical_ranking'],
|