@event4u/agent-config 9.20.0 → 9.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +86 -0
- package/dist/agent-src/commands/worktree/cleanup.md +17 -0
- package/dist/agent-src/commands/worktree/status.md +6 -0
- package/dist/agent-src/skills/worktree-lifecycle/SKILL.md +22 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/cli-delegate/{chunk-XGA2NTU4.js → chunk-3EWYC3YK.js} +40 -0
- package/dist/cli-delegate/cmd_handoff.js +1 -1
- package/dist/cli-delegate/cmd_memory_get.js +1 -1
- package/dist/cli-delegate/cmd_route_audit.js +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +5 -5
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +58 -2
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +47 -29
- package/package.json +1 -1
- package/src/config/gate-coverage.yml +22 -4
- package/src/config/gate-violation-baselines.json +3 -8
- package/src/scripts/_lib/bench_ab_activation.ts +368 -0
- package/src/scripts/_lib/gate_population.ts +70 -0
- package/src/scripts/_lib/roadmap_granularity.ts +157 -0
- package/src/scripts/_lib/scan_scope.ts +44 -0
- package/src/scripts/_lib/stdin.ts +82 -0
- package/src/scripts/audit_adr_coverage.ts +38 -0
- package/src/scripts/audit_auto_rules.ts +17 -3
- package/src/scripts/audit_command_surface.ts +51 -0
- package/src/scripts/audit_initial_context.ts +39 -3
- package/src/scripts/audit_mcp_tools.ts +25 -0
- package/src/scripts/audit_overlap.ts +24 -1
- package/src/scripts/audit_skill_descriptions.ts +19 -0
- package/src/scripts/{verify_before_complete_hook.ts → before_complete_hook.ts} +16 -3
- package/src/scripts/bench_ab_task_runner.ts +18 -0
- package/src/scripts/bench_ab_v2_run.ts +147 -6
- package/src/scripts/bench_ab_v2_stats.ts +74 -2
- package/src/scripts/check_adr_frontmatter.ts +23 -3
- package/src/scripts/check_always_budget.ts +24 -0
- package/src/scripts/check_artefact_checksums.ts +24 -1
- package/src/scripts/check_artefact_count_messaging.ts +20 -0
- package/src/scripts/check_augmentignore.ts +22 -15
- package/src/scripts/check_backstop_debt.ts +22 -0
- package/src/scripts/check_beta_review_markers.ts +22 -1
- package/src/scripts/check_bridge_derivation.ts +22 -1
- package/src/scripts/check_bundle_path_leakage.ts +26 -0
- package/src/scripts/check_ci_strict_superset.ts +21 -3
- package/src/scripts/check_claims.ts +22 -0
- package/src/scripts/check_cli_registry_budget_sync.ts +33 -4
- package/src/scripts/check_cluster_patterns.ts +26 -4
- package/src/scripts/check_command_count_messaging.ts +20 -4
- package/src/scripts/check_comparison.ts +18 -0
- package/src/scripts/check_condensed_paths.ts +23 -2
- package/src/scripts/check_consumer_scope_flip.ts +21 -2
- package/src/scripts/check_council_config_location.ts +20 -0
- package/src/scripts/check_council_layout.ts +22 -2
- package/src/scripts/check_council_references.ts +23 -1
- package/src/scripts/check_dependency_floors.ts +19 -3
- package/src/scripts/check_discovery_determinism.ts +26 -0
- package/src/scripts/check_enforcement_coverage.ts +20 -0
- package/src/scripts/check_evaluator_budgets.ts +23 -0
- package/src/scripts/check_finding_dispositions.ts +29 -0
- package/src/scripts/check_gate_coverage.ts +63 -22
- package/src/scripts/check_gate_paths.ts +30 -5
- package/src/scripts/check_generated_artefact_headers.ts +21 -0
- package/src/scripts/check_generator_output_coverage.ts +23 -0
- package/src/scripts/check_gitignore_freshness.ts +23 -0
- package/src/scripts/check_host_loadability.ts +34 -8
- package/src/scripts/check_installer_import_purity.ts +22 -8
- package/src/scripts/check_kernel_prefix_stability.ts +19 -1
- package/src/scripts/check_kernel_rule_bundle.ts +23 -0
- package/src/scripts/check_knowledge_cards.ts +13 -0
- package/src/scripts/check_knowledge_pages.ts +23 -0
- package/src/scripts/check_knowledge_sharing.ts +27 -0
- package/src/scripts/check_md_language.ts +24 -0
- package/src/scripts/check_media_deps.ts +23 -0
- package/src/scripts/check_memory.ts +26 -1
- package/src/scripts/check_memory_contradiction.ts +16 -1
- package/src/scripts/check_memory_proposal.ts +15 -0
- package/src/scripts/check_memory_similarity.ts +16 -0
- package/src/scripts/check_module_management_neutral.ts +19 -11
- package/src/scripts/check_no_conflict_markers.ts +22 -1
- package/src/scripts/check_no_external_sources.ts +26 -1
- package/src/scripts/check_no_local_settings_committed.ts +37 -10
- package/src/scripts/check_no_new_legacy_path.ts +30 -1
- package/src/scripts/check_one_off_location.ts +43 -0
- package/src/scripts/check_originality_freshness.ts +25 -1
- package/src/scripts/check_overlay_cascade_subdirs.ts +21 -0
- package/src/scripts/check_pack_size.ts +22 -0
- package/src/scripts/check_preamble_payload_budget.ts +35 -5
- package/src/scripts/check_proposal.ts +24 -1
- package/src/scripts/check_public_catalog_links.ts +27 -0
- package/src/scripts/check_public_links.ts +19 -0
- package/src/scripts/check_quality_regression.ts +19 -0
- package/src/scripts/check_reach_channels.ts +44 -0
- package/src/scripts/check_reach_staleness.ts +27 -0
- package/src/scripts/check_release_adjacent_health.ts +28 -0
- package/src/scripts/check_release_highlights.ts +34 -0
- package/src/scripts/check_release_includes_discovery.ts +25 -6
- package/src/scripts/check_release_pr_shape.ts +18 -1
- package/src/scripts/check_release_published.ts +21 -0
- package/src/scripts/check_release_surface_equality.ts +22 -0
- package/src/scripts/check_release_trunk_sync.ts +19 -0
- package/src/scripts/check_reply_consistency.ts +22 -0
- package/src/scripts/check_roadmap_trackable.ts +20 -0
- package/src/scripts/check_role_doc_links.ts +23 -0
- package/src/scripts/check_rule_invariants.ts +14 -0
- package/src/scripts/check_secret_leak.ts +47 -0
- package/src/scripts/check_skill_gaps.ts +32 -0
- package/src/scripts/check_skill_requires.ts +19 -0
- package/src/scripts/check_source_pointer_freshness.ts +20 -0
- package/src/scripts/check_structural_breaking.ts +29 -2
- package/src/scripts/check_surface_tiers.ts +34 -0
- package/src/scripts/check_template_pin_drift.ts +23 -1
- package/src/scripts/check_test_coverage_diff.ts +17 -1
- package/src/scripts/check_token_optimizer_freshness.ts +18 -3
- package/src/scripts/check_token_quality_golden.ts +21 -0
- package/src/scripts/check_token_regression.ts +25 -0
- package/src/scripts/check_tracked_but_ignored.ts +26 -0
- package/src/scripts/check_trigger_eval_presence.ts +23 -1
- package/src/scripts/check_trigger_evals.ts +36 -0
- package/src/scripts/check_trunk_drift.ts +23 -0
- package/src/scripts/check_update_banner.ts +24 -0
- package/src/scripts/gate_scope_classify.ts +513 -0
- package/src/scripts/hook_manifest.yaml +1 -1
- package/src/scripts/hooks/block_kernel_rule_writes.ts +1 -1
- package/src/scripts/hooks/concern_registry.ts +2 -2
- package/src/scripts/lint_agent_security.ts +19 -0
- package/src/scripts/lint_agent_skill_names.ts +30 -0
- package/src/scripts/lint_agents_layout.ts +20 -0
- package/src/scripts/lint_agents_md.ts +20 -0
- package/src/scripts/lint_archived_skills.ts +24 -3
- package/src/scripts/lint_behavioural_eval_freshness.ts +29 -3
- package/src/scripts/lint_bench_ab.ts +25 -0
- package/src/scripts/lint_bench_corpus.ts +20 -4
- package/src/scripts/lint_breaking_changes_index.ts +24 -0
- package/src/scripts/lint_budget_ownership.ts +20 -3
- package/src/scripts/lint_changelog_rollback.ts +24 -0
- package/src/scripts/lint_command_flow_coverage.ts +22 -0
- package/src/scripts/lint_command_tiers.ts +29 -5
- package/src/scripts/lint_commit_subjects.ts +15 -0
- package/src/scripts/lint_confusables.ts +24 -0
- package/src/scripts/lint_consumer_internal_refs.ts +24 -0
- package/src/scripts/lint_design_quality.ts +22 -0
- package/src/scripts/lint_design_slop.ts +23 -0
- package/src/scripts/lint_discovery_manifest.ts +23 -0
- package/src/scripts/lint_discovery_vocabulary.ts +19 -0
- package/src/scripts/lint_documented_commands.ts +23 -0
- package/src/scripts/lint_empty_roadmaps.ts +23 -0
- package/src/scripts/lint_eval_fixture_citations.ts +19 -0
- package/src/scripts/lint_eval_freshness.ts +35 -0
- package/src/scripts/lint_examples.ts +18 -3
- package/src/scripts/lint_explain_trace.ts +23 -0
- package/src/scripts/lint_featured_skills.ts +26 -1
- package/src/scripts/lint_flows.ts +19 -0
- package/src/scripts/lint_frontmatter_boilerplate.ts +20 -0
- package/src/scripts/lint_ghostwriter_source.ts +26 -6
- package/src/scripts/lint_glama_drift.ts +24 -0
- package/src/scripts/lint_global_paths.ts +24 -0
- package/src/scripts/lint_governed_writes.ts +20 -0
- package/src/scripts/lint_hidden_unicode.ts +25 -0
- package/src/scripts/lint_hook_concern_budget.ts +39 -6
- package/src/scripts/lint_hook_manifest.ts +22 -0
- package/src/scripts/lint_instruction_smuggling.ts +23 -0
- package/src/scripts/lint_knowledge_scale.ts +32 -2
- package/src/scripts/lint_legal_pack.ts +31 -0
- package/src/scripts/lint_marketplace.ts +26 -0
- package/src/scripts/lint_marketplace_install_completeness.ts +23 -0
- package/src/scripts/lint_mcp_config_security.ts +24 -0
- package/src/scripts/lint_mcp_registry_manifest.ts +22 -1
- package/src/scripts/lint_memory_tripwire.ts +35 -0
- package/src/scripts/lint_missions.ts +23 -7
- package/src/scripts/lint_model_tier_coverage.ts +18 -0
- package/src/scripts/lint_namespace_collisions.ts +19 -0
- package/src/scripts/lint_no_new_atomic_commands.ts +33 -0
- package/src/scripts/lint_no_python_twin_rationale.ts +18 -0
- package/src/scripts/lint_one_off_age.ts +38 -2
- package/src/scripts/lint_orchestration_dsl.ts +23 -3
- package/src/scripts/lint_orchestrator_auto_detect.ts +19 -12
- package/src/scripts/lint_originality.ts +22 -4
- package/src/scripts/lint_originality_shingles.ts +28 -3
- package/src/scripts/lint_output_slop.ts +19 -0
- package/src/scripts/lint_override_kernel_guard.ts +30 -0
- package/src/scripts/lint_pack_first_win.ts +23 -0
- package/src/scripts/lint_pack_risk_class.ts +19 -3
- package/src/scripts/lint_persistence.ts +19 -0
- package/src/scripts/lint_persona_governance.ts +18 -3
- package/src/scripts/lint_positioning.ts +23 -0
- package/src/scripts/lint_pre_migration_refs.ts +24 -0
- package/src/scripts/lint_profile_overlay_set_only.ts +24 -0
- package/src/scripts/lint_provenance.ts +21 -0
- package/src/scripts/lint_provenance_vocabulary.ts +24 -1
- package/src/scripts/lint_readme_jargon.ts +21 -3
- package/src/scripts/lint_readme_serial_comma.ts +21 -3
- package/src/scripts/lint_readme_size.ts +21 -3
- package/src/scripts/lint_regression.ts +28 -0
- package/src/scripts/lint_roadmap_blockers.ts +43 -0
- package/src/scripts/lint_roadmap_ci_steps.ts +45 -0
- package/src/scripts/lint_roadmap_complexity.ts +47 -0
- package/src/scripts/lint_roadmap_later_disposition.ts +23 -0
- package/src/scripts/lint_rule_interactions.ts +16 -2
- package/src/scripts/lint_rule_tiers.ts +19 -6
- package/src/scripts/lint_showcase_sessions.ts +19 -3
- package/src/scripts/lint_skill_descriptions.ts +19 -0
- package/src/scripts/lint_skill_frontmatter_safety.ts +24 -0
- package/src/scripts/lint_skill_originality.ts +18 -3
- package/src/scripts/lint_skill_scripts_readonly.ts +20 -0
- package/src/scripts/lint_skill_tools.ts +84 -145
- package/src/scripts/lint_spawn_payload.ts +27 -1
- package/src/scripts/lint_store_boundary.ts +19 -0
- package/src/scripts/lint_subagent_determinism.ts +23 -1
- package/src/scripts/lint_supported_tools_matrix.ts +18 -0
- package/src/scripts/lint_surface_matrix.ts +19 -3
- package/src/scripts/lint_ticket_buildable.ts +23 -0
- package/src/scripts/lint_topics_yaml.ts +19 -12
- package/src/scripts/lint_trigger_precision.ts +27 -0
- package/src/scripts/lint_trust_coherence.ts +21 -0
- package/src/scripts/lint_ui_stack_bundles.ts +17 -3
- package/src/scripts/lint_value_dashboard.ts +19 -3
- package/src/scripts/lint_versioned_cache.ts +19 -1
- package/src/scripts/lint_workflow_security.ts +22 -14
- package/src/scripts/pr_url_reminder_hook.ts +1 -1
- package/src/scripts/skill_collision_clusters.ts +37 -3
- package/src/scripts/skill_discovery.ts +20 -0
- package/src/scripts/skill_eval_coverage.ts +22 -0
- package/src/scripts/skill_overlap.ts +32 -6
- package/src/scripts/skill_preview.ts +31 -0
- package/src/scripts/skill_trigger_eval.ts +18 -0
- package/src/scripts/skill_usage_collect.ts +21 -0
- package/src/scripts/skill_usage_report.ts +21 -0
- package/src/scripts/sweep_dead_scan_roots.ts +11 -3
- package/src/scripts/verify_physical_move.ts +24 -0
- package/src/scripts/worktree_cleanup_check.ts +319 -4
- package/src/scripts/check_bite_sized_granularity.ts +0 -116
- package/src/scripts/lint_workspace_boundary.ts +0 -267
|
@@ -72,10 +72,15 @@
|
|
|
72
72
|
#
|
|
73
73
|
# What remains is NOT one line per gate. For the ~201 the count would have to be
|
|
74
74
|
# INVENTED (what is this gate's unit?) rather than published, and that judgement
|
|
75
|
-
# is per-gate. It
|
|
76
|
-
# unhardened-scan-scope` in src/config/gate-violation-baselines.json, which
|
|
77
|
-
# on a RISE and
|
|
78
|
-
#
|
|
75
|
+
# is per-gate. It was tracked as a vulnerability ratchet — `gate-hardening:
|
|
76
|
+
# unhardened-scan-scope` in src/config/gate-violation-baselines.json, which
|
|
77
|
+
# failed on a RISE and targeted 0 — under the road-to-gate-hardening-adoption
|
|
78
|
+
# charter. That charter closed 2026-08-05: the count reached 0 and the baseline
|
|
79
|
+
# entry was DELETED rather than zeroed, so `checkRatchet` now returns
|
|
80
|
+
# `unbaselined` with `ok = (actual === 0)`. Zero is mandatory, not a floor — any
|
|
81
|
+
# new gate written without a scope assertion reds immediately, and
|
|
82
|
+
# `tests/scripts/check_gate_coverage.test.ts` pins both the empty population and
|
|
83
|
+
# the entry's continued absence.
|
|
79
84
|
#
|
|
80
85
|
# `tests/scripts/check_gate_coverage.test.ts` fails the build the moment a gate
|
|
81
86
|
# starts emitting the line without being registered here, so the list cannot
|
|
@@ -441,3 +446,16 @@ gates:
|
|
|
441
446
|
note: >
|
|
442
447
|
Emits on stderr, not stdout: `--format json` owns this gate's stdout and a
|
|
443
448
|
count written there would corrupt the payload. The guard reads both streams.
|
|
449
|
+
|
|
450
|
+
- id: check_cli_registry_budget_sync
|
|
451
|
+
argv: []
|
|
452
|
+
min_scanned: 3
|
|
453
|
+
corpus: "the three named inputs it cross-checks — src/cli/registry.ts, src/config/evaluator-budgets.json, internal/bench/reports/evaluator-measurements.json"
|
|
454
|
+
status: enforced
|
|
455
|
+
note: >
|
|
456
|
+
Registered because its count became REAL, not to pad the manifest. The
|
|
457
|
+
gate always printed a hardcoded `scanned: 3`; hardening derived it from
|
|
458
|
+
the watch list that actually resolved, which is what makes a floor of 3
|
|
459
|
+
meaningful — it now falls when one of the three inputs is renamed away.
|
|
460
|
+
`argv: []` is the CI-identical invocation: both `taskfiles/ci-fast.yml`
|
|
461
|
+
and `.github/workflows/tests.yml` call it bare.
|
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$comment": "Violation ratchet for gates whose dead scan root was repaired. A repair REVEALS pre-existing debt; it does not create it. Each entry records the count revealed at repair time — the gate fails only when its count RISES. Lowering a count is a normal commit; raising one is a defect. Entries expire: a baseline that has not dropped within 56 days of `landed` (or of `reaffirmed.date`) fails its gate, because a number that never moves is suppression with extra steps. Mechanism + rationale: src/scripts/_lib/gate_baseline.ts. Decision: AI council 2026-08-02, roadmap road-to-gates-that-can-fail blocker `dead-gate-finding-triage`.",
|
|
3
3
|
"gates": {
|
|
4
|
-
"gate-hardening:unhardened-scan-scope": {
|
|
5
|
-
"count": 189,
|
|
6
|
-
"landed": "2026-08-04",
|
|
7
|
-
"note": "Gate scripts under src/scripts (population 223, the lint_/check_/audit_ prefix set) that neither route through _lib/scan_scope nor emit the machine-readable 'scanned:' line — the gates that can still report success over a moved or empty root without saying so. This is a VULNERABILITY count, not a coverage percentage: it measures how much of the original defect class is still reachable, it RISES whenever a new gate is written without a scope assertion, and its target is 0. That direction is the whole point — an adoption-reach metric ('hardened gates must increase') can never regress and would be threshold-lowering wearing a ratchet's clothes (AI council 2026-08-04, anthropic/claude-sonnet-4-5 + openai/gpt-4o, convergent on scope decomposition plus this discriminator). Clearing it is the charter of road-to-gate-hardening-adoption; the 56-day non-stagnation clause applies unchanged, so a number that never drops fails this gate rather than hardening into configuration. NEW ENTRY, not a raised threshold: this records newly-measured pre-existing exposure at first measurement — nothing was loosened."
|
|
8
|
-
},
|
|
9
4
|
"check_no_new_legacy_path:hardcoded-scan-roots": {
|
|
10
|
-
"count":
|
|
11
|
-
"landed": "2026-08-
|
|
12
|
-
"note": "Lines under src/scripts/**/*.{ts,mts,mjs} that pass the retired `.agent-src.uncondensed` directory into `path.join(...)` — a filesystem path CONSTRUCTED against the tree ADR-051 abandoned, which is the exact shape of all 14 dead gates. Deliberately narrow so the number is signal, not noise: 236 raw mentions under src/scripts → 147 string-literal lines → 67 path-constructing lines when the pass was written, after exempting src/scripts/_lib/agent_src.ts (the shared resolver, whose job IS owning the constant — gates that route through it survived the move). Detectors holding the literal as data to match, strip, or print are not counted, and no prose, docs, tests or fixtures are read: that full-tree shape was measured at 44 files / 213 hits and rejected as the wrong lint (see the note in check_no_new_legacy_path.ts::find_offenders). Recorded at 58 rather than 67 because Phase 1's scan-root repairs were landing on this branch while the pass was being written and had already removed 9 — a baseline above the truth silently permits new violations. This count goes to ZERO as the remaining repairs land; every repair is a lowering commit. Lowered 56→51 by the road-to-renewal-foundation Phase 1 dead-root sweep (check_always_budget, check_portability, check_references, lint_framework_leakage, lint_agents_md)."
|
|
5
|
+
"count": 49,
|
|
6
|
+
"landed": "2026-08-04",
|
|
7
|
+
"note": "Lines under src/scripts/**/*.{ts,mts,mjs} that pass the retired `.agent-src.uncondensed` directory into `path.join(...)` — a filesystem path CONSTRUCTED against the tree ADR-051 abandoned, which is the exact shape of all 14 dead gates. Deliberately narrow so the number is signal, not noise: 236 raw mentions under src/scripts → 147 string-literal lines → 67 path-constructing lines when the pass was written, after exempting src/scripts/_lib/agent_src.ts (the shared resolver, whose job IS owning the constant — gates that route through it survived the move). Detectors holding the literal as data to match, strip, or print are not counted, and no prose, docs, tests or fixtures are read: that full-tree shape was measured at 44 files / 213 hits and rejected as the wrong lint (see the note in check_no_new_legacy_path.ts::find_offenders). Recorded at 58 rather than 67 because Phase 1's scan-root repairs were landing on this branch while the pass was being written and had already removed 9 — a baseline above the truth silently permits new violations. This count goes to ZERO as the remaining repairs land; every repair is a lowering commit. Lowered 56→51 by the road-to-renewal-foundation Phase 1 dead-root sweep (check_always_budget, check_portability, check_references, lint_framework_leakage, lint_agents_md). Lowered 51→49 by the gate-hardening adoption PR: skill_collision_clusters and skill_overlap were repointed at the live skills tree via the shared resolver — two gates that had been reading the retired container and scoring 0 of 288 skills."
|
|
13
8
|
},
|
|
14
9
|
"audit_user_type_axis": {
|
|
15
10
|
"count": 1,
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Measurement-integrity primitives for the bench:ab v2 paired sweep.
|
|
3
|
+
*
|
|
4
|
+
* Implements deltas #1–#4 of the S0.3 harness-feasibility spike
|
|
5
|
+
* (`agents/evidence/investigations/solution-minimalism-phase0-spikes.md`
|
|
6
|
+
* § "Deltas required before a paid run") — the four cheap ones the roadmap's
|
|
7
|
+
* Phase-3 halt note says "land before any spend".
|
|
8
|
+
*
|
|
9
|
+
* Why this exists at all: the sweep has ALREADY produced a full set of invalid
|
|
10
|
+
* nulls once. Clones lived inside the repo, so the `vanilla` arm inherited the
|
|
11
|
+
* package through project scope and every prior null was void
|
|
12
|
+
* (`bench_ab_v2_run.ts` § arm-isolation note). The fix at the time was a `/tmp`
|
|
13
|
+
* path constant with no runtime assertion behind it, and the only activation
|
|
14
|
+
* field on a trial record — `injected_chars` — is the `String.length` of a file
|
|
15
|
+
* the harness itself wrote. It proves nothing about the model, and it is `0` by
|
|
16
|
+
* construction for the `package` arm, which arrives through global settings.
|
|
17
|
+
* If the plugin were disabled or version-drifted mid-sweep, every treatment run
|
|
18
|
+
* would silently degrade to `vanilla` and the report would look identical.
|
|
19
|
+
*
|
|
20
|
+
* Everything here is pure and host-free so it can be unit-tested without a
|
|
21
|
+
* model call. The sweep wires it in `bench_ab_v2_run.ts`.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** Four-bucket usage split as reported by the Claude CLI JSON envelope. */
|
|
25
|
+
export interface TokensBreakdown {
|
|
26
|
+
input_tokens: number;
|
|
27
|
+
output_tokens: number;
|
|
28
|
+
cache_read_input_tokens: number;
|
|
29
|
+
cache_creation_input_tokens: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Per-1M-token rates for one pricing tier (mirrors `_lib/bench_cost.TierRates`). */
|
|
33
|
+
export interface TierRates {
|
|
34
|
+
input: number;
|
|
35
|
+
output: number;
|
|
36
|
+
cache_write: number;
|
|
37
|
+
cache_read: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* How an arm is supposed to reach the model.
|
|
42
|
+
*
|
|
43
|
+
* - `plugin` — the package rides global settings (`setting_sources: null`).
|
|
44
|
+
* Invisible to `injected_chars`; only the prompt footprint can see it.
|
|
45
|
+
* - `text` — an `--append-system-prompt-file` injection the harness wrote.
|
|
46
|
+
* - `none` — the baseline: settings scoped away AND no text injection.
|
|
47
|
+
*/
|
|
48
|
+
export type ExpectedInjection = 'plugin' | 'text' | 'none';
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* `unknown` is deliberately not a violation: an errored or budget-capped run
|
|
52
|
+
* carries a zeroed usage block, so its footprint says nothing. Attrition is
|
|
53
|
+
* reported separately (delta #5) rather than laundered into an integrity fail.
|
|
54
|
+
*/
|
|
55
|
+
export type ActivationVerdict = 'ok' | 'violation' | 'unknown';
|
|
56
|
+
|
|
57
|
+
export interface Activation {
|
|
58
|
+
expected: ExpectedInjection;
|
|
59
|
+
prompt_tokens: number;
|
|
60
|
+
injected_chars: number;
|
|
61
|
+
verdict: ActivationVerdict;
|
|
62
|
+
reason: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface ActivationViolation {
|
|
66
|
+
task_id: string;
|
|
67
|
+
seed: number;
|
|
68
|
+
arm: string;
|
|
69
|
+
kind: 'text-injection-missing' | 'text-injection-unexpected' | 'collapsed-to-baseline' | 'model-mismatch';
|
|
70
|
+
detail: string;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface ActivationAudit {
|
|
74
|
+
checked: number;
|
|
75
|
+
skipped: number;
|
|
76
|
+
violations: ActivationViolation[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Minimum prompt-footprint ratio a plugin-bearing arm must hold over the paired
|
|
81
|
+
* baseline run before the pair is believed.
|
|
82
|
+
*
|
|
83
|
+
* Calibrated DOWNWARD from the three stored sweeps in S0.3's cost sheet, whose
|
|
84
|
+
* observed treatment/vanilla total-token ratios were 4.68× (sonnet, `package`),
|
|
85
|
+
* 1.71× (haiku, `rules-kernel-dc`) and 6.79× (haiku, `package`). 1.2 sits below
|
|
86
|
+
* the smallest of those by a wide margin on purpose: this flags a COLLAPSE to
|
|
87
|
+
* baseline — a disabled or drifted plugin — never ordinary run-to-run variance.
|
|
88
|
+
* Raising it toward the observed values would start failing legitimate runs;
|
|
89
|
+
* lowering it toward 1.0 would stop catching the failure it exists for.
|
|
90
|
+
*/
|
|
91
|
+
export const ACTIVATION_MIN_LIFT_RATIO = 1.2;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The footprint an injection actually moves: everything the model was fed.
|
|
95
|
+
*
|
|
96
|
+
* Output tokens are excluded — they measure how much the model wrote, which a
|
|
97
|
+
* bigger system prompt does not reliably change, so including them would blur
|
|
98
|
+
* the very signal this audit reads.
|
|
99
|
+
*/
|
|
100
|
+
export function prompt_tokens(b: TokensBreakdown | Record<string, never> | null | undefined): number {
|
|
101
|
+
if (!b) {
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
104
|
+
const t = b as Partial<TokensBreakdown>;
|
|
105
|
+
return _int(t.input_tokens) + _int(t.cache_read_input_tokens) + _int(t.cache_creation_input_tokens);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Classify an arm spec into the injection channel it is supposed to use. */
|
|
109
|
+
export function expected_injection(spec: { setting_sources: string | null; inject: string | null }): ExpectedInjection {
|
|
110
|
+
if (spec.inject) {
|
|
111
|
+
return 'text';
|
|
112
|
+
}
|
|
113
|
+
return spec.setting_sources === null ? 'plugin' : 'none';
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Per-trial verdict — the direction that IS decidable from one run.
|
|
118
|
+
*
|
|
119
|
+
* A text injection is checked both ways: an arm that declares one must carry
|
|
120
|
+
* it, and an arm that declares none must not. The plugin direction needs the
|
|
121
|
+
* paired baseline and is handled by `audit_activation`.
|
|
122
|
+
*/
|
|
123
|
+
export function activation_verdict(args: {
|
|
124
|
+
expected: ExpectedInjection;
|
|
125
|
+
tokens_breakdown: TokensBreakdown | Record<string, never> | null | undefined;
|
|
126
|
+
injected_chars: number;
|
|
127
|
+
errored: boolean;
|
|
128
|
+
}): Activation {
|
|
129
|
+
const pt = prompt_tokens(args.tokens_breakdown);
|
|
130
|
+
const base = { expected: args.expected, prompt_tokens: pt, injected_chars: args.injected_chars };
|
|
131
|
+
if (args.errored || pt === 0) {
|
|
132
|
+
return { ...base, verdict: 'unknown', reason: 'errored or zero usage — footprint unreadable' };
|
|
133
|
+
}
|
|
134
|
+
if (args.expected === 'text' && args.injected_chars <= 0) {
|
|
135
|
+
return { ...base, verdict: 'violation', reason: 'text arm carried no injection' };
|
|
136
|
+
}
|
|
137
|
+
if (args.expected !== 'text' && args.injected_chars > 0) {
|
|
138
|
+
return {
|
|
139
|
+
...base,
|
|
140
|
+
verdict: 'violation',
|
|
141
|
+
reason: `${args.expected} arm carried a ${args.injected_chars}-char injection`,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
return { ...base, verdict: 'ok', reason: 'expected channel present' };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Cross-arm audit over a finished (or partial) record set — the plugin
|
|
149
|
+
* direction plus every per-trial verdict already stamped on the records.
|
|
150
|
+
*
|
|
151
|
+
* A pair is only judged when BOTH sides are readable; unreadable pairs are
|
|
152
|
+
* counted as `skipped`, never silently passed.
|
|
153
|
+
*/
|
|
154
|
+
export function audit_activation(
|
|
155
|
+
records: readonly Record<string, unknown>[],
|
|
156
|
+
opts: { baseline_arm: string; lift_arms: readonly string[]; min_ratio?: number },
|
|
157
|
+
): ActivationAudit {
|
|
158
|
+
const minRatio = opts.min_ratio ?? ACTIVATION_MIN_LIFT_RATIO;
|
|
159
|
+
const violations: ActivationViolation[] = [];
|
|
160
|
+
let checked = 0;
|
|
161
|
+
let skipped = 0;
|
|
162
|
+
|
|
163
|
+
for (const rec of records) {
|
|
164
|
+
const taskId = String(rec['id'] ?? '');
|
|
165
|
+
const arms = _dict(rec['arms']);
|
|
166
|
+
|
|
167
|
+
for (const [arm, runsRaw] of Object.entries(arms)) {
|
|
168
|
+
for (const run of _list(runsRaw)) {
|
|
169
|
+
const act = _dict(run['activation']);
|
|
170
|
+
if (act['verdict'] === 'violation') {
|
|
171
|
+
const reason = String(act['reason'] ?? '');
|
|
172
|
+
violations.push({
|
|
173
|
+
task_id: taskId,
|
|
174
|
+
seed: _int(run['seed']),
|
|
175
|
+
arm,
|
|
176
|
+
kind: reason.includes('no injection') ? 'text-injection-missing' : 'text-injection-unexpected',
|
|
177
|
+
detail: reason,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const mc = _dict(run['model_check']);
|
|
181
|
+
if ('ok' in mc && !_truthy(mc['ok'])) {
|
|
182
|
+
violations.push({
|
|
183
|
+
task_id: taskId,
|
|
184
|
+
seed: _int(run['seed']),
|
|
185
|
+
arm,
|
|
186
|
+
kind: 'model-mismatch',
|
|
187
|
+
detail: String(mc['reason'] ?? ''),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const baseBySeed = new Map<number, number>();
|
|
194
|
+
for (const run of _list(arms[opts.baseline_arm])) {
|
|
195
|
+
const pt = _runPromptTokens(run);
|
|
196
|
+
if (!_truthy(run['errored']) && pt > 0) {
|
|
197
|
+
baseBySeed.set(_int(run['seed']), pt);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
for (const arm of opts.lift_arms) {
|
|
202
|
+
for (const run of _list(arms[arm])) {
|
|
203
|
+
const seed = _int(run['seed']);
|
|
204
|
+
const basePt = baseBySeed.get(seed);
|
|
205
|
+
const armPt = _runPromptTokens(run);
|
|
206
|
+
if (_truthy(run['errored']) || basePt === undefined || armPt === 0) {
|
|
207
|
+
skipped += 1;
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
checked += 1;
|
|
211
|
+
if (armPt < basePt * minRatio) {
|
|
212
|
+
violations.push({
|
|
213
|
+
task_id: taskId,
|
|
214
|
+
seed,
|
|
215
|
+
arm,
|
|
216
|
+
kind: 'collapsed-to-baseline',
|
|
217
|
+
detail:
|
|
218
|
+
`prompt footprint ${armPt} vs baseline ${basePt} ` +
|
|
219
|
+
`(ratio ${(armPt / basePt).toFixed(2)} < ${minRatio}) — ` +
|
|
220
|
+
'the treatment surface may not have reached the model',
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return { checked, skipped, violations };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ── delta #3 — model-id verification ────────────────────────────────────────
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Aliases the Claude CLI resolves server-side to whatever is current.
|
|
234
|
+
*
|
|
235
|
+
* A stored report already contains the bare alias `"sonnet"`, which makes that
|
|
236
|
+
* report unreproducible: nothing in it records which model actually answered.
|
|
237
|
+
*/
|
|
238
|
+
export const BARE_MODEL_ALIASES: readonly string[] = ['default', 'haiku', 'opus', 'sonnet', 'sonnet[1m]'];
|
|
239
|
+
|
|
240
|
+
/** True when `model` is an alias rather than a pinned, reproducible id. */
|
|
241
|
+
export function is_bare_alias(model: string): boolean {
|
|
242
|
+
return BARE_MODEL_ALIASES.includes(model.trim().toLowerCase());
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** Drop a trailing `-YYYYMMDD` build stamp so `claude-x-1` == `claude-x-1-20250101`. */
|
|
246
|
+
export function normalize_model_id(model: string): string {
|
|
247
|
+
return model.trim().toLowerCase().replace(/-\d{8}$/, '');
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Compare the requested model against the ids the CLI actually billed.
|
|
252
|
+
*
|
|
253
|
+
* `models_seen` comes from the envelope's `modelUsage` keys — the only place
|
|
254
|
+
* the run states which model answered. An empty list is `ok`: some envelopes
|
|
255
|
+
* omit the block, and refusing there would fail runs for a reporting gap
|
|
256
|
+
* rather than a real mismatch.
|
|
257
|
+
*/
|
|
258
|
+
export function verify_model_id(requested: string, models_seen: readonly string[]): { ok: boolean; reason: string } {
|
|
259
|
+
if (is_bare_alias(requested)) {
|
|
260
|
+
return { ok: false, reason: `requested model "${requested}" is a bare alias — pin a full model id` };
|
|
261
|
+
}
|
|
262
|
+
if (models_seen.length === 0) {
|
|
263
|
+
return { ok: true, reason: 'envelope reported no model usage — nothing to contradict' };
|
|
264
|
+
}
|
|
265
|
+
const want = normalize_model_id(requested);
|
|
266
|
+
const seen = models_seen.map(normalize_model_id);
|
|
267
|
+
if (seen.every((s) => s === want)) {
|
|
268
|
+
return { ok: true, reason: `billed model matches ${requested}` };
|
|
269
|
+
}
|
|
270
|
+
return {
|
|
271
|
+
ok: false,
|
|
272
|
+
reason: `requested ${requested} but envelope billed ${[...new Set(models_seen)].sort().join(', ')}`,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ── delta #4 — sweep-level cost ─────────────────────────────────────────────
|
|
277
|
+
|
|
278
|
+
/** Map a full model id onto a `pricing.yaml` tier, or null when unrecognised. */
|
|
279
|
+
export function tier_for_model(model: string): string | null {
|
|
280
|
+
const m = model.toLowerCase();
|
|
281
|
+
for (const tier of ['haiku', 'sonnet', 'opus']) {
|
|
282
|
+
if (m.includes(tier)) {
|
|
283
|
+
return tier;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Price one run's four buckets separately.
|
|
291
|
+
*
|
|
292
|
+
* The buckets differ in price by up to 125×, so a single blended rate over the
|
|
293
|
+
* summed total is not an approximation — it is a different number.
|
|
294
|
+
*/
|
|
295
|
+
export function cost_usd(b: TokensBreakdown | Record<string, never> | null | undefined, rates: TierRates): number {
|
|
296
|
+
if (!b) {
|
|
297
|
+
return 0;
|
|
298
|
+
}
|
|
299
|
+
const t = b as Partial<TokensBreakdown>;
|
|
300
|
+
return (
|
|
301
|
+
(_int(t.input_tokens) * rates.input +
|
|
302
|
+
_int(t.output_tokens) * rates.output +
|
|
303
|
+
_int(t.cache_creation_input_tokens) * rates.cache_write +
|
|
304
|
+
_int(t.cache_read_input_tokens) * rates.cache_read) /
|
|
305
|
+
1_000_000
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Cumulative sweep spend with a hard cap.
|
|
311
|
+
*
|
|
312
|
+
* `--max-budget-usd` is per RUN (default 1.0), so a 30×4×3 sweep at `--budget
|
|
313
|
+
* 3.5` has a $1,260 ceiling with nothing to stop it. This is the sweep-level
|
|
314
|
+
* counterpart: the first run that pushes cumulative spend over the cap aborts
|
|
315
|
+
* the sweep instead of discovering the overrun on the invoice.
|
|
316
|
+
*/
|
|
317
|
+
export class SweepBudget {
|
|
318
|
+
private _spent = 0;
|
|
319
|
+
|
|
320
|
+
constructor(
|
|
321
|
+
readonly cap_usd: number | null,
|
|
322
|
+
private readonly rates: TierRates | null,
|
|
323
|
+
) {}
|
|
324
|
+
|
|
325
|
+
get spent_usd(): number {
|
|
326
|
+
return this._spent;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/** Add one run's cost; return an abort reason when the cap is crossed. */
|
|
330
|
+
add(b: TokensBreakdown | Record<string, never> | null | undefined): string | null {
|
|
331
|
+
if (!this.rates) {
|
|
332
|
+
return null;
|
|
333
|
+
}
|
|
334
|
+
this._spent += cost_usd(b, this.rates);
|
|
335
|
+
if (this.cap_usd !== null && this._spent > this.cap_usd) {
|
|
336
|
+
return `sweep budget abort: cumulative $${this._spent.toFixed(2)} > cap $${this.cap_usd.toFixed(2)}`;
|
|
337
|
+
}
|
|
338
|
+
return null;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// ── internals ───────────────────────────────────────────────────────────────
|
|
343
|
+
|
|
344
|
+
function _int(v: unknown): number {
|
|
345
|
+
return typeof v === 'number' && Number.isFinite(v) ? Math.trunc(v) : 0;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function _dict(v: unknown): Record<string, unknown> {
|
|
349
|
+
return v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function _list(v: unknown): Record<string, unknown>[] {
|
|
353
|
+
return Array.isArray(v) ? (v as Record<string, unknown>[]) : [];
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function _truthy(v: unknown): boolean {
|
|
357
|
+
return Boolean(v);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Prompt footprint of one trial record — the stamped `activation` block when
|
|
362
|
+
* the sweep wrote one, else recomputed from the preserved breakdown (delta #2),
|
|
363
|
+
* so the audit also reads reports written before the stamp existed.
|
|
364
|
+
*/
|
|
365
|
+
function _runPromptTokens(run: Record<string, unknown>): number {
|
|
366
|
+
const stamped = _int(_dict(run['activation'])['prompt_tokens']);
|
|
367
|
+
return stamped > 0 ? stamped : prompt_tokens(run['tokens_breakdown'] as TokensBreakdown | undefined);
|
|
368
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* "What is a gate?" — asserted once, read by everything that needs the answer.
|
|
3
|
+
*
|
|
4
|
+
* Three call sites each carried their own regex and each got a different number
|
|
5
|
+
* (measured 2026-08-04):
|
|
6
|
+
*
|
|
7
|
+
* | Site | Pattern | Population |
|
|
8
|
+
* |---|---|---:|
|
|
9
|
+
* | `check_gate_coverage` (the ratchet) | `(lint\|check\|audit)_` minus `.d.ts` | 223 |
|
|
10
|
+
* | `sweep_dead_scan_roots` | `(lint\|audit\|check\|verify)_` | 225 |
|
|
11
|
+
* | the manifest-registration test | `(lint\|check\|audit\|skill)_` | 232 |
|
|
12
|
+
*
|
|
13
|
+
* Harmless while the numbers were only reported. Load-bearing the moment the
|
|
14
|
+
* ratchet read one of them, which it now does — and it read the narrowest:
|
|
15
|
+
*
|
|
16
|
+
* - it counted `check_secret_leak.test.ts`, a **test file**, which can never be
|
|
17
|
+
* hardened, so its target of 0 was unreachable by construction;
|
|
18
|
+
* - it missed 10 real gates the other two definitions caught — 8 × `skill_*`
|
|
19
|
+
* (including `skill_linter`, which already carries an enforced floor of 380 in
|
|
20
|
+
* the coverage manifest, so the manifest and the ratchet openly disagreed
|
|
21
|
+
* about whether it is a gate) and 2 × `verify_*`.
|
|
22
|
+
*
|
|
23
|
+
* ## Why a prefix filter and not manifest membership
|
|
24
|
+
*
|
|
25
|
+
* AI council 2026-08-04 (anthropic/claude-sonnet-4-5 + openai/gpt-4o, 3 rounds):
|
|
26
|
+
* making manifest registration the classifier inverts a dependency that runs the
|
|
27
|
+
* other way in practice. A new gate script lands BEFORE anyone registers it —
|
|
28
|
+
* that is precisely how the registration test found the drift — so a
|
|
29
|
+
* manifest-driven population would ignore every gate during the window when it
|
|
30
|
+
* is least proven. Classification is structural (this filter); registration is
|
|
31
|
+
* operational and layered on top (`enforced_manifest_ids`).
|
|
32
|
+
*
|
|
33
|
+
* The prefix set is a proxy for "this script certifies something and can lie
|
|
34
|
+
* about it", and an imperfect one: `skill_usage_report` reports rather than
|
|
35
|
+
* validates. It is kept because it is mechanical and over-inclusive in the safe
|
|
36
|
+
* direction — a reporting script that asserts its scan scope loses nothing,
|
|
37
|
+
* while a validator missing from the population is invisible exposure.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
/** Prefixes that mark a script as gate-shaped. */
|
|
41
|
+
const GATE_PREFIX = /^(lint|check|audit|skill|verify)_/;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Is this filename a gate script?
|
|
45
|
+
*
|
|
46
|
+
* Structural only — takes a bare filename, never touches disk, so it answers
|
|
47
|
+
* for a file that does not exist yet (a diff, a plan, a test fixture).
|
|
48
|
+
*/
|
|
49
|
+
export function matchesGatePattern(filename: string): boolean {
|
|
50
|
+
if (!filename.endsWith('.ts')) return false;
|
|
51
|
+
// A type declaration has no runtime behaviour to harden; a test file cannot
|
|
52
|
+
// be hardened at all, and counting one made the ratchet's own target
|
|
53
|
+
// unreachable.
|
|
54
|
+
if (filename.endsWith('.d.ts') || filename.endsWith('.test.ts')) return false;
|
|
55
|
+
return GATE_PREFIX.test(filename);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The gate scripts in `dir`, as bare ids (no `.ts`), sorted. Missing dir → `[]`. */
|
|
59
|
+
export function listGateScripts(dir: string, readdir: (d: string) => string[]): string[] {
|
|
60
|
+
let entries: string[];
|
|
61
|
+
try {
|
|
62
|
+
entries = readdir(dir);
|
|
63
|
+
} catch {
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
return entries
|
|
67
|
+
.filter(matchesGatePattern)
|
|
68
|
+
.map((f) => f.replace(/\.ts$/, ''))
|
|
69
|
+
.sort();
|
|
70
|
+
}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bite-sized task-granularity check for structural roadmaps.
|
|
3
|
+
*
|
|
4
|
+
* A `complexity: structural` roadmap's task bullets must name real work. A
|
|
5
|
+
* placeholder in a task bullet — `<something>`, TODO, FIXME, XXX, TBD, `???` —
|
|
6
|
+
* means the step was filed before it was decided, which is the failure the
|
|
7
|
+
* bite-sized standard exists to catch.
|
|
8
|
+
*
|
|
9
|
+
* ## Why this lives in `_lib/` and not in a gate of its own
|
|
10
|
+
*
|
|
11
|
+
* It used to be `src/scripts/check_bite_sized_granularity.ts`: a pure library
|
|
12
|
+
* with no CLI, no `main`, no exit code, and — measured — no production caller
|
|
13
|
+
* outside its own test. It was carried over from a retired Python module for
|
|
14
|
+
* API parity and never wired up, so a real check that no other gate covers ran
|
|
15
|
+
* nowhere, while the `check_` prefix kept it inside the gate population where
|
|
16
|
+
* the scan-scope ratchet counted it as a gate that could not assert a scan
|
|
17
|
+
* scope.
|
|
18
|
+
*
|
|
19
|
+
* AI council 2026-08-05 resolved it by folding rather than deleting: the checks
|
|
20
|
+
* are the only placeholder coverage in the tree (`lint_roadmap_complexity`
|
|
21
|
+
* covers the complexity tag, the lightweight caps, and the plate/horizon
|
|
22
|
+
* prohibition, and greps clean for every placeholder pattern), so deleting them
|
|
23
|
+
* would have discarded live value to reach a count. Folding into the gate that
|
|
24
|
+
* already owns the roadmap corpus adds no gate script, no CI job, and no
|
|
25
|
+
* manifest entry, and the population member disappears with the old file.
|
|
26
|
+
*
|
|
27
|
+
* ## Inline-code masking (the reason this is not a verbatim move)
|
|
28
|
+
*
|
|
29
|
+
* The original patterns matched inside backticked spans, which made CLI
|
|
30
|
+
* metasyntax a violation. Measured across 22 structural roadmaps, the only hit
|
|
31
|
+
* in the whole tree was `` `settings set <key> <value>` `` — argument
|
|
32
|
+
* metasyntax in a code span, on a step that is otherwise fully specified. The
|
|
33
|
+
* council flagged folding a check with an unaudited hit in an out-of-scope
|
|
34
|
+
* directory as a time-bomb: it would fire the day the gate's scope widened. So
|
|
35
|
+
* the fix is precision, not scope-limiting — inline code spans are masked
|
|
36
|
+
* before matching, the same technique `lint_readme_serial_comma` already uses
|
|
37
|
+
* on README prose. With masking the count is 0 across every structural roadmap,
|
|
38
|
+
* `later/` and `stubs/` included, so widening the gate's scope later cannot
|
|
39
|
+
* surprise anyone.
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
/** Placeholder patterns in (kind, regex) order. */
|
|
43
|
+
const PLACEHOLDER_PATTERNS: ReadonlyArray<[string, RegExp]> = [
|
|
44
|
+
['angle-placeholder', /<[a-z][a-z0-9 _\-/]*>/i],
|
|
45
|
+
['todo', /\bTODO\b/],
|
|
46
|
+
['fixme', /\bFIXME\b/],
|
|
47
|
+
['xxx', /\bXXX\b/],
|
|
48
|
+
['tbd', /\btbd\b/i],
|
|
49
|
+
['triple-question', /\?\?\?/],
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
const COMPLEXITY_PAT = /^complexity:\s*(lightweight|structural)\s*$/m;
|
|
53
|
+
|
|
54
|
+
const TASK_PREFIXES = ['- [ ]', '- [x]', '- [/]', '- [-]'] as const;
|
|
55
|
+
|
|
56
|
+
export interface Placeholder {
|
|
57
|
+
kind: string;
|
|
58
|
+
line: number;
|
|
59
|
+
text: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface GranularityResult {
|
|
63
|
+
complexity: string | null;
|
|
64
|
+
gated: boolean;
|
|
65
|
+
violations: Placeholder[];
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function frontmatter(text: string): string {
|
|
69
|
+
if (!text.startsWith('---\n')) {
|
|
70
|
+
return '';
|
|
71
|
+
}
|
|
72
|
+
const end = text.indexOf('\n---\n', 4);
|
|
73
|
+
return end !== -1 ? text.slice(4, end) : '';
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function read_complexity(text: string): string | null {
|
|
77
|
+
const fm = frontmatter(text);
|
|
78
|
+
if (fm === '') {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
const m = COMPLEXITY_PAT.exec(fm);
|
|
82
|
+
return m ? m[1]! : null;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const ANGLE_PLACEHOLDER = /<[a-z][a-z0-9 _\-/]*>/gi;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Blank inline code spans that are COMMAND TEMPLATES, so CLI argument
|
|
89
|
+
* metasyntax is not read as an undecided step.
|
|
90
|
+
*
|
|
91
|
+
* Blanking every backtick span (the first attempt) was too blunt, and the ported
|
|
92
|
+
* test corpus proved it: `` `<file>` `` in *"Edit `<file>` and add the new
|
|
93
|
+
* method"* is a real placeholder — the file is genuinely undecided — while
|
|
94
|
+
* `` `settings set <key> <value>` `` on a step that goes on to specify
|
|
95
|
+
* zod-validation, atomicity and key-class refusal is a command signature.
|
|
96
|
+
*
|
|
97
|
+
* The discriminator is what else the span holds: strip the placeholders and if
|
|
98
|
+
* nothing but whitespace is left, the span IS the placeholder (violation); if
|
|
99
|
+
* real content remains (`settings set`), it is a command template naming its
|
|
100
|
+
* arguments (masked). Only the angle pattern gets this treatment — a literal
|
|
101
|
+
* TODO / FIXME / XXX / TBD / `???` in a task bullet is unfinished work wherever
|
|
102
|
+
* it sits, backticks included.
|
|
103
|
+
*/
|
|
104
|
+
function mask_command_templates(line: string): string {
|
|
105
|
+
return line.replace(/`[^`]*`/g, (span) => {
|
|
106
|
+
const withoutPlaceholders = span.slice(1, -1).replace(ANGLE_PLACEHOLDER, '');
|
|
107
|
+
return withoutPlaceholders.trim() === '' ? span : ' ';
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function scan_placeholders(text: string): Placeholder[] {
|
|
112
|
+
const hits: Placeholder[] = [];
|
|
113
|
+
const lines = text.split('\n');
|
|
114
|
+
for (let idx = 0; idx < lines.length; idx++) {
|
|
115
|
+
const line = lines[idx]!;
|
|
116
|
+
const stripped = line.replace(/^\s+/, '');
|
|
117
|
+
if (!TASK_PREFIXES.some((p) => stripped.startsWith(p))) {
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
// Command-template masking applies to the angle pattern ONLY. A literal
|
|
121
|
+
// TODO / FIXME / XXX / TBD / `???` is unfinished work wherever it sits,
|
|
122
|
+
// so those patterns read the raw line.
|
|
123
|
+
const masked = mask_command_templates(line);
|
|
124
|
+
for (const [kind, pat] of PLACEHOLDER_PATTERNS) {
|
|
125
|
+
const probe = kind === 'angle-placeholder' ? masked : line;
|
|
126
|
+
if (pat.test(probe)) {
|
|
127
|
+
hits.push({ kind, line: idx + 1, text: line.replace(/\s+$/, '') });
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return hits;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Gated only for `complexity: structural`. Violations are empty when the gate
|
|
137
|
+
* is not active, regardless of placeholder presence.
|
|
138
|
+
*/
|
|
139
|
+
export function check_granularity(text: string): GranularityResult {
|
|
140
|
+
const complexity = read_complexity(text);
|
|
141
|
+
const gated = complexity === 'structural';
|
|
142
|
+
if (!gated) {
|
|
143
|
+
return { complexity, gated: false, violations: [] };
|
|
144
|
+
}
|
|
145
|
+
return { complexity, gated: true, violations: scan_placeholders(text) };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** One human-readable problem line per violation, for a gate's problem list. */
|
|
149
|
+
export function granularity_problems(text: string): string[] {
|
|
150
|
+
return check_granularity(text).violations.map(
|
|
151
|
+
(v) =>
|
|
152
|
+
`bite-sized granularity: line ${v.line} has a ${v.kind} placeholder in a ` +
|
|
153
|
+
`task bullet — a structural roadmap's steps name decided work: ${v.text.trim()}`,
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export { PLACEHOLDER_PATTERNS, COMPLEXITY_PAT };
|