@event4u/agent-config 9.14.0 → 9.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +110 -112
- package/README.md +8 -0
- package/dist/agent-src/commands/agents/audit.md +7 -7
- package/dist/agent-src/commands/agents/optimize.md +1 -1
- package/dist/agent-src/commands/analyze/reference-repo.md +1 -1
- package/dist/agent-src/commands/chat-history.md +1 -0
- package/dist/agent-src/commands/context.md +1 -0
- package/dist/agent-src/commands/council.md +1 -1
- package/dist/agent-src/commands/ghostwriter.md +1 -1
- package/dist/agent-src/commands/memory.md +2 -1
- package/dist/agent-src/commands/module.md +1 -0
- package/dist/agent-src/commands/optimize/agents-dir.md +1 -1
- package/dist/agent-src/commands/optimize/deep.md +5 -2
- package/dist/agent-src/commands/optimize/skills.md +5 -5
- package/dist/agent-src/commands/override.md +1 -0
- package/dist/agent-src/commands/prediction-pool.md +4 -4
- package/dist/agent-src/commands/research/deep.md +1 -1
- package/dist/agent-src/commands/roadmap.md +2 -1
- package/dist/agent-src/commands/rule-compliance-audit.md +1 -1
- package/dist/agent-src/commands/tdd.md +1 -0
- package/dist/agent-src/commands/tests.md +1 -0
- package/dist/agent-src/commands/upstream-contribute.md +5 -5
- package/dist/agent-src/commands/video/from-script.md +1 -1
- package/dist/agent-src/commands/video/from-song.md +1 -1
- package/dist/agent-src/commands/video/scene.md +1 -1
- package/dist/agent-src/commands/video/stitch.md +1 -1
- package/dist/agent-src/commands/video/storyboard.md +1 -1
- package/dist/agent-src/commands/worktree/create.md +2 -1
- package/dist/agent-src/contexts/augment-infrastructure.md +1 -1
- package/dist/agent-src/contexts/contracts/artifact-engagement-flow.md +5 -5
- package/dist/agent-src/contexts/contracts/command-suggestion-flow.md +3 -3
- package/dist/agent-src/contexts/contracts/emergency-triage-block.md +2 -2
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +4 -3
- package/dist/agent-src/contexts/execution/subagent-routing.md +19 -0
- package/dist/agent-src/contexts/override-system.md +1 -1
- package/dist/agent-src/ghostwriter/fictional-fixture-v1.md +1 -1
- package/dist/agent-src/rules/augment-edit-discipline.md +1 -1
- package/dist/agent-src/rules/autonomous-execution.md +2 -1
- package/dist/agent-src/rules/brand-consistency.md +1 -0
- package/dist/agent-src/rules/brand-source-of-truth.md +1 -0
- package/dist/agent-src/rules/code-comment-discipline.md +2 -0
- package/dist/agent-src/rules/communication-through-line.md +1 -0
- package/dist/agent-src/rules/content-quoting-floor.md +1 -0
- package/dist/agent-src/rules/cross-source-consistency.md +0 -1
- package/dist/agent-src/rules/downstream-changes.md +1 -0
- package/dist/agent-src/rules/external-code-graph-interop.md +1 -0
- package/dist/agent-src/rules/external-reference-deep-dive.md +1 -0
- package/dist/agent-src/rules/fast-path-marker-visibility.md +1 -0
- package/dist/agent-src/rules/finance-safety-floor.md +0 -1
- package/dist/agent-src/rules/icon-consistency.md +1 -0
- package/dist/agent-src/rules/image-likeness-and-rights.md +1 -0
- package/dist/agent-src/rules/improve-before-implement.md +1 -1
- package/dist/agent-src/rules/invite-challenge.md +1 -0
- package/dist/agent-src/rules/lethal-trifecta-guard.md +1 -0
- package/dist/agent-src/rules/low-impact-corpus-privacy-floor.md +1 -0
- package/dist/agent-src/rules/markdown-safe-codeblocks.md +1 -0
- package/dist/agent-src/rules/media-governance-routing.md +4 -3
- package/dist/agent-src/rules/media-sync-ground-truth.md +1 -0
- package/dist/agent-src/rules/no-attribution-footers.md +1 -0
- package/dist/agent-src/rules/no-decorative-emojis-in-git-surfaces.md +1 -0
- package/dist/agent-src/rules/no-pr-progress-comments.md +1 -0
- package/dist/agent-src/rules/onboarding-gate.md +1 -0
- package/dist/agent-src/rules/output-discipline.md +1 -0
- package/dist/agent-src/rules/prefer-enums-over-literals.md +1 -0
- package/dist/agent-src/rules/preservation-guard.md +1 -0
- package/dist/agent-src/rules/question-not-instruction.md +1 -0
- package/dist/agent-src/rules/roadmap-ci-steps-policy.md +1 -0
- package/dist/agent-src/rules/role-mode-adherence.md +1 -0
- package/dist/agent-src/rules/runtime-safety.md +1 -0
- package/dist/agent-src/rules/security-sensitive-stop.md +1 -0
- package/dist/agent-src/rules/session-canary.md +9 -2
- package/dist/agent-src/rules/source-confidentiality.md +1 -0
- package/dist/agent-src/rules/source-of-truth.md +1 -0
- package/dist/agent-src/rules/spreadsheet-source-quality.md +1 -0
- package/dist/agent-src/rules/think-before-action.md +1 -0
- package/dist/agent-src/rules/token-budget-discipline.md +1 -0
- package/dist/agent-src/rules/token-efficiency.md +1 -0
- package/dist/agent-src/rules/token-optimizer-maintenance.md +5 -5
- package/dist/agent-src/rules/tool-safety.md +1 -0
- package/dist/agent-src/rules/untrusted-input-defense.md +2 -0
- package/dist/agent-src/rules/user-interaction.md +1 -0
- package/dist/agent-src/rules/user-interrupt-priority.md +1 -0
- package/dist/agent-src/skills/adr-create/SKILL.md +1 -1
- package/dist/agent-src/skills/check-refs/SKILL.md +1 -1
- package/dist/agent-src/skills/competitive-positioning/SKILL.md +1 -1
- package/dist/agent-src/skills/condense-memory/SKILL.md +1 -1
- package/dist/agent-src/skills/context-authoring/SKILL.md +1 -1
- package/dist/agent-src/skills/copilot-agents-optimization/SKILL.md +1 -1
- package/dist/agent-src/skills/description-assist/SKILL.md +1 -1
- package/dist/agent-src/skills/github-ci/SKILL.md +1 -0
- package/dist/agent-src/skills/guideline-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/lint-skills/SKILL.md +1 -1
- package/dist/agent-src/skills/readme-reviewer/SKILL.md +1 -1
- package/dist/agent-src/skills/roadmap-management/SKILL.md +8 -1
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/skill-improvement-pipeline/SKILL.md +1 -1
- package/dist/agent-src/skills/skill-writing/SKILL.md +4 -4
- package/dist/agent-src/skills/token-optimizer/SKILL.md +4 -4
- package/dist/agent-src/skills/upstream-contribute/SKILL.md +6 -6
- package/dist/agent-src/skills/using-git-worktrees/SKILL.md +25 -0
- package/dist/agent-src/templates/AGENTS.md +1 -1
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/command.md +1 -1
- package/dist/agent-src/templates/copilot-instructions.md +3 -3
- package/dist/agent-src/templates/rule.md +2 -2
- package/dist/agent-src/templates/scripts/work_engine/orchestration.ts +1 -1
- package/dist/agent-src/templates/skill-archive-note.md +5 -5
- package/dist/agent-src/templates/skill.md +1 -1
- package/dist/cli/agent-config.js +66 -19
- package/dist/cli/agent-config.js.map +1 -1
- package/dist/cli/main.js +9 -4
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/python/workspace_drive.js +2 -0
- package/dist/cli/python/workspace_drive.js.map +1 -1
- package/dist/cli/python/workspace_drive_health.js +2 -0
- package/dist/cli/python/workspace_drive_health.js.map +1 -1
- package/dist/cli/python/workspace_hosts.js +2 -0
- package/dist/cli/python/workspace_hosts.js.map +1 -1
- package/dist/cli/python/workspace_sessions.js +2 -0
- package/dist/cli/python/workspace_sessions.js.map +1 -1
- package/dist/cli/registry.js +1 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/{chunk-A3HJ6NTI.js → chunk-63WSP6YZ.js} +2 -1
- package/dist/cli-delegate/{chunk-EJYIGZ3T.js → chunk-BKO2AZ7Z.js} +1 -1
- package/dist/cli-delegate/{chunk-GCVFLBP2.js → chunk-H574KOUH.js} +338 -494
- package/dist/cli-delegate/chunk-NXFWFAYV.js +339 -0
- package/dist/cli-delegate/{chunk-TEGJU2JX.js → chunk-ZXXJLL5K.js} +2 -2
- package/dist/cli-delegate/cmd_conformance.js +29 -23
- package/dist/cli-delegate/cmd_doctor.js +3 -3
- package/dist/cli-delegate/cmd_export.js +4 -4
- package/dist/cli-delegate/cmd_preflight.js +2 -2
- package/dist/cli-delegate/cmd_prune.js +4 -4
- package/dist/cli-delegate/cmd_settings_migrate.js +4 -4
- package/dist/cli-delegate/cmd_sync.js +4 -4
- package/dist/cli-delegate/cmd_uninstall.js +4 -4
- package/dist/cli-delegate/cmd_validate.js +4 -4
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +127 -95
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +729 -430
- package/dist/install/install.mjs +52 -11
- package/dist/install/rule_scope.js +18 -2
- package/dist/install/rule_scope.js.map +1 -1
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +518 -411
- package/dist/router.json +1 -1
- package/dist/scripts/_lib/claude_settings_hooks.js +30 -5
- package/dist/scripts/_lib/claude_settings_hooks.js.map +1 -1
- package/dist/scripts/_lib/scoped_projection.js +223 -0
- package/dist/scripts/_lib/scoped_projection.js.map +1 -0
- package/dist/scripts/_lib/surface_tiers.js +125 -0
- package/dist/scripts/_lib/surface_tiers.js.map +1 -0
- package/dist/scripts/_lib/token_count.js +166 -0
- package/dist/scripts/_lib/token_count.js.map +1 -0
- package/dist/scripts/_lib/trigger_eval_floors.js +8 -0
- package/dist/scripts/_lib/trigger_eval_floors.js.map +1 -1
- package/dist/scripts/condense.js +139 -9
- package/dist/scripts/condense.js.map +1 -1
- package/dist/scripts/project_thin_rules.js +450 -0
- package/dist/scripts/project_thin_rules.js.map +1 -0
- package/dist/server/routes/workspace.js +2 -0
- package/dist/server/routes/workspace.js.map +1 -1
- package/dist/server/schemas/settings.js +9 -3
- package/dist/server/schemas/settings.js.map +1 -1
- package/dist/ui/assets/index-DJY_vQVn.js +40 -0
- package/dist/ui/assets/index-DJY_vQVn.js.map +1 -0
- package/dist/ui/index.html +1 -1
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +1 -1
- package/package.json +11 -11
- package/src/agent-src/templates/AGENTS.md +1 -1
- package/src/agent-src/templates/scripts/work_engine/orchestration.ts +1 -1
- package/src/config/agent-settings.template.yml +59 -11
- package/src/config/ci-local-parity.yml +8 -0
- package/src/config/discovery/unassigned-artefacts.yml +23 -25
- package/src/config/evaluator-budgets.json +35 -5
- package/src/config/gate-coverage.yml +147 -0
- package/src/config/gate-violation-baselines.json +25 -0
- package/src/install/rule_scope.ts +27 -2
- package/src/scripts/_cli/cmd_conformance.ts +13 -24
- package/src/scripts/_dispatch.bash +10 -0
- package/src/scripts/_lib/claude_settings_hooks.ts +30 -4
- package/src/scripts/_lib/gate_baseline.ts +213 -0
- package/src/scripts/_lib/scoped_projection.ts +50 -0
- package/src/scripts/_lib/tier_budget_routing.ts +234 -0
- package/src/scripts/_lib/tool_adapter_registry.ts +8 -1
- package/src/scripts/_lib/trigger_eval_floors.ts +9 -0
- package/src/scripts/_lib/trigger_routers.ts +17 -5
- package/src/scripts/adr/regenerate_index.ts +4 -3
- package/src/scripts/ai-video/lib/parse-blueprint.sh +1 -1
- package/src/scripts/ai_council/compile_corpus.ts +1 -1
- package/src/scripts/audit_cloud_compatibility.ts +37 -2
- package/src/scripts/audit_initial_context.ts +53 -19
- package/src/scripts/audit_likelihood.ts +31 -2
- package/src/scripts/audit_user_type_axis.ts +58 -11
- package/src/scripts/bench_hook_latency.ts +172 -41
- package/src/scripts/build_mcp_catalog.ts +12 -1
- package/src/scripts/check_always_budget.ts +122 -11
- package/src/scripts/check_augment_description_cap.ts +24 -3
- package/src/scripts/check_ci_strict_superset.ts +150 -0
- package/src/scripts/check_cluster_patterns.ts +100 -0
- package/src/scripts/check_consumer_scope_flip.ts +26 -4
- package/src/scripts/check_evaluator_budgets.ts +198 -19
- package/src/scripts/check_gate_coverage.ts +337 -1
- package/src/scripts/check_generator_output_coverage.ts +2 -1
- package/src/scripts/check_iron_law_prominence.ts +1 -1
- package/src/scripts/check_kernel_rule_bundle.ts +1 -1
- package/src/scripts/check_no_new_legacy_path.ts +167 -3
- package/src/scripts/check_pack_size.ts +18 -4
- package/src/scripts/check_portability.ts +37 -7
- package/src/scripts/check_references.ts +19 -3
- package/src/scripts/check_release_pr_shape.ts +13 -0
- package/src/scripts/check_token_regression.ts +9 -2
- package/src/scripts/condense.ts +155 -12
- package/src/scripts/cost/budget.mjs +63 -2
- package/src/scripts/evaluator_umbrella.sh +34 -1
- package/src/scripts/generate_capability_matrix.ts +8 -0
- package/src/scripts/generate_router_coverage_from_matrix.ts +155 -0
- package/src/scripts/hook_manifest.yaml +17 -3
- package/src/scripts/hooks/block_config_weakening.ts +337 -0
- package/src/scripts/hooks/concern_registry.ts +2 -0
- package/src/scripts/hooks/probe_throttle.ts +75 -0
- package/src/scripts/install-hooks.sh +24 -2
- package/src/scripts/install.ts +2 -2
- package/src/scripts/lint_agents_md.ts +9 -3
- package/src/scripts/lint_archived_skills.ts +1 -1
- package/src/scripts/lint_artefact_frontmatter.ts +185 -37
- package/src/scripts/lint_command_routing.ts +21 -52
- package/src/scripts/lint_command_verbs.ts +47 -28
- package/src/scripts/lint_context_spine_usage.ts +38 -12
- package/src/scripts/lint_framework_leakage.ts +25 -6
- package/src/scripts/lint_handoffs.ts +49 -2
- package/src/scripts/lint_media_policy_linkage.ts +62 -30
- package/src/scripts/lint_namespace.ts +92 -7
- package/src/scripts/lint_new_skill_gate.ts +39 -31
- package/src/scripts/lint_originality_shingles.ts +7 -1
- package/src/scripts/lint_override_kernel_guard.ts +21 -3
- package/src/scripts/lint_pack_boundaries.ts +461 -91
- package/src/scripts/lint_pack_dependencies.ts +24 -57
- package/src/scripts/lint_persona_governance.ts +1 -1
- package/src/scripts/lint_role_experiences.ts +34 -5
- package/src/scripts/lint_trigger_precision.ts +142 -0
- package/src/scripts/mcp_server/consumer_tool_catalog.json +12 -201
- package/src/scripts/mcp_server/tools.ts +1 -1
- package/src/scripts/prepack-check.mjs +40 -1
- package/src/scripts/prepack_router_targets.d.mts +7 -0
- package/src/scripts/prepack_router_targets.mjs +93 -0
- package/src/scripts/print_required_checks.ts +72 -18
- package/src/scripts/profile_staleness_hook.ts +12 -0
- package/src/scripts/record_evaluator_measurements.mjs +105 -0
- package/src/scripts/recruit_preflight.sh +1 -1
- package/src/scripts/release.ts +105 -1
- package/src/scripts/router_target_paths.d.mts +4 -0
- package/src/scripts/router_target_paths.mjs +69 -0
- package/src/scripts/router_telemetry.ts +52 -1
- package/src/scripts/routing_doctor.ts +485 -0
- package/src/scripts/rule_trigger_eval.ts +403 -0
- package/src/scripts/schemas/rule.schema.json +5 -1
- package/src/scripts/session_canary_hook.ts +96 -5
- package/src/scripts/skill_linter.ts +9 -28
- package/src/scripts/smoke/skills.sh +1 -1
- package/src/scripts/sweep_dead_scan_roots.ts +356 -10
- package/src/scripts/trigger_coverage.ts +13 -2
- package/src/scripts/update_counts.ts +1 -1
- package/src/scripts/validate_telegraph_carveouts.ts +1 -1
- package/src/scripts/wrapper_freshness_hook.ts +14 -0
- package/src/server/schemas/settings.ts +19 -5
- package/dist/cli-delegate/chunk-TJIAM5FW.js +0 -133
- package/dist/ui/assets/index-BpSQt5vm.js +0 -40
- package/dist/ui/assets/index-BpSQt5vm.js.map +0 -1
|
@@ -91,11 +91,32 @@ function _globMdSorted(dir: string): string[] {
|
|
|
91
91
|
return out;
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
-
|
|
94
|
+
/**
|
|
95
|
+
* Injection seam for tests: `--root <dir>` overrides the scan root.
|
|
96
|
+
*
|
|
97
|
+
* Additive — no flag means the CI-identical bare call scans `RULES_DIR`, so the
|
|
98
|
+
* pinned CLI contract is unchanged. Without a seam the only way to exercise a
|
|
99
|
+
* real over-cap violation was to tamper with a committed rule, which is why this
|
|
100
|
+
* gate shipped with tests that never called `main()` at all.
|
|
101
|
+
*/
|
|
102
|
+
function _resolve_root(argv: readonly string[]): string {
|
|
103
|
+
for (let i = 0; i < argv.length; i++) {
|
|
104
|
+
const a = argv[i] as string;
|
|
105
|
+
if (a === '--root') {
|
|
106
|
+
const v = argv[i + 1];
|
|
107
|
+
if (v !== undefined) return path.resolve(v);
|
|
108
|
+
} else if (a.startsWith('--root=')) {
|
|
109
|
+
return path.resolve(a.slice('--root='.length));
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return RULES_DIR;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function main(argv: readonly string[] = []): number {
|
|
95
116
|
const failures: Array<[string, number, string]> = [];
|
|
96
117
|
let checked = 0;
|
|
97
118
|
|
|
98
|
-
for (const p of _globMdSorted(
|
|
119
|
+
for (const p of _globMdSorted(_resolve_root(argv))) {
|
|
99
120
|
const text = fs.readFileSync(p, 'utf-8');
|
|
100
121
|
const fm = parse_frontmatter(text);
|
|
101
122
|
if (fm['type'] !== 'auto') {
|
|
@@ -157,7 +178,7 @@ function _isCliEntry(): boolean {
|
|
|
157
178
|
}
|
|
158
179
|
|
|
159
180
|
if (_isCliEntry() || process.argv[1] === _HERE) {
|
|
160
|
-
process.exit(main());
|
|
181
|
+
process.exit(main(process.argv.slice(2)));
|
|
161
182
|
}
|
|
162
183
|
|
|
163
184
|
export { REPO_ROOT, RULES_DIR, DESC_CAP };
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* Assert `ci-strict` is a superset of `ci` BY CONSTRUCTION.
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS EXISTS (road-to-renewal-foundation Phase 1, council 2026-08-02
|
|
6
|
+
* decision A1, 2/2):
|
|
7
|
+
*
|
|
8
|
+
* `Taskfile.yml` used to carry `ci` (202 gate entries) and `ci-strict` (197)
|
|
9
|
+
* as two independently maintained literal lists. They drifted, and nothing
|
|
10
|
+
* detected it — `check_ci_local_parity.ts` deliberately scopes itself to
|
|
11
|
+
* `LOCAL_ROOTS = ['ci', 'consistency']`, so `ci-strict` was outside every
|
|
12
|
+
* assertion in the repo. Six gates ran in `ci` with no counterpart in
|
|
13
|
+
* `ci-strict` at all:
|
|
14
|
+
*
|
|
15
|
+
* preflight (fans out to 11 scripts, incl. check_no_new_legacy_path and
|
|
16
|
+
* check_kernel_rule_bundle), check-ci-local-parity,
|
|
17
|
+
* check-gitignore-freshness, check-generator-output-coverage,
|
|
18
|
+
* check-tracked-but-ignored, check-generated-artefact-headers
|
|
19
|
+
*
|
|
20
|
+
* — so the "release-tag gate" was strictly WEAKER than the everyday gate on
|
|
21
|
+
* ~16 concrete script runs. A stricter tier that checks less is the worst
|
|
22
|
+
* possible shape for a release gate: it is trusted more and proves less.
|
|
23
|
+
*
|
|
24
|
+
* The fix was structural, not a bigger checker: `ci-strict` now delegates to
|
|
25
|
+
* `ci` and adds only strict-only entries on top, so the superset property is
|
|
26
|
+
* tautological. This gate guards the ONE invariant that keeps it tautological
|
|
27
|
+
* — the first gate entry of `ci-strict` is `- task: ci`.
|
|
28
|
+
*
|
|
29
|
+
* It deliberately does NOT re-parse and diff both lists. That would recreate
|
|
30
|
+
* the very "maintain a second model of the same thing" problem the delegation
|
|
31
|
+
* removed, and it would pass happily on a re-forked pair of lists that merely
|
|
32
|
+
* happened to match on the day it ran.
|
|
33
|
+
*
|
|
34
|
+
* Exit codes: 0 ok · 1 invariant violated · 2 usage / environment error.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import * as fs from 'node:fs';
|
|
38
|
+
import * as path from 'node:path';
|
|
39
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
40
|
+
|
|
41
|
+
const _HERE = fileURLToPath(import.meta.url);
|
|
42
|
+
const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
43
|
+
const TASKFILE = path.join(REPO_ROOT, 'Taskfile.yml');
|
|
44
|
+
|
|
45
|
+
/** The delegation entry that makes the superset property tautological. */
|
|
46
|
+
const DELEGATION = '- task: ci';
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Extract the `cmds:` entries of a top-level task, in order.
|
|
50
|
+
*
|
|
51
|
+
* Minimal line scan rather than a YAML dependency: the shape is fixed
|
|
52
|
+
* (` <name>:` at two-space indent, ` cmds:` at four, ` - ...` at
|
|
53
|
+
* six) and the repo has no YAML anchors anywhere, so a parser buys nothing
|
|
54
|
+
* here and adds an import to a gate that must stay cheap.
|
|
55
|
+
*/
|
|
56
|
+
function taskCmds(source: string, taskName: string): string[] | null {
|
|
57
|
+
const lines = source.split('\n');
|
|
58
|
+
const header = ` ${taskName}:`;
|
|
59
|
+
let i = lines.findIndex((l) => l === header);
|
|
60
|
+
if (i === -1) {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
const out: string[] = [];
|
|
64
|
+
let inCmds = false;
|
|
65
|
+
for (i += 1; i < lines.length; i++) {
|
|
66
|
+
const line = lines[i]!;
|
|
67
|
+
if (line.trim() === '') {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
// Any line at two-space indent that is not deeper ends this task.
|
|
71
|
+
if (/^ {2}\S/.test(line)) {
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
if (/^ {4}cmds:\s*$/.test(line)) {
|
|
75
|
+
inCmds = true;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (!inCmds) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const m = /^ {6}(- .*)$/.exec(line);
|
|
82
|
+
if (m) {
|
|
83
|
+
out.push(m[1]!.trim());
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function main(): number {
|
|
90
|
+
let source: string;
|
|
91
|
+
try {
|
|
92
|
+
source = fs.readFileSync(TASKFILE, 'utf-8');
|
|
93
|
+
} catch (err) {
|
|
94
|
+
process.stderr.write(`❌ check_ci_strict_superset: cannot read ${TASKFILE}: ${String(err)}\n`);
|
|
95
|
+
return 2;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const strict = taskCmds(source, 'ci-strict');
|
|
99
|
+
if (strict === null) {
|
|
100
|
+
process.stderr.write(
|
|
101
|
+
'❌ check_ci_strict_superset: no `ci-strict:` task found in Taskfile.yml.\n' +
|
|
102
|
+
' The gate cannot verify an invariant on a task that does not exist.\n',
|
|
103
|
+
);
|
|
104
|
+
return 2;
|
|
105
|
+
}
|
|
106
|
+
if (taskCmds(source, 'ci') === null) {
|
|
107
|
+
process.stderr.write(
|
|
108
|
+
'❌ check_ci_strict_superset: no `ci:` task found in Taskfile.yml.\n',
|
|
109
|
+
);
|
|
110
|
+
return 2;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Skip the internal timing wrappers; they carry no gate semantics and
|
|
114
|
+
// `ci-strict` intentionally does not repeat them (`ci` already brackets
|
|
115
|
+
// the run, so the duration line prints once).
|
|
116
|
+
const gateEntries = strict.filter(
|
|
117
|
+
(e) => !e.startsWith('- defer:') && e !== '- task: _ci-start' && e !== '- task: _ci-end',
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
if (gateEntries.length === 0) {
|
|
121
|
+
process.stderr.write('❌ check_ci_strict_superset: `ci-strict` has no gate entries.\n');
|
|
122
|
+
return 1;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (gateEntries[0] !== DELEGATION) {
|
|
126
|
+
process.stderr.write(
|
|
127
|
+
'❌ check_ci_strict_superset: `ci-strict` no longer delegates to `ci`.\n' +
|
|
128
|
+
` expected first gate entry: \`${DELEGATION}\`\n` +
|
|
129
|
+
` found: \`${gateEntries[0]}\`\n\n` +
|
|
130
|
+
' `ci-strict` must run every `ci` gate plus strict-only extras. It used\n' +
|
|
131
|
+
' to be an independently maintained copy of the list and silently drifted\n' +
|
|
132
|
+
' to SIX fewer gates than `ci` — a release gate that proved less than the\n' +
|
|
133
|
+
' everyday gate. Delegation is what makes the superset property structural\n' +
|
|
134
|
+
' instead of asserted. If you need a strict variant of a gate, add the\n' +
|
|
135
|
+
' `-strict` task AFTER the delegation entry; never re-inline the list.\n',
|
|
136
|
+
);
|
|
137
|
+
return 1;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const extras = gateEntries.slice(1);
|
|
141
|
+
process.stdout.write(
|
|
142
|
+
`✅ ci-strict ⊇ ci by construction (delegates to \`ci\`, plus ${extras.length} strict-only ` +
|
|
143
|
+
`entr${extras.length === 1 ? 'y' : 'ies'}).\n`,
|
|
144
|
+
);
|
|
145
|
+
return 0;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
149
|
+
process.exit(main());
|
|
150
|
+
}
|
|
@@ -18,6 +18,19 @@
|
|
|
18
18
|
* must document the bare-invocation behaviour (menu / default route /
|
|
19
19
|
* detection fallback) per command-clusters.md § Bare invocation.
|
|
20
20
|
*
|
|
21
|
+
* Filesystem-enumeration addition (road-to-renewal-leverage, 2026-08-02): the
|
|
22
|
+
* checks above all read the CONTRACT and ask "does what it names exist?".
|
|
23
|
+
* Nothing asked the inverse — "does what exists get named?" — so a
|
|
24
|
+
* sub-command could sit on disk while appearing in neither its hub's
|
|
25
|
+
* `## Sub-commands` table nor its `routes_to:`, invisibly. Two instances were
|
|
26
|
+
* live when this was added (`/roadmap:materialize`, `/memory:learn-low-impact`)
|
|
27
|
+
* plus incomplete `routes_to` on 12 of 25 dispatch clusters.
|
|
28
|
+
* `check_completeness` closes that direction: for a `type: orchestrator`
|
|
29
|
+
* dispatcher, every on-disk `<sub>/command.md` sibling directory must be named
|
|
30
|
+
* in the table and listed in `routes_to`. Scoped to orchestrators on purpose —
|
|
31
|
+
* a few contract rows point at full working commands that merely happen to
|
|
32
|
+
* carry one nested sub-command, and those own no dispatch table.
|
|
33
|
+
*
|
|
21
34
|
* Exit codes: 0 = clean, 1 = pattern violations, 3 = internal error.
|
|
22
35
|
*/
|
|
23
36
|
|
|
@@ -170,6 +183,90 @@ function _repr(v: string | undefined): string {
|
|
|
170
183
|
return `'${v.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
|
|
171
184
|
}
|
|
172
185
|
|
|
186
|
+
/** Parse a `routes_to: [a, b]` frontmatter value into its entries. */
|
|
187
|
+
export function parse_routes_to(raw: string | undefined): string[] {
|
|
188
|
+
if (raw === undefined) {
|
|
189
|
+
return [];
|
|
190
|
+
}
|
|
191
|
+
const m = /^\[(.*)\]$/.exec(raw.trim());
|
|
192
|
+
if (!m) {
|
|
193
|
+
return [];
|
|
194
|
+
}
|
|
195
|
+
return (m[1] as string)
|
|
196
|
+
.split(',')
|
|
197
|
+
.map((s) => s.trim().replace(/^['"]|['"]$/g, ''))
|
|
198
|
+
.filter((s) => s.length > 0);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** Sub-command directory names that carry a `command.md`, sorted. */
|
|
202
|
+
export function on_disk_subs(dispatcher_path: string): string[] {
|
|
203
|
+
const dir = path.dirname(dispatcher_path);
|
|
204
|
+
let entries: fs.Dirent[];
|
|
205
|
+
try {
|
|
206
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
207
|
+
} catch {
|
|
208
|
+
return [];
|
|
209
|
+
}
|
|
210
|
+
return entries
|
|
211
|
+
.filter((e) => e.isDirectory() && _isFile(path.join(dir, e.name, 'command.md')))
|
|
212
|
+
.map((e) => e.name)
|
|
213
|
+
.sort();
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Does the body name `<sub>` as a sub-command of `<cluster>`?
|
|
218
|
+
*
|
|
219
|
+
* Both `` `/cluster:sub` `` and `` `/cluster sub` `` count — command-clusters.md
|
|
220
|
+
* § 157 makes the space form a first-class equivalent of the colon form, so
|
|
221
|
+
* requiring one of them would be inventing a rule the contract does not carry.
|
|
222
|
+
* The trailing class allows a row that decorates the name inside the same
|
|
223
|
+
* backticks (`` `/team review [--background]` ``).
|
|
224
|
+
*/
|
|
225
|
+
export function names_sub(body: string, cluster: string, sub: string): boolean {
|
|
226
|
+
const re = new RegExp('`/' + cluster + '[: ]' + sub + '(?![a-z0-9-])');
|
|
227
|
+
return re.test(body);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Every on-disk sub-command of an orchestrator must be named in its
|
|
232
|
+
* `## Sub-commands` table and listed in `routes_to:`.
|
|
233
|
+
*
|
|
234
|
+
* Non-orchestrator dispatchers are skipped: a handful of contract rows point
|
|
235
|
+
* at full working commands that merely happen to carry a nested sub-command,
|
|
236
|
+
* and those own no dispatch table to be complete against.
|
|
237
|
+
*/
|
|
238
|
+
export function check_completeness(
|
|
239
|
+
cluster: string,
|
|
240
|
+
dispatcher_path: string,
|
|
241
|
+
fm: Record<string, string>,
|
|
242
|
+
body: string,
|
|
243
|
+
): string[] {
|
|
244
|
+
if (fm['type'] !== 'orchestrator') {
|
|
245
|
+
return [];
|
|
246
|
+
}
|
|
247
|
+
const subs = on_disk_subs(dispatcher_path);
|
|
248
|
+
if (subs.length === 0) {
|
|
249
|
+
return [];
|
|
250
|
+
}
|
|
251
|
+
const errors: string[] = [];
|
|
252
|
+
const routes = parse_routes_to(fm['routes_to']);
|
|
253
|
+
for (const sub of subs) {
|
|
254
|
+
if (!names_sub(body, cluster, sub)) {
|
|
255
|
+
errors.push(
|
|
256
|
+
`sub-command \`${sub}\` exists on disk but is not named in the ` +
|
|
257
|
+
'`## Sub-commands` table',
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
if (!routes.includes(`${cluster}-${sub}`) && !routes.includes(sub)) {
|
|
261
|
+
errors.push(
|
|
262
|
+
`sub-command \`${sub}\` exists on disk but is missing from ` +
|
|
263
|
+
'`routes_to:`',
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return errors;
|
|
268
|
+
}
|
|
269
|
+
|
|
173
270
|
export function check_dispatcher(cluster: string, slug_map: Map<string, string>): FileReport {
|
|
174
271
|
const p = _resolve_command(cluster, slug_map);
|
|
175
272
|
if (p === null) {
|
|
@@ -241,6 +338,9 @@ export function check_dispatcher(cluster: string, slug_map: Map<string, string>)
|
|
|
241
338
|
}
|
|
242
339
|
}
|
|
243
340
|
|
|
341
|
+
// Completeness (2026-08-02): every sub-command that EXISTS must be named.
|
|
342
|
+
rep.errors.push(...check_completeness(cluster, p, fm, body));
|
|
343
|
+
|
|
244
344
|
// Bare-invocation story (Phase 4 / ADR-115): the Dispatch section must say
|
|
245
345
|
// what a bare `/<cluster>` does — a menu, a default route, or detection.
|
|
246
346
|
const dispatchIdx = body.indexOf('## Dispatch');
|
|
@@ -63,7 +63,7 @@ export interface FlipVerdict {
|
|
|
63
63
|
|
|
64
64
|
export function verify_flip(repoRoot = '.'): FlipVerdict {
|
|
65
65
|
const template = parseYaml(fs.readFileSync(path.join(repoRoot, TEMPLATE), 'utf-8')) as Record<string, unknown>;
|
|
66
|
-
const scope = ruleScopeFromSettings(template);
|
|
66
|
+
const scope = ruleScopeFromSettings(template, repoRoot);
|
|
67
67
|
if (scope.workspaces === null) {
|
|
68
68
|
throw new Error('template rule_workspaces is empty — nothing flipped, nothing to verify');
|
|
69
69
|
}
|
|
@@ -149,8 +149,26 @@ async function main(): Promise<number> {
|
|
|
149
149
|
},
|
|
150
150
|
pass: v.pass,
|
|
151
151
|
};
|
|
152
|
-
|
|
153
|
-
|
|
152
|
+
// WRITE IS OPT-IN (road-to-renewal-foundation Phase 1). This used to write
|
|
153
|
+
// unconditionally, which made a `check_*` script mutate a TRACKED, DATED
|
|
154
|
+
// artifact as a side effect of merely being executed. Two concrete costs,
|
|
155
|
+
// both hit in one session:
|
|
156
|
+
//
|
|
157
|
+
// * Any sweep that runs "every gate" — a coverage census, a timing pass,
|
|
158
|
+
// or the pooled umbrella runner this roadmap proposes — silently dirties
|
|
159
|
+
// the tree, and the next `task consistency` then reports derived-output
|
|
160
|
+
// drift that has nothing to do with the diff under test.
|
|
161
|
+
// * A dated report is a RECORD of a past measurement. Rewriting it in
|
|
162
|
+
// place with today's numbers destroys the thing its filename claims.
|
|
163
|
+
//
|
|
164
|
+
// Verification is unchanged and still runs on every invocation; only the
|
|
165
|
+
// persistence is gated. `--write` reproduces the old behaviour for a
|
|
166
|
+
// deliberate re-measure.
|
|
167
|
+
const wrote = process.argv.includes('--write');
|
|
168
|
+
if (wrote) {
|
|
169
|
+
fs.mkdirSync(path.dirname(REPORT), { recursive: true });
|
|
170
|
+
fs.writeFileSync(REPORT, `${JSON.stringify(report, null, 1)}\n`);
|
|
171
|
+
}
|
|
154
172
|
process.stdout.write(`excluded ${v.excluded.length} rules under scope [${v.scope.join(', ')}]\n`);
|
|
155
173
|
if (v.violations.length > 0) {
|
|
156
174
|
process.stdout.write(`❌ NON-MAINTAINER rules would drop: ${v.violations.join(' · ')}\n`);
|
|
@@ -162,7 +180,11 @@ async function main(): Promise<number> {
|
|
|
162
180
|
`legacy-all ${legacyRules} rules / ${legacyTok} tok → scoped ${scopedRules} rules / ${scopedTok} tok `
|
|
163
181
|
+ `(−${legacyTok - scopedTok} tok, −${report.delta.tokens_pct}%)\n`,
|
|
164
182
|
);
|
|
165
|
-
process.stdout.write(
|
|
183
|
+
process.stdout.write(
|
|
184
|
+
v.pass
|
|
185
|
+
? `✅ flip verified${wrote ? ` — wrote ${REPORT}` : ' (read-only; pass --write to persist the report)'}\n`
|
|
186
|
+
: '❌ flip NOT verified\n',
|
|
187
|
+
);
|
|
166
188
|
return v.pass ? 0 : 1;
|
|
167
189
|
}
|
|
168
190
|
|
|
@@ -3,17 +3,38 @@
|
|
|
3
3
|
* check_evaluator_budgets — enforce the outside-in evaluator budgets
|
|
4
4
|
* (road-to-credible-install Phase 6) against a measurements JSON.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
6
|
+
* Three failure modes per metric:
|
|
7
7
|
* 1. absolute: measured > budgets[metric].max
|
|
8
8
|
* 2. regression: measured > last_measured * (1 + regression_pct/100)
|
|
9
9
|
* — fails EVEN WHEN still under the absolute budget (the >10%
|
|
10
10
|
* creep rule), so slow rot cannot hide under a generous cap.
|
|
11
|
+
* 3. drift: a DETERMINISTIC metric disagrees with the committed
|
|
12
|
+
* measurement record the nightly publishes. `cli_help_command_count`
|
|
13
|
+
* drifted 74 → 80 against a value frozen at 79 with nobody seeing it;
|
|
14
|
+
* the record makes the day a number moves the day it shows in a diff
|
|
15
|
+
* (road-to-gates-that-can-fail Phase 5).
|
|
16
|
+
*
|
|
17
|
+
* ## Posture — warn on main, fail on release
|
|
18
|
+
*
|
|
19
|
+
* A hard fail on `main` turns every legitimate command addition into a
|
|
20
|
+
* blocked merge, which is how budgets get quietly raised with a cushion
|
|
21
|
+
* instead of consciously. So budget / regression / drift findings are
|
|
22
|
+
* ADVISORY outside a release context and BLOCKING inside one. This is not a
|
|
23
|
+
* weakening: before the Phase-4 path triggers, the umbrella ran on release
|
|
24
|
+
* PRs and the nightly only — the surfaces that now warn never saw the gate
|
|
25
|
+
* at all. The posture IS the delivery route the council adopted for nightly
|
|
26
|
+
* visibility, not a second mechanism beside it.
|
|
27
|
+
*
|
|
28
|
+
* Structural findings (a metric with NO measurement supplied) are ALWAYS
|
|
29
|
+
* blocking, in either posture: that is a harness reporting green on nothing,
|
|
30
|
+
* which is the whole class this roadmap exists to end.
|
|
11
31
|
*
|
|
12
32
|
* Usage:
|
|
13
33
|
* ./scripts-run src/scripts/check_evaluator_budgets --measurements <file.json>
|
|
34
|
+
* [--recorded <file.json>] [--posture warn|fail]
|
|
14
35
|
* (measurements: flat { metric: number } — produced by evaluator_umbrella.sh)
|
|
15
36
|
*
|
|
16
|
-
* Exit codes: 0 green · 1
|
|
37
|
+
* Exit codes: 0 green (or advisory findings only) · 1 blocking finding · 2 internal error.
|
|
17
38
|
*/
|
|
18
39
|
import * as fs from 'node:fs';
|
|
19
40
|
import * as path from 'node:path';
|
|
@@ -22,6 +43,9 @@ import { fileURLToPath } from 'node:url';
|
|
|
22
43
|
const REPO_ROOT = path.resolve(fileURLToPath(import.meta.url), '..', '..', '..');
|
|
23
44
|
const BUDGETS_PATH = path.join(REPO_ROOT, 'src', 'config', 'evaluator-budgets.json');
|
|
24
45
|
|
|
46
|
+
/** Committed measurement record the nightly refreshes (Phase 5 step 1). */
|
|
47
|
+
export const RECORD_REL = path.join('agents', 'evidence', 'metrics', 'evaluator-measurements.json');
|
|
48
|
+
|
|
25
49
|
interface BudgetEntry {
|
|
26
50
|
max: number;
|
|
27
51
|
last_measured: number;
|
|
@@ -30,51 +54,174 @@ interface BudgetEntry {
|
|
|
30
54
|
* (absolute cap is their real gate); sizes/counts keep the tight
|
|
31
55
|
* global default. */
|
|
32
56
|
regression_pct?: number;
|
|
57
|
+
/** `false` for wall-clock metrics: they cannot be compared run-to-run,
|
|
58
|
+
* so they are excluded from the drift check. Missing = deterministic. */
|
|
59
|
+
deterministic?: boolean;
|
|
33
60
|
}
|
|
34
61
|
interface BudgetsDoc {
|
|
35
62
|
regression_pct: number;
|
|
36
63
|
budgets: Record<string, BudgetEntry>;
|
|
37
64
|
}
|
|
38
65
|
|
|
39
|
-
|
|
66
|
+
/** Shape of the committed measurement record. */
|
|
67
|
+
export interface MeasurementRecord {
|
|
68
|
+
schema_version: number;
|
|
69
|
+
recorded_at: string;
|
|
70
|
+
git_sha?: string;
|
|
71
|
+
run_url?: string;
|
|
72
|
+
conditions?: string;
|
|
73
|
+
measurements: Record<string, number | string>;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export type Severity =
|
|
77
|
+
/** Always blocks — the harness itself is broken. */
|
|
78
|
+
| 'structural'
|
|
79
|
+
/** Blocks only under the `fail` posture. */
|
|
80
|
+
| 'advisory';
|
|
81
|
+
|
|
82
|
+
export interface Finding {
|
|
83
|
+
metric: string;
|
|
84
|
+
severity: Severity;
|
|
85
|
+
message: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function evaluateFindings(
|
|
40
89
|
budgetsDoc: BudgetsDoc,
|
|
41
90
|
measurements: Record<string, number>,
|
|
42
|
-
):
|
|
43
|
-
const
|
|
91
|
+
): Finding[] {
|
|
92
|
+
const findings: Finding[] = [];
|
|
44
93
|
for (const [metric, entry] of Object.entries(budgetsDoc.budgets)) {
|
|
45
94
|
const measured = measurements[metric];
|
|
46
95
|
if (measured === undefined || Number.isNaN(measured)) {
|
|
47
|
-
|
|
96
|
+
findings.push({
|
|
97
|
+
metric,
|
|
98
|
+
severity: 'structural',
|
|
99
|
+
message: `${metric}: no measurement supplied (method: ${entry.method})`,
|
|
100
|
+
});
|
|
48
101
|
continue;
|
|
49
102
|
}
|
|
50
103
|
if (measured > entry.max) {
|
|
51
|
-
|
|
104
|
+
findings.push({
|
|
105
|
+
metric,
|
|
106
|
+
severity: 'advisory',
|
|
107
|
+
message: `${metric}: measured ${measured} exceeds budget ${entry.max}`,
|
|
108
|
+
});
|
|
52
109
|
continue;
|
|
53
110
|
}
|
|
54
111
|
const pct = entry.regression_pct ?? budgetsDoc.regression_pct;
|
|
55
112
|
const ceiling = entry.last_measured * (1 + pct / 100);
|
|
56
113
|
if (entry.last_measured > 0 && measured > ceiling) {
|
|
57
|
-
|
|
58
|
-
|
|
114
|
+
findings.push({
|
|
115
|
+
metric,
|
|
116
|
+
severity: 'advisory',
|
|
117
|
+
message:
|
|
118
|
+
`${metric}: measured ${measured} regressed >${pct}% vs ` +
|
|
59
119
|
`last_measured ${entry.last_measured} (ceiling ${ceiling.toFixed(2)}) — ` +
|
|
60
120
|
'fails even under the absolute budget',
|
|
61
|
-
);
|
|
121
|
+
});
|
|
62
122
|
}
|
|
63
123
|
}
|
|
64
|
-
return
|
|
124
|
+
return findings;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Back-compat surface: the flat message list. Kept because the budget shape
|
|
129
|
+
* and the per-metric rules are pinned by tests through this function.
|
|
130
|
+
*/
|
|
131
|
+
export function evaluate(budgetsDoc: BudgetsDoc, measurements: Record<string, number>): string[] {
|
|
132
|
+
return evaluateFindings(budgetsDoc, measurements).map((f) => f.message);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Compare a fresh measurement set against the committed record.
|
|
137
|
+
*
|
|
138
|
+
* Only DETERMINISTIC metrics participate: comparing wall-clock timings
|
|
139
|
+
* run-to-run produces noise, and a noisy comparison gets ignored — which
|
|
140
|
+
* reproduces the failure the record exists to fix. A metric absent from
|
|
141
|
+
* either side is skipped (a missing measurement is already a structural
|
|
142
|
+
* finding above; a metric absent from the record is simply not recorded yet).
|
|
143
|
+
*/
|
|
144
|
+
export function driftFindings(
|
|
145
|
+
budgetsDoc: BudgetsDoc,
|
|
146
|
+
measurements: Record<string, number>,
|
|
147
|
+
record: MeasurementRecord | null,
|
|
148
|
+
): Finding[] {
|
|
149
|
+
if (record === null) return [];
|
|
150
|
+
const findings: Finding[] = [];
|
|
151
|
+
for (const [metric, entry] of Object.entries(budgetsDoc.budgets)) {
|
|
152
|
+
if (entry.deterministic === false) continue;
|
|
153
|
+
const fresh = measurements[metric];
|
|
154
|
+
const recorded = record.measurements[metric];
|
|
155
|
+
if (fresh === undefined || recorded === undefined) continue;
|
|
156
|
+
if (typeof recorded !== 'number') continue;
|
|
157
|
+
if (fresh === recorded) continue;
|
|
158
|
+
findings.push({
|
|
159
|
+
metric,
|
|
160
|
+
severity: 'advisory',
|
|
161
|
+
message:
|
|
162
|
+
`${metric}: measured ${fresh} but the committed record says ${recorded} ` +
|
|
163
|
+
`(recorded ${record.recorded_at}` +
|
|
164
|
+
`${record.git_sha ? ` @ ${record.git_sha.slice(0, 8)}` : ''}) — refresh ` +
|
|
165
|
+
`${RECORD_REL} in the PR that moves the number, or explain the move`,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return findings;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export type Posture = 'warn' | 'fail';
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Where are we, and does a budget finding block?
|
|
175
|
+
*
|
|
176
|
+
* Release context → `fail`. Local (no CI env) → `fail`, deliberately: a
|
|
177
|
+
* maintainer running this by hand — including `release.ts` — wants the strict
|
|
178
|
+
* answer, and a local run blocks nobody else. Everything else (main, the
|
|
179
|
+
* nightly, a feature PR) → `warn`.
|
|
180
|
+
*/
|
|
181
|
+
export function detectPosture(
|
|
182
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
183
|
+
override?: string,
|
|
184
|
+
): { posture: Posture; reason: string } {
|
|
185
|
+
if (override === 'warn' || override === 'fail') {
|
|
186
|
+
return { posture: override, reason: `--posture ${override}` };
|
|
187
|
+
}
|
|
188
|
+
const head = env['GITHUB_HEAD_REF'] ?? '';
|
|
189
|
+
const base = env['GITHUB_BASE_REF'] ?? '';
|
|
190
|
+
const ref = env['GITHUB_REF'] ?? '';
|
|
191
|
+
if (head.startsWith('release/')) {
|
|
192
|
+
return { posture: 'fail', reason: `release PR (head ${head})` };
|
|
193
|
+
}
|
|
194
|
+
if (base.startsWith('release/')) {
|
|
195
|
+
return { posture: 'fail', reason: `PR into a release branch (base ${base})` };
|
|
196
|
+
}
|
|
197
|
+
if (ref.startsWith('refs/heads/release/') || ref.startsWith('refs/tags/')) {
|
|
198
|
+
return { posture: 'fail', reason: `release ref (${ref})` };
|
|
199
|
+
}
|
|
200
|
+
if (!env['GITHUB_ACTIONS']) {
|
|
201
|
+
return { posture: 'fail', reason: 'local run (no CI env) — strict by default' };
|
|
202
|
+
}
|
|
203
|
+
return { posture: 'warn', reason: `non-release CI context (${ref || 'unknown ref'})` };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function argValue(argv: string[], flag: string): string | undefined {
|
|
207
|
+
const i = argv.indexOf(flag);
|
|
208
|
+
return i >= 0 ? argv[i + 1] : undefined;
|
|
65
209
|
}
|
|
66
210
|
|
|
67
211
|
export function main(argv: string[] = process.argv.slice(2)): number {
|
|
68
|
-
const
|
|
69
|
-
if (
|
|
70
|
-
process.stderr.write(
|
|
212
|
+
const measurementsPath = argValue(argv, '--measurements');
|
|
213
|
+
if (measurementsPath === undefined) {
|
|
214
|
+
process.stderr.write(
|
|
215
|
+
'usage: check_evaluator_budgets --measurements <file.json> ' +
|
|
216
|
+
'[--recorded <file.json>] [--posture warn|fail]\n',
|
|
217
|
+
);
|
|
71
218
|
return 2;
|
|
72
219
|
}
|
|
73
220
|
let budgetsDoc: BudgetsDoc;
|
|
74
221
|
let measurements: Record<string, number>;
|
|
75
222
|
try {
|
|
76
223
|
budgetsDoc = JSON.parse(fs.readFileSync(BUDGETS_PATH, 'utf-8')) as BudgetsDoc;
|
|
77
|
-
measurements = JSON.parse(fs.readFileSync(
|
|
224
|
+
measurements = JSON.parse(fs.readFileSync(measurementsPath, 'utf-8')) as Record<
|
|
78
225
|
string,
|
|
79
226
|
number
|
|
80
227
|
>;
|
|
@@ -82,13 +229,45 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
82
229
|
process.stderr.write(`check_evaluator_budgets: ${(e as Error).message}\n`);
|
|
83
230
|
return 2;
|
|
84
231
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
232
|
+
|
|
233
|
+
// The committed record is optional: a fresh checkout before the first
|
|
234
|
+
// nightly has none, and a missing record must not fake a green drift check.
|
|
235
|
+
const recordPath = argValue(argv, '--recorded') ?? path.join(REPO_ROOT, RECORD_REL);
|
|
236
|
+
let record: MeasurementRecord | null = null;
|
|
237
|
+
try {
|
|
238
|
+
record = JSON.parse(fs.readFileSync(recordPath, 'utf-8')) as MeasurementRecord;
|
|
239
|
+
} catch {
|
|
240
|
+
process.stdout.write(
|
|
241
|
+
`ℹ️ no measurement record at ${recordPath} — drift check skipped ` +
|
|
242
|
+
'(the nightly publishes it)\n',
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const { posture, reason } = detectPosture(process.env, argValue(argv, '--posture'));
|
|
247
|
+
const findings = [
|
|
248
|
+
...evaluateFindings(budgetsDoc, measurements),
|
|
249
|
+
...driftFindings(budgetsDoc, measurements, record),
|
|
250
|
+
];
|
|
251
|
+
const blocking = findings.filter((f) => f.severity === 'structural' || posture === 'fail');
|
|
252
|
+
const advisory = findings.filter((f) => !blocking.includes(f));
|
|
253
|
+
|
|
254
|
+
for (const f of advisory) {
|
|
255
|
+
process.stderr.write(`⚠️ evaluator budget (advisory, posture=${posture}): ${f.message}\n`);
|
|
256
|
+
}
|
|
257
|
+
for (const f of blocking) {
|
|
258
|
+
process.stderr.write(`❌ evaluator budget: ${f.message}\n`);
|
|
259
|
+
}
|
|
260
|
+
if (blocking.length > 0) {
|
|
261
|
+
process.stderr.write(
|
|
262
|
+
`check_evaluator_budgets: ${blocking.length} blocking finding(s) — ` +
|
|
263
|
+
`posture ${posture} (${reason})\n`,
|
|
264
|
+
);
|
|
88
265
|
return 1;
|
|
89
266
|
}
|
|
90
267
|
process.stdout.write(
|
|
91
|
-
`✅ evaluator budgets met (${Object.keys(budgetsDoc.budgets).length} metrics
|
|
268
|
+
`✅ evaluator budgets met (${Object.keys(budgetsDoc.budgets).length} metrics` +
|
|
269
|
+
`${advisory.length > 0 ? `, ${advisory.length} advisory finding(s)` : ''}) — ` +
|
|
270
|
+
`posture ${posture} (${reason})\n`,
|
|
92
271
|
);
|
|
93
272
|
return 0;
|
|
94
273
|
}
|