@event4u/agent-config 12.0.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +273 -0
- package/README.md +1 -1
- package/agents/templates/.ai-council.yml.example +46 -3
- package/dist/agent-src/commands/cost/report.md +23 -0
- package/dist/agent-src/contexts/execution/subagent-routing.md +148 -33
- package/dist/agent-src/contexts/model-recommendations.md +6 -4
- package/dist/agent-src/contexts/subagent-configuration.md +69 -7
- package/dist/agent-src/rules/autonomous-execution.md +1 -1
- package/dist/agent-src/rules/design-fidelity.md +1 -1
- package/dist/agent-src/rules/design-review-after-ui-write.md +10 -5
- package/dist/agent-src/rules/evaluator-independence.md +27 -5
- package/dist/agent-src/rules/git-history-discipline.md +5 -1
- package/dist/agent-src/rules/low-impact-corpus-privacy-floor.md +1 -0
- package/dist/agent-src/rules/missing-skill-recovery.md +89 -0
- package/dist/agent-src/rules/ui-audit-gate.md +9 -4
- package/dist/agent-src/scripts/archive_completed_roadmaps.ts +33 -0
- package/dist/agent-src/scripts/roadmap_gates.ts +30 -2
- package/dist/agent-src/scripts/update_roadmap_progress.ts +47 -1
- package/dist/agent-src/skills/authz-review/SKILL.md +4 -0
- package/dist/agent-src/skills/design-system-capture/SKILL.md +4 -0
- package/dist/agent-src/skills/design-system-capture/references/snapshot-preference-order.md +77 -0
- package/dist/agent-src/skills/merge-conflicts/SKILL.md +4 -0
- package/dist/agent-src/skills/prompt-optimizer/SKILL.md +17 -1
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +11 -11
- package/dist/agent-src/skills/systematic-debugging/SKILL.md +4 -0
- package/dist/agent-src/skills/threat-modeling/SKILL.md +3 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/roadmaps.md +41 -3
- package/dist/cli/registry.js +1 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/{chunk-NN4KW4HZ.js → chunk-3B72LSZL.js} +123 -2338
- package/dist/cli-delegate/{chunk-55WZN5VF.js → chunk-EVTY7ISR.js} +2 -2
- package/dist/cli-delegate/chunk-N7FEST6D.js +3895 -0
- package/dist/cli-delegate/{chunk-L565YFXH.js → chunk-XURJ4AQC.js} +566 -298
- package/dist/cli-delegate/cmd_conformance.js +4 -4
- package/dist/cli-delegate/cmd_doctor.js +2 -2
- package/dist/cli-delegate/cmd_export.js +2 -2
- package/dist/cli-delegate/cmd_handoff.js +1 -1
- package/dist/cli-delegate/cmd_prune.js +2 -2
- package/dist/cli-delegate/cmd_settings_migrate.js +2 -2
- package/dist/cli-delegate/cmd_settings_set.js +1 -1
- package/dist/cli-delegate/cmd_sync.js +2 -2
- package/dist/cli-delegate/cmd_uninstall.js +2 -2
- package/dist/cli-delegate/cmd_validate.js +2 -2
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +44 -23
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +3 -3
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +6 -5
- package/dist/discovery/trust-report.md +2 -2
- package/dist/discovery/workspaces.json +5 -4
- package/dist/hooks/dispatch.js +10625 -8541
- package/dist/install/install.mjs +754 -484
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +1302 -167
- package/dist/router.json +1 -1
- package/dist/scripts/_lib/model_tier.js +9 -0
- package/dist/scripts/_lib/model_tier.js.map +1 -1
- package/dist/scripts/condense.js +50 -0
- package/dist/scripts/condense.js.map +1 -1
- package/dist/server/schemas/settings.js +2 -2
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/MIGRATION.md +20 -1
- package/docs/guidelines/design-fidelity-mechanics.md +21 -0
- package/docs/guidelines/design-handover-extraction.md +87 -0
- package/docs/guidelines/prompt-templates.md +24 -1
- package/package.json +1 -1
- package/src/agent-src/scripts/archive_completed_roadmaps.ts +33 -0
- package/src/agent-src/scripts/roadmap_gates.ts +30 -2
- package/src/agent-src/scripts/update_roadmap_progress.ts +47 -1
- package/src/config/evaluator-budgets.json +9 -8
- package/src/config/gate-coverage.yml +63 -11
- package/src/config/gate-violation-baselines.json +12 -0
- package/src/config/hook-token-budget.json +15 -2
- package/src/config/preamble-payload-budget.json +28 -0
- package/src/scripts/_dispatch.bash +10 -0
- package/src/scripts/_lib/bench_ab_complexity.ts +439 -0
- package/src/scripts/_lib/bench_ab_safety_tier.ts +142 -0
- package/src/scripts/_lib/bench_ab_scoring_v2.ts +28 -3
- package/src/scripts/_lib/bench_ab_search_adherence.ts +423 -0
- package/src/scripts/_lib/bench_ab_size_claim.ts +206 -0
- package/src/scripts/_lib/bench_ab_workspace.ts +111 -0
- package/src/scripts/_lib/git_common_dir.ts +173 -0
- package/src/scripts/_lib/model_tier.ts +9 -0
- package/src/scripts/_lib/release_highlights.ts +108 -7
- package/src/scripts/_lib/skill_catalogue.ts +1115 -0
- package/src/scripts/_lib/tier_budget_routing.ts +31 -331
- package/src/scripts/ai_council/clients.ts +197 -18
- package/src/scripts/ai_council/config.ts +16 -1
- package/src/scripts/ai_council/low_impact.ts +14 -1
- package/src/scripts/ai_council/orchestrator.ts +14 -1
- package/src/scripts/ai_council/transport_resolver.ts +32 -2
- package/src/scripts/bench_ab_v2_complexity.ts +252 -0
- package/src/scripts/bench_ab_v2_run.ts +3 -42
- package/src/scripts/bench_ab_v2_safety.ts +296 -0
- package/src/scripts/bench_ab_v2_search.ts +343 -0
- package/src/scripts/bench_ab_v2_stats.ts +191 -0
- package/src/scripts/build_archive_index.ts +383 -0
- package/src/scripts/capture_skill_catalogue.ts +279 -317
- package/src/scripts/check_branch_freshness.ts +7 -1
- package/src/scripts/check_release_published.ts +6 -2
- package/src/scripts/check_release_trunk_sync.ts +18 -2
- package/src/scripts/check_source_size_budget.ts +428 -0
- package/src/scripts/check_suppression_hygiene.ts +5 -1
- package/src/scripts/condense.ts +50 -1
- package/src/scripts/cost/backfill_rates.mjs +298 -0
- package/src/scripts/cost/budget.mjs +12 -32
- package/src/scripts/cost_summary.ts +25 -0
- package/src/scripts/dispatch_r2_reviewer.ts +15 -5
- package/src/scripts/evidence_report.ts +15 -4
- package/src/scripts/external_sources_denylist.json +1 -0
- package/src/scripts/hook_manifest.yaml +75 -15
- package/src/scripts/hooks/concern_registry.ts +4 -0
- package/src/scripts/hooks/orchestration_record_hook.ts +21 -0
- package/src/scripts/hooks/skill_route_hook.ts +226 -0
- package/src/scripts/hooks/tool_result_bytes_hook.ts +219 -0
- package/src/scripts/hooks/ui_route_nudge_hook.ts +6 -3
- package/src/scripts/hot_context_hook.ts +13 -1
- package/src/scripts/install.ts +129 -0
- package/src/scripts/janitor.ts +77 -2
- package/src/scripts/lint_design_antipattern_parity.ts +40 -6
- package/src/scripts/lint_harvest_provenance.ts +35 -1
- package/src/scripts/lint_plan_risk_register.ts +22 -9
- package/src/scripts/lint_roadmap_blockers.ts +109 -6
- package/src/scripts/lint_rule_skill_pack_reach.ts +43 -0
- package/src/scripts/lint_scheduled_deprecations.ts +738 -0
- package/src/scripts/lint_skill_router_head.ts +21 -0
- package/src/scripts/mcp_server/consumer_tool_catalog.json +26 -3
- package/src/scripts/mcp_server/tool_catalog_source.ts +6 -24
- package/src/scripts/mcp_server/tools.ts +87 -0
- package/src/scripts/migration_status.ts +17 -1
- package/src/scripts/probe_review_binding_drift.ts +731 -0
- package/src/scripts/release.ts +100 -6
- package/src/scripts/routing_doctor.ts +50 -13
- package/src/scripts/schemas/command.schema.json +3 -3
- package/src/scripts/schemas/skill.schema.json +20 -2
- package/src/scripts/schemas/subagent.schema.json +2 -2
- package/src/scripts/session_eol_report.ts +81 -0
- package/src/scripts/trigger_coverage.ts +224 -3
- package/src/scripts/workspace_doctor.ts +464 -0
- package/src/scripts/worktree_cleanup_check.ts +2 -1
- package/src/server/schemas/settings.ts +2 -2
- package/dist/cli-delegate/chunk-AYKQGIBG.js +0 -1594
- package/src/config/budget-routing.json +0 -7
|
@@ -46,6 +46,15 @@ const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
|
46
46
|
const REPORTS_DIR = path.join(REPO_ROOT, 'internal', 'bench', 'reports', 'ab-v2');
|
|
47
47
|
const PRICING_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'pricing.yaml');
|
|
48
48
|
|
|
49
|
+
import { addPair, newPairedSample, searchClaimSection, searchClaimVerdict } from './_lib/bench_ab_search_adherence.js';
|
|
50
|
+
import {
|
|
51
|
+
evaluateSizeClaim,
|
|
52
|
+
renderSizeClaimSection,
|
|
53
|
+
type PairedContinuous,
|
|
54
|
+
type PairedRate,
|
|
55
|
+
type SizeClaimVerdict,
|
|
56
|
+
} from './_lib/bench_ab_size_claim.js';
|
|
57
|
+
|
|
49
58
|
type Dict = Record<string, unknown>;
|
|
50
59
|
|
|
51
60
|
const COMPARISONS: Array<[string, string, string]> = [
|
|
@@ -63,6 +72,10 @@ const COMPARISONS: Array<[string, string, string]> = [
|
|
|
63
72
|
['rules-kernel-dc', 'vanilla', 'kernel+downstream rules lift'],
|
|
64
73
|
['rules-balanced', 'vanilla', 'balanced-profile rules lift'],
|
|
65
74
|
['package', 'rules-kernel-dc', 'full package residual over kernel+downstream'],
|
|
75
|
+
// Phase-3 T1: the size claim is `package-ladder` vs `package`, and it is a
|
|
76
|
+
// metric PAIR — see `size_claim_verdict`. Arm-guarded like every row above,
|
|
77
|
+
// so a run without the ladder arm renders exactly as before.
|
|
78
|
+
['package-ladder', 'package', 'ladder size claim (T1/T2 pair)'],
|
|
66
79
|
];
|
|
67
80
|
|
|
68
81
|
// ── CPython math.erf / math.comb ports ────────────────────────────────────
|
|
@@ -312,6 +325,23 @@ export function compare(records: Dict[], arm_t: string, arm_b: string): Dict {
|
|
|
312
325
|
let dropped_baseline_only = 0;
|
|
313
326
|
let dropped_both = 0;
|
|
314
327
|
const dropped_buckets: Record<string, number> = {};
|
|
328
|
+
// Delta #11 — the metric PAIR (T1 size, T2 complexity) and the T4 safety
|
|
329
|
+
// disqualifier. Each is collected pair-wise and independently: a pair that
|
|
330
|
+
// carries added lines but no complexity contributes to T1's sample and not
|
|
331
|
+
// to T2's, which is what keeps an absent endpoint reported as absent rather
|
|
332
|
+
// than as a value.
|
|
333
|
+
const size_t: number[] = [];
|
|
334
|
+
const size_b: number[] = [];
|
|
335
|
+
const size_diffs: number[] = [];
|
|
336
|
+
const cx_t: number[] = [];
|
|
337
|
+
const cx_b: number[] = [];
|
|
338
|
+
const cx_diffs: number[] = [];
|
|
339
|
+
let saf_n = 0;
|
|
340
|
+
let saf_t = 0;
|
|
341
|
+
let saf_bl = 0;
|
|
342
|
+
let saf_disc_t = 0;
|
|
343
|
+
let saf_disc_b = 0;
|
|
344
|
+
const sea = newPairedSample(); // T5 — continuous in [0,1], collects like complexity.
|
|
315
345
|
for (const { rt, rb } of _pairs(records, arm_t, arm_b)) {
|
|
316
346
|
pairs_seen += 1;
|
|
317
347
|
const t_err = _pyTruthy(rt['errored']);
|
|
@@ -353,6 +383,36 @@ export function compare(records: Dict[], arm_t: string, arm_b: string): Dict {
|
|
|
353
383
|
dis_t_sum += dt;
|
|
354
384
|
dis_b_sum += db;
|
|
355
385
|
disn += 1;
|
|
386
|
+
|
|
387
|
+
const mt = _dictOr(rt['metrics']);
|
|
388
|
+
const mb = _dictOr(rb['metrics']);
|
|
389
|
+
|
|
390
|
+
const at = _numOrNull(mt['added_lines']);
|
|
391
|
+
const ab = _numOrNull(mb['added_lines']);
|
|
392
|
+
if (at !== null && ab !== null) {
|
|
393
|
+
size_t.push(at);
|
|
394
|
+
size_b.push(ab);
|
|
395
|
+
size_diffs.push(at - ab);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
const ct = _numOrNull(mt['median_cognitive_complexity']);
|
|
399
|
+
const cb = _numOrNull(mb['median_cognitive_complexity']);
|
|
400
|
+
if (ct !== null && cb !== null) {
|
|
401
|
+
cx_t.push(ct);
|
|
402
|
+
cx_b.push(cb);
|
|
403
|
+
cx_diffs.push(ct - cb);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
const st = _boolOrNull(mt['safety_tier_pass']);
|
|
407
|
+
const sb2 = _boolOrNull(mb['safety_tier_pass']);
|
|
408
|
+
if (st !== null && sb2 !== null) {
|
|
409
|
+
saf_n += 1;
|
|
410
|
+
saf_t += st ? 1 : 0;
|
|
411
|
+
saf_bl += sb2 ? 1 : 0;
|
|
412
|
+
if (st && !sb2) saf_disc_t += 1;
|
|
413
|
+
else if (sb2 && !st) saf_disc_b += 1;
|
|
414
|
+
}
|
|
415
|
+
addPair(sea, mt['search_adherence'], mb['search_adherence']);
|
|
356
416
|
}
|
|
357
417
|
}
|
|
358
418
|
void both1;
|
|
@@ -393,9 +453,80 @@ export function compare(records: Dict[], arm_t: string, arm_b: string): Dict {
|
|
|
393
453
|
drop_asymmetry: dropped_treatment_only - dropped_baseline_only,
|
|
394
454
|
dropped_by_status_bucket: dropped_buckets,
|
|
395
455
|
},
|
|
456
|
+
size: _paired_median_block(size_t, size_b, size_diffs),
|
|
457
|
+
complexity: _paired_median_block(cx_t, cx_b, cx_diffs),
|
|
458
|
+
safety: _paired_rate_block(saf_n, saf_t, saf_bl, saf_disc_t, saf_disc_b),
|
|
459
|
+
search: _paired_median_block(sea.t, sea.b, sea.diffs),
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* A paired continuous endpoint (added lines, cognitive complexity).
|
|
465
|
+
*
|
|
466
|
+
* `measured: false` when no analysed pair carried the metric on BOTH sides. It
|
|
467
|
+
* is not a zero and not a neutral value: `size_claim_verdict` reads it as
|
|
468
|
+
* "unmeasured" and refuses to report a win, which is the whole point of shipping
|
|
469
|
+
* the endpoint rather than defaulting it.
|
|
470
|
+
*/
|
|
471
|
+
function _paired_median_block(t: number[], b: number[], diffs: number[]): Dict {
|
|
472
|
+
if (diffs.length === 0) {
|
|
473
|
+
return { measured: false, n_pairs: 0 };
|
|
474
|
+
}
|
|
475
|
+
const mt = _median(t);
|
|
476
|
+
const mb = _median(b);
|
|
477
|
+
const wil = wilcoxon(diffs);
|
|
478
|
+
// Percent change of the medians. A zero baseline has no percent change —
|
|
479
|
+
// report null rather than Infinity, which would render as a win.
|
|
480
|
+
const pct = mb === 0 ? null : PF(_pyRound(((mt - mb) / Math.abs(mb)) * 100, 4));
|
|
481
|
+
return {
|
|
482
|
+
measured: true,
|
|
483
|
+
n_pairs: diffs.length,
|
|
484
|
+
median_treatment: PF(_pyRound(mt, 4)),
|
|
485
|
+
median_baseline: PF(_pyRound(mb, 4)),
|
|
486
|
+
median_delta: PF(_pyRound(mt - mb, 4)),
|
|
487
|
+
median_delta_pct: pct,
|
|
488
|
+
wilcoxon_p: PF(wil.p),
|
|
489
|
+
rank_biserial: PF(wil.rank_biserial),
|
|
490
|
+
n_nonzero: wil.n,
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/** A paired binary endpoint (the safety tier). Same `measured` contract. */
|
|
495
|
+
function _paired_rate_block(n: number, t: number, b: number, disc_t: number, disc_b: number): Dict {
|
|
496
|
+
if (n === 0) {
|
|
497
|
+
return { measured: false, n_pairs: 0 };
|
|
498
|
+
}
|
|
499
|
+
const p1 = t / n;
|
|
500
|
+
const p2 = b / n;
|
|
501
|
+
return {
|
|
502
|
+
measured: true,
|
|
503
|
+
n_pairs: n,
|
|
504
|
+
rate_treatment: PF(_pyRound(p1, 4)),
|
|
505
|
+
rate_baseline: PF(_pyRound(p2, 4)),
|
|
506
|
+
discordant_b_only_treatment: disc_t,
|
|
507
|
+
discordant_c_only_baseline: disc_b,
|
|
508
|
+
mcnemar_p: PF(_pyRound(mcnemar_exact(disc_t, disc_b), 4)),
|
|
396
509
|
};
|
|
397
510
|
}
|
|
398
511
|
|
|
512
|
+
/** Median over an already-collected sample; mean of the two middles when even. */
|
|
513
|
+
function _median(values: readonly number[]): number {
|
|
514
|
+
const s = [...values].sort((a, b) => a - b);
|
|
515
|
+
const mid = Math.floor(s.length / 2);
|
|
516
|
+
if (s.length % 2 === 1) return s[mid] as number;
|
|
517
|
+
return ((s[mid - 1] as number) + (s[mid] as number)) / 2;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function _numOrNull(v: unknown): number | null {
|
|
521
|
+
if (typeof v === 'number' && Number.isFinite(v)) return v;
|
|
522
|
+
return null;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function _boolOrNull(v: unknown): boolean | null {
|
|
526
|
+
if (typeof v === 'boolean') return v;
|
|
527
|
+
return null;
|
|
528
|
+
}
|
|
529
|
+
|
|
399
530
|
/**
|
|
400
531
|
* Recursion's NOVEL discipline lift over rules-only (ADR-106).
|
|
401
532
|
*
|
|
@@ -574,12 +705,64 @@ export function analyse(payload: Dict): Dict {
|
|
|
574
705
|
seeds: payload['seeds'] ?? null,
|
|
575
706
|
n_tasks: records.length,
|
|
576
707
|
comparisons: comps,
|
|
708
|
+
// One verdict per rendered comparison. Present even when every endpoint
|
|
709
|
+
// is absent — an `INCONCLUSIVE` row naming the missing endpoint is the
|
|
710
|
+
// reportable fact; omitting the block would read as "no size question
|
|
711
|
+
// was asked", which is a different claim.
|
|
712
|
+
size_claims: comps.map((c) => size_claim_verdict(c)),
|
|
713
|
+
search_claims: comps.map((c) => searchClaimVerdict(c)), // separate on purpose — see the T5 lib
|
|
577
714
|
status_buckets: bucket_rates(records, arms),
|
|
578
715
|
mean_tokens: mean_tokens_by_arm(records, arms),
|
|
579
716
|
cost: cost_by_arm(records, arms, payload['model'] ? String(payload['model']) : null, PRICING_PATH),
|
|
580
717
|
};
|
|
581
718
|
}
|
|
582
719
|
|
|
720
|
+
// ── the size claim is a PAIR, and safety is a disqualifier ─────────────────
|
|
721
|
+
//
|
|
722
|
+
// The verdict itself lives in `_lib/bench_ab_size_claim.ts`: it is decision
|
|
723
|
+
// logic over already-computed blocks, not statistics, and keeping it there makes
|
|
724
|
+
// it a pure function with no dependency on this pipeline. What stays here is the
|
|
725
|
+
// adapter — the `Dict`-shaped report objects this module speaks are coerced into
|
|
726
|
+
// the typed input that module takes.
|
|
727
|
+
|
|
728
|
+
/** Coerce one `compare()` block into the typed paired-continuous input. */
|
|
729
|
+
function _toContinuous(block: Dict): PairedContinuous {
|
|
730
|
+
return {
|
|
731
|
+
measured: block['measured'] === true,
|
|
732
|
+
n_pairs: _pyFloat(block['n_pairs']),
|
|
733
|
+
median_delta_pct: block['median_delta_pct'] == null ? null : _pyFloat(block['median_delta_pct']),
|
|
734
|
+
median_delta: _pyFloat(block['median_delta']),
|
|
735
|
+
wilcoxon_p: _pyFloat(block['wilcoxon_p']),
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
function _toRate(block: Dict): PairedRate {
|
|
740
|
+
return {
|
|
741
|
+
measured: block['measured'] === true,
|
|
742
|
+
n_pairs: _pyFloat(block['n_pairs']),
|
|
743
|
+
rate_treatment: _pyFloat(block['rate_treatment']),
|
|
744
|
+
rate_baseline: _pyFloat(block['rate_baseline']),
|
|
745
|
+
mcnemar_p: _pyFloat(block['mcnemar_p']),
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/** The Phase-3 size claim for one comparison. Contract: `evaluateSizeClaim`. */
|
|
750
|
+
export function size_claim_verdict(comparison: Dict): Dict {
|
|
751
|
+
return evaluateSizeClaim({
|
|
752
|
+
arm_treatment: comparison['arm_treatment'] == null ? null : String(comparison['arm_treatment']),
|
|
753
|
+
arm_baseline: comparison['arm_baseline'] == null ? null : String(comparison['arm_baseline']),
|
|
754
|
+
size: _toContinuous(_dictOr(comparison['size'])),
|
|
755
|
+
complexity: _toContinuous(_dictOr(comparison['complexity'])),
|
|
756
|
+
safety: _toRate(_dictOr(comparison['safety'])),
|
|
757
|
+
}) as unknown as Dict;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
function _size_claim_section(a: Dict): string[] {
|
|
761
|
+
const claims = Array.isArray(a['size_claims']) ? (a['size_claims'] as unknown as SizeClaimVerdict[]) : [];
|
|
762
|
+
return renderSizeClaimSection(claims);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
|
|
583
766
|
export function gate_verdict(analysis: Dict): Dict {
|
|
584
767
|
// L4 gate: PASS if ANY axis shows significant paired lift for package vs
|
|
585
768
|
// vanilla (McNemar p<0.05 OR Wilcoxon p<0.05 OR a status-bucket reduction).
|
|
@@ -604,6 +787,13 @@ export function gate_verdict(analysis: Dict): Dict {
|
|
|
604
787
|
capability_significant: cap_sig,
|
|
605
788
|
discipline_significant: dis_sig,
|
|
606
789
|
status_bucket_better: bucket_better,
|
|
790
|
+
// Goodhart guard, structural: this gate reads capability, discipline and
|
|
791
|
+
// status buckets ONLY. Size never enters it, so no arm can be ranked
|
|
792
|
+
// above another here by producing a smaller diff. The size question has
|
|
793
|
+
// exactly one home — `size_claim_verdict` — and that function refuses a
|
|
794
|
+
// win whenever the safety tier regressed or is unmeasured.
|
|
795
|
+
size_considered: false,
|
|
796
|
+
size_claim_owner: 'size_claim_verdict',
|
|
607
797
|
note:
|
|
608
798
|
'PASS = significant paired discipline/capability lift; ' +
|
|
609
799
|
'FALSIFIED only if also trivial across seeds (inspect n_pairs).',
|
|
@@ -660,6 +850,7 @@ export function to_markdown(analysis: Dict, payload: Dict): string {
|
|
|
660
850
|
L.push(`- discipline lift significant: \`${_pyStr(g['discipline_significant'])}\``);
|
|
661
851
|
L.push(`- status-bucket better (package vs vanilla): \`${_pyStr(g['status_bucket_better'])}\``);
|
|
662
852
|
L.push('');
|
|
853
|
+
L.push(..._size_claim_section(a), '', ...searchClaimSection(a));
|
|
663
854
|
if (g['verdict'] === 'PASS') {
|
|
664
855
|
L.push(
|
|
665
856
|
'> **Measurable discipline lift (significant).** On the scope-creep / ' +
|
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* Build a machine-readable index over the roadmap archive.
|
|
4
|
+
*
|
|
5
|
+
* "Has this already been tried, closed, or refuted?" is answered today by
|
|
6
|
+
* walking `agents/roadmaps/archive/` — 500 top-level files. This emits
|
|
7
|
+
* `INDEX.md` (human) and `index.json` (machine) beside them so the question
|
|
8
|
+
* costs one read instead of a walk.
|
|
9
|
+
*
|
|
10
|
+
* ## What is and is not extracted
|
|
11
|
+
*
|
|
12
|
+
* Every field is derived by a deterministic rule from the file's own bytes.
|
|
13
|
+
* There are NO model-written summaries: a verdict the frontmatter does not
|
|
14
|
+
* carry is emitted as `null` in JSON and `not extractable` in the table, so a
|
|
15
|
+
* reader can tell a missing verdict from an invented one. That is the
|
|
16
|
+
* roadmap's non-goal, enforced here rather than promised.
|
|
17
|
+
*
|
|
18
|
+
* `disposition` is the one derived field, and it is derived from the checkbox
|
|
19
|
+
* tally alone — including acceptance-criteria boxes, which are checkboxes in
|
|
20
|
+
* the same file and are not distinguished. It records how the file LOOKS at
|
|
21
|
+
* archival, never why it was archived:
|
|
22
|
+
*
|
|
23
|
+
* | tally | disposition |
|
|
24
|
+
* |---|---|
|
|
25
|
+
* | no checkbox anywhere | `not-extractable` |
|
|
26
|
+
* | any `[ ]` | `archived-with-open-steps` |
|
|
27
|
+
* | any `[-]`, none open | `closed-with-cancellations` |
|
|
28
|
+
* | any `[~]`, none open/cancelled | `completed-with-deferrals` |
|
|
29
|
+
* | otherwise | `completed` |
|
|
30
|
+
*
|
|
31
|
+
* ## Drift
|
|
32
|
+
*
|
|
33
|
+
* `--check` regenerates in memory and compares byte-for-byte against the two
|
|
34
|
+
* committed artefacts, following the shape `compile_router --check` already
|
|
35
|
+
* uses. A generated artefact nobody re-derives goes stale in one merge.
|
|
36
|
+
*
|
|
37
|
+
* The output carries no timestamp on purpose — a clock in a generated file
|
|
38
|
+
* makes every `--check` run a false red. The generation time lives in git
|
|
39
|
+
* history, the same convention `agents/roadmaps-progress.md` states.
|
|
40
|
+
*
|
|
41
|
+
* Reuses `validate_frontmatter`'s reader; this file adds no second parser.
|
|
42
|
+
*/
|
|
43
|
+
import * as fs from 'node:fs';
|
|
44
|
+
import * as path from 'node:path';
|
|
45
|
+
import process from 'node:process';
|
|
46
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
47
|
+
|
|
48
|
+
import { DeadScopeError, reportScanned } from './_lib/scan_scope.js';
|
|
49
|
+
import { parse_frontmatter } from './validate_frontmatter.js';
|
|
50
|
+
|
|
51
|
+
const _HERE = fileURLToPath(import.meta.url);
|
|
52
|
+
export const ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
53
|
+
|
|
54
|
+
const ARCHIVE_REL = 'agents/roadmaps/archive';
|
|
55
|
+
const INDEX_MD = 'INDEX.md';
|
|
56
|
+
const INDEX_JSON = 'index.json';
|
|
57
|
+
|
|
58
|
+
/** The generated artefacts themselves are never indexed. */
|
|
59
|
+
const SELF: ReadonlySet<string> = new Set([INDEX_MD, INDEX_JSON]);
|
|
60
|
+
|
|
61
|
+
/** `- [ ]` / `* [x]` / `- [~]` / `- [-]`, the four glyphs the roadmaps use. */
|
|
62
|
+
const CHECKBOX_RE = /^[ \t]*[-*][ \t]+\[([ xX~-])\][ \t]/;
|
|
63
|
+
|
|
64
|
+
/** `## Phase 3 — …` or `### Phase 0: …`. Two heading levels are in use. */
|
|
65
|
+
const PHASE_RE = /^#{2,3}[ \t]+Phase\b/;
|
|
66
|
+
|
|
67
|
+
/** The first level-1 heading, used as the title when frontmatter has none. */
|
|
68
|
+
const H1_RE = /^#[ \t]+(.+?)[ \t]*$/;
|
|
69
|
+
|
|
70
|
+
export type Disposition =
|
|
71
|
+
| 'completed'
|
|
72
|
+
| 'completed-with-deferrals'
|
|
73
|
+
| 'closed-with-cancellations'
|
|
74
|
+
| 'archived-with-open-steps'
|
|
75
|
+
| 'not-extractable';
|
|
76
|
+
|
|
77
|
+
export interface StepTally {
|
|
78
|
+
open: number;
|
|
79
|
+
done: number;
|
|
80
|
+
deferred: number;
|
|
81
|
+
cancelled: number;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export interface ArchiveEntry {
|
|
85
|
+
slug: string;
|
|
86
|
+
title: string | null;
|
|
87
|
+
disposition: Disposition;
|
|
88
|
+
phases: number;
|
|
89
|
+
steps: StepTally;
|
|
90
|
+
/** Frontmatter `verdict:`, verbatim. `null` = the file carries none. */
|
|
91
|
+
verdict: string | null;
|
|
92
|
+
/** Frontmatter `status:`, verbatim. Authoring status, not a closure. */
|
|
93
|
+
status: string | null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Frontmatter value → string, or `null` for absent / non-scalar. */
|
|
97
|
+
function _scalar(value: unknown): string | null {
|
|
98
|
+
if (typeof value === 'string') {
|
|
99
|
+
const t = value.trim();
|
|
100
|
+
return t === '' ? null : t;
|
|
101
|
+
}
|
|
102
|
+
if (typeof value === 'number' || typeof value === 'boolean') {
|
|
103
|
+
return String(value);
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Frontmatter, or `{}` when the file has none or carries a malformed block.
|
|
110
|
+
*
|
|
111
|
+
* A 500-file corpus assembled over months contains frontmatter the strict
|
|
112
|
+
* subset parser rejects; that is a property of the archive, not an error in
|
|
113
|
+
* this pass. Such a file still gets an entry — with its frontmatter-derived
|
|
114
|
+
* fields absent, which is exactly what "not extractable" means.
|
|
115
|
+
*/
|
|
116
|
+
function _frontmatter(text: string): Record<string, unknown> {
|
|
117
|
+
try {
|
|
118
|
+
const [data] = parse_frontmatter(text);
|
|
119
|
+
return data ?? {};
|
|
120
|
+
} catch {
|
|
121
|
+
return {};
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Body lines, i.e. everything after a frontmatter block if one is present. */
|
|
126
|
+
function _bodyLines(text: string): string[] {
|
|
127
|
+
const lines = text.split('\n');
|
|
128
|
+
if (lines[0]?.trimEnd() !== '---') {
|
|
129
|
+
return lines;
|
|
130
|
+
}
|
|
131
|
+
for (let i = 1; i < lines.length; i += 1) {
|
|
132
|
+
if (lines[i]?.trimEnd() === '---') {
|
|
133
|
+
return lines.slice(i + 1);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return lines;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function tally(bodyLines: readonly string[]): StepTally {
|
|
140
|
+
const t: StepTally = { open: 0, done: 0, deferred: 0, cancelled: 0 };
|
|
141
|
+
for (const line of bodyLines) {
|
|
142
|
+
const m = CHECKBOX_RE.exec(line);
|
|
143
|
+
if (m === null) {
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
switch (m[1]) {
|
|
147
|
+
case ' ':
|
|
148
|
+
t.open += 1;
|
|
149
|
+
break;
|
|
150
|
+
case '~':
|
|
151
|
+
t.deferred += 1;
|
|
152
|
+
break;
|
|
153
|
+
case '-':
|
|
154
|
+
t.cancelled += 1;
|
|
155
|
+
break;
|
|
156
|
+
default:
|
|
157
|
+
t.done += 1;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return t;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function disposition(t: StepTally): Disposition {
|
|
164
|
+
if (t.open + t.done + t.deferred + t.cancelled === 0) {
|
|
165
|
+
return 'not-extractable';
|
|
166
|
+
}
|
|
167
|
+
if (t.open > 0) {
|
|
168
|
+
return 'archived-with-open-steps';
|
|
169
|
+
}
|
|
170
|
+
if (t.cancelled > 0) {
|
|
171
|
+
return 'closed-with-cancellations';
|
|
172
|
+
}
|
|
173
|
+
if (t.deferred > 0) {
|
|
174
|
+
return 'completed-with-deferrals';
|
|
175
|
+
}
|
|
176
|
+
return 'completed';
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export function buildEntry(slug: string, text: string): ArchiveEntry {
|
|
180
|
+
const fm = _frontmatter(text);
|
|
181
|
+
const body = _bodyLines(text);
|
|
182
|
+
|
|
183
|
+
let title = _scalar(fm['title']);
|
|
184
|
+
if (title === null) {
|
|
185
|
+
for (const line of body) {
|
|
186
|
+
const m = H1_RE.exec(line);
|
|
187
|
+
if (m !== null) {
|
|
188
|
+
title = (m[1] ?? '').trim() || null;
|
|
189
|
+
break;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const steps = tally(body);
|
|
195
|
+
return {
|
|
196
|
+
slug,
|
|
197
|
+
title,
|
|
198
|
+
disposition: disposition(steps),
|
|
199
|
+
phases: body.filter((l) => PHASE_RE.test(l)).length,
|
|
200
|
+
steps,
|
|
201
|
+
verdict: _scalar(fm['verdict']),
|
|
202
|
+
status: _scalar(fm['status']),
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** Top-level `*.md` under the archive, sorted, excluding the generated pair. */
|
|
207
|
+
export function archiveFiles(dir: string): string[] {
|
|
208
|
+
let entries: fs.Dirent[];
|
|
209
|
+
try {
|
|
210
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
211
|
+
} catch {
|
|
212
|
+
return [];
|
|
213
|
+
}
|
|
214
|
+
return entries
|
|
215
|
+
.filter((e) => !e.isDirectory() && e.name.endsWith('.md') && !SELF.has(e.name))
|
|
216
|
+
.map((e) => e.name)
|
|
217
|
+
.sort();
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
export function buildIndex(dir: string): ArchiveEntry[] {
|
|
221
|
+
return archiveFiles(dir).map((name) =>
|
|
222
|
+
buildEntry(name.replace(/\.md$/, ''), fs.readFileSync(path.join(dir, name), 'utf8')),
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Table-cell text: pipes escaped, newlines flattened, never empty. */
|
|
227
|
+
function _cell(value: string | null): string {
|
|
228
|
+
if (value === null) {
|
|
229
|
+
return '_not extractable_';
|
|
230
|
+
}
|
|
231
|
+
return value.replace(/\|/g, '\\|').replace(/\s+/g, ' ').trim();
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
export function renderJson(entries: readonly ArchiveEntry[]): string {
|
|
235
|
+
return (
|
|
236
|
+
JSON.stringify(
|
|
237
|
+
{
|
|
238
|
+
generated_by: 'src/scripts/build_archive_index.ts',
|
|
239
|
+
source: ARCHIVE_REL,
|
|
240
|
+
count: entries.length,
|
|
241
|
+
entries,
|
|
242
|
+
},
|
|
243
|
+
null,
|
|
244
|
+
2,
|
|
245
|
+
) + '\n'
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
export function renderMarkdown(entries: readonly ArchiveEntry[]): string {
|
|
250
|
+
const byDisposition = new Map<Disposition, number>();
|
|
251
|
+
for (const e of entries) {
|
|
252
|
+
byDisposition.set(e.disposition, (byDisposition.get(e.disposition) ?? 0) + 1);
|
|
253
|
+
}
|
|
254
|
+
const summary = [...byDisposition.entries()]
|
|
255
|
+
.sort((a, b) => (a[0] < b[0] ? -1 : 1))
|
|
256
|
+
.map(([d, n]) => `${d} ${String(n)}`)
|
|
257
|
+
.join(' · ');
|
|
258
|
+
|
|
259
|
+
const out: string[] = [
|
|
260
|
+
'# Archive index',
|
|
261
|
+
'',
|
|
262
|
+
'> Auto-generated — do not edit. Regenerate with `task build-archive-index`;',
|
|
263
|
+
'> `task check-archive-index` fails on drift. Generation time lives in git',
|
|
264
|
+
'> history, not in this file (a clock here would make every drift check a',
|
|
265
|
+
'> false red).',
|
|
266
|
+
'',
|
|
267
|
+
`**${String(entries.length)} archived roadmaps** · ${summary}`,
|
|
268
|
+
'',
|
|
269
|
+
'Every column is extracted deterministically from the file itself. No',
|
|
270
|
+
'summary here is model-written: a verdict the frontmatter does not carry',
|
|
271
|
+
'reads _not extractable_ rather than being inferred. `Disposition` is',
|
|
272
|
+
'derived from the checkbox tally alone (acceptance-criteria boxes',
|
|
273
|
+
'included) and records how the file looks, never why it was archived —',
|
|
274
|
+
'so for anything the index marks _not extractable_, open the file.',
|
|
275
|
+
'',
|
|
276
|
+
'**Limits worth knowing before you rely on this.** Only a handful of',
|
|
277
|
+
'archived roadmaps carry a frontmatter `verdict:`, so the index answers',
|
|
278
|
+
'_tried?_ and _closed?_ and sends you to the file for _why_. The measured',
|
|
279
|
+
'saving is 82 % fewer files opened on a pre-registered question set — and',
|
|
280
|
+
'below roughly five candidate files it is a wash, so a single narrow',
|
|
281
|
+
'question is cheaper answered directly. Reading, method and full',
|
|
282
|
+
'distribution: [`archive-index-saving`](../../evidence/analysis/archive-index-saving.md).',
|
|
283
|
+
'',
|
|
284
|
+
'| Roadmap | Title | Disposition | Phases | Steps (done/total) | Verdict |',
|
|
285
|
+
'|---|---|---|---:|---|---|',
|
|
286
|
+
];
|
|
287
|
+
|
|
288
|
+
for (const e of entries) {
|
|
289
|
+
const total = e.steps.open + e.steps.done + e.steps.deferred + e.steps.cancelled;
|
|
290
|
+
const steps = total === 0 ? '—' : `${String(e.steps.done)}/${String(total)}`;
|
|
291
|
+
out.push(
|
|
292
|
+
`| [\`${e.slug}\`](${e.slug}.md) | ${_cell(e.title)} | ${e.disposition} | ` +
|
|
293
|
+
`${String(e.phases)} | ${steps} | ${_cell(e.verdict)} |`,
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
out.push('');
|
|
297
|
+
return out.join('\n');
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function _read(p: string): string | null {
|
|
301
|
+
try {
|
|
302
|
+
return fs.readFileSync(p, 'utf8');
|
|
303
|
+
} catch {
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
export function main(argv: readonly string[]): number {
|
|
309
|
+
const quiet = argv.includes('--quiet');
|
|
310
|
+
const dir = path.join(ROOT, ARCHIVE_REL);
|
|
311
|
+
|
|
312
|
+
const entries = buildIndex(dir);
|
|
313
|
+
try {
|
|
314
|
+
reportScanned({
|
|
315
|
+
gate: 'build_archive_index',
|
|
316
|
+
scanned: entries.length,
|
|
317
|
+
units: 'archived roadmap(s)',
|
|
318
|
+
roots: [ARCHIVE_REL],
|
|
319
|
+
});
|
|
320
|
+
} catch (err) {
|
|
321
|
+
if (err instanceof DeadScopeError) {
|
|
322
|
+
process.stderr.write(`${err.message}\n`);
|
|
323
|
+
return 1;
|
|
324
|
+
}
|
|
325
|
+
throw err;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const mdPath = path.join(dir, INDEX_MD);
|
|
329
|
+
const jsonPath = path.join(dir, INDEX_JSON);
|
|
330
|
+
const md = renderMarkdown(entries);
|
|
331
|
+
const json = renderJson(entries);
|
|
332
|
+
|
|
333
|
+
if (argv.includes('--check')) {
|
|
334
|
+
const stale = [
|
|
335
|
+
_read(mdPath) === md ? null : `${ARCHIVE_REL}/${INDEX_MD}`,
|
|
336
|
+
_read(jsonPath) === json ? null : `${ARCHIVE_REL}/${INDEX_JSON}`,
|
|
337
|
+
].filter((v): v is string => v !== null);
|
|
338
|
+
if (stale.length > 0) {
|
|
339
|
+
process.stderr.write(
|
|
340
|
+
`❌ archive index out of date (${stale.join(', ')}) — run \`task build-archive-index\`\n`,
|
|
341
|
+
);
|
|
342
|
+
return 1;
|
|
343
|
+
}
|
|
344
|
+
if (!quiet) {
|
|
345
|
+
process.stdout.write('✅ archive index is up to date\n');
|
|
346
|
+
}
|
|
347
|
+
return 0;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
fs.writeFileSync(mdPath, md, 'utf8');
|
|
351
|
+
fs.writeFileSync(jsonPath, json, 'utf8');
|
|
352
|
+
if (!quiet) {
|
|
353
|
+
process.stdout.write(
|
|
354
|
+
`✅ archive index — ${String(entries.length)} roadmap(s) → ` +
|
|
355
|
+
`${ARCHIVE_REL}/{${INDEX_MD},${INDEX_JSON}}\n`,
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
return 0;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function _isCliEntry(): boolean {
|
|
362
|
+
if (process.argv[1] === undefined) {
|
|
363
|
+
return false;
|
|
364
|
+
}
|
|
365
|
+
const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
|
|
366
|
+
if (import.meta.url === argvUrl) {
|
|
367
|
+
return true;
|
|
368
|
+
}
|
|
369
|
+
// A symlinked invocation (an installed projection, or macOS /var →
|
|
370
|
+
// /private/var temp dirs) makes the raw URLs differ: import.meta.url is
|
|
371
|
+
// the resolved real path while argv[1] keeps the symlink path.
|
|
372
|
+
try {
|
|
373
|
+
const here = fs.realpathSync(fileURLToPath(import.meta.url));
|
|
374
|
+
const argv = fs.realpathSync(path.resolve(process.argv[1]));
|
|
375
|
+
return here === argv;
|
|
376
|
+
} catch {
|
|
377
|
+
return false;
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
if (_isCliEntry() || process.argv[1] === _HERE) {
|
|
382
|
+
process.exit(main(process.argv.slice(2)));
|
|
383
|
+
}
|