@event4u/agent-config 9.6.0 → 9.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +207 -942
- package/README.md +21 -21
- package/dist/agent-src/README.md +2 -2
- package/dist/agent-src/commands/commit/in-chunks.md +8 -0
- package/dist/agent-src/commands/commit.md +17 -0
- package/dist/agent-src/commands/pr/create.md +4 -0
- package/dist/agent-src/contexts/override-system.md +75 -5
- package/dist/agent-src/rules/context-hygiene.md +2 -0
- package/dist/agent-src/rules/doc-screenshot-hygiene.md +97 -0
- package/dist/agent-src/rules/domain-safety-pii.md +1 -0
- package/dist/agent-src/rules/downstream-changes.md +31 -1
- package/dist/agent-src/rules/external-code-graph-interop.md +13 -3
- package/dist/agent-src/rules/framework-neutrality-in-generic-skills.md +2 -0
- package/dist/agent-src/rules/git-history-discipline.md +2 -0
- package/dist/agent-src/rules/language-and-tone.md +2 -0
- package/dist/agent-src/rules/lethal-trifecta-guard.md +3 -0
- package/dist/agent-src/rules/media-governance-routing.md +2 -0
- package/dist/agent-src/rules/minimal-safe-diff.md +2 -0
- package/dist/agent-src/rules/no-roadmap-references.md +3 -0
- package/dist/agent-src/rules/non-destructive-by-default.md +20 -0
- package/dist/agent-src/rules/onboarding-gate.md +2 -0
- package/dist/agent-src/rules/output-discipline.md +2 -0
- package/dist/agent-src/rules/persona-governance.md +2 -0
- package/dist/agent-src/rules/preservation-guard.md +3 -0
- package/dist/agent-src/rules/roadmap-progress-sync.md +2 -0
- package/dist/agent-src/rules/secret-vcs-guard.md +106 -0
- package/dist/agent-src/rules/security-sensitive-stop.md +1 -1
- package/dist/agent-src/rules/skill-quality.md +2 -0
- package/dist/agent-src/rules/source-confidentiality.md +2 -0
- package/dist/agent-src/rules/source-of-truth.md +2 -0
- package/dist/agent-src/rules/telegraph-speak.md +2 -0
- package/dist/agent-src/rules/token-optimizer-maintenance.md +2 -0
- package/dist/agent-src/rules/tool-safety.md +1 -0
- package/dist/agent-src/rules/verify-before-complete.md +2 -0
- package/dist/agent-src/skills/adr-create/SKILL.md +17 -1
- package/dist/agent-src/skills/agent-docs-writing/SKILL.md +34 -16
- package/dist/agent-src/skills/agents-md-thin-root/SKILL.md +9 -9
- package/dist/agent-src/skills/code-intelligence/SKILL.md +94 -0
- package/dist/agent-src/skills/gated-reach/SKILL.md +185 -0
- package/dist/agent-src/skills/gated-reach/evals/triggers.json +72 -0
- package/dist/agent-src/skills/image-analyser/SKILL.md +14 -12
- package/dist/agent-src/skills/image-editing/SKILL.md +1 -0
- package/dist/agent-src/skills/readme-writing/SKILL.md +6 -1
- package/dist/agent-src/skills/screenshot-hygiene/SKILL.md +171 -0
- package/dist/agent-src/skills/secrets-management/SKILL.md +44 -1
- package/dist/agent-src/skills/subagent-orchestration/prompts/adversarial-verification-council.md +29 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/do-and-judge-two-stage.md +29 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/do-and-judge.md +29 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/do-competitively.md +29 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/do-in-parallel.md +29 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/do-in-steps.md +29 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/do-in-worktrees.md +29 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/judge-with-debate.md +29 -0
- package/dist/agent-src/skills/verify-repair-loop/SKILL.md +4 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/github-workflows/doc-impact.yml +101 -0
- package/dist/agent-src/templates/github-workflows/secret-scan.yml +65 -0
- package/dist/cli/configRoot.js +112 -0
- package/dist/cli/configRoot.js.map +1 -0
- package/dist/cli/configRoot.test.js +97 -0
- package/dist/cli/configRoot.test.js.map +1 -0
- package/dist/cli/help.js +4 -1
- package/dist/cli/help.js.map +1 -1
- package/dist/cli/main.js +19 -2
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/registry.js +1 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +164 -47
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +4 -4
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +13 -8
- package/dist/discovery/trust-report.md +11 -11
- package/dist/discovery/workspaces.json +11 -6
- package/dist/install/install.mjs +16 -0
- package/dist/mcp/awesome-mcp-servers.row.md +1 -1
- package/dist/mcp/mcp-cloudflare-catalogue.json +1 -1
- package/dist/mcp/registry-manifest.json +4 -4
- package/dist/router.json +1 -1
- package/dist/server/app.js +10 -0
- package/dist/server/app.js.map +1 -1
- package/dist/server/routes/ping.js +18 -0
- package/dist/server/routes/ping.js.map +1 -1
- package/dist/server/schemas/settings.js +8 -0
- package/dist/server/schemas/settings.js.map +1 -1
- package/dist/server/serverInfo.js +6 -2
- package/dist/server/serverInfo.js.map +1 -1
- package/dist/server/token.js +11 -4
- package/dist/server/token.js.map +1 -1
- package/dist/server/writeRoot.js +15 -1
- package/dist/server/writeRoot.js.map +1 -1
- package/dist/shared/capabilities.js +33 -0
- package/dist/shared/capabilities.js.map +1 -0
- package/dist/shared/capabilities.test.js +39 -0
- package/dist/shared/capabilities.test.js.map +1 -0
- package/dist/ui/assets/{index-BiEZiZ31.js → index-CrmZ34Rg.js} +16 -16
- package/dist/ui/assets/index-CrmZ34Rg.js.map +1 -0
- package/dist/ui/index.html +14 -5
- package/docs/CLAIMS.md +64 -38
- package/docs/architecture.md +2 -2
- package/docs/archive/CHANGELOG-6.0.0-overview.md +953 -0
- package/docs/benchmark.md +170 -9
- package/docs/catalog.md +8 -3
- package/docs/comparison.yaml +44 -2
- package/docs/contracts/local-server-ports.md +53 -0
- package/docs/contracts/no-runtime-boundary.md +1 -1
- package/docs/contracts/subagent-boundary.md +15 -0
- package/docs/customization.md +14 -3
- package/docs/decisions/ADR-016-installer-architecture.md +2 -0
- package/docs/decisions/ADR-088-no-external-runtime-federation.md +1 -1
- package/docs/decisions/ADR-094-agent-memory-layer-removal.md +1 -1
- package/docs/decisions/ADR-109-subagent-v1-contract.md +2 -0
- package/docs/decisions/ADR-110-discipline-profile-resolution-locus.md +2 -0
- package/docs/decisions/ADR-113-ci-native-release-label-trigger.md +11 -3
- package/docs/decisions/ADR-122-adversarial-verification-council.md +30 -0
- package/docs/decisions/ADR-123-runtime-security-scope-and-spawn-hardening.md +31 -0
- package/docs/decisions/ADR-124-embedded-engine-doctrine.md +255 -0
- package/docs/decisions/ADR-125-doc-screenshot-anonymization.md +131 -0
- package/docs/decisions/ADR-126-internet-reach-operator-tooling.md +254 -0
- package/docs/decisions/ADR-127-enforcement-claims-must-resolve.md +150 -0
- package/docs/decisions/ADR-128-executable-evidence-and-measured-escalation.md +201 -0
- package/docs/decisions/INDEX.md +21 -0
- package/docs/decisions/engine-reclassification-2026-07.md +104 -0
- package/docs/distribution/awesome-list-submission.md +1 -1
- package/docs/distribution/registries.md +36 -27
- package/docs/enforcement-by-host.md +7 -0
- package/docs/featured-skills.md +2 -2
- package/docs/getting-started-by-role.md +1 -1
- package/docs/getting-started.md +1 -1
- package/docs/governance-advantage.md +1 -1
- package/docs/guides/gated-platform-reads.md +221 -0
- package/docs/positioning-evidence.md +1 -1
- package/docs/proof.md +92 -16
- package/docs/spawn-site-policy.md +72 -0
- package/docs/threat-model.md +2 -1
- package/package.json +6 -3
- package/src/agent-src/README.md +2 -2
- package/src/config/agent-settings.template.yml +43 -0
- package/src/config/reach-channels.yml +217 -0
- package/src/config/reach-prescriptions-intake.yml +200 -0
- package/src/scripts/_dispatch.bash +15 -0
- package/src/scripts/_lib/adversarial_bench_score.ts +106 -0
- package/src/scripts/_lib/exec_evidence.ts +251 -0
- package/src/scripts/_lib/fixtures/secret-detector/negatives.txt +10 -0
- package/src/scripts/_lib/fixtures/secret-detector/positives.txt +11 -0
- package/src/scripts/_lib/kernel_rules.ts +46 -0
- package/src/scripts/_lib/reddit_thread_parse.ts +598 -0
- package/src/scripts/_lib/risk_paths.ts +160 -0
- package/src/scripts/_lib/secret_detector.test.ts +103 -0
- package/src/scripts/_lib/secret_detector.ts +319 -0
- package/src/scripts/_lib/spawn_env.ts +24 -4
- package/src/scripts/_lib/tool_probe.ts +359 -0
- package/src/scripts/adr/regenerate_index.ts +81 -6
- package/src/scripts/audit_adr_coverage.ts +33 -0
- package/src/scripts/bench_adversarial_council.ts +292 -0
- package/src/scripts/build_proof.ts +162 -0
- package/src/scripts/check_adr_frontmatter.ts +195 -0
- package/src/scripts/check_artefact_count_messaging.ts +4 -0
- package/src/scripts/check_backstop_debt.ts +200 -0
- package/src/scripts/check_claims.ts +188 -8
- package/src/scripts/check_comparison.ts +12 -0
- package/src/scripts/check_enforcement_coverage.ts +592 -0
- package/src/scripts/check_kernel_rule_bundle.ts +16 -12
- package/src/scripts/check_no_new_legacy_path.ts +2 -0
- package/src/scripts/check_reach_channels.ts +407 -0
- package/src/scripts/check_reach_staleness.ts +364 -0
- package/src/scripts/check_secret_leak.test.ts +47 -0
- package/src/scripts/check_secret_leak.ts +290 -0
- package/src/scripts/check_source_pointer_freshness.ts +165 -0
- package/src/scripts/code_graph/build.ts +0 -0
- package/src/scripts/code_graph/cli.ts +207 -0
- package/src/scripts/code_graph/detect.ts +134 -0
- package/src/scripts/code_graph/extract.ts +386 -0
- package/src/scripts/code_graph/loader.ts +123 -0
- package/src/scripts/code_graph/query.ts +176 -0
- package/src/scripts/code_graph/sanitize.ts +30 -0
- package/src/scripts/code_graph/types.ts +83 -0
- package/src/scripts/code_graph/validate.ts +63 -0
- package/src/scripts/external_sources_denylist.json +3 -1
- package/src/scripts/generate_subagent_floor.ts +194 -0
- package/src/scripts/hook_manifest.yaml +13 -3
- package/src/scripts/hooks/code_graph_nudge_hook.ts +184 -0
- package/src/scripts/hooks/dispatch_hook.ts +5 -1
- package/src/scripts/iron_law_sha.ts +3 -12
- package/src/scripts/lint_override_kernel_guard.ts +219 -0
- package/src/scripts/lint_readme_serial_comma.ts +169 -0
- package/src/scripts/measure_rule_budget.ts +3 -12
- package/src/scripts/postinstall_gui.ts +129 -0
- package/src/scripts/reach_doctor.ts +1651 -0
- package/src/scripts/release.ts +24 -3
- package/src/scripts/risk_escalation_shadow.ts +160 -0
- package/src/scripts/schemas/agent-settings.schema.json +23 -0
- package/src/scripts/schemas/reach-channels.schema.json +134 -0
- package/src/scripts/schemas/reach-doctor-payload.schema.json +220 -0
- package/src/scripts/schemas/rule.schema.json +10 -0
- package/src/scripts/skill_linter.ts +55 -0
- package/src/scripts/validate_reach_prescriptions.ts +751 -0
- package/src/server/schemas/settings.ts +16 -0
- package/src/shared/capabilities.test.ts +43 -0
- package/src/shared/capabilities.ts +75 -0
- package/src/templates/minimal/agents-overrides-readme.md +20 -0
- package/dist/ui/assets/index-BiEZiZ31.js.map +0 -1
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two-stage residual-detection benchmark runner for the adversarial-council
|
|
3
|
+
* finding-coverage claim (ADR-122, road-to-adversarial-council-benchmark,
|
|
4
|
+
* design: docs/design/adversarial-council-eval.md).
|
|
5
|
+
*
|
|
6
|
+
* Protocol (documented here so the verdict is reproducible; 2 working providers
|
|
7
|
+
* — anthropic + openai — so the design is adapted to isolate panel VALUE without
|
|
8
|
+
* a structural-zero artifact):
|
|
9
|
+
*
|
|
10
|
+
* Stage 1 — define the residual. BOTH providers review the 12 defect fixtures
|
|
11
|
+
* under a NEUTRAL review prompt. Residual R = defects that NEITHER neutral pass
|
|
12
|
+
* caught (the judge-survivable pool: it survived a strong 2-vendor first read).
|
|
13
|
+
*
|
|
14
|
+
* Stage 2 — single skeptic vs cross-vendor panel, on R + the clean controls,
|
|
15
|
+
* under an ADVERSARIAL SKEPTIC prompt (the shipped Mode-9 posture):
|
|
16
|
+
* - single_judge = anthropic-skeptic alone.
|
|
17
|
+
* - panel = reconciled union of {anthropic, openai} skeptics.
|
|
18
|
+
* Because panel ⊇ single, lift ≥ 0 and measures the MARGINAL value the
|
|
19
|
+
* second vendor adds at a fixed skeptic posture — exactly "does a cross-vendor
|
|
20
|
+
* panel out-find a single skeptic on the residual, at no-worse FP".
|
|
21
|
+
*
|
|
22
|
+
* evaluateCouncilBench applies the LOCKED dual threshold → backed | honest-null.
|
|
23
|
+
*
|
|
24
|
+
* Limitation (recorded honestly): with 2 vendors this isolates the 2nd-vendor
|
|
25
|
+
* marginal contribution, not an N-vendor scaling curve. gemini/xai are stubbed
|
|
26
|
+
* or disabled in this CLI build.
|
|
27
|
+
*
|
|
28
|
+
* `--mock` runs the whole pipeline on canned responses (no spend) to pre-validate
|
|
29
|
+
* parsing + residual + scoring + gate in BOTH directions. `--run` fires the real
|
|
30
|
+
* paid cross-vendor calls (maintainer spend-gated).
|
|
31
|
+
*/
|
|
32
|
+
import * as fs from 'node:fs';
|
|
33
|
+
import * as path from 'node:path';
|
|
34
|
+
|
|
35
|
+
import {
|
|
36
|
+
AnthropicClient,
|
|
37
|
+
OpenAIClient,
|
|
38
|
+
load_anthropic_key,
|
|
39
|
+
load_openai_key,
|
|
40
|
+
type ExternalAIClient,
|
|
41
|
+
} from './ai_council/clients.js';
|
|
42
|
+
import { evaluateCouncilBench, type CouncilBenchInputs } from './_lib/adversarial_council_gate.js';
|
|
43
|
+
import {
|
|
44
|
+
caughtDefect,
|
|
45
|
+
isFalsePositive,
|
|
46
|
+
recall,
|
|
47
|
+
fpRate,
|
|
48
|
+
categoryFamily,
|
|
49
|
+
type Finding,
|
|
50
|
+
type GroundTruth,
|
|
51
|
+
} from './_lib/adversarial_bench_score.js';
|
|
52
|
+
|
|
53
|
+
const REPO_ROOT = process.cwd();
|
|
54
|
+
const CORPUS_PATH = path.join(REPO_ROOT, 'internal/bench/adversarial-council/corpus.json');
|
|
55
|
+
const SINGLE_SKEPTIC = 'anthropic'; // the single-judge baseline in stage 2
|
|
56
|
+
|
|
57
|
+
interface CorpusItem {
|
|
58
|
+
id: string;
|
|
59
|
+
class: string;
|
|
60
|
+
defect_category: string;
|
|
61
|
+
is_clean: boolean;
|
|
62
|
+
files: { path: string; content: string }[];
|
|
63
|
+
ground_truth: { defect_files: string[]; defect_summary: string };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function loadCorpus(): CorpusItem[] {
|
|
67
|
+
return JSON.parse(fs.readFileSync(CORPUS_PATH, 'utf8')).items as CorpusItem[];
|
|
68
|
+
}
|
|
69
|
+
function groundTruth(item: CorpusItem): GroundTruth {
|
|
70
|
+
return {
|
|
71
|
+
id: item.id,
|
|
72
|
+
is_clean: item.is_clean,
|
|
73
|
+
defect_files: item.ground_truth.defect_files,
|
|
74
|
+
defect_category: item.is_clean ? 'none' : categoryFamily(item.defect_category),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function codeBlock(item: CorpusItem): string {
|
|
78
|
+
return item.files.map((f) => `FILE: ${f.path}\n\`\`\`ts\n${f.content}\`\`\``).join('\n\n');
|
|
79
|
+
}
|
|
80
|
+
const JSON_SHAPE =
|
|
81
|
+
'{"findings":[{"file":"<file path>","category":"<defect category>","confidence":"high|medium|low","summary":"<one line>"}]}';
|
|
82
|
+
function neutralPrompt(item: CorpusItem): string {
|
|
83
|
+
return [
|
|
84
|
+
'You are a senior code reviewer. Review the TypeScript below for REAL defects and report only genuine bugs you are confident about. If it is correct, report an empty findings array — do not invent problems.',
|
|
85
|
+
`Respond with STRICT JSON only, exactly: ${JSON_SHAPE}`,
|
|
86
|
+
'',
|
|
87
|
+
codeBlock(item),
|
|
88
|
+
].join('\n');
|
|
89
|
+
}
|
|
90
|
+
function skepticPrompt(item: CorpusItem): string {
|
|
91
|
+
return [
|
|
92
|
+
'You are an adversarial security/correctness skeptic. Assume this code may hide a subtle, non-obvious defect that a normal review missed — a masked access-control/SSRF/traversal flaw, a boundary/logic inversion, a concurrency or resource-leak edge, a cross-file contract mismatch. Hunt for it. Report only defects you can concretely justify; if after hard scrutiny it is genuinely correct, report an empty array (do not fabricate).',
|
|
93
|
+
`Respond with STRICT JSON only, exactly: ${JSON_SHAPE}`,
|
|
94
|
+
'',
|
|
95
|
+
codeBlock(item),
|
|
96
|
+
].join('\n');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function parseFindings(text: string): Finding[] {
|
|
100
|
+
if (!text) return [];
|
|
101
|
+
const start = text.indexOf('{');
|
|
102
|
+
if (start < 0) return [];
|
|
103
|
+
let depth = 0;
|
|
104
|
+
let end = -1;
|
|
105
|
+
for (let i = start; i < text.length; i++) {
|
|
106
|
+
if (text[i] === '{') depth++;
|
|
107
|
+
else if (text[i] === '}' && --depth === 0) {
|
|
108
|
+
end = i;
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (end < 0) return [];
|
|
113
|
+
try {
|
|
114
|
+
const obj = JSON.parse(text.slice(start, end + 1)) as { findings?: unknown };
|
|
115
|
+
const arr = Array.isArray(obj.findings) ? obj.findings : [];
|
|
116
|
+
return arr
|
|
117
|
+
.filter((f): f is Record<string, unknown> => typeof f === 'object' && f !== null)
|
|
118
|
+
.map((f) => ({
|
|
119
|
+
file: String(f.file ?? ''),
|
|
120
|
+
category: String(f.category ?? ''),
|
|
121
|
+
confidence: (['high', 'medium', 'low'].includes(String(f.confidence)) ? String(f.confidence) : 'high') as
|
|
122
|
+
| 'high'
|
|
123
|
+
| 'medium'
|
|
124
|
+
| 'low',
|
|
125
|
+
}))
|
|
126
|
+
.filter((f) => f.file.length > 0);
|
|
127
|
+
} catch {
|
|
128
|
+
return [];
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
type Reviews = Record<string, Record<string, Finding[]>>; // fixtureId → member → findings
|
|
133
|
+
|
|
134
|
+
// Direct, INDEPENDENT client calls — NOT council_cli. The council `run`/`debate`
|
|
135
|
+
// transport imposes multi-round peer-review with cross-member visibility and
|
|
136
|
+
// returns prose ("## Round 2 … Reviewer A"), which (a) violates the claim's
|
|
137
|
+
// independent-skeptic requirement and (b) defeats JSON parsing. We call each
|
|
138
|
+
// provider's client once, in isolation, with a strict-JSON system prompt.
|
|
139
|
+
const SYSTEM_JSON =
|
|
140
|
+
'You are a code reviewer. Output ONLY one JSON object — no prose, no markdown, no code fences. ' +
|
|
141
|
+
'Shape exactly: {"findings":[{"file":"<path>","category":"<category>","confidence":"high|medium|low","summary":"<one line>"}]}. ' +
|
|
142
|
+
'If the code is correct, output {"findings":[]}. Do not include any text before or after the JSON object.';
|
|
143
|
+
|
|
144
|
+
let _clients: Record<string, ExternalAIClient> | null = null;
|
|
145
|
+
function getClients(): Record<string, ExternalAIClient> {
|
|
146
|
+
if (!_clients) {
|
|
147
|
+
_clients = {
|
|
148
|
+
anthropic: new AnthropicClient({ model: 'claude-sonnet-4-5', api_key: load_anthropic_key() }),
|
|
149
|
+
openai: new OpenAIClient({ model: 'gpt-4o', api_key: load_openai_key() }),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
return _clients;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async function directReview(item: CorpusItem, userPrompt: string): Promise<Record<string, Finding[]>> {
|
|
156
|
+
const cs = getClients();
|
|
157
|
+
const out: Record<string, Finding[]> = {};
|
|
158
|
+
for (const [name, client] of Object.entries(cs)) {
|
|
159
|
+
let text = '';
|
|
160
|
+
try {
|
|
161
|
+
const resp = await client.ask(SYSTEM_JSON, userPrompt, 1200);
|
|
162
|
+
text = resp.text ?? '';
|
|
163
|
+
} catch (e) {
|
|
164
|
+
process.stderr.write(` ${name} error on ${item.id}: ${(e as Error).message}\n`);
|
|
165
|
+
}
|
|
166
|
+
out[name] = parseFindings(text);
|
|
167
|
+
}
|
|
168
|
+
return out;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
interface Passes {
|
|
172
|
+
neutral: Reviews; // defect fixtures only
|
|
173
|
+
skeptic: Reviews; // residual + controls
|
|
174
|
+
residualIds: string[];
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async function realPasses(items: CorpusItem[]): Promise<Passes> {
|
|
178
|
+
const defects = items.filter((i) => !i.is_clean);
|
|
179
|
+
const controls = items.filter((i) => i.is_clean);
|
|
180
|
+
const neutral: Reviews = {};
|
|
181
|
+
process.stderr.write('stage 1 (neutral, independent per vendor) over 12 defects…\n');
|
|
182
|
+
for (const d of defects) {
|
|
183
|
+
neutral[d.id] = await directReview(d, neutralPrompt(d));
|
|
184
|
+
process.stderr.write(` neutral ${d.id}: ${Object.entries(neutral[d.id] ?? {}).map(([m, f]) => `${m}=${f.length}`).join(' ')}\n`);
|
|
185
|
+
}
|
|
186
|
+
const residual = defects.filter((d) => !Object.values(neutral[d.id] ?? {}).flat().some((f) => caughtDefect(groundTruth(d), [f])));
|
|
187
|
+
process.stderr.write(`residual (missed by BOTH neutral passes): ${residual.map((r) => r.id).join(', ') || '(none)'}\n`);
|
|
188
|
+
const skeptic: Reviews = {};
|
|
189
|
+
process.stderr.write('stage 2 (adversarial skeptic, independent per vendor) over residual + controls…\n');
|
|
190
|
+
for (const item of [...residual, ...controls]) {
|
|
191
|
+
skeptic[item.id] = await directReview(item, skepticPrompt(item));
|
|
192
|
+
process.stderr.write(` skeptic ${item.id}: ${Object.entries(skeptic[item.id] ?? {}).map(([m, f]) => `${m}=${f.length}`).join(' ')}\n`);
|
|
193
|
+
}
|
|
194
|
+
return { neutral, skeptic, residualIds: residual.map((r) => r.id) };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function mockPasses(items: CorpusItem[]): Passes {
|
|
198
|
+
// Deterministic proof that the pipeline discriminates: neutral 2-vendor pass
|
|
199
|
+
// catches the "easier" defects; a hard residual survives; the skeptic panel
|
|
200
|
+
// then recovers more of the residual than a single skeptic, at controlled FP.
|
|
201
|
+
const defects = items.filter((i) => !i.is_clean);
|
|
202
|
+
const controls = items.filter((i) => i.is_clean);
|
|
203
|
+
const neutralCatch = new Set(['sec-01', 'sec-02', 'inv-01', 'inv-02', 'state-03', 'mfi-01']); // 6 caught neutrally
|
|
204
|
+
// residual = the other 6: sec-03, inv-03, state-01, state-02, mfi-02, mfi-03
|
|
205
|
+
const anthropicSkepticCatch = new Set(['sec-03', 'state-01']); // single skeptic recovers 2/6
|
|
206
|
+
const openaiSkepticExtra = new Set(['inv-03', 'mfi-02', 'mfi-03']); // openai adds 3 more → panel 5/6
|
|
207
|
+
const hit = (i: CorpusItem): Finding[] => [
|
|
208
|
+
{ file: i.ground_truth.defect_files[0] ?? i.files[0]?.path ?? '', category: i.defect_category, confidence: 'high' as const },
|
|
209
|
+
];
|
|
210
|
+
const neutral: Reviews = {};
|
|
211
|
+
for (const d of defects) neutral[d.id] = { anthropic: neutralCatch.has(d.id) ? hit(d) : [], openai: [] };
|
|
212
|
+
const skeptic: Reviews = {};
|
|
213
|
+
const residual = defects.filter((d) => !neutralCatch.has(d.id));
|
|
214
|
+
for (const d of residual) {
|
|
215
|
+
skeptic[d.id] = {
|
|
216
|
+
anthropic: anthropicSkepticCatch.has(d.id) ? hit(d) : [],
|
|
217
|
+
openai: openaiSkepticExtra.has(d.id) ? hit(d) : [],
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
for (const c of controls)
|
|
221
|
+
skeptic[c.id] = { anthropic: [], openai: c.id === 'clean-02' ? [{ file: c.files[0]?.path ?? '', category: 'correctness', confidence: 'high' }] : [] };
|
|
222
|
+
return { neutral, skeptic, residualIds: residual.map((r) => r.id) };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function scorePasses(items: CorpusItem[], p: Passes): { inputs: CouncilBenchInputs; detail: Record<string, unknown> } {
|
|
226
|
+
const byId = new Map(items.map((i) => [i.id, i]));
|
|
227
|
+
const residual = p.residualIds.map((id) => byId.get(id)!);
|
|
228
|
+
const controls = items.filter((i) => i.is_clean);
|
|
229
|
+
|
|
230
|
+
const singleCaught: Record<string, boolean> = {};
|
|
231
|
+
const panelCaught: Record<string, boolean> = {};
|
|
232
|
+
for (const d of residual) {
|
|
233
|
+
const gt = groundTruth(d);
|
|
234
|
+
singleCaught[d.id] = caughtDefect(gt, p.skeptic[d.id]?.[SINGLE_SKEPTIC] ?? []);
|
|
235
|
+
panelCaught[d.id] = caughtDefect(gt, Object.values(p.skeptic[d.id] ?? {}).flat());
|
|
236
|
+
}
|
|
237
|
+
const singleFp: Record<string, boolean> = {};
|
|
238
|
+
const panelFp: Record<string, boolean> = {};
|
|
239
|
+
for (const c of controls) {
|
|
240
|
+
const gt = groundTruth(c);
|
|
241
|
+
singleFp[c.id] = isFalsePositive(gt, p.skeptic[c.id]?.[SINGLE_SKEPTIC] ?? []);
|
|
242
|
+
panelFp[c.id] = isFalsePositive(gt, Object.values(p.skeptic[c.id] ?? {}).flat());
|
|
243
|
+
}
|
|
244
|
+
const inputs: CouncilBenchInputs = {
|
|
245
|
+
single_judge_residual_recall: recall(singleCaught),
|
|
246
|
+
panel_residual_recall: recall(panelCaught),
|
|
247
|
+
single_judge_fp_rate: fpRate(singleFp),
|
|
248
|
+
panel_fp_rate: fpRate(panelFp),
|
|
249
|
+
fp_noise_margin: 1 / 3,
|
|
250
|
+
};
|
|
251
|
+
const detail = {
|
|
252
|
+
total_defects: items.filter((i) => !i.is_clean).length,
|
|
253
|
+
residual_ids: p.residualIds,
|
|
254
|
+
residual_size: residual.length,
|
|
255
|
+
single_skeptic: SINGLE_SKEPTIC,
|
|
256
|
+
single_residual_caught: Object.entries(singleCaught).filter(([, v]) => v).map(([k]) => k),
|
|
257
|
+
panel_residual_caught: Object.entries(panelCaught).filter(([, v]) => v).map(([k]) => k),
|
|
258
|
+
single_fp: Object.entries(singleFp).filter(([, v]) => v).map(([k]) => k),
|
|
259
|
+
panel_fp: Object.entries(panelFp).filter(([, v]) => v).map(([k]) => k),
|
|
260
|
+
};
|
|
261
|
+
return { inputs, detail };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function main(argv: string[]): Promise<number> {
|
|
265
|
+
const mock = argv.includes('--mock');
|
|
266
|
+
const run = argv.includes('--run');
|
|
267
|
+
if (!mock && !run) {
|
|
268
|
+
process.stderr.write('usage: bench_adversarial_council (--mock | --run)\n');
|
|
269
|
+
return 2;
|
|
270
|
+
}
|
|
271
|
+
const items = loadCorpus();
|
|
272
|
+
const passes = mock ? mockPasses(items) : await realPasses(items);
|
|
273
|
+
const { inputs, detail } = scorePasses(items, passes);
|
|
274
|
+
const verdict = evaluateCouncilBench(inputs);
|
|
275
|
+
const report = {
|
|
276
|
+
mode: mock ? 'mock' : 'run',
|
|
277
|
+
corpus: 'internal/bench/adversarial-council/corpus.json',
|
|
278
|
+
protocol:
|
|
279
|
+
'stage1 2-vendor neutral → residual (missed by both); stage2 single skeptic (anthropic) vs cross-vendor panel union on residual; FP on clean controls',
|
|
280
|
+
inputs,
|
|
281
|
+
detail,
|
|
282
|
+
verdict,
|
|
283
|
+
};
|
|
284
|
+
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
285
|
+
return 0;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
289
|
+
void main(process.argv.slice(2)).then((code) => process.exit(code));
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
export { parseFindings, scorePasses, main };
|
|
@@ -41,6 +41,105 @@ class ExitCode extends Error {
|
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* The prevented-failure table.
|
|
46
|
+
*
|
|
47
|
+
* Every row names a **failure mode that is prevented**, never a feature. That
|
|
48
|
+
* constraint is the whole discipline: "cross-platform" is a feature and says
|
|
49
|
+
* nothing; "the same rules do not silently differ between the agent you run
|
|
50
|
+
* today and the one you switch to" names what goes wrong without it.
|
|
51
|
+
*
|
|
52
|
+
* Every row cites a ledger claim, and `backs` is not decoration — a cited id
|
|
53
|
+
* that is missing or not `backed` throws, so the table cannot outlive its
|
|
54
|
+
* evidence. This is the guard against the failure it was modelled on: the
|
|
55
|
+
* reference ships an equivalent table whose headline efficiency figure appears
|
|
56
|
+
* only there, an order of magnitude below the cheapest scenario in its own
|
|
57
|
+
* generated benchmark. A number with nowhere to resolve is exactly what this
|
|
58
|
+
* ledger exists to catch, so the table is held to the ledger's own bar.
|
|
59
|
+
*
|
|
60
|
+
* A row with no resolving claim ships WITHOUT a number rather than with an
|
|
61
|
+
* approximate one.
|
|
62
|
+
*/
|
|
63
|
+
export const PREVENTED_FAILURES: { failure: string; mechanism: string; backs: string }[] = [
|
|
64
|
+
{
|
|
65
|
+
failure: 'A rule says MUST, and nothing anywhere enforces it — so the rule reads as a guarantee while being honour-system.',
|
|
66
|
+
mechanism:
|
|
67
|
+
'Each rule declares `enforced_by:`, and the check **resolves** it: a validator reachable from no taskfile, workflow, or hook manifest counts as unwired, not as covered. Undeclared counts as uncovered.',
|
|
68
|
+
backs: 'enforcement-coverage-resolved',
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
failure:
|
|
72
|
+
'A shipped artefact carries a hidden-Unicode or instruction-smuggling payload, and reaches consumers because only the source tree was scanned.',
|
|
73
|
+
mechanism:
|
|
74
|
+
'Source **and** the condensed projection are scanned in CI; a finding blocks the release before publish, not merely the merge.',
|
|
75
|
+
backs: 'shipped-artifacts-hidden-instruction-scanned',
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
failure:
|
|
79
|
+
'A count in public prose drifts from the source, and the marketing number quietly stops being true.',
|
|
80
|
+
mechanism:
|
|
81
|
+
'Counts are generated from source and drift-checked; the build fails on a count-shaped prose mention that disagrees, or on two different numbers for the same artefact kind.',
|
|
82
|
+
backs: 'skill-count',
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
failure:
|
|
86
|
+
'A host-coverage claim understates or overstates what actually works — for months, on the one surface everybody reads.',
|
|
87
|
+
mechanism: 'The number is pinned by a test over real detection, and the test is re-run as the claim evidence.',
|
|
88
|
+
backs: 'host-agent-count',
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
failure:
|
|
92
|
+
'A non-coding domain skill implies proven correctness it never had, because it was forged on a different domain and never checked against one.',
|
|
93
|
+
mechanism:
|
|
94
|
+
'Domain skills are labelled unvalidated until they pass a sourced domain-truth fixture, and the validated count is ratcheted so it cannot quietly fall.',
|
|
95
|
+
backs: 'domain-soundness-scoped',
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
failure:
|
|
99
|
+
'Behavioural-eval coverage regresses as skills are added, and the suite looks healthy because only the absolute number is reported.',
|
|
100
|
+
mechanism: 'Coverage is measured per tier and ratcheted in CI, so it can only rise; the gap is published rather than implied away.',
|
|
101
|
+
backs: 'eval-coverage-ratcheted',
|
|
102
|
+
},
|
|
103
|
+
];
|
|
104
|
+
|
|
105
|
+
export function renderPreventedFailures(ledger: ReturnType<typeof load_ledger>): string {
|
|
106
|
+
const L: string[] = [];
|
|
107
|
+
L.push('## What this prevents');
|
|
108
|
+
L.push('');
|
|
109
|
+
L.push('Each row is a failure mode, not a feature — and each cites the ledger');
|
|
110
|
+
L.push('claim that backs it. A row whose claim stopped resolving would fail this');
|
|
111
|
+
L.push('page\'s own generator, so the table cannot outlive its evidence.');
|
|
112
|
+
L.push('');
|
|
113
|
+
L.push('| Failure this prevents | How | Backed by |');
|
|
114
|
+
L.push('|---|---|---|');
|
|
115
|
+
for (const row of PREVENTED_FAILURES) {
|
|
116
|
+
const entry = ledger.get(row.backs);
|
|
117
|
+
if (!entry) {
|
|
118
|
+
throw new Error(
|
|
119
|
+
`build_proof: prevented-failure row cites claim '${row.backs}', which has no ledger entry. ` +
|
|
120
|
+
'Bind it in docs/CLAIMS.md or drop the row — the table may not carry an unbacked cell.',
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
if (entry.status !== 'backed') {
|
|
124
|
+
throw new Error(
|
|
125
|
+
`build_proof: prevented-failure row cites claim '${row.backs}', whose ledger status is ` +
|
|
126
|
+
`'${entry.status}', not 'backed'. The table may not carry an unbacked cell.`,
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
const reverifies = entry.evidence.trim().startsWith('exec:') ? ' ⟳' : '';
|
|
130
|
+
L.push(
|
|
131
|
+
`| ${row.failure.replace(/\|/g, '\\|')} | ${row.mechanism.replace(/\|/g, '\\|')} | ` +
|
|
132
|
+
`[\`${row.backs}\`](CLAIMS.md)${reverifies} |`,
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
L.push('');
|
|
136
|
+
L.push('⟳ — the claim carries `exec:` evidence: CI re-runs the command and');
|
|
137
|
+
L.push('compares its exit code, so a stale row turns the build red rather than');
|
|
138
|
+
L.push('ageing quietly into marketing.');
|
|
139
|
+
L.push('');
|
|
140
|
+
return L.join('\n');
|
|
141
|
+
}
|
|
142
|
+
|
|
44
143
|
function render(): string {
|
|
45
144
|
const ledger = load_ledger();
|
|
46
145
|
const entries = [...ledger.values()].sort((a, b) => a.id.localeCompare(b.id));
|
|
@@ -59,6 +158,7 @@ function render(): string {
|
|
|
59
158
|
L.push('every check below on a fresh checkout. This page is itself generated');
|
|
60
159
|
L.push('from those sources and fails CI if it drifts.');
|
|
61
160
|
L.push('');
|
|
161
|
+
L.push(renderPreventedFailures(ledger));
|
|
62
162
|
L.push('## See it run (< 60s, real output)');
|
|
63
163
|
L.push('');
|
|
64
164
|
L.push('');
|
|
@@ -86,6 +186,39 @@ function render(): string {
|
|
|
86
186
|
L.push('');
|
|
87
187
|
L.push(`**${backed.length} backed claim(s)** — all evidence pointers resolve in CI.`);
|
|
88
188
|
L.push('');
|
|
189
|
+
|
|
190
|
+
// Re-verifiable vs. merely-resolving. The distinction is the point: a
|
|
191
|
+
// pointer proves an artefact exists, not that the claim is still true. Any
|
|
192
|
+
// page that reported only the backed total would hide exactly the gap this
|
|
193
|
+
// ledger was built to expose.
|
|
194
|
+
const execBacked = backed.filter((e) => e.evidence.trim().startsWith('exec:'));
|
|
195
|
+
const pointerOnly = backed.filter((e) => !e.evidence.trim().startsWith('exec:'));
|
|
196
|
+
L.push('### How many of those re-derive themselves');
|
|
197
|
+
L.push('');
|
|
198
|
+
L.push(`**${execBacked.length} of ${backed.length}** backed claims carry \`exec:\` evidence —`);
|
|
199
|
+
L.push('CI re-runs the command and compares its exit code to the claim, so a');
|
|
200
|
+
L.push('stale one turns the build red. The other');
|
|
201
|
+
L.push(`**${pointerOnly.length}** rest on a pointer: CI checks that the artefact`);
|
|
202
|
+
L.push('exists and contains what it should, which cannot distinguish a live');
|
|
203
|
+
L.push('claim from one whose producer nobody has run in months.');
|
|
204
|
+
L.push('');
|
|
205
|
+
L.push('That residue is not an oversight, and it is listed rather than rounded');
|
|
206
|
+
L.push('away. A claim can only re-derive itself when a deterministic command');
|
|
207
|
+
L.push("carries the verdict in its exit code. These cannot, for the reasons");
|
|
208
|
+
L.push('given:');
|
|
209
|
+
L.push('');
|
|
210
|
+
L.push('| Claim | Why it cannot re-execute |');
|
|
211
|
+
L.push('|---|---|');
|
|
212
|
+
for (const e of pointerOnly) {
|
|
213
|
+
const ev = e.evidence.trim();
|
|
214
|
+
const why = /^https?:\/\//.test(ev)
|
|
215
|
+
? 'external cite — CI does not fetch the network'
|
|
216
|
+
: /benchmark|bench\/|results-/.test(ev)
|
|
217
|
+
? 'benchmark output — regenerating it needs paid or stochastic model calls no CI job can re-derive'
|
|
218
|
+
: 'prose or contract artefact — no exit code carries the verdict';
|
|
219
|
+
L.push(`| ${e.claim.replace(/\|/g, '\\|').slice(0, 110)} | ${why} |`);
|
|
220
|
+
}
|
|
221
|
+
L.push('');
|
|
89
222
|
L.push('Artefact counts in public prose (skills, commands, governed rules,');
|
|
90
223
|
L.push('guidelines, personas) are **generated from source and CI-drift-checked**:');
|
|
91
224
|
L.push('`update_counts.ts` writes the numbers, `check_artefact_count_messaging.ts`');
|
|
@@ -224,6 +357,35 @@ function render(): string {
|
|
|
224
357
|
L.push(`| ${claim} | ${ours} | ${theirs} | ${r.checkable ? '✅' : '—'} |`);
|
|
225
358
|
}
|
|
226
359
|
}
|
|
360
|
+
|
|
361
|
+
// Second lens on the SAME rows: what each one prevents, for a reader who has
|
|
362
|
+
// not asked "compared to what". Membership is a data property — a row appears
|
|
363
|
+
// here iff it carries a `failure_mode` — so nobody curates a short-list that
|
|
364
|
+
// then drifts from the table above. The pointer is the same pointer, resolved
|
|
365
|
+
// by the same gate, so no cell reaches this view unbound.
|
|
366
|
+
const fmRows = cmpRows.filter((r) => (r.failure_mode ?? '').trim() !== '');
|
|
367
|
+
if (fmRows.length > 0) {
|
|
368
|
+
L.push('');
|
|
369
|
+
L.push('### What each one prevents');
|
|
370
|
+
L.push('');
|
|
371
|
+
L.push('The same rows, read as failure modes rather than as comparisons. Each names');
|
|
372
|
+
L.push('something that goes wrong without the control and carries the identical');
|
|
373
|
+
L.push('resolvable pointer — a projection of the table above, not a second list to');
|
|
374
|
+
L.push('keep in sync.');
|
|
375
|
+
L.push('');
|
|
376
|
+
L.push('| Without it | The control | Evidence |');
|
|
377
|
+
L.push('|---|---|---|');
|
|
378
|
+
for (const r of fmRows) {
|
|
379
|
+
const fm = (r.failure_mode ?? '').replace(/\|/g, '\\|').trim();
|
|
380
|
+
const claim = r.claim.replace(/\|/g, '\\|');
|
|
381
|
+
const filePart = (r.our_evidence.split('#')[0] ?? r.our_evidence).split(':')[0] ?? r.our_evidence;
|
|
382
|
+
const ours = pointer_unresolved(r.our_evidence) === null
|
|
383
|
+
? `[\`${r.our_evidence}\`](../${filePart})`
|
|
384
|
+
: `\`${r.our_evidence}\``;
|
|
385
|
+
L.push(`| ${fm} | ${claim} | ${ours} |`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
227
389
|
L.push('');
|
|
228
390
|
L.push('## 5. Verify it yourself');
|
|
229
391
|
L.push('');
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* ADR frontmatter gate — and the `review_trigger` field that makes a decision
|
|
4
|
+
* revisitable on purpose rather than by archaeology.
|
|
5
|
+
*
|
|
6
|
+
* Two findings drive this. First, ADR frontmatter was **entirely unvalidated**:
|
|
7
|
+
* `validate_frontmatter.ts` covers skills, rules, commands, and personas, and
|
|
8
|
+
* nothing covered `docs/decisions/`. That is why `review_date` drifted into 11
|
|
9
|
+
* ADRs and back out again with nobody noticing.
|
|
10
|
+
*
|
|
11
|
+
* Second, of 128 ADRs, zero named the condition under which they should be
|
|
12
|
+
* reconsidered — while `decision-record` SKILL § step 4 already *mandates* a
|
|
13
|
+
* "Revisit-if" line and `decision-revisit-gate` already treats an ADR as a lock
|
|
14
|
+
* that must be re-openable. The doctrine was canon; the template just never
|
|
15
|
+
* emitted a slot for it, so 33 authors wrote it in prose and it stayed
|
|
16
|
+
* unfindable. This is a missing field, not a missing idea.
|
|
17
|
+
*
|
|
18
|
+
* `review_trigger` is a named CONDITION, never a date. "Review annually" is
|
|
19
|
+
* ignored by everyone and rots into ceremony; "when a second consumer reports a
|
|
20
|
+
* preservation surprise" fires exactly when it should.
|
|
21
|
+
*
|
|
22
|
+
* Grandfathering is by authorship date, not an allowlist: ADRs dated before the
|
|
23
|
+
* switch warn, ADRs dated on or after it fail. No list to maintain and no
|
|
24
|
+
* exemption to forget — a retrofit of 128 files would be busywork that teaches
|
|
25
|
+
* nobody anything.
|
|
26
|
+
*
|
|
27
|
+
* Usage:
|
|
28
|
+
* ./scripts-run src/scripts/check_adr_frontmatter # gate
|
|
29
|
+
* ./scripts-run src/scripts/check_adr_frontmatter --json
|
|
30
|
+
*
|
|
31
|
+
* Exit codes: 0 ok (warnings allowed) · 1 violation · 2 usage/env error.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import * as fs from 'node:fs';
|
|
35
|
+
import * as path from 'node:path';
|
|
36
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
37
|
+
|
|
38
|
+
const _HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
39
|
+
const REPO_ROOT = path.resolve(_HERE, '..', '..');
|
|
40
|
+
const ADR_DIR = path.join(REPO_ROOT, 'docs', 'decisions');
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Grandfathering switch. ADRs dated before this warn on a missing
|
|
44
|
+
* `review_trigger`; ADRs dated on or after it fail.
|
|
45
|
+
*/
|
|
46
|
+
export const REVIEW_TRIGGER_SINCE = '2026-07-25';
|
|
47
|
+
|
|
48
|
+
/** Fields every ADR must carry — the shape that already exists in practice. */
|
|
49
|
+
const REQUIRED = ['adr', 'status', 'date', 'decision'] as const;
|
|
50
|
+
|
|
51
|
+
const ALLOWED_STATUS = new Set([
|
|
52
|
+
'accepted',
|
|
53
|
+
'proposed',
|
|
54
|
+
'superseded',
|
|
55
|
+
'rejected',
|
|
56
|
+
'deprecated',
|
|
57
|
+
'draft',
|
|
58
|
+
]);
|
|
59
|
+
|
|
60
|
+
export interface AdrFinding {
|
|
61
|
+
file: string;
|
|
62
|
+
level: 'error' | 'warn';
|
|
63
|
+
message: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function parse_frontmatter(text: string): Record<string, string> | null {
|
|
67
|
+
if (!text.startsWith('---\n')) return null;
|
|
68
|
+
const end = text.indexOf('\n---\n', 4);
|
|
69
|
+
if (end === -1) return null;
|
|
70
|
+
const out: Record<string, string> = {};
|
|
71
|
+
let key: string | null = null;
|
|
72
|
+
for (const raw of text.slice(4, end).split('\n')) {
|
|
73
|
+
const line = raw.replace(/\s+$/, '');
|
|
74
|
+
if (!line || line.trimStart().startsWith('#')) continue;
|
|
75
|
+
// Continuation of a folded/indented value.
|
|
76
|
+
if (/^\s/.test(line) && key) {
|
|
77
|
+
out[key] = `${out[key]} ${line.trim()}`.trim();
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
const idx = line.indexOf(':');
|
|
81
|
+
if (idx === -1) continue;
|
|
82
|
+
key = line.slice(0, idx).trim();
|
|
83
|
+
out[key] = line
|
|
84
|
+
.slice(idx + 1)
|
|
85
|
+
.trim()
|
|
86
|
+
.replace(/^["'](.*)["']$/, '$1');
|
|
87
|
+
}
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A trigger must name a condition. A bare cadence is the failure mode. */
|
|
92
|
+
export function trigger_is_meaningful(value: string): boolean {
|
|
93
|
+
const v = value.trim();
|
|
94
|
+
if (v.length < 20) return false;
|
|
95
|
+
// "annually" / "every 6 months" / a bare date is a calendar, not a condition.
|
|
96
|
+
if (/^(annually|yearly|quarterly|monthly|every\s+\d+\s+\w+|\d{4}-\d{2}-\d{2})\.?$/i.test(v)) {
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
return true;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function check(dir: string = ADR_DIR): AdrFinding[] {
|
|
103
|
+
const findings: AdrFinding[] = [];
|
|
104
|
+
if (!fs.existsSync(dir)) return findings;
|
|
105
|
+
|
|
106
|
+
for (const name of fs.readdirSync(dir).sort()) {
|
|
107
|
+
if (!/^ADR-.*\.md$/.test(name)) continue;
|
|
108
|
+
const rel = path.relative(REPO_ROOT, path.join(dir, name));
|
|
109
|
+
const fm = parse_frontmatter(fs.readFileSync(path.join(dir, name), 'utf-8'));
|
|
110
|
+
|
|
111
|
+
if (fm === null) {
|
|
112
|
+
findings.push({ file: rel, level: 'error', message: 'no YAML frontmatter block' });
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
for (const req of REQUIRED) {
|
|
116
|
+
if (!fm[req]) findings.push({ file: rel, level: 'error', message: `missing \`${req}\`` });
|
|
117
|
+
}
|
|
118
|
+
if (fm['status'] && !ALLOWED_STATUS.has(fm['status'])) {
|
|
119
|
+
findings.push({
|
|
120
|
+
file: rel,
|
|
121
|
+
level: 'error',
|
|
122
|
+
message: `status \`${fm['status']}\` is not one of: ${[...ALLOWED_STATUS].sort().join(', ')}`,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const date = fm['date'] ?? '';
|
|
127
|
+
const grandfathered = date !== '' && date < REVIEW_TRIGGER_SINCE;
|
|
128
|
+
const trigger = fm['review_trigger'];
|
|
129
|
+
|
|
130
|
+
if (!trigger) {
|
|
131
|
+
findings.push({
|
|
132
|
+
file: rel,
|
|
133
|
+
level: grandfathered ? 'warn' : 'error',
|
|
134
|
+
message: grandfathered
|
|
135
|
+
? `no \`review_trigger\` (grandfathered — dated ${date}, before ${REVIEW_TRIGGER_SINCE})`
|
|
136
|
+
: `missing \`review_trigger\` — name the CONDITION that would reopen this decision`,
|
|
137
|
+
});
|
|
138
|
+
} else if (!trigger_is_meaningful(trigger)) {
|
|
139
|
+
findings.push({
|
|
140
|
+
file: rel,
|
|
141
|
+
level: grandfathered ? 'warn' : 'error',
|
|
142
|
+
message: `\`review_trigger\` is a cadence, not a condition: "${trigger}". A calendar review is ignored; an event fires.`,
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return findings;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function main(argv: string[]): number {
|
|
150
|
+
const as_json = argv.includes('--json');
|
|
151
|
+
const findings = check();
|
|
152
|
+
|
|
153
|
+
if (as_json) {
|
|
154
|
+
process.stdout.write(JSON.stringify({ findings }, null, 2) + '\n');
|
|
155
|
+
return 0;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const errors = findings.filter((f) => f.level === 'error');
|
|
159
|
+
const warns = findings.filter((f) => f.level === 'warn');
|
|
160
|
+
const total = fs.existsSync(ADR_DIR)
|
|
161
|
+
? fs.readdirSync(ADR_DIR).filter((f) => /^ADR-.*\.md$/.test(f)).length
|
|
162
|
+
: 0;
|
|
163
|
+
|
|
164
|
+
process.stdout.write(
|
|
165
|
+
`ADR frontmatter: ${total} ADR(s) · ${errors.length} error(s) · ` +
|
|
166
|
+
`${warns.length} grandfathered without a revisit condition\n`,
|
|
167
|
+
);
|
|
168
|
+
for (const e of errors) process.stderr.write(` ❌ ${e.file}: ${e.message}\n`);
|
|
169
|
+
|
|
170
|
+
if (errors.length > 0) {
|
|
171
|
+
process.stderr.write(
|
|
172
|
+
`\n❌ check_adr_frontmatter: ${errors.length} error(s). ` +
|
|
173
|
+
`ADRs dated ${REVIEW_TRIGGER_SINCE} or later must name a \`review_trigger\`.\n`,
|
|
174
|
+
);
|
|
175
|
+
return 1;
|
|
176
|
+
}
|
|
177
|
+
process.stdout.write('✅ check_adr_frontmatter: no errors\n');
|
|
178
|
+
return 0;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Main-guard (realpath-compared, mirrors the repo convention).
|
|
182
|
+
if (process.argv[1] !== undefined) {
|
|
183
|
+
try {
|
|
184
|
+
const here = fs.realpathSync(fileURLToPath(import.meta.url));
|
|
185
|
+
const argv1 = fs.realpathSync(path.resolve(process.argv[1]));
|
|
186
|
+
if (here === argv1) {
|
|
187
|
+
process.exit(main(process.argv.slice(2)));
|
|
188
|
+
}
|
|
189
|
+
} catch {
|
|
190
|
+
const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
|
|
191
|
+
if (import.meta.url === argvUrl) {
|
|
192
|
+
process.exit(main(process.argv.slice(2)));
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|