@event4u/agent-config 9.5.0 → 9.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +84 -942
- package/README.md +17 -17
- package/dist/agent-src/README.md +2 -2
- package/dist/agent-src/commands/agents/audit.md +2 -3
- package/dist/agent-src/commands/agents/optimize.md +2 -3
- package/dist/agent-src/commands/agents/user/review.md +4 -5
- package/dist/agent-src/commands/agents/user/show.md +4 -5
- package/dist/agent-src/commands/agents/user/update.md +4 -5
- package/dist/agent-src/commands/agents/user.md +2 -3
- package/dist/agent-src/commands/analytics/prune.md +2 -3
- package/dist/agent-src/commands/analytics/show.md +2 -3
- package/dist/agent-src/commands/analyze/decision.md +2 -3
- package/dist/agent-src/commands/analyze/incident.md +2 -3
- package/dist/agent-src/commands/analyze/near-miss.md +2 -3
- package/dist/agent-src/commands/analyze/postmortem.md +2 -3
- package/dist/agent-src/commands/analyze/premortem.md +2 -3
- package/dist/agent-src/commands/analyze/reference-repo.md +2 -3
- package/dist/agent-src/commands/brand/identity.md +2 -3
- package/dist/agent-src/commands/brand/review.md +2 -3
- package/dist/agent-src/commands/brand/strategy.md +2 -3
- package/dist/agent-src/commands/brand/tokens.md +2 -3
- package/dist/agent-src/commands/brand/voice.md +2 -3
- package/dist/agent-src/commands/bug/fix.md +2 -3
- package/dist/agent-src/commands/bug/investigate.md +2 -3
- package/dist/agent-src/commands/challenge-me/vision.md +2 -3
- package/dist/agent-src/commands/challenge-me/with-docs.md +2 -3
- package/dist/agent-src/commands/chat-history/import.md +2 -3
- package/dist/agent-src/commands/commit/in-chunks.md +2 -3
- package/dist/agent-src/commands/context/create.md +2 -3
- package/dist/agent-src/commands/context/refactor.md +2 -3
- package/dist/agent-src/commands/cost/report.md +2 -3
- package/dist/agent-src/commands/council/analysis.md +2 -3
- package/dist/agent-src/commands/council/debate.md +2 -3
- package/dist/agent-src/commands/council/design.md +2 -3
- package/dist/agent-src/commands/council/optimize.md +2 -3
- package/dist/agent-src/commands/council/pr.md +2 -3
- package/dist/agent-src/commands/feature/dev.md +2 -3
- package/dist/agent-src/commands/feature/explore.md +2 -3
- package/dist/agent-src/commands/feature/plan.md +2 -3
- package/dist/agent-src/commands/feature/refactor.md +2 -3
- package/dist/agent-src/commands/feature/roadmap.md +2 -3
- package/dist/agent-src/commands/fix/ci.md +2 -3
- package/dist/agent-src/commands/fix/comments.md +2 -3
- package/dist/agent-src/commands/fix/pr-comments.md +2 -3
- package/dist/agent-src/commands/fix/quality.md +2 -3
- package/dist/agent-src/commands/fix/route.md +2 -3
- package/dist/agent-src/commands/fix/seeder.md +2 -3
- package/dist/agent-src/commands/ghostwriter/delete.md +2 -3
- package/dist/agent-src/commands/ghostwriter/fetch.md +2 -3
- package/dist/agent-src/commands/ghostwriter/list.md +2 -3
- package/dist/agent-src/commands/ghostwriter/show.md +2 -3
- package/dist/agent-src/commands/ghostwriter/write.md +2 -3
- package/dist/agent-src/commands/image/analyse.md +2 -3
- package/dist/agent-src/commands/image/create.md +2 -3
- package/dist/agent-src/commands/image/verify.md +2 -3
- package/dist/agent-src/commands/knowledge/cross-repo.md +2 -3
- package/dist/agent-src/commands/knowledge/forget.md +2 -3
- package/dist/agent-src/commands/knowledge/ingest.md +2 -3
- package/dist/agent-src/commands/knowledge/list.md +2 -3
- package/dist/agent-src/commands/memory/add.md +2 -3
- package/dist/agent-src/commands/memory/learn-low-impact.md +2 -3
- package/dist/agent-src/commands/module/create.md +2 -3
- package/dist/agent-src/commands/module/explore.md +2 -3
- package/dist/agent-src/commands/optimize/agents-dir.md +2 -3
- package/dist/agent-src/commands/optimize/project.md +2 -3
- package/dist/agent-src/commands/optimize/prompt.md +2 -3
- package/dist/agent-src/commands/optimize/skills.md +2 -3
- package/dist/agent-src/commands/override/create.md +2 -3
- package/dist/agent-src/commands/override/manage.md +2 -3
- package/dist/agent-src/commands/post-as/ghostwriter.md +2 -3
- package/dist/agent-src/commands/post-as/me.md +2 -3
- package/dist/agent-src/commands/pr/create/description-only.md +2 -3
- package/dist/agent-src/commands/pr/create.md +1 -1
- package/dist/agent-src/commands/profile/activate.md +2 -3
- package/dist/agent-src/commands/profile/deactivate.md +2 -3
- package/dist/agent-src/commands/profile/show.md +2 -3
- package/dist/agent-src/commands/project/analyze.md +2 -3
- package/dist/agent-src/commands/project/health.md +2 -3
- package/dist/agent-src/commands/research/deep.md +2 -3
- package/dist/agent-src/commands/research/report.md +2 -3
- package/dist/agent-src/commands/review/changes.md +2 -3
- package/dist/agent-src/commands/review/routing.md +2 -3
- package/dist/agent-src/commands/roadmap/ai-council.md +2 -3
- package/dist/agent-src/commands/roadmap/create.md +2 -3
- package/dist/agent-src/commands/roadmap/materialize.md +2 -3
- package/dist/agent-src/commands/roadmap/process-full.md +2 -3
- package/dist/agent-src/commands/roadmap/process-phase.md +2 -3
- package/dist/agent-src/commands/roadmap/process-step.md +2 -3
- package/dist/agent-src/commands/skill/preview.md +2 -3
- package/dist/agent-src/commands/skills/discover.md +2 -3
- package/dist/agent-src/commands/sync/gitignore/fix.md +3 -4
- package/dist/agent-src/commands/tdd/green.md +2 -3
- package/dist/agent-src/commands/tdd/red.md +2 -3
- package/dist/agent-src/commands/tdd/refactor.md +2 -3
- package/dist/agent-src/commands/team/adversarial.md +2 -3
- package/dist/agent-src/commands/team/review.md +2 -3
- package/dist/agent-src/commands/team-knowledge/bootstrap.md +2 -3
- package/dist/agent-src/commands/team-knowledge/consolidate.md +2 -3
- package/dist/agent-src/commands/tests/create.md +2 -3
- package/dist/agent-src/commands/tests/e2e-heal.md +2 -3
- package/dist/agent-src/commands/tests/e2e-plan.md +2 -3
- package/dist/agent-src/commands/tests/execute.md +2 -3
- package/dist/agent-src/commands/video/from-script.md +2 -3
- package/dist/agent-src/commands/video/from-song.md +2 -3
- package/dist/agent-src/commands/video/scene.md +2 -3
- package/dist/agent-src/commands/video/stitch.md +2 -3
- package/dist/agent-src/commands/video/storyboard.md +2 -3
- package/dist/agent-src/commands/worktree/cleanup.md +2 -3
- package/dist/agent-src/commands/worktree/create.md +2 -3
- package/dist/agent-src/commands/worktree/status.md +2 -3
- package/dist/agent-src/commands/worktree/verify.md +2 -3
- package/dist/agent-src/contexts/contracts/command-suggestion-flow.md +10 -0
- package/dist/agent-src/rules/downstream-changes.md +31 -1
- package/dist/agent-src/skills/agent-docs-writing/SKILL.md +34 -16
- package/dist/agent-src/skills/agents-md-thin-root/SKILL.md +9 -9
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/github-workflows/doc-impact.yml +101 -0
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +113 -113
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/mcp/awesome-mcp-servers.row.md +1 -1
- package/dist/mcp/mcp-cloudflare-catalogue.json +1 -1
- package/dist/mcp/registry-manifest.json +3 -3
- package/docs/CLAIMS.md +2 -1
- package/docs/archive/CHANGELOG-6.0.0-overview.md +953 -0
- package/docs/benchmark.md +33 -9
- package/docs/decisions/ADR-122-adversarial-verification-council.md +30 -0
- package/docs/decisions/ADR-123-runtime-security-scope-and-spawn-hardening.md +142 -0
- package/docs/decisions/INDEX.md +1 -0
- package/docs/guidelines/agent-infra/artifact-drafting-protocol-mechanics.md +26 -0
- package/docs/spawn-site-policy.md +72 -0
- package/docs/threat-model.md +2 -1
- package/package.json +2 -2
- package/src/agent-src/README.md +2 -2
- package/src/scripts/_lib/adversarial_bench_score.ts +106 -0
- package/src/scripts/_lib/spawn_env.ts +120 -0
- package/src/scripts/ai_council/clients.ts +6 -0
- package/src/scripts/bench_adversarial_council.ts +292 -0
- package/src/scripts/check_no_new_legacy_path.ts +2 -0
- package/src/scripts/check_source_pointer_freshness.ts +165 -0
- package/src/scripts/hooks/dispatch_hook.ts +5 -1
- package/src/scripts/hot_context_hook.ts +2 -0
- package/src/scripts/lint_readme_serial_comma.ts +169 -0
- package/src/scripts/roadmap_progress_hook.ts +2 -0
|
@@ -44,6 +44,7 @@ import { spawnSync } from 'node:child_process';
|
|
|
44
44
|
import * as fs from 'node:fs';
|
|
45
45
|
import * as path from 'node:path';
|
|
46
46
|
|
|
47
|
+
import { hardenedSpawnEnv } from '../_lib/spawn_env.js';
|
|
47
48
|
import * as user_global_paths from '../_lib/user_global_paths.js';
|
|
48
49
|
import { appendEvent } from './events_log.js';
|
|
49
50
|
|
|
@@ -1101,6 +1102,11 @@ export abstract class CliClient extends ExternalAIClient {
|
|
|
1101
1102
|
const spawnOpts: Parameters<typeof spawnSync>[2] = {
|
|
1102
1103
|
encoding: 'utf-8',
|
|
1103
1104
|
timeout: Math.round(this.timeout_seconds * 1000),
|
|
1105
|
+
// Least-Agency: scrub code-execution-injection env vectors
|
|
1106
|
+
// (loader preload, git *_COMMAND, NODE_OPTIONS, …) so an
|
|
1107
|
+
// attacker-influenced parent env cannot RCE via the spawned CLI
|
|
1108
|
+
// or a `git` it invokes internally.
|
|
1109
|
+
env: hardenedSpawnEnv(),
|
|
1104
1110
|
};
|
|
1105
1111
|
if (stdinPayload !== null) {
|
|
1106
1112
|
spawnOpts.input = stdinPayload;
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two-stage residual-detection benchmark runner for the adversarial-council
|
|
3
|
+
* finding-coverage claim (ADR-122, road-to-adversarial-council-benchmark,
|
|
4
|
+
* design: docs/design/adversarial-council-eval.md).
|
|
5
|
+
*
|
|
6
|
+
* Protocol (documented here so the verdict is reproducible; 2 working providers
|
|
7
|
+
* — anthropic + openai — so the design is adapted to isolate panel VALUE without
|
|
8
|
+
* a structural-zero artifact):
|
|
9
|
+
*
|
|
10
|
+
* Stage 1 — define the residual. BOTH providers review the 12 defect fixtures
|
|
11
|
+
* under a NEUTRAL review prompt. Residual R = defects that NEITHER neutral pass
|
|
12
|
+
* caught (the judge-survivable pool: it survived a strong 2-vendor first read).
|
|
13
|
+
*
|
|
14
|
+
* Stage 2 — single skeptic vs cross-vendor panel, on R + the clean controls,
|
|
15
|
+
* under an ADVERSARIAL SKEPTIC prompt (the shipped Mode-9 posture):
|
|
16
|
+
* - single_judge = anthropic-skeptic alone.
|
|
17
|
+
* - panel = reconciled union of {anthropic, openai} skeptics.
|
|
18
|
+
* Because panel ⊇ single, lift ≥ 0 and measures the MARGINAL value the
|
|
19
|
+
* second vendor adds at a fixed skeptic posture — exactly "does a cross-vendor
|
|
20
|
+
* panel out-find a single skeptic on the residual, at no-worse FP".
|
|
21
|
+
*
|
|
22
|
+
* evaluateCouncilBench applies the LOCKED dual threshold → backed | honest-null.
|
|
23
|
+
*
|
|
24
|
+
* Limitation (recorded honestly): with 2 vendors this isolates the 2nd-vendor
|
|
25
|
+
* marginal contribution, not an N-vendor scaling curve. gemini/xai are stubbed
|
|
26
|
+
* or disabled in this CLI build.
|
|
27
|
+
*
|
|
28
|
+
* `--mock` runs the whole pipeline on canned responses (no spend) to pre-validate
|
|
29
|
+
* parsing + residual + scoring + gate in BOTH directions. `--run` fires the real
|
|
30
|
+
* paid cross-vendor calls (maintainer spend-gated).
|
|
31
|
+
*/
|
|
32
|
+
import * as fs from 'node:fs';
|
|
33
|
+
import * as path from 'node:path';
|
|
34
|
+
|
|
35
|
+
import {
|
|
36
|
+
AnthropicClient,
|
|
37
|
+
OpenAIClient,
|
|
38
|
+
load_anthropic_key,
|
|
39
|
+
load_openai_key,
|
|
40
|
+
type ExternalAIClient,
|
|
41
|
+
} from './ai_council/clients.js';
|
|
42
|
+
import { evaluateCouncilBench, type CouncilBenchInputs } from './_lib/adversarial_council_gate.js';
|
|
43
|
+
import {
|
|
44
|
+
caughtDefect,
|
|
45
|
+
isFalsePositive,
|
|
46
|
+
recall,
|
|
47
|
+
fpRate,
|
|
48
|
+
categoryFamily,
|
|
49
|
+
type Finding,
|
|
50
|
+
type GroundTruth,
|
|
51
|
+
} from './_lib/adversarial_bench_score.js';
|
|
52
|
+
|
|
53
|
+
const REPO_ROOT = process.cwd();
|
|
54
|
+
const CORPUS_PATH = path.join(REPO_ROOT, 'internal/bench/adversarial-council/corpus.json');
|
|
55
|
+
const SINGLE_SKEPTIC = 'anthropic'; // the single-judge baseline in stage 2
|
|
56
|
+
|
|
57
|
+
interface CorpusItem {
|
|
58
|
+
id: string;
|
|
59
|
+
class: string;
|
|
60
|
+
defect_category: string;
|
|
61
|
+
is_clean: boolean;
|
|
62
|
+
files: { path: string; content: string }[];
|
|
63
|
+
ground_truth: { defect_files: string[]; defect_summary: string };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function loadCorpus(): CorpusItem[] {
|
|
67
|
+
return JSON.parse(fs.readFileSync(CORPUS_PATH, 'utf8')).items as CorpusItem[];
|
|
68
|
+
}
|
|
69
|
+
function groundTruth(item: CorpusItem): GroundTruth {
|
|
70
|
+
return {
|
|
71
|
+
id: item.id,
|
|
72
|
+
is_clean: item.is_clean,
|
|
73
|
+
defect_files: item.ground_truth.defect_files,
|
|
74
|
+
defect_category: item.is_clean ? 'none' : categoryFamily(item.defect_category),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function codeBlock(item: CorpusItem): string {
|
|
78
|
+
return item.files.map((f) => `FILE: ${f.path}\n\`\`\`ts\n${f.content}\`\`\``).join('\n\n');
|
|
79
|
+
}
|
|
80
|
+
const JSON_SHAPE =
|
|
81
|
+
'{"findings":[{"file":"<file path>","category":"<defect category>","confidence":"high|medium|low","summary":"<one line>"}]}';
|
|
82
|
+
function neutralPrompt(item: CorpusItem): string {
|
|
83
|
+
return [
|
|
84
|
+
'You are a senior code reviewer. Review the TypeScript below for REAL defects and report only genuine bugs you are confident about. If it is correct, report an empty findings array — do not invent problems.',
|
|
85
|
+
`Respond with STRICT JSON only, exactly: ${JSON_SHAPE}`,
|
|
86
|
+
'',
|
|
87
|
+
codeBlock(item),
|
|
88
|
+
].join('\n');
|
|
89
|
+
}
|
|
90
|
+
function skepticPrompt(item: CorpusItem): string {
|
|
91
|
+
return [
|
|
92
|
+
'You are an adversarial security/correctness skeptic. Assume this code may hide a subtle, non-obvious defect that a normal review missed — a masked access-control/SSRF/traversal flaw, a boundary/logic inversion, a concurrency or resource-leak edge, a cross-file contract mismatch. Hunt for it. Report only defects you can concretely justify; if after hard scrutiny it is genuinely correct, report an empty array (do not fabricate).',
|
|
93
|
+
`Respond with STRICT JSON only, exactly: ${JSON_SHAPE}`,
|
|
94
|
+
'',
|
|
95
|
+
codeBlock(item),
|
|
96
|
+
].join('\n');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function parseFindings(text: string): Finding[] {
|
|
100
|
+
if (!text) return [];
|
|
101
|
+
const start = text.indexOf('{');
|
|
102
|
+
if (start < 0) return [];
|
|
103
|
+
let depth = 0;
|
|
104
|
+
let end = -1;
|
|
105
|
+
for (let i = start; i < text.length; i++) {
|
|
106
|
+
if (text[i] === '{') depth++;
|
|
107
|
+
else if (text[i] === '}' && --depth === 0) {
|
|
108
|
+
end = i;
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (end < 0) return [];
|
|
113
|
+
try {
|
|
114
|
+
const obj = JSON.parse(text.slice(start, end + 1)) as { findings?: unknown };
|
|
115
|
+
const arr = Array.isArray(obj.findings) ? obj.findings : [];
|
|
116
|
+
return arr
|
|
117
|
+
.filter((f): f is Record<string, unknown> => typeof f === 'object' && f !== null)
|
|
118
|
+
.map((f) => ({
|
|
119
|
+
file: String(f.file ?? ''),
|
|
120
|
+
category: String(f.category ?? ''),
|
|
121
|
+
confidence: (['high', 'medium', 'low'].includes(String(f.confidence)) ? String(f.confidence) : 'high') as
|
|
122
|
+
| 'high'
|
|
123
|
+
| 'medium'
|
|
124
|
+
| 'low',
|
|
125
|
+
}))
|
|
126
|
+
.filter((f) => f.file.length > 0);
|
|
127
|
+
} catch {
|
|
128
|
+
return [];
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
type Reviews = Record<string, Record<string, Finding[]>>; // fixtureId → member → findings
|
|
133
|
+
|
|
134
|
+
// Direct, INDEPENDENT client calls — NOT council_cli. The council `run`/`debate`
|
|
135
|
+
// transport imposes multi-round peer-review with cross-member visibility and
|
|
136
|
+
// returns prose ("## Round 2 … Reviewer A"), which (a) violates the claim's
|
|
137
|
+
// independent-skeptic requirement and (b) defeats JSON parsing. We call each
|
|
138
|
+
// provider's client once, in isolation, with a strict-JSON system prompt.
|
|
139
|
+
const SYSTEM_JSON =
|
|
140
|
+
'You are a code reviewer. Output ONLY one JSON object — no prose, no markdown, no code fences. ' +
|
|
141
|
+
'Shape exactly: {"findings":[{"file":"<path>","category":"<category>","confidence":"high|medium|low","summary":"<one line>"}]}. ' +
|
|
142
|
+
'If the code is correct, output {"findings":[]}. Do not include any text before or after the JSON object.';
|
|
143
|
+
|
|
144
|
+
let _clients: Record<string, ExternalAIClient> | null = null;
|
|
145
|
+
function getClients(): Record<string, ExternalAIClient> {
|
|
146
|
+
if (!_clients) {
|
|
147
|
+
_clients = {
|
|
148
|
+
anthropic: new AnthropicClient({ model: 'claude-sonnet-4-5', api_key: load_anthropic_key() }),
|
|
149
|
+
openai: new OpenAIClient({ model: 'gpt-4o', api_key: load_openai_key() }),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
return _clients;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async function directReview(item: CorpusItem, userPrompt: string): Promise<Record<string, Finding[]>> {
|
|
156
|
+
const cs = getClients();
|
|
157
|
+
const out: Record<string, Finding[]> = {};
|
|
158
|
+
for (const [name, client] of Object.entries(cs)) {
|
|
159
|
+
let text = '';
|
|
160
|
+
try {
|
|
161
|
+
const resp = await client.ask(SYSTEM_JSON, userPrompt, 1200);
|
|
162
|
+
text = resp.text ?? '';
|
|
163
|
+
} catch (e) {
|
|
164
|
+
process.stderr.write(` ${name} error on ${item.id}: ${(e as Error).message}\n`);
|
|
165
|
+
}
|
|
166
|
+
out[name] = parseFindings(text);
|
|
167
|
+
}
|
|
168
|
+
return out;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
interface Passes {
|
|
172
|
+
neutral: Reviews; // defect fixtures only
|
|
173
|
+
skeptic: Reviews; // residual + controls
|
|
174
|
+
residualIds: string[];
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async function realPasses(items: CorpusItem[]): Promise<Passes> {
|
|
178
|
+
const defects = items.filter((i) => !i.is_clean);
|
|
179
|
+
const controls = items.filter((i) => i.is_clean);
|
|
180
|
+
const neutral: Reviews = {};
|
|
181
|
+
process.stderr.write('stage 1 (neutral, independent per vendor) over 12 defects…\n');
|
|
182
|
+
for (const d of defects) {
|
|
183
|
+
neutral[d.id] = await directReview(d, neutralPrompt(d));
|
|
184
|
+
process.stderr.write(` neutral ${d.id}: ${Object.entries(neutral[d.id] ?? {}).map(([m, f]) => `${m}=${f.length}`).join(' ')}\n`);
|
|
185
|
+
}
|
|
186
|
+
const residual = defects.filter((d) => !Object.values(neutral[d.id] ?? {}).flat().some((f) => caughtDefect(groundTruth(d), [f])));
|
|
187
|
+
process.stderr.write(`residual (missed by BOTH neutral passes): ${residual.map((r) => r.id).join(', ') || '(none)'}\n`);
|
|
188
|
+
const skeptic: Reviews = {};
|
|
189
|
+
process.stderr.write('stage 2 (adversarial skeptic, independent per vendor) over residual + controls…\n');
|
|
190
|
+
for (const item of [...residual, ...controls]) {
|
|
191
|
+
skeptic[item.id] = await directReview(item, skepticPrompt(item));
|
|
192
|
+
process.stderr.write(` skeptic ${item.id}: ${Object.entries(skeptic[item.id] ?? {}).map(([m, f]) => `${m}=${f.length}`).join(' ')}\n`);
|
|
193
|
+
}
|
|
194
|
+
return { neutral, skeptic, residualIds: residual.map((r) => r.id) };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function mockPasses(items: CorpusItem[]): Passes {
|
|
198
|
+
// Deterministic proof that the pipeline discriminates: neutral 2-vendor pass
|
|
199
|
+
// catches the "easier" defects; a hard residual survives; the skeptic panel
|
|
200
|
+
// then recovers more of the residual than a single skeptic, at controlled FP.
|
|
201
|
+
const defects = items.filter((i) => !i.is_clean);
|
|
202
|
+
const controls = items.filter((i) => i.is_clean);
|
|
203
|
+
const neutralCatch = new Set(['sec-01', 'sec-02', 'inv-01', 'inv-02', 'state-03', 'mfi-01']); // 6 caught neutrally
|
|
204
|
+
// residual = the other 6: sec-03, inv-03, state-01, state-02, mfi-02, mfi-03
|
|
205
|
+
const anthropicSkepticCatch = new Set(['sec-03', 'state-01']); // single skeptic recovers 2/6
|
|
206
|
+
const openaiSkepticExtra = new Set(['inv-03', 'mfi-02', 'mfi-03']); // openai adds 3 more → panel 5/6
|
|
207
|
+
const hit = (i: CorpusItem): Finding[] => [
|
|
208
|
+
{ file: i.ground_truth.defect_files[0] ?? i.files[0]?.path ?? '', category: i.defect_category, confidence: 'high' as const },
|
|
209
|
+
];
|
|
210
|
+
const neutral: Reviews = {};
|
|
211
|
+
for (const d of defects) neutral[d.id] = { anthropic: neutralCatch.has(d.id) ? hit(d) : [], openai: [] };
|
|
212
|
+
const skeptic: Reviews = {};
|
|
213
|
+
const residual = defects.filter((d) => !neutralCatch.has(d.id));
|
|
214
|
+
for (const d of residual) {
|
|
215
|
+
skeptic[d.id] = {
|
|
216
|
+
anthropic: anthropicSkepticCatch.has(d.id) ? hit(d) : [],
|
|
217
|
+
openai: openaiSkepticExtra.has(d.id) ? hit(d) : [],
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
for (const c of controls)
|
|
221
|
+
skeptic[c.id] = { anthropic: [], openai: c.id === 'clean-02' ? [{ file: c.files[0]?.path ?? '', category: 'correctness', confidence: 'high' }] : [] };
|
|
222
|
+
return { neutral, skeptic, residualIds: residual.map((r) => r.id) };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function scorePasses(items: CorpusItem[], p: Passes): { inputs: CouncilBenchInputs; detail: Record<string, unknown> } {
|
|
226
|
+
const byId = new Map(items.map((i) => [i.id, i]));
|
|
227
|
+
const residual = p.residualIds.map((id) => byId.get(id)!);
|
|
228
|
+
const controls = items.filter((i) => i.is_clean);
|
|
229
|
+
|
|
230
|
+
const singleCaught: Record<string, boolean> = {};
|
|
231
|
+
const panelCaught: Record<string, boolean> = {};
|
|
232
|
+
for (const d of residual) {
|
|
233
|
+
const gt = groundTruth(d);
|
|
234
|
+
singleCaught[d.id] = caughtDefect(gt, p.skeptic[d.id]?.[SINGLE_SKEPTIC] ?? []);
|
|
235
|
+
panelCaught[d.id] = caughtDefect(gt, Object.values(p.skeptic[d.id] ?? {}).flat());
|
|
236
|
+
}
|
|
237
|
+
const singleFp: Record<string, boolean> = {};
|
|
238
|
+
const panelFp: Record<string, boolean> = {};
|
|
239
|
+
for (const c of controls) {
|
|
240
|
+
const gt = groundTruth(c);
|
|
241
|
+
singleFp[c.id] = isFalsePositive(gt, p.skeptic[c.id]?.[SINGLE_SKEPTIC] ?? []);
|
|
242
|
+
panelFp[c.id] = isFalsePositive(gt, Object.values(p.skeptic[c.id] ?? {}).flat());
|
|
243
|
+
}
|
|
244
|
+
const inputs: CouncilBenchInputs = {
|
|
245
|
+
single_judge_residual_recall: recall(singleCaught),
|
|
246
|
+
panel_residual_recall: recall(panelCaught),
|
|
247
|
+
single_judge_fp_rate: fpRate(singleFp),
|
|
248
|
+
panel_fp_rate: fpRate(panelFp),
|
|
249
|
+
fp_noise_margin: 1 / 3,
|
|
250
|
+
};
|
|
251
|
+
const detail = {
|
|
252
|
+
total_defects: items.filter((i) => !i.is_clean).length,
|
|
253
|
+
residual_ids: p.residualIds,
|
|
254
|
+
residual_size: residual.length,
|
|
255
|
+
single_skeptic: SINGLE_SKEPTIC,
|
|
256
|
+
single_residual_caught: Object.entries(singleCaught).filter(([, v]) => v).map(([k]) => k),
|
|
257
|
+
panel_residual_caught: Object.entries(panelCaught).filter(([, v]) => v).map(([k]) => k),
|
|
258
|
+
single_fp: Object.entries(singleFp).filter(([, v]) => v).map(([k]) => k),
|
|
259
|
+
panel_fp: Object.entries(panelFp).filter(([, v]) => v).map(([k]) => k),
|
|
260
|
+
};
|
|
261
|
+
return { inputs, detail };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function main(argv: string[]): Promise<number> {
|
|
265
|
+
const mock = argv.includes('--mock');
|
|
266
|
+
const run = argv.includes('--run');
|
|
267
|
+
if (!mock && !run) {
|
|
268
|
+
process.stderr.write('usage: bench_adversarial_council (--mock | --run)\n');
|
|
269
|
+
return 2;
|
|
270
|
+
}
|
|
271
|
+
const items = loadCorpus();
|
|
272
|
+
const passes = mock ? mockPasses(items) : await realPasses(items);
|
|
273
|
+
const { inputs, detail } = scorePasses(items, passes);
|
|
274
|
+
const verdict = evaluateCouncilBench(inputs);
|
|
275
|
+
const report = {
|
|
276
|
+
mode: mock ? 'mock' : 'run',
|
|
277
|
+
corpus: 'internal/bench/adversarial-council/corpus.json',
|
|
278
|
+
protocol:
|
|
279
|
+
'stage1 2-vendor neutral → residual (missed by both); stage2 single skeptic (anthropic) vs cross-vendor panel union on residual; FP on clean controls',
|
|
280
|
+
inputs,
|
|
281
|
+
detail,
|
|
282
|
+
verdict,
|
|
283
|
+
};
|
|
284
|
+
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
285
|
+
return 0;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
289
|
+
void main(process.argv.slice(2)).then((code) => process.exit(code));
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
export { parseFindings, scorePasses, main };
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
* - src/scripts/_lib/agent_src.py (the LEGACY_SRC constant)
|
|
25
25
|
* - src/scripts/check_references.py (forbidden-substring detector)
|
|
26
26
|
* - src/scripts/check_condensed_paths.py (forbidden-substring detector)
|
|
27
|
+
* - src/scripts/check_source_pointer_freshness.ts (forbidden-substring detector)
|
|
27
28
|
*
|
|
28
29
|
* Faithful-twin rule (Python→TypeScript migration): a `*.ts` file is also
|
|
29
30
|
* exempt when a same-stem `*.py` sibling exists AND already contains the
|
|
@@ -50,6 +51,7 @@ const EXEMPT: ReadonlySet<string> = new Set([
|
|
|
50
51
|
'src/scripts/check_references.py',
|
|
51
52
|
'src/scripts/check_condensed_paths.py',
|
|
52
53
|
'src/scripts/check_no_new_legacy_path.py', // this file documents the literal
|
|
54
|
+
'src/scripts/check_source_pointer_freshness.ts', // forbidden-substring detector for the legacy tree
|
|
53
55
|
]);
|
|
54
56
|
|
|
55
57
|
type TwinCheck = (curFile: string) => boolean;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* check_source_pointer_freshness — fail when an AUTHORING artifact asserts a
|
|
4
|
+
* RETIRED source-of-truth pointer as the current one.
|
|
5
|
+
*
|
|
6
|
+
* The doc-follows-code discipline (rule: downstream-changes § Doc-Impact;
|
|
7
|
+
* skill: agent-docs-writing § Doc-Impact) is a behavioral obligation. This is
|
|
8
|
+
* its deterministic backstop for the one drift class a plain script CAN catch
|
|
9
|
+
* without semantics: a doc that still names the retired `.agent-src.uncondensed/`
|
|
10
|
+
* tree as the source of truth after the source moved to `src/` (ADR-051).
|
|
11
|
+
*
|
|
12
|
+
* Scope is deliberately narrow (council 2026-07-21, do-not-build list): an
|
|
13
|
+
* explicit allowlist of the authoring files that MAKE a source-of-truth claim,
|
|
14
|
+
* not a whole-tree sweep — a gate that fires on every commit and gets
|
|
15
|
+
* routinely overridden is worse than none. Historical mentions (ADRs under
|
|
16
|
+
* docs/decisions/, archived roadmaps) are out of scope by construction; a
|
|
17
|
+
* genuinely-historical line inside an allowlisted file is exempted with an
|
|
18
|
+
* inline `<!-- pointer-freshness: historical -->` marker.
|
|
19
|
+
*
|
|
20
|
+
* Extend AUTHORING_FILES as the broader `.agent-src.uncondensed` cleanup lands
|
|
21
|
+
* (186 files still carry the token; see road-to-doc-follows-code Phase 4/5
|
|
22
|
+
* follow-up).
|
|
23
|
+
*
|
|
24
|
+
* Exit codes: 0 = clean, 1 = a retired pointer in an allowlisted authoring
|
|
25
|
+
* file, 2 = usage / self-test failure.
|
|
26
|
+
*
|
|
27
|
+
* Usage:
|
|
28
|
+
* tsx src/scripts/check_source_pointer_freshness.ts [--json] [--selftest]
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import * as fs from 'node:fs';
|
|
32
|
+
import * as path from 'node:path';
|
|
33
|
+
import process from 'node:process';
|
|
34
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
35
|
+
|
|
36
|
+
const _HERE = fileURLToPath(import.meta.url);
|
|
37
|
+
// parents[2] of src/scripts/<file> is the repo root.
|
|
38
|
+
const ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
39
|
+
|
|
40
|
+
// Authoring files that assert WHERE the source of truth lives. Each must NOT
|
|
41
|
+
// name the retired `.agent-src.uncondensed/` tree as current. Extend as the
|
|
42
|
+
// broader cleanup lands.
|
|
43
|
+
const AUTHORING_FILES: readonly string[] = [
|
|
44
|
+
'src/agent-src/README.md',
|
|
45
|
+
'src/skills/agents-md-thin-root/SKILL.md',
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
// Retired source-of-truth pointer. Matches both the bare token and the older
|
|
49
|
+
// monorepo-prefixed form `packages/core/.agent-src.uncondensed/`.
|
|
50
|
+
const RETIRED_POINTER = /\.agent-src\.uncondensed\//;
|
|
51
|
+
|
|
52
|
+
// A line carrying this marker is a deliberate historical reference — exempt.
|
|
53
|
+
const HISTORICAL_MARKER = '<!-- pointer-freshness: historical -->';
|
|
54
|
+
|
|
55
|
+
interface Hit {
|
|
56
|
+
file: string;
|
|
57
|
+
line: number;
|
|
58
|
+
text: string;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function _scanFile(rel: string): Hit[] {
|
|
62
|
+
const abs = path.join(ROOT, rel);
|
|
63
|
+
let text: string;
|
|
64
|
+
try {
|
|
65
|
+
if (!fs.statSync(abs).isFile()) {
|
|
66
|
+
return [];
|
|
67
|
+
}
|
|
68
|
+
text = fs.readFileSync(abs, 'utf-8');
|
|
69
|
+
} catch {
|
|
70
|
+
// A missing allowlisted file is a config drift in itself — surface it.
|
|
71
|
+
return [{ file: rel, line: 0, text: '(allowlisted authoring file not found)' }];
|
|
72
|
+
}
|
|
73
|
+
const hits: Hit[] = [];
|
|
74
|
+
const lines = text.split('\n');
|
|
75
|
+
for (let idx = 0; idx < lines.length; idx += 1) {
|
|
76
|
+
const line = lines[idx] as string;
|
|
77
|
+
if (line.includes(HISTORICAL_MARKER)) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
if (RETIRED_POINTER.test(line)) {
|
|
81
|
+
hits.push({ file: rel, line: idx + 1, text: line.trim().slice(0, 160) });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return hits;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Deterministic self-check of the matcher — no filesystem, no network. */
|
|
88
|
+
function _selftest(): number {
|
|
89
|
+
const stale = 'The source of truth is `.agent-src.uncondensed/`. Edit there.';
|
|
90
|
+
const monorepo = 'Editing `packages/core/.agent-src.uncondensed/templates/AGENTS.md`.';
|
|
91
|
+
const clean = 'The source of truth is `src/`. Edit there, then run `task sync`.';
|
|
92
|
+
const historical = `Retired path \`.agent-src.uncondensed/\` ${HISTORICAL_MARKER}`;
|
|
93
|
+
const cases: Array<[string, boolean]> = [
|
|
94
|
+
[stale, true],
|
|
95
|
+
[monorepo, true],
|
|
96
|
+
[clean, false],
|
|
97
|
+
[historical, false], // marker suppresses the match at scan time
|
|
98
|
+
];
|
|
99
|
+
let ok = true;
|
|
100
|
+
for (const [sample, shouldMatch] of cases) {
|
|
101
|
+
const exempt = sample.includes(HISTORICAL_MARKER);
|
|
102
|
+
const matched = !exempt && RETIRED_POINTER.test(sample);
|
|
103
|
+
if (matched !== shouldMatch) {
|
|
104
|
+
process.stderr.write(`selftest FAIL: ${JSON.stringify(sample)} expected match=${shouldMatch}, got ${matched}\n`);
|
|
105
|
+
ok = false;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (ok) {
|
|
109
|
+
process.stdout.write('✅ check_source_pointer_freshness selftest passed (4/4).\n');
|
|
110
|
+
return 0;
|
|
111
|
+
}
|
|
112
|
+
return 2;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function main(argv: readonly string[]): number {
|
|
116
|
+
if (argv.includes('--selftest')) {
|
|
117
|
+
return _selftest();
|
|
118
|
+
}
|
|
119
|
+
const asJson = argv.includes('--json');
|
|
120
|
+
const hits: Hit[] = [];
|
|
121
|
+
for (const rel of AUTHORING_FILES) {
|
|
122
|
+
hits.push(..._scanFile(rel));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (asJson) {
|
|
126
|
+
process.stdout.write(JSON.stringify({ ok: hits.length === 0, hits }, null, 2) + '\n');
|
|
127
|
+
} else if (hits.length > 0) {
|
|
128
|
+
process.stdout.write(`❌ ${hits.length} retired source-of-truth pointer(s) in authoring files:\n\n`);
|
|
129
|
+
for (const h of hits) {
|
|
130
|
+
process.stdout.write(` ${h.file}:${h.line} ${h.text}\n`);
|
|
131
|
+
}
|
|
132
|
+
process.stdout.write(
|
|
133
|
+
'\nThese name the retired `.agent-src.uncondensed/` tree as the source of\n' +
|
|
134
|
+
'truth. The source of truth is `src/` (ADR-051, rule: source-of-truth).\n' +
|
|
135
|
+
'Fix the pointer, or mark a genuinely-historical line with\n' +
|
|
136
|
+
`\`${HISTORICAL_MARKER}\`. See rule: downstream-changes § Doc-Impact.\n`,
|
|
137
|
+
);
|
|
138
|
+
} else {
|
|
139
|
+
process.stdout.write('✅ No retired source-of-truth pointers in authoring files.\n');
|
|
140
|
+
}
|
|
141
|
+
return hits.length > 0 ? 1 : 0;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function _isCliEntry(): boolean {
|
|
145
|
+
if (process.argv[1] === undefined) {
|
|
146
|
+
return false;
|
|
147
|
+
}
|
|
148
|
+
const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
|
|
149
|
+
if (import.meta.url === argvUrl) {
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
const here = fs.realpathSync(fileURLToPath(import.meta.url));
|
|
154
|
+
const argv = fs.realpathSync(path.resolve(process.argv[1]));
|
|
155
|
+
return here === argv;
|
|
156
|
+
} catch {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (_isCliEntry()) {
|
|
162
|
+
process.exit(main(process.argv.slice(2)));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export { main, _scanFile, _selftest, AUTHORING_FILES, RETIRED_POINTER, HISTORICAL_MARKER };
|
|
@@ -35,6 +35,7 @@ import path from "node:path";
|
|
|
35
35
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
36
36
|
import { parse as parseYaml } from "yaml";
|
|
37
37
|
|
|
38
|
+
import { hardenedSpawnEnv } from "../_lib/spawn_env.js";
|
|
38
39
|
import {
|
|
39
40
|
atomic_write_json,
|
|
40
41
|
feedback_dir,
|
|
@@ -500,7 +501,10 @@ function _run_concern(concern: ConcernDef, envelope: JsonObject): RunResult {
|
|
|
500
501
|
// Pass the package root so concerns can locate package-shipped
|
|
501
502
|
// distributed content (ADR-020 global-only consumers carry no
|
|
502
503
|
// project-local copy). REPO_ROOT is the dispatcher's own resolved root.
|
|
503
|
-
|
|
504
|
+
// Consumer-runtime concern dispatch: scrub code-execution-injection env
|
|
505
|
+
// vectors (loader / git-config / NODE_OPTIONS / …) before spawning each
|
|
506
|
+
// concern, then set the package-root marker as an explicit override.
|
|
507
|
+
const concern_env = hardenedSpawnEnv({ AGENT_CONFIG_PACKAGE_ROOT: REPO_ROOT });
|
|
504
508
|
|
|
505
509
|
const started = performance.now();
|
|
506
510
|
const { command, args: spawnArgs } = _resolve_tsx_invocation(script, cmd.slice(1));
|
|
@@ -35,6 +35,7 @@ import * as path from 'node:path';
|
|
|
35
35
|
import process from 'node:process';
|
|
36
36
|
import { fileURLToPath } from 'node:url';
|
|
37
37
|
|
|
38
|
+
import { hardenedSpawnEnv } from './_lib/spawn_env.js';
|
|
38
39
|
import { redact_low_impact_entry } from './ai_council/redact_low_impact_entry.js';
|
|
39
40
|
|
|
40
41
|
export const HOT_CONTEXT_REL = path.join('agents', 'runtime', 'state', 'hot-context.md');
|
|
@@ -85,6 +86,7 @@ export function _current_branch(root: string): string {
|
|
|
85
86
|
cwd: root,
|
|
86
87
|
encoding: 'utf-8',
|
|
87
88
|
timeout: 5000,
|
|
89
|
+
env: hardenedSpawnEnv(),
|
|
88
90
|
});
|
|
89
91
|
if (proc.status === 0) {
|
|
90
92
|
return (proc.stdout || '').trim() || 'unknown';
|