@event4u/agent-config 10.2.0 → 10.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +110 -276
- package/dist/agent-src/commands/analyze/reference-repo.md +62 -0
- package/dist/agent-src/commands/roadmap/create.md +4 -1
- package/dist/agent-src/personas/advisors/first-principles.md +6 -0
- package/dist/agent-src/rules/active-remediation.md +53 -2
- package/dist/agent-src/rules/code-provenance.md +66 -4
- package/dist/agent-src/rules/content-quoting-floor.md +6 -0
- package/dist/agent-src/rules/design-fidelity.md +49 -3
- package/dist/agent-src/skills/design-review/references/verification-automation.md +23 -1
- package/dist/agent-src/skills/fe-design/SKILL.md +27 -3
- package/dist/agent-src/skills/judge-synthesis/SKILL.md +30 -0
- package/dist/agent-src/skills/learning-to-rule-or-skill/SKILL.md +11 -0
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +8 -8
- package/dist/agent-src/skills/skill-writing/SKILL.md +4 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/persona.md +26 -0
- package/dist/agent-src/templates/roadmaps.md +27 -7
- package/dist/agent-src/templates/skill.md +35 -1
- package/dist/cli/commands/commands.candidates.test.js +257 -0
- package/dist/cli/commands/commands.candidates.test.js.map +1 -0
- package/dist/cli/commands/commands.js +199 -0
- package/dist/cli/commands/commands.js.map +1 -1
- package/dist/cli/main.js +1 -0
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/registry.js +1 -1
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/{chunk-E4RUZKXB.js → chunk-FV34EGEX.js} +1 -1
- package/dist/cli-delegate/cmd_conformance.js +1 -1
- package/dist/cli-delegate/cmd_doctor.js +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +13 -13
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +116 -24
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +1 -1
- package/dist/router.json +1 -1
- package/docs/guidelines/agent-infra/active-remediation-mechanics.md +54 -0
- package/docs/guidelines/design-antipatterns.md +79 -10
- package/docs/guidelines/design-fidelity-mechanics.md +120 -2
- package/package.json +1 -1
- package/src/config/gate-coverage.yml +42 -8
- package/src/config/gate-violation-baselines.json +6 -0
- package/src/config/review-prompt-binding-baseline.json +19 -0
- package/src/config/rule-enforcement-baseline.json +0 -1
- package/src/scripts/_dispatch.bash +1 -0
- package/src/scripts/ai_council/clients.ts +15 -1
- package/src/scripts/check_ci_local_parity.ts +60 -6
- package/src/scripts/check_review_prompt_binding.ts +553 -0
- package/src/scripts/check_suppression_hygiene.ts +19 -3
- package/src/scripts/design_slop_rules.test.ts +37 -6
- package/src/scripts/design_slop_rules.ts +150 -22
- package/src/scripts/dispatch_r2_reviewer.ts +15 -1
- package/src/scripts/hooks/ui_route_nudge_hook.ts +55 -2
- package/src/scripts/lint_design_antipattern_parity.ts +201 -0
- package/src/scripts/lint_harvest_provenance.ts +483 -0
- package/src/scripts/lint_skill_router_head.ts +254 -0
- package/src/scripts/report_consultation_rate.ts +126 -1
- package/src/scripts/run_skill_evals.ts +10 -5
- package/src/scripts/schemas/persona.schema.json +8 -0
- package/src/scripts/schemas/provenance-harvest.schema.json +39 -0
- package/src/scripts/sweep_dead_scan_roots.ts +51 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* Router-head gate — makes the published K6 cap true for NEW skills.
|
|
4
|
+
*
|
|
5
|
+
* `src/agent-src/templates/skill.md` has carried "K6: Under 400 lines" as a
|
|
6
|
+
* checklist item since long before this gate. A checklist item is a statement
|
|
7
|
+
* of intent that nothing checks; four skills exceed it today. This gate turns
|
|
8
|
+
* the existing contract into an enforced one, without re-litigating the number.
|
|
9
|
+
*
|
|
10
|
+
* The rule: a SKILL.md over MAX_HEAD_LINES must be a router head — an entry
|
|
11
|
+
* file (when-to-use, mode table, routing) plus per-mode bodies in `tasks/` or
|
|
12
|
+
* `references/` inside the skill folder. Over the cap with neither directory
|
|
13
|
+
* present is a monolith and fails.
|
|
14
|
+
*
|
|
15
|
+
* WHAT THIS GATE DOES NOT CLAIM. It is not an activation or token argument.
|
|
16
|
+
* The host loads SKILL.md whole on trigger, and whether it then follows a
|
|
17
|
+
* pointer into `tasks/` is host behaviour nothing in this tree observes — so
|
|
18
|
+
* "splitting saves tokens on trigger" is unmeasurable here and is not asserted.
|
|
19
|
+
* ADR-202's lesson binds: an activation claim goes behind a measured instrument
|
|
20
|
+
* or it is not made. The defensible claim is narrower and sufficient: the cap
|
|
21
|
+
* was already the repo's position, and now a new skill cannot quietly ignore it.
|
|
22
|
+
*
|
|
23
|
+
* GRANDFATHERING is a shrink-only ratchet. GRANDFATHERED below is seeded with
|
|
24
|
+
* the measured offender set. Entries may be REMOVED when a skill is
|
|
25
|
+
* restructured or drops under the cap; adding one is the move this gate exists
|
|
26
|
+
* to prevent, and the diff makes any addition visible in review. The list is an
|
|
27
|
+
* inline constant rather than a JSON baseline on purpose — a baseline file
|
|
28
|
+
* grows without anyone reading it, and this one should be read every time.
|
|
29
|
+
*
|
|
30
|
+
* Usage:
|
|
31
|
+
* ./scripts-run src/scripts/lint_skill_router_head
|
|
32
|
+
* ./scripts-run src/scripts/lint_skill_router_head --quiet
|
|
33
|
+
*
|
|
34
|
+
* Exit codes: 0 = clean · 2 = any finding OR usage error.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import * as fs from 'node:fs';
|
|
38
|
+
import * as path from 'node:path';
|
|
39
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
40
|
+
|
|
41
|
+
import { assertWatchlistResolves, DeadScopeError } from './_lib/scan_scope.js';
|
|
42
|
+
|
|
43
|
+
const _FILE = fileURLToPath(import.meta.url);
|
|
44
|
+
const _HERE = path.dirname(_FILE);
|
|
45
|
+
const REPO = path.resolve(_HERE, '..', '..');
|
|
46
|
+
const SKILLS_REL = 'src/skills';
|
|
47
|
+
|
|
48
|
+
/** The K6 cap, in lines. Mirrors src/agent-src/templates/skill.md § K6. */
|
|
49
|
+
export const MAX_HEAD_LINES = 400;
|
|
50
|
+
|
|
51
|
+
/** Directories a router head may route into. `references/` is the established
|
|
52
|
+
* convention (9 skills use it today); `tasks/` is permitted by the contract
|
|
53
|
+
* for per-mode procedure bodies. */
|
|
54
|
+
export const MODE_BODY_DIRS: readonly string[] = ['tasks', 'references'];
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Shrink-only allowlist — skills already over the cap when the gate landed.
|
|
58
|
+
* Measured, not guessed. REMOVE entries as skills are restructured; never add.
|
|
59
|
+
*/
|
|
60
|
+
export const GRANDFATHERED: readonly string[] = [
|
|
61
|
+
'ai-council', // 1055
|
|
62
|
+
'skill-writing', // 767
|
|
63
|
+
'roadmap-management', // 552
|
|
64
|
+
'quality-tools', // 445
|
|
65
|
+
];
|
|
66
|
+
|
|
67
|
+
export interface Finding {
|
|
68
|
+
readonly skill: string;
|
|
69
|
+
readonly lines: number;
|
|
70
|
+
readonly message: string;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Count lines the way `wc -l` does, so the gate and a shell check agree. */
|
|
74
|
+
export function countLines(text: string): number {
|
|
75
|
+
if (text.length === 0) return 0;
|
|
76
|
+
const n = text.split('\n').length;
|
|
77
|
+
return text.endsWith('\n') ? n - 1 : n;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Does this skill folder carry at least one mode-body directory? */
|
|
81
|
+
export function hasModeBodies(skillDir: string, dirs: readonly string[] = MODE_BODY_DIRS): boolean {
|
|
82
|
+
return dirs.some((d) => {
|
|
83
|
+
try {
|
|
84
|
+
return fs.statSync(path.join(skillDir, d)).isDirectory();
|
|
85
|
+
} catch {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface SkillHead {
|
|
92
|
+
readonly name: string;
|
|
93
|
+
readonly lines: number;
|
|
94
|
+
readonly hasBodies: boolean;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Read every `<skillsRoot>/<name>/SKILL.md`. */
|
|
98
|
+
export function collectSkillHeads(skillsRoot: string): SkillHead[] {
|
|
99
|
+
let entries: fs.Dirent[];
|
|
100
|
+
try {
|
|
101
|
+
entries = fs.readdirSync(skillsRoot, { withFileTypes: true });
|
|
102
|
+
} catch {
|
|
103
|
+
return [];
|
|
104
|
+
}
|
|
105
|
+
const heads: SkillHead[] = [];
|
|
106
|
+
for (const e of entries) {
|
|
107
|
+
if (!e.isDirectory()) continue;
|
|
108
|
+
const dir = path.join(skillsRoot, e.name);
|
|
109
|
+
const head = path.join(dir, 'SKILL.md');
|
|
110
|
+
let text: string;
|
|
111
|
+
try {
|
|
112
|
+
text = fs.readFileSync(head, 'utf-8');
|
|
113
|
+
} catch {
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
heads.push({ name: e.name, lines: countLines(text), hasBodies: hasModeBodies(dir) });
|
|
117
|
+
}
|
|
118
|
+
return heads.sort((a, b) => a.name.localeCompare(b.name));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* A head fails when it is over the cap, carries no mode-body directory, and is
|
|
123
|
+
* not grandfathered. Two separate results are reported so the caller can
|
|
124
|
+
* distinguish a live violation from allowlist drift.
|
|
125
|
+
*/
|
|
126
|
+
export function evaluate(
|
|
127
|
+
heads: readonly SkillHead[],
|
|
128
|
+
grandfathered: readonly string[] = GRANDFATHERED,
|
|
129
|
+
max: number = MAX_HEAD_LINES,
|
|
130
|
+
): { readonly findings: Finding[]; readonly staleAllowlist: string[] } {
|
|
131
|
+
const allow = new Set(grandfathered);
|
|
132
|
+
const findings: Finding[] = [];
|
|
133
|
+
const stillOver = new Set<string>();
|
|
134
|
+
|
|
135
|
+
for (const h of heads) {
|
|
136
|
+
if (h.lines <= max) continue;
|
|
137
|
+
stillOver.add(h.name);
|
|
138
|
+
if (allow.has(h.name)) continue;
|
|
139
|
+
if (h.hasBodies) continue;
|
|
140
|
+
findings.push({
|
|
141
|
+
skill: h.name,
|
|
142
|
+
lines: h.lines,
|
|
143
|
+
message:
|
|
144
|
+
`SKILL.md is ${h.lines} lines (cap ${max}) with no ${MODE_BODY_DIRS.map((d) => `${d}/`).join(' or ')} ` +
|
|
145
|
+
'directory — over the cap a skill restructures as a router head (entry head + one file per mode), ' +
|
|
146
|
+
'per templates/skill.md § The router-head contract. The allowlist is shrink-only and does not accept new entries.',
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// An allowlist entry for a skill that no longer exceeds the cap (or no
|
|
151
|
+
// longer exists) is the ratchet failing to tighten: the entry silently
|
|
152
|
+
// buys nothing and would re-permit a regression. Reported, not fatal —
|
|
153
|
+
// removing it is a one-line follow-up, not a reason to red the build.
|
|
154
|
+
const staleAllowlist = grandfathered.filter((g) => !stillOver.has(g));
|
|
155
|
+
|
|
156
|
+
return { findings, staleAllowlist };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ─── CLI ─────────────────────────────────────────────────────────────────────
|
|
160
|
+
|
|
161
|
+
class ExitCode extends Error {
|
|
162
|
+
readonly code: number;
|
|
163
|
+
constructor(code: number) {
|
|
164
|
+
super(`exit ${code}`);
|
|
165
|
+
this.code = code;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function parseArgs(argv: string[]): { readonly quiet: boolean } {
|
|
170
|
+
let quiet = false;
|
|
171
|
+
for (const a of argv) {
|
|
172
|
+
if (a === '--quiet') {
|
|
173
|
+
quiet = true;
|
|
174
|
+
} else if (a === '-h' || a === '--help') {
|
|
175
|
+
process.stdout.write('usage: lint_skill_router_head [--quiet]\n');
|
|
176
|
+
throw new ExitCode(0);
|
|
177
|
+
} else {
|
|
178
|
+
process.stderr.write(`❌ lint_skill_router_head: unrecognized argument: ${a}\n`);
|
|
179
|
+
throw new ExitCode(2);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return { quiet };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export function main(argv: string[] = process.argv.slice(2)): number {
|
|
186
|
+
const args = parseArgs(argv);
|
|
187
|
+
|
|
188
|
+
// The scanned root is the scope signal. A moved or renamed src/skills would
|
|
189
|
+
// otherwise produce zero heads and a clean green over nothing at all.
|
|
190
|
+
try {
|
|
191
|
+
assertWatchlistResolves({
|
|
192
|
+
gate: 'lint_skill_router_head',
|
|
193
|
+
candidates: [`${SKILLS_REL}/skill-writing/SKILL.md`],
|
|
194
|
+
repoRoot: REPO,
|
|
195
|
+
});
|
|
196
|
+
} catch (exc) {
|
|
197
|
+
if (exc instanceof DeadScopeError) {
|
|
198
|
+
process.stderr.write(`❌ ${exc.message}\n`);
|
|
199
|
+
return 2;
|
|
200
|
+
}
|
|
201
|
+
throw exc;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const heads = collectSkillHeads(path.join(REPO, SKILLS_REL));
|
|
205
|
+
const { findings, staleAllowlist } = evaluate(heads);
|
|
206
|
+
|
|
207
|
+
for (const s of staleAllowlist) {
|
|
208
|
+
process.stderr.write(
|
|
209
|
+
`⚠️ lint_skill_router_head: '${s}' is grandfathered but no longer over the cap — remove it from GRANDFATHERED so the ratchet keeps its teeth\n`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
if (findings.length > 0) {
|
|
214
|
+
for (const f of findings) {
|
|
215
|
+
process.stderr.write(`❌ ${SKILLS_REL}/${f.skill}/SKILL.md: ${f.message}\n`);
|
|
216
|
+
}
|
|
217
|
+
process.stderr.write(`❌ lint_skill_router_head: ${findings.length} finding(s)\n`);
|
|
218
|
+
return 2;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (!args.quiet) {
|
|
222
|
+
const over = heads.filter((h) => h.lines > MAX_HEAD_LINES).length;
|
|
223
|
+
process.stdout.write(
|
|
224
|
+
`✅ lint_skill_router_head: ${heads.length} skill head(s) scanned · ` +
|
|
225
|
+
`${over} over the ${MAX_HEAD_LINES}-line cap, all grandfathered or routed · ` +
|
|
226
|
+
`allowlist holds ${GRANDFATHERED.length} entry(ies), shrink-only\n`,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
return 0;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Robust "am I the entry script?" — realpath-compares argv[1] to this file. */
|
|
233
|
+
function _isCliEntry(): boolean {
|
|
234
|
+
const a = process.argv[1];
|
|
235
|
+
if (!a) return false;
|
|
236
|
+
if (a === _FILE || pathToFileURL(path.resolve(a)).href === import.meta.url) return true;
|
|
237
|
+
try {
|
|
238
|
+
return fs.realpathSync(a) === fs.realpathSync(_FILE);
|
|
239
|
+
} catch {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (_isCliEntry()) {
|
|
244
|
+
try {
|
|
245
|
+
process.exit(main());
|
|
246
|
+
} catch (exc) {
|
|
247
|
+
if (exc instanceof ExitCode) {
|
|
248
|
+
process.exit(exc.code);
|
|
249
|
+
}
|
|
250
|
+
throw exc;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export { REPO, SKILLS_REL, ExitCode };
|
|
@@ -48,6 +48,7 @@ import { fileURLToPath } from 'node:url';
|
|
|
48
48
|
|
|
49
49
|
import { projectStoreSlug } from './_lib/cc_transcript.js';
|
|
50
50
|
import {
|
|
51
|
+
isArtifactRead,
|
|
51
52
|
isConsultation,
|
|
52
53
|
isUiWrite,
|
|
53
54
|
type ToolEvent,
|
|
@@ -81,6 +82,24 @@ export interface SessionMeasurement {
|
|
|
81
82
|
consulted: number;
|
|
82
83
|
/** Those turns followed later by opening the review skill. */
|
|
83
84
|
reviewOpenedAfter: number;
|
|
85
|
+
/**
|
|
86
|
+
* Those turns that followed reading a provided design artifact.
|
|
87
|
+
*
|
|
88
|
+
* Shares this function's denominator on purpose — the two rates answer
|
|
89
|
+
* different questions over the SAME population ("did it consult the design
|
|
90
|
+
* surface" vs "did it read the handed-over source"), and measuring them on
|
|
91
|
+
* separate denominators is how two instruments about one behaviour stop
|
|
92
|
+
* being comparable.
|
|
93
|
+
*/
|
|
94
|
+
artifactRead: number;
|
|
95
|
+
/** True when this session read a provided artifact at any point. */
|
|
96
|
+
sawArtifact: boolean;
|
|
97
|
+
/**
|
|
98
|
+
* True when this session's FIRST UI write came after an artifact read —
|
|
99
|
+
* the metric named "artifact read before first UI write". Meaningless
|
|
100
|
+
* unless `sawArtifact`, which is why it is reported over that denominator.
|
|
101
|
+
*/
|
|
102
|
+
readBeforeFirstWrite: boolean;
|
|
84
103
|
}
|
|
85
104
|
|
|
86
105
|
export interface RateReport {
|
|
@@ -96,6 +115,16 @@ export interface RateReport {
|
|
|
96
115
|
uiWriteTurns: number;
|
|
97
116
|
consulted: number;
|
|
98
117
|
reviewOpenedAfter: number;
|
|
118
|
+
/** UI-write turns preceded by reading a provided design artifact. */
|
|
119
|
+
artifactRead: number;
|
|
120
|
+
/**
|
|
121
|
+
* Sessions that read a provided artifact at all — the ONLY honest
|
|
122
|
+
* denominator for the read-before-write rate. A session with no handover
|
|
123
|
+
* cannot fail to read one.
|
|
124
|
+
*/
|
|
125
|
+
handoverSessions: number;
|
|
126
|
+
/** Of those, the ones whose first UI write came after the read. */
|
|
127
|
+
handoverReadFirst: number;
|
|
99
128
|
}
|
|
100
129
|
|
|
101
130
|
/**
|
|
@@ -139,10 +168,20 @@ export function toolUseToEvent(part: Record<string, unknown>): ToolEvent | null
|
|
|
139
168
|
* consult for the file it already wrote.
|
|
140
169
|
*/
|
|
141
170
|
export function measureSession(timed: readonly TimedEvent[]): SessionMeasurement {
|
|
142
|
-
const out: SessionMeasurement = {
|
|
171
|
+
const out: SessionMeasurement = {
|
|
172
|
+
uiWriteTurns: 0,
|
|
173
|
+
consulted: 0,
|
|
174
|
+
reviewOpenedAfter: 0,
|
|
175
|
+
artifactRead: 0,
|
|
176
|
+
sawArtifact: false,
|
|
177
|
+
readBeforeFirstWrite: false,
|
|
178
|
+
};
|
|
143
179
|
let consulted = false;
|
|
180
|
+
let artifactRead = false;
|
|
144
181
|
/** turn -> was the session already consulted when that turn first wrote UI */
|
|
145
182
|
const writeTurns = new Map<number, boolean>();
|
|
183
|
+
/** turn -> had an artifact been read when that turn first wrote UI */
|
|
184
|
+
const artifactTurns = new Map<number, boolean>();
|
|
146
185
|
let lastReviewOpenAt = -1;
|
|
147
186
|
const reviewOpenPositions: number[] = [];
|
|
148
187
|
const writePositions: Array<{ position: number; turn: number }> = [];
|
|
@@ -151,10 +190,18 @@ export function measureSession(timed: readonly TimedEvent[]): SessionMeasurement
|
|
|
151
190
|
if (isUiWrite(event)) {
|
|
152
191
|
if (!writeTurns.has(turn)) {
|
|
153
192
|
writeTurns.set(turn, consulted);
|
|
193
|
+
artifactTurns.set(turn, artifactRead);
|
|
154
194
|
writePositions.push({ position, turn });
|
|
155
195
|
}
|
|
156
196
|
return;
|
|
157
197
|
}
|
|
198
|
+
// Evaluated INDEPENDENTLY of the consultation branch, never as an
|
|
199
|
+
// early return before it. The two predicates are not disjoint —
|
|
200
|
+
// `skills/design-review/references/design.html` satisfies both — so a
|
|
201
|
+
// `return` here would remove such an event from the consultation
|
|
202
|
+
// numerator and from the discharge-proxy latch. An earlier revision did
|
|
203
|
+
// exactly that while claiming in its comment to prevent it.
|
|
204
|
+
if (isArtifactRead(event)) artifactRead = true;
|
|
158
205
|
if (isConsultation(event)) {
|
|
159
206
|
consulted = true;
|
|
160
207
|
if (event.file.replace(/\\/g, '/').toLowerCase().includes(`skills/${REVIEW_SURFACE}/`)) {
|
|
@@ -166,6 +213,20 @@ export function measureSession(timed: readonly TimedEvent[]): SessionMeasurement
|
|
|
166
213
|
|
|
167
214
|
out.uiWriteTurns = writeTurns.size;
|
|
168
215
|
for (const wasConsulted of writeTurns.values()) if (wasConsulted) out.consulted += 1;
|
|
216
|
+
for (const wasRead of artifactTurns.values()) if (wasRead) out.artifactRead += 1;
|
|
217
|
+
out.sawArtifact = artifactRead;
|
|
218
|
+
// KNOWN IMPRECISION, accepted: ordering inside ONE assistant message comes
|
|
219
|
+
// from the `tool_use` part index, and parallel tool calls in a single
|
|
220
|
+
// message carry no real order. A read and a write issued together can
|
|
221
|
+
// therefore score either way. Not repaired here — a transcript carries no
|
|
222
|
+
// per-call timestamp, so the fix is a different data source rather than a
|
|
223
|
+
// different loop, and the error is bounded to same-message pairs. It is a
|
|
224
|
+
// third reason the published rate is a ceiling.
|
|
225
|
+
//
|
|
226
|
+
// The FIRST write's outcome — `artifactTurns` is insertion-ordered on the
|
|
227
|
+
// first UI write of each turn, so its first value is that write's state.
|
|
228
|
+
const firstWrite = artifactTurns.values().next();
|
|
229
|
+
out.readBeforeFirstWrite = firstWrite.done !== true && firstWrite.value === true;
|
|
169
230
|
// A single max comparison, not a scan per write: the proxy asks only whether
|
|
170
231
|
// ANY review-open follows, so the last one decides every earlier write.
|
|
171
232
|
void reviewOpenPositions;
|
|
@@ -216,6 +277,9 @@ export function measureStore(store: string, limit: number): RateReport {
|
|
|
216
277
|
uiWriteTurns: 0,
|
|
217
278
|
consulted: 0,
|
|
218
279
|
reviewOpenedAfter: 0,
|
|
280
|
+
artifactRead: 0,
|
|
281
|
+
handoverSessions: 0,
|
|
282
|
+
handoverReadFirst: 0,
|
|
219
283
|
};
|
|
220
284
|
// A store that is not there and a store with nothing in it are different
|
|
221
285
|
// findings, and only one of them is about the agent. Collapsing them into
|
|
@@ -255,6 +319,11 @@ export function measureStore(store: string, limit: number): RateReport {
|
|
|
255
319
|
report.uiWriteTurns += measured.uiWriteTurns;
|
|
256
320
|
report.consulted += measured.consulted;
|
|
257
321
|
report.reviewOpenedAfter += measured.reviewOpenedAfter;
|
|
322
|
+
report.artifactRead += measured.artifactRead;
|
|
323
|
+
if (measured.sawArtifact) {
|
|
324
|
+
report.handoverSessions += 1;
|
|
325
|
+
if (measured.readBeforeFirstWrite) report.handoverReadFirst += 1;
|
|
326
|
+
}
|
|
258
327
|
}
|
|
259
328
|
return report;
|
|
260
329
|
}
|
|
@@ -285,6 +354,62 @@ export function render(report: RateReport, threshold: number): string {
|
|
|
285
354
|
lines.push(
|
|
286
355
|
` discharge PROXY (review opened) ${pct(report.reviewOpenedAfter, report.uiWriteTurns)} (${report.reviewOpenedAfter}/${report.uiWriteTurns})`,
|
|
287
356
|
);
|
|
357
|
+
lines.push(
|
|
358
|
+
` artifact read (all UI turns) ${pct(report.artifactRead, report.uiWriteTurns)} (${report.artifactRead}/${report.uiWriteTurns})`,
|
|
359
|
+
);
|
|
360
|
+
lines.push(
|
|
361
|
+
` READ BEFORE FIRST WRITE ${pct(report.handoverReadFirst, report.handoverSessions)} (${report.handoverReadFirst}/${report.handoverSessions} handover session(s))`,
|
|
362
|
+
);
|
|
363
|
+
lines.push('');
|
|
364
|
+
lines.push(
|
|
365
|
+
' READ BEFORE FIRST WRITE is the rate to quote. Its denominator is',
|
|
366
|
+
);
|
|
367
|
+
lines.push(
|
|
368
|
+
' sessions that read a provided artifact at all — a session with no',
|
|
369
|
+
);
|
|
370
|
+
lines.push(
|
|
371
|
+
' handover cannot fail to read one, so including it would dilute the',
|
|
372
|
+
);
|
|
373
|
+
lines.push(
|
|
374
|
+
' rate toward the size of the estate rather than the behaviour.',
|
|
375
|
+
);
|
|
376
|
+
lines.push(
|
|
377
|
+
' That denominator is narrower than it sounds, twice: "artifact" means',
|
|
378
|
+
);
|
|
379
|
+
lines.push(
|
|
380
|
+
' a path ending design.html or under .claude/design-system/ — a handover',
|
|
381
|
+
);
|
|
382
|
+
lines.push(
|
|
383
|
+
' under any other name is not counted — and a session that read one but',
|
|
384
|
+
);
|
|
385
|
+
lines.push(
|
|
386
|
+
' wrote no UI is excluded, since it had nothing to be late for.',
|
|
387
|
+
);
|
|
388
|
+
lines.push(
|
|
389
|
+
' The line above it shares the consultation denominator and is kept for',
|
|
390
|
+
);
|
|
391
|
+
lines.push(
|
|
392
|
+
' comparability with that rate only.',
|
|
393
|
+
);
|
|
394
|
+
lines.push('');
|
|
395
|
+
lines.push(
|
|
396
|
+
' A CEILING, NOT A MEASUREMENT, for two reasons. A session handed an',
|
|
397
|
+
);
|
|
398
|
+
lines.push(
|
|
399
|
+
' artifact that NEVER reads it is invisible — nothing in a transcript',
|
|
400
|
+
);
|
|
401
|
+
lines.push(
|
|
402
|
+
' marks a handover that was ignored, and that case IS the defect. And a',
|
|
403
|
+
);
|
|
404
|
+
lines.push(
|
|
405
|
+
' search whose path argument happens to be the artifact counts as a read',
|
|
406
|
+
);
|
|
407
|
+
lines.push(
|
|
408
|
+
' although nothing was read. Both errors point the same way: the true',
|
|
409
|
+
);
|
|
410
|
+
lines.push(
|
|
411
|
+
' read-before-write rate is at most what is printed here.',
|
|
412
|
+
);
|
|
288
413
|
lines.push('');
|
|
289
414
|
|
|
290
415
|
if (report.sessionsWithUiWrite < threshold) {
|
|
@@ -10,10 +10,13 @@
|
|
|
10
10
|
* JSON files (json.dumps indent=2 + trailing newline). No behaviour
|
|
11
11
|
* changes.
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
13
|
+
* SKILLS_ROOT diverges from the mirrored .py source: it resolves the live
|
|
14
|
+
* skills tree through the shared resolver instead of replicating the retired
|
|
15
|
+
* container literal. Replicating it made every subcommand fail in
|
|
16
|
+
* `_skill_dir()` before doing any work — the twin was byte-faithful to a
|
|
17
|
+
* source that no longer exists and runnable against nothing. Documented per
|
|
18
|
+
* ADR-200 § 6 in `docs/migration/divergences/src-scripts-run_skill_evals.md`;
|
|
19
|
+
* an undocumented difference would be a regression by definition.
|
|
17
20
|
*
|
|
18
21
|
* Sub-agent SPAWNING is per-environment (Claude Code, Augment Code,
|
|
19
22
|
* council) and is left as a stub `_spawn_subagent(...)` that authors
|
|
@@ -25,11 +28,13 @@ import * as fs from 'node:fs';
|
|
|
25
28
|
import * as path from 'node:path';
|
|
26
29
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
27
30
|
|
|
31
|
+
import { SRC_SKILLS } from './_lib/agent_src.js';
|
|
32
|
+
|
|
28
33
|
const _HERE = fileURLToPath(import.meta.url);
|
|
29
34
|
// src/scripts/run_skill_evals.ts → parents[2] is the repo root (mirrors
|
|
30
35
|
// `Path(__file__).resolve().parent.parent.parent` in the .py).
|
|
31
36
|
export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
32
|
-
export const SKILLS_ROOT =
|
|
37
|
+
export const SKILLS_ROOT = SRC_SKILLS();
|
|
33
38
|
|
|
34
39
|
/** JSON value type for eval specs / grades. */
|
|
35
40
|
type JsonValue =
|
|
@@ -44,6 +44,14 @@
|
|
|
44
44
|
"type": "string",
|
|
45
45
|
"enum": ["package", "project"],
|
|
46
46
|
"default": "package"
|
|
47
|
+
},
|
|
48
|
+
"sources": {
|
|
49
|
+
"type": "array",
|
|
50
|
+
"items": {
|
|
51
|
+
"type": "string",
|
|
52
|
+
"pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"
|
|
53
|
+
},
|
|
54
|
+
"description": "Optional harvest_id list (provenance/harvests.jsonl) this persona may cite as external doctrine. Three distinct states, and the difference is the point: ABSENT = unscoped, today's behaviour for every persona that does not opt in; [] = explicitly declares it asserts only from its own reasoning, nothing harvested; a non-empty list = scoped to exactly those ids. Every declared id must resolve to a ledger row — checked by lint_harvest_provenance.ts, which treats a persona entry as a citation exactly like an inline harvest marker. Not to be confused with the singular `source` field above, which records package-vs-project origin."
|
|
47
55
|
}
|
|
48
56
|
}
|
|
49
57
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"$id": "provenance-harvest.schema.json",
|
|
4
|
+
"title": "Provenance ledger harvest record",
|
|
5
|
+
"description": "One line of provenance/harvests.jsonl — a single knowledge claim adopted from an external source into this repo's own artefacts. Sibling of provenance-borrow.schema.json, which covers borrowed CODE; this one covers borrowed IDEAS. Contract: provenance/README.md. Enforced by src/scripts/lint_harvest_provenance.ts (hand-rolled validator, following the provenance-borrow precedent — this file documents the contract; it is not loaded at runtime).",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"additionalProperties": false,
|
|
8
|
+
"required": ["harvest_id", "stated_in", "source_ref", "evidence_locator", "harvested_at", "verdict"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"harvest_id": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"pattern": "^[a-z0-9]+(-[a-z0-9]+)*$",
|
|
13
|
+
"description": "Kebab-case slug naming the harvested mechanism. This is the id an artefact cites; it must be unique across the ledger."
|
|
14
|
+
},
|
|
15
|
+
"stated_in": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"description": "Repo-relative path of the artefact asserting the claim, optionally suffixed ':<line>'. The path must exist."
|
|
18
|
+
},
|
|
19
|
+
"source_ref": {
|
|
20
|
+
"type": "string",
|
|
21
|
+
"description": "Either a public source pinned to a revision ('<url>@<sha>'), or an opaque token ('opaque:<id>' / 'ENC1:<payload>') when source-confidentiality keeps the source name out of the tracked tree. A bare URL with no revision is rejected — an unpinned reference cannot be re-verified."
|
|
22
|
+
},
|
|
23
|
+
"evidence_locator": {
|
|
24
|
+
"type": "string",
|
|
25
|
+
"minLength": 3,
|
|
26
|
+
"description": "Where inside the source the claim was read — a file:line, a section anchor, or an equivalent locator. Free-form because sources differ, but never empty: a claim whose evidence cannot be located again is not a citation."
|
|
27
|
+
},
|
|
28
|
+
"harvested_at": {
|
|
29
|
+
"type": "string",
|
|
30
|
+
"pattern": "^\\d{4}-\\d{2}-\\d{2}$",
|
|
31
|
+
"description": "ISO-8601 date (YYYY-MM-DD) the harvest landed."
|
|
32
|
+
},
|
|
33
|
+
"verdict": {
|
|
34
|
+
"type": "string",
|
|
35
|
+
"enum": ["adopt", "adapt"],
|
|
36
|
+
"description": "How the mechanism entered this repo. Only adopt and adapt produce a row — a rejected, already-present, or unclear finding is recorded in the analysis document, never in the ledger, because nothing in the tree cites it."
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -148,6 +148,36 @@ export function classify(rel: string): TriageClass {
|
|
|
148
148
|
return 'C';
|
|
149
149
|
}
|
|
150
150
|
|
|
151
|
+
/**
|
|
152
|
+
* Non-gate scripts eligible for the Class-A advisory pass.
|
|
153
|
+
*
|
|
154
|
+
* WHY A SECOND POPULATION EXISTS AT ALL. The gate population above is
|
|
155
|
+
* deliberately shared with the ratchet and the registration test, because three
|
|
156
|
+
* sites once carried their own regex and disagreed. That sharing is worth
|
|
157
|
+
* keeping — but it means the sweep's reach is a FILENAME PREFIX, and a script
|
|
158
|
+
* reading a retired container is dead whether or not its name starts with one
|
|
159
|
+
* of five words. Measured: `run_skill_evals.ts` roots its `SKILLS_ROOT` at the
|
|
160
|
+
* skills subtree of the container ADR-051 retired (see `CLASS_A_RETIRED`), and
|
|
161
|
+
* every subcommand fails in `_skill_dir()` before doing any work — while its
|
|
162
|
+
* sibling `skill_trigger_eval.ts`, same retired root and a `skill_` prefix, is
|
|
163
|
+
* reported. One visible, one not, on the prefix alone.
|
|
164
|
+
*
|
|
165
|
+
* So this pass widens the READING, never the shared population: it is
|
|
166
|
+
* Class-A-only (a closed set of retired containers, needing no allowlist to stay
|
|
167
|
+
* honest), it is advisory, and it feeds neither the ratchet count nor the exit
|
|
168
|
+
* code. Widening `matchesGatePattern` instead would move the ratchet base for a
|
|
169
|
+
* reason unrelated to any gate.
|
|
170
|
+
*/
|
|
171
|
+
export function isNonGateScript(filename: string): boolean {
|
|
172
|
+
if (!filename.endsWith('.ts')) return false;
|
|
173
|
+
if (filename.endsWith('.d.ts') || filename.endsWith('.test.ts')) return false;
|
|
174
|
+
// This file declares the retired containers as a classification table
|
|
175
|
+
// (`CLASS_A_RETIRED`); reading its own constant back as a dead root is a
|
|
176
|
+
// self-report, not a finding.
|
|
177
|
+
if (filename === 'sweep_dead_scan_roots.ts') return false;
|
|
178
|
+
return !isGateFile(filename);
|
|
179
|
+
}
|
|
180
|
+
|
|
151
181
|
export interface Finding {
|
|
152
182
|
rel: string;
|
|
153
183
|
names: string;
|
|
@@ -827,6 +857,20 @@ export function main(argv: string[]): number {
|
|
|
827
857
|
if (r.confirmed.length || r.unproven.length) results.set(f, r);
|
|
828
858
|
}
|
|
829
859
|
|
|
860
|
+
// Class-A advisory over the scripts the shared gate population cannot reach.
|
|
861
|
+
// Reuses the same extractor, so a prose mention or a report string is not a
|
|
862
|
+
// finding — only a literal join over a recognised root base is.
|
|
863
|
+
const advisory: string[] = [];
|
|
864
|
+
const advisoryPop = fs.existsSync(args.scripts)
|
|
865
|
+
? fs.readdirSync(args.scripts).filter(isNonGateScript).sort()
|
|
866
|
+
: [];
|
|
867
|
+
for (const f of advisoryPop) {
|
|
868
|
+
const r = analyze(fs.readFileSync(path.join(args.scripts, f), 'utf-8'), repoRoot);
|
|
869
|
+
for (const h of [...r.confirmed, ...r.unproven]) {
|
|
870
|
+
if (classify(h.rel) === 'A') advisory.push(` advisory [A] ${f}: ${h.rel} (${h.names})`);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
|
|
830
874
|
const used = new Set<Disposition>();
|
|
831
875
|
const disposed = (script: string, rel: string): Disposition | undefined => {
|
|
832
876
|
const hit = ledger.find((l) => l.script === script && l.rel === rel);
|
|
@@ -899,6 +943,13 @@ export function main(argv: string[]): number {
|
|
|
899
943
|
if (!args.quiet) {
|
|
900
944
|
process.stdout.write(`sweep_dead_scan_roots: ${pop.length} gate script(s) scanned\n`);
|
|
901
945
|
for (const l of lines) process.stdout.write(`${l}\n`);
|
|
946
|
+
if (advisory.length) {
|
|
947
|
+
process.stdout.write(
|
|
948
|
+
`sweep_dead_scan_roots: ${advisoryPop.length} non-gate script(s) scanned ` +
|
|
949
|
+
'(Class-A advisory — not counted, not gated)\n',
|
|
950
|
+
);
|
|
951
|
+
for (const l of advisory) process.stdout.write(`${l}\n`);
|
|
952
|
+
}
|
|
902
953
|
}
|
|
903
954
|
|
|
904
955
|
if (stale.length) {
|