@event4u/agent-config 14.2.0 → 14.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +286 -218
- package/agents/templates/.ai-council.yml.example +35 -0
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +83 -12
- package/dist/agent-src/rules/roadmap-progress-sync.md +48 -4
- package/dist/agent-src/scripts/resume_probe.ts +154 -9
- package/dist/agent-src/scripts/roadmap_gates.ts +210 -6
- package/dist/agent-src/skills/roadmap-management/SKILL.md +9 -4
- package/dist/agent-src/templates/agent-settings.md +70 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/scripts/telemetry/remote.ts +571 -0
- package/dist/agent-src/templates/scripts/telemetry/settings.ts +222 -44
- package/dist/cli/python/workspace_drive.js +8 -3
- package/dist/cli/python/workspace_drive.js.map +1 -1
- package/dist/cli/python/workspace_hosts.js +16 -12
- package/dist/cli/python/workspace_hosts.js.map +1 -1
- package/dist/cli/registry.js +2 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/{chunk-F3YUF7SA.js → chunk-3UCJWMMP.js} +1 -1
- package/dist/cli-delegate/chunk-F7DQWYR5.js +69 -0
- package/dist/cli-delegate/{chunk-7PZ4FSZB.js → chunk-FBKL6JWT.js} +61 -2
- package/dist/cli-delegate/{chunk-MNWRDMR5.js → chunk-PKOBQADP.js} +1 -1
- package/dist/cli-delegate/cmd_conformance.js +26 -11
- package/dist/cli-delegate/cmd_doctor.js +2 -2
- package/dist/cli-delegate/cmd_handoff.js +3 -3
- package/dist/cli-delegate/cmd_route_audit.js +2 -2
- package/dist/cli-delegate/cmd_settings_set.js +17 -3
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +4 -4
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +3684 -1719
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +1070 -408
- package/dist/scripts/_lib/host_launch.js +66 -0
- package/dist/scripts/_lib/host_launch.js.map +1 -0
- package/dist/server/io/yamlIO.js +175 -3
- package/dist/server/io/yamlIO.js.map +1 -1
- package/dist/server/routes/wizard.js +20 -1
- package/dist/server/routes/wizard.js.map +1 -1
- package/dist/shared/settingsConsent.js +25 -0
- package/dist/shared/settingsConsent.js.map +1 -1
- package/docs/contracts/settings-classes.md +11 -0
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +2 -1
- package/package.json +1 -1
- package/src/agent-src/scripts/resume_probe.ts +154 -9
- package/src/agent-src/scripts/roadmap_gates.ts +210 -6
- package/src/agent-src/templates/scripts/telemetry/remote.ts +571 -0
- package/src/agent-src/templates/scripts/telemetry/settings.ts +222 -44
- package/src/config/estate-count-budget.json +120 -0
- package/src/config/evaluator-budgets.json +4 -4
- package/src/config/gate-coverage.yml +32 -4
- package/src/config/gate-violation-baselines.json +63 -63
- package/src/config/gitignore-block.txt +5 -0
- package/src/config/hook-latency-budget.json +40 -1
- package/src/config/hook-token-budget.json +16 -0
- package/src/config/rule-stub-ceilings.json +11 -3
- package/src/scripts/_dispatch.bash +42 -1
- package/src/scripts/_lib/council_fallback_posture.ts +81 -0
- package/src/scripts/_lib/council_fallback_wiring.ts +126 -0
- package/src/scripts/_lib/council_settings_block.ts +152 -0
- package/src/scripts/_lib/headless_invocation.ts +285 -0
- package/src/scripts/_lib/host_launch.ts +72 -0
- package/src/scripts/_lib/map_to_object.ts +16 -0
- package/src/scripts/_lib/roadmap_checkboxes.ts +88 -0
- package/src/scripts/_lib/run_checkpoint.ts +409 -0
- package/src/scripts/_lib/session_register.ts +32 -1
- package/src/scripts/_lib/stdin.ts +35 -1
- package/src/scripts/_lib/turn_end_refusals.ts +12 -4
- package/src/scripts/_lib/unattended_guard.ts +371 -0
- package/src/scripts/ai_council/config.ts +27 -1
- package/src/scripts/ai_council/events_log.ts +12 -1
- package/src/scripts/ai_council/fallback_config.ts +114 -0
- package/src/scripts/ai_council/mid_flight_fallback.ts +295 -0
- package/src/scripts/ai_council/orchestrator.ts +251 -292
- package/src/scripts/ai_council/py_parity.ts +139 -0
- package/src/scripts/ai_council/response_render.ts +99 -0
- package/src/scripts/ai_council/spend_gate.ts +123 -0
- package/src/scripts/ai_council/transport_resolver.ts +81 -13
- package/src/scripts/bench_hook_injection.ts +100 -6
- package/src/scripts/bench_hook_latency.ts +442 -16
- package/src/scripts/check_estate_count.ts +924 -0
- package/src/scripts/compile_hook_manifest.ts +67 -0
- package/src/scripts/council_cli.ts +100 -125
- package/src/scripts/decision_memo.ts +325 -0
- package/src/scripts/hook_manifest.json +1 -0
- package/src/scripts/hook_manifest.yaml +172 -25
- package/src/scripts/hooks/concern_registry.ts +6 -0
- package/src/scripts/hooks/dispatch_hook.ts +183 -10
- package/src/scripts/hooks/dispatch_issues.ts +26 -0
- package/src/scripts/hooks/end_review_nudge_hook.ts +97 -24
- package/src/scripts/hooks/hook_stdin.ts +111 -5
- package/src/scripts/hooks/injection_budget.ts +595 -0
- package/src/scripts/hooks/payload_stub.ts +446 -0
- package/src/scripts/hooks/run_continuation_hook.ts +627 -0
- package/src/scripts/hooks/session_eol_hook.ts +52 -0
- package/src/scripts/hooks/telemetry_usage_hook.ts +339 -0
- package/src/scripts/hooks/tool_result_bytes_hook.ts +27 -0
- package/src/scripts/hooks/turn_end_gate_hook.ts +45 -7
- package/src/scripts/hooks_doctor.ts +16 -2
- package/src/scripts/interruption_report.ts +330 -13
- package/src/scripts/lint_hook_manifest.ts +167 -0
- package/src/scripts/roadmap_progress_hook.ts +254 -46
- package/src/scripts/run_supervise.ts +623 -0
- package/src/scripts/session_register_hook.ts +108 -8
- package/src/scripts/sessions_cli.ts +18 -6
- package/src/scripts/telemetry_disclosure_hook.ts +244 -0
- package/src/server/io/yamlIO.ts +167 -3
- package/src/shared/settingsConsent.ts +49 -2
- package/dist/cli-delegate/chunk-MMJL4TIX.js +0 -18
|
@@ -49,6 +49,8 @@ import * as fs from 'node:fs';
|
|
|
49
49
|
import * as path from 'node:path';
|
|
50
50
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
51
51
|
|
|
52
|
+
import { SESSION_ID_LEN, derive_session_tag } from './chat_history.js';
|
|
53
|
+
|
|
52
54
|
export const DEFAULT_WINDOW = 30;
|
|
53
55
|
|
|
54
56
|
export interface LedgerRow {
|
|
@@ -70,25 +72,79 @@ export interface HistoryTurn {
|
|
|
70
72
|
export interface RunReport {
|
|
71
73
|
run_id: string;
|
|
72
74
|
roadmap: string | null;
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
75
|
+
/**
|
|
76
|
+
* Contact counts, or `null` when this run has NO interruption-ledger
|
|
77
|
+
* entry — a session predating the ledger, or one whose stop hook never
|
|
78
|
+
* fired.
|
|
79
|
+
*
|
|
80
|
+
* R2 round 6, finding 8. These were built unconditionally from the ledger
|
|
81
|
+
* rows, so an absent run emitted `asks: 0, contacts: 0, halts: 0` — while
|
|
82
|
+
* this report's own note tells the reader such a run is "reported with the
|
|
83
|
+
* missing axis null". The aggregate median already excluded them; the ROWS
|
|
84
|
+
* did not, and a reader consuming rows would score an unmeasured run as
|
|
85
|
+
* zero contacts. That is precisely the arithmetic falsification criterion
|
|
86
|
+
* (3) of the pre-registered `user-out-of-loop-baseline` claim forbids —
|
|
87
|
+
* "scoring an unmeasured run as zero contacts is the arithmetic that would
|
|
88
|
+
* manufacture the result".
|
|
89
|
+
*/
|
|
90
|
+
asks: number | null;
|
|
91
|
+
handbacks: number | null;
|
|
92
|
+
contacts: number | null;
|
|
93
|
+
halts: number | null;
|
|
77
94
|
elapsed_minutes: number | null;
|
|
78
95
|
waiting_minutes: number | null;
|
|
79
96
|
working_minutes: number | null;
|
|
97
|
+
/** `engage` events for this run in `run-continuation.jsonl`. */
|
|
98
|
+
reengagements: number;
|
|
99
|
+
/** `halt-stall` events — the anti-stall rung firing, not the loop working. */
|
|
100
|
+
stall_halts: number;
|
|
101
|
+
/** Relaunches the supervisor has spent on this run. */
|
|
102
|
+
relaunches: number;
|
|
103
|
+
/** Decision memos this run wrote instead of contacting the user. */
|
|
104
|
+
memos: number;
|
|
80
105
|
}
|
|
81
106
|
|
|
107
|
+
/**
|
|
108
|
+
* Runs required before either pre-registered claim may be compared.
|
|
109
|
+
*
|
|
110
|
+
* Fixed BEFORE any data by `user-out-of-loop-baseline` and
|
|
111
|
+
* `roadmap-wall-clock-baseline` in `docs/CLAIMS.md`, both falsification
|
|
112
|
+
* criterion (3): "≥ 20 recorded runs before any comparison, else
|
|
113
|
+
* UNDERPOWERED". Named here so the report states the floor it is judging
|
|
114
|
+
* itself against rather than leaving a reader to look it up.
|
|
115
|
+
*/
|
|
116
|
+
export const POWER_FLOOR_RUNS = 20;
|
|
117
|
+
|
|
82
118
|
export interface Report {
|
|
83
119
|
window_requested: number;
|
|
84
120
|
sessions_found: number;
|
|
85
121
|
window_short: boolean;
|
|
86
122
|
synthetic_user_turns_excluded: number;
|
|
123
|
+
/** Runs carrying a contact count — the CONTACT axis's own denominator. */
|
|
124
|
+
contact_axis_runs: number;
|
|
125
|
+
/** Runs carrying timing — the WALL-CLOCK axis's own, much smaller, denominator. */
|
|
126
|
+
wall_clock_axis_runs: number;
|
|
127
|
+
/**
|
|
128
|
+
* Agent→user turnarounds behind `median_user_wait_minutes`.
|
|
129
|
+
*
|
|
130
|
+
* A THIRD denominator, and it is neither axis's: one run contributes as
|
|
131
|
+
* many waits as it had turnarounds, so this count is in gaps where the two
|
|
132
|
+
* above are in runs. Kept separate rather than folded into either.
|
|
133
|
+
*/
|
|
134
|
+
user_wait_gaps: number;
|
|
135
|
+
/** The pre-registered floor both claims fix before any comparison. */
|
|
136
|
+
power_floor_runs: number;
|
|
87
137
|
runs: RunReport[];
|
|
88
138
|
median_contacts_per_run: number | null;
|
|
89
139
|
median_user_wait_minutes: number | null;
|
|
90
140
|
median_elapsed_minutes: number | null;
|
|
91
141
|
median_working_minutes: number | null;
|
|
142
|
+
/** Phase 5.0 — the autonomy axis. */
|
|
143
|
+
median_reengagements_per_run: number | null;
|
|
144
|
+
/** Share of runs whose loop hit the stall rung. `null` with no runs. */
|
|
145
|
+
stall_halt_rate: number | null;
|
|
146
|
+
median_relaunches_per_run: number | null;
|
|
147
|
+
median_memos_per_run: number | null;
|
|
92
148
|
notes: string[];
|
|
93
149
|
}
|
|
94
150
|
|
|
@@ -107,6 +163,140 @@ export function isSyntheticUserText(text: string): boolean {
|
|
|
107
163
|
return SYNTHETIC_MARKERS.some((m) => text.includes(m));
|
|
108
164
|
}
|
|
109
165
|
|
|
166
|
+
/**
|
|
167
|
+
* Look one run's value up in a source keyed differently from the ledger.
|
|
168
|
+
*
|
|
169
|
+
* R2 review, finding 2. Three producers write three key spaces for the same
|
|
170
|
+
* run and the report joined them with exact lookups, so `reengagements`,
|
|
171
|
+
* `stall_halts`, `stall_halt_rate` and `relaunches` read 0 / 0 % for every
|
|
172
|
+
* real run while the data sat on disk:
|
|
173
|
+
*
|
|
174
|
+
* - the interruption ledger `derive_session_tag(id)` = 16 hex
|
|
175
|
+
* - run-continuation.jsonl `deriveSessionKey(id)` = 32 hex
|
|
176
|
+
* - supervise-relaunches the RAW `session_id`
|
|
177
|
+
*
|
|
178
|
+
* A zero meaning "no instrument" and a zero meaning "the key did not match"
|
|
179
|
+
* are indistinguishable in the output, and this report prints an explicit
|
|
180
|
+
* NO INSTRUMENT for two other axes precisely to avoid the first — so a silent
|
|
181
|
+
* second kind was the worse failure.
|
|
182
|
+
*
|
|
183
|
+
* Resolution is EXACT-FIRST and lives here rather than in the readers: a
|
|
184
|
+
* reader that rewrote its own keys would have to guess what an unrecognised
|
|
185
|
+
* string means, and guessing "hash it" turns a would-have-matched key into a
|
|
186
|
+
* guaranteed miss. The readers keep reporting what is on disk; the join owns
|
|
187
|
+
* the reconciliation, because only the join knows the ledger id it is
|
|
188
|
+
* looking for.
|
|
189
|
+
*
|
|
190
|
+
* The 32-hex key is a prefix-extension of the 16-hex tag ONLY because a
|
|
191
|
+
* session id carries no whitespace for `fingerprint` to normalise away.
|
|
192
|
+
* `interruption_report.test.ts` pins that equivalence rather than assuming it.
|
|
193
|
+
*/
|
|
194
|
+
export function lookupByRunId<T>(
|
|
195
|
+
source: ReadonlyMap<string, T> | Readonly<Record<string, T>>,
|
|
196
|
+
runId: string,
|
|
197
|
+
): T | undefined {
|
|
198
|
+
const get = (k: string): T | undefined =>
|
|
199
|
+
source instanceof Map ? source.get(k) : (source as Record<string, T>)[k];
|
|
200
|
+
// 1. The key as the ledger spells it. Covers a source that already agrees,
|
|
201
|
+
// and every fixture that writes one id everywhere.
|
|
202
|
+
const exact = get(runId);
|
|
203
|
+
if (exact !== undefined) return exact;
|
|
204
|
+
// 2. A 32-hex continuation key whose first SESSION_ID_LEN chars are this tag.
|
|
205
|
+
const keys: string[] = source instanceof Map ? [...source.keys()] : Object.keys(source);
|
|
206
|
+
for (const k of keys) {
|
|
207
|
+
if (k.length === 32 && /^[0-9a-f]+$/.test(k) && k.slice(0, SESSION_ID_LEN) === runId) {
|
|
208
|
+
return get(k);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
// 3. A raw session id that hashes to this tag.
|
|
212
|
+
for (const k of keys) {
|
|
213
|
+
if (derive_session_tag(k) === runId) return get(k);
|
|
214
|
+
}
|
|
215
|
+
return undefined;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Phase 5.0 — the autonomy axis, from three runtime sources.
|
|
220
|
+
*
|
|
221
|
+
* Each reader is best-effort and returns an empty map on any failure. That is
|
|
222
|
+
* correct rather than lenient: a run that predates the mechanism genuinely had
|
|
223
|
+
* zero re-engagements, and refusing to report the other axes because one
|
|
224
|
+
* source is absent would make the whole report unavailable on a fresh install.
|
|
225
|
+
*/
|
|
226
|
+
export function readContinuationEvents(
|
|
227
|
+
root: string,
|
|
228
|
+
): Map<string, { engage: number; stall: number }> {
|
|
229
|
+
const out = new Map<string, { engage: number; stall: number }>();
|
|
230
|
+
let text: string;
|
|
231
|
+
try {
|
|
232
|
+
text = fs.readFileSync(
|
|
233
|
+
path.join(root, 'agents', 'runtime', 'state', 'run-continuation.jsonl'),
|
|
234
|
+
'utf-8',
|
|
235
|
+
);
|
|
236
|
+
} catch {
|
|
237
|
+
return out;
|
|
238
|
+
}
|
|
239
|
+
for (const line of text.split('\n')) {
|
|
240
|
+
if (line.trim() === '') continue;
|
|
241
|
+
let rec: unknown;
|
|
242
|
+
try {
|
|
243
|
+
rec = JSON.parse(line);
|
|
244
|
+
} catch {
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
if (rec === null || typeof rec !== 'object' || Array.isArray(rec)) continue;
|
|
248
|
+
const o = rec as Record<string, unknown>;
|
|
249
|
+
const id = o['run_id'];
|
|
250
|
+
const ev = o['event'];
|
|
251
|
+
if (typeof id !== 'string' || typeof ev !== 'string') continue;
|
|
252
|
+
const cur = out.get(id) ?? { engage: 0, stall: 0 };
|
|
253
|
+
if (ev === 'engage') cur.engage += 1;
|
|
254
|
+
else if (ev === 'halt-stall') cur.stall += 1;
|
|
255
|
+
out.set(id, cur);
|
|
256
|
+
}
|
|
257
|
+
return out;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export function readRelaunchLedger(root: string): Record<string, number> {
|
|
261
|
+
try {
|
|
262
|
+
const raw: unknown = JSON.parse(
|
|
263
|
+
fs.readFileSync(
|
|
264
|
+
path.join(root, 'agents', 'runtime', 'state', 'supervise-relaunches.json'),
|
|
265
|
+
'utf-8',
|
|
266
|
+
),
|
|
267
|
+
);
|
|
268
|
+
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) return {};
|
|
269
|
+
const out: Record<string, number> = {};
|
|
270
|
+
for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
|
|
271
|
+
if (typeof v === 'number' && Number.isFinite(v) && v >= 0) out[k] = v;
|
|
272
|
+
}
|
|
273
|
+
return out;
|
|
274
|
+
} catch {
|
|
275
|
+
return {};
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
export function readMemoCounts(root: string): Record<string, number> {
|
|
280
|
+
const base = path.join(root, 'agents', 'runtime', 'state', 'decisions');
|
|
281
|
+
const out: Record<string, number> = {};
|
|
282
|
+
let runs: string[];
|
|
283
|
+
try {
|
|
284
|
+
runs = fs.readdirSync(base);
|
|
285
|
+
} catch {
|
|
286
|
+
return out;
|
|
287
|
+
}
|
|
288
|
+
for (const run of runs) {
|
|
289
|
+
try {
|
|
290
|
+
out[run] = fs
|
|
291
|
+
.readdirSync(path.join(base, run))
|
|
292
|
+
.filter((n) => /^\d{3}\.md$/.test(n)).length;
|
|
293
|
+
} catch {
|
|
294
|
+
// a file where a directory was expected — not a run
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return out;
|
|
298
|
+
}
|
|
299
|
+
|
|
110
300
|
export function median(values: readonly number[]): number | null {
|
|
111
301
|
if (values.length === 0) return null;
|
|
112
302
|
const sorted = [...values].sort((a, b) => a - b);
|
|
@@ -237,6 +427,13 @@ export function buildReport(root: string, windowRequested: number): Report {
|
|
|
237
427
|
// null wall clock rather than disappearing from the denominator.
|
|
238
428
|
for (const id of ledgerRuns) if (!windowedIds.has(id)) runIds.add(id);
|
|
239
429
|
|
|
430
|
+
// Phase 5.0 sources. All three are gitignored runtime state, all three are
|
|
431
|
+
// read best-effort, and an absent one yields zeros rather than an error —
|
|
432
|
+
// a run predating the mechanism genuinely had zero re-engagements.
|
|
433
|
+
const continuation = readContinuationEvents(root);
|
|
434
|
+
const relaunches = readRelaunchLedger(root);
|
|
435
|
+
const memosByRun = readMemoCounts(root);
|
|
436
|
+
|
|
240
437
|
const allGaps: number[] = [];
|
|
241
438
|
const runs: RunReport[] = [];
|
|
242
439
|
|
|
@@ -265,16 +462,31 @@ export function buildReport(root: string, windowRequested: number): Report {
|
|
|
265
462
|
if (elapsed !== null) working = Math.max(0, elapsed - waiting);
|
|
266
463
|
}
|
|
267
464
|
|
|
465
|
+
const cont = lookupByRunId(continuation, runId) ?? { engage: 0, stall: 0 };
|
|
466
|
+
const inLedger = ledgerRuns.has(runId);
|
|
268
467
|
runs.push({
|
|
269
468
|
run_id: runId,
|
|
270
469
|
roadmap: rows.find((r) => r.roadmap !== null)?.roadmap ?? null,
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
470
|
+
// `null`, not 0, when this run is absent from the ledger — see
|
|
471
|
+
// the field docs above and falsification criterion (3).
|
|
472
|
+
asks: inLedger ? asks : null,
|
|
473
|
+
handbacks: inLedger ? handbacks : null,
|
|
474
|
+
contacts: inLedger ? contacts.length : null,
|
|
475
|
+
halts: inLedger ? halts : null,
|
|
275
476
|
elapsed_minutes: elapsed,
|
|
276
477
|
waiting_minutes: waiting,
|
|
277
478
|
working_minutes: working,
|
|
479
|
+
reengagements: cont.engage,
|
|
480
|
+
stall_halts: cont.stall,
|
|
481
|
+
// By ROADMAP, not by run id: the relaunch ledger is keyed on the
|
|
482
|
+
// roadmap slug so its per-run cap survives the new session id a
|
|
483
|
+
// relaunch produces (see `run_supervise.RelaunchLedger`). Joining
|
|
484
|
+
// it on the run id would read 0 for every run — the same
|
|
485
|
+
// permanent-zero shape finding 2 fixed one source over.
|
|
486
|
+
relaunches: (rows.find((r) => r.roadmap !== null)?.roadmap ?? null) === null
|
|
487
|
+
? 0
|
|
488
|
+
: (relaunches[rows.find((r) => r.roadmap !== null)?.roadmap as string] ?? 0),
|
|
489
|
+
memos: lookupByRunId(memosByRun, runId) ?? 0,
|
|
278
490
|
});
|
|
279
491
|
}
|
|
280
492
|
|
|
@@ -303,10 +515,40 @@ export function buildReport(root: string, windowRequested: number): Report {
|
|
|
303
515
|
sessions_found: bySession.size,
|
|
304
516
|
window_short: bySession.size < windowRequested,
|
|
305
517
|
synthetic_user_turns_excluded: synthetic,
|
|
518
|
+
// Per-axis N against the pre-registered ≥ 20-run floor. The two axes
|
|
519
|
+
// read DIFFERENT sources — the ledger is a committed append-only file,
|
|
520
|
+
// the chat history is a rolling buffer — so their effective N diverges
|
|
521
|
+
// hard, and one banner over both cannot say which is underpowered.
|
|
522
|
+
//
|
|
523
|
+
// Measured 2026-08-19: **19** runs carried contacts and 4 carried
|
|
524
|
+
// timing, out of 21 runs in the window. NEITHER axis cleared the
|
|
525
|
+
// floor. The two runs in the gap carry timing and no ledger entry, so
|
|
526
|
+
// they raise the total and not the contact axis — which is the whole
|
|
527
|
+
// reason the total may not be read as either axis's N.
|
|
528
|
+
//
|
|
529
|
+
// R2 round 1, finding 1: this comment first said "21 runs carried
|
|
530
|
+
// contacts … the contact axis had cleared its own floor", i.e. it
|
|
531
|
+
// shipped the exact misreading this field exists to prevent, as a
|
|
532
|
+
// fact, inside the fix for it. The roadmap and CLAIMS.md in the same
|
|
533
|
+
// change said 19 and "it had not". Worth leaving on the record: the
|
|
534
|
+
// number is easy to get wrong even while writing the guard against
|
|
535
|
+
// getting it wrong, which is the argument for reading it off the
|
|
536
|
+
// field rather than off the total.
|
|
537
|
+
//
|
|
538
|
+
// `runsWithLedger` is reused rather than re-filtered (finding 4):
|
|
539
|
+
// `r.contacts` is null exactly when the run is absent from
|
|
540
|
+
// `ledgerRuns`, so a second predicate is a second thing to keep true.
|
|
541
|
+
contact_axis_runs: runsWithLedger,
|
|
542
|
+
wall_clock_axis_runs: runs.filter((r) => r.elapsed_minutes !== null).length,
|
|
543
|
+
user_wait_gaps: allGaps.length,
|
|
544
|
+
power_floor_runs: POWER_FLOOR_RUNS,
|
|
306
545
|
runs,
|
|
307
546
|
notes,
|
|
308
547
|
median_contacts_per_run: median(
|
|
309
|
-
runs
|
|
548
|
+
runs
|
|
549
|
+
.filter((r) => ledgerRuns.has(r.run_id))
|
|
550
|
+
.map((r) => r.contacts)
|
|
551
|
+
.filter((v): v is number => v !== null),
|
|
310
552
|
),
|
|
311
553
|
median_user_wait_minutes: median(allGaps),
|
|
312
554
|
median_elapsed_minutes: median(
|
|
@@ -315,6 +557,14 @@ export function buildReport(root: string, windowRequested: number): Report {
|
|
|
315
557
|
median_working_minutes: median(
|
|
316
558
|
runs.map((r) => r.working_minutes).filter((v): v is number => v !== null),
|
|
317
559
|
),
|
|
560
|
+
median_reengagements_per_run: median(runs.map((r) => r.reengagements)),
|
|
561
|
+
// A RATE, not a count: the question is what share of runs hit the
|
|
562
|
+
// anti-stall rung, and a raw count would rise with the window size and
|
|
563
|
+
// read as a regression when nothing changed.
|
|
564
|
+
stall_halt_rate:
|
|
565
|
+
runs.length === 0 ? null : runs.filter((r) => r.stall_halts > 0).length / runs.length,
|
|
566
|
+
median_relaunches_per_run: median(runs.map((r) => r.relaunches)),
|
|
567
|
+
median_memos_per_run: median(runs.map((r) => r.memos)),
|
|
318
568
|
};
|
|
319
569
|
}
|
|
320
570
|
|
|
@@ -322,6 +572,20 @@ function fmt(v: number | null, unit = ''): string {
|
|
|
322
572
|
return v === null ? 'n/a' : `${Math.round(v * 10) / 10}${unit}`;
|
|
323
573
|
}
|
|
324
574
|
|
|
575
|
+
/**
|
|
576
|
+
* The power verdict for ONE axis, printed on that axis's own header.
|
|
577
|
+
*
|
|
578
|
+
* `UNDERPOWERED` is the word the claims themselves use, so the reader meets
|
|
579
|
+
* the same term the falsification criterion is written in. The count is always
|
|
580
|
+
* shown, including when the floor is cleared — a bare "OK" would leave a
|
|
581
|
+
* reader unable to tell 20 from 200.
|
|
582
|
+
*/
|
|
583
|
+
function power(n: number, floor: number): string {
|
|
584
|
+
return n >= floor
|
|
585
|
+
? ` · n=${n} runs (≥ ${floor} floor cleared)`
|
|
586
|
+
: ` · n=${n} runs — ⚠️ UNDERPOWERED, floor is ${floor}`;
|
|
587
|
+
}
|
|
588
|
+
|
|
325
589
|
export function renderText(report: Report): string {
|
|
326
590
|
const lines: string[] = [];
|
|
327
591
|
lines.push('interruption_report — road-to-user-out-of-the-loop Phase 0');
|
|
@@ -332,19 +596,72 @@ export function renderText(report: Report): string {
|
|
|
332
596
|
);
|
|
333
597
|
lines.push(`synthetic user turns excluded: ${report.synthetic_user_turns_excluded}`);
|
|
334
598
|
lines.push('');
|
|
335
|
-
lines.push(
|
|
599
|
+
lines.push(`CONTACT AXIS${power(report.contact_axis_runs, report.power_floor_runs)}`);
|
|
336
600
|
lines.push(` median contacts per run: ${fmt(report.median_contacts_per_run)}`);
|
|
337
|
-
lines.push(` median user wait: ${fmt(report.median_user_wait_minutes, ' min')}`);
|
|
338
601
|
lines.push('');
|
|
339
|
-
lines.push(
|
|
602
|
+
lines.push(`WALL-CLOCK AXIS${power(report.wall_clock_axis_runs, report.power_floor_runs)}`);
|
|
340
603
|
lines.push(` median elapsed per run: ${fmt(report.median_elapsed_minutes, ' min')}`);
|
|
341
604
|
lines.push(` median agent working time: ${fmt(report.median_working_minutes, ' min')}`);
|
|
605
|
+
// `user wait` sits HERE and not under the contact axis, and it carries a
|
|
606
|
+
// third denominator of its own.
|
|
607
|
+
//
|
|
608
|
+
// R2 round 2, finding 1. It used to print under CONTACT AXIS, directly
|
|
609
|
+
// beneath that axis's ledger-sourced n — so the per-axis fix reproduced
|
|
610
|
+
// the very conflation it was written to remove, one line further down.
|
|
611
|
+
// `roadmap-wall-clock-baseline` settles which axis owns it: WAITING is
|
|
612
|
+
// named as that claim's own instrument, derived from chat-history
|
|
613
|
+
// timestamps, i.e. the source whose N is the small one.
|
|
614
|
+
//
|
|
615
|
+
// And the unit is gaps, not runs: one run contributes as many waits as it
|
|
616
|
+
// had agent→user turnarounds. Printing a run-count next to it would be the
|
|
617
|
+
// second half of the same error, so it carries its own count.
|
|
618
|
+
lines.push(
|
|
619
|
+
` median user wait: ${fmt(report.median_user_wait_minutes, ' min')}` +
|
|
620
|
+
` · n=${report.user_wait_gaps} gap(s), not runs`,
|
|
621
|
+
);
|
|
342
622
|
lines.push('');
|
|
343
|
-
lines.push(
|
|
623
|
+
lines.push('AUTONOMY AXIS (Phase 5.0)');
|
|
624
|
+
lines.push(` median re-engagements: ${fmt(report.median_reengagements_per_run)}`);
|
|
625
|
+
lines.push(
|
|
626
|
+
` stall-halt rate: ${
|
|
627
|
+
report.stall_halt_rate === null
|
|
628
|
+
? 'n/a'
|
|
629
|
+
: `${Math.round(report.stall_halt_rate * 1000) / 10}%`
|
|
630
|
+
}`,
|
|
631
|
+
);
|
|
632
|
+
lines.push(` median relaunches per run: ${fmt(report.median_relaunches_per_run)}`);
|
|
633
|
+
lines.push(` median memos per run: ${fmt(report.median_memos_per_run)}`);
|
|
634
|
+
// Two axes the roadmap names and this instrument CANNOT report. Stated as
|
|
635
|
+
// no-instrument rather than printed as 0, because a zero here would read
|
|
636
|
+
// as "measured, and it is zero" — the exact confusion between an absent
|
|
637
|
+
// record and an absent event that this suite records elsewhere.
|
|
638
|
+
// "cannot run YET" and "pre-registered" were both true when written and
|
|
639
|
+
// are both false now: the spawn is a published refusal
|
|
640
|
+
// (road-to-long-horizon-execution 4.0) and `unattended-demotion-gate` took
|
|
641
|
+
// its honest-null path in the same change-set that refused it. R2 round 2,
|
|
642
|
+
// finding 2 — `run_supervise.ts` had its identical wording rewritten and
|
|
643
|
+
// this sibling site in the same diff was missed, which is what the
|
|
644
|
+
// defect-pattern sweep exists to catch and did not.
|
|
645
|
+
lines.push(' unattended-vs-attended rework: NO INSTRUMENT, and permanently so — the');
|
|
646
|
+
lines.push(' unattended lane will not run: the spawn is a published REFUSAL, not an');
|
|
647
|
+
lines.push(' unbuilt feature (road-to-long-horizon-execution 4.0). `unattended-demotion-gate`');
|
|
648
|
+
lines.push(' took its honest-null path and is CLOSED — do not read it as a live threshold.');
|
|
649
|
+
lines.push(' memo revisit rate: NO INSTRUMENT — a memo carries no revisit marker,');
|
|
650
|
+
lines.push(' so a revisited decision is indistinguishable from one nobody reopened.');
|
|
651
|
+
lines.push('');
|
|
652
|
+
// The total carries its caveat inline, because the roadmap blames THIS
|
|
653
|
+
// line for the live misreading and round 1 left it untouched (R2 round 2,
|
|
654
|
+
// finding 11): a per-axis n printed thirty lines earlier does not reach a
|
|
655
|
+
// reader who scrolled to the number that looks like the sample size.
|
|
656
|
+
lines.push(
|
|
657
|
+
`runs: ${report.runs.length} · NOT either axis's N — see the per-axis n above ` +
|
|
658
|
+
`(contact ${report.contact_axis_runs}, wall-clock ${report.wall_clock_axis_runs})`,
|
|
659
|
+
);
|
|
344
660
|
for (const run of report.runs) {
|
|
345
661
|
lines.push(
|
|
346
662
|
` ${run.run_id} asks=${run.asks} handbacks=${run.handbacks} halts=${run.halts}` +
|
|
347
663
|
` elapsed=${fmt(run.elapsed_minutes)} working=${fmt(run.working_minutes)}` +
|
|
664
|
+
` re=${run.reengagements} stall=${run.stall_halts} relaunch=${run.relaunches} memo=${run.memos}` +
|
|
348
665
|
` roadmap=${run.roadmap ?? '-'}`,
|
|
349
666
|
);
|
|
350
667
|
}
|
|
@@ -179,11 +179,177 @@ function _check_concerns(manifest: YamlObject, errors: string[]): Set<string> {
|
|
|
179
179
|
}
|
|
180
180
|
}
|
|
181
181
|
}
|
|
182
|
+
// `needs_payload_bodies:` — the payload opt-in
|
|
183
|
+
// (`road-to-per-turn-hook-economy` step 2.1). Validated for the same
|
|
184
|
+
// reason `tools:` is, with the polarity reversed: the dispatcher ignores
|
|
185
|
+
// anything it does not recognise, so `true`, `"input"`, or a misspelled
|
|
186
|
+
// `[inputs]` all resolve to "keep nothing" and the concern silently
|
|
187
|
+
// receives stubs it was meant to opt out of. A concern that reads a body
|
|
188
|
+
// and gets a stub does not crash — it reads `undefined` and reports
|
|
189
|
+
// nothing, which is the failure this check exists to keep a YAML typo from
|
|
190
|
+
// introducing. The runtime deliberately does NOT widen on a malformed
|
|
191
|
+
// value; that would make this check unenforceable.
|
|
192
|
+
if ("needs_payload_bodies" in spec) {
|
|
193
|
+
const raw = spec["needs_payload_bodies"];
|
|
194
|
+
if (!Array.isArray(raw)) {
|
|
195
|
+
errors.push(
|
|
196
|
+
`concerns.${name}: 'needs_payload_bodies' must be a list of body ` +
|
|
197
|
+
`classes (got ${pyTypeName(raw)}) — use [input], [result], or ` +
|
|
198
|
+
`[input, result]; omit the key to receive stubs, which is the default`,
|
|
199
|
+
);
|
|
200
|
+
} else if (raw.length === 0) {
|
|
201
|
+
errors.push(
|
|
202
|
+
`concerns.${name}: 'needs_payload_bodies' is an empty list — omit ` +
|
|
203
|
+
`the key rather than declaring a need for nothing`,
|
|
204
|
+
);
|
|
205
|
+
} else {
|
|
206
|
+
for (const cls of raw) {
|
|
207
|
+
if (cls !== "input" && cls !== "result") {
|
|
208
|
+
errors.push(
|
|
209
|
+
`concerns.${name}: 'needs_payload_bodies' entries must be ` +
|
|
210
|
+
`'input' or 'result', got ${JSON.stringify(cls)}`,
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
182
216
|
names.add(name);
|
|
183
217
|
}
|
|
184
218
|
return names;
|
|
185
219
|
}
|
|
186
220
|
|
|
221
|
+
/** The two slots whose payload carries a stubbable tool body. */
|
|
222
|
+
const TOOL_BODY_EVENTS: ReadonlySet<string> = new Set([
|
|
223
|
+
"pre_tool_use",
|
|
224
|
+
"post_tool_use",
|
|
225
|
+
]);
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Payload keys per body class, mirroring `hooks/payload_stub.ts::BODY_KEYS`.
|
|
229
|
+
*
|
|
230
|
+
* Duplicated deliberately rather than imported: this gate must keep working if
|
|
231
|
+
* the dispatcher module moves, and the pair is pinned by a test that asserts
|
|
232
|
+
* the two lists agree. A silent divergence would make the gate scan for keys
|
|
233
|
+
* the dispatcher no longer stubs.
|
|
234
|
+
*/
|
|
235
|
+
const BODY_KEY_LITERALS: Readonly<Record<'input' | 'result', readonly string[]>> = {
|
|
236
|
+
input: ["tool_input", "toolInput"],
|
|
237
|
+
result: ["tool_response", "toolResponse", "tool_result", "toolUseResult"],
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
/** `// payload-bodies-waiver: <class> — <reason>` in a concern's own source. */
|
|
241
|
+
const WAIVER_RE = /payload-bodies-waiver:\s*(input|result)\s*[—:-]\s*(.+)/g;
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* A concern that READS a payload body must declare it
|
|
245
|
+
* (`road-to-per-turn-hook-economy` step 2.1).
|
|
246
|
+
*
|
|
247
|
+
* ## Why this is derived from source rather than from trust
|
|
248
|
+
*
|
|
249
|
+
* The first version of this check covered guards only, and the R2 review named
|
|
250
|
+
* the gap precisely: the failure mode the change itself introduces — a NEW
|
|
251
|
+
* advisory `post_tool_use` concern that reads `tool_response` and forgets the
|
|
252
|
+
* declaration — passed the lint, passed every test, and then read `undefined`
|
|
253
|
+
* in silence. Runtime cannot catch it either: the dispatcher's stub counter is
|
|
254
|
+
* a function of the declaration and the payload shape, so it says how often a
|
|
255
|
+
* concern ran without a body and never whether it wanted one.
|
|
256
|
+
*
|
|
257
|
+
* Authoring time is where the question is decidable, so the requirement is
|
|
258
|
+
* computed from the concern's own script: if it mentions a body key, it
|
|
259
|
+
* declares that class or says in the file why it does not.
|
|
260
|
+
*
|
|
261
|
+
* ## Over-detection is the safe direction, on purpose
|
|
262
|
+
*
|
|
263
|
+
* The scan is a literal match on the key name in quotes or after a dot, so a
|
|
264
|
+
* COMMENT mentioning `tool_response` also trips it. That is deliberate: a false
|
|
265
|
+
* positive costs one concern receiving a body it does not need, which is
|
|
266
|
+
* exactly the status quo before this phase; a false negative costs silence. The
|
|
267
|
+
* escape hatch is a `payload-bodies-waiver:` line naming the class and a
|
|
268
|
+
* reason, which keeps the claim next to the code that makes it instead of in a
|
|
269
|
+
* manifest a concern author may never open.
|
|
270
|
+
*
|
|
271
|
+
* ## The guard floor stays, and is stricter than the source scan
|
|
272
|
+
*
|
|
273
|
+
* `fail_closed` / `severity: blocking` on a tool slot requires `input`
|
|
274
|
+
* regardless of what the source scan finds, and a waiver cannot lift it: a
|
|
275
|
+
* guard served a stub has nothing to match and exits ALLOW. The dispatcher
|
|
276
|
+
* enforces the same floor independently (`_concern_body_classes` returns every
|
|
277
|
+
* class for a guard) and does not depend on this lint having run.
|
|
278
|
+
*
|
|
279
|
+
* Scoped to the two tool slots deliberately. `turn-end-gate` is blocking and
|
|
280
|
+
* binds `stop`, whose payload carries no tool body at all — requiring a
|
|
281
|
+
* declaration there would put a "give me the bodies" line on a concern that
|
|
282
|
+
* can never receive one, which documents nothing and misleads the next reader.
|
|
283
|
+
*/
|
|
284
|
+
function _check_guard_payload_bodies(
|
|
285
|
+
manifest: YamlObject,
|
|
286
|
+
errors: string[],
|
|
287
|
+
): void {
|
|
288
|
+
const concernsRaw = manifest["concerns"];
|
|
289
|
+
const platformsRaw = manifest["platforms"];
|
|
290
|
+
if (!isObject(concernsRaw) || !isObject(platformsRaw)) return;
|
|
291
|
+
const onToolSlot = new Set<string>();
|
|
292
|
+
for (const block of Object.values(platformsRaw)) {
|
|
293
|
+
if (!isObject(block)) continue;
|
|
294
|
+
for (const [event, names] of Object.entries(block)) {
|
|
295
|
+
if (!TOOL_BODY_EVENTS.has(event) || !Array.isArray(names)) continue;
|
|
296
|
+
for (const n of names) {
|
|
297
|
+
if (typeof n === "string") onToolSlot.add(n);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
for (const [name, spec] of Object.entries(concernsRaw)) {
|
|
302
|
+
if (!isObject(spec) || !onToolSlot.has(name)) continue;
|
|
303
|
+
const isGuard = spec["fail_closed"] === true || spec["severity"] === "blocking";
|
|
304
|
+
const declared = spec["needs_payload_bodies"];
|
|
305
|
+
const declaredSet = new Set(
|
|
306
|
+
Array.isArray(declared)
|
|
307
|
+
? declared.filter((c): c is string => typeof c === "string")
|
|
308
|
+
: [],
|
|
309
|
+
);
|
|
310
|
+
if (isGuard && !declaredSet.has("input")) {
|
|
311
|
+
errors.push(
|
|
312
|
+
`concerns.${name}: bound on a tool slot and fail_closed / ` +
|
|
313
|
+
`severity: blocking, so it must declare 'needs_payload_bodies' ` +
|
|
314
|
+
`including 'input' — a guard served a payload stub has nothing to ` +
|
|
315
|
+
`match and exits ALLOW`,
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
const script = spec["script"];
|
|
319
|
+
if (typeof script !== "string") continue;
|
|
320
|
+
let source = "";
|
|
321
|
+
try {
|
|
322
|
+
source = fs.readFileSync(path.join(REPO_ROOT, script), "utf8");
|
|
323
|
+
} catch {
|
|
324
|
+
continue; // missing script is already reported by _check_concerns
|
|
325
|
+
}
|
|
326
|
+
const waived = new Map<string, string>();
|
|
327
|
+
for (const m of source.matchAll(WAIVER_RE)) {
|
|
328
|
+
const cls = m[1];
|
|
329
|
+
const reason = (m[2] ?? "").trim();
|
|
330
|
+
if (cls !== undefined && reason) waived.set(cls, reason);
|
|
331
|
+
}
|
|
332
|
+
for (const [cls, keys] of Object.entries(BODY_KEY_LITERALS)) {
|
|
333
|
+
if (declaredSet.has(cls)) continue;
|
|
334
|
+
const hit = keys.find(
|
|
335
|
+
(k) =>
|
|
336
|
+
source.includes(`'${k}'`) ||
|
|
337
|
+
source.includes(`"${k}"`) ||
|
|
338
|
+
source.includes(`.${k}`),
|
|
339
|
+
);
|
|
340
|
+
if (hit === undefined) continue;
|
|
341
|
+
if (waived.has(cls)) continue;
|
|
342
|
+
errors.push(
|
|
343
|
+
`concerns.${name}: ${script} references '${hit}' but does not declare ` +
|
|
344
|
+
`'${cls}' in needs_payload_bodies — it would receive a stub and read ` +
|
|
345
|
+
`undefined in silence. Declare the class, or put ` +
|
|
346
|
+
`\`payload-bodies-waiver: ${cls} — <reason>\` in the script saying ` +
|
|
347
|
+
`why the body is not needed`,
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
187
353
|
function _check_platforms(
|
|
188
354
|
manifest: YamlObject,
|
|
189
355
|
concernNames: Set<string>,
|
|
@@ -425,6 +591,7 @@ export function lint(manifestPath: string, strict: boolean): number {
|
|
|
425
591
|
const warnings: string[] = [];
|
|
426
592
|
const concernNames = _check_concerns(manifest, errors);
|
|
427
593
|
const bound = _check_platforms(manifest, concernNames, errors, warnings);
|
|
594
|
+
_check_guard_payload_bodies(manifest, errors);
|
|
428
595
|
_check_roles(manifest, concernNames, errors);
|
|
429
596
|
_check_aliases(manifest, errors);
|
|
430
597
|
_check_orphan_trampolines(manifest, errors);
|