@wildorder/nightshift 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/dist/agent-runner.d.ts +56 -4
- package/dist/agent-runner.d.ts.map +1 -1
- package/dist/agent-runner.js +253 -28
- package/dist/agent-runner.js.map +1 -1
- package/dist/author.d.ts +8 -0
- package/dist/author.d.ts.map +1 -1
- package/dist/author.js +84 -57
- package/dist/author.js.map +1 -1
- package/dist/causal-analysis.d.ts +212 -0
- package/dist/causal-analysis.d.ts.map +1 -0
- package/dist/causal-analysis.js +733 -0
- package/dist/causal-analysis.js.map +1 -0
- package/dist/decider-review.d.ts +6 -1
- package/dist/decider-review.d.ts.map +1 -1
- package/dist/decider-review.js +19 -7
- package/dist/decider-review.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/preflight.d.ts +3 -0
- package/dist/preflight.d.ts.map +1 -1
- package/dist/preflight.js +73 -57
- package/dist/preflight.js.map +1 -1
- package/dist/prompt-telemetry.d.ts +64 -0
- package/dist/prompt-telemetry.d.ts.map +1 -0
- package/dist/prompt-telemetry.js +112 -0
- package/dist/prompt-telemetry.js.map +1 -0
- package/dist/provider-telemetry.d.ts +106 -0
- package/dist/provider-telemetry.d.ts.map +1 -0
- package/dist/provider-telemetry.js +423 -0
- package/dist/provider-telemetry.js.map +1 -0
- package/dist/run-analytics-report.d.ts +173 -0
- package/dist/run-analytics-report.d.ts.map +1 -0
- package/dist/run-analytics-report.js +650 -0
- package/dist/run-analytics-report.js.map +1 -0
- package/dist/run-analytics.d.ts +738 -0
- package/dist/run-analytics.d.ts.map +1 -0
- package/dist/run-analytics.js +545 -0
- package/dist/run-analytics.js.map +1 -0
- package/dist/run-program.d.ts +69 -1
- package/dist/run-program.d.ts.map +1 -1
- package/dist/run-program.js +1254 -714
- package/dist/run-program.js.map +1 -1
- package/dist/whole-program-review.d.ts +3 -0
- package/dist/whole-program-review.d.ts.map +1 -1
- package/dist/whole-program-review.js +8 -1
- package/dist/whole-program-review.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,650 @@
|
|
|
1
|
+
import { BUCKETS, KNOWN_OUTCOMES, } from "./run-analytics.js";
|
|
2
|
+
function compareIntervals(a, b) {
|
|
3
|
+
return a.start - b.start || a.end - b.end || a.span.id.localeCompare(b.span.id);
|
|
4
|
+
}
|
|
5
|
+
function attemptKeyOf(dims) {
|
|
6
|
+
if (dims.workstream === undefined || dims.attemptSeat === undefined || dims.attemptIndex === undefined) {
|
|
7
|
+
return undefined;
|
|
8
|
+
}
|
|
9
|
+
return `${dims.workstream} ${dims.attemptSeat} ${dims.attemptIndex}`;
|
|
10
|
+
}
|
|
11
|
+
function bump(map, key, durationMs) {
|
|
12
|
+
const cur = map.get(key) ?? { count: 0, durationMs: 0 };
|
|
13
|
+
cur.count += 1;
|
|
14
|
+
cur.durationMs += durationMs;
|
|
15
|
+
map.set(key, cur);
|
|
16
|
+
}
|
|
17
|
+
function rankedFromGroups(map) {
|
|
18
|
+
return [...map.entries()]
|
|
19
|
+
.map(([key, v]) => ({ key, durationMs: v.durationMs, count: v.count }))
|
|
20
|
+
.sort((a, b) => b.durationMs - a.durationMs || a.key.localeCompare(b.key));
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Walks a span's `parentId` chain until it reaches a top-level span id.
|
|
24
|
+
* Returns `undefined` for an orphan chain (a dangling `parentId` — already
|
|
25
|
+
* flagged as its own diagnostic — or a chain that never reaches a top-level
|
|
26
|
+
* ancestor), never throws, and is bounded against a cyclic `parentId` a
|
|
27
|
+
* malformed artifact might otherwise loop on forever.
|
|
28
|
+
*/
|
|
29
|
+
function findTopLevelAncestor(span, byId, topLevelIds) {
|
|
30
|
+
let parentId = span.parentId;
|
|
31
|
+
let hops = 0;
|
|
32
|
+
while (parentId !== undefined && hops < 10_000) {
|
|
33
|
+
if (topLevelIds.has(parentId))
|
|
34
|
+
return parentId;
|
|
35
|
+
const parent = byId.get(parentId);
|
|
36
|
+
if (parent === undefined)
|
|
37
|
+
return undefined;
|
|
38
|
+
parentId = parent.parentId;
|
|
39
|
+
hops += 1;
|
|
40
|
+
}
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Sweeps the sorted endpoints of every top-level interval and assigns each
|
|
45
|
+
* covered elementary sub-interval to exactly one owner — the covering span
|
|
46
|
+
* that sorts first by `(startOffsetMs, endOffsetMs, id)` — so
|
|
47
|
+
* `Σ(bucket durations)` equals the interval union exactly, even when
|
|
48
|
+
* top-level spans overlap (D2). Any sub-interval covered by more than one
|
|
49
|
+
* span is recorded as an `overlap` diagnostic.
|
|
50
|
+
*/
|
|
51
|
+
function sweepBucketOwnership(intervals, diagnostics) {
|
|
52
|
+
const totals = new Map(BUCKETS.map((bucket) => [bucket, 0]));
|
|
53
|
+
if (intervals.length === 0)
|
|
54
|
+
return totals;
|
|
55
|
+
const boundarySet = new Set();
|
|
56
|
+
for (const interval of intervals) {
|
|
57
|
+
boundarySet.add(interval.start);
|
|
58
|
+
boundarySet.add(interval.end);
|
|
59
|
+
}
|
|
60
|
+
const boundaries = [...boundarySet].sort((a, b) => a - b);
|
|
61
|
+
const seenOverlaps = new Set();
|
|
62
|
+
for (let i = 0; i < boundaries.length - 1; i++) {
|
|
63
|
+
const a = boundaries[i];
|
|
64
|
+
const b = boundaries[i + 1];
|
|
65
|
+
if (b <= a)
|
|
66
|
+
continue;
|
|
67
|
+
const covering = intervals.filter((interval) => interval.start <= a && interval.end >= b);
|
|
68
|
+
if (covering.length === 0)
|
|
69
|
+
continue;
|
|
70
|
+
covering.sort(compareIntervals);
|
|
71
|
+
const owner = covering[0];
|
|
72
|
+
totals.set(owner.span.bucket, (totals.get(owner.span.bucket) ?? 0) + (b - a));
|
|
73
|
+
if (covering.length > 1) {
|
|
74
|
+
const ids = covering.map((interval) => interval.span.id).sort();
|
|
75
|
+
const key = ids.join(",");
|
|
76
|
+
if (!seenOverlaps.has(key)) {
|
|
77
|
+
seenOverlaps.add(key);
|
|
78
|
+
diagnostics.push({
|
|
79
|
+
kind: "overlap",
|
|
80
|
+
message: `top-level spans overlap in [${a}, ${b}]ms: ${ids.join(", ")} — ` +
|
|
81
|
+
`attributed once, to ${owner.span.id} (${owner.span.bucket})`,
|
|
82
|
+
spanIds: ids,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return totals;
|
|
88
|
+
}
|
|
89
|
+
const AGENT_ATTEMPT_STAGES = new Set(["implementer", "recovery", "informed-retry", "fix-now", "test-critique-fix"]);
|
|
90
|
+
const AGENT_REPEAT_REASONS = new Set([
|
|
91
|
+
"verify-failure",
|
|
92
|
+
"informed-by-diagnosis",
|
|
93
|
+
"critique-finding",
|
|
94
|
+
"fix-now-finding",
|
|
95
|
+
]);
|
|
96
|
+
const ALWAYS_RERUN_VERIFY_PHASES = new Set(["post-recovery", "post-informed-retry", "test-critique-fix", "fix-now"]);
|
|
97
|
+
const VERIFY_COMMAND_STAGES = new Set(["verification-command", "baseline-verification"]);
|
|
98
|
+
const TELEMETRY_KINDS = new Set(["provider-telemetry", "prompt-component-size"]);
|
|
99
|
+
/**
|
|
100
|
+
* Deterministically reconciles a `RunAnalytics` artifact into conserved
|
|
101
|
+
* wall-clock and repetition analytics (SC-09). Pure: no clock, no I/O, no
|
|
102
|
+
* randomness. A shuffled `spans`/`points` array yields a byte-identical
|
|
103
|
+
* summary, because every grouping keys off span/point fields — never array
|
|
104
|
+
* position — and every list is explicitly sorted before it is returned.
|
|
105
|
+
*/
|
|
106
|
+
export function aggregateRunAnalytics(artifact) {
|
|
107
|
+
const diagnostics = [];
|
|
108
|
+
const totalElapsedMs = artifact.anchors.finalizedOffsetMs ?? 0;
|
|
109
|
+
const byId = new Map(artifact.spans.map((span) => [span.id, span]));
|
|
110
|
+
for (const span of artifact.spans) {
|
|
111
|
+
if (span.parentId !== undefined && !byId.has(span.parentId)) {
|
|
112
|
+
diagnostics.push({
|
|
113
|
+
kind: "unresolved-parent",
|
|
114
|
+
message: `span ${span.id} (${span.stage}) has parentId "${span.parentId}", which does not resolve`,
|
|
115
|
+
spanIds: [span.id],
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
if (span.endOffsetMs !== undefined && span.endOffsetMs < span.startOffsetMs) {
|
|
119
|
+
diagnostics.push({
|
|
120
|
+
kind: "invalid-interval",
|
|
121
|
+
message: `span ${span.id} (${span.stage}) has endOffsetMs < startOffsetMs`,
|
|
122
|
+
spanIds: [span.id],
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const runSpan = artifact.spans.find((span) => span.stage === "run");
|
|
127
|
+
const runId = runSpan?.id;
|
|
128
|
+
if (runSpan === undefined) {
|
|
129
|
+
diagnostics.push({
|
|
130
|
+
kind: "unresolved-parent",
|
|
131
|
+
message: "no run-stage span was found; treating root spans (no parentId) as top-level",
|
|
132
|
+
spanIds: [],
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
const topLevelSpans = artifact.spans.filter((span) => span.stage !== "run" && (runId !== undefined ? span.parentId === runId : span.parentId === undefined));
|
|
136
|
+
const topLevelIntervals = topLevelSpans.map((span) => {
|
|
137
|
+
if (span.endOffsetMs === undefined) {
|
|
138
|
+
diagnostics.push({
|
|
139
|
+
kind: "open-span",
|
|
140
|
+
// No raw span id in the human-facing message (the report never
|
|
141
|
+
// prints internal ids — `spanIds` below carries it for machine
|
|
142
|
+
// consumers/tests instead): this fires on effectively every real
|
|
143
|
+
// run, since `report-assembly` is deliberately still open when the
|
|
144
|
+
// snapshot is taken (§2.3/§3.2), not just on a genuine crash.
|
|
145
|
+
message: `the \`${span.stage}\` span has no endOffsetMs — open (still in progress, or interrupted) at ` +
|
|
146
|
+
`snapshot/finalize time, clamped to the total elapsed ${Math.round(totalElapsedMs)}ms`,
|
|
147
|
+
spanIds: [span.id],
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
const rawEnd = span.endOffsetMs ?? totalElapsedMs;
|
|
151
|
+
let end = rawEnd;
|
|
152
|
+
if (end > totalElapsedMs) {
|
|
153
|
+
diagnostics.push({
|
|
154
|
+
kind: "interval-exceeds-total",
|
|
155
|
+
message: `span ${span.id} (${span.stage}) ends at ${end}ms, past the total elapsed ${totalElapsedMs}ms`,
|
|
156
|
+
spanIds: [span.id],
|
|
157
|
+
});
|
|
158
|
+
end = totalElapsedMs;
|
|
159
|
+
}
|
|
160
|
+
const start = span.startOffsetMs;
|
|
161
|
+
return { span, start, end: Math.max(start, end) };
|
|
162
|
+
});
|
|
163
|
+
const bucketTotals = sweepBucketOwnership(topLevelIntervals, diagnostics);
|
|
164
|
+
let attributedMs = 0;
|
|
165
|
+
for (const bucket of BUCKETS) {
|
|
166
|
+
if (bucket !== "unattributed")
|
|
167
|
+
attributedMs += bucketTotals.get(bucket) ?? 0;
|
|
168
|
+
}
|
|
169
|
+
const unattributedMs = Math.max(0, totalElapsedMs - attributedMs);
|
|
170
|
+
bucketTotals.set("unattributed", unattributedMs);
|
|
171
|
+
const spanCountByBucket = new Map(BUCKETS.map((bucket) => [bucket, 0]));
|
|
172
|
+
for (const interval of topLevelIntervals) {
|
|
173
|
+
spanCountByBucket.set(interval.span.bucket, (spanCountByBucket.get(interval.span.bucket) ?? 0) + 1);
|
|
174
|
+
}
|
|
175
|
+
const buckets = BUCKETS.map((bucket) => ({
|
|
176
|
+
bucket,
|
|
177
|
+
durationMs: bucketTotals.get(bucket) ?? 0,
|
|
178
|
+
percent: totalElapsedMs > 0 ? ((bucketTotals.get(bucket) ?? 0) / totalElapsedMs) * 100 : 0,
|
|
179
|
+
spanCount: spanCountByBucket.get(bucket) ?? 0,
|
|
180
|
+
}));
|
|
181
|
+
// Drill-downs: every non-top-level, non-run span, grouped under its
|
|
182
|
+
// nearest top-level ancestor — detail, never summed into any bucket.
|
|
183
|
+
const topLevelIds = new Set(topLevelIntervals.map((interval) => interval.span.id));
|
|
184
|
+
const drillGroups = new Map();
|
|
185
|
+
for (const span of artifact.spans) {
|
|
186
|
+
if (span.id === runId || topLevelIds.has(span.id))
|
|
187
|
+
continue;
|
|
188
|
+
const owner = findTopLevelAncestor(span, byId, topLevelIds);
|
|
189
|
+
if (owner === undefined)
|
|
190
|
+
continue;
|
|
191
|
+
const durationMs = Math.max(0, (span.endOffsetMs ?? totalElapsedMs) - span.startOffsetMs);
|
|
192
|
+
const list = drillGroups.get(owner) ?? [];
|
|
193
|
+
list.push({
|
|
194
|
+
id: span.id,
|
|
195
|
+
stage: span.stage,
|
|
196
|
+
...(span.dimensions.workstream !== undefined ? { workstream: span.dimensions.workstream } : {}),
|
|
197
|
+
...(span.dimensions.role !== undefined ? { role: span.dimensions.role } : {}),
|
|
198
|
+
...(span.dimensions.outcome !== undefined ? { outcome: span.dimensions.outcome } : {}),
|
|
199
|
+
durationMs,
|
|
200
|
+
});
|
|
201
|
+
drillGroups.set(owner, list);
|
|
202
|
+
}
|
|
203
|
+
const drillDowns = [...drillGroups.entries()]
|
|
204
|
+
.map(([ownerSpanId, spans]) => ({
|
|
205
|
+
ownerSpanId,
|
|
206
|
+
ownerStage: byId.get(ownerSpanId)?.stage ?? "",
|
|
207
|
+
spans: [...spans].sort((a, b) => b.durationMs - a.durationMs || a.id.localeCompare(b.id)),
|
|
208
|
+
}))
|
|
209
|
+
.sort((a, b) => a.ownerSpanId.localeCompare(b.ownerSpanId));
|
|
210
|
+
// Dispositions: the five point-backed outcomes join `attempt-outcome`
|
|
211
|
+
// points to their spawn span on (workstream, attemptSeat, attemptIndex);
|
|
212
|
+
// `interrupted` is span-derived (D3) since WS-02 emits no point for it.
|
|
213
|
+
// Restricted to the actual attempt-spawn stages: a `verification-command`
|
|
214
|
+
// span carries the very same (workstream, attemptSeat, attemptIndex) triple
|
|
215
|
+
// as the attempt that provoked it (WS-02 §3.4/§3.6, so a rerun pass can be
|
|
216
|
+
// correlated back to its trigger) — without this filter it would collide
|
|
217
|
+
// with, and could nondeterministically shadow, the real spawn span.
|
|
218
|
+
const spawnByAttemptKey = new Map();
|
|
219
|
+
for (const interval of topLevelIntervals) {
|
|
220
|
+
if (!AGENT_ATTEMPT_STAGES.has(interval.span.stage))
|
|
221
|
+
continue;
|
|
222
|
+
const key = attemptKeyOf(interval.span.dimensions);
|
|
223
|
+
if (key === undefined)
|
|
224
|
+
continue;
|
|
225
|
+
const existing = spawnByAttemptKey.get(key);
|
|
226
|
+
if (existing === undefined || interval.span.id < existing.span.id)
|
|
227
|
+
spawnByAttemptKey.set(key, interval);
|
|
228
|
+
}
|
|
229
|
+
const dispositionTotals = new Map(KNOWN_OUTCOMES.map((outcome) => [outcome, { count: 0, durationMs: 0 }]));
|
|
230
|
+
const coveredAttemptKeys = new Set();
|
|
231
|
+
for (const point of artifact.points) {
|
|
232
|
+
if (point.kind !== "attempt-outcome")
|
|
233
|
+
continue;
|
|
234
|
+
const outcome = point.dimensions?.outcome ?? "unknown";
|
|
235
|
+
const key = attemptKeyOf(point.dimensions ?? {});
|
|
236
|
+
if (key !== undefined)
|
|
237
|
+
coveredAttemptKeys.add(key);
|
|
238
|
+
const matched = key === undefined ? undefined : spawnByAttemptKey.get(key);
|
|
239
|
+
const durationMs = matched === undefined ? 0 : matched.end - matched.start;
|
|
240
|
+
bump(dispositionTotals, outcome, durationMs);
|
|
241
|
+
}
|
|
242
|
+
let interruptedCount = 0;
|
|
243
|
+
let interruptedMs = 0;
|
|
244
|
+
for (const [key, interval] of spawnByAttemptKey) {
|
|
245
|
+
if (coveredAttemptKeys.has(key))
|
|
246
|
+
continue;
|
|
247
|
+
const isOpen = interval.span.endOffsetMs === undefined;
|
|
248
|
+
if (interval.span.dimensions.outcome === "interrupted" || isOpen) {
|
|
249
|
+
interruptedCount += 1;
|
|
250
|
+
interruptedMs += interval.end - interval.start;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
dispositionTotals.set("interrupted", { count: interruptedCount, durationMs: interruptedMs });
|
|
254
|
+
// `bump` above may have created outcome keys outside the known vocabulary
|
|
255
|
+
// (SC-04's open outcome vocabulary) — surfaced after the known six, sorted.
|
|
256
|
+
const extraOutcomes = [...dispositionTotals.keys()]
|
|
257
|
+
.filter((outcome) => !KNOWN_OUTCOMES.includes(outcome))
|
|
258
|
+
.sort();
|
|
259
|
+
const dispositions = [...KNOWN_OUTCOMES, ...extraOutcomes].map((outcome) => ({
|
|
260
|
+
outcome,
|
|
261
|
+
...(dispositionTotals.get(outcome) ?? { count: 0, durationMs: 0 }),
|
|
262
|
+
}));
|
|
263
|
+
// Rerun cost, grouped by trigger — never a single "waste" number (SC-06).
|
|
264
|
+
const agentRerunByReason = new Map();
|
|
265
|
+
let agentRerunTotalMs = 0;
|
|
266
|
+
for (const interval of topLevelIntervals) {
|
|
267
|
+
if (!AGENT_ATTEMPT_STAGES.has(interval.span.stage))
|
|
268
|
+
continue;
|
|
269
|
+
const reason = interval.span.dimensions.attemptReason;
|
|
270
|
+
if (reason === undefined || !AGENT_REPEAT_REASONS.has(reason))
|
|
271
|
+
continue;
|
|
272
|
+
const durationMs = interval.end - interval.start;
|
|
273
|
+
bump(agentRerunByReason, reason, durationMs);
|
|
274
|
+
agentRerunTotalMs += durationMs;
|
|
275
|
+
}
|
|
276
|
+
const verifyByWorkstreamPhase = new Map();
|
|
277
|
+
for (const interval of topLevelIntervals) {
|
|
278
|
+
if (interval.span.stage !== "verification-command")
|
|
279
|
+
continue;
|
|
280
|
+
const workstream = interval.span.dimensions.workstream ?? "";
|
|
281
|
+
const phase = interval.span.dimensions.attemptReason ?? "";
|
|
282
|
+
const key = `${workstream} ${phase}`;
|
|
283
|
+
const list = verifyByWorkstreamPhase.get(key) ?? [];
|
|
284
|
+
list.push(interval);
|
|
285
|
+
verifyByWorkstreamPhase.set(key, list);
|
|
286
|
+
}
|
|
287
|
+
const verifyRerunByPhase = new Map();
|
|
288
|
+
const verifyRerunByCommand = new Map();
|
|
289
|
+
let verifyRerunTotalMs = 0;
|
|
290
|
+
for (const [key, group] of verifyByWorkstreamPhase) {
|
|
291
|
+
const phase = key.split(" ")[1] ?? "";
|
|
292
|
+
const sorted = [...group].sort(compareIntervals);
|
|
293
|
+
sorted.forEach((interval, index) => {
|
|
294
|
+
// `post-implementer` verifies the initial attempt — only a rerun if it
|
|
295
|
+
// somehow fires again for the same workstream; every other phase this
|
|
296
|
+
// runner emits (post-recovery, post-informed-retry, test-critique-fix,
|
|
297
|
+
// fix-now) exists only because a prior attempt failed, so every
|
|
298
|
+
// occurrence counts (WS-02's phase vocabulary, §3.4).
|
|
299
|
+
const isRerun = phase === "post-implementer" ? index > 0 : ALWAYS_RERUN_VERIFY_PHASES.has(phase);
|
|
300
|
+
if (!isRerun)
|
|
301
|
+
return;
|
|
302
|
+
const durationMs = interval.end - interval.start;
|
|
303
|
+
verifyRerunTotalMs += durationMs;
|
|
304
|
+
bump(verifyRerunByPhase, phase, durationMs);
|
|
305
|
+
bump(verifyRerunByCommand, interval.span.dimensions.verifyCommand ?? "(unknown)", durationMs);
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
// Rankings — largest measured contributors, never graded (§3.5). Only
|
|
309
|
+
// top-level spans participate, so parent/child durations are never summed.
|
|
310
|
+
const stageGroups = new Map();
|
|
311
|
+
const workstreamGroups = new Map();
|
|
312
|
+
const roleGroups = new Map();
|
|
313
|
+
const verifyCommandGroups = new Map();
|
|
314
|
+
let largestAgentAttempt;
|
|
315
|
+
for (const interval of topLevelIntervals) {
|
|
316
|
+
const durationMs = interval.end - interval.start;
|
|
317
|
+
bump(stageGroups, interval.span.stage, durationMs);
|
|
318
|
+
bump(workstreamGroups, interval.span.dimensions.workstream ?? "run-level", durationMs);
|
|
319
|
+
if (interval.span.dimensions.role !== undefined)
|
|
320
|
+
bump(roleGroups, interval.span.dimensions.role, durationMs);
|
|
321
|
+
if (VERIFY_COMMAND_STAGES.has(interval.span.stage) && interval.span.dimensions.verifyCommand !== undefined) {
|
|
322
|
+
const command = interval.span.dimensions.verifyCommand;
|
|
323
|
+
const cur = verifyCommandGroups.get(command) ?? { count: 0, durationMs: 0, largestMs: 0 };
|
|
324
|
+
cur.count += 1;
|
|
325
|
+
cur.durationMs += durationMs;
|
|
326
|
+
cur.largestMs = Math.max(cur.largestMs, durationMs);
|
|
327
|
+
verifyCommandGroups.set(command, cur);
|
|
328
|
+
}
|
|
329
|
+
if (AGENT_ATTEMPT_STAGES.has(interval.span.stage) && interval.span.dimensions.attemptSeat !== undefined) {
|
|
330
|
+
if (largestAgentAttempt === undefined ||
|
|
331
|
+
durationMs > largestAgentAttempt.durationMs ||
|
|
332
|
+
(durationMs === largestAgentAttempt.durationMs && interval.span.id < largestAgentAttempt.spanId)) {
|
|
333
|
+
largestAgentAttempt = {
|
|
334
|
+
spanId: interval.span.id,
|
|
335
|
+
stage: interval.span.stage,
|
|
336
|
+
...(interval.span.dimensions.workstream !== undefined ? { workstream: interval.span.dimensions.workstream } : {}),
|
|
337
|
+
...(interval.span.dimensions.attemptSeat !== undefined ? { attemptSeat: interval.span.dimensions.attemptSeat } : {}),
|
|
338
|
+
...(interval.span.dimensions.attemptIndex !== undefined ? { attemptIndex: interval.span.dimensions.attemptIndex } : {}),
|
|
339
|
+
durationMs,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
const byVerifyCommand = [...verifyCommandGroups.entries()]
|
|
345
|
+
.map(([command, v]) => ({ command, invocations: v.count, totalMs: v.durationMs, largestMs: v.largestMs }))
|
|
346
|
+
.sort((a, b) => b.totalMs - a.totalMs || a.command.localeCompare(b.command));
|
|
347
|
+
// Observability coverage (§3.6) — over WS-03's own points only.
|
|
348
|
+
const coverageCounts = { observed: 0, estimated: 0, unavailable: 0, incomplete: 0 };
|
|
349
|
+
const unavailableRoles = new Set();
|
|
350
|
+
const providerPoints = [...artifact.points]
|
|
351
|
+
.filter((point) => point.kind === "provider-telemetry")
|
|
352
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
353
|
+
const roleInfo = new Map();
|
|
354
|
+
for (const point of artifact.points) {
|
|
355
|
+
if (!TELEMETRY_KINDS.has(point.kind))
|
|
356
|
+
continue;
|
|
357
|
+
coverageCounts[point.coverage] += 1;
|
|
358
|
+
if (point.kind === "provider-telemetry" && point.coverage === "unavailable" && point.dimensions?.role !== undefined) {
|
|
359
|
+
unavailableRoles.add(point.dimensions.role);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
for (const point of providerPoints) {
|
|
363
|
+
const role = point.dimensions?.role;
|
|
364
|
+
if (role === undefined || (point.label !== "provider" && point.label !== "model"))
|
|
365
|
+
continue;
|
|
366
|
+
const info = roleInfo.get(role) ?? { role };
|
|
367
|
+
const field = point.label === "provider" ? "provider" : "model";
|
|
368
|
+
if (info[field] === undefined) {
|
|
369
|
+
info[field] = { coverage: point.coverage, ...(point.detail !== undefined ? { value: point.detail } : {}) };
|
|
370
|
+
}
|
|
371
|
+
roleInfo.set(role, info);
|
|
372
|
+
}
|
|
373
|
+
const byRoleCoverage = [...roleInfo.values()].sort((a, b) => a.role.localeCompare(b.role));
|
|
374
|
+
// Provider/model duration totals (§3.5): each role's already-computed
|
|
375
|
+
// duration total (`roleGroups`) is attributed to the provider/model
|
|
376
|
+
// identity WS-03 observed for that role — labelled `"unavailable"`, never
|
|
377
|
+
// dropped or blanked, when a role's provider telemetry is unavailable
|
|
378
|
+
// (e.g. a non-Claude role), so an entire role's measured time is never
|
|
379
|
+
// silently missing from these totals.
|
|
380
|
+
const providerGroups = new Map();
|
|
381
|
+
const modelGroups = new Map();
|
|
382
|
+
function labelFor(field) {
|
|
383
|
+
return field === undefined || field.coverage === "unavailable" ? "unavailable" : (field.value ?? "unavailable");
|
|
384
|
+
}
|
|
385
|
+
for (const [role, group] of roleGroups) {
|
|
386
|
+
const info = roleInfo.get(role);
|
|
387
|
+
const providerKey = labelFor(info?.provider);
|
|
388
|
+
const modelKey = labelFor(info?.model);
|
|
389
|
+
for (const [target, key] of [
|
|
390
|
+
[providerGroups, providerKey],
|
|
391
|
+
[modelGroups, modelKey],
|
|
392
|
+
]) {
|
|
393
|
+
const cur = target.get(key) ?? { count: 0, durationMs: 0 };
|
|
394
|
+
cur.count += group.count;
|
|
395
|
+
cur.durationMs += group.durationMs;
|
|
396
|
+
target.set(key, cur);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
const bytesByLabel = new Map();
|
|
400
|
+
for (const point of artifact.points) {
|
|
401
|
+
if (point.kind !== "prompt-component-size" || point.unit !== "bytes")
|
|
402
|
+
continue;
|
|
403
|
+
bytesByLabel.set(point.label ?? "(unlabeled)", (bytesByLabel.get(point.label ?? "(unlabeled)") ?? 0) + (point.value ?? 0));
|
|
404
|
+
}
|
|
405
|
+
const promptComponents = [...bytesByLabel.entries()]
|
|
406
|
+
.map(([label, bytes]) => ({ label, bytes }))
|
|
407
|
+
.sort((a, b) => b.bytes - a.bytes || a.label.localeCompare(b.label))
|
|
408
|
+
.slice(0, 5);
|
|
409
|
+
const overheadPoints = artifact.points.filter((point) => point.kind === "analytics-overhead");
|
|
410
|
+
const analyticsOverheadMs = overheadPoints.length > 0 ? overheadPoints.reduce((sum, point) => sum + (point.value ?? 0), 0) : undefined;
|
|
411
|
+
const publishPoints = artifact.points.filter((point) => point.kind === "publish-duration");
|
|
412
|
+
const publish = publishPoints.length > 0
|
|
413
|
+
? { durationMs: publishPoints.reduce((sum, point) => sum + (point.value ?? 0), 0), invocations: publishPoints.length }
|
|
414
|
+
: undefined;
|
|
415
|
+
const evidenceMap = new Map();
|
|
416
|
+
for (const span of artifact.spans) {
|
|
417
|
+
for (const ref of span.dimensions.evidence ?? [])
|
|
418
|
+
evidenceMap.set(`${ref.kind}:${ref.ref}`, ref);
|
|
419
|
+
}
|
|
420
|
+
for (const point of artifact.points) {
|
|
421
|
+
for (const ref of point.evidence ?? [])
|
|
422
|
+
evidenceMap.set(`${ref.kind}:${ref.ref}`, ref);
|
|
423
|
+
}
|
|
424
|
+
const sortedEvidence = [...evidenceMap.values()].sort((a, b) => a.ref.localeCompare(b.ref));
|
|
425
|
+
const evidence = sortedEvidence.slice(0, 8).map((ref) => ({
|
|
426
|
+
kind: ref.kind,
|
|
427
|
+
locality: ref.locality,
|
|
428
|
+
ref: ref.ref,
|
|
429
|
+
...(ref.note !== undefined ? { note: ref.note } : {}),
|
|
430
|
+
}));
|
|
431
|
+
// Diagnostics are pushed as each condition is discovered while walking
|
|
432
|
+
// `artifact.spans` in array order (unresolved-parent, invalid-interval,
|
|
433
|
+
// interval-exceeds-total); `sweepBucketOwnership`'s overlap diagnostics are
|
|
434
|
+
// already order-independent (found via sorted interval boundaries), but the
|
|
435
|
+
// other three are not — sorted here so the full summary stays
|
|
436
|
+
// order-independent (SC-09): a shuffled `spans` array must never change
|
|
437
|
+
// which diagnostics were found or their relative order, only the pass that
|
|
438
|
+
// happened to find them first.
|
|
439
|
+
diagnostics.sort((a, b) => a.kind.localeCompare(b.kind) ||
|
|
440
|
+
a.spanIds.join(",").localeCompare(b.spanIds.join(",")) ||
|
|
441
|
+
a.message.localeCompare(b.message));
|
|
442
|
+
return {
|
|
443
|
+
totalElapsedMs,
|
|
444
|
+
attributedMs,
|
|
445
|
+
unattributedMs,
|
|
446
|
+
buckets,
|
|
447
|
+
diagnostics,
|
|
448
|
+
drillDowns,
|
|
449
|
+
dispositions,
|
|
450
|
+
rerun: {
|
|
451
|
+
agent: { totalMs: agentRerunTotalMs, byReason: rankedFromGroups(agentRerunByReason).map((r) => ({ key: r.key, count: r.count, durationMs: r.durationMs })) },
|
|
452
|
+
verification: {
|
|
453
|
+
totalMs: verifyRerunTotalMs,
|
|
454
|
+
byPhase: rankedFromGroups(verifyRerunByPhase).map((r) => ({ key: r.key, count: r.count, durationMs: r.durationMs })),
|
|
455
|
+
byCommand: rankedFromGroups(verifyRerunByCommand).map((r) => ({ key: r.key, count: r.count, durationMs: r.durationMs })),
|
|
456
|
+
},
|
|
457
|
+
},
|
|
458
|
+
rankings: {
|
|
459
|
+
byStage: rankedFromGroups(stageGroups),
|
|
460
|
+
byWorkstream: rankedFromGroups(workstreamGroups),
|
|
461
|
+
byRole: rankedFromGroups(roleGroups),
|
|
462
|
+
byProvider: rankedFromGroups(providerGroups),
|
|
463
|
+
byModel: rankedFromGroups(modelGroups),
|
|
464
|
+
byVerifyCommand,
|
|
465
|
+
...(largestAgentAttempt !== undefined ? { largestAgentAttempt } : {}),
|
|
466
|
+
},
|
|
467
|
+
coverage: { counts: coverageCounts, unavailableRoles: [...unavailableRoles].sort(), byRole: byRoleCoverage },
|
|
468
|
+
promptComponents,
|
|
469
|
+
...(analyticsOverheadMs !== undefined ? { analyticsOverheadMs } : {}),
|
|
470
|
+
...(publish !== undefined ? { publish } : {}),
|
|
471
|
+
evidence,
|
|
472
|
+
evidenceTotalCount: evidenceMap.size,
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
// ---------------------------------------------------------------------------
|
|
476
|
+
// Rendering
|
|
477
|
+
// ---------------------------------------------------------------------------
|
|
478
|
+
function formatDuration(ms) {
|
|
479
|
+
if (!Number.isFinite(ms) || ms < 0)
|
|
480
|
+
ms = 0;
|
|
481
|
+
if (ms < 1000)
|
|
482
|
+
return `${Math.round(ms)}ms`;
|
|
483
|
+
const totalSeconds = ms / 1000;
|
|
484
|
+
if (totalSeconds < 60)
|
|
485
|
+
return `${totalSeconds.toFixed(1)}s`;
|
|
486
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
487
|
+
const seconds = Math.round(totalSeconds - minutes * 60);
|
|
488
|
+
return `${minutes}m ${String(seconds).padStart(2, "0")}s`;
|
|
489
|
+
}
|
|
490
|
+
function plural(count, noun) {
|
|
491
|
+
return `${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* Renders the aggregated summary into the "Where the time went" section
|
|
495
|
+
* (SC-10, SC-11) — human-first, concise, and never a score or a grade.
|
|
496
|
+
*/
|
|
497
|
+
export function renderWhereTheTimeWent(summary) {
|
|
498
|
+
const lines = ["## Where the time went", ""];
|
|
499
|
+
lines.push(`Total elapsed: **${formatDuration(summary.totalElapsedMs)}** — run start to the analytics snapshot taken ` +
|
|
500
|
+
"just before finalize and the report commit. The report write, the report commit, and publish are " +
|
|
501
|
+
"excluded from this total and, when observed, recorded separately (see Publish below).", "");
|
|
502
|
+
lines.push("### Time by bucket", "");
|
|
503
|
+
for (const bucket of summary.buckets) {
|
|
504
|
+
lines.push(`- **${bucket.bucket}**: ${formatDuration(bucket.durationMs)} (${bucket.percent.toFixed(1)}%, ${plural(bucket.spanCount, "span")})`);
|
|
505
|
+
}
|
|
506
|
+
lines.push("");
|
|
507
|
+
lines.push("### Largest contributors", "");
|
|
508
|
+
const topStage = summary.rankings.byStage[0];
|
|
509
|
+
lines.push(topStage
|
|
510
|
+
? `- Stage: \`${topStage.key}\` — ${formatDuration(topStage.durationMs)} across ${plural(topStage.count, "span")}`
|
|
511
|
+
: "- Stage: none observed.");
|
|
512
|
+
const topWorkstream = summary.rankings.byWorkstream[0];
|
|
513
|
+
lines.push(topWorkstream
|
|
514
|
+
? `- Workstream: ${topWorkstream.key} — ${formatDuration(topWorkstream.durationMs)}`
|
|
515
|
+
: "- Workstream: none observed.");
|
|
516
|
+
const attempt = summary.rankings.largestAgentAttempt;
|
|
517
|
+
lines.push(attempt
|
|
518
|
+
? `- Largest single agent attempt: \`${attempt.stage}\`` +
|
|
519
|
+
(attempt.workstream ? ` (${attempt.workstream}` : " (") +
|
|
520
|
+
`${attempt.attemptSeat ? `, ${attempt.attemptSeat}` : ""}` +
|
|
521
|
+
`${attempt.attemptIndex !== undefined ? ` #${attempt.attemptIndex}` : ""})` +
|
|
522
|
+
` — ${formatDuration(attempt.durationMs)}`
|
|
523
|
+
: "- Largest single agent attempt: none observed.");
|
|
524
|
+
const topCommand = summary.rankings.byVerifyCommand[0];
|
|
525
|
+
lines.push(topCommand
|
|
526
|
+
? `- Largest verification command: \`${topCommand.command}\` — ${formatDuration(topCommand.totalMs)} across ` +
|
|
527
|
+
`${plural(topCommand.invocations, "invocation")} (largest single run ${formatDuration(topCommand.largestMs)})`
|
|
528
|
+
: "- Largest verification command: none observed.");
|
|
529
|
+
lines.push("");
|
|
530
|
+
lines.push("### Attempts", "");
|
|
531
|
+
for (const disposition of summary.dispositions) {
|
|
532
|
+
lines.push(`- ${disposition.outcome}: ${plural(disposition.count, "attempt")} (${formatDuration(disposition.durationMs)})`);
|
|
533
|
+
}
|
|
534
|
+
lines.push("", `Agent time spent re-running after a prior attempt: **${formatDuration(summary.rerun.agent.totalMs)}**, by trigger:`);
|
|
535
|
+
if (summary.rerun.agent.byReason.length === 0) {
|
|
536
|
+
lines.push("- none observed");
|
|
537
|
+
}
|
|
538
|
+
else {
|
|
539
|
+
for (const group of summary.rerun.agent.byReason) {
|
|
540
|
+
lines.push(`- ${group.key}: ${plural(group.count, "attempt")} (${formatDuration(group.durationMs)})`);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
lines.push("");
|
|
544
|
+
lines.push("### Verification", "");
|
|
545
|
+
lines.push(`Verification rerun time (passes caused by a retry, recovery, or fix — never the first pass over fresh ` +
|
|
546
|
+
`work): **${formatDuration(summary.rerun.verification.totalMs)}**`);
|
|
547
|
+
if (summary.rerun.verification.byPhase.length > 0) {
|
|
548
|
+
lines.push("", "By phase:");
|
|
549
|
+
for (const group of summary.rerun.verification.byPhase) {
|
|
550
|
+
lines.push(`- ${group.key}: ${plural(group.count, "invocation")} (${formatDuration(group.durationMs)})`);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
if (summary.rankings.byVerifyCommand.length > 0) {
|
|
554
|
+
lines.push("", "Every configured command, all invocations:");
|
|
555
|
+
for (const command of summary.rankings.byVerifyCommand) {
|
|
556
|
+
lines.push(`- \`${command.command}\`: ${plural(command.invocations, "invocation")}, ${formatDuration(command.totalMs)} total`);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
lines.push("");
|
|
560
|
+
lines.push("### Observability gaps", "");
|
|
561
|
+
lines.push(`Provider/prompt telemetry: ${summary.coverage.counts.observed} observed, ${summary.coverage.counts.estimated} ` +
|
|
562
|
+
`estimated, ${summary.coverage.counts.unavailable} unavailable, ${summary.coverage.counts.incomplete} incomplete.`);
|
|
563
|
+
if (summary.coverage.unavailableRoles.length > 0) {
|
|
564
|
+
lines.push(`Roles reporting unavailable provider telemetry (expected coverage for a non-Claude provider, never a ` +
|
|
565
|
+
`missing-time signal — their elapsed spans are still fully attributed above): ` +
|
|
566
|
+
`${summary.coverage.unavailableRoles.join(", ")}.`);
|
|
567
|
+
}
|
|
568
|
+
if (summary.rankings.byProvider.length > 0) {
|
|
569
|
+
lines.push("", "Time by provider (`unavailable` when a role's provider telemetry is unavailable, never a dropped total):");
|
|
570
|
+
for (const p of summary.rankings.byProvider) {
|
|
571
|
+
lines.push(`- ${p.key}: ${formatDuration(p.durationMs)} across ${plural(p.count, "span")}`);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
if (summary.rankings.byModel.length > 0) {
|
|
575
|
+
lines.push("", "Time by model:");
|
|
576
|
+
for (const m of summary.rankings.byModel) {
|
|
577
|
+
lines.push(`- ${m.key}: ${formatDuration(m.durationMs)} across ${plural(m.count, "span")}`);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (summary.promptComponents.length > 0) {
|
|
581
|
+
lines.push("", "Largest initial-context components: " +
|
|
582
|
+
summary.promptComponents.map((c) => `${c.label} (${c.bytes.toLocaleString("en-US")} bytes)`).join(", ") +
|
|
583
|
+
".");
|
|
584
|
+
}
|
|
585
|
+
if (summary.diagnostics.length > 0) {
|
|
586
|
+
lines.push("", "Interval-consistency notes:");
|
|
587
|
+
for (const diagnostic of summary.diagnostics)
|
|
588
|
+
lines.push(`- ${diagnostic.message}`);
|
|
589
|
+
}
|
|
590
|
+
if (summary.analyticsOverheadMs !== undefined) {
|
|
591
|
+
lines.push("", `Analytics-recorder overhead (a diagnostic footnote, not part of the total above): ${formatDuration(summary.analyticsOverheadMs)}.`);
|
|
592
|
+
}
|
|
593
|
+
lines.push("");
|
|
594
|
+
lines.push("### Evidence", "");
|
|
595
|
+
if (summary.evidence.length === 0) {
|
|
596
|
+
lines.push("No canonical evidence references were recorded for this run.");
|
|
597
|
+
}
|
|
598
|
+
else {
|
|
599
|
+
for (const ref of summary.evidence) {
|
|
600
|
+
lines.push(`- local ${ref.kind}: \`${ref.ref}\`${ref.note ? ` — ${ref.note}` : ""}`);
|
|
601
|
+
}
|
|
602
|
+
if (summary.evidenceTotalCount > summary.evidence.length) {
|
|
603
|
+
lines.push(`(showing ${summary.evidence.length} of ${summary.evidenceTotalCount} recorded references)`);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
lines.push("", "Every reference above resolves only on the machine that ran this build — it is local evidence, not a " +
|
|
607
|
+
"durable CI artifact.", "");
|
|
608
|
+
if (summary.publish !== undefined) {
|
|
609
|
+
lines.push("### Publish", "", `Publish ran after this report was already committed, so it is never reconciled into the total above: ` +
|
|
610
|
+
`${formatDuration(summary.publish.durationMs)} across ${plural(summary.publish.invocations, "invocation")}.`, "");
|
|
611
|
+
}
|
|
612
|
+
return lines;
|
|
613
|
+
}
|
|
614
|
+
function unavailableSection(reason) {
|
|
615
|
+
return ["## Where the time went", "", `Analytics unavailable — ${reason}.`, ""];
|
|
616
|
+
}
|
|
617
|
+
/**
|
|
618
|
+
* The run's own fail-open isolation boundary for analytics (D4, SC-13):
|
|
619
|
+
* always returns a `## Where the time went` section, never throws. A throw
|
|
620
|
+
* from `aggregate` (real or injected via `deps`, for tests), an `undefined`
|
|
621
|
+
* snapshot, or a throw while *obtaining* the snapshot all degrade to an
|
|
622
|
+
* explicit "analytics unavailable" placeholder — never suppress the rest of
|
|
623
|
+
* the report, which is built independently of this function's output.
|
|
624
|
+
*
|
|
625
|
+
* `artifact` accepts either a plain value (the common case, and every
|
|
626
|
+
* existing pure test) or a thunk. The thunk form matters at the real call
|
|
627
|
+
* site: `recorder.snapshot()` is documented as non-throwing, but evaluating
|
|
628
|
+
* it as a plain argument — `renderAnalyticsSection(recorder.snapshot())` —
|
|
629
|
+
* would run it *before* this function's own try/catch is reached, so any
|
|
630
|
+
* violation of that contract (a bug, or a test double standing in for the
|
|
631
|
+
* recorder) would still escape this boundary and, through the caller's own
|
|
632
|
+
* try/catch, suppress the whole report. Wrapping it in a thunk —
|
|
633
|
+
* `renderAnalyticsSection(() => recorder.snapshot())` — keeps that call
|
|
634
|
+
* inside the try below, so this function's "never throws" is true no matter
|
|
635
|
+
* what actually produced the artifact.
|
|
636
|
+
*/
|
|
637
|
+
export function renderAnalyticsSection(artifact, deps = {}) {
|
|
638
|
+
const aggregate = deps.aggregate ?? aggregateRunAnalytics;
|
|
639
|
+
try {
|
|
640
|
+
const resolved = typeof artifact === "function" ? artifact() : artifact;
|
|
641
|
+
if (resolved === undefined) {
|
|
642
|
+
return unavailableSection("no analytics snapshot was available when the report was rendered");
|
|
643
|
+
}
|
|
644
|
+
return renderWhereTheTimeWent(aggregate(resolved));
|
|
645
|
+
}
|
|
646
|
+
catch (error) {
|
|
647
|
+
return unavailableSection(error.message);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
//# sourceMappingURL=run-analytics-report.js.map
|