forgedock 1.32.1 → 1.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +7 -3
- package/bin/engine/phases.mjs +3 -3
- package/bin/engine-cli.mjs +56 -11
- package/bin/forgedock.mjs +21 -1
- package/bin/lean/basesync.mjs +98 -0
- package/bin/lean/basesync.test.mjs +151 -0
- package/bin/lean/batch.test.mjs +103 -2
- package/bin/lean/budget.mjs +7 -1
- package/bin/lean/budget.test.mjs +9 -1
- package/bin/lean/conformance.mjs +46 -10
- package/bin/lean/conformance.test.mjs +50 -2
- package/bin/lean/depcache.mjs +23 -3
- package/bin/lean/depcache.test.mjs +42 -1
- package/bin/lean/effects.mjs +1 -1
- package/bin/lean/execution.mjs +102 -12
- package/bin/lean/execution.test.mjs +101 -1
- package/bin/lean/hostkill.mjs +8 -0
- package/bin/lean/lean.mjs +49 -21
- package/bin/lean/lean.test.mjs +133 -7
- package/bin/lean/lightlane.mjs +38 -13
- package/bin/lean/lightlane.test.mjs +35 -1
- package/bin/lean/orchestrate.mjs +63 -17
- package/bin/lean/prompts.mjs +19 -9
- package/bin/lean/recovery.test.mjs +26 -0
- package/bin/lean/sandbox-provider.contract.test.mjs +74 -1
- package/bin/lean/sandbox-remote.mjs +40 -7
- package/bin/lean/seqlease.mjs +49 -2
- package/bin/lean/seqlease.test.mjs +14 -0
- package/bin/lean/sequence.mjs +54 -6
- package/bin/lean/sequence.test.mjs +83 -1
- package/bin/pool/agent.mjs +435 -0
- package/bin/pool/cli.mjs +110 -0
- package/bin/pool/docker.mjs +56 -0
- package/bin/pool/http.mjs +55 -0
- package/bin/pool/router.mjs +299 -0
- package/bin/pool/store.mjs +25 -0
- package/bin/pool/version.mjs +44 -0
- package/bin/review-staging.mjs +14 -2
- package/bin/runner.mjs +1 -0
- package/bin/stats/aggregate.mjs +6 -0
- package/bin/stats/efficiency.mjs +103 -17
- package/bin/stats/efficiency.test.mjs +103 -4
- package/bin/stats/index.mjs +40 -6
- package/bin/stats/local-source.mjs +1 -1
- package/bin/stats/pool.mjs +51 -0
- package/bin/stats/pool.test.mjs +39 -0
- package/bin/stats/query.mjs +2 -2
- package/bin/stats/stats.test.mjs +15 -1
- package/commands/orchestrate.md +3 -3
- package/commands/review-pr-agents/protocols.md +1 -1
- package/commands/review-pr-staging.md +1 -1
- package/docs/CONFIG.md +23 -9
- package/docs/OUTCOME-CODES.md +5 -3
- package/docs/STATS.md +36 -1
- package/package.json +2 -1
- package/pool/README.md +13 -0
- package/pool/image/Dockerfile +46 -0
- package/pool/image/claude-shim +8 -0
- package/pool/image/cs-cred +12 -0
- package/pool/image/entrypoint.sh +61 -0
- package/pool/image/gh-shim +6 -0
- package/pool/install/install-agent-linux.sh +53 -0
- package/pool/install/install-agent.ps1 +225 -0
- package/pool/install/install-agent.sh +101 -0
- package/pool/install/install-router.sh +67 -0
- package/pool/install/keepalive.ps1 +36 -0
- package/pool/install/uninstall-agent.ps1 +20 -0
- package/scripts/hotspots.sh +33 -21
- package/scripts/hotspots.test.sh +22 -3
- package/scripts/review-finalize.sh +17 -1
- package/scripts/review-finalize.test.sh +41 -0
- package/scripts/review-pr-domain-map.test.sh +2 -2
- package/scripts/review-pr-staging-scope.test.sh +4 -2
- package/scripts/review-prepare.sh +62 -27
- package/scripts/review-prepare.test.sh +53 -0
- package/scripts/review-staging-pre.sh +29 -9
- package/scripts/review-staging-pre.test.sh +18 -0
package/bin/review-staging.mjs
CHANGED
|
@@ -85,6 +85,9 @@ export function parseArgs(argv) {
|
|
|
85
85
|
return o;
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
/** Largest diff slice substituted into a brief; a larger one is referenced by its file (#4573). */
|
|
89
|
+
const INLINE_SLICE_MAX = 16000;
|
|
90
|
+
|
|
88
91
|
/** Replace every `[KEY]` in one pass: a substituted value is never rescanned, so diff text cannot inject placeholders. */
|
|
89
92
|
export function renderBrief(template, values) {
|
|
90
93
|
return template.replace(/\[([A-Z][A-Z0-9_]*)\]/g, (m, k) => (Object.hasOwn(values, k) && values[k] != null ? String(values[k]) : m));
|
|
@@ -273,6 +276,8 @@ async function main(argv, io) {
|
|
|
273
276
|
const agents = ctx.roster?.agents?.length ? ctx.roster.agents : ["security"];
|
|
274
277
|
const launchedFile = join(sd, `${pr}_launched_review_agents`);
|
|
275
278
|
io.writeFile(launchedFile, "");
|
|
279
|
+
// The head this list describes: review-prepare trusts the list as --delta proof only for this head.
|
|
280
|
+
io.writeFile(`${launchedFile}.head`, `${head}\n`);
|
|
276
281
|
const briefDir = join(sd, `review-staging-${pr}`);
|
|
277
282
|
io.mkdir(briefDir);
|
|
278
283
|
const briefs = [];
|
|
@@ -282,6 +287,8 @@ async function main(argv, io) {
|
|
|
282
287
|
const slice = ctx.slices?.[domain];
|
|
283
288
|
let diff = "";
|
|
284
289
|
try { diff = slice?.diff_file ? io.readFile(slice.diff_file) : ""; } catch { diff = ""; }
|
|
290
|
+
// #4573: the persona templates repeat [DOMAIN_DIFF_SLICE] many times, so only a small slice is inlined; a large one is a pointer to the on-disk slice file.
|
|
291
|
+
if (diff.length > INLINE_SLICE_MAX) diff = `(diff slice too large to inline: ${diff.length} chars. Read the file ${slice.diff_file} instead, e.g. grep -n or sed -n 'X,Yp' on it, never cat it whole. Files listed as OMITTED at its top are read from the head checkout.)`;
|
|
285
292
|
const values = {
|
|
286
293
|
PR_NUMBER: pr, REVIEW_SHA: head, REVIEW_SHA_SHORT: sha7, AGENT_DOMAIN: domain, GH_REPO: c.ghRepo,
|
|
287
294
|
TITLE: ctx.pr?.title ?? "", FILE_LIST: (slice?.paths ?? ctx.full_files ?? []).join(" "), SERVICES: (ctx.classification?.services ?? []).join(" "),
|
|
@@ -405,7 +412,11 @@ async function finalize(a, io) {
|
|
|
405
412
|
if (!/^[A-Za-z0-9._/-]+$/.test(codeBranch)) codeBranch = c.stagingBranch;
|
|
406
413
|
|
|
407
414
|
const findings = readJson(io, join(outDir, "findings.json"))?.findings ?? [];
|
|
408
|
-
|
|
415
|
+
// A finding whose own filed issue is already closed (file-prepare: skip-resolved, delta pass) is resolved: it never blocks (#4569).
|
|
416
|
+
const resolvedBy = new Map(worklist.items.filter((i) => i.action === "skip-resolved").map((i) => [i.id, i.dedup_of]));
|
|
417
|
+
const resolved = findings.filter((f) => resolvedBy.has(f.id) && upper(f.confidence) === "CONFIRMED" && ["CRITICAL", "HIGH"].includes(upper(f.severity)))
|
|
418
|
+
.map((f) => ({ id: f.id, severity: upper(f.severity), title: f.title ?? "", issue: resolvedBy.get(f.id) }));
|
|
419
|
+
const confirmed = (sev) => findings.filter((f) => !resolvedBy.has(f.id) && upper(f.confidence) === "CONFIRMED" && upper(f.severity) === sev).length;
|
|
409
420
|
const outOfScope = Number(/out_of_scope=(\d+)/.exec(readText(io, join(outDir, "note-disposition.md")) ?? "")?.[1] ?? 0);
|
|
410
421
|
const counts = {
|
|
411
422
|
findings_kept: Number(x.FINDINGS_KEPT ?? 0), findings_dropped_stale: Number(x.FINDINGS_DROPPED_STALE ?? 0), findings_preexisting: Number(x.FINDINGS_PREEXISTING ?? 0),
|
|
@@ -415,7 +426,7 @@ async function finalize(a, io) {
|
|
|
415
426
|
const items = worklist.items;
|
|
416
427
|
const state = {
|
|
417
428
|
pr, head, bundle, codeBranch, scopeMode, delta, testGate: { verdict: tg.verdict, reason: tg.reason }, counts, notes,
|
|
418
|
-
confirmedCritical: confirmed("CRITICAL"), confirmedHigh: confirmed("HIGH"),
|
|
429
|
+
confirmedCritical: confirmed("CRITICAL"), confirmedHigh: confirmed("HIGH"), resolved,
|
|
419
430
|
toFile: items.filter((i) => i.action === "file"), skipped: items.filter((i) => String(i.action).startsWith("skip-")),
|
|
420
431
|
};
|
|
421
432
|
io.writeFile(join(sd, `${pr}_finalize.json`), JSON.stringify(state));
|
|
@@ -500,6 +511,7 @@ async function summary(a, io) {
|
|
|
500
511
|
`- findings_kept: ${n.findings_kept ?? 0}; dropped_stale: ${n.findings_dropped_stale ?? 0}; pre-existing: ${n.findings_preexisting ?? 0}; notes_demoted_no_scenario: ${n.notes_demoted_no_scenario ?? 0}`,
|
|
501
512
|
`- findings_filed: ${n.findings_filed ?? 0}${filed.length ? ` (${filed.join(", ")})` : ""}; out_of_scope: ${n.out_of_scope ?? 0}`,
|
|
502
513
|
`- notes_fixed: 0; notes_listed: ${n.notes_listed ?? 0}; notes_dropped: ${n.notes_dropped ?? 0}; notes_capped: ${n.notes_capped ?? 0}; notes_reflag: ${n.notes_reflag ?? 0}`,
|
|
514
|
+
`- resolved (excluded from the verdict): ${(st.resolved ?? []).length ? st.resolved.map((r) => `${r.severity} ${r.id} ${r.title} (closed #${r.issue})`.replace(/\s+/g, " ")).join("; ") : "(none)"}`,
|
|
503
515
|
`- deduped: ${deduped.length ? deduped.join("; ") : "(none)"}`,
|
|
504
516
|
...(st.delta?.from ? [`- delta pass: ${String(st.delta.from).slice(0, 7)}..${head.slice(0, 7)} (${(st.delta.files ?? []).length} file(s)); earlier pass at ${st.delta.from} credited (its findings re-read by extract)`] : []),
|
|
505
517
|
"",
|
package/bin/runner.mjs
CHANGED
|
@@ -1809,6 +1809,7 @@ export function runCliBackend({
|
|
|
1809
1809
|
` Invocation: ${bin} ${argvSummary} (cwd: ${cwd})`,
|
|
1810
1810
|
);
|
|
1811
1811
|
err.code = "CLI_BACKEND_FAILED";
|
|
1812
|
+
err.status = result.status ?? null; err.signal = result.signal ?? null; // #4518: lets bin/lean/hostkill.mjs tell an external kill
|
|
1812
1813
|
// forge#2241: attach the CLI's reported session-limit reset time (when
|
|
1813
1814
|
// present) so callers (bin/engine.mjs) can surface *when* a
|
|
1814
1815
|
// quota-exhaustion failure will clear without reading raw logs. Only
|
package/bin/stats/aggregate.mjs
CHANGED
|
@@ -295,6 +295,12 @@ export function computeAlerts(lanes = [], opts = {}) {
|
|
|
295
295
|
const ex = failed.flatMap((l) => l.deps).find((d) => (d.eco ?? "unknown") === eco && d.status === "failed");
|
|
296
296
|
add(`DEPS_PREPARED failed (${eco})`, failed.length, used.length, c.depsFailedPct / 100, ex?.reason ?? null);
|
|
297
297
|
}
|
|
298
|
+
// #4522: a broken snapshot falls back to a fresh install; the lane has dependencies, but the cache still needs a look.
|
|
299
|
+
const fell = used.filter((l) => l.deps.some((d) => (d.eco ?? "unknown") === eco && d.status === "materialize-fallback"));
|
|
300
|
+
if (fell.length / used.length > c.depsFailedPct / 100) {
|
|
301
|
+
const ex = fell.flatMap((l) => l.deps).find((d) => (d.eco ?? "unknown") === eco && d.status === "materialize-fallback");
|
|
302
|
+
add(`DEPS_PREPARED materialize-fallback (${eco})`, fell.length, used.length, c.depsFailedPct / 100, ex?.reason ?? null);
|
|
303
|
+
}
|
|
298
304
|
}
|
|
299
305
|
const deferred = win.filter((l) => l.deferred.length);
|
|
300
306
|
if (deferred.length / win.length > c.deferredPct / 100) add("deferred-to-ci", deferred.length, win.length, c.deferredPct / 100, deferred[deferred.length - 1].deferred[0]);
|
package/bin/stats/efficiency.mjs
CHANGED
|
@@ -49,6 +49,8 @@ const costOf = (e) => num(e.cost_usd_delta) ?? num(e.cost_usd);
|
|
|
49
49
|
const sum = (xs) => xs.reduce((a, b) => a + (num(b) ?? 0), 0);
|
|
50
50
|
const r = (x, p = 1e4) => (num(x) == null ? null : Math.round(x * p) / p);
|
|
51
51
|
const per = (a, b) => (b > 0 ? a / b : null);
|
|
52
|
+
/** #4491: GitHub slugs are case-insensitive (`--repo o/forgedock` matches a plan's `O/ForgeDock`). */
|
|
53
|
+
export const sameRepo = (a, b) => String(a ?? "").toLowerCase() === String(b ?? "").toLowerCase();
|
|
52
54
|
|
|
53
55
|
function parseLog(text) {
|
|
54
56
|
const out = [];
|
|
@@ -67,13 +69,14 @@ export function issueSlices(events) {
|
|
|
67
69
|
let cur = null;
|
|
68
70
|
for (const e of Array.isArray(events) ? events : []) {
|
|
69
71
|
if (!e || typeof e !== "object") continue;
|
|
70
|
-
if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, shape: "full", attempts: [] }; out.push(cur); continue; }
|
|
72
|
+
if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, shape: "full", conformance: null, attempts: [] }; out.push(cur); continue; }
|
|
71
73
|
if (!cur) continue;
|
|
72
74
|
if (e.event === "LIGHT_DONE") cur.shape = "light";
|
|
75
|
+
else if (e.event === "CONFORMANCE" && typeof e.state === "string") cur.conformance = e.state; // #4491: the slice's last verdict
|
|
73
76
|
else if (e.event === "LIGHT_UPGRADE") cur.shape = "upgraded";
|
|
74
77
|
if (e.event === "PHASE_ATTEMPT_USAGE") {
|
|
75
78
|
cur.attempts.push({ phase: e.phase ?? "other", trigger: e.trigger ?? null, restart_of: e.restart_of ?? null, model: e.model ?? null,
|
|
76
|
-
turns: num(e.turns), cacheRead: num(e.usage?.cache_read_input_tokens), cost: costOf(e),
|
|
79
|
+
turns: num(e.turns), cacheRead: num(e.usage?.cache_read_input_tokens), cost: costOf(e), durationMs: num(e.duration_ms), // #4491
|
|
77
80
|
...(e.tools && typeof e.tools === "object" ? { calls: num(e.tools.calls), toolTurns: num(e.tools.turns), rereads: num(e.tools.rereads) } : {}) }); // #4410
|
|
78
81
|
} else if (e.event === "RUN_TERMINAL") {
|
|
79
82
|
cur.terminal = { reason: e.reason ?? null, code: e.code ?? null, ts: e.ts ?? null, size: SIZES.includes(e.turns?.size) ? e.turns.size : null };
|
|
@@ -137,10 +140,17 @@ const median = (xs) => {
|
|
|
137
140
|
return s.length % 2 ? s[h] : r((s[h - 1] + s[h]) / 2, 100);
|
|
138
141
|
};
|
|
139
142
|
|
|
143
|
+
/** #4491: a lane's implement wall in minutes, the sum of its implement attempts' duration_ms; null when none carries it. */
|
|
144
|
+
function laneWall(impl) {
|
|
145
|
+
const xs = impl.filter((a) => a.durationMs != null);
|
|
146
|
+
return xs.length ? r(sum(xs.map((a) => a.durationMs)) / 60000, 100) : null;
|
|
147
|
+
}
|
|
148
|
+
const wallP50 = (xs) => median(xs.map((x) => x.implWall).filter((w) => w != null));
|
|
149
|
+
|
|
140
150
|
const group = (xs) => {
|
|
141
151
|
const turns = sum(xs.map((x) => x.implTurns)), cache = sum(xs.map((x) => x.implCache));
|
|
142
152
|
return { n: xs.length, cost_usd: r(sum(xs.map((x) => x.cost)), 1e6), usd_per_merge: r(per(sum(xs.map((x) => x.cost)), xs.length)),
|
|
143
|
-
turns_per_merge: r(per(turns, xs.length), 10), context_per_turn: r(per(cache, turns), 1) };
|
|
153
|
+
turns_per_merge: r(per(turns, xs.length), 10), context_per_turn: r(per(cache, turns), 1), implement_wall_p50_min: wallP50(xs) };
|
|
144
154
|
};
|
|
145
155
|
|
|
146
156
|
/**
|
|
@@ -149,7 +159,7 @@ const group = (xs) => {
|
|
|
149
159
|
*/
|
|
150
160
|
export function efficiencyMetrics({ entries = [], logs = new Map(), issues = null, repo = null, now = 0, notes = [] } = {}) {
|
|
151
161
|
const merged = [];
|
|
152
|
-
const inScope = (row, settleAt) => issueSlices(logs.get(row.issue) ?? []).filter((s) => (!repo || !s.repo || s.repo
|
|
162
|
+
const inScope = (row, settleAt) => issueSlices(logs.get(row.issue) ?? []).filter((s) => (!repo || !s.repo || sameRepo(s.repo, repo)) && (!settleAt || !s.startTs || s.startTs <= settleAt));
|
|
153
163
|
// #4452: light/full split and upgrade rate over every lane of every row (all outcomes); cost is the #4407 delta sum
|
|
154
164
|
const shapes = Object.fromEntries(LANE_SHAPES.map((k) => [k, { n: 0, cost_usd: 0 }]));
|
|
155
165
|
for (const { row, settleAt } of entries) {
|
|
@@ -169,7 +179,7 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
169
179
|
const term = [...slices].reverse().find((s) => s.terminal?.reason === "merged")?.terminal;
|
|
170
180
|
merged.push({ issue: row.issue, slices: slices.length, cost: sum(atts.map((a) => a.cost)), implCost: sum(impl.map((a) => a.cost)),
|
|
171
181
|
reviewCost: sum(atts.filter((a) => a.phase === "review").map((a) => a.cost)), implTurns: sum(impl.map((a) => a.turns)), implCache: sum(impl.map((a) => a.cacheRead)),
|
|
172
|
-
size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl), shape: slices.at(-1)?.shape ?? null });
|
|
182
|
+
size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl), shape: slices.at(-1)?.shape ?? null, implWall: laneWall(impl) });
|
|
173
183
|
}
|
|
174
184
|
const all = group(merged);
|
|
175
185
|
const bySize = {};
|
|
@@ -182,6 +192,9 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
182
192
|
}
|
|
183
193
|
}
|
|
184
194
|
const retries = Object.fromEntries(RETRY_CAUSES.map((c) => [c, sum(merged.map((m) => m.retries[c]))]));
|
|
195
|
+
// #4491: merge rate over the rows, conformance pass rate over rows whose lanes logged a verdict (each row's last one)
|
|
196
|
+
const rows = entries.filter(({ row }) => Number.isSafeInteger(row?.issue));
|
|
197
|
+
const verdicts = rows.map(({ row, settleAt }) => inScope(row, settleAt).map((s) => s.conformance).filter(Boolean).at(-1)).filter(Boolean);
|
|
185
198
|
const withTools = merged.filter((m) => m.tools);
|
|
186
199
|
const total = sum(Object.values(retries));
|
|
187
200
|
// no possible action: parked post-deploy follow-ups and promotion-blocked lanes (#4389), over all rows
|
|
@@ -209,6 +222,8 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
209
222
|
merged: merged.length, cost_usd: all.cost_usd, implement_usd: r(sum(merged.map((m) => m.implCost)), 1e6), review_usd: r(sum(merged.map((m) => m.reviewCost)), 1e6),
|
|
210
223
|
usd_per_merge: all.usd_per_merge, turns_per_merge: all.turns_per_merge, context_per_turn: all.context_per_turn,
|
|
211
224
|
implement_turns: sum(merged.map((m) => m.implTurns)), implement_cache_read: sum(merged.map((m) => m.implCache)),
|
|
225
|
+
implement_wall_p50_min: all.implement_wall_p50_min, merge_rate: r(per(merged.length, rows.length)),
|
|
226
|
+
conformance_pass_rate: r(per(verdicts.filter((v) => v === "conforms").length, verdicts.length)),
|
|
212
227
|
by_size: bySize,
|
|
213
228
|
by_model: Object.fromEntries(Object.entries(byModel).map(([k, g]) => [k, { merges: g.merges.size, attempts: g.attempts, turns: g.turns, cost_usd: r(g.cost, 1e6),
|
|
214
229
|
usd_per_merge: r(per(g.cost, g.merges.size)), turns_per_merge: r(per(g.turns, g.merges.size), 10), context_per_turn: r(per(g.cache, g.turns), 1) }])),
|
|
@@ -219,7 +234,8 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
219
234
|
tool_calls_per_turn_p50: median(withTools.map((m) => m.tools.per_turn)), rereads_per_merge: withTools.length ? r(per(sum(withTools.map((m) => m.tools.rereads)), withTools.length), 100) : null,
|
|
220
235
|
per_issue: merged.map((m) => ({ issue: m.issue, slices: m.slices, cost_usd: r(m.cost, 1e6), implement_usd: r(m.implCost, 1e6), implement_turns: m.implTurns,
|
|
221
236
|
implement_cache_read: m.implCache, size: m.size, retries: m.retries, ...(m.shape ? { shape: m.shape } : {}), ...(m.info ? { quality: issueQuality(m.info.body).tag } : {}),
|
|
222
|
-
...(m.tools ? { implement_tool_calls_per_turn: m.tools.per_turn, implement_rereads: m.tools.rereads } : {})
|
|
237
|
+
...(m.tools ? { implement_tool_calls_per_turn: m.tools.per_turn, implement_rereads: m.tools.rereads } : {}),
|
|
238
|
+
...(m.implWall != null ? { implement_wall_min: m.implWall } : {}) })),
|
|
223
239
|
notes,
|
|
224
240
|
};
|
|
225
241
|
}
|
|
@@ -274,26 +290,77 @@ export async function batchEfficiency({ batchDir, runsDir, repo = null, gh = nul
|
|
|
274
290
|
return { batch: plan.id ?? null, ...(await computeScope({ entries, runsDir, repo: repo ?? plan.repo ?? null, gh, now })) };
|
|
275
291
|
}
|
|
276
292
|
|
|
277
|
-
/**
|
|
278
|
-
|
|
293
|
+
/** The repo's batches by creation time, and the repo slug as the plans spell it (run-log file names are case-sensitive, #4491). */
|
|
294
|
+
function repoBatches(batchRoot, repo) {
|
|
279
295
|
const batches = (existsSync(batchRoot) ? listBatches(batchRoot) : []).map((id) => {
|
|
280
296
|
const dir = join(batchRoot, id), plan = readJson(join(dir, "plan.json")) ?? {};
|
|
281
297
|
return { dir, plan, created: Date.parse(plan.created ?? "") };
|
|
282
|
-
}).filter((b) => Number.isFinite(b.created) && (!repo || b.plan.repo
|
|
298
|
+
}).filter((b) => Number.isFinite(b.created) && (!repo || sameRepo(b.plan.repo, repo))).sort((a, b) => a.created - b.created);
|
|
299
|
+
return { batches, repo: repo ? (batches.find((b) => typeof b.plan.repo === "string")?.plan.repo ?? repo) : repo };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** Rows of the batches created in [from, to]; an issue in several batches counts once (latest row). */
|
|
303
|
+
function cohortEntries(batches, from, to, toInclusive = true) {
|
|
304
|
+
const inside = batches.filter((b) => b.created >= from && (toInclusive ? b.created <= to : b.created < to));
|
|
305
|
+
const latest = new Map();
|
|
306
|
+
for (const b of inside) {
|
|
307
|
+
const at = settleTimes(readLogRows(b.dir));
|
|
308
|
+
for (const row of batchRows(b.dir)) if (Number.isSafeInteger(row?.issue)) latest.set(row.issue, { row, settleAt: at.get(row.issue) ?? null });
|
|
309
|
+
}
|
|
310
|
+
return { batches: inside.length, entries: [...latest.values()] };
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Trailing windows: the union of the repo's batches created in each window; an issue in several batches counts once (latest row). */
|
|
314
|
+
export async function windowEfficiency({ batchRoot, runsDir, repo = null, gh = null, now = Date.now(), windows = [WEEK_MS] }) {
|
|
315
|
+
const { batches, repo: slug } = repoBatches(batchRoot, repo);
|
|
283
316
|
const out = {};
|
|
284
317
|
for (const w of windows) {
|
|
285
|
-
const
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
318
|
+
const c = cohortEntries(batches, now - w, now);
|
|
319
|
+
out[windowLabel(w)] = { batches: c.batches, ...(await computeScope({ entries: c.entries, runsDir, repo: slug, gh, now })) };
|
|
320
|
+
}
|
|
321
|
+
return out;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
export const COMPARE_KEYS = Object.freeze(["turns_per_merge", "implement_wall_p50_min", "tool_calls_per_turn_p50", "rereads_per_merge", "merge_rate", "conformance_pass_rate"]);
|
|
325
|
+
const delta = (a, b) => Object.fromEntries(COMPARE_KEYS.map((k) => [k, num(a?.[k]) == null || num(b?.[k]) == null ? null : r(b[k] - a[k], 100)]));
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* #4491: full-shape merged lanes per size bucket present in both cohorts (light lanes, #4452, cut turns on their own).
|
|
329
|
+
* A merged row with no in-scope run log carries no shape and drops out, so its zero turns do not pull a bucket down.
|
|
330
|
+
*/
|
|
331
|
+
function matchedCohorts(pre, post) {
|
|
332
|
+
const full = (m) => (m.per_issue ?? []).filter((x) => x.shape === "full");
|
|
333
|
+
const bySize = (xs) => xs.reduce((o, x) => ((o[x.size] ??= []).push(x), o), {});
|
|
334
|
+
const a = bySize(full(pre)), b = bySize(full(post)), out = {};
|
|
335
|
+
const g = (xs) => {
|
|
336
|
+
const tools = xs.filter((x) => x.implement_tool_calls_per_turn != null);
|
|
337
|
+
return { n: xs.length, turns_per_merge: r(per(sum(xs.map((x) => x.implement_turns)), xs.length), 10),
|
|
338
|
+
implement_wall_p50_min: median(xs.map((x) => x.implement_wall_min).filter((w) => w != null)),
|
|
339
|
+
tool_calls_per_turn_p50: median(tools.map((x) => x.implement_tool_calls_per_turn)),
|
|
340
|
+
rereads_per_merge: tools.length ? r(per(sum(tools.map((x) => x.implement_rereads)), tools.length), 100) : null };
|
|
341
|
+
};
|
|
342
|
+
for (const size of Object.keys(a).sort()) {
|
|
343
|
+
if (!b[size]) continue;
|
|
344
|
+
const pa = g(a[size]), pb = g(b[size]);
|
|
345
|
+
out[size] = { pre: pa, post: pb, delta: delta(pa, pb) };
|
|
293
346
|
}
|
|
294
347
|
return out;
|
|
295
348
|
}
|
|
296
349
|
|
|
350
|
+
/**
|
|
351
|
+
* #4491: before/after A/B around a deploy `pivot` (ms): post = the repo's batches created in [pivot, min(now, pivot + span)],
|
|
352
|
+
* pre = the same span ending just before the pivot. `span` defaults to the time since the pivot, so both cohorts are equally long.
|
|
353
|
+
*/
|
|
354
|
+
export async function compareEfficiency({ batchRoot, runsDir, repo = null, gh = null, pivot, now = Date.now(), span = null }) {
|
|
355
|
+
if (!Number.isFinite(pivot) || pivot > now) throw new Error("compare: the pivot must be a time in the past");
|
|
356
|
+
const s = Math.max(0, Math.min(Number.isFinite(span) ? span : Infinity, now - pivot));
|
|
357
|
+
const { batches, repo: slug } = repoBatches(batchRoot, repo);
|
|
358
|
+
const a = cohortEntries(batches, pivot - s, pivot, false), b = cohortEntries(batches, pivot, pivot + s);
|
|
359
|
+
const pre = { batches: a.batches, ...(await computeScope({ entries: a.entries, runsDir, repo: slug, gh, now })) };
|
|
360
|
+
const post = { batches: b.batches, ...(await computeScope({ entries: b.entries, runsDir, repo: slug, gh, now })) };
|
|
361
|
+
return { pivot: new Date(pivot).toISOString(), span_days: r(s / 86400000, 100), pre, post, delta: delta(pre, post), matched: matchedCohorts(pre, post) };
|
|
362
|
+
}
|
|
363
|
+
|
|
297
364
|
export const windowLabel = (ms) => `${Math.round(ms / 86400000)}d`;
|
|
298
365
|
/** "7d,30d" -> [ms]; throws on anything else. */
|
|
299
366
|
export function parseWindows(s) {
|
|
@@ -331,5 +398,24 @@ export function formatEfficiencyLines(m, label = "") {
|
|
|
331
398
|
const ln = m.lanes; // #4452
|
|
332
399
|
lines.push(ln ? `lanes: ${LANE_SHAPES.map((k) => `${k} ${ln[k]?.n ?? 0} (${usd(ln[k]?.cost_usd ?? 0)})`).join(", ")}; upgrade rate ${ln.upgrade_rate == null ? "n/a"
|
|
333
400
|
: `${Math.round(ln.upgrade_rate * 100)}% (${ln.upgraded?.n ?? 0}/${(ln.light?.n ?? 0) + (ln.upgraded?.n ?? 0)})`}` : "lanes: n/a");
|
|
401
|
+
lines.push(rates(m)); // #4491
|
|
402
|
+
return lines;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
const pct = (x) => (num(x) == null ? "n/a" : `${Math.round(x * 100)}%`);
|
|
406
|
+
const rates = (m) => `implement wall p50 ${m.implement_wall_p50_min == null ? "n/a" : `${f(m.implement_wall_p50_min, 1)} min`}, merge rate ${pct(m.merge_rate)}, conformance pass ${pct(m.conformance_pass_rate)}`;
|
|
407
|
+
const sign = (x, d, unit = "") => (num(x) == null ? "n/a" : `${x > 0 ? "+" : ""}${x.toFixed(d)}${unit}`);
|
|
408
|
+
const deltaLine = (d, withRates = true) => `turns/merge ${sign(d.turns_per_merge, 1)}, implement wall p50 ${sign(d.implement_wall_p50_min, 1, " min")}, calls/turn p50 ${sign(d.tool_calls_per_turn_p50, 2)}, `
|
|
409
|
+
+ `re-reads/merge ${sign(d.rereads_per_merge, 2)}${withRates ? `, merge rate ${sign(d.merge_rate == null ? null : d.merge_rate * 100, 0, " pts")}, conformance pass ${sign(d.conformance_pass_rate == null ? null : d.conformance_pass_rate * 100, 0, " pts")}` : ""}`;
|
|
410
|
+
const cohortLine = (k, m) => `compare ${k}: n=${m.n ?? m.merged} turns/merge ${f(m.turns_per_merge, 1)}, implement wall p50 ${m.implement_wall_p50_min == null ? "n/a" : `${f(m.implement_wall_p50_min, 1)} min`}, `
|
|
411
|
+
+ `calls/turn p50 ${f(m.tool_calls_per_turn_p50)}, re-reads/merge ${f(m.rereads_per_merge)}${"merge_rate" in m ? `, merge rate ${pct(m.merge_rate)}, conformance pass ${pct(m.conformance_pass_rate)}` : ""}`;
|
|
412
|
+
|
|
413
|
+
/** #4491: the `--compare` report: pre/post cohorts, their delta, then the full-shape per-size matched cohorts. */
|
|
414
|
+
export function formatCompareLines(c) {
|
|
415
|
+
const lines = [`compare: pivot ${c.pivot}, ${f(c.span_days, 1)} days each side (pre ${c.pre.batches} batches, post ${c.post.batches} batches)`,
|
|
416
|
+
cohortLine("pre", c.pre), cohortLine("post", c.post), `compare delta: ${deltaLine(c.delta)}`];
|
|
417
|
+
const sizes = Object.entries(c.matched ?? {});
|
|
418
|
+
if (!sizes.length) lines.push("compare matched (full lanes per size): n/a");
|
|
419
|
+
for (const [size, g] of sizes) lines.push(`compare matched ${size}: pre n=${g.pre.n} post n=${g.post.n}; delta ${deltaLine(g.delta, false)}`);
|
|
334
420
|
return lines;
|
|
335
421
|
}
|
|
@@ -9,9 +9,10 @@ import { tmpDir } from "../tests/_tmp.mjs";
|
|
|
9
9
|
import { TRIGGERS } from "../lean/fixround.mjs";
|
|
10
10
|
import {
|
|
11
11
|
TRIGGER_CAUSE, RETRY_CAUSES, WEEK_MS, codeCause, issueSlices, sliceRetries, issueQuality, acceptanceOf, efficiencyMetrics,
|
|
12
|
-
batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows,
|
|
12
|
+
batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows, compareEfficiency, formatCompareLines,
|
|
13
13
|
} from "./efficiency.mjs";
|
|
14
|
-
import { parseArgs, runStats } from "./index.mjs";
|
|
14
|
+
import { comparePivot, parseArgs, runStats } from "./index.mjs";
|
|
15
|
+
import { runLogPath } from "../engine/runlog.mjs";
|
|
15
16
|
|
|
16
17
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
17
18
|
const FX = join(here, "fixtures", "efficiency-20261022");
|
|
@@ -115,7 +116,8 @@ describe("#4402 efficiencyMetrics", () => {
|
|
|
115
116
|
});
|
|
116
117
|
it("renders n/a, never NaN, for an empty cohort", () => {
|
|
117
118
|
const lines = formatEfficiencyLines(efficiencyMetrics({}));
|
|
118
|
-
assert.equal(lines.length,
|
|
119
|
+
assert.equal(lines.length, 10);
|
|
120
|
+
assert.equal(lines[9], "implement wall p50 n/a, merge rate n/a, conformance pass n/a"); // #4491, appended
|
|
119
121
|
assert.ok(!lines.some((l) => /NaN|undefined/.test(l)), lines.join("\n"));
|
|
120
122
|
assert.match(lines[0], /^efficiency: merged 0, n\/a\/merge/);
|
|
121
123
|
assert.equal(lines[6], "acceptance (7d after promotion): n/a");
|
|
@@ -152,7 +154,7 @@ describe("#4452 light/full split and upgrade rate", () => {
|
|
|
152
154
|
});
|
|
153
155
|
it("the batch report prints the split and the upgrade rate", () => {
|
|
154
156
|
const lines = formatEfficiencyLines(m);
|
|
155
|
-
assert.equal(lines.
|
|
157
|
+
assert.equal(lines.find((l) => l.startsWith("lanes:")), "lanes: light 2 ($0.65), upgraded 2 ($0.90), full 1 ($1.00); upgrade rate 50% (2/4)");
|
|
156
158
|
});
|
|
157
159
|
});
|
|
158
160
|
|
|
@@ -264,3 +266,100 @@ describe("#4402 forgedock stats --efficiency", () => {
|
|
|
264
266
|
assert.match(text, /^no action: 32 lanes, \$10\.13 \(blocked-promotion 32\)$/m);
|
|
265
267
|
});
|
|
266
268
|
});
|
|
269
|
+
|
|
270
|
+
describe("#4491 before/after compare", () => {
|
|
271
|
+
const PIVOT = Date.parse("2026-10-20T00:00:00Z"), SLUG = "Own/Repo";
|
|
272
|
+
const tools = (calls, turns, rereads) => ({ tools: { calls, turns, per_turn: calls / turns, rereads, top: [] } });
|
|
273
|
+
// pre batch (10-18): issues 1, 2 merged (M, full), 3 failed; post batch (10-21): issues 4 merged (M, full), 5 merged light, 6 merged (L, full);
|
|
274
|
+
// 8 (pre) and 7 (post) merged with no run log, so both sides hold a size-unknown, shapeless lane
|
|
275
|
+
const lane = (turns, wallMin, t, conf, size = "M", extra = []) => [start("2026-10-01T00:00:00Z", SLUG), att("implement", { trigger: "build", turns, duration_ms: wallMin * 60000, ...t }),
|
|
276
|
+
...extra, ...(conf ? [{ event: "CONFORMANCE", state: conf }] : []), term("merged", "merged", { turns: { size } })];
|
|
277
|
+
const setup = () => {
|
|
278
|
+
const root = tmpDir("eff-cmp-root-"), runs = tmpDir("eff-cmp-runs-");
|
|
279
|
+
const batch = (id, created, rows) => {
|
|
280
|
+
mkdirSync(join(root, id), { recursive: true });
|
|
281
|
+
writeFileSync(join(root, id, "plan.json"), JSON.stringify({ v: 1, id, repo: SLUG, created }));
|
|
282
|
+
writeFileSync(join(root, id, "report.json"), JSON.stringify({ rows }));
|
|
283
|
+
};
|
|
284
|
+
batch("pre", "2026-10-18T00:00:00Z", [{ issue: 1, outcome: "merged" }, { issue: 2, outcome: "merged" }, { issue: 3, outcome: "failed" }, { issue: 8, outcome: "merged" }]);
|
|
285
|
+
batch("old", "2026-10-10T00:00:00Z", [{ issue: 9, outcome: "merged" }]); // outside the 3-day span
|
|
286
|
+
batch("post", "2026-10-21T00:00:00Z", [{ issue: 4, outcome: "merged" }, { issue: 5, outcome: "merged" }, { issue: 6, outcome: "merged" }, { issue: 7, outcome: "merged" }]);
|
|
287
|
+
const logs = {
|
|
288
|
+
1: lane(40, 30, tools(40, 40, 2), "conforms"), 2: lane(60, 50, tools(60, 60, 4), "nonconforming"),
|
|
289
|
+
3: [start("2026-10-01T00:00:00Z", SLUG), att("implement", { trigger: "build" }), { event: "CONFORMANCE", state: "nonconforming" }, term("failed", "ci.red")],
|
|
290
|
+
4: lane(20, 20, tools(30, 20, 0), "conforms"), 5: lane(5, 2, tools(5, 5, 0), "conforms", "S", [{ event: "LIGHT_DONE" }]), 6: lane(30, 25, tools(30, 30, 1), null, "L"),
|
|
291
|
+
9: lane(99, 99, tools(99, 99, 9), "conforms"),
|
|
292
|
+
};
|
|
293
|
+
for (const [n, evs] of Object.entries(logs)) writeFileSync(runLogPath(runs, Number(n), SLUG), evs.map((e) => JSON.stringify(e)).join("\n") + "\n");
|
|
294
|
+
return { root, runs };
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
it("compare splits equal-span cohorts at the pivot with implement wall, merge and conformance rates, and deltas", async () => {
|
|
298
|
+
const { root, runs } = setup();
|
|
299
|
+
const c = await compareEfficiency({ batchRoot: root, runsDir: runs, repo: SLUG, pivot: PIVOT, now: PIVOT + 3 * DAY });
|
|
300
|
+
assert.equal(c.span_days, 3);
|
|
301
|
+
assert.deepEqual([c.pre.batches, c.pre.merged, c.post.batches, c.post.merged], [1, 3, 1, 4]); // 8 and 7 merged without a run log
|
|
302
|
+
assert.equal(c.pre.implement_wall_p50_min, 40); // median of 30 and 50
|
|
303
|
+
assert.equal(c.post.implement_wall_p50_min, 20);
|
|
304
|
+
assert.equal(c.pre.merge_rate, 0.75);
|
|
305
|
+
assert.equal(c.post.merge_rate, 1);
|
|
306
|
+
assert.equal(c.pre.conformance_pass_rate, 0.3333); // 1 of 3 verdicts
|
|
307
|
+
assert.equal(c.post.conformance_pass_rate, 1); // issue 6 has no verdict
|
|
308
|
+
assert.equal(c.delta.turns_per_merge, -19.5); // (40 + 60 + 0) / 3 = 33.3 -> (20 + 5 + 30 + 0) / 4 = 13.8
|
|
309
|
+
assert.equal(c.delta.implement_wall_p50_min, -20);
|
|
310
|
+
assert.equal(c.delta.merge_rate, 0.25);
|
|
311
|
+
assert.equal(c.delta.tool_calls_per_turn_p50, 0);
|
|
312
|
+
});
|
|
313
|
+
it("compare matches full lanes per size present on both sides (light, log-less lanes and unmatched sizes drop out)", async () => {
|
|
314
|
+
const { root, runs } = setup();
|
|
315
|
+
const c = await compareEfficiency({ batchRoot: root, runsDir: runs, repo: SLUG, pivot: PIVOT, now: PIVOT + 3 * DAY });
|
|
316
|
+
assert.deepEqual(Object.keys(c.matched), ["M"]); // issues 7 and 8 (no log, size unknown) are in no bucket
|
|
317
|
+
assert.deepEqual(c.matched.M.pre, { n: 2, turns_per_merge: 50, implement_wall_p50_min: 40, tool_calls_per_turn_p50: 1, rereads_per_merge: 3 });
|
|
318
|
+
assert.deepEqual(c.matched.M.post, { n: 1, turns_per_merge: 20, implement_wall_p50_min: 20, tool_calls_per_turn_p50: 1.5, rereads_per_merge: 0 });
|
|
319
|
+
assert.equal(c.matched.M.delta.tool_calls_per_turn_p50, 0.5);
|
|
320
|
+
assert.equal(c.matched.M.delta.turns_per_merge, -30);
|
|
321
|
+
});
|
|
322
|
+
it("compare report lines", async () => {
|
|
323
|
+
const { root, runs } = setup();
|
|
324
|
+
const lines = formatCompareLines(await compareEfficiency({ batchRoot: root, runsDir: runs, repo: SLUG, pivot: PIVOT, now: PIVOT + 3 * DAY }));
|
|
325
|
+
assert.equal(lines[0], "compare: pivot 2026-10-20T00:00:00.000Z, 3.0 days each side (pre 1 batches, post 1 batches)");
|
|
326
|
+
assert.match(lines[1], /^compare pre: n=3 turns\/merge 33\.3, implement wall p50 40\.0 min, calls\/turn p50 1\.00, re-reads\/merge 3\.00, merge rate 75%, conformance pass 33%$/);
|
|
327
|
+
assert.match(lines[3], /^compare delta: turns\/merge -?\d+\.\d, implement wall p50 -20\.0 min, .*merge rate \+25 pts, conformance pass \+67 pts$/);
|
|
328
|
+
assert.match(lines[4], /^compare matched M: pre n=2 post n=1; delta turns\/merge -30\.0, implement wall p50 -20\.0 min, calls\/turn p50 \+0\.50, re-reads\/merge -3\.00$/);
|
|
329
|
+
assert.ok(!lines.some((l) => /NaN|undefined/.test(l)), lines.join("\n"));
|
|
330
|
+
});
|
|
331
|
+
it("case-insensitive repo: a lowercase --repo matches the plans and reads the run logs under their spelling", async () => {
|
|
332
|
+
const { root, runs } = setup();
|
|
333
|
+
const c = await compareEfficiency({ batchRoot: root, runsDir: runs, repo: "own/repo", pivot: PIVOT, now: PIVOT + 3 * DAY });
|
|
334
|
+
assert.deepEqual([c.pre.merged, c.post.merged, c.pre.implement_wall_p50_min], [3, 4, 40]);
|
|
335
|
+
const w = (await windowEfficiency({ batchRoot: root, runsDir: runs, repo: "OWN/REPO", now: PIVOT + 3 * DAY, windows: [7 * DAY] }))["7d"];
|
|
336
|
+
assert.deepEqual([w.batches, w.merged], [2, 7]);
|
|
337
|
+
});
|
|
338
|
+
it("compare rejects a pivot in the future and prints a bare n/a delta for empty cohorts", async () => {
|
|
339
|
+
await assert.rejects(compareEfficiency({ batchRoot: tmpDir("eff-cmp-x-"), runsDir: tmpDir("eff-cmp-y-"), pivot: NOW + DAY, now: NOW }), /pivot/);
|
|
340
|
+
const empty = formatCompareLines(await compareEfficiency({ batchRoot: tmpDir("eff-cmp-x-"), runsDir: tmpDir("eff-cmp-y-"), pivot: NOW - DAY, now: NOW }));
|
|
341
|
+
assert.equal(empty[3], "compare delta: turns/merge n/a, implement wall p50 n/a, calls/turn p50 n/a, re-reads/merge n/a, merge rate n/a, conformance pass n/a");
|
|
342
|
+
});
|
|
343
|
+
it("compare and offline flags: parsing, pivot resolution and the offline CLI skips gh", async () => {
|
|
344
|
+
const o = parseArgs(["--efficiency", "--compare", "v1.32.0", "--offline"]);
|
|
345
|
+
assert.deepEqual([o.compare, o.offline], ["v1.32.0", true]);
|
|
346
|
+
assert.throws(() => parseArgs(["--compare", "2026-10-20"]), /need --efficiency/);
|
|
347
|
+
assert.throws(() => parseArgs(["--offline"]), /need --efficiency/);
|
|
348
|
+
assert.throws(() => parseArgs(["--efficiency", "--compare", "last-week"]), /--compare must be/);
|
|
349
|
+
assert.equal(comparePivot("2026-10-20T00:00:00Z"), PIVOT);
|
|
350
|
+
assert.equal(comparePivot("v1.32.0", [{ v: "1.32.0", t: PIVOT / 1000 }]), PIVOT);
|
|
351
|
+
assert.throws(() => comparePivot("9.9.9", []), /no release time known/);
|
|
352
|
+
const { root, runs } = setup();
|
|
353
|
+
const out = [], w = process.stdout.write;
|
|
354
|
+
process.stdout.write = (t) => { out.push(String(t)); return true; };
|
|
355
|
+
let code;
|
|
356
|
+
try {
|
|
357
|
+
code = await runStats(["--efficiency", "--window", "7d", "--compare", "2026-10-20T00:00:00Z", "--offline", "--repo", "own/repo"],
|
|
358
|
+
{ batchRoot: root, runsDir: runs, now: () => PIVOT + 3 * DAY, releaseTimes: [], gh: async () => { throw new Error("gh must not be called offline"); } });
|
|
359
|
+
} finally { process.stdout.write = w; }
|
|
360
|
+
assert.equal(code, 0);
|
|
361
|
+
const text = out.join("");
|
|
362
|
+
assert.match(text, /^compare: pivot 2026-10-20T00:00:00\.000Z/m);
|
|
363
|
+
assert.match(text, /^issue quality: n\/a \(no gh\/repo\)$/m);
|
|
364
|
+
});
|
|
365
|
+
});
|
package/bin/stats/index.mjs
CHANGED
|
@@ -15,14 +15,16 @@ import { readLanes, readLocalLaneRuns, readLocalRuns, runsDir } from "./local-so
|
|
|
15
15
|
import { parseForgeYaml } from "../forge-utils.mjs";
|
|
16
16
|
import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
|
|
17
17
|
import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers, exploreAb, renderExploreAb, laneProfileReport, renderLaneProfiles } from "./query.mjs";
|
|
18
|
-
import { batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows } from "./efficiency.mjs";
|
|
18
|
+
import { batchEfficiency, windowEfficiency, compareEfficiency, formatEfficiencyLines, formatCompareLines, parseWindows } from "./efficiency.mjs";
|
|
19
|
+
import { poolUtilisation, formatPoolLines } from "./pool.mjs";
|
|
19
20
|
import { batchRoot } from "../lean/batch.mjs";
|
|
20
21
|
|
|
21
22
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
22
23
|
|
|
23
24
|
export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <owner/repo>] [--json] [--refresh]
|
|
24
25
|
[--compact] [--no-local] [--no-cache] [--min-n N] [--release-times FILE]
|
|
25
|
-
forgedock stats --
|
|
26
|
+
forgedock stats --pool [--window 7d,30d] [--repo <owner/repo>] [--json]
|
|
27
|
+
forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--compare <date|version>] [--offline] [--repo <owner/repo>] [--json]
|
|
26
28
|
|
|
27
29
|
Version-over-version report: Cost, Time, Flow, Review funnel, Gates and Issue flow per (version, repo), with a
|
|
28
30
|
delta against the previous version and a low-n flag for samples under 10. Missing data prints n/a.
|
|
@@ -56,11 +58,18 @@ Explorer A/B (forge#4418; local run logs only, every run of each log, no GitHub
|
|
|
56
58
|
--explore-ab builder turns, builder cache-read, merge and conformance rates per size (S, M, L) and
|
|
57
59
|
lean.explore mode (on, off), plus the same issues run in both modes; --json for JSON
|
|
58
60
|
|
|
61
|
+
Pool (forge#4549; local batch reports, no gh):
|
|
62
|
+
--pool remote lane minutes, lanes and share per pool agent over --window (default 7d,30d)
|
|
63
|
+
|
|
59
64
|
Efficiency (forge#4402; local batch dirs and run logs, gh only for merged issues' bodies, labels and probes):
|
|
60
65
|
--efficiency $/merge, turns/merge, context/turn per size and model, retries/merge by cause, no-action
|
|
61
66
|
lanes, issue quality vs cost and the acceptance pass rate 7 days after promotion
|
|
62
67
|
--batch <id> also report one batch (~/.forge/batches/<id>, FORGEDOCK_BATCH_DIR)
|
|
63
68
|
--window <list> trailing windows over the repo's batches (default 7d,30d)
|
|
69
|
+
--compare <date|v> before/after A/B around a deploy (date or release tag): equal-span cohorts of batches either side,
|
|
70
|
+
turns/merge, implement wall p50, calls/turn p50, re-reads/merge, merge and conformance rates,
|
|
71
|
+
their delta and full-lane cohorts matched per size (forge#4491)
|
|
72
|
+
--offline skip the GitHub crawl for merged issues (issue quality and acceptance print n/a)
|
|
64
73
|
|
|
65
74
|
Builder model tiers (forge#4394; local run logs only, no GitHub crawl):
|
|
66
75
|
--tiers merge, conformance and post-merge-rework rates, first-pass yield and cost per builder tier
|
|
@@ -73,7 +82,7 @@ Lane profile baseline (forge#4436; local run logs only, every run of each issue,
|
|
|
73
82
|
`;
|
|
74
83
|
|
|
75
84
|
export function parseArgs(argv) {
|
|
76
|
-
const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, batch: null, windows: null, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
|
|
85
|
+
const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
|
|
77
86
|
const val = (i, name) => {
|
|
78
87
|
const v = argv[i + 1];
|
|
79
88
|
if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
|
|
@@ -95,8 +104,15 @@ export function parseArgs(argv) {
|
|
|
95
104
|
else if (name === "--lane-profile") o.laneProfile = true;
|
|
96
105
|
else if (name === "--issues") { const v = take(); if (!/^[1-9]\d*(,[1-9]\d*)*$/.test(v)) throw new Error("--issues must be a comma list of issue numbers"); o.issues = v.split(",").map(Number); }
|
|
97
106
|
else if (name === "--efficiency") o.efficiency = true;
|
|
107
|
+
else if (name === "--pool") o.pool = true;
|
|
98
108
|
else if (name === "--batch") { const v = take(); if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v)) throw new Error(`invalid --batch id: ${v}`); o.batch = v; }
|
|
99
109
|
else if (name === "--window") o.windows = parseWindows(take());
|
|
110
|
+
else if (name === "--compare") { // #4491
|
|
111
|
+
const v = take(), bad = () => new Error(`--compare must be a date (YYYY-MM-DD[THH:MM:SSZ]) or a version (1.32.0): ${v}`);
|
|
112
|
+
try { if (!v || !Object.keys(parseSince(v)).length) throw bad(); } catch { throw bad(); }
|
|
113
|
+
o.compare = v;
|
|
114
|
+
}
|
|
115
|
+
else if (name === "--offline") o.offline = true;
|
|
100
116
|
else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); }
|
|
101
117
|
else if (name === "--json") o.json = true;
|
|
102
118
|
else if (name === "--refresh") o.refresh = true;
|
|
@@ -106,7 +122,9 @@ export function parseArgs(argv) {
|
|
|
106
122
|
else if (name === "--help" || name === "-h") o.help = true;
|
|
107
123
|
else throw new Error(`unknown argument: ${a}`);
|
|
108
124
|
}
|
|
109
|
-
if (
|
|
125
|
+
if (o.batch && !o.efficiency) throw new Error("--batch and --window need --efficiency");
|
|
126
|
+
if (o.windows && !o.efficiency && !o.pool) throw new Error("--batch and --window need --efficiency");
|
|
127
|
+
if ((o.compare || o.offline) && !o.efficiency) throw new Error("--compare and --offline need --efficiency");
|
|
110
128
|
return o;
|
|
111
129
|
}
|
|
112
130
|
|
|
@@ -147,6 +165,15 @@ export function sinceToTimestamp(since, releaseTimes = []) {
|
|
|
147
165
|
return { since: null, note: null };
|
|
148
166
|
}
|
|
149
167
|
|
|
168
|
+
/** #4491: `--compare` pivot in ms: a date as is, a version as its exact tag time (no crawl margin); throws when unknown. */
|
|
169
|
+
export function comparePivot(v, releaseTimes = []) {
|
|
170
|
+
const p = parseSince(v);
|
|
171
|
+
if (p.sinceMs != null) return p.sinceMs;
|
|
172
|
+
const hit = releaseTimes.find((r) => r.v === p.sinceVersion);
|
|
173
|
+
if (!hit) throw new Error(`--compare: no release time known for ${p.sinceVersion} (pass --release-times or a date)`);
|
|
174
|
+
return hit.t * 1000;
|
|
175
|
+
}
|
|
176
|
+
|
|
150
177
|
/** Throttled stderr progress printer (never stdout, so --json stays parseable). */
|
|
151
178
|
export function makeProgressPrinter(write = (t) => process.stderr.write(t), now = Date.now) {
|
|
152
179
|
let last = 0, lastPhase = null;
|
|
@@ -215,11 +242,16 @@ export async function buildReport(opts, deps = {}) {
|
|
|
215
242
|
if (opts.releaseTimes) releaseTimes = parseReleaseTimes(readFileSync(opts.releaseTimes, "utf8"));
|
|
216
243
|
else if (deps.releaseTimes) releaseTimes = deps.releaseTimes;
|
|
217
244
|
else releaseTimes = gitTagTimes();
|
|
245
|
+
if (opts.pool) { // forge#4549: local batch reports only
|
|
246
|
+
const pool = poolUtilisation({ batchRoot: deps.batchRoot ?? batchRoot(env), repo, windows: opts.windows ?? parseWindows("7d,30d"), now: deps.now?.() ?? Date.now() });
|
|
247
|
+
return { repo, query: { pool } };
|
|
248
|
+
}
|
|
218
249
|
if (opts.efficiency) { // forge#4402: local batch dirs + run logs; gh only for merged issues
|
|
219
|
-
const root = deps.batchRoot ?? batchRoot(env), rd = deps.runsDir ?? runsDir(env), now = deps.now?.() ?? Date.now(), gh = deps.gh ?? defaultGh;
|
|
250
|
+
const root = deps.batchRoot ?? batchRoot(env), rd = deps.runsDir ?? runsDir(env), now = deps.now?.() ?? Date.now(), gh = opts.offline ? null : (deps.gh ?? defaultGh);
|
|
220
251
|
const efficiency = {};
|
|
221
252
|
if (opts.batch) efficiency.batch = await batchEfficiency({ batchDir: join(root, opts.batch), runsDir: rd, repo, gh, now });
|
|
222
253
|
efficiency.windows = await windowEfficiency({ batchRoot: root, runsDir: rd, repo, gh, now, windows: opts.windows ?? parseWindows("7d,30d") });
|
|
254
|
+
if (opts.compare) efficiency.compare = await compareEfficiency({ batchRoot: root, runsDir: rd, repo, gh, now, pivot: comparePivot(opts.compare, releaseTimes) });
|
|
223
255
|
return { repo, query: { efficiency } };
|
|
224
256
|
}
|
|
225
257
|
if (opts.contextAb) { // forge#4275: run logs only
|
|
@@ -276,10 +308,12 @@ export async function runStats(argv, deps = {}) {
|
|
|
276
308
|
const report = await buildReport(opts, { onProgress: makeProgressPrinter(), ...deps });
|
|
277
309
|
if (report.query?.lane_profile && !opts.json) { process.stdout.write(`${renderLaneProfiles(report.query.lane_profile)}\n`); return 0; }
|
|
278
310
|
if (report.query?.tiers && !opts.json) { process.stdout.write(`${renderTiers(report.query.tiers)}\n`); return 0; }
|
|
311
|
+
if (report.query?.pool && !opts.json) { process.stdout.write(`${formatPoolLines(report.query.pool).join("\n")}\n`); return 0; }
|
|
279
312
|
if (report.query?.efficiency && !opts.json) {
|
|
280
313
|
const e = report.query.efficiency;
|
|
281
314
|
const lines = [...(e.batch ? formatEfficiencyLines(e.batch, `batch ${e.batch.batch ?? opts.batch}`) : []),
|
|
282
|
-
...Object.entries(e.windows).flatMap(([w, m]) => formatEfficiencyLines(m, `${w}, ${m.batches} batch${m.batches === 1 ? "" : "es"}`))
|
|
315
|
+
...Object.entries(e.windows).flatMap(([w, m]) => formatEfficiencyLines(m, `${w}, ${m.batches} batch${m.batches === 1 ? "" : "es"}`)),
|
|
316
|
+
...(e.compare ? formatCompareLines(e.compare) : [])];
|
|
283
317
|
process.stdout.write(`${lines.join("\n")}\n`);
|
|
284
318
|
return 0;
|
|
285
319
|
}
|
|
@@ -58,7 +58,7 @@ export function summariseLog(events, fallbackIssue = null) {
|
|
|
58
58
|
const usage = ev.filter((e) => e.event === "PHASE_ATTEMPT_USAGE");
|
|
59
59
|
const builderCalls = usage.filter((e) => e.phase === "implement");
|
|
60
60
|
const reviewCalls = usage.filter((e) => e.phase === "review").length;
|
|
61
|
-
const contextPack = { mode: term?.context_pack ?? cp?.mode ?? "off", laneType: term?.lane_type ?? cp?.lane_type ?? null, ab: (term?.context_ab ?? cp?.context_ab) === true };
|
|
61
|
+
const contextPack = { mode: term?.context_pack ?? cp?.mode ?? "off", laneType: term?.lane_type ?? cp?.lane_type ?? null, ab: (term?.context_ab ?? cp?.context_ab) === true, light: term?.light === true };
|
|
62
62
|
const startTs = start.ts ?? null;
|
|
63
63
|
const endTs = finished ? ev[t].ts ?? null : null;
|
|
64
64
|
const a = Date.parse(startTs ?? ""); const b = Date.parse(endTs ?? "");
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
/**
|
|
4
|
+
* #4549: pool utilisation for `forgedock stats --pool`: remote lane minutes, lanes and share per pool agent, from the
|
|
5
|
+
* repo's local batch report.json rows (`sandbox.by_agent`, written by the orchestrator). No network, no model.
|
|
6
|
+
* Live capacity stays in `forgedock pool status`.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { listBatches } from "../lean/batch.mjs";
|
|
11
|
+
import { sameRepo, windowLabel } from "./efficiency.mjs";
|
|
12
|
+
|
|
13
|
+
const readJson = (f) => { try { return JSON.parse(readFileSync(f, "utf8")); } catch { return null; } };
|
|
14
|
+
const round1 = (x) => Math.round(x * 10) / 10;
|
|
15
|
+
|
|
16
|
+
/** Per window: the repo's batches created in it; an issue in several batches counts once (latest row). */
|
|
17
|
+
export function poolUtilisation({ batchRoot, repo = null, windows, now = Date.now() }) {
|
|
18
|
+
const batches = (existsSync(batchRoot) ? listBatches(batchRoot) : []).map((id) => {
|
|
19
|
+
const dir = join(batchRoot, id), plan = readJson(join(dir, "plan.json")) ?? {};
|
|
20
|
+
return { dir, plan, created: Date.parse(plan.created ?? "") };
|
|
21
|
+
}).filter((b) => Number.isFinite(b.created) && (!repo || sameRepo(b.plan.repo, repo))).sort((a, b) => a.created - b.created);
|
|
22
|
+
const out = {};
|
|
23
|
+
for (const w of windows) {
|
|
24
|
+
const inWin = batches.filter((b) => b.created <= now && now - b.created <= w);
|
|
25
|
+
const latest = new Map();
|
|
26
|
+
for (const b of inWin) for (const row of readJson(join(b.dir, "report.json"))?.rows ?? []) if (Number.isSafeInteger(row?.issue)) latest.set(row.issue, row);
|
|
27
|
+
const agents = {};
|
|
28
|
+
let remoteLanes = 0, minutes = 0;
|
|
29
|
+
for (const row of latest.values()) {
|
|
30
|
+
if (!row.sandbox) continue;
|
|
31
|
+
remoteLanes++; minutes += Number(row.sandbox.minutes) || 0;
|
|
32
|
+
for (const [a, m] of Object.entries(row.sandbox.by_agent ?? {})) {
|
|
33
|
+
const e = (agents[a] ??= { minutes: 0, lanes: 0 });
|
|
34
|
+
e.minutes += Number(m) || 0; e.lanes++;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
const attributed = Object.values(agents).reduce((s, e) => s + e.minutes, 0);
|
|
38
|
+
for (const e of Object.values(agents)) { e.share = attributed > 0 ? Math.round((e.minutes / attributed) * 1000) / 1000 : null; e.minutes = round1(e.minutes); }
|
|
39
|
+
out[windowLabel(w)] = { batches: inWin.length, remote_lanes: remoteLanes, minutes: round1(minutes), agents };
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function formatPoolLines(windows) {
|
|
45
|
+
return Object.entries(windows).flatMap(([w, m]) => {
|
|
46
|
+
const head = `pool ${w}: ${m.batches} batch${m.batches === 1 ? "" : "es"}, ${m.remote_lanes} remote lane${m.remote_lanes === 1 ? "" : "s"}, ${m.minutes.toFixed(1)} min`;
|
|
47
|
+
const rows = Object.entries(m.agents).sort((a, b) => b[1].minutes - a[1].minutes || a[0].localeCompare(b[0]))
|
|
48
|
+
.map(([a, e]) => ` ${a}: ${e.minutes.toFixed(1)} min, ${e.lanes} lane${e.lanes === 1 ? "" : "s"}${e.share == null ? "" : `, ${Math.round(e.share * 100)}% of pool`}`);
|
|
49
|
+
return [head, ...(rows.length ? rows : [" per-agent minutes: n/a"])];
|
|
50
|
+
});
|
|
51
|
+
}
|