forgedock 1.33.0 → 1.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/lean/conformance.mjs +26 -3
- package/bin/lean/conformance.test.mjs +21 -0
- package/bin/lean/prompts.mjs +2 -2
- package/bin/review-staging.mjs +14 -2
- package/bin/stats/efficiency.mjs +103 -17
- package/bin/stats/efficiency.test.mjs +103 -4
- package/bin/stats/index.mjs +27 -5
- package/bin/stats/pool.mjs +2 -2
- package/commands/review-pr-agents/protocols.md +1 -1
- package/commands/review-pr-staging.md +1 -1
- package/docs/STATS.md +29 -1
- package/package.json +1 -1
- package/scripts/review-finalize.sh +17 -1
- package/scripts/review-finalize.test.sh +41 -0
- package/scripts/review-pr-domain-map.test.sh +2 -2
- package/scripts/review-pr-staging-scope.test.sh +4 -2
- package/scripts/review-prepare.sh +62 -27
- package/scripts/review-prepare.test.sh +53 -0
- package/scripts/review-staging-pre.sh +29 -9
- package/scripts/review-staging-pre.test.sh +18 -0
package/bin/lean/conformance.mjs
CHANGED
|
@@ -264,6 +264,28 @@ export function validateContract(c, issueText, { repro = false } = {}) {
|
|
|
264
264
|
/** forge#4519: a shell pattern as in scripts/hotspots.sh `case` (`*` crosses `/`, `?` one character), anchored. */
|
|
265
265
|
const shellGlobRe = (g) => new RegExp(`^${String(g).replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".")}$`);
|
|
266
266
|
|
|
267
|
+
const AMEND_KINDS = ["add-deliverable", "add-requirement", ...TRACKED];
|
|
268
|
+
const AMEND_SLIPS = { "amend-requirement": "defer", limitation: "defer" };
|
|
269
|
+
|
|
270
|
+
/** forge#4574: common builder slips: `type` for `kind`, amend-requirement/limitation as defer, an `R#` in `id` as the requirement. */
|
|
271
|
+
function normalizeAmendment(a) {
|
|
272
|
+
if (!a || typeof a !== "object" || Array.isArray(a)) return a;
|
|
273
|
+
const n = { ...a };
|
|
274
|
+
if (blank(n.kind) && !blank(n.type)) n.kind = n.type;
|
|
275
|
+
if (Object.hasOwn(AMEND_SLIPS, n.kind)) n.kind = AMEND_SLIPS[n.kind];
|
|
276
|
+
if (TRACKED.includes(n.kind) && blank(n.requirement) && /^R\d+$/.test(String(n.id ?? ""))) n.requirement = n.id;
|
|
277
|
+
return n;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** The rejection reason, naming the valid kinds or the missing field so the builder's correction round can fix it. */
|
|
281
|
+
function amendmentProblem(a, tag) {
|
|
282
|
+
const kind = a?.kind;
|
|
283
|
+
const need = kind === "add-deliverable" ? "needs a path" : kind === "add-requirement" ? "needs requirement {id, quote}"
|
|
284
|
+
: TRACKED.includes(kind) ? "needs requirement: \"R#\"" : null;
|
|
285
|
+
const why = need ? `amendment kind '${kind}' ${need}` : `unknown kind '${kind ?? ""}'; valid kinds are ${AMEND_KINDS.join(", ")}`;
|
|
286
|
+
return `unknown or incomplete amendment: ${why}: ${tag}`;
|
|
287
|
+
}
|
|
288
|
+
|
|
267
289
|
/**
|
|
268
290
|
* Apply result.json amendments to the frozen contract. Kinds: add-deliverable {path}, add-requirement {requirement},
|
|
269
291
|
* defer|post-deploy {requirement: id}. Every amendment needs a why. Returns {contract, applied[], toTrack[], problems[]}.
|
|
@@ -275,8 +297,9 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
|
|
|
275
297
|
c.deliverables = c.deliverables ?? [];
|
|
276
298
|
const applied = [], toTrack = [], problems = [];
|
|
277
299
|
const genRes = (Array.isArray(generatedPaths) ? generatedPaths : []).filter((g) => typeof g === "string" && g).map(shellGlobRe);
|
|
278
|
-
for (const
|
|
279
|
-
const
|
|
300
|
+
for (const raw of Array.isArray(amendments) ? amendments : []) {
|
|
301
|
+
const a = normalizeAmendment(raw);
|
|
302
|
+
const tag = JSON.stringify(raw ?? null).slice(0, 400);
|
|
280
303
|
if (blank(a?.why)) { problems.push(`amendment without why: ${tag}`); continue; }
|
|
281
304
|
if (a.kind === "add-deliverable" && !blank(a.path)) {
|
|
282
305
|
if (genRes.some((re) => re.test(cleanPath(a.path)))) {
|
|
@@ -301,7 +324,7 @@ export function applyAmendments(frozen, amendments, issueText = null, { generate
|
|
|
301
324
|
const r = c.requirements.find((x) => x.id === a.requirement);
|
|
302
325
|
if (!r) { problems.push(`amendment names unknown requirement ${a.requirement}`); continue; }
|
|
303
326
|
if (!TRACKED.includes(r.disposition)) { r.disposition = a.kind; r.why = a.why; r.amended = true; toTrack.push(r); }
|
|
304
|
-
} else { problems.push(
|
|
327
|
+
} else { problems.push(amendmentProblem(a, tag)); continue; }
|
|
305
328
|
applied.push({ kind: a.kind, target: a.path ?? a.requirement?.id ?? a.requirement, why: String(a.why).trim() });
|
|
306
329
|
}
|
|
307
330
|
return { contract: c, applied, toTrack, problems };
|
|
@@ -60,6 +60,27 @@ describe("amendments", () => {
|
|
|
60
60
|
});
|
|
61
61
|
});
|
|
62
62
|
|
|
63
|
+
describe("#4574 amendment slips", () => {
|
|
64
|
+
it("type for kind, amend-requirement and limitation apply as defer; an R# id fills requirement", () => {
|
|
65
|
+
const a = applyAmendments(C, [{ kind: "amend-requirement", id: "R2", why: "stored hashed" }, { id: "A1", requirement: "R1", type: "limitation", why: "plugin limit" }]);
|
|
66
|
+
assert.deepEqual(a.problems, []);
|
|
67
|
+
assert.deepEqual(a.toTrack.map((r) => r.id).sort(), ["R1", "R2"]);
|
|
68
|
+
assert.ok(a.contract.requirements.every((r) => r.disposition === "defer"));
|
|
69
|
+
});
|
|
70
|
+
it("an unknown kind names the valid kinds and shows the full amendment", () => {
|
|
71
|
+
const why = "x".repeat(300);
|
|
72
|
+
const a = applyAmendments(C, [{ kind: "rewrite", why }]);
|
|
73
|
+
assert.match(a.problems[0], /unknown kind 'rewrite'; valid kinds are add-deliverable, add-requirement, defer, post-deploy/);
|
|
74
|
+
assert.ok(a.problems[0].includes(why));
|
|
75
|
+
});
|
|
76
|
+
it("an Object.prototype key as kind is an unknown kind, not a slip", () => {
|
|
77
|
+
assert.match(applyAmendments(C, [{ kind: "constructor", why: "w" }]).problems[0], /unknown kind 'constructor'; valid kinds are add-deliverable/);
|
|
78
|
+
});
|
|
79
|
+
it("an incomplete known kind names the missing field", () => {
|
|
80
|
+
assert.match(applyAmendments(C, [{ kind: "defer", why: "w" }]).problems[0], /'defer' needs requirement/);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
63
84
|
describe("precheck (deterministic, no reviewer)", () => {
|
|
64
85
|
const res = { requirements: [{ id: "R1", status: "implemented" }, { id: "R2", status: "implemented" }], validation: [{ id: "V1", exit: 0 }] };
|
|
65
86
|
it("passes a complete, validated, in-scope change", () => assert.deepEqual(precheck({ contract: C, result: res, changedFiles: files }), { ok: true, failures: [], superseded: [], ciCovered: [] }));
|
package/bin/lean/prompts.mjs
CHANGED
|
@@ -174,7 +174,7 @@ ${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${factsRule(facts)}
|
|
|
174
174
|
The contract changes only through typed amendments in ${RESULT_FILE}, each with a why, which the engine posts on the issue:
|
|
175
175
|
- {"kind":"add-deliverable","path":"<file>","why":""}: a file you must change that the contract did not list.
|
|
176
176
|
- {"kind":"add-requirement","requirement":{"id":"R9","quote":"<verbatim from the issue>","disposition":"deliver|defer|post-deploy"},"why":""}: an issue requirement the contract missed.
|
|
177
|
-
- {"kind":"defer|post-deploy","requirement":"R#","why":""}: a contracted requirement that becomes separate work or a post-deploy check. The engine files it as a tracked follow-up.
|
|
177
|
+
- {"kind":"defer|post-deploy","requirement":"R#","why":""}: a contracted requirement that becomes separate work or a post-deploy check. The engine files it as a tracked follow-up. Not "type"; other kinds are rejected.
|
|
178
178
|
Never report a deliver requirement as "deferred" or "follow-up" without an amendment.
|
|
179
179
|
A new dependency between workspace packages in a package.json that no deliverable names is blocked by the engine: add it through add-deliverable with a why naming the package, or do not add it.
|
|
180
180
|
If the repository has scripts/check-devdocs-freshness.sh, a behaviour change under the paths it checks must update the matching dossier (devdocs/modules/*.md, per devdocs/index.yaml) in the same change. Run it against the base before writing the result; the engine runs it too.
|
|
@@ -280,7 +280,7 @@ export function conformanceResumePrompt(failures, mode = "local", lane = "") {
|
|
|
280
280
|
const baseMoved = lane && failures.some((f) => f?.kind === "validate_failed")
|
|
281
281
|
? `\n\nIf a validate command fails only because origin/${lane} moved on after this branch started (the change itself is fine), bring the branch up to date with \`git merge --no-edit origin/${lane}\` (already fetched; never rebase or force), resolve any conflicts, then re-run every validate command and record the results.`
|
|
282
282
|
: "";
|
|
283
|
-
return `The change does not conform to the frozen contract and the issue:\n${failures.slice(0, 30).map((f) => `- ${f.code}: ${String(f.detail).slice(0,
|
|
283
|
+
return `The change does not conform to the frozen contract and the issue:\n${failures.slice(0, 30).map((f) => `- ${f.code}: ${String(f.detail).slice(0, 900)}`).join("\n")}${baseMoved}\n\nFix each item in the code, or record a typed amendment with a why in ${RESULT_FILE}. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
|
|
284
284
|
}
|
|
285
285
|
|
|
286
286
|
/** forge#4159: the one resume for package.json edges between workspace packages the contract does not name. */
|
package/bin/review-staging.mjs
CHANGED
|
@@ -85,6 +85,9 @@ export function parseArgs(argv) {
|
|
|
85
85
|
return o;
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
/** Largest diff slice substituted into a brief; a larger one is referenced by its file (#4573). */
|
|
89
|
+
const INLINE_SLICE_MAX = 16000;
|
|
90
|
+
|
|
88
91
|
/** Replace every `[KEY]` in one pass: a substituted value is never rescanned, so diff text cannot inject placeholders. */
|
|
89
92
|
export function renderBrief(template, values) {
|
|
90
93
|
return template.replace(/\[([A-Z][A-Z0-9_]*)\]/g, (m, k) => (Object.hasOwn(values, k) && values[k] != null ? String(values[k]) : m));
|
|
@@ -273,6 +276,8 @@ async function main(argv, io) {
|
|
|
273
276
|
const agents = ctx.roster?.agents?.length ? ctx.roster.agents : ["security"];
|
|
274
277
|
const launchedFile = join(sd, `${pr}_launched_review_agents`);
|
|
275
278
|
io.writeFile(launchedFile, "");
|
|
279
|
+
// The head this list describes: review-prepare trusts the list as --delta proof only for this head.
|
|
280
|
+
io.writeFile(`${launchedFile}.head`, `${head}\n`);
|
|
276
281
|
const briefDir = join(sd, `review-staging-${pr}`);
|
|
277
282
|
io.mkdir(briefDir);
|
|
278
283
|
const briefs = [];
|
|
@@ -282,6 +287,8 @@ async function main(argv, io) {
|
|
|
282
287
|
const slice = ctx.slices?.[domain];
|
|
283
288
|
let diff = "";
|
|
284
289
|
try { diff = slice?.diff_file ? io.readFile(slice.diff_file) : ""; } catch { diff = ""; }
|
|
290
|
+
// #4573: the persona templates repeat [DOMAIN_DIFF_SLICE] many times, so only a small slice is inlined; a large one is a pointer to the on-disk slice file.
|
|
291
|
+
if (diff.length > INLINE_SLICE_MAX) diff = `(diff slice too large to inline: ${diff.length} chars. Read the file ${slice.diff_file} instead, e.g. grep -n or sed -n 'X,Yp' on it, never cat it whole. Files listed as OMITTED at its top are read from the head checkout.)`;
|
|
285
292
|
const values = {
|
|
286
293
|
PR_NUMBER: pr, REVIEW_SHA: head, REVIEW_SHA_SHORT: sha7, AGENT_DOMAIN: domain, GH_REPO: c.ghRepo,
|
|
287
294
|
TITLE: ctx.pr?.title ?? "", FILE_LIST: (slice?.paths ?? ctx.full_files ?? []).join(" "), SERVICES: (ctx.classification?.services ?? []).join(" "),
|
|
@@ -405,7 +412,11 @@ async function finalize(a, io) {
|
|
|
405
412
|
if (!/^[A-Za-z0-9._/-]+$/.test(codeBranch)) codeBranch = c.stagingBranch;
|
|
406
413
|
|
|
407
414
|
const findings = readJson(io, join(outDir, "findings.json"))?.findings ?? [];
|
|
408
|
-
|
|
415
|
+
// A finding whose own filed issue is already closed (file-prepare: skip-resolved, delta pass) is resolved: it never blocks (#4569).
|
|
416
|
+
const resolvedBy = new Map(worklist.items.filter((i) => i.action === "skip-resolved").map((i) => [i.id, i.dedup_of]));
|
|
417
|
+
const resolved = findings.filter((f) => resolvedBy.has(f.id) && upper(f.confidence) === "CONFIRMED" && ["CRITICAL", "HIGH"].includes(upper(f.severity)))
|
|
418
|
+
.map((f) => ({ id: f.id, severity: upper(f.severity), title: f.title ?? "", issue: resolvedBy.get(f.id) }));
|
|
419
|
+
const confirmed = (sev) => findings.filter((f) => !resolvedBy.has(f.id) && upper(f.confidence) === "CONFIRMED" && upper(f.severity) === sev).length;
|
|
409
420
|
const outOfScope = Number(/out_of_scope=(\d+)/.exec(readText(io, join(outDir, "note-disposition.md")) ?? "")?.[1] ?? 0);
|
|
410
421
|
const counts = {
|
|
411
422
|
findings_kept: Number(x.FINDINGS_KEPT ?? 0), findings_dropped_stale: Number(x.FINDINGS_DROPPED_STALE ?? 0), findings_preexisting: Number(x.FINDINGS_PREEXISTING ?? 0),
|
|
@@ -415,7 +426,7 @@ async function finalize(a, io) {
|
|
|
415
426
|
const items = worklist.items;
|
|
416
427
|
const state = {
|
|
417
428
|
pr, head, bundle, codeBranch, scopeMode, delta, testGate: { verdict: tg.verdict, reason: tg.reason }, counts, notes,
|
|
418
|
-
confirmedCritical: confirmed("CRITICAL"), confirmedHigh: confirmed("HIGH"),
|
|
429
|
+
confirmedCritical: confirmed("CRITICAL"), confirmedHigh: confirmed("HIGH"), resolved,
|
|
419
430
|
toFile: items.filter((i) => i.action === "file"), skipped: items.filter((i) => String(i.action).startsWith("skip-")),
|
|
420
431
|
};
|
|
421
432
|
io.writeFile(join(sd, `${pr}_finalize.json`), JSON.stringify(state));
|
|
@@ -500,6 +511,7 @@ async function summary(a, io) {
|
|
|
500
511
|
`- findings_kept: ${n.findings_kept ?? 0}; dropped_stale: ${n.findings_dropped_stale ?? 0}; pre-existing: ${n.findings_preexisting ?? 0}; notes_demoted_no_scenario: ${n.notes_demoted_no_scenario ?? 0}`,
|
|
501
512
|
`- findings_filed: ${n.findings_filed ?? 0}${filed.length ? ` (${filed.join(", ")})` : ""}; out_of_scope: ${n.out_of_scope ?? 0}`,
|
|
502
513
|
`- notes_fixed: 0; notes_listed: ${n.notes_listed ?? 0}; notes_dropped: ${n.notes_dropped ?? 0}; notes_capped: ${n.notes_capped ?? 0}; notes_reflag: ${n.notes_reflag ?? 0}`,
|
|
514
|
+
`- resolved (excluded from the verdict): ${(st.resolved ?? []).length ? st.resolved.map((r) => `${r.severity} ${r.id} ${r.title} (closed #${r.issue})`.replace(/\s+/g, " ")).join("; ") : "(none)"}`,
|
|
503
515
|
`- deduped: ${deduped.length ? deduped.join("; ") : "(none)"}`,
|
|
504
516
|
...(st.delta?.from ? [`- delta pass: ${String(st.delta.from).slice(0, 7)}..${head.slice(0, 7)} (${(st.delta.files ?? []).length} file(s)); earlier pass at ${st.delta.from} credited (its findings re-read by extract)`] : []),
|
|
505
517
|
"",
|
package/bin/stats/efficiency.mjs
CHANGED
|
@@ -49,6 +49,8 @@ const costOf = (e) => num(e.cost_usd_delta) ?? num(e.cost_usd);
|
|
|
49
49
|
const sum = (xs) => xs.reduce((a, b) => a + (num(b) ?? 0), 0);
|
|
50
50
|
const r = (x, p = 1e4) => (num(x) == null ? null : Math.round(x * p) / p);
|
|
51
51
|
const per = (a, b) => (b > 0 ? a / b : null);
|
|
52
|
+
/** #4491: GitHub slugs are case-insensitive (`--repo o/forgedock` matches a plan's `O/ForgeDock`). */
|
|
53
|
+
export const sameRepo = (a, b) => String(a ?? "").toLowerCase() === String(b ?? "").toLowerCase();
|
|
52
54
|
|
|
53
55
|
function parseLog(text) {
|
|
54
56
|
const out = [];
|
|
@@ -67,13 +69,14 @@ export function issueSlices(events) {
|
|
|
67
69
|
let cur = null;
|
|
68
70
|
for (const e of Array.isArray(events) ? events : []) {
|
|
69
71
|
if (!e || typeof e !== "object") continue;
|
|
70
|
-
if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, shape: "full", attempts: [] }; out.push(cur); continue; }
|
|
72
|
+
if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, shape: "full", conformance: null, attempts: [] }; out.push(cur); continue; }
|
|
71
73
|
if (!cur) continue;
|
|
72
74
|
if (e.event === "LIGHT_DONE") cur.shape = "light";
|
|
75
|
+
else if (e.event === "CONFORMANCE" && typeof e.state === "string") cur.conformance = e.state; // #4491: the slice's last verdict
|
|
73
76
|
else if (e.event === "LIGHT_UPGRADE") cur.shape = "upgraded";
|
|
74
77
|
if (e.event === "PHASE_ATTEMPT_USAGE") {
|
|
75
78
|
cur.attempts.push({ phase: e.phase ?? "other", trigger: e.trigger ?? null, restart_of: e.restart_of ?? null, model: e.model ?? null,
|
|
76
|
-
turns: num(e.turns), cacheRead: num(e.usage?.cache_read_input_tokens), cost: costOf(e),
|
|
79
|
+
turns: num(e.turns), cacheRead: num(e.usage?.cache_read_input_tokens), cost: costOf(e), durationMs: num(e.duration_ms), // #4491
|
|
77
80
|
...(e.tools && typeof e.tools === "object" ? { calls: num(e.tools.calls), toolTurns: num(e.tools.turns), rereads: num(e.tools.rereads) } : {}) }); // #4410
|
|
78
81
|
} else if (e.event === "RUN_TERMINAL") {
|
|
79
82
|
cur.terminal = { reason: e.reason ?? null, code: e.code ?? null, ts: e.ts ?? null, size: SIZES.includes(e.turns?.size) ? e.turns.size : null };
|
|
@@ -137,10 +140,17 @@ const median = (xs) => {
|
|
|
137
140
|
return s.length % 2 ? s[h] : r((s[h - 1] + s[h]) / 2, 100);
|
|
138
141
|
};
|
|
139
142
|
|
|
143
|
+
/** #4491: a lane's implement wall in minutes, the sum of its implement attempts' duration_ms; null when none carries it. */
|
|
144
|
+
function laneWall(impl) {
|
|
145
|
+
const xs = impl.filter((a) => a.durationMs != null);
|
|
146
|
+
return xs.length ? r(sum(xs.map((a) => a.durationMs)) / 60000, 100) : null;
|
|
147
|
+
}
|
|
148
|
+
const wallP50 = (xs) => median(xs.map((x) => x.implWall).filter((w) => w != null));
|
|
149
|
+
|
|
140
150
|
const group = (xs) => {
|
|
141
151
|
const turns = sum(xs.map((x) => x.implTurns)), cache = sum(xs.map((x) => x.implCache));
|
|
142
152
|
return { n: xs.length, cost_usd: r(sum(xs.map((x) => x.cost)), 1e6), usd_per_merge: r(per(sum(xs.map((x) => x.cost)), xs.length)),
|
|
143
|
-
turns_per_merge: r(per(turns, xs.length), 10), context_per_turn: r(per(cache, turns), 1) };
|
|
153
|
+
turns_per_merge: r(per(turns, xs.length), 10), context_per_turn: r(per(cache, turns), 1), implement_wall_p50_min: wallP50(xs) };
|
|
144
154
|
};
|
|
145
155
|
|
|
146
156
|
/**
|
|
@@ -149,7 +159,7 @@ const group = (xs) => {
|
|
|
149
159
|
*/
|
|
150
160
|
export function efficiencyMetrics({ entries = [], logs = new Map(), issues = null, repo = null, now = 0, notes = [] } = {}) {
|
|
151
161
|
const merged = [];
|
|
152
|
-
const inScope = (row, settleAt) => issueSlices(logs.get(row.issue) ?? []).filter((s) => (!repo || !s.repo || s.repo
|
|
162
|
+
const inScope = (row, settleAt) => issueSlices(logs.get(row.issue) ?? []).filter((s) => (!repo || !s.repo || sameRepo(s.repo, repo)) && (!settleAt || !s.startTs || s.startTs <= settleAt));
|
|
153
163
|
// #4452: light/full split and upgrade rate over every lane of every row (all outcomes); cost is the #4407 delta sum
|
|
154
164
|
const shapes = Object.fromEntries(LANE_SHAPES.map((k) => [k, { n: 0, cost_usd: 0 }]));
|
|
155
165
|
for (const { row, settleAt } of entries) {
|
|
@@ -169,7 +179,7 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
169
179
|
const term = [...slices].reverse().find((s) => s.terminal?.reason === "merged")?.terminal;
|
|
170
180
|
merged.push({ issue: row.issue, slices: slices.length, cost: sum(atts.map((a) => a.cost)), implCost: sum(impl.map((a) => a.cost)),
|
|
171
181
|
reviewCost: sum(atts.filter((a) => a.phase === "review").map((a) => a.cost)), implTurns: sum(impl.map((a) => a.turns)), implCache: sum(impl.map((a) => a.cacheRead)),
|
|
172
|
-
size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl), shape: slices.at(-1)?.shape ?? null });
|
|
182
|
+
size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl), shape: slices.at(-1)?.shape ?? null, implWall: laneWall(impl) });
|
|
173
183
|
}
|
|
174
184
|
const all = group(merged);
|
|
175
185
|
const bySize = {};
|
|
@@ -182,6 +192,9 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
182
192
|
}
|
|
183
193
|
}
|
|
184
194
|
const retries = Object.fromEntries(RETRY_CAUSES.map((c) => [c, sum(merged.map((m) => m.retries[c]))]));
|
|
195
|
+
// #4491: merge rate over the rows, conformance pass rate over rows whose lanes logged a verdict (each row's last one)
|
|
196
|
+
const rows = entries.filter(({ row }) => Number.isSafeInteger(row?.issue));
|
|
197
|
+
const verdicts = rows.map(({ row, settleAt }) => inScope(row, settleAt).map((s) => s.conformance).filter(Boolean).at(-1)).filter(Boolean);
|
|
185
198
|
const withTools = merged.filter((m) => m.tools);
|
|
186
199
|
const total = sum(Object.values(retries));
|
|
187
200
|
// no possible action: parked post-deploy follow-ups and promotion-blocked lanes (#4389), over all rows
|
|
@@ -209,6 +222,8 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
209
222
|
merged: merged.length, cost_usd: all.cost_usd, implement_usd: r(sum(merged.map((m) => m.implCost)), 1e6), review_usd: r(sum(merged.map((m) => m.reviewCost)), 1e6),
|
|
210
223
|
usd_per_merge: all.usd_per_merge, turns_per_merge: all.turns_per_merge, context_per_turn: all.context_per_turn,
|
|
211
224
|
implement_turns: sum(merged.map((m) => m.implTurns)), implement_cache_read: sum(merged.map((m) => m.implCache)),
|
|
225
|
+
implement_wall_p50_min: all.implement_wall_p50_min, merge_rate: r(per(merged.length, rows.length)),
|
|
226
|
+
conformance_pass_rate: r(per(verdicts.filter((v) => v === "conforms").length, verdicts.length)),
|
|
212
227
|
by_size: bySize,
|
|
213
228
|
by_model: Object.fromEntries(Object.entries(byModel).map(([k, g]) => [k, { merges: g.merges.size, attempts: g.attempts, turns: g.turns, cost_usd: r(g.cost, 1e6),
|
|
214
229
|
usd_per_merge: r(per(g.cost, g.merges.size)), turns_per_merge: r(per(g.turns, g.merges.size), 10), context_per_turn: r(per(g.cache, g.turns), 1) }])),
|
|
@@ -219,7 +234,8 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
219
234
|
tool_calls_per_turn_p50: median(withTools.map((m) => m.tools.per_turn)), rereads_per_merge: withTools.length ? r(per(sum(withTools.map((m) => m.tools.rereads)), withTools.length), 100) : null,
|
|
220
235
|
per_issue: merged.map((m) => ({ issue: m.issue, slices: m.slices, cost_usd: r(m.cost, 1e6), implement_usd: r(m.implCost, 1e6), implement_turns: m.implTurns,
|
|
221
236
|
implement_cache_read: m.implCache, size: m.size, retries: m.retries, ...(m.shape ? { shape: m.shape } : {}), ...(m.info ? { quality: issueQuality(m.info.body).tag } : {}),
|
|
222
|
-
...(m.tools ? { implement_tool_calls_per_turn: m.tools.per_turn, implement_rereads: m.tools.rereads } : {})
|
|
237
|
+
...(m.tools ? { implement_tool_calls_per_turn: m.tools.per_turn, implement_rereads: m.tools.rereads } : {}),
|
|
238
|
+
...(m.implWall != null ? { implement_wall_min: m.implWall } : {}) })),
|
|
223
239
|
notes,
|
|
224
240
|
};
|
|
225
241
|
}
|
|
@@ -274,26 +290,77 @@ export async function batchEfficiency({ batchDir, runsDir, repo = null, gh = nul
|
|
|
274
290
|
return { batch: plan.id ?? null, ...(await computeScope({ entries, runsDir, repo: repo ?? plan.repo ?? null, gh, now })) };
|
|
275
291
|
}
|
|
276
292
|
|
|
277
|
-
/**
|
|
278
|
-
|
|
293
|
+
/** The repo's batches by creation time, and the repo slug as the plans spell it (run-log file names are case-sensitive, #4491). */
|
|
294
|
+
function repoBatches(batchRoot, repo) {
|
|
279
295
|
const batches = (existsSync(batchRoot) ? listBatches(batchRoot) : []).map((id) => {
|
|
280
296
|
const dir = join(batchRoot, id), plan = readJson(join(dir, "plan.json")) ?? {};
|
|
281
297
|
return { dir, plan, created: Date.parse(plan.created ?? "") };
|
|
282
|
-
}).filter((b) => Number.isFinite(b.created) && (!repo || b.plan.repo
|
|
298
|
+
}).filter((b) => Number.isFinite(b.created) && (!repo || sameRepo(b.plan.repo, repo))).sort((a, b) => a.created - b.created);
|
|
299
|
+
return { batches, repo: repo ? (batches.find((b) => typeof b.plan.repo === "string")?.plan.repo ?? repo) : repo };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** Rows of the batches created in [from, to]; an issue in several batches counts once (latest row). */
|
|
303
|
+
function cohortEntries(batches, from, to, toInclusive = true) {
|
|
304
|
+
const inside = batches.filter((b) => b.created >= from && (toInclusive ? b.created <= to : b.created < to));
|
|
305
|
+
const latest = new Map();
|
|
306
|
+
for (const b of inside) {
|
|
307
|
+
const at = settleTimes(readLogRows(b.dir));
|
|
308
|
+
for (const row of batchRows(b.dir)) if (Number.isSafeInteger(row?.issue)) latest.set(row.issue, { row, settleAt: at.get(row.issue) ?? null });
|
|
309
|
+
}
|
|
310
|
+
return { batches: inside.length, entries: [...latest.values()] };
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Trailing windows: the union of the repo's batches created in each window; an issue in several batches counts once (latest row). */
|
|
314
|
+
export async function windowEfficiency({ batchRoot, runsDir, repo = null, gh = null, now = Date.now(), windows = [WEEK_MS] }) {
|
|
315
|
+
const { batches, repo: slug } = repoBatches(batchRoot, repo);
|
|
283
316
|
const out = {};
|
|
284
317
|
for (const w of windows) {
|
|
285
|
-
const
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
318
|
+
const c = cohortEntries(batches, now - w, now);
|
|
319
|
+
out[windowLabel(w)] = { batches: c.batches, ...(await computeScope({ entries: c.entries, runsDir, repo: slug, gh, now })) };
|
|
320
|
+
}
|
|
321
|
+
return out;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
export const COMPARE_KEYS = Object.freeze(["turns_per_merge", "implement_wall_p50_min", "tool_calls_per_turn_p50", "rereads_per_merge", "merge_rate", "conformance_pass_rate"]);
|
|
325
|
+
const delta = (a, b) => Object.fromEntries(COMPARE_KEYS.map((k) => [k, num(a?.[k]) == null || num(b?.[k]) == null ? null : r(b[k] - a[k], 100)]));
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* #4491: full-shape merged lanes per size bucket present in both cohorts (light lanes, #4452, cut turns on their own).
|
|
329
|
+
* A merged row with no in-scope run log carries no shape and drops out, so its zero turns do not pull a bucket down.
|
|
330
|
+
*/
|
|
331
|
+
function matchedCohorts(pre, post) {
|
|
332
|
+
const full = (m) => (m.per_issue ?? []).filter((x) => x.shape === "full");
|
|
333
|
+
const bySize = (xs) => xs.reduce((o, x) => ((o[x.size] ??= []).push(x), o), {});
|
|
334
|
+
const a = bySize(full(pre)), b = bySize(full(post)), out = {};
|
|
335
|
+
const g = (xs) => {
|
|
336
|
+
const tools = xs.filter((x) => x.implement_tool_calls_per_turn != null);
|
|
337
|
+
return { n: xs.length, turns_per_merge: r(per(sum(xs.map((x) => x.implement_turns)), xs.length), 10),
|
|
338
|
+
implement_wall_p50_min: median(xs.map((x) => x.implement_wall_min).filter((w) => w != null)),
|
|
339
|
+
tool_calls_per_turn_p50: median(tools.map((x) => x.implement_tool_calls_per_turn)),
|
|
340
|
+
rereads_per_merge: tools.length ? r(per(sum(tools.map((x) => x.implement_rereads)), tools.length), 100) : null };
|
|
341
|
+
};
|
|
342
|
+
for (const size of Object.keys(a).sort()) {
|
|
343
|
+
if (!b[size]) continue;
|
|
344
|
+
const pa = g(a[size]), pb = g(b[size]);
|
|
345
|
+
out[size] = { pre: pa, post: pb, delta: delta(pa, pb) };
|
|
293
346
|
}
|
|
294
347
|
return out;
|
|
295
348
|
}
|
|
296
349
|
|
|
350
|
+
/**
|
|
351
|
+
* #4491: before/after A/B around a deploy `pivot` (ms): post = the repo's batches created in [pivot, min(now, pivot + span)],
|
|
352
|
+
* pre = the same span ending just before the pivot. `span` defaults to the time since the pivot, so both cohorts are equally long.
|
|
353
|
+
*/
|
|
354
|
+
export async function compareEfficiency({ batchRoot, runsDir, repo = null, gh = null, pivot, now = Date.now(), span = null }) {
|
|
355
|
+
if (!Number.isFinite(pivot) || pivot > now) throw new Error("compare: the pivot must be a time in the past");
|
|
356
|
+
const s = Math.max(0, Math.min(Number.isFinite(span) ? span : Infinity, now - pivot));
|
|
357
|
+
const { batches, repo: slug } = repoBatches(batchRoot, repo);
|
|
358
|
+
const a = cohortEntries(batches, pivot - s, pivot, false), b = cohortEntries(batches, pivot, pivot + s);
|
|
359
|
+
const pre = { batches: a.batches, ...(await computeScope({ entries: a.entries, runsDir, repo: slug, gh, now })) };
|
|
360
|
+
const post = { batches: b.batches, ...(await computeScope({ entries: b.entries, runsDir, repo: slug, gh, now })) };
|
|
361
|
+
return { pivot: new Date(pivot).toISOString(), span_days: r(s / 86400000, 100), pre, post, delta: delta(pre, post), matched: matchedCohorts(pre, post) };
|
|
362
|
+
}
|
|
363
|
+
|
|
297
364
|
export const windowLabel = (ms) => `${Math.round(ms / 86400000)}d`;
|
|
298
365
|
/** "7d,30d" -> [ms]; throws on anything else. */
|
|
299
366
|
export function parseWindows(s) {
|
|
@@ -331,5 +398,24 @@ export function formatEfficiencyLines(m, label = "") {
|
|
|
331
398
|
const ln = m.lanes; // #4452
|
|
332
399
|
lines.push(ln ? `lanes: ${LANE_SHAPES.map((k) => `${k} ${ln[k]?.n ?? 0} (${usd(ln[k]?.cost_usd ?? 0)})`).join(", ")}; upgrade rate ${ln.upgrade_rate == null ? "n/a"
|
|
333
400
|
: `${Math.round(ln.upgrade_rate * 100)}% (${ln.upgraded?.n ?? 0}/${(ln.light?.n ?? 0) + (ln.upgraded?.n ?? 0)})`}` : "lanes: n/a");
|
|
401
|
+
lines.push(rates(m)); // #4491
|
|
402
|
+
return lines;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
const pct = (x) => (num(x) == null ? "n/a" : `${Math.round(x * 100)}%`);
|
|
406
|
+
const rates = (m) => `implement wall p50 ${m.implement_wall_p50_min == null ? "n/a" : `${f(m.implement_wall_p50_min, 1)} min`}, merge rate ${pct(m.merge_rate)}, conformance pass ${pct(m.conformance_pass_rate)}`;
|
|
407
|
+
const sign = (x, d, unit = "") => (num(x) == null ? "n/a" : `${x > 0 ? "+" : ""}${x.toFixed(d)}${unit}`);
|
|
408
|
+
const deltaLine = (d, withRates = true) => `turns/merge ${sign(d.turns_per_merge, 1)}, implement wall p50 ${sign(d.implement_wall_p50_min, 1, " min")}, calls/turn p50 ${sign(d.tool_calls_per_turn_p50, 2)}, `
|
|
409
|
+
+ `re-reads/merge ${sign(d.rereads_per_merge, 2)}${withRates ? `, merge rate ${sign(d.merge_rate == null ? null : d.merge_rate * 100, 0, " pts")}, conformance pass ${sign(d.conformance_pass_rate == null ? null : d.conformance_pass_rate * 100, 0, " pts")}` : ""}`;
|
|
410
|
+
const cohortLine = (k, m) => `compare ${k}: n=${m.n ?? m.merged} turns/merge ${f(m.turns_per_merge, 1)}, implement wall p50 ${m.implement_wall_p50_min == null ? "n/a" : `${f(m.implement_wall_p50_min, 1)} min`}, `
|
|
411
|
+
+ `calls/turn p50 ${f(m.tool_calls_per_turn_p50)}, re-reads/merge ${f(m.rereads_per_merge)}${"merge_rate" in m ? `, merge rate ${pct(m.merge_rate)}, conformance pass ${pct(m.conformance_pass_rate)}` : ""}`;
|
|
412
|
+
|
|
413
|
+
/** #4491: the `--compare` report: pre/post cohorts, their delta, then the full-shape per-size matched cohorts. */
|
|
414
|
+
export function formatCompareLines(c) {
|
|
415
|
+
const lines = [`compare: pivot ${c.pivot}, ${f(c.span_days, 1)} days each side (pre ${c.pre.batches} batches, post ${c.post.batches} batches)`,
|
|
416
|
+
cohortLine("pre", c.pre), cohortLine("post", c.post), `compare delta: ${deltaLine(c.delta)}`];
|
|
417
|
+
const sizes = Object.entries(c.matched ?? {});
|
|
418
|
+
if (!sizes.length) lines.push("compare matched (full lanes per size): n/a");
|
|
419
|
+
for (const [size, g] of sizes) lines.push(`compare matched ${size}: pre n=${g.pre.n} post n=${g.post.n}; delta ${deltaLine(g.delta, false)}`);
|
|
334
420
|
return lines;
|
|
335
421
|
}
|
|
@@ -9,9 +9,10 @@ import { tmpDir } from "../tests/_tmp.mjs";
|
|
|
9
9
|
import { TRIGGERS } from "../lean/fixround.mjs";
|
|
10
10
|
import {
|
|
11
11
|
TRIGGER_CAUSE, RETRY_CAUSES, WEEK_MS, codeCause, issueSlices, sliceRetries, issueQuality, acceptanceOf, efficiencyMetrics,
|
|
12
|
-
batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows,
|
|
12
|
+
batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows, compareEfficiency, formatCompareLines,
|
|
13
13
|
} from "./efficiency.mjs";
|
|
14
|
-
import { parseArgs, runStats } from "./index.mjs";
|
|
14
|
+
import { comparePivot, parseArgs, runStats } from "./index.mjs";
|
|
15
|
+
import { runLogPath } from "../engine/runlog.mjs";
|
|
15
16
|
|
|
16
17
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
17
18
|
const FX = join(here, "fixtures", "efficiency-20261022");
|
|
@@ -115,7 +116,8 @@ describe("#4402 efficiencyMetrics", () => {
|
|
|
115
116
|
});
|
|
116
117
|
it("renders n/a, never NaN, for an empty cohort", () => {
|
|
117
118
|
const lines = formatEfficiencyLines(efficiencyMetrics({}));
|
|
118
|
-
assert.equal(lines.length,
|
|
119
|
+
assert.equal(lines.length, 10);
|
|
120
|
+
assert.equal(lines[9], "implement wall p50 n/a, merge rate n/a, conformance pass n/a"); // #4491, appended
|
|
119
121
|
assert.ok(!lines.some((l) => /NaN|undefined/.test(l)), lines.join("\n"));
|
|
120
122
|
assert.match(lines[0], /^efficiency: merged 0, n\/a\/merge/);
|
|
121
123
|
assert.equal(lines[6], "acceptance (7d after promotion): n/a");
|
|
@@ -152,7 +154,7 @@ describe("#4452 light/full split and upgrade rate", () => {
|
|
|
152
154
|
});
|
|
153
155
|
it("the batch report prints the split and the upgrade rate", () => {
|
|
154
156
|
const lines = formatEfficiencyLines(m);
|
|
155
|
-
assert.equal(lines.
|
|
157
|
+
assert.equal(lines.find((l) => l.startsWith("lanes:")), "lanes: light 2 ($0.65), upgraded 2 ($0.90), full 1 ($1.00); upgrade rate 50% (2/4)");
|
|
156
158
|
});
|
|
157
159
|
});
|
|
158
160
|
|
|
@@ -264,3 +266,100 @@ describe("#4402 forgedock stats --efficiency", () => {
|
|
|
264
266
|
assert.match(text, /^no action: 32 lanes, \$10\.13 \(blocked-promotion 32\)$/m);
|
|
265
267
|
});
|
|
266
268
|
});
|
|
269
|
+
|
|
270
|
+
describe("#4491 before/after compare", () => {
|
|
271
|
+
const PIVOT = Date.parse("2026-10-20T00:00:00Z"), SLUG = "Own/Repo";
|
|
272
|
+
const tools = (calls, turns, rereads) => ({ tools: { calls, turns, per_turn: calls / turns, rereads, top: [] } });
|
|
273
|
+
// pre batch (10-18): issues 1, 2 merged (M, full), 3 failed; post batch (10-21): issues 4 merged (M, full), 5 merged light, 6 merged (L, full);
|
|
274
|
+
// 8 (pre) and 7 (post) merged with no run log, so both sides hold a size-unknown, shapeless lane
|
|
275
|
+
const lane = (turns, wallMin, t, conf, size = "M", extra = []) => [start("2026-10-01T00:00:00Z", SLUG), att("implement", { trigger: "build", turns, duration_ms: wallMin * 60000, ...t }),
|
|
276
|
+
...extra, ...(conf ? [{ event: "CONFORMANCE", state: conf }] : []), term("merged", "merged", { turns: { size } })];
|
|
277
|
+
const setup = () => {
|
|
278
|
+
const root = tmpDir("eff-cmp-root-"), runs = tmpDir("eff-cmp-runs-");
|
|
279
|
+
const batch = (id, created, rows) => {
|
|
280
|
+
mkdirSync(join(root, id), { recursive: true });
|
|
281
|
+
writeFileSync(join(root, id, "plan.json"), JSON.stringify({ v: 1, id, repo: SLUG, created }));
|
|
282
|
+
writeFileSync(join(root, id, "report.json"), JSON.stringify({ rows }));
|
|
283
|
+
};
|
|
284
|
+
batch("pre", "2026-10-18T00:00:00Z", [{ issue: 1, outcome: "merged" }, { issue: 2, outcome: "merged" }, { issue: 3, outcome: "failed" }, { issue: 8, outcome: "merged" }]);
|
|
285
|
+
batch("old", "2026-10-10T00:00:00Z", [{ issue: 9, outcome: "merged" }]); // outside the 3-day span
|
|
286
|
+
batch("post", "2026-10-21T00:00:00Z", [{ issue: 4, outcome: "merged" }, { issue: 5, outcome: "merged" }, { issue: 6, outcome: "merged" }, { issue: 7, outcome: "merged" }]);
|
|
287
|
+
const logs = {
|
|
288
|
+
1: lane(40, 30, tools(40, 40, 2), "conforms"), 2: lane(60, 50, tools(60, 60, 4), "nonconforming"),
|
|
289
|
+
3: [start("2026-10-01T00:00:00Z", SLUG), att("implement", { trigger: "build" }), { event: "CONFORMANCE", state: "nonconforming" }, term("failed", "ci.red")],
|
|
290
|
+
4: lane(20, 20, tools(30, 20, 0), "conforms"), 5: lane(5, 2, tools(5, 5, 0), "conforms", "S", [{ event: "LIGHT_DONE" }]), 6: lane(30, 25, tools(30, 30, 1), null, "L"),
|
|
291
|
+
9: lane(99, 99, tools(99, 99, 9), "conforms"),
|
|
292
|
+
};
|
|
293
|
+
for (const [n, evs] of Object.entries(logs)) writeFileSync(runLogPath(runs, Number(n), SLUG), evs.map((e) => JSON.stringify(e)).join("\n") + "\n");
|
|
294
|
+
return { root, runs };
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
it("compare splits equal-span cohorts at the pivot with implement wall, merge and conformance rates, and deltas", async () => {
|
|
298
|
+
const { root, runs } = setup();
|
|
299
|
+
const c = await compareEfficiency({ batchRoot: root, runsDir: runs, repo: SLUG, pivot: PIVOT, now: PIVOT + 3 * DAY });
|
|
300
|
+
assert.equal(c.span_days, 3);
|
|
301
|
+
assert.deepEqual([c.pre.batches, c.pre.merged, c.post.batches, c.post.merged], [1, 3, 1, 4]); // 8 and 7 merged without a run log
|
|
302
|
+
assert.equal(c.pre.implement_wall_p50_min, 40); // median of 30 and 50
|
|
303
|
+
assert.equal(c.post.implement_wall_p50_min, 20);
|
|
304
|
+
assert.equal(c.pre.merge_rate, 0.75);
|
|
305
|
+
assert.equal(c.post.merge_rate, 1);
|
|
306
|
+
assert.equal(c.pre.conformance_pass_rate, 0.3333); // 1 of 3 verdicts
|
|
307
|
+
assert.equal(c.post.conformance_pass_rate, 1); // issue 6 has no verdict
|
|
308
|
+
assert.equal(c.delta.turns_per_merge, -19.5); // (40 + 60 + 0) / 3 = 33.3 -> (20 + 5 + 30 + 0) / 4 = 13.8
|
|
309
|
+
assert.equal(c.delta.implement_wall_p50_min, -20);
|
|
310
|
+
assert.equal(c.delta.merge_rate, 0.25);
|
|
311
|
+
assert.equal(c.delta.tool_calls_per_turn_p50, 0);
|
|
312
|
+
});
|
|
313
|
+
it("compare matches full lanes per size present on both sides (light, log-less lanes and unmatched sizes drop out)", async () => {
|
|
314
|
+
const { root, runs } = setup();
|
|
315
|
+
const c = await compareEfficiency({ batchRoot: root, runsDir: runs, repo: SLUG, pivot: PIVOT, now: PIVOT + 3 * DAY });
|
|
316
|
+
assert.deepEqual(Object.keys(c.matched), ["M"]); // issues 7 and 8 (no log, size unknown) are in no bucket
|
|
317
|
+
assert.deepEqual(c.matched.M.pre, { n: 2, turns_per_merge: 50, implement_wall_p50_min: 40, tool_calls_per_turn_p50: 1, rereads_per_merge: 3 });
|
|
318
|
+
assert.deepEqual(c.matched.M.post, { n: 1, turns_per_merge: 20, implement_wall_p50_min: 20, tool_calls_per_turn_p50: 1.5, rereads_per_merge: 0 });
|
|
319
|
+
assert.equal(c.matched.M.delta.tool_calls_per_turn_p50, 0.5);
|
|
320
|
+
assert.equal(c.matched.M.delta.turns_per_merge, -30);
|
|
321
|
+
});
|
|
322
|
+
it("compare report lines", async () => {
|
|
323
|
+
const { root, runs } = setup();
|
|
324
|
+
const lines = formatCompareLines(await compareEfficiency({ batchRoot: root, runsDir: runs, repo: SLUG, pivot: PIVOT, now: PIVOT + 3 * DAY }));
|
|
325
|
+
assert.equal(lines[0], "compare: pivot 2026-10-20T00:00:00.000Z, 3.0 days each side (pre 1 batches, post 1 batches)");
|
|
326
|
+
assert.match(lines[1], /^compare pre: n=3 turns\/merge 33\.3, implement wall p50 40\.0 min, calls\/turn p50 1\.00, re-reads\/merge 3\.00, merge rate 75%, conformance pass 33%$/);
|
|
327
|
+
assert.match(lines[3], /^compare delta: turns\/merge -?\d+\.\d, implement wall p50 -20\.0 min, .*merge rate \+25 pts, conformance pass \+67 pts$/);
|
|
328
|
+
assert.match(lines[4], /^compare matched M: pre n=2 post n=1; delta turns\/merge -30\.0, implement wall p50 -20\.0 min, calls\/turn p50 \+0\.50, re-reads\/merge -3\.00$/);
|
|
329
|
+
assert.ok(!lines.some((l) => /NaN|undefined/.test(l)), lines.join("\n"));
|
|
330
|
+
});
|
|
331
|
+
it("case-insensitive repo: a lowercase --repo matches the plans and reads the run logs under their spelling", async () => {
|
|
332
|
+
const { root, runs } = setup();
|
|
333
|
+
const c = await compareEfficiency({ batchRoot: root, runsDir: runs, repo: "own/repo", pivot: PIVOT, now: PIVOT + 3 * DAY });
|
|
334
|
+
assert.deepEqual([c.pre.merged, c.post.merged, c.pre.implement_wall_p50_min], [3, 4, 40]);
|
|
335
|
+
const w = (await windowEfficiency({ batchRoot: root, runsDir: runs, repo: "OWN/REPO", now: PIVOT + 3 * DAY, windows: [7 * DAY] }))["7d"];
|
|
336
|
+
assert.deepEqual([w.batches, w.merged], [2, 7]);
|
|
337
|
+
});
|
|
338
|
+
it("compare rejects a pivot in the future and prints a bare n/a delta for empty cohorts", async () => {
|
|
339
|
+
await assert.rejects(compareEfficiency({ batchRoot: tmpDir("eff-cmp-x-"), runsDir: tmpDir("eff-cmp-y-"), pivot: NOW + DAY, now: NOW }), /pivot/);
|
|
340
|
+
const empty = formatCompareLines(await compareEfficiency({ batchRoot: tmpDir("eff-cmp-x-"), runsDir: tmpDir("eff-cmp-y-"), pivot: NOW - DAY, now: NOW }));
|
|
341
|
+
assert.equal(empty[3], "compare delta: turns/merge n/a, implement wall p50 n/a, calls/turn p50 n/a, re-reads/merge n/a, merge rate n/a, conformance pass n/a");
|
|
342
|
+
});
|
|
343
|
+
it("compare and offline flags: parsing, pivot resolution and the offline CLI skips gh", async () => {
|
|
344
|
+
const o = parseArgs(["--efficiency", "--compare", "v1.32.0", "--offline"]);
|
|
345
|
+
assert.deepEqual([o.compare, o.offline], ["v1.32.0", true]);
|
|
346
|
+
assert.throws(() => parseArgs(["--compare", "2026-10-20"]), /need --efficiency/);
|
|
347
|
+
assert.throws(() => parseArgs(["--offline"]), /need --efficiency/);
|
|
348
|
+
assert.throws(() => parseArgs(["--efficiency", "--compare", "last-week"]), /--compare must be/);
|
|
349
|
+
assert.equal(comparePivot("2026-10-20T00:00:00Z"), PIVOT);
|
|
350
|
+
assert.equal(comparePivot("v1.32.0", [{ v: "1.32.0", t: PIVOT / 1000 }]), PIVOT);
|
|
351
|
+
assert.throws(() => comparePivot("9.9.9", []), /no release time known/);
|
|
352
|
+
const { root, runs } = setup();
|
|
353
|
+
const out = [], w = process.stdout.write;
|
|
354
|
+
process.stdout.write = (t) => { out.push(String(t)); return true; };
|
|
355
|
+
let code;
|
|
356
|
+
try {
|
|
357
|
+
code = await runStats(["--efficiency", "--window", "7d", "--compare", "2026-10-20T00:00:00Z", "--offline", "--repo", "own/repo"],
|
|
358
|
+
{ batchRoot: root, runsDir: runs, now: () => PIVOT + 3 * DAY, releaseTimes: [], gh: async () => { throw new Error("gh must not be called offline"); } });
|
|
359
|
+
} finally { process.stdout.write = w; }
|
|
360
|
+
assert.equal(code, 0);
|
|
361
|
+
const text = out.join("");
|
|
362
|
+
assert.match(text, /^compare: pivot 2026-10-20T00:00:00\.000Z/m);
|
|
363
|
+
assert.match(text, /^issue quality: n\/a \(no gh\/repo\)$/m);
|
|
364
|
+
});
|
|
365
|
+
});
|
package/bin/stats/index.mjs
CHANGED
|
@@ -15,7 +15,7 @@ import { readLanes, readLocalLaneRuns, readLocalRuns, runsDir } from "./local-so
|
|
|
15
15
|
import { parseForgeYaml } from "../forge-utils.mjs";
|
|
16
16
|
import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
|
|
17
17
|
import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers, exploreAb, renderExploreAb, laneProfileReport, renderLaneProfiles } from "./query.mjs";
|
|
18
|
-
import { batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows } from "./efficiency.mjs";
|
|
18
|
+
import { batchEfficiency, windowEfficiency, compareEfficiency, formatEfficiencyLines, formatCompareLines, parseWindows } from "./efficiency.mjs";
|
|
19
19
|
import { poolUtilisation, formatPoolLines } from "./pool.mjs";
|
|
20
20
|
import { batchRoot } from "../lean/batch.mjs";
|
|
21
21
|
|
|
@@ -24,7 +24,7 @@ const here = dirname(fileURLToPath(import.meta.url));
|
|
|
24
24
|
export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <owner/repo>] [--json] [--refresh]
|
|
25
25
|
[--compact] [--no-local] [--no-cache] [--min-n N] [--release-times FILE]
|
|
26
26
|
forgedock stats --pool [--window 7d,30d] [--repo <owner/repo>] [--json]
|
|
27
|
-
forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--repo <owner/repo>] [--json]
|
|
27
|
+
forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--compare <date|version>] [--offline] [--repo <owner/repo>] [--json]
|
|
28
28
|
|
|
29
29
|
Version-over-version report: Cost, Time, Flow, Review funnel, Gates and Issue flow per (version, repo), with a
|
|
30
30
|
delta against the previous version and a low-n flag for samples under 10. Missing data prints n/a.
|
|
@@ -66,6 +66,10 @@ Efficiency (forge#4402; local batch dirs and run logs, gh only for merged issues
|
|
|
66
66
|
lanes, issue quality vs cost and the acceptance pass rate 7 days after promotion
|
|
67
67
|
--batch <id> also report one batch (~/.forge/batches/<id>, FORGEDOCK_BATCH_DIR)
|
|
68
68
|
--window <list> trailing windows over the repo's batches (default 7d,30d)
|
|
69
|
+
--compare <date|v> before/after A/B around a deploy (date or release tag): equal-span cohorts of batches either side,
|
|
70
|
+
turns/merge, implement wall p50, calls/turn p50, re-reads/merge, merge and conformance rates,
|
|
71
|
+
their delta and full-lane cohorts matched per size (forge#4491)
|
|
72
|
+
--offline skip the GitHub crawl for merged issues (issue quality and acceptance print n/a)
|
|
69
73
|
|
|
70
74
|
Builder model tiers (forge#4394; local run logs only, no GitHub crawl):
|
|
71
75
|
--tiers merge, conformance and post-merge-rework rates, first-pass yield and cost per builder tier
|
|
@@ -78,7 +82,7 @@ Lane profile baseline (forge#4436; local run logs only, every run of each issue,
|
|
|
78
82
|
`;
|
|
79
83
|
|
|
80
84
|
export function parseArgs(argv) {
|
|
81
|
-
const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
|
|
85
|
+
const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
|
|
82
86
|
const val = (i, name) => {
|
|
83
87
|
const v = argv[i + 1];
|
|
84
88
|
if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
|
|
@@ -103,6 +107,12 @@ export function parseArgs(argv) {
|
|
|
103
107
|
else if (name === "--pool") o.pool = true;
|
|
104
108
|
else if (name === "--batch") { const v = take(); if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v)) throw new Error(`invalid --batch id: ${v}`); o.batch = v; }
|
|
105
109
|
else if (name === "--window") o.windows = parseWindows(take());
|
|
110
|
+
else if (name === "--compare") { // #4491
|
|
111
|
+
const v = take(), bad = () => new Error(`--compare must be a date (YYYY-MM-DD[THH:MM:SSZ]) or a version (1.32.0): ${v}`);
|
|
112
|
+
try { if (!v || !Object.keys(parseSince(v)).length) throw bad(); } catch { throw bad(); }
|
|
113
|
+
o.compare = v;
|
|
114
|
+
}
|
|
115
|
+
else if (name === "--offline") o.offline = true;
|
|
106
116
|
else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); }
|
|
107
117
|
else if (name === "--json") o.json = true;
|
|
108
118
|
else if (name === "--refresh") o.refresh = true;
|
|
@@ -114,6 +124,7 @@ export function parseArgs(argv) {
|
|
|
114
124
|
}
|
|
115
125
|
if (o.batch && !o.efficiency) throw new Error("--batch and --window need --efficiency");
|
|
116
126
|
if (o.windows && !o.efficiency && !o.pool) throw new Error("--batch and --window need --efficiency");
|
|
127
|
+
if ((o.compare || o.offline) && !o.efficiency) throw new Error("--compare and --offline need --efficiency");
|
|
117
128
|
return o;
|
|
118
129
|
}
|
|
119
130
|
|
|
@@ -154,6 +165,15 @@ export function sinceToTimestamp(since, releaseTimes = []) {
|
|
|
154
165
|
return { since: null, note: null };
|
|
155
166
|
}
|
|
156
167
|
|
|
168
|
+
/** #4491: `--compare` pivot in ms: a date as is, a version as its exact tag time (no crawl margin); throws when unknown. */
|
|
169
|
+
export function comparePivot(v, releaseTimes = []) {
|
|
170
|
+
const p = parseSince(v);
|
|
171
|
+
if (p.sinceMs != null) return p.sinceMs;
|
|
172
|
+
const hit = releaseTimes.find((r) => r.v === p.sinceVersion);
|
|
173
|
+
if (!hit) throw new Error(`--compare: no release time known for ${p.sinceVersion} (pass --release-times or a date)`);
|
|
174
|
+
return hit.t * 1000;
|
|
175
|
+
}
|
|
176
|
+
|
|
157
177
|
/** Throttled stderr progress printer (never stdout, so --json stays parseable). */
|
|
158
178
|
export function makeProgressPrinter(write = (t) => process.stderr.write(t), now = Date.now) {
|
|
159
179
|
let last = 0, lastPhase = null;
|
|
@@ -227,10 +247,11 @@ export async function buildReport(opts, deps = {}) {
|
|
|
227
247
|
return { repo, query: { pool } };
|
|
228
248
|
}
|
|
229
249
|
if (opts.efficiency) { // forge#4402: local batch dirs + run logs; gh only for merged issues
|
|
230
|
-
const root = deps.batchRoot ?? batchRoot(env), rd = deps.runsDir ?? runsDir(env), now = deps.now?.() ?? Date.now(), gh = deps.gh ?? defaultGh;
|
|
250
|
+
const root = deps.batchRoot ?? batchRoot(env), rd = deps.runsDir ?? runsDir(env), now = deps.now?.() ?? Date.now(), gh = opts.offline ? null : (deps.gh ?? defaultGh);
|
|
231
251
|
const efficiency = {};
|
|
232
252
|
if (opts.batch) efficiency.batch = await batchEfficiency({ batchDir: join(root, opts.batch), runsDir: rd, repo, gh, now });
|
|
233
253
|
efficiency.windows = await windowEfficiency({ batchRoot: root, runsDir: rd, repo, gh, now, windows: opts.windows ?? parseWindows("7d,30d") });
|
|
254
|
+
if (opts.compare) efficiency.compare = await compareEfficiency({ batchRoot: root, runsDir: rd, repo, gh, now, pivot: comparePivot(opts.compare, releaseTimes) });
|
|
234
255
|
return { repo, query: { efficiency } };
|
|
235
256
|
}
|
|
236
257
|
if (opts.contextAb) { // forge#4275: run logs only
|
|
@@ -291,7 +312,8 @@ export async function runStats(argv, deps = {}) {
|
|
|
291
312
|
if (report.query?.efficiency && !opts.json) {
|
|
292
313
|
const e = report.query.efficiency;
|
|
293
314
|
const lines = [...(e.batch ? formatEfficiencyLines(e.batch, `batch ${e.batch.batch ?? opts.batch}`) : []),
|
|
294
|
-
...Object.entries(e.windows).flatMap(([w, m]) => formatEfficiencyLines(m, `${w}, ${m.batches} batch${m.batches === 1 ? "" : "es"}`))
|
|
315
|
+
...Object.entries(e.windows).flatMap(([w, m]) => formatEfficiencyLines(m, `${w}, ${m.batches} batch${m.batches === 1 ? "" : "es"}`)),
|
|
316
|
+
...(e.compare ? formatCompareLines(e.compare) : [])];
|
|
295
317
|
process.stdout.write(`${lines.join("\n")}\n`);
|
|
296
318
|
return 0;
|
|
297
319
|
}
|
package/bin/stats/pool.mjs
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import { existsSync, readFileSync } from "node:fs";
|
|
9
9
|
import { join } from "node:path";
|
|
10
10
|
import { listBatches } from "../lean/batch.mjs";
|
|
11
|
-
import { windowLabel } from "./efficiency.mjs";
|
|
11
|
+
import { sameRepo, windowLabel } from "./efficiency.mjs";
|
|
12
12
|
|
|
13
13
|
const readJson = (f) => { try { return JSON.parse(readFileSync(f, "utf8")); } catch { return null; } };
|
|
14
14
|
const round1 = (x) => Math.round(x * 10) / 10;
|
|
@@ -18,7 +18,7 @@ export function poolUtilisation({ batchRoot, repo = null, windows, now = Date.no
|
|
|
18
18
|
const batches = (existsSync(batchRoot) ? listBatches(batchRoot) : []).map((id) => {
|
|
19
19
|
const dir = join(batchRoot, id), plan = readJson(join(dir, "plan.json")) ?? {};
|
|
20
20
|
return { dir, plan, created: Date.parse(plan.created ?? "") };
|
|
21
|
-
}).filter((b) => Number.isFinite(b.created) && (!repo || b.plan.repo
|
|
21
|
+
}).filter((b) => Number.isFinite(b.created) && (!repo || sameRepo(b.plan.repo, repo))).sort((a, b) => a.created - b.created);
|
|
22
22
|
const out = {};
|
|
23
23
|
for (const w of windows) {
|
|
24
24
|
const inWin = batches.filter((b) => b.created <= now && now - b.created <= w);
|
|
@@ -18,7 +18,7 @@ Canonical source: `docs/spec/review-protocol.md` — sync changes there first.
|
|
|
18
18
|
|
|
19
19
|
**Rationale**: Passing the full changeset to every agent inflates per-child input cost on large PRs. A billing agent reviewing auth code produces noise, not signal. Scoped inputs reduce cost and improve focus.
|
|
20
20
|
|
|
21
|
-
**
|
|
21
|
+
**Empty slice rule** (forge#4573): if a domain's file pattern matches nothing in the PR, its slice is a one-line note, never a copy of the full diff (that duplicated the whole diff across domains). Slices hold whole file blocks only: a file too large for the slice is listed as `# OMITTED` at the top and read from the head checkout. A staging brief may carry a pointer to the slice file instead of the text when the slice is large; read that file with bounded `grep`/`sed -n`.
|
|
22
22
|
|
|
23
23
|
**Security agent exception**: The General Security agent ALWAYS receives the full diff — security vulnerabilities are cross-cutting and cannot be safely scoped to a file-path filter.
|
|
24
24
|
|
|
@@ -129,7 +129,7 @@ node "$FD_ROOT/bin/forgedock.mjs" review-staging ${PR_ARG_NUM:+"$PR_ARG_NUM"} ${
|
|
|
129
129
|
|
|
130
130
|
Exit 0 prints the manifest. `REVIEW_STAGING: PR #N head <sha>` gives the PR number and the full head SHA. `BUNDLE_PRS:` lists the bundle member PRs. `SCOPE_MODE=` gives the 0B.1 mode. `ROSTER:` gives the tier and reason (record them in the final summary). `CONTEXT_FILE=` names the 0C context. There is one `DISPATCH <domain> <brief file>` line per roster domain. Exit 1 is a stop: the gate printed `RESULT: BLOCK DEPLOY` (report the blocking finding list), or a `REVIEW_RESULT: status: BLOCKED` line names the blocker. Report it and do not continue. A PR with no gate comment at all indicates a gate bypass, not a clean pass.
|
|
131
131
|
|
|
132
|
-
**Delta pass** (forge#4360): after a fix lands on staging mid-review, re-run with `--delta <previous full head SHA>`. Only that range is reviewed, by the domains it touches (a `DELTA:` manifest line), and the earlier pass is credited. Dispatch only the DISPATCH lines.
|
|
132
|
+
**Delta pass** (forge#4360): after a fix lands on staging mid-review, re-run with `--delta <previous full head SHA>`. Only that range is reviewed, by the domains it touches (a `DELTA:` manifest line), and the earlier pass is credited. The earlier pass counts as complete when it was finalized, or when every launched domain (the PR-keyed launched-agent file) delivered findings at that head with no degraded marker (forge#4573). Dispatch only the DISPATCH lines.
|
|
133
133
|
|
|
134
134
|
---
|
|
135
135
|
|
package/docs/STATS.md
CHANGED
|
@@ -37,7 +37,7 @@ With no local state at all, `forgedock stats --repo <r>` reproduces the report f
|
|
|
37
37
|
## Efficiency (`--efficiency`, forge#4402)
|
|
38
38
|
|
|
39
39
|
```bash
|
|
40
|
-
npx forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--repo <owner/repo>] [--json]
|
|
40
|
+
npx forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--compare <date|version>] [--offline] [--repo <owner/repo>] [--json]
|
|
41
41
|
```
|
|
42
42
|
|
|
43
43
|
Per batch (`--batch`, a dir under `~/.forge/batches`, `FORGEDOCK_BATCH_DIR`) and over trailing windows (`--window`,
|
|
@@ -78,6 +78,34 @@ Limits: logs written before #4362 carry no trigger, so implement attempts beyond
|
|
|
78
78
|
`unknown`. Logs written before #4407 sum running session totals. Lanes run outside a batch are in no window. `gh` (for
|
|
79
79
|
merged issues only) is the one network call; when it fails, quality and acceptance print `n/a` with the reason.
|
|
80
80
|
|
|
81
|
+
Every scope also prints, as its last line (forge#4491):
|
|
82
|
+
|
|
83
|
+
- **Implement wall p50**: the median over merged lanes of the summed `duration_ms` of their implement attempts, in
|
|
84
|
+
minutes.
|
|
85
|
+
- **Merge rate**: merged ÷ the cohort's issue rows.
|
|
86
|
+
- **Conformance pass**: the share of rows whose lanes logged a `CONFORMANCE` verdict and whose last verdict is
|
|
87
|
+
`conforms`.
|
|
88
|
+
|
|
89
|
+
`--repo` matches batch plans case-insensitively, as GitHub slugs are. Run logs are read under the plans' spelling.
|
|
90
|
+
`--offline` skips the GitHub crawl, so issue quality and acceptance print `n/a`. The crawl is two `gh` calls per merged
|
|
91
|
+
issue, which can take minutes over a large window.
|
|
92
|
+
|
|
93
|
+
### Before/after A/B (`--compare`, forge#4491)
|
|
94
|
+
|
|
95
|
+
`--compare <date|version>` compares two cohorts of the repo's batches around a deploy. The pivot is a date or ISO time,
|
|
96
|
+
or a release version resolved to its exact tag time (`--release-times`, else the checkout's `v*` tags).
|
|
97
|
+
|
|
98
|
+
- **post**: batches created from the pivot to now.
|
|
99
|
+
- **pre**: batches created in the same span before the pivot. Both sides are equally long.
|
|
100
|
+
|
|
101
|
+
It prints for each cohort turns/merge, implement wall p50, implement calls/turn p50, re-reads/merge, merge rate and
|
|
102
|
+
conformance pass, and then the post − pre delta.
|
|
103
|
+
|
|
104
|
+
`matched` repeats the comparison on full-shape merged lanes only, per size bucket present on both sides. Light lanes
|
|
105
|
+
(#4452) cut turns on their own, so the unmatched delta mixes the lane mix into any prompt or tool change. With `--json`
|
|
106
|
+
the result is under `query.efficiency.compare`. Cohorts are split by batch `plan.created`, as `--window` is; a lane run
|
|
107
|
+
outside a batch is in neither.
|
|
108
|
+
|
|
81
109
|
## Pool utilisation (`--pool`, forge#4549)
|
|
82
110
|
|
|
83
111
|
`forgedock stats --pool [--window 7d,30d] [--json]` reads the repo's local batch `report.json` rows and prints, per window,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "forgedock",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.34.0",
|
|
4
4
|
"description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"forgedock": "bin/forgedock.mjs"
|
|
@@ -942,10 +942,12 @@ do_file_prepare() {
|
|
|
942
942
|
# Title (defense in depth: strip command substitution so the raw title stays readable if it round-trips through an eval-based consumer).
|
|
943
943
|
ISSUE_TITLE="fix: ${F_TITLE} (review finding — PR #${PR_NUMBER})"
|
|
944
944
|
ISSUE_TITLE=$(printf '%s' "$ISSUE_TITLE" | tr '\t\n`' " '" | sed 's/\$(/$ (/g')
|
|
945
|
+
# The title is passed WITHOUT the constant "(review finding — PR #N)" suffix: its tokens (review, finding, the PR number) would make
|
|
946
|
+
# any two findings of one PR a >=3-token match and map a finding to its neighbour's issue (#4569).
|
|
945
947
|
# Step 0: deterministic title dedup (scripts/issue-dedup.sh; also excludes the issue this PR fixes: it is about to be closed, so a
|
|
946
948
|
# finding matched to it is tracked nowhere). Unresolved helper: "not deduped"; NEVER exit 1 (which means "duplicate, skip").
|
|
947
949
|
if [ -n "$DEDUP_SCRIPT" ]; then
|
|
948
|
-
DEDUP_RESULT=$(bash "$DEDUP_SCRIPT" "$
|
|
950
|
+
DEDUP_RESULT=$(bash "$DEDUP_SCRIPT" "fix: ${F_TITLE}" "-R $REPO" ${MERGE_ISSUE:+--exclude "$MERGE_ISSUE"} 2>&1); DEDUP_EXIT=$?
|
|
949
951
|
else
|
|
950
952
|
DEDUP_RESULT=""; DEDUP_EXIT=0
|
|
951
953
|
fi
|
|
@@ -965,6 +967,20 @@ do_file_prepare() {
|
|
|
965
967
|
else
|
|
966
968
|
# A CLOSED review-finding at the same location +-5 lines is a regression: create with a warning and elevate to priority:P1.
|
|
967
969
|
[ -n "$F_FILE" ] && REGRESSION=$(line_match "$CLOSED_JSON" "$F_FILE" "$F_LINE" "$F_TITLE" 0)
|
|
970
|
+
# DELTA: the earlier pass's finding is credited and its own issue is closed (fixed): resolved, never re-filed (#4569).
|
|
971
|
+
# Only an issue this PR's review filed counts (title suffix `PR #N)` or a `**Source**: PR #N` line); a closed finding of
|
|
972
|
+
# another PR at the same location stays a regression, or any closed issue nearby could drop a new blocker from the verdict.
|
|
973
|
+
OWN_CLOSED=""
|
|
974
|
+
if [ -n "$REGRESSION" ] && [ "$REVIEW_MODE" = "DELTA" ]; then
|
|
975
|
+
OWN_CLOSED=$(printf '%s' "$CLOSED_JSON" | jq -r --argjson n "$REGRESSION" --arg pr "$PR_NUMBER" '
|
|
976
|
+
.[] | select(.number == $n)
|
|
977
|
+
| select((.title // "" | endswith("PR #" + $pr + ")")) or ((.body // "") | test("(^|\\n)\\*\\*Source\\*\\*: PR #" + $pr + "([^0-9]|$)")))
|
|
978
|
+
| .number' 2>/dev/null | head -1)
|
|
979
|
+
fi
|
|
980
|
+
if [ -n "$OWN_CLOSED" ]; then
|
|
981
|
+
ACTION="skip-resolved"; DEDUP_OF="$REGRESSION"; REGRESSION=""
|
|
982
|
+
echo "RESOLVED: ${F_FILE}:${F_LINE} was fixed in closed issue #${DEDUP_OF} (delta pass)"
|
|
983
|
+
fi
|
|
968
984
|
[ -z "$REGRESSION" ] || echo "REGRESSION: Previously fixed in #${REGRESSION} — elevating priority"
|
|
969
985
|
fi
|
|
970
986
|
fi
|
|
@@ -738,6 +738,47 @@ mkfind G-1 CONFIRMED MEDIUM a.py 10 "Duplicate of the linked issue"; putfind 1
|
|
|
738
738
|
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
739
739
|
run "$GOOD" file-prepare; want "linked issue excluded" '.items[0].action' worklist.json "file"
|
|
740
740
|
|
|
741
|
+
# #4569: the title suffix never makes two findings of one PR dedup to each other; a DELTA closed match is resolved, a FULL one a regression
|
|
742
|
+
fp_setup
|
|
743
|
+
echo '[{"number":60,"title":"fix: credential helper hands token to any host (review finding — PR #7)","body":"see z.py:90"}]' > "$FIX/issues-open.json"
|
|
744
|
+
echo '[{"number":61,"title":"fix: DCO exemption only on staging (review finding — PR #7)","body":"see c.py:5"}]' > "$FIX/issues-closed.json"
|
|
745
|
+
mkfind K-1 CONFIRMED HIGH c.py 5 "DCO exemption only on staging"
|
|
746
|
+
putfind 1
|
|
747
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
748
|
+
run "$GOOD" file-prepare
|
|
749
|
+
want "full mode: closed own issue is a regression, not skipped" '.items[0] | [.action, (.regression_of | tostring)] | join(",")' worklist.json "file,61"
|
|
750
|
+
fp_setup; mkctx 1 DELTA "$OLD_SHA" 12
|
|
751
|
+
echo '[{"number":60,"title":"fix: credential helper hands token to any host (review finding — PR #7)","body":"see z.py:90"}]' > "$FIX/issues-open.json"
|
|
752
|
+
echo '[{"number":61,"title":"fix: DCO exemption only on staging (review finding — PR #7)","body":"see c.py:5"}]' > "$FIX/issues-closed.json"
|
|
753
|
+
mkfind K-1 CONFIRMED HIGH c.py 5 "DCO exemption only on staging"; putfind 1
|
|
754
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
755
|
+
run "$GOOD" file-prepare
|
|
756
|
+
want "delta mode: closed own issue is resolved by that issue" '.items[0] | [.action, (.dedup_of | tostring), (.regression_of | tostring)] | join(",")' worklist.json "skip-resolved,61,null"
|
|
757
|
+
# a closed finding of ANOTHER PR at the same location is no proof this finding was fixed: it stays a regression and blocks
|
|
758
|
+
fp_setup; mkctx 1 DELTA "$OLD_SHA" 12
|
|
759
|
+
echo '[]' > "$FIX/issues-open.json"
|
|
760
|
+
echo '[{"number":62,"title":"fix: unrelated old defect (review finding — PR #3)","body":"see c.py:7"}]' > "$FIX/issues-closed.json"
|
|
761
|
+
mkfind K-3 CONFIRMED HIGH c.py 5 "Brand new defect"; putfind 1
|
|
762
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
763
|
+
run "$GOOD" file-prepare
|
|
764
|
+
want "delta mode: another PR's closed issue is a regression, never resolved" '.items[0] | [.action, (.regression_of | tostring)] | join(",")' worklist.json "file,62"
|
|
765
|
+
# a **Source**: PR #N line marks the issue as this PR's own
|
|
766
|
+
fp_setup; mkctx 1 DELTA "$OLD_SHA" 12
|
|
767
|
+
echo '[]' > "$FIX/issues-open.json"
|
|
768
|
+
printf '[{"number":63,"title":"fix: renamed title","body":"**Source**: PR #7 — x\\nsee c.py:5"}]' > "$FIX/issues-closed.json"
|
|
769
|
+
mkfind K-4 CONFIRMED HIGH c.py 5 "Renamed defect"; putfind 1
|
|
770
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
771
|
+
run "$GOOD" file-prepare
|
|
772
|
+
want "delta mode: Source line marks own closed issue" '.items[0] | [.action, (.dedup_of | tostring)] | join(",")' worklist.json "skip-resolved,63"
|
|
773
|
+
# the dedup helper receives the title without the shared suffix
|
|
774
|
+
DEDUP_LOG="$WORK/dedup-args"; : > "$DEDUP_LOG"
|
|
775
|
+
printf '#!/usr/bin/env bash\nprintf "%%s\\n" "$1" >> "%s"\nexit 0\n' "$DEDUP_LOG" > "$GOOD/scripts/issue-dedup.sh"
|
|
776
|
+
fp_setup; mkfind K-2 CONFIRMED MEDIUM d.py 7 "Some other defect"; putfind 1
|
|
777
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
778
|
+
run "$GOOD" file-prepare
|
|
779
|
+
[ "$(cat "$DEDUP_LOG")" = "fix: Some other defect" ] && ok || bad "dedup helper gets the suffix-less title: '$(cat "$DEDUP_LOG")'"
|
|
780
|
+
mkroot "$GOOD"
|
|
781
|
+
|
|
741
782
|
# pattern recurrence: exempt severities skip, a prior instance in the same subsystem consolidates
|
|
742
783
|
fp_setup
|
|
743
784
|
echo '[{"number":50,"body":"x commands/work-on/z.md <!-- FORGE:PATTERN: slug-one -->","labels":[],"stateReason":"COMPLETED"}]' > "$FIX/issues-all.json"
|
|
@@ -93,7 +93,7 @@ done
|
|
|
93
93
|
if grep -qF 'marker domain' "$PERSONAS/protocols.md"; then ok; else bad "protocols.md does not define the marker domain"; fi
|
|
94
94
|
|
|
95
95
|
# Test 7: the roster regexes in delta_roster must equal the per-domain file regexes of the diff slices (both in scripts/review-prepare.sh since #3841),
|
|
96
|
-
# and the slice function must fall back to the full file list
|
|
96
|
+
# and the slice function must NOT fall back to the full file list or full diff for a domain with no hit (#4573)
|
|
97
97
|
for pair in auth:auth billing:billing concurrency:concurrency database:database infra:infra scraper:scraper frontend:frontend api:api; do
|
|
98
98
|
var=${pair%%:*}; dom=${pair#*:}
|
|
99
99
|
a=$(grep -E "^ slice $var " "$PREPARE" | head -n 1 | sed -n 's/.* "\([^"]*\)"$/\1/p')
|
|
@@ -101,7 +101,7 @@ for pair in auth:auth billing:billing concurrency:concurrency database:database
|
|
|
101
101
|
if [ -z "$a" ] || [ -z "$b" ]; then bad "could not extract $var pattern (slice='$a' delta_roster='$b')"
|
|
102
102
|
elif [ "$a" = "$b" ]; then ok; else bad "$var regex drifted: slice='$a' delta_roster='$b'"; fi
|
|
103
103
|
done
|
|
104
|
-
if grep -qF '[ -n "$paths" ] || paths="$SLICE_FILES_SRC"' "$PREPARE"
|
|
104
|
+
if grep -qF '[ -n "$paths" ] || paths="$SLICE_FILES_SRC"' "$PREPARE"; then bad "prepare slice() must not fall back to the full file list"; elif grep -qF '[ -n "$body" ] || body="$FULL_DIFF"' "$PREPARE"; then bad "prepare slice() must not fall back to the full diff"; elif grep -qF '[ -n "$body" ] || body="# No diff blocks' "$PREPARE"; then ok; else bad "prepare slice() lacks the empty-slice note"; fi
|
|
105
105
|
|
|
106
106
|
echo "review-pr-domain-map: $pass passed, $fail failed"
|
|
107
107
|
[ "$fail" -eq 0 ]
|
|
@@ -287,14 +287,16 @@ gh11() {
|
|
|
287
287
|
esac
|
|
288
288
|
}
|
|
289
289
|
[ "$(grep -c '^# BEGIN bundle-pr-kind$' "$PRE")" -eq 2 ] && [ "$(awk '/^# BEGIN bundle-pr-kind$/{f=1;n++} f{print > ("'"$TMP"'/kind" n ".sh")} /^# END bundle-pr-kind$/{f=0}' "$PRE"; cmp -s "$TMP/kind1.sh" "$TMP/kind2.sh" && echo same)" = same ] && ok || bad "0A and 0B.1 must carry identical bundle_pr_kind blocks"
|
|
290
|
+
awk '/^# BEGIN gh-retry$/{f=1;next} /^# END gh-retry$/{f=0} f' "$PRE" > "$TMP/ghretry.sh" # bundle_pr_kind calls gh_retry (forge#4572)
|
|
291
|
+
[ -s "$TMP/ghretry.sh" ] && ok || bad "gh-retry block not found"
|
|
290
292
|
awk '/^# BEGIN bundle-pr-set$/{f=1;next} /^# END bundle-pr-set$/{f=0} f' "$PRE" | sed 's/{GH_REPO}/o\/r/g' > "$TMP/prset.sh"
|
|
291
293
|
[ -s "$TMP/prset.sh" ] && ok || bad "bundle-pr-set block not found in Phase 0A"
|
|
292
|
-
OUT11=$( gh() { gh11 "$@"; }; GH_REPO=o/r; BUNDLE_PRS="5 6 28668 77 90"; MERGE_PRS="90"; . "$TMP/prset.sh"; echo $ALL_PR_NUMBERS "|$SKIPPED_ISSUES" )
|
|
294
|
+
OUT11=$( gh() { gh11 "$@"; }; GH_REPO=o/r; BUNDLE_PRS="5 6 28668 77 90"; MERGE_PRS="90"; FORGE_GH_RETRY_SLEEP=0; . "$TMP/ghretry.sh"; . "$TMP/prset.sh"; echo $ALL_PR_NUMBERS "|$SKIPPED_ISSUES" )
|
|
293
295
|
[ "$OUT11" = "5 6 77 90 | 28668" ] && ok || bad "0A must drop the confirmed issue and keep PRs, merge PRs and unclassifiable numbers (got '$OUT11')"
|
|
294
296
|
awk '/^# BEGIN bundle-scope-loop$/{f=1;next} /^# END bundle-scope-loop$/{f=0} f' "$PRE" > "$TMP/loop.sh"
|
|
295
297
|
[ -s "$TMP/loop.sh" ] && ok || bad "bundle-scope-loop block not found in 0B.1"
|
|
296
298
|
scope11() { # $1 = ALL_PR_NUMBERS; prints "<SCOPE_MODE> <REVIEWED_PR_COUNT>" then the loop output
|
|
297
|
-
( gh() { gh11 "$@"; }; bundle_delta_credit() { DELTA_COMMITS=""; DELTA_FULL=0; }; . "$TMP/kind1.sh"
|
|
299
|
+
( gh() { gh11 "$@"; }; bundle_delta_credit() { DELTA_COMMITS=""; DELTA_FULL=0; }; FORGE_GH_RETRY_SLEEP=0; . "$TMP/ghretry.sh"; . "$TMP/kind1.sh"
|
|
298
300
|
SCOPE_MODE=full; REVIEWED_COMMITS=""; REVIEWED_MERGES=""; REVIEWED_PR_COUNT=0; _PF="$TMP/pf11"; : > "$_PF"
|
|
299
301
|
TRUSTED_SCRIPT="$TC" ALL_PR_NUMBERS="$1" PR_NUMBER=9 GH_REPO=o/r _SD="$TMP" REVIEW_DELTA_SCRIPT=""
|
|
300
302
|
LOG11=$(. "$TMP/loop.sh"; echo "@@$SCOPE_MODE $REVIEWED_PR_COUNT"); echo "${LOG11##*@@}"; echo "${LOG11%@@*}" )
|
|
@@ -473,9 +473,18 @@ if [ -n "$DELTA_FROM_OPT" ]; then
|
|
|
473
473
|
# an agent body with a findings block for <from>, a post-panel-guard proof naming <from> (staging summary, synthesized
|
|
474
474
|
# block, or a # PR Review Summary whose Agents count matches) and no degraded marker after its first agent body.
|
|
475
475
|
_dv="error"
|
|
476
|
+
# forge#4573: a panel that delivered every launched domain at <from> but was never finalized (the head moved) is complete too.
|
|
477
|
+
# The launched list is the PR-keyed file the dispatch persisted; empty or unreadable means no such proof. It counts only when
|
|
478
|
+
# its .head sidecar names <from>: a later dispatch at another head rewrites the list, so it no longer describes the panel at <from>.
|
|
479
|
+
_dlf="${FORGE_SCRATCHPAD:-$PWD/.forge-scratch}/${PR_NUMBER}_launched_review_agents"
|
|
480
|
+
_dl="[]"
|
|
481
|
+
if [ "$(sed -n 1p "${_dlf}.head" 2>/dev/null)" = "$DELTA_FROM_OPT" ]; then
|
|
482
|
+
_dl=$(grep -E '^[a-z0-9-]+$' "$_dlf" 2>/dev/null | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))' 2>/dev/null) || _dl="[]"
|
|
483
|
+
[ -n "$_dl" ] || _dl="[]"
|
|
484
|
+
fi
|
|
476
485
|
if [ -n "$TRUSTED_SCRIPT" ] && [ -n "$COMMENTS_JSON" ]; then
|
|
477
486
|
_db=$(printf '%s' "$COMMENTS_JSON" | bash "$TRUSTED_SCRIPT" bodies '^<!-- FORGE:REVIEW-AGENT:[a-z0-9-]+ -->|^<!-- REVIEW-FINDINGS-SYNTHESIZED-START -->|# PR Review Summary|^<!-- FORGE:GATE_(PASS|FAILURE) -->|<!-- FORGE:GATE_FAILURE:TYPE=review-panel-integrity|<!-- FORGE:REVIEW_DEGRADED|<!-- FORGE:REVIEW_BLOCKED' 2>/dev/null) \
|
|
478
|
-
&& _dv=$(printf '%s\n' "$_db" | jq -s -r --arg f "$DELTA_FROM_OPT" '
|
|
487
|
+
&& _dv=$(printf '%s\n' "$_db" | jq -s -r --arg f "$DELTA_FROM_OPT" --argjson L "$_dl" '
|
|
479
488
|
def shas($t; $re): [ $t | scan($re) | .[0] ];
|
|
480
489
|
def one($a): if ($a | length) == 1 then $a[0] else null end;
|
|
481
490
|
def rsha: "(?:^|\\n)Reviewed-SHA: ([0-9a-f]{40}|[0-9a-f]{64})\\r?(?:\\n|$)";
|
|
@@ -494,13 +503,15 @@ if [ -n "$DELTA_FROM_OPT" ]; then
|
|
|
494
503
|
and one(shas($t; "\\*\\*Agents\\*\\*: ([0-9]+)")) == ($nd | tostring))) ] | length) as $proof
|
|
495
504
|
| ([ $E[] | select(.key > $first)
|
|
496
505
|
| select(.value | test("<!-- FORGE:GATE_FAILURE:TYPE=review-panel-integrity|<!-- FORGE:REVIEW_DEGRADED|<!-- FORGE:REVIEW_BLOCKED")) ] | length) as $deg
|
|
497
|
-
| if $deg > 0 then "degraded" elif $proof
|
|
506
|
+
| if $deg > 0 then "degraded" elif $proof > 0 then "ok"
|
|
507
|
+
elif ($L | length) > 0 and ([ $L[] | . as $d | select($AG | any(.d == $d)) ] | length) == ($L | length) then "ok"
|
|
508
|
+
else "no-proof" end
|
|
498
509
|
end' 2>/dev/null) || _dv="error"
|
|
499
510
|
fi
|
|
500
511
|
case "$_dv" in
|
|
501
512
|
ok) REVIEW_MODE="DELTA"; DELTA_FROM="$DELTA_FROM_OPT" ;;
|
|
502
513
|
no-agent) block "--delta-from ${DELTA_FROM_OPT}: no trusted FORGE:REVIEW-AGENT comment with a findings block for that head, no earlier pass to credit: run a full review" ;;
|
|
503
|
-
no-proof) block "--delta-from ${DELTA_FROM_OPT}: no trusted proof that the earlier panel completed (staging summary, synthesized block or PR Review Summary for that head): run a full review" ;;
|
|
514
|
+
no-proof) block "--delta-from ${DELTA_FROM_OPT}: no trusted proof that the earlier panel completed (staging summary, synthesized block or PR Review Summary for that head, nor every launched domain delivered at it): run a full review" ;;
|
|
504
515
|
degraded) block "--delta-from ${DELTA_FROM_OPT}: the earlier pass is degraded (review-panel-integrity / REVIEW_DEGRADED / REVIEW_BLOCKED after its agents): run a full review" ;;
|
|
505
516
|
*) block "--delta-from ${DELTA_FROM_OPT}: could not read the trusted review comments, no earlier pass to credit: run a full review" ;;
|
|
506
517
|
esac
|
|
@@ -587,19 +598,31 @@ if [ "$REVIEW_MODE" != "BASE_SYNC_ONLY" ]; then
|
|
|
587
598
|
# slice <domain> <diff-header regex|ALL> <file-path regex|ALL>
|
|
588
599
|
slice() {
|
|
589
600
|
local dom="$1" dre="$2" fre="$3" body paths tmp df="$OUT_DIR/slice-${REVIEW_SHA_SHORT:-unresolved}-$1.diff"
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
601
|
+
# forge#4573: whole `diff --git` blocks only. A block over the per-file cap, or one that would push the slice over the total
|
|
602
|
+
# cap, is listed as omitted (never cut mid-file). No fallback to the full diff: an unmatched domain gets a note, not a copy.
|
|
603
|
+
# The security slice (ALL) lists .github/ blocks last so CI files cannot crowd out source.
|
|
604
|
+
body=$(printf '%s\n' "$FULL_DIFF" | awk -v re="$dre" -v cap="${REVIEW_SLICE_CAP:-102400}" -v fcap="${REVIEW_SLICE_FILE_CAP:-20000}" '
|
|
605
|
+
/^diff --git/ { n++; hdr[n] = $0; blk[n] = "" }
|
|
606
|
+
n { blk[n] = blk[n] $0 "\n" }
|
|
607
|
+
END {
|
|
608
|
+
for (pass = 1; pass <= 2; pass++) for (i = 1; i <= n; i++) {
|
|
609
|
+
ci = (hdr[i] ~ /^diff --git a\/\.github\//) ? 2 : 1
|
|
610
|
+
if (re == "ALL" ? ci != pass : (pass == 2 || hdr[i] !~ re)) continue
|
|
611
|
+
l = length(blk[i])
|
|
612
|
+
if (l > fcap || used + l > cap) { split(hdr[i], h, " "); om = om "# OMITTED (too large for this slice, read it from the head checkout): " substr(h[3], 3) "\n"; continue }
|
|
613
|
+
out = out blk[i]; used += l
|
|
614
|
+
}
|
|
615
|
+
printf "%s%s", om, out
|
|
616
|
+
}')
|
|
617
|
+
[ -n "$body" ] || body="# No diff blocks for this domain in the review range."
|
|
595
618
|
if [ "$fre" = "ALL" ]; then paths="$SLICE_FILES_SRC"
|
|
596
|
-
else paths=$(printf '%s\n' "$SLICE_FILES_SRC" | grep -iE "$fre" || echo "");
|
|
597
|
-
# 100K-char cap, same as the runner's tool-result cap
|
|
598
|
-
if [ "${#body}" -gt 102400 ]; then body="$(printf '%s' "$body" | head -c 102400)"$'\n'"... [TRUNCATED — diff exceeded 100K chars; focus on the files listed]"; fi
|
|
619
|
+
else paths=$(printf '%s\n' "$SLICE_FILES_SRC" | grep -iE "$fre" || echo ""); fi
|
|
599
620
|
# Publish atomically: temp file in $OUT_DIR (same filesystem) then rename, so a concurrent reader sees nothing or the whole file.
|
|
600
621
|
tmp=$(mktemp "$OUT_DIR/.slice-XXXXXX") || { echo "review-prepare: cannot create temp slice in $OUT_DIR" >&2; exit 2; }
|
|
601
622
|
if printf '%s\n' "$body" > "$tmp" && chmod 644 "$tmp"; then mv -f "$tmp" "$df"; else rm -f "$tmp"; echo "review-prepare: cannot write slice $df" >&2; exit 2; fi
|
|
602
|
-
|
|
623
|
+
# paths can exceed the 128 KiB per-argument limit on a huge PR: pass it as a file, not argv
|
|
624
|
+
printf '%s' "$paths" > "$TMP_DIR/slice-paths" || { echo "review-prepare: cannot write $TMP_DIR/slice-paths" >&2; exit 2; }
|
|
625
|
+
SLICES_JSON=$(printf '%s' "$SLICES_JSON" | jq -c --arg d "$dom" --arg df "$df" --rawfile p "$TMP_DIR/slice-paths" '. + {($d): {paths: ($p | split("\n") | map(select(length > 0))), diff_file: $df}}')
|
|
603
626
|
}
|
|
604
627
|
slice security ALL ALL
|
|
605
628
|
slice auth '^diff --git.*/(auth|session|jwt|oauth|permission|middleware|login|token)' "auth|session|login|logout|jwt|oauth|token|user|permission|middleware"
|
|
@@ -617,43 +640,55 @@ fi
|
|
|
617
640
|
STATUS="OK"; [ -n "$BLOCKERS" ] && STATUS="BLOCKED"
|
|
618
641
|
j_lines() { printf '%s' "${1}" | jq -R -s -c 'split("\n") | map(select(length > 0))'; }
|
|
619
642
|
j_words() { printf '%s' "${1}" | jq -R -s -c 'split(" ") | map(select(length > 0))'; }
|
|
643
|
+
# Everything that scales with the PR goes to the final jq through a file (--slurpfile/--rawfile): argv caps one argument at 128 KiB.
|
|
644
|
+
jf_lines() { j_lines "$2" > "$TMP_DIR/$1.json"; }
|
|
645
|
+
jf_words() { j_words "$2" > "$TMP_DIR/$1.json"; }
|
|
646
|
+
jf_raw() { printf '%s' "$2" > "$TMP_DIR/$1.txt"; }
|
|
647
|
+
{ jf_lines blockers "$BLOCKERS" && jf_lines p2 "$P2_FINDINGS" && jf_lines asm "$ASSUMPTIONS" && jf_lines dfiles "$DELTA_FILES" && jf_lines full_files "$FILES" \
|
|
648
|
+
&& printf '%s' "$SLICES_JSON" > "$TMP_DIR/slices.json" \
|
|
649
|
+
&& jf_raw churn "$CHURN_CONTEXT" && jf_raw pctx "$PROJECT_CONTEXT" && jf_raw tmono "$MONOLITHIC_CATALOG" \
|
|
650
|
+
&& jf_raw d_auth "$DOMAIN_CONTEXT_AUTH" && jf_raw d_billing "$DOMAIN_CONTEXT_BILLING" && jf_raw d_conc "$DOMAIN_CONTEXT_CONCURRENCY" \
|
|
651
|
+
&& jf_raw d_infra "$DOMAIN_CONTEXT_INFRA" && jf_raw d_db "$DOMAIN_CONTEXT_DATABASE" && jf_raw d_fe "$DOMAIN_CONTEXT_FRONTEND" \
|
|
652
|
+
&& jf_raw d_sec "$DOMAIN_CONTEXT_SECURITY" && jf_raw d_api "$DOMAIN_CONTEXT_API" && jf_raw d_scr "$DOMAIN_CONTEXT_SCRAPING" \
|
|
653
|
+
&& jf_raw i_auth "$IDX_AUTH" && jf_raw i_billing "$IDX_BILLING" && jf_raw i_db "$IDX_DATABASE" && jf_raw i_api "$IDX_API" \
|
|
654
|
+
&& jf_raw i_fe "$IDX_FRONTEND" && jf_raw i_infra "$IDX_INFRA"; } || { echo "review-prepare: cannot write context inputs under $TMP_DIR" >&2; exit 2; }
|
|
620
655
|
CTX=$(jq -n \
|
|
621
|
-
--arg status "$STATUS" --
|
|
656
|
+
--arg status "$STATUS" --slurpfile blockers "$TMP_DIR/blockers.json" \
|
|
622
657
|
--argjson pr_number "$PR_NUMBER" --arg repo "$REPO" --arg title "$TITLE" --arg author "$PR_AUTHOR" --arg head "$HEAD" --arg base "$BASE" \
|
|
623
658
|
--arg sha "$REVIEW_SHA" --arg sha7 "$REVIEW_SHA_SHORT" --argjson adds "$ADDITIONS" --argjson dels "$DELETIONS" --arg nfiles "${NFILES:-}" --arg first "$IS_FIRST_TIME_CONTRIBUTOR" \
|
|
624
659
|
--arg s2m "$IS_STAGING_TO_MAIN" --arg m2s "$IS_MILESTONE_TO_STAGING" --arg rfb "$REQUIRES_FULL_BUILD" --arg thorough "$THOROUGH" \
|
|
625
660
|
--arg mh "$MERGE_HEALTH" --arg mhs "$MERGE_HEALTH_STATE" --argjson mhr "$MERGE_RETRY" \
|
|
626
661
|
--argjson services "$(j_words "$SERVICES")" --argjson domains "$(j_words "$DOMAINS")" \
|
|
627
|
-
--arg tsb "$TS_BUILD" --arg pyf "$PY_FORMAT" --arg bcs "$BUILD_CONFIG_STATUS" --
|
|
628
|
-
--
|
|
662
|
+
--arg tsb "$TS_BUILD" --arg pyf "$PY_FORMAT" --arg bcs "$BUILD_CONFIG_STATUS" --slurpfile p2 "$TMP_DIR/p2.json" \
|
|
663
|
+
--slurpfile asm "$TMP_DIR/asm.json" --argjson risk "$(j_words "$RISK")" --rawfile churn "$TMP_DIR/churn.txt" --arg churn_esc "$CHURN_ESCALATION" --arg danger "$DANGER" \
|
|
629
664
|
--argjson sa "$SCORE_AUTH" --argjson sb "$SCORE_BILLING" --argjson sc "$SCORE_CONCURRENCY" --argjson sd "$SCORE_DATABASE" --argjson si "$SCORE_INFRA" \
|
|
630
665
|
--argjson ss "$SCORE_SECURITY" --argjson ssc "$SCORE_SCRAPING" --argjson sf "$SCORE_FRONTEND" --argjson sap "$SCORE_API" \
|
|
631
666
|
--arg cissue "$ISSUE_NUM" --arg cflags "$CONTRACT_RISK_FLAGS" \
|
|
632
667
|
--arg rdepth "$REVIEW_DEPTH" --arg rweight "$REVIEW_WEIGHT" --arg rtier "$ROSTER_TIER" --arg rreason "$ROSTER_REASON" --argjson agents "$(j_words "$SELECTED_AGENTS")" \
|
|
633
|
-
--arg dmode "$REVIEW_MODE" --arg dfrom "$DELTA_FROM" --arg dcarried "$CARRIED_FROM" --argjson drounds "$FULL_ROUNDS" --
|
|
634
|
-
--arg pname "$PROJECT_NAME" --
|
|
635
|
-
--
|
|
636
|
-
--
|
|
637
|
-
--arg tsrc "$TEMPLATE_SOURCE" --arg tbase "$TEMPLATE_BASE" --
|
|
638
|
-
--
|
|
639
|
-
--
|
|
640
|
-
{schema_version: 1, status: $status, blockers: $blockers,
|
|
668
|
+
--arg dmode "$REVIEW_MODE" --arg dfrom "$DELTA_FROM" --arg dcarried "$CARRIED_FROM" --argjson drounds "$FULL_ROUNDS" --slurpfile dfiles "$TMP_DIR/dfiles.json" \
|
|
669
|
+
--arg pname "$PROJECT_NAME" --rawfile pctx "$TMP_DIR/pctx.txt" --arg scraping_on "$SCRAPING_ON" --arg rpath "$REPO_PATH" \
|
|
670
|
+
--rawfile d_auth "$TMP_DIR/d_auth.txt" --rawfile d_billing "$TMP_DIR/d_billing.txt" --rawfile d_conc "$TMP_DIR/d_conc.txt" --rawfile d_infra "$TMP_DIR/d_infra.txt" \
|
|
671
|
+
--rawfile d_db "$TMP_DIR/d_db.txt" --rawfile d_fe "$TMP_DIR/d_fe.txt" --rawfile d_sec "$TMP_DIR/d_sec.txt" --rawfile d_api "$TMP_DIR/d_api.txt" --rawfile d_scr "$TMP_DIR/d_scr.txt" \
|
|
672
|
+
--arg tsrc "$TEMPLATE_SOURCE" --arg tbase "$TEMPLATE_BASE" --rawfile tmono "$TMP_DIR/tmono.txt" \
|
|
673
|
+
--rawfile i_auth "$TMP_DIR/i_auth.txt" --rawfile i_billing "$TMP_DIR/i_billing.txt" --rawfile i_db "$TMP_DIR/i_db.txt" --rawfile i_api "$TMP_DIR/i_api.txt" --rawfile i_fe "$TMP_DIR/i_fe.txt" --rawfile i_infra "$TMP_DIR/i_infra.txt" \
|
|
674
|
+
--slurpfile slices "$TMP_DIR/slices.json" --slurpfile full_files "$TMP_DIR/full_files.json" '
|
|
675
|
+
{schema_version: 1, status: $status, blockers: $blockers[0],
|
|
641
676
|
pr: {number: $pr_number, repo: $repo, title: $title, author: $author, head_ref: $head, base_ref: $base, review_sha: $sha, review_sha_short: $sha7,
|
|
642
677
|
additions: $adds, deletions: $dels, changed_files: ($nfiles | tonumber? // null), first_time_contributor: ($first == "true")},
|
|
643
678
|
lane: {is_staging_to_main: ($s2m == "true"), is_milestone_to_staging: ($m2s == "true"), requires_full_build: ($rfb == "true"), thorough: ($thorough == "true")},
|
|
644
679
|
merge_health: {mergeable: $mh, state: $mhs, retries: $mhr},
|
|
645
680
|
classification: {services: $services, domains: $domains},
|
|
646
|
-
phase2: {build_config: {typescript_build: $tsb, python_format: $pyf, status: $bcs}, findings: $p2},
|
|
647
|
-
broken_assumptions: $asm, risk_signals: $risk, churn_context: $churn, churn_escalation: ($churn_esc == "true"), danger_zone: ($danger == "true"),
|
|
681
|
+
phase2: {build_config: {typescript_build: $tsb, python_format: $pyf, status: $bcs}, findings: $p2[0]},
|
|
682
|
+
broken_assumptions: $asm[0], risk_signals: $risk, churn_context: $churn, churn_escalation: ($churn_esc == "true"), danger_zone: ($danger == "true"),
|
|
648
683
|
scores: {AUTH: $sa, BILLING: $sb, CONCURRENCY: $sc, DATABASE: $sd, INFRA: $si, SECURITY: $ss, SCRAPING: $ssc, FRONTEND: $sf, API: $sap},
|
|
649
684
|
contract: {issue: ($cissue | tonumber? // null), flags: $cflags},
|
|
650
685
|
roster: {tier: $rtier, review_depth: $rdepth, review_weight: $rweight, reason: $rreason, agents: $agents},
|
|
651
|
-
delta: {mode: $dmode, from: $dfrom, carried_from: $dcarried, full_rounds: $drounds, files: $dfiles},
|
|
686
|
+
delta: {mode: $dmode, from: $dfrom, carried_from: $dcarried, full_rounds: $drounds, files: $dfiles[0]},
|
|
652
687
|
config: {project_name: $pname, project_context: $pctx, scraping_configured: ($scraping_on == "true"), repo_path: $rpath,
|
|
653
688
|
domains: {auth: $d_auth, billing: $d_billing, concurrency: $d_conc, infra: $d_infra, database: $d_db, frontend: $d_fe, security: $d_sec, api: $d_api, scraping: $d_scr}},
|
|
654
689
|
templates: {source: $tsrc, base: $tbase, monolithic_catalog: $tmono},
|
|
655
690
|
index_slices: {auth: $i_auth, billing: $i_billing, database: $i_db, api: $i_api, frontend: $i_fe, infrastructure: $i_infra},
|
|
656
|
-
slices: $slices, full_files: $full_files}') || { echo "review-prepare: could not assemble the context JSON" >&2; exit 2; }
|
|
691
|
+
slices: $slices[0], full_files: $full_files[0]}') || { echo "review-prepare: could not assemble the context JSON" >&2; exit 2; }
|
|
657
692
|
if [ -n "$OUT" ]; then printf '%s\n' "$CTX" > "$OUT" || { echo "review-prepare: cannot write $OUT" >&2; exit 2; }; else printf '%s\n' "$CTX"; fi
|
|
658
693
|
[ "$STATUS" = "OK" ] && exit 0
|
|
659
694
|
exit 3
|
|
@@ -116,6 +116,28 @@ prep "$ROOT"; eq "garbage state normalized" .merge_health.state UNKNOWN
|
|
|
116
116
|
setup feat/x staging "README.md" "$DIFF_PLAIN"; echo 0 > "$FIX/prior"
|
|
117
117
|
prep "$ROOT"; eq "first-time contributor" .pr.first_time_contributor true
|
|
118
118
|
|
|
119
|
+
# --- 2b. slices are whole-file blocks, bounded, never the full-diff fallback (#4573) ---
|
|
120
|
+
BIG_LINES=$(awk 'BEGIN { for (i = 0; i < 1500; i++) print "-old line " i " of the huge generated file" }')
|
|
121
|
+
DIFF_BIG="diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
|
|
122
|
+
+ci: 1
|
|
123
|
+
diff --git a/services/api/big_scrape.py b/services/api/big_scrape.py
|
|
124
|
+
$BIG_LINES
|
|
125
|
+
diff --git a/services/api/auth.py b/services/api/auth.py
|
|
126
|
+
+def f(): jwt = 1"
|
|
127
|
+
setup feat/x staging "services/api/big_scrape.py
|
|
128
|
+
services/api/auth.py
|
|
129
|
+
.github/workflows/ci.yml" "$DIFF_BIG"
|
|
130
|
+
prep "$ROOT"
|
|
131
|
+
SEC_SLICE=$(jqv '.slices.security.diff_file')
|
|
132
|
+
if grep -q '^# OMITTED.*services/api/big_scrape.py' "$SEC_SLICE" && ! grep -q 'huge generated file' "$SEC_SLICE"; then ok; else bad "oversize file must be listed as omitted, not cut into the slice"; fi
|
|
133
|
+
if grep -q '^+def f(): jwt = 1' "$SEC_SLICE"; then ok; else bad "small blocks stay whole next to an omitted one"; fi
|
|
134
|
+
if [ "$(grep -n 'services/api/auth.py b/' "$SEC_SLICE" | head -1 | cut -d: -f1)" -lt "$(grep -n '\.github/workflows/ci.yml b/' "$SEC_SLICE" | head -1 | cut -d: -f1)" ]; then ok; else bad "security slice lists .github blocks last"; fi
|
|
135
|
+
BILL_SLICE=$(jqv '.slices.billing.diff_file')
|
|
136
|
+
if grep -q 'auth.py\|big_scrape' "$BILL_SLICE"; then bad "unmatched domain must not fall back to the full diff"; else ok; fi
|
|
137
|
+
eq "unmatched domain has no paths" '.slices.billing.paths | length' 0
|
|
138
|
+
REVIEW_SLICE_CAP=100 prep "$ROOT"
|
|
139
|
+
if [ "$(wc -c < "$(jqv '.slices.security.diff_file')")" -lt 600 ]; then ok; else bad "total cap bounds the slice"; fi
|
|
140
|
+
|
|
119
141
|
# --- 3. classification + scores ---
|
|
120
142
|
setup feat/x staging "services/api/auth.py
|
|
121
143
|
services/api/billing.py" "$DIFF_AUTH_BILL"
|
|
@@ -252,6 +274,26 @@ prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "degraded mark
|
|
|
252
274
|
{ cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
253
275
|
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "partial earlier pass (no post-guard proof) blocks" 3
|
|
254
276
|
eq "partial earlier pass blocker" '.blockers | join(" ") | test("no trusted proof that the earlier panel completed")' true
|
|
277
|
+
# forge#4573: an unfinalized panel is complete when every launched domain delivered findings at <from> and nothing is degraded
|
|
278
|
+
export FORGE_SCRATCHPAD="$WORK/sd"; mkdir -p "$FORGE_SCRATCHPAD"
|
|
279
|
+
printf 'security\n' > "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
280
|
+
printf '%s\n' "$SHA_A" > "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
281
|
+
{ cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
282
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "complete but unfinalized panel is accepted (#4573)" 0; eq "unfinalized panel mode" .delta.mode DELTA
|
|
283
|
+
printf '%s\n' "$SHA_B" > "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
284
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "a launched list written for another head is no proof" 3
|
|
285
|
+
rm -f "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
286
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "a launched list with no head sidecar is no proof" 3
|
|
287
|
+
printf '%s\n' "$SHA_A" > "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
288
|
+
printf 'security\nauth\n' > "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
289
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "a launched domain with no findings keeps the delta blocked" 3
|
|
290
|
+
printf 'security\n' > "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
291
|
+
{ cmt "$(agent_body "$SHA_A")"; cmt "<!-- FORGE:REVIEW_DEGRADED -->"; } | comments_of
|
|
292
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "degraded unfinalized panel stays blocked" 3
|
|
293
|
+
rm -f "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
294
|
+
{ cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
295
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "no launched list: unfinalized panel stays blocked" 3
|
|
296
|
+
unset FORGE_SCRATCHPAD
|
|
255
297
|
{ cmt "$(agent_body "$SHA_A")"; cmt "$(staging_summary "$SHA_A")" NONE; } | comments_of
|
|
256
298
|
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "untrusted staging summary is no proof" 3
|
|
257
299
|
{ cmt "$(staging_summary "$SHA_B")"; cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
@@ -435,5 +477,16 @@ if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; th
|
|
|
435
477
|
if grep -q "code index timed out after 1s" "$WORK/stderr"; then ok; else bad "index timeout must be reported on stderr"; fi
|
|
436
478
|
fi
|
|
437
479
|
|
|
480
|
+
# --- 12. oversized slices JSON (>128 KiB, argv per-argument limit) still assembles ---
|
|
481
|
+
BIG_FILES=$(awk 'BEGIN{for(i=1;i<=1100;i++) printf "services/api/auth/very/long/directory/name/to/pad/the/path/out/for/the/test/module_%05d_handler.py\n", i}')
|
|
482
|
+
BIG_DIFF='diff --git a/services/api/auth/x.py b/services/api/auth/x.py
|
|
483
|
+
+jwt = 1'
|
|
484
|
+
setup feat/x staging "$BIG_FILES" "$BIG_DIFF"
|
|
485
|
+
prep "$ROOT"
|
|
486
|
+
rc_is "oversized slices JSON" 0
|
|
487
|
+
eq "oversized slices JSON is over 128 KiB" '.slices | tojson | length > 131072' true
|
|
488
|
+
eq "oversized security slice keeps every path" '.slices.security.paths | length' 1100
|
|
489
|
+
eq "oversized full_files keeps every path" '.full_files | length' 1100
|
|
490
|
+
|
|
438
491
|
echo "review-prepare tests: $pass passed, $fail failed"
|
|
439
492
|
[ "$fail" -eq 0 ]
|
|
@@ -50,6 +50,31 @@ if [ "$CMD" != gate ]; then
|
|
|
50
50
|
export FORGE_SCRATCHPAD="$SD"
|
|
51
51
|
fi
|
|
52
52
|
_SD="${SD:-${FORGE_SCRATCHPAD:-$PWD/.forge-scratch}}"
|
|
53
|
+
|
|
54
|
+
# BEGIN gh-retry
|
|
55
|
+
# gh_retry [--search] CMD...: run CMD up to FORGE_GH_RETRY_ATTEMPTS (default 6) times with exponential backoff from
|
|
56
|
+
# FORGE_GH_RETRY_SLEEP seconds (default 2, capped at 16 per wait; --search triples the base for the 30 req/min search API).
|
|
57
|
+
# Prints stdout of the first success and returns 0; returns 1 once every attempt failed, so callers still fail closed/open as before (forge#4572).
|
|
58
|
+
gh_retry() {
|
|
59
|
+
local mult=1 n=0 max="${FORGE_GH_RETRY_ATTEMPTS:-6}" base="${FORGE_GH_RETRY_SLEEP:-2}" w out
|
|
60
|
+
[ "${1:-}" = --search ] && { mult=3; shift; }
|
|
61
|
+
case "$max" in ''|*[!0-9]*|0) max=6 ;; esac
|
|
62
|
+
case "$base" in ''|*[!0-9]*) base=2 ;; esac
|
|
63
|
+
while :; do
|
|
64
|
+
out=$("$@") && { printf '%s' "$out"; return 0; }
|
|
65
|
+
n=$((n+1)); [ "$n" -ge "$max" ] && return 1
|
|
66
|
+
w=$((base * mult * (1 << (n-1)))); [ "$w" -gt $((16*mult)) ] && w=$((16*mult))
|
|
67
|
+
[ "$w" -gt 0 ] && sleep "$w"
|
|
68
|
+
done
|
|
69
|
+
}
|
|
70
|
+
# END gh-retry
|
|
71
|
+
# One review-finding search fetch; fails (so gh_retry retries) on a gh error or non-array output.
|
|
72
|
+
fetch_findings() {
|
|
73
|
+
local raw
|
|
74
|
+
raw=$(gh issue list -R "$GH_REPO" --label "review-finding" --state open --search "PR #${1}" --limit 20 --json number,title,body 2>/dev/null) || return 1
|
|
75
|
+
printf '%s' "$raw" | jq -e 'type == "array"' >/dev/null 2>&1 || return 1
|
|
76
|
+
printf '%s' "$raw"
|
|
77
|
+
}
|
|
53
78
|
ALL_PR_NUMBERS=""
|
|
54
79
|
if [ "$CMD" != bundle ]; then
|
|
55
80
|
[ -n "$PRS_FILE" ] || { echo "review-staging-pre: --prs-file is required" >&2; exit 2; }
|
|
@@ -81,7 +106,7 @@ MERGE_PRS=$(git log origin/$DEFAULT_BRANCH..origin/$STAGING_BRANCH --merges --on
|
|
|
81
106
|
# Merge-subject numbers are PRs by construction; an unclassifiable number is kept (fail open: it may be a PR). forge#4127
|
|
82
107
|
# BEGIN bundle-pr-kind
|
|
83
108
|
# Prints "pr" or "issue" for issue number ${2} in repo ${1}; prints nothing and returns 1 when the read fails or is malformed.
|
|
84
|
-
bundle_pr_kind() { local k; k=$(gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
109
|
+
bundle_pr_kind() { local k; k=$(gh_retry gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
85
110
|
# END bundle-pr-kind
|
|
86
111
|
SUBJECT_PRS=""; SKIPPED_ISSUES=""
|
|
87
112
|
for n in $BUNDLE_PRS; do
|
|
@@ -115,7 +140,7 @@ DEGRADED_REVIEWS=""
|
|
|
115
140
|
READ_FAILURES=""
|
|
116
141
|
for pr_num in $ALL_PR_NUMBERS; do
|
|
117
142
|
# Capture each gh read and test its own status (as the scope step does): a failed read is never "not degraded" / "no findings".
|
|
118
|
-
IS_REVIEW_DEGRADED=$(gh pr view "$pr_num" -R $GH_REPO --json labels \
|
|
143
|
+
IS_REVIEW_DEGRADED=$(gh_retry gh pr view "$pr_num" -R $GH_REPO --json labels \
|
|
119
144
|
--jq '[.labels[].name] | any(. == "review-degraded")' 2>/dev/null) || IS_REVIEW_DEGRADED=unreadable
|
|
120
145
|
case "$IS_REVIEW_DEGRADED" in true|false) ;; *) READ_FAILURES="$READ_FAILURES #${pr_num}(labels)" ;; esac
|
|
121
146
|
if [ "$IS_REVIEW_DEGRADED" = "true" ]; then
|
|
@@ -128,12 +153,7 @@ for pr_num in $ALL_PR_NUMBERS; do
|
|
|
128
153
|
# narrow it with the anchored FINDING_PR_FILTER: title suffix `PR #N)` or a `**Source**: PR #N` line.
|
|
129
154
|
FINDING_PR_FILTER='[.[] | select((.title | endswith("PR #" + $n + ")")) or ((.body // "") | test("(^|\\n)\\*\\*Source\\*\\*: PR #" + $n + "([^0-9]|$)")))] | .[] | " - #\(.number): \(.title)"' # FINDING_PR_FILTER
|
|
130
155
|
OPEN_FINDINGS=""
|
|
131
|
-
if _RAW=$(
|
|
132
|
-
--label "review-finding" \
|
|
133
|
-
--state open \
|
|
134
|
-
--search "PR #${pr_num}" \
|
|
135
|
-
--limit 20 \
|
|
136
|
-
--json number,title,body 2>/dev/null) && printf '%s' "$_RAW" | jq -e 'type == "array"' >/dev/null 2>&1; then
|
|
156
|
+
if _RAW=$(gh_retry --search fetch_findings "$pr_num"); then
|
|
137
157
|
OPEN_FINDINGS=$(printf '%s' "$_RAW" | jq -r --arg n "${pr_num}" "$FINDING_PR_FILTER" 2>/dev/null) || READ_FAILURES="$READ_FAILURES #${pr_num}(findings)" # uses FINDING_PR_FILTER
|
|
138
158
|
else
|
|
139
159
|
READ_FAILURES="$READ_FAILURES #${pr_num}(findings)"
|
|
@@ -312,7 +332,7 @@ bundle_delta_credit() {
|
|
|
312
332
|
# END bundle-delta-credit
|
|
313
333
|
# BEGIN bundle-pr-kind
|
|
314
334
|
# Prints "pr" or "issue" for issue number ${2} in repo ${1}; prints nothing and returns 1 when the read fails or is malformed.
|
|
315
|
-
bundle_pr_kind() { local k; k=$(gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
335
|
+
bundle_pr_kind() { local k; k=$(gh_retry gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
316
336
|
# END bundle-pr-kind
|
|
317
337
|
# BEGIN bundle-scope-loop
|
|
318
338
|
_WHY=""
|
|
@@ -13,6 +13,7 @@ bad() { FAILN=$((FAILN+1)); echo "FAIL: $1"; }
|
|
|
13
13
|
command -v jq >/dev/null 2>&1 || { echo "review-staging-pre.test.sh: jq missing, skipped"; exit 0; }
|
|
14
14
|
command -v git >/dev/null 2>&1 || { echo "review-staging-pre.test.sh: git missing, skipped"; exit 0; }
|
|
15
15
|
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
16
|
+
export FORGE_GH_RETRY_SLEEP=0 # no real backoff in tests
|
|
16
17
|
export FORGEDOCK_HOME="$ROOT" # resolve helpers from this checkout, not an ambient install
|
|
17
18
|
|
|
18
19
|
# (1) the blocks live only here: present in the script, absent from the spec
|
|
@@ -34,6 +35,9 @@ cat > "$TMP/bin/gh" <<'STUB'
|
|
|
34
35
|
F="$FIX"; a="$*"
|
|
35
36
|
# fail-labels / fail-issues: the degraded-label / review-finding read fails (rate limit, 5xx, expired auth)
|
|
36
37
|
case "$a" in "pr view"*"--json labels"*) [ -f "$F/fail-labels" ] && exit 1 ;; "issue list"*) [ -f "$F/fail-issues" ] && exit 1 ;; esac
|
|
38
|
+
# flaky-<kind>: fail the first N reads of that kind (N = file content), then succeed (transient error)
|
|
39
|
+
flaky() { [ -f "$F/flaky-$1" ] || return 0; local c; c=$(cat "$F/count-$1" 2>/dev/null || echo 0); echo $((c+1)) > "$F/count-$1"; [ "$c" -lt "$(cat "$F/flaky-$1")" ] && exit 1; return 0; }
|
|
40
|
+
case "$a" in "pr view"*"--json labels"*) flaky labels ;; "issue list"*) flaky issues ;; "api repos/o/r/issues/"*) case "$a" in *"/comments"*) ;; *) flaky kind ;; esac ;; esac
|
|
37
41
|
case "$1 $2" in
|
|
38
42
|
"pr comment") while [ $# -gt 0 ]; do [ "$1" = --body ] && { printf '%s\n@@\n' "$2" >> "$F/posted"; }; shift; done; exit 0 ;;
|
|
39
43
|
"pr diff") [ -f "$F/files-$3" ] && { cat "$F/files-$3"; exit 0; }; exit 1 ;;
|
|
@@ -70,6 +74,10 @@ run() { (cd "$TMP/work" && bash "$PRE" "$@"); }
|
|
|
70
74
|
|
|
71
75
|
# (3) bundle: the merge-subject PR is kept, the issue cited in a subject is skipped, the list is persisted
|
|
72
76
|
newfix bundle; : > "$FIX/issue-77"
|
|
77
|
+
newfix bundleflaky; : > "$FIX/issue-77"; echo 2 > "$FIX/flaky-kind"
|
|
78
|
+
OUT=$(run bundle --repo o/r --default main --staging staging --pr 9 --sd "$TMP/sd" 2>&1); RC=$?
|
|
79
|
+
case "$OUT" in *"skipped):"*"77"*) ok ;; *) bad "a transient bundle_pr_kind failure must be retried so issue #77 is still skipped (got '$OUT')" ;; esac
|
|
80
|
+
newfix bundle; : > "$FIX/issue-77"
|
|
73
81
|
OUT=$(run bundle --repo o/r --default main --staging staging --pr 9 --sd "$TMP/sd" 2>&1); RC=$?
|
|
74
82
|
[ "$RC" -eq 0 ] && ok || bad "bundle must exit 0 (got $RC: $OUT)"
|
|
75
83
|
[ "$(cat "$TMP/sd/9_bundle-prs.txt" 2>/dev/null)" = 5 ] && ok || bad "bundle must persist exactly PR 5 (got '$(cat "$TMP/sd/9_bundle-prs.txt" 2>/dev/null)')"
|
|
@@ -106,6 +114,16 @@ OUT=$(run gate --repo o/r --pr 9 --prs-file "$TMP/prs2" 2>&1); RC=$?
|
|
|
106
114
|
case "$OUT" in *"RESULT: BLOCK DEPLOY"*) ok ;; *) bad "gh read failure blocks: must print RESULT: BLOCK DEPLOY (got '$OUT')" ;; esac
|
|
107
115
|
! grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "gh read failure blocks: must not post GATE_PASS"
|
|
108
116
|
grep -q 'FORGE:GATE_FAILURE:TYPE=gate-read-failure' "$FIX/posted" 2>/dev/null && ok || bad "gh read failure blocks: must post TYPE=gate-read-failure"
|
|
117
|
+
# forge#4572: a transient read failure is retried and does not block; a persistent one (above) still blocks.
|
|
118
|
+
newfix flaky; echo 2 > "$FIX/flaky-labels"; echo 3 > "$FIX/flaky-issues"
|
|
119
|
+
OUT=$(run gate --repo o/r --pr 9 --prs-file "$TMP/prs2" 2>&1); RC=$?
|
|
120
|
+
[ "$RC" -eq 0 ] && grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "transient label/finding read failures must be retried, not block (rc $RC: $OUT)"
|
|
121
|
+
newfix flakyfew; echo 5 > "$FIX/flaky-issues"
|
|
122
|
+
run gate --repo o/r --pr 9 --prs-file "$TMP/prs2" >/dev/null 2>&1; RC=$?
|
|
123
|
+
[ "$RC" -eq 0 ] && ok || bad "5 transient failures fit inside the default 6 attempts (rc $RC)"
|
|
124
|
+
newfix flakymany; echo 6 > "$FIX/flaky-issues"
|
|
125
|
+
run gate --repo o/r --pr 9 --prs-file "$TMP/prs" >/dev/null 2>&1; RC=$?
|
|
126
|
+
[ "$RC" -eq 1 ] && ! grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "failures beyond the attempt budget must still block (rc $RC)"
|
|
109
127
|
newfix issuefail; : > "$FIX/fail-issues"
|
|
110
128
|
run gate --repo o/r --pr 9 --prs-file "$TMP/prs" >/dev/null 2>&1; RC=$?
|
|
111
129
|
[ "$RC" -eq 1 ] && ! grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "a failed review-finding read alone must block without GATE_PASS (rc $RC)"
|