forgedock 1.31.0 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine-cli.mjs +51 -9
- package/bin/hooks/reread-nudge.mjs +65 -0
- package/bin/lean/batch.test.mjs +44 -1
- package/bin/lean/context.mjs +12 -1
- package/bin/lean/context.test.mjs +18 -1
- package/bin/lean/effects.mjs +13 -2
- package/bin/lean/lean.mjs +67 -15
- package/bin/lean/lean.test.mjs +177 -1
- package/bin/lean/lightlane.mjs +219 -0
- package/bin/lean/lightlane.test.mjs +237 -0
- package/bin/lean/orchestrate.mjs +45 -6
- package/bin/lean/precheck.test.mjs +5 -0
- package/bin/lean/prefilter.mjs +69 -8
- package/bin/lean/prefilter.test.mjs +109 -2
- package/bin/lean/prompts.mjs +3 -0
- package/bin/lean/sched.mjs +16 -1
- package/bin/lean/sched.test.mjs +48 -0
- package/bin/lean/schedd.test.mjs +21 -0
- package/bin/stats/efficiency.mjs +22 -6
- package/bin/stats/efficiency.test.mjs +35 -2
- package/bin/stats/index.mjs +25 -3
- package/bin/stats/local-source.mjs +59 -5
- package/bin/stats/query.mjs +133 -3
- package/bin/stats/stats.test.mjs +181 -5
- package/docs/CONFIG.md +3 -3
- package/docs/STATS.md +13 -0
- package/docs/eval/corpus/lightlane-v1.json +55 -0
- package/docs/eval/lightlane-bench.md +91 -0
- package/package.json +1 -1
- package/scripts/bench-lightlane.mjs +286 -0
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
4
|
import assert from "node:assert/strict";
|
|
5
|
-
import { prefilterIssue, flakyReason, makePrefilter, postDeployGate, postDeployOnly, deferredFrom, formatParkedLine, lazyDefaultBranch, ownLogins, resolveOwnLogins, formatPrefilterSummary, claimsFix, ciFailureRun, namedPaths, isTrustedIssueComment, DEFAULT_PREFILTER_MODEL } from "./prefilter.mjs";
|
|
5
|
+
import { prefilterIssue, flakyReason, makePrefilter, postDeployGate, postDeployOnly, deferredFrom, formatParkedLine, lazyDefaultBranch, ownLogins, resolveOwnLogins, formatPrefilterSummary, claimsFix, ciFailureRun, namedPaths, isTrustedIssueComment, DEFAULT_PREFILTER_MODEL, parseMinAge, itemWindowsMs, landedAt } from "./prefilter.mjs";
|
|
6
|
+
import { writeFileSync } from "node:fs";
|
|
7
|
+
import { tmpDir } from "../tests/_tmp.mjs";
|
|
8
|
+
import { execFileSync } from "node:child_process";
|
|
9
|
+
import { join } from "node:path";
|
|
6
10
|
import { parsePrefilterVerdict, prefilterPrompt } from "./prompts.mjs";
|
|
7
11
|
import { orchestrate, formatTable, formatOutcomeLists } from "./orchestrate.mjs";
|
|
8
|
-
import { triageFromCli } from "../engine-cli.mjs";
|
|
12
|
+
import { triageFromCli, readPostDeployMinAge } from "../engine-cli.mjs";
|
|
9
13
|
|
|
10
14
|
const fence = (o) => "```json\n" + JSON.stringify(o) + "\n```";
|
|
11
15
|
|
|
@@ -477,6 +481,109 @@ describe("post-deploy precondition gate (#4389)", () => {
|
|
|
477
481
|
});
|
|
478
482
|
});
|
|
479
483
|
|
|
484
|
+
describe("post_deploy.min_age soak window (#4425)", () => {
|
|
485
|
+
const MERGE = "a".repeat(40), PROMO = "b".repeat(40), MAIN = "c".repeat(40);
|
|
486
|
+
const H = 3600, LANDED = 1_800_000_000; // unix seconds of the promotion merge
|
|
487
|
+
const item = (id, why) => `- [ ] \`${id}: a health row exists\` — post-deploy: ${why}`;
|
|
488
|
+
const followup = (items) => `## Acceptance Criteria\n\n${items.join("\n")}\n\n<!-- FORGE:DEFERRED_FROM: #40 -->\n`;
|
|
489
|
+
const parentPr = [{ number: 90, title: "fix(x): health row (#40)", body: "", mergeCommit: { oid: MERGE } }];
|
|
490
|
+
// the parent fix reached main through promotion merge PROMO (first parent MAIN) at LANDED
|
|
491
|
+
const soakWorld = (o = {}) => {
|
|
492
|
+
const w = world({ mergedPrs: parentPr, ancestry: { [MERGE]: 0 }, ...o });
|
|
493
|
+
const git = w.io.git;
|
|
494
|
+
w.io.git = async (a, opts) => {
|
|
495
|
+
if (a[0] === "log" && a[1] === "--first-parent") { w.gitCalls.push(a); return o.walk ?? `${"d".repeat(40)} ${PROMO} ${LANDED + 9 * H}\n${PROMO} ${MAIN} ${MERGE} ${LANDED}\n${MAIN} ${"e".repeat(40)} ${LANDED - H}\n`; }
|
|
496
|
+
if (a[0] === "rev-list") { w.gitCalls.push(a); return o.below ?? `${"d".repeat(40)}\n${PROMO}\n`; }
|
|
497
|
+
if (a[0] === "log" && a[1] === "-1") { w.gitCalls.push(a); return String(LANDED - 5 * H); }
|
|
498
|
+
return git(a, opts);
|
|
499
|
+
};
|
|
500
|
+
return w;
|
|
501
|
+
};
|
|
502
|
+
const gate = (w, body, extra = {}) => postDeployGate({ iss: { number: 41, body }, repo: "o/r", lane: "staging", defaultBranch: "main", cwd: "/repo", io: w.io, ...extra });
|
|
503
|
+
|
|
504
|
+
it("parses min_age and the windows each post-deploy item's why states (largest wins, none -> null)", () => {
|
|
505
|
+
assert.equal(parseMinAge("24h"), 24 * H * 1000); assert.equal(parseMinAge("7d"), 7 * 24 * H * 1000); assert.equal(parseMinAge("2w"), 14 * 24 * H * 1000);
|
|
506
|
+
assert.equal(parseMinAge("30m"), 30 * 60e3); assert.equal(parseMinAge("0"), 0); assert.equal(parseMinAge(90), 90 * 60e3);
|
|
507
|
+
assert.equal(parseMinAge("soon"), null); assert.equal(parseMinAge(null), null);
|
|
508
|
+
assert.deepEqual(itemWindowsMs(followup([item("R1", "after 24h of traffic, then 7 days of retention"), item("R2", "only in prod")])), [7 * 24 * H * 1000, null]);
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
it("landedAt: the promotion merge time, or the SHA's own time when it is on the mainline", async () => {
|
|
512
|
+
const w = soakWorld();
|
|
513
|
+
assert.equal(await landedAt({ sha: MERGE, def: "main", io: w.io, cwd: "/repo" }), LANDED);
|
|
514
|
+
assert.deepEqual(w.gitCalls.slice(0, 2), [["rev-list", "--ancestry-path", `${MERGE}..origin/main`], ["log", "--first-parent", "--format=%H %P %ct", `${MERGE}..origin/main`]]);
|
|
515
|
+
const own = soakWorld({ walk: `${PROMO} ${MERGE} ${LANDED}\n`, below: `${PROMO}\n` });
|
|
516
|
+
assert.equal(await landedAt({ sha: MERGE, def: "main", io: own.io, cwd: "/repo" }), LANDED - 5 * H);
|
|
517
|
+
const tip = soakWorld({ walk: "", below: "" });
|
|
518
|
+
assert.equal(await landedAt({ sha: MERGE, def: "main", io: tip.io, cwd: "/repo" }), LANDED - 5 * H);
|
|
519
|
+
});
|
|
520
|
+
|
|
521
|
+
it("landedAt on a real repo: the promotion merge that brought a staging fix in through its second parent", async () => {
|
|
522
|
+
const d = tmpDir("landed-");
|
|
523
|
+
const run = (args, t) => execFileSync("git", args, { cwd: d, encoding: "utf8", env: { ...process.env, GIT_AUTHOR_DATE: `@${t} +0000`, GIT_COMMITTER_DATE: `@${t} +0000`,
|
|
524
|
+
GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t", GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" } }).trim();
|
|
525
|
+
run(["init", "-q", "-b", "main"], 1);
|
|
526
|
+
run(["commit", "-q", "--allow-empty", "-m", "base"], LANDED - 10 * H);
|
|
527
|
+
run(["checkout", "-q", "-b", "staging"], 1);
|
|
528
|
+
run(["commit", "-q", "--allow-empty", "-m", "fix"], LANDED - 8 * H);
|
|
529
|
+
const fix = run(["rev-parse", "HEAD"], 1);
|
|
530
|
+
run(["checkout", "-q", "main"], 1);
|
|
531
|
+
run(["commit", "-q", "--allow-empty", "-m", "hotfix on main"], LANDED - 2 * H);
|
|
532
|
+
run(["merge", "-q", "--no-ff", "-m", "promote", "staging"], LANDED);
|
|
533
|
+
run(["commit", "-q", "--allow-empty", "-m", "later"], LANDED + 5 * H);
|
|
534
|
+
run(["update-ref", "refs/remotes/origin/main", "HEAD"], 1);
|
|
535
|
+
const io = { git: async (a, o) => run(a, 1) + "\n" };
|
|
536
|
+
assert.equal(await landedAt({ sha: fix, def: "main", io, cwd: d }), LANDED);
|
|
537
|
+
assert.equal(await landedAt({ sha: fix.slice(0, 12), def: "main", io, cwd: d }), LANDED);
|
|
538
|
+
assert.equal(await landedAt({ sha: run(["rev-parse", "HEAD~1^1"], 1), def: "main", io, cwd: d }), LANDED - 2 * H); // on the mainline itself
|
|
539
|
+
assert.equal(await landedAt({ sha: "f".repeat(40), def: "main", io, cwd: d }), null);
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
it("parks awaiting-min-age until max(item window ?? min_age) after landing, then is ready", async () => {
|
|
543
|
+
const body = followup([item("R1", "wait 7d of production data"), item("R2", "only in prod")]);
|
|
544
|
+
const g = await gate(soakWorld(), body, { minAge: parseMinAge("24h"), now: (LANDED + 48 * H) * 1000 });
|
|
545
|
+
assert.equal(g.state, "awaiting-min-age");
|
|
546
|
+
assert.equal(g.eligibleAt, new Date((LANDED + 7 * 24 * H) * 1000).toISOString());
|
|
547
|
+
assert.match(g.detail, /^fix of #40 on main since .*; min age 7d until /);
|
|
548
|
+
// no stated window: the min_age floor applies
|
|
549
|
+
const short = followup([item("R2", "only in prod")]);
|
|
550
|
+
assert.equal((await gate(soakWorld(), short, { minAge: parseMinAge("24h"), now: (LANDED + 23 * H) * 1000 })).state, "awaiting-min-age");
|
|
551
|
+
assert.equal((await gate(soakWorld(), short, { minAge: parseMinAge("24h"), now: (LANDED + 25 * H) * 1000 })).state, "ready");
|
|
552
|
+
assert.equal((await gate(soakWorld(), body, { minAge: 0, now: (LANDED + 8 * 24 * H) * 1000 })).state, "ready");
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
it("unset min_age changes nothing (no landing-time git calls); an unknown landing time fails open", async () => {
|
|
556
|
+
const w = soakWorld();
|
|
557
|
+
assert.equal((await gate(w, followup([item("R1", "wait 7d")]))).state, "ready");
|
|
558
|
+
assert.ok(!w.gitCalls.some((a) => a[0] === "log" || a[0] === "rev-list"));
|
|
559
|
+
const broken = soakWorld();
|
|
560
|
+
const git = broken.io.git;
|
|
561
|
+
broken.io.git = async (a, o) => { if (a[0] === "rev-list") throw Object.assign(new Error("bad"), { code: 128 }); return git(a, o); };
|
|
562
|
+
const g = await gate(broken, followup([item("R1", "wait 7d")]), { minAge: 0 });
|
|
563
|
+
assert.equal(g.state, "unresolved"); assert.match(g.detail, /landing time of aaaaaaaaaaaa on main unknown/);
|
|
564
|
+
});
|
|
565
|
+
|
|
566
|
+
it("prefilterIssue parks it with the awaiting-promotion label, one comment, and an 'awaiting min age:' report line", async () => {
|
|
567
|
+
const w = soakWorld({ issues: { 41: { body: followup([item("R1", "wait 7d")]) } } });
|
|
568
|
+
w.io.now = () => (LANDED + H) * 1000;
|
|
569
|
+
const r = await pf(w, 41, { defaultBranch: "main", minAge: parseMinAge("24h") });
|
|
570
|
+
assert.equal(r.parked, true); assert.equal(r.kind, "awaiting-min-age"); assert.equal(r.cost_usd, 0);
|
|
571
|
+
assert.deepEqual(w.edits, [[41, "--add-label", "awaiting-promotion"]]);
|
|
572
|
+
assert.equal(w.comments.length, 1); assert.match(w.comments[0][1], /Parked: awaiting its post-deploy soak window/);
|
|
573
|
+
assert.equal(formatParkedLine([{ issue: 41, prefilter: r }, { issue: 42, prefilter: { parked: true, kind: "awaiting-promotion", lane: "staging", default: "main" } }]),
|
|
574
|
+
"awaiting promotion: #42 (fixes on staging, not on main)\nawaiting min age: #41 (fix on main, soak window not elapsed)");
|
|
575
|
+
});
|
|
576
|
+
|
|
577
|
+
it("readPostDeployMinAge reads forge.yaml post_deploy.min_age; absent or invalid -> null", async () => {
|
|
578
|
+
const d = tmpDir("minage-");
|
|
579
|
+
assert.equal(await readPostDeployMinAge(d), null);
|
|
580
|
+
writeFileSync(join(d, "forge.yaml"), "post_deploy:\n min_age: \"24h\"\n");
|
|
581
|
+
assert.equal(await readPostDeployMinAge(d), 24 * H * 1000);
|
|
582
|
+
writeFileSync(join(d, "forge.yaml"), "post_deploy:\n min_age: later\n");
|
|
583
|
+
assert.equal(await readPostDeployMinAge(d), null);
|
|
584
|
+
});
|
|
585
|
+
});
|
|
586
|
+
|
|
480
587
|
describe("#4467: isTrustedIssueComment", () => {
|
|
481
588
|
it("trusts the App in gh issue view and REST shapes and trusted associations; not other contributors", () => {
|
|
482
589
|
assert.equal(isTrustedIssueComment({ author: { login: "rapiercraft-forgedock" }, authorAssociation: "CONTRIBUTOR" }), true);
|
package/bin/lean/prompts.mjs
CHANGED
|
@@ -24,6 +24,9 @@ export const FORGEDOCK_CLI = fileURLToPath(new URL("../forgedock.mjs", import.me
|
|
|
24
24
|
const HOST_RULES = `Host rules (many lanes share this machine): run only targeted tests (specific files or -k filters), never the full suite. Never create extra checkouts, copies or base clones of the repo; compare with git show or git diff origin/<base>. Never install browsers. Keep scratch files under $TMPDIR. Cap verification workers at $FORGE_HOST_PARALLELISM when set: turbo --concurrency, vitest/jest --maxWorkers, playwright --workers, node --test --test-concurrency (eslint via turbo). For any typecheck, lint or test you run, first run \`node ${CHECKS_CLI} --base origin/<base>\` after your edits and run the scoped commands it prints instead of the full tsc/eslint/test scripts (it widens itself when impact is unknown: lockfile, tsconfig, CI config); once they pass, rerun it with --publish to share the incremental caches. Lint changed packages only, never dependents (turbo ...[base], nx affected). node_modules/.venv may be hardlinked read-only from a shared cache: after changing a lockfile, or when an install fails with EACCES, delete that dir first (rm -rf is safe), then install; never chmod or edit files inside it.
|
|
25
25
|
Quiet output (tool results are re-read every later turn): run tests quietly (-q or a dot/summary reporter); on failure read \`tail -40\` plus the failing test ids (grep FAILED / not ok / FAIL), never the whole log. Send output over 8 KB to a $TMPDIR file; tail or grep it. Issue independent read-only calls (reads, greps, git log/blame, gh) in the same turn or one shell call. Read each file's ranges once (sed -n a,bp or Read offset/limit, never a whole large file) and keep notes; don't re-read it.`;
|
|
26
26
|
|
|
27
|
+
/** #4450: the host-rules block of the builder prompts (toolchain, sparse and repo-facts lines included), reused by the light-lane prompt. */
|
|
28
|
+
export const hostRules = ({ toolchain = null, sparse = null, facts = null } = {}) => `${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${factsRule(facts)}`;
|
|
29
|
+
|
|
27
30
|
/** Verify-by-CI (#4167): in ci/hybrid mode the engine runs the repo's CI on GitHub runners; the host runs only light checks. */
|
|
28
31
|
const LIGHT = "run only formatters and linters on the files you changed";
|
|
29
32
|
const VERIFY_RUN = { ci: `${LIGHT} (no type-checks, test suites or test containers)`, hybrid: "run the targeted validate commands" };
|
package/bin/lean/sched.mjs
CHANGED
|
@@ -17,6 +17,9 @@
|
|
|
17
17
|
* (`forgedock sched cap N`), else the caller's forge.yaml `orchestration.global_parallel`, else none. Fail-open: an
|
|
18
18
|
* unreadable registry or a lock not taken within `lockWaitMs` grants (`fallback: true`), since a stuck registry would
|
|
19
19
|
* stall every batch on the host.
|
|
20
|
+
* Per-issue lock (#4481), asked for per request with `exclusive: true` in the entry (not stored; the batch runner sets it):
|
|
21
|
+
* such a key is never granted while a live slot of another batch holds the same (non-null) repo and issue, so two batches that both include issue N (a resumed batch and a new one) never run two lanes for N at once;
|
|
22
|
+
* `tryAcquire` answers `{ok:false, conflict:<batch>}` and drops the key from the queue (it would hold a grant rank).
|
|
20
23
|
*/
|
|
21
24
|
import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeSync } from "node:fs";
|
|
22
25
|
import { homedir } from "node:os";
|
|
@@ -50,6 +53,14 @@ const posInt = (k) => (Number.isInteger(k) && k >= 1 ? k : null);
|
|
|
50
53
|
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
51
54
|
const byRank = ([ka, a], [kb, b]) => (a.priority - b.priority) || (a.seq - b.seq) || (ka < kb ? -1 : ka > kb ? 1 : 0);
|
|
52
55
|
const groupOf = (k, e) => e?.repo ?? e?.batch ?? k;
|
|
56
|
+
/** #4481: the batch of another live slot running the same repo and issue as an `exclusive` `entry`, or null (no repo: never). */
|
|
57
|
+
const holderOf = (s, key, entry) => {
|
|
58
|
+
if (entry?.exclusive !== true || !entry.repo || entry.issue == null) return null;
|
|
59
|
+
for (const [k, e] of Object.entries(s.slots)) {
|
|
60
|
+
if (k !== key && e?.repo === entry.repo && e?.issue === entry.issue && (e.batch ?? null) !== (entry.batch ?? null)) return e.batch ?? k;
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
};
|
|
53
64
|
|
|
54
65
|
/**
|
|
55
66
|
* Queue entries `[key, entry]` in grant order: priority ascending, then a weighted fair share across repos within each
|
|
@@ -88,7 +99,7 @@ export function grantOrder(s) {
|
|
|
88
99
|
|
|
89
100
|
/**
|
|
90
101
|
* Handle on the registry. `pid` is this dispatcher; `alive(pid)` is the liveness probe (injected in tests).
|
|
91
|
-
* @returns {{dir, tryAcquire, offer, withdraw, release, setLanePid, adoptSlot, cap, setCap, weights, setWeight, list}}
|
|
102
|
+
* @returns {{dir, tryAcquire, holder, offer, withdraw, release, setLanePid, adoptSlot, cap, setCap, weights, setWeight, list}}
|
|
92
103
|
*/
|
|
93
104
|
export function openSched(dir = schedDir(), { now = () => Date.now(), pid = process.pid, alive = pidAlive, lockWaitMs = LOCK_WAIT_MS } = {}) {
|
|
94
105
|
const file = join(dir, "registry.json"), lockFile = join(dir, "registry.lock");
|
|
@@ -167,6 +178,8 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
|
|
|
167
178
|
try {
|
|
168
179
|
return mutate((s) => {
|
|
169
180
|
if (s.slots[key]) return { ok: true };
|
|
181
|
+
const holder = holderOf(s, key, entry);
|
|
182
|
+
if (holder !== null) { delete s.queue[key]; return { ok: false, conflict: holder, reason: `#${entry.issue} has a live lane in batch ${holder}` }; }
|
|
170
183
|
enqueue(s, key, entry);
|
|
171
184
|
const cap = effCap(s, callerCap);
|
|
172
185
|
if (cap !== null) {
|
|
@@ -182,6 +195,8 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
|
|
|
182
195
|
});
|
|
183
196
|
} catch (e) { return { ok: true, fallback: true, reason: e?.message ?? String(e) }; }
|
|
184
197
|
},
|
|
198
|
+
/** #4481: the other batch holding a live slot for `{repo, issue}` (not `batch`), or null; null on any error. */
|
|
199
|
+
holder: (entry = {}) => { try { return mutate((s) => holderOf(s, null, entry)); } catch { return null; } },
|
|
185
200
|
/** Queue entries `[{key, batch, repo, issue, priority}]` (their FIFO position is kept on a re-offer). */
|
|
186
201
|
offer: (entries) => { try { mutate((s) => { for (const e of entries) enqueue(s, e.key, e); }); } catch { /* fail-open */ } },
|
|
187
202
|
withdraw: (keys) => { if (!keys.length) return; try { mutate((s) => { for (const k of keys) delete s.queue[k]; }); } catch { /* fail-open */ } },
|
package/bin/lean/sched.test.mjs
CHANGED
|
@@ -434,3 +434,51 @@ describe("longest-first admission (#4409)", () => {
|
|
|
434
434
|
assert.equal(estCalls, 0);
|
|
435
435
|
});
|
|
436
436
|
});
|
|
437
|
+
|
|
438
|
+
describe("#4481: one lane per repo and issue across batches", () => {
|
|
439
|
+
it("the registry refuses a second batch's key for a live repo+issue slot, names the batch and drops it from the queue", () => {
|
|
440
|
+
const { handle, live } = world();
|
|
441
|
+
const a = handle(201), b = handle(202);
|
|
442
|
+
assert.equal(a.tryAcquire("A#5", { batch: "A", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true);
|
|
443
|
+
const r = b.tryAcquire("B#5", { batch: "B", repo: "o/r", issue: 5, exclusive: true }, 4);
|
|
444
|
+
assert.deepEqual([r.ok, r.conflict], [false, "A"]);
|
|
445
|
+
assert.match(r.reason, /#5 has a live lane in batch A/);
|
|
446
|
+
assert.deepEqual(b.list().queue, [], "a refused key holds no grant rank");
|
|
447
|
+
assert.equal(b.holder({ batch: "B", repo: "o/r", issue: 5, exclusive: true }), "A");
|
|
448
|
+
assert.equal(a.holder({ batch: "A", repo: "o/r", issue: 5, exclusive: true }), null, "its own batch is no conflict");
|
|
449
|
+
assert.equal(b.tryAcquire("B#6", { batch: "B", repo: "o/r", issue: 6, exclusive: true }, 4).ok, true, "another issue");
|
|
450
|
+
assert.equal(b.tryAcquire("C#5", { batch: "C", repo: "o/other", issue: 5, exclusive: true }, 4).ok, true, "another repo");
|
|
451
|
+
assert.equal(b.tryAcquire("D#5", { batch: "D", repo: null, issue: 5, exclusive: true }, 4).ok, true, "no repo: unchanged");
|
|
452
|
+
assert.equal(a.tryAcquire("A#5", { batch: "A", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true, "a granted key stays granted");
|
|
453
|
+
assert.equal(b.tryAcquire("E#5", { batch: "E", repo: "o/r", issue: 5 }, null).ok, true, "not exclusive: unchanged");
|
|
454
|
+
b.release("E#5");
|
|
455
|
+
live.delete(201); // A's dispatcher dies with no lane: its slot is reclaimed
|
|
456
|
+
assert.equal(b.tryAcquire("B#5", { batch: "B", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true);
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
it("two batches that both include issue N never run two lanes for N; the second defers naming the first and runs its other issues", async () => {
|
|
460
|
+
const { handle } = world();
|
|
461
|
+
const regA = handle(301), regB = handle(302);
|
|
462
|
+
const active = new Map(), maxActive = new Map(), ranB = [];
|
|
463
|
+
let releaseA;
|
|
464
|
+
const gateA = new Promise((r) => { releaseA = r; });
|
|
465
|
+
const lane = (n) => { active.set(n, (active.get(n) ?? 0) + 1); maxActive.set(n, Math.max(maxActive.get(n) ?? 0, active.get(n))); };
|
|
466
|
+
const pA = orchestrate({ issues: [5], readBody: async () => "", pollMs: 2, batchId: "bA", sched: { registry: regA, batch: "bA", repo: "o/r", cap: null, exclusive: true },
|
|
467
|
+
runOne: async (n) => { lane(n); await gateA; active.set(n, active.get(n) - 1); return { terminalReason: "merged", cost_usd: 0 }; } });
|
|
468
|
+
await until(() => active.get(5) === 1, "batch A's lane");
|
|
469
|
+
const rowsB = [];
|
|
470
|
+
const pB = orchestrate({ issues: [5, 6], readBody: async () => "", pollMs: 2, batchId: "bB", sched: { registry: regB, batch: "bB", repo: "o/r", cap: null, exclusive: true },
|
|
471
|
+
onUpdate: (r) => rowsB.push(r),
|
|
472
|
+
runOne: async (n) => { lane(n); ranB.push(n); active.set(n, active.get(n) - 1); return { terminalReason: "merged", cost_usd: 0 }; } });
|
|
473
|
+
await until(() => ranB.includes(6), "batch B's other issue");
|
|
474
|
+
const deferred = rowsB.filter((r) => r.issue === 5 && r.outcome === "deferred");
|
|
475
|
+
assert.equal(deferred.length, 1, "one row per other batch");
|
|
476
|
+
assert.match(deferred[0].detail, /live lane in batch bA/);
|
|
477
|
+
assert.ok(!ranB.includes(5), "no second lane while batch A runs #5");
|
|
478
|
+
releaseA();
|
|
479
|
+
await pA;
|
|
480
|
+
const done = await pB;
|
|
481
|
+
assert.deepEqual(done.map((r) => [r.issue, r.outcome]).sort(), [[5, "merged"], [6, "merged"]]);
|
|
482
|
+
assert.equal(maxActive.get(5), 1);
|
|
483
|
+
});
|
|
484
|
+
});
|
package/bin/lean/schedd.test.mjs
CHANGED
|
@@ -214,6 +214,27 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
|
|
|
214
214
|
} finally { h.cleanup(); }
|
|
215
215
|
});
|
|
216
216
|
|
|
217
|
+
it("#4481: a batch cancelled through the daemon is not resumed by a schedd restart", async () => {
|
|
218
|
+
const h = host();
|
|
219
|
+
try {
|
|
220
|
+
const statusIds = async () => (await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" })).batches.map((x) => x.id);
|
|
221
|
+
const kill = async (d) => { const gone = new Promise((r) => d.on("exit", r)); d.kill("SIGKILL"); await gone; };
|
|
222
|
+
let d = await h.daemon();
|
|
223
|
+
await withRoot(h, () => capture(async () => {
|
|
224
|
+
const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
|
|
225
|
+
await until(() => h.launched().length === 1 && h.lanePids().length === 1);
|
|
226
|
+
const [idA] = listBatches(h.root);
|
|
227
|
+
assert.equal((await orchestrateFromCli(["--cancel", idA], h.client)).cancelled, true);
|
|
228
|
+
await a;
|
|
229
|
+
await kill(d);
|
|
230
|
+
d = await h.daemon();
|
|
231
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
232
|
+
assert.deepEqual(await statusIds(), [], "the restarted daemon hosts no cancelled batch");
|
|
233
|
+
assert.deepEqual(h.launched(), [1]);
|
|
234
|
+
}));
|
|
235
|
+
} finally { h.cleanup(); }
|
|
236
|
+
});
|
|
237
|
+
|
|
217
238
|
it("single instance: a live daemon keeps its lock; a dead owner's lock and socket are taken over", async () => {
|
|
218
239
|
const h = host();
|
|
219
240
|
try {
|
package/bin/stats/efficiency.mjs
CHANGED
|
@@ -59,14 +59,18 @@ function parseLog(text) {
|
|
|
59
59
|
return out;
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
-
|
|
62
|
+
export const LANE_SHAPES = Object.freeze(["light", "upgraded", "full"]);
|
|
63
|
+
|
|
64
|
+
/** Every RUN_START..RUN_TERMINAL slice of one run log (all of them, not only the last). `shape` (#4452): light (LIGHT_DONE), upgraded (LIGHT_UPGRADE) or full. */
|
|
63
65
|
export function issueSlices(events) {
|
|
64
66
|
const out = [];
|
|
65
67
|
let cur = null;
|
|
66
68
|
for (const e of Array.isArray(events) ? events : []) {
|
|
67
69
|
if (!e || typeof e !== "object") continue;
|
|
68
|
-
if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, attempts: [] }; out.push(cur); continue; }
|
|
70
|
+
if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, shape: "full", attempts: [] }; out.push(cur); continue; }
|
|
69
71
|
if (!cur) continue;
|
|
72
|
+
if (e.event === "LIGHT_DONE") cur.shape = "light";
|
|
73
|
+
else if (e.event === "LIGHT_UPGRADE") cur.shape = "upgraded";
|
|
70
74
|
if (e.event === "PHASE_ATTEMPT_USAGE") {
|
|
71
75
|
cur.attempts.push({ phase: e.phase ?? "other", trigger: e.trigger ?? null, restart_of: e.restart_of ?? null, model: e.model ?? null,
|
|
72
76
|
turns: num(e.turns), cacheRead: num(e.usage?.cache_read_input_tokens), cost: costOf(e),
|
|
@@ -145,9 +149,18 @@ const group = (xs) => {
|
|
|
145
149
|
*/
|
|
146
150
|
export function efficiencyMetrics({ entries = [], logs = new Map(), issues = null, repo = null, now = 0, notes = [] } = {}) {
|
|
147
151
|
const merged = [];
|
|
152
|
+
const inScope = (row, settleAt) => issueSlices(logs.get(row.issue) ?? []).filter((s) => (!repo || !s.repo || s.repo === repo) && (!settleAt || !s.startTs || s.startTs <= settleAt));
|
|
153
|
+
// #4452: light/full split and upgrade rate over every lane of every row (all outcomes); cost is the #4407 delta sum
|
|
154
|
+
const shapes = Object.fromEntries(LANE_SHAPES.map((k) => [k, { n: 0, cost_usd: 0 }]));
|
|
155
|
+
for (const { row, settleAt } of entries) {
|
|
156
|
+
if (!Number.isSafeInteger(row?.issue)) continue;
|
|
157
|
+
for (const s of inScope(row, settleAt)) { const g = shapes[s.shape] ?? shapes.full; g.n++; g.cost_usd += sum(s.attempts.map((a) => a.cost)); }
|
|
158
|
+
}
|
|
159
|
+
for (const g of Object.values(shapes)) g.cost_usd = r(g.cost_usd, 1e6);
|
|
160
|
+
const lanes = { ...shapes, upgrade_rate: r(per(shapes.upgraded.n, shapes.light.n + shapes.upgraded.n)) };
|
|
148
161
|
for (const { row, settleAt } of entries) {
|
|
149
162
|
if (row?.outcome !== "merged") continue;
|
|
150
|
-
const slices =
|
|
163
|
+
const slices = inScope(row, settleAt);
|
|
151
164
|
const atts = slices.flatMap((s) => s.attempts), impl = atts.filter((a) => a.phase === "implement");
|
|
152
165
|
const retries = Object.fromEntries(RETRY_CAUSES.map((c) => [c, 0]));
|
|
153
166
|
for (const s of slices) for (const [c, k] of Object.entries(sliceRetries(s))) retries[c] += k;
|
|
@@ -156,7 +169,7 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
156
169
|
const term = [...slices].reverse().find((s) => s.terminal?.reason === "merged")?.terminal;
|
|
157
170
|
merged.push({ issue: row.issue, slices: slices.length, cost: sum(atts.map((a) => a.cost)), implCost: sum(impl.map((a) => a.cost)),
|
|
158
171
|
reviewCost: sum(atts.filter((a) => a.phase === "review").map((a) => a.cost)), implTurns: sum(impl.map((a) => a.turns)), implCache: sum(impl.map((a) => a.cacheRead)),
|
|
159
|
-
size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl) });
|
|
172
|
+
size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl), shape: slices.at(-1)?.shape ?? null });
|
|
160
173
|
}
|
|
161
174
|
const all = group(merged);
|
|
162
175
|
const bySize = {};
|
|
@@ -202,10 +215,10 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
|
|
|
202
215
|
retries: { total, per_merge: r(per(total, merged.length), 100), by_cause: retries, lane_reruns: sum(merged.map((m) => Math.max(0, m.slices - 1))) },
|
|
203
216
|
no_action: { lanes: sum(Object.values(noAction).map((g) => g.n)), cost_usd: r(sum(Object.values(noAction).map((g) => g.cost_usd)), 1e6),
|
|
204
217
|
by_kind: Object.fromEntries(Object.entries(noAction).map(([k, g]) => [k, { ...g, cost_usd: r(g.cost_usd, 1e6) }])) },
|
|
205
|
-
quality, acceptance,
|
|
218
|
+
quality, acceptance, lanes,
|
|
206
219
|
tool_calls_per_turn_p50: median(withTools.map((m) => m.tools.per_turn)), rereads_per_merge: withTools.length ? r(per(sum(withTools.map((m) => m.tools.rereads)), withTools.length), 100) : null,
|
|
207
220
|
per_issue: merged.map((m) => ({ issue: m.issue, slices: m.slices, cost_usd: r(m.cost, 1e6), implement_usd: r(m.implCost, 1e6), implement_turns: m.implTurns,
|
|
208
|
-
implement_cache_read: m.implCache, size: m.size, retries: m.retries, ...(m.info ? { quality: issueQuality(m.info.body).tag } : {}),
|
|
221
|
+
implement_cache_read: m.implCache, size: m.size, retries: m.retries, ...(m.shape ? { shape: m.shape } : {}), ...(m.info ? { quality: issueQuality(m.info.body).tag } : {}),
|
|
209
222
|
...(m.tools ? { implement_tool_calls_per_turn: m.tools.per_turn, implement_rereads: m.tools.rereads } : {}) })),
|
|
210
223
|
notes,
|
|
211
224
|
};
|
|
@@ -315,5 +328,8 @@ export function formatEfficiencyLines(m, label = "") {
|
|
|
315
328
|
lines.push(a ? `acceptance (7d after promotion): pass ${a.pass}/${a.pass + a.fail}${a.pass_rate == null ? "" : ` (${Math.round(a.pass_rate * 100)}%)`}, pending ${a.pending}, unprobed ${a.unprobed}`
|
|
316
329
|
: "acceptance (7d after promotion): n/a");
|
|
317
330
|
lines.push(`tools: calls/turn p50 ${f(m.tool_calls_per_turn_p50)}, re-reads (>=3x same path)/merge ${f(m.rereads_per_merge)}`); // #4410
|
|
331
|
+
const ln = m.lanes; // #4452
|
|
332
|
+
lines.push(ln ? `lanes: ${LANE_SHAPES.map((k) => `${k} ${ln[k]?.n ?? 0} (${usd(ln[k]?.cost_usd ?? 0)})`).join(", ")}; upgrade rate ${ln.upgrade_rate == null ? "n/a"
|
|
333
|
+
: `${Math.round(ln.upgrade_rate * 100)}% (${ln.upgraded?.n ?? 0}/${(ln.light?.n ?? 0) + (ln.upgraded?.n ?? 0)})`}` : "lanes: n/a");
|
|
318
334
|
return lines;
|
|
319
335
|
}
|
|
@@ -115,11 +115,44 @@ describe("#4402 efficiencyMetrics", () => {
|
|
|
115
115
|
});
|
|
116
116
|
it("renders n/a, never NaN, for an empty cohort", () => {
|
|
117
117
|
const lines = formatEfficiencyLines(efficiencyMetrics({}));
|
|
118
|
-
assert.equal(lines.length,
|
|
118
|
+
assert.equal(lines.length, 9);
|
|
119
119
|
assert.ok(!lines.some((l) => /NaN|undefined/.test(l)), lines.join("\n"));
|
|
120
120
|
assert.match(lines[0], /^efficiency: merged 0, n\/a\/merge/);
|
|
121
121
|
assert.equal(lines[6], "acceptance (7d after promotion): n/a");
|
|
122
122
|
assert.equal(lines[7], "tools: calls/turn p50 n/a, re-reads (>=3x same path)/merge n/a");
|
|
123
|
+
assert.equal(lines[8], "lanes: light 0 ($0.00), upgraded 0 ($0.00), full 0 ($0.00); upgrade rate n/a");
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
describe("#4452 light/full split and upgrade rate", () => {
|
|
128
|
+
// issue 1: light, merged; 2: light then upgraded in the same run, merged; 3: full, merged; 4: light, failed; 5: upgraded, failed
|
|
129
|
+
const logs = new Map([
|
|
130
|
+
[1, [start("2026-10-22T10:00:00Z"), { event: "LIGHT_ELIGIBILITY", light: true }, att("implement", { trigger: "build", cost_usd: 0.9, cost_usd_delta: 0.4 }), { event: "LIGHT_DONE" }, att("review", { cost_usd: 0.05 }), term("merged", "merged")]],
|
|
131
|
+
[2, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.3 }), { event: "LIGHT_UPGRADE", reasons: [{ code: "diff_lines" }] }, att("implement", { trigger: "plan", cost_usd_delta: 0.5 }), term("merged", "merged")]],
|
|
132
|
+
[3, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "plan", cost_usd_delta: 0.6 }), att("implement", { trigger: "build", cost_usd: 1.2, cost_usd_delta: 0.4 }), term("merged", "merged")]],
|
|
133
|
+
[4, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.2 }), { event: "LIGHT_DONE" }, term("failed", "ci.red")]],
|
|
134
|
+
[5, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.1 }), { event: "LIGHT_UPGRADE" }, term("failed", "conformance.unimplemented")]],
|
|
135
|
+
]);
|
|
136
|
+
const entries = [1, 2, 3, 4, 5].map((issue) => ({ row: { issue, outcome: issue <= 3 ? "merged" : "failed" }, settleAt: null }));
|
|
137
|
+
const m = efficiencyMetrics({ entries, logs, repo: "o/r" });
|
|
138
|
+
|
|
139
|
+
it("tags each lane slice by shape from LIGHT_DONE / LIGHT_UPGRADE", () => {
|
|
140
|
+
assert.deepEqual(issueSlices(logs.get(1)).map((s) => s.shape), ["light"]);
|
|
141
|
+
assert.deepEqual(issueSlices(logs.get(2)).map((s) => s.shape), ["upgraded"]);
|
|
142
|
+
assert.deepEqual(issueSlices(logs.get(3)).map((s) => s.shape), ["full"]);
|
|
143
|
+
});
|
|
144
|
+
it("splits every lane (all outcomes) by shape with true #4407 delta costs and computes the upgrade rate", () => {
|
|
145
|
+
assert.deepEqual(m.lanes, {
|
|
146
|
+
light: { n: 2, cost_usd: 0.65 }, // 0.4 delta (not the 0.9 running total) + 0.05 review + 0.2
|
|
147
|
+
upgraded: { n: 2, cost_usd: 0.9 },
|
|
148
|
+
full: { n: 1, cost_usd: 1 }, // 0.6 + 0.4 delta, not 1.2
|
|
149
|
+
upgrade_rate: 0.5,
|
|
150
|
+
});
|
|
151
|
+
assert.deepEqual(m.per_issue.map((x) => [x.issue, x.shape]), [[1, "light"], [2, "upgraded"], [3, "full"]]);
|
|
152
|
+
});
|
|
153
|
+
it("the batch report prints the split and the upgrade rate", () => {
|
|
154
|
+
const lines = formatEfficiencyLines(m);
|
|
155
|
+
assert.equal(lines.at(-1), "lanes: light 2 ($0.65), upgraded 2 ($0.90), full 1 ($1.00); upgrade rate 50% (2/4)");
|
|
123
156
|
});
|
|
124
157
|
});
|
|
125
158
|
|
|
@@ -142,7 +175,7 @@ describe("#4410 tool calls per turn and re-reads", () => {
|
|
|
142
175
|
const p = Object.fromEntries(m.per_issue.map((x) => [x.issue, x]));
|
|
143
176
|
assert.equal(p[1].implement_tool_calls_per_turn, 1.5); assert.equal(p[1].implement_rereads, 2);
|
|
144
177
|
assert.equal("implement_tool_calls_per_turn" in p[4], false);
|
|
145
|
-
assert.equal(formatEfficiencyLines(m).
|
|
178
|
+
assert.equal(formatEfficiencyLines(m).find((l) => l.startsWith("tools:")), "tools: calls/turn p50 1.50, re-reads (>=3x same path)/merge 1.00");
|
|
146
179
|
});
|
|
147
180
|
it("null without tool data (older logs)", () => {
|
|
148
181
|
const e = efficiencyMetrics({ entries: [entries[3]], logs, repo: "o/r", now: NOW });
|
package/bin/stats/index.mjs
CHANGED
|
@@ -11,10 +11,10 @@ import { dirname, join } from "node:path";
|
|
|
11
11
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
12
12
|
import { aggregate, computeAlerts, DEFAULT_ALERTS, DEFAULT_MIN_N } from "./aggregate.mjs";
|
|
13
13
|
import { defaultGh, fetchGithubRecords, isValidRepo, toModel } from "./github-source.mjs";
|
|
14
|
-
import { readLanes, readLocalRuns, runsDir } from "./local-source.mjs";
|
|
14
|
+
import { readLanes, readLocalLaneRuns, readLocalRuns, runsDir } from "./local-source.mjs";
|
|
15
15
|
import { parseForgeYaml } from "../forge-utils.mjs";
|
|
16
16
|
import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
|
|
17
|
-
import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers } from "./query.mjs";
|
|
17
|
+
import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers, exploreAb, renderExploreAb, laneProfileReport, renderLaneProfiles } from "./query.mjs";
|
|
18
18
|
import { batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows } from "./efficiency.mjs";
|
|
19
19
|
import { batchRoot } from "../lean/batch.mjs";
|
|
20
20
|
|
|
@@ -52,6 +52,10 @@ Context-pack A/B (forge#4275; local run logs only, no GitHub crawl):
|
|
|
52
52
|
--context-ab first-pass yield, builder turns, cache tokens and cost per lane type (rerun, child,
|
|
53
53
|
first) and context_pack mode (on, off, shadow); --json for JSON
|
|
54
54
|
|
|
55
|
+
Explorer A/B (forge#4418; local run logs only, every run of each log, no GitHub crawl):
|
|
56
|
+
--explore-ab builder turns, builder cache-read, merge and conformance rates per size (S, M, L) and
|
|
57
|
+
lean.explore mode (on, off), plus the same issues run in both modes; --json for JSON
|
|
58
|
+
|
|
55
59
|
Efficiency (forge#4402; local batch dirs and run logs, gh only for merged issues' bodies, labels and probes):
|
|
56
60
|
--efficiency $/merge, turns/merge, context/turn per size and model, retries/merge by cause, no-action
|
|
57
61
|
lanes, issue quality vs cost and the acceptance pass rate 7 days after promotion
|
|
@@ -61,10 +65,15 @@ Efficiency (forge#4402; local batch dirs and run logs, gh only for merged issues
|
|
|
61
65
|
Builder model tiers (forge#4394; local run logs only, no GitHub crawl):
|
|
62
66
|
--tiers merge, conformance and post-merge-rework rates, first-pass yield and cost per builder tier
|
|
63
67
|
(mid, mid->top escalated, top) and routing class; --json for JSON
|
|
68
|
+
|
|
69
|
+
Lane profile baseline (forge#4436; local run logs only, every run of each issue, no GitHub crawl):
|
|
70
|
+
--lane-profile first-turn baseline_tokens per lane profile (full, memory-off, lane) and phase, and the
|
|
71
|
+
conformance rate per profile; --json for JSON
|
|
72
|
+
--issues <list> pin the report to these issues (comma list), for a before/after on the same issue set
|
|
64
73
|
`;
|
|
65
74
|
|
|
66
75
|
export function parseArgs(argv) {
|
|
67
|
-
const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, tiers: false, efficiency: false, batch: null, windows: null, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
|
|
76
|
+
const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, batch: null, windows: null, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
|
|
68
77
|
const val = (i, name) => {
|
|
69
78
|
const v = argv[i + 1];
|
|
70
79
|
if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
|
|
@@ -81,7 +90,10 @@ export function parseArgs(argv) {
|
|
|
81
90
|
else if (name === "--edge") { const v = take(); if (!parseEdge(v)) throw new Error("--edge must be FROM:TO"); o.edge = v; }
|
|
82
91
|
else if (name === "--milestone") o.milestone = take();
|
|
83
92
|
else if (name === "--context-ab") o.contextAb = true;
|
|
93
|
+
else if (name === "--explore-ab") o.exploreAb = true;
|
|
84
94
|
else if (name === "--tiers") o.tiers = true;
|
|
95
|
+
else if (name === "--lane-profile") o.laneProfile = true;
|
|
96
|
+
else if (name === "--issues") { const v = take(); if (!/^[1-9]\d*(,[1-9]\d*)*$/.test(v)) throw new Error("--issues must be a comma list of issue numbers"); o.issues = v.split(",").map(Number); }
|
|
85
97
|
else if (name === "--efficiency") o.efficiency = true;
|
|
86
98
|
else if (name === "--batch") { const v = take(); if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v)) throw new Error(`invalid --batch id: ${v}`); o.batch = v; }
|
|
87
99
|
else if (name === "--window") o.windows = parseWindows(take());
|
|
@@ -214,10 +226,18 @@ export async function buildReport(opts, deps = {}) {
|
|
|
214
226
|
const runs = (deps.localRuns ?? readLocalRuns(runsDir(env))).filter((r) => r.repo == null || r.repo === repo);
|
|
215
227
|
return { repo, sources: { local: { runs: runs.length } }, query: { context_ab: contextPackAb(runs) } };
|
|
216
228
|
}
|
|
229
|
+
if (opts.exploreAb) { // forge#4418: run logs only, every RUN_START slice
|
|
230
|
+
const runs = (deps.localRuns ?? readLocalLaneRuns(runsDir(env))).filter((r) => r.repo == null || r.repo === repo);
|
|
231
|
+
return { repo, sources: { local: { runs: runs.length } }, query: { explore_ab: exploreAb(runs) } };
|
|
232
|
+
}
|
|
217
233
|
if (opts.tiers) { // forge#4394: run logs only
|
|
218
234
|
const runs = (deps.localRuns ?? readLocalRuns(runsDir(env))).filter((r) => r.repo == null || r.repo === repo);
|
|
219
235
|
return { repo, sources: { local: { runs: runs.length } }, query: { tiers: builderTierReport(runs) } };
|
|
220
236
|
}
|
|
237
|
+
if (opts.laneProfile) { // forge#4436: run logs only, every RUN_START slice
|
|
238
|
+
const runs = (deps.localRuns ?? readLocalRuns(runsDir(env), { allSlices: true })).filter((r) => r.repo == null || r.repo === repo);
|
|
239
|
+
return { repo, sources: { local: { runs: runs.length } }, query: { lane_profile: laneProfileReport(runs, { issues: opts.issues }) } };
|
|
240
|
+
}
|
|
221
241
|
const bound = sinceToTimestamp(opts.since, releaseTimes);
|
|
222
242
|
if (bound.note) (deps.warn ?? ((m) => process.stderr.write(`forgedock stats: ${m}\n`)))(bound.note);
|
|
223
243
|
const gh = await fetchGithubRecords({ repo, since: bound.since, refresh: opts.refresh, useCache: opts.cache, gh: deps.gh, env,
|
|
@@ -254,6 +274,7 @@ export async function runStats(argv, deps = {}) {
|
|
|
254
274
|
}
|
|
255
275
|
try {
|
|
256
276
|
const report = await buildReport(opts, { onProgress: makeProgressPrinter(), ...deps });
|
|
277
|
+
if (report.query?.lane_profile && !opts.json) { process.stdout.write(`${renderLaneProfiles(report.query.lane_profile)}\n`); return 0; }
|
|
257
278
|
if (report.query?.tiers && !opts.json) { process.stdout.write(`${renderTiers(report.query.tiers)}\n`); return 0; }
|
|
258
279
|
if (report.query?.efficiency && !opts.json) {
|
|
259
280
|
const e = report.query.efficiency;
|
|
@@ -263,6 +284,7 @@ export async function runStats(argv, deps = {}) {
|
|
|
263
284
|
return 0;
|
|
264
285
|
}
|
|
265
286
|
if (report.query?.context_ab && !opts.json) { process.stdout.write(`${renderContextAb(report.query.context_ab)}\n`); return 0; }
|
|
287
|
+
if (report.query?.explore_ab && !opts.json) { process.stdout.write(`${renderExploreAb(report.query.explore_ab)}\n`); return 0; }
|
|
266
288
|
process.stdout.write(`${opts.json || report.query ? renderJson(report) : opts.compact ? renderCompact(report) : renderMarkdown(report)}\n`);
|
|
267
289
|
return 0;
|
|
268
290
|
} catch (err) {
|
|
@@ -58,7 +58,7 @@ export function summariseLog(events, fallbackIssue = null) {
|
|
|
58
58
|
const usage = ev.filter((e) => e.event === "PHASE_ATTEMPT_USAGE");
|
|
59
59
|
const builderCalls = usage.filter((e) => e.phase === "implement");
|
|
60
60
|
const reviewCalls = usage.filter((e) => e.phase === "review").length;
|
|
61
|
-
const contextPack = { mode: term?.context_pack ?? cp?.mode ?? "off", laneType: term?.lane_type ?? cp?.lane_type ?? null };
|
|
61
|
+
const contextPack = { mode: term?.context_pack ?? cp?.mode ?? "off", laneType: term?.lane_type ?? cp?.lane_type ?? null, ab: (term?.context_ab ?? cp?.context_ab) === true };
|
|
62
62
|
const startTs = start.ts ?? null;
|
|
63
63
|
const endTs = finished ? ev[t].ts ?? null : null;
|
|
64
64
|
const a = Date.parse(startTs ?? ""); const b = Date.parse(endTs ?? "");
|
|
@@ -76,7 +76,15 @@ export function summariseLog(events, fallbackIssue = null) {
|
|
|
76
76
|
pitfallsMatched: Number.isInteger(term?.pitfalls_matched) ? term.pitfalls_matched
|
|
77
77
|
: Array.isArray(cp?.pitfalls) ? cp.pitfalls.filter((p) => p?.match === "files").length : null,
|
|
78
78
|
builderTurns: builderCalls.reduce((x, e) => x + n(e.turns), 0),
|
|
79
|
+
// #4418: the explorer A/B tag (null before #4391 wrote it) and the main-session (plan + build) cache-read
|
|
80
|
+
explore: term?.explore === "on" || term?.explore === "off" ? term.explore : null,
|
|
81
|
+
builderCacheRead: builderCalls.reduce((x, e) => x + n(e.usage?.cache_read_input_tokens), 0),
|
|
79
82
|
size: term?.turns?.size ?? null, // #4409: S|M|L from the lane's turn record, buckets the admission estimate history
|
|
83
|
+
// #4444: cache-read alone (per-merge A/B metric) and the implement attempt (builder calls other than the plan call)
|
|
84
|
+
cacheReadTokens: usage.reduce((x, e) => x + n(e.usage?.cache_read_input_tokens), 0),
|
|
85
|
+
implementAttempt: implementAttemptOf(builderCalls),
|
|
86
|
+
// #4444: the implement attempt of the earliest finished first/child run in this log, the same-issue baseline of a rerun
|
|
87
|
+
firstRunAttempt: firstRunAttemptOf(events.slice(0, s)),
|
|
80
88
|
cacheTokens: usage.reduce((x, e) => x + n(e.usage?.cache_read_input_tokens) + n(e.usage?.cache_creation_input_tokens), 0),
|
|
81
89
|
costUsd: usage.reduce((x, e) => x + n(costOf(e)), 0),
|
|
82
90
|
// #4394: the builder tier (RUN_TERMINAL builder_*, else the BUILDER_ROUTE / BUILDER_ESCALATED events); null before routing
|
|
@@ -88,9 +96,19 @@ export function summariseLog(events, fallbackIssue = null) {
|
|
|
88
96
|
reworkedMerges: earlierMergedRoutes(events.slice(0, s)),
|
|
89
97
|
// first pass: merged off one plan + one implement call, at most one review, no CI fix round
|
|
90
98
|
firstPass: finished && term?.reason === "merged" && builderCalls.length <= 2 && reviewCalls <= 1 && !ev.some((e) => e.event === "CI_FIX_ROUND"),
|
|
99
|
+
// #4436: the lane profile (#4395) of this slice's sessions ("mixed" when they differ, null before the field) and the
|
|
100
|
+
// fresh sessions' first-turn baseline_tokens per phase (resumed sessions carry none)
|
|
101
|
+
laneProfile: profileOf(usage),
|
|
102
|
+
baselines: usage.reduce((acc, e) => { if (Number.isFinite(e.baseline_tokens)) (acc[e.phase ?? "other"] ??= []).push(e.baseline_tokens); return acc; }, {}),
|
|
91
103
|
};
|
|
92
104
|
}
|
|
93
105
|
|
|
106
|
+
/** #4436: the one lane_profile of the usage records, "mixed" when they differ, null when none carries it. */
|
|
107
|
+
function profileOf(usage) {
|
|
108
|
+
const ps = new Set(usage.map((e) => e.lane_profile).filter((p) => typeof p === "string" && p));
|
|
109
|
+
return ps.size === 0 ? null : ps.size === 1 ? [...ps][0] : "mixed";
|
|
110
|
+
}
|
|
111
|
+
|
|
94
112
|
/** #4394: {tier, class, escalated, model} of a lane, or null when it computed no builder route. */
|
|
95
113
|
function routeOf(term, ev) {
|
|
96
114
|
if (typeof term?.builder_tier === "string") {
|
|
@@ -101,6 +119,31 @@ function routeOf(term, ev) {
|
|
|
101
119
|
return { tier: r.tier ?? null, class: r.class ?? null, escalated: ev.some((e) => e.event === "BUILDER_ESCALATED"), model: r.model ?? null };
|
|
102
120
|
}
|
|
103
121
|
|
|
122
|
+
/** #4444: {turns, output_tokens} of the builder calls other than the plan call; null when there is none. */
|
|
123
|
+
function implementAttemptOf(builderCalls) {
|
|
124
|
+
const traced = builderCalls.some((e) => typeof e.trigger === "string");
|
|
125
|
+
const calls = traced ? builderCalls.filter((e) => e.trigger !== "plan") : builderCalls.slice(1);
|
|
126
|
+
if (!calls.length) return null;
|
|
127
|
+
return { turns: calls.reduce((x, e) => x + n(e.turns), 0), output_tokens: calls.reduce((x, e) => x + n(e.usage?.output_tokens), 0) };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** #4444: implementAttemptOf for the earliest finished RUN_START slice of `events` whose lane type was first or child. */
|
|
131
|
+
function firstRunAttemptOf(events) {
|
|
132
|
+
let slice = null;
|
|
133
|
+
const slices = [];
|
|
134
|
+
for (const e of events) {
|
|
135
|
+
if (e.event === "RUN_START") { slice = []; slices.push(slice); }
|
|
136
|
+
if (slice) slice.push(e);
|
|
137
|
+
}
|
|
138
|
+
for (const sl of slices) {
|
|
139
|
+
const term = [...sl].reverse().find((e) => e.event === "RUN_TERMINAL");
|
|
140
|
+
if (!["first", "child"].includes(term?.lane_type)) continue;
|
|
141
|
+
const att = implementAttemptOf(sl.filter((e) => e.event === "PHASE_ATTEMPT_USAGE" && e.phase === "implement"));
|
|
142
|
+
if (att) return att;
|
|
143
|
+
}
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
146
|
+
|
|
104
147
|
/** #4394: the routes ({tier, class, escalated, model}) of the RUN_START slices in `events` that ended merged with a route. */
|
|
105
148
|
function earlierMergedRoutes(events) {
|
|
106
149
|
const out = [];
|
|
@@ -119,21 +162,32 @@ function earlierMergedRoutes(events) {
|
|
|
119
162
|
return out;
|
|
120
163
|
}
|
|
121
164
|
|
|
122
|
-
/**
|
|
123
|
-
|
|
165
|
+
/**
|
|
166
|
+
* All run summaries in `dir` (a missing dir or an unreadable log yields nothing, never an error). One per log (its last
|
|
167
|
+
* RUN_START slice); with `allSlices` (#4436) one per RUN_START slice, so a before and an after run of one issue both count.
|
|
168
|
+
*/
|
|
169
|
+
export function readLocalRuns(dir = runsDir(), { allSlices = false } = {}) {
|
|
124
170
|
if (!existsSync(dir)) return [];
|
|
125
171
|
const runs = [];
|
|
126
172
|
for (const f of readdirSync(dir).sort()) {
|
|
127
173
|
if (!f.endsWith(".jsonl")) continue;
|
|
128
174
|
try {
|
|
129
175
|
const m = /(?:^|__)(\d+)\.jsonl$/.exec(f); // #4432: `<issue>.jsonl` or `<owner>__<repo>__<issue>.jsonl`
|
|
130
|
-
const
|
|
131
|
-
|
|
176
|
+
const events = parseLog(readFileSync(join(dir, f), "utf8"));
|
|
177
|
+
const starts = allSlices ? events.flatMap((e, i) => (e.event === "RUN_START" ? [i] : [])) : [events.length];
|
|
178
|
+
const mtimeTs = new Date(statSync(join(dir, f)).mtimeMs).toISOString();
|
|
179
|
+
starts.forEach((_, k) => {
|
|
180
|
+
const sum = summariseLog(allSlices ? events.slice(0, starts[k + 1] ?? events.length) : events, m ? Number(m[1]) : null);
|
|
181
|
+
if (sum) runs.push({ ...sum, mtimeTs });
|
|
182
|
+
});
|
|
132
183
|
} catch { /* unreadable log */ }
|
|
133
184
|
}
|
|
134
185
|
return runs;
|
|
135
186
|
}
|
|
136
187
|
|
|
188
|
+
/** #4418: every RUN_START slice (readLocalRuns allSlices), so an on run and an off run of the same issue both count. */
|
|
189
|
+
export const readLocalLaneRuns = (dir = runsDir()) => readLocalRuns(dir, { allSlices: true });
|
|
190
|
+
|
|
137
191
|
/**
|
|
138
192
|
* forge#4354: one lane per RUN_START..next-RUN_START slice (every slice, not only the last), with the fail-soft events the
|
|
139
193
|
* alert pass rates: DEPS_PREPARED rows, TOOLCHAIN_WARNING texts, deferred-to-ci precheck commands and the terminal code.
|