forgedock 1.30.0 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +3 -2
- package/bin/engine/runlog.mjs +82 -12
- package/bin/engine/telemetry-seam.mjs +11 -4
- package/bin/engine/telemetry-seam.test.mjs +18 -0
- package/bin/engine-cli.mjs +409 -38
- package/bin/engine.mjs +47 -37
- package/bin/forgedock.mjs +35 -3
- package/bin/gc.mjs +50 -17
- package/bin/hooks/reread-nudge.mjs +65 -0
- package/bin/labels.json +5 -0
- package/bin/lean/apishape.mjs +170 -0
- package/bin/lean/apishape.test.mjs +187 -0
- package/bin/lean/batch.mjs +32 -0
- package/bin/lean/batch.test.mjs +338 -2
- package/bin/lean/budget.mjs +89 -0
- package/bin/lean/budget.test.mjs +106 -0
- package/bin/lean/checks.mjs +32 -9
- package/bin/lean/checks.test.mjs +78 -1
- package/bin/lean/conformance.e2e.test.mjs +27 -5
- package/bin/lean/conformance.mjs +150 -15
- package/bin/lean/conformance.test.mjs +175 -3
- package/bin/lean/context.mjs +128 -44
- package/bin/lean/context.test.mjs +157 -5
- package/bin/lean/decompose.mjs +90 -0
- package/bin/lean/decompose.test.mjs +105 -0
- package/bin/lean/depcache.mjs +11 -7
- package/bin/lean/depcache.test.mjs +81 -0
- package/bin/lean/e2e.mjs +102 -0
- package/bin/lean/e2e.test.mjs +86 -0
- package/bin/lean/effects.mjs +48 -15
- package/bin/lean/explore.mjs +47 -0
- package/bin/lean/explore.test.mjs +61 -0
- package/bin/lean/facts.mjs +84 -0
- package/bin/lean/facts.test.mjs +66 -0
- package/bin/lean/fixround.mjs +90 -0
- package/bin/lean/fixround.test.mjs +117 -0
- package/bin/lean/heavy.mjs +35 -2
- package/bin/lean/heavy.test.mjs +32 -2
- package/bin/lean/lean.mjs +712 -134
- package/bin/lean/lean.test.mjs +1626 -37
- package/bin/lean/ledger.mjs +5 -3
- package/bin/lean/ledger.test.mjs +18 -0
- package/bin/lean/light.mjs +135 -0
- package/bin/lean/light.test.mjs +173 -0
- package/bin/lean/lightlane.mjs +219 -0
- package/bin/lean/lightlane.test.mjs +237 -0
- package/bin/lean/limits.test.mjs +36 -1
- package/bin/lean/orchestrate.mjs +303 -41
- package/bin/lean/pitfalls.mjs +136 -0
- package/bin/lean/pitfalls.test.mjs +117 -0
- package/bin/lean/postmerge.mjs +85 -0
- package/bin/lean/postmerge.test.mjs +79 -0
- package/bin/lean/precheck.mjs +138 -13
- package/bin/lean/precheck.test.mjs +118 -1
- package/bin/lean/prefilter.mjs +220 -5
- package/bin/lean/prefilter.test.mjs +260 -4
- package/bin/lean/preflight.mjs +108 -3
- package/bin/lean/preflight.test.mjs +48 -1
- package/bin/lean/probe.mjs +297 -0
- package/bin/lean/probe.test.mjs +210 -0
- package/bin/lean/prompts.mjs +104 -13
- package/bin/lean/render.mjs +7 -3
- package/bin/lean/resume.mjs +67 -0
- package/bin/lean/resume.test.mjs +84 -0
- package/bin/lean/review-trigger.mjs +12 -0
- package/bin/lean/review-trigger.test.mjs +17 -1
- package/bin/lean/routing.mjs +74 -0
- package/bin/lean/routing.test.mjs +67 -0
- package/bin/lean/sched.mjs +16 -1
- package/bin/lean/sched.test.mjs +175 -1
- package/bin/lean/schedd.mjs +55 -9
- package/bin/lean/schedd.test.mjs +170 -3
- package/bin/lean/selfreview.mjs +37 -0
- package/bin/lean/selfreview.test.mjs +51 -0
- package/bin/lean/seqlease.mjs +40 -0
- package/bin/lean/seqlease.test.mjs +101 -0
- package/bin/lean/sequence.mjs +296 -13
- package/bin/lean/sequence.test.mjs +263 -2
- package/bin/lean/toolchain.mjs +60 -15
- package/bin/lean/toolchain.test.mjs +120 -3
- package/bin/lean/toolstats.mjs +72 -0
- package/bin/lean/toolstats.test.mjs +47 -0
- package/bin/lean/verifier.mjs +232 -0
- package/bin/lean/verifier.test.mjs +163 -0
- package/bin/lean/verify.mjs +72 -10
- package/bin/lean/verify.test.mjs +68 -3
- package/bin/review-staging.mjs +533 -0
- package/bin/runner.mjs +82 -13
- package/bin/stats/aggregate.mjs +87 -2
- package/bin/stats/efficiency.mjs +335 -0
- package/bin/stats/efficiency.test.mjs +266 -0
- package/bin/stats/github-source.mjs +8 -3
- package/bin/stats/index.mjs +91 -6
- package/bin/stats/local-source.mjs +145 -8
- package/bin/stats/query.mjs +181 -3
- package/bin/stats/render.mjs +22 -0
- package/bin/stats/stats.test.mjs +403 -8
- package/commands/orchestrate.md +3 -1
- package/commands/review-pr-agents/protocols.md +1 -1
- package/commands/review-pr-staging.md +84 -831
- package/commands/review-pr.md +1 -1
- package/commands/work-on/build/implement.md +1 -0
- package/commands/work-on/build/validate.md +11 -1
- package/docs/CONFIG.md +80 -5
- package/docs/OUTCOME-CODES.md +2 -1
- package/docs/STATS.md +66 -1
- package/docs/TELEMETRY.md +1 -1
- package/docs/eval/corpus/lightlane-v1.json +55 -0
- package/docs/eval/lightlane-bench.md +91 -0
- package/docs/spec/review-protocol.md +1 -1
- package/package.json +1 -1
- package/scripts/bench-lightlane.mjs +286 -0
- package/scripts/ci-local/lib.mjs +20 -0
- package/scripts/ci-local/plan.mjs +2 -12
- package/scripts/ci-local/scoping.mjs +10 -1
- package/scripts/ci-local.test.sh +2 -0
- package/scripts/fixtures/apishape/other/go-change/base/x.go +3 -0
- package/scripts/fixtures/apishape/other/go-change/head/x.go +3 -0
- package/scripts/fixtures/apishape/python/generator-function/base/seq.py +3 -0
- package/scripts/fixtures/apishape/python/generator-function/head/seq.py +2 -0
- package/scripts/fixtures/apishape/python/ichunked/base/more.py +17 -0
- package/scripts/fixtures/apishape/python/ichunked/head/more.py +20 -0
- package/scripts/fixtures/apishape/python/private-edit/base/calc.py +6 -0
- package/scripts/fixtures/apishape/python/private-edit/head/calc.py +9 -0
- package/scripts/fixtures/apishape/python/removed/base/ops.py +6 -0
- package/scripts/fixtures/apishape/python/removed/head/ops.py +2 -0
- package/scripts/fixtures/apishape/python/renamed/base/text.py +2 -0
- package/scripts/fixtures/apishape/python/renamed/head/text.py +2 -0
- package/scripts/fixtures/apishape/python/signature/base/shapes.py +2 -0
- package/scripts/fixtures/apishape/python/signature/head/shapes.py +2 -0
- package/scripts/fixtures/apishape/python/sync-async/base/client.py +7 -0
- package/scripts/fixtures/apishape/python/sync-async/head/client.py +7 -0
- package/scripts/fixtures/apishape/ts/generator-function/base/seq.js +3 -0
- package/scripts/fixtures/apishape/ts/generator-function/head/seq.js +5 -0
- package/scripts/fixtures/apishape/ts/private-edit/base/calc.ts +7 -0
- package/scripts/fixtures/apishape/ts/private-edit/head/calc.ts +9 -0
- package/scripts/fixtures/apishape/ts/removed/base/ops.ts +7 -0
- package/scripts/fixtures/apishape/ts/removed/head/ops.ts +3 -0
- package/scripts/fixtures/apishape/ts/renamed/base/text.ts +3 -0
- package/scripts/fixtures/apishape/ts/renamed/head/text.ts +3 -0
- package/scripts/fixtures/apishape/ts/signature/base/shapes.ts +3 -0
- package/scripts/fixtures/apishape/ts/signature/head/shapes.ts +3 -0
- package/scripts/fixtures/apishape/ts/sync-async/base/client.mjs +5 -0
- package/scripts/fixtures/apishape/ts/sync-async/head/client.mjs +5 -0
- package/scripts/forge-root.test.sh +2 -2
- package/scripts/publish-staging-sync.test.sh +62 -4
- package/scripts/review-finalize.sh +17 -6
- package/scripts/review-finalize.test.sh +47 -0
- package/scripts/review-pr-arguments.test.sh +13 -1
- package/scripts/review-pr-delta-wiring.test.sh +2 -2
- package/scripts/review-pr-gate2.test.sh +6 -14
- package/scripts/review-pr-staging-finding-filter.test.sh +4 -4
- package/scripts/review-pr-staging-scope.test.sh +39 -33
- package/scripts/review-prepare.sh +73 -6
- package/scripts/review-prepare.test.sh +85 -0
- package/scripts/review-staging-post.sh +293 -0
- package/scripts/review-staging-post.test.sh +143 -0
- package/scripts/review-staging-pre.sh +401 -0
- package/scripts/review-staging-pre.test.sh +150 -0
- package/scripts/symbol-index.mjs +26 -7
- package/scripts/symbol-index.py +55 -9
- package/scripts/trusted-comments.test.sh +1 -1
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
/**
|
|
5
|
+
* scripts/bench-lightlane.mjs — three-way benchmark: light lane vs full lane vs a plain single-session control (#4453)
|
|
6
|
+
*
|
|
7
|
+
* Runs the same issue set through three arms and grades every arm with the upstream oracle tests (the test files of
|
|
8
|
+
* the real upstream fix, overlaid on the arm's head). Zero dependencies; every process call goes through an
|
|
9
|
+
* injectable `run`, so tests use fakes. Instructions: docs/eval/lightlane-bench.md. Corpus: docs/eval/corpus/.
|
|
10
|
+
*
|
|
11
|
+
* Usage:
|
|
12
|
+
* node scripts/bench-lightlane.mjs calibrate --corpus <corpus.json>
|
|
13
|
+
* node scripts/bench-lightlane.mjs run --corpus <corpus.json> --config <runcfg.json> --out <rows.jsonl> [--arms light,full,control] [--issues id,...] [--keep]
|
|
14
|
+
* node scripts/bench-lightlane.mjs aggregate <rows.jsonl|->
|
|
15
|
+
*
|
|
16
|
+
* Arms:
|
|
17
|
+
* light, full: `forgedock run-issue <n> --repo <sandbox> --result-file <f>` on a sandbox repo whose forge.yaml sets
|
|
18
|
+
* lean.light.mode on (light) or off (full). Cost is the run result's cost_usd (the sum of the #4407 per-session
|
|
19
|
+
* deltas); model time is model_ms (summed session time, #4451).
|
|
20
|
+
* control: `claude -p <prompt> --output-format json --model <model>` in a fresh clone at the base; cost is
|
|
21
|
+
* total_cost_usd, wall and model time are duration_ms (one session).
|
|
22
|
+
*
|
|
23
|
+
* The harness never writes to GitHub itself; the only GitHub writes are those of the light/full arms' own run-issue.
|
|
24
|
+
*
|
|
25
|
+
* Exit codes: 0 ok | 1 invalid input or a failed run step | 2 a calibration or a target failed (output still emitted)
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { spawn } from "node:child_process";
|
|
29
|
+
import { appendFileSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
30
|
+
import { tmpdir } from "node:os";
|
|
31
|
+
import { join } from "node:path";
|
|
32
|
+
import { pathToFileURL } from "node:url";
|
|
33
|
+
import { detectApiShape } from "../bin/lean/apishape.mjs";
|
|
34
|
+
|
|
35
|
+
export const ARMS = ["light", "full", "control"];
|
|
36
|
+
export const TARGETS = Object.freeze({ cost_vs_control: 1.2, wall_vs_model: 1.3 });
|
|
37
|
+
const SHA_RE = /^[0-9a-f]{40}$/;
|
|
38
|
+
const CLAIMED_REASONS = new Set(["merged", "ci-pending"]);
|
|
39
|
+
const OUT_CAP = 1 << 20;
|
|
40
|
+
|
|
41
|
+
/** Spawn `cmd args` and collect its output (the last 1 MiB of each stream); never throws. */
|
|
42
|
+
export function benchDefaultRun(cmd, args, { cwd, env } = {}) {
|
|
43
|
+
return new Promise((resolve) => {
|
|
44
|
+
let stdout = "", stderr = "";
|
|
45
|
+
const cap = (s) => (s.length > OUT_CAP ? s.slice(-OUT_CAP) : s);
|
|
46
|
+
let p;
|
|
47
|
+
try { p = spawn(cmd, args, { cwd, env: env ?? process.env, stdio: ["ignore", "pipe", "pipe"] }); } catch (e) { resolve({ code: 127, stdout: "", stderr: String(e?.message ?? e) }); return; }
|
|
48
|
+
p.stdout.on("data", (d) => { stdout = cap(stdout + d); });
|
|
49
|
+
p.stderr.on("data", (d) => { stderr = cap(stderr + d); });
|
|
50
|
+
p.on("error", (e) => resolve({ code: e?.code === "ENOENT" ? 127 : 1, stdout, stderr: stderr + String(e?.message ?? e) }));
|
|
51
|
+
p.on("close", (code) => resolve({ code: code ?? 1, stdout, stderr }));
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** A parsed corpus, or a thrown Error naming the first bad field. */
|
|
56
|
+
export function validateCorpus(c) {
|
|
57
|
+
if (!c || typeof c !== "object") throw new Error("corpus: not an object");
|
|
58
|
+
for (const k of ["clone_url", "suite_command", "control_prompt"]) if (typeof c[k] !== "string" || !c[k].trim()) throw new Error(`corpus: missing ${k}`);
|
|
59
|
+
if (!SHA_RE.test(String(c.base))) throw new Error("corpus: base must be a full 40-hex sha");
|
|
60
|
+
if (!Array.isArray(c.issues) || !c.issues.length) throw new Error("corpus: issues must be a non-empty array");
|
|
61
|
+
const ids = new Set();
|
|
62
|
+
for (const i of c.issues) {
|
|
63
|
+
if (!i?.id || ids.has(i.id)) throw new Error(`corpus: missing or duplicate issue id ${i?.id}`);
|
|
64
|
+
ids.add(i.id);
|
|
65
|
+
if (!SHA_RE.test(String(i.fix_sha))) throw new Error(`corpus: ${i.id}: fix_sha must be a full 40-hex sha`);
|
|
66
|
+
if (typeof i.title !== "string" || typeof i.body !== "string") throw new Error(`corpus: ${i.id}: title and body are required`);
|
|
67
|
+
if (!Array.isArray(i.oracle?.files) || !i.oracle.files.length || typeof i.oracle.command !== "string") throw new Error(`corpus: ${i.id}: oracle {files, command} is required`);
|
|
68
|
+
}
|
|
69
|
+
return c;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The run config for `arms`, or a thrown Error: light/full need {repo, issues: {<corpus id>: n}}, control needs {model}. */
|
|
73
|
+
export function validateRunConfig(cfg, corpus, arms) {
|
|
74
|
+
const a = cfg?.arms;
|
|
75
|
+
if (!a || typeof a !== "object") throw new Error("config: missing arms");
|
|
76
|
+
for (const arm of arms) {
|
|
77
|
+
const x = a[arm];
|
|
78
|
+
if (!x) throw new Error(`config: missing arms.${arm}`);
|
|
79
|
+
if (arm === "control") { if (typeof x.model !== "string" || !x.model) throw new Error("config: arms.control.model is required"); continue; }
|
|
80
|
+
if (typeof x.repo !== "string" || !/^[\w.-]+\/[\w.-]+$/.test(x.repo)) throw new Error(`config: arms.${arm}.repo must be owner/name`);
|
|
81
|
+
for (const i of corpus.issues) if (!Number.isSafeInteger(x.issues?.[i.id]) || x.issues[i.id] <= 0) throw new Error(`config: arms.${arm}.issues.${i.id} must be the sandbox issue number`);
|
|
82
|
+
}
|
|
83
|
+
return cfg;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** The control prompt for an issue: `{title}` and `{body}` replaced. */
|
|
87
|
+
export const fillPrompt = (tpl, issue) => String(tpl).replaceAll("{title}", issue.title).replaceAll("{body}", issue.body);
|
|
88
|
+
|
|
89
|
+
const sh = (run, command, cwd) => run("sh", ["-c", command], { cwd });
|
|
90
|
+
async function git(run, args, cwd) {
|
|
91
|
+
const r = await run("git", args, { cwd });
|
|
92
|
+
if (r.code !== 0) throw new Error(`git ${args.join(" ")} failed (${r.code}): ${String(r.stderr).trim().slice(0, 300)}`);
|
|
93
|
+
return String(r.stdout).trim();
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Overlay the oracle test files from the fix commit onto the checkout in `dir` and run the oracle; true when it passes. */
|
|
97
|
+
async function runOracle(run, dir, issue) {
|
|
98
|
+
await git(run, ["checkout", "-q", issue.fix_sha, "--", ...issue.oracle.files], dir);
|
|
99
|
+
return (await sh(run, issue.oracle.command, dir)).code === 0;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** For each corpus issue: the oracle must fail on the base and pass on the fix. -> [{id, red_on_base, green_on_fix}] */
|
|
103
|
+
export async function calibrate({ corpus, run = benchDefaultRun, tmp }) {
|
|
104
|
+
const dir = join(tmp, "upstream");
|
|
105
|
+
await git(run, ["clone", "-q", corpus.clone_url, dir], tmp);
|
|
106
|
+
const out = [];
|
|
107
|
+
for (const issue of corpus.issues) {
|
|
108
|
+
await git(run, ["checkout", "-q", "-f", corpus.base], dir);
|
|
109
|
+
const red = !(await runOracle(run, dir, issue));
|
|
110
|
+
await git(run, ["checkout", "-q", "-f", issue.fix_sha], dir);
|
|
111
|
+
const green = (await sh(run, issue.oracle.command, dir)).code === 0;
|
|
112
|
+
out.push({ id: issue.id, red_on_base: red, green_on_fix: green });
|
|
113
|
+
}
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const num = (x) => (Number.isFinite(Number(x)) && x !== null && x !== "" ? Number(x) : null);
|
|
118
|
+
const readJson = (p) => { try { return JSON.parse(readFileSync(p, "utf8")); } catch { return null; } };
|
|
119
|
+
|
|
120
|
+
/** Grade `head` from the repo in `src`: own suite, then the oracle overlay, then the API shape from `base`. */
|
|
121
|
+
async function grade({ run, src, head, base, corpus, issue, tmp, tag, apiShape }) {
|
|
122
|
+
await git(run, ["branch", "-f", "bench-head", head], src);
|
|
123
|
+
const dir = join(tmp, `${tag}-grade`);
|
|
124
|
+
await git(run, ["clone", "-q", "-b", "bench-head", src, dir], tmp);
|
|
125
|
+
await git(run, ["fetch", "-q", corpus.clone_url, issue.fix_sha], dir);
|
|
126
|
+
const changed = (await run("git", ["diff", "--quiet", base, "HEAD"], { cwd: dir })).code === 1;
|
|
127
|
+
let shape;
|
|
128
|
+
try { const r = await apiShape({ cwd: dir, base, head: "HEAD" }); shape = { status: r?.status ?? "unknown", changes: Array.isArray(r?.changes) ? r.changes : [] }; }
|
|
129
|
+
catch (e) { shape = { status: "error", changes: [], error: String(e?.message ?? e).slice(0, 200) }; }
|
|
130
|
+
const suite = (await sh(run, corpus.suite_command, dir)).code === 0;
|
|
131
|
+
const oracle = await runOracle(run, dir, issue);
|
|
132
|
+
return { changed, suite_pass: suite, oracle_pass: oracle, api_shape: shape };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Run one arm on one corpus issue and grade it -> one row (see aggregate). `now` and `apiShape` are injectable. */
|
|
136
|
+
export async function runArm({ arm, issue, corpus, cfg, run = benchDefaultRun, tmp, now = () => Date.now(), apiShape = detectApiShape }) {
|
|
137
|
+
const tag = `${arm}-${issue.id}`, a = cfg.arms[arm];
|
|
138
|
+
const row = { arm, id: issue.id, cost_usd: null, wall_ms: null, model_ms: null, harness_wall_ms: null, claimed_fix: false, oracle_pass: false, suite_pass: null,
|
|
139
|
+
api_shape: null, light: null, light_eligibility: null, light_upgrade: null, terminal: null };
|
|
140
|
+
const t0 = now();
|
|
141
|
+
let src = null, head = null, base = corpus.base, claimed = false;
|
|
142
|
+
if (arm === "control") {
|
|
143
|
+
src = join(tmp, `${tag}-work`);
|
|
144
|
+
await git(run, ["clone", "-q", corpus.clone_url, src], tmp);
|
|
145
|
+
await git(run, ["checkout", "-q", "-b", "bench-control", corpus.base], src);
|
|
146
|
+
const [bin, ...pre] = Array.isArray(a.command) && a.command.length ? a.command : ["claude"];
|
|
147
|
+
const r = await run(bin, [...pre, "-p", fillPrompt(corpus.control_prompt, issue), "--output-format", "json", "--model", a.model, ...(a.args ?? [])], { cwd: src });
|
|
148
|
+
row.harness_wall_ms = now() - t0;
|
|
149
|
+
let j = null;
|
|
150
|
+
try { j = JSON.parse(String(r.stdout).trim().split("\n").pop()); } catch { /* reported as terminal below */ }
|
|
151
|
+
row.cost_usd = num(j?.total_cost_usd); row.wall_ms = num(j?.duration_ms); row.model_ms = num(j?.duration_ms);
|
|
152
|
+
row.terminal = r.code !== 0 ? `exit ${r.code}` : !j ? "no-json" : j.is_error ? "error" : "ok";
|
|
153
|
+
claimed = row.terminal === "ok";
|
|
154
|
+
await git(run, ["add", "-A"], src);
|
|
155
|
+
await git(run, ["-c", "user.name=bench", "-c", "user.email=bench@localhost", "commit", "-q", "--allow-empty", "--no-verify", "-m", `control ${issue.id}`], src);
|
|
156
|
+
head = await git(run, ["rev-parse", "HEAD"], src);
|
|
157
|
+
} else {
|
|
158
|
+
const n = a.issues[issue.id], rf = join(tmp, `${tag}-result.json`);
|
|
159
|
+
const [bin, ...pre] = Array.isArray(a.command) && a.command.length ? a.command : ["forgedock", "run-issue"];
|
|
160
|
+
const r = await run(bin, [...pre, String(n), "--repo", a.repo, "--result-file", rf, ...(a.args ?? [])], {});
|
|
161
|
+
row.harness_wall_ms = now() - t0;
|
|
162
|
+
const res = readJson(rf);
|
|
163
|
+
row.terminal = res?.terminalReason ?? `exit ${r.code}`;
|
|
164
|
+
row.cost_usd = num(res?.cost_usd); row.wall_ms = num(res?.wall_ms); row.model_ms = num(res?.model_ms);
|
|
165
|
+
row.light = typeof res?.light === "boolean" ? res.light : null;
|
|
166
|
+
row.light_eligibility = res?.light_eligibility ?? null;
|
|
167
|
+
row.light_upgrade = Array.isArray(res?.light_upgrade) ? res.light_upgrade : null;
|
|
168
|
+
claimed = CLAIMED_REASONS.has(row.terminal);
|
|
169
|
+
base = a.base ?? corpus.base;
|
|
170
|
+
src = join(tmp, `${tag}-src`);
|
|
171
|
+
await git(run, ["clone", "-q", a.clone_url ?? `https://github.com/${a.repo}.git`, src], tmp);
|
|
172
|
+
if (res?.pr) {
|
|
173
|
+
const pr = String(res.pr).match(/(\d+)\s*$/)?.[1];
|
|
174
|
+
const v = await run("gh", ["pr", "view", pr, "-R", a.repo, "--json", "headRefOid", "--jq", ".headRefOid"], {});
|
|
175
|
+
await git(run, ["fetch", "-q", "origin", `refs/pull/${pr}/head`], src);
|
|
176
|
+
head = SHA_RE.test(String(v.stdout).trim()) ? String(v.stdout).trim() : await git(run, ["rev-parse", "FETCH_HEAD"], src);
|
|
177
|
+
} else if (res?.branch) {
|
|
178
|
+
await git(run, ["fetch", "-q", "origin", res.branch], src);
|
|
179
|
+
head = await git(run, ["rev-parse", "FETCH_HEAD"], src);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
if (!head) return row; // no PR and no branch: nothing to grade; oracle_pass stays false
|
|
183
|
+
const g = await grade({ run, src, head, base, corpus, issue, tmp, tag, apiShape });
|
|
184
|
+
return { ...row, claimed_fix: claimed && g.changed, oracle_pass: g.oracle_pass, suite_pass: g.suite_pass, api_shape: g.api_shape };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const sum = (xs) => xs.reduce((s, x) => s + x, 0);
|
|
188
|
+
const nums = (rows, k) => rows.map((r) => r[k]).filter((x) => Number.isFinite(x));
|
|
189
|
+
const rate = (rows, f) => (rows.length ? rows.filter(f).length / rows.length : null);
|
|
190
|
+
const r4 = (x) => (x == null ? null : Math.round(x * 10000) / 10000);
|
|
191
|
+
|
|
192
|
+
/** Per-arm metrics over rows (pure). */
|
|
193
|
+
export function armMetrics(rows) {
|
|
194
|
+
const cost = nums(rows, "cost_usd"), wall = nums(rows, "wall_ms"), model = nums(rows, "model_ms");
|
|
195
|
+
return {
|
|
196
|
+
n: rows.length,
|
|
197
|
+
cost_usd: { sum: r4(sum(cost)), mean: cost.length ? r4(sum(cost) / cost.length) : null },
|
|
198
|
+
wall_ms: sum(wall), model_ms: sum(model), wall_over_model: sum(model) > 0 ? r4(sum(wall) / sum(model)) : null,
|
|
199
|
+
oracle_pass_rate: r4(rate(rows, (r) => r.oracle_pass === true)),
|
|
200
|
+
false_fix_rate: r4(rate(rows, (r) => r.claimed_fix === true && r.oracle_pass !== true)),
|
|
201
|
+
suite_pass_rate: r4(rate(rows, (r) => r.suite_pass === true)),
|
|
202
|
+
api_shape_changed: rows.filter((r) => r.api_shape?.status === "changed").length,
|
|
203
|
+
undetected_api_shape: rows.filter((r) => r.arm === "light" && r.light === true && r.api_shape?.status === "changed").length,
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Rows -> {arms, light_eligible, targets} (pure). Cost and wall are judged on the light rows whose eligibility verdict is
|
|
209
|
+
* `eligible` (cost against the control rows of the same issues); oracle, false fix and undetected API shape on all light
|
|
210
|
+
* rows. A target is {pass: true|false|null, value, limit}; null when an arm it needs has no rows.
|
|
211
|
+
*/
|
|
212
|
+
export function aggregate(rows) {
|
|
213
|
+
const by = Object.fromEntries(ARMS.map((a) => [a, rows.filter((r) => r.arm === a)]));
|
|
214
|
+
const arms = Object.fromEntries(ARMS.map((a) => [a, armMetrics(by[a])]));
|
|
215
|
+
const elig = by.light.filter((r) => r.light_eligibility === "eligible");
|
|
216
|
+
const eligIds = new Set(elig.map((r) => r.id));
|
|
217
|
+
const ctlElig = by.control.filter((r) => eligIds.has(r.id));
|
|
218
|
+
const le = armMetrics(elig), ce = armMetrics(ctlElig), L = arms.light, C = arms.control;
|
|
219
|
+
const t = (ok, value, limit) => ({ pass: ok == null ? null : ok, value, limit });
|
|
220
|
+
const costRatio = elig.length && ctlElig.length && ce.cost_usd.sum > 0 ? r4(le.cost_usd.sum / ce.cost_usd.sum) : null;
|
|
221
|
+
const targets = {
|
|
222
|
+
cost_le_1_2x_control: t(costRatio == null ? null : costRatio <= TARGETS.cost_vs_control, costRatio, TARGETS.cost_vs_control),
|
|
223
|
+
wall_le_1_3x_model: t(le.wall_over_model == null ? null : le.wall_over_model <= TARGETS.wall_vs_model, le.wall_over_model, TARGETS.wall_vs_model),
|
|
224
|
+
oracle_ge_control: t(L.n && C.n ? L.oracle_pass_rate >= C.oracle_pass_rate : null, L.oracle_pass_rate, C.oracle_pass_rate),
|
|
225
|
+
false_fix_le_control: t(L.n && C.n ? L.false_fix_rate <= C.false_fix_rate : null, L.false_fix_rate, C.false_fix_rate),
|
|
226
|
+
zero_undetected_api_shape: t(L.n ? L.undetected_api_shape === 0 : null, L.undetected_api_shape, 0),
|
|
227
|
+
};
|
|
228
|
+
return { arms, light_eligible: { ...le, control_cost_usd: ce.cost_usd.sum, ids: [...eligIds].sort() }, targets };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** JSONL (or a JSON array) -> rows. */
|
|
232
|
+
export function parseRows(text) {
|
|
233
|
+
const s = String(text).trim();
|
|
234
|
+
if (!s) return [];
|
|
235
|
+
if (s.startsWith("[")) return JSON.parse(s);
|
|
236
|
+
return s.split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l));
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const flag = (argv, name) => { const i = argv.indexOf(name); return i >= 0 && argv[i + 1] && !argv[i + 1].startsWith("--") ? argv[i + 1] : null; };
|
|
240
|
+
const list = (v, all) => (v ? v.split(",").map((s) => s.trim()).filter(Boolean) : all);
|
|
241
|
+
|
|
242
|
+
export async function main(argv = process.argv.slice(2), { run = benchDefaultRun, out = process.stdout, err = process.stderr } = {}) {
|
|
243
|
+
const cmd = argv[0];
|
|
244
|
+
try {
|
|
245
|
+
if (cmd === "aggregate") {
|
|
246
|
+
const src = argv[1];
|
|
247
|
+
const text = !src || src === "-" ? readFileSync(0, "utf8") : readFileSync(src, "utf8");
|
|
248
|
+
const agg = aggregate(parseRows(text));
|
|
249
|
+
out.write(JSON.stringify(agg, null, 2) + "\n");
|
|
250
|
+
return Object.values(agg.targets).some((x) => x.pass === false) ? 2 : 0;
|
|
251
|
+
}
|
|
252
|
+
if (cmd !== "calibrate" && cmd !== "run") { err.write("usage: bench-lightlane.mjs calibrate|run|aggregate (see the header)\n"); return 1; }
|
|
253
|
+
const cp = flag(argv, "--corpus");
|
|
254
|
+
if (!cp) throw new Error("--corpus is required");
|
|
255
|
+
const corpus = validateCorpus(JSON.parse(readFileSync(cp, "utf8")));
|
|
256
|
+
const tmp = mkdtempSync(join(tmpdir(), "bench-lightlane-"));
|
|
257
|
+
try {
|
|
258
|
+
if (cmd === "calibrate") {
|
|
259
|
+
const res = await calibrate({ corpus, run, tmp });
|
|
260
|
+
out.write(JSON.stringify(res, null, 2) + "\n");
|
|
261
|
+
return res.every((r) => r.red_on_base && r.green_on_fix) ? 0 : 2;
|
|
262
|
+
}
|
|
263
|
+
const arms = list(flag(argv, "--arms"), ARMS), rowsOut = flag(argv, "--out"), cfgPath = flag(argv, "--config");
|
|
264
|
+
if (!rowsOut || !cfgPath) throw new Error("--config and --out are required");
|
|
265
|
+
const bad = arms.filter((a) => !ARMS.includes(a));
|
|
266
|
+
if (bad.length) throw new Error(`unknown arm(s): ${bad.join(", ")}`);
|
|
267
|
+
const cfg = validateRunConfig(JSON.parse(readFileSync(cfgPath, "utf8")), corpus, arms);
|
|
268
|
+
const ids = list(flag(argv, "--issues"), corpus.issues.map((i) => i.id));
|
|
269
|
+
for (const issue of corpus.issues.filter((i) => ids.includes(i.id))) {
|
|
270
|
+
for (const arm of arms) {
|
|
271
|
+
const row = await runArm({ arm, issue, corpus, cfg, run, tmp });
|
|
272
|
+
appendFileSync(rowsOut, JSON.stringify(row) + "\n");
|
|
273
|
+
err.write(`${arm} ${issue.id}: oracle ${row.oracle_pass ? "pass" : "fail"}, $${row.cost_usd ?? "?"}, ${row.terminal}\n`);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
return 0;
|
|
277
|
+
} finally {
|
|
278
|
+
if (!argv.includes("--keep")) rmSync(tmp, { recursive: true, force: true });
|
|
279
|
+
}
|
|
280
|
+
} catch (e) {
|
|
281
|
+
err.write(`ERROR: ${e?.message ?? e}\n`);
|
|
282
|
+
return 1;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) process.exitCode = await main();
|
package/scripts/ci-local/lib.mjs
CHANGED
|
@@ -93,6 +93,26 @@ export function readAtRef(repo, ref, file) {
|
|
|
93
93
|
return r.code === 0 ? r.out : null;
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/**
|
|
97
|
+
* forge#4040 (SEC-2): forge.yaml from a TRUSTED copy only, never from the PR's own tree: {file, text} ('' when none).
|
|
98
|
+
* In order: the main checkout's forge.yaml (a worktree's git common dir points at it); for a plain checkout, the base
|
|
99
|
+
* ref's tracked copy, or an untracked/ignored working-tree copy (not part of the PR). A tracked worktree copy is never read.
|
|
100
|
+
*/
|
|
101
|
+
export function trustedForgeYaml(repo, baseRef, baseSha) {
|
|
102
|
+
let docText = ''; let file = '';
|
|
103
|
+
const gc = git(repo, ['rev-parse', '--git-common-dir']).out.trim();
|
|
104
|
+
const top = fs.realpathSync(path.resolve(git(repo, ['rev-parse', '--show-toplevel']).out.trim() || repo));
|
|
105
|
+
let main = '';
|
|
106
|
+
if (gc) { const abs = path.resolve(repo, gc); if (path.basename(abs) === '.git') main = fs.realpathSync(path.dirname(abs)); }
|
|
107
|
+
if (main && main !== top && fs.existsSync(path.join(main, 'forge.yaml'))) { file = path.join(main, 'forge.yaml'); docText = fs.readFileSync(file, 'utf8'); }
|
|
108
|
+
else if (!main || main === top) {
|
|
109
|
+
const tracked = git(repo, ['ls-files', '--error-unmatch', '--', 'forge.yaml']).code === 0;
|
|
110
|
+
if (tracked) { const t = baseSha ? readAtRef(repo, baseRef, 'forge.yaml') : null; if (t) { file = `${baseRef}:forge.yaml`; docText = t; } }
|
|
111
|
+
else if (fs.existsSync(path.join(repo, 'forge.yaml'))) { file = path.join(repo, 'forge.yaml'); docText = fs.readFileSync(file, 'utf8'); }
|
|
112
|
+
}
|
|
113
|
+
return { file, text: docText };
|
|
114
|
+
}
|
|
115
|
+
|
|
96
116
|
export function listAtRef(repo, ref) {
|
|
97
117
|
const r = git(repo, ['ls-tree', '-r', '--name-only', ref]);
|
|
98
118
|
return r.code === 0 ? r.out.split('\n').filter(Boolean) : [];
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import fs from 'node:fs';
|
|
9
9
|
import path from 'node:path';
|
|
10
10
|
import {
|
|
11
|
-
Degraded, git, run, resolveRef, fileHash, readAtRef, listAtRef, changedFiles, headSha, sha256, hostClass, homeForgeDir, parseYaml, globToRegex,
|
|
11
|
+
Degraded, git, run, resolveRef, fileHash, readAtRef, listAtRef, changedFiles, headSha, sha256, hostClass, homeForgeDir, parseYaml, globToRegex, trustedForgeYaml,
|
|
12
12
|
} from './lib.mjs';
|
|
13
13
|
import { makeScopeCtx, scopeCommand } from './scoping.mjs';
|
|
14
14
|
import { missingTool } from './adapters/common.mjs';
|
|
@@ -31,17 +31,7 @@ const CI_ADAPTERS = [gha, gitlab, circle, azure, bitbucket, buildkite];
|
|
|
31
31
|
// base ref's tracked copy, or an untracked/ignored working-tree copy (not part of the PR). A tracked forge.yaml in the
|
|
32
32
|
// worktree is never read.
|
|
33
33
|
function loadConfig(repo, baseRef, baseSha) {
|
|
34
|
-
|
|
35
|
-
const gc = git(repo, ['rev-parse', '--git-common-dir']).out.trim();
|
|
36
|
-
const top = fs.realpathSync(path.resolve(git(repo, ['rev-parse', '--show-toplevel']).out.trim() || repo));
|
|
37
|
-
let main = '';
|
|
38
|
-
if (gc) { const abs = path.resolve(repo, gc); if (path.basename(abs) === '.git') main = fs.realpathSync(path.dirname(abs)); }
|
|
39
|
-
if (main && main !== top && fs.existsSync(path.join(main, 'forge.yaml'))) { file = path.join(main, 'forge.yaml'); docText = fs.readFileSync(file, 'utf8'); }
|
|
40
|
-
else if (!main || main === top) {
|
|
41
|
-
const tracked = git(repo, ['ls-files', '--error-unmatch', '--', 'forge.yaml']).code === 0;
|
|
42
|
-
if (tracked) { const t = baseSha ? readAtRef(repo, baseRef, 'forge.yaml') : null; if (t) { file = `${baseRef}:forge.yaml`; docText = t; } }
|
|
43
|
-
else if (fs.existsSync(path.join(repo, 'forge.yaml'))) { file = path.join(repo, 'forge.yaml'); docText = fs.readFileSync(file, 'utf8'); }
|
|
44
|
-
}
|
|
34
|
+
const { file, text: docText } = trustedForgeYaml(repo, baseRef, baseSha);
|
|
45
35
|
if (!docText) return { file: '', exclude_jobs: [], include_jobs: [], extra_commands: [], services: 'auto', mode: 'advisory' };
|
|
46
36
|
let doc; try { doc = parseYaml(docText); } catch (e) { if (e instanceof Degraded) throw e; doc = null; }
|
|
47
37
|
const c = (doc && doc.verification && doc.verification.ci_local) || {};
|
|
@@ -589,6 +589,8 @@ function scopeTsc(rest, cmd, cwdRel, sctx, join) {
|
|
|
589
589
|
return scoped(join([rest[0], '-b', ...owners.map((d) => fromCwd(cwdRel, d)), ...flags.filter((f) => f !== '--noEmit')]), owners.length, null, 'tsc-project', { projects: owners });
|
|
590
590
|
}
|
|
591
591
|
|
|
592
|
+
const isLintTask = (t) => t === 'lint' || t.startsWith('lint:') || t.endsWith(':lint');
|
|
593
|
+
|
|
592
594
|
function scopeMono(tool, rest, cmd, sctx, join) {
|
|
593
595
|
if (sctx.widen) return full(cmd, sctx.widen, { selector: `${tool}-affected` });
|
|
594
596
|
const base = sctx.from || sctx.baseRef;
|
|
@@ -597,7 +599,14 @@ function scopeMono(tool, rest, cmd, sctx, join) {
|
|
|
597
599
|
if (rest[1] === 'run-many' && !out.some((t) => t === '-t' || t.startsWith('--target'))) return full(cmd, '', { selector: 'unscoped:nx-no-target' });
|
|
598
600
|
return scoped(join([...out, `--base=${base}`]), null, null, 'nx-affected');
|
|
599
601
|
}
|
|
600
|
-
if (tool === 'turbo' && rest[1] === 'run') {
|
|
602
|
+
if (tool === 'turbo' && rest[1] === 'run') {
|
|
603
|
+
if (rest.some((t) => t.startsWith('--filter'))) return full(cmd, '', { selector: 'unscoped:explicit-filter' });
|
|
604
|
+
// #4381: lint is per-file, so a lint-only run covers the changed packages, not their dependents (`...[base]`).
|
|
605
|
+
const end = rest.findIndex((t, i) => i > 1 && t.startsWith('-'));
|
|
606
|
+
const tasks = rest.slice(2, end < 0 ? rest.length : end);
|
|
607
|
+
if (tasks.length && tasks.every(isLintTask)) return scoped(join([...rest, `--filter=[${base}]`]), null, null, 'turbo-changed');
|
|
608
|
+
return scoped(join([...rest, `--filter=...[${base}]`]), null, null, 'turbo-affected');
|
|
609
|
+
}
|
|
601
610
|
return null;
|
|
602
611
|
}
|
|
603
612
|
|
package/scripts/ci-local.test.sh
CHANGED
|
@@ -23,6 +23,8 @@ export FORGE_TELEMETRY=off
|
|
|
23
23
|
export FORGE_CI_LOCAL_HOME="$T/home"
|
|
24
24
|
export FORGE_CI_LOCAL_SLOTS=4
|
|
25
25
|
unset FORGE_CI_LOCAL_SERVICES FORGE_CI_LOCAL_NO_YQ FORGE_CI_LOCAL_BUDGET_SECONDS FORGE_CI_LOCAL_JOB_TIMEOUT FORGE_CI_LOCAL_NO_CACHE
|
|
26
|
+
# A lane's FORGE_HOST_PARALLELISM cap (#4182) would serialise the parallelism fixtures below; capParallel has its own test.
|
|
27
|
+
unset FORGE_HOST_PARALLELISM
|
|
26
28
|
HOST=$(uname -s)
|
|
27
29
|
|
|
28
30
|
pass=0; fail=0
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from itertools import islice
|
|
2
|
+
|
|
3
|
+
__all__ = ["ichunked"]
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _take(iterator, n):
|
|
7
|
+
return list(islice(iterator, n))
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def ichunked(iterable, n):
|
|
11
|
+
"""Yield successive chunks of length n from iterable."""
|
|
12
|
+
iterator = iter(iterable)
|
|
13
|
+
while True:
|
|
14
|
+
chunk = _take(iterator, n)
|
|
15
|
+
if not chunk:
|
|
16
|
+
return
|
|
17
|
+
yield iter(chunk)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from itertools import islice
|
|
2
|
+
|
|
3
|
+
__all__ = ["ichunked"]
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _take(iterator, n):
|
|
7
|
+
return list(islice(iterator, n))
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _chunks(iterator, n):
|
|
11
|
+
while True:
|
|
12
|
+
chunk = _take(iterator, n)
|
|
13
|
+
if not chunk:
|
|
14
|
+
return
|
|
15
|
+
yield iter(chunk)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def ichunked(iterable, n):
|
|
19
|
+
"""Return an iterator over successive chunks of length n from iterable."""
|
|
20
|
+
return _chunks(iter(iterable), n)
|
|
@@ -416,9 +416,9 @@ FR="$ROOT/scripts/lib/forge-resolve.sh"; FB="$ROOT/scripts/lib/fence-bootstrap.s
|
|
|
416
416
|
[ -f "$FR" ] && [ -f "$FB" ] && ok || bad "scripts/lib/forge-resolve.sh and scripts/lib/fence-bootstrap.sh must exist"
|
|
417
417
|
sed 's/^[[:space:]]*//' "$STG_SPEC" | grep '^type forge_resolve >/dev/null' | sort | uniq -c > "$T/loc_uniq"
|
|
418
418
|
[ "$(wc -l < "$T/loc_uniq" | tr -d ' ')" = 1 ] && ok || bad "forge_resolve locator line must be byte-identical across every stub in review-pr-staging.md"
|
|
419
|
-
[ "$(awk '{print $1}' "$T/loc_uniq")" -ge
|
|
419
|
+
[ "$(awk '{print $1}' "$T/loc_uniq")" -ge 12 ] && ok || bad "expected a locator on every bootstrap (10) and resolver (2) stub (the pre/post-dispatch fences moved to forgedock review-staging, #4359)"
|
|
420
420
|
grep -qE 'PWD|\$\(pwd\)|REPO_PATH' "$T/locator.sh" && bad "stub locator must not consult the cwd or repo under review" || ok
|
|
421
|
-
[ "$(grep -c '# FENCE_BOOTSTRAP$' "$STG_SPEC")" =
|
|
421
|
+
[ "$(grep -c '# FENCE_BOOTSTRAP$' "$STG_SPEC")" = 10 ] && ok || bad "expected 10 fence bootstrap stubs"
|
|
422
422
|
[ "$(grep '# FENCE_BOOTSTRAP$' "$STG_SPEC" | sort -u | wc -l | tr -d ' ')" = 1 ] && ok || bad "fence bootstrap stubs must be byte-identical"
|
|
423
423
|
grep '# FENCE_BOOTSTRAP$' "$STG_SPEC" | sed 1q | grep -qF 'exit 1' && ok || bad "fence bootstrap stub must fail closed (exit 1) when the helper is unresolved"
|
|
424
424
|
# the spec carries none of the old inlined blocks
|