forgedock 1.24.5 → 1.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +11 -0
- package/bin/engine/phases.mjs +22 -6
- package/bin/engine/runlog.mjs +1 -1
- package/bin/engine-cli.mjs +206 -22
- package/bin/engine.mjs +3 -1
- package/bin/forgedock.mjs +17 -5
- package/bin/lean/batch.mjs +147 -0
- package/bin/lean/batch.test.mjs +184 -0
- package/bin/lean/lane.mjs +85 -0
- package/bin/lean/lane.test.mjs +91 -0
- package/bin/lean/lean.mjs +122 -9
- package/bin/lean/lean.test.mjs +370 -12
- package/bin/lean/orchestrate.mjs +229 -29
- package/bin/lean/prefilter.mjs +148 -0
- package/bin/lean/prefilter.test.mjs +228 -0
- package/bin/lean/prompts.mjs +80 -3
- package/bin/runner.mjs +25 -3
- package/commands/orchestrate.md +64 -21
- package/commands/review-pr-staging.md +28 -3
- package/docs/CONFIG.md +2 -0
- package/docs/OUTCOME-CODES.md +11 -4
- package/package.json +1 -1
- package/scripts/code-index.sh +61 -40
- package/scripts/code-index.test.sh +73 -0
- package/scripts/review-pr-staging-scope.test.sh +36 -1
- package/scripts/review-prepare.sh +16 -3
- package/scripts/review-prepare.test.sh +16 -0
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
/**
|
|
4
|
+
* Durable batch state and detached lanes for `forgedock orchestrate` (#4121).
|
|
5
|
+
*
|
|
6
|
+
* A batch lives in `~/.forge/batches/<id>/` (or `FORGEDOCK_BATCH_DIR`): `plan.json` (the command), `state.json` (the
|
|
7
|
+
* scheduler state, rewritten after every transition), `log.jsonl` (one line per lane update), `parallel` (live K) and
|
|
8
|
+
* `lanes/<issue>.{json,log}` (each lane's result and output). Every write is tmp + fsync + rename, and the root must be
|
|
9
|
+
* on disk, never tmpfs. A lane is its own session (`setsid` through `detached`), so killing the dispatcher leaves it
|
|
10
|
+
* running; a resumed dispatcher re-attaches to it by pid, checking `/proc/<pid>/cmdline` against its result path.
|
|
11
|
+
*/
|
|
12
|
+
import { appendFileSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeSync } from "node:fs";
|
|
13
|
+
import { spawn } from "node:child_process";
|
|
14
|
+
import { homedir } from "node:os";
|
|
15
|
+
import { dirname, join, resolve, sep } from "node:path";
|
|
16
|
+
|
|
17
|
+
export const RAM_FS = new Set(["tmpfs", "ramfs"]);
|
|
18
|
+
const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
19
|
+
|
|
20
|
+
export function batchRoot(env = process.env) {
|
|
21
|
+
return env.FORGEDOCK_BATCH_DIR || join(homedir(), ".forge", "batches");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** `[{mount, type}]` from /proc/self/mounts; empty when unreadable (non-Linux). */
|
|
25
|
+
export function readMounts(file = "/proc/self/mounts") {
|
|
26
|
+
try {
|
|
27
|
+
return readFileSync(file, "utf8").split("\n").filter(Boolean).map((l) => {
|
|
28
|
+
const [, mount, type] = l.split(" ");
|
|
29
|
+
return { mount: mount.replace(/\\(\d{3})/g, (_, o) => String.fromCharCode(parseInt(o, 8))), type };
|
|
30
|
+
});
|
|
31
|
+
} catch { return []; }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Filesystem type of the mount that holds `path` (longest mount-point prefix); null when unknown. */
|
|
35
|
+
export function fsTypeOf(path, mounts = readMounts()) {
|
|
36
|
+
const p = resolve(path);
|
|
37
|
+
let best = null;
|
|
38
|
+
for (const m of mounts) {
|
|
39
|
+
const under = m.mount === "/" || p === m.mount || p.startsWith(m.mount.endsWith(sep) ? m.mount : m.mount + sep);
|
|
40
|
+
if (under && (!best || m.mount.length > best.mount.length)) best = m;
|
|
41
|
+
}
|
|
42
|
+
return best?.type ?? null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Throws when `path` is on a RAM filesystem: a full tmpfs truncated an earlier state file to 0 bytes. */
|
|
46
|
+
export function assertDiskPath(path, mounts = readMounts()) {
|
|
47
|
+
const type = fsTypeOf(path, mounts);
|
|
48
|
+
if (RAM_FS.has(type)) throw new Error(`batch state must be on disk, but ${path} is on ${type}; set FORGEDOCK_BATCH_DIR to a disk path`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Write `data` to `file` atomically: a reader sees the old or the new content, never a truncated file. */
|
|
52
|
+
export function writeAtomic(file, data) {
|
|
53
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
54
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
55
|
+
const fd = openSync(tmp, "w");
|
|
56
|
+
try { writeSync(fd, data); fsyncSync(fd); } finally { closeSync(fd); }
|
|
57
|
+
renameSync(tmp, file);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const readJson = (file) => { try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; } };
|
|
61
|
+
|
|
62
|
+
export function newBatchId(now = Date.now(), pid = process.pid) {
|
|
63
|
+
return `b${new Date(now).toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15)}-${pid}`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Handle on one batch directory. */
|
|
67
|
+
export function openBatch(root, id) {
|
|
68
|
+
if (!ID_RE.test(String(id ?? ""))) throw new Error(`invalid batch id ${JSON.stringify(id)}`);
|
|
69
|
+
const dir = join(root, id);
|
|
70
|
+
const f = (name) => join(dir, name);
|
|
71
|
+
return {
|
|
72
|
+
id, dir,
|
|
73
|
+
exists: () => existsSync(f("plan.json")),
|
|
74
|
+
readPlan: () => readJson(f("plan.json")),
|
|
75
|
+
writePlan: (plan) => writeAtomic(f("plan.json"), JSON.stringify(plan, null, 2) + "\n"),
|
|
76
|
+
store: {
|
|
77
|
+
load: () => readJson(f("state.json")),
|
|
78
|
+
save: (state) => writeAtomic(f("state.json"), JSON.stringify(state) + "\n"),
|
|
79
|
+
log: (row) => { try { appendFileSync(f("log.jsonl"), JSON.stringify({ at: new Date().toISOString(), ...row }) + "\n"); } catch { /* the log is advisory */ } },
|
|
80
|
+
},
|
|
81
|
+
/** Live K: the `parallel` file, read on every scheduler pass; null when missing or invalid. */
|
|
82
|
+
readParallel: () => { try { const k = parseInt(readFileSync(f("parallel"), "utf8"), 10); return Number.isInteger(k) && k >= 1 ? k : null; } catch { return null; } },
|
|
83
|
+
setParallel: (k) => {
|
|
84
|
+
if (!Number.isInteger(k) || k < 1) throw new Error("parallel must be a positive integer");
|
|
85
|
+
writeAtomic(f("parallel"), `${k}\n`);
|
|
86
|
+
},
|
|
87
|
+
resultFile: (n) => f(join("lanes", `${n}.json`)),
|
|
88
|
+
logFile: (n) => f(join("lanes", `${n}.log`)),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** True when `pid` is alive and (where /proc exists) its command line names `marker`, so a reused pid is not adopted. */
|
|
93
|
+
export function isLaneAlive(pid, marker, { kill = process.kill.bind(process), procDir = "/proc" } = {}) {
|
|
94
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
95
|
+
try { kill(pid, 0); } catch (e) { if (e?.code !== "EPERM") return false; }
|
|
96
|
+
if (!existsSync(procDir)) return true;
|
|
97
|
+
let cmd;
|
|
98
|
+
try { cmd = readFileSync(join(procDir, String(pid), "cmdline"), "utf8"); } catch { return false; }
|
|
99
|
+
return !marker || cmd.split("\0").includes(marker);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Start one lane in its own session: `<bin> <script> run-issue <issue> ...args --result-file <lanes/N.json>`, output to
|
|
104
|
+
* `lanes/N.log`. A stale result from an earlier attempt is removed first. @returns {{pid:number}}
|
|
105
|
+
*/
|
|
106
|
+
export function launchLane({ batch, issue, args = [], bin = process.execPath, script, cwd = process.cwd(), env = process.env, spawnFn = spawn }) {
|
|
107
|
+
const rf = batch.resultFile(issue);
|
|
108
|
+
mkdirSync(dirname(rf), { recursive: true });
|
|
109
|
+
rmSync(rf, { force: true });
|
|
110
|
+
const out = openSync(batch.logFile(issue), "a");
|
|
111
|
+
try {
|
|
112
|
+
const child = spawnFn(bin, [script, "run-issue", String(issue), ...args, "--result-file", rf], { cwd, env, detached: true, stdio: ["ignore", out, out] });
|
|
113
|
+
child.on?.("error", () => { /* surfaces as a dead pid without a result */ });
|
|
114
|
+
child.unref?.();
|
|
115
|
+
return { pid: child.pid };
|
|
116
|
+
} finally { closeSync(out); }
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Poll a lane until its result file appears; a lane that died without one is an engine-error. */
|
|
120
|
+
export async function waitLane({ batch, issue, pid, alive = isLaneAlive, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), pollMs = 5000 }) {
|
|
121
|
+
const rf = batch.resultFile(issue);
|
|
122
|
+
for (;;) {
|
|
123
|
+
const r = readJson(rf);
|
|
124
|
+
if (r) return r;
|
|
125
|
+
if (!alive(pid, rf)) {
|
|
126
|
+
const last = readJson(rf); // it may have written the result just before exiting
|
|
127
|
+
return last ?? { terminalReason: "engine-error", detail: `lane pid ${pid} exited without a result; see ${batch.logFile(issue)}` };
|
|
128
|
+
}
|
|
129
|
+
await sleep(pollMs);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Lane seams for `orchestrate()`: `runOne` launches a detached lane, `adopt` re-attaches to a live one (else null). */
|
|
134
|
+
export function detachedLanes({ batch, laneArgs, script, cwd, env, bin, alive = isLaneAlive, pollMs, spawnFn }) {
|
|
135
|
+
return {
|
|
136
|
+
runOne: async (n, lane, ctx = {}) => {
|
|
137
|
+
const { pid } = launchLane({ batch, issue: n, args: laneArgs(n, lane), script, cwd, env, bin, spawnFn });
|
|
138
|
+
ctx.onStart?.({ pid });
|
|
139
|
+
return waitLane({ batch, issue: n, pid, alive, pollMs });
|
|
140
|
+
},
|
|
141
|
+
adopt: async (n, rec) => {
|
|
142
|
+
const rf = batch.resultFile(n);
|
|
143
|
+
if (!existsSync(rf) && !alive(rec.pid, rf)) return null;
|
|
144
|
+
return waitLane({ batch, issue: n, pid: rec.pid, alive, pollMs });
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
import { describe, it } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { spawn } from "node:child_process";
|
|
6
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { tmpDir } from "../tests/_tmp.mjs";
|
|
9
|
+
import { orchestrate, deferPolicy, DEFER_BACKOFF_S } from "./orchestrate.mjs";
|
|
10
|
+
import { openBatch, fsTypeOf, assertDiskPath, isLaneAlive, writeAtomic } from "./batch.mjs";
|
|
11
|
+
import { orchestrateFromCli, runFromCli, readOrchestration } from "../engine-cli.mjs";
|
|
12
|
+
|
|
13
|
+
const quiet = async (fn) => { const l = console.log; console.log = () => {}; try { return await fn(); } finally { console.log = l; } };
|
|
14
|
+
const until = async (cond, ms = 15000) => {
|
|
15
|
+
const end = Date.now() + ms;
|
|
16
|
+
while (!cond()) { if (Date.now() > end) throw new Error("timed out waiting"); await new Promise((r) => setTimeout(r, 25)); }
|
|
17
|
+
};
|
|
18
|
+
// A fake clock: `sleep` advances it, so delays are asserted without waiting.
|
|
19
|
+
const fakeClock = () => { const c = { t: 1_000_000 }; return { c, now: () => c.t, sleep: async (ms) => { c.t += ms; } }; };
|
|
20
|
+
|
|
21
|
+
describe("durable batch runner (#4121)", () => {
|
|
22
|
+
it("killing the dispatcher mid-batch and --resume re-adopts live lanes and finishes without duplicate launches", async () => {
|
|
23
|
+
const work = tmpDir("batch-int-");
|
|
24
|
+
const root = join(work, "batches");
|
|
25
|
+
const launches = join(work, "launches"), release = join(work, "release");
|
|
26
|
+
writeFileSync(launches, "");
|
|
27
|
+
// Stub lane: records its launch, waits for the release file, then writes its result like `run-issue --result-file`.
|
|
28
|
+
const stub = join(work, "stub-lane.mjs");
|
|
29
|
+
writeFileSync(stub, `import { appendFileSync, existsSync, writeFileSync, renameSync } from "node:fs";
|
|
30
|
+
const a = process.argv.slice(2); const n = Number(a[1]); const rf = a[a.indexOf("--result-file") + 1];
|
|
31
|
+
appendFileSync(${JSON.stringify(launches)}, n + "\\n");
|
|
32
|
+
const t = setInterval(() => { if (!existsSync(${JSON.stringify(release)})) return; clearInterval(t);
|
|
33
|
+
writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "merged", pr: 100 + n, cost_usd: 0 })); renameSync(rf + ".tmp", rf); }, 20);
|
|
34
|
+
`);
|
|
35
|
+
const depsSrc = `{ batchRoot: ${JSON.stringify(root)}, batchId: "bt1", allowTmpfs: true, laneScript: ${JSON.stringify(stub)}, pollMs: 30,
|
|
36
|
+
orchestration: { breaker: null, maxDeferrals: 3 }, io: { gh: async () => "o/r\\n" },
|
|
37
|
+
leanIo: { gh: async (a) => a.includes("milestone") ? JSON.stringify({ milestone: null }) : a[0] === "repo" ? "main\\n" : JSON.stringify({ body: a[2] === "3" ? "Depends on #1" : "" }),
|
|
38
|
+
git: async (a) => (a[0] === "ls-remote" ? "abc\\trefs/heads/staging\\n" : "") } }`;
|
|
39
|
+
const dispatcher = join(work, "dispatcher.mjs");
|
|
40
|
+
writeFileSync(dispatcher, `import { orchestrateFromCli } from ${JSON.stringify(new URL("../engine-cli.mjs", import.meta.url).href)};
|
|
41
|
+
await orchestrateFromCli(["1", "2", "3", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], ${depsSrc});
|
|
42
|
+
`);
|
|
43
|
+
const batch = openBatch(root, "bt1");
|
|
44
|
+
const pids = () => Object.values(batch.store.load()?.running ?? {}).map((r) => r.pid).filter(Boolean);
|
|
45
|
+
let lanePids = [];
|
|
46
|
+
try {
|
|
47
|
+
const child = spawn(process.execPath, [dispatcher], { stdio: "ignore" });
|
|
48
|
+
const exited = new Promise((r) => child.on("exit", r));
|
|
49
|
+
await until(() => readFileSync(launches, "utf8").trim().split("\n").filter(Boolean).length === 2 && pids().length === 2);
|
|
50
|
+
lanePids = pids();
|
|
51
|
+
child.kill("SIGKILL");
|
|
52
|
+
await exited;
|
|
53
|
+
// The lanes run in their own session: the dispatcher's death did not take them down.
|
|
54
|
+
for (const pid of lanePids) assert.equal(isLaneAlive(pid, null), true, `lane ${pid} survived`);
|
|
55
|
+
const deps = (await import(`data:text/javascript,export default ${encodeURIComponent(depsSrc)}`)).default;
|
|
56
|
+
const resumed = quiet(() => orchestrateFromCli(["--resume", "bt1"], deps));
|
|
57
|
+
setTimeout(() => writeFileSync(release, ""), 200);
|
|
58
|
+
const res = await resumed;
|
|
59
|
+
assert.deepEqual(res.rows.map((r) => [r.issue, r.outcome, r.pr]), [[1, "merged", 101], [2, "merged", 102], [3, "merged", 103]]);
|
|
60
|
+
assert.deepEqual(readFileSync(launches, "utf8").trim().split("\n").map(Number).sort(), [1, 2, 3]); // no duplicate launch
|
|
61
|
+
assert.deepEqual(batch.store.load().running, {});
|
|
62
|
+
} finally {
|
|
63
|
+
writeFileSync(release, "");
|
|
64
|
+
for (const pid of lanePids) { try { process.kill(pid, "SIGKILL"); } catch { /* gone */ } }
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it("deferred codes are requeued with their own delay; non-host deferrals stop at the cap", async () => {
|
|
69
|
+
const { now, sleep } = fakeClock();
|
|
70
|
+
const seq = [
|
|
71
|
+
{ code: "host.busy", retryInSeconds: 60 },
|
|
72
|
+
{ code: "rate_limited", resetAtEpochMs: null, retryInSeconds: 600 },
|
|
73
|
+
{ code: "merge.transient" },
|
|
74
|
+
{ code: "push.transient" },
|
|
75
|
+
{ code: "cli.idle" },
|
|
76
|
+
];
|
|
77
|
+
const starts = [];
|
|
78
|
+
const rows = await orchestrate({ issues: [1], readBody: async () => "", now, sleep, maxDeferrals: 3,
|
|
79
|
+
runOne: async () => { starts.push(now()); const d = seq.shift(); return { terminalReason: "deferred", detail: d.code, ...d }; } });
|
|
80
|
+
const gaps = starts.slice(1).map((t, i) => (t - starts[i]) / 1000);
|
|
81
|
+
assert.deepEqual(gaps, [60, 600, DEFER_BACKOFF_S * 2, DEFER_BACKOFF_S * 4]);
|
|
82
|
+
assert.equal(rows[0].outcome, "deferred");
|
|
83
|
+
assert.match(rows[0].detail, /gave up after 3 deferrals/);
|
|
84
|
+
// the usage-limit reset time and a lease expiry are honoured
|
|
85
|
+
assert.deepEqual(deferPolicy({ code: "rate_limited", resetAtEpochMs: 5000 }, { now: 1000 }), { delayMs: 4000, hold: true });
|
|
86
|
+
assert.equal(deferPolicy({ code: "lease-held", detail: "leased by x until 1970-01-01T00:01:40.000Z" }, { now: 0 }).delayMs, 105000);
|
|
87
|
+
assert.deepEqual(deferPolicy({ code: "host.busy" }, { now: 0, deferrals: 99 }), { delayMs: 60000, host: true });
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("a usage limit holds every admission until its reset; a deferred issue's dependents wait instead of being skipped", async () => {
|
|
91
|
+
const { now, sleep, c } = fakeClock();
|
|
92
|
+
const t0 = c.t, starts = {};
|
|
93
|
+
let limited = true;
|
|
94
|
+
const rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, now, sleep, readBody: async (n) => (n === 3 ? "Blocked by #1" : ""),
|
|
95
|
+
runOne: async (n) => {
|
|
96
|
+
(starts[n] ??= []).push(now());
|
|
97
|
+
if (n === 1 && limited) { limited = false; return { terminalReason: "deferred", code: "rate_limited", resetAtEpochMs: t0 + 600_000 }; }
|
|
98
|
+
return { terminalReason: "merged" };
|
|
99
|
+
} });
|
|
100
|
+
assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "merged"]);
|
|
101
|
+
assert.ok(starts[2][0] >= t0 + 600_000, "no admission during the hold");
|
|
102
|
+
assert.equal(starts[1].length, 2);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it("the breaker stops admissions after N engine-errors in M minutes and lets running lanes finish", async () => {
|
|
106
|
+
const started = [];
|
|
107
|
+
let releaseSlow;
|
|
108
|
+
const slow = new Promise((r) => { releaseSlow = r; });
|
|
109
|
+
// #3 runs slowly; #1 fails (relaunch queued behind #2, #4), then #2 fails: two errors in the window trip the breaker.
|
|
110
|
+
const rows = await orchestrate({ issues: [3, 1, 2, 4], parallel: 2, breaker: { errors: 2, window_min: 30 }, readBody: async () => "",
|
|
111
|
+
onUpdate: (r) => { if (r.outcome === "breaker") setTimeout(releaseSlow, 10); },
|
|
112
|
+
runOne: async (n) => {
|
|
113
|
+
started.push(n);
|
|
114
|
+
if (n === 3) { await slow; return { terminalReason: "merged", pr: 103 }; }
|
|
115
|
+
throw new Error(`boom ${n}`);
|
|
116
|
+
} });
|
|
117
|
+
assert.deepEqual(started, [3, 1, 2]);
|
|
118
|
+
assert.deepEqual(rows.map((r) => r.outcome), ["merged", "held", "held", "held"]);
|
|
119
|
+
assert.match(rows[1].detail, /circuit breaker: 2 engine-errors within 30 min/);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it("an engine-error is relaunched once; a resume after the breaker continues the held issues", async () => {
|
|
123
|
+
const saved = { state: null };
|
|
124
|
+
const store = { load: () => saved.state, save: (s) => { saved.state = JSON.parse(JSON.stringify(s)); } };
|
|
125
|
+
const runs = [];
|
|
126
|
+
let fail = true;
|
|
127
|
+
const runOne = async (n) => { runs.push(n); if (fail) throw new Error("x"); return { terminalReason: "merged" }; };
|
|
128
|
+
let rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, breaker: { errors: 2, window_min: 30 }, store, readBody: async () => "", runOne });
|
|
129
|
+
assert.deepEqual(runs, [1, 2]); // #1's relaunch is queued behind #2, whose error trips the breaker
|
|
130
|
+
assert.deepEqual(rows.map((r) => r.outcome), ["held", "held", "held"]);
|
|
131
|
+
fail = false;
|
|
132
|
+
saved.state = { ...saved.state, breaker: null, errorTimes: [] }; // what --resume does
|
|
133
|
+
rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, breaker: { errors: 2, window_min: 30 }, store, readBody: async () => "", runOne });
|
|
134
|
+
assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "merged"]);
|
|
135
|
+
assert.deepEqual(runs, [1, 2, 3, 1, 2]);
|
|
136
|
+
// without a breaker an engine-error is relaunched exactly once, then final
|
|
137
|
+
const again = [];
|
|
138
|
+
const [row] = await orchestrate({ issues: [9], readBody: async () => "", runOne: async (n) => { again.push(n); throw new Error("boom"); } });
|
|
139
|
+
assert.deepEqual([again, row.outcome], [[9, 9], "engine-error"]);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("concurrency follows the live K", async () => {
|
|
143
|
+
let k = 1, active = 0, maxActive = 0;
|
|
144
|
+
const rows = await orchestrate({ issues: [1, 2, 3, 4, 5, 6, 7], parallel: 1, pollMs: 5, getParallel: () => k, readBody: async () => "",
|
|
145
|
+
onUpdate: () => { k = 3; },
|
|
146
|
+
runOne: async () => { active++; maxActive = Math.max(maxActive, active); await new Promise((r) => setTimeout(r, 15)); active--; return { terminalReason: "merged" }; } });
|
|
147
|
+
assert.equal(maxActive, 3);
|
|
148
|
+
assert.equal(rows.length, 7);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it("batch files: atomic writes, live K via --set-parallel, tmpfs refused, pid reuse not adopted", async () => {
|
|
152
|
+
const root = tmpDir("batch-files-");
|
|
153
|
+
const b = openBatch(root, "b1");
|
|
154
|
+
b.writePlan({ argv: ["1"] });
|
|
155
|
+
await quiet(() => orchestrateFromCli(["--set-parallel", "b1", "7"], { batchRoot: root }));
|
|
156
|
+
assert.equal(b.readParallel(), 7);
|
|
157
|
+
await assert.rejects(orchestrateFromCli(["--set-parallel", "nope", "2"], { batchRoot: root }), /no batch nope/);
|
|
158
|
+
await assert.rejects(orchestrateFromCli(["--set-parallel", "b1", "0"], { batchRoot: root }), /positive integer/);
|
|
159
|
+
assert.throws(() => openBatch(root, "../x"), /invalid batch id/);
|
|
160
|
+
writeAtomic(join(root, "f.json"), "{}");
|
|
161
|
+
assert.equal(readFileSync(join(root, "f.json"), "utf8"), "{}");
|
|
162
|
+
const mounts = [{ mount: "/", type: "ext4" }, { mount: "/tmp", type: "tmpfs" }, { mount: "/home", type: "btrfs" }];
|
|
163
|
+
assert.equal(fsTypeOf("/tmp/x/y", mounts), "tmpfs");
|
|
164
|
+
assert.equal(fsTypeOf("/tmpx/y", mounts), "ext4");
|
|
165
|
+
assert.throws(() => assertDiskPath("/tmp/batches", mounts), /must be on disk/);
|
|
166
|
+
assert.doesNotThrow(() => assertDiskPath("/home/u/.forge/batches", mounts));
|
|
167
|
+
assert.equal(isLaneAlive(process.pid, "--definitely-not-in-my-cmdline"), false);
|
|
168
|
+
assert.equal(isLaneAlive(2 ** 22 + 12345, null), false);
|
|
169
|
+
assert.deepEqual(readOrchestration(root), { breaker: { errors: 3, window_min: 30 }, maxDeferrals: 3 });
|
|
170
|
+
writeFileSync(join(root, "forge.yaml"), "orchestration:\n breaker:\n errors: 5\n window_min: 10\n max_deferrals: 2\n");
|
|
171
|
+
assert.deepEqual(readOrchestration(root), { breaker: { errors: 5, window_min: 10 }, maxDeferrals: 2 });
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("run-issue --result-file writes the lane result, also when the run throws", async () => {
|
|
175
|
+
const d = tmpDir("batch-rf-");
|
|
176
|
+
const rf = join(d, "lanes", "7.json");
|
|
177
|
+
const base = { dir: d, emitTelemetry: async () => {}, leanIo: {}, io: { gh: async () => "o/r\n" } };
|
|
178
|
+
await quiet(() => runFromCli(["7", "--repo", "o/r", "--result-file", rf], { ...base, leanModule: { runCore: async () => ({ terminalReason: "merged", pr: 9 }) } }));
|
|
179
|
+
assert.deepEqual(JSON.parse(readFileSync(rf, "utf8")), { terminalReason: "merged", pr: 9 });
|
|
180
|
+
await assert.rejects(quiet(() => runFromCli(["7", "--repo", "o/r", "--result-file", rf], { ...base, leanModule: { runCore: async () => { throw new Error("kaboom"); } } })), /kaboom/);
|
|
181
|
+
assert.equal(JSON.parse(readFileSync(rf, "utf8")).terminalReason, "engine-error");
|
|
182
|
+
assert.ok(!existsSync(`${rf}.tmp-${process.pid}`));
|
|
183
|
+
});
|
|
184
|
+
});
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
/**
|
|
4
|
+
* Lane resolution for core work-on (#4125). The lane (base and merge branch) comes from the issue, not the caller:
|
|
5
|
+
* a milestone maps to forge.yaml `branches.feature_pattern` with `{slug}` from the milestone title, otherwise
|
|
6
|
+
* `branches.staging`. A requested `--lane` is an assertion; the default/production branch is never a lane; the branch
|
|
7
|
+
* must exist on origin. Every failure is fail-closed with a typed `lane.*` code. Reads are injected through `io`.
|
|
8
|
+
*/
|
|
9
|
+
import { readFileSync } from "node:fs";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import { parseForgeYaml } from "../forge-utils.mjs";
|
|
12
|
+
import { OUTCOME_CODES } from "../engine/outcome-codes.mjs";
|
|
13
|
+
|
|
14
|
+
/** The documented forge.yaml conventions (forge.yaml.example, scripts/classify-lane.sh); always verified on origin. */
|
|
15
|
+
export const DEFAULT_STAGING_BRANCH = "staging";
|
|
16
|
+
export const DEFAULT_FEATURE_PATTERN = "milestone/{slug}";
|
|
17
|
+
const LANE_RE = /^[A-Za-z0-9._][A-Za-z0-9._/-]*$/;
|
|
18
|
+
|
|
19
|
+
/** Same slug as scripts/classify-lane.sh: lowercase, spaces to hyphens, keep [a-z0-9-], collapse and trim hyphens. */
|
|
20
|
+
export function milestoneSlug(title) {
|
|
21
|
+
return String(title ?? "").toLowerCase().replace(/ /g, "-").replace(/[^a-z0-9-]/g, "").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** forge.yaml `branches` from the checkout: {default, staging, featurePattern}; missing keys are null. */
|
|
25
|
+
export function readBranches(cwd = process.cwd()) {
|
|
26
|
+
let b = {};
|
|
27
|
+
try { b = parseForgeYaml(readFileSync(join(cwd, "forge.yaml"), "utf8"))?.branches ?? {}; } catch { b = {}; }
|
|
28
|
+
const s = (v) => (typeof v === "string" && v.trim() ? v.trim() : null);
|
|
29
|
+
return { default: s(b.default), staging: s(b.staging), featurePattern: s(b.feature_pattern) };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** True when `lane` is one of the default (production) branches. */
|
|
33
|
+
export function isDefaultLane(lane, defaults) {
|
|
34
|
+
return !!lane && (defaults ?? []).includes(lane);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The repo's default branches: forge.yaml `branches.default` plus GitHub's default branch. Throws when GitHub's is unreadable. */
|
|
38
|
+
export async function defaultBranches({ repo, io, branches = {} }) {
|
|
39
|
+
const gh = String(await io.gh(["repo", "view", repo, "--json", "defaultBranchRef", "--jq", ".defaultBranchRef.name"])).trim();
|
|
40
|
+
if (!gh) throw new Error("GitHub reported no default branch");
|
|
41
|
+
return [...new Set([branches.default, gh].filter(Boolean))];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @returns {Promise<{ok:true, lane:string, expected:string, milestone:?string, overridden:boolean, defaults:string[]}
|
|
46
|
+
* | {ok:false, code:string, detail:string}>}
|
|
47
|
+
*/
|
|
48
|
+
export async function resolveLane({ issue, repo, io, cwd, branches = {}, requested = null, override = false }) {
|
|
49
|
+
const refuse = (code, detail) => ({ ok: false, code, detail });
|
|
50
|
+
const staging = branches.staging || DEFAULT_STAGING_BRANCH;
|
|
51
|
+
const pattern = branches.featurePattern || DEFAULT_FEATURE_PATTERN;
|
|
52
|
+
if (!pattern.includes("{slug}")) return refuse(OUTCOME_CODES.LANE_UNRESOLVED, `forge.yaml branches.feature_pattern "${pattern}" has no {slug}`);
|
|
53
|
+
let milestone = null, defaults;
|
|
54
|
+
try {
|
|
55
|
+
milestone = JSON.parse(String(await io.gh(["issue", "view", String(issue), "-R", repo, "--json", "milestone"])))?.milestone?.title || null;
|
|
56
|
+
} catch (e) { return refuse(OUTCOME_CODES.LANE_UNRESOLVED, `cannot read the milestone of #${issue}: ${String(e?.message ?? e).slice(0, 200)}`); }
|
|
57
|
+
try { defaults = await defaultBranches({ repo, io, branches }); } catch (e) {
|
|
58
|
+
return refuse(OUTCOME_CODES.LANE_UNRESOLVED, `cannot read the default branch of ${repo}: ${String(e?.message ?? e).slice(0, 200)}`);
|
|
59
|
+
}
|
|
60
|
+
let expected = staging;
|
|
61
|
+
if (milestone) {
|
|
62
|
+
const slug = milestoneSlug(milestone);
|
|
63
|
+
if (!slug) return refuse(OUTCOME_CODES.LANE_UNRESOLVED, `milestone "${milestone}" gives an empty branch slug; rename it to include ASCII letters or digits`);
|
|
64
|
+
expected = pattern.replace("{slug}", slug);
|
|
65
|
+
}
|
|
66
|
+
const why = milestone ? `milestone "${milestone}"` : "no milestone (branches.staging)";
|
|
67
|
+
if (isDefaultLane(requested, defaults)) {
|
|
68
|
+
return refuse(OUTCOME_CODES.LANE_DEFAULT_BRANCH, `--lane ${requested} is the default branch; the core path never targets it (promotion is a human-merged release PR). #${issue} belongs on ${expected} (${why})`);
|
|
69
|
+
}
|
|
70
|
+
if (isDefaultLane(expected, defaults)) {
|
|
71
|
+
return refuse(OUTCOME_CODES.LANE_DEFAULT_BRANCH, `#${issue} resolves to ${expected} (${why}), which is the default branch; set forge.yaml branches.staging to a non-default branch`);
|
|
72
|
+
}
|
|
73
|
+
let lane = expected, overridden = false;
|
|
74
|
+
if (requested && requested !== expected) {
|
|
75
|
+
if (!override) return refuse(OUTCOME_CODES.LANE_MISMATCH, `--lane ${requested} does not match #${issue}'s lane ${expected} (${why}); omit --lane, or pass --lane-override to force it`);
|
|
76
|
+
lane = requested; overridden = true;
|
|
77
|
+
}
|
|
78
|
+
if (!LANE_RE.test(lane) || lane.includes("..")) return refuse(OUTCOME_CODES.LANE_UNRESOLVED, `invalid lane ${JSON.stringify(lane)}`);
|
|
79
|
+
let out;
|
|
80
|
+
try { out = String(await io.git(["ls-remote", "--heads", "origin", `refs/heads/${lane}`], { cwd })); } catch (e) {
|
|
81
|
+
return refuse(OUTCOME_CODES.LANE_UNRESOLVED, `cannot check origin for ${lane}: ${String(e?.message ?? e).slice(0, 200)}`);
|
|
82
|
+
}
|
|
83
|
+
if (!out.trim()) return refuse(OUTCOME_CODES.LANE_BRANCH_MISSING, `lane branch ${lane} (${why}) does not exist on origin; create it first (e.g. /milestone start) or fix the milestone title`);
|
|
84
|
+
return { ok: true, lane, expected, milestone, overridden, defaults };
|
|
85
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
import { describe, it } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { writeFileSync } from "node:fs";
|
|
6
|
+
import { tmpDir } from "../tests/_tmp.mjs";
|
|
7
|
+
import { milestoneSlug, readBranches, resolveLane } from "./lane.mjs";
|
|
8
|
+
|
|
9
|
+
/** Fake gh/git: the issue's milestone, GitHub's default branch, and the branches on origin. */
|
|
10
|
+
function io({ milestone = null, ghDefault = "main", onOrigin = ["staging", "main"], ghFails = null, gitFails = false } = {}) {
|
|
11
|
+
const calls = [];
|
|
12
|
+
return {
|
|
13
|
+
calls,
|
|
14
|
+
gh: async (a) => {
|
|
15
|
+
calls.push(["gh", ...a]);
|
|
16
|
+
if (ghFails && a[0] === ghFails) throw new Error("HTTP 502");
|
|
17
|
+
if (a[0] === "issue" && a[1] === "view") return JSON.stringify({ milestone: milestone ? { title: milestone } : null });
|
|
18
|
+
if (a[0] === "repo" && a[1] === "view") return `${ghDefault}\n`;
|
|
19
|
+
throw new Error(`unexpected gh ${a.join(" ")}`);
|
|
20
|
+
},
|
|
21
|
+
git: async (a) => {
|
|
22
|
+
calls.push(["git", ...a]);
|
|
23
|
+
if (gitFails) throw new Error("network down");
|
|
24
|
+
const b = a.at(-1).replace(/^refs\/heads\//, "");
|
|
25
|
+
return onOrigin.includes(b) ? `abc\trefs/heads/${b}\n` : "";
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
const resolve = (o, extra = {}) => resolveLane({ issue: 5, repo: "o/r", io: io(o), cwd: "/repo", ...extra });
|
|
30
|
+
|
|
31
|
+
describe("lane resolution (#4125)", () => {
|
|
32
|
+
it("slug matches scripts/classify-lane.sh", () => {
|
|
33
|
+
assert.equal(milestoneSlug("Adaptive Unblocking: Control Loop"), "adaptive-unblocking-control-loop");
|
|
34
|
+
assert.equal(milestoneSlug("User Auth v2"), "user-auth-v2");
|
|
35
|
+
assert.equal(milestoneSlug(" -- Foo (Bar) -- "), "foo-bar");
|
|
36
|
+
assert.equal(milestoneSlug("🚀✨"), "");
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it("a milestone issue resolves to feature_pattern with the slug; no milestone resolves to staging", async () => {
|
|
40
|
+
const m = await resolve({ milestone: "Adaptive Unblocking Control Loop", onOrigin: ["staging", "milestone/adaptive-unblocking-control-loop"] });
|
|
41
|
+
assert.deepEqual([m.ok, m.lane, m.overridden], [true, "milestone/adaptive-unblocking-control-loop", false]);
|
|
42
|
+
const s = await resolve({});
|
|
43
|
+
assert.deepEqual([s.ok, s.lane], [true, "staging"]);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("forge.yaml branches.staging and feature_pattern are honoured", async () => {
|
|
47
|
+
const branches = { staging: "develop", featurePattern: "feature/{slug}" };
|
|
48
|
+
assert.equal((await resolve({ onOrigin: ["develop"] }, { branches })).lane, "develop");
|
|
49
|
+
assert.equal((await resolve({ milestone: "X Y", onOrigin: ["feature/x-y"] }, { branches })).lane, "feature/x-y");
|
|
50
|
+
const bad = await resolve({}, { branches: { featurePattern: "milestone/fixed" } });
|
|
51
|
+
assert.equal(bad.code, "lane.unresolved");
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("--lane is an assertion: a mismatch is refused, --lane-override accepts it", async () => {
|
|
55
|
+
const o = { milestone: "M1", onOrigin: ["staging", "milestone/m1"] };
|
|
56
|
+
const r = await resolve(o, { requested: "staging" });
|
|
57
|
+
assert.equal(r.ok, false); assert.equal(r.code, "lane.mismatch");
|
|
58
|
+
assert.match(r.detail, /--lane staging does not match #5's lane milestone\/m1/);
|
|
59
|
+
assert.equal((await resolve(o, { requested: "milestone/m1" })).ok, true);
|
|
60
|
+
const ov = await resolve(o, { requested: "staging", override: true });
|
|
61
|
+
assert.deepEqual([ov.ok, ov.lane, ov.expected, ov.overridden], [true, "staging", "milestone/m1", true]);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("the default branch is never a lane, even with an override or when staging is configured as it", async () => {
|
|
65
|
+
for (const requested of ["main", "trunk"]) {
|
|
66
|
+
const r = await resolve({ ghDefault: "trunk" }, { requested, override: true, branches: { default: "main" } });
|
|
67
|
+
assert.equal(r.code, "lane.default_branch", requested);
|
|
68
|
+
}
|
|
69
|
+
assert.equal((await resolve({}, { branches: { staging: "main" } })).code, "lane.default_branch");
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("a missing lane branch fails closed with lane.branch_missing", async () => {
|
|
73
|
+
const r = await resolve({ milestone: "Not Created", onOrigin: ["staging"] });
|
|
74
|
+
assert.equal(r.code, "lane.branch_missing");
|
|
75
|
+
assert.match(r.detail, /milestone\/not-created/);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("unreadable milestone, default branch or origin, and an empty slug, fail closed with lane.unresolved", async () => {
|
|
79
|
+
assert.equal((await resolve({ ghFails: "issue" })).code, "lane.unresolved");
|
|
80
|
+
assert.equal((await resolve({ ghFails: "repo" })).code, "lane.unresolved");
|
|
81
|
+
assert.equal((await resolve({ gitFails: true })).code, "lane.unresolved");
|
|
82
|
+
assert.equal((await resolve({ milestone: "🚀" })).code, "lane.unresolved");
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("readBranches reads forge.yaml branches; missing keys are null", () => {
|
|
86
|
+
const d = tmpDir("lane-");
|
|
87
|
+
writeFileSync(`${d}/forge.yaml`, 'branches:\n default: "main"\n staging: "develop"\n feature_pattern: "milestone/{slug}"\n');
|
|
88
|
+
assert.deepEqual(readBranches(d), { default: "main", staging: "develop", featurePattern: "milestone/{slug}" });
|
|
89
|
+
assert.deepEqual(readBranches(tmpDir("lane-")), { default: null, staging: null, featurePattern: null });
|
|
90
|
+
});
|
|
91
|
+
});
|