forgedock 1.32.1 → 1.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +7 -3
- package/bin/engine/phases.mjs +3 -3
- package/bin/engine-cli.mjs +56 -11
- package/bin/forgedock.mjs +21 -1
- package/bin/lean/basesync.mjs +98 -0
- package/bin/lean/basesync.test.mjs +151 -0
- package/bin/lean/batch.test.mjs +103 -2
- package/bin/lean/budget.mjs +7 -1
- package/bin/lean/budget.test.mjs +9 -1
- package/bin/lean/conformance.mjs +20 -7
- package/bin/lean/conformance.test.mjs +29 -2
- package/bin/lean/depcache.mjs +23 -3
- package/bin/lean/depcache.test.mjs +42 -1
- package/bin/lean/effects.mjs +1 -1
- package/bin/lean/execution.mjs +102 -12
- package/bin/lean/execution.test.mjs +101 -1
- package/bin/lean/hostkill.mjs +8 -0
- package/bin/lean/lean.mjs +49 -21
- package/bin/lean/lean.test.mjs +133 -7
- package/bin/lean/lightlane.mjs +38 -13
- package/bin/lean/lightlane.test.mjs +35 -1
- package/bin/lean/orchestrate.mjs +63 -17
- package/bin/lean/prompts.mjs +17 -7
- package/bin/lean/recovery.test.mjs +26 -0
- package/bin/lean/sandbox-provider.contract.test.mjs +74 -1
- package/bin/lean/sandbox-remote.mjs +40 -7
- package/bin/lean/seqlease.mjs +49 -2
- package/bin/lean/seqlease.test.mjs +14 -0
- package/bin/lean/sequence.mjs +54 -6
- package/bin/lean/sequence.test.mjs +83 -1
- package/bin/pool/agent.mjs +435 -0
- package/bin/pool/cli.mjs +110 -0
- package/bin/pool/docker.mjs +56 -0
- package/bin/pool/http.mjs +55 -0
- package/bin/pool/router.mjs +299 -0
- package/bin/pool/store.mjs +25 -0
- package/bin/pool/version.mjs +44 -0
- package/bin/runner.mjs +1 -0
- package/bin/stats/aggregate.mjs +6 -0
- package/bin/stats/index.mjs +14 -2
- package/bin/stats/local-source.mjs +1 -1
- package/bin/stats/pool.mjs +51 -0
- package/bin/stats/pool.test.mjs +39 -0
- package/bin/stats/query.mjs +2 -2
- package/bin/stats/stats.test.mjs +15 -1
- package/commands/orchestrate.md +3 -3
- package/docs/CONFIG.md +23 -9
- package/docs/OUTCOME-CODES.md +5 -3
- package/docs/STATS.md +7 -0
- package/package.json +2 -1
- package/pool/README.md +13 -0
- package/pool/image/Dockerfile +46 -0
- package/pool/image/claude-shim +8 -0
- package/pool/image/cs-cred +12 -0
- package/pool/image/entrypoint.sh +61 -0
- package/pool/image/gh-shim +6 -0
- package/pool/install/install-agent-linux.sh +53 -0
- package/pool/install/install-agent.ps1 +225 -0
- package/pool/install/install-agent.sh +101 -0
- package/pool/install/install-router.sh +67 -0
- package/pool/install/keepalive.ps1 +36 -0
- package/pool/install/uninstall-agent.ps1 +20 -0
- package/scripts/hotspots.sh +33 -21
- package/scripts/hotspots.test.sh +22 -3
package/bin/lean/prompts.mjs
CHANGED
|
@@ -215,18 +215,27 @@ export function reproPlanNote(on) {
|
|
|
215
215
|
return `\n\nThis issue is a bug. When it is testable, ${CONTRACT_FILE} must also carry "repro_test":{"path":"<test file>","id":"<test name>","command":"<exact command that runs only that test>"}: a test that fails on the base and passes after the fix. The engine proves it with scripts/acceptance-red-green.sh (head's changed test files are overlaid on the base) and fails the run if it is not red on base and green on head. When it is not testable (specs, docs, workflows, environment), write "repro_test":"none","repro_test_reason":"<one line why>" instead.`;
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
+
/** forge#4519: the repo's generated, bot-owned globs (forge.yaml maintenance.generated_paths via hotspots.sh); "" when none. */
|
|
219
|
+
function generatedNote(facts) {
|
|
220
|
+
const globs = Array.isArray(facts?.generated_paths) ? facts.generated_paths.filter((g) => typeof g === "string" && g) : [];
|
|
221
|
+
if (!globs.length) return "";
|
|
222
|
+
return `\n\nGenerated paths (forge.yaml maintenance.generated_paths, owned by the repo's generator): ${globs.join(", ")}. Never list or amend them in as deliverables and never commit changes to them; the engine refuses such amendments.`;
|
|
223
|
+
}
|
|
224
|
+
|
|
218
225
|
/** forge#4195: hotspot facts for the plan prompt: one sentence plus one line per over-threshold file; "" when none. */
|
|
219
226
|
export function hotspotPlanNote(facts) {
|
|
220
|
-
|
|
227
|
+
const gen = generatedNote(facts);
|
|
228
|
+
if (!facts || facts.enabled === false || !Array.isArray(facts.files)) return gen;
|
|
221
229
|
const hot = facts.files.filter((f) => f?.over && !f.excluded);
|
|
222
|
-
if (!hot.length) return
|
|
223
|
-
return `\n\nHotspots (scripts/hotspots.sh): an Affected File below is over ${facts.file_lines} lines, so the contract must put new logic in a new or neighbouring module and give the hotspot only wiring (an import and a call); growing it by more than ${facts.growth_lines} lines needs "hotspot-exempt: <reason>" in that deliverable's change.\n${hot.map((f) => `- ${f.path}: ${f.lines} lines, ${f.commits_90d} commits/90d`).join("\n")}`;
|
|
230
|
+
if (!hot.length) return gen;
|
|
231
|
+
return gen + `\n\nHotspots (scripts/hotspots.sh): an Affected File below is over ${facts.file_lines} lines, so the contract must put new logic in a new or neighbouring module and give the hotspot only wiring (an import and a call); growing it by more than ${facts.growth_lines} lines needs "hotspot-exempt: <reason>" in that deliverable's change.\n${hot.map((f) => `- ${f.path}: ${f.lines} lines, ${f.commits_90d} commits/90d`).join("\n")}`;
|
|
224
232
|
}
|
|
225
233
|
|
|
226
234
|
/** forge#4195: the builder's growth rule with the effective thresholds; "" when the check is off. */
|
|
227
235
|
export function hotspotBuildRule(facts) {
|
|
228
|
-
|
|
229
|
-
|
|
236
|
+
const gen = generatedNote(facts);
|
|
237
|
+
if (!facts || facts.enabled === false || !(facts.file_lines > 0) || !(facts.growth_lines > 0)) return gen;
|
|
238
|
+
return gen + `\n\nDon't grow a file over ${facts.file_lines} lines by more than ${facts.growth_lines} lines. Put new code next to it.`;
|
|
230
239
|
}
|
|
231
240
|
|
|
232
241
|
/** #4322: tests that enumerate the siblings of a module the issue names (bin/lean/enumerating.mjs), for the plan; "" when none. */
|
|
@@ -248,7 +257,7 @@ export function missingEntriesPrompt(missing, mode = "local") {
|
|
|
248
257
|
return `These tests enumerate the siblings of symbols or files this change adds, but do not name them (CI will fail on the stale list):\n${missing.slice(0, 30).map((m) => `- ${m.symbol} (${m.module}) -> ${m.test}:${m.line}`).join("\n")}\n\nFor each, add the symbol to that test's list, ${NOT_MEMBER}. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
|
|
249
258
|
}
|
|
250
259
|
|
|
251
|
-
const RENUMBER_RULE = "Renumber the lane's files, never the base's: rename each with git mv to its new number, keep the SQL (or other migration source) byte-identical, change a numbered document (e.g. an ADR) only in its own number (its title), regenerate the snapshot and journal with the repo's own migration generator (never hand- or text-merge the journal), keep journal entries strictly ordered, and update every reference to the old name, including the series index (e.g. docs/adr/README.md) so it lists both the base's entry and yours.";
|
|
260
|
+
const RENUMBER_RULE = "Renumber the lane's files, never the base's: rename each with git mv to its new number, keep the SQL (or other migration source) byte-identical except a first-line header naming its own number/tag, which follows the new one, change a numbered document (e.g. an ADR) only in its own number (its title), regenerate the snapshot and journal with the repo's own migration generator (never hand- or text-merge the journal), keep journal entries strictly ordered, and update every reference to the old name, including the series index (e.g. docs/adr/README.md) so it lists both the base's entry and yours.";
|
|
252
261
|
|
|
253
262
|
/** #4322: lane-added sequence-numbered files whose number the base already uses, at hand-off or after a base-sync. */
|
|
254
263
|
export function sequencePrompt(collisions, lane, mode = "local") {
|
|
@@ -301,7 +310,8 @@ const BLOCK_RULES = `"blocked" needs a typed "block" in ${RESULT_FILE}, one of:
|
|
|
301
310
|
- {"kind":"dependency","issues":[N]}: an open upstream issue must land first. The engine requeues this issue behind it.
|
|
302
311
|
- {"kind":"split-required","subissues":[{"title":"","body":"","depends_on":[0],"files":["<path>"]}],"size":{"files":N,"lines":N}}: too big for one change. The engine files the sub-issues (depends_on = indexes of earlier sub-issues; files = the files or dirs each will edit, also accepted as "paths") only when every sub-issue's files are disjoint from the others' or the whole change's size estimate exceeds ${SPLIT_MIN_FILES} files or ${SPLIT_MIN_LINES} lines; otherwise keep one lane: an ordered checklist in the contract with a validate per item. In an admitted split, when siblings would edit the same hub (a barrel or index export list, a union type, a shared test fixture, an index doc), pre-stage its slots in the earliest sub-issue and make the others depend on it, or give each sibling its own per-feature file; the engine serialises sub-issues that still share a file.
|
|
303
312
|
- {"kind":"human","why":"${HUMAN_BLOCK_REASONS.join("|")}"}: only an external action agents cannot take, or a product decision too critical for agents.
|
|
304
|
-
"An upstream issue must land first", "needs a split" and "needs prod data that read-only monitoring can read" are never "human"
|
|
313
|
+
"An upstream issue must land first", "needs a split" and "needs prod data that read-only monitoring can read" are never "human".
|
|
314
|
+
Remote sandbox lacking a host-only capability (sops, prod_monitor, ops_credentials): name it in the summary; the host reruns the lane.`;
|
|
305
315
|
|
|
306
316
|
export function blockCorrectionPrompt(problem) {
|
|
307
317
|
return `Your ${RESULT_FILE} says "blocked", but ${String(problem).slice(0, 1000)}. That is not a block that needs a human.\n\n${BLOCK_RULES}\n\nEither do the work (fix, commit, update the stage notes and ${RESULT_FILE}) or write a valid typed "block". Do not push or touch GitHub.`;
|
|
@@ -56,6 +56,32 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
56
56
|
assert.equal(lines.length, 1); assert.match(lines[0], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
|
|
57
57
|
});
|
|
58
58
|
|
|
59
|
+
it("#4505 budget-exceeded: one recovery lane from the checkpoint at the next size; dependents stay queued; no relabel while it recovers", async () => {
|
|
60
|
+
const BE = (detail, more = {}) => ({ terminalReason: "budget-exceeded", code: "budget.exceeded", detail, branch: "feat/issue-1-lean", ...more });
|
|
61
|
+
const relabels = [];
|
|
62
|
+
const { rows, calls, decisions } = await batch({ 1: [BE("Implement turn budget spent (size S, 25 turns; 30 used)"), { terminalReason: "merged", pr: 101 }], 2: [{ terminalReason: "merged", pr: 102 }] },
|
|
63
|
+
{ 2: "**Depends on**: #1" }, { relabelNeedsHuman: async (x) => { relabels.push(x); } });
|
|
64
|
+
const r = byIssue(rows);
|
|
65
|
+
assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover), [null, { code: "budget.exceeded", attempt: 1, final: true }]);
|
|
66
|
+
assert.equal(r[1].outcome, "merged"); assert.equal(r[1].recovered, "budget.exceeded");
|
|
67
|
+
assert.equal(r[2].outcome, "merged", "the dependent stayed queued and ran");
|
|
68
|
+
assert.ok(calls.findIndex((c) => c.n === 2) > calls.findLastIndex((c) => c.n === 1), "the dependent ran after the recovered merge");
|
|
69
|
+
assert.ok(decisions.some((d) => d.issue === 1 && /recovery lane for budget\.exceeded .*dependents stay queued/.test(d.what)));
|
|
70
|
+
assert.equal(relabels.length, 0, "never labelled needs-human");
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("#4505 an unrecovered budget-exceeded (recovery off) is relabelled once by the runner, parks dependents and is listed unrecovered", async () => {
|
|
74
|
+
const relabels = [];
|
|
75
|
+
const { rows, calls } = await batch({ 1: [{ terminalReason: "budget-exceeded", code: "budget.exceeded", detail: "Implement turn budget spent (size S)" }], 2: [] },
|
|
76
|
+
{ 2: "**Depends on**: #1" }, { maxAttempts: 0, relabelNeedsHuman: async (x) => { relabels.push(x); } });
|
|
77
|
+
const r = byIssue(rows);
|
|
78
|
+
assert.equal(calls.filter((c) => c.n === 1).length, 1);
|
|
79
|
+
assert.equal(r[1].outcome, "budget-exceeded");
|
|
80
|
+
assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /budget\.exceeded[\s\S]*size S/);
|
|
81
|
+
assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
|
|
82
|
+
assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
|
|
83
|
+
});
|
|
84
|
+
|
|
59
85
|
it("human codes (product-decision, spend) are never recovered, still skip dependents and are the only escalations, asked once while other lanes keep running", async () => {
|
|
60
86
|
const { rows, calls, decisions } = await batch({
|
|
61
87
|
1: [NH("builder.blocked", "Builder is blocked (product-decision): pick a pricing model")],
|
|
@@ -54,7 +54,7 @@ function fakeTransport({ launchError = null } = {}) {
|
|
|
54
54
|
function remoteFixture(batch, { launchError = null } = {}) {
|
|
55
55
|
const transport = fakeTransport({ launchError });
|
|
56
56
|
const provider = remoteProvider({ batch, repo: "o/r", settings: { image: "img:1", cacheStore: "s3://cache", tokenEnv: "FD_TOKEN", secrets: ["GH_TOKEN"] }, transport,
|
|
57
|
-
env: { GH_TOKEN: SECRET, FD_TOKEN: TOKEN }, credentials: (n) => ({ GH_TOKEN: SECRET, FD_TOKEN: TOKEN })[n] ?? null });
|
|
57
|
+
env: { GH_TOKEN: SECRET, FD_TOKEN: TOKEN }, credentials: (n) => ({ GH_TOKEN: SECRET, FD_TOKEN: TOKEN })[n] ?? null, gitIdentity: () => ({}) });
|
|
58
58
|
return {
|
|
59
59
|
provider, transport,
|
|
60
60
|
finish: (h, result) => { const b = transport.boxes.get(h.id); b.log += `lane done; token ${SECRET} and ${TOKEN}\n`; b.result = result; b.state = "exited"; b.minutes = 3.5; },
|
|
@@ -147,6 +147,23 @@ describe("remote adapter (#4254)", () => {
|
|
|
147
147
|
assert.equal(transport.specs[0].env.GH_TOKEN, "from-keychain");
|
|
148
148
|
});
|
|
149
149
|
|
|
150
|
+
it("the sandbox commits as the dispatching host's git identity; a lane env identity wins", async () => {
|
|
151
|
+
const batch = openBatch(join(tmpDir("remote-git-"), "batches"), "bt");
|
|
152
|
+
const transport = fakeTransport();
|
|
153
|
+
const p = remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null, gitIdentity: () => ({ name: "bot[bot]", email: "1+bot[bot]@users.noreply.github.com" }) });
|
|
154
|
+
await p.start({ issue: 1, args: [], env: {} });
|
|
155
|
+
const env = transport.specs[0].env;
|
|
156
|
+
for (const role of ["AUTHOR", "COMMITTER"]) {
|
|
157
|
+
assert.equal(env[`GIT_${role}_NAME`], "bot[bot]");
|
|
158
|
+
assert.equal(env[`GIT_${role}_EMAIL`], "1+bot[bot]@users.noreply.github.com");
|
|
159
|
+
}
|
|
160
|
+
await p.start({ issue: 2, args: [], env: { GIT_AUTHOR_NAME: "lane", GIT_AUTHOR_EMAIL: "lane@example.invalid" } });
|
|
161
|
+
assert.equal(transport.specs[1].env.GIT_AUTHOR_EMAIL, "lane@example.invalid");
|
|
162
|
+
const none = remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null, gitIdentity: () => ({ name: "", email: "" }) });
|
|
163
|
+
await none.start({ issue: 3, args: [], env: {} });
|
|
164
|
+
assert.equal("GIT_AUTHOR_EMAIL" in transport.specs[2].env, false);
|
|
165
|
+
});
|
|
166
|
+
|
|
150
167
|
it("streams the lane log into the batch log, redacted; billable minutes on the handle and on the lane result", async () => {
|
|
151
168
|
const { provider, finish, batch } = setup();
|
|
152
169
|
const lanes = providerLanes({ provider: { ...provider, start: async (a) => { const h = await provider.start(a); setTimeout(() => finish(h, { terminalReason: "merged", pr: 2 }), 5); return h; } },
|
|
@@ -255,3 +272,59 @@ describe("gc reaps a remote sandbox left by a dead dispatcher (#4254)", () => {
|
|
|
255
272
|
assert.equal(existsSync(led.path), false);
|
|
256
273
|
});
|
|
257
274
|
});
|
|
275
|
+
|
|
276
|
+
describe("pool provider (#4546)", () => {
|
|
277
|
+
const setupPool = () => {
|
|
278
|
+
const dir = tmpDir("pool-cfg-");
|
|
279
|
+
const cfg = join(dir, "router.json");
|
|
280
|
+
writeFileSync(cfg, JSON.stringify({ listen: { host: "100.64.0.1", port: 8123 } }));
|
|
281
|
+
writeFileSync(join(dir, "forgedock.token"), "pool-token-789\n");
|
|
282
|
+
return { dir, cfg };
|
|
283
|
+
};
|
|
284
|
+
const launchOnce = async (makeProvider) => {
|
|
285
|
+
const calls = [];
|
|
286
|
+
const real = globalThis.fetch;
|
|
287
|
+
globalThis.fetch = async (url, init) => { calls.push({ url: String(url), init }); return { status: 201, ok: true, headers: { get: () => null }, text: async () => "{}" }; };
|
|
288
|
+
try { await makeProvider().start({ issue: 5, args: ["--lane", "main"], env: {}, secrets: [] }); } finally { globalThis.fetch = real; }
|
|
289
|
+
return calls[0];
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
it("defaults the endpoint and token to the local router and injects the cwd forge.yaml", async () => {
|
|
293
|
+
const { cfg } = setupPool();
|
|
294
|
+
const cwd = tmpDir("pool-cwd-");
|
|
295
|
+
writeFileSync(join(cwd, "forge.yaml"), "verify:\n mode: ci\n");
|
|
296
|
+
const batch = openBatch(tmpDir("pool-b-"), "bt");
|
|
297
|
+
const make = () => providerByName("pool", { batch, repo: "o/r", cwd, exec: parseExecution({ mode: "remote", provider: "pool" }), env: {}, poolConfigPath: cfg });
|
|
298
|
+
assert.equal(make().name, "pool");
|
|
299
|
+
const c = await launchOnce(make);
|
|
300
|
+
assert.equal(c.url, "http://100.64.0.1:8123/sandboxes");
|
|
301
|
+
assert.equal(c.init.headers.authorization, "Bearer pool-token-789");
|
|
302
|
+
const spec = JSON.parse(c.init.body);
|
|
303
|
+
assert.equal(Buffer.from(spec.env.CS_FORGE_YAML_B64, "base64").toString(), "verify:\n mode: ci\n");
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
it("an explicit endpoint wins; no forge.yaml adds no env; a missing router config is misconfigured", async () => {
|
|
307
|
+
const { dir } = setupPool();
|
|
308
|
+
const batch = openBatch(tmpDir("pool-b2-"), "bt");
|
|
309
|
+
const c = await launchOnce(() => providerByName("pool", { batch, repo: "o/r", cwd: dir, exec: parseExecution({ provider: "pool", endpoint: "https://pool.example/v1" }), env: {}, poolConfigPath: join(dir, "router.json") }));
|
|
310
|
+
assert.equal(c.url, "https://pool.example/v1/sandboxes");
|
|
311
|
+
assert.equal(JSON.parse(c.init.body).env.CS_FORGE_YAML_B64, undefined);
|
|
312
|
+
assert.throws(() => providerByName("pool", { batch, repo: "o/r", cwd: dir, exec: parseExecution({ provider: "pool" }), env: {}, poolConfigPath: join(dir, "none.json") }), (e) => e.code === "execution.provider.misconfigured");
|
|
313
|
+
});
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
describe("remote capacity (GET /status, #4540)", () => {
|
|
317
|
+
const mk = (res) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url) => { if (res instanceof Error) throw res; assert.equal(url, "https://pool.example/v1/status"); return { status: res.status, ok: res.status < 400, text: async () => res.body ?? "", headers: { get: () => null } }; } });
|
|
318
|
+
it("free integer is reported; 404, malformed, negative and transport errors are unknown (null)", async () => {
|
|
319
|
+
assert.deepEqual(await mk({ status: 200, body: '{"free":3}' }).capacity(), { free: 3 });
|
|
320
|
+
assert.deepEqual(await mk({ status: 200, body: '{"free":0}' }).capacity(), { free: 0 });
|
|
321
|
+
for (const r of [{ status: 404 }, { status: 200, body: "nope" }, { status: 200, body: '{"free":"2"}' }, { status: 200, body: '{"free":-1}' }, new Error("down")]) assert.equal(await mk(r).capacity(), null);
|
|
322
|
+
});
|
|
323
|
+
it("remoteProvider.capacity delegates, and is null without a transport capability or on a throw", async () => {
|
|
324
|
+
const batch = openBatch(join(tmpDir("cap-"), "batches"), "bt");
|
|
325
|
+
const p = (transport) => remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null });
|
|
326
|
+
assert.deepEqual(await p({ capacity: async () => ({ free: 1 }) }).capacity(), { free: 1 });
|
|
327
|
+
assert.equal(await p({}).capacity(), null);
|
|
328
|
+
assert.equal(await p({ capacity: async () => { throw new Error("x"); } }).capacity(), null);
|
|
329
|
+
});
|
|
330
|
+
});
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* `httpTransport` is the one shipped transport: a small JSON protocol (docs/CONFIG.md `execution`) that a runner pool
|
|
16
16
|
* or a session gateway fronts. Tests use an in-memory transport.
|
|
17
17
|
*/
|
|
18
|
+
import { execFileSync } from "node:child_process";
|
|
18
19
|
import { appendFileSync, mkdirSync } from "node:fs";
|
|
19
20
|
import { dirname } from "node:path";
|
|
20
21
|
import { safeSegment } from "./ledger.mjs";
|
|
@@ -41,8 +42,9 @@ export function isActionsEndpoint(endpoint) {
|
|
|
41
42
|
/**
|
|
42
43
|
* Generic HTTP transport: POST {endpoint}/sandboxes (body = launch spec), GET /sandboxes/:id -> {state},
|
|
43
44
|
* GET /sandboxes/:id/logs?cursor=N -> {text, cursor}, GET /sandboxes/:id/result -> lane result (404 = none yet),
|
|
44
|
-
* GET /sandboxes/:id/usage -> {minutes}, DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
|
|
45
|
+
* GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
|
|
45
46
|
* 429/503 on launch is a retryable capacity error (`Retry-After` seconds). Error text never carries the token.
|
|
47
|
+
* Optional `GET /status -> {free: N}` is the pool's free slots (#4540); anything else (404, error, malformed) is unknown (null).
|
|
46
48
|
*/
|
|
47
49
|
export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fetch }) {
|
|
48
50
|
const root = String(endpoint ?? "").replace(/\/+$/, "");
|
|
@@ -70,8 +72,19 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
|
|
|
70
72
|
},
|
|
71
73
|
async logs(id, cursor = 0) { const r = await call("GET", `${p(id)}/logs?cursor=${encodeURIComponent(cursor)}`); return r.ok ? { text: String(r.data?.text ?? ""), cursor: r.data?.cursor ?? cursor } : { text: "", cursor }; },
|
|
72
74
|
async result(id) { const r = await call("GET", `${p(id)}/result`); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
|
|
73
|
-
async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes) } : null; },
|
|
75
|
+
async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes), ...(typeof r.data.agent === "string" && r.data.agent ? { agent: r.data.agent } : {}) } : null; }, // agent: #4549, optional
|
|
74
76
|
async destroy(id) { const r = await call("DELETE", p(id)); if (r.ok || r.status === 404) return true; throw fail("destroy", r); },
|
|
77
|
+
async capacity() {
|
|
78
|
+
// `free` (#4540) and agent liveness `online`/`total` (#4547) are each optional; none valid is unknown.
|
|
79
|
+
try {
|
|
80
|
+
const r = await call("GET", "/status");
|
|
81
|
+
if (!r.ok) return null;
|
|
82
|
+
const ok = (v) => Number.isInteger(v) && v >= 0;
|
|
83
|
+
const { free, online, total } = r.data ?? {};
|
|
84
|
+
const out = { ...(ok(free) ? { free } : {}), ...(ok(online) ? { online } : {}), ...(ok(total) ? { total } : {}) };
|
|
85
|
+
return Object.keys(out).length ? out : null;
|
|
86
|
+
} catch { return null; }
|
|
87
|
+
},
|
|
75
88
|
};
|
|
76
89
|
}
|
|
77
90
|
|
|
@@ -79,7 +92,23 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
|
|
|
79
92
|
* The remote provider. `settings` is parseExecution's output (image, cacheStore, tokenEnv, secrets); `env` is the
|
|
80
93
|
* dispatcher's env (secret values for redaction on adopt); `credentials(name)` resolves a secret (env, then keychain).
|
|
81
94
|
*/
|
|
82
|
-
|
|
95
|
+
/** The dispatching host's git identity (`git config user.name/email`), so sandbox commits carry the same author as local lanes. */
|
|
96
|
+
export function hostGitIdentity() {
|
|
97
|
+
const get = (k) => { try { return execFileSync("git", ["config", k], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { return ""; } };
|
|
98
|
+
return { name: get("user.name"), email: get("user.email") };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** GIT_AUTHOR_* and GIT_COMMITTER_* for the sandbox, unless the lane env already sets them; empty when the host has no identity. */
|
|
102
|
+
export function gitIdentityEnv(identity, laneEnv = {}) {
|
|
103
|
+
const out = {};
|
|
104
|
+
for (const role of ["AUTHOR", "COMMITTER"]) {
|
|
105
|
+
const n = laneEnv[`GIT_${role}_NAME`] || identity?.name, e = laneEnv[`GIT_${role}_EMAIL`] || identity?.email;
|
|
106
|
+
if (n && e) { out[`GIT_${role}_NAME`] = n; out[`GIT_${role}_EMAIL`] = e; }
|
|
107
|
+
}
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function remoteProvider({ batch, repo, settings = {}, name = "remote", transport, env = process.env, credentials = (name) => resolveCredential(name, { env }), now = () => Date.now(), gitIdentity = hostGitIdentity }) {
|
|
83
112
|
if (!transport) throw new Error("remote provider needs a transport");
|
|
84
113
|
const idFor = (issue) => `fdrsbx-${safeSegment(batch.id)}-${issue}`;
|
|
85
114
|
const errs = new Map(); // sandbox id -> consecutive status errors
|
|
@@ -89,7 +118,7 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
89
118
|
if (settings.tokenEnv) remember(credentials(settings.tokenEnv));
|
|
90
119
|
const minutesOf = async (h) => {
|
|
91
120
|
let m = null;
|
|
92
|
-
try { const u = await transport.usage?.(h.id); if (u && Number.isFinite(Number(u.minutes))) m = Number(u.minutes); } catch { m = null; }
|
|
121
|
+
try { const u = await transport.usage?.(h.id); if (u && Number.isFinite(Number(u.minutes))) m = Number(u.minutes); if (typeof u?.agent === "string" && u.agent) h.agent = u.agent; } catch { m = null; }
|
|
93
122
|
if (m === null && Number.isFinite(h.startedAt)) m = Math.round(((now() - h.startedAt) / 60000) * 10) / 10;
|
|
94
123
|
if (m !== null) h.minutes = Math.max(h.minutes ?? 0, m);
|
|
95
124
|
};
|
|
@@ -102,7 +131,7 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
102
131
|
} catch { /* the next poll retries */ }
|
|
103
132
|
};
|
|
104
133
|
return {
|
|
105
|
-
name
|
|
134
|
+
name,
|
|
106
135
|
idFor,
|
|
107
136
|
async start({ issue, args = [], env: laneEnv = env, secrets = settings.secrets ?? [] }) {
|
|
108
137
|
const id = idFor(issue);
|
|
@@ -115,12 +144,13 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
115
144
|
id, issue, batch: batch.id, image: settings.image ?? null,
|
|
116
145
|
repo: { slug: repo, url: repo ? `https://github.com/${repo}.git` : null, ref: li >= 0 ? args[li + 1] ?? null : null },
|
|
117
146
|
cache: settings.cacheStore ? { store: settings.cacheStore, key: safeSegment(String(repo ?? "repo")) } : null,
|
|
118
|
-
|
|
147
|
+
// The host's git identity: without it the lane image falls back to a placeholder author that fails DCO.
|
|
148
|
+
env: { ...gitIdentityEnv(gitIdentity(), laneEnv ?? {}), ...scopedEnv({ env: resolved, secrets, batchId: batch.id, repo }), ...(settings.forgeYaml ? { CS_FORGE_YAML_B64: Buffer.from(settings.forgeYaml).toString("base64") } : {}) }, // #4546: the dispatcher's forge.yaml (gitignored in the repo)
|
|
119
149
|
command: ["forgedock", "run-issue", String(issue), ...args, "--result-file", resultFile], result_file: resultFile,
|
|
120
150
|
};
|
|
121
151
|
const startedAt = now();
|
|
122
152
|
await transport.launch(spec);
|
|
123
|
-
return { provider:
|
|
153
|
+
return { provider: name, id, issue, startedAt, minutes: 0 };
|
|
124
154
|
},
|
|
125
155
|
async status(h) {
|
|
126
156
|
let s;
|
|
@@ -134,6 +164,9 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
134
164
|
if (r) { await pullLogs(h); await minutesOf(h); }
|
|
135
165
|
return r ?? null;
|
|
136
166
|
},
|
|
167
|
+
async capacity() {
|
|
168
|
+
try { return (await transport.capacity?.()) ?? null; } catch { return null; }
|
|
169
|
+
},
|
|
137
170
|
async teardown(h) {
|
|
138
171
|
await minutesOf(h);
|
|
139
172
|
try { return (await transport.destroy(h.id)) !== false; } catch { return false; }
|
package/bin/lean/seqlease.mjs
CHANGED
|
@@ -6,23 +6,70 @@
|
|
|
6
6
|
* Each key is a one-slot `heavy.mjs` semaphore in its own registry dir (`schedDir()/seq-lease/<key>/heavy.json`):
|
|
7
7
|
* file-locked, FIFO, dead holders dropped by pid. The heavy-step registry and its env caps are never touched.
|
|
8
8
|
* Fail-open: a wait over `maxWaitS` (or an unreadable registry) proceeds without the lease; CI and base-sync stay the gate.
|
|
9
|
+
* #4526: number reservations. `reserve` picks the numbers of a renumber under a short host lock per (repo, base, dir) in
|
|
10
|
+
* its own registry (`schedDir()/seq-pick/<key>/`, never the lease above: its holder renumbers while it holds it) and
|
|
11
|
+
* records them in `reserved.json` until `unreserve` (run end), the holder's death or 24h, so concurrent lanes renumbering
|
|
12
|
+
* against the same base get distinct numbers without holding a lock across regenerate and push. Fail-open.
|
|
9
13
|
*/
|
|
10
14
|
import { createHash } from "node:crypto";
|
|
15
|
+
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
11
16
|
import { join } from "node:path";
|
|
12
17
|
import { acquireHeavy, openHeavy } from "./heavy.mjs";
|
|
13
|
-
import { schedDir } from "./sched.mjs";
|
|
18
|
+
import { pidAlive, schedDir } from "./sched.mjs";
|
|
14
19
|
|
|
15
20
|
/** The lease key of one sequence dir of one lane branch of one repo. */
|
|
16
21
|
export const sequenceLeaseKey = (repo, lane, dir) => createHash("sha1").update(`${repo}\0${lane}\0${dir}`).digest("hex").slice(0, 16);
|
|
17
22
|
|
|
18
23
|
const NO_MEM = Object.freeze({ cores: () => 2, memAvailableMb: () => null });
|
|
24
|
+
const RESERVE_TTL_MS = 24 * 3600 * 1000;
|
|
19
25
|
|
|
20
26
|
/**
|
|
21
27
|
* `acquire({repo, lane, dirs, issue, maxWaitS})` takes every dir's lease in sorted order (no lock-order deadlock).
|
|
22
28
|
* @returns {{acquire: (e:object) => Promise<{waitMs:number, timedOut:boolean, release:Function}>}}
|
|
23
29
|
*/
|
|
24
|
-
export function makeSequenceLeaseIo({ dir = join(schedDir(), "seq-lease"), open = openHeavy, acquire = acquireHeavy, pollMs } = {}) {
|
|
30
|
+
export function makeSequenceLeaseIo({ dir = join(schedDir(), "seq-lease"), pickDir = join(schedDir(), "seq-pick"), open = openHeavy, acquire = acquireHeavy, pollMs, alive = pidAlive, now = () => Date.now() } = {}) {
|
|
31
|
+
const reservedDirs = new Map(); // `${repo}\0${lane}\0${issue}` -> dirs reserved in, for unreserve
|
|
32
|
+
/** Run `fn(entries) => entries'` on one key's reservations under its pick lock; returns what `fn` set as `out`. */
|
|
33
|
+
const locked = async ({ repo, lane, dir: d }, fn) => {
|
|
34
|
+
const kd = join(pickDir, sequenceLeaseKey(repo, lane, d)), file = join(kd, "reserved.json");
|
|
35
|
+
const h = open(kd, { cfg: { max: 1, max_wait_s: 60 }, env: {}, host: NO_MEM });
|
|
36
|
+
const g = await acquire(h, { phase: "sequence-pick" }, ...(pollMs ? [{ pollMs }] : []));
|
|
37
|
+
try {
|
|
38
|
+
let entries = [];
|
|
39
|
+
try { const o = JSON.parse(readFileSync(file, "utf8")); if (Array.isArray(o?.entries)) entries = o.entries; } catch { /* none yet */ }
|
|
40
|
+
const t = now();
|
|
41
|
+
entries = entries.filter((e) => Array.isArray(e?.numbers) && alive(e.pid) && t - Number(e.at ?? 0) < RESERVE_TTL_MS);
|
|
42
|
+
const r = fn(entries);
|
|
43
|
+
mkdirSync(kd, { recursive: true });
|
|
44
|
+
writeFileSync(`${file}.tmp`, `${JSON.stringify({ entries: r.entries }, null, 2)}\n`);
|
|
45
|
+
renameSync(`${file}.tmp`, file);
|
|
46
|
+
return r.out;
|
|
47
|
+
} finally { try { g.release(); } catch { /* fail-open */ } }
|
|
48
|
+
};
|
|
25
49
|
return {
|
|
50
|
+
/**
|
|
51
|
+
* #4526: `count` consecutive numbers above max(floor, every other issue's live reservation) in (repo, lane, dir),
|
|
52
|
+
* recorded for `issue` (its earlier entries there replaced). @returns {Promise<{numbers:number[]}|null>} null on error.
|
|
53
|
+
*/
|
|
54
|
+
async reserve({ repo, lane, dir: d, issue = null, floor = 0, count = 1, pid = process.pid } = {}) {
|
|
55
|
+
try {
|
|
56
|
+
const k = `${repo}\0${lane}\0${issue}`;
|
|
57
|
+
reservedDirs.set(k, new Set([...(reservedDirs.get(k) ?? []), d]));
|
|
58
|
+
return await locked({ repo, lane, dir: d }, (entries) => {
|
|
59
|
+
const others = entries.filter((e) => String(e.issue) !== String(issue));
|
|
60
|
+
let n = Math.max(Number(floor) || 0, ...others.flatMap((e) => e.numbers.map(Number)).filter(Number.isFinite));
|
|
61
|
+
const numbers = Array.from({ length: Math.max(0, count) }, () => ++n);
|
|
62
|
+
return { entries: [...others, { issue, pid, numbers, at: now() }], out: { numbers } };
|
|
63
|
+
});
|
|
64
|
+
} catch { return null; }
|
|
65
|
+
},
|
|
66
|
+
/** #4526: drop `issue`'s reservations in every dir of (repo, lane) it reserved in this process. Never throws. */
|
|
67
|
+
async unreserve({ repo, lane, issue = null } = {}) {
|
|
68
|
+
for (const d of [...(reservedDirs.get(`${repo}\0${lane}\0${issue}`) ?? [])]) {
|
|
69
|
+
try { await locked({ repo, lane, dir: d }, (entries) => ({ entries: entries.filter((e) => String(e.issue) !== String(issue)) })); } catch { /* fail-open */ }
|
|
70
|
+
}
|
|
71
|
+
reservedDirs.delete(`${repo}\0${lane}\0${issue}`);
|
|
72
|
+
},
|
|
26
73
|
async acquire({ repo, lane, dirs = [], issue = null, maxWaitS = 1800 } = {}) {
|
|
27
74
|
const held = [];
|
|
28
75
|
let waitMs = 0, timedOut = false;
|
|
@@ -39,6 +39,20 @@ describe("sequence lease (#4386)", () => {
|
|
|
39
39
|
assert.equal(b.timedOut, true);
|
|
40
40
|
b.release(); a.release();
|
|
41
41
|
});
|
|
42
|
+
|
|
43
|
+
it("#4526: reserve gives each issue distinct numbers above the floor and other issues' live reservations; dead or unreserved entries free them", async () => {
|
|
44
|
+
const dead = new Set();
|
|
45
|
+
const io = makeSequenceLeaseIo({ dir: tmpDir("seqlease-"), pickDir: tmpDir("seqpick-"), pollMs: 20, alive: (pid) => !dead.has(pid) });
|
|
46
|
+
const k = { repo: "o/r", lane: "staging", dir: "db" };
|
|
47
|
+
const [a, b] = await Promise.all([io.reserve({ ...k, issue: 1, floor: 51, count: 1, pid: 101 }), io.reserve({ ...k, issue: 2, floor: 51, count: 2, pid: 102 })]);
|
|
48
|
+
assert.deepEqual([...a.numbers, ...b.numbers].sort((x, y) => x - y), [52, 53, 54]);
|
|
49
|
+
assert.deepEqual((await io.reserve({ ...k, issue: 1, floor: 51, count: 1, pid: 101 })).numbers, [Math.max(...b.numbers) + 1], "own entry replaced, others kept");
|
|
50
|
+
assert.deepEqual((await io.reserve({ ...k, dir: "docs/adr", issue: 3, floor: 51, pid: 103 })).numbers, [52], "other dir: own registry");
|
|
51
|
+
assert.deepEqual((await io.reserve({ ...k, lane: "main", issue: 3, floor: 51, pid: 103 })).numbers, [52], "other base: own registry");
|
|
52
|
+
dead.add(102);
|
|
53
|
+
await io.unreserve({ repo: "o/r", lane: "staging", issue: 1 });
|
|
54
|
+
assert.deepEqual((await io.reserve({ ...k, issue: 4, floor: 51, pid: 104 })).numbers, [52], "dead holder and unreserved issue dropped");
|
|
55
|
+
});
|
|
42
56
|
});
|
|
43
57
|
|
|
44
58
|
const env = { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" };
|
package/bin/lean/sequence.mjs
CHANGED
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
* #4343: a leading calendar date (YYYY-MM-DD, YYYYMMDD) is not a number, and a directory only counts when it holds a
|
|
12
12
|
* series on the base: a migration journal (`<dir>/meta/_journal.json`, `<dir>/_journal.json`), or >=2 distinct numbers
|
|
13
13
|
* of the lane file's width that fill at least half of their min..max range (sparse issue-numbered docs are no series).
|
|
14
|
+
* #4526: renumberLane takes its numbers from a host reservation (`reserve`, seqlease.mjs) so concurrent lanes get distinct
|
|
15
|
+
* ones; verifyRenumber accepts a first-line header that only swaps the file's own tag (`headerOnlyChange`); a hand-written
|
|
16
|
+
* migration the generator writes nothing for gets the generator's `--custom` snapshot and keeps the lane's SQL.
|
|
14
17
|
*/
|
|
15
18
|
import { execFile } from "node:child_process";
|
|
16
19
|
import { randomUUID } from "node:crypto";
|
|
@@ -123,6 +126,21 @@ export function numberOnlyChange(was, now, from, to) {
|
|
|
123
126
|
return new RegExp(`^${parts.map((p, i) => (i % 2 ? alt(p) : esc(p))).join("")}$`).test(String(now));
|
|
124
127
|
}
|
|
125
128
|
|
|
129
|
+
/**
|
|
130
|
+
* #4526: everything after the first line is byte-identical, and the first line of `now` is that of `was` with the old
|
|
131
|
+
* stem (`fromPath` without extension) mapped to the new one, then the old number (padded or bare, not next to another
|
|
132
|
+
* digit) to the new number: a migration's `-- Migration 0067_x` or an ADR's `# 0029. Title` header following a renumber.
|
|
133
|
+
*/
|
|
134
|
+
export function headerOnlyChange(was, now, fromPath, toPath) {
|
|
135
|
+
const from = numOf(fromPath), to = numOf(toPath);
|
|
136
|
+
if (!from || !to) return false;
|
|
137
|
+
const split = (t) => { const s = String(t), i = s.indexOf("\n"); return i < 0 ? [s, ""] : [s.slice(0, i), s.slice(i)]; };
|
|
138
|
+
const [h1, r1] = split(was), [h2, r2] = split(now);
|
|
139
|
+
if (r1 !== r2) return false;
|
|
140
|
+
const mapped = h1.replaceAll(stemOf(fromPath), stemOf(toPath)).replace(new RegExp(`(?<!\\d)0*${Number(from)}(?!\\d)`, "g"), (tok) => (tok.length >= to.length ? to : String(Number(to))));
|
|
141
|
+
return mapped === h2;
|
|
142
|
+
}
|
|
143
|
+
|
|
126
144
|
/** Lane-added numbered files with their blobs, recorded before a renumber so verifyRenumber can prove the SQL kept. */
|
|
127
145
|
export const snapshotLane = ({ git, base }) => laneNumbered(git, base).catch(() => []);
|
|
128
146
|
|
|
@@ -173,7 +191,7 @@ export async function verifyRenumber({ git, base, before = [], moves = null }) {
|
|
|
173
191
|
for (const f of peers) {
|
|
174
192
|
if (kept || f.path === b.path) continue;
|
|
175
193
|
const text = await g(["cat-file", "-p", f.blob]);
|
|
176
|
-
kept = numberOnlyChange(was, text, numOf(b.path), numOf(f.path)) || (rest(f.path) === rest(b.path) && text === mapped);
|
|
194
|
+
kept = numberOnlyChange(was, text, numOf(b.path), numOf(f.path)) || (rest(f.path) === rest(b.path) && text === mapped) || headerOnlyChange(was, text, b.path, f.path);
|
|
177
195
|
}
|
|
178
196
|
if (!kept) problems.push(`${b.path}: content changed or file gone after renumbering (keep it byte-identical, rename only)`);
|
|
179
197
|
}
|
|
@@ -269,7 +287,8 @@ const writeJson = (p, o) => writeFileSync(p, `${JSON.stringify(o, null, 2)}\n`);
|
|
|
269
287
|
* where it started. Never throws.
|
|
270
288
|
* @returns {Promise<{ok:true, moves:Array<{from,to}>, commit:string|null}|{unsupported:string}>}
|
|
271
289
|
*/
|
|
272
|
-
export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerate = null, run = null, uuid = randomUUID }) {
|
|
290
|
+
export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerate = null, regenerateCustom = null, run = null, uuid = randomUUID, reserve = null }) {
|
|
291
|
+
const custom = regenerateCustom ?? (regenerate ? `${regenerate} --custom` : null);
|
|
273
292
|
const g = out(git);
|
|
274
293
|
const start = (await g(["rev-parse", "HEAD"])).trim();
|
|
275
294
|
const written = [];
|
|
@@ -286,6 +305,16 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
|
|
|
286
305
|
if (!collisions.length) return { ok: true, moves: [], commit: null };
|
|
287
306
|
const odd = collisions.find((c) => !SUPPORTED_RE.test(c.lane));
|
|
288
307
|
if (odd) return { unsupported: `${odd.lane}: no deterministic renumber for .${basename(odd.lane).split(".").pop()} migrations` };
|
|
308
|
+
// #4526: numbers above every other live lane's host reservation, not only above the base (fail-open: base-derived).
|
|
309
|
+
if (reserve) {
|
|
310
|
+
for (const dir of [...new Set(collisions.map((c) => c.dir))].sort()) {
|
|
311
|
+
const cs = collisions.filter((c) => c.dir === dir), width = cs[0].next.length;
|
|
312
|
+
let got = null;
|
|
313
|
+
try { got = await reserve({ dir, floor: Number(cs[0].next) - 1, count: cs.length }); } catch { /* fail-open */ }
|
|
314
|
+
const nums = Array.isArray(got?.numbers) ? got.numbers.map(Number) : [];
|
|
315
|
+
if (nums.length === cs.length && nums.every(Number.isInteger)) cs.forEach((c, i) => { c.next = String(nums[i]).padStart(width, "0"); });
|
|
316
|
+
}
|
|
317
|
+
}
|
|
289
318
|
const moves = collisions.map((c) => ({ from: c.lane, to: posix.join(c.dir, c.next + basename(c.lane).slice(c.number.length)), dir: c.dir, oldNum: c.number, newNum: c.next }));
|
|
290
319
|
// Plan the journals and snapshots before touching anything, so an unsupported migration leaves the branch as it was.
|
|
291
320
|
const baseList = lines(await g(["-c", "core.quotepath=off", "ls-tree", "-r", "--name-only", base]));
|
|
@@ -296,12 +325,14 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
|
|
|
296
325
|
if (!jp) continue;
|
|
297
326
|
const meta = dirname(jp), mine = moves.filter((m) => m.dir === dir && /\.sql$/i.test(m.from));
|
|
298
327
|
const snaps = [];
|
|
328
|
+
let chained = false; // #4526: after a schema-changing move, a data-only one's predecessor is not the base tip
|
|
299
329
|
for (const m of mine) {
|
|
300
330
|
const laneSnap = await json(g, `${lane}:${posix.join(meta, `${m.oldNum}_snapshot.json`)}`);
|
|
301
331
|
if (!laneSnap) continue;
|
|
302
332
|
const laneMeta = lines(await g(["-c", "core.quotepath=off", "ls-tree", "--name-only", lane, `${meta}/`])).filter((p) => SNAPSHOT_RE.test(p) && Number(numOf(p)) < Number(m.oldNum));
|
|
303
333
|
const pred = laneMeta.sort((a, b) => Number(numOf(a)) - Number(numOf(b))).at(-1);
|
|
304
|
-
const dataOnly = Boolean(pred) && strip(await json(g, `${lane}:${pred}`)) === strip(laneSnap);
|
|
334
|
+
const dataOnly = !chained && Boolean(pred) && strip(await json(g, `${lane}:${pred}`)) === strip(laneSnap);
|
|
335
|
+
if (!dataOnly) chained = true;
|
|
305
336
|
if (!dataOnly && !regenerate) return { unsupported: `schema-changing migration ${m.from} needs lean.sequence.regenerate (its snapshot cannot be cloned)` };
|
|
306
337
|
if (!dataOnly) needRegen = true;
|
|
307
338
|
snaps.push({ m, dataOnly });
|
|
@@ -339,7 +370,8 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
|
|
|
339
370
|
const tags = new Set(baseEntries.map((e) => e?.tag));
|
|
340
371
|
let idx = Math.max(-1, ...baseEntries.map((e) => Number(e?.idx)).filter(Number.isFinite));
|
|
341
372
|
const add = (Array.isArray(lj.entries) ? lj.entries : []).filter((e) => !tags.has(e?.tag)).map((e) => ({ ...e, idx: ++idx, tag: stem.get(e.tag) ?? e.tag }));
|
|
342
|
-
|
|
373
|
+
p.journal = { ...bj, entries: [...baseEntries, ...add] };
|
|
374
|
+
writeJson(join(cwd, p.jp), p.journal);
|
|
343
375
|
touched.add(p.jp);
|
|
344
376
|
const baseSnaps = baseList.filter((x) => dirname(x) === p.meta && SNAPSHOT_RE.test(x)).sort((a, b) => Number(numOf(a)) - Number(numOf(b)));
|
|
345
377
|
let tip = baseSnaps.length ? await json(g, `${base}:${baseSnaps.at(-1)}`) : null;
|
|
@@ -360,11 +392,27 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
|
|
|
360
392
|
}
|
|
361
393
|
if (needRegen) {
|
|
362
394
|
try { await (run ?? defaultRun)(regenerate, { cwd }); } catch (e) { return await fail(`lean.sequence.regenerate failed: ${String(e?.stderr || e?.message || e).trim().split("\n")[0]}`); }
|
|
395
|
+
const exists = (p) => { try { readFileSync(join(cwd, p)); return true; } catch { return false; } };
|
|
363
396
|
for (const p of plans) {
|
|
364
397
|
touched.add(p.meta);
|
|
365
|
-
|
|
398
|
+
// #4526: a hand-written migration (no schema diff) gets nothing from the regular run: the generator's --custom
|
|
399
|
+
// writes an empty migration and its snapshot at that number (its SQL and later ones withheld), then the lane's
|
|
400
|
+
// SQL and the planned journal come back.
|
|
401
|
+
for (const { m, dataOnly } of [...p.snaps].sort((a, b) => Number(a.m.newNum) - Number(b.m.newNum))) {
|
|
366
402
|
const want = posix.join(p.meta, `${m.newNum}_snapshot.json`);
|
|
367
|
-
if (
|
|
403
|
+
if (dataOnly || exists(want)) continue;
|
|
404
|
+
if (!custom) return await fail(`lean.sequence.regenerate did not write ${want}`);
|
|
405
|
+
const held = moves.filter((w) => w.dir === p.dir && /\.sql$/i.test(w.to) && Number(w.newNum) >= Number(m.newNum));
|
|
406
|
+
const heldTags = new Set(held.map((w) => stemOf(w.to)));
|
|
407
|
+
for (const w of held) rmSync(join(cwd, w.to), { force: true });
|
|
408
|
+
writeJson(join(cwd, p.jp), { ...p.journal, entries: p.journal.entries.filter((e) => !heldTags.has(e?.tag)) });
|
|
409
|
+
try { await (run ?? defaultRun)(custom, { cwd }); } catch (e) { return await fail(`lean.sequence.regenerate --custom failed: ${String(e?.stderr || e?.message || e).trim().split("\n")[0]}`); }
|
|
410
|
+
for (const f of lines(await g(["-c", "core.quotepath=off", "ls-files", "--others", "--exclude-standard", "--", `${p.dir}/`]))) {
|
|
411
|
+
if (dirname(f) === p.dir && /\.sql$/i.test(f) && numOf(f) === m.newNum && f !== m.to) rmSync(join(cwd, f), { force: true });
|
|
412
|
+
}
|
|
413
|
+
for (const w of held) writeFileSync(join(cwd, w.to), await g(["cat-file", "blob", `${lane}:${w.from}`]));
|
|
414
|
+
writeJson(join(cwd, p.jp), p.journal);
|
|
415
|
+
if (!exists(want)) return await fail(`lean.sequence.regenerate did not write ${want}`);
|
|
368
416
|
}
|
|
369
417
|
}
|
|
370
418
|
}
|
|
@@ -8,7 +8,8 @@ import { execFileSync } from "node:child_process";
|
|
|
8
8
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
9
9
|
import { dirname, join } from "node:path";
|
|
10
10
|
import { tmpDir } from "../tests/_tmp.mjs";
|
|
11
|
-
import { sequenceCollisions, journalProblems, snapshotLane, verifyRenumber, numberOnlyChange, renumberLane, resolveSequenceConflicts } from "./sequence.mjs";
|
|
11
|
+
import { sequenceCollisions, journalProblems, snapshotLane, verifyRenumber, numberOnlyChange, headerOnlyChange, renumberLane, resolveSequenceConflicts } from "./sequence.mjs";
|
|
12
|
+
import { makeSequenceLeaseIo } from "./seqlease.mjs";
|
|
12
13
|
import { sequencePrompt, sequenceSyncNote } from "./prompts.mjs";
|
|
13
14
|
|
|
14
15
|
const sh = (cwd, ...a) => execFileSync("git", a, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" } });
|
|
@@ -301,6 +302,87 @@ describe("deterministic renumber (#4386)", () => {
|
|
|
301
302
|
// both add the same numbered name in a directory without a series
|
|
302
303
|
await conflict({ "src/a.ts": "1\n" }, { "src/500_error.html": "LANE\n" }, { "src/500_error.html": "BASE\n" }, "src/500_error.html");
|
|
303
304
|
});
|
|
305
|
+
|
|
306
|
+
it("#4526: two lanes renumbering at once against the same base get distinct numbers (host reservation), and free them at the end", async () => {
|
|
307
|
+
const io = makeSequenceLeaseIo({ dir: tmpDir("seqlease-"), pickDir: tmpDir("seqpick-"), pollMs: 20 });
|
|
308
|
+
const reserveFor = (issue) => (x) => io.reserve({ repo: "o/r", lane: "main", issue, ...x });
|
|
309
|
+
const lanes = [drizzle(), drizzle()];
|
|
310
|
+
const rs = await Promise.all(lanes.map((d, i) => renumberLane({ git: gitIn(d), base: "main", cwd: d, uuid: () => `u${i}`, reserve: reserveFor(i + 1) })));
|
|
311
|
+
for (const r of rs) assert.ok(r.ok, r.unsupported);
|
|
312
|
+
assert.deepEqual(rs.map((r) => r.moves[0].to).sort(), ["db/0052_nclt.sql", "db/0053_nclt.sql"]);
|
|
313
|
+
for (const d of lanes) {
|
|
314
|
+
assert.deepEqual(await journalProblems({ git: gitIn(d), dirs: ["db"] }), []);
|
|
315
|
+
assert.deepEqual(await sequenceCollisions({ git: gitIn(d), base: "main" }), []);
|
|
316
|
+
}
|
|
317
|
+
await io.unreserve({ repo: "o/r", lane: "main", issue: 1 });
|
|
318
|
+
await io.unreserve({ repo: "o/r", lane: "main", issue: 2 });
|
|
319
|
+
const d = drizzle();
|
|
320
|
+
assert.deepEqual((await renumberLane({ git: gitIn(d), base: "main", cwd: d, reserve: reserveFor(3) })).moves, [{ from: "db/0049_nclt.sql", to: "db/0052_nclt.sql" }]);
|
|
321
|
+
const failing = drizzle(); // a broken reservation keeps the base-derived number (fail-open)
|
|
322
|
+
assert.deepEqual((await renumberLane({ git: gitIn(failing), base: "main", cwd: failing, reserve: async () => { throw new Error("x"); } })).moves[0].to, "db/0052_nclt.sql");
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
it("#4526: a renumber that changes only the header line's tag passes verifyRenumber; any other change does not", async () => {
|
|
326
|
+
const lane = (header, body = "INSERT INTO t VALUES (1);\n") => {
|
|
327
|
+
const d = repo({ "db/0047_a.sql": "a;\n", "db/0048_init.sql": "b;\n" }, { "db/0049_x.sql": "-- Migration 0049_x\nINSERT INTO t VALUES (1);\n" }, { "db/0049_a.sql": "c;\n" });
|
|
328
|
+
return { d, header, body };
|
|
329
|
+
};
|
|
330
|
+
for (const [header, body, ok] of [["-- Migration 0050_brave_hulk", undefined, true], ["-- Migration 0050_brave_hulk (edited)", undefined, false], ["-- Migration 0050_brave_hulk", "INSERT INTO t VALUES (2);\n", false]]) {
|
|
331
|
+
const { d } = lane(header);
|
|
332
|
+
const before = await snapshotLane({ git: gitIn(d), base: "main" });
|
|
333
|
+
sh(d, "merge", "--no-edit", "main");
|
|
334
|
+
sh(d, "mv", "db/0049_x.sql", "db/0050_brave_hulk.sql");
|
|
335
|
+
put(d, { "db/0050_brave_hulk.sql": `${header}\n${body ?? "INSERT INTO t VALUES (1);\n"}` });
|
|
336
|
+
commit(d, "renumber");
|
|
337
|
+
const v = await verifyRenumber({ git: gitIn(d), base: "main", before });
|
|
338
|
+
assert.equal(v.ok, ok, `${header} / ${body}: ${v.problems.join("; ")}`);
|
|
339
|
+
if (!ok) assert.match(v.problems.join("\n"), /db\/0049_x\.sql: content changed/);
|
|
340
|
+
}
|
|
341
|
+
assert.equal(headerOnlyChange("# 0029. Title\nbody\n", "# 0030. Title\nbody\n", "docs/adr/0029-title.md", "docs/adr/0030-title.md"), true);
|
|
342
|
+
assert.equal(headerOnlyChange("# 29. Title\nbody\n", "# 30. Title\nbody\n", "docs/adr/0029-title.md", "docs/adr/0030-title.md"), true);
|
|
343
|
+
assert.equal(headerOnlyChange("-- 0067_x\nSELECT 1;\n", "-- 0068_x\nSELECT 1;\n", "db/0067_x.sql", "db/0068_x.sql"), true);
|
|
344
|
+
assert.equal(headerOnlyChange("-- 0067_x\nSELECT 1;\n", "-- 0068_x\nSELECT 2;\n", "db/0067_x.sql", "db/0068_x.sql"), false);
|
|
345
|
+
assert.equal(headerOnlyChange("-- 0067_x 10067\nSELECT 1;\n", "-- 0068_x 10068\nSELECT 1;\n", "db/0067_x.sql", "db/0068_x.sql"), false, "another number containing the tag is not the tag");
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
it("#4526: a hand-written migration with no schema diff after a schema-changing one gets the generator's --custom snapshot and keeps its SQL byte-for-byte", async () => {
|
|
349
|
+
const FN_SQL = "-- hand-written: no schema diff\nCREATE FUNCTION f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql;\n";
|
|
350
|
+
const d = repo(
|
|
351
|
+
{ "db/0048_init.sql": "CREATE TABLE t (id int);\n", "db/meta/_journal.json": journal("0048_init"), "db/meta/0048_snapshot.json": snap("s48", "s47") },
|
|
352
|
+
{ "db/0049_s.sql": "CREATE TABLE s (id int);\n", "db/0050_fn.sql": FN_SQL, "db/meta/_journal.json": journal("0048_init", "0049_s", "0050_fn"),
|
|
353
|
+
"db/meta/0049_snapshot.json": snap("lane49", "s48", { t: 1, s: 1 }), "db/meta/0050_snapshot.json": snap("lane50", "lane49", { t: 1, s: 1 }) },
|
|
354
|
+
{
|
|
355
|
+
"db/0049_a.sql": "a;\n", "db/0050_b.sql": "b;\n", "db/0051_c.sql": "CREATE TABLE u (id int);\n",
|
|
356
|
+
"db/meta/_journal.json": journal("0048_init", "0049_a", "0050_b", "0051_c"),
|
|
357
|
+
"db/meta/0049_snapshot.json": snap("s49", "s48"), "db/meta/0050_snapshot.json": snap("s50", "s49"), "db/meta/0051_snapshot.json": snap("s51", "s50", { t: 1, u: 1 }),
|
|
358
|
+
},
|
|
359
|
+
);
|
|
360
|
+
const before = await snapshotLane({ git: gitIn(d), base: "main" });
|
|
361
|
+
const calls = [];
|
|
362
|
+
/** A drizzle-kit stand-in: the regular run writes only the schema-changing snapshot; --custom adds an empty migration at the journal's next number. */
|
|
363
|
+
const run = async (cmd, { cwd }) => {
|
|
364
|
+
calls.push(cmd);
|
|
365
|
+
if (!cmd.endsWith("--custom")) { writeFileSync(join(cwd, "db/meta/0052_snapshot.json"), snap("gen52", "s51", { t: 1, u: 1, s: 1 })); return; }
|
|
366
|
+
assert.ok(!existsSync(join(cwd, "db/0053_fn.sql")), "the lane's SQL is withheld while the generator runs");
|
|
367
|
+
const j = JSON.parse(readFileSync(join(cwd, "db/meta/_journal.json"), "utf8"));
|
|
368
|
+
const n = String(Number(j.entries.at(-1).tag.slice(0, 4)) + 1).padStart(4, "0");
|
|
369
|
+
writeFileSync(join(cwd, `db/${n}_wise_owl.sql`), "-- Custom SQL migration file, put your code below! --\n");
|
|
370
|
+
writeFileSync(join(cwd, `db/meta/${n}_snapshot.json`), snap("custom", "gen52", { t: 1, u: 1, s: 1 }));
|
|
371
|
+
j.entries.push({ idx: j.entries.length, tag: `${n}_wise_owl`, when: 9 });
|
|
372
|
+
writeFileSync(join(cwd, "db/meta/_journal.json"), JSON.stringify(j));
|
|
373
|
+
};
|
|
374
|
+
const r = await renumberLane({ git: gitIn(d), base: "main", cwd: d, regenerate: "pnpm db:generate", run });
|
|
375
|
+
assert.ok(r.ok, r.unsupported);
|
|
376
|
+
assert.deepEqual(r.moves, [{ from: "db/0049_s.sql", to: "db/0052_s.sql" }, { from: "db/0050_fn.sql", to: "db/0053_fn.sql" }]);
|
|
377
|
+
assert.deepEqual(calls, ["pnpm db:generate", "pnpm db:generate --custom"]);
|
|
378
|
+
assert.equal(show(d, "db/0053_fn.sql"), FN_SQL, "hand-written SQL byte-identical");
|
|
379
|
+
assert.ok(!existsSync(join(d, "db/0053_wise_owl.sql")), "the generator's empty migration is dropped");
|
|
380
|
+
assert.equal(JSON.parse(show(d, "db/meta/0053_snapshot.json")).id, "custom");
|
|
381
|
+
assert.deepEqual(JSON.parse(show(d, "db/meta/_journal.json")).entries.map((e) => e.tag), ["0048_init", "0049_a", "0050_b", "0051_c", "0052_s", "0053_fn"]);
|
|
382
|
+
assert.deepEqual(await journalProblems({ git: gitIn(d), dirs: ["db"] }), []);
|
|
383
|
+
assert.equal(sh(d, "status", "--porcelain").trim(), "");
|
|
384
|
+
assert.deepEqual(await verifyRenumber({ git: gitIn(d), base: "main", before, moves: r.moves }), { ok: true, problems: [] });
|
|
385
|
+
});
|
|
304
386
|
});
|
|
305
387
|
|
|
306
388
|
describe("sequence-numbered documents: two siblings each add an ADR (#4385)", () => {
|