@intentius/chant 0.124.56 → 0.124.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/dist/ci-audit/index.d.ts.map +1 -1
  2. package/dist/ci-audit/index.js +1 -2
  3. package/dist/ci-product.d.ts.map +1 -1
  4. package/dist/ci-product.js +4 -5
  5. package/dist/cli/handlers/lifecycle.d.ts.map +1 -1
  6. package/dist/cli/handlers/lifecycle.js +14 -0
  7. package/dist/cli/handlers/pr.d.ts.map +1 -1
  8. package/dist/cli/handlers/pr.js +90 -35
  9. package/dist/components/auto-release.d.ts +5 -2
  10. package/dist/components/auto-release.d.ts.map +1 -1
  11. package/dist/components/auto-release.js +9 -4
  12. package/dist/components/capability.d.ts +10 -0
  13. package/dist/components/capability.d.ts.map +1 -1
  14. package/dist/components/deploy-units.d.ts +12 -0
  15. package/dist/components/deploy-units.d.ts.map +1 -1
  16. package/dist/components/deploy-units.js +20 -1
  17. package/dist/components/pr-pipeline.d.ts +8 -2
  18. package/dist/components/pr-pipeline.d.ts.map +1 -1
  19. package/dist/components/pr-pipeline.js +11 -3
  20. package/dist/components/row-lease.d.ts +22 -1
  21. package/dist/components/row-lease.d.ts.map +1 -1
  22. package/dist/components/row-lease.js +32 -6
  23. package/dist/e2e-harness/index.d.ts.map +1 -1
  24. package/dist/e2e-harness/index.js +1 -2
  25. package/dist/lexicon.d.ts +9 -0
  26. package/dist/lexicon.d.ts.map +1 -1
  27. package/dist/lifecycle/git.d.ts +39 -3
  28. package/dist/lifecycle/git.d.ts.map +1 -1
  29. package/dist/lifecycle/git.js +223 -15
  30. package/dist/lifecycle/lock-rows.d.ts +23 -0
  31. package/dist/lifecycle/lock-rows.d.ts.map +1 -1
  32. package/dist/lifecycle/lock-rows.js +47 -8
  33. package/dist/lifecycle/resource-lock.d.ts +70 -0
  34. package/dist/lifecycle/resource-lock.d.ts.map +1 -1
  35. package/dist/lifecycle/resource-lock.js +241 -5
  36. package/dist/op/gate.js +21 -4
  37. package/dist/op/op-waves-release-port.js +1 -1
  38. package/dist/op/op-waves-release.d.ts +2 -1
  39. package/dist/op/op-waves-release.d.ts.map +1 -1
  40. package/dist/op/op-waves-run.d.ts +8 -0
  41. package/dist/op/op-waves-run.d.ts.map +1 -1
  42. package/dist/op/op-waves-run.js +101 -29
  43. package/dist/op/op-waves.d.ts +18 -0
  44. package/dist/op/op-waves.d.ts.map +1 -1
  45. package/dist/op/op-waves.js +16 -0
  46. package/dist/pr-loop.d.ts.map +1 -1
  47. package/dist/pr-loop.js +1 -2
  48. package/dist/release-floors.d.ts +1 -1
  49. package/dist/release-floors.js +1 -1
  50. package/dist/wave-runtime.mjs +310 -300
  51. package/package.json +1 -1
  52. package/src/ci-audit/index.ts +1 -2
  53. package/src/ci-product.ts +4 -5
  54. package/src/cli/handlers/lifecycle.ts +13 -0
  55. package/src/cli/handlers/pr-before-merge.test.ts +1 -1
  56. package/src/cli/handlers/pr-deploy-inputs.test.ts +1 -1
  57. package/src/cli/handlers/pr-endpoint.test.ts +1 -1
  58. package/src/cli/handlers/pr-green.test.ts +1 -1
  59. package/src/cli/handlers/pr-plan-comment-targets.test.ts +1 -1
  60. package/src/cli/handlers/pr-plan-flags.test.ts +1 -1
  61. package/src/cli/handlers/pr-plan-targets.test.ts +1 -1
  62. package/src/cli/handlers/pr-record.test.ts +34 -2
  63. package/src/cli/handlers/pr-waves-blocked.test.ts +1 -1
  64. package/src/cli/handlers/pr-waves.test.ts +1 -1
  65. package/src/cli/handlers/pr.test.ts +1 -1
  66. package/src/cli/handlers/pr.ts +86 -31
  67. package/src/components/auto-release.test.ts +4 -2
  68. package/src/components/auto-release.ts +9 -5
  69. package/src/components/capability.ts +10 -0
  70. package/src/components/deploy-units.test.ts +18 -1
  71. package/src/components/deploy-units.ts +26 -1
  72. package/src/components/pr-pipeline.ts +12 -3
  73. package/src/components/row-lease.test.ts +61 -1
  74. package/src/components/row-lease.ts +48 -7
  75. package/src/e2e-harness/index.ts +1 -2
  76. package/src/lexicon.ts +9 -0
  77. package/src/lifecycle/git.test.ts +155 -0
  78. package/src/lifecycle/git.ts +213 -17
  79. package/src/lifecycle/lock-rows.test.ts +6 -1
  80. package/src/lifecycle/lock-rows.ts +58 -7
  81. package/src/lifecycle/resource-lock-wait.test.ts +292 -0
  82. package/src/lifecycle/resource-lock.ts +261 -7
  83. package/src/op/gate.test.ts +7 -5
  84. package/src/op/gate.ts +20 -4
  85. package/src/op/op-waves-after-plan.test.ts +1 -2
  86. package/src/op/op-waves-ledger-lock.test.ts +208 -0
  87. package/src/op/op-waves-release-port.ts +1 -1
  88. package/src/op/op-waves-release.ts +1 -1
  89. package/src/op/op-waves-run.ts +104 -29
  90. package/src/op/op-waves.ts +18 -0
  91. package/src/pr-loop-bundle.test.ts +0 -2
  92. package/src/pr-loop.ts +1 -2
  93. package/src/release-floors.ts +1 -1
@@ -0,0 +1,292 @@
1
+ /**
2
+ * The ruling for ci-products (2026-10-28), "Ledger lock": a stock Pulumi
3
+ * stack or a Terraform root waits for its row on chant's ledger, oldest
4
+ * commit first, up to the job's timeout, and plunge stacks on the resource
5
+ * grain lease their resource rows and never wait on another stack.
6
+ *
7
+ * Each test has a bare remote and one clone per writer, as two pushes'
8
+ * wave jobs have on a forge. The waits use a pause the test drives, so the
9
+ * order of reads is fixed and nothing sleeps.
10
+ */
11
+ import { afterAll, describe, expect, test } from "vitest";
12
+ import { execFileSync } from "node:child_process";
13
+ import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
14
+ import { tmpdir } from "node:os";
15
+ import { join } from "node:path";
16
+ import { rootRowKey, stackRowKey, urnRowKey, type LockRow } from "./lock-rows";
17
+ import {
18
+ leaseLockRows,
19
+ lockWaitSeconds,
20
+ DEFAULT_LOCK_WAIT_SECONDS,
21
+ parseLockRows,
22
+ readLockRows,
23
+ renderLockRows,
24
+ waiterBefore,
25
+ withWaiter,
26
+ withRowsTaken,
27
+ type LockRowHolder,
28
+ } from "./resource-lock";
29
+
30
+ const dirs: string[] = [];
31
+ afterAll(() => {
32
+ for (const d of dirs) rmSync(d, { recursive: true, force: true });
33
+ });
34
+
35
+ function tmp(prefix: string): string {
36
+ const d = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
37
+ dirs.push(d);
38
+ return d;
39
+ }
40
+
41
+ function git(cwd: string, ...args: string[]): string {
42
+ return execFileSync("git", args, { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] }).trim();
43
+ }
44
+
45
+ /** A bare remote with one commit on main, and `n` clones of it, each a writer. */
46
+ function writers(n: number): { remote: string; clones: string[] } {
47
+ const remote = tmp("chant-lock-wait-remote-");
48
+ git(remote, "init", "-q", "--bare", "-b", "main");
49
+ const seed = tmp("chant-lock-wait-seed-");
50
+ git(seed, "init", "-q", "-b", "main");
51
+ git(seed, "config", "user.email", "seed@chant.dev");
52
+ git(seed, "config", "user.name", "Seed");
53
+ git(seed, "config", "commit.gpgsign", "false");
54
+ writeFileSync(join(seed, "README.md"), "# t\n");
55
+ git(seed, "add", "README.md");
56
+ git(seed, "commit", "-q", "-m", "init");
57
+ git(seed, "remote", "add", "origin", remote);
58
+ git(seed, "push", "-q", "origin", "main");
59
+ const clones: string[] = [];
60
+ for (let i = 0; i < n; i++) {
61
+ const c = tmp(`chant-lock-wait-clone-${i}-`);
62
+ git(c, "clone", "-q", remote, ".");
63
+ git(c, "config", "user.email", `w${i}@chant.dev`);
64
+ git(c, "config", "user.name", `Writer ${i}`);
65
+ git(c, "config", "commit.gpgsign", "false");
66
+ clones.push(c);
67
+ }
68
+ return { remote, clones };
69
+ }
70
+
71
+ const holder = (run: string, sha?: string): LockRowHolder => ({ run, at: new Date().toISOString(), ...(sha ? { sha } : {}) });
72
+ const alive = async () => "alive" as const;
73
+ const stackRow = (stack: string): LockRow => ({ key: stackRowKey("shop", stack), target: `shop/${stack}`, address: `shop/${stack}` });
74
+ const urnRow = (stack: string, name: string): LockRow => {
75
+ const urn = `urn:pulumi:${stack}::shop::aws:sqs/queue:Queue::${name}`;
76
+ return { key: urnRowKey(urn), target: `shop/${stack}`, address: urn };
77
+ };
78
+
79
+ /** A promise and its resolver. */
80
+ function signal(): { promise: Promise<void>; fire: () => void } {
81
+ let fire!: () => void;
82
+ const promise = new Promise<void>((r) => (fire = r));
83
+ return { promise, fire };
84
+ }
85
+
86
+ describe("the queue in the rows file", () => {
87
+ test("a waiter round-trips, and the file leaves `waiting` out when nobody waits", () => {
88
+ const empty = parseLockRows(null);
89
+ expect(renderLockRows(empty)).not.toContain("waiting");
90
+ const queued = withWaiter(empty, { run: "12", at: "2026-10-28T10:00:00.000Z", keys: ["stack shop/staging"], commitAt: "2026-10-28T09:59:00Z" });
91
+ const back = parseLockRows(renderLockRows(queued));
92
+ expect(back.waiting).toEqual({ "12": { run: "12", at: "2026-10-28T10:00:00.000Z", keys: ["stack shop/staging"], commitAt: "2026-10-28T09:59:00Z" } });
93
+ // Taking the rows takes the run out of the queue.
94
+ const took = withRowsTaken(back, ["stack shop/staging"], holder("12"), []);
95
+ expect(took.waiting).toBeUndefined();
96
+ expect(took.rows).toEqual({ "stack shop/staging": "12" });
97
+ });
98
+
99
+ test("the older commit goes first, then the lower run id, then the commit sha; no commit time goes last", () => {
100
+ expect(waiterBefore({ run: "20", commitAt: "2026-10-28T10:00:00Z" }, { run: "9", commitAt: "2026-10-28T10:05:00Z" })).toBe(true);
101
+ expect(waiterBefore({ run: "9", commitAt: "2026-10-28T10:00:00Z" }, { run: "20", commitAt: "2026-10-28T10:00:00Z" })).toBe(true);
102
+ expect(waiterBefore({ run: "20", commitAt: "2026-10-28T10:00:00Z" }, { run: "9", commitAt: "2026-10-28T10:00:00Z" })).toBe(false);
103
+ expect(waiterBefore({ run: "local:a:1", commitAt: "2026-10-28T10:00:00Z" }, { run: "local:b:1", commitAt: "2026-10-28T10:00:00Z" })).toBe(true);
104
+ expect(waiterBefore({ run: "7", commitAt: "2026-10-28T10:00:00Z", sha: "aaa" }, { run: "7", commitAt: "2026-10-28T10:00:00Z", sha: "bbb" })).toBe(true);
105
+ expect(waiterBefore({ run: "1" }, { run: "2", commitAt: "2026-10-28T10:00:00Z" })).toBe(false);
106
+ });
107
+
108
+ test("CHANT_LOCK_WAIT sets the wait; the default sits under a Forgejo runner's three hours", () => {
109
+ expect(lockWaitSeconds({ CHANT_LOCK_WAIT: "600" })).toBe(600);
110
+ expect(lockWaitSeconds({})).toBe(DEFAULT_LOCK_WAIT_SECONDS);
111
+ expect(DEFAULT_LOCK_WAIT_SECONDS).toBeLessThan(3 * 3600);
112
+ });
113
+
114
+ test("a root row is the root's key, apart from a choudoufu root's resource rows", () => {
115
+ expect(rootRowKey("stacks/net")).toBe("root stacks/net");
116
+ });
117
+ });
118
+
119
+ describe("two writers on a bare-remote ledger", () => {
120
+ test("the second writer waits for the stack row the first holds, then takes it once the first lets go", async () => {
121
+ const { remote, clones } = writers(2);
122
+ const [a, b] = clones as [string, string];
123
+ const first = await leaseLockRows("pr-apply", [stackRow("staging")], { cwd: a, alive, holder: holder("101") });
124
+ expect(first.ok).toBe(true);
125
+ if (!first.ok) return;
126
+
127
+ const lines: string[] = [];
128
+ let pauses = 0;
129
+ let queuedOnRemote: unknown;
130
+ const second = await leaseLockRows("pr-apply", [stackRow("staging")], {
131
+ cwd: b,
132
+ alive,
133
+ holder: holder("102"),
134
+ wait: {
135
+ deadline: Date.now() + 60_000,
136
+ pollMs: 0,
137
+ commitAt: "2026-10-28T10:05:00Z",
138
+ log: (l) => lines.push(l),
139
+ sleep: async () => {
140
+ pauses++;
141
+ if (pauses === 1) {
142
+ // While B waits, the remote's file names it in the queue, behind A's hold.
143
+ queuedOnRemote = parseLockRows(git(remote, "show", "chant/lifecycle:_locks/rows/pr-apply.json")).waiting;
144
+ expect(await first.lease.release()).toBe(true);
145
+ }
146
+ },
147
+ },
148
+ });
149
+ expect(second.ok).toBe(true);
150
+ expect(pauses).toBe(1);
151
+ expect(queuedOnRemote).toMatchObject({ "102": { run: "102", keys: ["stack shop/staging"], commitAt: "2026-10-28T10:05:00Z" } });
152
+ expect(lines.join("\n")).toContain("run 101 is applying shop/staging");
153
+ const now = await readLockRows("pr-apply", { cwd: a });
154
+ expect(now.rows).toEqual({ "stack shop/staging": "102" });
155
+ expect(now.waiting).toBeUndefined();
156
+ });
157
+
158
+ test("two plunge stacks on the resource grain lease their URN rows side by side and never wait", async () => {
159
+ const { clones } = writers(2);
160
+ const [a, b] = clones as [string, string];
161
+ // Two pushes on one plunge stack that change disjoint resources (pw-disjoint), and a second stack beside them.
162
+ const events = await leaseLockRows("pr-apply", [urnRow("dev", "events")], { cwd: a, alive, holder: holder("201") });
163
+ const worker = await leaseLockRows("pr-apply", [urnRow("dev", "worker-3"), urnRow("prod-us", "orders")], { cwd: b, alive, holder: holder("202") });
164
+ expect(events.ok).toBe(true);
165
+ expect(worker.ok).toBe(true);
166
+ const now = await readLockRows("pr-apply", { cwd: a });
167
+ expect(Object.values(now.rows).sort()).toEqual(["201", "202", "202"]);
168
+ expect(now.waiting).toBeUndefined();
169
+ // A resource-grain take on a URN another run holds is refused at once, held, with no wait.
170
+ const clash = await leaseLockRows("pr-apply", [urnRow("dev", "events")], { cwd: b, alive, holder: holder("203") });
171
+ expect(clash.ok).toBe(false);
172
+ if (!clash.ok) expect(clash.timedOut).toBeUndefined();
173
+ });
174
+
175
+ test("a wait that reaches the job's deadline fails by name and leaves the queue", async () => {
176
+ const { clones } = writers(2);
177
+ const [a, b] = clones as [string, string];
178
+ const first = await leaseLockRows("pr-apply", [stackRow("prod-eu")], { cwd: a, alive, holder: holder("301") });
179
+ expect(first.ok).toBe(true);
180
+ let clock = 1_000_000;
181
+ let pauses = 0;
182
+ const second = await leaseLockRows("pr-apply", [stackRow("prod-eu")], {
183
+ cwd: b,
184
+ alive,
185
+ holder: holder("302"),
186
+ wait: {
187
+ deadline: clock + 30_000,
188
+ pollMs: 10_000,
189
+ now: () => clock,
190
+ sleep: async (ms) => {
191
+ pauses++;
192
+ clock += ms;
193
+ },
194
+ },
195
+ });
196
+ expect(second.ok).toBe(false);
197
+ if (second.ok) return;
198
+ expect(second.timedOut).toBe(true);
199
+ expect(second.message).toMatch(/^waited 30s for stack shop\/prod-eu of pr-apply and gave up at the job's deadline: run 301 is applying shop\/prod-eu/);
200
+ expect(pauses).toBe(3);
201
+ const now = await readLockRows("pr-apply", { cwd: a });
202
+ expect(now.rows).toEqual({ "stack shop/prod-eu": "301" });
203
+ expect(now.waiting).toBeUndefined();
204
+ });
205
+
206
+ test("of two waiters, the older commit takes the row first, whichever joined the queue first", async () => {
207
+ const { clones } = writers(3);
208
+ const [a, b, c] = clones as [string, string, string];
209
+ const holding = await leaseLockRows("pr-apply", [stackRow("staging")], { cwd: a, alive, holder: holder("401") });
210
+ expect(holding.ok).toBe(true);
211
+ if (!holding.ok) return;
212
+
213
+ const order: string[] = [];
214
+ const cJoined = signal();
215
+ const aReleased = signal();
216
+ const bSawC = signal();
217
+ const cTook = signal();
218
+ const cReleased = signal();
219
+ const bLines: string[] = [];
220
+ let bPauses = 0;
221
+ let cPauses = 0;
222
+ let older: Promise<Awaited<ReturnType<typeof leaseLockRows>>> | undefined;
223
+
224
+ const startOlder = () =>
225
+ leaseLockRows("pr-apply", [stackRow("staging")], {
226
+ cwd: c,
227
+ alive,
228
+ holder: holder("403"),
229
+ wait: {
230
+ deadline: Date.now() + 60_000,
231
+ pollMs: 0,
232
+ commitAt: "2026-10-28T10:00:00Z",
233
+ sleep: async () => {
234
+ cPauses++;
235
+ if (cPauses === 1) {
236
+ cJoined.fire();
237
+ await aReleased.promise;
238
+ // C reads again only once B has read the free row and stayed behind C.
239
+ await bSawC.promise;
240
+ }
241
+ },
242
+ },
243
+ });
244
+
245
+ // B, the newer commit, joins the queue first.
246
+ const newer = leaseLockRows("pr-apply", [stackRow("staging")], {
247
+ cwd: b,
248
+ alive,
249
+ holder: holder("402"),
250
+ wait: {
251
+ deadline: Date.now() + 60_000,
252
+ pollMs: 0,
253
+ commitAt: "2026-10-28T10:05:00Z",
254
+ log: (l) => bLines.push(l),
255
+ sleep: async () => {
256
+ bPauses++;
257
+ if (bPauses === 1) {
258
+ older = startOlder().then((r) => {
259
+ order.push("403");
260
+ cTook.fire();
261
+ return r;
262
+ });
263
+ await cJoined.promise;
264
+ expect(await holding.lease.release()).toBe(true);
265
+ aReleased.fire();
266
+ } else if (bPauses === 2) {
267
+ bSawC.fire();
268
+ await cTook.promise;
269
+ } else if (bPauses === 3) {
270
+ await cReleased.promise;
271
+ }
272
+ },
273
+ },
274
+ }).then((r) => {
275
+ order.push("402");
276
+ return r;
277
+ });
278
+
279
+ await cTook.promise;
280
+ const c1 = await older!;
281
+ expect(c1.ok).toBe(true);
282
+ if (c1.ok) expect(await c1.lease.release()).toBe(true);
283
+ cReleased.fire();
284
+ const b1 = await newer;
285
+ expect(b1.ok).toBe(true);
286
+ expect(order).toEqual(["403", "402"]);
287
+ expect(bLines.join("\n")).toContain("run 403 waits ahead of it with an older commit");
288
+ const now = await readLockRows("pr-apply", { cwd: a });
289
+ expect(now.rows).toEqual({ "stack shop/staging": "402" });
290
+ expect(now.waiting).toBeUndefined();
291
+ });
292
+ });
@@ -24,6 +24,19 @@
24
24
  * for one release), or, for a run on this host outside CI, when its
25
25
  * process is. Anything nobody can answer counts as alive.
26
26
  * - A release never throws: a row left held is taken over once its run is gone.
27
+ * - A take may wait (ruling for ci-products 2026-10-28, "Ledger lock"): the
28
+ * `target` and `repo` grains, a stock Pulumi stack's `stack` row, a
29
+ * Terraform root's `root` row or the `repo` row. A waiting run adds itself
30
+ * to `waiting` in the file, and takes its rows only once no live run holds
31
+ * any of them and no live waiter for a row that meets one of them comes
32
+ * first. First is the older commit (`commitAt`, the commit time of the
33
+ * commit the run applies), then the lower run id (both numbers, else by
34
+ * text), then the commit sha: the run that started first among runs of one
35
+ * commit. A waiter that is gone is dropped by the next writer, as a holder
36
+ * is. A wait that reaches its deadline (`CHANT_LOCK_WAIT` seconds, which
37
+ * the waves pipelines set from the job's timeout) leaves the queue and
38
+ * fails with {@link LockWaitTimeoutError}, the table's `held`, exit 5. A
39
+ * take on the resource grain never waits: a held row refuses it at once.
27
40
  *
28
41
  * The writes go through chant's ledger code (./git.ts) with the same
29
42
  * read-then-CAS and push the target locks use, in place of terragucci's raw
@@ -35,6 +48,7 @@
35
48
  * exit 5, and names the run that holds it.
36
49
  */
37
50
  import { hostname } from "node:os";
51
+ import { getRuntime } from "../runtime-adapter";
38
52
  import { pushLifecycleStatus, readBlobBySha, readPathSha, RefCASConflictError, writeBlobToPath } from "./git";
39
53
  import { lockRowsConflict, sortedLockRows, type LockRow } from "./lock-rows";
40
54
  import { dropRefusedWrite, fetchForLocks, localTip, LOCKS_DIR, targetLockGateProblem } from "./target-lock";
@@ -57,12 +71,21 @@ export interface LockRowHolder {
57
71
  sha?: string;
58
72
  }
59
73
 
74
+ /** A run waiting for rows: who it is, the rows it waits for, and the commit time that orders it. */
75
+ export interface LockRowWaiter extends LockRowHolder {
76
+ keys: string[];
77
+ /** The commit time (ISO 8601) of the commit the run applies. Absent sorts after every waiter that has one. */
78
+ commitAt?: string;
79
+ }
80
+
60
81
  /** `_locks/rows/<gate>.json`. */
61
82
  export interface LockRowsFile {
62
83
  version: 1;
63
84
  holders: Record<string, LockRowHolder>;
64
85
  /** Row key to the run that holds it. */
65
86
  rows: Record<string, string>;
87
+ /** The runs waiting for rows (ruling for ci-products 2026-10-28), by run. Left out of the file when empty. */
88
+ waiting?: Record<string, LockRowWaiter>;
66
89
  }
67
90
 
68
91
  /** A row another run holds, conflicting with `wanted`. */
@@ -161,7 +184,24 @@ export function parseLockRows(text: string | null): LockRowsFile {
161
184
  }
162
185
  const rows: Record<string, string> = {};
163
186
  for (const [k, id] of Object.entries(doc.rows as Record<string, unknown>)) if (typeof id === "string" && holders[id]) rows[k] = id;
164
- return { version: 1, holders, rows };
187
+ const waiting: Record<string, LockRowWaiter> = {};
188
+ const rawWaiting = (doc as { waiting?: unknown }).waiting;
189
+ if (rawWaiting && typeof rawWaiting === "object") {
190
+ for (const [id, raw] of Object.entries(rawWaiting as Record<string, Record<string, unknown>>)) {
191
+ if (typeof raw?.run !== "string" || typeof raw.at !== "string" || !Array.isArray(raw.keys)) continue;
192
+ const keys = (raw.keys as unknown[]).filter((k): k is string => typeof k === "string");
193
+ if (keys.length === 0) continue;
194
+ waiting[id] = {
195
+ run: raw.run,
196
+ at: raw.at,
197
+ keys,
198
+ ...(typeof raw.url === "string" ? { url: raw.url } : {}),
199
+ ...(typeof raw.sha === "string" ? { sha: raw.sha } : {}),
200
+ ...(typeof raw.commitAt === "string" ? { commitAt: raw.commitAt } : {}),
201
+ };
202
+ }
203
+ }
204
+ return { version: 1, holders, rows, ...(Object.keys(waiting).length > 0 ? { waiting } : {}) };
165
205
  }
166
206
  } catch {
167
207
  // An unreadable file holds no rows.
@@ -172,7 +212,8 @@ export function parseLockRows(text: string | null): LockRowsFile {
172
212
  /** The file's text, keys sorted, so two writes of the same rows are the same blob. */
173
213
  export function renderLockRows(file: LockRowsFile): string {
174
214
  const sort = <T>(r: Record<string, T>) => Object.fromEntries(Object.keys(r).sort().map((k) => [k, r[k]]));
175
- return `${JSON.stringify({ version: 1, holders: sort(file.holders), rows: sort(file.rows) }, null, 2)}\n`;
215
+ const waiting = file.waiting && Object.keys(file.waiting).length > 0 ? { waiting: sort(file.waiting) } : {};
216
+ return `${JSON.stringify({ version: 1, holders: sort(file.holders), rows: sort(file.rows), ...waiting }, null, 2)}\n`;
176
217
  }
177
218
 
178
219
  /** Who else holds a row that meets any of `keys`, split into runs still going and runs that are gone. */
@@ -197,19 +238,68 @@ export async function heldLockRowsIn(file: LockRowsFile, keys: readonly string[]
197
238
  * `gone`. Pure, so the decision can be checked without git.
198
239
  */
199
240
  export function withRowsTaken(file: LockRowsFile, keys: readonly string[], me: LockRowHolder, gone: readonly string[]): LockRowsFile {
200
- const next: LockRowsFile = { version: 1, holders: { ...file.holders }, rows: { ...file.rows } };
241
+ const next = copyRows(file);
201
242
  for (const id of gone) {
202
243
  delete next.holders[id];
203
244
  for (const [k, h] of Object.entries(next.rows)) if (h === id) delete next.rows[k];
245
+ delete next.waiting?.[id];
204
246
  }
205
- next.holders[me.run] = me;
247
+ delete next.waiting?.[me.run];
248
+ next.holders[me.run] = { run: me.run, at: me.at, ...(me.url ? { url: me.url } : {}), ...(me.sha ? { sha: me.sha } : {}) };
206
249
  for (const k of keys) next.rows[k] = me.run;
250
+ return tidyWaiting(next);
251
+ }
252
+
253
+ function copyRows(file: LockRowsFile): LockRowsFile {
254
+ return { version: 1, holders: { ...file.holders }, rows: { ...file.rows }, ...(file.waiting ? { waiting: { ...file.waiting } } : {}) };
255
+ }
256
+
257
+ function tidyWaiting(file: LockRowsFile): LockRowsFile {
258
+ if (file.waiting && Object.keys(file.waiting).length === 0) delete file.waiting;
259
+ return file;
260
+ }
261
+
262
+ /** The rows file with `me` in the queue for `keys`, less the waiters in `gone`. Pure. */
263
+ export function withWaiter(file: LockRowsFile, me: LockRowWaiter, gone: readonly string[] = []): LockRowsFile {
264
+ const next = copyRows(file);
265
+ next.waiting = { ...(next.waiting ?? {}) };
266
+ for (const id of gone) delete next.waiting[id];
267
+ next.waiting[me.run] = { ...me, keys: [...new Set(me.keys)].sort() };
207
268
  return next;
208
269
  }
209
270
 
271
+ /** The rows file with `me` out of the queue. Pure. */
272
+ export function withoutWaiter(file: LockRowsFile, me: string): LockRowsFile {
273
+ const next = copyRows(file);
274
+ delete next.waiting?.[me];
275
+ return tidyWaiting(next);
276
+ }
277
+
278
+ /**
279
+ * Whether waiter `a` goes before waiter `b`: the older commit first, then the
280
+ * lower run id (as numbers when both are, else as text), then the commit
281
+ * sha. A waiter with no commit time goes after every one that has one.
282
+ */
283
+ export function waiterBefore(a: Pick<LockRowWaiter, "run" | "commitAt" | "sha">, b: Pick<LockRowWaiter, "run" | "commitAt" | "sha">): boolean {
284
+ const time = (w: { commitAt?: string }) => {
285
+ const t = w.commitAt ? new Date(w.commitAt).getTime() : NaN;
286
+ return Number.isFinite(t) ? t : Number.POSITIVE_INFINITY;
287
+ };
288
+ const ta = time(a);
289
+ const tb = time(b);
290
+ if (ta !== tb) return ta < tb;
291
+ if (a.run !== b.run) {
292
+ const na = Number(a.run);
293
+ const nb = Number(b.run);
294
+ if (/^\d+$/.test(a.run) && /^\d+$/.test(b.run) && na !== nb) return na < nb;
295
+ return a.run < b.run;
296
+ }
297
+ return (a.sha ?? "") < (b.sha ?? "");
298
+ }
299
+
210
300
  /** The rows file after `me` lets go of `keys`; its holder entry goes once it holds no row. */
211
301
  export function withRowsReleased(file: LockRowsFile, keys: readonly string[], me: string): LockRowsFile {
212
- const next: LockRowsFile = { version: 1, holders: { ...file.holders }, rows: { ...file.rows } };
302
+ const next = copyRows(file);
213
303
  for (const k of keys) if (next.rows[k] === me) delete next.rows[k];
214
304
  if (!Object.values(next.rows).includes(me)) delete next.holders[me];
215
305
  return next;
@@ -292,6 +382,161 @@ export async function takeLockRows(gate: string, rows: readonly LockRow[], me: L
292
382
  throw new Error(`could not hold the rows of ${gate}: ${STATE_BRANCH} kept moving; check that the job may push to it`);
293
383
  }
294
384
 
385
+ /** A wait for rows reached its deadline (ruling for ci-products 2026-10-28): nothing applied. Exit 5, as a held row is. */
386
+ export class LockWaitTimeoutError extends Error {
387
+ readonly lockRowsHeld = true;
388
+ constructor(
389
+ message: string,
390
+ public readonly held: HeldLockRow[],
391
+ public readonly ahead: LockRowWaiter[],
392
+ public readonly waitedSeconds: number,
393
+ ) {
394
+ super(message);
395
+ this.name = "LockWaitTimeoutError";
396
+ }
397
+ }
398
+
399
+ /** How a take waits for its rows. */
400
+ export interface LockRowsWait {
401
+ /** When the wait gives up, in ms since the epoch. */
402
+ deadline: number;
403
+ /** The commit time (ISO 8601) of the commit the run applies, which orders the queue. */
404
+ commitAt?: string;
405
+ /** How long between reads of the rows file, in ms. Default `CHANT_LOCK_POLL` seconds, else 10 s. */
406
+ pollMs?: number;
407
+ /** Where the wait's lines go. Default: none. */
408
+ log?: (line: string) => void;
409
+ /** The clock and the pause, for tests. */
410
+ now?: () => number;
411
+ sleep?: (ms: number) => Promise<void>;
412
+ }
413
+
414
+ /** The default wait when a job names no timeout: just under three hours, a Forgejo runner's default job timeout. */
415
+ export const DEFAULT_LOCK_WAIT_SECONDS = 3 * 3600 - 300;
416
+
417
+ /** How long a take waits, in seconds: `CHANT_LOCK_WAIT`, else {@link DEFAULT_LOCK_WAIT_SECONDS}. */
418
+ export function lockWaitSeconds(env: Record<string, string | undefined>): number {
419
+ const n = Number(env.CHANT_LOCK_WAIT);
420
+ return env.CHANT_LOCK_WAIT !== undefined && Number.isFinite(n) && n >= 0 ? n : DEFAULT_LOCK_WAIT_SECONDS;
421
+ }
422
+
423
+ /** The poll interval in ms: `CHANT_LOCK_POLL` seconds, else 10 s. */
424
+ export function lockPollMs(env: Record<string, string | undefined>): number {
425
+ const n = Number(env.CHANT_LOCK_POLL);
426
+ return env.CHANT_LOCK_POLL !== undefined && Number.isFinite(n) && n > 0 ? n * 1000 : 10_000;
427
+ }
428
+
429
+ /** The commit time of `rev` in `cwd` (ISO 8601), or undefined when git cannot say. */
430
+ export async function commitTimeOf(rev: string, cwd?: string): Promise<string | undefined> {
431
+ const out = await getRuntime().spawn(["git", "show", "-s", "--format=%cI", rev], { cwd });
432
+ const t = out.exitCode === 0 ? out.stdout.trim() : "";
433
+ return t && Number.isFinite(new Date(t).getTime()) ? t : undefined;
434
+ }
435
+
436
+ /** The live waiters for a row that meets one of `keys` who go before `me`, and the waiters that are gone. */
437
+ export async function waitersAhead(file: LockRowsFile, keys: readonly string[], me: LockRowWaiter, alive: LockLiveness): Promise<{ ahead: LockRowWaiter[]; gone: string[] }> {
438
+ const ahead: LockRowWaiter[] = [];
439
+ const gone: string[] = [];
440
+ for (const [id, w] of Object.entries(file.waiting ?? {})) {
441
+ if (id === me.run || w.run === me.run) continue;
442
+ if (!w.keys.some((k) => keys.some((mine) => lockRowsConflict(k, mine)))) continue;
443
+ const verdict = await alive(w);
444
+ if (verdict === "dead") {
445
+ gone.push(id);
446
+ continue;
447
+ }
448
+ if (waiterBefore(w, me)) ahead.push(w);
449
+ }
450
+ return { ahead: ahead.sort((a, b) => (waiterBefore(a, b) ? -1 : 1)), gone: gone.sort() };
451
+ }
452
+
453
+ function describeWait(held: readonly HeldLockRow[], ahead: readonly LockRowWaiter[], rows: readonly LockRow[]): string {
454
+ const parts: string[] = [];
455
+ if (held.length > 0) parts.push(describeHeldLockRows(held, rows));
456
+ if (ahead.length > 0) parts.push(`${ahead.map((w) => `run ${w.run}${w.url ? ` (${w.url})` : ""}`).join(", ")} ${ahead.length === 1 ? "waits" : "wait"} ahead of it with an older commit`);
457
+ return parts.join("; ");
458
+ }
459
+
460
+ /**
461
+ * Take `rows` for `me`, waiting in the queue until no live run holds any of
462
+ * them and no live waiter that goes first ({@link waiterBefore}) waits for a
463
+ * row that meets one of them. Throws {@link LockWaitTimeoutError} at the
464
+ * deadline, after leaving the queue.
465
+ */
466
+ export async function waitLockRows(gate: string, rows: readonly LockRow[], me: LockRowHolder, wait: LockRowsWait, opts: LockRowsOptions = {}): Promise<{ ok: true; tookOver: LockRowHolder[]; pushed: boolean; waitedSeconds: number; waited: boolean }> {
467
+ checkGate(gate);
468
+ const keys = sortedLockRows(rows).map((r) => r.key);
469
+ if (keys.length === 0) return { ok: true, tookOver: [], pushed: false, waitedSeconds: 0, waited: false };
470
+ const alive = opts.alive ?? forgeLockLiveness(process.env);
471
+ const now = wait.now ?? Date.now;
472
+ const sleep = wait.sleep ?? pause;
473
+ const poll = wait.pollMs ?? lockPollMs(process.env);
474
+ const stale = lockStaleSeconds(process.env) * 1000;
475
+ const started = now();
476
+ const waiter = (): LockRowWaiter => ({ ...me, keys, ...(wait.commitAt ? { commitAt: wait.commitAt } : {}) });
477
+ let queued: LockRowWaiter | undefined;
478
+ let said = "";
479
+ await fetchForLocks(opts.cwd);
480
+ for (let raced = 0; ; ) {
481
+ const { sha, file } = await readCurrent(gate, opts.cwd);
482
+ const mine = queued ?? waiter();
483
+ const { live, gone } = await heldLockRowsIn(file, keys, me.run, alive);
484
+ const { ahead, gone: goneWaiters } = await waitersAhead(file, keys, mine, alive);
485
+ if (live.length === 0 && ahead.length === 0) {
486
+ const next = withRowsTaken(file, keys, me, gone);
487
+ for (const id of goneWaiters) delete next.waiting?.[id];
488
+ const took = gone.length ? `, taken over from run ${gone.join(", ")}` : "";
489
+ const wrote = await writeAndPush(gate, sha, tidyWaiting(next), `Hold ${keys.length} row${keys.length === 1 ? "" : "s"} of ${gate} for run ${me.run}${took}`, opts.cwd);
490
+ if (wrote) return { ok: true, tookOver: gone.map((id) => file.holders[id]!), pushed: wrote.pushed, waitedSeconds: Math.round((now() - started) / 1000), waited: queued !== undefined };
491
+ if (++raced >= ATTEMPTS) throw new Error(`could not hold the rows of ${gate}: ${STATE_BRANCH} kept moving; check that the job may push to it`);
492
+ await sleep(200 + Math.floor(Math.random() * 800));
493
+ continue;
494
+ }
495
+ const why = describeWait(live, ahead, rows);
496
+ if (now() >= wait.deadline) {
497
+ await leaveQueue(gate, me.run, opts.cwd);
498
+ const waited = Math.round((now() - started) / 1000);
499
+ throw new LockWaitTimeoutError(`waited ${waited}s for ${keys.join(", ")} of ${gate} and gave up at the job's deadline: ${why}`, live, ahead, waited);
500
+ }
501
+ const inFile = file.waiting?.[me.run];
502
+ const refresh = inFile ? now() - new Date(inFile.at).getTime() > stale / 2 : true;
503
+ if (!inFile || refresh || goneWaiters.length > 0) {
504
+ const entry: LockRowWaiter = { ...waiter(), at: new Date(now()).toISOString() };
505
+ const wrote = await writeAndPush(gate, sha, withWaiter(file, entry, goneWaiters), `Run ${me.run} waits for ${keys.length} row${keys.length === 1 ? "" : "s"} of ${gate}`, opts.cwd);
506
+ if (!wrote) {
507
+ if (++raced >= ATTEMPTS) throw new Error(`could not join the queue for the rows of ${gate}: ${STATE_BRANCH} kept moving; check that the job may push to it`);
508
+ await sleep(200 + Math.floor(Math.random() * 800));
509
+ continue;
510
+ }
511
+ queued = entry;
512
+ } else {
513
+ queued ??= inFile;
514
+ }
515
+ raced = 0;
516
+ if (why !== said) {
517
+ wait.log?.(`waits for ${keys.join(", ")} of ${gate}: ${why}`);
518
+ said = why;
519
+ }
520
+ await sleep(Math.max(0, Math.min(poll, wait.deadline - now())));
521
+ await fetchForLocks(opts.cwd);
522
+ }
523
+ }
524
+
525
+ /** Take run `me` out of `gate`'s queue. Never throws. */
526
+ async function leaveQueue(gate: string, me: string, cwd?: string): Promise<void> {
527
+ try {
528
+ await fetchForLocks(cwd);
529
+ for (let attempt = 0; attempt < ATTEMPTS; attempt++) {
530
+ const { sha, file } = await readCurrent(gate, cwd);
531
+ if (!file.waiting?.[me]) return;
532
+ if (await writeAndPush(gate, sha, withoutWaiter(file, me), `Run ${me} stops waiting for rows of ${gate}`, cwd)) return;
533
+ await pause(200 + Math.floor(Math.random() * 800));
534
+ }
535
+ } catch {
536
+ // A waiter left behind is dropped once its run is gone.
537
+ }
538
+ }
539
+
295
540
  /** Let go of `keys` where run `me` still holds them. Never throws: a row left held is taken over once the run is gone. */
296
541
  export async function releaseLockRows(gate: string, keys: readonly string[], me: string, opts: { cwd?: string } = {}): Promise<boolean> {
297
542
  try {
@@ -342,12 +587,21 @@ export interface LockRowsLease {
342
587
  export async function leaseLockRows(
343
588
  gate: string,
344
589
  rows: readonly LockRow[],
345
- opts: LockRowsOptions & { holder?: LockRowHolder } = {},
346
- ): Promise<{ ok: true; lease: LockRowsLease; tookOver: LockRowHolder[] } | { ok: false; held: HeldLockRow[]; message: string }> {
590
+ opts: LockRowsOptions & { holder?: LockRowHolder; wait?: LockRowsWait } = {},
591
+ ): Promise<{ ok: true; lease: LockRowsLease; tookOver: LockRowHolder[]; waitedSeconds?: number; waited?: boolean } | { ok: false; held: HeldLockRow[]; message: string; timedOut?: true }> {
347
592
  const holder = opts.holder ?? lockRowHolder(process.env);
348
593
  const keys = sortedLockRows(rows).map((r) => r.key);
349
594
  const lease: LockRowsLease = { gate, holder, keys, release: () => (keys.length === 0 ? Promise.resolve(true) : releaseLockRows(gate, keys, holder.run, { ...(opts.cwd ? { cwd: opts.cwd } : {}) })) };
350
595
  if (keys.length === 0) return { ok: true, lease, tookOver: [] };
596
+ if (opts.wait) {
597
+ try {
598
+ const waited = await waitLockRows(gate, rows, holder, opts.wait, opts);
599
+ return { ok: true, lease, tookOver: waited.tookOver, ...(waited.waited ? { waited: true, waitedSeconds: waited.waitedSeconds } : {}) };
600
+ } catch (err) {
601
+ if (err instanceof LockWaitTimeoutError) return { ok: false, held: err.held, message: err.message, timedOut: true };
602
+ throw err;
603
+ }
604
+ }
351
605
  const taken = await takeLockRows(gate, rows, holder, opts);
352
606
  if (!taken.ok) return { ok: false, held: taken.held, message: describeHeldLockRows(taken.held, rows) };
353
607
  return { ok: true, lease, tookOver: taken.tookOver };
@@ -571,12 +571,14 @@ describe("op/gate — a rejected ledger push is reported, not swallowed (#2310)"
571
571
  timestamp: "2026-09-09T05:10:00.000Z",
572
572
  expiresAt: "2026-09-10T05:10:00.000Z",
573
573
  });
574
- expect(third.pushed).toBe(false);
575
- expect(third.pushWarning).toBeTruthy();
576
- expect(third.pushWarning).toMatch(/chant\/lifecycle remote branch has moved|stale|rejected|non-fast-forward/i);
577
- // The append itself still landed locally — a correct local answer, per
578
- // the #2309 author's reasoning; only the report was missing.
574
+ // plywood#22 finding 9: the refused push replays the third fact on the
575
+ // remote's tip and pushes again, so it lands beside the other writer's.
576
+ expect(third.pushed).toBe(true);
577
+ expect(third.pushWarning).toBeUndefined();
579
578
  expect(third.record.gate).toBe("approve-third-gate");
579
+ const onRemote = git(["log", "-p", "chant/lifecycle"], remote).stdout;
580
+ expect(onRemote).toContain("approve-other-gate");
581
+ expect(onRemote).toContain("approve-third-gate");
580
582
  });
581
583
 
582
584
  test("evaluateGate carries the rejection onto the gate check it returns, recorded but not pushed", async () => {