@intentius/chant 0.124.58 → 0.124.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/dist/ci-product.d.ts.map +1 -1
  2. package/dist/ci-product.js +1 -2
  3. package/dist/cli/handlers/lifecycle-drift.d.ts +2 -1
  4. package/dist/cli/handlers/lifecycle-drift.d.ts.map +1 -1
  5. package/dist/cli/handlers/lifecycle-drift.js +4 -2
  6. package/dist/cli/handlers/lock.d.ts.map +1 -1
  7. package/dist/cli/handlers/lock.js +45 -2
  8. package/dist/cli/main.d.ts.map +1 -1
  9. package/dist/cli/main.js +5 -1
  10. package/dist/cli/mcp/product-tools.d.ts.map +1 -1
  11. package/dist/cli/mcp/product-tools.js +1 -2
  12. package/dist/cli/parse-args.d.ts.map +1 -1
  13. package/dist/cli/parse-args.js +18 -0
  14. package/dist/cli/registry.d.ts +6 -0
  15. package/dist/cli/registry.d.ts.map +1 -1
  16. package/dist/components/capability.d.ts +11 -0
  17. package/dist/components/capability.d.ts.map +1 -1
  18. package/dist/drift-record.d.ts +5 -0
  19. package/dist/drift-record.d.ts.map +1 -1
  20. package/dist/e2e-harness/index.d.ts.map +1 -1
  21. package/dist/e2e-harness/index.js +1 -2
  22. package/dist/lifecycle/lock-beat.d.ts +27 -0
  23. package/dist/lifecycle/lock-beat.d.ts.map +1 -0
  24. package/dist/lifecycle/lock-beat.js +84 -0
  25. package/dist/lifecycle/resource-lock.d.ts +54 -1
  26. package/dist/lifecycle/resource-lock.d.ts.map +1 -1
  27. package/dist/lifecycle/resource-lock.js +128 -7
  28. package/dist/lifecycle/wave-plans.d.ts +19 -0
  29. package/dist/lifecycle/wave-plans.d.ts.map +1 -1
  30. package/dist/lifecycle/wave-plans.js +25 -1
  31. package/dist/op/activities/activity-contracts.d.ts +4 -4
  32. package/dist/op/gate-review.d.ts +1 -1
  33. package/dist/op/gate-review.d.ts.map +1 -1
  34. package/dist/op/index.d.ts +2 -1
  35. package/dist/op/index.d.ts.map +1 -1
  36. package/dist/op/index.js +2 -1
  37. package/dist/op/op-waves-capability.d.ts +2 -0
  38. package/dist/op/op-waves-capability.d.ts.map +1 -1
  39. package/dist/op/op-waves-capability.js +1 -0
  40. package/dist/op/op-waves-notify.d.ts.map +1 -1
  41. package/dist/op/op-waves-notify.js +11 -0
  42. package/dist/op/op-waves-preview.d.ts +45 -0
  43. package/dist/op/op-waves-preview.d.ts.map +1 -0
  44. package/dist/op/op-waves-preview.js +37 -0
  45. package/dist/op/op-waves-run.d.ts +46 -1
  46. package/dist/op/op-waves-run.d.ts.map +1 -1
  47. package/dist/op/op-waves-run.js +124 -7
  48. package/dist/op/op-waves.d.ts +12 -1
  49. package/dist/op/op-waves.d.ts.map +1 -1
  50. package/dist/op/op-waves.js +9 -2
  51. package/dist/release-floors.d.ts +1 -1
  52. package/dist/release-floors.js +1 -1
  53. package/dist/template-request/template-request.d.ts.map +1 -1
  54. package/dist/template-request/template-request.js +1 -2
  55. package/dist/wave-runtime.mjs +299 -299
  56. package/dist/workspace/drift-cli.d.ts +13 -1
  57. package/dist/workspace/drift-cli.d.ts.map +1 -1
  58. package/dist/workspace/drift-cli.js +19 -3
  59. package/package.json +1 -1
  60. package/src/ci-product.ts +1 -2
  61. package/src/cli/handlers/lifecycle-drift.test.ts +52 -0
  62. package/src/cli/handlers/lifecycle-drift.ts +4 -2
  63. package/src/cli/handlers/lock.ts +41 -2
  64. package/src/cli/main.ts +5 -1
  65. package/src/cli/mcp/product-tools.test.ts +1 -2
  66. package/src/cli/mcp/product-tools.ts +1 -2
  67. package/src/cli/parse-args.ts +12 -0
  68. package/src/cli/registry.ts +6 -0
  69. package/src/components/capability.ts +11 -0
  70. package/src/drift-record.ts +5 -0
  71. package/src/e2e-harness/index.ts +1 -2
  72. package/src/lifecycle/ledger-readers.test.ts +10 -0
  73. package/src/lifecycle/lock-beat.ts +99 -0
  74. package/src/lifecycle/resource-lock-liveness.test.ts +289 -0
  75. package/src/lifecycle/resource-lock.ts +143 -8
  76. package/src/lifecycle/wave-plans.ts +41 -1
  77. package/src/op/gate-review.test.ts +1 -1
  78. package/src/op/gate-review.ts +1 -1
  79. package/src/op/index.ts +2 -0
  80. package/src/op/op-waves-capability.ts +3 -0
  81. package/src/op/op-waves-notify.ts +11 -0
  82. package/src/op/op-waves-preview.test.ts +284 -0
  83. package/src/op/op-waves-preview.ts +70 -0
  84. package/src/op/op-waves-run.ts +166 -8
  85. package/src/op/op-waves.test.ts +11 -1
  86. package/src/op/op-waves.ts +20 -3
  87. package/src/release-floors.ts +1 -1
  88. package/src/template-request/template-request.test.ts +1 -2
  89. package/src/template-request/template-request.ts +1 -2
  90. package/src/workspace/drift-cli.test.ts +35 -1
  91. package/src/workspace/drift-cli.ts +21 -3
@@ -0,0 +1,289 @@
1
+ /**
2
+ * #4660: a hold left by a job that failed, lost its lock push or was
3
+ * cancelled must not block later waves forever, and a hold of a live job
4
+ * must still block. A bare remote and one clone per writer, as in
5
+ * resource-lock-wait.test.ts; the clock and the pause are the test's.
6
+ */
7
+ import { afterAll, describe, expect, test } from "vitest";
8
+ import { execFileSync } from "node:child_process";
9
+ import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
10
+ import { tmpdir } from "node:os";
11
+ import { join } from "node:path";
12
+ import { stackRowKey, type LockRow } from "./lock-rows";
13
+ import { runBeatLoop } from "./lock-beat";
14
+ import {
15
+ beatLockRows,
16
+ clearRunLockRows,
17
+ forgeLockLiveness,
18
+ leaseLockRows,
19
+ readLockRows,
20
+ takeLockRows,
21
+ waitLockRows,
22
+ withWaiter,
23
+ type LockRowHolder,
24
+ } from "./resource-lock";
25
+
26
+ const dirs: string[] = [];
27
+ afterAll(() => {
28
+ for (const d of dirs) rmSync(d, { recursive: true, force: true });
29
+ });
30
+
31
+ function tmp(prefix: string): string {
32
+ const d = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
33
+ dirs.push(d);
34
+ return d;
35
+ }
36
+
37
+ function git(cwd: string, ...args: string[]): string {
38
+ return execFileSync("git", args, { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] }).trim();
39
+ }
40
+
41
+ function writers(n: number): { remote: string; clones: string[] } {
42
+ const remote = tmp("chant-lock-live-remote-");
43
+ git(remote, "init", "-q", "--bare", "-b", "main");
44
+ const seed = tmp("chant-lock-live-seed-");
45
+ git(seed, "init", "-q", "-b", "main");
46
+ git(seed, "config", "user.email", "seed@chant.dev");
47
+ git(seed, "config", "user.name", "Seed");
48
+ git(seed, "config", "commit.gpgsign", "false");
49
+ writeFileSync(join(seed, "README.md"), "# t\n");
50
+ git(seed, "add", "README.md");
51
+ git(seed, "commit", "-q", "-m", "init");
52
+ git(seed, "remote", "add", "origin", remote);
53
+ git(seed, "push", "-q", "origin", "main");
54
+ const clones: string[] = [];
55
+ for (let i = 0; i < n; i++) {
56
+ const c = tmp(`chant-lock-live-clone-${i}-`);
57
+ git(c, "clone", "-q", remote, ".");
58
+ git(c, "config", "user.email", `w${i}@chant.dev`);
59
+ git(c, "config", "user.name", `Writer ${i}`);
60
+ git(c, "config", "commit.gpgsign", "false");
61
+ clones.push(c);
62
+ }
63
+ return { remote, clones };
64
+ }
65
+
66
+ const row: LockRow = { key: stackRowKey("shop", "staging"), target: "shop/staging", address: "shop/staging" };
67
+ const holder = (run: string, at = new Date()): LockRowHolder => ({ run, at: at.toISOString() });
68
+
69
+ /** The forge cannot be asked (no token), so only the hold's own token and age speak. */
70
+ const liveness = (clock: () => number) => forgeLockLiveness({}, (async () => ({ ok: false, json: async () => ({}) })) as never, clock);
71
+
72
+ const MIN = 60_000;
73
+
74
+ describe("a failed job", () => {
75
+ test("lets go of its hold when its lease is released, so the next wave takes the row at once", async () => {
76
+ const { clones: [a, b] } = writers(2);
77
+ const first = await leaseLockRows("pr-apply", [row], { cwd: a, holder: holder("25"), beat: () => () => {} });
78
+ expect(first.ok).toBe(true);
79
+ expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "25" });
80
+ if (first.ok) expect(await first.lease.release()).toBe(true);
81
+ expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({});
82
+ const next = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(Date.now) });
83
+ expect(next.ok).toBe(true);
84
+ });
85
+
86
+ test("the beat stops before the rows are let go", async () => {
87
+ const { clones: [a] } = writers(1);
88
+ const order: string[] = [];
89
+ const first = await leaseLockRows("pr-apply", [row], { cwd: a, holder: holder("25"), beat: () => () => order.push("beat stopped") });
90
+ if (!first.ok) throw new Error("not held");
91
+ await first.lease.release();
92
+ order.push("released");
93
+ expect(order).toEqual(["beat stopped", "released"]);
94
+ });
95
+ });
96
+
97
+ describe("a job that lost its lock push", () => {
98
+ test("leaves no hold on the remote, so the next wave is not blocked", async () => {
99
+ const { clones: [a, b] } = writers(2);
100
+ // The push of chant/lifecycle from `a` cannot reach the remote.
101
+ git(a, "remote", "set-url", "--push", "origin", join(tmpdir(), "chant-lock-live-no-such-remote"));
102
+ await expect(leaseLockRows("pr-apply", [row], { cwd: a, holder: holder("25") })).rejects.toThrow();
103
+ expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({});
104
+ const next = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(Date.now) });
105
+ expect(next.ok).toBe(true);
106
+ });
107
+ });
108
+
109
+ describe("a job cancelled or killed with no cleanup", () => {
110
+ test("its hold is taken over once its beat is older than CHANT_LOCK_BEAT_STALE, with no token for the forge", async () => {
111
+ const { clones: [a, b] } = writers(2);
112
+ const t0 = Date.now();
113
+ // Run 26 took the row and beat once; then it was cancelled and never ran its cleanup.
114
+ const dead = await takeLockRows("pr-apply", [row], holder("26", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
115
+ expect(dead.ok).toBe(true);
116
+ expect(await beatLockRows("pr-apply", "26", new Date(t0 + MIN).toISOString(), { cwd: a })).toBe("held");
117
+
118
+ // Nine minutes after the last beat the hold still blocks.
119
+ const early = await takeLockRows("pr-apply", [row], holder("44"), { cwd: b, alive: liveness(() => t0 + 10 * MIN) });
120
+ expect(early.ok).toBe(false);
121
+
122
+ // Past ten minutes it is gone, and the next wave takes the row over.
123
+ const late = await takeLockRows("pr-apply", [row], holder("44"), { cwd: b, alive: liveness(() => t0 + 12 * MIN) });
124
+ expect(late.ok).toBe(true);
125
+ if (late.ok) expect(late.tookOver.map((h) => h.run)).toEqual(["26"]);
126
+ expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "44" });
127
+ });
128
+
129
+ test("a queued wave waits no longer than the beat, then takes the row", async () => {
130
+ const { clones: [a, b] } = writers(2);
131
+ const t0 = Date.now();
132
+ await takeLockRows("pr-apply", [row], holder("26", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
133
+ await beatLockRows("pr-apply", "26", new Date(t0).toISOString(), { cwd: a });
134
+ let clock = t0 + 5 * MIN;
135
+ const got = await waitLockRows(
136
+ "pr-apply",
137
+ [row],
138
+ holder("44", new Date(clock)),
139
+ {
140
+ deadline: t0 + 60 * MIN,
141
+ pollMs: MIN,
142
+ now: () => clock,
143
+ // Each pause is a minute on the test's clock.
144
+ sleep: async (ms) => {
145
+ clock += ms;
146
+ },
147
+ },
148
+ { cwd: b, alive: liveness(() => clock) },
149
+ );
150
+ expect(got.ok).toBe(true);
151
+ expect(got.tookOver.map((h) => h.run)).toEqual(["26"]);
152
+ // It waited for the beat to go stale, about five more minutes, and not for the 60 minute deadline.
153
+ expect(clock - t0).toBeLessThan(15 * MIN);
154
+ });
155
+
156
+ test("a queued run that stopped refreshing its entry does not hold up the waves behind it", async () => {
157
+ const { clones: [a, b] } = writers(2);
158
+ const t0 = Date.now();
159
+ // Run 19 holds the row, and run 25 queues behind it with the older commit. Run 25 is then killed: its pause throws, so it never leaves the queue.
160
+ await takeLockRows("pr-apply", [row], holder("19", new Date(t0 - 10 * MIN)), { cwd: a, alive: liveness(() => t0 - 10 * MIN) });
161
+ await beatLockRows("pr-apply", "19", new Date(t0 - 10 * MIN).toISOString(), { cwd: a });
162
+ await expect(
163
+ waitLockRows(
164
+ "pr-apply",
165
+ [row],
166
+ holder("25", new Date(t0 - 10 * MIN)),
167
+ { deadline: t0 + 60 * MIN, commitAt: "2026-10-30T20:00:00Z", now: () => t0 - 10 * MIN, sleep: async () => Promise.reject(new Error("killed")) },
168
+ { cwd: a, alive: liveness(() => t0 - 10 * MIN) },
169
+ ),
170
+ ).rejects.toThrow("killed");
171
+ expect(Object.keys((await readLockRows("pr-apply", { cwd: b })).waiting ?? {})).toEqual(["25"]);
172
+ // Run 19 finishes and lets go.
173
+ await clearRunLockRows("pr-apply", "19", { cwd: a });
174
+
175
+ const wave = (clock: number) =>
176
+ waitLockRows(
177
+ "pr-apply",
178
+ [row],
179
+ holder("31", new Date(clock)),
180
+ { deadline: clock, commitAt: "2026-10-30T21:00:00Z", now: () => clock, sleep: async () => {} },
181
+ { cwd: b, alive: liveness(() => clock) },
182
+ );
183
+ // A minute on, run 25 may still be alive and goes first.
184
+ await expect(wave(t0 - 9 * MIN)).rejects.toThrow(/run 25/);
185
+ // Ten minutes after its last refresh it is gone, and run 31 takes the row.
186
+ const got = await wave(t0);
187
+ expect(got.ok).toBe(true);
188
+ expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "31" });
189
+ });
190
+ });
191
+
192
+ describe("a live job", () => {
193
+ test("keeps blocking while its beat is fresh, even when its hold is older than the lease", async () => {
194
+ const { clones: [a, b] } = writers(2);
195
+ const t0 = Date.now();
196
+ await takeLockRows("pr-apply", [row], holder("19", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
197
+ // Three hours on, an apply that still beats.
198
+ const later = t0 + 3 * 3600_000;
199
+ await beatLockRows("pr-apply", "19", new Date(later - MIN).toISOString(), { cwd: a });
200
+ const blocked = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(() => later) });
201
+ expect(blocked.ok).toBe(false);
202
+ if (!blocked.ok) expect(blocked.held.map((h) => h.holder.run)).toEqual(["19"]);
203
+ expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "19" });
204
+ });
205
+
206
+ test("a hold without a beat keeps the old rules: it blocks inside the lease", async () => {
207
+ const { clones: [a, b] } = writers(2);
208
+ const t0 = Date.now();
209
+ await takeLockRows("pr-apply", [row], holder("19", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
210
+ const blocked = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(() => t0 + 30 * MIN) });
211
+ expect(blocked.ok).toBe(false);
212
+ });
213
+
214
+ test("a wait that gives up names the hold and the command that frees it", async () => {
215
+ const { clones: [a, b] } = writers(2);
216
+ const t0 = Date.now();
217
+ await takeLockRows("pr-apply", [row], holder("19", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
218
+ await beatLockRows("pr-apply", "19", new Date(t0).toISOString(), { cwd: a });
219
+ const got = await leaseLockRows("pr-apply", [row], {
220
+ cwd: b,
221
+ holder: holder("31"),
222
+ alive: liveness(() => t0),
223
+ wait: { deadline: t0, now: () => t0, sleep: async () => {} },
224
+ });
225
+ expect(got.ok).toBe(false);
226
+ if (!got.ok) {
227
+ expect(got.timedOut).toBe(true);
228
+ expect(got.message).toContain("run 19");
229
+ expect(got.message).toContain("last beat");
230
+ expect(got.message).toContain("chant unlock --run <id> --gate pr-apply");
231
+ }
232
+ });
233
+ });
234
+
235
+ describe("a hold cleared by hand", () => {
236
+ test("chant unlock --run frees the rows and the queue entry of a run, whatever its liveness", async () => {
237
+ const { clones: [a, b] } = writers(2);
238
+ await takeLockRows("pr-apply", [row], holder("26"), { cwd: a, alive: liveness(Date.now) });
239
+ const cleared = await clearRunLockRows("pr-apply", "26", { cwd: b });
240
+ expect(cleared.released).toEqual([row.key]);
241
+ const now = await readLockRows("pr-apply", { cwd: a });
242
+ expect(now.rows).toEqual({});
243
+ expect(now.holders).toEqual({});
244
+ // Nothing left to clear.
245
+ expect((await clearRunLockRows("pr-apply", "26", { cwd: b })).released).toEqual([]);
246
+ });
247
+ });
248
+
249
+ describe("the heartbeat loop", () => {
250
+ const base = { gate: "pr-apply", run: "26", parent: 1, intervalMs: 60_000, sleep: async () => {}, now: () => 0 };
251
+
252
+ test("beats until the hold is gone", async () => {
253
+ let n = 0;
254
+ const end = await runBeatLoop({ ...base, parentAlive: () => true, beat: async () => (++n < 3 ? "held" : "gone") });
255
+ expect(end).toBe("released");
256
+ expect(n).toBe(3);
257
+ });
258
+
259
+ test("ends with its parent process, and touches nothing after", async () => {
260
+ let n = 0;
261
+ let alive = true;
262
+ const end = await runBeatLoop({
263
+ ...base,
264
+ parentAlive: () => alive,
265
+ beat: async () => {
266
+ if (++n === 2) alive = false;
267
+ return "held";
268
+ },
269
+ });
270
+ expect(end).toBe("parent-gone");
271
+ expect(n).toBe(2);
272
+ });
273
+
274
+ test("a failed touch is tried again at the next interval", async () => {
275
+ let n = 0;
276
+ const lines: string[] = [];
277
+ const end = await runBeatLoop({
278
+ ...base,
279
+ parentAlive: () => true,
280
+ log: (l) => lines.push(l),
281
+ beat: async () => {
282
+ if (++n === 1) throw new Error("push refused");
283
+ return "gone";
284
+ },
285
+ });
286
+ expect(end).toBe("released");
287
+ expect(lines[0]).toContain("push refused");
288
+ });
289
+ });
@@ -50,6 +50,7 @@
50
50
  import { hostname } from "node:os";
51
51
  import { getRuntime } from "../runtime-adapter";
52
52
  import { pushLifecycleStatus, readBlobBySha, readPathSha, RefCASConflictError, writeBlobToPath } from "./git";
53
+ import { ownBeatStarter } from "./lock-beat";
53
54
  import { lockRowsConflict, sortedLockRows, type LockRow } from "./lock-rows";
54
55
  import { dropRefusedWrite, fetchForLocks, localTip, LOCKS_DIR, targetLockGateProblem } from "./target-lock";
55
56
 
@@ -59,6 +60,15 @@ export const LOCK_ROWS_DIR = `${LOCKS_DIR}/rows`;
59
60
  /** A lease older than this many seconds is taken over (`CHANT_LOCK_STALE`). */
60
61
  export const DEFAULT_LOCK_STALE_SECONDS = 7200;
61
62
 
63
+ /** A hold whose beat is older than this many seconds is gone (`CHANT_LOCK_BEAT_STALE`, #4660). */
64
+ export const DEFAULT_LOCK_BEAT_STALE_SECONDS = 600;
65
+
66
+ /** How often a live apply touches its hold, in seconds (`CHANT_LOCK_BEAT`, #4660). */
67
+ export const DEFAULT_LOCK_BEAT_SECONDS = 60;
68
+
69
+ /** A queued run that has not refreshed its entry for this many seconds is gone (`CHANT_LOCK_WAITER_STALE`, #4660). */
70
+ export const DEFAULT_LOCK_WAITER_STALE_SECONDS = 300;
71
+
62
72
  const STATE_BRANCH = "chant/lifecycle";
63
73
  const ATTEMPTS = 30;
64
74
 
@@ -69,6 +79,15 @@ export interface LockRowHolder {
69
79
  /** The run's page, for a refusal that names it. */
70
80
  url?: string;
71
81
  sha?: string;
82
+ /**
83
+ * The liveness token (#4660): the last time a heartbeat of the run's own
84
+ * process touched its hold. A holder with a beat is judged by the beat
85
+ * alone, so a job that is cancelled or killed without running its cleanup
86
+ * frees its rows once the beat is {@link lockBeatStaleSeconds} old, while a
87
+ * job that still beats keeps them however long its apply runs. A holder
88
+ * without one (an older chant, terragucci) keeps the rules below.
89
+ */
90
+ beat?: string;
72
91
  }
73
92
 
74
93
  /** A run waiting for rows: who it is, the rows it waits for, and the commit time that orders it. */
@@ -114,6 +133,26 @@ export function lockStaleSeconds(env: Record<string, string | undefined>): numbe
114
133
  return DEFAULT_LOCK_STALE_SECONDS;
115
134
  }
116
135
 
136
+ function positiveSeconds(env: Record<string, string | undefined>, name: string, fallback: number): number {
137
+ const n = Number(env[name]);
138
+ return env[name] !== undefined && Number.isFinite(n) && n > 0 ? n : fallback;
139
+ }
140
+
141
+ /** How old a hold's beat may be before the hold is gone, in seconds: `CHANT_LOCK_BEAT_STALE`, else {@link DEFAULT_LOCK_BEAT_STALE_SECONDS}. */
142
+ export function lockBeatStaleSeconds(env: Record<string, string | undefined>): number {
143
+ return positiveSeconds(env, "CHANT_LOCK_BEAT_STALE", DEFAULT_LOCK_BEAT_STALE_SECONDS);
144
+ }
145
+
146
+ /** Seconds between beats: `CHANT_LOCK_BEAT`, else {@link DEFAULT_LOCK_BEAT_SECONDS}. */
147
+ export function lockBeatSeconds(env: Record<string, string | undefined>): number {
148
+ return positiveSeconds(env, "CHANT_LOCK_BEAT", DEFAULT_LOCK_BEAT_SECONDS);
149
+ }
150
+
151
+ /** How old a queued run's entry may be before it is gone, in seconds: `CHANT_LOCK_WAITER_STALE`, else {@link DEFAULT_LOCK_WAITER_STALE_SECONDS}. */
152
+ export function lockWaiterStaleSeconds(env: Record<string, string | undefined>): number {
153
+ return positiveSeconds(env, "CHANT_LOCK_WAITER_STALE", DEFAULT_LOCK_WAITER_STALE_SECONDS);
154
+ }
155
+
117
156
  /** Who this apply is, as the rows name it: the forge's run, or this host and process outside CI. */
118
157
  export function lockRowHolder(env: Record<string, string | undefined>, now: Date = new Date()): LockRowHolder {
119
158
  const gitlab = Boolean(env.CI_PIPELINE_ID);
@@ -137,7 +176,13 @@ type Fetch = (url: string, init?: { headers?: Record<string, string> }) => Promi
137
176
  */
138
177
  export function forgeLockLiveness(env: Record<string, string | undefined>, fetch: Fetch = globalThis.fetch as unknown as Fetch, now: () => number = Date.now): LockLiveness {
139
178
  const stale = lockStaleSeconds(env);
179
+ const beatStale = lockBeatStaleSeconds(env);
140
180
  return async (h) => {
181
+ // #4660: a hold that carries a beat is judged by it alone. The forge may call a cancelled run finished while its process still applies, and an apply may outlast the lease; the beat says whether the process is there.
182
+ if (h.beat !== undefined) {
183
+ const beat = new Date(h.beat).getTime();
184
+ if (Number.isFinite(beat)) return now() - beat >= beatStale * 1000 ? "dead" : "alive";
185
+ }
141
186
  const since = new Date(h.at).getTime();
142
187
  if (Number.isFinite(since) && now() - since >= stale * 1000) return "dead";
143
188
  const local = /^local:(.*):(\d+)$/.exec(h.run);
@@ -180,7 +225,7 @@ export function parseLockRows(text: string | null): LockRowsFile {
180
225
  const holders: Record<string, LockRowHolder> = {};
181
226
  for (const [id, raw] of Object.entries(doc.holders as Record<string, Record<string, unknown>>)) {
182
227
  if (typeof raw?.run !== "string" || typeof raw.at !== "string") continue;
183
- holders[id] = { run: raw.run, at: raw.at, ...(typeof raw.url === "string" ? { url: raw.url } : {}), ...(typeof raw.sha === "string" ? { sha: raw.sha } : {}) };
228
+ holders[id] = { run: raw.run, at: raw.at, ...(typeof raw.url === "string" ? { url: raw.url } : {}), ...(typeof raw.sha === "string" ? { sha: raw.sha } : {}), ...(typeof raw.beat === "string" ? { beat: raw.beat } : {}) };
184
229
  }
185
230
  const rows: Record<string, string> = {};
186
231
  for (const [k, id] of Object.entries(doc.rows as Record<string, unknown>)) if (typeof id === "string" && holders[id]) rows[k] = id;
@@ -434,13 +479,16 @@ export async function commitTimeOf(rev: string, cwd?: string): Promise<string |
434
479
  }
435
480
 
436
481
  /** The live waiters for a row that meets one of `keys` who go before `me`, and the waiters that are gone. */
437
- export async function waitersAhead(file: LockRowsFile, keys: readonly string[], me: LockRowWaiter, alive: LockLiveness): Promise<{ ahead: LockRowWaiter[]; gone: string[] }> {
482
+ export async function waitersAhead(file: LockRowsFile, keys: readonly string[], me: LockRowWaiter, alive: LockLiveness, now: () => number = Date.now): Promise<{ ahead: LockRowWaiter[]; gone: string[] }> {
438
483
  const ahead: LockRowWaiter[] = [];
439
484
  const gone: string[] = [];
485
+ const waiterStale = lockWaiterStaleSeconds(process.env) * 1000;
440
486
  for (const [id, w] of Object.entries(file.waiting ?? {})) {
441
487
  if (id === me.run || w.run === me.run) continue;
442
488
  if (!w.keys.some((k) => keys.some((mine) => lockRowsConflict(k, mine)))) continue;
443
- const verdict = await alive(w);
489
+ // #4660: a waiter refreshes its entry every few poll rounds, so one that has not is gone (cancelled or killed), whatever the forge says.
490
+ const entered = new Date(w.at).getTime();
491
+ const verdict = Number.isFinite(entered) && now() - entered >= waiterStale ? "dead" : await alive(w);
444
492
  if (verdict === "dead") {
445
493
  gone.push(id);
446
494
  continue;
@@ -450,6 +498,13 @@ export async function waitersAhead(file: LockRowsFile, keys: readonly string[],
450
498
  return { ahead: ahead.sort((a, b) => (waiterBefore(a, b) ? -1 : 1)), gone: gone.sort() };
451
499
  }
452
500
 
501
+ /** When each live holder took its rows and last beat, for a wait that gave up (#4660). */
502
+ function holdAges(held: readonly HeldLockRow[]): string {
503
+ const runs = [...new Map(held.map((h) => [h.holder.run, h.holder])).values()];
504
+ if (runs.length === 0) return "";
505
+ return ` (${runs.map((h) => `run ${h.run} took its rows at ${h.at}${h.beat ? `, last beat ${h.beat}` : ", no beat"}`).join("; ")})`;
506
+ }
507
+
453
508
  function describeWait(held: readonly HeldLockRow[], ahead: readonly LockRowWaiter[], rows: readonly LockRow[]): string {
454
509
  const parts: string[] = [];
455
510
  if (held.length > 0) parts.push(describeHeldLockRows(held, rows));
@@ -481,7 +536,7 @@ export async function waitLockRows(gate: string, rows: readonly LockRow[], me: L
481
536
  const { sha, file } = await readCurrent(gate, opts.cwd);
482
537
  const mine = queued ?? waiter();
483
538
  const { live, gone } = await heldLockRowsIn(file, keys, me.run, alive);
484
- const { ahead, gone: goneWaiters } = await waitersAhead(file, keys, mine, alive);
539
+ const { ahead, gone: goneWaiters } = await waitersAhead(file, keys, mine, alive, now);
485
540
  if (live.length === 0 && ahead.length === 0) {
486
541
  const next = withRowsTaken(file, keys, me, gone);
487
542
  for (const id of goneWaiters) delete next.waiting?.[id];
@@ -496,10 +551,10 @@ export async function waitLockRows(gate: string, rows: readonly LockRow[], me: L
496
551
  if (now() >= wait.deadline) {
497
552
  await leaveQueue(gate, me.run, opts.cwd);
498
553
  const waited = Math.round((now() - started) / 1000);
499
- throw new LockWaitTimeoutError(`waited ${waited}s for ${keys.join(", ")} of ${gate} and gave up at the job's deadline: ${why}`, live, ahead, waited);
554
+ throw new LockWaitTimeoutError(`waited ${waited}s for ${keys.join(", ")} of ${gate} and gave up at the job's deadline: ${why}${holdAges(live)}. If a run named here is gone, chant unlock --run <id> --gate ${gate} frees its rows`, live, ahead, waited);
500
555
  }
501
556
  const inFile = file.waiting?.[me.run];
502
- const refresh = inFile ? now() - new Date(inFile.at).getTime() > stale / 2 : true;
557
+ const refresh = inFile ? now() - new Date(inFile.at).getTime() > Math.min(stale / 2, (lockWaiterStaleSeconds(process.env) * 1000) / 3) : true;
503
558
  if (!inFile || refresh || goneWaiters.length > 0) {
504
559
  const entry: LockRowWaiter = { ...waiter(), at: new Date(now()).toISOString() };
505
560
  const wrote = await writeAndPush(gate, sha, withWaiter(file, entry, goneWaiters), `Run ${me.run} waits for ${keys.length} row${keys.length === 1 ? "" : "s"} of ${gate}`, opts.cwd);
@@ -556,6 +611,63 @@ export async function releaseLockRows(gate: string, keys: readonly string[], me:
556
611
  return false;
557
612
  }
558
613
 
614
+ /** The rows file after run `run`'s beat is set to `at`. Unchanged when the run holds no row. */
615
+ export function withRowsBeat(file: LockRowsFile, run: string, at: string): LockRowsFile {
616
+ const holder = file.holders[run];
617
+ if (!holder || !Object.values(file.rows).includes(run)) return file;
618
+ const next = copyRows(file);
619
+ next.holders[run] = { ...holder, beat: at };
620
+ return next;
621
+ }
622
+
623
+ /**
624
+ * Touch run `run`'s hold (#4660): the heartbeat of a live apply. `held` while
625
+ * the run still holds a row, `gone` once it holds none (released, or taken
626
+ * over after its beat went stale), so the heartbeat stops.
627
+ */
628
+ export async function beatLockRows(gate: string, run: string, at: string, opts: { cwd?: string } = {}): Promise<"held" | "gone"> {
629
+ checkGate(gate);
630
+ await fetchForLocks(opts.cwd);
631
+ for (let attempt = 0; attempt < ATTEMPTS; attempt++) {
632
+ const { sha, file } = await readCurrent(gate, opts.cwd);
633
+ if (!Object.values(file.rows).includes(run)) return "gone";
634
+ const wrote = await writeAndPush(gate, sha, withRowsBeat(file, run, at), `Run ${run} still holds its rows of ${gate}`, opts.cwd);
635
+ if (wrote) return "held";
636
+ await pause(200 + Math.floor(Math.random() * 800));
637
+ }
638
+ throw new Error(`could not touch the hold of run ${run} on ${gate}: ${STATE_BRANCH} kept moving`);
639
+ }
640
+
641
+ /** The rows file after every hold and queue entry of `run` is dropped, whether or not the run is gone. */
642
+ export function withRunCleared(file: LockRowsFile, run: string): LockRowsFile {
643
+ const next = copyRows(file);
644
+ delete next.holders[run];
645
+ for (const [k, id] of Object.entries(next.rows)) if (id === run) delete next.rows[k];
646
+ delete next.waiting?.[run];
647
+ return tidyWaiting(next);
648
+ }
649
+
650
+ /**
651
+ * Let go of everything run `run` holds or waits for on `gate`, by hand
652
+ * (#4660, `chant unlock --run`). It does not ask whether the run is alive:
653
+ * the person who runs it says so. Throws when the branch cannot be read or
654
+ * pushed, so a hold that is still there is never reported as cleared.
655
+ */
656
+ export async function clearRunLockRows(gate: string, run: string, opts: { cwd?: string } = {}): Promise<{ released: string[]; waited: boolean; pushed: boolean }> {
657
+ checkGate(gate);
658
+ await fetchForLocks(opts.cwd);
659
+ for (let attempt = 0; attempt < ATTEMPTS; attempt++) {
660
+ const { sha, file } = await readCurrent(gate, opts.cwd);
661
+ const released = Object.entries(file.rows).filter(([, id]) => id === run).map(([k]) => k).sort();
662
+ const waited = file.waiting?.[run] !== undefined;
663
+ if (released.length === 0 && !file.holders[run] && !waited) return { released, waited, pushed: false };
664
+ const wrote = await writeAndPush(gate, sha, withRunCleared(file, run), `Clear the hold of run ${run} on ${gate}`, opts.cwd);
665
+ if (wrote) return { released, waited, pushed: wrote.pushed };
666
+ await pause(200 + Math.floor(Math.random() * 800));
667
+ }
668
+ throw new Error(`could not clear run ${run} from ${gate}: ${STATE_BRANCH} kept moving; check that the job may push to it`);
669
+ }
670
+
559
671
  /** One line naming the runs that hold rows, and what of this apply they hold, for a log line and a refusal. */
560
672
  export function describeHeldLockRows(held: readonly HeldLockRow[], rows: readonly LockRow[]): string {
561
673
  const address = new Map(rows.map((r) => [r.key, `${r.target}: ${r.address}`]));
@@ -578,6 +690,9 @@ export interface LockRowsLease {
578
690
  release(): Promise<boolean>;
579
691
  }
580
692
 
693
+ /** Starts the heartbeat of a hold (#4660) and returns how to stop it. */
694
+ export type LockBeatStarter = (hold: { gate: string; holder: LockRowHolder; cwd?: string }) => () => void;
695
+
581
696
  /**
582
697
  * Lease `rows` for the apply about to run (#4552): take them, or say who
583
698
  * holds them. With no rows, a lease that holds nothing. Throws when the
@@ -587,15 +702,34 @@ export interface LockRowsLease {
587
702
  export async function leaseLockRows(
588
703
  gate: string,
589
704
  rows: readonly LockRow[],
590
- opts: LockRowsOptions & { holder?: LockRowHolder; wait?: LockRowsWait } = {},
705
+ opts: LockRowsOptions & { holder?: LockRowHolder; wait?: LockRowsWait; beat?: LockBeatStarter } = {},
591
706
  ): Promise<{ ok: true; lease: LockRowsLease; tookOver: LockRowHolder[]; waitedSeconds?: number; waited?: boolean } | { ok: false; held: HeldLockRow[]; message: string; timedOut?: true }> {
592
707
  const holder = opts.holder ?? lockRowHolder(process.env);
593
708
  const keys = sortedLockRows(rows).map((r) => r.key);
594
- const lease: LockRowsLease = { gate, holder, keys, release: () => (keys.length === 0 ? Promise.resolve(true) : releaseLockRows(gate, keys, holder.run, { ...(opts.cwd ? { cwd: opts.cwd } : {}) })) };
709
+ let stopBeat: (() => void) | undefined;
710
+ const lease: LockRowsLease = {
711
+ gate,
712
+ holder,
713
+ keys,
714
+ release: () => {
715
+ // The beat stops first, so it cannot touch a hold that is being let go.
716
+ stopBeat?.();
717
+ stopBeat = undefined;
718
+ return keys.length === 0 ? Promise.resolve(true) : releaseLockRows(gate, keys, holder.run, { ...(opts.cwd ? { cwd: opts.cwd } : {}) });
719
+ },
720
+ };
595
721
  if (keys.length === 0) return { ok: true, lease, tookOver: [] };
722
+ const startBeat = () => {
723
+ try {
724
+ stopBeat = (opts.beat ?? ownBeatStarter())?.({ gate, holder, ...(opts.cwd ? { cwd: opts.cwd } : {}) });
725
+ } catch {
726
+ // No beat: the hold keeps the forge and lease rules.
727
+ }
728
+ };
596
729
  if (opts.wait) {
597
730
  try {
598
731
  const waited = await waitLockRows(gate, rows, holder, opts.wait, opts);
732
+ startBeat();
599
733
  return { ok: true, lease, tookOver: waited.tookOver, ...(waited.waited ? { waited: true, waitedSeconds: waited.waitedSeconds } : {}) };
600
734
  } catch (err) {
601
735
  if (err instanceof LockWaitTimeoutError) return { ok: false, held: err.held, message: err.message, timedOut: true };
@@ -604,5 +738,6 @@ export async function leaseLockRows(
604
738
  }
605
739
  const taken = await takeLockRows(gate, rows, holder, opts);
606
740
  if (!taken.ok) return { ok: false, held: taken.held, message: describeHeldLockRows(taken.held, rows) };
741
+ startBeat();
607
742
  return { ok: true, lease, tookOver: taken.tookOver };
608
743
  }
@@ -25,6 +25,20 @@ export interface HeadPlans {
25
25
  waves: Array<HeadPlanWave>;
26
26
  }
27
27
 
28
+ /**
29
+ * One run's plan at the head (plywood#55), kept so a merged wave can compare
30
+ * the resources the pull request changed, and only those, with the ones it
31
+ * plans now. `resources` is the digest of each resource the plan changes, by
32
+ * address; a run whose plan gave none is compared as a whole by `planDigest`.
33
+ * `empty` is a plan that changes nothing.
34
+ */
35
+ export interface HeadPlanMember {
36
+ member: string;
37
+ planDigest: string;
38
+ resources?: Record<string, string>;
39
+ empty?: true;
40
+ }
41
+
28
42
  /**
29
43
  * One wave's plan at the head. `changes` and `destructive` (#4630) are the
30
44
  * plan's facts a gate policy reads: some run changes something, some run's
@@ -44,6 +58,12 @@ export interface HeadPlanWave {
44
58
  * approval against it.
45
59
  */
46
60
  changeSet?: string;
61
+ /**
62
+ * Each run's plan at the head (plywood#55). A record from chant 0.124.59
63
+ * has none: it holds the wave's set digest only, and a merged wave
64
+ * compares that whole digest, as it did.
65
+ */
66
+ members?: HeadPlanMember[];
47
67
  }
48
68
 
49
69
  /** Where {@link HeadPlans} for `head` live on `chant/lifecycle`: `_wave-plans/<name>/<head>.json`. */
@@ -71,8 +91,10 @@ export function parseHeadPlans(text: string | null): HeadPlans | null {
71
91
  const waves: HeadPlans["waves"] = [];
72
92
  for (const w of r.waves) {
73
93
  if (w === null || typeof w !== "object") return null;
74
- const { wave, name, digest, changes, destructive, changeSet } = w as Record<string, unknown>;
94
+ const { wave, name, digest, changes, destructive, changeSet, members } = w as Record<string, unknown>;
75
95
  if (typeof wave !== "number" || typeof name !== "string" || typeof digest !== "string") return null;
96
+ const runs = members === undefined ? undefined : parseHeadPlanMembers(members);
97
+ if (runs === null) return null;
76
98
  waves.push({
77
99
  wave,
78
100
  name,
@@ -80,11 +102,29 @@ export function parseHeadPlans(text: string | null): HeadPlans | null {
80
102
  ...(typeof changes === "boolean" ? { changes } : {}),
81
103
  ...(typeof destructive === "boolean" ? { destructive } : {}),
82
104
  ...(typeof changeSet === "string" && changeSet !== "" ? { changeSet } : {}),
105
+ ...(runs ? { members: runs } : {}),
83
106
  });
84
107
  }
85
108
  return { version: 1, name: r.name, head: r.head, timestamp: r.timestamp, waves };
86
109
  }
87
110
 
111
+ function parseHeadPlanMembers(value: unknown): HeadPlanMember[] | null {
112
+ if (!Array.isArray(value)) return null;
113
+ const out: HeadPlanMember[] = [];
114
+ for (const m of value) {
115
+ if (m === null || typeof m !== "object") return null;
116
+ const { member, planDigest, resources, empty } = m as Record<string, unknown>;
117
+ if (typeof member !== "string" || typeof planDigest !== "string") return null;
118
+ let digests: Record<string, string> | undefined;
119
+ if (resources !== undefined) {
120
+ if (resources === null || typeof resources !== "object" || Array.isArray(resources) || !Object.values(resources).every((d) => typeof d === "string")) return null;
121
+ digests = resources as Record<string, string>;
122
+ }
123
+ out.push({ member, planDigest, ...(digests ? { resources: digests } : {}), ...(empty === true ? { empty: true as const } : {}) });
124
+ }
125
+ return out;
126
+ }
127
+
88
128
  /** The record for `head` of the spec `name`, or null when none was recorded or it cannot be read. */
89
129
  export async function readHeadPlans(port: LedgerReadPort, name: string, head: string): Promise<HeadPlans | null> {
90
130
  const { dir, file } = headPlansPath(name, head);
@@ -178,7 +178,7 @@ describe("a pr-review wave (#3684)", () => {
178
178
  let written: unknown;
179
179
  const recorded = await recordOpWaveHeadPlans({ spec: spec("pr-review"), cwd: h.cwd, exec: h.exec, head: HEAD, now: "t", write: async (p) => void (written = p) });
180
180
  // #4630: the record keeps whether the wave changes anything and whether it is destructive.
181
- expect(recorded.waves).toEqual([{ wave: 1, name: "prod", digest: set(digest("a")), changes: true, destructive: false }]);
181
+ expect(recorded.waves).toEqual([{ wave: 1, name: "prod", digest: set(digest("a")), changes: true, destructive: false, members: [{ member: "prod", planDigest: digest("a") }] }]);
182
182
  expect(written).toEqual(recorded);
183
183
  });
184
184
  });
@@ -71,7 +71,7 @@ export function bothRules(a: GateApprovalRule | null, b: GateApprovalRule | null
71
71
  // ── pr-review ─────────────────────────────────────────────────────────────
72
72
 
73
73
  // The record and its reader live apart, so a bundle can import them without git (#3865).
74
- export { HEAD_PLANS_DIR, headPlansPath, type HeadPlans } from "../lifecycle/wave-plans";
74
+ export { HEAD_PLANS_DIR, headPlansPath, type HeadPlans, type HeadPlanMember } from "../lifecycle/wave-plans";
75
75
 
76
76
 
77
77
  /** The merged pull request's review, as a `pr-review` gate reads it. */
package/src/op/index.ts CHANGED
@@ -81,6 +81,7 @@ export {
81
81
  opWavePlanPath,
82
82
  opWavePlansPath,
83
83
  OP_WAVE_PLAN_PATH_FLOOR,
84
+ OP_WAVES_PREVIEW_FLOOR,
84
85
  opWaveRunsPath,
85
86
  opWaveSetDigest,
86
87
  parseOpWavesSpec,
@@ -91,6 +92,7 @@ export {
91
92
  runOpWaveApplyHooks,
92
93
  opWaveResourceLines,
93
94
  } from "./op-waves-run";
95
+ export { movedSincePreview, shortAddress, type PlannedRunDigests } from "./op-waves-preview";
94
96
  export type {
95
97
  OpWaveBaseRules,
96
98
  OpWaveDecision,