@intentius/chant 0.124.58 → 0.124.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ci-product.d.ts.map +1 -1
- package/dist/ci-product.js +1 -2
- package/dist/cli/handlers/lifecycle-drift.d.ts +2 -1
- package/dist/cli/handlers/lifecycle-drift.d.ts.map +1 -1
- package/dist/cli/handlers/lifecycle-drift.js +4 -2
- package/dist/cli/handlers/lock.d.ts.map +1 -1
- package/dist/cli/handlers/lock.js +45 -2
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +5 -1
- package/dist/cli/mcp/product-tools.d.ts.map +1 -1
- package/dist/cli/mcp/product-tools.js +1 -2
- package/dist/cli/parse-args.d.ts.map +1 -1
- package/dist/cli/parse-args.js +18 -0
- package/dist/cli/registry.d.ts +6 -0
- package/dist/cli/registry.d.ts.map +1 -1
- package/dist/drift-record.d.ts +5 -0
- package/dist/drift-record.d.ts.map +1 -1
- package/dist/e2e-harness/index.d.ts.map +1 -1
- package/dist/e2e-harness/index.js +1 -2
- package/dist/lifecycle/lock-beat.d.ts +27 -0
- package/dist/lifecycle/lock-beat.d.ts.map +1 -0
- package/dist/lifecycle/lock-beat.js +84 -0
- package/dist/lifecycle/resource-lock.d.ts +54 -1
- package/dist/lifecycle/resource-lock.d.ts.map +1 -1
- package/dist/lifecycle/resource-lock.js +128 -7
- package/dist/op/activities/activity-contracts.d.ts +4 -4
- package/dist/op/index.d.ts +1 -1
- package/dist/op/index.d.ts.map +1 -1
- package/dist/op/index.js +1 -1
- package/dist/op/op-waves-notify.d.ts.map +1 -1
- package/dist/op/op-waves-notify.js +11 -0
- package/dist/op/op-waves-run.d.ts +28 -1
- package/dist/op/op-waves-run.d.ts.map +1 -1
- package/dist/op/op-waves-run.js +92 -7
- package/dist/op/op-waves.d.ts +12 -1
- package/dist/op/op-waves.d.ts.map +1 -1
- package/dist/op/op-waves.js +9 -2
- package/dist/release-floors.d.ts +1 -1
- package/dist/release-floors.js +1 -1
- package/dist/template-request/template-request.d.ts.map +1 -1
- package/dist/template-request/template-request.js +1 -2
- package/dist/wave-runtime.mjs +298 -298
- package/dist/workspace/drift-cli.d.ts +13 -1
- package/dist/workspace/drift-cli.d.ts.map +1 -1
- package/dist/workspace/drift-cli.js +19 -3
- package/package.json +1 -1
- package/src/ci-product.ts +1 -2
- package/src/cli/handlers/lifecycle-drift.test.ts +52 -0
- package/src/cli/handlers/lifecycle-drift.ts +4 -2
- package/src/cli/handlers/lock.ts +41 -2
- package/src/cli/main.ts +5 -1
- package/src/cli/mcp/product-tools.test.ts +1 -2
- package/src/cli/mcp/product-tools.ts +1 -2
- package/src/cli/parse-args.ts +12 -0
- package/src/cli/registry.ts +6 -0
- package/src/drift-record.ts +5 -0
- package/src/e2e-harness/index.ts +1 -2
- package/src/lifecycle/lock-beat.ts +99 -0
- package/src/lifecycle/resource-lock-liveness.test.ts +289 -0
- package/src/lifecycle/resource-lock.ts +143 -8
- package/src/op/index.ts +1 -0
- package/src/op/op-waves-notify.ts +11 -0
- package/src/op/op-waves-preview.test.ts +167 -0
- package/src/op/op-waves-run.ts +116 -8
- package/src/op/op-waves.ts +20 -3
- package/src/release-floors.ts +1 -1
- package/src/template-request/template-request.test.ts +1 -2
- package/src/template-request/template-request.ts +1 -2
- package/src/workspace/drift-cli.test.ts +35 -1
- package/src/workspace/drift-cli.ts +21 -3
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #4660: a hold left by a job that failed, lost its lock push or was
|
|
3
|
+
* cancelled must not block later waves forever, and a hold of a live job
|
|
4
|
+
* must still block. A bare remote and one clone per writer, as in
|
|
5
|
+
* resource-lock-wait.test.ts; the clock and the pause are the test's.
|
|
6
|
+
*/
|
|
7
|
+
import { afterAll, describe, expect, test } from "vitest";
|
|
8
|
+
import { execFileSync } from "node:child_process";
|
|
9
|
+
import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
10
|
+
import { tmpdir } from "node:os";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { stackRowKey, type LockRow } from "./lock-rows";
|
|
13
|
+
import { runBeatLoop } from "./lock-beat";
|
|
14
|
+
import {
|
|
15
|
+
beatLockRows,
|
|
16
|
+
clearRunLockRows,
|
|
17
|
+
forgeLockLiveness,
|
|
18
|
+
leaseLockRows,
|
|
19
|
+
readLockRows,
|
|
20
|
+
takeLockRows,
|
|
21
|
+
waitLockRows,
|
|
22
|
+
withWaiter,
|
|
23
|
+
type LockRowHolder,
|
|
24
|
+
} from "./resource-lock";
|
|
25
|
+
|
|
26
|
+
const dirs: string[] = [];
|
|
27
|
+
afterAll(() => {
|
|
28
|
+
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
function tmp(prefix: string): string {
|
|
32
|
+
const d = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
|
|
33
|
+
dirs.push(d);
|
|
34
|
+
return d;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function git(cwd: string, ...args: string[]): string {
|
|
38
|
+
return execFileSync("git", args, { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] }).trim();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function writers(n: number): { remote: string; clones: string[] } {
|
|
42
|
+
const remote = tmp("chant-lock-live-remote-");
|
|
43
|
+
git(remote, "init", "-q", "--bare", "-b", "main");
|
|
44
|
+
const seed = tmp("chant-lock-live-seed-");
|
|
45
|
+
git(seed, "init", "-q", "-b", "main");
|
|
46
|
+
git(seed, "config", "user.email", "seed@chant.dev");
|
|
47
|
+
git(seed, "config", "user.name", "Seed");
|
|
48
|
+
git(seed, "config", "commit.gpgsign", "false");
|
|
49
|
+
writeFileSync(join(seed, "README.md"), "# t\n");
|
|
50
|
+
git(seed, "add", "README.md");
|
|
51
|
+
git(seed, "commit", "-q", "-m", "init");
|
|
52
|
+
git(seed, "remote", "add", "origin", remote);
|
|
53
|
+
git(seed, "push", "-q", "origin", "main");
|
|
54
|
+
const clones: string[] = [];
|
|
55
|
+
for (let i = 0; i < n; i++) {
|
|
56
|
+
const c = tmp(`chant-lock-live-clone-${i}-`);
|
|
57
|
+
git(c, "clone", "-q", remote, ".");
|
|
58
|
+
git(c, "config", "user.email", `w${i}@chant.dev`);
|
|
59
|
+
git(c, "config", "user.name", `Writer ${i}`);
|
|
60
|
+
git(c, "config", "commit.gpgsign", "false");
|
|
61
|
+
clones.push(c);
|
|
62
|
+
}
|
|
63
|
+
return { remote, clones };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const row: LockRow = { key: stackRowKey("shop", "staging"), target: "shop/staging", address: "shop/staging" };
|
|
67
|
+
const holder = (run: string, at = new Date()): LockRowHolder => ({ run, at: at.toISOString() });
|
|
68
|
+
|
|
69
|
+
/** The forge cannot be asked (no token), so only the hold's own token and age speak. */
|
|
70
|
+
const liveness = (clock: () => number) => forgeLockLiveness({}, (async () => ({ ok: false, json: async () => ({}) })) as never, clock);
|
|
71
|
+
|
|
72
|
+
const MIN = 60_000;
|
|
73
|
+
|
|
74
|
+
describe("a failed job", () => {
|
|
75
|
+
test("lets go of its hold when its lease is released, so the next wave takes the row at once", async () => {
|
|
76
|
+
const { clones: [a, b] } = writers(2);
|
|
77
|
+
const first = await leaseLockRows("pr-apply", [row], { cwd: a, holder: holder("25"), beat: () => () => {} });
|
|
78
|
+
expect(first.ok).toBe(true);
|
|
79
|
+
expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "25" });
|
|
80
|
+
if (first.ok) expect(await first.lease.release()).toBe(true);
|
|
81
|
+
expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({});
|
|
82
|
+
const next = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(Date.now) });
|
|
83
|
+
expect(next.ok).toBe(true);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("the beat stops before the rows are let go", async () => {
|
|
87
|
+
const { clones: [a] } = writers(1);
|
|
88
|
+
const order: string[] = [];
|
|
89
|
+
const first = await leaseLockRows("pr-apply", [row], { cwd: a, holder: holder("25"), beat: () => () => order.push("beat stopped") });
|
|
90
|
+
if (!first.ok) throw new Error("not held");
|
|
91
|
+
await first.lease.release();
|
|
92
|
+
order.push("released");
|
|
93
|
+
expect(order).toEqual(["beat stopped", "released"]);
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
describe("a job that lost its lock push", () => {
|
|
98
|
+
test("leaves no hold on the remote, so the next wave is not blocked", async () => {
|
|
99
|
+
const { clones: [a, b] } = writers(2);
|
|
100
|
+
// The push of chant/lifecycle from `a` cannot reach the remote.
|
|
101
|
+
git(a, "remote", "set-url", "--push", "origin", join(tmpdir(), "chant-lock-live-no-such-remote"));
|
|
102
|
+
await expect(leaseLockRows("pr-apply", [row], { cwd: a, holder: holder("25") })).rejects.toThrow();
|
|
103
|
+
expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({});
|
|
104
|
+
const next = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(Date.now) });
|
|
105
|
+
expect(next.ok).toBe(true);
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
describe("a job cancelled or killed with no cleanup", () => {
|
|
110
|
+
test("its hold is taken over once its beat is older than CHANT_LOCK_BEAT_STALE, with no token for the forge", async () => {
|
|
111
|
+
const { clones: [a, b] } = writers(2);
|
|
112
|
+
const t0 = Date.now();
|
|
113
|
+
// Run 26 took the row and beat once; then it was cancelled and never ran its cleanup.
|
|
114
|
+
const dead = await takeLockRows("pr-apply", [row], holder("26", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
|
|
115
|
+
expect(dead.ok).toBe(true);
|
|
116
|
+
expect(await beatLockRows("pr-apply", "26", new Date(t0 + MIN).toISOString(), { cwd: a })).toBe("held");
|
|
117
|
+
|
|
118
|
+
// Nine minutes after the last beat the hold still blocks.
|
|
119
|
+
const early = await takeLockRows("pr-apply", [row], holder("44"), { cwd: b, alive: liveness(() => t0 + 10 * MIN) });
|
|
120
|
+
expect(early.ok).toBe(false);
|
|
121
|
+
|
|
122
|
+
// Past ten minutes it is gone, and the next wave takes the row over.
|
|
123
|
+
const late = await takeLockRows("pr-apply", [row], holder("44"), { cwd: b, alive: liveness(() => t0 + 12 * MIN) });
|
|
124
|
+
expect(late.ok).toBe(true);
|
|
125
|
+
if (late.ok) expect(late.tookOver.map((h) => h.run)).toEqual(["26"]);
|
|
126
|
+
expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "44" });
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("a queued wave waits no longer than the beat, then takes the row", async () => {
|
|
130
|
+
const { clones: [a, b] } = writers(2);
|
|
131
|
+
const t0 = Date.now();
|
|
132
|
+
await takeLockRows("pr-apply", [row], holder("26", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
|
|
133
|
+
await beatLockRows("pr-apply", "26", new Date(t0).toISOString(), { cwd: a });
|
|
134
|
+
let clock = t0 + 5 * MIN;
|
|
135
|
+
const got = await waitLockRows(
|
|
136
|
+
"pr-apply",
|
|
137
|
+
[row],
|
|
138
|
+
holder("44", new Date(clock)),
|
|
139
|
+
{
|
|
140
|
+
deadline: t0 + 60 * MIN,
|
|
141
|
+
pollMs: MIN,
|
|
142
|
+
now: () => clock,
|
|
143
|
+
// Each pause is a minute on the test's clock.
|
|
144
|
+
sleep: async (ms) => {
|
|
145
|
+
clock += ms;
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
{ cwd: b, alive: liveness(() => clock) },
|
|
149
|
+
);
|
|
150
|
+
expect(got.ok).toBe(true);
|
|
151
|
+
expect(got.tookOver.map((h) => h.run)).toEqual(["26"]);
|
|
152
|
+
// It waited for the beat to go stale, about five more minutes, and not for the 60 minute deadline.
|
|
153
|
+
expect(clock - t0).toBeLessThan(15 * MIN);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("a queued run that stopped refreshing its entry does not hold up the waves behind it", async () => {
|
|
157
|
+
const { clones: [a, b] } = writers(2);
|
|
158
|
+
const t0 = Date.now();
|
|
159
|
+
// Run 19 holds the row, and run 25 queues behind it with the older commit. Run 25 is then killed: its pause throws, so it never leaves the queue.
|
|
160
|
+
await takeLockRows("pr-apply", [row], holder("19", new Date(t0 - 10 * MIN)), { cwd: a, alive: liveness(() => t0 - 10 * MIN) });
|
|
161
|
+
await beatLockRows("pr-apply", "19", new Date(t0 - 10 * MIN).toISOString(), { cwd: a });
|
|
162
|
+
await expect(
|
|
163
|
+
waitLockRows(
|
|
164
|
+
"pr-apply",
|
|
165
|
+
[row],
|
|
166
|
+
holder("25", new Date(t0 - 10 * MIN)),
|
|
167
|
+
{ deadline: t0 + 60 * MIN, commitAt: "2026-10-30T20:00:00Z", now: () => t0 - 10 * MIN, sleep: async () => Promise.reject(new Error("killed")) },
|
|
168
|
+
{ cwd: a, alive: liveness(() => t0 - 10 * MIN) },
|
|
169
|
+
),
|
|
170
|
+
).rejects.toThrow("killed");
|
|
171
|
+
expect(Object.keys((await readLockRows("pr-apply", { cwd: b })).waiting ?? {})).toEqual(["25"]);
|
|
172
|
+
// Run 19 finishes and lets go.
|
|
173
|
+
await clearRunLockRows("pr-apply", "19", { cwd: a });
|
|
174
|
+
|
|
175
|
+
const wave = (clock: number) =>
|
|
176
|
+
waitLockRows(
|
|
177
|
+
"pr-apply",
|
|
178
|
+
[row],
|
|
179
|
+
holder("31", new Date(clock)),
|
|
180
|
+
{ deadline: clock, commitAt: "2026-10-30T21:00:00Z", now: () => clock, sleep: async () => {} },
|
|
181
|
+
{ cwd: b, alive: liveness(() => clock) },
|
|
182
|
+
);
|
|
183
|
+
// A minute on, run 25 may still be alive and goes first.
|
|
184
|
+
await expect(wave(t0 - 9 * MIN)).rejects.toThrow(/run 25/);
|
|
185
|
+
// Ten minutes after its last refresh it is gone, and run 31 takes the row.
|
|
186
|
+
const got = await wave(t0);
|
|
187
|
+
expect(got.ok).toBe(true);
|
|
188
|
+
expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "31" });
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
describe("a live job", () => {
|
|
193
|
+
test("keeps blocking while its beat is fresh, even when its hold is older than the lease", async () => {
|
|
194
|
+
const { clones: [a, b] } = writers(2);
|
|
195
|
+
const t0 = Date.now();
|
|
196
|
+
await takeLockRows("pr-apply", [row], holder("19", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
|
|
197
|
+
// Three hours on, an apply that still beats.
|
|
198
|
+
const later = t0 + 3 * 3600_000;
|
|
199
|
+
await beatLockRows("pr-apply", "19", new Date(later - MIN).toISOString(), { cwd: a });
|
|
200
|
+
const blocked = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(() => later) });
|
|
201
|
+
expect(blocked.ok).toBe(false);
|
|
202
|
+
if (!blocked.ok) expect(blocked.held.map((h) => h.holder.run)).toEqual(["19"]);
|
|
203
|
+
expect((await readLockRows("pr-apply", { cwd: b })).rows).toEqual({ [row.key]: "19" });
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
test("a hold without a beat keeps the old rules: it blocks inside the lease", async () => {
|
|
207
|
+
const { clones: [a, b] } = writers(2);
|
|
208
|
+
const t0 = Date.now();
|
|
209
|
+
await takeLockRows("pr-apply", [row], holder("19", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
|
|
210
|
+
const blocked = await takeLockRows("pr-apply", [row], holder("31"), { cwd: b, alive: liveness(() => t0 + 30 * MIN) });
|
|
211
|
+
expect(blocked.ok).toBe(false);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test("a wait that gives up names the hold and the command that frees it", async () => {
|
|
215
|
+
const { clones: [a, b] } = writers(2);
|
|
216
|
+
const t0 = Date.now();
|
|
217
|
+
await takeLockRows("pr-apply", [row], holder("19", new Date(t0)), { cwd: a, alive: liveness(() => t0) });
|
|
218
|
+
await beatLockRows("pr-apply", "19", new Date(t0).toISOString(), { cwd: a });
|
|
219
|
+
const got = await leaseLockRows("pr-apply", [row], {
|
|
220
|
+
cwd: b,
|
|
221
|
+
holder: holder("31"),
|
|
222
|
+
alive: liveness(() => t0),
|
|
223
|
+
wait: { deadline: t0, now: () => t0, sleep: async () => {} },
|
|
224
|
+
});
|
|
225
|
+
expect(got.ok).toBe(false);
|
|
226
|
+
if (!got.ok) {
|
|
227
|
+
expect(got.timedOut).toBe(true);
|
|
228
|
+
expect(got.message).toContain("run 19");
|
|
229
|
+
expect(got.message).toContain("last beat");
|
|
230
|
+
expect(got.message).toContain("chant unlock --run <id> --gate pr-apply");
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
describe("a hold cleared by hand", () => {
|
|
236
|
+
test("chant unlock --run frees the rows and the queue entry of a run, whatever its liveness", async () => {
|
|
237
|
+
const { clones: [a, b] } = writers(2);
|
|
238
|
+
await takeLockRows("pr-apply", [row], holder("26"), { cwd: a, alive: liveness(Date.now) });
|
|
239
|
+
const cleared = await clearRunLockRows("pr-apply", "26", { cwd: b });
|
|
240
|
+
expect(cleared.released).toEqual([row.key]);
|
|
241
|
+
const now = await readLockRows("pr-apply", { cwd: a });
|
|
242
|
+
expect(now.rows).toEqual({});
|
|
243
|
+
expect(now.holders).toEqual({});
|
|
244
|
+
// Nothing left to clear.
|
|
245
|
+
expect((await clearRunLockRows("pr-apply", "26", { cwd: b })).released).toEqual([]);
|
|
246
|
+
});
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
describe("the heartbeat loop", () => {
|
|
250
|
+
const base = { gate: "pr-apply", run: "26", parent: 1, intervalMs: 60_000, sleep: async () => {}, now: () => 0 };
|
|
251
|
+
|
|
252
|
+
test("beats until the hold is gone", async () => {
|
|
253
|
+
let n = 0;
|
|
254
|
+
const end = await runBeatLoop({ ...base, parentAlive: () => true, beat: async () => (++n < 3 ? "held" : "gone") });
|
|
255
|
+
expect(end).toBe("released");
|
|
256
|
+
expect(n).toBe(3);
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
test("ends with its parent process, and touches nothing after", async () => {
|
|
260
|
+
let n = 0;
|
|
261
|
+
let alive = true;
|
|
262
|
+
const end = await runBeatLoop({
|
|
263
|
+
...base,
|
|
264
|
+
parentAlive: () => alive,
|
|
265
|
+
beat: async () => {
|
|
266
|
+
if (++n === 2) alive = false;
|
|
267
|
+
return "held";
|
|
268
|
+
},
|
|
269
|
+
});
|
|
270
|
+
expect(end).toBe("parent-gone");
|
|
271
|
+
expect(n).toBe(2);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
test("a failed touch is tried again at the next interval", async () => {
|
|
275
|
+
let n = 0;
|
|
276
|
+
const lines: string[] = [];
|
|
277
|
+
const end = await runBeatLoop({
|
|
278
|
+
...base,
|
|
279
|
+
parentAlive: () => true,
|
|
280
|
+
log: (l) => lines.push(l),
|
|
281
|
+
beat: async () => {
|
|
282
|
+
if (++n === 1) throw new Error("push refused");
|
|
283
|
+
return "gone";
|
|
284
|
+
},
|
|
285
|
+
});
|
|
286
|
+
expect(end).toBe("released");
|
|
287
|
+
expect(lines[0]).toContain("push refused");
|
|
288
|
+
});
|
|
289
|
+
});
|
|
@@ -50,6 +50,7 @@
|
|
|
50
50
|
import { hostname } from "node:os";
|
|
51
51
|
import { getRuntime } from "../runtime-adapter";
|
|
52
52
|
import { pushLifecycleStatus, readBlobBySha, readPathSha, RefCASConflictError, writeBlobToPath } from "./git";
|
|
53
|
+
import { ownBeatStarter } from "./lock-beat";
|
|
53
54
|
import { lockRowsConflict, sortedLockRows, type LockRow } from "./lock-rows";
|
|
54
55
|
import { dropRefusedWrite, fetchForLocks, localTip, LOCKS_DIR, targetLockGateProblem } from "./target-lock";
|
|
55
56
|
|
|
@@ -59,6 +60,15 @@ export const LOCK_ROWS_DIR = `${LOCKS_DIR}/rows`;
|
|
|
59
60
|
/** A lease older than this many seconds is taken over (`CHANT_LOCK_STALE`). */
|
|
60
61
|
export const DEFAULT_LOCK_STALE_SECONDS = 7200;
|
|
61
62
|
|
|
63
|
+
/** A hold whose beat is older than this many seconds is gone (`CHANT_LOCK_BEAT_STALE`, #4660). */
|
|
64
|
+
export const DEFAULT_LOCK_BEAT_STALE_SECONDS = 600;
|
|
65
|
+
|
|
66
|
+
/** How often a live apply touches its hold, in seconds (`CHANT_LOCK_BEAT`, #4660). */
|
|
67
|
+
export const DEFAULT_LOCK_BEAT_SECONDS = 60;
|
|
68
|
+
|
|
69
|
+
/** A queued run that has not refreshed its entry for this many seconds is gone (`CHANT_LOCK_WAITER_STALE`, #4660). */
|
|
70
|
+
export const DEFAULT_LOCK_WAITER_STALE_SECONDS = 300;
|
|
71
|
+
|
|
62
72
|
const STATE_BRANCH = "chant/lifecycle";
|
|
63
73
|
const ATTEMPTS = 30;
|
|
64
74
|
|
|
@@ -69,6 +79,15 @@ export interface LockRowHolder {
|
|
|
69
79
|
/** The run's page, for a refusal that names it. */
|
|
70
80
|
url?: string;
|
|
71
81
|
sha?: string;
|
|
82
|
+
/**
|
|
83
|
+
* The liveness token (#4660): the last time a heartbeat of the run's own
|
|
84
|
+
* process touched its hold. A holder with a beat is judged by the beat
|
|
85
|
+
* alone, so a job that is cancelled or killed without running its cleanup
|
|
86
|
+
* frees its rows once the beat is {@link lockBeatStaleSeconds} old, while a
|
|
87
|
+
* job that still beats keeps them however long its apply runs. A holder
|
|
88
|
+
* without one (an older chant, terragucci) keeps the rules below.
|
|
89
|
+
*/
|
|
90
|
+
beat?: string;
|
|
72
91
|
}
|
|
73
92
|
|
|
74
93
|
/** A run waiting for rows: who it is, the rows it waits for, and the commit time that orders it. */
|
|
@@ -114,6 +133,26 @@ export function lockStaleSeconds(env: Record<string, string | undefined>): numbe
|
|
|
114
133
|
return DEFAULT_LOCK_STALE_SECONDS;
|
|
115
134
|
}
|
|
116
135
|
|
|
136
|
+
function positiveSeconds(env: Record<string, string | undefined>, name: string, fallback: number): number {
|
|
137
|
+
const n = Number(env[name]);
|
|
138
|
+
return env[name] !== undefined && Number.isFinite(n) && n > 0 ? n : fallback;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** How old a hold's beat may be before the hold is gone, in seconds: `CHANT_LOCK_BEAT_STALE`, else {@link DEFAULT_LOCK_BEAT_STALE_SECONDS}. */
|
|
142
|
+
export function lockBeatStaleSeconds(env: Record<string, string | undefined>): number {
|
|
143
|
+
return positiveSeconds(env, "CHANT_LOCK_BEAT_STALE", DEFAULT_LOCK_BEAT_STALE_SECONDS);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** Seconds between beats: `CHANT_LOCK_BEAT`, else {@link DEFAULT_LOCK_BEAT_SECONDS}. */
|
|
147
|
+
export function lockBeatSeconds(env: Record<string, string | undefined>): number {
|
|
148
|
+
return positiveSeconds(env, "CHANT_LOCK_BEAT", DEFAULT_LOCK_BEAT_SECONDS);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** How old a queued run's entry may be before it is gone, in seconds: `CHANT_LOCK_WAITER_STALE`, else {@link DEFAULT_LOCK_WAITER_STALE_SECONDS}. */
|
|
152
|
+
export function lockWaiterStaleSeconds(env: Record<string, string | undefined>): number {
|
|
153
|
+
return positiveSeconds(env, "CHANT_LOCK_WAITER_STALE", DEFAULT_LOCK_WAITER_STALE_SECONDS);
|
|
154
|
+
}
|
|
155
|
+
|
|
117
156
|
/** Who this apply is, as the rows name it: the forge's run, or this host and process outside CI. */
|
|
118
157
|
export function lockRowHolder(env: Record<string, string | undefined>, now: Date = new Date()): LockRowHolder {
|
|
119
158
|
const gitlab = Boolean(env.CI_PIPELINE_ID);
|
|
@@ -137,7 +176,13 @@ type Fetch = (url: string, init?: { headers?: Record<string, string> }) => Promi
|
|
|
137
176
|
*/
|
|
138
177
|
export function forgeLockLiveness(env: Record<string, string | undefined>, fetch: Fetch = globalThis.fetch as unknown as Fetch, now: () => number = Date.now): LockLiveness {
|
|
139
178
|
const stale = lockStaleSeconds(env);
|
|
179
|
+
const beatStale = lockBeatStaleSeconds(env);
|
|
140
180
|
return async (h) => {
|
|
181
|
+
// #4660: a hold that carries a beat is judged by it alone. The forge may call a cancelled run finished while its process still applies, and an apply may outlast the lease; the beat says whether the process is there.
|
|
182
|
+
if (h.beat !== undefined) {
|
|
183
|
+
const beat = new Date(h.beat).getTime();
|
|
184
|
+
if (Number.isFinite(beat)) return now() - beat >= beatStale * 1000 ? "dead" : "alive";
|
|
185
|
+
}
|
|
141
186
|
const since = new Date(h.at).getTime();
|
|
142
187
|
if (Number.isFinite(since) && now() - since >= stale * 1000) return "dead";
|
|
143
188
|
const local = /^local:(.*):(\d+)$/.exec(h.run);
|
|
@@ -180,7 +225,7 @@ export function parseLockRows(text: string | null): LockRowsFile {
|
|
|
180
225
|
const holders: Record<string, LockRowHolder> = {};
|
|
181
226
|
for (const [id, raw] of Object.entries(doc.holders as Record<string, Record<string, unknown>>)) {
|
|
182
227
|
if (typeof raw?.run !== "string" || typeof raw.at !== "string") continue;
|
|
183
|
-
holders[id] = { run: raw.run, at: raw.at, ...(typeof raw.url === "string" ? { url: raw.url } : {}), ...(typeof raw.sha === "string" ? { sha: raw.sha } : {}) };
|
|
228
|
+
holders[id] = { run: raw.run, at: raw.at, ...(typeof raw.url === "string" ? { url: raw.url } : {}), ...(typeof raw.sha === "string" ? { sha: raw.sha } : {}), ...(typeof raw.beat === "string" ? { beat: raw.beat } : {}) };
|
|
184
229
|
}
|
|
185
230
|
const rows: Record<string, string> = {};
|
|
186
231
|
for (const [k, id] of Object.entries(doc.rows as Record<string, unknown>)) if (typeof id === "string" && holders[id]) rows[k] = id;
|
|
@@ -434,13 +479,16 @@ export async function commitTimeOf(rev: string, cwd?: string): Promise<string |
|
|
|
434
479
|
}
|
|
435
480
|
|
|
436
481
|
/** The live waiters for a row that meets one of `keys` who go before `me`, and the waiters that are gone. */
|
|
437
|
-
export async function waitersAhead(file: LockRowsFile, keys: readonly string[], me: LockRowWaiter, alive: LockLiveness): Promise<{ ahead: LockRowWaiter[]; gone: string[] }> {
|
|
482
|
+
export async function waitersAhead(file: LockRowsFile, keys: readonly string[], me: LockRowWaiter, alive: LockLiveness, now: () => number = Date.now): Promise<{ ahead: LockRowWaiter[]; gone: string[] }> {
|
|
438
483
|
const ahead: LockRowWaiter[] = [];
|
|
439
484
|
const gone: string[] = [];
|
|
485
|
+
const waiterStale = lockWaiterStaleSeconds(process.env) * 1000;
|
|
440
486
|
for (const [id, w] of Object.entries(file.waiting ?? {})) {
|
|
441
487
|
if (id === me.run || w.run === me.run) continue;
|
|
442
488
|
if (!w.keys.some((k) => keys.some((mine) => lockRowsConflict(k, mine)))) continue;
|
|
443
|
-
|
|
489
|
+
// #4660: a waiter refreshes its entry every few poll rounds, so one that has not is gone (cancelled or killed), whatever the forge says.
|
|
490
|
+
const entered = new Date(w.at).getTime();
|
|
491
|
+
const verdict = Number.isFinite(entered) && now() - entered >= waiterStale ? "dead" : await alive(w);
|
|
444
492
|
if (verdict === "dead") {
|
|
445
493
|
gone.push(id);
|
|
446
494
|
continue;
|
|
@@ -450,6 +498,13 @@ export async function waitersAhead(file: LockRowsFile, keys: readonly string[],
|
|
|
450
498
|
return { ahead: ahead.sort((a, b) => (waiterBefore(a, b) ? -1 : 1)), gone: gone.sort() };
|
|
451
499
|
}
|
|
452
500
|
|
|
501
|
+
/** When each live holder took its rows and last beat, for a wait that gave up (#4660). */
|
|
502
|
+
function holdAges(held: readonly HeldLockRow[]): string {
|
|
503
|
+
const runs = [...new Map(held.map((h) => [h.holder.run, h.holder])).values()];
|
|
504
|
+
if (runs.length === 0) return "";
|
|
505
|
+
return ` (${runs.map((h) => `run ${h.run} took its rows at ${h.at}${h.beat ? `, last beat ${h.beat}` : ", no beat"}`).join("; ")})`;
|
|
506
|
+
}
|
|
507
|
+
|
|
453
508
|
function describeWait(held: readonly HeldLockRow[], ahead: readonly LockRowWaiter[], rows: readonly LockRow[]): string {
|
|
454
509
|
const parts: string[] = [];
|
|
455
510
|
if (held.length > 0) parts.push(describeHeldLockRows(held, rows));
|
|
@@ -481,7 +536,7 @@ export async function waitLockRows(gate: string, rows: readonly LockRow[], me: L
|
|
|
481
536
|
const { sha, file } = await readCurrent(gate, opts.cwd);
|
|
482
537
|
const mine = queued ?? waiter();
|
|
483
538
|
const { live, gone } = await heldLockRowsIn(file, keys, me.run, alive);
|
|
484
|
-
const { ahead, gone: goneWaiters } = await waitersAhead(file, keys, mine, alive);
|
|
539
|
+
const { ahead, gone: goneWaiters } = await waitersAhead(file, keys, mine, alive, now);
|
|
485
540
|
if (live.length === 0 && ahead.length === 0) {
|
|
486
541
|
const next = withRowsTaken(file, keys, me, gone);
|
|
487
542
|
for (const id of goneWaiters) delete next.waiting?.[id];
|
|
@@ -496,10 +551,10 @@ export async function waitLockRows(gate: string, rows: readonly LockRow[], me: L
|
|
|
496
551
|
if (now() >= wait.deadline) {
|
|
497
552
|
await leaveQueue(gate, me.run, opts.cwd);
|
|
498
553
|
const waited = Math.round((now() - started) / 1000);
|
|
499
|
-
throw new LockWaitTimeoutError(`waited ${waited}s for ${keys.join(", ")} of ${gate} and gave up at the job's deadline: ${why}`, live, ahead, waited);
|
|
554
|
+
throw new LockWaitTimeoutError(`waited ${waited}s for ${keys.join(", ")} of ${gate} and gave up at the job's deadline: ${why}${holdAges(live)}. If a run named here is gone, chant unlock --run <id> --gate ${gate} frees its rows`, live, ahead, waited);
|
|
500
555
|
}
|
|
501
556
|
const inFile = file.waiting?.[me.run];
|
|
502
|
-
const refresh = inFile ? now() - new Date(inFile.at).getTime() > stale / 2 : true;
|
|
557
|
+
const refresh = inFile ? now() - new Date(inFile.at).getTime() > Math.min(stale / 2, (lockWaiterStaleSeconds(process.env) * 1000) / 3) : true;
|
|
503
558
|
if (!inFile || refresh || goneWaiters.length > 0) {
|
|
504
559
|
const entry: LockRowWaiter = { ...waiter(), at: new Date(now()).toISOString() };
|
|
505
560
|
const wrote = await writeAndPush(gate, sha, withWaiter(file, entry, goneWaiters), `Run ${me.run} waits for ${keys.length} row${keys.length === 1 ? "" : "s"} of ${gate}`, opts.cwd);
|
|
@@ -556,6 +611,63 @@ export async function releaseLockRows(gate: string, keys: readonly string[], me:
|
|
|
556
611
|
return false;
|
|
557
612
|
}
|
|
558
613
|
|
|
614
|
+
/** The rows file after run `run`'s beat is set to `at`. Unchanged when the run holds no row. */
|
|
615
|
+
export function withRowsBeat(file: LockRowsFile, run: string, at: string): LockRowsFile {
|
|
616
|
+
const holder = file.holders[run];
|
|
617
|
+
if (!holder || !Object.values(file.rows).includes(run)) return file;
|
|
618
|
+
const next = copyRows(file);
|
|
619
|
+
next.holders[run] = { ...holder, beat: at };
|
|
620
|
+
return next;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Touch run `run`'s hold (#4660): the heartbeat of a live apply. `held` while
|
|
625
|
+
* the run still holds a row, `gone` once it holds none (released, or taken
|
|
626
|
+
* over after its beat went stale), so the heartbeat stops.
|
|
627
|
+
*/
|
|
628
|
+
export async function beatLockRows(gate: string, run: string, at: string, opts: { cwd?: string } = {}): Promise<"held" | "gone"> {
|
|
629
|
+
checkGate(gate);
|
|
630
|
+
await fetchForLocks(opts.cwd);
|
|
631
|
+
for (let attempt = 0; attempt < ATTEMPTS; attempt++) {
|
|
632
|
+
const { sha, file } = await readCurrent(gate, opts.cwd);
|
|
633
|
+
if (!Object.values(file.rows).includes(run)) return "gone";
|
|
634
|
+
const wrote = await writeAndPush(gate, sha, withRowsBeat(file, run, at), `Run ${run} still holds its rows of ${gate}`, opts.cwd);
|
|
635
|
+
if (wrote) return "held";
|
|
636
|
+
await pause(200 + Math.floor(Math.random() * 800));
|
|
637
|
+
}
|
|
638
|
+
throw new Error(`could not touch the hold of run ${run} on ${gate}: ${STATE_BRANCH} kept moving`);
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/** The rows file after every hold and queue entry of `run` is dropped, whether or not the run is gone. */
|
|
642
|
+
export function withRunCleared(file: LockRowsFile, run: string): LockRowsFile {
|
|
643
|
+
const next = copyRows(file);
|
|
644
|
+
delete next.holders[run];
|
|
645
|
+
for (const [k, id] of Object.entries(next.rows)) if (id === run) delete next.rows[k];
|
|
646
|
+
delete next.waiting?.[run];
|
|
647
|
+
return tidyWaiting(next);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Let go of everything run `run` holds or waits for on `gate`, by hand
|
|
652
|
+
* (#4660, `chant unlock --run`). It does not ask whether the run is alive:
|
|
653
|
+
* the person who runs it says so. Throws when the branch cannot be read or
|
|
654
|
+
* pushed, so a hold that is still there is never reported as cleared.
|
|
655
|
+
*/
|
|
656
|
+
export async function clearRunLockRows(gate: string, run: string, opts: { cwd?: string } = {}): Promise<{ released: string[]; waited: boolean; pushed: boolean }> {
|
|
657
|
+
checkGate(gate);
|
|
658
|
+
await fetchForLocks(opts.cwd);
|
|
659
|
+
for (let attempt = 0; attempt < ATTEMPTS; attempt++) {
|
|
660
|
+
const { sha, file } = await readCurrent(gate, opts.cwd);
|
|
661
|
+
const released = Object.entries(file.rows).filter(([, id]) => id === run).map(([k]) => k).sort();
|
|
662
|
+
const waited = file.waiting?.[run] !== undefined;
|
|
663
|
+
if (released.length === 0 && !file.holders[run] && !waited) return { released, waited, pushed: false };
|
|
664
|
+
const wrote = await writeAndPush(gate, sha, withRunCleared(file, run), `Clear the hold of run ${run} on ${gate}`, opts.cwd);
|
|
665
|
+
if (wrote) return { released, waited, pushed: wrote.pushed };
|
|
666
|
+
await pause(200 + Math.floor(Math.random() * 800));
|
|
667
|
+
}
|
|
668
|
+
throw new Error(`could not clear run ${run} from ${gate}: ${STATE_BRANCH} kept moving; check that the job may push to it`);
|
|
669
|
+
}
|
|
670
|
+
|
|
559
671
|
/** One line naming the runs that hold rows, and what of this apply they hold, for a log line and a refusal. */
|
|
560
672
|
export function describeHeldLockRows(held: readonly HeldLockRow[], rows: readonly LockRow[]): string {
|
|
561
673
|
const address = new Map(rows.map((r) => [r.key, `${r.target}: ${r.address}`]));
|
|
@@ -578,6 +690,9 @@ export interface LockRowsLease {
|
|
|
578
690
|
release(): Promise<boolean>;
|
|
579
691
|
}
|
|
580
692
|
|
|
693
|
+
/** Starts the heartbeat of a hold (#4660) and returns how to stop it. */
|
|
694
|
+
export type LockBeatStarter = (hold: { gate: string; holder: LockRowHolder; cwd?: string }) => () => void;
|
|
695
|
+
|
|
581
696
|
/**
|
|
582
697
|
* Lease `rows` for the apply about to run (#4552): take them, or say who
|
|
583
698
|
* holds them. With no rows, a lease that holds nothing. Throws when the
|
|
@@ -587,15 +702,34 @@ export interface LockRowsLease {
|
|
|
587
702
|
export async function leaseLockRows(
|
|
588
703
|
gate: string,
|
|
589
704
|
rows: readonly LockRow[],
|
|
590
|
-
opts: LockRowsOptions & { holder?: LockRowHolder; wait?: LockRowsWait } = {},
|
|
705
|
+
opts: LockRowsOptions & { holder?: LockRowHolder; wait?: LockRowsWait; beat?: LockBeatStarter } = {},
|
|
591
706
|
): Promise<{ ok: true; lease: LockRowsLease; tookOver: LockRowHolder[]; waitedSeconds?: number; waited?: boolean } | { ok: false; held: HeldLockRow[]; message: string; timedOut?: true }> {
|
|
592
707
|
const holder = opts.holder ?? lockRowHolder(process.env);
|
|
593
708
|
const keys = sortedLockRows(rows).map((r) => r.key);
|
|
594
|
-
|
|
709
|
+
let stopBeat: (() => void) | undefined;
|
|
710
|
+
const lease: LockRowsLease = {
|
|
711
|
+
gate,
|
|
712
|
+
holder,
|
|
713
|
+
keys,
|
|
714
|
+
release: () => {
|
|
715
|
+
// The beat stops first, so it cannot touch a hold that is being let go.
|
|
716
|
+
stopBeat?.();
|
|
717
|
+
stopBeat = undefined;
|
|
718
|
+
return keys.length === 0 ? Promise.resolve(true) : releaseLockRows(gate, keys, holder.run, { ...(opts.cwd ? { cwd: opts.cwd } : {}) });
|
|
719
|
+
},
|
|
720
|
+
};
|
|
595
721
|
if (keys.length === 0) return { ok: true, lease, tookOver: [] };
|
|
722
|
+
const startBeat = () => {
|
|
723
|
+
try {
|
|
724
|
+
stopBeat = (opts.beat ?? ownBeatStarter())?.({ gate, holder, ...(opts.cwd ? { cwd: opts.cwd } : {}) });
|
|
725
|
+
} catch {
|
|
726
|
+
// No beat: the hold keeps the forge and lease rules.
|
|
727
|
+
}
|
|
728
|
+
};
|
|
596
729
|
if (opts.wait) {
|
|
597
730
|
try {
|
|
598
731
|
const waited = await waitLockRows(gate, rows, holder, opts.wait, opts);
|
|
732
|
+
startBeat();
|
|
599
733
|
return { ok: true, lease, tookOver: waited.tookOver, ...(waited.waited ? { waited: true, waitedSeconds: waited.waitedSeconds } : {}) };
|
|
600
734
|
} catch (err) {
|
|
601
735
|
if (err instanceof LockWaitTimeoutError) return { ok: false, held: err.held, message: err.message, timedOut: true };
|
|
@@ -604,5 +738,6 @@ export async function leaseLockRows(
|
|
|
604
738
|
}
|
|
605
739
|
const taken = await takeLockRows(gate, rows, holder, opts);
|
|
606
740
|
if (!taken.ok) return { ok: false, held: taken.held, message: describeHeldLockRows(taken.held, rows) };
|
|
741
|
+
startBeat();
|
|
607
742
|
return { ok: true, lease, tookOver: taken.tookOver };
|
|
608
743
|
}
|
package/src/op/index.ts
CHANGED
|
@@ -114,6 +114,17 @@ export function opWaveNotifyFields(
|
|
|
114
114
|
reason: result.outcome.reason ?? `another run${holder ? `, ${holder.run},` : ""} holds what the apply changes; apply again once it is done`,
|
|
115
115
|
};
|
|
116
116
|
}
|
|
117
|
+
if (result.exitCode === CHANT_EXIT.moved && decision?.refused?.reason === "moved") {
|
|
118
|
+
// The plans moved since the pull request previewed them (`approvals.preview`), or were never previewed.
|
|
119
|
+
return {
|
|
120
|
+
...base,
|
|
121
|
+
event: "refused",
|
|
122
|
+
status: "moved",
|
|
123
|
+
summary: `${label} refused: ${decision.preview?.status === "unrecorded" ? "its plans were not previewed" : "its plans moved since the pull request previewed them"}`,
|
|
124
|
+
targets: decision.refused.runs,
|
|
125
|
+
reason: decision.refused.message ?? "the plans are not the ones the pull request previewed, so nothing was applied",
|
|
126
|
+
};
|
|
127
|
+
}
|
|
117
128
|
if (result.exitCode === CHANT_EXIT.moved) {
|
|
118
129
|
return {
|
|
119
130
|
...base,
|