@jonmilner/local-ci 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/check-lock.d.ts +8 -2
- package/dist/check-lock.js +5 -3
- package/dist/lock.d.ts +11 -1
- package/dist/lock.js +102 -12
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -61,7 +61,7 @@ A repo without a `package.json` sets `LOCAL_CI_COMMAND`, for example on the hook
|
|
|
61
61
|
|
|
62
62
|
Logs, the record of passed commits and the release lock live in `~/.local/state/local-ci/<repo>.log`, `<repo>.passed` and `<repo>.release.lock` (`$XDG_STATE_HOME` if set). A lock whose process has died is taken over.
|
|
63
63
|
|
|
64
|
-
One check command runs at a time on the machine, across every repo, so overlapping checks can't starve each other's tests: `pre-push`, `status`, the `status` runs `renovate` starts, and `local-ci run` each take `check.lock` in the same directory around the command they run (not the status posts). Nothing else takes it: a bare `pnpm check` or `pnpm test` runs beside every other check, so start a full check by hand with `local-ci run -- pnpm check`. The lock only runs checks one at a time; it does not wait for memory pressure or swap to clear. A run that finds it held prints `local-ci: waiting for the check lock, held by pid <pid> (<repo> <short-sha>)` (`<repo> <command>` for `local-ci run`, with the command cut to 60 characters), again whenever another run takes it first, and polls every second until the holder exits or dies; Ctrl-C stops the wait. After the wait it skips a commit that passed meanwhile and re-applies the clean-tree rule. The lock goes when the check passes, fails or is killed by a signal. Ctrl-C, SIGTERM, SIGHUP or SIGQUIT (Ctrl-\\) to local-ci passes the signal to the check's whole process group (so a check script's children get it too), waits for the check to exit, then releases the lock and dies of the signal, so a check never outlives its lock. A lock whose holder died (SIGKILL, a crash) or that predates the last boot is taken over by the next run. A local-ci check nested inside a check (one that pushes, say) sees its ancestor holds the lock and runs without waiting. A check that finds a background job's lock (see below) runs beside the job instead of waiting and leaves that lock alone, printing `local-ci: running beside background job pid <pid> (<holder>), without the check lock`; it holds `check.lock.beside` meanwhile, so another check waits for it rather than overlapping, and signals still stop it and release that lock as above. Each check run appends its duration and outcome to the log when it ends, e.g. `[<end time>] abc1234: pnpm check passed in 71.2s` (or `failed in`, or `did not finish after` when the check is killed by a signal or can't start); a commit that already passed is skipped and logs nothing. Ctrl-C stops the check, which logs `did not finish after`.
|
|
64
|
+
One check command runs at a time on the machine, across every repo, so overlapping checks can't starve each other's tests: `pre-push`, `status`, the `status` runs `renovate` starts, and `local-ci run` each take `check.lock` in the same directory around the command they run (not the status posts). Nothing else takes it: a bare `pnpm check` or `pnpm test` runs beside every other check, so start a full check by hand with `local-ci run -- pnpm check`. The lock only runs checks one at a time; it does not wait for memory pressure or swap to clear. A run that finds it held prints `local-ci: waiting for the check lock, held by pid <pid> (<repo> <short-sha>)` (`<repo> <command>` for `local-ci run`, with the command cut to 60 characters), again whenever another run takes it first, and polls every second until the holder exits or dies; Ctrl-C stops the wait. Checks don't queue, so the next one to poll after a release goes first; background jobs wait in arrival order (see [below](#taking-the-check-lock-from-your-own-scripts)). After the wait it skips a commit that passed meanwhile and re-applies the clean-tree rule. The lock goes when the check passes, fails or is killed by a signal. Ctrl-C, SIGTERM, SIGHUP or SIGQUIT (Ctrl-\\) to local-ci passes the signal to the check's whole process group (so a check script's children get it too), waits for the check to exit, then releases the lock and dies of the signal, so a check never outlives its lock. A lock whose holder died (SIGKILL, a crash) or that predates the last boot is taken over by the next run. A local-ci check nested inside a check (one that pushes, say) sees its ancestor holds the lock and runs without waiting. A check that finds a background job's lock (see below) runs beside the job instead of waiting and leaves that lock alone, printing `local-ci: running beside background job pid <pid> (<holder>), without the check lock`; it holds `check.lock.beside` meanwhile, so another check waits for it rather than overlapping, and signals still stop it and release that lock as above. Each check run appends its duration and outcome to the log when it ends, e.g. `[<end time>] abc1234: pnpm check passed in 71.2s` (or `failed in`, or `did not finish after` when the check is killed by a signal or can't start); a commit that already passed is skipped and logs nothing. Ctrl-C stops the check, which logs `did not finish after`.
|
|
65
65
|
|
|
66
66
|
### Taking the check lock from your own scripts
|
|
67
67
|
|
|
@@ -75,6 +75,6 @@ await withCheckLock("my-repo browser", async () => {
|
|
|
75
75
|
});
|
|
76
76
|
```
|
|
77
77
|
|
|
78
|
-
`withCheckLock(holder, fn, options?)` waits while another holder has the lock, logging `local-ci: waiting for the check lock, held by pid <pid> (<holder>)` to stderr as the CLI does, runs `fn`, and releases the lock when `fn` returns or throws. `holder` is the label other waiters see. `lockCheck(holder, options?)` does the take and returns the release function, for callers that can't wrap their work in a function. `options.file` overrides the lock path and `options.onWait` replaces the wait message. The path is `check.lock` in the state directory above, so no repo config is needed. A call nested under an ancestor process that holds the lock runs without waiting and leaves the lock alone. Pass `options.background: true` for a long job run after a push, such as a browser suite: it still waits for a running check, but `local-ci` checks (and other non-background callers) run beside it instead of waiting, so a push never sits in its hook behind its run. Background jobs
|
|
78
|
+
`withCheckLock(holder, fn, options?)` waits while another holder has the lock, logging `local-ci: waiting for the check lock, held by pid <pid> (<holder>)` to stderr as the CLI does, runs `fn`, and releases the lock when `fn` returns or throws. `holder` is the label other waiters see. `lockCheck(holder, options?)` does the take and returns the release function, for callers that can't wrap their work in a function. `options.file` overrides the lock path and `options.onWait` replaces the wait message. The path is `check.lock` in the state directory above, so no repo config is needed. A call nested under an ancestor process that holds the lock runs without waiting and leaves the lock alone. Pass `options.background: true` for a long job run after a push, such as a browser suite: it still waits for a running check, but `local-ci` checks (and other non-background callers) run beside it instead of waiting, so a push never sits in its hook behind its run. Background jobs queue behind each other in arrival order: each waiter writes a ticket file to `check.lock.queue/` with its pid, arrival time and head start, and takes the lock only when no live ticket sorts before its own. `options.headStartMs` (milliseconds, default 0) moves a background job up by that much: it sorts by arrival minus head start, so it passes jobs that arrived less than that long before it, and no others. A ticket whose process died, that predates the last boot, or that can't be parsed is removed by the next waiter that finds it, so a killed waiter can't wedge the queue. A job held back only by a ticket, with the lock free, says so: `local-ci: waiting for the check lock, held by pid <pid> (queued ahead)`. Checks take no ticket and never wait behind queued background jobs; when the lock is free, whichever polls first takes it. Versions before the queue ignore tickets, so a waiter from such a version (another repo pinned to an older local-ci) can still take the lock out of turn. A check run beside a background job takes `check.lock.beside` instead, so checks still run one at a time: a second check started during the job waits for the first, and whoever takes `check.lock` after the background job ends (another background job or a check) waits for a check still running beside it. A background job waiting that way holds `check.lock` as a check would, without the `background` line, until the beside check ends, so checks arriving meanwhile wait in turn rather than run beside a job that has not started. A check nested in a beside check runs under its lock without waiting. A beside lock whose holder died is taken over, as `check.lock` is. Versions before the beside lock ignore it, so a check from such a version (another repo pinned to an older local-ci) can still run beside a newer beside check. The lock file marks such a lock with a third line, `background`; a file without it (as older versions wrote) is waited on as before. SIGINT, SIGTERM or SIGHUP releases the lock and ends the process, so a job that needs its own signal cleanup should run it before calling the release itself. The package has an `exports` map, so deep imports other than `./lock` no longer resolve.
|
|
79
79
|
|
|
80
80
|
Part of [jonmilner/config](https://github.com/jonmilner/config#readme).
|
package/dist/check-lock.d.ts
CHANGED
|
@@ -16,6 +16,11 @@ export interface LockCheckOptions {
|
|
|
16
16
|
* waits for a running check, but checks run beside it instead of waiting for it to finish.
|
|
17
17
|
*/
|
|
18
18
|
background?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* With `background`, how many milliseconds earlier than its arrival this job counts as having
|
|
21
|
+
* arrived when queueing behind other background jobs. Default 0. Ignored for a check.
|
|
22
|
+
*/
|
|
23
|
+
headStartMs?: number;
|
|
19
24
|
}
|
|
20
25
|
/** The machine-wide check lock's path, in `$XDG_STATE_HOME/local-ci` (default `~/.local/state`). */
|
|
21
26
|
export declare function checkLockPath(env?: NodeJS.ProcessEnv): string;
|
|
@@ -25,9 +30,10 @@ export declare function checkLockPath(env?: NodeJS.ProcessEnv): string;
|
|
|
25
30
|
* that holds the lock runs without waiting, and its release leaves the lock alone. A check (not
|
|
26
31
|
* `background`) that finds a background job holding it runs beside the job under a second lock,
|
|
27
32
|
* `<file>.beside`, instead: a second such check waits for it, and whoever takes the check lock
|
|
28
|
-
* next waits for it to clear.
|
|
33
|
+
* next waits for it to clear. Background jobs take the lock in arrival order, less any
|
|
34
|
+
* `headStartMs`; a check doesn't queue. SIGINT, SIGTERM, SIGHUP and SIGQUIT release the lock and end the
|
|
29
35
|
* process.
|
|
30
36
|
*/
|
|
31
|
-
export declare function lockCheck(holder: string, { file, onWait, background }?: LockCheckOptions): Promise<() => void>;
|
|
37
|
+
export declare function lockCheck(holder: string, { file, onWait, background, headStartMs }?: LockCheckOptions): Promise<() => void>;
|
|
32
38
|
/** Runs `fn` under the machine-wide check lock, so it queues behind `local-ci` checks and vice versa. */
|
|
33
39
|
export declare function withCheckLock<T>(holder: string, fn: () => Promise<T>, options?: LockCheckOptions): Promise<T>;
|
package/dist/check-lock.js
CHANGED
|
@@ -75,10 +75,11 @@ function besideLockPath(file) {
|
|
|
75
75
|
* that holds the lock runs without waiting, and its release leaves the lock alone. A check (not
|
|
76
76
|
* `background`) that finds a background job holding it runs beside the job under a second lock,
|
|
77
77
|
* `<file>.beside`, instead: a second such check waits for it, and whoever takes the check lock
|
|
78
|
-
* next waits for it to clear.
|
|
78
|
+
* next waits for it to clear. Background jobs take the lock in arrival order, less any
|
|
79
|
+
* `headStartMs`; a check doesn't queue. SIGINT, SIGTERM, SIGHUP and SIGQUIT release the lock and end the
|
|
79
80
|
* process.
|
|
80
81
|
*/
|
|
81
|
-
export async function lockCheck(holder, { file = checkLockPath(), onWait, background = false } = {}) {
|
|
82
|
+
export async function lockCheck(holder, { file = checkLockPath(), onWait, background = false, headStartMs = 0 } = {}) {
|
|
82
83
|
const sidecar = besideLockPath(file);
|
|
83
84
|
let mine;
|
|
84
85
|
// A check nested in a check (one that pushes, say) runs under its ancestor's lock rather than
|
|
@@ -98,9 +99,10 @@ export async function lockCheck(holder, { file = checkLockPath(), onWait, backgr
|
|
|
98
99
|
for (;;) {
|
|
99
100
|
let beside = false;
|
|
100
101
|
const skip = (held) => nested(held) || (beside = besides(held));
|
|
102
|
+
const queue = background ? { headStartMs } : undefined;
|
|
101
103
|
// Taken as a check's lock even for a background job, so checks wait for it rather than run
|
|
102
104
|
// beside it until it starts its run.
|
|
103
|
-
const unlock = await waitForLock(file, { ...options, skip });
|
|
105
|
+
const unlock = await waitForLock(file, { ...options, skip, queue });
|
|
104
106
|
if (unlock) {
|
|
105
107
|
const release = releaseOnSignal(unlock);
|
|
106
108
|
try {
|
package/dist/lock.d.ts
CHANGED
|
@@ -38,8 +38,18 @@ export interface WaitForLockOptions extends TakeLockOptions {
|
|
|
38
38
|
holder?: string;
|
|
39
39
|
/** A holder this caller need not wait behind: the wait ends, returning `undefined`, untaken. */
|
|
40
40
|
skip?: (held: LockHeld) => boolean;
|
|
41
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* Called when the wait starts, and again whenever another process takes the lock first. A queued
|
|
43
|
+
* waiter held back only by another waiter's ticket gets that waiter, with the holder `queued ahead`.
|
|
44
|
+
*/
|
|
42
45
|
onWait?: (held: LockHeld) => void;
|
|
46
|
+
/**
|
|
47
|
+
* Waits in line for the lock: this caller takes it only when no live waiter's arrival minus head
|
|
48
|
+
* start sorts before its own. Waiters without this option don't queue and aren't held back.
|
|
49
|
+
*/
|
|
50
|
+
queue?: {
|
|
51
|
+
headStartMs: number;
|
|
52
|
+
};
|
|
43
53
|
intervalMs?: number;
|
|
44
54
|
sleep?: (ms: number) => Promise<void>;
|
|
45
55
|
}
|
package/dist/lock.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { linkSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
|
|
2
3
|
import { uptime } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
3
5
|
import { defaultSleep } from "./github.js";
|
|
4
6
|
/** Whether a process with this pid is running. */
|
|
5
7
|
export function alive(pid) {
|
|
@@ -131,27 +133,115 @@ export function takeLock(file, what = "release", holder, options = {}) {
|
|
|
131
133
|
rmSync(mine, { force: true });
|
|
132
134
|
}
|
|
133
135
|
}
|
|
136
|
+
const queueDir = (file) => `${file}.queue`;
|
|
137
|
+
/** Earliest arrival minus head start first; arrival, then name, settle a tie the same for everyone. */
|
|
138
|
+
function ticketOrder(a, b) {
|
|
139
|
+
return (a.arrival - a.headStartMs - (b.arrival - b.headStartMs) ||
|
|
140
|
+
a.arrival - b.arrival ||
|
|
141
|
+
(a.name < b.name ? -1 : 1));
|
|
142
|
+
}
|
|
143
|
+
/** Joins the queue for `file`, arriving now. */
|
|
144
|
+
function takeTicket(file, headStartMs) {
|
|
145
|
+
if (!Number.isFinite(headStartMs) || headStartMs < 0) {
|
|
146
|
+
throw new RangeError(`headStartMs must be a finite number of 0 or more, got ${headStartMs}`);
|
|
147
|
+
}
|
|
148
|
+
const dir = queueDir(file);
|
|
149
|
+
mkdirSync(dir, { recursive: true });
|
|
150
|
+
const ticket = {
|
|
151
|
+
name: `${process.pid}-${randomUUID()}`,
|
|
152
|
+
pid: process.pid,
|
|
153
|
+
arrival: Date.now(),
|
|
154
|
+
headStartMs,
|
|
155
|
+
};
|
|
156
|
+
// Renaming publishes it whole; the temporary name is beside the queue so nobody lists it.
|
|
157
|
+
const unpublished = `${dir}.${ticket.name}`;
|
|
158
|
+
writeFileSync(unpublished, `${ticket.pid}\n${ticket.arrival}\n${ticket.headStartMs}`);
|
|
159
|
+
renameSync(unpublished, join(dir, ticket.name));
|
|
160
|
+
return ticket;
|
|
161
|
+
}
|
|
162
|
+
/** The ticket stored at `path`, or `undefined` for one that is gone or can't be read. */
|
|
163
|
+
function readTicket(path, name) {
|
|
164
|
+
let content;
|
|
165
|
+
try {
|
|
166
|
+
content = readFileSync(path, "utf8");
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
return undefined;
|
|
170
|
+
}
|
|
171
|
+
const [pid = NaN, arrival = NaN, headStartMs = NaN] = content
|
|
172
|
+
.split("\n")
|
|
173
|
+
.map((field) => (field === "" ? NaN : Number(field)));
|
|
174
|
+
return { name, pid, arrival, headStartMs };
|
|
175
|
+
}
|
|
176
|
+
/** The live ticket that goes next before `mine`, if any, removing dead and pre-boot ones it finds. */
|
|
177
|
+
function ticketAhead(file, mine) {
|
|
178
|
+
const dir = queueDir(file);
|
|
179
|
+
let names;
|
|
180
|
+
try {
|
|
181
|
+
names = readdirSync(dir);
|
|
182
|
+
}
|
|
183
|
+
catch (error) {
|
|
184
|
+
if (error.code === "ENOENT")
|
|
185
|
+
return undefined;
|
|
186
|
+
throw error;
|
|
187
|
+
}
|
|
188
|
+
let ahead;
|
|
189
|
+
for (const name of names) {
|
|
190
|
+
if (name === mine.name)
|
|
191
|
+
continue;
|
|
192
|
+
const path = join(dir, name);
|
|
193
|
+
const ticket = readTicket(path, name);
|
|
194
|
+
if (!ticket)
|
|
195
|
+
continue;
|
|
196
|
+
if (!alive(ticket.pid) ||
|
|
197
|
+
beforeBoot(path) ||
|
|
198
|
+
!Number.isFinite(ticket.arrival) ||
|
|
199
|
+
!Number.isFinite(ticket.headStartMs)) {
|
|
200
|
+
rmSync(path, { force: true });
|
|
201
|
+
}
|
|
202
|
+
else if (ticketOrder(ticket, ahead ?? mine) < 0) {
|
|
203
|
+
ahead = ticket;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return ahead;
|
|
207
|
+
}
|
|
134
208
|
/**
|
|
135
209
|
* Takes the lock, polling while a live process holds it, and returns its release, or `undefined`
|
|
136
210
|
* once it finds a holder `skip` accepts.
|
|
137
211
|
*/
|
|
138
212
|
export async function waitForLock(file, options) {
|
|
139
|
-
const { what, holder, skip, onWait, intervalMs = 1_000, sleep = defaultSleep } = options;
|
|
213
|
+
const { what, holder, skip, onWait, queue, intervalMs = 1_000, sleep = defaultSleep } = options;
|
|
214
|
+
const ticket = queue && takeTicket(file, queue.headStartMs);
|
|
140
215
|
let last;
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
216
|
+
try {
|
|
217
|
+
for (;;) {
|
|
218
|
+
const ahead = ticket && ticketAhead(file, ticket);
|
|
219
|
+
let held;
|
|
220
|
+
if (ahead) {
|
|
221
|
+
held = lockHolder(file, what);
|
|
222
|
+
}
|
|
223
|
+
else {
|
|
224
|
+
try {
|
|
225
|
+
return takeLock(file, what, holder, { background: options.background });
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
if (!(error instanceof LockHeld))
|
|
229
|
+
throw error;
|
|
230
|
+
held = error;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
if (held && skip?.(held))
|
|
149
234
|
return undefined;
|
|
150
|
-
const
|
|
235
|
+
const blocker = held ?? new LockHeld(what, file, ahead.pid, "queued ahead");
|
|
236
|
+
const seen = `${blocker.pid}\n${blocker.holder}`;
|
|
151
237
|
if (seen !== last)
|
|
152
|
-
onWait?.(
|
|
238
|
+
onWait?.(blocker);
|
|
153
239
|
last = seen;
|
|
154
240
|
await sleep(intervalMs);
|
|
155
241
|
}
|
|
156
242
|
}
|
|
243
|
+
finally {
|
|
244
|
+
if (ticket)
|
|
245
|
+
rmSync(join(queueDir(file), ticket.name), { force: true });
|
|
246
|
+
}
|
|
157
247
|
}
|