clearcote 0.40.1 → 0.41.1

package/dist/download.js

@@ -11,14 +11,17 @@
 //                            fingerprint. Lets you stay current without bumping the SDK. Falls back
 //                            to the pinned release if GitHub is unreachable.
 // A hash mismatch (either mode) is always a hard failure and the partial download is deleted.
-import { execFileSync, spawnSync } from "node:child_process";
-import { createHash } from "node:crypto";
-import { chmodSync, closeSync, createReadStream, createWriteStream, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { execFile, spawnSync } from "node:child_process";
+import { createHash, randomBytes } from "node:crypto";
+import { chmodSync, closeSync, createReadStream, createWriteStream, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, renameSync, rmSync, statSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs";
 import { rm } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
+import { setTimeout as sleep } from "node:timers/promises";
+import { promisify } from "node:util";
+import { Worker } from "node:worker_threads";
 import extract from "extract-zip";
 import { RELEASE, REPO, SIGNING_KEY_FPR, platformRelease, CATALOG_URL, CATALOG_FALLBACK } from "./release.js";
 /**
@@ -336,11 +339,26 @@ export function writeManifest(base, browserDir) {
         }
     };
     walk(browserDir);
+    // Written to a temporary file and renamed into place, so no reader ever sees half a manifest.
+    const final = path.join(base, MANIFEST);
+    const tmp = `${final}.tmp-${randomBytes(4).toString("hex")}`;
     try {
-        writeFileSync(path.join(base, MANIFEST), JSON.stringify({ files }));
+        writeFileSync(tmp, JSON.stringify({ files }));
+        for (let attempt = 0;; attempt++) {
+            try {
+                renameSync(tmp, final);
+                return;
+            }
+            catch (e) {
+                // Windows: a reader has the old one open this instant
+                if (process.platform !== "win32" || attempt >= 9 || errCode(e) !== "EPERM")
+                    throw e;
+                Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50);
+            }
+        }
     }
     catch {
-        /* a manifest is an optimisation, never a reason to fail an install */
+        rmSync(tmp, { force: true }); // a manifest is an optimisation, never a reason to fail an install
     }
 }
 /** Return `"<file> — <problem>"` for every missing / truncated entry. */
@@ -437,27 +455,563 @@ export function checkInstall(exe) {
  * A damaged tree returns null (after wiping it) so the caller re-downloads: the `.verified` marker
  * only records that the archive hashed correctly AT INSTALL TIME, and files can be eaten later.
  */
-export function cachedBinary(base, binary, quiet) {
+export function cachedBinary(base, binary, quiet, repair = true, full) {
     if (!existsSync(path.join(base, ".verified")))
         return null;
     const browserDir = path.join(base, "browser");
     const cached = findFile(browserDir, binary);
-    if (!cached)
-        return null;
-    const problems = verifyInstall(browserDir, base);
-    if (!problems.length)
+    // The binary itself can be gone too: antivirus quarantine, or browser/ deleted by hand.
+    const problems = cached ? verifyInstall(browserDir, base, full) : [`${binary} — missing`];
+    if (cached && !problems.length)
         return cached;
+    // `repair` also moves the damaged tree out of the way; false when not holding the install lock, so
+    // only the process that will re-install touches it.
+    if (!repair)
+        return null;
     log(quiet, `cached browser is damaged (${problems[0]}) — re-downloading`);
-    rmSync(browserDir, { recursive: true, force: true });
+    // Moved aside first: if a running browser still holds its files, this stops here with "close it", with the
+    // tree still marked, instead of leaving an unmarked tree that no later install can move.
+    moveAside(base, browserDir);
     for (const marker of [".verified", MANIFEST])
         rmSync(path.join(base, marker), { force: true });
     return null;
 }
-/** Download + verify a resolved release into `base`, returning the extracted browser-binary path. */
+// Install lock. One protocol, the same in Python (download.py), Node (download.ts) and .NET
+// (Download.cs) -- change all three together. Any mix of processes installing one build into
+// <cache>/<tag> takes turns:
+//  1. The lock is the file <tag>/.install-lock, created with O_CREAT|O_EXCL (atomic on every OS and in
+//     all three languages; Node has no flock). It holds JSON: pid, start, host, boot, pidns, nonce, sdk,
+//     created. "start" is the process start time where the SDK can read it ("linux:" from /proc; "win:"
+//     the creation time, Python and .NET), else null; "created" is when the lock was written (ms, Unix).
+//  2. The holder touches the file (mtime) every 5 s while it works, from a thread of its own.
+//  3. Waiters poll every 0.25 s. A record from this machine (same host, boot id and PID namespace) is stale
+//     when its pid is dead, when that pid now belongs to a newer process (another start time; on Windows
+//     created after the lock), or -- only without a boot id (Windows, macOS) -- when it was written before
+//     the machine last started. A pid confirmed to be the holder is never stale, however old its heartbeat.
+//     When that cannot be confirmed, and for any other lock: stale once it (mtime + content) has not changed
+//     for 120 s by the waiter's own clock -- 10 s when not valid JSON or its mtime is over 15 min old -- so
+//     clocks that disagree do not matter. A stale lock is broken while holding <tag>/.install-lock.break
+//     (also O_EXCL; a breaker unchanged for 30 s by the waiter's own clock is removed): if the lock is still
+//     exactly what was judged stale, delete it; then delete the breaker.
+//  4. The holder re-checks the cache and uses a verified build if one is there now; a tree marked
+//     .verified that fails the check (files or the binary gone) is moved aside by the holder and only then
+//     unmarked -- if its files are in use, the install stops at once ("close it and try again"). Otherwise
+//     it downloads and extracts into <tag>/.tmp-<nonce>/. Then, only while the lock is still its own: it
+//     uses a tree that appeared meanwhile and passes the full check, else moves browser/ aside
+//     (<tag>/.trash-*; after taking over a holder not confirmed dead, it waits within the install wait while
+//     that tree is still in use), moves its tree to <tag>/browser, checks the lock and writes .manifest.json
+//     (temporary file, then rename), checks the lock once more and only then writes .verified. A tree that
+//     passes the check is never moved or written over by an install. Leftovers (.tmp-*, .trash-*, a breaker
+//     unchanged throughout the install) are swept.
+//  5. Release deletes the lock only while it still holds our nonce. Waiting gives up after 30 min with an
+//     error that names the holder (pid, host) and the lock file.
+export const INSTALL_LOCK = ".install-lock";
+/** Install-lock timings in ms (see the protocol above). Tests may shorten them. */
+export const installLockTiming = {
+    pollMs: 250, heartbeatMs: 5_000, staleMs: 120_000, unreadableMs: 10_000, ancientMs: 15 * 60_000, breakerStaleMs: 30_000, waitMs: 30 * 60_000,
+    inUsePollMs: 1_000,
+};
+/** Test seam: called right before an install writes .verified. Left empty in production. */
+export const installTestHooks = {};
+const BOOT_MARGIN_MS = 5 * 60_000; // the clock can still be corrected in the first minutes after a start
+const errCode = (e) => e?.code;
+function bootId() {
+    try {
+        return readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim() || null;
+    }
+    catch {
+        return null;
+    }
+}
+function pidNamespace() {
+    try {
+        return readlinkSync("/proc/self/ns/pid");
+    }
+    catch {
+        return null;
+    }
+}
+/** When this machine last started (ms, Unix time), or null. */
+function bootTimeMs() {
+    try {
+        const line = readFileSync("/proc/stat", "utf8").split("\n").find((l) => l.startsWith("btime "));
+        if (line)
+            return Number(line.split(/\s+/)[1]) * 1000;
+    }
+    catch {
+        /* not Linux */
+    }
+    const up = os.uptime();
+    return up > 0 ? Date.now() - up * 1000 : null;
+}
+/** A marker that differs once `pid` belongs to another process: its start time from /proc (Linux). */
+function processStart(pid) {
+    try {
+        const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
+        const f = stat.slice(stat.lastIndexOf(")") + 2).split(/\s+/);
+        return f[19] ? `linux:${f[19]}` : null;
+    }
+    catch {
+        return null;
+    }
+}
+function pidAlive(pid) {
+    try {
+        process.kill(pid, 0); // signal 0: an existence check, also on Windows
+        return true;
+    }
+    catch (e) {
+        return errCode(e) === "EPERM"; // exists, but not ours to signal
+    }
+}
+function readLock(p) {
+    let mtimeMs;
+    let raw;
+    try {
+        mtimeMs = statSync(p).mtimeMs;
+        raw = readFileSync(p, "utf8");
+    }
+    catch (e) {
+        return { state: errCode(e) === "ENOENT" ? "gone" : "busy", rec: null, mtimeMs: 0, raw: null };
+    }
+    try {
+        const rec = JSON.parse(raw);
+        if (rec && typeof rec === "object" && !Array.isArray(rec))
+            return { state: "ok", rec: rec, mtimeMs, raw };
+    }
+    catch {
+        /* not (yet) valid JSON */
+    }
+    return { state: "invalid", rec: null, mtimeMs, raw };
+}
+/** What a waiter compares to tell whether the lock changed since it last looked. */
+const lockKey = (s) => `${s.state}\u0000${s.mtimeMs}\u0000${s.raw ?? ""}`;
+/** The record is from this machine (same host, boot id and PID namespace) and names a usable pid. */
+function fromHere(rec) {
+    const { pid, host } = rec;
+    return (typeof host === "string" && host.toLowerCase() === os.hostname().toLowerCase() &&
+        (rec.boot ?? null) === bootId() && (rec.pidns ?? null) === pidNamespace() &&
+        typeof pid === "number" && Number.isInteger(pid) && pid > 0 && pid < 2 ** 31);
+}
+/** For a record from this machine: "gone", "alive" (certainly still the holder) or "unknown". Node cannot
+ * read another process's start time on Windows, so there a live pid is always "unknown". */
+function ownerState(rec) {
+    const created = typeof rec.created === "number" ? rec.created : null;
+    const boot = bootTimeMs();
+    // written before this machine last started (a boot id, where there is one, tells that already)
+    if ((rec.boot ?? null) === null && created !== null && boot !== null && created < boot - BOOT_MARGIN_MS)
+        return "gone";
+    if (!pidAlive(rec.pid))
+        return "gone";
+    const now = processStart(rec.pid);
+    if (now?.startsWith("linux:")) {
+        if (typeof rec.start === "string" && rec.start.startsWith("linux:"))
+            return now === rec.start ? "alive" : "gone";
+        return "unknown";
+    }
+    return "unknown";
+}
+/** `unchangedForMs`: how long the lock has stayed exactly as it is, by this process's own clock. */
+function lockStale(rec, mtimeMs, unchangedForMs) {
+    if (rec && fromHere(rec)) {
+        const state = ownerState(rec);
+        if (state !== "unknown")
+            return state === "gone";
+    }
+    const quick = !rec || Date.now() - mtimeMs > installLockTiming.ancientMs;
+    return unchangedForMs >= (quick ? Math.min(installLockTiming.staleMs, installLockTiming.unreadableMs) : installLockTiming.staleMs);
+}
+function fileKey(p) {
+    try {
+        const st = statSync(p);
+        return `${st.mtimeMs}\u0000${st.size}`;
+    }
+    catch {
+        return null;
+    }
+}
+/** A breaker is held for milliseconds: one that stays exactly the same for the breaker window, by this
+ * process's own clock, was left by a process that died holding it. Its timestamp alone proves nothing. */
+function clearStaleBreaker(breaker, watch) {
+    const key = fileKey(breaker);
+    const now = performance.now();
+    if (key === null || key !== watch.key) {
+        watch.key = key;
+        watch.at = now;
+    }
+    else if (now - watch.at >= installLockTiming.breakerStaleMs) {
+        rmSync(breaker, { force: true });
+        watch.key = null;
+    }
+}
+/** Delete the lock if it is still exactly what was judged stale, under the breaker file. True when the lock is gone afterwards. */
+function breakStaleLock(p, judged, watch) {
+    const breaker = `${p}.break`;
+    try {
+        closeSync(openSync(breaker, "wx"));
+    }
+    catch {
+        try {
+            clearStaleBreaker(breaker, watch);
+        }
+        catch {
+            /* gone meanwhile */
+        }
+        return false;
+    }
+    try {
+        const s = readLock(p);
+        if (lockKey(s) === judged) {
+            unlinkSync(p);
+            return true;
+        }
+        return s.state === "gone";
+    }
+    catch {
+        return false;
+    }
+    finally {
+        rmSync(breaker, { force: true });
+    }
+}
+const SDK_NAMES = { python: "Python", node: "Node", dotnet: ".NET" };
+function lockHolder(rec) {
+    if (!rec || !rec.pid)
+        return "a process that left no details";
+    const sdk = SDK_NAMES[String(rec.sdk)] ?? String(rec.sdk ?? "unknown");
+    return `process ${String(rec.pid)} on ${typeof rec.host === "string" && rec.host ? rec.host : "an unknown machine"} (Clearcote ${sdk} SDK)`;
+}
+function waitWords(ms) {
+    const [n, unit] = ms < 120_000 ? [Math.max(1, Math.round(ms / 1000)), "second"] : [Math.round(ms / 60_000), "minute"];
+    return `${n} ${unit}${n === 1 ? "" : "s"}`;
+}
+// The heartbeat runs in a worker thread, so a long synchronous step on the main thread (reading the whole
+// tree on Windows, extraction, a large delete) cannot stop it.
+const HEARTBEAT_WORKER = `
+const { workerData } = require("node:worker_threads");
+const { readFileSync, utimesSync } = require("node:fs");
+setInterval(() => {
+  try {
+    if (JSON.parse(readFileSync(workerData.path, "utf8")).nonce !== workerData.nonce) return;
+    const now = new Date();
+    utimesSync(workerData.path, now, now);
+  } catch {
+    /* next beat */
+  }
+}, workerData.everyMs);
+`;
+function startHeartbeat(p, nonce, beat) {
+    let timer;
+    let stopped = false;
+    // A worker that cannot start, fails or exits early: beat from this thread's timer instead.
+    const fallback = () => {
+        if (stopped || timer)
+            return;
+        timer = setInterval(beat, installLockTiming.heartbeatMs);
+        timer.unref();
+    };
+    let worker;
+    try {
+        const source = installTestHooks.heartbeatWorkerSource ?? HEARTBEAT_WORKER;
+        worker = new Worker(source, { eval: true, workerData: { path: p, nonce, everyMs: installLockTiming.heartbeatMs } });
+        worker.on("error", fallback);
+        worker.on("exit", fallback);
+        worker.unref();
+    }
+    catch {
+        fallback();
+    }
+    return async () => {
+        stopped = true;
+        if (timer)
+            clearInterval(timer);
+        await worker?.terminate().catch(() => 0);
+    };
+}
+function holdLock(p, nonce, tookOver) {
+    let lost = false;
+    const held = () => {
+        if (lost)
+            return false;
+        const s = readLock(p);
+        if (s.state !== "busy" && s.rec?.nonce !== nonce)
+            lost = true;
+        return !lost;
+    };
+    const beat = () => {
+        if (!held())
+            return;
+        try {
+            const now = new Date();
+            utimesSync(p, now, now);
+        }
+        catch {
+            /* next beat */
+        }
+    };
+    const stopHeartbeat = startHeartbeat(p, nonce, beat);
+    return {
+        path: p,
+        nonce,
+        held,
+        beat,
+        tookOver,
+        async release() {
+            await stopHeartbeat();
+            for (let i = 0; i < 40; i++) { // another process may be reading the file this instant (Windows refuses the delete)
+                const s = readLock(p);
+                if (s.state === "gone" || (s.state !== "busy" && s.rec?.nonce !== nonce))
+                    return;
+                if (s.state === "ok") {
+                    try {
+                        unlinkSync(p);
+                        return;
+                    }
+                    catch (e) {
+                        if (errCode(e) === "ENOENT")
+                            return;
+                    }
+                }
+                await sleep(50);
+            }
+        },
+    };
+}
+/** Wait for, then take, the install lock of build directory `base` (see the protocol above). */
+export async function acquireInstallLock(base, opts = {}) {
+    mkdirSync(base, { recursive: true });
+    const p = path.join(base, INSTALL_LOCK);
+    const nonce = randomBytes(16).toString("hex");
+    const body = JSON.stringify({
+        pid: process.pid, start: processStart(process.pid), host: os.hostname(), boot: bootId(), pidns: pidNamespace(), nonce, sdk: "node", created: Date.now(),
+    });
+    const wait = opts.timeoutMs ?? installLockTiming.waitMs;
+    const deadline = performance.now() + wait;
+    let told = false;
+    let refused = 0;
+    let seen = ""; // the lock as last read, and since when (our clock) it has looked like that
+    let seenAt = 0;
+    const breakerWatch = { key: null, at: 0 };
+    let tookOver = false;
+    for (;;) {
+        let fd = null;
+        try {
+            fd = openSync(p, "wx");
+        }
+        catch (e) {
+            const c = errCode(e);
+            // Windows: the previous lock file is still being deleted
+            if (c !== "EEXIST" && !(process.platform === "win32" && (c === "EPERM" || c === "EACCES" || c === "EBUSY")))
+                throw e;
+            if (c !== "EEXIST")
+                refused++;
+        }
+        if (fd !== null) {
+            try {
+                writeSync(fd, body);
+            }
+            catch (e) {
+                closeSync(fd);
+                rmSync(p, { force: true });
+                throw e;
+            }
+            closeSync(fd);
+            return holdLock(p, nonce, tookOver);
+        }
+        tookOver = false;
+        const s = readLock(p);
+        if (s.state !== "gone")
+            refused = 0;
+        else if (refused > 40)
+            throw new Error(`Clearcote cannot create ${p}: permission denied`); // not a race: a real error
+        const now = performance.now();
+        const key = lockKey(s);
+        if (key !== seen) {
+            seen = key;
+            seenAt = now;
+        }
+        if ((s.state === "ok" || s.state === "invalid") && lockStale(s.rec, s.mtimeMs, now - seenAt) && breakStaleLock(p, key, breakerWatch)) {
+            log(opts.quiet ?? true, `removed an abandoned install lock (${lockHolder(s.rec)})`);
+            tookOver = !(s.rec && fromHere(s.rec) && ownerState(s.rec) === "gone");
+            continue;
+        }
+        if (now >= deadline) {
+            const here = s.state === "ok" && s.rec && fromHere(s.rec) ? ownerState(s.rec) : null;
+            const note = here === "alive" ? ", which is still running" : here === "unknown" ? "; that process number may now belong to another program" : "";
+            throw new Error(`Gave up after ${waitWords(wait)} waiting for another program to finish installing the Clearcote browser in\n    ${base}\n` +
+                `The other installer is ${lockHolder(s.rec)}${note}. If no Clearcote program is installing this build, delete this file and try again:\n    ${p}`);
+        }
+        if (!told && s.state === "ok") {
+            log(opts.quiet ?? true, `another program is installing this browser build (${lockHolder(s.rec)}); waiting for it`);
+            told = true;
+        }
+        await sleep(installLockTiming.pollMs);
+    }
+}
+/** Another process took the lock over, or changed the build meanwhile: check again under the lock. */
+class LockLost extends Error {
+}
+/** Synchronous sleep for the short Windows retries below. */
+const pause = (ms) => {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+};
+function renameDir(src, dst) {
+    for (let attempt = 0;; attempt++) {
+        try {
+            installTestHooks.beforeRename?.(src, dst);
+            renameSync(src, dst);
+            return;
+        }
+        catch (e) {
+            // Windows: a scanner can hold a file in the fresh tree for a moment
+            const c = errCode(e);
+            if (process.platform !== "win32" || attempt >= 9 || !(c === "EPERM" || c === "EACCES" || c === "EBUSY"))
+                throw e;
+            pause(200);
+        }
+    }
+}
+/** Move a tree that must be replaced out of the way, then delete it if nothing is using it. */
+/** A tree that must be moved aside is in use (Windows will not move a folder whose files are open). */
+class InUseError extends Error {
+}
+function moveAside(base, browserDir) {
+    const trash = path.join(base, `.trash-${randomBytes(6).toString("hex")}`);
+    try {
+        renameDir(browserDir, trash);
+    }
+    catch (e) {
+        if (errCode(e) === "ENOENT")
+            return;
+        throw new InUseError(`Clearcote cannot replace the browser files in\n    ${browserDir}\nbecause another program is still using them (${e.message}).\n` +
+            "Close every program that uses this Clearcote browser, then try again.");
+    }
+    try {
+        rmSync(trash, { recursive: true, force: true });
+    }
+    catch {
+        /* what is still in use stays until the next install sweeps it */
+    }
+}
+/**
+ * Move an unverified browser/ aside. After taking over a holder that was not confirmed dead (it was suspended),
+ * its half-installed tree can stay in use until it resumes and finds the lock gone: wait for that, within the
+ * install wait, instead of failing. In any other case an in-use tree fails at once.
+ */
+async function moveAsideWhenFree(base, browserDir, lock, quiet) {
+    const deadline = performance.now() + installLockTiming.waitMs;
+    let told = false;
+    for (;;) {
+        try {
+            moveAside(base, browserDir);
+            return;
+        }
+        catch (e) {
+            if (!(e instanceof InUseError) || !lock.tookOver || performance.now() >= deadline)
+                throw e;
+            if (!lock.held())
+                throw new LockLost();
+            if (!told) {
+                log(quiet, `waiting for the installer that lost the lock to let go of ${browserDir}`);
+                told = true;
+            }
+            await sleep(installLockTiming.inUsePollMs);
+        }
+    }
+}
+/**
+ * Remove what an install that stopped part-way left behind. Only called by the lock holder. Returns the
+ * breaker file's state now (and when, by this process's clock). Given its state at the start of the install,
+ * removes a breaker that has not changed since, once that is the breaker window or more: one left by a
+ * process killed while it broke a stale lock.
+ */
+function sweepLeftovers(base, asset, breakerSeen) {
+    const breaker = path.join(base, `${INSTALL_LOCK}.break`);
+    const seen = { key: fileKey(breaker), at: performance.now() };
+    let names = [];
+    try {
+        names = readdirSync(base);
+    }
+    catch {
+        return seen;
+    }
+    for (const name of names) {
+        if (name === ".incoming" || name.startsWith(".tmp-") || name.startsWith(".trash-") || name.startsWith(`${MANIFEST}.tmp-`)) {
+            try {
+                rmSync(path.join(base, name), { recursive: true, force: true });
+            }
+            catch {
+                /* best-effort */
+            }
+        }
+    }
+    if (breakerSeen && seen.key !== null && seen.key === breakerSeen.key && seen.at - breakerSeen.at >= installLockTiming.breakerStaleMs) {
+        try {
+            rmSync(breaker, { force: true });
+        }
+        catch {
+            /* best-effort */
+        }
+    }
+    if (asset) {
+        try {
+            rmSync(path.join(base, asset), { force: true }); // where earlier versions downloaded the archive
+        }
+        catch {
+            /* best-effort */
+        }
+    }
+    return seen;
+}
+/**
+ * Download + verify a resolved release into `base`, returning the extracted browser-binary path.
+ *
+ * Holds `base`'s install lock: a second process that missed the cache at the same moment waits, then uses
+ * the tree the first one finished instead of downloading over it.
+ */
 async function fetchAndVerify(rel, base, opts) {
+    const binary = rel.binary || "chrome.exe";
+    for (let attempt = 0; attempt < 3; attempt++) {
+        const lock = await acquireInstallLock(base, { quiet: opts.quiet ?? false });
+        try {
+            const cached = cachedBinary(base, binary, opts.quiet);
+            if (cached) {
+                log(opts.quiet, `installed by another process meanwhile: ${cached}`);
+                return cached;
+            }
+            try {
+                return await installLocked(rel, base, opts, lock);
+            }
+            catch (e) {
+                if (!(e instanceof LockLost) && lock.held())
+                    throw e;
+            }
+            log(opts.quiet, "another process took over or changed this install; checking again");
+        }
+        finally {
+            await lock.release();
+        }
+    }
+    throw new Error(`Clearcote could not install ${rel.tag}: other processes kept taking over the install in ${base}`);
+}
+/**
+ * The install itself; the caller holds the install lock. Everything is written under `base/.tmp-<nonce>`
+ * first, and the finished tree is moved into place only while the lock is still ours.
+ */
+async function installLocked(rel, base, opts, lock) {
+    const breakerSeen = sweepLeftovers(base, rel.asset);
+    const tmp = path.join(base, `.tmp-${lock.nonce}`);
+    mkdirSync(tmp, { recursive: true });
+    try {
+        return await installInto(rel, base, tmp, opts, lock, breakerSeen);
+    }
+    finally {
+        await rm(tmp, { recursive: true, force: true }).catch(() => { });
+    }
+}
+async function installInto(rel, base, tmp, opts, lock, breakerSeen) {
     const browserDir = path.join(base, "browser");
-    mkdirSync(base, { recursive: true });
-    const zipPath = path.join(base, rel.asset);
+    const zipPath = path.join(tmp, rel.asset);
     log(opts.quiet, `fetching Clearcote ${rel.version} (${rel.tag}${rel.unpinned ? ", latest" : ""}, ~${(rel.size / 1e6).toFixed(0)} MB)`);
     await downloadTo(rel.url, zipPath, rel.size, opts.quiet);
     log(opts.quiet, "verifying SHA-256");
@@ -470,7 +1024,7 @@ async function fetchAndVerify(rel, base, opts) {
     if (rel.unpinned && rel.ascUrl) {
         const sumsBody = await fetchText(`https://github.com/${REPO}/releases/download/${rel.tag}/SHA256SUMS.txt`).catch(() => "");
         if (sumsBody) {
-            const verdict = await gpgVerifyAsync(rel, sumsBody, base, opts.quiet);
+            const verdict = await gpgVerifyAsync(rel, sumsBody, tmp, opts.quiet);
             if (verdict === "failed") {
                 await rm(zipPath, { force: true });
                 throw new Error(`Clearcote ${rel.tag}: GPG signature verification FAILED against the pinned key ${SIGNING_KEY_FPR} — refusing to use it.`);
@@ -480,25 +1034,25 @@ async function fetchAndVerify(rel, base, opts) {
         }
     }
     log(opts.quiet, "extracting");
-    await rm(browserDir, { recursive: true, force: true });
-    // Extract to a sibling temp dir, then atomically move it into place so `browser/` only ever
+    // Extract next to the target, then move the finished tree into place so `browser/` only ever
     // appears once fully written (no partial tree a launch could race), and — on Windows — we can
     // pre-scan the finished tree before any launch (below).
-    const incoming = path.join(base, ".incoming");
-    await rm(incoming, { recursive: true, force: true });
+    const incoming = path.join(tmp, "browser");
     mkdirSync(incoming, { recursive: true });
     if (rel.asset.endsWith(".tar.xz") || rel.archive === "tar.xz") {
-        // Node has no stdlib xz; the system `tar` (always present on Linux) auto-detects .xz.
-        execFileSync("tar", ["-xf", zipPath, "-C", incoming]);
+        // Node has no stdlib xz; the system `tar` (always present on Linux) auto-detects .xz. Async, so the
+        // lock's heartbeat keeps running during a long extraction.
+        await promisify(execFile)("tar", ["-xf", zipPath, "-C", incoming]);
     }
     else {
         await extract(zipPath, { dir: incoming });
     }
-    renameSync(incoming, browserDir);
+    await rm(zipPath, { force: true }); // reclaim ~250 MB; keep only the extracted tree
     const binaryName = rel.binary || "chrome.exe";
-    const exe = findFile(browserDir, binaryName);
-    if (!exe)
+    const found = findFile(incoming, binaryName);
+    if (!found)
         throw new Error(`Clearcote archive verified but ${binaryName} was not found inside it.`);
+    let exe = found;
     if (rel.exeSha256) {
         const exeHash = await sha256File(exe);
         if (exeHash.toLowerCase() !== rel.exeSha256.toLowerCase()) {
@@ -525,16 +1079,42 @@ async function fetchAndVerify(rel, base, opts) {
             }
         }
     }
+    // Move the tree into place only while the lock is still ours, and never over a tree that passes the full
+    // check: one that appeared since the cache check (another installer finished it) may already be running,
+    // so use it. One marked verified that fails the check is damaged, and moved aside (we hold the lock).
+    lock.beat();
+    if (!lock.held())
+        throw new LockLost();
+    const done = cachedBinary(base, binaryName, opts.quiet, true, true);
+    if (done) {
+        log(opts.quiet, `installed by another process meanwhile: ${done}`);
+        return done;
+    }
+    if (existsSync(path.join(base, ".verified")))
+        throw new LockLost(); // marked verified meanwhile by another process: start over
+    rmSync(path.join(base, MANIFEST), { force: true });
+    if (existsSync(browserDir))
+        await moveAsideWhenFree(base, browserDir, lock, opts.quiet); // not verified: an install that stopped part-way
+    renameDir(incoming, browserDir);
+    exe = path.join(browserDir, path.relative(incoming, exe));
     if (process.platform === "win32") {
         // Pre-scan so real-time AV finishes with the freshly-extracted binaries before the first launch
         // — closes the chrome_elf.dll scan race that otherwise poisons the path (see warmFiles).
         warmFiles(browserDir);
     }
     // Record the finished tree BEFORE the .verified marker, so a launch never sees "verified" with
-    // no manifest to check it against.
+    // no manifest to check it against -- and only while the lock is still ours, so a late manifest never
+    // lands on a tree the new holder is putting in place.
+    if (!lock.held())
+        throw new LockLost();
     writeManifest(base, browserDir);
+    // Mark it verified only while the lock is still ours: a process that took the lock over meanwhile then
+    // finds an unverified tree it may replace, not a verified one it must leave alone.
+    installTestHooks.beforeVerified?.();
+    if (!lock.held())
+        throw new LockLost();
     writeFileSync(path.join(base, ".verified"), `${rel.sha256}\n`);
-    await rm(zipPath, { force: true }); // reclaim ~250 MB; keep only the extracted tree
+    sweepLeftovers(base, undefined, breakerSeen);
     log(opts.quiet, `ready: ${exe}`);
     return exe;
 }
@@ -678,7 +1258,7 @@ export async function ensureVersion(selector, opts = {}) {
     }
     const base = path.join(opts.cacheDir || defaultCacheRoot(), plan.rel.tag);
     {
-        const cached = cachedBinary(base, plan.rel.binary || "chrome", opts.quiet);
+        const cached = cachedBinary(base, plan.rel.binary || "chrome", opts.quiet, false);
         if (cached)
             return cached;
     }
@@ -757,7 +1337,7 @@ export async function proEnsureBinary(licenseKey, opts = {}) {
     };
     const base = path.join(opts.cacheDir || defaultCacheRoot(), rel.tag);
     {
-        const cached = cachedBinary(base, rel.binary || "chrome.exe", opts.quiet);
+        const cached = cachedBinary(base, rel.binary || "chrome.exe", opts.quiet, false);
         if (cached)
             return cached;
     }
@@ -785,7 +1365,7 @@ export async function ensureBinary(opts = {}) {
     }
     const base = path.join(cacheRoot, rel.tag);
     {
-        const cached = cachedBinary(base, rel.binary || "chrome.exe", opts.quiet);
+        const cached = cachedBinary(base, rel.binary || "chrome.exe", opts.quiet, false);
         if (cached)
             return cached;
     }