@north-light/crouter 0.3.193 → 0.3.194

package/scripts/install-runtime.mjs

@@ -2,6 +2,7 @@
 import { spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { chmod, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, readlink, rename, rm, symlink, writeFile } from 'node:fs/promises';
+import { hostname } from 'node:os';
 import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { ENTRY_NAMES, GENERATION_ID, MANIFEST_KEYS, PACKAGE_NAME, MANIFEST_SCHEMA, contained, isSealed, publishedMarkerPath, runtimeHome, sortInventory, validateManifestShape } from '../bin/runtime-generation-manifest.mjs';
@@ -11,9 +12,20 @@ const PACKAGE_ROOT = process.argv[2] ? resolve(process.argv[2]) : resolve(dirnam
 const RUNTIME_HOME = runtimeHome();
 const GENERATIONS = join(RUNTIME_HOME, 'generations');
 const LOCK = join(RUNTIME_HOME, 'install.lock');
+const LOCK_OWNER_FILE = 'owner.json';
 const RUNTIME_LOCK = 'runtime.lock.json';
 const KEEP_RECENT_GENERATIONS = 3;
 const FS_CONCURRENCY = 64;
+// How long an install.lock with no readable owner record is given the
+// benefit of the doubt (the owner is mid-write, or on an ancient version of
+// this script that never wrote one) before it is treated as abandoned.
+const STALE_LOCK_RECORD_THRESHOLD_MS = 15 * 60 * 1000;
+// How long a `.staging-*`/`.archive-*` temp dir sits before the sweep in
+// step 4 treats it as wreckage from an earlier killed install rather than
+// work in progress under a lock we don't hold (impossible while we hold the
+// lock, but a conservative margin costs nothing).
+const STALE_TEMP_DIR_THRESHOLD_MS = 60 * 60 * 1000;
+const LOCK_ACQUIRE_MAX_ATTEMPTS = 50;
 
 function run(command, args, cwd, capture = false, env = process.env) {
   return new Promise((resolveRun, reject) => {
@@ -235,14 +247,162 @@ async function pruneGenerations() {
   }
 }
 
-async function install() {
-  await mkdir(GENERATIONS, { recursive: true });
+/** A process's start time as `ps` reports it — the cheap, boot-unique-enough
+ *  fellow-traveler of a pid that lets a reused pid be told apart from the
+ *  process that actually wrote a lock owner record. Empty string when the
+ *  pid does not exist (or `ps` cannot be run), never rejects. */
+async function pidStartTime(pid) {
+  const output = await probe('ps', ['-o', 'lstart=', '-p', String(pid)], RUNTIME_HOME);
+  return output.trim();
+}
+
+/** Whether `pid` currently identifies a live process, from this user's
+ *  vantage point. `EPERM` means the pid exists but is owned by someone
+ *  else — still alive, just not ours to signal. */
+function pidAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === 'EPERM';
+  }
+}
+
+/** Write the lock owner record via temp-file + rename so a concurrent reader
+ *  never observes a half-written file. Must be called only once `LOCK` is
+ *  known to be ours (immediately after `mkdir(LOCK)` succeeds). */
+async function writeLockOwner(lockDir) {
+  const owner = { pid: process.pid, startTime: await pidStartTime(process.pid), hostname: hostname(), startedAt: new Date().toISOString() };
+  const temporary = join(lockDir, `.${LOCK_OWNER_FILE}.${process.pid}-${Date.now()}.tmp`);
+  await writeFile(temporary, JSON.stringify(owner));
+  await rename(temporary, join(lockDir, LOCK_OWNER_FILE));
+}
+
+async function readLockOwner(lockDir) {
+  try {
+    const owner = JSON.parse(await readFile(join(lockDir, LOCK_OWNER_FILE), 'utf8'));
+    return Number.isInteger(owner?.pid) ? owner : null;
+  } catch {
+    return null;
+  }
+}
+
+/** True only when `owner`'s pid is both alive AND (when a start time was
+ *  recorded) still the same process that wrote the record — a recycled pid
+ *  must not masquerade as a live owner. */
+async function ownerIsAlive(owner) {
+  if (!owner || !pidAlive(owner.pid)) return false;
+  if (!owner.startTime) return true;
+  const currentStartTime = await pidStartTime(owner.pid);
+  return currentStartTime === '' || currentStartTime === owner.startTime;
+}
+
+/** Take `LOCK`, reclaiming it first if its owner is provably dead. A dead
+ *  owner is reclaimed unconditionally; a missing/corrupt owner record is
+ *  reclaimed only once the lock dir is older than a conservative threshold
+ *  (it may belong to a process that has not written its record yet). The
+ *  reclaim itself is race-safe: the stale dir is renamed to a private,
+ *  unique path first, and only the process whose rename wins takes the
+ *  fresh lock — a loser simply retries the ordinary contended path rather
+ *  than deleting a dir a concurrent winner might still be reading. */
+async function acquireLock() {
+  for (let attempt = 0; attempt < LOCK_ACQUIRE_MAX_ATTEMPTS; attempt++) {
+    try {
+      await mkdir(LOCK);
+      await writeLockOwner(LOCK);
+      return;
+    } catch (error) {
+      if (error?.code !== 'EEXIST') throw error;
+      const owner = await readLockOwner(LOCK);
+      if (owner && (await ownerIsAlive(owner))) {
+        throw new Error(`runtime installer lock is held by a live process: pid=${owner.pid} started=${owner.startTime || 'unknown'} host=${owner.hostname || 'unknown'}; refusing to install concurrently`);
+      }
+      if (!owner) {
+        let stat;
+        try {
+          stat = await lstat(LOCK);
+        } catch (statError) {
+          if (statError?.code === 'ENOENT') continue; // lock vanished between mkdir and lstat; retry
+          throw statError;
+        }
+        const ageMs = Date.now() - stat.mtimeMs;
+        if (ageMs < STALE_LOCK_RECORD_THRESHOLD_MS) {
+          throw new Error(`runtime installer lock exists without a readable owner record and is only ${Math.round(ageMs / 1000)}s old: ${LOCK}; inspect and remove only after confirming no installation is running`);
+        }
+      }
+      const graveyard = join(RUNTIME_HOME, `.lock-stale-${process.pid}-${Date.now()}`);
+      try {
+        await rename(LOCK, graveyard);
+      } catch (renameError) {
+        if (renameError?.code === 'ENOENT') continue; // another process already reclaimed it; fall back to the contended path
+        throw renameError;
+      }
+      process.stderr.write(`runtime installer: reclaimed a stale lock (dead owner pid=${owner?.pid ?? 'unknown'})\n`);
+      await removeStage(graveyard);
+      // Loop back and take the now-vacant lock.
+    }
+  }
+  throw new Error(`runtime installer lock: gave up after ${LOCK_ACQUIRE_MAX_ATTEMPTS} contended attempts: ${LOCK}`);
+}
+
+/** Sweep `.staging-*`/`.archive-*` temp dirs left behind by an install that
+ *  was killed before its own `finally` (or signal handler) could run. Only
+ *  called once this process holds `LOCK`, so any dir old enough to clear
+ *  the threshold cannot belong to a still-running install. */
+async function sweepOrphanTempDirs() {
+  let entries;
   try {
-    await mkdir(LOCK);
+    entries = await readdir(RUNTIME_HOME, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !/^\.(?:staging|archive)-/.test(entry.name)) continue;
+    const absolute = join(RUNTIME_HOME, entry.name);
+    let stat;
+    try {
+      stat = await lstat(absolute);
+    } catch {
+      continue;
+    }
+    if (Date.now() - stat.mtimeMs < STALE_TEMP_DIR_THRESHOLD_MS) continue;
+    process.stderr.write(`runtime installer: sweeping orphan temp dir from an earlier aborted install: ${absolute}\n`);
+    await removeStage(absolute);
+  }
+}
+
+// Populated as install() proceeds so a SIGINT/SIGTERM handler can undo
+// exactly what the normal `finally` below would have undone. SIGKILL
+// remains uncatchable, which is exactly why acquireLock() can reclaim.
+const cleanupState = { lockTaken: false, stage: undefined, archiveDir: undefined, publishedRoot: undefined, committed: false, temporary: undefined, id: undefined, exiting: false };
+
+async function cleanupOnSignal(signal) {
+  if (cleanupState.exiting) return;
+  cleanupState.exiting = true;
+  process.stderr.write(`runtime installer: received ${signal}, cleaning up before exit\n`);
+  try {
+    if (cleanupState.temporary) await rm(cleanupState.temporary, { force: true });
+    if (cleanupState.stage) await removeStage(cleanupState.stage);
+    if (cleanupState.publishedRoot && !cleanupState.committed) {
+      await removeStage(cleanupState.publishedRoot);
+      if (cleanupState.id) await rm(publishedMarkerPath(GENERATIONS, cleanupState.id), { force: true });
+    }
+    if (cleanupState.archiveDir) await rm(cleanupState.archiveDir, { recursive: true, force: true });
+    if (cleanupState.lockTaken) await rm(LOCK, { recursive: true, force: true });
   } catch (error) {
-    if (error?.code === 'EEXIST') throw new Error(`runtime installer lock exists: ${LOCK}; inspect and remove only after confirming no installation is running`);
-    throw error;
+    process.stderr.write(`runtime installer: cleanup on ${signal} failed: ${error?.message ?? error}\n`);
   }
+  process.exit(1);
+}
+
+process.on('SIGINT', () => { cleanupOnSignal('SIGINT'); });
+process.on('SIGTERM', () => { cleanupOnSignal('SIGTERM'); });
+
+async function install() {
+  await mkdir(GENERATIONS, { recursive: true });
+  await acquireLock();
+  cleanupState.lockTaken = true;
+  await sweepOrphanTempDirs();
   let stage;
   let archiveDir;
   let publishedRoot;
@@ -251,9 +411,11 @@ async function install() {
   let id;
   try {
     archiveDir = await mkdtemp(join(RUNTIME_HOME, '.archive-'));
+    cleanupState.archiveDir = archiveDir;
     const archive = await pack(archiveDir);
     const archiveHash = digest(await readFile(archive));
     stage = await mkdtemp(join(RUNTIME_HOME, '.staging-'));
+    cleanupState.stage = stage;
     await run('tar', ['-xzf', archive, '--strip-components=1', '-C', stage], PACKAGE_ROOT);
     await copyFile(join(stage, RUNTIME_LOCK), join(stage, 'package-lock.json'));
     // Humanloop's postinstall only prewarms its external termrender cache; runtime rendering retries lazily with a plaintext fallback.
@@ -265,6 +427,7 @@ async function install() {
     const pkg = JSON.parse(await readFile(join(stage, 'package.json'), 'utf8'));
     const files = sortInventory(await inventory(stage));
     id = digest(`${archiveHash}\n${digest(await readFile(join(stage, 'package-lock.json')))}\n${digest(JSON.stringify(files))}\n${process.version}\n${process.platform}\n${process.arch}\ninstaller-schema-1`);
+    cleanupState.id = id;
     const finalRoot = join(GENERATIONS, id);
     const manifest = {
       schema: MANIFEST_SCHEMA,
@@ -294,8 +457,10 @@ async function install() {
       await chmod(stage, 0o755);
       await rename(stage, finalRoot);
       publishedRoot = finalRoot;
+      cleanupState.publishedRoot = publishedRoot;
       await chmod(finalRoot, 0o555);
       stage = undefined;
+      cleanupState.stage = undefined;
       await validatePublished(finalRoot, id);
     }
     // Only after the full-hash check above has succeeded for this exact id —
@@ -303,9 +468,11 @@ async function install() {
     await markPublished(GENERATIONS, id);
     const selected = join(RUNTIME_HOME, 'selected');
     temporary = join(RUNTIME_HOME, `.selected-${process.pid}-${Date.now()}`);
+    cleanupState.temporary = temporary;
     await symlink(`generations/${id}`, temporary);
     await rename(temporary, selected);
     committed = true;
+    cleanupState.committed = true;
     try {
       await pruneGenerations();
     } catch (error) {