@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/server/threads/manageThreads.js

@@ -7,7 +7,7 @@ const { realExit } = require('./workerProcessGuard.ts');
 
 const { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } = require('worker_threads');
 const { spawn, spawnSync } = require('node:child_process');
-const { readFileSync } = require('node:fs');
+const { readdirSync, readFileSync, readlinkSync } = require('node:fs');
 const { setTimeout: delay } = require('node:timers/promises');
 const { join, isAbsolute, extname } = require('path');
 const { pathToFileURL } = require('url');
@@ -21,7 +21,16 @@ const { randomBytes } = require('crypto');
 const { _assignPackageExport } = require('../../globals.js');
 const { PACKAGE_ROOT } = require('../../utility/packageUtils.js');
 const { resolvePreloadModules } = require('./resolvePreload.ts');
+const { resolveThreadHeapMemoryMb } = require('./threadHeapMemory.ts');
 const { getConfigPath } = require('../../config/configUtils.ts');
+const { resolveWatchTarget } = require('../../utility/watchPath.ts');
+const {
+	DIRECTORY_POLLING_FALLBACK_OPTIONS,
+	claimLostNativeWatchError,
+	guardedWatch,
+	isWatcherExhaustionError,
+	warnWatcherFallback,
+} = require('../../utility/watcherFallback.ts');
 let importModules;
 function getImportModules() {
 	if (importModules === undefined)
@@ -42,7 +51,6 @@ function getRequireModules() {
 		);
 	return requireModules;
 }
-const chokidar = require('chokidar');
 const isBun = typeof globalThis.Bun !== 'undefined';
 const MB = 1024 * 1024;
 const workers = []; // these are our child workers that we are managing
@@ -54,8 +62,11 @@ const MAX_UNEXPECTED_RESTARTS = 50;
 // down mid-disposal crashes the process, so give that teardown more room before the forced backstop.
 let threadTerminationTimeout = process.env.DEV_MODE === 'true' || process.env.DEV_MODE === '1' ? 30000 : 10000;
 const RESTART_TYPE = 'restart';
+const RESTART_PROGRESS_HEARTBEAT_MS = 15000;
 const REQUEST_THREAD_INFO = 'request_thread_info';
 const RESOURCE_REPORT = 'resource_report';
+const OS_THREAD_ID = 'os-thread-id';
+const STUCK_WORKER_REPORT = 'stuck-worker-report';
 const THREAD_INFO = 'thread_info';
 const ADDED_PORT = 'added-port';
 const ACKNOWLEDGEMENT = 'ack';
@@ -154,16 +165,14 @@ module.exports = {
 	beginProcessShutdown,
 	registerWorkerDataProvider,
 	onThreadExit,
+	hasThreadExited,
+	notifyThreadExit,
 	registerProcessGroup,
 	unregisterProcessGroup,
+	isProcessGroupAlive,
 	isThreadRunning,
 	waitUntilConfirmedGone,
 	restartNumber: workerData?.restartNumber || 1,
-	// Identifies this process incarnation, where the PID cannot: a container reuses PID 1. Minted once
-	// on the main thread and carried to workers, so live siblings agree on it — one derived per thread
-	// would not. `undefined` on a worker started without it; consumers must fall back, not treat that
-	// as a mismatch.
-	processIncarnation: workerData ? workerData.processIncarnation : randomBytes(8).toString('hex'),
 };
 
 connectedPorts.onMessageByType = onMessageByType;
@@ -230,7 +239,6 @@ const RESERVED_WORKER_DATA_KEYS = [
 	'workerCount',
 	'name',
 	'restartNumber',
-	'processIncarnation',
 	'ticketKeys',
 	'noServerStart',
 	'__proto__', // never a legitimate payload name; spread would define it as an own property
@@ -303,6 +311,15 @@ if (!parentPort) {
 	onMessageByType(RESOURCE_REPORT, (message, worker) => {
 		if (worker) recordResourceReport(worker, message);
 	});
+	onMessageByType(OS_THREAD_ID, (message, worker) => {
+		if (worker) worker.osThreadId = message.osThreadId;
+	});
+	onMessageByType(STUCK_WORKER_REPORT, (message) => {
+		for (const threadId of message.threadIds) {
+			const worker = workers.find((worker) => worker.threadId === threadId);
+			if (worker) logStuckWorkerDiagnostics(worker);
+		}
+	});
 	onMessageByType(AWAIT_PROCESS_GROUP_TERMINATION, async (message, worker) => {
 		if (!worker) return;
 		await (pendingProcessGroupTerminations.get(message.ownerThreadId) ?? Promise.resolve());
@@ -314,6 +331,7 @@ if (!parentPort) {
 }
 // postMessage type listeners that are registered in other ways or can be registered later
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, null);
+listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.SCHEMA, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.USER, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST, null);
@@ -351,7 +369,7 @@ function startWorker(path, options = {}) {
 	// and lower than total memory
 	availableMemory = Math.min(availableMemory, totalmem(), 20000 * MB);
 	const maxOldMemory =
-		envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_MAXHEAPMEMORY) ??
+		resolveThreadHeapMemoryMb(envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_MAXHEAPMEMORY)) ??
 		Math.max(Math.floor(availableMemory / MB / (10 + (options.threadCount || 1) / 4)), 512);
 	// Max young memory space (semi-space for scavenger) is 1/128 of max memory (limited to 16-64). For most of our m5
 	// machines this will be 64MB (less for t3's). This is based on recommendations from:
@@ -415,7 +433,6 @@ function startWorker(path, options = {}) {
 			workerCount: (workerCount = options.threadCount),
 			name: options.name,
 			restartNumber: module.exports.restartNumber,
-			processIncarnation: module.exports.processIncarnation,
 			ticketKeys: getTicketKeys(),
 		},
 		transferList: portsToSend,
@@ -454,7 +471,10 @@ function startWorker(path, options = {}) {
 			if (worker.unexpectedRestarts < MAX_UNEXPECTED_RESTARTS) {
 				options.unexpectedRestarts = worker.unexpectedRestarts + 1;
 				startWorker(path, options);
-			} else harperLogger.error(`Thread has been restarted ${worker.restarts} times and will not be restarted`);
+			} else {
+				harperLogger.error(`Thread has been restarted ${worker.unexpectedRestarts} times and will not be restarted`);
+				options.onRestartExhausted?.(worker);
+			}
 		}
 	});
 	workers.push(worker);
@@ -474,16 +494,24 @@ const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP];
  * threads at the same time we shutdown new ones. However, we usually want to limit how many we do at once to avoid
  * excessive load and to keep things responsive. This parameter throttles the restarts to minimize load from
  * thread startups.
- * @returns {Promise<void>}
+ * @param onProgress Called each time a worker has been replaced (or its replacement has been given
+ * up on), so a caller waiting on a wide pool can tell a slow restart from a stalled one.
+ * @returns {Promise<{workersKeptOnOldCode: number, replacementsNotStarted: number}|{declined: true}|undefined>}
+ * from the main thread, how many workers were left running the old code and how many replacements
+ * never reported that they started — or `{declined: true}` when the process is already shutting down;
+ * from a worker, nothing — the restart is handed to the main thread.
  */
 
 async function restartWorkers(
 	name = null,
 	maxWorkersDown = Math.max(Math.floor(workerCount / 8), 1), // restart 1/8 of the threads at a time, but at least 1
-	startReplacementThreads = true
+	startReplacementThreads = true,
+	onProgress = null
 ) {
 	if (isMainThread) {
-		if (processShuttingDown && startReplacementThreads) return;
+		// Declining is not the same as delegating: a caller reporting on the restart must not read this
+		// as "another thread is completing it".
+		if (processShuttingDown && startReplacementThreads) return { declined: true };
 		try {
 			// we do this because it is possible for a component to chdir to itself, get re-deployed and then the cwd
 			// inode link is invalid and it can cause a lot of problems. But process.cwd() still returns the path, for
@@ -498,7 +526,16 @@ async function restartWorkers(
 		// This is here to prevent circular dependencies
 		if (startReplacementThreads) {
 			const { loadRootComponents } = require('../loadRootComponents.js');
-			await loadRootComponents();
+			// Installing and loading every root component reports nothing and can outlast a caller's idle
+			// window on its own (a cold npm cache, a large dependency graph), so beat while it runs. The
+			// caller's absolute ceiling is what bounds a load that never finishes.
+			const loading = setInterval(() => onProgress?.(), RESTART_PROGRESS_HEARTBEAT_MS).unref();
+			try {
+				await loadRootComponents();
+			} finally {
+				clearInterval(loading);
+			}
+			onProgress?.();
 		}
 
 		module.exports.restartNumber++;
@@ -508,7 +545,17 @@ async function restartWorkers(
 			maxWorkersDown = maxWorkersDown * workers.length;
 		}
 		// make a copy of the workers before iterating them, as the workers array mutates a lot during this
-		let waitingToFinish = []; // promises for workers we have shut down and are waiting to exit
+		let waitingToFinish = []; // promises for workers we are replacing, spliced as each is replaced
+		// Every replacement that was started without being awaited first, so this function can still
+		// resolve only once each one is accepting connections.
+		let replacementsStarting = [];
+		// A replacement that never came up leaves the pool in one of two very different states, and a
+		// caller waiting on this restart needs them apart: the pre-start path keeps the *old* worker
+		// serving old code, while a replacement that exits after its predecessor is already gone only
+		// costs capacity until startWorker's auto-restart brings a fresh one up.
+		let workersKeptOnOldCode = 0;
+		let replacementsNotStarted = 0;
+		let replacementsFailedToStart = 0;
 		// We can only start the replacement *before* the old worker releases its port when the OS lets
 		// both listen on the same port at once (SO_REUSEPORT). Without that — Windows (no SO_REUSEPORT),
 		// macOS (unreliable SO_REUSEPORT, so workers bind exclusively), and Bun — the replacement can't
@@ -520,7 +567,9 @@ async function restartWorkers(
 		// thread keeps serving the HTTP ports throughout. This ordering is also what lets
 		// listenOnPorts() treat a dedicated listener's EADDRINUSE as an external conflict.
 		const canPreStartReplacement = process.platform !== 'win32' && process.platform !== 'darwin' && !isBun;
-		for (let worker of workers.slice(0)) {
+		const restarting = workers.slice(0);
+		for (let index = 0; index < restarting.length; index++) {
+			const worker = restarting[index];
 			// Terminal shutdown: stop replacing workers mid-loop — the guard for every replacement start below.
 			if (processShuttingDown && startReplacementThreads) break;
 			if ((name && worker.name !== name) || worker.wasShutdown) continue; // filter by type, if specified
@@ -587,6 +636,8 @@ async function restartWorkers(
 					// Replacement didn't come up — keep the existing worker serving. Restore its auto-restart
 					// protection if it is still alive (it may have exited on its own during the wait).
 					if (workers.includes(worker)) worker.wasShutdown = false;
+					workersKeptOnOldCode++;
+					onProgress?.();
 					continue;
 				}
 			}
@@ -607,7 +658,16 @@ async function restartWorkers(
 			// Overlapping types we couldn't pre-start (Windows/Bun): start the replacement now that the old
 			// worker is releasing its port. server.close() stops accepting immediately, so the port frees up
 			// well before the replacement finishes booting and binds.
-			if (overlapping && startReplacementThreads && !canPreStartReplacement && !processShuttingDown) worker.startCopy();
+			let replacementStarting;
+			if (overlapping && startReplacementThreads && !canPreStartReplacement && !processShuttingDown) {
+				replacementStarting = whenWorkerStarted(worker.startCopy()).then((started) => {
+					if (!started) replacementsFailedToStart++;
+					onProgress?.();
+					return started;
+				});
+				replacementsStarting.push(replacementStarting);
+			}
+
 			let whenDone = new Promise((resolve) => {
 				// in case the exit inside the thread doesn't timeout, force it from the outside
 				const armTerminate = (delay) =>
@@ -634,29 +694,62 @@ async function restartWorkers(
 					// buggy/rogue message can't defer the force-kill unboundedly; a shrink (drain-done reset)
 					// passes through untouched. See boundedTerminateDelay for the arithmetic + its unit tests.
 					const { boundedTerminateDelay, getShutdownDrainCeilingMs } = require('../../components/shutdownDrain.ts');
-					timeout = armTerminate(
-						boundedTerminateDelay(deadlineMs, Date.now(), threadTerminationTimeout * 2, getShutdownDrainCeilingMs())
+					const delay = boundedTerminateDelay(
+						deadlineMs,
+						Date.now(),
+						threadTerminationTimeout * 2,
+						getShutdownDrainCeilingMs()
 					);
+					timeout = armTerminate(delay);
+					// The worker is telling us it has work still moving and how long it may take, so pass that
+					// on: a caller waiting on the restart must not treat a live drain as a stalled one.
+					onProgress?.(Date.now() + delay);
 				};
 				worker.on('exit', () => {
 					clearTimeout(timeout);
+					onProgress?.();
 					worker.extendTerminateDeadline = undefined;
-					const index = waitingToFinish.indexOf(whenDone);
-					if (index > -1) waitingToFinish.splice(index, 1);
 					// non-overlapping types have no advance replacement, so start it once the old one is gone
 					if (!overlapping && startReplacementThreads && !processShuttingDown) worker.startCopy();
 					resolve();
 				});
 			});
-			waitingToFinish.push(whenDone);
+			// A worker counts as replaced only once its replacement is accepting connections, not merely
+			// once it has exited. This promise is held unawaited between throttle points, so it must not
+			// be able to reject.
+			const replaced = (replacementStarting ? Promise.all([whenDone, replacementStarting]) : whenDone)
+				.catch((error) => harperLogger.warn('Error waiting for a worker to be replaced', error))
+				.then(() => {
+					const at = waitingToFinish.indexOf(replaced);
+					if (at > -1) waitingToFinish.splice(at, 1);
+				});
+			waitingToFinish.push(replaced);
 			if (waitingToFinish.length >= maxWorkersDown) {
-				// throttle how many workers are draining/down at once to limit load
+				// throttle how many workers are down at once to limit load
 				await Promise.race(waitingToFinish);
 			}
+			// Readiness throttling bounds how many workers are down at once, but not how many *fail*: with
+			// replacements that never come up, walking the rest of the pool would leave nothing serving.
+			// The workers not yet touched are still running the old code, which beats none running at all.
+			if (replacementsFailedToStart >= maxWorkersDown) {
+				const untouched = restarting
+					.slice(index + 1)
+					// a worker that exited on its own mid-restart is spliced out of `workers` and
+					// auto-restarted onto the new code (see the exit handler above); it is not still on
+					// the previous code even though this loop never got to it.
+					.filter((other) => (!name || other.name === name) && !other.wasShutdown && workers.includes(other)).length;
+				harperLogger.error(
+					`${replacementsFailedToStart} replacement worker thread(s) did not start; stopping this restart with ${untouched} worker(s) still on the previous code`
+				);
+				workersKeptOnOldCode += untouched;
+				break;
+			}
 		}
-		// seems appropriate to wait for this to finish, but the API doesn't actually wait for this function
-		// to finish, so not that important
 		await Promise.all(waitingToFinish);
+		// A caller awaiting this needs it to mean "the pool is serving the new code", so wait out the
+		// replacements that could only be started once their predecessor released its exclusive ports.
+		replacementsNotStarted = (await Promise.all(replacementsStarting)).filter((started) => !started).length;
+		return { workersKeptOnOldCode, replacementsNotStarted };
 	} else {
 		parentPort.postMessage({
 			type: RESTART_TYPE,
@@ -664,6 +757,51 @@ async function restartWorkers(
 		});
 	}
 }
+/**
+ * Resolve once a newly started worker reports that it is accepting connections, or gives up on it.
+ * There is no old worker left to fall back on here, so a replacement that fails to start is left to
+ * startWorker's own auto-restart handling; this only stops waiting on it.
+ * @param newWorker The replacement worker
+ * @returns {Promise<boolean>} whether the worker reported that it started
+ */
+function whenWorkerStarted(newWorker) {
+	return new Promise((resolve) => {
+		const cleanup = () => {
+			clearTimeout(timeout);
+			newWorker.off('message', startListener);
+			newWorker.off('exit', exitListener);
+		};
+		const timeout = setTimeout(
+			() => {
+				harperLogger.error('Replacement worker did not start in time', newWorker.threadId);
+				cleanup();
+				// Its predecessor is already gone, so a replacement wedged in boot is a worker slot serving
+				// nothing until the process restarts. Stop it and let startWorker's exit handling replace it.
+				if (isBun) {
+					// terminate() triggers a NAPI segfault in Bun; ask the worker to self-exit instead.
+					try {
+						newWorker.postMessage({ type: FORCE_EXIT });
+					} catch {}
+				} else newWorker.terminate();
+				resolve(false);
+			},
+			Math.max(threadTerminationTimeout * 2, 60000)
+		).unref();
+		const startListener = (message) => {
+			if (message.type === hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED) {
+				cleanup();
+				resolve(true);
+			}
+		};
+		const exitListener = () => {
+			harperLogger.warn('Replacement worker exited before starting', newWorker.threadId);
+			cleanup();
+			resolve(false);
+		};
+		newWorker.on('message', startListener);
+		newWorker.on('exit', exitListener);
+	});
+}
 function shutdownWorkers(name) {
 	return restartWorkers(name, Infinity, false);
 }
@@ -791,18 +929,154 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS)
 				timer = undefined;
 				const stuck = [];
 				for (let ackHandler of [...pending]) {
-					stuck.push(ackHandler.port?.threadId);
+					stuck.push(ackHandler.port);
 					ackHandler(); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves
 				}
 				harperLogger.warn(
-					`ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort`
+					`ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.map((port) => port?.threadId).join(', ')} within ${timeout}ms; proceeding best-effort`
 				);
+				if (isMainThread) for (let port of stuck) logStuckWorkerDiagnostics(port);
+				else if (parentPort) {
+					// Only main holds the Worker objects (and tids), so a worker-originated timeout is sampled there.
+					const threadIds = stuck.map((port) => port?.threadId).filter((threadId) => threadId > 0);
+					if (threadIds.length > 0) parentPort.postMessage({ type: STUCK_WORKER_REPORT, threadIds });
+				}
 			}, timeout);
 			timer.unref?.();
 		}
 	});
 }
 
+// Linux only: /proc/thread-self resolves to <pid>/task/<tid> for the calling thread.
+function getOsThreadId() {
+	try {
+		const tid = Number(readlinkSync('/proc/thread-self').split('/').pop());
+		return Number.isInteger(tid) && tid > 0 ? tid : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+function readTaskFile(tid, name) {
+	try {
+		return readFileSync(`/proc/self/task/${tid}/${name}`, 'utf8').trim();
+	} catch {
+		return undefined;
+	}
+}
+
+// Kernel-side view of one thread. Every field is best-effort and reported individually, since
+// wchan/syscall need ptrace read access that a hardened container may deny while stat is open.
+function finiteOrUndefined(value) {
+	const number = Number(value);
+	return Number.isFinite(number) ? number : undefined;
+}
+
+function readOsThreadState(tid) {
+	const thread = { tid };
+	const stat = readTaskFile(tid, 'stat');
+	if (stat !== undefined) {
+		// Fields after the parenthesized comm, so state is [0], utime/stime [11]/[12], starttime [19].
+		const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+		thread.state = fields[0];
+		const utime = finiteOrUndefined(fields[11]);
+		const stime = finiteOrUndefined(fields[12]);
+		if (utime !== undefined && stime !== undefined) thread.cpuTicks = utime + stime;
+		thread.startTime = fields[19];
+	}
+	const wchan = readTaskFile(tid, 'wchan');
+	if (wchan !== undefined) thread.wchan = wchan;
+	// First token only: the syscall number, "running", or -1 (blocked outside a syscall). The
+	// remainder is argument registers and the stack/instruction pointers, which don't belong in a log.
+	const syscall = readTaskFile(tid, 'syscall');
+	if (syscall !== undefined) thread.syscall = syscall.split(' ')[0];
+	const status = readTaskFile(tid, 'status');
+	if (status !== undefined) {
+		thread.voluntarySwitches = finiteOrUndefined(/^voluntary_ctxt_switches:\s*(\d+)/m.exec(status)?.[1]);
+		thread.nonvoluntarySwitches = finiteOrUndefined(/^nonvoluntary_ctxt_switches:\s*(\d+)/m.exec(status)?.[1]);
+	}
+	return thread;
+}
+
+function snapshotWorkerThread(worker) {
+	const snapshot = { at: Date.now() };
+	if (worker.resources?.updated) snapshot.sinceResourceReport = snapshot.at - worker.resources.updated;
+	const eventLoop = worker.performance?.eventLoopUtilization?.();
+	if (eventLoop) snapshot.eventLoop = { idle: eventLoop.idle, active: eventLoop.active };
+	if (worker.osThreadId) snapshot.osThread = readOsThreadState(worker.osThreadId);
+	return snapshot;
+}
+
+function describeThreadState(thread) {
+	return `state=${thread.state ?? '?'} wchan=${thread.wchan ?? '?'} syscall=${thread.syscall ?? '?'}`;
+}
+
+function describeSnapshot(snapshot) {
+	const parts = [
+		snapshot.sinceResourceReport === undefined
+			? 'no resource report received'
+			: `last resource report ${snapshot.sinceResourceReport}ms ago`,
+	];
+	if (snapshot.eventLoop)
+		parts.push(
+			`event loop active ${Math.round(snapshot.eventLoop.active)}ms idle ${Math.round(snapshot.eventLoop.idle)}ms`
+		);
+	const thread = snapshot.osThread;
+	if (!thread) parts.push('os thread state unavailable');
+	else
+		parts.push(
+			`os tid ${thread.tid} ${describeThreadState(thread)} cpuTicks=${thread.cpuTicks ?? '?'} ctxtSwitches=${thread.voluntarySwitches ?? '?'}/${thread.nonvoluntarySwitches ?? '?'}`
+		);
+	return parts.join('; ');
+}
+
+function describeDelta(before, after) {
+	return Number.isFinite(before) && Number.isFinite(after) ? `+${after - before}` : '?';
+}
+
+function describeProgress(first, second) {
+	const parts = [];
+	if (first.eventLoop && second.eventLoop)
+		parts.push(
+			`event loop active +${Math.round(second.eventLoop.active - first.eventLoop.active)}ms idle +${Math.round(second.eventLoop.idle - first.eventLoop.idle)}ms`
+		);
+	const before = first.osThread;
+	const after = second.osThread;
+	if (before && after)
+		parts.push(
+			`cpuTicks ${describeDelta(before.cpuTicks, after.cpuTicks)} ctxtSwitches ${describeDelta(before.voluntarySwitches, after.voluntarySwitches)}/${describeDelta(before.nonvoluntarySwitches, after.nonvoluntarySwitches)} ${describeThreadState(after)}`
+		);
+	return parts.join('; ');
+}
+
+// A blocked event loop cannot report itself; two kernel-state samples a second apart separate
+// "parked on a lock" (no CPU ticks, no context switches) from "spinning".
+const STUCK_WORKER_SAMPLE_INTERVAL_MS = 1000;
+const STUCK_WORKER_DIAGNOSTIC_COOLDOWN_MS = 30000;
+function logStuckWorkerDiagnostics(worker) {
+	if (!worker || !workers.includes(worker)) return;
+	const now = Date.now();
+	if (worker.stuckDiagnosticAt !== undefined && now - worker.stuckDiagnosticAt < STUCK_WORKER_DIAGNOSTIC_COOLDOWN_MS)
+		return;
+	worker.stuckDiagnosticAt = now;
+	const threadId = worker.threadId; // Node resets it to -1 once the worker exits
+	const first = snapshotWorkerThread(worker);
+	harperLogger.warn(`Worker thread ${threadId} at ack timeout: ${describeSnapshot(first)}`);
+	if (!first.eventLoop && !first.osThread) return;
+	setTimeout(() => {
+		if (!workers.includes(worker)) {
+			harperLogger.warn(`Worker thread ${threadId} exited before its follow-up sample`);
+			return;
+		}
+		const second = snapshotWorkerThread(worker);
+		// A recycled tid after a thread exit would attribute another thread's activity to this worker.
+		if (second.osThread && second.osThread.startTime !== first.osThread?.startTime) second.osThread = undefined;
+		harperLogger.warn(
+			`Worker thread ${threadId} over the next ${second.at - first.at}ms: ${describeProgress(first, second) || 'no further state available'}`
+		);
+	}, STUCK_WORKER_SAMPLE_INTERVAL_MS).unref();
+}
+
 function sendThreadInfo(targetWorker) {
 	targetWorker.postMessage({
 		type: THREAD_INFO,
@@ -875,6 +1149,8 @@ if (parentPort && workerData?.addPorts) {
 	// parentPort so sendToThread(0, ...) and similar lookups can route back to main.
 	parentPort.threadId = 0;
 	addPort(parentPort);
+	const osThreadId = getOsThreadId();
+	if (osThreadId !== undefined) parentPort.postMessage({ type: OS_THREAD_ID, osThreadId });
 	for (let i = 0, l = workerData.addPorts.length; i < l; i++) {
 		let port = workerData.addPorts[i];
 		port.threadId = workerData.addThreadIds[i];
@@ -923,17 +1199,23 @@ if (parentPort && workerData?.addPorts) {
 			}
 		});
 	let awaitTerminationRequestId = 0;
-	awaitProcessGroupTermination = (ownerThreadId) =>
+	awaitProcessGroupTermination = (ownerThreadId, signal) =>
 		new Promise((resolve) => {
 			// Deliberately no timeout: a contender must not reclaim a dead owner's lock while its
 			// process group might still be alive and mutating files, so this mirrors the unbounded
-			// wait Application.ts's waitForConfirmedTermination uses for the same reason.
+			// wait Application.ts's waitForConfirmedTermination uses for the same reason. A caller
+			// that needs its own bound (isThreadRunning) passes `signal` so this listener still gets
+			// torn down when that caller gives up, instead of leaking for the life of the worker.
 			const requestId = ++awaitTerminationRequestId;
 			parentPort.on('message', receiveConfirmation);
+			signal?.addEventListener('abort', cleanup, { once: true });
 			parentPort.postMessage({ type: AWAIT_PROCESS_GROUP_TERMINATION, ownerThreadId, requestId });
+			function cleanup() {
+				parentPort.off('message', receiveConfirmation);
+			}
 			function receiveConfirmation(message) {
 				if (message.type === PROCESS_GROUP_TERMINATION_CONFIRMED && message.requestId === requestId) {
-					parentPort.off('message', receiveConfirmation);
+					cleanup();
 					resolve();
 				}
 			}
@@ -966,33 +1248,142 @@ const processGroupsByThread = new Map();
 // window component-preparation locking exists to close.
 const pendingProcessGroupTerminations = new Map();
 const PROCESS_GROUP_TERMINATION_POLL_MS = 25;
+const ZOMBIE_GROUP_MEMBER_SCAN_INTERVAL_MS = 1000;
+const PROCESS_GROUP_LIVENESS_WARNING_MS = 30000;
+// Bounds isThreadRunning's own wait (below); Application.ts's waitForConfirmedTermination stays
+// deliberately unbounded.
+const THREAD_RUNNING_TERMINATION_BACKSTOP_MS = PROCESS_GROUP_LIVENESS_WARNING_MS;
+const zombieGroupScanTimes = new Map();
+const processGroupLivenessStates = new Map();
+
+function processGroupExists(processGroupId) {
+	try {
+		process.kill(-processGroupId, 0);
+		return true;
+	} catch (error) {
+		return error.code === 'EPERM';
+	}
+}
+
+function processProbeError(error) {
+	return error?.code ?? error?.message ?? String(error ?? 'unknown error');
+}
 
-// A process-group leader spawned by a worker thread's own event loop is normally reaped by that
-// same loop when it dies. But the only caller of this check (waitForProcessGroupExit, for a
-// *dead* owner thread) runs after that loop is already gone, so nothing will ever reap it here:
-// kill(pid, 0) keeps succeeding forever against the zombie's still-allocated PID slot. A zombie
-// can no longer touch the filesystem, so treat a confirmed zombie as dead rather than poll
-// forever for a reap that will never come. Reproduced by killing a worker mid-install: the
-// installer's process-group leader becomes a permanent `Z` in /proc without this check.
-function isZombieProcessGroupLeader(processGroupId) {
-	if (process.platform !== 'linux') return false;
+function processGroupLeaderState(processGroupId, platform, readStat) {
+	if (platform !== 'linux') {
+		return { state: 'unknown', reason: `zombie-process detection is unavailable on ${platform}` };
+	}
 	let stat;
 	try {
-		stat = readFileSync(`/proc/${processGroupId}/stat`, 'utf8');
-	} catch {
-		return false;
+		stat = readStat(`/proc/${processGroupId}/stat`, 'utf8');
+	} catch (error) {
+		if (error.code === 'ENOENT' || error.code === 'ESRCH') return { state: 'missing' };
+		return {
+			state: 'unknown',
+			reason: `reading /proc/${processGroupId}/stat failed (${processProbeError(error)})`,
+		};
 	}
-	// Format is "pid (comm) state ..."; comm can contain spaces/parens, so anchor on the last ')'.
-	return stat[stat.lastIndexOf(')') + 2] === 'Z';
+	const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+	if (Number(fields[2]) !== processGroupId) return { state: 'missing' };
+	return { state: fields[0] === 'Z' ? 'zombie' : 'alive' };
 }
 
-function processGroupIsAlive(processGroupId) {
+function scanLinuxProcessGroup(processGroupId, readDirectory, readStat) {
+	let processIds;
 	try {
-		process.kill(-processGroupId, 0);
+		processIds = readDirectory('/proc');
 	} catch (error) {
-		return error.code === 'EPERM';
+		return {
+			isAlive: null,
+			reason: `reading /proc failed (${processProbeError(error)})`,
+		};
 	}
-	return !isZombieProcessGroupLeader(processGroupId);
+	for (const processId of processIds) {
+		if (!/^\d+$/.test(processId)) continue;
+		let stat;
+		try {
+			stat = readStat(`/proc/${processId}/stat`, 'utf8');
+		} catch (error) {
+			if (error.code === 'ENOENT' || error.code === 'ESRCH') continue; // already gone
+			// EACCES/EPERM (e.g. hidepid/ProtectProc) can only happen on a pid we don't own — we
+			// spawned our own group's members as this same user, so their stat is always readable —
+			// so it's not one of ours. Anything else is genuinely unknown, not "not ours".
+			if (error.code === 'EACCES' || error.code === 'EPERM') continue;
+			return {
+				isAlive: null,
+				reason: `reading /proc/${processId}/stat failed (${processProbeError(error)})`,
+			};
+		}
+		const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+		const pgrp = Number(fields[2]);
+		if (!Number.isFinite(pgrp)) {
+			return { isAlive: null, reason: `/proc/${processId}/stat had an unparseable process-group field` };
+		}
+		if (pgrp !== processGroupId) continue;
+		if (fields[0] !== 'Z') return { isAlive: true, reason: `a live member was observed at pid ${processId}` };
+	}
+	return { isAlive: false };
+}
+
+function keepProcessGroupAlive(processGroupId, reason, observationTime, warn) {
+	let livenessState = processGroupLivenessStates.get(processGroupId);
+	if (!livenessState) {
+		livenessState = { observedAt: observationTime, reason, warned: false };
+		processGroupLivenessStates.set(processGroupId, livenessState);
+	} else {
+		livenessState.reason = reason;
+	}
+	if (!livenessState.warned && observationTime - livenessState.observedAt >= PROCESS_GROUP_LIVENESS_WARNING_MS) {
+		livenessState.warned = true;
+		warn(
+			`Process group ${processGroupId} termination remains unconfirmed after ${PROCESS_GROUP_LIVENESS_WARNING_MS}ms: ${livenessState.reason}`
+		);
+	}
+	return true;
+}
+
+function clearProcessGroupLivenessState(processGroupId) {
+	zombieGroupScanTimes.delete(processGroupId);
+	processGroupLivenessStates.delete(processGroupId);
+}
+
+function isProcessGroupAlive(processGroupId, options) {
+	const platform = options?.platform ?? process.platform;
+	const groupExists = options?.processGroupExists ?? processGroupExists;
+	const readDirectory = options?.readDirectory ?? readdirSync;
+	const readStat = options?.readStat ?? readFileSync;
+	const observationTime = options?.now?.() ?? performance.now();
+	const warn = options?.warn ?? ((message) => harperLogger.warn(message));
+	if (!groupExists(processGroupId)) {
+		clearProcessGroupLivenessState(processGroupId);
+		return false;
+	}
+	const leaderState = processGroupLeaderState(processGroupId, platform, readStat);
+	if (leaderState.state === 'alive')
+		return keepProcessGroupAlive(processGroupId, 'the process-group leader is still alive', observationTime, warn);
+	if (leaderState.state === 'unknown')
+		return keepProcessGroupAlive(processGroupId, leaderState.reason, observationTime, warn);
+	const scanTime = observationTime;
+	const lastScan = zombieGroupScanTimes.get(processGroupId);
+	if (lastScan !== undefined && scanTime >= lastScan && scanTime - lastScan < ZOMBIE_GROUP_MEMBER_SCAN_INTERVAL_MS) {
+		const reason = processGroupLivenessStates.get(processGroupId)?.reason ?? 'the previous Linux scan is still current';
+		return keepProcessGroupAlive(processGroupId, reason, observationTime, warn);
+	}
+	zombieGroupScanTimes.set(processGroupId, scanTime);
+	let scanResult = scanLinuxProcessGroup(processGroupId, readDirectory, readStat);
+	if (scanResult.isAlive !== false)
+		return keepProcessGroupAlive(processGroupId, scanResult.reason, observationTime, warn);
+	// A second snapshot catches a member omitted after forking inside the first readdir/stat window.
+	// It narrows rather than closes the race because the second synchronous pass has the same gap.
+	scanResult = scanLinuxProcessGroup(processGroupId, readDirectory, readStat);
+	if (scanResult.isAlive !== false)
+		return keepProcessGroupAlive(processGroupId, scanResult.reason, observationTime, warn);
+	clearProcessGroupLivenessState(processGroupId);
+	return false;
+}
+
+function processGroupIsAlive(processGroupId) {
+	return isProcessGroupAlive(processGroupId);
 }
 
 async function waitForProcessGroupExit(processGroupId) {
@@ -1074,6 +1465,7 @@ function addProcessGroup(ownerThreadId, processGroupId) {
 }
 
 function removeProcessGroup(ownerThreadId, processGroupId) {
+	clearProcessGroupLivenessState(processGroupId);
 	const processGroups = processGroupsByThread.get(ownerThreadId);
 	if (!processGroups) return;
 	processGroups.delete(processGroupId);
@@ -1125,12 +1517,48 @@ function unregisterProcessGroup(processGroupId) {
 	else parentPort?.postMessage({ type: UNREGISTER_PROCESS_GROUP, processGroupId });
 }
 
+class ProcessGroupTerminationUnconfirmedError extends Error {
+	code = 'ERR_PROCESS_GROUP_TERMINATION_UNCONFIRMED';
+	constructor(ownerThreadId, timeoutMs) {
+		super(`Thread ${ownerThreadId}'s process-group termination could not be confirmed after ${timeoutMs}ms`);
+		this.name = 'ProcessGroupTerminationUnconfirmedError';
+	}
+}
+
+// Bounds awaitProcessGroupTermination for isThreadRunning below. Throws rather than resolving to
+// "not running": componentPreparationLock's ownerIsAlive/ownerLivenessConfirmed already treat a
+// throwing isOwnerAlive as "can't confirm — don't steal the claim, don't renew the deadline", so
+// the lock's own bounded wait is what eventually fails the waiter.
+async function awaitConfirmedProcessGroupTermination(ownerThreadId) {
+	const abortController = new AbortController();
+	let timedOut = false;
+	const timer = setTimeout(() => {
+		timedOut = true;
+		abortController.abort();
+	}, THREAD_RUNNING_TERMINATION_BACKSTOP_MS);
+	timer.unref?.();
+	try {
+		await Promise.race([
+			awaitProcessGroupTermination(ownerThreadId, abortController.signal),
+			new Promise((resolve) => abortController.signal.addEventListener('abort', resolve, { once: true })),
+		]);
+	} finally {
+		clearTimeout(timer);
+	}
+	if (timedOut) {
+		harperLogger.warn(
+			`Thread ${ownerThreadId}'s process-group termination could not be confirmed after ${THREAD_RUNNING_TERMINATION_BACKSTOP_MS}ms`
+		);
+		throw new ProcessGroupTerminationUnconfirmedError(ownerThreadId, THREAD_RUNNING_TERMINATION_BACKSTOP_MS);
+	}
+}
+
 async function isThreadRunning(ownerThreadId, timeoutMs = THREAD_INFO_REQUEST_TIMEOUT_MS) {
 	if (ownerThreadId === threadId || ownerThreadId === 0) return true;
 	if ((await getThreadInfo(timeoutMs)).some((worker) => worker.threadId === ownerThreadId)) return true;
 	// The thread itself is gone, but it may still own process groups whose forced termination is
 	// in flight — wait for that to be confirmed before reporting the owner as reclaimable.
-	await awaitProcessGroupTermination(ownerThreadId);
+	await awaitConfirmedProcessGroupTermination(ownerThreadId);
 	return false;
 }
 
@@ -1140,6 +1568,15 @@ if (isMainThread) {
 	});
 }
 
+/**
+ * Whether a thread has already been reported dead. Sync, unlike `isThreadRunning`, because it only
+ * reads the tombstone `notifyThreadExit` records below — callers on the exit path need an answer
+ * without awaiting process-group confirmation.
+ */
+function hasThreadExited(threadId) {
+	return notifiedDeadThreadIds.has(threadId);
+}
+
 function notifyThreadExit(deadThreadId) {
 	if (deadThreadId == null || notifiedDeadThreadIds.has(deadThreadId)) return;
 	notifiedDeadThreadIds.add(deadThreadId);
@@ -1247,23 +1684,49 @@ if (isMainThread) {
 	const ignoredPaths = ['node_modules', '.git'];
 	const watchDir = async (dir, beforeRestartCallback) => {
 		if (beforeRestartCallback) beforeRestart = beforeRestartCallback;
-		chokidar
-			.watch(dir, {
+		const watchTarget = resolveWatchTarget(dir);
+		let usingPolling = watchTarget.mustPoll;
+		let liveWatcher;
+		const openWatcher = () => {
+			const opened = (liveWatcher = guardedWatch(watchTarget.path, {
 				persistent: false,
+				...(usingPolling ? DIRECTORY_POLLING_FALLBACK_OPTIONS : {}),
 				ignored: (path) => {
 					return ignoredPaths.some((ignoredPath) => path.includes(ignoredPath));
 				},
-			})
-			.on('change', (path) => {
-				changedFiles.add(path);
-				if (queuedRestart) clearTimeout(queuedRestart);
-				queuedRestart = setTimeout(async () => {
-					if (beforeRestart) await beforeRestart();
-					await restartWorkers();
-					console.log('Reloaded Harper components, changed files:', Array.from(changedFiles));
-					changedFiles.clear();
-				}, 100);
-			});
+			}));
+			opened
+				// This runs on the thread that owns every worker, and chokidar emits 'error' unguarded for
+				// anything but ENOENT/ENOTDIR.
+				.on('error', (error) => {
+					if (claimLostNativeWatchError(error)) return;
+					if (isWatcherExhaustionError(error)) {
+						if (usingPolling || liveWatcher !== opened) return;
+						warnWatcherFallback(dir);
+						usingPolling = true;
+						Promise.resolve()
+							.then(() => opened.close())
+							.catch(() => {})
+							.then(openWatcher)
+							.catch((reopenError) =>
+								console.error(`Could not reopen the ${dir} component-reload watch on polling:`, reopenError)
+							);
+						return;
+					}
+					console.error(`Error watching ${dir} for component reloads:`, error);
+				})
+				.on('change', (path) => {
+					changedFiles.add(path);
+					if (queuedRestart) clearTimeout(queuedRestart);
+					queuedRestart = setTimeout(async () => {
+						if (beforeRestart) await beforeRestart();
+						await restartWorkers();
+						console.log('Reloaded Harper components, changed files:', Array.from(changedFiles));
+						changedFiles.clear();
+					}, 100);
+				});
+		};
+		openWatcher();
 	};
 	module.exports.watchDir = watchDir;
 	if (process.env.WATCH_DIR) watchDir(process.env.WATCH_DIR);