@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/dist/server/threads/manageThreads.js

@@ -5,7 +5,7 @@
 const { realExit } = require("./workerProcessGuard.js");
 const { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } = require('worker_threads');
 const { spawn, spawnSync } = require('node:child_process');
-const { readFileSync } = require('node:fs');
+const { readdirSync, readFileSync, readlinkSync } = require('node:fs');
 const { setTimeout: delay } = require('node:timers/promises');
 const { join, isAbsolute, extname } = require('path');
 const { pathToFileURL } = require('url');
@@ -19,7 +19,10 @@ const { randomBytes } = require('crypto');
 const { _assignPackageExport } = require('../../globals.js');
 const { PACKAGE_ROOT } = require('../../utility/packageUtils.js');
 const { resolvePreloadModules } = require("./resolvePreload.js");
+const { resolveThreadHeapMemoryMb } = require("./threadHeapMemory.js");
 const { getConfigPath } = require("../../config/configUtils.js");
+const { resolveWatchTarget } = require("../../utility/watchPath.js");
+const { DIRECTORY_POLLING_FALLBACK_OPTIONS, claimLostNativeWatchError, guardedWatch, isWatcherExhaustionError, warnWatcherFallback, } = require("../../utility/watcherFallback.js");
 let importModules;
 function getImportModules() {
     if (importModules === undefined)
@@ -32,7 +35,6 @@ function getRequireModules() {
         requireModules = resolvePreloadModules(envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_PRELOADREQUIRE), getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), 'threads.preloadRequire');
     return requireModules;
 }
-const chokidar = require('chokidar');
 const isBun = typeof globalThis.Bun !== 'undefined';
 const MB = 1024 * 1024;
 const workers = []; // these are our child workers that we are managing
@@ -44,9 +46,14 @@ const MAX_UNEXPECTED_RESTARTS = 50;
 // down mid-disposal crashes the process, so give that teardown more room before the forced backstop.
 let threadTerminationTimeout = process.env.DEV_MODE === 'true' || process.env.DEV_MODE === '1' ? 30000 : 10000;
 const RESTART_TYPE = 'restart';
+const RESTART_PROGRESS_HEARTBEAT_MS = 15000;
 const REQUEST_THREAD_INFO = 'request_thread_info';
+const REQUEST_RUNNING_ISOLATED_APPLICATIONS = 'request-running-isolated-applications';
 const RESOURCE_REPORT = 'resource_report';
+const OS_THREAD_ID = 'os-thread-id';
+const STUCK_WORKER_REPORT = 'stuck-worker-report';
 const THREAD_INFO = 'thread_info';
+const RUNNING_ISOLATED_APPLICATIONS = 'running-isolated-applications';
 const ADDED_PORT = 'added-port';
 const ACKNOWLEDGEMENT = 'ack';
 const REMOVE_PORT = 'remove-port';
@@ -64,6 +71,7 @@ const AWAIT_PROCESS_GROUP_TERMINATION = 'await-process-group-termination';
 const PROCESS_GROUP_TERMINATION_CONFIRMED = 'process-group-termination-confirmed';
 const THREAD_INFO_REQUEST_TIMEOUT_MS = 1000;
 let getThreadInfo;
+let getRunningIsolatedApplications;
 let awaitProcessGroupTermination;
 // Worker-side backstop that force-exits if the graceful shutdown sequence doesn't finish in time.
 let selfExitTimer;
@@ -127,6 +135,19 @@ module.exports = {
     workers,
     setMonitorListener,
     onMessageFromWorkers,
+    setIsolatedWorkerReconciler,
+    setRunningIsolatedApplicationsGetter,
+    isApplicationPrimaryWorker,
+    applicationWorkerIndex,
+    runsApplicationCodeSingletons,
+    isDedicatedWorker,
+    markBranchStorePath,
+    ownsStoreMaintenance,
+    ownsStoreExpiration,
+    workersForApplication,
+    stopWorker,
+    encodeRestartScope,
+    decodeRestartScope,
     onMessageByType,
     broadcast,
     broadcastWithAcknowledgement,
@@ -141,8 +162,11 @@ module.exports = {
     beginProcessShutdown,
     registerWorkerDataProvider,
     onThreadExit,
+    hasThreadExited,
+    notifyThreadExit,
     registerProcessGroup,
     unregisterProcessGroup,
+    isProcessGroupAlive,
     isThreadRunning,
     waitUntilConfirmedGone,
     restartNumber: workerData?.restartNumber || 1,
@@ -172,6 +196,9 @@ connectedPorts.sendToThread = function (threadId, message) {
         throw err;
     }
 };
+// Direct thread-to-thread send, so a worker can reach a sibling (e.g. the record lock owner worker)
+// without a hop through main. Returns false when no port for the thread is connected.
+module.exports.sendToThread = connectedPorts.sendToThread;
 module.exports.whenThreadsStarted = new Promise((resolve) => {
     module.exports.threadsHaveStarted = resolve;
 });
@@ -186,6 +213,106 @@ function setTerminateTimeout(newTimeout) {
 function getWorkerIndex() {
     return workerData ? workerData.workerIndex : isMainWorker ? 0 : undefined;
 }
+/**
+ * The worker that owns `applicationName`'s per-application singletons (its scheduled jobs, its data
+ * files): pool worker 0 for a shared application and for root-level plugins (which pass no name), the
+ * dedicated worker for an isolated one -- never index 0, so node-wide duties stay on the pool, and
+ * never a dedicated worker for anything but its own application.
+ */
+function isApplicationPrimaryWorker(applicationName) {
+    if (workerData?.isolatedApplication !== undefined)
+        return applicationName === workerData.isolatedApplication;
+    return getWorkerIndex() === 0;
+}
+/**
+ * Stores that exist only on this thread: the branch stores an isolated application opened in its
+ * dedicated worker. Registered by openBranchDatabase, so the ownership predicates below can tell them
+ * from the shared stores every thread has open.
+ */
+const branchStorePaths = new Set();
+function markBranchStorePath(path, isBranch = true) {
+    if (isBranch)
+        branchStorePaths.add(path);
+    else
+        branchStorePaths.delete(path);
+}
+/**
+ * Whether this thread runs the "last worker" per-store maintenance for `storePath` (TTL scans, storage
+ * reclamation, audit cleanup). Shared stores belong to the last pool worker; a dedicated worker
+ * maintains only its own branch stores, which no other thread has open.
+ */
+function ownsStoreMaintenance(storePath) {
+    if (workerData?.isolatedApplication !== undefined)
+        return branchStorePaths.has(storePath);
+    return getWorkerIndex() === getWorkerCount() - 1;
+}
+/** The "worker 0" counterpart: expiration eviction for `storePath`. */
+function ownsStoreExpiration(storePath) {
+    if (workerData?.isolatedApplication !== undefined)
+        return branchStorePaths.has(storePath);
+    return getWorkerIndex() === 0;
+}
+/**
+ * Whether a singleton set up by APPLICATION CODE runs here (a caching table's `sourcedFrom`
+ * subscription): pool worker 0, or a dedicated worker -- the only code that runs on one is its own
+ * application's, so whatever it sets up is that application's to run. Not for root-level plugins,
+ * which load on every thread and must use isApplicationPrimaryWorker(name).
+ */
+function runsApplicationCodeSingletons() {
+    return getWorkerIndex() === 0 || workerData?.isolatedApplication !== undefined;
+}
+/** Whether this thread is a worker dedicated to one isolated application. */
+function isDedicatedWorker() {
+    return workerData?.isolatedApplication !== undefined;
+}
+/** The worker index as the application sees it: a dedicated worker is its application's worker 0. */
+function applicationWorkerIndex() {
+    return workerData?.isolatedApplication !== undefined ? 0 : getWorkerIndex();
+}
+/** Every started worker dedicated to `application`, including a replacement still booting. */
+function workersForApplication(application) {
+    return workers.filter((worker) => worker.application === application);
+}
+/**
+ * Stop one worker for good: shut it down, honour the drain extension it may ask for, force it after
+ * the same backstop the rolling restart uses (FORCE_EXIT on Bun, where terminate() segfaults), and
+ * resolve once it has exited. Marked as shut down first so its exit does not start a replacement.
+ */
+function stopWorker(worker) {
+    worker.wasShutdown = true;
+    return new Promise((resolve) => {
+        const armTerminate = (delay) => setTimeout(() => {
+            harperLogger.warn('Thread did not voluntarily terminate, terminating from the outside', worker.threadId);
+            if (isBun) {
+                try {
+                    worker.postMessage({ type: FORCE_EXIT });
+                }
+                catch { }
+            }
+            else {
+                worker.terminate();
+            }
+        }, delay).unref();
+        let timeout = armTerminate(threadTerminationTimeout * 2);
+        worker.extendTerminateDeadline = (deadlineMs) => {
+            clearTimeout(timeout);
+            const { boundedTerminateDelay, getShutdownDrainCeilingMs } = require("../../components/shutdownDrain.js");
+            timeout = armTerminate(boundedTerminateDelay(deadlineMs, Date.now(), threadTerminationTimeout * 2, getShutdownDrainCeilingMs()));
+        };
+        worker.on('exit', () => {
+            clearTimeout(timeout);
+            worker.extendTerminateDeadline = undefined;
+            resolve();
+        });
+        try {
+            worker.postMessage({ restartNumber: module.exports.restartNumber, type: hdbTerms.ITC_EVENT_TYPES.SHUTDOWN });
+        }
+        catch {
+            clearTimeout(timeout);
+            resolve(); // already gone
+        }
+    });
+}
 function getWorkerCount() {
     return workerData ? workerData.workerCount : isMainWorker ? 1 : undefined;
 }
@@ -221,6 +348,7 @@ const RESERVED_WORKER_DATA_KEYS = [
     'processIncarnation',
     'ticketKeys',
     'noServerStart',
+    'isolatedApplication',
     '__proto__', // never a legitimate payload name; spread would define it as an own property
 ];
 const workerDataProviders = new Map();
@@ -280,14 +408,17 @@ function getTicketKeys() {
     ticketKeys = isMainThread ? randomBytes(48) : workerData.ticketKeys;
     return ticketKeys;
 }
+// What application code sees: a dedicated worker is its application's only worker, index 0 of 1, so
+// the documented partitioning idiom (`hash % server.workerCount === server.workerIndex`) covers
+// everything there. Node-wide duties keep using the raw getWorkerIndex/getWorkerCount.
 Object.defineProperty(server, 'workerIndex', {
     get() {
-        return getWorkerIndex();
+        return applicationWorkerIndex();
     },
 });
 Object.defineProperty(server, 'workerCount', {
     get() {
-        return getWorkerCount();
+        return workerData?.isolatedApplication !== undefined ? 1 : getWorkerCount();
     },
 });
 if (!parentPort) {
@@ -295,10 +426,29 @@ if (!parentPort) {
         if (worker)
             sendThreadInfo(worker);
     });
+    onMessageByType(REQUEST_RUNNING_ISOLATED_APPLICATIONS, (message, worker) => {
+        if (worker)
+            worker.postMessage({
+                type: RUNNING_ISOLATED_APPLICATIONS,
+                requestId: message.requestId,
+                applications: runningIsolatedApplicationsGetter(),
+            });
+    });
     onMessageByType(RESOURCE_REPORT, (message, worker) => {
         if (worker)
             recordResourceReport(worker, message);
     });
+    onMessageByType(OS_THREAD_ID, (message, worker) => {
+        if (worker)
+            worker.osThreadId = message.osThreadId;
+    });
+    onMessageByType(STUCK_WORKER_REPORT, (message) => {
+        for (const threadId of message.threadIds) {
+            const worker = workers.find((worker) => worker.threadId === threadId);
+            if (worker)
+                logStuckWorkerDiagnostics(worker);
+        }
+    });
     onMessageByType(AWAIT_PROCESS_GROUP_TERMINATION, async (message, worker) => {
         if (!worker)
             return;
@@ -311,6 +461,7 @@ if (!parentPort) {
 }
 // postMessage type listeners that are registered in other ways or can be registered later
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, null);
+listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.SCHEMA, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.USER, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST, null);
@@ -321,11 +472,12 @@ listenersByType.set(hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_RESPONSE, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_REGISTERED, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_REQUEST, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_RESPONSE, null);
-// getThreadInfo/awaitProcessGroupTermination register their own one-shot parentPort listener per
+// These request/response functions register their own one-shot parentPort listener per
 // call rather than going through onMessageByType, so without this, every reply would also reach
 // addPort's permanent dispatcher as an "unregistered" type: notifyMessageListeners would warn and
 // queue each one in messagesQueuedByType forever, and a lock-wait polls every 50ms.
 listenersByType.set(THREAD_INFO, null);
+listenersByType.set(RUNNING_ISOLATED_APPLICATIONS, null);
 listenersByType.set(PROCESS_GROUP_TERMINATION_CONFIRMED, null);
 function startWorker(path, options = {}) {
     if (processShuttingDown) {
@@ -346,8 +498,8 @@ function startWorker(path, options = {}) {
     let availableMemory = process.constrainedMemory?.() || totalmem(); // used constrained memory if it is available
     // and lower than total memory
     availableMemory = Math.min(availableMemory, totalmem(), 20000 * MB);
-    const maxOldMemory = envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_MAXHEAPMEMORY) ??
-        Math.max(Math.floor(availableMemory / MB / (10 + (options.threadCount || 1) / 4)), 512);
+    const maxOldMemory = resolveThreadHeapMemoryMb(envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_MAXHEAPMEMORY)) ??
+        Math.max(Math.floor(availableMemory / MB / (10 + (options.heapShareCount || options.threadCount || 1) / 4)), 512);
     // Max young memory space (semi-space for scavenger) is 1/128 of max memory (limited to 16-64). For most of our m5
     // machines this will be 64MB (less for t3's). This is based on recommendations from:
     // https://www.alibabacloud.com/blog/node-js-application-troubleshooting-manual---comprehensive-gc-problems-and-optimization594965
@@ -391,6 +543,9 @@ function startWorker(path, options = {}) {
         for (const requirePath of getRequireModules())
             execArgv.push('--require', requirePath);
     }
+    // Only a start that declares the serving topology may write it; see DESIGN.md on the two workerCounts.
+    if (typeof options.threadCount === 'number')
+        workerCount = options.threadCount;
     const worker = new Worker(isAbsolute(path) ? path : join(PACKAGE_ROOT, path), {
         resourceLimits: {
             maxOldGenerationSizeMb: maxOldMemory,
@@ -405,8 +560,9 @@ function startWorker(path, options = {}) {
             addThreadIds: channelsToConnect.map((channel) => channel.existingPort.threadId),
             addPortIsJobWorkers: channelsToConnect.map((channel) => channel.existingPort.isJobWorker === true),
             workerIndex: options.workerIndex,
-            workerCount: (workerCount = options.threadCount),
+            workerCount: options.threadCount,
             name: options.name,
+            isolatedApplication: options.application,
             restartNumber: module.exports.restartNumber,
             processIncarnation: module.exports.processIncarnation,
             ticketKeys: getTicketKeys(),
@@ -439,14 +595,19 @@ function startWorker(path, options = {}) {
     });
     worker.on('exit', (_code) => {
         workers.splice(workers.indexOf(worker), 1);
-        if (!processShuttingDown && !worker.wasShutdown && options.autoRestart !== false) {
+        if (!processShuttingDown &&
+            !worker.wasShutdown &&
+            options.autoRestart !== false &&
+            options.shouldAutoRestart?.(worker) !== false) {
             // if this wasn't an intentional shutdown, restart now (unless we have tried too many times)
             if (worker.unexpectedRestarts < MAX_UNEXPECTED_RESTARTS) {
                 options.unexpectedRestarts = worker.unexpectedRestarts + 1;
                 startWorker(path, options);
             }
-            else
-                harperLogger.error(`Thread has been restarted ${worker.restarts} times and will not be restarted`);
+            else {
+                harperLogger.error(`Thread has been restarted ${worker.unexpectedRestarts} times and will not be restarted`);
+                options.onRestartExhausted?.(worker);
+            }
         }
     });
     workers.push(worker);
@@ -454,6 +615,7 @@ function startWorker(path, options = {}) {
     if (options.onStarted)
         options.onStarted(worker); // notify that it is ready
     worker.name = options.name;
+    worker.application = options.application; // the isolated application this worker is dedicated to, if any
     return worker;
 }
 const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP];
@@ -465,13 +627,27 @@ const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP];
  * threads at the same time we shutdown new ones. However, we usually want to limit how many we do at once to avoid
  * excessive load and to keep things responsive. This parameter throttles the restarts to minimize load from
  * thread startups.
- * @returns {Promise<void>}
+ * @param onProgress Called each time a worker has been replaced (or its replacement has been given
+ * up on), so a caller waiting on a wide pool can tell a slow restart from a stalled one.
+ * @returns {Promise<{workersKeptOnOldCode: number, replacementsNotStarted: number}|{declined: true}|undefined>}
+ * from the main thread, how many workers were left running the old code and how many replacements
+ * never reported that they started — or `{declined: true}` when the process is already shutting down;
+ * from a worker, nothing — the restart is handed to the main thread.
+ */
+/**
+ * `application` selects which workers of the type restart: omitted preserves the historic all-worker
+ * behavior, explicit undefined restarts the shared pool only, a name restarts only that application's
+ * dedicated worker, and '*' restarts all.
  */
 async function restartWorkers(name = null, maxWorkersDown = Math.max(Math.floor(workerCount / 8), 1), // restart 1/8 of the threads at a time, but at least 1
-startReplacementThreads = true) {
+startReplacementThreads = true, onProgress = null, application = undefined) {
+    if (arguments.length < 5)
+        application = '*';
     if (isMainThread) {
+        // Declining is not the same as delegating: a caller reporting on the restart must not read this
+        // as "another thread is completing it".
         if (processShuttingDown && startReplacementThreads)
-            return;
+            return { declined: true };
         try {
             // we do this because it is possible for a component to chdir to itself, get re-deployed and then the cwd
             // inode link is invalid and it can cause a lot of problems. But process.cwd() still returns the path, for
@@ -481,22 +657,65 @@ startReplacementThreads = true) {
         catch (e) {
             harperLogger.error('Unable to reestablish current working directory', e);
         }
+        const freshlyStarted = new Set(); // dedicated workers the reconcile just started: already on the new code
         // problematic cyclic dependency, bind late
         const { resetRestartNeeded } = require("../../components/requestRestart.js");
-        resetRestartNeeded();
+        // One process-wide bit cannot say WHICH application is pending, so it may only be cleared by a
+        // restart that demonstrably covers every worker the bit could stand for. A restart scoped to one
+        // application leaves the whole pool on its old code; a pool restart replaces every pool worker and
+        // reconciles dedicated slots, but leaves an already-running dedicated worker on its old modules.
+        // Clearing from either would report restartRequired: false while a deployed component is still
+        // loaded nowhere. So: all workers always, the pool only while no dedicated worker is running --
+        // which is every node that uses no isolated application, i.e. the historic behavior unchanged.
+        const coversEveryWorker = application === '*' || (application === undefined && !workers.some((worker) => worker.application));
+        if (coversEveryWorker)
+            resetRestartNeeded();
         // This is here to prevent circular dependencies
         if (startReplacementThreads) {
             const { loadRootComponents } = require('../loadRootComponents.js');
-            await loadRootComponents();
+            // Installing and loading every root component reports nothing and can outlast a caller's idle
+            // window on its own (a cold npm cache, a large dependency graph), so beat while it runs. The
+            // caller's absolute ceiling is what bounds a load that never finishes.
+            const loading = setInterval(() => onProgress?.(), RESTART_PROGRESS_HEARTBEAT_MS).unref();
+            try {
+                await loadRootComponents();
+                // isolated applications added or removed by the reload get their dedicated worker started or
+                // stopped; a crash-looping newcomer can take a while, so the heartbeat runs through this too
+                try {
+                    for (const application of (await isolatedWorkerReconciler?.()) ?? [])
+                        freshlyStarted.add(application);
+                }
+                catch (error) {
+                    harperLogger.error('Could not reconcile isolated application workers', error);
+                }
+            }
+            finally {
+                clearInterval(loading);
+            }
+            onProgress?.();
         }
         module.exports.restartNumber++;
-        if (maxWorkersDown < 1) {
+        // `Infinity` is shutdownWorkers' "all at once" sentinel and must survive.
+        if (typeof maxWorkersDown !== 'number' || Number.isNaN(maxWorkersDown)) {
+            maxWorkersDown = 1;
+        }
+        else if (maxWorkersDown < 1) {
             // we accept a ratio of workers, and compute absolute maximum being down at a time from the total number of
             // threads
             maxWorkersDown = maxWorkersDown * workers.length;
         }
         // make a copy of the workers before iterating them, as the workers array mutates a lot during this
-        let waitingToFinish = []; // promises for workers we have shut down and are waiting to exit
+        let waitingToFinish = []; // promises for workers we are replacing, spliced as each is replaced
+        // Every replacement that was started without being awaited first, so this function can still
+        // resolve only once each one is accepting connections.
+        let replacementsStarting = [];
+        // A replacement that never came up leaves the pool in one of two very different states, and a
+        // caller waiting on this restart needs them apart: the pre-start path keeps the *old* worker
+        // serving old code, while a replacement that exits after its predecessor is already gone only
+        // costs capacity until startWorker's auto-restart brings a fresh one up.
+        let workersKeptOnOldCode = 0;
+        let replacementsNotStarted = 0;
+        let replacementsFailedToStart = 0;
         // We can only start the replacement *before* the old worker releases its port when the OS lets
         // both listen on the same port at once (SO_REUSEPORT). Without that — Windows (no SO_REUSEPORT),
         // macOS (unreliable SO_REUSEPORT, so workers bind exclusively), and Bun — the replacement can't
@@ -507,14 +726,21 @@ startReplacementThreads = true) {
         // replacement — an unavoidable brief gap, but only for worker-owned listeners, since the main
         // thread keeps serving the HTTP ports throughout. This ordering is also what lets
         // listenOnPorts() treat a dedicated listener's EADDRINUSE as an external conflict.
-        const canPreStartReplacement = process.platform !== 'win32' && process.platform !== 'darwin' && !isBun;
-        for (let worker of workers.slice(0)) {
+        const platformCanPreStartReplacement = process.platform !== 'win32' && process.platform !== 'darwin' && !isBun;
+        const restarting = workers.slice(0);
+        for (let index = 0; index < restarting.length; index++) {
+            const worker = restarting[index];
             // Terminal shutdown: stop replacing workers mid-loop — the guard for every replacement start below.
             if (processShuttingDown && startReplacementThreads)
                 break;
             if ((name && worker.name !== name) || worker.wasShutdown)
                 continue; // filter by type, if specified
+            if (application !== '*' && worker.application !== application)
+                continue; // and by isolated application
+            if (worker.application && freshlyStarted.has(worker.application))
+                continue;
             const overlapping = OVERLAPPING_RESTART_TYPES.indexOf(worker.name) > -1;
+            const canPreStartReplacement = platformCanPreStartReplacement && !worker.application;
             if (overlapping && startReplacementThreads && canPreStartReplacement) {
                 // Overlapping restart: start the replacement and wait until it is accepting connections
                 // *before* shutting down the worker it replaces. The replacement joins the (SO_REUSEPORT)
@@ -569,6 +795,8 @@ startReplacementThreads = true) {
                     // protection if it is still alive (it may have exited on its own during the wait).
                     if (workers.includes(worker))
                         worker.wasShutdown = false;
+                    workersKeptOnOldCode++;
+                    onProgress?.();
                     continue;
                 }
             }
@@ -591,8 +819,16 @@ startReplacementThreads = true) {
             // Overlapping types we couldn't pre-start (Windows/Bun): start the replacement now that the old
             // worker is releasing its port. server.close() stops accepting immediately, so the port frees up
             // well before the replacement finishes booting and binds.
-            if (overlapping && startReplacementThreads && !canPreStartReplacement && !processShuttingDown)
-                worker.startCopy();
+            let replacementStarting;
+            if (overlapping && startReplacementThreads && !canPreStartReplacement && !processShuttingDown) {
+                replacementStarting = whenWorkerStarted(worker.startCopy()).then((started) => {
+                    if (!started)
+                        replacementsFailedToStart++;
+                    onProgress?.();
+                    return started;
+                });
+                replacementsStarting.push(replacementStarting);
+            }
             let whenDone = new Promise((resolve) => {
                 // in case the exit inside the thread doesn't timeout, force it from the outside
                 const armTerminate = (delay) => setTimeout(() => {
@@ -620,39 +856,113 @@ startReplacementThreads = true) {
                     // buggy/rogue message can't defer the force-kill unboundedly; a shrink (drain-done reset)
                     // passes through untouched. See boundedTerminateDelay for the arithmetic + its unit tests.
                     const { boundedTerminateDelay, getShutdownDrainCeilingMs } = require("../../components/shutdownDrain.js");
-                    timeout = armTerminate(boundedTerminateDelay(deadlineMs, Date.now(), threadTerminationTimeout * 2, getShutdownDrainCeilingMs()));
+                    const delay = boundedTerminateDelay(deadlineMs, Date.now(), threadTerminationTimeout * 2, getShutdownDrainCeilingMs());
+                    timeout = armTerminate(delay);
+                    // The worker is telling us it has work still moving and how long it may take, so pass that
+                    // on: a caller waiting on the restart must not treat a live drain as a stalled one.
+                    onProgress?.(Date.now() + delay);
                 };
                 worker.on('exit', () => {
                     clearTimeout(timeout);
+                    onProgress?.();
                     worker.extendTerminateDeadline = undefined;
-                    const index = waitingToFinish.indexOf(whenDone);
-                    if (index > -1)
-                        waitingToFinish.splice(index, 1);
                     // non-overlapping types have no advance replacement, so start it once the old one is gone
                     if (!overlapping && startReplacementThreads && !processShuttingDown)
                         worker.startCopy();
                     resolve();
                 });
             });
-            waitingToFinish.push(whenDone);
+            // A worker counts as replaced only once its replacement is accepting connections, not merely
+            // once it has exited. This promise is held unawaited between throttle points, so it must not
+            // be able to reject.
+            const replaced = (replacementStarting ? Promise.all([whenDone, replacementStarting]) : whenDone)
+                .catch((error) => harperLogger.warn('Error waiting for a worker to be replaced', error))
+                .then(() => {
+                const at = waitingToFinish.indexOf(replaced);
+                if (at > -1)
+                    waitingToFinish.splice(at, 1);
+            });
+            waitingToFinish.push(replaced);
             if (waitingToFinish.length >= maxWorkersDown) {
-                // throttle how many workers are draining/down at once to limit load
+                // throttle how many workers are down at once to limit load
                 await Promise.race(waitingToFinish);
             }
+            // Readiness throttling bounds how many workers are down at once, but not how many *fail*: with
+            // replacements that never come up, walking the rest of the pool would leave nothing serving.
+            // The workers not yet touched are still running the old code, which beats none running at all.
+            if (replacementsFailedToStart >= maxWorkersDown) {
+                const untouched = restarting
+                    .slice(index + 1)
+                    // a worker that exited on its own mid-restart is spliced out of `workers` and
+                    // auto-restarted onto the new code (see the exit handler above); it is not still on
+                    // the previous code even though this loop never got to it.
+                    .filter((other) => (!name || other.name === name) && !other.wasShutdown && workers.includes(other)).length;
+                harperLogger.error(`${replacementsFailedToStart} replacement worker thread(s) did not start; stopping this restart with ${untouched} worker(s) still on the previous code`);
+                workersKeptOnOldCode += untouched;
+                break;
+            }
         }
-        // seems appropriate to wait for this to finish, but the API doesn't actually wait for this function
-        // to finish, so not that important
         await Promise.all(waitingToFinish);
+        // A caller awaiting this needs it to mean "the pool is serving the new code", so wait out the
+        // replacements that could only be started once their predecessor released its exclusive ports.
+        replacementsNotStarted = (await Promise.all(replacementsStarting)).filter((started) => !started).length;
+        return { workersKeptOnOldCode, replacementsNotStarted };
     }
     else {
         parentPort.postMessage({
             type: RESTART_TYPE,
             workerType: name,
+            scope: encodeRestartScope(application),
         });
     }
 }
+/**
+ * Resolve once a newly started worker reports that it is accepting connections, or gives up on it.
+ * There is no old worker left to fall back on here, so a replacement that fails to start is left to
+ * startWorker's own auto-restart handling; this only stops waiting on it.
+ * @param newWorker The replacement worker
+ * @returns {Promise<boolean>} whether the worker reported that it started
+ */
+function whenWorkerStarted(newWorker) {
+    return new Promise((resolve) => {
+        const cleanup = () => {
+            clearTimeout(timeout);
+            newWorker.off('message', startListener);
+            newWorker.off('exit', exitListener);
+        };
+        const timeout = setTimeout(() => {
+            harperLogger.error('Replacement worker did not start in time', newWorker.threadId);
+            cleanup();
+            // Its predecessor is already gone, so a replacement wedged in boot is a worker slot serving
+            // nothing until the process restarts. Stop it and let startWorker's exit handling replace it.
+            if (isBun) {
+                // terminate() triggers a NAPI segfault in Bun; ask the worker to self-exit instead.
+                try {
+                    newWorker.postMessage({ type: FORCE_EXIT });
+                }
+                catch { }
+            }
+            else
+                newWorker.terminate();
+            resolve(false);
+        }, Math.max(threadTerminationTimeout * 2, 60000)).unref();
+        const startListener = (message) => {
+            if (message.type === hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED) {
+                cleanup();
+                resolve(true);
+            }
+        };
+        const exitListener = () => {
+            harperLogger.warn('Replacement worker exited before starting', newWorker.threadId);
+            cleanup();
+            resolve(false);
+        };
+        newWorker.on('message', startListener);
+        newWorker.on('exit', exitListener);
+    });
+}
 function shutdownWorkers(name) {
-    return restartWorkers(name, Infinity, false);
+    return restartWorkers(name, Infinity, false, null, '*');
 }
 function beginProcessShutdown() {
     processShuttingDown = true;
@@ -674,6 +984,27 @@ async function shutdownWorkersNow(name) {
         await Promise.all(workers.map((worker) => worker.terminate()));
     }
 }
+/**
+ * The restart scope on the wire: a worker cannot post `undefined` and have it mean "the pool" (a
+ * message with no scope at all means "everything", as every pre-existing sender intends).
+ */
+function encodeRestartScope(application) {
+    return application === undefined ? '' : application; // '' is not a legal application name
+}
+function decodeRestartScope(message) {
+    if (message.scope === undefined)
+        return '*';
+    return message.scope === '' ? undefined : message.scope;
+}
+let isolatedWorkerReconciler = null;
+let runningIsolatedApplicationsGetter = () => [];
+/** Registered by socketRouter: starts and stops dedicated workers to match the root config's isolated applications. */
+function setIsolatedWorkerReconciler(reconcile) {
+    isolatedWorkerReconciler = reconcile;
+}
+function setRunningIsolatedApplicationsGetter(getter) {
+    runningIsolatedApplicationsGetter = getter;
+}
 const messageListeners = [];
 function onMessageFromWorkers(listener) {
     messageListeners.push(listener);
@@ -784,15 +1115,146 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS)
                 timer = undefined;
                 const stuck = [];
                 for (let ackHandler of [...pending]) {
-                    stuck.push(ackHandler.port?.threadId);
+                    stuck.push(ackHandler.port);
                     ackHandler(); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves
                 }
-                harperLogger.warn(`ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort`);
+                harperLogger.warn(`ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.map((port) => port?.threadId).join(', ')} within ${timeout}ms; proceeding best-effort`);
+                if (isMainThread)
+                    for (let port of stuck)
+                        logStuckWorkerDiagnostics(port);
+                else if (parentPort) {
+                    // Only main holds the Worker objects (and tids), so a worker-originated timeout is sampled there.
+                    const threadIds = stuck.map((port) => port?.threadId).filter((threadId) => threadId > 0);
+                    if (threadIds.length > 0)
+                        parentPort.postMessage({ type: STUCK_WORKER_REPORT, threadIds });
+                }
             }, timeout);
             timer.unref?.();
         }
     });
 }
+// Linux only: /proc/thread-self resolves to <pid>/task/<tid> for the calling thread.
+function getOsThreadId() {
+    try {
+        const tid = Number(readlinkSync('/proc/thread-self').split('/').pop());
+        return Number.isInteger(tid) && tid > 0 ? tid : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function readTaskFile(tid, name) {
+    try {
+        return readFileSync(`/proc/self/task/${tid}/${name}`, 'utf8').trim();
+    }
+    catch {
+        return undefined;
+    }
+}
+// Kernel-side view of one thread. Every field is best-effort and reported individually, since
+// wchan/syscall need ptrace read access that a hardened container may deny while stat is open.
+function finiteOrUndefined(value) {
+    const number = Number(value);
+    return Number.isFinite(number) ? number : undefined;
+}
+function readOsThreadState(tid) {
+    const thread = { tid };
+    const stat = readTaskFile(tid, 'stat');
+    if (stat !== undefined) {
+        // Fields after the parenthesized comm, so state is [0], utime/stime [11]/[12], starttime [19].
+        const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+        thread.state = fields[0];
+        const utime = finiteOrUndefined(fields[11]);
+        const stime = finiteOrUndefined(fields[12]);
+        if (utime !== undefined && stime !== undefined)
+            thread.cpuTicks = utime + stime;
+        thread.startTime = fields[19];
+    }
+    const wchan = readTaskFile(tid, 'wchan');
+    if (wchan !== undefined)
+        thread.wchan = wchan;
+    // First token only: the syscall number, "running", or -1 (blocked outside a syscall). The
+    // remainder is argument registers and the stack/instruction pointers, which don't belong in a log.
+    const syscall = readTaskFile(tid, 'syscall');
+    if (syscall !== undefined)
+        thread.syscall = syscall.split(' ')[0];
+    const status = readTaskFile(tid, 'status');
+    if (status !== undefined) {
+        thread.voluntarySwitches = finiteOrUndefined(/^voluntary_ctxt_switches:\s*(\d+)/m.exec(status)?.[1]);
+        thread.nonvoluntarySwitches = finiteOrUndefined(/^nonvoluntary_ctxt_switches:\s*(\d+)/m.exec(status)?.[1]);
+    }
+    return thread;
+}
+function snapshotWorkerThread(worker) {
+    const snapshot = { at: Date.now() };
+    if (worker.resources?.updated)
+        snapshot.sinceResourceReport = snapshot.at - worker.resources.updated;
+    const eventLoop = worker.performance?.eventLoopUtilization?.();
+    if (eventLoop)
+        snapshot.eventLoop = { idle: eventLoop.idle, active: eventLoop.active };
+    if (worker.osThreadId)
+        snapshot.osThread = readOsThreadState(worker.osThreadId);
+    return snapshot;
+}
+function describeThreadState(thread) {
+    return `state=${thread.state ?? '?'} wchan=${thread.wchan ?? '?'} syscall=${thread.syscall ?? '?'}`;
+}
+function describeSnapshot(snapshot) {
+    const parts = [
+        snapshot.sinceResourceReport === undefined
+            ? 'no resource report received'
+            : `last resource report ${snapshot.sinceResourceReport}ms ago`,
+    ];
+    if (snapshot.eventLoop)
+        parts.push(`event loop active ${Math.round(snapshot.eventLoop.active)}ms idle ${Math.round(snapshot.eventLoop.idle)}ms`);
+    const thread = snapshot.osThread;
+    if (!thread)
+        parts.push('os thread state unavailable');
+    else
+        parts.push(`os tid ${thread.tid} ${describeThreadState(thread)} cpuTicks=${thread.cpuTicks ?? '?'} ctxtSwitches=${thread.voluntarySwitches ?? '?'}/${thread.nonvoluntarySwitches ?? '?'}`);
+    return parts.join('; ');
+}
+function describeDelta(before, after) {
+    return Number.isFinite(before) && Number.isFinite(after) ? `+${after - before}` : '?';
+}
+function describeProgress(first, second) {
+    const parts = [];
+    if (first.eventLoop && second.eventLoop)
+        parts.push(`event loop active +${Math.round(second.eventLoop.active - first.eventLoop.active)}ms idle +${Math.round(second.eventLoop.idle - first.eventLoop.idle)}ms`);
+    const before = first.osThread;
+    const after = second.osThread;
+    if (before && after)
+        parts.push(`cpuTicks ${describeDelta(before.cpuTicks, after.cpuTicks)} ctxtSwitches ${describeDelta(before.voluntarySwitches, after.voluntarySwitches)}/${describeDelta(before.nonvoluntarySwitches, after.nonvoluntarySwitches)} ${describeThreadState(after)}`);
+    return parts.join('; ');
+}
+// A blocked event loop cannot report itself; two kernel-state samples a second apart separate
+// "parked on a lock" (no CPU ticks, no context switches) from "spinning".
+const STUCK_WORKER_SAMPLE_INTERVAL_MS = 1000;
+const STUCK_WORKER_DIAGNOSTIC_COOLDOWN_MS = 30000;
+function logStuckWorkerDiagnostics(worker) {
+    if (!worker || !workers.includes(worker))
+        return;
+    const now = Date.now();
+    if (worker.stuckDiagnosticAt !== undefined && now - worker.stuckDiagnosticAt < STUCK_WORKER_DIAGNOSTIC_COOLDOWN_MS)
+        return;
+    worker.stuckDiagnosticAt = now;
+    const threadId = worker.threadId; // Node resets it to -1 once the worker exits
+    const first = snapshotWorkerThread(worker);
+    harperLogger.warn(`Worker thread ${threadId} at ack timeout: ${describeSnapshot(first)}`);
+    if (!first.eventLoop && !first.osThread)
+        return;
+    setTimeout(() => {
+        if (!workers.includes(worker)) {
+            harperLogger.warn(`Worker thread ${threadId} exited before its follow-up sample`);
+            return;
+        }
+        const second = snapshotWorkerThread(worker);
+        // A recycled tid after a thread exit would attribute another thread's activity to this worker.
+        if (second.osThread && second.osThread.startTime !== first.osThread?.startTime)
+            second.osThread = undefined;
+        harperLogger.warn(`Worker thread ${threadId} over the next ${second.at - first.at}ms: ${describeProgress(first, second) || 'no further state available'}`);
+    }, STUCK_WORKER_SAMPLE_INTERVAL_MS).unref();
+}
 function sendThreadInfo(targetWorker) {
     targetWorker.postMessage({
         type: THREAD_INFO,
@@ -804,6 +1266,7 @@ function getChildWorkerInfo() {
     return workers.map((worker) => ({
         threadId: worker.threadId,
         name: worker.name,
+        application: worker.application,
         heapTotal: worker.resources?.heapTotal,
         heapUsed: worker.resources?.heapUsed,
         externalMemory: worker.resources?.external,
@@ -864,6 +1327,9 @@ if (parentPort && workerData?.addPorts) {
     // parentPort so sendToThread(0, ...) and similar lookups can route back to main.
     parentPort.threadId = 0;
     addPort(parentPort);
+    const osThreadId = getOsThreadId();
+    if (osThreadId !== undefined)
+        parentPort.postMessage({ type: OS_THREAD_ID, osThreadId });
     for (let i = 0, l = workerData.addPorts.length; i < l; i++) {
         let port = workerData.addPorts[i];
         port.threadId = workerData.addThreadIds[i];
@@ -912,17 +1378,56 @@ if (parentPort && workerData?.addPorts) {
             }, timeoutMs);
         }
     });
+    let nextRunningIsolatedApplicationsRequestId = 0;
+    getRunningIsolatedApplications = (timeoutMs) => new Promise((resolve, reject) => {
+        const requestId = ++nextRunningIsolatedApplicationsRequestId;
+        let timeout;
+        parentPort.on('message', receiveApplications);
+        try {
+            parentPort.postMessage({ type: REQUEST_RUNNING_ISOLATED_APPLICATIONS, requestId });
+        }
+        catch (error) {
+            cleanup();
+            reject(error);
+            return;
+        }
+        function receiveApplications(message) {
+            if (message.type === RUNNING_ISOLATED_APPLICATIONS && message.requestId === requestId) {
+                cleanup();
+                resolve(message.applications);
+            }
+        }
+        function cleanup() {
+            if (timeout)
+                clearTimeout(timeout);
+            parentPort.off('message', receiveApplications);
+        }
+        if (timeoutMs != null) {
+            timeout = setTimeout(() => {
+                cleanup();
+                const error = new Error(`Timed out waiting for isolated application topology after ${timeoutMs}ms`);
+                error.code = 'ERR_ISOLATED_APPLICATIONS_TIMEOUT';
+                reject(error);
+            }, timeoutMs);
+        }
+    });
     let awaitTerminationRequestId = 0;
-    awaitProcessGroupTermination = (ownerThreadId) => new Promise((resolve) => {
+    awaitProcessGroupTermination = (ownerThreadId, signal) => new Promise((resolve) => {
         // Deliberately no timeout: a contender must not reclaim a dead owner's lock while its
         // process group might still be alive and mutating files, so this mirrors the unbounded
-        // wait Application.ts's waitForConfirmedTermination uses for the same reason.
+        // wait Application.ts's waitForConfirmedTermination uses for the same reason. A caller
+        // that needs its own bound (isThreadRunning) passes `signal` so this listener still gets
+        // torn down when that caller gives up, instead of leaking for the life of the worker.
         const requestId = ++awaitTerminationRequestId;
         parentPort.on('message', receiveConfirmation);
+        signal?.addEventListener('abort', cleanup, { once: true });
         parentPort.postMessage({ type: AWAIT_PROCESS_GROUP_TERMINATION, ownerThreadId, requestId });
+        function cleanup() {
+            parentPort.off('message', receiveConfirmation);
+        }
         function receiveConfirmation(message) {
             if (message.type === PROCESS_GROUP_TERMINATION_CONFIRMED && message.requestId === requestId) {
-                parentPort.off('message', receiveConfirmation);
+                cleanup();
                 resolve();
             }
         }
@@ -930,9 +1435,17 @@ if (parentPort && workerData?.addPorts) {
 }
 else {
     getThreadInfo = getChildWorkerInfo;
+    getRunningIsolatedApplications = isMainThread
+        ? () => runningIsolatedApplicationsGetter()
+        : () => {
+            const error = new Error('No channel to the main thread for isolated application topology');
+            error.code = 'ERR_ISOLATED_APPLICATIONS_UNAVAILABLE';
+            return Promise.reject(error);
+        };
     awaitProcessGroupTermination = (ownerThreadId) => pendingProcessGroupTerminations.get(ownerThreadId) ?? Promise.resolve();
 }
 module.exports.getThreadInfo = getThreadInfo;
+module.exports.getRunningIsolatedApplications = getRunningIsolatedApplications;
 // Listeners notified when a connected thread's port closes (worker exit/restart), so
 // modules holding per-thread state (e.g. registeredOperations' registry and in-flight
 // forwards) can clean up.
@@ -953,34 +1466,144 @@ const processGroupsByThread = new Map();
 // window component-preparation locking exists to close.
 const pendingProcessGroupTerminations = new Map();
 const PROCESS_GROUP_TERMINATION_POLL_MS = 25;
-// A process-group leader spawned by a worker thread's own event loop is normally reaped by that
-// same loop when it dies. But the only caller of this check (waitForProcessGroupExit, for a
-// *dead* owner thread) runs after that loop is already gone, so nothing will ever reap it here:
-// kill(pid, 0) keeps succeeding forever against the zombie's still-allocated PID slot. A zombie
-// can no longer touch the filesystem, so treat a confirmed zombie as dead rather than poll
-// forever for a reap that will never come. Reproduced by killing a worker mid-install: the
-// installer's process-group leader becomes a permanent `Z` in /proc without this check.
-function isZombieProcessGroupLeader(processGroupId) {
-    if (process.platform !== 'linux')
-        return false;
+const ZOMBIE_GROUP_MEMBER_SCAN_INTERVAL_MS = 1000;
+const PROCESS_GROUP_LIVENESS_WARNING_MS = 30000;
+// Bounds isThreadRunning's own wait (below); Application.ts's waitForConfirmedTermination stays
+// deliberately unbounded.
+const THREAD_RUNNING_TERMINATION_BACKSTOP_MS = PROCESS_GROUP_LIVENESS_WARNING_MS;
+const zombieGroupScanTimes = new Map();
+const processGroupLivenessStates = new Map();
+function processGroupExists(processGroupId) {
+    try {
+        process.kill(-processGroupId, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+}
+function processProbeError(error) {
+    return error?.code ?? error?.message ?? String(error ?? 'unknown error');
+}
+function processGroupLeaderState(processGroupId, platform, readStat) {
+    if (platform !== 'linux') {
+        return { state: 'unknown', reason: `zombie-process detection is unavailable on ${platform}` };
+    }
     let stat;
     try {
-        stat = readFileSync(`/proc/${processGroupId}/stat`, 'utf8');
+        stat = readStat(`/proc/${processGroupId}/stat`, 'utf8');
     }
-    catch {
-        return false;
+    catch (error) {
+        if (error.code === 'ENOENT' || error.code === 'ESRCH')
+            return { state: 'missing' };
+        return {
+            state: 'unknown',
+            reason: `reading /proc/${processGroupId}/stat failed (${processProbeError(error)})`,
+        };
     }
-    // Format is "pid (comm) state ..."; comm can contain spaces/parens, so anchor on the last ')'.
-    return stat[stat.lastIndexOf(')') + 2] === 'Z';
+    const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+    if (Number(fields[2]) !== processGroupId)
+        return { state: 'missing' };
+    return { state: fields[0] === 'Z' ? 'zombie' : 'alive' };
 }
-function processGroupIsAlive(processGroupId) {
+function scanLinuxProcessGroup(processGroupId, readDirectory, readStat) {
+    let processIds;
     try {
-        process.kill(-processGroupId, 0);
+        processIds = readDirectory('/proc');
     }
     catch (error) {
-        return error.code === 'EPERM';
+        return {
+            isAlive: null,
+            reason: `reading /proc failed (${processProbeError(error)})`,
+        };
+    }
+    for (const processId of processIds) {
+        if (!/^\d+$/.test(processId))
+            continue;
+        let stat;
+        try {
+            stat = readStat(`/proc/${processId}/stat`, 'utf8');
+        }
+        catch (error) {
+            if (error.code === 'ENOENT' || error.code === 'ESRCH')
+                continue; // already gone
+            // EACCES/EPERM (e.g. hidepid/ProtectProc) can only happen on a pid we don't own — we
+            // spawned our own group's members as this same user, so their stat is always readable —
+            // so it's not one of ours. Anything else is genuinely unknown, not "not ours".
+            if (error.code === 'EACCES' || error.code === 'EPERM')
+                continue;
+            return {
+                isAlive: null,
+                reason: `reading /proc/${processId}/stat failed (${processProbeError(error)})`,
+            };
+        }
+        const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+        const pgrp = Number(fields[2]);
+        if (!Number.isFinite(pgrp)) {
+            return { isAlive: null, reason: `/proc/${processId}/stat had an unparseable process-group field` };
+        }
+        if (pgrp !== processGroupId)
+            continue;
+        if (fields[0] !== 'Z')
+            return { isAlive: true, reason: `a live member was observed at pid ${processId}` };
     }
-    return !isZombieProcessGroupLeader(processGroupId);
+    return { isAlive: false };
+}
+function keepProcessGroupAlive(processGroupId, reason, observationTime, warn) {
+    let livenessState = processGroupLivenessStates.get(processGroupId);
+    if (!livenessState) {
+        livenessState = { observedAt: observationTime, reason, warned: false };
+        processGroupLivenessStates.set(processGroupId, livenessState);
+    }
+    else {
+        livenessState.reason = reason;
+    }
+    if (!livenessState.warned && observationTime - livenessState.observedAt >= PROCESS_GROUP_LIVENESS_WARNING_MS) {
+        livenessState.warned = true;
+        warn(`Process group ${processGroupId} termination remains unconfirmed after ${PROCESS_GROUP_LIVENESS_WARNING_MS}ms: ${livenessState.reason}`);
+    }
+    return true;
+}
+function clearProcessGroupLivenessState(processGroupId) {
+    zombieGroupScanTimes.delete(processGroupId);
+    processGroupLivenessStates.delete(processGroupId);
+}
+function isProcessGroupAlive(processGroupId, options) {
+    const platform = options?.platform ?? process.platform;
+    const groupExists = options?.processGroupExists ?? processGroupExists;
+    const readDirectory = options?.readDirectory ?? readdirSync;
+    const readStat = options?.readStat ?? readFileSync;
+    const observationTime = options?.now?.() ?? performance.now();
+    const warn = options?.warn ?? ((message) => harperLogger.warn(message));
+    if (!groupExists(processGroupId)) {
+        clearProcessGroupLivenessState(processGroupId);
+        return false;
+    }
+    const leaderState = processGroupLeaderState(processGroupId, platform, readStat);
+    if (leaderState.state === 'alive')
+        return keepProcessGroupAlive(processGroupId, 'the process-group leader is still alive', observationTime, warn);
+    if (leaderState.state === 'unknown')
+        return keepProcessGroupAlive(processGroupId, leaderState.reason, observationTime, warn);
+    const scanTime = observationTime;
+    const lastScan = zombieGroupScanTimes.get(processGroupId);
+    if (lastScan !== undefined && scanTime >= lastScan && scanTime - lastScan < ZOMBIE_GROUP_MEMBER_SCAN_INTERVAL_MS) {
+        const reason = processGroupLivenessStates.get(processGroupId)?.reason ?? 'the previous Linux scan is still current';
+        return keepProcessGroupAlive(processGroupId, reason, observationTime, warn);
+    }
+    zombieGroupScanTimes.set(processGroupId, scanTime);
+    let scanResult = scanLinuxProcessGroup(processGroupId, readDirectory, readStat);
+    if (scanResult.isAlive !== false)
+        return keepProcessGroupAlive(processGroupId, scanResult.reason, observationTime, warn);
+    // A second snapshot catches a member omitted after forking inside the first readdir/stat window.
+    // It narrows rather than closes the race because the second synchronous pass has the same gap.
+    scanResult = scanLinuxProcessGroup(processGroupId, readDirectory, readStat);
+    if (scanResult.isAlive !== false)
+        return keepProcessGroupAlive(processGroupId, scanResult.reason, observationTime, warn);
+    clearProcessGroupLivenessState(processGroupId);
+    return false;
+}
+function processGroupIsAlive(processGroupId) {
+    return isProcessGroupAlive(processGroupId);
 }
 async function waitForProcessGroupExit(processGroupId) {
     while (processGroupIsAlive(processGroupId))
@@ -1054,6 +1677,7 @@ function addProcessGroup(ownerThreadId, processGroupId) {
     processGroups.add(processGroupId);
 }
 function removeProcessGroup(ownerThreadId, processGroupId) {
+    clearProcessGroupLivenessState(processGroupId);
     const processGroups = processGroupsByThread.get(ownerThreadId);
     if (!processGroups)
         return;
@@ -1107,6 +1731,39 @@ function unregisterProcessGroup(processGroupId) {
     else
         parentPort?.postMessage({ type: UNREGISTER_PROCESS_GROUP, processGroupId });
 }
+class ProcessGroupTerminationUnconfirmedError extends Error {
+    code = 'ERR_PROCESS_GROUP_TERMINATION_UNCONFIRMED';
+    constructor(ownerThreadId, timeoutMs) {
+        super(`Thread ${ownerThreadId}'s process-group termination could not be confirmed after ${timeoutMs}ms`);
+        this.name = 'ProcessGroupTerminationUnconfirmedError';
+    }
+}
+// Bounds awaitProcessGroupTermination for isThreadRunning below. Throws rather than resolving to
+// "not running": componentPreparationLock's ownerIsAlive/ownerLivenessConfirmed already treat a
+// throwing isOwnerAlive as "can't confirm — don't steal the claim, don't renew the deadline", so
+// the lock's own bounded wait is what eventually fails the waiter.
+async function awaitConfirmedProcessGroupTermination(ownerThreadId) {
+    const abortController = new AbortController();
+    let timedOut = false;
+    const timer = setTimeout(() => {
+        timedOut = true;
+        abortController.abort();
+    }, THREAD_RUNNING_TERMINATION_BACKSTOP_MS);
+    timer.unref?.();
+    try {
+        await Promise.race([
+            awaitProcessGroupTermination(ownerThreadId, abortController.signal),
+            new Promise((resolve) => abortController.signal.addEventListener('abort', resolve, { once: true })),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+    if (timedOut) {
+        harperLogger.warn(`Thread ${ownerThreadId}'s process-group termination could not be confirmed after ${THREAD_RUNNING_TERMINATION_BACKSTOP_MS}ms`);
+        throw new ProcessGroupTerminationUnconfirmedError(ownerThreadId, THREAD_RUNNING_TERMINATION_BACKSTOP_MS);
+    }
+}
 async function isThreadRunning(ownerThreadId, timeoutMs = THREAD_INFO_REQUEST_TIMEOUT_MS) {
     if (ownerThreadId === threadId || ownerThreadId === 0)
         return true;
@@ -1114,7 +1771,7 @@ async function isThreadRunning(ownerThreadId, timeoutMs = THREAD_INFO_REQUEST_TI
         return true;
     // The thread itself is gone, but it may still own process groups whose forced termination is
     // in flight — wait for that to be confirmed before reporting the owner as reclaimable.
-    await awaitProcessGroupTermination(ownerThreadId);
+    await awaitConfirmedProcessGroupTermination(ownerThreadId);
     return false;
 }
 if (isMainThread) {
@@ -1123,6 +1780,14 @@ if (isMainThread) {
             terminateProcessGroupsForThread(ownerThreadId);
     });
 }
+/**
+ * Whether a thread has already been reported dead. Sync, unlike `isThreadRunning`, because it only
+ * reads the tombstone `notifyThreadExit` records below — callers on the exit path need an answer
+ * without awaiting process-group confirmation.
+ */
+function hasThreadExited(threadId) {
+    return notifiedDeadThreadIds.has(threadId);
+}
 function notifyThreadExit(deadThreadId) {
     if (deadThreadId == null || notifiedDeadThreadIds.has(deadThreadId))
         return;
@@ -1246,25 +1911,51 @@ if (isMainThread) {
     const watchDir = async (dir, beforeRestartCallback) => {
         if (beforeRestartCallback)
             beforeRestart = beforeRestartCallback;
-        chokidar
-            .watch(dir, {
-            persistent: false,
-            ignored: (path) => {
-                return ignoredPaths.some((ignoredPath) => path.includes(ignoredPath));
-            },
-        })
-            .on('change', (path) => {
-            changedFiles.add(path);
-            if (queuedRestart)
-                clearTimeout(queuedRestart);
-            queuedRestart = setTimeout(async () => {
-                if (beforeRestart)
-                    await beforeRestart();
-                await restartWorkers();
-                console.log('Reloaded Harper components, changed files:', Array.from(changedFiles));
-                changedFiles.clear();
-            }, 100);
-        });
+        const watchTarget = resolveWatchTarget(dir);
+        let usingPolling = watchTarget.mustPoll;
+        let liveWatcher;
+        const openWatcher = () => {
+            const opened = (liveWatcher = guardedWatch(watchTarget.path, {
+                persistent: false,
+                ...(usingPolling ? DIRECTORY_POLLING_FALLBACK_OPTIONS : {}),
+                ignored: (path) => {
+                    return ignoredPaths.some((ignoredPath) => path.includes(ignoredPath));
+                },
+            }));
+            opened
+                // This runs on the thread that owns every worker, and chokidar emits 'error' unguarded for
+                // anything but ENOENT/ENOTDIR.
+                .on('error', (error) => {
+                if (claimLostNativeWatchError(error))
+                    return;
+                if (isWatcherExhaustionError(error)) {
+                    if (usingPolling || liveWatcher !== opened)
+                        return;
+                    warnWatcherFallback(dir);
+                    usingPolling = true;
+                    Promise.resolve()
+                        .then(() => opened.close())
+                        .catch(() => { })
+                        .then(openWatcher)
+                        .catch((reopenError) => console.error(`Could not reopen the ${dir} component-reload watch on polling:`, reopenError));
+                    return;
+                }
+                console.error(`Error watching ${dir} for component reloads:`, error);
+            })
+                .on('change', (path) => {
+                changedFiles.add(path);
+                if (queuedRestart)
+                    clearTimeout(queuedRestart);
+                queuedRestart = setTimeout(async () => {
+                    if (beforeRestart)
+                        await beforeRestart();
+                    await restartWorkers(undefined, undefined, true, null, '*');
+                    console.log('Reloaded Harper components, changed files:', Array.from(changedFiles));
+                    changedFiles.clear();
+                }, 100);
+            });
+        };
+        openWatcher();
     };
     module.exports.watchDir = watchDir;
     if (process.env.WATCH_DIR)
@@ -1282,4 +1973,7 @@ else {
         realExit(0);
     });
 }
+// Required here, not from logging, which must never import this module; last in the file because it
+// calls onThreadExit.
+require("./logRotationTransport.js");
 //# sourceMappingURL=manageThreads.js.map