@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.1

package/dist/server/threads/manageThreads.js

@@ -48,10 +48,12 @@ let threadTerminationTimeout = process.env.DEV_MODE === 'true' || process.env.DE
 const RESTART_TYPE = 'restart';
 const RESTART_PROGRESS_HEARTBEAT_MS = 15000;
 const REQUEST_THREAD_INFO = 'request_thread_info';
+const REQUEST_RUNNING_ISOLATED_APPLICATIONS = 'request-running-isolated-applications';
 const RESOURCE_REPORT = 'resource_report';
 const OS_THREAD_ID = 'os-thread-id';
 const STUCK_WORKER_REPORT = 'stuck-worker-report';
 const THREAD_INFO = 'thread_info';
+const RUNNING_ISOLATED_APPLICATIONS = 'running-isolated-applications';
 const ADDED_PORT = 'added-port';
 const ACKNOWLEDGEMENT = 'ack';
 const REMOVE_PORT = 'remove-port';
@@ -69,6 +71,7 @@ const AWAIT_PROCESS_GROUP_TERMINATION = 'await-process-group-termination';
 const PROCESS_GROUP_TERMINATION_CONFIRMED = 'process-group-termination-confirmed';
 const THREAD_INFO_REQUEST_TIMEOUT_MS = 1000;
 let getThreadInfo;
+let getRunningIsolatedApplications;
 let awaitProcessGroupTermination;
 // Worker-side backstop that force-exits if the graceful shutdown sequence doesn't finish in time.
 let selfExitTimer;
@@ -132,6 +135,19 @@ module.exports = {
     workers,
     setMonitorListener,
     onMessageFromWorkers,
+    setIsolatedWorkerReconciler,
+    setRunningIsolatedApplicationsGetter,
+    isApplicationPrimaryWorker,
+    applicationWorkerIndex,
+    runsApplicationCodeSingletons,
+    isDedicatedWorker,
+    markBranchStorePath,
+    ownsStoreMaintenance,
+    ownsStoreExpiration,
+    workersForApplication,
+    stopWorker,
+    encodeRestartScope,
+    decodeRestartScope,
     onMessageByType,
     broadcast,
     broadcastWithAcknowledgement,
@@ -154,6 +170,11 @@ module.exports = {
     isThreadRunning,
     waitUntilConfirmedGone,
     restartNumber: workerData?.restartNumber || 1,
+    // Identifies this process incarnation, where the PID cannot: a container reuses PID 1. Minted once
+    // on the main thread and carried to workers, so live siblings agree on it — one derived per thread
+    // would not. `undefined` on a worker started without it; consumers must fall back, not treat that
+    // as a mismatch.
+    processIncarnation: workerData ? workerData.processIncarnation : randomBytes(8).toString('hex'),
 };
 connectedPorts.onMessageByType = onMessageByType;
 connectedPorts.sendToThread = function (threadId, message) {
@@ -175,6 +196,9 @@ connectedPorts.sendToThread = function (threadId, message) {
         throw err;
     }
 };
+// Direct thread-to-thread send, so a worker can reach a sibling (e.g. the record lock owner worker)
+// without a hop through main. Returns false when no port for the thread is connected.
+module.exports.sendToThread = connectedPorts.sendToThread;
 module.exports.whenThreadsStarted = new Promise((resolve) => {
     module.exports.threadsHaveStarted = resolve;
 });
@@ -189,6 +213,106 @@ function setTerminateTimeout(newTimeout) {
 function getWorkerIndex() {
     return workerData ? workerData.workerIndex : isMainWorker ? 0 : undefined;
 }
+/**
+ * The worker that owns `applicationName`'s per-application singletons (its scheduled jobs, its data
+ * files): pool worker 0 for a shared application and for root-level plugins (which pass no name), the
+ * dedicated worker for an isolated one -- never index 0, so node-wide duties stay on the pool, and
+ * never a dedicated worker for anything but its own application.
+ */
+function isApplicationPrimaryWorker(applicationName) {
+    if (workerData?.isolatedApplication !== undefined)
+        return applicationName === workerData.isolatedApplication;
+    return getWorkerIndex() === 0;
+}
+/**
+ * Stores that exist only on this thread: the branch stores an isolated application opened in its
+ * dedicated worker. Registered by openBranchDatabase, so the ownership predicates below can tell them
+ * from the shared stores every thread has open.
+ */
+const branchStorePaths = new Set();
+function markBranchStorePath(path, isBranch = true) {
+    if (isBranch)
+        branchStorePaths.add(path);
+    else
+        branchStorePaths.delete(path);
+}
+/**
+ * Whether this thread runs the "last worker" per-store maintenance for `storePath` (TTL scans, storage
+ * reclamation, audit cleanup). Shared stores belong to the last pool worker; a dedicated worker
+ * maintains only its own branch stores, which no other thread has open.
+ */
+function ownsStoreMaintenance(storePath) {
+    if (workerData?.isolatedApplication !== undefined)
+        return branchStorePaths.has(storePath);
+    return getWorkerIndex() === getWorkerCount() - 1;
+}
+/** The "worker 0" counterpart: expiration eviction for `storePath`. */
+function ownsStoreExpiration(storePath) {
+    if (workerData?.isolatedApplication !== undefined)
+        return branchStorePaths.has(storePath);
+    return getWorkerIndex() === 0;
+}
+/**
+ * Whether a singleton set up by APPLICATION CODE runs here (a caching table's `sourcedFrom`
+ * subscription): pool worker 0, or a dedicated worker -- the only code that runs on one is its own
+ * application's, so whatever it sets up is that application's to run. Not for root-level plugins,
+ * which load on every thread and must use isApplicationPrimaryWorker(name).
+ */
+function runsApplicationCodeSingletons() {
+    return getWorkerIndex() === 0 || workerData?.isolatedApplication !== undefined;
+}
+/** Whether this thread is a worker dedicated to one isolated application. */
+function isDedicatedWorker() {
+    return workerData?.isolatedApplication !== undefined;
+}
+/** The worker index as the application sees it: a dedicated worker is its application's worker 0. */
+function applicationWorkerIndex() {
+    return workerData?.isolatedApplication !== undefined ? 0 : getWorkerIndex();
+}
+/** Every started worker dedicated to `application`, including a replacement still booting. */
+function workersForApplication(application) {
+    return workers.filter((worker) => worker.application === application);
+}
+/**
+ * Stop one worker for good: shut it down, honour the drain extension it may ask for, force it after
+ * the same backstop the rolling restart uses (FORCE_EXIT on Bun, where terminate() segfaults), and
+ * resolve once it has exited. Marked as shut down first so its exit does not start a replacement.
+ */
+function stopWorker(worker) {
+    worker.wasShutdown = true;
+    return new Promise((resolve) => {
+        const armTerminate = (delay) => setTimeout(() => {
+            harperLogger.warn('Thread did not voluntarily terminate, terminating from the outside', worker.threadId);
+            if (isBun) {
+                try {
+                    worker.postMessage({ type: FORCE_EXIT });
+                }
+                catch { }
+            }
+            else {
+                worker.terminate();
+            }
+        }, delay).unref();
+        let timeout = armTerminate(threadTerminationTimeout * 2);
+        worker.extendTerminateDeadline = (deadlineMs) => {
+            clearTimeout(timeout);
+            const { boundedTerminateDelay, getShutdownDrainCeilingMs } = require("../../components/shutdownDrain.js");
+            timeout = armTerminate(boundedTerminateDelay(deadlineMs, Date.now(), threadTerminationTimeout * 2, getShutdownDrainCeilingMs()));
+        };
+        worker.on('exit', () => {
+            clearTimeout(timeout);
+            worker.extendTerminateDeadline = undefined;
+            resolve();
+        });
+        try {
+            worker.postMessage({ restartNumber: module.exports.restartNumber, type: hdbTerms.ITC_EVENT_TYPES.SHUTDOWN });
+        }
+        catch {
+            clearTimeout(timeout);
+            resolve(); // already gone
+        }
+    });
+}
 function getWorkerCount() {
     return workerData ? workerData.workerCount : isMainWorker ? 1 : undefined;
 }
@@ -221,8 +345,10 @@ const RESERVED_WORKER_DATA_KEYS = [
     'workerCount',
     'name',
     'restartNumber',
+    'processIncarnation',
     'ticketKeys',
     'noServerStart',
+    'isolatedApplication',
     '__proto__', // never a legitimate payload name; spread would define it as an own property
 ];
 const workerDataProviders = new Map();
@@ -282,14 +408,17 @@ function getTicketKeys() {
     ticketKeys = isMainThread ? randomBytes(48) : workerData.ticketKeys;
     return ticketKeys;
 }
+// What application code sees: a dedicated worker is its application's only worker, index 0 of 1, so
+// the documented partitioning idiom (`hash % server.workerCount === server.workerIndex`) covers
+// everything there. Node-wide duties keep using the raw getWorkerIndex/getWorkerCount.
 Object.defineProperty(server, 'workerIndex', {
     get() {
-        return getWorkerIndex();
+        return applicationWorkerIndex();
     },
 });
 Object.defineProperty(server, 'workerCount', {
     get() {
-        return getWorkerCount();
+        return workerData?.isolatedApplication !== undefined ? 1 : getWorkerCount();
     },
 });
 if (!parentPort) {
@@ -297,6 +426,14 @@ if (!parentPort) {
         if (worker)
             sendThreadInfo(worker);
     });
+    onMessageByType(REQUEST_RUNNING_ISOLATED_APPLICATIONS, (message, worker) => {
+        if (worker)
+            worker.postMessage({
+                type: RUNNING_ISOLATED_APPLICATIONS,
+                requestId: message.requestId,
+                applications: runningIsolatedApplicationsGetter(),
+            });
+    });
     onMessageByType(RESOURCE_REPORT, (message, worker) => {
         if (worker)
             recordResourceReport(worker, message);
@@ -335,11 +472,12 @@ listenersByType.set(hdbTerms.ITC_EVENT_TYPES.MIDDLEWARE_CHAINS_RESPONSE, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_REGISTERED, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_REQUEST, null);
 listenersByType.set(hdbTerms.ITC_EVENT_TYPES.OPERATION_EXECUTE_RESPONSE, null);
-// getThreadInfo/awaitProcessGroupTermination register their own one-shot parentPort listener per
+// These request/response functions register their own one-shot parentPort listener per
 // call rather than going through onMessageByType, so without this, every reply would also reach
 // addPort's permanent dispatcher as an "unregistered" type: notifyMessageListeners would warn and
 // queue each one in messagesQueuedByType forever, and a lock-wait polls every 50ms.
 listenersByType.set(THREAD_INFO, null);
+listenersByType.set(RUNNING_ISOLATED_APPLICATIONS, null);
 listenersByType.set(PROCESS_GROUP_TERMINATION_CONFIRMED, null);
 function startWorker(path, options = {}) {
     if (processShuttingDown) {
@@ -361,7 +499,7 @@ function startWorker(path, options = {}) {
     // and lower than total memory
     availableMemory = Math.min(availableMemory, totalmem(), 20000 * MB);
     const maxOldMemory = resolveThreadHeapMemoryMb(envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_MAXHEAPMEMORY)) ??
-        Math.max(Math.floor(availableMemory / MB / (10 + (options.threadCount || 1) / 4)), 512);
+        Math.max(Math.floor(availableMemory / MB / (10 + (options.heapShareCount || options.threadCount || 1) / 4)), 512);
     // Max young memory space (semi-space for scavenger) is 1/128 of max memory (limited to 16-64). For most of our m5
     // machines this will be 64MB (less for t3's). This is based on recommendations from:
     // https://www.alibabacloud.com/blog/node-js-application-troubleshooting-manual---comprehensive-gc-problems-and-optimization594965
@@ -405,6 +543,9 @@ function startWorker(path, options = {}) {
         for (const requirePath of getRequireModules())
             execArgv.push('--require', requirePath);
     }
+    // Only a start that declares the serving topology may write it; see DESIGN.md on the two workerCounts.
+    if (typeof options.threadCount === 'number')
+        workerCount = options.threadCount;
     const worker = new Worker(isAbsolute(path) ? path : join(PACKAGE_ROOT, path), {
         resourceLimits: {
             maxOldGenerationSizeMb: maxOldMemory,
@@ -419,9 +560,11 @@ function startWorker(path, options = {}) {
             addThreadIds: channelsToConnect.map((channel) => channel.existingPort.threadId),
             addPortIsJobWorkers: channelsToConnect.map((channel) => channel.existingPort.isJobWorker === true),
             workerIndex: options.workerIndex,
-            workerCount: (workerCount = options.threadCount),
+            workerCount: options.threadCount,
             name: options.name,
+            isolatedApplication: options.application,
             restartNumber: module.exports.restartNumber,
+            processIncarnation: module.exports.processIncarnation,
             ticketKeys: getTicketKeys(),
         },
         transferList: portsToSend,
@@ -452,7 +595,10 @@ function startWorker(path, options = {}) {
     });
     worker.on('exit', (_code) => {
         workers.splice(workers.indexOf(worker), 1);
-        if (!processShuttingDown && !worker.wasShutdown && options.autoRestart !== false) {
+        if (!processShuttingDown &&
+            !worker.wasShutdown &&
+            options.autoRestart !== false &&
+            options.shouldAutoRestart?.(worker) !== false) {
             // if this wasn't an intentional shutdown, restart now (unless we have tried too many times)
             if (worker.unexpectedRestarts < MAX_UNEXPECTED_RESTARTS) {
                 options.unexpectedRestarts = worker.unexpectedRestarts + 1;
@@ -469,6 +615,7 @@ function startWorker(path, options = {}) {
     if (options.onStarted)
         options.onStarted(worker); // notify that it is ready
     worker.name = options.name;
+    worker.application = options.application; // the isolated application this worker is dedicated to, if any
     return worker;
 }
 const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP];
@@ -487,8 +634,15 @@ const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP];
  * never reported that they started — or `{declined: true}` when the process is already shutting down;
  * from a worker, nothing — the restart is handed to the main thread.
  */
+/**
+ * `application` selects which workers of the type restart: omitted preserves the historic all-worker
+ * behavior, explicit undefined restarts the shared pool only, a name restarts only that application's
+ * dedicated worker, and '*' restarts all.
+ */
 async function restartWorkers(name = null, maxWorkersDown = Math.max(Math.floor(workerCount / 8), 1), // restart 1/8 of the threads at a time, but at least 1
-startReplacementThreads = true, onProgress = null) {
+startReplacementThreads = true, onProgress = null, application = undefined) {
+    if (arguments.length < 5)
+        application = '*';
     if (isMainThread) {
         // Declining is not the same as delegating: a caller reporting on the restart must not read this
         // as "another thread is completing it".
@@ -503,9 +657,19 @@ startReplacementThreads = true, onProgress = null) {
         catch (e) {
             harperLogger.error('Unable to reestablish current working directory', e);
         }
+        const freshlyStarted = new Set(); // dedicated workers the reconcile just started: already on the new code
         // problematic cyclic dependency, bind late
         const { resetRestartNeeded } = require("../../components/requestRestart.js");
-        resetRestartNeeded();
+        // One process-wide bit cannot say WHICH application is pending, so it may only be cleared by a
+        // restart that demonstrably covers every worker the bit could stand for. A restart scoped to one
+        // application leaves the whole pool on its old code; a pool restart replaces every pool worker and
+        // reconciles dedicated slots, but leaves an already-running dedicated worker on its old modules.
+        // Clearing from either would report restartRequired: false while a deployed component is still
+        // loaded nowhere. So: all workers always, the pool only while no dedicated worker is running --
+        // which is every node that uses no isolated application, i.e. the historic behavior unchanged.
+        const coversEveryWorker = application === '*' || (application === undefined && !workers.some((worker) => worker.application));
+        if (coversEveryWorker)
+            resetRestartNeeded();
         // This is here to prevent circular dependencies
         if (startReplacementThreads) {
             const { loadRootComponents } = require('../loadRootComponents.js');
@@ -515,6 +679,15 @@ startReplacementThreads = true, onProgress = null) {
             const loading = setInterval(() => onProgress?.(), RESTART_PROGRESS_HEARTBEAT_MS).unref();
             try {
                 await loadRootComponents();
+                // isolated applications added or removed by the reload get their dedicated worker started or
+                // stopped; a crash-looping newcomer can take a while, so the heartbeat runs through this too
+                try {
+                    for (const application of (await isolatedWorkerReconciler?.()) ?? [])
+                        freshlyStarted.add(application);
+                }
+                catch (error) {
+                    harperLogger.error('Could not reconcile isolated application workers', error);
+                }
             }
             finally {
                 clearInterval(loading);
@@ -522,7 +695,11 @@ startReplacementThreads = true, onProgress = null) {
             onProgress?.();
         }
         module.exports.restartNumber++;
-        if (maxWorkersDown < 1) {
+        // `Infinity` is shutdownWorkers' "all at once" sentinel and must survive.
+        if (typeof maxWorkersDown !== 'number' || Number.isNaN(maxWorkersDown)) {
+            maxWorkersDown = 1;
+        }
+        else if (maxWorkersDown < 1) {
             // we accept a ratio of workers, and compute absolute maximum being down at a time from the total number of
             // threads
             maxWorkersDown = maxWorkersDown * workers.length;
@@ -549,7 +726,7 @@ startReplacementThreads = true, onProgress = null) {
         // replacement — an unavoidable brief gap, but only for worker-owned listeners, since the main
         // thread keeps serving the HTTP ports throughout. This ordering is also what lets
         // listenOnPorts() treat a dedicated listener's EADDRINUSE as an external conflict.
-        const canPreStartReplacement = process.platform !== 'win32' && process.platform !== 'darwin' && !isBun;
+        const platformCanPreStartReplacement = process.platform !== 'win32' && process.platform !== 'darwin' && !isBun;
         const restarting = workers.slice(0);
         for (let index = 0; index < restarting.length; index++) {
             const worker = restarting[index];
@@ -558,7 +735,12 @@ startReplacementThreads = true, onProgress = null) {
                 break;
             if ((name && worker.name !== name) || worker.wasShutdown)
                 continue; // filter by type, if specified
+            if (application !== '*' && worker.application !== application)
+                continue; // and by isolated application
+            if (worker.application && freshlyStarted.has(worker.application))
+                continue;
             const overlapping = OVERLAPPING_RESTART_TYPES.indexOf(worker.name) > -1;
+            const canPreStartReplacement = platformCanPreStartReplacement && !worker.application;
             if (overlapping && startReplacementThreads && canPreStartReplacement) {
                 // Overlapping restart: start the replacement and wait until it is accepting connections
                 // *before* shutting down the worker it replaces. The replacement joins the (SO_REUSEPORT)
@@ -730,6 +912,7 @@ startReplacementThreads = true, onProgress = null) {
         parentPort.postMessage({
             type: RESTART_TYPE,
             workerType: name,
+            scope: encodeRestartScope(application),
         });
     }
 }
@@ -779,7 +962,7 @@ function whenWorkerStarted(newWorker) {
     });
 }
 function shutdownWorkers(name) {
-    return restartWorkers(name, Infinity, false);
+    return restartWorkers(name, Infinity, false, null, '*');
 }
 function beginProcessShutdown() {
     processShuttingDown = true;
@@ -801,6 +984,27 @@ async function shutdownWorkersNow(name) {
         await Promise.all(workers.map((worker) => worker.terminate()));
     }
 }
+/**
+ * The restart scope on the wire: a worker cannot post `undefined` and have it mean "the pool" (a
+ * message with no scope at all means "everything", as every pre-existing sender intends).
+ */
+function encodeRestartScope(application) {
+    return application === undefined ? '' : application; // '' is not a legal application name
+}
+function decodeRestartScope(message) {
+    if (message.scope === undefined)
+        return '*';
+    return message.scope === '' ? undefined : message.scope;
+}
+let isolatedWorkerReconciler = null;
+let runningIsolatedApplicationsGetter = () => [];
+/** Registered by socketRouter: starts and stops dedicated workers to match the root config's isolated applications. */
+function setIsolatedWorkerReconciler(reconcile) {
+    isolatedWorkerReconciler = reconcile;
+}
+function setRunningIsolatedApplicationsGetter(getter) {
+    runningIsolatedApplicationsGetter = getter;
+}
 const messageListeners = [];
 function onMessageFromWorkers(listener) {
     messageListeners.push(listener);
@@ -1062,6 +1266,7 @@ function getChildWorkerInfo() {
     return workers.map((worker) => ({
         threadId: worker.threadId,
         name: worker.name,
+        application: worker.application,
         heapTotal: worker.resources?.heapTotal,
         heapUsed: worker.resources?.heapUsed,
         externalMemory: worker.resources?.external,
@@ -1173,6 +1378,39 @@ if (parentPort && workerData?.addPorts) {
             }, timeoutMs);
         }
     });
+    let nextRunningIsolatedApplicationsRequestId = 0;
+    getRunningIsolatedApplications = (timeoutMs) => new Promise((resolve, reject) => {
+        const requestId = ++nextRunningIsolatedApplicationsRequestId;
+        let timeout;
+        parentPort.on('message', receiveApplications);
+        try {
+            parentPort.postMessage({ type: REQUEST_RUNNING_ISOLATED_APPLICATIONS, requestId });
+        }
+        catch (error) {
+            cleanup();
+            reject(error);
+            return;
+        }
+        function receiveApplications(message) {
+            if (message.type === RUNNING_ISOLATED_APPLICATIONS && message.requestId === requestId) {
+                cleanup();
+                resolve(message.applications);
+            }
+        }
+        function cleanup() {
+            if (timeout)
+                clearTimeout(timeout);
+            parentPort.off('message', receiveApplications);
+        }
+        if (timeoutMs != null) {
+            timeout = setTimeout(() => {
+                cleanup();
+                const error = new Error(`Timed out waiting for isolated application topology after ${timeoutMs}ms`);
+                error.code = 'ERR_ISOLATED_APPLICATIONS_TIMEOUT';
+                reject(error);
+            }, timeoutMs);
+        }
+    });
     let awaitTerminationRequestId = 0;
     awaitProcessGroupTermination = (ownerThreadId, signal) => new Promise((resolve) => {
         // Deliberately no timeout: a contender must not reclaim a dead owner's lock while its
@@ -1197,9 +1435,17 @@ if (parentPort && workerData?.addPorts) {
 }
 else {
     getThreadInfo = getChildWorkerInfo;
+    getRunningIsolatedApplications = isMainThread
+        ? () => runningIsolatedApplicationsGetter()
+        : () => {
+            const error = new Error('No channel to the main thread for isolated application topology');
+            error.code = 'ERR_ISOLATED_APPLICATIONS_UNAVAILABLE';
+            return Promise.reject(error);
+        };
     awaitProcessGroupTermination = (ownerThreadId) => pendingProcessGroupTerminations.get(ownerThreadId) ?? Promise.resolve();
 }
 module.exports.getThreadInfo = getThreadInfo;
+module.exports.getRunningIsolatedApplications = getRunningIsolatedApplications;
 // Listeners notified when a connected thread's port closes (worker exit/restart), so
 // modules holding per-thread state (e.g. registeredOperations' registry and in-flight
 // forwards) can clean up.
@@ -1703,7 +1949,7 @@ if (isMainThread) {
                 queuedRestart = setTimeout(async () => {
                     if (beforeRestart)
                         await beforeRestart();
-                    await restartWorkers();
+                    await restartWorkers(undefined, undefined, true, null, '*');
                     console.log('Reloaded Harper components, changed files:', Array.from(changedFiles));
                     changedFiles.clear();
                 }, 100);
@@ -1727,4 +1973,7 @@ else {
         realExit(0);
     });
 }
+// Required here, not from logging, which must never import this module; last in the file because it
+// calls onThreadExit.
+require("./logRotationTransport.js");
 //# sourceMappingURL=manageThreads.js.map