@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/dist/components/operations.js

@@ -1,6 +1,7 @@
 'use strict';
 const path = require('node:path');
-const { isMainThread } = require('node:worker_threads');
+const { isMainThread, parentPort } = require('node:worker_threads');
+const { isDeepStrictEqual } = require('node:util');
 const fs = require('fs-extra');
 const fg = require('fast-glob');
 const normalize = require('normalize-path');
@@ -17,11 +18,22 @@ const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
 const manageThreads = require('../server/threads/manageThreads.js');
 const { packageDirectory, scanPackageDirectory, streamPackagedDirectory, } = require("../components/packageComponent.js");
 const { Resources } = require("../resources/Resources.js");
-const { Application, prepareApplication, ASIDE_STAGING_DIR } = require("./Application.js");
-const { COMPONENT_PREPARATION_LOCK_DIR } = require("./componentPreparationLock.js");
+const { Application, prepareApplication, ASIDE_STAGING_DIR, DEPLOY_STAGING_DIR, dropComponentDirectory, } = require("./Application.js");
+const { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } = require("./componentPreparationLock.js");
 const { server } = require("../server/Server.js");
 const { DeploymentRecorder, awaitDeploymentRow, readPayloadBlobWithRetry, coerceTimeoutMs, DEFAULT_AWAIT_ROW_TIMEOUT_MS, } = require("./deploymentRecorder.js");
 const { ProgressEmitter } = require("../server/serverHelpers/progressEmitter.js");
+const DROP_COMPONENT_LOCK_TIMEOUT_MS = 5 * 60 * 1000;
+const ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS = 5000;
+function componentDropLockOptions(project) {
+    return {
+        timeoutMs: DROP_COMPONENT_LOCK_TIMEOUT_MS,
+        onWait: (owner) => log.info(`Waiting to drop ${project} while component preparation is in progress` +
+            (owner ? ` in process ${owner.pid}, thread ${owner.threadId}` : '')),
+        onReleaseError: (error) => log.error(`Failed to release the component preparation lock for ${project}:`, error),
+        isOwnerAlive: (owner) => owner.pid !== process.pid || manageThreads.isThreadRunning(owner.threadId),
+    };
+}
 /**
  * Read the settings.js file and return the
  *
@@ -172,12 +184,10 @@ async function addComponent(req) {
         throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
     }
     log.trace(`adding component`);
-    const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
     const { project, install_command, install_timeout, install_allow_scripts } = req;
     const template = req.template || 'https://github.com/harperdb/application-template';
     try {
-        const projectDir = path.join(cfDir, project);
-        fs.mkdirSync(projectDir, { recursive: true });
+        await fs.mkdir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), { recursive: true });
         const application = new Application({
             name: project,
             packageIdentifier: template,
@@ -230,8 +240,13 @@ async function dropCustomFunctionProject(req) {
     }
     try {
         const projectDir = path.join(cfDir, project);
-        fs.rmSync(projectDir, { recursive: true });
-        let response = await server.replication.replicateOperation(req);
+        const stagingDir = path.join(cfDir, ASIDE_STAGING_DIR, project);
+        if (!(await fs.pathExists(projectDir)) && !(await fs.pathExists(stagingDir)))
+            await fs.stat(projectDir);
+        await withComponentPreparationLock(projectDir, async () => {
+            await dropComponentDirectory(projectDir, project, log);
+        }, componentDropLockOptions(project));
+        const response = await server.replication.replicateOperation(req);
         response.message = `Successfully deleted project: ${project}`;
         return response;
     }
@@ -320,6 +335,115 @@ async function packageComponent(req) {
     // return the package payload as base64-encoded string
     return { project, payload };
 }
+/**
+ * Load the built candidate to surface load-time errors. A load-ERROR PROBE, not a safety guarantee: it runs
+ * the component's own top-level code with incomplete side-effect isolation.
+ *
+ * A no-op on the main thread, and the operations API deploys there — so operator deploys are unvalidated
+ * (#2315 step 2). What this guarantees is ORDER: where validation runs, a rejected candidate never goes live.
+ */
+// `componentLoader.setErrorReporter` is ONE process-global callback, so two components validating
+// concurrently on the same worker cross-attribute their failures: B installs its reporter while A is
+// loading, A's load error lands in B, and A then activates broken code while B rejects a good candidate.
+// Validation is serialized (it is already the slow path) and the previous reporter is restored, so the
+// global is only ever owned by one in-flight validation.
+let validationChain = Promise.resolve();
+async function validateComponentLoads(candidateDirPath, emit) {
+    const run = validationChain.then(() => validateComponentLoadsExclusive(candidateDirPath, emit), () => validateComponentLoadsExclusive(candidateDirPath, emit));
+    validationChain = run.then(() => { }, () => { });
+    return run;
+}
+async function validateComponentLoadsExclusive(candidateDirPath, emit) {
+    // now we attempt to actually load the component in case there is
+    // an error we can immediately detect and report, but app code should not run on the main thread
+    if (!isMainThread && !process.env.HARPER_SAFE_MODE) {
+        const pseudoResources = new Resources();
+        pseudoResources.isWorker = true;
+        const componentLoader = require("./componentLoader.js").default || require("./componentLoader.js");
+        const { trackScopeClose } = require("./scopeShutdown.js");
+        let lastError;
+        const priorErrorReporter = componentLoader.getErrorReporter?.();
+        componentLoader.setErrorReporter((error) => (lastError = error));
+        emit('phase', { phase: 'load', status: 'start' });
+        // This load exists only to surface load-time errors early; the Scopes it creates are
+        // throwaway. They are collected (instead of registered for worker-shutdown auto-close) so we
+        // can close them here once validation completes — otherwise each deploy leaks the Scope's
+        // deploy-lifecycle listeners on this worker, eventually tripping MaxListenersExceededWarning
+        // (#1462).
+        const validationScopes = new Set();
+        // Process-wide `server.*` registrations (registerOperation, setMcpQuotaHandler) are not owned by
+        // a Scope, so a candidate's top-level registration during this throwaway load would otherwise
+        // outlive it and pollute the live worker on a failed/rolled-back deploy. The guard makes those
+        // registration methods no-op for the duration of the load.
+        const { runWithDeployValidationGuard } = require("../server/serverHelpers/deployValidationState.js");
+        // The candidate loads under the REAL component's name, so a candidate that throws would mark the live
+        // component ERROR. Its status writes are diverted into the guard's throwaway sink instead — see
+        // `deployValidationState.ts` for why this is context-scoped rather than captured and reverted here.
+        const componentName = path.basename(candidateDirPath);
+        // Extension modules the candidate load pulls in are registered in the loader's module registry keyed by
+        // module, so forgetting only the candidate's realpath leaves those behind — one set per deploy. Their
+        // identities are collected by the load itself; diffing the global registry instead would delete a live
+        // module registered by an interleaving real load, since validations serialize only with each other.
+        const validationModules = new Set();
+        const validation = runWithDeployValidationGuard(async () => {
+            try {
+                await componentLoader.loadComponent(candidateDirPath, pseudoResources, undefined, {
+                    collectScopes: validationScopes,
+                    collectLoadedModules: validationModules,
+                });
+            }
+            finally {
+                const closeResults = await Promise.allSettled(Array.from(validationScopes, (scope) => scope.close()));
+                const failedCloses = closeResults.filter((result) => result.status === 'rejected');
+                for (const result of failedCloses) {
+                    log.warn('Failed to close a deploy-validation Scope', result.reason);
+                }
+                // A rejected close is a REJECTED VALIDATION, not a warning. `Scope.close()` stops at the
+                // throwing listener, so its remaining internal listener removal and subscription-hold release
+                // never run and the throwaway scope stays partially live — one leak per deploy, on the worker
+                // that serves the component.
+                if (failedCloses.length) {
+                    throw new AggregateError(failedCloses.map((result) => result.reason), `Could not tear down deploy validation for ${componentName}: ${failedCloses.length} scope(s) failed to close`);
+                }
+            }
+        });
+        // Track the load+close so a concurrent worker shutdown waits for these scopes to finish
+        // disposing — a plugin may start a native runtime in handleApplication — before realExit.
+        trackScopeClose(validation);
+        try {
+            await validation;
+        }
+        finally {
+            componentLoader.setErrorReporter(priorErrorReporter);
+            // The candidate path is unique per deploy, so leaving it in the loader's realpath registry leaks
+            // one dead entry per deploy for the life of the process.
+            componentLoader.forgetLoadedPath?.(candidateDirPath);
+            componentLoader.forgetLoadedModules?.(validationModules);
+        }
+        emit('phase', { phase: 'load', status: 'done' });
+        if (lastError)
+            throw lastError;
+    }
+}
+const BRANCH_STORAGE_RETAINED = '. Any branched database storage this application owns was left in place; drop it again with restart: true to discard that data';
+/** Report a restart outcome the operation's own success message cannot convey. */
+function logRestartOutcome(restart, what) {
+    const { RESTART_IDLE_TIMEOUT_MS, RESTART_WAIT_CEILING_MS } = require("./awaitRestart.js");
+    if (restart.replacementsNotStarted)
+        log.warn(`${restart.replacementsNotStarted} replacement worker thread(s) did not report starting after ${what}; the pool is short until they are restarted`);
+    if (restart.error)
+        log.error(`Restart after ${what} failed`, restart.error);
+    else if (restart.workersKeptOnOldCode)
+        log.warn(`${restart.workersKeptOnOldCode} worker thread(s) could not be replaced after ${what} and are still running the previous code`);
+    else if (restart.declined)
+        log.warn(`No restart was performed after ${what}: the process is already shutting down`);
+    else if (restart.handedOff)
+        log.debug?.(`The restart after ${what} was handed to the main thread, which reports its own completion`);
+    else if (restart.stalled)
+        log.warn(`The restart after ${what} stopped making progress for ${RESTART_IDLE_TIMEOUT_MS}ms; worker threads may still be running the previous code`);
+    else if (!restart.completed)
+        log.warn(`The restart after ${what} was still running after ${RESTART_WAIT_CEILING_MS}ms; worker threads may still be running the previous code`);
+}
 /**
  * Can deploy a component in multiple ways. If a 'package' is provided all it will do is write that package to
  * harperdb-config, when HDB is restarted the package will be installed in hdb/nodeModules. If a base64 encoded string is passed it
@@ -342,12 +466,53 @@ async function deployComponent(req) {
     // replicated ciphertext (reference, not embed); already-reference entries pass through, and with
     // no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers
     // re-running a replicated deploy already carry references and never re-ingest.
+    const { isIsolatedApplication } = require("../server/threads/isolatedApplications.js");
+    const requestedIsolation = req.isolated;
+    const isReplicatedExecution = typeof req._deploymentId === 'string';
+    const mode = req.deployment_id ? 'activate' : req.activate === false ? 'stage' : 'deploy';
+    const isActivation = mode === 'activate';
+    // this thread's cached config may predate an earlier deploy that changed the entry without a restart
+    env.initSync(true);
+    const initiallyIsolated = isIsolatedApplication(req.project);
+    if (requestedIsolation !== undefined && !req.package) {
+        throw handleHDBError(new Error(), `'isolated' is only supported for package deployments; set it on the application's root config entry instead`, HTTP_STATUS_CODES.BAD_REQUEST);
+    }
+    // Only the originating node rejects admission: replicated peers must commit the same desired config,
+    // then report any node-local inability to run it through component lifecycle status.
+    const assertIsolationAdmission = async (isolated) => {
+        if (!isolated || isReplicatedExecution)
+            return;
+        const { isolatedApplicationRefusal, isolatedApplicationCapacityRefusal, presentIsolatedApplicationNames, } = require("../server/threads/isolatedApplications.js");
+        let runningApplications;
+        try {
+            runningApplications = await manageThreads.getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS);
+        }
+        catch (error) {
+            throw handleHDBError(error, `Cannot deploy '${req.project}' as an isolated application: the main thread's worker topology is unavailable: ${error.message}`, HTTP_STATUS_CODES.SERVICE_UNAVAILABLE);
+        }
+        runningApplications = new Set([
+            ...runningApplications,
+            ...presentIsolatedApplicationNames().filter((application) => !isolatedApplicationRefusal(application)),
+        ]);
+        const refusal = isolatedApplicationRefusal(req.project) ?? isolatedApplicationCapacityRefusal(req.project, runningApplications);
+        if (refusal) {
+            throw handleHDBError(new Error(), `Cannot deploy '${req.project}' as an isolated application: ${refusal}`, HTTP_STATUS_CODES.CONFLICT);
+        }
+    };
+    // Skipped for an activation: the artifact's descriptor is the authority for the isolation it wants, and
+    // `admitIsolation` runs the real check under the preparation lock. Admitting the CURRENT value here would
+    // refuse an activation whose artifact turns isolation OFF for a component whose present isolated state is
+    // itself the refusal — a 409 for the request that would fix it.
+    if (!isActivation)
+        await assertIsolationAdmission(requestedIsolation ?? initiallyIsolated);
     const { ingestCredentials, resolveCredentials } = require("./secretOperations.js");
-    req.credentials = await ingestCredentials(req, req.credentials, req.project);
+    // An activation resolves, fetches and installs nothing, so there is no credential for it to carry.
+    // The validator rejects one; this keeps the ingest itself off the path rather than relying on that.
+    if (!isActivation)
+        req.credentials = await ingestCredentials(req, req.credentials, req.project);
     // References are safe to persist (config + deployment row) and replicate; a no-custody literal
     // token is not — it is used only for this node's install below, then stripped before replication.
     const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined);
-    // Write to root config if the request contains a package identifier
     if (req.package) {
         // Check if trying to overwrite a core component (requires force)
         // Lazy-load to avoid circular dependency with componentLoader
@@ -355,25 +520,9 @@ async function deployComponent(req) {
         if (TRUSTED_RESOURCE_PLUGINS[req.project] && !req.force) {
             throw handleHDBError(new Error(), `Cannot deploy component with name '${req.project}': this is a protected core component name. Use force: true to overwrite.`, HTTP_STATUS_CODES.CONFLICT);
         }
-        const applicationConfig = { package: req.package };
-        // Avoid writing an empty `install:` block
-        if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) {
-            applicationConfig.install = {
-                command: req.install_command,
-                timeout: req.install_timeout,
-                allowInstallScripts: req.install_allow_scripts,
-            };
-        }
-        if (req.urlPath !== undefined)
-            applicationConfig.urlPath = req.urlPath;
-        if (req.host !== undefined)
-            applicationConfig.host = req.host;
-        // Persist credential references (never tokens) so every cold install of this component —
-        // reboot, new peer, rollback — re-resolves the credential from the store.
-        if (credentialReferences.length)
-            applicationConfig.credentials = credentialReferences;
-        await configUtils.addConfig(req.project, applicationConfig);
     }
+    let wasIsolated;
+    let nowIsolated;
     // Create a hdb_deployment row up front so the deploy is observable and auditable
     // even if the CLI disconnects. The row also holds the payload in a Blob attribute,
     // which doubles as the source for peer replication and (later) rollback.
@@ -382,7 +531,6 @@ async function deployComponent(req) {
     // recording so we don't accumulate one row per node for the same deploy. The row
     // reaches peers via the table's standard replication; the peer-side branch below
     // reads payload_blob back from there.
-    const isReplicatedExecution = typeof req._deploymentId === 'string';
     // An SSE-bound caller already attached a ProgressEmitter (created in the server
     // handler so it can also drive the response stream). Reuse it; otherwise spin up a
     // fresh emitter so the recorder still gets phase events for non-SSE deploys.
@@ -395,6 +543,7 @@ async function deployComponent(req) {
             project: req.project,
             package_identifier: req.package ?? null,
             user: req.hdb_user?.username,
+            activated_from: req.deployment_id ?? null,
             restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null,
             // Reference form only — the rollback source for re-resolving the credential.
             credentials: credentialReferences.length ? credentialReferences : null,
@@ -417,12 +566,19 @@ async function deployComponent(req) {
     // Bounded ring buffer of install stdout/stderr so a non-SSE caller sees the tail
     // in the thrown error. SSE callers still stream every line live.
     const installCapture = createInstallCapture();
+    let stagedOnOrigin = false;
     try {
         // On the origin, tee the tarball (Buffer or Readable from the multipart parser)
         // through a hash-and-size tap into the row's payload_blob, then re-source extraction
         // from the persisted blob. When `system` replicates, the blob becomes the channel
         // peers read from; when it doesn't, the blob stays local for audit and rollback.
-        if (recorder && req.payload != null) {
+        if (isActivation) {
+            // A replicated activation has exactly the shape the peer-side blob branch below keys on — neither
+            // `package` nor `payload` — so without this branch every peer would wait out `deployment_timeout`
+            // for a payload its row never had while the origin swapped successfully.
+            extractionPayload = undefined;
+        }
+        else if (recorder && req.payload != null) {
             await recorder.ingestPayload(req.payload);
             extractionPayload = recorder.row.payload_blob.stream();
         }
@@ -458,9 +614,13 @@ async function deployComponent(req) {
         if (isReplicatedExecution) {
             credentialsWaitMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS);
         }
-        const resolvedCredentials = await resolveCredentials(req.credentials, req.project, {
-            waitMs: credentialsWaitMs,
-        });
+        // Skipped for an activation, which runs no pack, clone or install: waiting for an hdb_secret row to
+        // replicate in would only delay a swap that has nothing to authenticate to.
+        const resolvedCredentials = isActivation
+            ? undefined
+            : await resolveCredentials(req.credentials, req.project, {
+                waitMs: credentialsWaitMs,
+            });
         const application = new Application({
             name: req.project,
             payload: extractionPayload,
@@ -492,52 +652,102 @@ async function deployComponent(req) {
             req.credentials = credentialReferences;
         else
             delete req.credentials;
+        // Validated INSIDE preparation, between build and swap, so a component that installs cleanly but
+        // throws at load never becomes live.
+        //
+        // Phase ORDER is unchanged for clients, but `prepare:done` means "candidate built", not "swap
+        // committed" — so a later failure arrives after both phases reported success. The operation's error
+        // is the authority on whether the deploy landed, not the phase stream.
         emit('phase', { phase: 'prepare', status: 'start' });
-        await prepareApplication(application);
-        emit('phase', { phase: 'prepare', status: 'done' });
-        // now we attempt to actually load the component in case there is
-        // an error we can immediately detect and report, but app code should not run on the main thread
-        if (!isMainThread && !process.env.HARPER_SAFE_MODE) {
-            const pseudoResources = new Resources();
-            pseudoResources.isWorker = true;
-            const componentLoader = require("./componentLoader.js").default || require("./componentLoader.js");
-            const { trackScopeClose } = require("./scopeShutdown.js");
-            let lastError;
-            componentLoader.setErrorReporter((error) => (lastError = error));
-            emit('phase', { phase: 'load', status: 'start' });
-            // This load exists only to surface load-time errors early; the Scopes it creates are
-            // throwaway. They are collected (instead of registered for worker-shutdown auto-close) so we
-            // can close them here once validation completes — otherwise each deploy leaks the Scope's
-            // deploy-lifecycle listeners on this worker, eventually tripping MaxListenersExceededWarning
-            // (#1462).
-            const validationScopes = new Set();
-            // Process-wide `server.*` registrations (registerOperation, setMcpQuotaHandler) are not owned by
-            // a Scope, so a candidate's top-level registration during this throwaway load would otherwise
-            // outlive it and pollute the live worker on a failed/rolled-back deploy. The guard makes those
-            // registration methods no-op for the duration of the load.
-            const { runWithDeployValidationGuard } = require("../server/serverHelpers/deployValidationState.js");
-            const validation = runWithDeployValidationGuard(async () => {
-                try {
-                    await componentLoader.loadComponent(application.dirPath, pseudoResources, undefined, {
-                        collectScopes: validationScopes,
-                    });
-                }
-                finally {
-                    const closeResults = await Promise.allSettled(Array.from(validationScopes, (scope) => scope.close()));
-                    for (const result of closeResults) {
-                        if (result.status === 'rejected')
-                            log.warn('Failed to close a deploy-validation Scope', result.reason);
+        // The root-config entry a package deploy publishes. A stage records it with the artifact instead of
+        // publishing it, so a staged release nobody activated cannot leave config naming it — and the
+        // activation that eventually publishes it does so in the same order a normal deploy does.
+        let stagedRootConfig = null;
+        await prepareApplication(application, {
+            // `.deploy-staging/<artifactId>`. The public deployment id, so the id the caller was handed is
+            // the id a later `deployment_id` request can name; an activation names the artifact's own id,
+            // not the new row's.
+            artifactId: req.deployment_id ?? req._deploymentId,
+            mode,
+            describeArtifact: () => ({ rootConfig: stagedRootConfig, isolated: Boolean(nowIsolated) }),
+            // Returns its own undo. Publication happens BEFORE the swap, in the same order a normal deploy
+            // uses, so an activation that fails pre-commit would otherwise leave config naming a release that
+            // is not live — and `installApplications()` would resolve and install that package from scratch at
+            // the next boot, over a component whose certified artifact is sitting right there. Restoring is
+            // best-effort and not durable (that is #2315 step 3's work); it is still the difference between a
+            // failed activation that changed nothing and one that re-points the component.
+            publishRootConfig: async (entry) => {
+                const previous = configUtils.getConfigObj()?.[req.project];
+                await configUtils.addConfig(req.project, entry);
+                env.initSync(true);
+                // Read BACK, not `entry` — see `publishedEntryStillStands`.
+                const published = configUtils.getConfigObj()?.[req.project];
+                return async () => {
+                    if (!publishedEntryStillStands(published, () => env.initSync(true), () => configUtils.getConfigObj()?.[req.project])) {
+                        return;
                     }
+                    if (previous === undefined)
+                        configUtils.deleteConfigFromFile([req.project]);
+                    else
+                        await configUtils.addConfig(req.project, previous);
+                    env.initSync(true);
+                };
+            },
+            admitIsolation: async (descriptor) => {
+                env.initSync(true);
+                wasIsolated = isIsolatedApplication(req.project);
+                // A package artifact restores its own root-config entry, so its recorded intent is what will
+                // be in force. A payload artifact publishes no config, so the effective configuration is the
+                // only authority there — a saved `false` must not admit past an isolation the operator set
+                // between the stage and this call.
+                nowIsolated = descriptor.rootConfig ? descriptor.isolated : wasIsolated;
+                await assertIsolationAdmission(nowIsolated);
+            },
+            beforePrepare: async () => {
+                if (isActivation)
+                    return;
+                env.initSync(true);
+                wasIsolated = isIsolatedApplication(req.project);
+                nowIsolated = requestedIsolation ?? wasIsolated;
+                await assertIsolationAdmission(nowIsolated);
+                if (!isReplicatedExecution && req.package)
+                    req.isolated = nowIsolated;
+                if (!req.package)
+                    return;
+                const applicationConfig = { package: req.package };
+                if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) {
+                    applicationConfig.install = {
+                        command: req.install_command,
+                        timeout: req.install_timeout,
+                        allowInstallScripts: req.install_allow_scripts,
+                    };
                 }
-            });
-            // Track the load+close so a concurrent worker shutdown waits for these scopes to finish
-            // disposing — a plugin may start a native runtime in handleApplication — before realExit.
-            trackScopeClose(validation);
-            await validation;
-            emit('phase', { phase: 'load', status: 'done' });
-            if (lastError)
-                throw lastError;
-        }
+                if (req.urlPath !== undefined)
+                    applicationConfig.urlPath = req.urlPath;
+                if (req.host !== undefined)
+                    applicationConfig.host = req.host;
+                if (req.branchedDatabases !== undefined)
+                    applicationConfig.branchedDatabases = req.branchedDatabases;
+                if (nowIsolated)
+                    applicationConfig.isolated = true;
+                if (credentialReferences.length)
+                    applicationConfig.credentials = credentialReferences;
+                if (mode === 'stage') {
+                    stagedRootConfig = applicationConfig;
+                    return;
+                }
+                await configUtils.addConfig(req.project, applicationConfig);
+                env.initSync(true);
+            },
+            validateCandidate: async (candidateDirPath) => {
+                emit('phase', { phase: 'prepare', status: 'done' });
+                await validateComponentLoads(candidateDirPath, emit);
+            },
+        });
+        // The build is certified on disk from here on, so every later failure — a peer result, or a rejection
+        // thrown by the replication layer itself — still leaves an artifact this id can activate.
+        if (mode === 'stage')
+            stagedOnOrigin = true;
         const rollingRestart = req.restart === 'rolling';
         // if doing a rolling restart set restart to false so that other nodes don't also restart.
         req.restart = rollingRestart ? false : req.restart;
@@ -580,10 +790,28 @@ async function deployComponent(req) {
             // when the per-peer callback already fired for these.
             recorder.recordPeers(response.replicated);
         }
-        if (req.restart === true) {
+        // A still-isolated application restarts only its own dedicated worker. Everything else restarts the
+        // pool: a shared application, and either direction of an isolation flip, where the reconcile in
+        // restartWorkers starts or stops the moving application's own worker.
+        const restartScope = wasIsolated && nowIsolated ? application.name : undefined;
+        if (mode === 'stage') {
+            // No restart and no restart-required flag: nothing about the running component changed. The
+            // marker is what tells the origin which peers understood the request — see the confirmation
+            // check below — and it is set on every node, origin included, since a peer returns this same
+            // object as its operation response.
+            response.staged = true;
+            response.message = `Staged: ${application.name}`;
+        }
+        else if (req.restart === true) {
             emit('phase', { phase: 'restart', status: 'start' });
-            manageThreads.restartWorkers('http');
+            // Workers not yet replaced keep serving the pre-deploy component set, and where the OS lets
+            // replacements share a port they keep accepting connections for the whole rolling restart, so
+            // a caller that reads success as "the component is live" can be served by a worker that has
+            // never heard of it.
+            const { awaitRestart } = require("./awaitRestart.js");
+            const restart = await awaitRestart((onProgress) => manageThreads.restartWorkers('http', undefined, undefined, onProgress, restartScope));
             emit('phase', { phase: 'restart', status: 'done' });
+            logRestartOutcome(restart, `deploying ${application.name}`);
             response.message = `Successfully deployed: ${application.name}, restarting Harper`;
         }
         else if (rollingRestart) {
@@ -592,6 +820,8 @@ async function deployComponent(req) {
             const jobResponse = await serverUtilities.executeJob({
                 operation: 'restart_service',
                 service: 'http',
+                scope: manageThreads.encodeRestartScope(restartScope),
+                scopeFallback: restartScope === undefined ? undefined : manageThreads.encodeRestartScope(undefined),
                 replicated: true,
             });
             emit('phase', { phase: 'restart', status: 'done' });
@@ -611,10 +841,8 @@ async function deployComponent(req) {
             // An existing component's watched files are handled by Scope/EntryHandler. Package
             // metadata is deliberately outside most plugin globs, so compare it across the atomic
             // swap as well: a dependency or module-entry change also invalidates loaded code.
-            if (application.isNewComponent || application.packageMetadataChanged) {
-                const { requestRestart } = require("./requestRestart.js");
-                requestRestart();
-            }
+            const { requestRestartAfterDeploy } = require("./requestRestart.js");
+            requestRestartAfterDeploy(application.isNewComponent, application.packageMetadataChanged, wasIsolated, nowIsolated);
             response.message = `Successfully deployed: ${application.name}`;
         }
         // Replication failures don't reject replicateOperation — they surface as 'failed'
@@ -629,10 +857,32 @@ async function deployComponent(req) {
                 const detail = failedPeers
                     .map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`)
                     .join(', ');
-                throw new ServerError(`Component '${application.name}' was deployed on the origin node but failed to replicate to ` +
-                    `${failedPeers.length} of ${recorder.row.peer_results.length} peer node(s): ${detail}. ` +
-                    `See deployment ${recorder.deploymentId} (get_deployment) for details, or pass ` +
-                    `ignore_replication_errors: true to treat replication failures as non-fatal.`);
+                const replicationError = new ServerError(`Component '${application.name}' was ${mode === 'stage' ? 'staged' : 'deployed'} on the origin node ` +
+                    `but failed to replicate to ${failedPeers.length} of ${recorder.row.peer_results.length} peer ` +
+                    `node(s): ${detail}. See deployment ${recorder.deploymentId} (get_deployment) for details, or ` +
+                    `pass ignore_replication_errors: true to treat replication failures as non-fatal.`);
+                throw replicationError;
+            }
+            // A peer running a build that predates staged deploys accepts `activate: false` as an unknown
+            // field and performs its ordinary deploy, so it is now serving a release the operator asked only
+            // to stage. Nothing can prevent that — the operations channel carries no peer version or
+            // capability — so the origin reports it, from the one signal that separates the two cases: a node
+            // that staged says so in its response. Read from the raw aggregate, because the recorder's peer
+            // normalization keeps only node/status/error.
+            //
+            // The wording stays at "did not confirm". An unreachable peer and one that dropped the request
+            // look identical from here, and telling an operator those nodes went live would be a guess.
+            if (mode === 'stage') {
+                const unconfirmed = unconfirmedStagingPeers(response?.replicated);
+                if (unconfirmed.length > 0) {
+                    const detail = unconfirmed.map((peer) => peer.node ?? 'unknown').join(', ');
+                    const unconfirmedError = new ServerError(`Component '${application.name}' was staged on the origin node, but ${unconfirmed.length} peer ` +
+                        `node(s) did not confirm staging: ${detail}. Either they are unreachable, or they run a build ` +
+                        `that predates staged deploys — which treats this request as an ordinary deploy, so the ` +
+                        `release is already serving there. Check those nodes before activating deployment ` +
+                        `${recorder.deploymentId}, or pass ignore_replication_errors: true to accept the difference.`);
+                    throw unconfirmedError;
+                }
             }
         }
         if (recorder) {
@@ -645,15 +895,27 @@ async function deployComponent(req) {
             // the tarball when it's still the artifact you'd debug or retry with: failed deploys
             // don't reach this branch, and a deploy that reached here only because
             // ignore_replication_errors masked failed peers keeps its payload for those peers.
+            // Never for a stage: the artifact on disk is not yet a release anyone can re-derive, and retention
+            // can evict it (it shares `deployment_stagingRetention_maxCount` with incidental dormant builds),
+            // so dropping the tarball here is how a staged release becomes unrecoverable — no tree, no bytes,
+            // and a row still reporting `staged`. An operator can still reclaim it deliberately with
+            // delete_deployment_payload once they accept that.
             const payloadSize = recorder.row.payload_size;
             const retentionMaxSize = getPayloadRetentionMaxSize();
-            if (typeof payloadSize === 'number' && payloadSize > retentionMaxSize && recorder.getFailedPeers().length === 0) {
+            if (mode !== 'stage' &&
+                typeof payloadSize === 'number' &&
+                payloadSize > retentionMaxSize &&
+                recorder.getFailedPeers().length === 0) {
                 const freed = recorder.dropPayload();
                 if (freed > 0)
                     emit('payload_dropped', { payload_size: freed, max_size: retentionMaxSize });
             }
-            emit('phase', { phase: 'success', status: 'done' });
-            await recorder.finish('success');
+            emit('phase', { phase: mode === 'stage' ? 'staged' : 'success', status: 'done' });
+            await recorder.finish(mode === 'stage' ? 'staged' : 'success');
+            if (mode === 'stage') {
+                response.message =
+                    `Staged: ${application.name}. Deploy it with deploy_component ` + `deployment_id=${recorder.deploymentId}`;
+            }
         }
         return response;
     }
@@ -697,7 +959,7 @@ async function deployComponent(req) {
         // install output, and failed_peers the caller needs.
         if (recorder) {
             try {
-                await recorder.finish('failed', err);
+                await recorder.finish(stagedOnOrigin ? 'staged' : 'failed', err);
             }
             catch (finishErr) {
                 log.warn('Failed to record deployment failure row', finishErr);
@@ -706,6 +968,42 @@ async function deployComponent(req) {
         throw outErr;
     }
 }
+/**
+ * Whether the root-config entry an activation published is still the one on disk — the condition for that
+ * activation's failure to take it back. A `set_configuration` acknowledged while the swap was retrying is
+ * not this operation's to overwrite, and restoring a snapshot taken before it would silently drop it.
+ *
+ * Split out from the caller so it can be tested without a deploy: this is the only guard on a hazard the
+ * preparation lock does not cover, and both ways it can regress fail silently. `refreshConfig` is called
+ * BEFORE the read and is load-bearing — `set_configuration` writes the config file without refreshing this
+ * process's config object, so a cached read compares against a value that predates the very change the
+ * guard exists to protect and always concludes nothing moved. It narrows the window rather than closing
+ * it; serializing config publication is #2315 step 3.
+ *
+ * `published` must be the entry as read back after publication, not as passed in: only two values that went
+ * through the same write-and-parse round trip are comparable, and comparing against the argument would read
+ * any serialization difference as a concurrent change and skip every undo.
+ */
+function publishedEntryStillStands(published, refreshConfig, readCurrentEntry) {
+    refreshConfig();
+    return isDeepStrictEqual(readCurrentEntry(), published);
+}
+/**
+ * Peers that did not answer a stage with `staged: true` — either unreachable, or running a build that
+ * predates staged deploys and therefore treated the request as an ordinary deploy.
+ *
+ * Split out from the caller so it can be tested without a cluster: this is the only safety net for the
+ * mixed-version hazard #2315 records as accepted, and a regression in it fails silently. The entry shape
+ * belongs to harper-pro's replicator rather than this repo — `normalizePeerResult` already tolerates more
+ * than one — so the marker is read flat or from a wrapped body; assuming flat would report every peer in a
+ * fully-upgraded cluster as unconfirmed.
+ */
+function unconfirmedStagingPeers(replicated) {
+    if (!Array.isArray(replicated))
+        return [];
+    const confirmed = (peer) => peer?.staged === true || peer?.value?.staged === true || peer?.body?.staged === true;
+    return replicated.filter((peer) => peer && !confirmed(peer));
+}
 // Ring buffer of install stdout/stderr lines, capped by both line count and bytes so
 // a chatty install can't unbounded-grow the error response. snapshot() reports whether
 // the head was dropped so callers can flag truncation.
@@ -774,8 +1072,12 @@ async function getComponents() {
             const list = await fs.readdir(dir, { withFileTypes: true });
             for (let item of list) {
                 const itemName = item.name;
+                // Deny-list, not a dot-prefix skip: component CONTENTS legitimately include dot-files
+                // (`.aiignore`, `.env.example`) that callers expect to see, so only Harper's own
+                // bookkeeping directories are excluded by name.
                 if (itemName === 'node_modules' ||
                     itemName === ASIDE_STAGING_DIR ||
+                    itemName === DEPLOY_STAGING_DIR ||
                     itemName === COMPONENT_PREPARATION_LOCK_DIR)
                     continue;
                 const itemPath = path.join(dir, itemName);
@@ -1054,31 +1356,90 @@ async function dropComponent(req) {
     }
     const { project, file } = req;
     const projectPath = req.file ? path.join(project, file) : project;
-    const pathToComponent = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
-    const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project);
-    if (await fs.pathExists(componentSymlink)) {
-        await fs.unlink(componentSymlink);
-    }
-    if (await fs.pathExists(pathToComponent)) {
-        await fs.remove(pathToComponent);
-    }
-    // Remove the component from the package.json file
-    const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json');
-    if (await fs.pathExists(packageJsonPath)) {
-        const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
-        if (packageJson?.dependencies?.[project]) {
-            delete packageJson.dependencies[project];
+    const componentsRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+    const componentPath = path.join(componentsRoot, project);
+    const pathToComponent = path.join(componentsRoot, projectPath);
+    let restartScope;
+    let response;
+    await withComponentPreparationLock(componentPath, async () => {
+        if (req.restart === true) {
+            let runningApplications;
+            try {
+                runningApplications = await manageThreads.getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS);
+            }
+            catch (error) {
+                throw handleHDBError(error, `Cannot drop '${project}' with a restart: the main thread's worker topology is unavailable: ${error.message}`, HTTP_STATUS_CODES.SERVICE_UNAVAILABLE);
+            }
+            if (runningApplications.includes(project))
+                restartScope = project;
         }
-        await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8');
-    }
-    configUtils.deleteConfigFromFile([project]);
-    let response = await server.replication.replicateOperation(req);
-    if (req.restart === true) {
-        manageThreads.restartWorkers('http');
+        const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project);
+        if (!file && (await fs.pathExists(componentSymlink))) {
+            await fs.unlink(componentSymlink);
+        }
+        if (!file) {
+            await dropComponentDirectory(componentPath, project, log);
+        }
+        else if (await fs.pathExists(pathToComponent)) {
+            await fs.remove(pathToComponent);
+        }
+        const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json');
+        if (!file && (await fs.pathExists(packageJsonPath))) {
+            const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+            if (packageJson?.dependencies?.[project]) {
+                delete packageJson.dependencies[project];
+            }
+            await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8');
+        }
+        if (!file)
+            configUtils.deleteConfigFromFile([project]);
+        if (!file && isMainThread)
+            env.initSync(true);
+        response = await server.replication.replicateOperation(req);
+        const { applicationHasBranchStorage, removeBranchesForApplication } = require("../resources/branchDatabase.js");
+        const branched = !file && applicationHasBranchStorage(project);
+        if (req.restart !== true) {
+            response.message = `Successfully dropped: ${projectPath}`;
+            if (branched)
+                response.message += BRANCH_STORAGE_RETAINED;
+            return;
+        }
+        // The branches go after the restart, under this lock; a worker is one of the threads being
+        // replaced, so it hands both to the main thread.
         response.message = `Successfully dropped: ${projectPath}, restarting Harper`;
-    }
-    else
-        response.message = `Successfully dropped: ${projectPath}`;
+        if (!isMainThread) {
+            parentPort.postMessage({
+                type: hdbTerms.ITC_EVENT_TYPES.RESTART,
+                workerType: 'http',
+                scope: manageThreads.encodeRestartScope(restartScope),
+                removeBranchesFor: file ? undefined : project,
+            });
+            if (branched) {
+                response.message +=
+                    '; the main thread removes its branched database storage after the restart and logs the outcome';
+            }
+            return;
+        }
+        const { awaitRestart } = require("./awaitRestart.js");
+        const restart = await awaitRestart((onProgress) => manageThreads.restartWorkers('http', undefined, undefined, onProgress, restartScope));
+        logRestartOutcome(restart, `dropping ${projectPath}`);
+        if (!branched)
+            return;
+        if (restart.completed && !restart.workersKeptOnOldCode) {
+            try {
+                await removeBranchesForApplication(project);
+            }
+            catch (error) {
+                log.error(`Branched database storage of ${project} could not be removed`, error);
+                throw new Error(`Successfully dropped: ${projectPath}, but its branched database storage could not be removed and ` +
+                    `was left in place: ${error.message}. Drop it again with restart: true to retry.`);
+            }
+        }
+        else {
+            log.warn(`Branched database storage of ${project} was left in place: the restart did not complete`);
+            response.message += BRANCH_STORAGE_RETAINED;
+        }
+    }, componentDropLockOptions(project));
     return response;
 }
 exports.customFunctionsStatus = customFunctionsStatus;
@@ -1090,6 +1451,8 @@ exports.addComponent = addComponent;
 exports.dropCustomFunctionProject = dropCustomFunctionProject;
 exports.packageComponent = packageComponent;
 exports.deployComponent = deployComponent;
+exports.unconfirmedStagingPeers = unconfirmedStagingPeers;
+exports.publishedEntryStillStands = publishedEntryStillStands;
 exports.getComponents = getComponents;
 exports.getComponentFile = getComponentFile;
 exports.setComponentFile = setComponentFile;