@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/components/operations.js

@@ -1,7 +1,8 @@
 'use strict';
 
 const path = require('node:path');
-const { isMainThread } = require('node:worker_threads');
+const { isMainThread, parentPort } = require('node:worker_threads');
+const { isDeepStrictEqual } = require('node:util');
 const fs = require('fs-extra');
 const fg = require('fast-glob');
 const normalize = require('normalize-path');
@@ -29,8 +30,14 @@ const {
 	streamPackagedDirectory,
 } = require('../components/packageComponent.ts');
 const { Resources } = require('../resources/Resources.ts');
-const { Application, prepareApplication, ASIDE_STAGING_DIR } = require('./Application.ts');
-const { COMPONENT_PREPARATION_LOCK_DIR } = require('./componentPreparationLock.ts');
+const {
+	Application,
+	prepareApplication,
+	ASIDE_STAGING_DIR,
+	DEPLOY_STAGING_DIR,
+	dropComponentDirectory,
+} = require('./Application.ts');
+const { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } = require('./componentPreparationLock.ts');
 const { server } = require('../server/Server.ts');
 const {
 	DeploymentRecorder,
@@ -41,6 +48,22 @@ const {
 } = require('./deploymentRecorder.ts');
 const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts');
 
+const DROP_COMPONENT_LOCK_TIMEOUT_MS = 5 * 60 * 1000;
+const ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS = 5000;
+
+function componentDropLockOptions(project) {
+	return {
+		timeoutMs: DROP_COMPONENT_LOCK_TIMEOUT_MS,
+		onWait: (owner) =>
+			log.info(
+				`Waiting to drop ${project} while component preparation is in progress` +
+					(owner ? ` in process ${owner.pid}, thread ${owner.threadId}` : '')
+			),
+		onReleaseError: (error) => log.error(`Failed to release the component preparation lock for ${project}:`, error),
+		isOwnerAlive: (owner) => owner.pid !== process.pid || manageThreads.isThreadRunning(owner.threadId),
+	};
+}
+
 /**
  * Read the settings.js file and return the
  *
@@ -238,14 +261,12 @@ async function addComponent(req) {
 	}
 
 	log.trace(`adding component`);
-	const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const { project, install_command, install_timeout, install_allow_scripts } = req;
 
 	const template = req.template || 'https://github.com/harperdb/application-template';
 
 	try {
-		const projectDir = path.join(cfDir, project);
-		fs.mkdirSync(projectDir, { recursive: true });
+		await fs.mkdir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), { recursive: true });
 		const application = new Application({
 			name: project,
 			packageIdentifier: template,
@@ -310,8 +331,16 @@ async function dropCustomFunctionProject(req) {
 
 	try {
 		const projectDir = path.join(cfDir, project);
-		fs.rmSync(projectDir, { recursive: true });
-		let response = await server.replication.replicateOperation(req);
+		const stagingDir = path.join(cfDir, ASIDE_STAGING_DIR, project);
+		if (!(await fs.pathExists(projectDir)) && !(await fs.pathExists(stagingDir))) await fs.stat(projectDir);
+		await withComponentPreparationLock(
+			projectDir,
+			async () => {
+				await dropComponentDirectory(projectDir, project, log);
+			},
+			componentDropLockOptions(project)
+		);
+		const response = await server.replication.replicateOperation(req);
 		response.message = `Successfully deleted project: ${project}`;
 		return response;
 	} catch (err) {
@@ -410,6 +439,134 @@ async function packageComponent(req) {
 	return { project, payload };
 }
 
+/**
+ * Load the built candidate to surface load-time errors. A load-ERROR PROBE, not a safety guarantee: it runs
+ * the component's own top-level code with incomplete side-effect isolation.
+ *
+ * A no-op on the main thread, and the operations API deploys there — so operator deploys are unvalidated
+ * (#2315 step 2). What this guarantees is ORDER: where validation runs, a rejected candidate never goes live.
+ */
+// `componentLoader.setErrorReporter` is ONE process-global callback, so two components validating
+// concurrently on the same worker cross-attribute their failures: B installs its reporter while A is
+// loading, A's load error lands in B, and A then activates broken code while B rejects a good candidate.
+// Validation is serialized (it is already the slow path) and the previous reporter is restored, so the
+// global is only ever owned by one in-flight validation.
+let validationChain = Promise.resolve();
+
+async function validateComponentLoads(candidateDirPath, emit) {
+	const run = validationChain.then(
+		() => validateComponentLoadsExclusive(candidateDirPath, emit),
+		() => validateComponentLoadsExclusive(candidateDirPath, emit)
+	);
+	validationChain = run.then(
+		() => {},
+		() => {}
+	);
+	return run;
+}
+
+async function validateComponentLoadsExclusive(candidateDirPath, emit) {
+	// now we attempt to actually load the component in case there is
+	// an error we can immediately detect and report, but app code should not run on the main thread
+	if (!isMainThread && !process.env.HARPER_SAFE_MODE) {
+		const pseudoResources = new Resources();
+		pseudoResources.isWorker = true;
+
+		const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts');
+		const { trackScopeClose } = require('./scopeShutdown.ts');
+		let lastError;
+		const priorErrorReporter = componentLoader.getErrorReporter?.();
+		componentLoader.setErrorReporter((error) => (lastError = error));
+		emit('phase', { phase: 'load', status: 'start' });
+		// This load exists only to surface load-time errors early; the Scopes it creates are
+		// throwaway. They are collected (instead of registered for worker-shutdown auto-close) so we
+		// can close them here once validation completes — otherwise each deploy leaks the Scope's
+		// deploy-lifecycle listeners on this worker, eventually tripping MaxListenersExceededWarning
+		// (#1462).
+		const validationScopes = new Set();
+		// Process-wide `server.*` registrations (registerOperation, setMcpQuotaHandler) are not owned by
+		// a Scope, so a candidate's top-level registration during this throwaway load would otherwise
+		// outlive it and pollute the live worker on a failed/rolled-back deploy. The guard makes those
+		// registration methods no-op for the duration of the load.
+		const { runWithDeployValidationGuard } = require('../server/serverHelpers/deployValidationState.ts');
+		// The candidate loads under the REAL component's name, so a candidate that throws would mark the live
+		// component ERROR. Its status writes are diverted into the guard's throwaway sink instead — see
+		// `deployValidationState.ts` for why this is context-scoped rather than captured and reverted here.
+		const componentName = path.basename(candidateDirPath);
+		// Extension modules the candidate load pulls in are registered in the loader's module registry keyed by
+		// module, so forgetting only the candidate's realpath leaves those behind — one set per deploy. Their
+		// identities are collected by the load itself; diffing the global registry instead would delete a live
+		// module registered by an interleaving real load, since validations serialize only with each other.
+		const validationModules = new Set();
+		const validation = runWithDeployValidationGuard(async () => {
+			try {
+				await componentLoader.loadComponent(candidateDirPath, pseudoResources, undefined, {
+					collectScopes: validationScopes,
+					collectLoadedModules: validationModules,
+				});
+			} finally {
+				const closeResults = await Promise.allSettled(Array.from(validationScopes, (scope) => scope.close()));
+				const failedCloses = closeResults.filter((result) => result.status === 'rejected');
+				for (const result of failedCloses) {
+					log.warn('Failed to close a deploy-validation Scope', result.reason);
+				}
+				// A rejected close is a REJECTED VALIDATION, not a warning. `Scope.close()` stops at the
+				// throwing listener, so its remaining internal listener removal and subscription-hold release
+				// never run and the throwaway scope stays partially live — one leak per deploy, on the worker
+				// that serves the component.
+				if (failedCloses.length) {
+					throw new AggregateError(
+						failedCloses.map((result) => result.reason),
+						`Could not tear down deploy validation for ${componentName}: ${failedCloses.length} scope(s) failed to close`
+					);
+				}
+			}
+		});
+		// Track the load+close so a concurrent worker shutdown waits for these scopes to finish
+		// disposing — a plugin may start a native runtime in handleApplication — before realExit.
+		trackScopeClose(validation);
+		try {
+			await validation;
+		} finally {
+			componentLoader.setErrorReporter(priorErrorReporter);
+			// The candidate path is unique per deploy, so leaving it in the loader's realpath registry leaks
+			// one dead entry per deploy for the life of the process.
+			componentLoader.forgetLoadedPath?.(candidateDirPath);
+			componentLoader.forgetLoadedModules?.(validationModules);
+		}
+		emit('phase', { phase: 'load', status: 'done' });
+
+		if (lastError) throw lastError;
+	}
+}
+
+const BRANCH_STORAGE_RETAINED =
+	'. Any branched database storage this application owns was left in place; drop it again with restart: true to discard that data';
+
+/** Report a restart outcome the operation's own success message cannot convey. */
+function logRestartOutcome(restart, what) {
+	const { RESTART_IDLE_TIMEOUT_MS, RESTART_WAIT_CEILING_MS } = require('./awaitRestart.ts');
+	if (restart.replacementsNotStarted)
+		log.warn(
+			`${restart.replacementsNotStarted} replacement worker thread(s) did not report starting after ${what}; the pool is short until they are restarted`
+		);
+	if (restart.error) log.error(`Restart after ${what} failed`, restart.error);
+	else if (restart.workersKeptOnOldCode)
+		log.warn(
+			`${restart.workersKeptOnOldCode} worker thread(s) could not be replaced after ${what} and are still running the previous code`
+		);
+	else if (restart.declined) log.warn(`No restart was performed after ${what}: the process is already shutting down`);
+	else if (restart.handedOff)
+		log.debug?.(`The restart after ${what} was handed to the main thread, which reports its own completion`);
+	else if (restart.stalled)
+		log.warn(
+			`The restart after ${what} stopped making progress for ${RESTART_IDLE_TIMEOUT_MS}ms; worker threads may still be running the previous code`
+		);
+	else if (!restart.completed)
+		log.warn(
+			`The restart after ${what} was still running after ${RESTART_WAIT_CEILING_MS}ms; worker threads may still be running the previous code`
+		);
+}
 /**
  * Can deploy a component in multiple ways. If a 'package' is provided all it will do is write that package to
  * harperdb-config, when HDB is restarted the package will be installed in hdb/nodeModules. If a base64 encoded string is passed it
@@ -433,13 +590,67 @@ async function deployComponent(req) {
 	// replicated ciphertext (reference, not embed); already-reference entries pass through, and with
 	// no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers
 	// re-running a replicated deploy already carry references and never re-ingest.
+	const { isIsolatedApplication } = require('../server/threads/isolatedApplications.ts');
+	const requestedIsolation = req.isolated;
+	const isReplicatedExecution = typeof req._deploymentId === 'string';
+	const mode = req.deployment_id ? 'activate' : req.activate === false ? 'stage' : 'deploy';
+	const isActivation = mode === 'activate';
+	// this thread's cached config may predate an earlier deploy that changed the entry without a restart
+	env.initSync(true);
+	const initiallyIsolated = isIsolatedApplication(req.project);
+	if (requestedIsolation !== undefined && !req.package) {
+		throw handleHDBError(
+			new Error(),
+			`'isolated' is only supported for package deployments; set it on the application's root config entry instead`,
+			HTTP_STATUS_CODES.BAD_REQUEST
+		);
+	}
+	// Only the originating node rejects admission: replicated peers must commit the same desired config,
+	// then report any node-local inability to run it through component lifecycle status.
+	const assertIsolationAdmission = async (isolated) => {
+		if (!isolated || isReplicatedExecution) return;
+		const {
+			isolatedApplicationRefusal,
+			isolatedApplicationCapacityRefusal,
+			presentIsolatedApplicationNames,
+		} = require('../server/threads/isolatedApplications.ts');
+		let runningApplications;
+		try {
+			runningApplications = await manageThreads.getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS);
+		} catch (error) {
+			throw handleHDBError(
+				error,
+				`Cannot deploy '${req.project}' as an isolated application: the main thread's worker topology is unavailable: ${error.message}`,
+				HTTP_STATUS_CODES.SERVICE_UNAVAILABLE
+			);
+		}
+		runningApplications = new Set([
+			...runningApplications,
+			...presentIsolatedApplicationNames().filter((application) => !isolatedApplicationRefusal(application)),
+		]);
+		const refusal =
+			isolatedApplicationRefusal(req.project) ?? isolatedApplicationCapacityRefusal(req.project, runningApplications);
+		if (refusal) {
+			throw handleHDBError(
+				new Error(),
+				`Cannot deploy '${req.project}' as an isolated application: ${refusal}`,
+				HTTP_STATUS_CODES.CONFLICT
+			);
+		}
+	};
+	// Skipped for an activation: the artifact's descriptor is the authority for the isolation it wants, and
+	// `admitIsolation` runs the real check under the preparation lock. Admitting the CURRENT value here would
+	// refuse an activation whose artifact turns isolation OFF for a component whose present isolated state is
+	// itself the refusal — a 409 for the request that would fix it.
+	if (!isActivation) await assertIsolationAdmission(requestedIsolation ?? initiallyIsolated);
 	const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts');
-	req.credentials = await ingestCredentials(req, req.credentials, req.project);
+	// An activation resolves, fetches and installs nothing, so there is no credential for it to carry.
+	// The validator rejects one; this keeps the ingest itself off the path rather than relying on that.
+	if (!isActivation) req.credentials = await ingestCredentials(req, req.credentials, req.project);
 	// References are safe to persist (config + deployment row) and replicate; a no-custody literal
 	// token is not — it is used only for this node's install below, then stripped before replication.
 	const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined);
 
-	// Write to root config if the request contains a package identifier
 	if (req.package) {
 		// Check if trying to overwrite a core component (requires force)
 		// Lazy-load to avoid circular dependency with componentLoader
@@ -451,23 +662,9 @@ async function deployComponent(req) {
 				HTTP_STATUS_CODES.CONFLICT
 			);
 		}
-
-		const applicationConfig = { package: req.package };
-		// Avoid writing an empty `install:` block
-		if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) {
-			applicationConfig.install = {
-				command: req.install_command,
-				timeout: req.install_timeout,
-				allowInstallScripts: req.install_allow_scripts,
-			};
-		}
-		if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath;
-		if (req.host !== undefined) applicationConfig.host = req.host;
-		// Persist credential references (never tokens) so every cold install of this component —
-		// reboot, new peer, rollback — re-resolves the credential from the store.
-		if (credentialReferences.length) applicationConfig.credentials = credentialReferences;
-		await configUtils.addConfig(req.project, applicationConfig);
 	}
+	let wasIsolated;
+	let nowIsolated;
 
 	// Create a hdb_deployment row up front so the deploy is observable and auditable
 	// even if the CLI disconnects. The row also holds the payload in a Blob attribute,
@@ -477,7 +674,6 @@ async function deployComponent(req) {
 	// recording so we don't accumulate one row per node for the same deploy. The row
 	// reaches peers via the table's standard replication; the peer-side branch below
 	// reads payload_blob back from there.
-	const isReplicatedExecution = typeof req._deploymentId === 'string';
 	// An SSE-bound caller already attached a ProgressEmitter (created in the server
 	// handler so it can also drive the response stream). Reuse it; otherwise spin up a
 	// fresh emitter so the recorder still gets phase events for non-SSE deploys.
@@ -489,6 +685,7 @@ async function deployComponent(req) {
 				project: req.project,
 				package_identifier: req.package ?? null,
 				user: req.hdb_user?.username,
+				activated_from: req.deployment_id ?? null,
 				restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null,
 				// Reference form only — the rollback source for re-resolving the credential.
 				credentials: credentialReferences.length ? credentialReferences : null,
@@ -513,12 +710,18 @@ async function deployComponent(req) {
 	// Bounded ring buffer of install stdout/stderr so a non-SSE caller sees the tail
 	// in the thrown error. SSE callers still stream every line live.
 	const installCapture = createInstallCapture();
+	let stagedOnOrigin = false;
 	try {
 		// On the origin, tee the tarball (Buffer or Readable from the multipart parser)
 		// through a hash-and-size tap into the row's payload_blob, then re-source extraction
 		// from the persisted blob. When `system` replicates, the blob becomes the channel
 		// peers read from; when it doesn't, the blob stays local for audit and rollback.
-		if (recorder && req.payload != null) {
+		if (isActivation) {
+			// A replicated activation has exactly the shape the peer-side blob branch below keys on — neither
+			// `package` nor `payload` — so without this branch every peer would wait out `deployment_timeout`
+			// for a payload its row never had while the origin swapped successfully.
+			extractionPayload = undefined;
+		} else if (recorder && req.payload != null) {
 			await recorder.ingestPayload(req.payload);
 			extractionPayload = recorder.row.payload_blob.stream();
 		} else if (isReplicatedExecution && req.payload == null && !req.package) {
@@ -554,9 +757,13 @@ async function deployComponent(req) {
 		if (isReplicatedExecution) {
 			credentialsWaitMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS);
 		}
-		const resolvedCredentials = await resolveCredentials(req.credentials, req.project, {
-			waitMs: credentialsWaitMs,
-		});
+		// Skipped for an activation, which runs no pack, clone or install: waiting for an hdb_secret row to
+		// replicate in would only delay a swap that has nothing to authenticate to.
+		const resolvedCredentials = isActivation
+			? undefined
+			: await resolveCredentials(req.credentials, req.project, {
+					waitMs: credentialsWaitMs,
+				});
 
 		const application = new Application({
 			name: req.project,
@@ -587,52 +794,97 @@ async function deployComponent(req) {
 		if (credentialReferences.length) req.credentials = credentialReferences;
 		else delete req.credentials;
 
+		// Validated INSIDE preparation, between build and swap, so a component that installs cleanly but
+		// throws at load never becomes live.
+		//
+		// Phase ORDER is unchanged for clients, but `prepare:done` means "candidate built", not "swap
+		// committed" — so a later failure arrives after both phases reported success. The operation's error
+		// is the authority on whether the deploy landed, not the phase stream.
 		emit('phase', { phase: 'prepare', status: 'start' });
-		await prepareApplication(application);
-		emit('phase', { phase: 'prepare', status: 'done' });
-
-		// now we attempt to actually load the component in case there is
-		// an error we can immediately detect and report, but app code should not run on the main thread
-		if (!isMainThread && !process.env.HARPER_SAFE_MODE) {
-			const pseudoResources = new Resources();
-			pseudoResources.isWorker = true;
-
-			const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts');
-			const { trackScopeClose } = require('./scopeShutdown.ts');
-			let lastError;
-			componentLoader.setErrorReporter((error) => (lastError = error));
-			emit('phase', { phase: 'load', status: 'start' });
-			// This load exists only to surface load-time errors early; the Scopes it creates are
-			// throwaway. They are collected (instead of registered for worker-shutdown auto-close) so we
-			// can close them here once validation completes — otherwise each deploy leaks the Scope's
-			// deploy-lifecycle listeners on this worker, eventually tripping MaxListenersExceededWarning
-			// (#1462).
-			const validationScopes = new Set();
-			// Process-wide `server.*` registrations (registerOperation, setMcpQuotaHandler) are not owned by
-			// a Scope, so a candidate's top-level registration during this throwaway load would otherwise
-			// outlive it and pollute the live worker on a failed/rolled-back deploy. The guard makes those
-			// registration methods no-op for the duration of the load.
-			const { runWithDeployValidationGuard } = require('../server/serverHelpers/deployValidationState.ts');
-			const validation = runWithDeployValidationGuard(async () => {
-				try {
-					await componentLoader.loadComponent(application.dirPath, pseudoResources, undefined, {
-						collectScopes: validationScopes,
-					});
-				} finally {
-					const closeResults = await Promise.allSettled(Array.from(validationScopes, (scope) => scope.close()));
-					for (const result of closeResults) {
-						if (result.status === 'rejected') log.warn('Failed to close a deploy-validation Scope', result.reason);
+		// The root-config entry a package deploy publishes. A stage records it with the artifact instead of
+		// publishing it, so a staged release nobody activated cannot leave config naming it — and the
+		// activation that eventually publishes it does so in the same order a normal deploy does.
+		let stagedRootConfig = null;
+		await prepareApplication(application, {
+			// `.deploy-staging/<artifactId>`. The public deployment id, so the id the caller was handed is
+			// the id a later `deployment_id` request can name; an activation names the artifact's own id,
+			// not the new row's.
+			artifactId: req.deployment_id ?? req._deploymentId,
+			mode,
+			describeArtifact: () => ({ rootConfig: stagedRootConfig, isolated: Boolean(nowIsolated) }),
+			// Returns its own undo. Publication happens BEFORE the swap, in the same order a normal deploy
+			// uses, so an activation that fails pre-commit would otherwise leave config naming a release that
+			// is not live — and `installApplications()` would resolve and install that package from scratch at
+			// the next boot, over a component whose certified artifact is sitting right there. Restoring is
+			// best-effort and not durable (that is #2315 step 3's work); it is still the difference between a
+			// failed activation that changed nothing and one that re-points the component.
+			publishRootConfig: async (entry) => {
+				const previous = configUtils.getConfigObj()?.[req.project];
+				await configUtils.addConfig(req.project, entry);
+				env.initSync(true);
+				// Read BACK, not `entry` — see `publishedEntryStillStands`.
+				const published = configUtils.getConfigObj()?.[req.project];
+				return async () => {
+					if (
+						!publishedEntryStillStands(
+							published,
+							() => env.initSync(true),
+							() => configUtils.getConfigObj()?.[req.project]
+						)
+					) {
+						return;
 					}
+					if (previous === undefined) configUtils.deleteConfigFromFile([req.project]);
+					else await configUtils.addConfig(req.project, previous);
+					env.initSync(true);
+				};
+			},
+			admitIsolation: async (descriptor) => {
+				env.initSync(true);
+				wasIsolated = isIsolatedApplication(req.project);
+				// A package artifact restores its own root-config entry, so its recorded intent is what will
+				// be in force. A payload artifact publishes no config, so the effective configuration is the
+				// only authority there — a saved `false` must not admit past an isolation the operator set
+				// between the stage and this call.
+				nowIsolated = descriptor.rootConfig ? descriptor.isolated : wasIsolated;
+				await assertIsolationAdmission(nowIsolated);
+			},
+			beforePrepare: async () => {
+				if (isActivation) return;
+				env.initSync(true);
+				wasIsolated = isIsolatedApplication(req.project);
+				nowIsolated = requestedIsolation ?? wasIsolated;
+				await assertIsolationAdmission(nowIsolated);
+				if (!isReplicatedExecution && req.package) req.isolated = nowIsolated;
+				if (!req.package) return;
+				const applicationConfig = { package: req.package };
+				if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) {
+					applicationConfig.install = {
+						command: req.install_command,
+						timeout: req.install_timeout,
+						allowInstallScripts: req.install_allow_scripts,
+					};
 				}
-			});
-			// Track the load+close so a concurrent worker shutdown waits for these scopes to finish
-			// disposing — a plugin may start a native runtime in handleApplication — before realExit.
-			trackScopeClose(validation);
-			await validation;
-			emit('phase', { phase: 'load', status: 'done' });
-
-			if (lastError) throw lastError;
-		}
+				if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath;
+				if (req.host !== undefined) applicationConfig.host = req.host;
+				if (req.branchedDatabases !== undefined) applicationConfig.branchedDatabases = req.branchedDatabases;
+				if (nowIsolated) applicationConfig.isolated = true;
+				if (credentialReferences.length) applicationConfig.credentials = credentialReferences;
+				if (mode === 'stage') {
+					stagedRootConfig = applicationConfig;
+					return;
+				}
+				await configUtils.addConfig(req.project, applicationConfig);
+				env.initSync(true);
+			},
+			validateCandidate: async (candidateDirPath) => {
+				emit('phase', { phase: 'prepare', status: 'done' });
+				await validateComponentLoads(candidateDirPath, emit);
+			},
+		});
+		// The build is certified on disk from here on, so every later failure — a peer result, or a rejection
+		// thrown by the replication layer itself — still leaves an artifact this id can activate.
+		if (mode === 'stage') stagedOnOrigin = true;
 		const rollingRestart = req.restart === 'rolling';
 		// if doing a rolling restart set restart to false so that other nodes don't also restart.
 		req.restart = rollingRestart ? false : req.restart;
@@ -675,10 +927,29 @@ async function deployComponent(req) {
 			// when the per-peer callback already fired for these.
 			recorder.recordPeers(response.replicated);
 		}
-		if (req.restart === true) {
+		// A still-isolated application restarts only its own dedicated worker. Everything else restarts the
+		// pool: a shared application, and either direction of an isolation flip, where the reconcile in
+		// restartWorkers starts or stops the moving application's own worker.
+		const restartScope = wasIsolated && nowIsolated ? application.name : undefined;
+		if (mode === 'stage') {
+			// No restart and no restart-required flag: nothing about the running component changed. The
+			// marker is what tells the origin which peers understood the request — see the confirmation
+			// check below — and it is set on every node, origin included, since a peer returns this same
+			// object as its operation response.
+			response.staged = true;
+			response.message = `Staged: ${application.name}`;
+		} else if (req.restart === true) {
 			emit('phase', { phase: 'restart', status: 'start' });
-			manageThreads.restartWorkers('http');
+			// Workers not yet replaced keep serving the pre-deploy component set, and where the OS lets
+			// replacements share a port they keep accepting connections for the whole rolling restart, so
+			// a caller that reads success as "the component is live" can be served by a worker that has
+			// never heard of it.
+			const { awaitRestart } = require('./awaitRestart.ts');
+			const restart = await awaitRestart((onProgress) =>
+				manageThreads.restartWorkers('http', undefined, undefined, onProgress, restartScope)
+			);
 			emit('phase', { phase: 'restart', status: 'done' });
+			logRestartOutcome(restart, `deploying ${application.name}`);
 			response.message = `Successfully deployed: ${application.name}, restarting Harper`;
 		} else if (rollingRestart) {
 			const serverUtilities = require('../server/serverHelpers/serverUtilities.ts');
@@ -686,6 +957,8 @@ async function deployComponent(req) {
 			const jobResponse = await serverUtilities.executeJob({
 				operation: 'restart_service',
 				service: 'http',
+				scope: manageThreads.encodeRestartScope(restartScope),
+				scopeFallback: restartScope === undefined ? undefined : manageThreads.encodeRestartScope(undefined),
 				replicated: true,
 			});
 			emit('phase', { phase: 'restart', status: 'done' });
@@ -705,10 +978,13 @@ async function deployComponent(req) {
 			// An existing component's watched files are handled by Scope/EntryHandler. Package
 			// metadata is deliberately outside most plugin globs, so compare it across the atomic
 			// swap as well: a dependency or module-entry change also invalidates loaded code.
-			if (application.isNewComponent || application.packageMetadataChanged) {
-				const { requestRestart } = require('./requestRestart.ts');
-				requestRestart();
-			}
+			const { requestRestartAfterDeploy } = require('./requestRestart.ts');
+			requestRestartAfterDeploy(
+				application.isNewComponent,
+				application.packageMetadataChanged,
+				wasIsolated,
+				nowIsolated
+			);
 			response.message = `Successfully deployed: ${application.name}`;
 		}
 
@@ -724,12 +1000,37 @@ async function deployComponent(req) {
 				const detail = failedPeers
 					.map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`)
 					.join(', ');
-				throw new ServerError(
-					`Component '${application.name}' was deployed on the origin node but failed to replicate to ` +
-						`${failedPeers.length} of ${recorder.row.peer_results.length} peer node(s): ${detail}. ` +
-						`See deployment ${recorder.deploymentId} (get_deployment) for details, or pass ` +
-						`ignore_replication_errors: true to treat replication failures as non-fatal.`
+				const replicationError = new ServerError(
+					`Component '${application.name}' was ${mode === 'stage' ? 'staged' : 'deployed'} on the origin node ` +
+						`but failed to replicate to ${failedPeers.length} of ${recorder.row.peer_results.length} peer ` +
+						`node(s): ${detail}. See deployment ${recorder.deploymentId} (get_deployment) for details, or ` +
+						`pass ignore_replication_errors: true to treat replication failures as non-fatal.`
 				);
+				throw replicationError;
+			}
+
+			// A peer running a build that predates staged deploys accepts `activate: false` as an unknown
+			// field and performs its ordinary deploy, so it is now serving a release the operator asked only
+			// to stage. Nothing can prevent that — the operations channel carries no peer version or
+			// capability — so the origin reports it, from the one signal that separates the two cases: a node
+			// that staged says so in its response. Read from the raw aggregate, because the recorder's peer
+			// normalization keeps only node/status/error.
+			//
+			// The wording stays at "did not confirm". An unreachable peer and one that dropped the request
+			// look identical from here, and telling an operator those nodes went live would be a guess.
+			if (mode === 'stage') {
+				const unconfirmed = unconfirmedStagingPeers(response?.replicated);
+				if (unconfirmed.length > 0) {
+					const detail = unconfirmed.map((peer) => peer.node ?? 'unknown').join(', ');
+					const unconfirmedError = new ServerError(
+						`Component '${application.name}' was staged on the origin node, but ${unconfirmed.length} peer ` +
+							`node(s) did not confirm staging: ${detail}. Either they are unreachable, or they run a build ` +
+							`that predates staged deploys — which treats this request as an ordinary deploy, so the ` +
+							`release is already serving there. Check those nodes before activating deployment ` +
+							`${recorder.deploymentId}, or pass ignore_replication_errors: true to accept the difference.`
+					);
+					throw unconfirmedError;
+				}
 			}
 		}
 
@@ -743,14 +1044,28 @@ async function deployComponent(req) {
 			// the tarball when it's still the artifact you'd debug or retry with: failed deploys
 			// don't reach this branch, and a deploy that reached here only because
 			// ignore_replication_errors masked failed peers keeps its payload for those peers.
+			// Never for a stage: the artifact on disk is not yet a release anyone can re-derive, and retention
+			// can evict it (it shares `deployment_stagingRetention_maxCount` with incidental dormant builds),
+			// so dropping the tarball here is how a staged release becomes unrecoverable — no tree, no bytes,
+			// and a row still reporting `staged`. An operator can still reclaim it deliberately with
+			// delete_deployment_payload once they accept that.
 			const payloadSize = recorder.row.payload_size;
 			const retentionMaxSize = getPayloadRetentionMaxSize();
-			if (typeof payloadSize === 'number' && payloadSize > retentionMaxSize && recorder.getFailedPeers().length === 0) {
+			if (
+				mode !== 'stage' &&
+				typeof payloadSize === 'number' &&
+				payloadSize > retentionMaxSize &&
+				recorder.getFailedPeers().length === 0
+			) {
 				const freed = recorder.dropPayload();
 				if (freed > 0) emit('payload_dropped', { payload_size: freed, max_size: retentionMaxSize });
 			}
-			emit('phase', { phase: 'success', status: 'done' });
-			await recorder.finish('success');
+			emit('phase', { phase: mode === 'stage' ? 'staged' : 'success', status: 'done' });
+			await recorder.finish(mode === 'stage' ? 'staged' : 'success');
+			if (mode === 'stage') {
+				response.message =
+					`Staged: ${application.name}. Deploy it with deploy_component ` + `deployment_id=${recorder.deploymentId}`;
+			}
 		}
 		return response;
 	} catch (err) {
@@ -791,7 +1106,7 @@ async function deployComponent(req) {
 		// install output, and failed_peers the caller needs.
 		if (recorder) {
 			try {
-				await recorder.finish('failed', err);
+				await recorder.finish(stagedOnOrigin ? 'staged' : 'failed', err);
 			} catch (finishErr) {
 				log.warn('Failed to record deployment failure row', finishErr);
 			}
@@ -800,6 +1115,43 @@ async function deployComponent(req) {
 	}
 }
 
+/**
+ * Whether the root-config entry an activation published is still the one on disk — the condition for that
+ * activation's failure to take it back. A `set_configuration` acknowledged while the swap was retrying is
+ * not this operation's to overwrite, and restoring a snapshot taken before it would silently drop it.
+ *
+ * Split out from the caller so it can be tested without a deploy: this is the only guard on a hazard the
+ * preparation lock does not cover, and both ways it can regress fail silently. `refreshConfig` is called
+ * BEFORE the read and is load-bearing — `set_configuration` writes the config file without refreshing this
+ * process's config object, so a cached read compares against a value that predates the very change the
+ * guard exists to protect and always concludes nothing moved. It narrows the window rather than closing
+ * it; serializing config publication is #2315 step 3.
+ *
+ * `published` must be the entry as read back after publication, not as passed in: only two values that went
+ * through the same write-and-parse round trip are comparable, and comparing against the argument would read
+ * any serialization difference as a concurrent change and skip every undo.
+ */
+function publishedEntryStillStands(published, refreshConfig, readCurrentEntry) {
+	refreshConfig();
+	return isDeepStrictEqual(readCurrentEntry(), published);
+}
+
+/**
+ * Peers that did not answer a stage with `staged: true` — either unreachable, or running a build that
+ * predates staged deploys and therefore treated the request as an ordinary deploy.
+ *
+ * Split out from the caller so it can be tested without a cluster: this is the only safety net for the
+ * mixed-version hazard #2315 records as accepted, and a regression in it fails silently. The entry shape
+ * belongs to harper-pro's replicator rather than this repo — `normalizePeerResult` already tolerates more
+ * than one — so the marker is read flat or from a wrapped body; assuming flat would report every peer in a
+ * fully-upgraded cluster as unconfirmed.
+ */
+function unconfirmedStagingPeers(replicated) {
+	if (!Array.isArray(replicated)) return [];
+	const confirmed = (peer) => peer?.staged === true || peer?.value?.staged === true || peer?.body?.staged === true;
+	return replicated.filter((peer) => peer && !confirmed(peer));
+}
+
 // Ring buffer of install stdout/stderr lines, capped by both line count and bytes so
 // a chatty install can't unbounded-grow the error response. snapshot() reports whether
 // the head was dropped so callers can flag truncation.
@@ -869,9 +1221,13 @@ async function getComponents() {
 			const list = await fs.readdir(dir, { withFileTypes: true });
 			for (let item of list) {
 				const itemName = item.name;
+				// Deny-list, not a dot-prefix skip: component CONTENTS legitimately include dot-files
+				// (`.aiignore`, `.env.example`) that callers expect to see, so only Harper's own
+				// bookkeeping directories are excluded by name.
 				if (
 					itemName === 'node_modules' ||
 					itemName === ASIDE_STAGING_DIR ||
+					itemName === DEPLOY_STAGING_DIR ||
 					itemName === COMPONENT_PREPARATION_LOCK_DIR
 				)
 					continue;
@@ -1168,33 +1524,99 @@ async function dropComponent(req) {
 
 	const { project, file } = req;
 	const projectPath = req.file ? path.join(project, file) : project;
-	const pathToComponent = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath);
-
-	const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project);
-	if (await fs.pathExists(componentSymlink)) {
-		await fs.unlink(componentSymlink);
-	}
+	const componentsRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	const componentPath = path.join(componentsRoot, project);
+	const pathToComponent = path.join(componentsRoot, projectPath);
+	let restartScope;
+
+	let response;
+	await withComponentPreparationLock(
+		componentPath,
+		async () => {
+			if (req.restart === true) {
+				let runningApplications;
+				try {
+					runningApplications = await manageThreads.getRunningIsolatedApplications(
+						ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS
+					);
+				} catch (error) {
+					throw handleHDBError(
+						error,
+						`Cannot drop '${project}' with a restart: the main thread's worker topology is unavailable: ${error.message}`,
+						HTTP_STATUS_CODES.SERVICE_UNAVAILABLE
+					);
+				}
+				if (runningApplications.includes(project)) restartScope = project;
+			}
+			const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project);
+			if (!file && (await fs.pathExists(componentSymlink))) {
+				await fs.unlink(componentSymlink);
+			}
 
-	if (await fs.pathExists(pathToComponent)) {
-		await fs.remove(pathToComponent);
-	}
+			if (!file) {
+				await dropComponentDirectory(componentPath, project, log);
+			} else if (await fs.pathExists(pathToComponent)) {
+				await fs.remove(pathToComponent);
+			}
 
-	// Remove the component from the package.json file
-	const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json');
-	if (await fs.pathExists(packageJsonPath)) {
-		const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
-		if (packageJson?.dependencies?.[project]) {
-			delete packageJson.dependencies[project];
-		}
-		await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8');
-	}
+			const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json');
+			if (!file && (await fs.pathExists(packageJsonPath))) {
+				const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+				if (packageJson?.dependencies?.[project]) {
+					delete packageJson.dependencies[project];
+				}
+				await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8');
+			}
 
-	configUtils.deleteConfigFromFile([project]);
-	let response = await server.replication.replicateOperation(req);
-	if (req.restart === true) {
-		manageThreads.restartWorkers('http');
-		response.message = `Successfully dropped: ${projectPath}, restarting Harper`;
-	} else response.message = `Successfully dropped: ${projectPath}`;
+			if (!file) configUtils.deleteConfigFromFile([project]);
+			if (!file && isMainThread) env.initSync(true);
+			response = await server.replication.replicateOperation(req);
+			const { applicationHasBranchStorage, removeBranchesForApplication } = require('../resources/branchDatabase.ts');
+			const branched = !file && applicationHasBranchStorage(project);
+			if (req.restart !== true) {
+				response.message = `Successfully dropped: ${projectPath}`;
+				if (branched) response.message += BRANCH_STORAGE_RETAINED;
+				return;
+			}
+			// The branches go after the restart, under this lock; a worker is one of the threads being
+			// replaced, so it hands both to the main thread.
+			response.message = `Successfully dropped: ${projectPath}, restarting Harper`;
+			if (!isMainThread) {
+				parentPort.postMessage({
+					type: hdbTerms.ITC_EVENT_TYPES.RESTART,
+					workerType: 'http',
+					scope: manageThreads.encodeRestartScope(restartScope),
+					removeBranchesFor: file ? undefined : project,
+				});
+				if (branched) {
+					response.message +=
+						'; the main thread removes its branched database storage after the restart and logs the outcome';
+				}
+				return;
+			}
+			const { awaitRestart } = require('./awaitRestart.ts');
+			const restart = await awaitRestart((onProgress) =>
+				manageThreads.restartWorkers('http', undefined, undefined, onProgress, restartScope)
+			);
+			logRestartOutcome(restart, `dropping ${projectPath}`);
+			if (!branched) return;
+			if (restart.completed && !restart.workersKeptOnOldCode) {
+				try {
+					await removeBranchesForApplication(project);
+				} catch (error) {
+					log.error(`Branched database storage of ${project} could not be removed`, error);
+					throw new Error(
+						`Successfully dropped: ${projectPath}, but its branched database storage could not be removed and ` +
+							`was left in place: ${error.message}. Drop it again with restart: true to retry.`
+					);
+				}
+			} else {
+				log.warn(`Branched database storage of ${project} was left in place: the restart did not complete`);
+				response.message += BRANCH_STORAGE_RETAINED;
+			}
+		},
+		componentDropLockOptions(project)
+	);
 	return response;
 }
 
@@ -1207,6 +1629,8 @@ exports.addComponent = addComponent;
 exports.dropCustomFunctionProject = dropCustomFunctionProject;
 exports.packageComponent = packageComponent;
 exports.deployComponent = deployComponent;
+exports.unconfirmedStagingPeers = unconfirmedStagingPeers;
+exports.publishedEntryStillStands = publishedEntryStillStands;
 exports.getComponents = getComponents;
 exports.getComponentFile = getComponentFile;
 exports.setComponentFile = setComponentFile;