@omnitron-dev/omnitron 0.3.1 → 0.3.2

package/dist/services/remote-deployer.service.js

@@ -1,70 +1,32 @@
-/**
- * RemoteDeployer — SSH-based artifact deployment to remote/cluster nodes
- *
- * Deployment pipeline for a single node:
- *   1. Connect via SSH (xec SSHAdapter or child_process ssh)
- *   2. Check if omnitron daemon is installed on remote
- *   3. Transfer app artifact (tarball) via SFTP/SCP
- *   4. Extract artifact on remote node
- *   5. Signal remote omnitron daemon to reload/restart apps
- *   6. Verify health on remote
- *
- * For cluster stacks, this runs in parallel across all nodes.
- *
- * Artifact path on remote: /opt/omnitron/artifacts/<project>/<app>/<version>/
- * Remote daemon config: /etc/omnitron/omnitron.config.ts
- *
- * NEVER EXECUTED, as of 2026-09-14. Measured rather than assumed: no log in
- * `~/.omnitron/logs` carries a single line this class emits ("Starting
- * deployment to node", "Slave provisioned", "Deployment successful"), no
- * artifact directory exists, and — decisively — no project config declares
- * `stacks.nodes`, which `startRemoteStack` requires before it can reach here.
- * So both entry points are unreachable in every current configuration.
- *
- * This matters to anyone reading the code below and taking it for a working
- * path. It is careful code — arguments are quoted, path segments validated,
- * heredocs replaced with base64 — and none of that has ever met a real host.
- * Its first run will be its first test. Treat a green read of this file as
- * evidence about intent, not about behaviour.
- *
- * The first thing that first run found, 2026-09-14: **it could not log in.**
- * Every command here went through `ssh -o BatchMode=yes` with an optional
- * `-i <keyfile>`, and `BatchMode=yes` disables every interactive method —
- * password and key passphrase both. The console's Add Node dialog collects
- * exactly those, encrypts them in the daemon's vault, and the health monitor
- * uses them through `ExecutionService.ssh()` on every check round. So the
- * product had two SSH implementations: one that can present what the operator
- * gave it, and one — this — that cannot, on the path where it matters most.
- *
- * Measured against the test host, whose SSH answers a password in 311 ms:
- *
- *   ssh -o BatchMode=yes root@<host> 'echo ok'
- *   → root@<host>: Permission denied (publickey,password).
- *
- * That is the answer to "why can a machine added in the console not be
- * deployed to": not policy, not a missing feature — the deployer could not
- * authenticate as the operator had arranged. It goes through the same
- * `ExecutionService` as the checks now, and takes a `DeployTarget` carrying
- * the credentials rather than an `IStackNode` whose `ISSHConfig` has nowhere
- * to put them.
- */
+import path from 'node:path';
+import { stackExecutionKey } from '../shared/dto/stack-execution.js';
+import { deploymentProgressKey } from '../shared/release-reading.js';
+import { redactDiagnosticText } from '@omnitron-dev/titan/diagnostics';
+import { assertFrozenLaunch, installFrozenApplicationCommand } from '../release/frozen-installation.js';
+import { releaseOptionsForEnv } from '../release/target-runtime.js';
+import { installedReleaseCheckCommand, validOrigin } from '../release/runtime-provenance.js';
 import { PLATFORM_PROBE, parsePlatformProbe, planProvisioning, describePlan, } from './remote-provisioner.js';
 import { installSteps, installPreflightCommand, versionDir, activateSteps, pruneSteps } from './bundle-builder.js';
 import { reachabilityRule, reachabilityCommand } from '../infrastructure/gateway-reachability.js';
 import { readNodeHealth, readNodeStatus } from '../project/node-app-health.js';
-import { decideRedeploy, decideNodeStackStart, decideSlaveDaemonRestart, artifactChanged, readChecksumCommand, parseRecordedChecksum, ARTIFACT_CHECKSUM_FILE, NODE_CONFIG_HASH_FILE, } from './redeploy-decision.js';
+import { decideRedeploy, decideNodeStackStart, decideSlaveDaemonRestart, artifactChanged, readChecksumCommand, parseRecordedChecksum, ARTIFACT_CHECKSUM_FILE, } from './redeploy-decision.js';
 import { deliveredProbeCommand, parseDeliveredProbe, checkDelivered, } from '../release/delivered.js';
 import crypto from 'node:crypto';
 import { AsyncLocalStorage } from 'node:async_hooks';
-import { assertControlledDaemonPort, assertServingBundle, bundleProbeCommand, contractPreflightCommand, executeHostUpgrade, hookCommand, inventoryCommand, planHostUpgrade, receiptCommand, verifyRestoredInventory, updateContractBindingsCommand, runtimeProbeCommand, runtimeStartCommand, runtimeStopCommand, appHealthCommand, pendingReceiptsCommand, recoverHostUpgrade, stoppedDaemonCommand, } from './host-upgrade.js';
+import { assertControlledDaemonPort, assertServingBundle, bundleProbeCommand, contractPreflightCommand, executeHostUpgrade, hookCommand, inventoryCommand, planHostUpgrade, receiptCommand, verifyRestoredInventory, updateContractBindingsCommand, runtimeProbeCommand, runtimeStartCommand, runtimeStopCommand, appHealthCommand, pendingReceiptsCommand, recoverHostUpgrade, stoppedDaemonCommand, bootstrapPreflightCommand, HOST_BOOTSTRAP_PREFLIGHT_TIMEOUT_MS, assertResidentApplicationRuntime, assertSameApplicationRuntime, } from './host-upgrade.js';
 import { gunzipSync } from 'node:zlib';
 import { shellEscape } from '../shared/shell-escape.js';
 import { fromDataChannel, throughDataChannel } from '../execution/data-channel.js';
+import { commandFailure } from '../execution/command-failure.js';
+import { activateManagedTarget, managedHostCommand, planManagedRollout, reconcileManagedTarget } from './managed-rollout.js';
 /** A node that could not be asked at all — the SSH call itself failed. */
 const UNREADABLE = -3;
 import { withNodeLeases } from './node-deploy-lease.js';
 import { hostOperationBarrierCommand } from './host-operation-barrier.js';
 import { migrationOutput } from './migration-output.js';
+import { prepareRemoteMigration } from './migration-process.js';
+import { migrationDatabaseEnvironment, validateMigrationDestinations, validateMigrationOwners, orderMigrationPlacements } from './migration-target.js';
+import { prepareNodeConfigPublication, restoreNodeConfigCommand } from './node-registration.js';
 /**
  * A name that is safe to place in a remote filesystem path.
  *
@@ -103,6 +65,8 @@ function sshTargetOf(target) {
     const ssh = { host: target.host };
     if (target.sshPort != null)
         ssh.port = target.sshPort;
+    if (target.expectedHostFingerprint)
+        ssh.expectedHostFingerprint = target.expectedHostFingerprint;
     if (target.username)
         ssh.username = target.username;
     if (target.privateKey)
@@ -164,6 +128,12 @@ export function withNodeCredentials(target, registered) {
         merged.sshPort = registered.port;
     if (!merged.username && registered.username)
         merged.username = registered.username;
+    // Vault credentials and approval belong to one registered SSH account and port.
+    if (target.host !== registered.host || (merged.sshPort ?? 22) !== (registered.port ?? 22)
+        || (merged.username ?? 'root') !== (registered.username ?? 'root'))
+        return merged;
+    if (registered.expectedHostFingerprint)
+        merged.expectedHostFingerprint = registered.expectedHostFingerprint;
     if (!merged.privateKey && registered.privateKey)
         merged.privateKey = registered.privateKey;
     if (!merged.passphrase && registered.passphrase)
@@ -182,33 +152,13 @@ export function withNodeCredentials(target, registered) {
 const SLAVE_START_TIMEOUT_MS = 120_000;
 /** How often it is asked, while it is starting. */
 const DAEMON_POLL_MS = 3_000;
+const transferScope = new AsyncLocalStorage();
+export function withDeploymentProgress(scope, work) {
+    return transferScope.run(scope, work);
+}
 // =============================================================================
 // RemoteDeployer
 // =============================================================================
-/**
- * The parts of a postgres URL, for the environment a migrator expects.
- *
- * Returns null rather than guessing at a URL it cannot read: a migration run
- * against half-parsed connection details is worse than one that does not run,
- * because it can succeed somewhere nobody meant.
- */
-function parseDatabaseUrl(url) {
-    try {
-        const u = new URL(url);
-        if (!u.protocol.startsWith('postgres'))
-            return null;
-        return {
-            host: u.hostname,
-            port: u.port || '5432',
-            database: decodeURIComponent(u.pathname.replace(/^\//, '')),
-            user: decodeURIComponent(u.username),
-            password: decodeURIComponent(u.password),
-        };
-    }
-    catch {
-        return null;
-    }
-}
 /**
  * Is this a gzip stream of a COMPLETE tar archive?
  *
@@ -247,9 +197,7 @@ export class RemoteDeployer {
      * transferring, installing, the message naming the failing step — reached a
      * handler that re-emitted it to nobody.
      *
-     * Keyed by node and app, so this is bounded by the fleet rather than by how
-     * often anyone deploys, and the entry for an app is always its current
-     * state rather than a scroll an operator has to read to the end of.
+     * Each project, stack, node and app retains its latest observation across retries.
      */
     lastProgress = new Map();
     constructor(logger, execution) {
@@ -275,12 +223,22 @@ export class RemoteDeployer {
         const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
         this.logger.info({ node: nodeKey, app: artifact.app, version: artifact.version }, 'Starting deployment to node');
         try {
+            if (artifact.release && (!validOrigin(artifact.release) || artifact.release.app !== artifact.app ||
+                artifact.release.artifactSha256 !== artifact.tarballSha256 || artifact.release.artifactBytes !== artifact.size))
+                throw new Error('Release origin does not identify the supplied artifact');
+            const nodeOptions = artifact.release ? releaseOptionsForEnv(options?.launchEnv) : '';
+            let inspected;
             // 1. Verify SSH connectivity
             this.emitProgress(nodeKey, artifact.app, 'pending', 0, 'Connecting via SSH...');
             await this.verifySSH(target);
+            if (artifact.release) {
+                assertFrozenLaunch(artifact.launch, artifact.release);
+                inspected = await this.inspectInstalledBundle(target);
+                assertResidentApplicationRuntime(inspected.applicationRuntime);
+            }
             // 2. Ensure remote directory structure
-            const remotePath = artifactDirOnNode(project, artifact.app, artifact.version);
-            await this.sshExec(target, `mkdir -p ${shellEscape(remotePath)}`);
+            let directory = `${artifact.version}-${artifact.tarballSha256 ?? crypto.randomUUID()}`;
+            let remotePath = artifactDirOnNode(project, artifact.app, directory);
             // 2a. What the node already has.
             //
             // A master's boot-time autostart deploys every remote stack again, so
@@ -296,7 +254,10 @@ export class RemoteDeployer {
             // could pass: the escape hatch this comment named had no door.
             if (options?.force !== true) {
                 const recorded = parseRecordedChecksum(await this.sshExec(target, readChecksumCommand(shellEscape(remotePath))));
-                if (artifactChanged(recorded, artifact.checksum) === false) {
+                const sameInputs = artifactChanged(recorded, artifact.checksum) === false;
+                const binding = inspected?.applicationRuntime;
+                if (sameInputs && (!artifact.release || (binding &&
+                    (await this.sshExec(target, installedReleaseCheckCommand(remotePath, artifact.release, binding, nodeOptions), 120_000)).trim() === 'verified'))) {
                     const duration = Date.now() - startTime;
                     this.emitProgress(nodeKey, artifact.app, 'success', 75, 'Already installed — unchanged');
                     this.logger.info({ node: nodeKey, app: artifact.app, version: artifact.version, sha: recorded.slice(0, 12) }, 'The node already has this artifact — not transferring it again');
@@ -307,9 +268,20 @@ export class RemoteDeployer {
                         status: 'success',
                         duration,
                         unchanged: true,
+                        directory,
+                        ...(artifact.release && binding ? { migrationNode: binding.executable } : {}),
                     };
                 }
             }
+            // Never clear a version directory: a running process can still load files from it.
+            // A failed attempt, or an explicit reinstall, gets another private directory.
+            const occupied = await this.sshExec(target, `test -e ${shellEscape(remotePath)} && echo occupied || echo absent`);
+            if (options?.force === true || occupied.trim() !== 'absent') {
+                directory += `-${crypto.randomUUID()}`;
+                remotePath = artifactDirOnNode(project, artifact.app, directory);
+            }
+            const parent = artifactDirOnNode(project, artifact.app, artifact.version).replace(/\/[^/]+$/, '');
+            await this.sshExec(target, `umask 077; mkdir -p ${shellEscape(parent)} && mkdir ${shellEscape(remotePath)}`);
             // 3. Transfer artifact
             this.emitProgress(nodeKey, artifact.app, 'transferring', 20, 'Transferring artifact...');
             const remoteFile = `${remotePath}/${artifact.app}-${artifact.version}.tar.gz`;
@@ -349,116 +321,112 @@ export class RemoteDeployer {
             else {
                 this.logger.warn({ node: nodeKey, app: artifact.app }, 'Artifact carries no tarball hash — the transfer was not verified');
             }
-            // 4. Extract on remote, into an empty directory.
-            //
-            // `tar -xzf` over whatever was there before leaves everything the new
-            // archive does not name — and what was there before is a `node_modules`
-            // of a different SHAPE. The first deployment after this builder changed
-            // put an npm manifest beside a pnpm tree and `npm install` answered
-            //
-            //     npm error Cannot read properties of null (reading 'edgesOut')
-            //
-            // which is npm walking a dependency graph through symlinks it did not
-            // create. The same reasoning as `installSteps`, which removes the
-            // version directory before unpacking for exactly this: an install
-            // reconciling against a tree from another tool is not this version.
-            //
-            // Everything except the archive just uploaded, and only inside the
-            // artifact's own version directory — every segment of which was checked
-            // by `assertRemotePathSegment` above.
-            this.emitProgress(nodeKey, artifact.app, 'extracting', 50, 'Extracting artifact...');
+            // This newly reserved directory is not referenced by any registered application.
+            this.emitProgress(nodeKey, artifact.app, 'extracting', 50, 'Extracting candidate artifact...');
             const archiveName = `${artifact.app}-${artifact.version}.tar.gz`;
-            await this.sshExec(target, `cd ${shellEscape(remotePath)} && ` +
-                `find . -mindepth 1 -maxdepth 1 ! -name ${shellEscape(archiveName)} -exec rm -rf {} + && ` +
-                `tar -xzf ${shellEscape(archiveName)}`);
-            // 5. Install the dependencies ON THE NODE.
-            //
-            // Three shapes of this step, and the two that were wrong are worth
-            // keeping written down because each looked finished.
-            //
-            // It first ran `npm install --production --ignore-scripts 2>/dev/null
-            // || true`. That could not work even once — the manifests use pnpm's
-            // `workspace:` protocol, npm answers `EUNSUPPORTEDPROTOCOL` — and the
-            // `|| true` meant nobody found out.
-            //
-            // It then trusted the artifact to arrive complete, from `pnpm deploy`,
-            // and checked `test -d node_modules`. The artifact was NOT complete:
-            // every `@omnitron-dev/*` package was a symlink into a home directory
-            // on the master, twenty-three of them, and a directory of dangling
-            // symlinks passes `test -d`. A symlink to `/opt/omnitron/current`
-            // patched the one the first import happened to name; the other
-            // twenty-two were waiting behind it.
-            //
-            // Now the artifact carries `vendor/*.tgz` and a manifest that installs
-            // them, exactly like the daemon's own bundle, and the install runs
-            // here. That is not a cost to be minimised — it is the only step that
-            // happens on a machine that knows it is Linux, which is what decides
-            // between `@esbuild/darwin-arm64` and `@esbuild/linux-x64`.
-            this.emitProgress(nodeKey, artifact.app, 'installing', 60, 'Installing dependencies on the node...');
-            try {
-                await this.sshExec(target, `cd ${shellEscape(remotePath)} && npm install --omit=dev --no-audit --no-fund`, 900_000);
+            if (!artifact.release)
+                await this.sshExec(target, `cd ${shellEscape(remotePath)} && tar -xzf ${shellEscape(archiveName)}`);
+            const daemonBundle = inspected ?? await this.inspectInstalledBundle(target);
+            const runtime = daemonBundle.applicationRuntime;
+            if (artifact.release) {
+                assertResidentApplicationRuntime(runtime);
+                assertFrozenLaunch(artifact.launch, artifact.release);
+                assertSameApplicationRuntime(runtime, (await this.inspectInstalledBundle(target)).applicationRuntime);
+                this.emitProgress(nodeKey, artifact.app, 'extracting', 65, 'Verifying the unchanged application SDK and execution target');
+                const result = await this.sshExec(target, installFrozenApplicationCommand(remotePath, archiveName, artifact.launch, artifact.release, runtime, nodeOptions, daemonBundle.root), 240_000);
+                if (JSON.parse(result.trim()) !== nodeOptions)
+                    throw new Error('Release Node options differ from the effective application definition');
             }
-            catch (err) {
-                const duration = Date.now() - startTime;
-                const detail = `dependency install failed on the node: ${err.message.slice(0, 400)}`;
-                this.emitProgress(nodeKey, artifact.app, 'failed', 60, detail);
-                this.logger.error({ node: nodeKey, app: artifact.app, path: remotePath }, detail);
-                return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'failed', duration, error: detail };
+            else {
+                // 5. Install the dependencies ON THE NODE.
+                //
+                // Three shapes of this step, and the two that were wrong are worth
+                // keeping written down because each looked finished.
+                //
+                // It first ran `npm install --production --ignore-scripts 2>/dev/null
+                // || true`. That could not work even once — the manifests use pnpm's
+                // `workspace:` protocol, npm answers `EUNSUPPORTEDPROTOCOL` — and the
+                // `|| true` meant nobody found out.
+                //
+                // It then trusted the artifact to arrive complete, from `pnpm deploy`,
+                // and checked `test -d node_modules`. The artifact was NOT complete:
+                // every `@omnitron-dev/*` package was a symlink into a home directory
+                // on the master, twenty-three of them, and a directory of dangling
+                // symlinks passes `test -d`. A symlink to `/opt/omnitron/current`
+                // patched the one the first import happened to name; the other
+                // twenty-two were waiting behind it.
+                //
+                // Now the artifact carries `vendor/*.tgz` and a manifest that installs
+                // them, exactly like the daemon's own bundle, and the install runs
+                // here. That is not a cost to be minimised — it is the only step that
+                // happens on a machine that knows it is Linux, which is what decides
+                // between `@esbuild/darwin-arm64` and `@esbuild/linux-x64`.
+                this.emitProgress(nodeKey, artifact.app, 'installing', 60, 'Installing dependencies on the node...');
+                try {
+                    await this.sshExec(target, `cd ${shellEscape(remotePath)} && npm install --omit=dev --engine-strict --no-audit --no-fund`, 900_000);
+                }
+                catch (err) {
+                    const duration = Date.now() - startTime;
+                    const detail = `dependency install failed on the node: ${err.message.slice(0, 400)}`;
+                    this.emitProgress(nodeKey, artifact.app, 'failed', 60, detail);
+                    this.logger.error({ node: nodeKey, app: artifact.app, path: remotePath }, detail);
+                    return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'failed', duration, error: detail };
+                }
+                // 5a. One physical copy of every package the daemon shares with the app.
+                //
+                // A supervised app does not run alone in its process: omnitron's own
+                // `bootstrap-process.js` is the entry point, and the app's modules are
+                // loaded into it. Titan keeps identities at module scope — the
+                // `Container` class among them — so two physical installations of
+                // `@omnitron-dev/titan` in one process means `@Inject(Container)`
+                // resolves against a class that is not the one the daemon registered,
+                // and the failure reads `Dependency 'Container' not found`.
+                //
+                // `bootstrap-process.ts` checks for exactly this and refuses to start,
+                // which is how a freshly installed artifact answered:
+                //
+                //     Titan package identity mismatch detected between omnitron daemon
+                //     and app 'geo'. Daemon resolves @omnitron-dev/titan from
+                //     /opt/omnitron/versions/…; App resolves … from /opt/omnitron/
+                //     artifacts/sample/geo/0.0.1/node_modules/@omnitron-dev/titan
+                //
+                // That is the correct refusal, and the answer is not to install less:
+                // the vendored copies are what let `npm install` resolve a coherent
+                // tree in the first place. They are replaced afterwards by links to
+                // the daemon's own, computed ON THE NODE from what the daemon actually
+                // has, so the set stays right as the daemon's closure changes rather
+                // than being a list here that drifts from it.
+                //
+                // `@omnitron-dev/omnitron` is deliberately NOT one of them, and this is
+                // the second time that lesson has been paid for. The daemon IS that
+                // package, so it is not inside its own `node_modules`, and linking to
+                // the install root instead gives node a directory whose package.json
+                // carries `name`, `version`, `type` and `dependencies` and NO `main`
+                // and NO `exports` — `bundleRootManifest` writes an install manifest,
+                // not a library manifest. Resolution then falls back to `index.js`,
+                // which is not there:
+                //
+                //     Cannot find package
+                //     '.../node_modules/@omnitron-dev/omnitron/index.js'
+                //
+                // — the exact error this whole line of work started from. The vendored
+                // copy that `pnpm pack` produces from `apps/omnitron` has the real
+                // manifest, so it is left alone. Its identity is not shared across the
+                // process boundary the way titan's is: `defineSystem` returns data that
+                // the daemon's bootstrap reads, not a class anything does `instanceof`
+                // against.
+                //
+                // `rm -rf` before each `ln -sfn`: linking onto an existing DIRECTORY
+                // puts the link INSIDE it, leaving a directory with no package.json
+                // that node reports as a missing `index.js` — a message about a file
+                // that was never the problem.
+                const daemonRoot = runtime?.root ?? daemonBundle.root;
+                await this.sshExec(target, `set -e; APP=${shellEscape(`${remotePath}/node_modules/@omnitron-dev`)}; ` +
+                    `mkdir -p "$APP"; ` +
+                    `for p in ${shellEscape(daemonRoot)}/node_modules/@omnitron-dev/*; do ` +
+                    `[ -e "$p" ] || continue; n=$(basename "$p"); ` +
+                    `rm -rf "$APP/$n"; ln -sfn "$p" "$APP/$n"; done`);
             }
-            // 5a. One physical copy of every package the daemon shares with the app.
-            //
-            // A supervised app does not run alone in its process: omnitron's own
-            // `bootstrap-process.js` is the entry point, and the app's modules are
-            // loaded into it. Titan keeps identities at module scope — the
-            // `Container` class among them — so two physical installations of
-            // `@omnitron-dev/titan` in one process means `@Inject(Container)`
-            // resolves against a class that is not the one the daemon registered,
-            // and the failure reads `Dependency 'Container' not found`.
-            //
-            // `bootstrap-process.ts` checks for exactly this and refuses to start,
-            // which is how a freshly installed artifact answered:
-            //
-            //     Titan package identity mismatch detected between omnitron daemon
-            //     and app 'geo'. Daemon resolves @omnitron-dev/titan from
-            //     /opt/omnitron/versions/…; App resolves … from /opt/omnitron/
-            //     artifacts/sample/geo/0.0.1/node_modules/@omnitron-dev/titan
-            //
-            // That is the correct refusal, and the answer is not to install less:
-            // the vendored copies are what let `npm install` resolve a coherent
-            // tree in the first place. They are replaced afterwards by links to
-            // the daemon's own, computed ON THE NODE from what the daemon actually
-            // has, so the set stays right as the daemon's closure changes rather
-            // than being a list here that drifts from it.
-            //
-            // `@omnitron-dev/omnitron` is deliberately NOT one of them, and this is
-            // the second time that lesson has been paid for. The daemon IS that
-            // package, so it is not inside its own `node_modules`, and linking to
-            // the install root instead gives node a directory whose package.json
-            // carries `name`, `version`, `type` and `dependencies` and NO `main`
-            // and NO `exports` — `bundleRootManifest` writes an install manifest,
-            // not a library manifest. Resolution then falls back to `index.js`,
-            // which is not there:
-            //
-            //     Cannot find package
-            //     '.../node_modules/@omnitron-dev/omnitron/index.js'
-            //
-            // — the exact error this whole line of work started from. The vendored
-            // copy that `pnpm pack` produces from `apps/omnitron` has the real
-            // manifest, so it is left alone. Its identity is not shared across the
-            // process boundary the way titan's is: `defineSystem` returns data that
-            // the daemon's bootstrap reads, not a class anything does `instanceof`
-            // against.
-            //
-            // `rm -rf` before each `ln -sfn`: linking onto an existing DIRECTORY
-            // puts the link INSIDE it, leaving a directory with no package.json
-            // that node reports as a missing `index.js` — a message about a file
-            // that was never the problem.
-            const daemonRoot = (await this.inspectInstalledBundle(target)).root;
-            await this.sshExec(target, `set -e; APP=${shellEscape(`${remotePath}/node_modules/@omnitron-dev`)}; ` +
-                `mkdir -p "$APP"; ` +
-                `for p in ${shellEscape(daemonRoot)}/node_modules/@omnitron-dev/*; do ` +
-                `[ -e "$p" ] || continue; n=$(basename "$p"); ` +
-                `rm -rf "$APP/$n"; ln -sfn "$p" "$APP/$n"; done`);
             // The install either produced a tree that resolves or it did not, and
             // `test -d node_modules` cannot tell those apart — it was true for the
             // whole time nothing worked. Asking node to RESOLVE the package the
@@ -470,16 +438,17 @@ export class RemoteDeployer {
             // map publishes what it lists and nothing else. A probe that fails on
             // a healthy tree is worse than none — it would have failed every
             // deployment from here on, for the one reason that is not a fault.
-            this.emitProgress(nodeKey, artifact.app, 'extracting', 70, 'Checking dependencies...');
-            const resolved = await this.sshExec(target, `cd ${shellEscape(remotePath)} && node -e ${shellEscape("const r=require('module').createRequire(process.cwd()+'/package.json');" +
-                "r.resolve('@omnitron-dev/omnitron');r.resolve('@omnitron-dev/titan')")} >/dev/null 2>&1 && echo resolves || echo broken`).catch(() => 'broken');
-            if (resolved.trim() !== 'resolves') {
-                const duration = Date.now() - startTime;
-                const detail = 'the installed tree cannot resolve @omnitron-dev/omnitron or @omnitron-dev/titan, ' +
-                    'so the app will fail at its first import';
-                this.emitProgress(nodeKey, artifact.app, 'failed', 70, detail);
-                this.logger.error({ node: nodeKey, app: artifact.app, path: remotePath }, detail);
-                return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'failed', duration, error: detail };
+            if (!artifact.release) {
+                this.emitProgress(nodeKey, artifact.app, 'extracting', 70, 'Checking dependencies...');
+                const resolved = await this.sshExec(target, `cd ${shellEscape(remotePath)} && node -e ${shellEscape("const r=require('module').createRequire(process.cwd()+'/package.json');" +
+                    "r.resolve('@omnitron-dev/omnitron');r.resolve('@omnitron-dev/titan')")} >/dev/null 2>&1 && echo resolves || echo broken`).catch(() => 'broken');
+                if (resolved.trim() !== 'resolves') {
+                    const duration = Date.now() - startTime;
+                    const detail = 'The installed tree cannot resolve @omnitron-dev/omnitron or @omnitron-dev/titan';
+                    this.emitProgress(nodeKey, artifact.app, 'failed', 70, detail);
+                    this.logger.error({ node: nodeKey, app: artifact.app, path: remotePath }, detail);
+                    return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'failed', duration, error: detail };
+                }
             }
             // 5b. No record yet. The next deployment reads `.artifact-sha256` as
             // «the app runs this», and at this point it does not: the files are
@@ -504,35 +473,12 @@ export class RemoteDeployer {
                     version: artifact.version,
                     status: 'success',
                     duration,
+                    directory,
                     ...(artifact.checksum ? { checksum: artifact.checksum } : {}),
+                    ...(artifact.release && runtime ? { migrationNode: runtime.executable } : {}),
                 };
             }
-            this.emitProgress(nodeKey, artifact.app, 'restarting', 80, 'Restarting app on remote...');
-            const started = await this.signalRemoteDaemon(target, artifact.app);
-            // 7. Verify health
-            this.emitProgress(nodeKey, artifact.app, 'verifying', 90, 'Verifying health...');
-            const health = await this.verifyHealth(target, artifact.app, project);
-            const duration = Date.now() - startTime;
-            // An artifact on disk is not a running application. Reporting success
-            // for the transfer alone is what let a fleet show six deployed apps and
-            // run none of them.
-            if (!started.ok || !health.online) {
-                const why = !started.ok ? started.detail : health.detail;
-                this.emitProgress(nodeKey, artifact.app, 'failed', 90, `artifact installed, app not running: ${why}`);
-                this.logger.error({ node: nodeKey, app: artifact.app, version: artifact.version, duration, detail: why }, 'Artifact installed, but the app is not running on the node');
-                return {
-                    node: nodeKey,
-                    app: artifact.app,
-                    version: artifact.version,
-                    status: 'failed',
-                    duration,
-                    error: `artifact installed, app not running: ${why}`,
-                };
-            }
-            await this.recordRunningArtifact(target, remotePath, artifact.app, artifact.checksum);
-            this.emitProgress(nodeKey, artifact.app, 'success', 100, `Deployed in ${Math.round(duration / 1000)}s`);
-            this.logger.info({ node: nodeKey, app: artifact.app, version: artifact.version, duration }, 'Deployment successful');
-            return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'success', duration };
+            throw new Error('Artifact installed in a new directory; activate it through deployToStack with application definitions');
         }
         catch (err) {
             const duration = Date.now() - startTime;
@@ -546,6 +492,29 @@ export class RemoteDeployer {
      * Deploy artifacts to all nodes in a stack (parallel).
      */
     async deployToStack(targets, artifacts, project, options) {
+        if (options?.apps?.length) {
+            if (!options.stack)
+                throw new Error('Application deployment requires an explicit source stack before transferring artifacts');
+            stackExecutionKey({ project, stack: options.stack });
+        }
+        const migrations = options?.migrations ?? [];
+        validateMigrationOwners(migrations, options?.apps ?? []);
+        targets = orderMigrationPlacements(migrations, targets.map(target => ({ target,
+            apps: (options?.apps ?? []).filter(app => !target.apps || target.apps.includes(app.name)),
+        }))).map(placement => placement.target);
+        const destinations = migrations.map(request => {
+            if (!artifacts.some(artifact => artifact.app === request.app))
+                throw Error(`No artifact for migration owner ${request.app}`);
+            const app = options?.apps?.find(entry => entry.name === request.app);
+            if (!app)
+                throw Error(`Missing migration owner ${request.app}`);
+            const env = { ...app.env, ...options?.appEnv?.[request.app] };
+            const database = migrationDatabaseEnvironment(request.app, env['DATABASE_URL'] ?? '', request.database, request.mode);
+            if (request.mode === 'config' && !artifacts.find(artifact => artifact.app === request.app)?.release)
+                throw Error(`Explicit migration config for ${request.app} requires a verified release artifact`);
+            return { app: request.app, database };
+        });
+        validateMigrationDestinations(destinations);
         const concurrency = options?.concurrency ?? 3;
         const results = [];
         // Build deployment matrix: each app to each node (or node-specific apps)
@@ -558,6 +527,13 @@ export class RemoteDeployer {
                 tasks.push({ target, artifact });
             }
         }
+        const owners = new Map();
+        for (const { target, artifact } of tasks)
+            if (artifact.release && !owners.has(target)) {
+                const runtime = (await this.inspectInstalledBundle(target)).applicationRuntime;
+                assertResidentApplicationRuntime(runtime);
+                owners.set(target, runtime);
+            }
         // Execute with concurrency limit
         const executing = new Set();
         for (const task of tasks) {
@@ -566,6 +542,10 @@ export class RemoteDeployer {
                 // started before the node knows it exists.
                 const result = await this.deployToNode(task.target, task.artifact, project, {
                     startAfterInstall: !options?.apps?.length,
+                    ...(task.artifact.release ? { launchEnv: {
+                            ...options?.apps?.find(app => app.name === task.artifact.app)?.env,
+                            ...options?.appEnv?.[task.artifact.app],
+                        } } : {}),
                     ...(options?.force === true ? { force: true } : {}),
                 });
                 results.push(result);
@@ -590,11 +570,38 @@ export class RemoteDeployer {
                 const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
                 const landed = results
                     .filter((r) => r.node === nodeKey && r.status === 'success')
-                    .map((r) => ({ app: r.app, version: r.version }));
-                const registration = await this.registerNodeApps(target, project, options.apps, landed, options.appEnv, options.alertSink);
-                // Schema before the apps that read it — and an app whose schema did
-                // not arrive does not get its new code started on the old one.
-                const migrationFailed = await this.migrateNodeApps(target, project, landed, options.appEnv);
+                    .map((r) => ({ app: r.app, version: r.version, ...(artifacts.find(a => a.app === r.app)?.launch ? { launch: artifacts.find(a => a.app === r.app).launch } : {}), ...(r.migrationNode ? { migrationNode: r.migrationNode } : {}), ...(r.directory ? { directory: r.directory } : {}) }));
+                const expected = tasks.filter(task => task.target === target).length;
+                if (landed.length !== expected) {
+                    for (const result of results.filter(r => r.node === nodeKey && r.status === 'success')) {
+                        result.status = 'failed';
+                        result.error = 'Another artifact failed installation; existing node configuration and processes were retained';
+                        this.emitProgress(nodeKey, result.app, 'failed', 75, result.error);
+                    }
+                    continue;
+                }
+                // Do not publish a new boot configuration until every candidate is installed and migrated.
+                const migrationFailed = await this.migrateNodeApps(target, project, landed, migrations.filter(request => landed.some(entry => entry.app === request.app)), artifacts, Object.fromEntries(options.apps.map(app => [app.name, { ...app.env, ...options.appEnv?.[app.name] }])));
+                if (migrationFailed.size) {
+                    for (const result of results.filter(r => r.node === nodeKey && r.status === 'success')) {
+                        result.status = 'failed';
+                        result.error = `database migrations failed; existing code retained: ${migrationFailed.get(result.app) ?? 'another application failed'}`;
+                        this.emitProgress(nodeKey, result.app, 'failed', 80, result.error);
+                    }
+                    continue;
+                }
+                let registration;
+                try {
+                    registration = await this.registerNodeApps(target, project, options.apps, landed, options.appEnv, options.alertSink, options.stack, owners.get(target));
+                }
+                catch (error) {
+                    for (const result of results.filter(r => r.node === nodeKey && r.status === 'success')) {
+                        result.status = 'failed';
+                        result.error = `Node registration failed: ${error.message}`;
+                        this.emitProgress(nodeKey, result.app, 'failed', 85, result.error);
+                    }
+                    continue;
+                }
                 // And the path from the gateway to them, which is the one crossing
                 // no configuration can open by itself. See `gateway-reachability`.
                 if (options.stack)
@@ -637,12 +644,24 @@ export class RemoteDeployer {
                         this.logger.info({ node: result.node, app: entry.app, version: entry.version, because: decision.because }, 'Left running — this deployment changes nothing for this app');
                         continue;
                     }
+                    const expectedOwner = owners.get(target);
+                    if (expectedOwner) {
+                        try {
+                            assertSameApplicationRuntime(expectedOwner, (await this.inspectInstalledBundle(target)).applicationRuntime);
+                        }
+                        catch (error) {
+                            result.status = 'failed';
+                            result.error = error.message;
+                            this.emitProgress(result.node, entry.app, 'failed', 90, result.error);
+                            continue;
+                        }
+                    }
                     const started = await this.signalRemoteDaemon(target, entry.app);
                     const health = started.ok
                         ? await this.verifyHealth(target, entry.app, project)
                         : { online: false, detail: started.detail };
                     if (health.online) {
-                        await this.recordRunningArtifact(target, artifactDirOnNode(project, entry.app, entry.version), entry.app, result.checksum);
+                        await this.recordRunningArtifact(target, artifactDirOnNode(project, entry.app, entry.directory ?? entry.version), entry.app, result.checksum);
                         this.emitProgress(result.node, entry.app, 'success', 100, 'Running');
                         // Said, because its opposite is. «Left running» above logs when
                         // this deployment leaves an application alone; restarting one
@@ -739,129 +758,80 @@ export class RemoteDeployer {
             this.logger.warn({ node: target.host, network, error: err.message.slice(0, 200) }, 'Could not check the gateway’s path to the applications');
         }
     }
-    /**
-     * Run each app's migrations, on the node, against the node's database.
-     *
-     * They had never run there. `ProjectService` migrates by executing
-     * `apps/<name>/src/database/migrate.ts` from the PROJECT PATH against
-     * `localhost` — both of which name the machine the master is on. A node has
-     * no project sources and a different localhost, so a remote stack's
-     * databases were created and left empty.
-     *
-     * Measured on the test node after six apps installed and five started:
-     * every one of the six databases existed and held ZERO tables. Five apps
-     * came up anyway, because they touch no table at boot; `payments` does, and
-     * said so —
-     *
-     *     Failed to call @PostConstruct 'bootstrap' on 'PlatformRevenueService':
-     *     relation "accounts" does not exist
-     *
-     * — which is the only reason anyone found out. A platform with no schema
-     * that reports five of six apps healthy is a worse state than one that
-     * refuses to start.
-     *
-     * The artifact already carries what is needed: `dist/database/migrate.js`
-     * beside `dist/database/migrations/` — 160 compiled files for payments. The
-     * same build the app itself runs from, so there is no second copy of the
-     * schema to drift.
-     *
-     * Returns the apps whose migrations failed, with the reason, and the
-     * caller does not start their new code.
-     *
-     * This used to say: «Failure is reported and does not stop the deployment:
-     * an app whose migrations fail will fail its own start with a message
-     * about the table it wanted, which is more specific than anything this
-     * step could say.» True for one of the two ways it goes. An app that
-     * fails to START is caught by `verifyHealth` and marked failed — that
-     * branch was covered, and the sentence is right about it. An app that
-     * starts fine and fails on the first REQUEST that touches the missing
-     * table passes `verifyHealth`, the stack reports `started`, and the only
-     * trace is one ERROR line in the master's log. The justification covered
-     * the branch that was already safe.
-     *
-     * Idempotence is what makes refusing cheap: the migrator records each
-     * migration it applied, in a transaction with the migration itself, so
-     * running the batch again after the cause is fixed applies exactly what
-     * is missing. That holds because `HardenedMigrationRunner` defaults
-     * `useTransactions` to true (`titan-database/src/migration/
-     * hardened-runner.ts:171`) and no sample `migrate.ts` overrides it — it is
-     * conditional in the runner (`:502`), so a migrator that turned it off
-     * would make a refused deployment leave a half-applied migration behind.
-     */
-    async migrateNodeApps(target, project, landed, appEnv) {
+    /** Explicit owners only. Preflight every program and destination before the first
+     * migration; recheck sealed origin immediately before execution. No discovery,
+     * source fallback or retry after an ambiguous database outcome. */
+    async migrateNodeApps(target, project, landed, requests, artifacts, appEnv) {
         const failed = new Map();
-        for (const entry of landed) {
-            const databaseUrl = appEnv?.[entry.app]?.['DATABASE_URL'];
-            if (!databaseUrl)
-                continue;
-            const dir = `/opt/omnitron/artifacts/${assertRemotePathSegment('project name', project)}` +
-                `/${assertRemotePathSegment('app name', entry.app)}` +
-                `/${assertRemotePathSegment('version', entry.version)}`;
-            const script = `${dir}/dist/database/migrate.js`;
-            // A node that could not be asked is not an app without a migrator.
-            // This command exits 0 either way, so `sshExec` throws only when the
-            // question never arrived — and that was answered `'no'`: migrations
-            // skipped, new code started on the old schema. The worst case of the
-            // docblock above, reached through a dropped SSH connection instead of
-            // a failed migration, and refused the same way.
-            let present;
+        const planned = [];
+        let runtime;
+        for (const request of requests) {
             try {
-                present = await this.sshExec(target, `test -f ${shellEscape(script)} && echo yes || echo no`);
+                const entry = landed.find(candidate => candidate.app === request.app);
+                if (!entry)
+                    throw Error(`Migration owner ${request.app} did not install`);
+                const directory = artifactDirOnNode(project, entry.app, entry.directory ?? entry.version);
+                const script = `${directory}/${request.release}`;
+                const artifact = artifacts.find(candidate => candidate.app === entry.app);
+                let verify;
+                let nodeOptions = '', environmentApp = request.app;
+                if (artifact?.release) {
+                    runtime ??= (await this.inspectInstalledBundle(target)).applicationRuntime;
+                    if (!runtime || runtime.executable !== entry.migrationNode)
+                        throw Error('Migration execution runtime changed');
+                    const binding = runtime, origin = artifact.release;
+                    nodeOptions = releaseOptionsForEnv(appEnv[request.app]);
+                    environmentApp = origin.runtimeOrigin?.producerApp ?? request.app;
+                    verify = async () => {
+                        if ((await this.sshExec(target, installedReleaseCheckCommand(directory, origin, binding, nodeOptions), 120_000)).trim() !== 'verified')
+                            throw Error(`Migration release origin changed for ${request.app}`);
+                    };
+                    await verify();
+                }
+                else if (request.mode === 'config')
+                    throw Error('Explicit migration config requires a verified release artifact');
+                const database = migrationDatabaseEnvironment(environmentApp, appEnv[request.app]?.['DATABASE_URL'] ?? '', request.database, request.mode);
+                // test -f alone accepts escaping parent symlinks. Every path component must
+                // identify the physical program inside the selected artifact directory.
+                const check = `const fs=require('node:fs');if(fs.realpathSync(${JSON.stringify(directory)})!==${JSON.stringify(directory)}||fs.realpathSync(${JSON.stringify(script)})!==${JSON.stringify(script)})process.exit(1)`;
+                const executable = entry.migrationNode ? `env -u NODE_OPTIONS -u NODE_PATH ${shellEscape(entry.migrationNode)}` : 'node';
+                const present = await this.sshExec(target, `test -f ${shellEscape(script)} && ${executable} -e ${shellEscape(check)} && echo yes || echo no`);
+                if (present.trim() !== 'yes')
+                    throw Error(`Declared migration program is unavailable: ${request.app}/${request.release}`);
+                const env = { ...appEnv[request.app], ...database.env, ...(artifact?.release ? { NODE_OPTIONS: nodeOptions, NODE_PATH: '' } : {}) };
+                if (request.mode === 'config') {
+                    const preflight = prepareRemoteMigration({ cwd: directory, script, mode: 'config', validate: true,
+                        ...(entry.migrationNode ? { node: entry.migrationNode } : {}) }, env);
+                    await this.sshExec(target, preflight.command, 60_000, preflight.stdin);
+                }
+                planned.push({ request, directory, script, ...(entry.migrationNode ? { node: entry.migrationNode } : {}),
+                    env, ...(verify ? { verify } : {}) });
             }
-            catch (err) {
-                const message = `the node could not be asked whether there are any to apply — ${err.message}`;
-                this.logger.error({ node: target.host, app: entry.app, error: message.slice(0, 400) }, 'Could not ask the node for the app\'s migrations — its new code will not be started');
-                failed.set(entry.app, message.slice(0, 300));
-                continue;
+            catch (error) {
+                const message = redactDiagnosticText(error.message);
+                failed.set(request.app, message);
+                this.logger.error({ node: target.host, app: request.app, database: request.database, error: message }, 'Migration preparation failed — no database migrations were executed');
             }
-            if (present.trim() !== 'yes')
-                continue;
-            this.emitProgress(`${target.host}:${target.daemonPort ?? 9700}`, entry.app, 'installing', 75, 'Applying database migrations...');
-            // Both spellings, because the migrators read the second one.
-            //
-            // `apps/payments/src/database/migrate.ts` declares
-            // `envPrefix: 'PAYMENTS__DATABASE'` and builds its connection from
-            // `PAYMENTS__DATABASE__HOST` / `__PORT` / `__USER` / `__PASSWORD` /
-            // `__DATABASE`. Given only `DATABASE_URL` it falls back to its own
-            // defaults and reports `password authentication failed for user
-            // "postgres"` — against a password that is correct and that it never
-            // read. All six apps failed that way, which is what a convention
-            // nobody wrote down costs when a second caller appears.
-            //
-            // `ProjectService` passes both, with the comment "App-specific env
-            // vars (various naming conventions)". This is the second caller.
-            const parsed = parseDatabaseUrl(databaseUrl);
-            const prefix = entry.app.toUpperCase().replace(/-/g, '_');
-            const dbEnv = parsed
-                ? {
-                    [`${prefix}__DATABASE__HOST`]: parsed.host,
-                    [`${prefix}__DATABASE__PORT`]: parsed.port,
-                    [`${prefix}__DATABASE__DATABASE`]: parsed.database,
-                    [`${prefix}__DATABASE__USER`]: parsed.user,
-                    [`${prefix}__DATABASE__PASSWORD`]: parsed.password,
-                }
-                : {};
+        }
+        if (failed.size)
+            return failed;
+        for (const item of planned) {
+            const { request } = item;
+            this.emitProgress(`${target.host}:${target.daemonPort ?? 9700}`, request.app, 'installing', 75, `Applying ${request.service}/${request.database} migrations...`);
             try {
-                // `cd` into the artifact so the script resolves its own imports, and
-                // every value in the environment rather than the command line, where
-                // it would be in every process listing on the host.
-                const assignments = Object.entries({ DATABASE_URL: databaseUrl, ...dbEnv })
-                    .map(([k, v]) => `${k}=${shellEscape(v)}`)
-                    .join(' ');
-                // What the migrations said, in order, before the verdict (see
-                // `migration-output.ts`). The masked channel on purpose: these lines
-                // are for reading, not parsing, and a migrator that ever printed a
-                // credential must not put it in the master's log.
-                const said = await this.sshExec(target, `cd ${shellEscape(dir)} && ${assignments} node ${shellEscape(script)}`, 600_000);
-                for (const line of migrationOutput(said)) {
-                    this.logger.info({ node: target.host, app: entry.app, line }, 'Migration said');
-                }
-                this.logger.info({ node: target.host, app: entry.app }, 'Database migrations applied on the node');
+                await item.verify?.();
+                const migration = prepareRemoteMigration({ cwd: item.directory, script: item.script, mode: request.mode,
+                    ...(item.node ? { node: item.node } : {}) }, item.env);
+                const said = await this.sshExec(target, migration.command, 600_000, migration.stdin);
+                for (const line of migrationOutput(said).map(redactDiagnosticText))
+                    this.logger.info({ node: target.host, app: request.app, database: request.database, line }, 'Migration said');
+                this.logger.info({ node: target.host, app: request.app, database: request.database }, 'Database migrations applied on the node');
             }
-            catch (err) {
-                const message = err.message;
-                this.logger.error({ node: target.host, app: entry.app, error: message.slice(0, 400) }, 'Database migrations failed on the node — its new code will not be started');
-                failed.set(entry.app, message.slice(0, 300));
+            catch (error) {
+                const message = redactDiagnosticText(error.message);
+                failed.set(request.app, message);
+                this.logger.error({ node: target.host, app: request.app, database: request.database, error: message }, 'Database migrations failed on the node — candidate applications will not be started');
+                break;
             }
         }
         return failed;
@@ -876,39 +846,34 @@ export class RemoteDeployer {
      * the file back: the body holds every generated credential on the node,
      * and it should cross the wire once, in the direction it has to.
      */
-    async registerNodeApps(target, project, apps, landed, appEnv, alertSink) {
+    async registerNodeApps(target, project, apps, landed, appEnv, alertSink, sourceStack, expectedOwner) {
         if (landed.length === 0) {
             this.logger.warn({ host: target.host, project }, 'No artifact reached this node — nothing to register');
             return { changed: false };
         }
+        if (!sourceStack)
+            throw new Error('Node registration requires an explicit source stack');
         const { renderNodeAppConfig } = await import('../project/node-app-config.js');
         const dir = `/opt/omnitron/projects/${assertRemotePathSegment('project name', project)}`;
         const body = renderNodeAppConfig({
             project,
             artifactRoot: '/opt/omnitron/artifacts',
+            sourceStack,
             apps,
             artifacts: landed,
             appEnv,
             alertSink,
         });
-        const bodyHash = crypto.createHash('sha256').update(body).digest('hex');
+        const operation = crypto.randomUUID();
+        let published = false, publicationAttempted = false;
         try {
-            await this.sshExec(target, `mkdir -p ${shellEscape(dir)}`);
-            const previous = parseRecordedChecksum(await this.sshExec(target, readChecksumCommand(shellEscape(dir), NODE_CONFIG_HASH_FILE)));
-            const changed = previous !== bodyHash;
-            // Through a here-document: the config carries braces, quotes and
-            // newlines, and a single-quoted argument would need every quote in it
-            // escaped by hand — which is how a generated file acquires a syntax
-            // error nobody can see in the source that generated it.
-            const configPath = `${dir}/omnitron.config.mjs`;
-            // `umask 077` around the write, not a `chmod` after it: the file exists
-            // with its contents between the two, and what it contains is the
-            // generated password of every service on this node. It was written 0644
-            // for as long as it has existed — measured on the test node — which was
-            // harmless only because it held no credentials to leak.
-            await this.sshExec(target, `umask 077 && cat > ${shellEscape(configPath)} <<'OMNITRON_EOF'\n${body}\nOMNITRON_EOF`);
-            await this.sshExec(target, `chmod 600 ${shellEscape(configPath)}`);
-            await this.sshExec(target, `printf %s ${shellEscape(bodyHash)} > ${shellEscape(`${dir}/${NODE_CONFIG_HASH_FILE}`)}`);
+            if (expectedOwner)
+                assertSameApplicationRuntime(expectedOwner, (await this.inspectInstalledBundle(target)).applicationRuntime);
+            const publication = prepareNodeConfigPublication({ directory: dir, project, body, operation, artifacts: landed });
+            publicationAttempted = true;
+            const result = await this.sshExec(target, publication.command, 60_000, publication.stdin);
+            published = true;
+            const { changed } = JSON.parse(result.trim());
             const added = await this.sshExec(target, `omnitron project add ${shellEscape(project)} ${shellEscape(dir)} 2>&1`);
             // Registering the project is not starting it. The node reads the
             // definitions and waits: measured, `omnitron status` answered
@@ -928,6 +893,8 @@ export class RemoteDeployer {
                 apps: landed.map((l) => l.app),
                 online: await this.appsOnline(target, project),
             });
+            if (expectedOwner)
+                assertSameApplicationRuntime(expectedOwner, (await this.inspectInstalledBundle(target)).applicationRuntime);
             const out = start.action === 'start'
                 ? await this.sshExec(target, `omnitron stack start ${shellEscape(project)} ${shellEscape(NODE_STACK)} 2>&1`, 300_000)
                 : '';
@@ -940,7 +907,7 @@ export class RemoteDeployer {
             const refused = /\bno .*found\b|\bfailed\b|\berror\b|not found|cannot/i.test(`${added}\n${out}`);
             if (refused) {
                 this.logger.error({ host: target.host, project, dir, detail: `${added}\n${out}`.trim().slice(0, 300) }, 'The node refused the apps it was given — its artifacts are installed and it will run none of them');
-                return { changed };
+                throw new Error('Node CLI refused project registration or stack start');
             }
             this.logger.info({
                 host: target.host,
@@ -958,9 +925,18 @@ export class RemoteDeployer {
         }
         catch (err) {
             this.logger.error({ host: target.host, project, dir, error: err.message }, 'Could not tell the node what to run — its artifacts are installed and its daemon does not know the apps');
-            // Unknown, and unknown must not read as "nothing moved": an app whose
-            // config may have changed is an app to restart.
-            return { changed: true };
+            // Restore durable boot definitions if publication or registration was interrupted.
+            // Do not claim process rollback: some applications may already have started.
+            try {
+                if (publicationAttempted)
+                    await this.sshExec(target, restoreNodeConfigCommand(dir, operation));
+                if (published)
+                    await this.sshExec(target, `omnitron project add ${shellEscape(project)} ${shellEscape(dir)} 2>&1`);
+            }
+            catch (restoreError) {
+                throw new Error(`Registration and configuration recovery failed; inspect ${dir}/.registration-${operation}`, { cause: restoreError });
+            }
+            throw err;
         }
     }
     /**
@@ -1283,6 +1259,54 @@ export class RemoteDeployer {
     async assertNoPendingHostOperations(target, prefix = '/opt/omnitron', allowance) {
         await this.sshExecData(target, hostOperationBarrierCommand(prefix, allowance));
     }
+    async readManagedHost(target, request) {
+        return JSON.parse(await this.sshExecData(target, managedHostCommand(request), 480_000));
+    }
+    assertManagedTarget(target) {
+        assertControlledDaemonPort(target.daemonPort);
+        if (!target.expectedHostFingerprint)
+            throw Error('Managed kernel rollout requires a pinned SSH host key');
+    }
+    async checkManagedTarget(target) {
+        await this.assertNoPendingHostOperations(target, '/opt/omnitron');
+        const pending = JSON.parse(await this.sshExecData(target, pendingReceiptsCommand('/opt/omnitron')));
+        if (pending.length)
+            throw Error('An unfinished host upgrade must be reconciled before kernel rollout');
+    }
+    async inspectManagedRollout(target, commit, options = {}) {
+        this.assertManagedTarget(target);
+        if (!/^[0-9a-f]{40}$/.test(commit))
+            throw Error('A full lowercase Git commit is required');
+        const inspect = () => planManagedRollout(request => this.readManagedHost(target, request), commit);
+        if (!options.stage) {
+            await this.checkManagedTarget(target);
+            return inspect();
+        }
+        return this.underLease(target, 'stage managed kernel', async () => {
+            await this.checkManagedTarget(target);
+            const { stageCatalogRuntime } = await import('./kernel-distribution.js');
+            const expected = await stageCatalogRuntime(this, target, commit, options.excludeOwnerId);
+            const plan = await inspect();
+            if (plan.candidate !== expected)
+                throw Error('Installed runtime differs from the qualified catalog artifact');
+            return plan;
+        });
+    }
+    async activateManagedRollout(target, plan, beforeDispatch) {
+        this.assertManagedTarget(target);
+        return this.underLease(target, 'managed kernel handoff', async () => {
+            await this.checkManagedTarget(target);
+            await activateManagedTarget(request => this.readManagedHost(target, request), plan, beforeDispatch);
+        });
+    }
+    async reconcileManagedRollout(target, plan, options) {
+        this.assertManagedTarget(target);
+        return this.underLease(target, 'reconcile managed kernel handoff', async () => {
+            await this.checkManagedTarget(target);
+            const selection = await reconcileManagedTarget(request => this.readManagedHost(target, request), plan);
+            return options?.detailed || selection === 'candidate' ? selection : 'previous';
+        });
+    }
     /** Require a settled host and the actual serving immutable daemon build. */
     async inspectInstalledBundle(target, prefix = '/opt/omnitron', allowance) {
         assertControlledDaemonPort(target.daemonPort);
@@ -1290,11 +1314,19 @@ export class RemoteDeployer {
         const pending = JSON.parse(await this.sshExecData(target, pendingReceiptsCommand(prefix)));
         if (pending.length)
             throw new Error(`Unfinished host upgrade ${pending.map(r => r.id).join(', ')}; recover it before deploying an app`);
+        const managed = await this.readManagedHost(target, { operation: 'inspect' });
+        if (managed)
+            return managed.bundle;
         const bundle = JSON.parse(await this.sshExecData(target, bundleProbeCommand(prefix)));
         const status = JSON.parse(await this.sshExecData(target, inventoryCommand(bundle, false)));
         assertServingBundle(bundle, status);
-        await this.sshExecData(target, runtimeProbeCommand(bundle, status.pid));
-        return bundle;
+        const host = JSON.parse(await this.sshExecData(target, runtimeProbeCommand(bundle, status.pid)));
+        const execution = host.applicationRuntime ?? (host.kind !== 'managed' && host.executable ? {
+            root: bundle.root, executable: host.executable, commit: bundle.commit,
+        } : undefined);
+        if (execution && (!path.isAbsolute(execution.root) || !path.isAbsolute(execution.executable) || !/^[a-f0-9]{40}$/.test(execution.commit)))
+            throw new Error('Invalid execution owner identity');
+        return { ...bundle, ...(execution ? { applicationRuntime: execution } : {}) };
     }
     async activateBundle(target, version, prefix = '/opt/omnitron', keepVersions = 3, verifyPlatform, allowance) {
         assertControlledDaemonPort(target.daemonPort);
@@ -1315,6 +1347,8 @@ export class RemoteDeployer {
                     return false;
                 }
                 const previous = await this.inspectInstalledBundle(target, prefix, allowance);
+                if (previous.managed)
+                    throw Error('Independent owner detected. Use a sealed managed kernel rollout for this node');
                 const candidate = JSON.parse(await this.sshExecData(target, bundleProbeCommand(prefix, version)));
                 await this.sshExec(target, `${shellEscape(`${candidate.root}/dist/cli/omnitron.js`)} --version`, 120_000);
                 const inventory = JSON.parse(await this.sshExecData(target, inventoryCommand(previous, true)));
@@ -1326,6 +1360,8 @@ export class RemoteDeployer {
                     if (plan.online.includes(app.name))
                         await this.sshExec(target, appHealthCommand(app, previous), 330_000);
                 }
+                this.emitProgress(nodeKey, '*', 'verifying', 0, 'Checking candidate bootstrap readers before stopping applications');
+                await this.sshExecData(target, bootstrapPreflightCommand(plan), HOST_BOOTSTRAP_PREFLIGHT_TIMEOUT_MS + 10_000);
                 const id = `${new Date().toISOString().replace(/[:.]/g, '-')}-${crypto.randomUUID()}`;
                 const receiptPath = `${prefix}/upgrade-receipts/${id}.json`;
                 const receipt = await executeHostUpgrade(plan, id, this.hostUpgradeActions(target, prefix, keepVersions, plan, verifyPlatform));
@@ -1472,10 +1508,10 @@ export class RemoteDeployer {
         const lease = this.deploymentLeases.getStore()?.get(`${target.host}:${target.sshPort ?? 22}`);
         return lease ? { command: lease.guard(command, timeout), timeout: timeout + 45_000 } : { command, timeout };
     }
-    async sshExec(target, command, timeout = 60_000) {
+    async sshExec(target, command, timeout = 60_000, stdin) {
         await this.confirmLease(target, 'remote command');
         const guarded = this.guardedCommand(target, command, timeout);
-        const result = await this.execution.ssh(sshTargetOf(target), guarded.command, { timeout: guarded.timeout });
+        const result = await this.execution.ssh(sshTargetOf(target), guarded.command, { timeout: guarded.timeout, ...(stdin !== undefined && { stdin }) });
         if (result.exitCode !== 0) {
             const detail = result.stderr || result.stdout || `exit ${result.exitCode}`;
             throw new Error(`ssh ${target.username ?? 'root'}@${target.host}: ${detail}`);
@@ -1486,7 +1522,8 @@ export class RemoteDeployer {
     async sshExecData(target, command, timeout = 60_000) {
         const result = await this.readFromNode(target, command, timeout);
         if (result.code !== 0) {
-            throw new Error(`ssh ${target.username ?? 'root'}@${target.host}: ${result.stderr || result.stdout || `exit ${result.code}`}`);
+            const reason = redactDiagnosticText(commandFailure(result.stderr, `exit ${result.code}`));
+            throw new Error(`ssh ${target.username ?? 'root'}@${target.host}: ${reason}`);
         }
         return result.stdout.trim();
     }
@@ -1596,7 +1633,6 @@ export class RemoteDeployer {
         const { createHash } = await import('node:crypto');
         const fsp = await import('node:fs/promises');
         const os = await import('node:os');
-        const path = await import('node:path');
         const stat = await fsp.stat(localDir).catch(() => null);
         if (!stat?.isDirectory())
             throw new Error(`No such directory to serve: ${localDir}`);
@@ -1811,6 +1847,7 @@ export class RemoteDeployer {
         catch (err) {
             this.logger.warn({ host: target.host, fresh, error: err.message.slice(0, 400) }, 'Could not carry the previous build forward — open tabs will reload onto the new one');
         }
+        return undefined;
     }
     async signalRemoteDaemon(target, appName) {
         try {
@@ -1924,8 +1961,8 @@ export class RemoteDeployer {
     // Private — Progress
     // ===========================================================================
     emitProgress(node, app, status, progress, message) {
-        const event = { node, app, status, progress, message };
-        this.lastProgress.set(`${node}\u0000${app}`, { ...event, at: new Date().toISOString() });
+        const event = { ...transferScope.getStore(), node, app, status, progress, message: redactDiagnosticText(message), at: new Date().toISOString() };
+        this.lastProgress.set(deploymentProgressKey(event), event);
         for (const handler of this.progressHandlers) {
             try {
                 handler(event);