@omnitron-dev/omnitron 0.2.0 → 0.3.0

package/dist/services/remote-deployer.service.js

@@ -13,23 +13,244 @@
  *
  * Artifact path on remote: /opt/omnitron/artifacts/<project>/<app>/<version>/
  * Remote daemon config: /etc/omnitron/omnitron.config.ts
+ *
+ * NEVER EXECUTED, as of 2026-09-14. Measured rather than assumed: no log in
+ * `~/.omnitron/logs` carries a single line this class emits ("Starting
+ * deployment to node", "Slave provisioned", "Deployment successful"), no
+ * artifact directory exists, and — decisively — no project config declares
+ * `stacks.nodes`, which `startRemoteStack` requires before it can reach here.
+ * So both entry points are unreachable in every current configuration.
+ *
+ * This matters to anyone reading the code below and taking it for a working
+ * path. It is careful code — arguments are quoted, path segments validated,
+ * heredocs replaced with base64 — and none of that has ever met a real host.
+ * Its first run will be its first test. Treat a green read of this file as
+ * evidence about intent, not about behaviour.
+ *
+ * The first thing that first run found, 2026-09-14: **it could not log in.**
+ * Every command here went through `ssh -o BatchMode=yes` with an optional
+ * `-i <keyfile>`, and `BatchMode=yes` disables every interactive method —
+ * password and key passphrase both. The console's Add Node dialog collects
+ * exactly those, encrypts them in the daemon's vault, and the health monitor
+ * uses them through `ExecutionService.ssh()` on every check round. So the
+ * product had two SSH implementations: one that can present what the operator
+ * gave it, and one — this — that cannot, on the path where it matters most.
+ *
+ * Measured against the test host, whose SSH answers a password in 311 ms:
+ *
+ *   ssh -o BatchMode=yes root@<host> 'echo ok'
+ *   → root@<host>: Permission denied (publickey,password).
+ *
+ * That is the answer to "why can a machine added in the console not be
+ * deployed to": not policy, not a missing feature — the deployer could not
+ * authenticate as the operator had arranged. It goes through the same
+ * `ExecutionService` as the checks now, and takes a `DeployTarget` carrying
+ * the credentials rather than an `IStackNode` whose `ISSHConfig` has nowhere
+ * to put them.
  */
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
-const exec = promisify(execFile);
-/** Escape a string for safe use inside a single-quoted shell argument. */
-function shellEscape(s) {
-    return "'" + s.replace(/'/g, "'\\''") + "'";
+import { PLATFORM_PROBE, parsePlatformProbe, planProvisioning, describePlan, } from './remote-provisioner.js';
+import { installSteps, activateSteps, pruneSteps } from './bundle-builder.js';
+import { reachabilityRule, reachabilityCommand } from '../infrastructure/gateway-reachability.js';
+import { readNodeHealth, readNodeStatus } from '../project/node-app-health.js';
+import { decideRedeploy, decideNodeStackStart, decideSlaveDaemonRestart, artifactChanged, readChecksumCommand, parseRecordedChecksum, ARTIFACT_CHECKSUM_FILE, NODE_CONFIG_HASH_FILE, } from './redeploy-decision.js';
+import { deliveredProbeCommand, parseDeliveredProbe, checkDelivered, } from '../release/delivered.js';
+import crypto from 'node:crypto';
+import { gunzipSync } from 'node:zlib';
+import { shellEscape } from '../shared/shell-escape.js';
+import { fromDataChannel, throughDataChannel } from '../execution/data-channel.js';
+/** A node that could not be asked at all — the SSH call itself failed. */
+const UNREADABLE = -3;
+import { withNodeLeases } from './node-deploy-lease.js';
+import { migrationOutput } from './migration-output.js';
+/**
+ * A name that is safe to place in a remote filesystem path.
+ *
+ * Project, app and version names are interpolated into
+ * `/opt/omnitron/artifacts/<project>/<app>/<version>` and the result is sent
+ * to a remote shell running as the SSH user, which defaults to root. Quoting
+ * makes that safe against injection but not against the path itself: a
+ * project literally named `../..` resolves outside the artifact root, and
+ * `mkdir -p` then creates it there. Quoting is not a substitute for the
+ * value meaning what the path assumes it means.
+ *
+ * These names come from the project registry and the console's own forms —
+ * operator-supplied, not attacker-supplied in the ordinary case. It is
+ * defence in depth, and the cost of it is a regular expression.
+ *
+ * @throws Error naming the segment and what is allowed.
+ */
+export function assertRemotePathSegment(kind, value) {
+    if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) || value.includes('..')) {
+        throw new Error(`Refusing to deploy: ${kind} ${JSON.stringify(value)} is not usable in a remote path. ` +
+            `Allowed: letters, digits, dot, dash and underscore, starting with a letter or digit, and no "..".`);
+    }
+    return value;
+}
+/**
+ * Where an app's artifact lives on a node — one place, because the install
+ * writes there and the step that starts the app records there.
+ */
+export function artifactDirOnNode(project, app, version) {
+    return (`/opt/omnitron/artifacts/${assertRemotePathSegment('project name', project)}` +
+        `/${assertRemotePathSegment('app name', app)}` +
+        `/${assertRemotePathSegment('version', version)}`);
+}
+/** The credentials half, in the shape `ExecutionService` takes. */
+function sshTargetOf(target) {
+    const ssh = { host: target.host };
+    if (target.sshPort != null)
+        ssh.port = target.sshPort;
+    if (target.username)
+        ssh.username = target.username;
+    if (target.privateKey)
+        ssh.privateKey = target.privateKey;
+    if (target.passphrase)
+        ssh.passphrase = target.passphrase;
+    if (target.password)
+        ssh.password = target.password;
+    return ssh;
+}
+/** Project a node declared in a project config into a deploy target. */
+export function stackNodeToDeployTarget(node) {
+    const target = { host: node.host };
+    if (node.port != null)
+        target.daemonPort = node.port;
+    if (node.ssh?.port != null)
+        target.sshPort = node.ssh.port;
+    if (node.ssh?.user)
+        target.username = node.ssh.user;
+    // A path, read by the engine. `ISSHConfig` has no passphrase field, so a
+    // key declared here must be one that needs none — which is worth knowing
+    // when a deployment from a config fails and the same node works from the
+    // console.
+    if (node.ssh?.privateKey)
+        target.privateKey = node.ssh.privateKey;
+    if (node.apps)
+        target.apps = node.apps;
+    if (node.label)
+        target.label = node.label;
+    return target;
 }
+/**
+ * Fill a stack node's connection details from the node registry.
+ *
+ * A stack says WHICH host an app runs on; the node registry says HOW to
+ * reach it — the user, the port, and the key or password, which are held
+ * encrypted in the daemon's vault because they are credentials and a config
+ * file in a repository is not where those go.
+ *
+ * Nothing joined the two. `stackNodeToDeployTarget` read only the stack's
+ * own `ssh` block, so deploying to a machine that the console had already
+ * registered, provisioned and been connected to failed at the first
+ * connection:
+ *
+ *     Invalid SSH options: Either privateKey or password must be provided
+ *
+ * — for a node the same daemon was, at that moment, holding an SSH tunnel to.
+ *
+ * The stack's explicit values win where it has them: an operator who wrote
+ * `ssh.user` in the stack config meant it, and a registry entry is the
+ * default, not an override. Everything the stack leaves out comes from the
+ * registry.
+ */
+export function withNodeCredentials(target, registered) {
+    if (!registered)
+        return target;
+    const merged = { ...target };
+    if (merged.sshPort == null && registered.port != null)
+        merged.sshPort = registered.port;
+    if (!merged.username && registered.username)
+        merged.username = registered.username;
+    if (!merged.privateKey && registered.privateKey)
+        merged.privateKey = registered.privateKey;
+    if (!merged.passphrase && registered.passphrase)
+        merged.passphrase = registered.passphrase;
+    if (!merged.password && registered.password)
+        merged.password = registered.password;
+    return merged;
+}
+/**
+ * How long a freshly provisioned slave has to answer.
+ *
+ * A daemon start is an application boot: a DI container, a module graph, a
+ * SQLite open. On a loaded host this is tens of seconds, and the old
+ * three-second sleep was shorter than the work by an order of magnitude.
+ */
+const SLAVE_START_TIMEOUT_MS = 120_000;
+/** How often it is asked, while it is starting. */
+const DAEMON_POLL_MS = 3_000;
 // =============================================================================
 // RemoteDeployer
 // =============================================================================
+/**
+ * The parts of a postgres URL, for the environment a migrator expects.
+ *
+ * Returns null rather than guessing at a URL it cannot read: a migration run
+ * against half-parsed connection details is worse than one that does not run,
+ * because it can succeed somewhere nobody meant.
+ */
+function parseDatabaseUrl(url) {
+    try {
+        const u = new URL(url);
+        if (!u.protocol.startsWith('postgres'))
+            return null;
+        return {
+            host: u.hostname,
+            port: u.port || '5432',
+            database: decodeURIComponent(u.pathname.replace(/^\//, '')),
+            user: decodeURIComponent(u.username),
+            password: decodeURIComponent(u.password),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Is this a gzip stream of a COMPLETE tar archive?
+ *
+ * Two ways to be short, told apart: a gzip stream cut off fails to
+ * decompress («unexpected end of file»), and a tar cut off inside a gzip
+ * that finished — `tar` killed, `gzip` still closing its own stream —
+ * decompresses to something that is not a whole number of 512-byte blocks,
+ * or does not end in the two zero blocks every tar writer closes an archive
+ * with. Answers `true` or the reason it is not.
+ */
+export function wholeTarGz(archive) {
+    let tar;
+    try {
+        tar = gunzipSync(archive);
+    }
+    catch (err) {
+        return `the gzip stream is broken: ${err.message}`;
+    }
+    if (tar.length < 1024 || tar.length % 512 !== 0)
+        return `the tar inside is ${tar.length} bytes, not whole blocks`;
+    return tar.subarray(tar.length - 1024).every((b) => b === 0) ? true : 'the tar inside has no end-of-archive marker';
+}
 export class RemoteDeployer {
     logger;
+    execution;
     /** Active deployment progress handlers */
     progressHandlers = [];
-    constructor(logger) {
+    /**
+     * The last thing that happened to each app on each node.
+     *
+     * A deployment publishes progress as it goes and the console polls; without
+     * somewhere for an event to WAIT, those two never meet, and a deployment
+     * that takes a quarter of an hour showed the operator one row that said
+     * `deploying` from beginning to end. Everything that said WHERE it was —
+     * transferring, installing, the message naming the failing step — reached a
+     * handler that re-emitted it to nobody.
+     *
+     * Keyed by node and app, so this is bounded by the fleet rather than by how
+     * often anyone deploys, and the entry for an app is always its current
+     * state rather than a scroll an operator has to read to the end of.
+     */
+    lastProgress = new Map();
+    constructor(logger, execution) {
         this.logger = logger;
+        this.execution = execution;
     }
     /**
      * Register a progress handler for real-time deployment updates.
@@ -45,34 +266,266 @@ export class RemoteDeployer {
     /**
      * Deploy an artifact to a single remote node.
      */
-    async deployToNode(node, artifact, project) {
+    async deployToNode(target, artifact, project, options) {
         const startTime = Date.now();
-        const nodeKey = `${node.host}:${node.port ?? 9700}`;
+        const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
         this.logger.info({ node: nodeKey, app: artifact.app, version: artifact.version }, 'Starting deployment to node');
         try {
             // 1. Verify SSH connectivity
             this.emitProgress(nodeKey, artifact.app, 'pending', 0, 'Connecting via SSH...');
-            await this.verifySSH(node);
+            await this.verifySSH(target);
             // 2. Ensure remote directory structure
-            const remotePath = `/opt/omnitron/artifacts/${project}/${artifact.app}/${artifact.version}`;
-            await this.sshExec(node, `mkdir -p ${shellEscape(remotePath)}`);
+            const remotePath = artifactDirOnNode(project, artifact.app, artifact.version);
+            await this.sshExec(target, `mkdir -p ${shellEscape(remotePath)}`);
+            // 2a. What the node already has.
+            //
+            // A master's boot-time autostart deploys every remote stack again, so
+            // the same artifact was rebuilt, transferred, unpacked, `npm
+            // install`ed and its app restarted on every master restart — four
+            // times in one afternoon here, twenty-four application restarts on a
+            // node whose files were identical each time.
+            //
+            // The comparison is the artifact's own sha256 against the one the node
+            // recorded when the app last came up healthy on it. `options.force`
+            // deploys anyway, for an operator who has reason to doubt the record —
+            // `omnitron stack start --reinstall`, which until 2026-09-24 nothing
+            // could pass: the escape hatch this comment named had no door.
+            if (options?.force !== true) {
+                const recorded = parseRecordedChecksum(await this.sshExec(target, readChecksumCommand(shellEscape(remotePath))));
+                if (artifactChanged(recorded, artifact.checksum) === false) {
+                    const duration = Date.now() - startTime;
+                    this.emitProgress(nodeKey, artifact.app, 'success', 75, 'Already installed — unchanged');
+                    this.logger.info({ node: nodeKey, app: artifact.app, version: artifact.version, sha: recorded.slice(0, 12) }, 'The node already has this artifact — not transferring it again');
+                    return {
+                        node: nodeKey,
+                        app: artifact.app,
+                        version: artifact.version,
+                        status: 'success',
+                        duration,
+                        unchanged: true,
+                    };
+                }
+            }
             // 3. Transfer artifact
             this.emitProgress(nodeKey, artifact.app, 'transferring', 20, 'Transferring artifact...');
             const remoteFile = `${remotePath}/${artifact.app}-${artifact.version}.tar.gz`;
-            await this.scpTransfer(node, artifact.path, remoteFile);
-            // 4. Extract on remote
+            await this.scpTransfer(target, artifact.path, remoteFile);
+            // 3b. And ask the node whether what it has is what was sent.
+            //
+            // This used to go straight to `tar -xzf`. The `.artifact-sha256`
+            // written further down reads like this check and is not one: it holds
+            // `artifact.checksum`, the hash of the build INPUTS, it is written
+            // AFTER the unpack, and it is compared against the next build rather
+            // than against the file. So a truncated transfer reached tar, which
+            // failed with whatever tar says about a corrupt archive, and one that
+            // happened to end on a record boundary unpacked a subset and reported
+            // success.
+            //
+            // Skipped only when the artifact carries no tarball hash — an
+            // artifact found on disk by `listArtifacts` rather than built here.
+            // That is an absence of information, not a verdict, and it is said out
+            // loud rather than passed over in silence.
+            if (artifact.tarballSha256) {
+                const probe = await this.sshExec(target, deliveredProbeCommand(remoteFile));
+                const verdict = checkDelivered({ sha256: artifact.tarballSha256, bytes: artifact.size }, parseDeliveredProbe(probe));
+                if (!verdict.ok) {
+                    const duration = Date.now() - startTime;
+                    this.emitProgress(nodeKey, artifact.app, 'failed', 30, verdict.because);
+                    this.logger.error({ node: nodeKey, app: artifact.app, file: remoteFile }, `Artifact transfer failed verification — ${verdict.because}`);
+                    return {
+                        node: nodeKey,
+                        app: artifact.app,
+                        version: artifact.version,
+                        status: 'failed',
+                        duration,
+                        error: verdict.because,
+                    };
+                }
+            }
+            else {
+                this.logger.warn({ node: nodeKey, app: artifact.app }, 'Artifact carries no tarball hash — the transfer was not verified');
+            }
+            // 4. Extract on remote, into an empty directory.
+            //
+            // `tar -xzf` over whatever was there before leaves everything the new
+            // archive does not name — and what was there before is a `node_modules`
+            // of a different SHAPE. The first deployment after this builder changed
+            // put an npm manifest beside a pnpm tree and `npm install` answered
+            //
+            //     npm error Cannot read properties of null (reading 'edgesOut')
+            //
+            // which is npm walking a dependency graph through symlinks it did not
+            // create. The same reasoning as `installSteps`, which removes the
+            // version directory before unpacking for exactly this: an install
+            // reconciling against a tree from another tool is not this version.
+            //
+            // Everything except the archive just uploaded, and only inside the
+            // artifact's own version directory — every segment of which was checked
+            // by `assertRemotePathSegment` above.
             this.emitProgress(nodeKey, artifact.app, 'extracting', 50, 'Extracting artifact...');
-            await this.sshExec(node, `cd ${shellEscape(remotePath)} && tar -xzf ${shellEscape(`${artifact.app}-${artifact.version}.tar.gz`)}`);
-            // 5. Install production dependencies
-            this.emitProgress(nodeKey, artifact.app, 'extracting', 65, 'Installing dependencies...');
-            await this.sshExec(node, `cd ${shellEscape(remotePath)} && npm install --production --ignore-scripts 2>/dev/null || true`);
-            // 6. Signal remote daemon to restart the app
+            const archiveName = `${artifact.app}-${artifact.version}.tar.gz`;
+            await this.sshExec(target, `cd ${shellEscape(remotePath)} && ` +
+                `find . -mindepth 1 -maxdepth 1 ! -name ${shellEscape(archiveName)} -exec rm -rf {} + && ` +
+                `tar -xzf ${shellEscape(archiveName)}`);
+            // 5. Install the dependencies ON THE NODE.
+            //
+            // Three shapes of this step, and the two that were wrong are worth
+            // keeping written down because each looked finished.
+            //
+            // It first ran `npm install --production --ignore-scripts 2>/dev/null
+            // || true`. That could not work even once — the manifests use pnpm's
+            // `workspace:` protocol, npm answers `EUNSUPPORTEDPROTOCOL` — and the
+            // `|| true` meant nobody found out.
+            //
+            // It then trusted the artifact to arrive complete, from `pnpm deploy`,
+            // and checked `test -d node_modules`. The artifact was NOT complete:
+            // every `@omnitron-dev/*` package was a symlink into a home directory
+            // on the master, twenty-three of them, and a directory of dangling
+            // symlinks passes `test -d`. A symlink to `/opt/omnitron/current`
+            // patched the one the first import happened to name; the other
+            // twenty-two were waiting behind it.
+            //
+            // Now the artifact carries `vendor/*.tgz` and a manifest that installs
+            // them, exactly like the daemon's own bundle, and the install runs
+            // here. That is not a cost to be minimised — it is the only step that
+            // happens on a machine that knows it is Linux, which is what decides
+            // between `@esbuild/darwin-arm64` and `@esbuild/linux-x64`.
+            this.emitProgress(nodeKey, artifact.app, 'installing', 60, 'Installing dependencies on the node...');
+            try {
+                await this.sshExec(target, `cd ${shellEscape(remotePath)} && npm install --omit=dev --no-audit --no-fund`, 900_000);
+            }
+            catch (err) {
+                const duration = Date.now() - startTime;
+                const detail = `dependency install failed on the node: ${err.message.slice(0, 400)}`;
+                this.emitProgress(nodeKey, artifact.app, 'failed', 60, detail);
+                this.logger.error({ node: nodeKey, app: artifact.app, path: remotePath }, detail);
+                return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'failed', duration, error: detail };
+            }
+            // 5a. One physical copy of every package the daemon shares with the app.
+            //
+            // A supervised app does not run alone in its process: omnitron's own
+            // `bootstrap-process.js` is the entry point, and the app's modules are
+            // loaded into it. Titan keeps identities at module scope — the
+            // `Container` class among them — so two physical installations of
+            // `@omnitron-dev/titan` in one process means `@Inject(Container)`
+            // resolves against a class that is not the one the daemon registered,
+            // and the failure reads `Dependency 'Container' not found`.
+            //
+            // `bootstrap-process.ts` checks for exactly this and refuses to start,
+            // which is how a freshly installed artifact answered:
+            //
+            //     Titan package identity mismatch detected between omnitron daemon
+            //     and app 'geo'. Daemon resolves @omnitron-dev/titan from
+            //     /opt/omnitron/versions/…; App resolves … from /opt/omnitron/
+            //     artifacts/daos/geo/0.0.1/node_modules/@omnitron-dev/titan
+            //
+            // That is the correct refusal, and the answer is not to install less:
+            // the vendored copies are what let `npm install` resolve a coherent
+            // tree in the first place. They are replaced afterwards by links to
+            // the daemon's own, computed ON THE NODE from what the daemon actually
+            // has, so the set stays right as the daemon's closure changes rather
+            // than being a list here that drifts from it.
+            //
+            // `@omnitron-dev/omnitron` is deliberately NOT one of them, and this is
+            // the second time that lesson has been paid for. The daemon IS that
+            // package, so it is not inside its own `node_modules`, and linking to
+            // the install root instead gives node a directory whose package.json
+            // carries `name`, `version`, `type` and `dependencies` and NO `main`
+            // and NO `exports` — `bundleRootManifest` writes an install manifest,
+            // not a library manifest. Resolution then falls back to `index.js`,
+            // which is not there:
+            //
+            //     Cannot find package
+            //     '.../node_modules/@omnitron-dev/omnitron/index.js'
+            //
+            // — the exact error this whole line of work started from. The vendored
+            // copy that `pnpm pack` produces from `apps/omnitron` has the real
+            // manifest, so it is left alone. Its identity is not shared across the
+            // process boundary the way titan's is: `defineSystem` returns data that
+            // the daemon's bootstrap reads, not a class anything does `instanceof`
+            // against.
+            //
+            // `rm -rf` before each `ln -sfn`: linking onto an existing DIRECTORY
+            // puts the link INSIDE it, leaving a directory with no package.json
+            // that node reports as a missing `index.js` — a message about a file
+            // that was never the problem.
+            const daemonRoot = '/opt/omnitron/current';
+            await this.sshExec(target, `set -e; APP=${shellEscape(`${remotePath}/node_modules/@omnitron-dev`)}; ` +
+                `mkdir -p "$APP"; ` +
+                `for p in ${shellEscape(daemonRoot)}/node_modules/@omnitron-dev/*; do ` +
+                `[ -e "$p" ] || continue; n=$(basename "$p"); ` +
+                `rm -rf "$APP/$n"; ln -sfn "$p" "$APP/$n"; done`);
+            // The install either produced a tree that resolves or it did not, and
+            // `test -d node_modules` cannot tell those apart — it was true for the
+            // whole time nothing worked. Asking node to RESOLVE the package the
+            // app imports first is the smallest question with a real answer.
+            //
+            // The package, not a file inside it: my first version of this resolved
+            // `@omnitron-dev/omnitron/package.json` and a correct install answered
+            // `ERR_PACKAGE_PATH_NOT_EXPORTED`, because a package with an `exports`
+            // map publishes what it lists and nothing else. A probe that fails on
+            // a healthy tree is worse than none — it would have failed every
+            // deployment from here on, for the one reason that is not a fault.
+            this.emitProgress(nodeKey, artifact.app, 'extracting', 70, 'Checking dependencies...');
+            const resolved = await this.sshExec(target, `cd ${shellEscape(remotePath)} && node -e ${shellEscape("const r=require('module').createRequire(process.cwd()+'/package.json');" +
+                "r.resolve('@omnitron-dev/omnitron');r.resolve('@omnitron-dev/titan')")} >/dev/null 2>&1 && echo resolves || echo broken`).catch(() => 'broken');
+            if (resolved.trim() !== 'resolves') {
+                const duration = Date.now() - startTime;
+                const detail = 'the installed tree cannot resolve @omnitron-dev/omnitron or @omnitron-dev/titan, ' +
+                    'so the app will fail at its first import';
+                this.emitProgress(nodeKey, artifact.app, 'failed', 70, detail);
+                this.logger.error({ node: nodeKey, app: artifact.app, path: remotePath }, detail);
+                return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'failed', duration, error: detail };
+            }
+            // 5b. No record yet. The next deployment reads `.artifact-sha256` as
+            // «the app runs this», and at this point it does not: the files are
+            // unpacked, the process still runs what it started with. The record
+            // used to be written here, and on daos/test (2026-09-24) a deployment
+            // that stalled after installing three artifacts left them recorded;
+            // the retry read «installed, running» and left all three on the code
+            // they had started with an hour earlier. It is written once the app
+            // has come up healthy on these files — `recordRunningArtifact`.
+            // 6. Installed. Starting is a SEPARATE phase, and the order matters:
+            // a node cannot start an app it has no definition for, and the
+            // definitions are written once all the artifacts have landed. Starting
+            // here made the two mutually exclusive — the restart failed with
+            // `Unknown app`, the result was marked failed, and the registration,
+            // which only writes apps whose deployment succeeded, wrote nothing.
+            if (options?.startAfterInstall === false) {
+                const duration = Date.now() - startTime;
+                this.emitProgress(nodeKey, artifact.app, 'success', 75, 'Installed');
+                return {
+                    node: nodeKey,
+                    app: artifact.app,
+                    version: artifact.version,
+                    status: 'success',
+                    duration,
+                    ...(artifact.checksum ? { checksum: artifact.checksum } : {}),
+                };
+            }
             this.emitProgress(nodeKey, artifact.app, 'restarting', 80, 'Restarting app on remote...');
-            await this.signalRemoteDaemon(node, artifact.app);
+            const started = await this.signalRemoteDaemon(target, artifact.app);
             // 7. Verify health
             this.emitProgress(nodeKey, artifact.app, 'verifying', 90, 'Verifying health...');
-            await this.verifyHealth(node, artifact.app);
+            const health = await this.verifyHealth(target, artifact.app, project);
             const duration = Date.now() - startTime;
+            // An artifact on disk is not a running application. Reporting success
+            // for the transfer alone is what let a fleet show six deployed apps and
+            // run none of them.
+            if (!started.ok || !health.online) {
+                const why = !started.ok ? started.detail : health.detail;
+                this.emitProgress(nodeKey, artifact.app, 'failed', 90, `artifact installed, app not running: ${why}`);
+                this.logger.error({ node: nodeKey, app: artifact.app, version: artifact.version, duration, detail: why }, 'Artifact installed, but the app is not running on the node');
+                return {
+                    node: nodeKey,
+                    app: artifact.app,
+                    version: artifact.version,
+                    status: 'failed',
+                    duration,
+                    error: `artifact installed, app not running: ${why}`,
+                };
+            }
+            await this.recordRunningArtifact(target, remotePath, artifact.app, artifact.checksum);
             this.emitProgress(nodeKey, artifact.app, 'success', 100, `Deployed in ${Math.round(duration / 1000)}s`);
             this.logger.info({ node: nodeKey, app: artifact.app, version: artifact.version, duration }, 'Deployment successful');
             return { node: nodeKey, app: artifact.app, version: artifact.version, status: 'success', duration };
@@ -88,24 +541,29 @@ export class RemoteDeployer {
     /**
      * Deploy artifacts to all nodes in a stack (parallel).
      */
-    async deployToStack(nodes, artifacts, project, options) {
+    async deployToStack(targets, artifacts, project, options) {
         const concurrency = options?.concurrency ?? 3;
         const results = [];
         // Build deployment matrix: each app to each node (or node-specific apps)
         const tasks = [];
-        for (const node of nodes) {
+        for (const target of targets) {
             for (const artifact of artifacts) {
                 // If node has explicit app list, only deploy matching apps
-                if (node.apps && !node.apps.includes(artifact.app))
+                if (target.apps && !target.apps.includes(artifact.app))
                     continue;
-                tasks.push({ node, artifact });
+                tasks.push({ target, artifact });
             }
         }
         // Execute with concurrency limit
         const executing = new Set();
         for (const task of tasks) {
             const promise = (async () => {
-                const result = await this.deployToNode(task.node, task.artifact, project);
+                // Install only when the definitions are coming: the app cannot be
+                // started before the node knows it exists.
+                const result = await this.deployToNode(task.target, task.artifact, project, {
+                    startAfterInstall: !options?.apps?.length,
+                    ...(options?.force === true ? { force: true } : {}),
+                });
                 results.push(result);
             })();
             executing.add(promise);
@@ -115,8 +573,392 @@ export class RemoteDeployer {
             }
         }
         await Promise.all(executing);
+        // Tell each node what it now has. After the transfers, because a config
+        // naming an artifact that has not landed yet is a config the node will
+        // fail on — and only for the apps that actually arrived, which is what
+        // `results` knows and the artifact list does not.
+        if (options?.apps?.length) {
+            for (const target of targets) {
+                // The SAME key `deployToNode` builds — `host:daemonPort`. Comparing
+                // against the SSH port here matched nothing, so the registration and
+                // the start phase would both have run over an empty list and reported
+                // nothing wrong.
+                const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
+                const landed = results
+                    .filter((r) => r.node === nodeKey && r.status === 'success')
+                    .map((r) => ({ app: r.app, version: r.version }));
+                const registration = await this.registerNodeApps(target, project, options.apps, landed, options.appEnv, options.alertSink);
+                // Schema before the apps that read it — and an app whose schema did
+                // not arrive does not get its new code started on the old one.
+                const migrationFailed = await this.migrateNodeApps(target, project, landed, options.appEnv);
+                // And the path from the gateway to them, which is the one crossing
+                // no configuration can open by itself. See `gateway-reachability`.
+                if (options.stack)
+                    await this.openGatewayPath(target, project, options.stack);
+                // What the node is running right now, asked once for the whole node
+                // rather than once per app: the decision below needs it for every
+                // app, and `omnitron status --json` answers about all of them.
+                const runningNow = await this.appsOnline(target, project);
+                // Now that the node knows what these apps are, start them. Their
+                // result is upgraded in place, so a caller reading `results` sees
+                // running apps rather than installed files.
+                for (const entry of landed) {
+                    const result = results.find((r) => r.app === entry.app && r.node === nodeKey);
+                    if (!result)
+                        continue;
+                    // Before anything decides to restart it. The code that was just
+                    // unpacked expects the schema its migrations produce; starting it
+                    // on the schema they failed to produce is the one outcome worse
+                    // than not deploying — it can come up healthy and fail on a request.
+                    const schemaError = migrationFailed.get(entry.app);
+                    if (schemaError !== undefined) {
+                        result.status = 'failed';
+                        result.error = `database migrations failed, so the new code was not started: ${schemaError}`;
+                        this.emitProgress(result.node, entry.app, 'failed', 80, result.error);
+                        continue;
+                    }
+                    // An app whose artifact did not move, whose configuration did not
+                    // move, and which is running, has nothing that a restart would
+                    // change — and a restart is downtime. See `decideRedeploy`.
+                    const decision = decideRedeploy({
+                        // `unchanged` is set by `deployToNode` when the node's recorded
+                        // checksum matched the artifact's; anything else means files
+                        // were written.
+                        artifactChanged: result.unchanged === true ? false : true,
+                        online: runningNow.has(entry.app),
+                        configChanged: registration.changed,
+                    });
+                    if (decision.action === 'leave') {
+                        this.emitProgress(result.node, entry.app, 'success', 100, 'Running — nothing to change');
+                        this.logger.info({ node: result.node, app: entry.app, version: entry.version, because: decision.because }, 'Left running — this deployment changes nothing for this app');
+                        continue;
+                    }
+                    const started = await this.signalRemoteDaemon(target, entry.app);
+                    const health = started.ok
+                        ? await this.verifyHealth(target, entry.app, project)
+                        : { online: false, detail: started.detail };
+                    if (health.online) {
+                        await this.recordRunningArtifact(target, artifactDirOnNode(project, entry.app, entry.version), entry.app, result.checksum);
+                        this.emitProgress(result.node, entry.app, 'success', 100, 'Running');
+                        // Said, because its opposite is. «Left running» above logs when
+                        // this deployment leaves an application alone; restarting one
+                        // logged nothing at all, so a log full of six «Left running»
+                        // lines and a log of six silent restarts differ only by the
+                        // absence of something — and absence is what a reader cannot
+                        // see. Measured 2026-09-22: six artifacts delivered and every
+                        // application restarted, with nothing in the log between
+                        // «Opened the gateway's path» and «Stack started» to say so.
+                        this.logger.info({
+                            node: result.node,
+                            app: entry.app,
+                            version: entry.version,
+                            because: decision.because,
+                        }, 'Restarted onto the artifact this deployment installed');
+                        continue;
+                    }
+                    // Installed and not running is a failure, and it is the state this
+                    // fleet reported as success for as long as it has existed.
+                    result.status = 'failed';
+                    result.error = `artifact installed, app not running: ${started.ok ? health.detail : started.detail}`;
+                    this.emitProgress(result.node, entry.app, 'failed', 90, result.error);
+                    this.logger.error({ node: result.node, app: entry.app, version: entry.version, detail: result.error }, 'Artifact installed, but the app is not running on the node');
+                }
+            }
+        }
         return results;
     }
+    /**
+     * Give a node the config that tells it what to run.
+     *
+     * Written, then registered. Registering a path with no config in it leaves
+     * the daemon with a project it cannot read, which reports as a broken
+     * project rather than a missing file.
+     *
+     * Reported and not thrown: artifacts that transferred are on the node
+     * either way, and failing the whole deployment over the registration would
+     * discard a transfer that succeeded. The operator needs to know the apps
+     * are not runnable, which is what the error says.
+     */
+    /**
+     * Let the gateway reach the applications it proxies to.
+     *
+     * The gateway is a container and the apps are host processes, so every
+     * `/api/*` request crosses out of docker and into the host's INPUT chain.
+     * On a hardened host that chain drops it, and the gateway answers a
+     * correct, unhelpful `503` three seconds later while the upstream it could
+     * not reach is listening and healthy on the same machine.
+     *
+     * Measured on the test node with everything else already right: apps on
+     * `0.0.0.0:3001-3007`, answering `404` to the host itself, timing out from
+     * the gateway, `ufw` active with `22/tcp` its only rule and INPUT policy
+     * DROP.
+     *
+     * Derived, not configured: the subnet comes from the network the gateway
+     * is actually on, so a stack cannot be given a rule for the wrong one, and
+     * a subnet that is not a private range is refused outright rather than
+     * turned into a firewall rule nobody can explain.
+     *
+     * Never fatal. A host whose firewall cannot be read is a host that gets a
+     * message; stopping a deployment over it would trade a visible 503 for an
+     * invisible one.
+     */
+    async openGatewayPath(target, project, stack) {
+        const network = `${project}-${stack}_default`;
+        try {
+            // No `2>/dev/null || true`: a network that cannot be inspected is the
+            // reason the path stays shut, and discarding it leaves an operator
+            // with a 503 and no sentence. The failure is caught below and named.
+            const subnet = (await this.sshExec(target, `docker network inspect ${shellEscape(network)} --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}'`)).trim();
+            if (!subnet) {
+                this.logger.warn({ node: target.host, network }, 'The stack network reports no subnet — the gateway’s path to the applications was left as it was');
+                return;
+            }
+            const rule = reachabilityRule(subnet, project, stack);
+            if (!rule) {
+                this.logger.warn({ node: target.host, network, subnet }, 'Refusing to open a path from this subnet — it is not a private range');
+                return;
+            }
+            const outcome = (await this.sshExec(target, reachabilityCommand(rule))).trim();
+            if (outcome === 'allowed') {
+                this.logger.info({ node: target.host, subnet: rule.subnet, ports: `${rule.fromPort}-${rule.toPort}` }, 'Opened the gateway’s path to the applications on this host');
+            }
+            else if (outcome === 'failed') {
+                this.logger.error({ node: target.host, subnet: rule.subnet }, 'Could not open the gateway’s path — every /api/* request will answer 503 while the apps are healthy');
+            }
+            else if (outcome.startsWith('ufw-unreadable')) {
+                // Not `ufw-inactive`: nothing is known about the firewall, and if it
+                // is active it is dropping the gateway's traffic right now.
+                this.logger.warn({ node: target.host, subnet: rule.subnet, reason: outcome.slice('ufw-unreadable'.length).replace(/^:\s*/, '') }, 'Could not read the node’s firewall — if it is active, the gateway’s path to the applications is closed and /api/* will answer 503');
+            }
+        }
+        catch (err) {
+            this.logger.warn({ node: target.host, network, error: err.message.slice(0, 200) }, 'Could not check the gateway’s path to the applications');
+        }
+    }
+    /**
+     * Run each app's migrations, on the node, against the node's database.
+     *
+     * They had never run there. `ProjectService` migrates by executing
+     * `apps/<name>/src/database/migrate.ts` from the PROJECT PATH against
+     * `localhost` — both of which name the machine the master is on. A node has
+     * no project sources and a different localhost, so a remote stack's
+     * databases were created and left empty.
+     *
+     * Measured on the test node after six apps installed and five started:
+     * every one of the six databases existed and held ZERO tables. Five apps
+     * came up anyway, because they touch no table at boot; `paysys` does, and
+     * said so —
+     *
+     *     Failed to call @PostConstruct 'bootstrap' on 'PlatformRevenueService':
+     *     relation "accounts" does not exist
+     *
+     * — which is the only reason anyone found out. A platform with no schema
+     * that reports five of six apps healthy is a worse state than one that
+     * refuses to start.
+     *
+     * The artifact already carries what is needed: `dist/database/migrate.js`
+     * beside `dist/database/migrations/` — 160 compiled files for paysys. The
+     * same build the app itself runs from, so there is no second copy of the
+     * schema to drift.
+     *
+     * Returns the apps whose migrations failed, with the reason, and the
+     * caller does not start their new code.
+     *
+     * This used to say: «Failure is reported and does not stop the deployment:
+     * an app whose migrations fail will fail its own start with a message
+     * about the table it wanted, which is more specific than anything this
+     * step could say.» True for one of the two ways it goes. An app that
+     * fails to START is caught by `verifyHealth` and marked failed — that
+     * branch was covered, and the sentence is right about it. An app that
+     * starts fine and fails on the first REQUEST that touches the missing
+     * table passes `verifyHealth`, the stack reports `started`, and the only
+     * trace is one ERROR line in the master's log. The justification covered
+     * the branch that was already safe.
+     *
+     * Idempotence is what makes refusing cheap: the migrator records each
+     * migration it applied, in a transaction with the migration itself, so
+     * running the batch again after the cause is fixed applies exactly what
+     * is missing. That holds because `HardenedMigrationRunner` defaults
+     * `useTransactions` to true (`titan-database/src/migration/
+     * hardened-runner.ts:171`) and no daos `migrate.ts` overrides it — it is
+     * conditional in the runner (`:502`), so a migrator that turned it off
+     * would make a refused deployment leave a half-applied migration behind.
+     */
+    async migrateNodeApps(target, project, landed, appEnv) {
+        const failed = new Map();
+        for (const entry of landed) {
+            const databaseUrl = appEnv?.[entry.app]?.['DATABASE_URL'];
+            if (!databaseUrl)
+                continue;
+            const dir = `/opt/omnitron/artifacts/${assertRemotePathSegment('project name', project)}` +
+                `/${assertRemotePathSegment('app name', entry.app)}` +
+                `/${assertRemotePathSegment('version', entry.version)}`;
+            const script = `${dir}/dist/database/migrate.js`;
+            // A node that could not be asked is not an app without a migrator.
+            // This command exits 0 either way, so `sshExec` throws only when the
+            // question never arrived — and that was answered `'no'`: migrations
+            // skipped, new code started on the old schema. The worst case of the
+            // docblock above, reached through a dropped SSH connection instead of
+            // a failed migration, and refused the same way.
+            let present;
+            try {
+                present = await this.sshExec(target, `test -f ${shellEscape(script)} && echo yes || echo no`);
+            }
+            catch (err) {
+                const message = `the node could not be asked whether there are any to apply — ${err.message}`;
+                this.logger.error({ node: target.host, app: entry.app, error: message.slice(0, 400) }, 'Could not ask the node for the app\'s migrations — its new code will not be started');
+                failed.set(entry.app, message.slice(0, 300));
+                continue;
+            }
+            if (present.trim() !== 'yes')
+                continue;
+            this.emitProgress(`${target.host}:${target.daemonPort ?? 9700}`, entry.app, 'installing', 75, 'Applying database migrations...');
+            // Both spellings, because the migrators read the second one.
+            //
+            // `apps/paysys/src/database/migrate.ts` declares
+            // `envPrefix: 'PAYSYS__DATABASE'` and builds its connection from
+            // `PAYSYS__DATABASE__HOST` / `__PORT` / `__USER` / `__PASSWORD` /
+            // `__DATABASE`. Given only `DATABASE_URL` it falls back to its own
+            // defaults and reports `password authentication failed for user
+            // "postgres"` — against a password that is correct and that it never
+            // read. All six apps failed that way, which is what a convention
+            // nobody wrote down costs when a second caller appears.
+            //
+            // `ProjectService` passes both, with the comment "App-specific env
+            // vars (various naming conventions)". This is the second caller.
+            const parsed = parseDatabaseUrl(databaseUrl);
+            const prefix = entry.app.toUpperCase().replace(/-/g, '_');
+            const dbEnv = parsed
+                ? {
+                    [`${prefix}__DATABASE__HOST`]: parsed.host,
+                    [`${prefix}__DATABASE__PORT`]: parsed.port,
+                    [`${prefix}__DATABASE__DATABASE`]: parsed.database,
+                    [`${prefix}__DATABASE__USER`]: parsed.user,
+                    [`${prefix}__DATABASE__PASSWORD`]: parsed.password,
+                }
+                : {};
+            try {
+                // `cd` into the artifact so the script resolves its own imports, and
+                // every value in the environment rather than the command line, where
+                // it would be in every process listing on the host.
+                const assignments = Object.entries({ DATABASE_URL: databaseUrl, ...dbEnv })
+                    .map(([k, v]) => `${k}=${shellEscape(v)}`)
+                    .join(' ');
+                // What the migrations said, in order, before the verdict (see
+                // `migration-output.ts`). The masked channel on purpose: these lines
+                // are for reading, not parsing, and a migrator that ever printed a
+                // credential must not put it in the master's log.
+                const said = await this.sshExec(target, `cd ${shellEscape(dir)} && ${assignments} node ${shellEscape(script)}`, 600_000);
+                for (const line of migrationOutput(said)) {
+                    this.logger.info({ node: target.host, app: entry.app, line }, 'Migration said');
+                }
+                this.logger.info({ node: target.host, app: entry.app }, 'Database migrations applied on the node');
+            }
+            catch (err) {
+                const message = err.message;
+                this.logger.error({ node: target.host, app: entry.app, error: message.slice(0, 400) }, 'Database migrations failed on the node — its new code will not be started');
+                failed.set(entry.app, message.slice(0, 300));
+            }
+        }
+        return failed;
+    }
+    /**
+     * Write the node's app definitions, and say whether they changed.
+     *
+     * The caller decides whether to restart the applications, and "the
+     * configuration moved" is half of that decision — identical files with a
+     * different environment is a different deployment. Compared by the hash of
+     * the rendered body against one written beside it, rather than by reading
+     * the file back: the body holds every generated credential on the node,
+     * and it should cross the wire once, in the direction it has to.
+     */
+    async registerNodeApps(target, project, apps, landed, appEnv, alertSink) {
+        if (landed.length === 0) {
+            this.logger.warn({ host: target.host, project }, 'No artifact reached this node — nothing to register');
+            return { changed: false };
+        }
+        const { renderNodeAppConfig } = await import('../project/node-app-config.js');
+        const dir = `/opt/omnitron/projects/${assertRemotePathSegment('project name', project)}`;
+        const body = renderNodeAppConfig({
+            project,
+            artifactRoot: '/opt/omnitron/artifacts',
+            apps,
+            artifacts: landed,
+            appEnv,
+            alertSink,
+        });
+        const bodyHash = crypto.createHash('sha256').update(body).digest('hex');
+        try {
+            await this.sshExec(target, `mkdir -p ${shellEscape(dir)}`);
+            const previous = parseRecordedChecksum(await this.sshExec(target, readChecksumCommand(shellEscape(dir), NODE_CONFIG_HASH_FILE)));
+            const changed = previous !== bodyHash;
+            // Through a here-document: the config carries braces, quotes and
+            // newlines, and a single-quoted argument would need every quote in it
+            // escaped by hand — which is how a generated file acquires a syntax
+            // error nobody can see in the source that generated it.
+            const configPath = `${dir}/omnitron.config.mjs`;
+            // `umask 077` around the write, not a `chmod` after it: the file exists
+            // with its contents between the two, and what it contains is the
+            // generated password of every service on this node. It was written 0644
+            // for as long as it has existed — measured on the test node — which was
+            // harmless only because it held no credentials to leak.
+            await this.sshExec(target, `umask 077 && cat > ${shellEscape(configPath)} <<'OMNITRON_EOF'\n${body}\nOMNITRON_EOF`);
+            await this.sshExec(target, `chmod 600 ${shellEscape(configPath)}`);
+            await this.sshExec(target, `printf %s ${shellEscape(bodyHash)} > ${shellEscape(`${dir}/${NODE_CONFIG_HASH_FILE}`)}`);
+            const added = await this.sshExec(target, `omnitron project add ${shellEscape(project)} ${shellEscape(dir)} 2>&1`);
+            // Registering the project is not starting it. The node reads the
+            // definitions and waits: measured, `omnitron status` answered
+            // `appsTotal: 0` with all six apps listed in the file it had just been
+            // given, and `omnitron restart main` still said `Unknown app: main`.
+            //
+            // `startStack` is how a project's apps are started, which is why the
+            // generated config carries a stack for them to be in.
+            const { NODE_STACK } = await import('../project/node-app-config.js');
+            // ...and whether it has to be told at all. This command starts
+            // everything the node's config lists, so it overruled the per-app
+            // decision taken below it — measured 2026-09-22, six «Left running»
+            // lines about six applications this same deployment had restarted
+            // twenty seconds earlier. See `decideNodeStackStart`.
+            const start = decideNodeStackStart({
+                configChanged: changed,
+                apps: landed.map((l) => l.app),
+                online: await this.appsOnline(target, project),
+            });
+            const out = start.action === 'start'
+                ? await this.sshExec(target, `omnitron stack start ${shellEscape(project)} ${shellEscape(NODE_STACK)} 2>&1`, 300_000)
+                : '';
+            void added;
+            // The node's CLI prints its refusals and exits zero — `omnitron project
+            // add` answered `No omnitron.config.ts found at …` and this logged
+            // `The node now knows what to run` over it, twice, while the node knew
+            // nothing. An exit code that does not carry the outcome means the
+            // OUTPUT is the outcome.
+            const refused = /\bno .*found\b|\bfailed\b|\berror\b|not found|cannot/i.test(`${added}\n${out}`);
+            if (refused) {
+                this.logger.error({ host: target.host, project, dir, detail: `${added}\n${out}`.trim().slice(0, 300) }, 'The node refused the apps it was given — its artifacts are installed and it will run none of them');
+                return { changed };
+            }
+            this.logger.info({
+                host: target.host,
+                project,
+                dir,
+                apps: landed.map((l) => l.app),
+                changed,
+                started: start.action === 'start',
+                because: start.because,
+                detail: out.trim().slice(0, 200),
+            }, start.action === 'start'
+                ? 'The node now knows what to run'
+                : 'The node is already running these apps with this configuration — its stack was not restarted');
+            return { changed };
+        }
+        catch (err) {
+            this.logger.error({ host: target.host, project, dir, error: err.message }, 'Could not tell the node what to run — its artifacts are installed and its daemon does not know the apps');
+            // Unknown, and unknown must not read as "nothing moved": an app whose
+            // config may have changed is an app to restart.
+            return { changed: true };
+        }
+    }
     /**
      * Provision a remote node as a slave daemon.
      *
@@ -134,191 +976,788 @@ export class RemoteDeployer {
      * @param masterHost - The master daemon's reachable address (from slave's perspective)
      * @param masterPort - The master daemon's fleet TCP port
      */
-    async provisionSlaveNode(node, masterHost, masterPort, project) {
-        const nodeKey = `${node.host}:${node.port ?? 9700}`;
+    async provisionSlaveNode(target, 
+    /**
+     * Where this node should dial to reach the master, or null when there is
+     * nowhere — a master behind NAT, which is where remote stacks are
+     * started from.
+     *
+     * Null is not a failure. Replication is master-PULL: the master opens
+     * the connection, over SSH when the node's daemon port is closed, and
+     * drains the node's buffer. `omnitron up --slave` takes the address
+     * optionally for exactly this reason.
+     */
+    masterHost, masterPort, project, options = {}) {
+        const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
         try {
             // 1. Verify SSH access
             this.emitProgress(nodeKey, '*', 'pending', 0, 'Connecting via SSH...');
-            await this.verifySSH(node);
-            // 2. Ensure runtime (Node.js or Bun)
-            this.emitProgress(nodeKey, '*', 'extracting', 10, 'Checking runtime...');
-            const hasNode = await this.sshExec(node, 'which node 2>/dev/null || which bun 2>/dev/null || echo ""').catch(() => '');
-            if (!hasNode.trim()) {
-                this.logger.info({ host: node.host }, 'Installing Node.js on remote node...');
-                this.emitProgress(nodeKey, '*', 'extracting', 15, 'Installing Node.js...');
-                try {
-                    // Install Node.js via official installer (works on most Linux distros)
-                    await this.sshExec(node, 'curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs 2>/dev/null || (curl -fsSL https://rpm.nodesource.com/setup_22.x | bash - && yum install -y nodejs) 2>/dev/null || (apk add --no-cache nodejs npm)', 120_000);
-                }
-                catch (err) {
-                    this.logger.error({ host: node.host, error: err.message }, 'Failed to install Node.js');
-                    return false;
-                }
+            await this.verifySSH(target);
+            // 2-3. Prepare the host: whatever it is, and whatever it is missing.
+            //
+            // This was two probes and a `||` chain of package-manager installs that
+            // only worked on three Linux families, added a vendor repository to the
+            // host as a side effect, and on failure reported the error from the
+            // LAST alternative — `apk` on a Debian machine that has never had it.
+            //
+            // Now the host is asked what it is in one round trip, a plan is derived
+            // from the answer, and the plan is a value: it can be logged before it
+            // runs and read in a test without a machine. See
+            // `remote-provisioner.ts` for what it decides and why.
+            this.emitProgress(nodeKey, '*', 'extracting', 10, 'Inspecting the host...');
+            const facts = parsePlatformProbe(await this.sshExec(target, PLATFORM_PROBE, 60_000));
+            const plan = planProvisioning(facts, {
+                ...(options.nodeVersion ? { nodeVersion: options.nodeVersion } : {}),
+                ...(options.installRuntime === false ? { usePackageManager: false } : {}),
+            });
+            this.logger.info({
+                host: target.host,
+                os: facts.os,
+                arch: facts.arch,
+                distro: facts.distro,
+                packageManager: facts.packageManager,
+                node: facts.node,
+                omnitron: facts.omnitron,
+                plan: describePlan(plan),
+            }, 'Host inspected');
+            if (plan.refusal) {
+                this.logger.error({ host: target.host, reason: plan.refusal }, 'Cannot prepare this host');
+                this.emitProgress(nodeKey, '*', 'failed', 0, plan.refusal);
+                return false;
             }
-            // 3. Install omnitron
-            this.emitProgress(nodeKey, '*', 'extracting', 30, 'Installing omnitron...');
-            const hasOmnitron = await this.sshExec(node, 'which omnitron 2>/dev/null || echo ""').catch(() => '');
-            if (!hasOmnitron.trim()) {
+            let progress = 15;
+            const stride = plan.steps.length > 0 ? Math.floor(30 / plan.steps.length) : 0;
+            for (const step of plan.steps) {
+                this.emitProgress(nodeKey, '*', 'extracting', progress, step.what);
+                if (step.touchesPackageManager) {
+                    // Said at warning level: this is the one kind of step that changes
+                    // how the machine gets its software, and the machines this reaches
+                    // are not blank.
+                    this.logger.warn({ host: target.host, step: step.what }, 'Using the host package manager');
+                }
                 try {
-                    await this.sshExec(node, 'npm install -g @omnitron-dev/omnitron', 120_000);
+                    await this.sshExec(target, step.command, step.timeoutMs);
                 }
                 catch (err) {
-                    this.logger.error({ host: node.host, error: err.message }, 'Failed to install omnitron');
+                    // The step that failed, by name, rather than the last alternative's
+                    // error message.
+                    this.logger.error({ host: target.host, step: step.what, error: err.message }, 'Host preparation step failed');
+                    this.emitProgress(nodeKey, '*', 'failed', progress, `${step.what}: ${err.message}`);
                     return false;
                 }
+                progress += stride;
             }
-            // 4. Generate slave config
+            // 4. Configure and start the slave, through omnitron's own setup path.
+            //
+            // This wrote `/etc/omnitron/omnitron.config.ts` containing `role`,
+            // `master` and a `daemon` block, then ran a bare `omnitron up` beside
+            // it. Every one of those keys was read by nothing.
+            //
+            // `IEcosystemConfig` — the schema of that file — has no `role`, no
+            // `master` and no `daemon`. The daemon boots from
+            // `~/.omnitron/config.json`, whose `SavedDaemonConfig` carried role and
+            // master and nothing about transports. So a provisioned slave would
+            // have come up as a MASTER, bound to loopback, with no master address
+            // and no sync: three of the four things this step exists to arrange,
+            // silently not arranged.
+            //
+            // `omnitron up --slave <host>:<port>` is the path that writes what the
+            // daemon reads, and it is the same one an operator uses by hand. One
+            // way to configure a slave instead of two, and the one that is
+            // exercised.
             this.emitProgress(nodeKey, '*', 'extracting', 50, 'Configuring slave daemon...');
-            const configDir = '/etc/omnitron';
-            const configContent = this.generateSlaveConfig(masterHost, masterPort, node.port ?? 9700, project);
-            await this.sshExec(node, `mkdir -p ${shellEscape(configDir)}`);
-            // Write config via heredoc (avoids shell escaping issues with complex content)
-            await this.sshExec(node, `cat > ${shellEscape(configDir + '/omnitron.config.ts')} << 'OMNITRON_EOF'\n${configContent}\nOMNITRON_EOF`);
-            // 5. Start slave daemon (or restart if already running)
-            this.emitProgress(nodeKey, '*', 'restarting', 70, 'Starting slave daemon...');
-            await this.sshExec(node, `cd ${shellEscape(configDir)} && (omnitron down 2>/dev/null; omnitron up) &`).catch(() => {
-                // Background start — may "fail" because SSH returns before daemon fully starts
+            assertRemotePathSegment('project name', project);
+            const masterAddr = masterHost ? `${masterHost}:${masterPort}` : null;
+            // 5. Start the slave daemon — if it is not already the one we want.
+            //
+            // `omnitron down` stops the daemon AND every application under it, so
+            // this step is the most expensive thing a deployment does to a node
+            // that needed nothing. See `decideSlaveDaemonRestart`: it took three
+            // deployments to attribute, because the node-side `stack start` was
+            // restarting them too and either one alone explains the pids.
+            const daemonNow = readNodeStatus((await this.askNodeStatus(target)).raw);
+            const daemonDecision = decideSlaveDaemonRestart({
+                hostChanged: plan.steps.length > 0,
+                daemon: daemonNow
+                    ? {
+                        running: typeof daemonNow.pid === 'number' || typeof daemonNow.uptime === 'number',
+                        ...(daemonNow.role ? { role: daemonNow.role } : {}),
+                    }
+                    : null,
             });
-            // 6. Wait briefly and verify daemon is running
+            if (daemonDecision.action === 'leave') {
+                this.emitProgress(nodeKey, '*', 'restarting', 70, 'Slave daemon already running');
+                this.logger.info({ host: target.host, because: daemonDecision.because, apps: daemonNow?.apps.length ?? 0 }, 'The node daemon was left alone — this deployment changes nothing under it');
+            }
+            else {
+                // Said as loudly as the other branch. A decision that only speaks
+                // when it declines is a decision an operator cannot audit: this one
+                // took three deployments to attribute and a fourth to explain,
+                // because «restart» left nothing behind but the restart itself.
+                this.logger.info({
+                    host: target.host,
+                    because: daemonDecision.because,
+                    steps: plan.steps.length,
+                    role: daemonNow?.role ?? null,
+                    pid: daemonNow?.pid ?? null,
+                    uptime: daemonNow?.uptime ?? null,
+                }, 'Taking the node daemon down and back up — and everything under it');
+                this.emitProgress(nodeKey, '*', 'restarting', 70, 'Starting slave daemon...');
+                await this.sshExec(target, masterAddr
+                    ? `omnitron down 2>/dev/null; omnitron up --slave ${shellEscape(masterAddr)} --no-infra`
+                    // `--slave` with no address still sets the role; the node buffers
+                    // locally and waits to be pulled from, which is what it would do
+                    // with an address it cannot reach anyway.
+                    : 'omnitron down 2>/dev/null; omnitron up --slave --no-infra', 180_000).catch((err) => {
+                    // Reported, not swallowed. The verification below tells us whether the
+                    // daemon came up; this tells us what it said on the way.
+                    this.logger.warn({ host: target.host, error: err.message }, 'Slave start command did not return cleanly — verifying anyway');
+                });
+            }
+            // 6. Verify the daemon is running, and mean it.
+            //
+            // This slept three seconds, pinged once, and on `unreachable` logged
+            // "may still be starting" — then reported success regardless. Measured
+            // on the first real run: the start command failed with
+            // `omnitron: command not found`, the ping found nothing, and the
+            // function returned `true`. A caller that trusts that goes on to ship
+            // artifacts to a node with no daemon, and the console shows a node
+            // that was "provisioned" and answers nothing.
+            //
+            // A daemon that is starting will answer within the window; one that is
+            // not there will not answer within any window. Polling tells them
+            // apart, which one sample cannot.
             this.emitProgress(nodeKey, '*', 'verifying', 90, 'Verifying slave daemon...');
-            await new Promise((resolve) => setTimeout(resolve, 3000));
+            const started = await this.awaitDaemon(target, SLAVE_START_TIMEOUT_MS);
+            if (!started.ok) {
+                this.logger.error({ host: target.host, waitedMs: SLAVE_START_TIMEOUT_MS, detail: started.detail }, 'Slave daemon did not come up');
+                this.emitProgress(nodeKey, '*', 'failed', 90, `the slave daemon did not answer within ${Math.round(SLAVE_START_TIMEOUT_MS / 1000)}s: ${started.detail}`);
+                return false;
+            }
+            this.emitProgress(nodeKey, '*', 'success', 100, `Slave provisioned — ${started.detail}`);
+            this.logger.info({ host: target.host, masterHost, masterPort, daemon: started.detail }, 'Slave node provisioned');
+            return true;
+        }
+        catch (err) {
+            this.emitProgress(nodeKey, '*', 'failed', 0, err.message);
+            this.logger.error({ host: target.host, error: err.message }, 'Failed to provision slave node');
+            return false;
+        }
+    }
+    // ===========================================================================
+    // Private — SSH Operations
+    // ===========================================================================
+    /**
+     * Wait for the remote daemon to answer, or report why it did not.
+     *
+     * `omnitron ping` on the far side prints its pid, uptime and version when
+     * the daemon is up, and fails otherwise. The last failure is carried out of
+     * the loop so a caller can say what the host said rather than only that
+     * time ran out.
+     */
+    async awaitDaemon(target, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        let last = 'no answer';
+        while (Date.now() < deadline) {
             try {
-                const pingResult = await this.sshExec(node, 'omnitron ping 2>/dev/null || echo "unreachable"', 10_000);
-                if (pingResult.includes('unreachable')) {
-                    this.logger.warn({ host: node.host }, 'Slave daemon not yet responding — may still be starting');
+                const answer = await this.sshExec(target, 'omnitron ping', 15_000);
+                // "It printed something" is not "it is running". `omnitron ping`
+                // used to exit 0 while printing that the daemon is not running —
+                // fixed, but a node runs whatever version it has, and this code
+                // upgrades nodes from older ones. The answer is read, not assumed.
+                if (/daemon is running/i.test(answer)) {
+                    return { ok: true, detail: answer.split('\n').filter(Boolean).pop().trim() };
                 }
+                last = answer.split('\n').filter(Boolean).pop()?.trim() || 'the ping printed nothing';
             }
-            catch {
-                this.logger.warn({ host: node.host }, 'Could not verify slave daemon — it may still be starting');
+            catch (err) {
+                last = err.message;
             }
-            this.emitProgress(nodeKey, '*', 'success', 100, 'Slave provisioned');
-            this.logger.info({ host: node.host, masterHost, masterPort }, 'Slave node provisioned');
+            await new Promise((resolve) => setTimeout(resolve, DAEMON_POLL_MS));
+        }
+        return { ok: false, detail: last };
+    }
+    /**
+     * Install a locally built omnitron on a node, beside whatever it is
+     * running.
+     *
+     * The counterpart to installing from the registry, and the only channel
+     * that can put THIS tree on a node: the published package is whatever was
+     * last released, which on 2026-09-14 was five months and 224 commits
+     * behind, under the same version number.
+     *
+     * Nothing is made live here. The archive is transferred, unpacked into its
+     * own version directory, installed, and run — and `activate` is a separate
+     * call, so a caller that stops after this has spent disk and changed
+     * nothing about what the node is serving.
+     */
+    async installBundle(target, archivePath, version, prefix = '/opt/omnitron') {
+        const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
+        const layout = { prefix, version };
+        // Under the prefix, not `/tmp`: a bundle is tens of megabytes and `/tmp`
+        // is a tmpfs on many hosts, where a large transfer competes with memory.
+        const remoteArchive = `${prefix}/releases/${version}.tar.gz`;
+        try {
+            this.emitProgress(nodeKey, '*', 'pending', 0, 'Connecting via SSH...');
+            await this.verifySSH(target);
+            this.emitProgress(nodeKey, '*', 'transferring', 10, `Transferring ${version}...`);
+            await this.sshExec(target, `mkdir -p ${shellEscape(`${prefix}/releases`)}`);
+            await this.scpTransfer(target, archivePath, remoteArchive);
+            let progress = 30;
+            for (const step of installSteps(layout, remoteArchive)) {
+                this.emitProgress(nodeKey, '*', 'extracting', progress, step.what);
+                const output = await this.sshExec(target, step.command, step.timeoutMs);
+                if (step.what.includes('runs')) {
+                    // The version the node reports has to be the version we shipped.
+                    // A mismatch means the archive, the directory and the manifest
+                    // disagree — and the one thing a fleet upgrade cannot tolerate is
+                    // a node that reports a version it is not running.
+                    const reported = output.trim().split('\n').pop()?.trim();
+                    if (reported !== version) {
+                        throw new Error(`The installed copy reports ${JSON.stringify(reported)}, not ${JSON.stringify(version)}.`);
+                    }
+                }
+                progress += 20;
+            }
+            this.emitProgress(nodeKey, '*', 'success', 100, `${version} installed (not yet current)`);
+            this.logger.info({ host: target.host, version, prefix }, 'Bundle installed beside the running version');
             return true;
         }
         catch (err) {
-            this.emitProgress(nodeKey, '*', 'failed', 0, err.message);
-            this.logger.error({ host: node.host, error: err.message }, 'Failed to provision slave node');
+            const message = err.message;
+            this.emitProgress(nodeKey, '*', 'failed', 0, message);
+            this.logger.error({ host: target.host, version, error: message }, 'Bundle install failed');
             return false;
         }
     }
     /**
-     * Generate omnitron.config.ts content for a slave daemon.
+     * `activateBundle` with the platform held. `omnitron down; up` stops every
+     * app on the node, so each stack gateway there holds its platform
+     * (`maintenance-lock.ts`) — warned first — until its apps answer through it
+     * again, for up to three minutes after the restart; past that the lock is
+     * left to expire by itself. A node with no gateway activates as before.
      */
-    generateSlaveConfig(masterHost, masterPort, slavePort, project) {
-        return `/**
- * Omnitron Slave Configuration
- * Auto-generated by master during remote/cluster deployment.
- * Project: ${project}
- */
-export default {
-  name: '${project}',
-  apps: [],
-  role: 'slave',
-  master: { host: '${masterHost}', port: ${masterPort} },
-  sync: {
-    interval: 30000,
-    batchSize: 1000,
-  },
-  daemon: {
-    socketPath: '~/.omnitron/daemon.sock',
-    port: ${slavePort},
-    host: '0.0.0.0',
-    httpPort: 9800,
-    pidFile: '~/.omnitron/daemon.pid',
-    stateFile: '~/.omnitron/daemon.state',
-  },
-  supervision: {
-    strategy: 'one_for_one',
-    maxRestarts: 10,
-    window: 60000,
-    backoff: { type: 'exponential', initial: 1000, max: 30000, factor: 2 },
-  },
-  monitoring: {
-    healthCheck: { interval: 30000, timeout: 10000 },
-    metrics: { interval: 15000, retention: 86400000 },
-  },
-  logging: {
-    level: 'info',
-    directory: '~/.omnitron/logs',
-    maxSize: '50m',
-    maxFiles: 10,
-    compress: false,
-  },
-  env: 'production',
-};
-`;
+    async activateBundleHeld(target, version, prefix = '/opt/omnitron', keepVersions = 3) {
+        const { runningGatewaySites, MaintenanceLock, DEFAULT_MAINTENANCE_TIMING } = await import('./maintenance-lock.js');
+        const run = this.leaseRunner(target);
+        const locks = [];
+        for (const site of await runningGatewaySites(run, `omnitron ${version}`)) {
+            const lock = await MaintenanceLock.open(site, this.logger, { ...DEFAULT_MAINTENANCE_TIMING, probeWithinMs: 180_000 });
+            if (lock)
+                locks.push(lock);
+        }
+        try {
+            const activated = await this.activateBundle(target, version, prefix, keepVersions);
+            while (locks.length > 0) {
+                const lifted = await locks.shift().liftWhenAnswering(['/', '/api/main/health']);
+                if (!lifted.lifted) {
+                    this.logger.error({ host: target.host, version, failed: lifted.failed }, 'After the upgrade the platform does not answer through its gateway — maintenance expires by itself');
+                }
+            }
+            return activated;
+        }
+        finally {
+            for (const lock of locks)
+                await lock.lift('failed').catch(() => undefined);
+        }
     }
-    // ===========================================================================
-    // Private — SSH Operations
-    // ===========================================================================
-    async verifySSH(node) {
-        await this.sshExec(node, 'echo ok', 10_000);
-    }
-    async sshExec(node, command, timeout = 60_000) {
-        const sshArgs = this.buildSSHArgs(node);
-        sshArgs.push(command);
-        const { stdout } = await exec('ssh', sshArgs, { timeout });
-        return stdout.trim();
-    }
-    async scpTransfer(node, localPath, remotePath) {
-        const user = node.ssh?.user ?? 'root';
-        const port = node.ssh?.port ?? 22;
-        const scpArgs = [
-            '-o', 'StrictHostKeyChecking=accept-new',
-            '-o', 'ConnectTimeout=10',
-            '-P', String(port),
-        ];
-        if (node.ssh?.privateKey) {
-            scpArgs.push('-i', node.ssh.privateKey);
-        }
-        scpArgs.push(localPath, `${user}@${node.host}:${remotePath}`);
-        await exec('scp', scpArgs, { timeout: 300_000 }); // 5 min for large artifacts
-    }
-    async signalRemoteDaemon(node, appName) {
+    /**
+     * Make an installed version the one the node runs, and restart into it.
+     *
+     * Separate from `installBundle` for the reason the layout exists: this is
+     * the only step that changes what runs.
+     */
+    async activateBundle(target, version, prefix = '/opt/omnitron', keepVersions = 3) {
+        const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
+        const layout = { prefix, version };
         try {
-            // Try RPC restart via omnitron CLI on remote
-            await this.sshExec(node, `omnitron restart ${shellEscape(appName)} 2>/dev/null || true`);
+            for (const step of activateSteps(layout)) {
+                this.emitProgress(nodeKey, '*', 'restarting', 40, step.what);
+                await this.sshExec(target, step.command, step.timeoutMs);
+            }
+            this.emitProgress(nodeKey, '*', 'restarting', 60, 'Restarting the daemon into the new version');
+            // By absolute path into `current`, not by name. A bare `omnitron` is
+            // whatever the node's PATH resolves — which, until the step above ran,
+            // was a different installation entirely. Naming the copy we just
+            // activated is the difference between restarting the new version and
+            // restarting whatever was there.
+            const cli = shellEscape(`${prefix}/current/dist/cli/omnitron.js`);
+            await this.sshExec(target, `${cli} down 2>/dev/null; ${cli} up --no-infra`, 180_000).catch((err) => {
+                this.logger.warn({ host: target.host, error: err.message }, 'Restart command did not return cleanly — verifying anyway');
+            });
+            this.emitProgress(nodeKey, '*', 'verifying', 80, 'Verifying the daemon answers');
+            const started = await this.awaitDaemon(target, 120_000);
+            if (!started.ok) {
+                // Reported, and NOT rolled back automatically. A daemon that will not
+                // start is a decision for whoever is watching: the previous version
+                // is still on disk and one `ln -sfn` away, and guessing that a
+                // rollback is wanted can be as wrong as guessing it is not.
+                this.logger.error({ host: target.host, version, detail: started.detail }, 'The new version did not answer — the previous one is still installed');
+                this.emitProgress(nodeKey, '*', 'failed', 80, `${version} did not answer: ${started.detail}. Roll back with ` +
+                    `\`ln -sfn ${prefix}/versions/<previous> ${prefix}/current\` and restart.`);
+                return false;
+            }
+            // Only once the new version is serving. Pruning before this could
+            // remove the version a rollback needs.
+            for (const step of pruneSteps(layout, keepVersions)) {
+                await this.sshExec(target, step.command, step.timeoutMs).catch((err) => {
+                    // Retention failing is not the upgrade failing.
+                    this.logger.warn({ host: target.host, error: err.message }, 'Version retention failed');
+                });
+            }
+            this.emitProgress(nodeKey, '*', 'success', 100, `Running ${version} — ${started.detail}`);
+            this.logger.info({ host: target.host, version, daemon: started.detail }, 'Node upgraded');
+            return true;
+        }
+        catch (err) {
+            const message = err.message;
+            this.emitProgress(nodeKey, '*', 'failed', 0, message);
+            this.logger.error({ host: target.host, version, error: message }, 'Activation failed');
+            return false;
+        }
+    }
+    async verifySSH(target) {
+        await this.sshExec(target, 'echo ok', 10_000);
+    }
+    /**
+     * Run a command on the node, through the daemon's one SSH implementation.
+     *
+     * Throws on a non-zero exit, which is what the callers above are written
+     * against — they use `|| echo ""` where they want to see a failure as an
+     * answer, and expect a throw everywhere else. `ExecutionService.ssh` reports
+     * failure in `exitCode` rather than raising, so the check has to be here;
+     * without it every `if (!result.trim())` in this file would read a failed
+     * command as an empty answer, and the two probes in `provisionSlaveNode`
+     * would install a runtime onto a host that already has one.
+     */
+    /**
+     * How a node deploy lease reaches its node: the lease module writes the
+     * scripts, the deployer owns the SSH. See `node-deploy-lease.ts`.
+     */
+    leaseRunner(target) {
+        // Through the data channel: a lease answer carries the holder's record,
+        // `{"token":…}`, which the masker rewrote into `"token": [REDACTED]` on
+        // every read — a refusal could never say who held the node.
+        return (script) => this.sshExecData(target, script, 30_000);
+    }
+    /**
+     * Run `work` holding the machine's deploy lease — for what changes a node
+     * outside a stack deployment. `fleet upgrade`'s install and activate went
+     * to the node with no lease at all, and are exactly a second writer: the
+     * activation restarts the node's daemon under whatever else is deploying.
+     */
+    async underLease(target, purpose, work) {
+        const machine = `${target.host}:${target.sshPort ?? 22}`;
+        return withNodeLeases([{ node: machine, machine, run: this.leaseRunner(target) }], purpose, this.logger, async (leases) => {
+            if (!leases.has(machine)) {
+                throw new Error(`Could not reach ${machine} to lease it for ${purpose}: ${leases.unreachable.get(machine) ?? 'no answer'}`);
+            }
+            return work();
+        });
+    }
+    async sshExec(target, command, timeout = 60_000) {
+        const result = await this.execution.ssh(sshTargetOf(target), command, { timeout });
+        if (result.exitCode !== 0) {
+            const detail = result.stderr || result.stdout || `exit ${result.exitCode}`;
+            throw new Error(`ssh ${target.username ?? 'root'}@${target.host}: ${detail}`);
         }
-        catch {
-            // Non-critical — daemon may not be running
-            this.logger.debug({ host: node.host, app: appName }, 'Remote daemon restart signal failed');
+        return result.stdout.trim();
+    }
+    /** `sshExec` for a command whose stdout is data — see `readFromNode`. */
+    async sshExecData(target, command, timeout = 60_000) {
+        const result = await this.readFromNode(target, command, timeout);
+        if (result.code !== 0) {
+            throw new Error(`ssh ${target.username ?? 'root'}@${target.host}: ${result.stderr || result.stdout || `exit ${result.code}`}`);
+        }
+        return result.stdout.trim();
+    }
+    /**
+     * Run a command on a node and hand back what it said, exit code included.
+     *
+     * `sshExec` turns every non-zero exit into a throw, which is right for a
+     * deployment step — a step either happened or it did not. A PROBE's exit
+     * code is information: the attestation producer exits 1 when probes failed
+     * (a fact to keep) and 2 when it could not measure at all (nothing to
+     * keep), and a throw would flatten the two into one.
+     */
+    async runOnNode(target, command, timeoutMs = 600_000) {
+        const result = await this.execution.ssh(sshTargetOf(target), command, { timeout: timeoutMs });
+        return { stdout: result.stdout, stderr: result.stderr, code: result.exitCode ?? -1 };
+    }
+    /**
+     * `runOnNode` for a command whose stdout is DATA — a record to parse, a
+     * line to store — carried past the transport's masker
+     * (`execution/data-channel.ts`). Its callers are named there and here: the
+     * lease runner (holder records), the attestation (the producer's line), the
+     * operator tool (its answer line). What it returns is unmasked; none of
+     * them logs it whole.
+     *
+     * A channel that came back as something other than its encoding is a
+     * failure with its own code, -2, and its words on stderr — never data.
+     */
+    async readFromNode(target, command, timeoutMs = 600_000) {
+        const result = await this.execution.ssh(sshTargetOf(target), throughDataChannel(command), { timeout: timeoutMs });
+        const data = fromDataChannel(result.stdout);
+        if (!data.ok) {
+            return { stdout: '', stderr: [data.because, result.stderr].filter(Boolean).join(' | '), code: result.exitCode === 0 ? -2 : (result.exitCode ?? -1) };
         }
+        return { stdout: data.text, stderr: result.stderr, code: result.exitCode ?? -1 };
     }
-    async verifyHealth(node, appName) {
-        // Simple health check: ping remote daemon and check app status
+    async scpTransfer(target, localPath, remotePath) {
+        await this.execution.uploadFile(sshTargetOf(target), localPath, remotePath);
+    }
+    /**
+     * Ask the node's daemon to run the app whose artifact just landed.
+     *
+     * This was `omnitron restart <app> 2>/dev/null || true`, and the `|| true`
+     * was hiding the whole of the remote deployment's last mile.
+     *
+     * Measured on the test node, running the command by hand:
+     *
+     *     omnitron restart main  →  Failed: Unknown app: main
+     *
+     * The node's daemon has no app definitions — its own log says `No projects
+     * registered`, and `omnitron status --json` answers `appsTotal: 0` — because
+     * nothing in this deployment ever tells it about the apps. Artifacts land
+     * at `/opt/omnitron/artifacts/<project>/<app>/<version>/` complete with
+     * `config`, `dist` and `package.json`, and a project is "a directory with
+     * omnitron.config.ts", which an artifact is not.
+     *
+     * That gap is not closed here — it is a design question about how a node
+     * learns what to run. What is closed here is its INVISIBILITY: the shell
+     * discarded the only sentence that said so, and the caller reported
+     * `Deployment successful`.
+     */
+    /**
+       * Ship a built frontend to a node and say where it landed.
+       *
+       * The gateway's config travels inside the provisioning RPC — six files, 65
+       * KB. A frontend build is 32 MB, and an RPC argument is the wrong shape for
+       * that: it is held whole in memory on both sides and blocks the call it
+       * rides on. So it goes the way artifacts already go — tar over SSH, extract
+       * on the far side — which is the mechanism this class exists for.
+       *
+       * Idempotent by content: the archive's hash names the directory, so a build
+       * that has not changed is transferred once and every later provisioning pass
+       * finds it already there. A build that HAS changed lands beside the old one
+       * and the gateway's spec hash changes with the path, which is what makes the
+       * container pick it up.
+       */
+    async uploadStaticBundle(target, localDir, remoteRoot, 
+    /**
+     * The stack's link inside `remoteRoot`, `current-<project>-<stack>`
+     * (`gatewayStaticLinkName`). Given, the delivery points it at the build it
+     * just unpacked — which is what lets the gateway mount the ROOT once and
+     * never be recreated for a new build. Omitted, nothing is linked and the
+     * delivery is exactly what it was.
+     */
+    link) {
+        const { createHash } = await import('node:crypto');
+        const fsp = await import('node:fs/promises');
+        const os = await import('node:os');
+        const path = await import('node:path');
+        const stat = await fsp.stat(localDir).catch(() => null);
+        if (!stat?.isDirectory())
+            throw new Error(`No such directory to serve: ${localDir}`);
+        const staging = await fsp.mkdtemp(path.join(os.tmpdir(), 'omnitron-static-'));
+        const archive = path.join(staging, 'static.tar.gz');
+        try {
+            // `-C` so the archive holds the directory's CONTENTS, not a path from
+            // this machine: the node mounts what is inside, and a leading
+            // `apps/portal/dist/` would put every file one level too deep.
+            // `COPYFILE_DISABLE=1`: macOS `tar` writes an AppleDouble sidecar for
+            // every file with an extended attribute, and this archive is unpacked
+            // straight into the gateway's web root. Measured there:
+            // `._index.html`, `._assets`, `._docs-static` — 163 bytes each, served
+            // to anyone who asks for them.
+            // `--no-xattrs` for the same reason as `COPYFILE_DISABLE`, one layer
+            // down. macOS stamps `com.apple.provenance` on files it has seen, and
+            // bsdtar carries it as a `LIBARCHIVE.xattr.*` header that GNU tar on
+            // the node does not know. Measured: 29 319 characters of
+            // `tar: Ignoring unknown extended header keyword` in one error, with
+            // the four lines that named the actual failure at the very end. The
+            // warnings were harmless and the noise was not — it is what a reader
+            // sees first, and it says nothing.
+            // `| gzip -n` rather than tar's own `-z`, for the digest below.
+            //
+            // gzip writes the time of compression into its header, so the same
+            // directory packed twice a second apart produces two different sums —
+            // and this sum is what the remote directory is NAMED after. Measured:
+            // two archives of one file, one second apart, gave 2 distinct sha256
+            // with `-czf` and 1 with `| gzip -n`.
+            //
+            // Two consequences, and the second is the load-bearing one. A peer
+            // repacking the bundle to check what the node holds could compare
+            // only its LENGTH, because the sums were guaranteed to differ — an
+            // «almost proof» where a proof was one flag away. And the skip on an
+            // unchanged build never fired: every deployment minted a new digest,
+            // so `test -d <digest>` was always false and the same bytes crossed
+            // the link again. (omni-74, after their size-matched, sum-mismatched
+            // comparison.)
+            const packed = await this.execution.exec(`COPYFILE_DISABLE=1 tar --no-xattrs -cf - -C ${shellEscape(localDir)} . | gzip -n > ${shellEscape(archive)}`);
+            // Read, and judged whole, BEFORE it is summed. `exec` never throws, and
+            // this pipeline's status is gzip's alone, so a `tar` that stopped
+            // halfway left a short archive that was then summed, sent, checked
+            // against its own short sum — and failed only on the node: «gzip:
+            // stdin: unexpected end of file» (daos-202609221949-23dce2f6 to test,
+            // 2026-09-22). The sum proves the file ARRIVED as it left; nothing
+            // proved it left whole.
+            const content = await fsp.readFile(archive).catch(() => null);
+            const verdict = content ? wholeTarGz(content) : 'no archive was written';
+            if (packed.exitCode !== 0 || verdict !== true) {
+                const words = (packed.stderr || '').trim().split('\n').slice(-3).join(' | ');
+                throw new Error(`Packing ${localDir} for the node failed (exit ${packed.exitCode}; ${verdict === true ? 'archive whole' : verdict})` +
+                    (words ? `: ${words}` : '') +
+                    ' — nothing was sent');
+            }
+            const bytes = content.length;
+            const digest = createHash('sha256').update(content).digest('hex').slice(0, 16);
+            const remoteDir = `${remoteRoot}/${digest}`;
+            const remoteFile = `${remoteDir}.tar.gz`;
+            // Already there: the same build was sent before. Transferring it again
+            // costs 32 MB over a link that may be an SSH tunnel, for no change.
+            //
+            // The question is «is the BUNDLE there», and `test -d` answered «is
+            // the DIRECTORY there» — two different statements, and the first
+            // deployment made them disagree. `mkdir -p` had succeeded, the
+            // transfer had landed short, and `tar` had died on the truncated
+            // stream: the directory existed, holding nothing or part of a tree.
+            // The next run would have seen it, answered «the node already has
+            // this build», and skipped the delivery — reporting a saved transfer
+            // where there was a missing portal. (omni-74 saw this coming before
+            // the second attempt reached the step.)
+            //
+            // So readiness is a RECORD, written last, exactly as the app
+            // artifacts do it with `.artifact-sha256`: a marker that cannot
+            // outlive a failed unpack because it is written after one succeeds.
+            //
+            // Not `test -s index.html` either, which was the first idea: that is
+            // a sign the CONTENT is present, and half a tree contains an
+            // index.html too. The marker is a sign the OPERATION completed.
+            // (omni-74 drew that distinction.)
+            // It sits beside the directory rather than inside it — everything
+            // inside is served to anyone who asks.
+            const marker = `${remoteDir}.delivered`;
+            const exists = await this.sshExec(target, `test -s ${shellEscape(marker)} && test -d ${shellEscape(remoteDir)} && echo yes || echo no`).catch(() => 'no');
+            //
+            // What «already there» excuses is the TRANSFER, not the deployment.
+            // This used to `return` here, and under the old mount that was exact:
+            // the gateway mounted the build directory itself, so a container
+            // recreated with this digest as its source served this digest, and
+            // nothing else had to happen. Serving through
+            // `current-<project>-<stack>` moves that fact into the link, and a
+            // deployment that does not point the link has not deployed.
+            //
+            // The case is a ROLLBACK, and only a rollback: forward releases carry
+            // new content, so a new digest, so this is false. Deploy A, deploy B,
+            // then ask for A again — A's bundle and its marker are still on the
+            // node, the early return fired, and the link stayed on B. The gateway
+            // went on serving B while the release said A, with nothing anywhere
+            // saying so. Which is the version you least want wrong.
+            const alreadyThere = exists.trim() === 'yes';
+            // Whatever a previous attempt left half-written is not a starting
+            // point: unpacking onto it would merge two trees.
+            if (!alreadyThere) {
+                await this.sshExec(target, `rm -rf ${shellEscape(remoteDir)} ${shellEscape(marker)}`).catch(() => undefined);
+                await this.sshExec(target, `mkdir -p ${shellEscape(remoteDir)}`);
+                await this.execution.uploadFile(sshTargetOf(target), archive, remoteFile);
+                // `uploadFile` already refuses a file that landed SHORT. That catches
+                // the likely failure and not the other one: a transfer that completes
+                // and corrupts in the middle has the right length and the wrong
+                // bytes, and `tar` would report it three steps later as a broken
+                // archive rather than as a bad transfer.
+                //
+                // The sum costs nothing to obtain here, because it is already the
+                // name of the directory this is going into. `sha256sum` on the node,
+                // `shasum -a 256` where that is what exists — asking for both and
+                // taking whichever answers beats assuming the platform.
+                // (omni-74's objection: the size check was one class of failure, and
+                // the stronger check needed no new information.)
+                const sum = await this.sshExec(target, `sha256sum ${shellEscape(remoteFile)} 2>/dev/null || shasum -a 256 ${shellEscape(remoteFile)}`);
+                const landed = sum.trim().split(/\s+/)[0]?.slice(0, 16);
+                if (landed !== digest) {
+                    await this.sshExec(target, `rm -f ${shellEscape(remoteFile)}`).catch(() => undefined);
+                    throw new Error(`The static bundle arrived corrupt: sent sha256 ${digest}, the node has ${landed ?? 'no readable sum'}. ` +
+                        'Nothing was unpacked and the partial file was removed.');
+                }
+                await this.sshExec(target, `tar -xzf ${shellEscape(remoteFile)} -C ${shellEscape(remoteDir)} && rm -f ${shellEscape(remoteFile)}`);
+            }
+            // A tab opened before this deployment asks for chunks of the build it
+            // loaded; they are carried into this one, and builds nobody serves go
+            // (`static-carry-forward.ts`). Before the marker: it is part of what
+            // «delivered» means now.
+            await this.carryPreviousBuildForward(target, remoteRoot, remoteDir);
+            // AFTER the carry-forward, because that reads the link to find the build
+            // open tabs loaded — the one being replaced.
+            if (link)
+                await this.pointStaticLinkAt(target, remoteRoot, link, digest);
+            // Last, and only now: the unpack is what this records.
+            if (!alreadyThere)
+                await this.sshExec(target, `printf %s ${shellEscape(digest)} > ${shellEscape(marker)}`);
+            this.logger.info({ host: target.host, remoteDir, bytes: alreadyThere ? 0 : bytes, transferred: !alreadyThere }, alreadyThere ? 'The node already had this build — pointed at it' : 'Static bundle delivered to the node');
+            return { remoteDir, bytes: alreadyThere ? 0 : bytes };
+        }
+        finally {
+            await fsp.rm(staging, { recursive: true, force: true }).catch(() => undefined);
+        }
+    }
+    /**
+     * Point a stack's `current-<project>-<stack>` at the build just unpacked.
+     *
+     * This is the whole reason the gateway stops being recreated on a
+     * deployment: it mounts the static ROOT, whose path never changes, and
+     * serves `/var/www/portal/<link>`. nginx resolves `root` per request and
+     * this config sets no `open_file_cache`, so the swap is picked up with no
+     * reload and no restart. What it replaces cost **1 s of 000** on the
+     * gateway, measured on the release-4 deployment (81 s total).
+     *
+     * RELATIVE, so it resolves inside the container as well as on the host — the
+     * host path is not the container path.
+     *
+     * And swapped with `mv -T`, never `ln -sfn`: `-f` unlinks before it
+     * symlinks, and a request arriving in that window gets a 404 from a web root
+     * that does not exist. `rename(2)` has no window at all. The temporary name
+     * carries the digest so two deliveries racing on one node cannot collide on
+     * it.
+     */
+    async pointStaticLinkAt(target, root, link, digest) {
+        const tmp = `${root}/${link}.${digest}.tmp`;
+        await this.sshExec(target, `ln -s ${shellEscape(digest)} ${shellEscape(tmp)} && mv -T ${shellEscape(tmp)} ${shellEscape(`${root}/${link}`)}`);
+        this.logger.info({ host: target.host, link, digest }, 'The gateway now serves this build through its link');
+    }
+    /**
+     * Hard-link the assets of the build the running gateways serve into the
+     * one just unpacked, and remove builds nobody serves
+     * (`static-carry-forward.ts`). Never fails the delivery: without it an open
+     * tab reloads onto the new build instead of continuing, which the portal
+     * handles; with a failed delivery nobody gets a portal at all.
+     */
+    async carryPreviousBuildForward(target, root, fresh) {
+        try {
+            const { GATEWAY_MOUNTS_COMMAND, readGatewayMounts, carryForwardScript } = await import('./static-carry-forward.js');
+            const mounts = readGatewayMounts(await this.sshExec(target, GATEWAY_MOUNTS_COMMAND).catch(() => ''));
+            const script = carryForwardScript({ root, fresh, serving: mounts.serving, mounted: mounts.mounted, keep: 5 });
+            const said = await this.sshExec(target, `bash -c ${shellEscape(script)}`, 300_000);
+            this.logger.info({ host: target.host, fresh, serving: mounts.serving, said: said.trim() }, 'The build open tabs loaded is carried forward; builds nobody serves are removed');
+        }
+        catch (err) {
+            this.logger.warn({ host: target.host, fresh, error: err.message.slice(0, 400) }, 'Could not carry the previous build forward — open tabs will reload onto the new one');
+        }
+    }
+    async signalRemoteDaemon(target, appName) {
+        try {
+            // No `2>/dev/null`, no `|| true`: the failure IS the information.
+            const out = await this.sshExec(target, `omnitron restart ${shellEscape(appName)} 2>&1`);
+            const failed = /unknown app|failed|not found|no such/i.test(out);
+            if (failed) {
+                this.logger.error({ host: target.host, app: appName, detail: out.trim().slice(0, 300) }, 'The node refused to start this app — its artifact is installed and its daemon does not know the app');
+                return { ok: false, detail: out.trim().slice(0, 300) };
+            }
+            return { ok: true, detail: out.trim().slice(0, 200) };
+        }
+        catch (err) {
+            const detail = err.message;
+            this.logger.error({ host: target.host, app: appName, error: detail }, 'Could not reach the node to start this app');
+            return { ok: false, detail };
+        }
+    }
+    /**
+     * Record that the app now runs this artifact — written only after it came
+     * up healthy on it, never at unpack time, so a deployment interrupted
+     * between installing and starting leaves the old record and the next one
+     * ships and starts again. No checksum, no record: the next deployment then
+     * cannot tell, and «cannot tell» deploys.
+     *
+     * A write that fails is said and not fatal: the app runs; the next
+     * deployment transfers once more than it needed to.
+     */
+    async recordRunningArtifact(target, remotePath, app, checksum) {
+        if (!checksum)
+            return;
+        try {
+            await this.sshExec(target, `printf %s ${shellEscape(checksum)} > ${shellEscape(`${remotePath}/${ARTIFACT_CHECKSUM_FILE}`)}`);
+        }
+        catch (err) {
+            this.logger.warn({ node: target.host, app, error: err.message }, 'The app runs its new artifact and the node could not record which — the next deployment ships it again');
+        }
+    }
+    /**
+     * The apps this node reports as online, by their bare names.
+     *
+     * One question for the whole node. The per-app check beside this one asks
+     * the same command, and asking it six times to decide six restarts is six
+     * SSH round trips for one answer.
+     *
+     * An empty set on any failure: not knowing what is running is not a claim
+     * that nothing is, and the caller's fallback — restart it — is the safe
+     * reading of "we could not tell".
+     */
+    async appsOnline(target, project) {
         try {
-            const status = await this.sshExec(node, `omnitron status --json 2>/dev/null || echo "{}"`, 15_000);
-            const parsed = JSON.parse(status);
-            if (parsed?.apps) {
-                const app = parsed.apps.find((a) => a.name === appName);
-                if (app?.status === 'online')
-                    return;
+            const answer = await this.askNodeStatus(target);
+            const status = readNodeStatus(answer.raw);
+            if (!status) {
+                this.logger.warn({ node: target.host, project, said: answer.words }, 'The node did not answer what it is running — every app will be restarted');
+                return new Set();
             }
+            return new Set(status.apps
+                .filter((app) => app.status === 'online' && typeof app.name === 'string')
+                .map((app) => {
+                const name = app.name;
+                return name.includes('/') ? name.slice(name.lastIndexOf('/') + 1) : name;
+            }));
+        }
+        catch (err) {
+            this.logger.warn({ node: target.host, project, error: err.message }, 'Could not read what this node is running — every app will be restarted');
+            return new Set();
         }
-        catch {
-            // Health check is best-effort
+    }
+    /**
+     * Whether the app is actually running on the node.
+     *
+     * This returned `void` on every path — a match, a mismatch, a parse failure,
+     * an unreachable node — under a comment reading "Health check is
+     * best-effort". A verification that cannot fail is not a verification, and
+     * it is worse than none: its presence in the sequence is what persuades a
+     * reader that the deployment was checked.
+     *
+     * The reading itself is in `readNodeHealth`, where the test can reach it.
+     */
+    async verifyHealth(target, appName, project) {
+        const answer = await this.askNodeStatus(target);
+        if (answer.code === UNREADABLE) {
+            return { online: false, detail: `could not read the node's status: ${answer.words}` };
         }
+        const health = readNodeHealth(answer.raw, appName, project);
+        return health.online || !answer.words ? health : { online: false, detail: `${health.detail}: ${answer.words}` };
     }
-    buildSSHArgs(node) {
-        const user = node.ssh?.user ?? 'root';
-        const port = node.ssh?.port ?? 22;
-        const args = [
-            '-o', 'StrictHostKeyChecking=accept-new',
-            '-o', 'ConnectTimeout=10',
-            '-o', 'BatchMode=yes',
-            '-p', String(port),
-        ];
-        if (node.ssh?.privateKey) {
-            args.push('-i', node.ssh.privateKey);
+    /**
+     * `omnitron status --json` on a node: its stdout is DATA, so it comes
+     * through the data channel — past the transport's masker, which rewrites a
+     * value such as `"token": …` into something that is no longer JSON — and
+     * its stderr stays words, masked, and separate. `2>&1` used to fold the two
+     * together, so an answer that could not be parsed was quoted into the log
+     * whole. Three readers: the daemon check before a restart, `appsOnline`,
+     * `verifyHealth`. None logs `raw`.
+     *
+     * `UNREADABLE` when the node could not be asked at all, with the reason as
+     * the words.
+     */
+    async askNodeStatus(target) {
+        try {
+            const answer = await this.readFromNode(target, 'omnitron status --json', 15_000);
+            return { raw: answer.stdout, words: answer.stderr.trim().split('\n')[0] ?? '', code: answer.code };
+        }
+        catch (err) {
+            return { raw: '', words: err.message, code: UNREADABLE };
         }
-        args.push(`${user}@${node.host}`);
-        return args;
     }
     // ===========================================================================
     // Private — Progress
     // ===========================================================================
     emitProgress(node, app, status, progress, message) {
         const event = { node, app, status, progress, message };
+        this.lastProgress.set(`${node}\u0000${app}`, { ...event, at: new Date().toISOString() });
         for (const handler of this.progressHandlers) {
             try {
                 handler(event);
@@ -328,5 +1767,15 @@ export default {
             }
         }
     }
+    /**
+     * What each app on each node is doing, newest first.
+     *
+     * For a console that polls. Returned as a list rather than the map so a
+     * caller cannot hold a reference that keeps changing underneath it while
+     * it renders.
+     */
+    getProgress() {
+        return [...this.lastProgress.values()].sort((a, b) => (a.at < b.at ? 1 : a.at > b.at ? -1 : 0));
+    }
 }
 //# sourceMappingURL=remote-deployer.service.js.map