@omnitron-dev/omnitron 0.3.0 → 0.3.1

package/dist/services/remote-deployer.service.js

@@ -49,18 +49,21 @@
  * to put them.
  */
 import { PLATFORM_PROBE, parsePlatformProbe, planProvisioning, describePlan, } from './remote-provisioner.js';
-import { installSteps, activateSteps, pruneSteps } from './bundle-builder.js';
+import { installSteps, installPreflightCommand, versionDir, activateSteps, pruneSteps } from './bundle-builder.js';
 import { reachabilityRule, reachabilityCommand } from '../infrastructure/gateway-reachability.js';
 import { readNodeHealth, readNodeStatus } from '../project/node-app-health.js';
 import { decideRedeploy, decideNodeStackStart, decideSlaveDaemonRestart, artifactChanged, readChecksumCommand, parseRecordedChecksum, ARTIFACT_CHECKSUM_FILE, NODE_CONFIG_HASH_FILE, } from './redeploy-decision.js';
 import { deliveredProbeCommand, parseDeliveredProbe, checkDelivered, } from '../release/delivered.js';
 import crypto from 'node:crypto';
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { assertControlledDaemonPort, assertServingBundle, bundleProbeCommand, contractPreflightCommand, executeHostUpgrade, hookCommand, inventoryCommand, planHostUpgrade, receiptCommand, verifyRestoredInventory, updateContractBindingsCommand, runtimeProbeCommand, runtimeStartCommand, runtimeStopCommand, appHealthCommand, pendingReceiptsCommand, recoverHostUpgrade, stoppedDaemonCommand, } from './host-upgrade.js';
 import { gunzipSync } from 'node:zlib';
 import { shellEscape } from '../shared/shell-escape.js';
 import { fromDataChannel, throughDataChannel } from '../execution/data-channel.js';
 /** A node that could not be asked at all — the SSH call itself failed. */
 const UNREADABLE = -3;
 import { withNodeLeases } from './node-deploy-lease.js';
+import { hostOperationBarrierCommand } from './host-operation-barrier.js';
 import { migrationOutput } from './migration-output.js';
 /**
  * A name that is safe to place in a remote filesystem path.
@@ -231,6 +234,7 @@ export function wholeTarGz(archive) {
 export class RemoteDeployer {
     logger;
     execution;
+    deploymentLeases = new AsyncLocalStorage();
     /** Active deployment progress handlers */
     progressHandlers = [];
     /**
@@ -417,7 +421,7 @@ export class RemoteDeployer {
             //     Titan package identity mismatch detected between omnitron daemon
             //     and app 'geo'. Daemon resolves @omnitron-dev/titan from
             //     /opt/omnitron/versions/…; App resolves … from /opt/omnitron/
-            //     artifacts/daos/geo/0.0.1/node_modules/@omnitron-dev/titan
+            //     artifacts/sample/geo/0.0.1/node_modules/@omnitron-dev/titan
             //
             // That is the correct refusal, and the answer is not to install less:
             // the vendored copies are what let `npm install` resolve a coherent
@@ -449,7 +453,7 @@ export class RemoteDeployer {
             // puts the link INSIDE it, leaving a directory with no package.json
             // that node reports as a missing `index.js` — a message about a file
             // that was never the problem.
-            const daemonRoot = '/opt/omnitron/current';
+            const daemonRoot = (await this.inspectInstalledBundle(target)).root;
             await this.sshExec(target, `set -e; APP=${shellEscape(`${remotePath}/node_modules/@omnitron-dev`)}; ` +
                 `mkdir -p "$APP"; ` +
                 `for p in ${shellEscape(daemonRoot)}/node_modules/@omnitron-dev/*; do ` +
@@ -480,7 +484,7 @@ export class RemoteDeployer {
             // 5b. No record yet. The next deployment reads `.artifact-sha256` as
             // «the app runs this», and at this point it does not: the files are
             // unpacked, the process still runs what it started with. The record
-            // used to be written here, and on daos/test (2026-09-24) a deployment
+            // used to be written here, and on sample/test (2026-09-24) a deployment
             // that stalled after installing three artifacts left them recorded;
             // the retry read «installed, running» and left all three on the code
             // they had started with an hour earlier. It is written once the app
@@ -746,7 +750,7 @@ export class RemoteDeployer {
      *
      * Measured on the test node after six apps installed and five started:
      * every one of the six databases existed and held ZERO tables. Five apps
-     * came up anyway, because they touch no table at boot; `paysys` does, and
+     * came up anyway, because they touch no table at boot; `payments` does, and
      * said so —
      *
      *     Failed to call @PostConstruct 'bootstrap' on 'PlatformRevenueService':
@@ -757,7 +761,7 @@ export class RemoteDeployer {
      * refuses to start.
      *
      * The artifact already carries what is needed: `dist/database/migrate.js`
-     * beside `dist/database/migrations/` — 160 compiled files for paysys. The
+     * beside `dist/database/migrations/` — 160 compiled files for payments. The
      * same build the app itself runs from, so there is no second copy of the
      * schema to drift.
      *
@@ -780,7 +784,7 @@ export class RemoteDeployer {
      * running the batch again after the cause is fixed applies exactly what
      * is missing. That holds because `HardenedMigrationRunner` defaults
      * `useTransactions` to true (`titan-database/src/migration/
-     * hardened-runner.ts:171`) and no daos `migrate.ts` overrides it — it is
+     * hardened-runner.ts:171`) and no sample `migrate.ts` overrides it — it is
      * conditional in the runner (`:502`), so a migrator that turned it off
      * would make a refused deployment leave a half-applied migration behind.
      */
@@ -815,9 +819,9 @@ export class RemoteDeployer {
             this.emitProgress(`${target.host}:${target.daemonPort ?? 9700}`, entry.app, 'installing', 75, 'Applying database migrations...');
             // Both spellings, because the migrators read the second one.
             //
-            // `apps/paysys/src/database/migrate.ts` declares
-            // `envPrefix: 'PAYSYS__DATABASE'` and builds its connection from
-            // `PAYSYS__DATABASE__HOST` / `__PORT` / `__USER` / `__PASSWORD` /
+            // `apps/payments/src/database/migrate.ts` declares
+            // `envPrefix: 'PAYMENTS__DATABASE'` and builds its connection from
+            // `PAYMENTS__DATABASE__HOST` / `__PORT` / `__USER` / `__PASSWORD` /
             // `__DATABASE`. Given only `DATABASE_URL` it falls back to its own
             // defaults and reports `password authentication failed for user
             // "postgres"` — against a password that is correct and that it never
@@ -1194,17 +1198,20 @@ export class RemoteDeployer {
     async installBundle(target, archivePath, version, prefix = '/opt/omnitron') {
         const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
         const layout = { prefix, version };
-        // Under the prefix, not `/tmp`: a bundle is tens of megabytes and `/tmp`
-        // is a tmpfs on many hosts, where a large transfer competes with memory.
-        const remoteArchive = `${prefix}/releases/${version}.tar.gz`;
         try {
+            // Reject path traversal before any connection, upload or filesystem mutation.
+            versionDir(layout);
+            const fs = await import('node:fs/promises');
+            const archiveSha256 = crypto.createHash('sha256').update(await fs.readFile(archivePath)).digest('hex');
+            // Content-addressed transfer cannot replace an earlier archive with different bytes.
+            const remoteArchive = `${prefix}/releases/${version}.${archiveSha256}.tar.gz`;
             this.emitProgress(nodeKey, '*', 'pending', 0, 'Connecting via SSH...');
             await this.verifySSH(target);
             this.emitProgress(nodeKey, '*', 'transferring', 10, `Transferring ${version}...`);
-            await this.sshExec(target, `mkdir -p ${shellEscape(`${prefix}/releases`)}`);
+            await this.sshExec(target, installPreflightCommand(layout));
             await this.scpTransfer(target, archivePath, remoteArchive);
             let progress = 30;
-            for (const step of installSteps(layout, remoteArchive)) {
+            for (const step of installSteps(layout, remoteArchive, archiveSha256)) {
                 this.emitProgress(nodeKey, '*', 'extracting', progress, step.what);
                 const output = await this.sshExec(target, step.command, step.timeoutMs);
                 if (step.what.includes('runs')) {
@@ -1238,83 +1245,171 @@ export class RemoteDeployer {
      * left to expire by itself. A node with no gateway activates as before.
      */
     async activateBundleHeld(target, version, prefix = '/opt/omnitron', keepVersions = 3) {
-        const { runningGatewaySites, MaintenanceLock, DEFAULT_MAINTENANCE_TIMING } = await import('./maintenance-lock.js');
-        const run = this.leaseRunner(target);
-        const locks = [];
-        for (const site of await runningGatewaySites(run, `omnitron ${version}`)) {
-            const lock = await MaintenanceLock.open(site, this.logger, { ...DEFAULT_MAINTENANCE_TIMING, probeWithinMs: 180_000 });
-            if (lock)
-                locks.push(lock);
-        }
-        try {
-            const activated = await this.activateBundle(target, version, prefix, keepVersions);
-            while (locks.length > 0) {
-                const lifted = await locks.shift().liftWhenAnswering(['/', '/api/main/health']);
-                if (!lifted.lifted) {
-                    this.logger.error({ host: target.host, version, failed: lifted.failed }, 'After the upgrade the platform does not answer through its gateway — maintenance expires by itself');
+        assertControlledDaemonPort(target.daemonPort);
+        return this.underLease(target, `held host upgrade to ${version}`, async () => {
+            await this.assertNoPendingHostOperations(target, prefix);
+            const { runningGatewaySites, MaintenanceLock, DEFAULT_MAINTENANCE_TIMING } = await import('./maintenance-lock.js');
+            const run = (script) => this.sshExecData(target, script, 30_000);
+            const locks = [];
+            for (const site of await runningGatewaySites(run, `omnitron ${version}`)) {
+                const lock = await MaintenanceLock.open(site, this.logger, { ...DEFAULT_MAINTENANCE_TIMING, probeWithinMs: 180_000 });
+                if (lock)
+                    locks.push(lock);
+            }
+            try {
+                let activated = await this.activateBundle(target, version, prefix, keepVersions, async () => {
+                    for (const lock of locks) {
+                        const failed = await lock.probe(['/', '/api/main/health']);
+                        if (failed.length)
+                            throw new Error(`Gateway verification failed: ${failed.join(', ')}`);
+                    }
+                });
+                while (locks.length > 0) {
+                    const lifted = await locks.shift().liftWhenAnswering(['/', '/api/main/health']);
+                    if (!lifted.lifted) {
+                        activated = false;
+                        this.logger.error({ host: target.host, version, failed: lifted.failed }, 'After the upgrade the platform does not answer through its gateway — maintenance expires by itself');
+                    }
                 }
+                return activated;
             }
-            return activated;
-        }
-        finally {
-            for (const lock of locks)
-                await lock.lift('failed').catch(() => undefined);
-        }
+            finally {
+                for (const lock of locks)
+                    await lock.lift('failed').catch(() => undefined);
+            }
+        });
     }
-    /**
-     * Make an installed version the one the node runs, and restart into it.
-     *
-     * Separate from `installBundle` for the reason the layout exists: this is
-     * the only step that changes what runs.
-     */
-    async activateBundle(target, version, prefix = '/opt/omnitron', keepVersions = 3) {
-        const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
-        const layout = { prefix, version };
-        try {
-            for (const step of activateSteps(layout)) {
-                this.emitProgress(nodeKey, '*', 'restarting', 40, step.what);
-                await this.sshExec(target, step.command, step.timeoutMs);
+    /** Check all durable operator workflows; a recovery can allow only its exact active journal. */
+    async assertNoPendingHostOperations(target, prefix = '/opt/omnitron', allowance) {
+        await this.sshExecData(target, hostOperationBarrierCommand(prefix, allowance));
+    }
+    /** Require a settled host and the actual serving immutable daemon build. */
+    async inspectInstalledBundle(target, prefix = '/opt/omnitron', allowance) {
+        assertControlledDaemonPort(target.daemonPort);
+        await this.assertNoPendingHostOperations(target, prefix, allowance);
+        const pending = JSON.parse(await this.sshExecData(target, pendingReceiptsCommand(prefix)));
+        if (pending.length)
+            throw new Error(`Unfinished host upgrade ${pending.map(r => r.id).join(', ')}; recover it before deploying an app`);
+        const bundle = JSON.parse(await this.sshExecData(target, bundleProbeCommand(prefix)));
+        const status = JSON.parse(await this.sshExecData(target, inventoryCommand(bundle, false)));
+        assertServingBundle(bundle, status);
+        await this.sshExecData(target, runtimeProbeCommand(bundle, status.pid));
+        return bundle;
+    }
+    async activateBundle(target, version, prefix = '/opt/omnitron', keepVersions = 3, verifyPlatform, allowance) {
+        assertControlledDaemonPort(target.daemonPort);
+        if (!/^[A-Za-z0-9][A-Za-z0-9._+-]*$/.test(version) || version.includes('..'))
+            throw new Error('Invalid installed bundle version');
+        if (!Number.isSafeInteger(keepVersions) || keepVersions < 1)
+            throw new Error('keepVersions must be a positive integer');
+        return this.underLease(target, `host upgrade to ${version}`, async () => {
+            const nodeKey = `${target.host}:${target.daemonPort ?? 9700}`;
+            try {
+                await this.assertNoPendingHostOperations(target, prefix, allowance);
+                const pending = JSON.parse(await this.sshExecData(target, pendingReceiptsCommand(prefix)));
+                if (pending.length) {
+                    if (pending.length !== 1)
+                        throw new Error('Multiple unfinished host upgrades; recover an explicit receipt ID first');
+                    const recovered = await this.recoverBundleUpgrade(target, pending[0].id, prefix, verifyPlatform, allowance);
+                    this.logger.warn({ receipt: pending[0].id, recovered }, 'Recovered the unfinished host upgrade; retry the requested upgrade separately');
+                    return false;
+                }
+                const previous = await this.inspectInstalledBundle(target, prefix, allowance);
+                const candidate = JSON.parse(await this.sshExecData(target, bundleProbeCommand(prefix, version)));
+                await this.sshExec(target, `${shellEscape(`${candidate.root}/dist/cli/omnitron.js`)} --version`, 120_000);
+                const inventory = JSON.parse(await this.sshExecData(target, inventoryCommand(previous, true)));
+                inventory.runtime = JSON.parse(await this.sshExecData(target, runtimeProbeCommand(previous, inventory.status.pid)));
+                const plan = planHostUpgrade(previous, candidate, inventory);
+                await this.sshExec(target, contractPreflightCommand(plan.apps));
+                for (const app of plan.apps) {
+                    await this.sshExec(target, hookCommand(app, previous, 'verify'), 330_000);
+                    if (plan.online.includes(app.name))
+                        await this.sshExec(target, appHealthCommand(app, previous), 330_000);
+                }
+                const id = `${new Date().toISOString().replace(/[:.]/g, '-')}-${crypto.randomUUID()}`;
+                const receiptPath = `${prefix}/upgrade-receipts/${id}.json`;
+                const receipt = await executeHostUpgrade(plan, id, this.hostUpgradeActions(target, prefix, keepVersions, plan, verifyPlatform));
+                const ok = receipt.outcome === 'upgraded';
+                this.emitProgress(nodeKey, '*', ok ? 'success' : 'failed', 100, `${receipt.outcome}; recovery receipt: ${receiptPath}`);
+                this.logger[ok ? 'info' : 'error']({ host: target.host, version, receipt: receiptPath, outcome: receipt.outcome }, 'Controlled host upgrade finished');
+                return ok;
             }
-            this.emitProgress(nodeKey, '*', 'restarting', 60, 'Restarting the daemon into the new version');
-            // By absolute path into `current`, not by name. A bare `omnitron` is
-            // whatever the node's PATH resolves — which, until the step above ran,
-            // was a different installation entirely. Naming the copy we just
-            // activated is the difference between restarting the new version and
-            // restarting whatever was there.
-            const cli = shellEscape(`${prefix}/current/dist/cli/omnitron.js`);
-            await this.sshExec(target, `${cli} down 2>/dev/null; ${cli} up --no-infra`, 180_000).catch((err) => {
-                this.logger.warn({ host: target.host, error: err.message }, 'Restart command did not return cleanly — verifying anyway');
-            });
-            this.emitProgress(nodeKey, '*', 'verifying', 80, 'Verifying the daemon answers');
-            const started = await this.awaitDaemon(target, 120_000);
-            if (!started.ok) {
-                // Reported, and NOT rolled back automatically. A daemon that will not
-                // start is a decision for whoever is watching: the previous version
-                // is still on disk and one `ln -sfn` away, and guessing that a
-                // rollback is wanted can be as wrong as guessing it is not.
-                this.logger.error({ host: target.host, version, detail: started.detail }, 'The new version did not answer — the previous one is still installed');
-                this.emitProgress(nodeKey, '*', 'failed', 80, `${version} did not answer: ${started.detail}. Roll back with ` +
-                    `\`ln -sfn ${prefix}/versions/<previous> ${prefix}/current\` and restart.`);
+            catch (error) {
+                const message = error.message;
+                this.emitProgress(nodeKey, '*', 'failed', 0, message);
+                this.logger.error({ host: target.host, version, error: message }, 'Host upgrade refused or preparation failed');
                 return false;
             }
-            // Only once the new version is serving. Pruning before this could
-            // remove the version a rollback needs.
-            for (const step of pruneSteps(layout, keepVersions)) {
-                await this.sshExec(target, step.command, step.timeoutMs).catch((err) => {
-                    // Retention failing is not the upgrade failing.
-                    this.logger.warn({ host: target.host, error: err.message }, 'Version retention failed');
-                });
-            }
-            this.emitProgress(nodeKey, '*', 'success', 100, `Running ${version} — ${started.detail}`);
-            this.logger.info({ host: target.host, version, daemon: started.detail }, 'Node upgraded');
-            return true;
-        }
-        catch (err) {
-            const message = err.message;
-            this.emitProgress(nodeKey, '*', 'failed', 0, message);
-            this.logger.error({ host: target.host, version, error: message }, 'Activation failed');
-            return false;
-        }
+        });
+    }
+    /** Recover a recorded interrupted upgrade under the same node lease; never prunes or changes app release versions. */
+    async recoverBundleUpgrade(target, id, prefix = '/opt/omnitron', verifyPlatform, allowance) {
+        assertControlledDaemonPort(target.daemonPort);
+        return this.underLease(target, `recover host upgrade ${id}`, async () => {
+            await this.assertNoPendingHostOperations(target, prefix, allowance);
+            const records = JSON.parse(await this.sshExecData(target, pendingReceiptsCommand(prefix, id)));
+            if (records.length !== 1 || records[0].id !== id)
+                throw new Error('No unfinished host upgrade with that receipt ID');
+            const receipt = records[0];
+            const plan = planHostUpgrade(receipt.plan.previous, receipt.plan.candidate, receipt.plan.inventory);
+            const installed = JSON.parse(await this.sshExecData(target, bundleProbeCommand(prefix, plan.previous.version)));
+            if (JSON.stringify(installed) !== JSON.stringify(plan.previous))
+                throw new Error('Recorded previous bundle changed; recovery refused');
+            await this.sshExec(target, contractPreflightCommand(plan.apps));
+            const result = await recoverHostUpgrade(receipt, this.hostUpgradeActions(target, prefix, 3, plan, verifyPlatform));
+            this.logger[result.outcome === 'rolled-back' ? 'info' : 'error']({ receipt: `${prefix}/upgrade-receipts/${id}.json`, outcome: result.outcome }, 'Host upgrade recovery finished');
+            return result.outcome === 'rolled-back';
+        });
+    }
+    hostUpgradeActions(target, prefix, keepVersions, plan, verifyPlatform) {
+        return {
+            save: async (receipt) => { await this.sshExec(target, receiptCommand(prefix, receipt)); },
+            stop: async (bundle) => {
+                await this.sshExec(target, runtimeStopCommand(plan.inventory.runtime, bundle), 180_000);
+                await this.sshExec(target, stoppedDaemonCommand(bundle), 30_000);
+            },
+            activate: async (bundle) => {
+                for (const step of activateSteps({ prefix, version: bundle.version }))
+                    await this.sshExec(target, step.command, step.timeoutMs);
+            },
+            rebind: async (app, bundle) => { await this.sshExec(target, hookCommand(app, bundle, 'rebind'), 330_000); },
+            start: async (bundle) => {
+                await this.sshExec(target, runtimeStartCommand(plan.inventory.runtime, plan.previous, bundle, [plan.previous.root, plan.candidate.root]), 180_000);
+            },
+            verify: async (bundle) => {
+                const deadline = Date.now() + 180_000;
+                let lastError;
+                do {
+                    try {
+                        const status = JSON.parse(await this.sshExecData(target, inventoryCommand(bundle, false), 30_000));
+                        verifyRestoredInventory(plan, bundle, status);
+                        const runtime = JSON.parse(await this.sshExecData(target, runtimeProbeCommand(bundle, status.pid)));
+                        if (runtime.kind !== plan.inventory.runtime.kind || (runtime.kind === 'systemd' && plan.inventory.runtime.kind === 'systemd' &&
+                            (runtime.scope !== plan.inventory.runtime.scope || runtime.enabled !== plan.inventory.runtime.enabled)))
+                            throw new Error('Supervisor state was not restored');
+                        for (const app of plan.apps) {
+                            await this.sshExec(target, hookCommand(app, bundle, 'verify'), 330_000);
+                            if (plan.online.includes(app.name))
+                                await this.sshExec(target, appHealthCommand(app, bundle), 330_000);
+                        }
+                        if (verifyPlatform)
+                            await verifyPlatform();
+                        await this.sshExec(target, updateContractBindingsCommand(plan, bundle));
+                        return;
+                    }
+                    catch (error) {
+                        lastError = error;
+                    }
+                    if (Date.now() < deadline)
+                        await new Promise(resolve => setTimeout(resolve, 2_000));
+                } while (Date.now() < deadline);
+                throw new Error(`Host and application verification failed: ${String(lastError)}`);
+            },
+            prune: async (protectedRoots) => {
+                for (const step of pruneSteps({ prefix, version: plan.candidate.version }, keepVersions, protectedRoots))
+                    await this.sshExec(target, step.command, step.timeoutMs);
+            },
+        };
     }
     async verifySSH(target) {
         await this.sshExec(target, 'echo ok', 10_000);
@@ -1338,7 +1433,14 @@ export class RemoteDeployer {
         // Through the data channel: a lease answer carries the holder's record,
         // `{"token":…}`, which the masker rewrote into `"token": [REDACTED]` on
         // every read — a refusal could never say who held the node.
-        return (script) => this.sshExecData(target, script, 30_000);
+        return async (script) => {
+            // Confirmation itself must use the raw channel, never recursively ask
+            // for another confirmation of the lease being acquired or renewed.
+            const result = await this.readFromNodeRaw(target, script, 30_000);
+            if (result.code !== 0)
+                throw new Error(`Lease transport failed: ${result.stderr || result.stdout}`);
+            return result.stdout.trim();
+        };
     }
     /**
      * Run `work` holding the machine's deploy lease — for what changes a node
@@ -1348,15 +1450,32 @@ export class RemoteDeployer {
      */
     async underLease(target, purpose, work) {
         const machine = `${target.host}:${target.sshPort ?? 22}`;
+        const held = this.deploymentLeases.getStore();
+        if (held?.has(machine))
+            return work();
         return withNodeLeases([{ node: machine, machine, run: this.leaseRunner(target) }], purpose, this.logger, async (leases) => {
             if (!leases.has(machine)) {
                 throw new Error(`Could not reach ${machine} to lease it for ${purpose}: ${leases.unreachable.get(machine) ?? 'no answer'}`);
             }
-            return work();
+            const context = new Map(held);
+            context.set(machine, { confirm: step => leases.confirm(machine, step), guard: (command, timeoutMs) => leases.guard(machine, command, timeoutMs) });
+            return this.deploymentLeases.run(context, work);
         });
     }
+    /** Confirm a held lease before the next node operation; outside a lease this is a no-op. */
+    async confirmLease(target, step) {
+        const machine = `${target.host}:${target.sshPort ?? 22}`;
+        await this.deploymentLeases.getStore()?.get(machine)?.confirm(step);
+    }
+    /** The node holds its execution fence for the command lifetime, beyond a lost controller. */
+    guardedCommand(target, command, timeout) {
+        const lease = this.deploymentLeases.getStore()?.get(`${target.host}:${target.sshPort ?? 22}`);
+        return lease ? { command: lease.guard(command, timeout), timeout: timeout + 45_000 } : { command, timeout };
+    }
     async sshExec(target, command, timeout = 60_000) {
-        const result = await this.execution.ssh(sshTargetOf(target), command, { timeout });
+        await this.confirmLease(target, 'remote command');
+        const guarded = this.guardedCommand(target, command, timeout);
+        const result = await this.execution.ssh(sshTargetOf(target), guarded.command, { timeout: guarded.timeout });
         if (result.exitCode !== 0) {
             const detail = result.stderr || result.stdout || `exit ${result.exitCode}`;
             throw new Error(`ssh ${target.username ?? 'root'}@${target.host}: ${detail}`);
@@ -1381,7 +1500,9 @@ export class RemoteDeployer {
      * keep), and a throw would flatten the two into one.
      */
     async runOnNode(target, command, timeoutMs = 600_000) {
-        const result = await this.execution.ssh(sshTargetOf(target), command, { timeout: timeoutMs });
+        await this.confirmLease(target, 'remote command');
+        const guarded = this.guardedCommand(target, command, timeoutMs);
+        const result = await this.execution.ssh(sshTargetOf(target), guarded.command, { timeout: guarded.timeout });
         return { stdout: result.stdout, stderr: result.stderr, code: result.exitCode ?? -1 };
     }
     /**
@@ -1396,6 +1517,11 @@ export class RemoteDeployer {
      * failure with its own code, -2, and its words on stderr — never data.
      */
     async readFromNode(target, command, timeoutMs = 600_000) {
+        await this.confirmLease(target, 'remote data command');
+        const guarded = this.guardedCommand(target, command, timeoutMs);
+        return this.readFromNodeRaw(target, guarded.command, guarded.timeout);
+    }
+    async readFromNodeRaw(target, command, timeoutMs) {
         const result = await this.execution.ssh(sshTargetOf(target), throughDataChannel(command), { timeout: timeoutMs });
         const data = fromDataChannel(result.stdout);
         if (!data.ok) {
@@ -1403,8 +1529,23 @@ export class RemoteDeployer {
         }
         return { stdout: data.text, stderr: result.stderr, code: result.exitCode ?? -1 };
     }
+    /** Use this inside underLease so uploads share the command confirmation boundary. */
+    async uploadToNode(target, localPath, remotePath) {
+        await this.confirmLease(target, 'file upload');
+        const lease = this.deploymentLeases.getStore()?.get(`${target.host}:${target.sshPort ?? 22}`);
+        if (!lease) {
+            await this.execution.uploadFile(sshTargetOf(target), localPath, remotePath);
+            return;
+        }
+        await this.execution.uploadFile(sshTargetOf(target), localPath, remotePath, {
+            runControl: async (command, timeout) => {
+                const result = await this.runOnNode(target, command, timeout);
+                return { stdout: result.stdout, stderr: result.stderr, exitCode: result.code };
+            },
+        });
+    }
     async scpTransfer(target, localPath, remotePath) {
-        await this.execution.uploadFile(sshTargetOf(target), localPath, remotePath);
+        await this.uploadToNode(target, localPath, remotePath);
     }
     /**
      * Ask the node's daemon to run the app whose artifact just landed.
@@ -1499,7 +1640,7 @@ export class RemoteDeployer {
             // this pipeline's status is gzip's alone, so a `tar` that stopped
             // halfway left a short archive that was then summed, sent, checked
             // against its own short sum — and failed only on the node: «gzip:
-            // stdin: unexpected end of file» (daos-202609221949-23dce2f6 to test,
+            // stdin: unexpected end of file» (sample-202609221949-23dce2f6 to test,
             // 2026-09-22). The sum proves the file ARRIVED as it left; nothing
             // proved it left whole.
             const content = await fsp.readFile(archive).catch(() => null);
@@ -1560,7 +1701,7 @@ export class RemoteDeployer {
             if (!alreadyThere) {
                 await this.sshExec(target, `rm -rf ${shellEscape(remoteDir)} ${shellEscape(marker)}`).catch(() => undefined);
                 await this.sshExec(target, `mkdir -p ${shellEscape(remoteDir)}`);
-                await this.execution.uploadFile(sshTargetOf(target), archive, remoteFile);
+                await this.uploadToNode(target, archive, remoteFile);
                 // `uploadFile` already refuses a file that landed SHORT. That catches
                 // the likely failure and not the other one: a transfer that completes
                 // and corrupts in the middle has the right length and the wrong
@@ -1625,14 +1766,41 @@ export class RemoteDeployer {
         await this.sshExec(target, `ln -s ${shellEscape(digest)} ${shellEscape(tmp)} && mv -T ${shellEscape(tmp)} ${shellEscape(`${root}/${link}`)}`);
         this.logger.info({ host: target.host, link, digest }, 'The gateway now serves this build through its link');
     }
-    /**
-     * Hard-link the assets of the build the running gateways serve into the
-     * one just unpacked, and remove builds nobody serves
-     * (`static-carry-forward.ts`). Never fails the delivery: without it an open
-     * tab reloads onto the new build instead of continuing, which the portal
-     * handles; with a failed delivery nobody gets a portal at all.
-     */
-    async carryPreviousBuildForward(target, root, fresh) {
+    async carryPreviousBuildForward(target, input, fresh) {
+        if (typeof input !== 'string') {
+            const { immutableAssetsRoot, immutableAssetsScript } = await import('./static-carry-forward.js');
+            const expectedRoot = immutableAssetsRoot(input.owner);
+            // The native execution fence serializes every publisher, including reentrant calls.
+            return this.underLease(target, 'retain immutable public assets', async () => {
+                await this.assertNoPendingHostOperations(target);
+                const fs = await import('node:fs/promises');
+                const { tmpdir } = await import('node:os');
+                const { join } = await import('node:path');
+                const { randomUUID } = await import('node:crypto');
+                const temporary = await fs.mkdtemp(join(tmpdir(), 'omnitron-static-assets-'));
+                const local = join(temporary, 'request.json');
+                const remote = '/tmp/omnitron-static-assets-' + randomUUID() + '.json';
+                try {
+                    await fs.writeFile(local, JSON.stringify(input), { mode: 0o600, flag: 'wx' });
+                    await this.uploadToNode(target, local, remote);
+                    const output = await this.sshExec(target, immutableAssetsScript(remote), 300_000);
+                    const receipt = JSON.parse(output);
+                    if (receipt.assetsRoot !== expectedRoot || !/^[a-f0-9]{64}$/.test(receipt.manifestSha256) ||
+                        ![receipt.files, receipt.added, receipt.reused, receipt.bytes].every((n) => Number.isSafeInteger(n) && n >= 0)) {
+                        throw new Error('Invalid immutable asset publication receipt');
+                    }
+                    this.logger.info({ host: target.host, owner: input.owner, ...receipt }, 'Immutable public asset bytes retained');
+                    return receipt;
+                }
+                finally {
+                    await fs.rm(temporary, { recursive: true, force: true });
+                    await this.sshExec(target, 'rm -f -- ' + shellEscape(remote)).catch(() => undefined);
+                }
+            });
+        }
+        const root = input;
+        if (!fresh)
+            throw new Error('Missing legacy static build');
         try {
             const { GATEWAY_MOUNTS_COMMAND, readGatewayMounts, carryForwardScript } = await import('./static-carry-forward.js');
             const mounts = readGatewayMounts(await this.sshExec(target, GATEWAY_MOUNTS_COMMAND).catch(() => ''));