@0xmaxma/claude-gateway 1.7.9 → 1.8.0

package/dist/apps/installer.js

@@ -34,6 +34,8 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AppInstaller = void 0;
+exports.isPermissionError = isPermissionError;
+exports.commonAncestorDir = commonAncestorDir;
 exports.parseComposePs = parseComposePs;
 exports.mapContainerStatesToAppStatus = mapContainerStatesToAppStatus;
 const fs = __importStar(require("node:fs"));
@@ -65,6 +67,10 @@ const VOLUME_TAR_TIMEOUT_MS = 300000;
 // OCI image used for the throwaway tar helper. Small, ubiquitous, already a
 // transitive dependency of most stacks, so it is almost always cache-warm.
 const BACKUP_HELPER_IMAGE = 'alpine';
+// Ceiling for the root helper that moves a bind path the gateway user cannot
+// rename itself. The move is a single rename(2) — instant — so this only bounds
+// a stuck container (or a cold pull of the helper image) during an update swap.
+const BIND_MOVE_TIMEOUT_MS = 120000;
 // Per-app ceiling for the boot-time `compose up --wait` during restore. Runs in
 // the background (non-blocking), so this only bounds how long a hung container
 // keeps its child process alive — not the gateway's responsiveness. Shorter than
@@ -96,6 +102,90 @@ function isValidTimezone(tz) {
 // Disallow '..' in owner/repo segments — prevents path traversal via edge-case git URL parsing.
 const GITHUB_URL_RE = /^https:\/\/github\.com\/(?!.*\.\.)[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*(\.git)?$/;
 // ─── Installer ────────────────────────────────────────────────────────────────
+const UUID_RE_SRC = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}';
+/**
+ * The staging checkout `runUpdate` creates beside an app's install path,
+ * `.cg-update-<app>-<uuid>`. It holds a fresh clone and nothing else, so a
+ * crash leftover is pure garbage and safe to reclaim. The UUID means the
+ * pattern cannot match an app directory named by a user.
+ */
+const STALE_UPDATE_DIR_RE = new RegExp(`^\\.cg-update-.+-${UUID_RE_SRC}$`);
+/**
+ * The release snapshots `runUpdate` renames an install path to during the swap:
+ * `<appDir>-old-<uuid>` (previous release) and `<appDir>-failed-<uuid>` (the
+ * release that failed). Deliberately **not** swept.
+ *
+ * Both can hold the only copy of live bind-mount data. `-failed-` is kept on
+ * purpose when a rollback cannot move a bind path back, and a crash between the
+ * swap's two renames leaves an `-old-` still holding the paths that had not
+ * moved yet. Sweeping them would delete a database — through {@link rmrf}'s
+ * `sudo rm -rf` fallback, which exists precisely to defeat the container
+ * ownership that made the data unmovable in the first place. They are reported
+ * at boot instead, so a leak is visible rather than silent.
+ */
+const RELEASE_SNAPSHOT_DIR_RE = new RegExp(`-(?:old|failed)-${UUID_RE_SRC}$`);
+/**
+ * A filesystem call that failed because the gateway user lacks the rights, not
+ * because the path is wrong. Containers running as their image's uid leave
+ * files and directories the gateway user can neither delete nor rename, and
+ * those are the two codes the kernel reports for it.
+ */
+function isPermissionError(err) {
+    const code = err?.code;
+    return code === 'EACCES' || code === 'EPERM';
+}
+/**
+ * Deepest directory containing both paths, never either path itself.
+ *
+ * Used to pick a **single** mount for the root helper that moves a bind path:
+ * mounting each parent separately would put the two paths on different mount
+ * points, where `rename(2)` fails EXDEV and `mv` degrades to a copy.
+ *
+ * The comparison stops one component short of the shorter path, so the result
+ * is always a strict ancestor of both — for siblings (what the callers here
+ * pass) that is exact. When one path contains the other it is the *shallower*
+ * path's parent, i.e. broader than strictly needed; never narrower, which would
+ * put a path outside the mount.
+ */
+function commonAncestorDir(a, b) {
+    const left = path.resolve(a).split(path.sep);
+    const right = path.resolve(b).split(path.sep);
+    const shared = [];
+    for (let i = 0; i < Math.min(left.length, right.length) - 1; i++) {
+        if (left[i] !== right[i])
+            break;
+        shared.push(left[i]);
+    }
+    return shared.join(path.sep) || path.sep;
+}
+/**
+ * Rows from a `docker compose … --format json` call. Compose has emitted both a
+ * single JSON array and one object per line across its 2.x line, so accept
+ * either rather than silently reading a newer/older daemon as "nothing here".
+ */
+function parseJsonRows(stdout) {
+    const text = stdout.trim();
+    if (!text)
+        return [];
+    try {
+        const parsed = JSON.parse(text);
+        return Array.isArray(parsed) ? parsed : [parsed];
+    }
+    catch { /* not a single document — try line-delimited below */ }
+    const rows = [];
+    for (const line of text.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            rows.push(JSON.parse(trimmed));
+        }
+        catch { /* skip a line that is not JSON */ }
+    }
+    return rows;
+}
+/** Private tag the update holds on an image so a rollback can put it back. */
+const ROLLBACK_TAG_PREFIX = 'cg-rollback-';
 class AppInstaller {
     constructor(registry, registryClient, callbacks, spawn = defaultSpawn, appsDir, agentManager, spawnAsync = defaultAsyncSpawn, housekeepingConfig = {}, appBackupConfig, backupsDir) {
         this.registry = registry;
@@ -720,23 +810,29 @@ class AppInstaller {
      * read-only `~/.claude/projects`) are intentionally excluded. Best-effort:
      * returns `[]` on any failure, mirroring {@link discoverVolumes}.
      */
-    discoverBindMounts(appName, appDir) {
+    /**
+     * The paths a generated compose file may have resolved this app dir to.
+     * Local-dev installs symlink the app dir into appsDir, and the generated
+     * compose resolves bind sources against the symlink's *realpath*, so both the
+     * symlink path and its target must be treated as the app root.
+     */
+    bindMountBases(appDir) {
+        const bases = [appDir];
+        try {
+            const real = fs.realpathSync(appDir);
+            if (real !== appDir)
+                bases.push(real);
+        }
+        catch {
+            /* app dir unreadable — fall back to the literal path */
+        }
+        return bases;
+    }
+    discoverBindMounts(appName, appDir, onError = 'empty') {
+        const bases = this.bindMountBases(appDir);
         try {
             const { stdout } = this.run(['docker', 'compose', '-p', appName, 'config', '--format', 'json'], appDir, 30000);
             const parsed = JSON.parse(stdout);
-            // Local-dev installs symlink the app dir into appsDir, and the generated
-            // compose resolves bind sources against the symlink's *realpath*. Match a
-            // source that sits under either the symlink path or its target, so those
-            // bind mounts are not wrongly excluded.
-            const bases = [appDir];
-            try {
-                const real = fs.realpathSync(appDir);
-                if (real !== appDir)
-                    bases.push(real);
-            }
-            catch {
-                /* app dir unreadable — fall back to the literal path */
-            }
             const rels = new Set();
             for (const svc of Object.values(parsed.services ?? {})) {
                 for (const vol of svc.volumes ?? []) {
@@ -753,8 +849,18 @@ class AppInstaller {
             }
             return Array.from(rels).sort();
         }
-        catch {
-            return [];
+        catch (err) {
+            if (onError === 'empty')
+                return [];
+            // Fail closed: the caller is about to move this app's directory, so an
+            // unknown bind set would silently strand live state. Only an app whose
+            // stored compose anchors nothing to the app dir (under either base) is
+            // safe to treat as bind-free.
+            const composePath = path.join(appDir, 'docker-compose.yml');
+            const compose = fs.existsSync(composePath) ? fs.readFileSync(composePath, 'utf-8') : '';
+            if (!bases.some((b) => compose.includes(b)))
+                return [];
+            throw new Error(`Cannot safely discover bind mounts for update: ${err.message}`);
         }
     }
     /**
@@ -823,6 +929,60 @@ class AppInstaller {
      * Returns a cancel function. No-op (returns a noop canceller) when both caps
      * are disabled. The timer is `unref`'d so it never holds the event loop open.
      */
+    /**
+     * Reclaim the `.cg-update-*` staging checkout a crashed or killed update left
+     * behind. Staging moved next to the install path so the swap is a
+     * same-filesystem rename, which also means `/tmp` cleanup no longer collects
+     * it — without this sweep a mid-update crash leaks a full app checkout
+     * forever.
+     *
+     * Release snapshots (`-old-`/`-failed-`) are **reported, never removed**: see
+     * {@link RELEASE_SNAPSHOT_DIR_RE}. They can hold the only copy of a live bind
+     * mount, and this sweep deletes with root.
+     *
+     * **Boot only.** An update in flight owns directories matching these names,
+     * so this must run before any update can start. Never throws; a directory it
+     * cannot remove is reported and skipped.
+     */
+    async sweepStaleUpdateDirs() {
+        const swept = [];
+        let names;
+        try {
+            names = fs.readdirSync(this.appsDir);
+        }
+        catch {
+            return swept; // no apps dir yet
+        }
+        // An installPath is authoritative — never remove a directory an app still
+        // points at, however its name happens to look.
+        let live;
+        try {
+            live = new Set((await this.registry.list()).map((a) => a.installPath));
+        }
+        catch {
+            return swept; // registry unreadable — do not guess
+        }
+        for (const name of names) {
+            const full = path.join(this.appsDir, name);
+            if (live.has(full))
+                continue;
+            if (RELEASE_SNAPSHOT_DIR_RE.test(name)) {
+                console.warn(`[installer] keeping app release snapshot "${full}" — it may hold the only copy of a `
+                    + 'bind mount an update could not move back. Recover or delete it by hand.');
+                continue;
+            }
+            if (!STALE_UPDATE_DIR_RE.test(name))
+                continue;
+            try {
+                this.rmrf(full);
+                swept.push(full);
+            }
+            catch (err) {
+                console.warn(`[installer] failed to sweep stale update dir "${full}": ${err.message}`);
+            }
+        }
+        return swept;
+    }
     startBackupCleanup() {
         const { retention, maxAgeDays, cleanupHour, cleanupTimezone } = this.appBackupConfig;
         if (retention <= 0 && maxAgeDays <= 0)
@@ -1357,7 +1517,12 @@ class AppInstaller {
                 this.log(job, `WARNING: pre-update backup failed (continuing): ${err instanceof Error ? err.message : String(err)}`);
             }
         }
-        const tmpDir = path.join(os.tmpdir(), `cg-update-${appName}-${crypto.randomUUID()}`);
+        // Stage beside the durable install path so the directory swap and bind-data
+        // moves are same-filesystem renames, not cross-device copies/failures.
+        const tmpDir = path.join(path.dirname(entry.installPath), `.cg-update-${appName}-${crypto.randomUUID()}`);
+        // Rollback tags held on the images this update is about to build over.
+        // Declared out here so every exit path can drop them again.
+        let preservedImages = [];
         try {
             // ── Shallow fetch of specific commit into tmp dir ─────────────────────
             this.log(job, `Cloning ${target.repo}`);
@@ -1392,12 +1557,20 @@ class AppInstaller {
                 version: newVersion,
                 commit: target.newCommit,
                 installPath: tmpDir,
-                ...(generated.agentDeclaration !== null ? { agentDeclaration: generated.agentDeclaration } : {}),
+                agentDeclaration: generated.agentDeclaration,
                 ...(agentPaths ? { agentPaths } : {}),
             };
             if (generated.agentDeclaration && this.agentManager && agentPaths) {
                 this.agentManager.injectAgentService(newEntry);
             }
+            // Pin the images the app is running before the build takes their tags
+            // over. A `build:` service's new image reuses the old tag
+            // (`<project>-<service>`), so once the build succeeds that tag names the
+            // new release: rolling only the source back would bring the app up on the
+            // failed release's image and crash-loop on a "successful" rollback. This
+            // has to happen before the build — afterwards the old image has no
+            // reference left to grab it by.
+            preservedImages = this.preserveRunningImages(appName, entry.installPath, job);
             // ── Build new images in tmp dir ───────────────────────────────────────
             this.log(job, 'Building new images');
             this.run(['docker', 'compose', '-p', appName, 'build'], tmpDir, 600000);
@@ -1409,30 +1582,99 @@ class AppInstaller {
                     this.log(job, 'MEMORY.md backed up');
                 }
             }
+            // Discover the app-owned bind paths while the old compose file still points
+            // at the durable install directory. They must move with the release swap so
+            // cleanup cannot delete live state. This query fails closed, so it runs
+            // *before* routes and sockets come down — a throw here would otherwise
+            // leave the app running but unreachable, with no path back.
+            const oldBindMounts = this.discoverBindMounts(appName, entry.installPath, 'throw');
             // ── Deregister old routes before taking down containers ───────────────
             this.callbacks.deregisterRoutes(appName);
             this.callbacks.stopSockets(appName);
-            // Capture the current (old) image IDs while the old stack is still up, so
-            // we can reclaim exactly those images after the update — without a
-            // `compose down` that would collide with the new stack (issue #283).
+            // Capture the current (old) image IDs while the old compose file is still
+            // the live one, so the reclaim below targets exactly this app's images.
             const oldImageIds = this.captureComposeImageIds(appName, entry.installPath);
-            // Also capture the app's declared image refs (repo:tag) so a superseded
-            // *pulled* tag can be reclaimed after the update (issue #302).
             const oldImageRefs = this.captureComposeImageRefs(appName, entry.installPath);
-            // ── Bring old containers down (keeps images for rollback) ─────────────
+            // ── Swap dirs before starting the new containers ───────────────────────
+            // The new compose file's bind sources are anchored to finalDir. Starting
+            // before this swap would bind the old directory inode then delete it.
             this.log(job, 'Stopping old containers');
             this.run(['docker', 'compose', '-p', appName, 'down'], entry.installPath, 120000);
-            // ── Start new containers ──────────────────────────────────────────────
-            this.log(job, 'Starting new containers');
+            this.log(job, 'Swapping app directories');
+            const finalDir = entry.installPath;
+            const oldBackupDir = `${finalDir}-old-${crypto.randomUUID()}`;
+            let swapped = false;
+            let reachedContainerStart = false;
+            let failedDir = null;
+            const movedBindMounts = [];
             try {
-                this.composeUp(appName, tmpDir, job);
+                fs.renameSync(finalDir, oldBackupDir);
+                fs.renameSync(tmpDir, finalDir);
+                swapped = true;
+                this.moveBindMounts(oldBackupDir, finalDir, oldBindMounts, job, movedBindMounts);
+                // The source checkout is now permanent. Rewrite the compose file with
+                // finalDir as its base so every bind source and build context resolves
+                // to the durable path — this call is kept for that side effect; its
+                // return value necessarily matches `generated` (same app.yaml). The
+                // rewrite drops the injected agent service, so re-inject it after.
+                const finalComposePath = path.join(finalDir, 'docker-compose.yml');
+                const finalYaml = (0, compose_generator_1.parseAppYaml)(fs.readFileSync(path.join(finalDir, 'app.yaml'), 'utf-8'), finalDir);
+                (0, compose_generator_1.generateCompose)(finalYaml, appName, finalDir, finalComposePath);
+                if (generated.agentDeclaration && this.agentManager && agentPaths) {
+                    this.agentManager.injectAgentService({ ...newEntry, installPath: finalDir });
+                }
+                // ── Start new containers ────────────────────────────────────────────
+                this.log(job, 'Starting new containers');
+                reachedContainerStart = true;
+                this.composeUp(appName, finalDir, job);
             }
             catch (upErr) {
-                // Rollback: bring old containers back up from old install path
-                this.log(job, 'New containers failed — rolling back to previous version');
+                // The catch spans the whole swap, so a failure here is not necessarily a
+                // container failure — saying it is sends diagnosis to the wrong
+                // subsystem (a bind-mount move that threw looked exactly like a crashed
+                // container in the job log).
+                this.log(job, reachedContainerStart
+                    ? 'New containers failed — rolling back to previous version'
+                    : `Update failed during the directory swap — rolling back to previous version: ${upErr.message}`);
                 let rollbackFailed = false;
+                failedDir = `${finalDir}-failed-${crypto.randomUUID()}`;
                 try {
-                    this.run(['docker', 'compose', '-p', appName, 'up', '-d'], entry.installPath, 120000);
+                    let unrestored = [];
+                    if (swapped) {
+                        fs.renameSync(finalDir, failedDir);
+                        fs.renameSync(oldBackupDir, finalDir);
+                        unrestored = this.restoreBindMounts(failedDir, finalDir, movedBindMounts, job);
+                    }
+                    else if (fs.existsSync(oldBackupDir)) {
+                        fs.renameSync(oldBackupDir, finalDir);
+                    }
+                    // Point every tag back at the image the app was actually running
+                    // before this update built over it, so the restored source and the
+                    // restored image are the same release. Falls back to rebuilding from
+                    // the restored source when an old image is no longer on disk.
+                    //
+                    // This has to happen even when the bind restore below refuses to
+                    // start the app: the source is already back on the previous release,
+                    // so `<app>-<service>:latest` must name that release's build before
+                    // *anything* starts it. Leaving it on the failed release's build
+                    // means an operator who finishes the recovery by hand brings old
+                    // source up on a new image — the crash-loop `preserveRunningImages`
+                    // exists to prevent.
+                    const rebuild = !this.restorePreservedImages(preservedImages, finalDir, job);
+                    if (unrestored.length > 0) {
+                        // Throwing here is deliberate: it skips both the container start
+                        // below and the safeRmrf of failedDir, so the only copy of that
+                        // data stays on disk. Starting the app on a half-restored
+                        // directory is worse than not starting it — postgres on an empty
+                        // pgdata initialises a fresh cluster and then looks healthy.
+                        throw new Error(`live bind-mount data for ${unrestored.map((r) => `"${r}"`).join(', ')} could not be moved back `
+                            + `— it is still in "${failedDir}", which has been kept. Move those paths back into `
+                            + `"${finalDir}" before starting the app.`);
+                    }
+                    const upArgs = ['docker', 'compose', '-p', appName, 'up', '-d'];
+                    if (rebuild)
+                        upArgs.push('--build');
+                    this.run(upArgs, finalDir, rebuild ? 600000 : 120000);
                     this.callbacks.registerRoutes(appName, entry.ports.map((p) => ({
                         name: p.name,
                         service: p.service,
@@ -1442,38 +1684,41 @@ class AppInstaller {
                         rateLimit: p.rateLimit,
                     })));
                     await this.registry.updateStatus(appName, 'running');
+                    if (failedDir !== null)
+                        this.safeRmrf(failedDir, job, 'failed update dir');
                 }
                 catch (rollbackErr) {
                     rollbackFailed = true;
                     this.log(job, `ROLLBACK FAILED — app "${appName}" may be in a broken state: ${rollbackErr.message}`);
                 }
-                this.safeRmrf(tmpDir, job, 'update temp dir');
                 if (rollbackFailed) {
+                    // Keep the private `cg-rollback-*` tags: they are the only remaining
+                    // reference to the pre-update build, and the outer catch drops them.
+                    // Clearing the list here is what stops that — an incomplete rollback
+                    // is finished by hand, and that recovery needs the old image to still
+                    // exist. Untagged, containerd is free to reclaim it.
+                    if (preservedImages.some((i) => i.backupRef)) {
+                        this.log(job, `Pre-update images kept for manual recovery: ${preservedImages.filter((i) => i.backupRef).map((i) => `"${i.backupRef}" (for "${i.ref}")`).join(', ')}`);
+                    }
+                    preservedImages = [];
+                    // The success path above sets 'running'. Leaving the registry on the
+                    // pre-update 'running' here would report a healthy app that is not
+                    // running at all — and this branch now includes the case where the
+                    // app was deliberately not restarted.
+                    await this.registry.updateStatus(appName, 'error').catch(() => { });
                     throw new Error(`Update failed and rollback also failed — app "${appName}" may be in a broken state. Check job logs for details.`);
                 }
                 throw upErr;
             }
-            // Capture the new stack's image IDs (still at tmpDir) so image reclamation
-            // below never removes an image the new containers depend on (e.g. when the
-            // old and new versions happen to share a base/image).
-            const newImageIds = this.captureComposeImageIds(appName, tmpDir);
-            const newImageRefs = this.captureComposeImageRefs(appName, tmpDir);
-            // ── Swap dirs ─────────────────────────────────────────────────────────
-            // Swap in place at the recorded install path — NOT path.join(appsDir, appName).
-            // For legacy installs the on-disk dir is named after the source repo/URL
-            // basename, so `installPath` basename can differ from the app name. Using
-            // the app name here throws ENOENT (issue #275). `entry.installPath` is the
-            // authoritative location the `down`/rollback steps above already use.
-            this.log(job, 'Swapping app directories');
-            const finalDir = entry.installPath;
-            const oldBackupDir = `${finalDir}-old-${crypto.randomUUID()}`;
-            fs.renameSync(finalDir, oldBackupDir);
-            fs.renameSync(tmpDir, finalDir);
-            // ── Restore MEMORY.md ─────────────────────────────────────────────────
-            if (memoryBackup !== null && generated.agentDeclaration && this.agentManager) {
-                this.agentManager.restoreMemory(generated.agentDeclaration.name, memoryBackup);
-                this.log(job, 'MEMORY.md restored');
-            }
+            // The new stack is up: the previous images are no longer a rollback
+            // target. Untag them before the reclaim below, which removes by image ID
+            // and would be refused while a second reference exists.
+            this.dropPreservedImageTags(preservedImages);
+            preservedImages = [];
+            // Capture the new stack's image IDs from its permanent compose file so
+            // cleanup below never removes an image the new containers depend on.
+            const newImageIds = this.captureComposeImageIds(appName, finalDir);
+            const newImageRefs = this.captureComposeImageRefs(appName, finalDir);
             // ── Update registry ───────────────────────────────────────────────────
             const finalEntry = {
                 ...newEntry,
@@ -1482,10 +1727,28 @@ class AppInstaller {
                 status: 'running',
             };
             await this.registry.upsert(finalEntry);
-            // ── Re-create agent symlink + config.json entry ───────────────────────
+            // ── Re-create, rename, or remove the app-agent registration ───────────
+            // `upsertAgent` keys off the *new* agent name, so a release that drops or
+            // renames its agent would otherwise strand the previous workspace symlink
+            // and config.json entry. Both transitions are the same deregistration.
+            const oldAgentName = entry.agentDeclaration?.name ?? null;
+            const newAgentName = generated.agentDeclaration?.name ?? null;
+            if (this.agentManager && oldAgentName !== null && oldAgentName !== newAgentName) {
+                await this.agentManager.deleteAgentByName(oldAgentName);
+                this.log(job, newAgentName === null
+                    ? `Agent "${oldAgentName}" removed`
+                    : `Agent "${oldAgentName}" deregistered (renamed to "${newAgentName}")`);
+            }
             if (generated.agentDeclaration && this.agentManager) {
                 await this.agentManager.upsertAgent(finalEntry);
                 this.log(job, `Agent "${generated.agentDeclaration.name}" re-registered`);
+                // MEMORY.md must be written *after* registration: restoreMemory resolves
+                // the workspace through config.json, so on a rename the new name is not
+                // resolvable until upsertAgent has written its entry.
+                if (memoryBackup !== null) {
+                    this.agentManager.restoreMemory(generated.agentDeclaration.name, memoryBackup);
+                    this.log(job, 'MEMORY.md restored');
+                }
                 await this.callbacks.reinitializeAgent?.(generated.agentDeclaration.name);
             }
             // ── Re-register proxy routes + sockets ───────────────────────────────
@@ -1534,6 +1797,7 @@ class AppInstaller {
             this.log(job, `Update complete → ${newVersion}`);
         }
         catch (err) {
+            this.dropPreservedImageTags(preservedImages);
             if (fs.existsSync(tmpDir)) {
                 this.safeRmrf(tmpDir, job, 'update temp dir');
             }
@@ -1950,7 +2214,7 @@ class AppInstaller {
             const code = err.code;
             // Containers running as root leave root-owned files (e.g. postgres pgdata)
             // that the gateway user cannot delete — fs.rmSync surfaces EACCES or EPERM.
-            if (code === 'EACCES' || code === 'EPERM') {
+            if (isPermissionError(err)) {
                 console.warn(`[installer] ${code} removing "${dirPath}" — falling back to sudo rm -rf`);
                 this.run(['sudo', 'rm', '-rf', dirPath]);
             }
@@ -1975,6 +2239,247 @@ class AppInstaller {
             }
         }
     }
+    /**
+     * Create `targetDir` under `baseDir`, refusing to traverse or create through a
+     * symlink. The check runs **before** each segment is created — a `mkdir -p`
+     * that ran first would already have materialised directories on the far side
+     * of a symlink, leaving the guard with nothing left to prevent.
+     *
+     * Scope: this guards the **destination** side — the freshly checked-out
+     * release, which is the side an app repo controls. A bind path that was
+     * already a symlink in the *previous* app dir is carried across as-is; that
+     * preserves an escape the operator set up themselves rather than creating
+     * one, and is the behaviour every release before this one had.
+     */
+    ensureDirWithinNoSymlink(baseDir, targetDir) {
+        const relative = path.relative(baseDir, targetDir);
+        if (path.isAbsolute(relative) || relative.split(path.sep).includes('..')) {
+            throw new Error(`Bind-mount path escapes the app directory: "${targetDir}"`);
+        }
+        let current = baseDir;
+        for (const segment of relative.split(path.sep).filter(Boolean)) {
+            current = path.join(current, segment);
+            let stat;
+            try {
+                stat = fs.lstatSync(current);
+            }
+            catch {
+                stat = null;
+            }
+            if (stat === null) {
+                fs.mkdirSync(current);
+                continue;
+            }
+            if (stat.isSymbolicLink()) {
+                throw new Error(`Updated app bind-mount source must not be a symlink: "${current}"`);
+            }
+            if (!stat.isDirectory()) {
+                throw new Error(`Updated app bind-mount path is not a directory: "${current}"`);
+            }
+        }
+    }
+    /** Reject a discovered bind path that does not stay inside both app roots. */
+    isSafeBindRel(fromDir, toDir, rel) {
+        if (rel.length === 0 || path.isAbsolute(rel))
+            return false;
+        const fromBase = fromDir.endsWith(path.sep) ? fromDir : fromDir + path.sep;
+        const toBase = toDir.endsWith(path.sep) ? toDir : toDir + path.sep;
+        return path.resolve(fromDir, rel).startsWith(fromBase)
+            && path.resolve(toDir, rel).startsWith(toBase);
+    }
+    /**
+     * Move app-owned relative bind data across an update directory swap.
+     * Renaming preserves the database's ownership and inode, unlike copying.
+     *
+     * A release legitimately ships content at a bind path (a `.gitkeep`, seed
+     * files, a tracked `init.sql`), so a collision is normal, not an error. Live
+     * state always wins — it is the data the issue exists to protect — but a
+     * directory collision is **merged** entry by entry so release-provided files
+     * the previous version never had still land. Every rename performed is
+     * recorded, app-relative and in order, so a rollback can replay it backwards.
+     */
+    moveBindMounts(fromDir, toDir, rels, job, moved) {
+        for (const rel of rels) {
+            if (!this.isSafeBindRel(fromDir, toDir, rel)) {
+                this.log(job, `Warning: skipping bind-mount path outside the app directory: "${rel}"`);
+                continue;
+            }
+            if (!fs.existsSync(path.resolve(fromDir, rel)))
+                continue; // never created
+            this.moveBindEntry(fromDir, toDir, rel, job, moved);
+        }
+    }
+    moveBindEntry(fromDir, toDir, rel, job, moved) {
+        const source = path.resolve(fromDir, rel);
+        const destination = path.resolve(toDir, rel);
+        this.ensureDirWithinNoSymlink(toDir, path.dirname(destination));
+        let destStat;
+        try {
+            destStat = fs.lstatSync(destination);
+        }
+        catch {
+            destStat = null;
+        }
+        if (destStat === null) {
+            this.moveBindPath(source, destination, rel, job);
+            moved?.push(rel);
+            return;
+        }
+        if (destStat.isDirectory() && fs.lstatSync(source).isDirectory()) {
+            // Merge: recurse so a release-only file inside the directory survives.
+            //
+            // Entry-by-entry on purpose. Swapping the trees (move the live dir over
+            // wholesale, then re-apply the release's own files on top) would be one
+            // rename instead of one per live file, but `moved` would no longer be an
+            // exact inverse: a rollback would carry those re-applied release files
+            // into the restored previous app dir. Exact rollback beats the renames,
+            // which are same-filesystem and only walk a directory the release also
+            // ships content in.
+            let entries;
+            try {
+                entries = fs.readdirSync(source);
+            }
+            catch (err) {
+                // The live directory is one the app's container owns and the gateway
+                // user cannot even list (postgres leaves pgdata 0700). There is no way
+                // to merge into it entry by entry, so preserve it wholesale — the same
+                // trade the non-directory collision below already makes.
+                if (!isPermissionError(err))
+                    throw err;
+                entries = null;
+            }
+            if (entries !== null) {
+                // Each child that needs the root helper costs its own container. That
+                // is bounded by the live directory's own entry count, and only reached
+                // when a release ships tracked content *inside* a path whose children
+                // the gateway user cannot rename — rare enough not to trade the exact
+                // rollback above for one wholesale move.
+                for (const name of entries) {
+                    this.moveBindEntry(fromDir, toDir, `${rel}/${name}`, job, moved);
+                }
+                return;
+            }
+        }
+        this.log(job, `Warning: preserved existing bind-mount data at "${rel}" — the updated release's copy of that path `
+            + `was discarded (${this.describeDiscarded(destination)})`);
+        fs.rmSync(destination, { recursive: true, force: true });
+        this.moveBindPath(source, destination, rel, job);
+        moved?.push(rel);
+    }
+    /**
+     * Name what a preserve-the-live-copy collision is about to delete.
+     *
+     * The discarded side is the release's own checkout, so it is always readable
+     * by the gateway user even when the live side is not. Without this the log
+     * says only that "the release's copy was discarded" — which reads as a
+     * `.gitkeep` and hides the case that matters: a release shipping a new
+     * `init.sql` or entrypoint script *inside* a path whose live copy wins, where
+     * the file is dropped on every update and nothing ever says which.
+     */
+    describeDiscarded(destination, limit = 10) {
+        let names;
+        try {
+            names = fs.statSync(destination).isDirectory()
+                ? fs.readdirSync(destination).sort()
+                : [path.basename(destination)];
+        }
+        catch {
+            return 'contents unreadable';
+        }
+        if (names.length === 0)
+            return 'it was empty';
+        const shown = names.slice(0, limit).map((n) => `"${n}"`).join(', ');
+        return names.length > limit
+            ? `${shown} and ${names.length - limit} more`
+            : shown;
+    }
+    /**
+     * Undo {@link moveBindMounts}. Returns the paths it could **not** move back:
+     * those still live only under `fromDir`, so the caller must keep that
+     * directory rather than clean it up.
+     *
+     * A path that is missing at the source, or already present at the
+     * destination, is not a failure — it needs no move and is not reported.
+     */
+    restoreBindMounts(fromDir, toDir, moved, job) {
+        const unrestored = [];
+        for (const rel of [...moved].reverse()) {
+            if (!this.isSafeBindRel(fromDir, toDir, rel)) {
+                this.log(job, `Warning: skipping bind-mount path outside the app directory: "${rel}"`);
+                unrestored.push(rel);
+                continue;
+            }
+            const source = path.resolve(fromDir, rel);
+            const destination = path.resolve(toDir, rel);
+            if (!fs.existsSync(source) || fs.existsSync(destination))
+                continue;
+            try {
+                this.ensureDirWithinNoSymlink(toDir, path.dirname(destination));
+                this.moveBindPath(source, destination, rel, job);
+            }
+            catch (err) {
+                this.log(job, `Warning: could not restore bind-mount path "${rel}": ${err.message}`);
+                unrestored.push(rel);
+            }
+        }
+        return unrestored;
+    }
+    /**
+     * Move one bind-mount path between app directories.
+     *
+     * `rename(2)` that moves a **directory** to a different parent needs write
+     * permission on the directory itself — the kernel has to update its `..`
+     * entry — not just on the two parents, which is exactly what the swap does. A bind mount the app's own container created is owned by that
+     * image's uid (postgres initdb leaves its data directory mode 0700), so the
+     * gateway user cannot rename it and the update swap failed with EACCES on
+     * every attempt.
+     *
+     * The same host-vs-container ownership split is already handled elsewhere in
+     * this file — backups tar through a root helper container, {@link rmrf} falls
+     * back to `sudo rm -rf`. The swap was the one path still doing it bare, so
+     * the fallback here matches the backup mechanism: a throwaway root container.
+     *
+     * The helper mounts the **nearest common ancestor** of the two paths rather
+     * than each parent separately, and that is load-bearing: two bind mounts are
+     * two mount points, so `rename(2)` across them fails EXDEV and busybox `mv`
+     * silently degrades to copy-then-unlink. Measured by inode, two mounts give a
+     * new inode (a copy) where one mount preserves it (a real rename). For a
+     * database directory the degraded path means duplicating it on disk and
+     * losing the atomicity the swap depends on.
+     *
+     * The single mount is therefore the apps root, not one app's directory — the
+     * two app directories are siblings under it. The helper is a throwaway
+     * container whose only command is the `mv`, and it is reached only after the
+     * gateway user's own rename was refused.
+     */
+    moveBindPath(source, destination, rel, job) {
+        try {
+            fs.renameSync(source, destination);
+            return;
+        }
+        catch (err) {
+            if (!isPermissionError(err))
+                throw err;
+        }
+        const base = commonAncestorDir(source, destination);
+        const from = path.relative(base, source);
+        const to = path.relative(base, destination);
+        if (base === path.parse(base).root || base.includes(':') || from === '' || to === ''
+            || from.split(path.sep).includes('..') || to.split(path.sep).includes('..')) {
+            // A colon would be read as the mount separator by `docker -v`, silently
+            // mounting something other than what was asked for.
+            throw new Error(`Cannot move bind-mount path "${rel}" as root: unsafe mount base "${base}"`);
+        }
+        this.log(job, `Bind-mount path "${rel}" is not writable by the gateway user (created by the app's container) `
+            + '— moving it with a root helper container');
+        this.run([
+            'docker', 'run', '--rm',
+            '-v', `${base}:/mnt`,
+            BACKUP_HELPER_IMAGE,
+            'mv', path.posix.join('/mnt', from.split(path.sep).join('/')),
+            path.posix.join('/mnt', to.split(path.sep).join('/')),
+        ], undefined, BIND_MOVE_TIMEOUT_MS);
+    }
     /**
      * Stop conflicting containers then run `docker compose up -d --wait`.
      * Captures container logs into the job on failure before rethrowing.
@@ -2076,6 +2581,104 @@ class AppInstaller {
      * container back down (issue #283). Returns a de-duplicated list of image
      * IDs, or `[]` on any error (nothing to reclaim / docker unavailable).
      */
+    /**
+     * Tag every image the app is *currently* running under a private
+     * `<repo>:cg-rollback-<id>` name, and return the pairs.
+     *
+     * A `build:` service's new image reuses the tag of the one in production
+     * (`<project>-<service>:latest`), so once the update's build succeeds that
+     * tag names the new release. Rolling only the source directory back would
+     * bring the app up on the failed release's image. Holding a second tag keeps
+     * the old image addressable *and* alive — under the containerd image store an
+     * untagged image is not kept around as a `<none>` image to find later.
+     *
+     * Only images this app builds are preserved: compose names them
+     * `<project>-<service>`, and a pulled tag (`postgres:16-alpine`) is never
+     * overwritten by a build, so it needs no protection. Best-effort throughout —
+     * anything that cannot be preserved is simply absent from the result, and the
+     * rollback rebuilds from the restored source instead.
+     */
+    preserveRunningImages(appName, dir, job) {
+        let rows;
+        try {
+            const { stdout } = this.run(['docker', 'compose', '-p', appName, 'images', '--format', 'json'], dir, 15000);
+            rows = parseJsonRows(stdout);
+        }
+        catch {
+            return [];
+        }
+        const preserved = [];
+        const seen = new Set();
+        for (const row of rows) {
+            if (typeof row !== 'object' || row === null)
+                continue;
+            const r = row;
+            const id = typeof r.ID === 'string' ? r.ID.trim() : '';
+            const repo = typeof r.Repository === 'string' ? r.Repository.trim() : '';
+            const tag = typeof r.Tag === 'string' && r.Tag.trim() ? r.Tag.trim() : 'latest';
+            if (!id || !repo.startsWith(`${appName}-`))
+                continue;
+            const ref = `${repo}:${tag}`;
+            if (seen.has(ref))
+                continue;
+            seen.add(ref);
+            const backupRef = `${repo}:${ROLLBACK_TAG_PREFIX}${id.replace(/^sha256:/, '').slice(0, 12)}`;
+            try {
+                this.run(['docker', 'image', 'tag', id, backupRef], dir, 15000);
+                preserved.push({ ref, backupRef });
+            }
+            catch (err) {
+                // Recorded with no backup reference: the rollback must know this tag is
+                // unprotected and rebuild, rather than read an empty list as "no built
+                // images, nothing at risk".
+                preserved.push({ ref, backupRef: '' });
+                this.log(job, `Warning: could not preserve image "${ref}" for rollback (${err instanceof Error ? err.message : String(err)})`);
+            }
+        }
+        return preserved;
+    }
+    /**
+     * Put each preserved image back on the reference the update's build took
+     * over. Returns true when nothing is at risk (an empty list — the app builds
+     * no images) or every reference is back on its original image; false when at
+     * least one could not be, so the caller rebuilds from the rolled-back source
+     * rather than starting the failed release's build.
+     */
+    restorePreservedImages(images, dir, job) {
+        let allRestored = true;
+        for (const { ref, backupRef } of images) {
+            if (!backupRef) {
+                allRestored = false;
+                this.log(job, `Image "${ref}" was not preserved — rebuilding from the rolled-back source`);
+                continue;
+            }
+            try {
+                this.run(['docker', 'image', 'tag', backupRef, ref], dir, 15000);
+                this.log(job, `Restored image "${ref}" to the pre-update build`);
+            }
+            catch (err) {
+                allRestored = false;
+                this.log(job, `Warning: could not restore image "${ref}" (${err instanceof Error ? err.message : String(err)}) — rebuilding from the rolled-back source instead`);
+            }
+        }
+        return allRestored;
+    }
+    /**
+     * Drop the private rollback tags. Removing a reference only untags the image
+     * while another tag remains, so this never deletes an image the app still
+     * uses. Must run before the post-update image reclaim: an extra tag would
+     * make `docker image rm <id>` refuse.
+     */
+    dropPreservedImageTags(images) {
+        for (const { backupRef } of images) {
+            if (!backupRef)
+                continue;
+            try {
+                this.run(['docker', 'image', 'rm', backupRef], os.tmpdir(), 15000);
+            }
+            catch { /* already gone — non-fatal */ }
+        }
+    }
     captureComposeImageIds(appName, dir) {
         try {
             const { stdout } = this.run(['docker', 'compose', '-p', appName, 'images', '--quiet'], dir, 15000);