@pleri/olam-cli 0.1.12 → 0.1.13

package/dist/commands/upgrade.js

@@ -16,6 +16,11 @@ import { spawnSync } from 'node:child_process';
 import pc from 'picocolors';
 import { printError, printSuccess, printInfo, printWarning, printHeader } from '../output.js';
 import { buildComposeEnv, readAuthSecret, runCompose } from './host-cp.js';
+import { acquireLock, releaseLock, formatRefusalMessage, LOCK_FILE_PATH } from './upgrade-lock.js';
+import { appendUpgradeLog } from './upgrade-log.js';
+import { handleHistory, parseHistoryOpts } from './upgrade-history.js';
+import { AuthContainerController } from '@olam/core/src/auth/index.js';
+const AUTH_HEALTH_URL = 'http://127.0.0.1:9999/health';
 /**
  * Check whether node_modules is in sync with package-lock.json.
  *
@@ -71,11 +76,23 @@ export function validateRepoRoot(cwd) {
 }
 /** Normalise raw Commander option object into typed opts. */
 export function parseUpgradeOpts(raw) {
+    const rawN = raw.n;
+    const historyN = typeof rawN === 'number'
+        ? rawN
+        : typeof rawN === 'string'
+            ? Number.parseInt(rawN, 10)
+            : 10;
     return {
         yes: raw.yes === true,
         skipImage: raw.skipImage === true,
         skipInstall: raw.skipInstall === true,
         branch: raw.branch ?? null,
+        rollback: raw.rollback === true,
+        force: raw.force === true,
+        noCache: raw.noCache === true,
+        history: raw.history === true,
+        historyN: Number.isFinite(historyN) && historyN > 0 ? historyN : 10,
+        historyJson: raw.json === true,
     };
 }
 /**
@@ -122,6 +139,321 @@ function hasGitUpstream(cwd) {
     });
     return result.status === 0;
 }
+/**
+ * Capture HEAD SHA via `git rev-parse HEAD`. Returns null on failure.
+ *
+ * Phase 2a — A2: must be invoked AFTER `git pull --ff-only` so the captured
+ * SHA reflects the state we're upgrading TO (not the pre-pull state). The
+ * pull's whole purpose is to advance HEAD; capturing before would refuse the
+ * CLI's own pull as drift at A6's swap-boundary check.
+ *
+ * The returned SHA is sticky for the rest of the run (no per-step re-reads);
+ * A6 / B4 re-read once at the swap boundary to detect operator-driven mid-flight
+ * `git checkout` / `git reset` that happen DURING the build window.
+ */
+export function captureHeadSha(cwd) {
+    const result = spawnSync('git', ['rev-parse', 'HEAD'], {
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'pipe'],
+        cwd,
+    });
+    if (result.status !== 0)
+        return null;
+    const sha = (result.stdout ?? '').trim();
+    // git rev-parse HEAD returns 40-char lowercase hex; defensive validation.
+    if (!/^[0-9a-f]{40}$/.test(sha))
+        return null;
+    return sha;
+}
+/** Abbreviate a 40-char SHA to 8 chars for human-readable output. */
+export function abbreviateSha(sha) {
+    return sha.slice(0, 8);
+}
+/**
+ * Check whether a docker image tag exists locally (Phase 2b — B1).
+ *
+ * Uses `docker image inspect` which exits 0 only when ALL specified
+ * images exist locally. Single-image variant for the rollback pre-flight.
+ */
+export function imageExists(tag) {
+    try {
+        const result = spawnSync('docker', ['image', 'inspect', '--format', '{{.Id}}', tag], {
+            encoding: 'utf-8',
+            stdio: ['ignore', 'pipe', 'ignore'],
+        });
+        return result.status === 0;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Pre-flight check for `olam upgrade --rollback` (Phase 2b — B1).
+ *
+ * Verifies that all three `:olam-rollback` tags exist. Returns an error
+ * message naming the missing image(s) when any are absent — typically
+ * the first-upgrade case where no prior canonical existed for one or
+ * more components, leaving the rollback set incoherent (see audit A6-001).
+ *
+ * Returns null when all three are present (rollback is safe to proceed).
+ */
+export function checkRollbackSetExists(plan) {
+    const missing = plan.filter((p) => !imageExists(p.rollback)).map((p) => p.rollback);
+    if (missing.length === 0)
+        return null;
+    return missing.join(', ');
+}
+/**
+ * Run docker create + docker inspect for a single image.
+ *
+ * Returns ok=true when:
+ *   - `docker create <image>` exits 0 (image manifest valid, layers downloadable).
+ *   - `docker inspect <image> --format '{{.Config.Labels.olam_build_sha}}'`
+ *     returns the expected `targetSha`.
+ *
+ * The container created by `docker create` is removed via `docker rm` even on
+ * failure paths (best-effort cleanup; orphans are harmless and pruned by the
+ * daemon's GC eventually).
+ */
+export function smokeImage(image, targetSha) {
+    // 1. docker create — allocates the container; doesn't start the entrypoint.
+    const createResult = spawnSync('docker', ['create', '--name', `olam-smoke-${Date.now()}`, image], {
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    if (createResult.status !== 0) {
+        return {
+            image,
+            ok: false,
+            bakedSha: null,
+            error: `docker create failed: ${(createResult.stderr ?? '').trim()}`,
+        };
+    }
+    const containerId = (createResult.stdout ?? '').trim();
+    // 2. docker inspect — read the OLAM_BUILD_SHA label.
+    const inspectResult = spawnSync('docker', ['inspect', '--format', '{{index .Config.Labels "olam_build_sha"}}', image], {
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    // 3. Cleanup — best-effort; ignore exit code.
+    if (containerId.length > 0) {
+        spawnSync('docker', ['rm', '-f', containerId], {
+            encoding: 'utf-8',
+            stdio: ['ignore', 'ignore', 'ignore'],
+        });
+    }
+    if (inspectResult.status !== 0) {
+        return {
+            image,
+            ok: false,
+            bakedSha: null,
+            error: `docker inspect failed: ${(inspectResult.stderr ?? '').trim()}`,
+        };
+    }
+    const bakedSha = (inspectResult.stdout ?? '').trim();
+    // Empty output means the label is absent — that's a build-corrupt signal.
+    if (bakedSha.length === 0) {
+        return {
+            image,
+            ok: false,
+            bakedSha: null,
+            error: 'olam_build_sha label is missing or empty',
+        };
+    }
+    // Allow either full 40-char SHA or "unknown" (build-host-cp.sh writes
+    // "unknown" when git rev-parse fails). Match against targetSha for
+    // success.
+    if (bakedSha !== targetSha) {
+        return {
+            image,
+            ok: false,
+            bakedSha,
+            error: `baked SHA ${abbreviateSha(bakedSha)} ≠ target SHA ${abbreviateSha(targetSha)}`,
+        };
+    }
+    return { image, ok: true, bakedSha };
+}
+export const PRODUCTION_SWAP_PLAN = [
+    { transient: 'olam-auth:olam-next', canonical: 'olam-auth:local', rollback: 'olam-auth:olam-rollback' },
+    { transient: 'olam-devbox:olam-next', canonical: 'olam-devbox:latest', rollback: 'olam-devbox:olam-rollback' },
+    { transient: 'olam-host-cp:olam-next', canonical: 'olam-host-cp:latest', rollback: 'olam-host-cp:olam-rollback' },
+];
+/**
+ * Run `docker tag <source> <dest>`. Returns ok=false with stderr trimmed
+ * on failure (e.g. source image absent). No retry — caller decides.
+ *
+ * Per audit A6-003: spawnSync may throw synchronously under fork pressure
+ * (libuv clone(2) failures). The try/catch ensures performAtomicSwap can
+ * always proceed to its summary phase — a thrown exception escaping
+ * dockerTag would leak the upgrade lock and produce no SwapResult, which
+ * confuses both the operator AND Phase 2b's --rollback recovery path.
+ */
+export function dockerTag(source, dest) {
+    try {
+        const result = spawnSync('docker', ['tag', source, dest], {
+            encoding: 'utf-8',
+            stdio: ['ignore', 'ignore', 'pipe'],
+        });
+        if (result.status === 0 && result.error === undefined)
+            return { ok: true };
+        return {
+            ok: false,
+            error: (result.stderr ?? '').trim() || result.error?.message || 'docker tag failed',
+        };
+    }
+    catch (err) {
+        return {
+            ok: false,
+            error: err instanceof Error ? `spawnSync threw: ${err.message}` : 'spawnSync threw',
+        };
+    }
+}
+/**
+ * Atomic-ish 3-image set swap.
+ *
+ * Six sequential `docker tag` ops in two phases:
+ *
+ *   Phase 1 (rollback-save):
+ *     1. canonical → :olam-rollback (image 1)
+ *     2. canonical → :olam-rollback (image 2)
+ *     3. canonical → :olam-rollback (image 3)
+ *
+ *   Phase 2 (canonical-advance):
+ *     4. :olam-next → canonical (image 1)
+ *     5. :olam-next → canonical (image 2)
+ *     6. :olam-next → canonical (image 3)
+ *
+ * Invariants:
+ *
+ * - **First-upgrade tolerance**: any of steps 1-3 may fail with "no such
+ *   image" if the operator has never had a canonical-tagged image. Those
+ *   failures are NON-FATAL (recorded in rollbackError but not aborted) —
+ *   `:olam-rollback` simply doesn't exist for that image; Phase 2b's
+ *   `--rollback` pre-flight detects this and refuses.
+ *
+ * - **Canonical-advance fatality**: any failure in steps 4-6 is fatal.
+ *   The swap is partially advanced; canonical tags are now mixed (some
+ *   at SHA-Y, some at SHA-X). Operator runs `olam upgrade --rollback`
+ *   (Phase 2b) which uses the FULL `:olam-rollback` set written in Phase 1
+ *   to restore coherent prior state.
+ *
+ * - **SIGKILL recovery**: if killed during Phase 1, partial `:olam-rollback`
+ *   exists but canonical is intact — operator's next `olam upgrade` succeeds
+ *   normally (the partial `:olam-rollback` is overwritten by the next
+ *   successful run). If killed during Phase 2, canonical is mixed —
+ *   operator must `olam upgrade --rollback` to recover.
+ *
+ * The "atomic-ish" qualifier: `docker tag` is per-image atomic (POSIX rename
+ * of a symbolic name), but the SET of 3 canonical tags is updated sequentially
+ * across ~1s wall-clock. Sub-second window is acceptable for solo-dev/dogfood
+ * per the plan's local-dev/dogfood priority axis.
+ */
+export function performAtomicSwap(plan) {
+    const steps = plan.map((p) => ({
+        image: p.canonical,
+        rollbackSaved: false,
+        canonicalAdvanced: false,
+    }));
+    // Phase 1: preserve previous-good as :olam-rollback (steps 1-3).
+    // Non-fatal failures: missing canonical (first-upgrade) is acceptable.
+    for (let i = 0; i < plan.length; i++) {
+        const p = plan[i];
+        const r = dockerTag(p.canonical, p.rollback);
+        steps[i] = {
+            ...steps[i],
+            rollbackSaved: r.ok,
+            ...(r.error !== undefined && { rollbackError: r.error }),
+        };
+    }
+    // Phase 2: advance canonical to :olam-next (steps 4-6).
+    // FATAL failure: canonical is mixed. Recovery via `olam upgrade --rollback`.
+    let advanceFailed = false;
+    let firstFailureIdx = -1;
+    for (let i = 0; i < plan.length; i++) {
+        const p = plan[i];
+        if (advanceFailed) {
+            // Skip remaining advances after first failure — leaves canonical mixed
+            // but caller's recovery path is `olam upgrade --rollback`, not
+            // continue-and-hope.
+            steps[i] = { ...steps[i], canonicalAdvanced: false };
+            continue;
+        }
+        const r = dockerTag(p.transient, p.canonical);
+        steps[i] = {
+            ...steps[i],
+            canonicalAdvanced: r.ok,
+            ...(r.error !== undefined && { canonicalError: r.error }),
+        };
+        if (!r.ok) {
+            advanceFailed = true;
+            firstFailureIdx = i;
+        }
+    }
+    const allAdvanced = steps.every((s) => s.canonicalAdvanced);
+    const noneAdvanced = steps.every((s) => !s.canonicalAdvanced);
+    const partialAdvance = !allAdvanced && !noneAdvanced;
+    const rollbackCoherent = steps.every((s) => s.rollbackSaved);
+    let summary;
+    if (allAdvanced) {
+        const rollbacks = steps.filter((s) => s.rollbackSaved).length;
+        summary = `Swapped ${plan.length} canonical tags; ${rollbacks} :olam-rollback preserved`;
+    }
+    else if (partialAdvance) {
+        const advanced = steps.filter((s) => s.canonicalAdvanced).length;
+        const failedStep = steps[firstFailureIdx];
+        // Audit A6-001: only recommend --rollback when the rollback set is COHERENT.
+        // Otherwise the operator would either partially restore or hit Phase 2b's
+        // pre-flight refusal — misleading either way.
+        const recoveryHint = rollbackCoherent
+            ? `Run \`olam upgrade --rollback\` to restore coherent prior state.`
+            : `Rollback set INCOHERENT (${steps.filter((s) => s.rollbackSaved).length} of ${plan.length} :olam-rollback tags written). Manual recovery required: inspect images and re-tag canonical from a known-good source.`;
+        summary = `PARTIAL: ${advanced} of ${plan.length} canonical tags advanced before failure on ${failedStep?.image}: ${failedStep?.canonicalError}. ${recoveryHint}`;
+    }
+    else {
+        const failedStep = steps[firstFailureIdx];
+        summary = `Failed on first canonical-advance (${failedStep?.image}): ${failedStep?.canonicalError}. Canonical tags untouched.`;
+    }
+    return {
+        ok: allAdvanced,
+        steps,
+        partialAdvance,
+        rollbackCoherent,
+        summary,
+    };
+}
+/**
+ * Inverse of performAtomicSwap — restore canonical from :olam-rollback
+ * (Phase 2b — B1). Three sequential `docker tag` ops:
+ *
+ *   docker tag olam-auth:olam-rollback     olam-auth:local
+ *   docker tag olam-devbox:olam-rollback   olam-devbox:latest
+ *   docker tag olam-host-cp:olam-rollback  olam-host-cp:latest
+ *
+ * No two-phase ceremony — the source `:olam-rollback` set is already a
+ * coherent prior-good captured by a previous successful `olam upgrade`,
+ * so we don't need to preserve current canonical (it's known-broken,
+ * which is why we're rolling back).
+ *
+ * Caller MUST pre-flight via `checkRollbackSetExists()` before invoking.
+ * Behavior on missing source is per-image fatal (returns ok=false +
+ * error naming the missing image).
+ */
+export function performRollbackSwap(plan) {
+    const results = [];
+    for (const p of plan) {
+        const r = dockerTag(p.rollback, p.canonical);
+        results.push({
+            image: p.canonical,
+            ok: r.ok,
+            ...(r.error !== undefined && { error: r.error }),
+        });
+    }
+    const allOk = results.every((r) => r.ok);
+    const summary = allOk
+        ? `Rolled back ${plan.length} canonical tags from :olam-rollback`
+        : `PARTIAL rollback: ${results.filter((r) => r.ok).length} of ${plan.length} succeeded; failed: ${results.filter((r) => !r.ok).map((r) => r.image).join(', ')}`;
+    return { ok: allOk, results, summary };
+}
 async function confirm(message) {
     if (!process.stdin.isTTY)
         return true;
@@ -151,6 +483,144 @@ async function waitForHealth(timeoutMs = 10_000) {
     }
     return false;
 }
+/**
+ * Poll /api/version/status until all three component `.running` SHAs match
+ * `targetSha`, or until `timeoutMs` elapses. Returns the final snapshot.
+ *
+ * Phase 2a — A8: this is the success criterion for the entire upgrade.
+ * After A6's atomic swap + A7's recreate, the new images should report
+ * the new SHA via OLAM_BUILD_SHA baked at build time. Round-trip through
+ * Phase 1's detection path closes the loop.
+ *
+ * Returns:
+ *   - { matched: true, snapshot } when all three SHAs equal targetSha within timeout.
+ *   - { matched: false, snapshot } when timeout expires; caller decides
+ *     whether to warn (recreate succeeded but propagation slow) or error.
+ *   - { matched: false, snapshot: null } when /api/version/status is
+ *     unreachable for the entire timeout (host-cp didn't come back up).
+ */
+export async function waitForVersionMatch(targetSha, timeoutMs = 60_000, pollIntervalMs = 1_000) {
+    const deadline = Date.now() + timeoutMs;
+    let lastSnapshot = null;
+    while (Date.now() < deadline) {
+        try {
+            const res = await fetch('http://127.0.0.1:19000/api/version/status', {
+                signal: AbortSignal.timeout(2_000),
+            });
+            if (res.ok) {
+                const snapshot = (await res.json());
+                lastSnapshot = snapshot;
+                if (snapshot.hostCp?.running === targetSha &&
+                    snapshot.authService?.running === targetSha &&
+                    snapshot.devbox?.running === targetSha) {
+                    return { matched: true, snapshot };
+                }
+            }
+        }
+        catch {
+            // host-cp not yet ready or transient network blip
+        }
+        await new Promise((r) => setTimeout(r, pollIntervalMs));
+    }
+    return { matched: false, snapshot: lastSnapshot };
+}
+/**
+ * Format a version-snapshot mismatch into a readable per-component diff
+ * (Phase 2a — A8). Used in the timeout-warn path so the operator sees
+ * which component is lagging.
+ */
+export function formatVersionMismatch(targetSha, snapshot) {
+    if (!snapshot)
+        return 'No /api/version/status response received within timeout.';
+    const lines = [];
+    for (const [name, comp] of [
+        ['host-cp', snapshot.hostCp],
+        ['auth-service', snapshot.authService],
+        ['devbox', snapshot.devbox],
+    ]) {
+        const match = comp?.running === targetSha;
+        lines.push(`  ${match ? '✓' : '✗'} ${name}: running=${abbreviateSha(comp?.running ?? 'unknown')} target=${abbreviateSha(targetSha)}`);
+    }
+    return lines.join('\n');
+}
+/**
+ * Block until auth-service /health responds or timeout expires (Phase 2a — A7).
+ *
+ * Mirrors auth-upgrade.ts's waitForAuthHealth — kept inline to avoid a
+ * circular dep. When auth-upgrade.ts is refactored later (Phase G+),
+ * extract a shared helper.
+ */
+async function waitForAuthHealthLocal(timeoutMs = 15_000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        try {
+            const res = await fetch(AUTH_HEALTH_URL, { signal: AbortSignal.timeout(2000) });
+            if (res.ok)
+                return true;
+        }
+        catch {
+            // not up yet
+        }
+        await new Promise((r) => setTimeout(r, 500));
+    }
+    return false;
+}
+/**
+ * Recreate the auth-service container against the freshly-tagged
+ * `olam-auth:local` image (Phase 2a — A7).
+ *
+ * Mirrors auth-upgrade.ts:237-275: docker stop → docker rm →
+ * AuthContainerController.start(). Auth-service is NOT in compose.yaml
+ * (it runs via the controller's docker run with secret injection), so
+ * we cannot reuse `docker compose --force-recreate auth-service` —
+ * compose would fail with "no such service: auth-service" (verified
+ * during pass-2 review F2 audit).
+ *
+ * Errors:
+ *   - docker stop / docker rm errors are swallowed (container may not
+ *     be running or may not exist; both are recoverable states).
+ *   - AuthContainerController.start() throws on real failures (image
+ *     missing, port conflict, secret missing); caller catches and
+ *     reports.
+ *
+ * Returns true on successful recreate + /health response within 15s.
+ */
+async function recreateAuthService() {
+    const start = Date.now();
+    try {
+        // Step 1: stop + remove. Errors swallowed — container may be absent/stopped.
+        spawnSync('docker', ['stop', 'olam-auth'], {
+            encoding: 'utf-8',
+            stdio: ['ignore', 'ignore', 'ignore'],
+        });
+        spawnSync('docker', ['rm', 'olam-auth'], {
+            encoding: 'utf-8',
+            stdio: ['ignore', 'ignore', 'ignore'],
+        });
+        // Step 2: start the new container via the controller (handles secret
+        // injection; reads OLAM_AUTH_SECRET from env or ~/.olam/auth-secret).
+        const controller = new AuthContainerController();
+        controller.start();
+        // Step 3: wait for /health.
+        const healthy = await waitForAuthHealthLocal(15_000);
+        const durationMs = Date.now() - start;
+        if (!healthy) {
+            return {
+                ok: false,
+                durationMs,
+                error: 'auth-service /health did not respond within 15s after recreate',
+            };
+        }
+        return { ok: true, durationMs };
+    }
+    catch (err) {
+        return {
+            ok: false,
+            durationMs: Date.now() - start,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
 function readBundleHash(cwd) {
     const indexPath = path.join(cwd, 'packages/control-plane/public/index.html');
     if (!fs.existsSync(indexPath))
@@ -197,6 +667,174 @@ async function handleUpgrade(opts) {
             return;
         }
     }
+    // Phase 2c — C2: --history reads the audit log; no state changes.
+    if (opts.history) {
+        handleHistory(parseHistoryOpts({ n: opts.historyN, json: opts.historyJson }));
+        return;
+    }
+    // Phase 2b — B1: rollback path. Branches at the top so --rollback skips
+    // the entire git-pull/build/swap sequence and only retags + recreates.
+    if (opts.rollback) {
+        return await handleRollback();
+    }
+    // 3b. Acquire CLI lock (Phase 2a — A1). Refuses if a live upgrade is in flight;
+    //     auto-recovers stale locks (parse-error / empty / dead-pid / >5 min).
+    const lock = acquireLock();
+    if (!lock.acquired) {
+        printError(formatRefusalMessage(lock, LOCK_FILE_PATH));
+        process.exitCode = 1;
+        return;
+    }
+    // SIGINT / SIGTERM handler — release the lock before terminating so the
+    // operator's next invocation doesn't have to wait for stale-recovery
+    // (audit A1-005). `process.once` so a second Ctrl-C terminates immediately.
+    let signalReleased = false;
+    const releaseOnSignal = (signal) => {
+        if (signalReleased)
+            return;
+        signalReleased = true;
+        try {
+            releaseLock();
+        }
+        catch {
+            // best-effort
+        }
+        // Standard shell exit code for signal-induced termination: 128 + signal-number.
+        process.exit(signal === 'SIGINT' ? 130 : 143);
+    };
+    process.once('SIGINT', releaseOnSignal);
+    process.once('SIGTERM', releaseOnSignal);
+    // Phase 2c — C1: collect a JSONL log row. Mutated as the upgrade progresses;
+    // appended once on the way out (success OR failure) so --history surfaces
+    // every attempt, not just successful ones.
+    const logRow = {
+        started_at: Date.now(),
+        durations_ms: {},
+        sha_target: '',
+        failed_step: null,
+        status: 'failed', // default; flipped to 'success' on clean exit
+    };
+    try {
+        await runUpgradeStepsWithLockHeld(opts, cwd, logRow);
+        if (process.exitCode !== 1)
+            logRow.status = 'success';
+    }
+    finally {
+        const ended_at = Date.now();
+        const row = {
+            ts: new Date(ended_at).toISOString(),
+            started_at: logRow.started_at,
+            ended_at,
+            sha_target: logRow.sha_target,
+            status: logRow.status,
+            failed_step: logRow.failed_step,
+            durations_ms: logRow.durations_ms,
+        };
+        appendUpgradeLog(row);
+        releaseLock();
+        process.removeListener('SIGINT', releaseOnSignal);
+        process.removeListener('SIGTERM', releaseOnSignal);
+    }
+}
+/**
+ * Phase 2b — B1: handle `olam upgrade --rollback`.
+ *
+ * Pre-flights all three :olam-rollback tags exist; refuses with exit 1 if
+ * any missing. Else atomically retags :olam-rollback → canonical for all
+ * three images, then recreates host-cp (compose) + auth-service (controller).
+ * No git pull, no build, no smoke — the rollback target is a known-good
+ * image set captured by a previous successful upgrade.
+ *
+ * Acquires the same upgrade lock as the regular path so concurrent
+ * --rollback + --normal-upgrade refuse at the file-mutex layer.
+ */
+async function handleRollback() {
+    printHeader('olam upgrade --rollback');
+    // 1. Pre-flight — verify rollback set exists.
+    const missing = checkRollbackSetExists(PRODUCTION_SWAP_PLAN);
+    if (missing !== null) {
+        printError(`No rollback-set available — missing :olam-rollback tag(s): ${missing}\n\n` +
+            'A rollback-set is created by the FIRST successful `olam upgrade`. If this\n' +
+            'is your first install, run `olam upgrade` to populate the rollback set.\n' +
+            'If a previous upgrade was incomplete, the rollback set may be partial;\n' +
+            'manually inspect images with `docker images olam-*:olam-rollback`.');
+        process.exitCode = 1;
+        return;
+    }
+    // 2. Acquire lock (same primitive as the upgrade path).
+    const lock = acquireLock();
+    if (!lock.acquired) {
+        printError(formatRefusalMessage(lock, LOCK_FILE_PATH));
+        process.exitCode = 1;
+        return;
+    }
+    let signalReleased = false;
+    const releaseOnSignal = (signal) => {
+        if (signalReleased)
+            return;
+        signalReleased = true;
+        try {
+            releaseLock();
+        }
+        catch {
+            /* best-effort */
+        }
+        process.exit(signal === 'SIGINT' ? 130 : 143);
+    };
+    process.once('SIGINT', releaseOnSignal);
+    process.once('SIGTERM', releaseOnSignal);
+    try {
+        // 3. Inverse swap: 3 docker tag ops.
+        process.stdout.write(`  ${pc.dim('rollback retag (3 ops)'.padEnd(34))}`);
+        const swapStart = Date.now();
+        const swapResult = performRollbackSwap(PRODUCTION_SWAP_PLAN);
+        const swapDur = `${((Date.now() - swapStart) / 1000).toFixed(1)}s`;
+        process.stdout.write(`${swapResult.ok ? pc.green('✓') : pc.red('✗')} ${swapDur}\n`);
+        if (!swapResult.ok) {
+            printError(`Rollback retag failed: ${swapResult.summary}`);
+            process.exitCode = 1;
+            return;
+        }
+        printInfo('Rollback', swapResult.summary);
+        // 4. Recreate containers (host-cp via compose, auth via controller).
+        const cwd = process.cwd();
+        const composeFile = path.join(cwd, 'packages/host-cp/compose.yaml');
+        const authSecret = readAuthSecret();
+        process.stdout.write(`  ${pc.dim('docker compose recreate host-cp'.padEnd(34))}`);
+        const composeStart = Date.now();
+        const composeResult = runCompose(['up', '-d', '--force-recreate', 'host-cp'], composeFile, buildComposeEnv(authSecret));
+        const composeDur = `${((Date.now() - composeStart) / 1000).toFixed(1)}s`;
+        process.stdout.write(`${composeResult.ok ? pc.green('✓') : pc.red('✗')} ${composeDur}\n`);
+        if (!composeResult.ok) {
+            printError(`Rollback compose recreate failed:\n${composeResult.stderr}\n` +
+                'Canonical tags are at :olam-rollback (good); container restart pending. ' +
+                'Manually: `docker compose -f packages/host-cp/compose.yaml up -d --force-recreate host-cp`.');
+            process.exitCode = 1;
+            return;
+        }
+        process.stdout.write(`  ${pc.dim('recreate auth-service'.padEnd(34))}`);
+        const authResult = await recreateAuthService();
+        const authDur = `${(authResult.durationMs / 1000).toFixed(1)}s`;
+        process.stdout.write(`${authResult.ok ? pc.green('✓') : pc.red('✗')} ${authDur}\n`);
+        if (!authResult.ok) {
+            printError(`Auth-service recreate failed: ${authResult.error ?? 'unknown'}`);
+            process.exitCode = 1;
+            return;
+        }
+        process.stdout.write('\n');
+        printSuccess('Rollback complete — canonical tags restored from :olam-rollback');
+    }
+    finally {
+        releaseLock();
+        process.removeListener('SIGINT', releaseOnSignal);
+        process.removeListener('SIGTERM', releaseOnSignal);
+    }
+}
+/**
+ * Internal — runs all state-changing upgrade steps inside the lock.
+ * Extracted so handleUpgrade can wrap in try/finally without indenting the body.
+ */
+async function runUpgradeStepsWithLockHeld(opts, cwd, logRow) {
     // 4a. Branch switch (--branch).
     if (opts.branch !== null) {
         if (isGitDirty(cwd)) {
@@ -241,6 +879,21 @@ async function handleUpgrade(opts) {
         process.exitCode = 1;
         return;
     }
+    // Phase 2a — A2: capture HEAD SHA AFTER pull (sticky for the run).
+    // The pull is what we're upgrading to; capturing before would self-refuse
+    // at A6's swap-boundary drift check. _targetSha is consumed by A6 (atomic
+    // swap) and B4 (drift refusal). Phase 2a A2 just stashes it; A6/B4 land it
+    // load-bearing.
+    const _targetSha = captureHeadSha(cwd);
+    logRow.sha_target = _targetSha ?? '';
+    if (_targetSha === null) {
+        logRow.failed_step = 'capture HEAD SHA';
+        printError('Failed to capture HEAD SHA via `git rev-parse HEAD`. Aborting upgrade.\n' +
+            'Re-run from a clean git checkout; ensure `git rev-parse HEAD` returns a 40-char SHA.');
+        process.exitCode = 1;
+        return;
+    }
+    printInfo('Target SHA', abbreviateSha(_targetSha));
     // Step c: npm install (skip when in sync or --skip-install).
     const installDecision = shouldSkipInstall(opts, cwd);
     if (installDecision.skip) {
@@ -290,15 +943,133 @@ async function handleUpgrade(opts) {
         printTimings(timings);
         return;
     }
-    // Step f: docker image build.
-    const buildScript = path.join(cwd, 'packages/adapters/src/docker/build-host-cp.sh');
-    const imageResult = runStep('bash build-host-cp.sh', 'bash', [buildScript], { cwd });
-    timings.push({ label: 'docker image build', durationMs: imageResult.durationMs });
-    if (!imageResult.ok) {
-        printError(`Docker image build failed:\n${imageResult.stderr}`);
+    // Note: A4-A8 step durations are captured in `timings`; the per-step
+    // durations_ms snapshot on the log row reflects the full timings array
+    // at the end of the run (logRow.durations_ms is updated below at each
+    // significant boundary so a mid-run failure is recorded with what we know).
+    // Phase 2a — A4: sequential build invocation (auth → devbox → host-cp)
+    // with OLAM_TAG=olam-next so each script tags its image transiently per
+    // A3's retag block. Order is load-bearing: auth first minimises P3's
+    // in-flight 401 window when the recreate (A7) restarts auth before
+    // host-cp. Devbox uses inherit-stdio (live tee) per audit F13 since
+    // its cold-cache build dominates the 12-22 min budget and silent
+    // capture is indistinguishable from a hang.
+    // Phase 2b — B3: --no-cache passes through to all three build scripts.
+    // The build scripts honor DOCKER_BUILD_NO_CACHE via the build-arg env mechanism
+    // documented in their shell. (B3 implementation: forward the env; build
+    // scripts treat unset as default cache enabled.)
+    const olamTagEnv = { OLAM_TAG: 'olam-next' };
+    if (opts.noCache) {
+        olamTagEnv.DOCKER_BUILD_NO_CACHE = '1';
+    }
+    const buildScripts = [
+        { label: 'bash build-auth.sh', relPath: 'packages/adapters/src/docker/build-auth.sh', tee: false },
+        { label: 'bash build-devbox.sh', relPath: 'packages/adapters/src/docker/build-devbox.sh', tee: true },
+        { label: 'bash build-host-cp.sh', relPath: 'packages/adapters/src/docker/build-host-cp.sh', tee: false },
+    ];
+    for (const step of buildScripts) {
+        const scriptPath = path.join(cwd, step.relPath);
+        if (step.tee) {
+            // Live-tee variant: stdio: 'inherit' so docker build's apt/bundle/npm
+            // progress reaches the operator's terminal in real-time. No stdout
+            // capture means we can't include stderr in the failure message —
+            // operator already saw the failure inline.
+            process.stdout.write(`  ${pc.dim(step.label.padEnd(34))}\n`);
+            const start = Date.now();
+            const result = spawnSync('bash', [scriptPath], {
+                stdio: 'inherit',
+                cwd,
+                env: { ...process.env, ...olamTagEnv },
+            });
+            const durationMs = Date.now() - start;
+            const ok = result.status === 0 && result.error === undefined;
+            const dur = `${(durationMs / 1000).toFixed(1)}s`;
+            process.stdout.write(`  ${pc.dim(step.label.padEnd(34))}${ok ? pc.green('✓') : pc.red('✗')} ${dur}\n`);
+            timings.push({ label: step.label, durationMs });
+            if (!ok) {
+                printError(`${step.label} failed (see output above for details).`);
+                process.exitCode = 1;
+                return;
+            }
+        }
+        else {
+            const result = runStep(step.label, 'bash', [scriptPath], {
+                cwd,
+                env: olamTagEnv,
+            });
+            timings.push({ label: step.label, durationMs: result.durationMs });
+            logRow.durations_ms[step.label] = result.durationMs;
+            if (!result.ok) {
+                logRow.failed_step = step.label;
+                printError(`${step.label} failed:\n${result.stderr.split('\n').slice(-3).join('\n')}`);
+                process.exitCode = 1;
+                return;
+            }
+        }
+    }
+    // Snapshot durations to logRow so a later-step failure preserves what we know.
+    for (const t of timings)
+        logRow.durations_ms[t.label] = t.durationMs;
+    // Phase 2a — A5: smoke each :olam-next image via docker create + inspect.
+    // Catches build-corrupt cases (manifest invalid, OLAM_BUILD_SHA label
+    // missing, baked SHA != target SHA) before A6's atomic swap touches
+    // canonical tags. Sub-second per image; no port bind.
+    const smokeStart = Date.now();
+    process.stdout.write(`  ${pc.dim('smoke (docker create + inspect)'.padEnd(34))}`);
+    const smokeImages = [
+        'olam-auth:olam-next',
+        'olam-devbox:olam-next',
+        'olam-host-cp:olam-next',
+    ];
+    const smokeResults = smokeImages.map((img) => smokeImage(img, _targetSha));
+    const smokeFailures = smokeResults.filter((r) => !r.ok);
+    const smokeDurationMs = Date.now() - smokeStart;
+    const smokeDur = `${(smokeDurationMs / 1000).toFixed(1)}s`;
+    process.stdout.write(`${smokeFailures.length === 0 ? pc.green('✓') : pc.red('✗')} ${smokeDur}\n`);
+    timings.push({ label: 'smoke', durationMs: smokeDurationMs });
+    if (smokeFailures.length > 0) {
+        printError(`Smoke failed for ${smokeFailures.length} of ${smokeResults.length} images:\n` +
+            smokeFailures.map((r) => `  - ${r.image}: ${r.error}`).join('\n') +
+            '\nCanonical tags (`:latest`/`:local`) untouched. Investigate the failed image(s),' +
+            ' then re-run `olam upgrade` (--no-cache if cache-poisoning suspected).');
         process.exitCode = 1;
         return;
     }
+    // Phase 2b — B4: SHA drift check at swap boundary.
+    // Re-read HEAD via `git rev-parse HEAD` and compare to A2's captured
+    // _targetSha. If different, refuse the swap unless --force.
+    const swapBoundarySha = captureHeadSha(cwd);
+    if (swapBoundarySha !== null && swapBoundarySha !== _targetSha && !opts.force) {
+        printError(`HEAD drifted during build window:\n` +
+            `  captured (after pull): ${abbreviateSha(_targetSha)}\n` +
+            `  current at swap:       ${abbreviateSha(swapBoundarySha)}\n\n` +
+            'Operator-driven `git checkout` or `git reset` triggered drift.\n' +
+            'Recovery options:\n' +
+            '  • Re-run `olam upgrade` (will rebuild against current HEAD).\n' +
+            '  • Pass `--force` to swap anyway (canonical advances to the\n' +
+            '    captured-at-pull SHA, NOT current HEAD).');
+        process.exitCode = 1;
+        return;
+    }
+    // Phase 2a — A6: atomic 6-tag swap.
+    // Phase 1 of swap: preserve previous-good as :olam-rollback (3 ops).
+    // Phase 2 of swap: advance canonical to :olam-next (3 ops).
+    // Sub-second wall-clock; SIGKILL during Phase 2 is recoverable via
+    // `olam upgrade --rollback` (Phase 2b) since :olam-rollback is fully
+    // populated before any canonical tag is touched.
+    process.stdout.write(`  ${pc.dim('atomic 6-tag swap'.padEnd(34))}`);
+    const swapStart = Date.now();
+    const swapResult = performAtomicSwap(PRODUCTION_SWAP_PLAN);
+    const swapDurationMs = Date.now() - swapStart;
+    const swapDur = `${(swapDurationMs / 1000).toFixed(1)}s`;
+    process.stdout.write(`${swapResult.ok ? pc.green('✓') : pc.red('✗')} ${swapDur}\n`);
+    timings.push({ label: 'atomic swap', durationMs: swapDurationMs });
+    if (!swapResult.ok) {
+        printError(`Atomic swap failed: ${swapResult.summary}`);
+        process.exitCode = 1;
+        return;
+    }
+    printInfo('Swap', swapResult.summary);
     // Step g: docker compose up -d --force-recreate.
     const composeFile = path.join(cwd, 'packages/host-cp/compose.yaml');
     process.stdout.write(`  ${pc.dim('docker compose recreate'.padEnd(34))}`);
@@ -310,11 +1081,36 @@ async function handleUpgrade(opts) {
     process.stdout.write(`${composeOk ? pc.green('✓') : pc.red('✗')} ${composeDur}\n`);
     timings.push({ label: 'container recreate', durationMs: composeDurationMs });
     if (!composeOk) {
-        printError(`docker compose up --force-recreate failed:\n${composeResult.stderr}`);
+        // Audit A6-002: at this point canonical tags are at NEW SHA but the stack
+        // failed to start. Operator needs to know --rollback is one command away.
+        printError(`docker compose up --force-recreate failed:\n${composeResult.stderr}\n\n` +
+            'Canonical tags advanced to new SHA but the stack failed to start.\n' +
+            'Recovery options:\n' +
+            '  • Run `olam upgrade --rollback` to restore the prior :olam-rollback set, then investigate.\n' +
+            '  • Manually `docker logs olam-host-cp` to diagnose; if recoverable, retry recreate without rollback.');
+        process.exitCode = 1;
+        return;
+    }
+    // Phase 2a — A7: recreate auth-service via AuthContainerController.
+    // Auth is NOT in compose.yaml; reusing the auth-upgrade.ts recreate pattern
+    // (docker stop → docker rm → controller.start() → wait /health). The 25s
+    // in-flight 401 window for active world API calls during this recreate is
+    // documented in the operator's confirmation prompt (P3 mitigation).
+    process.stdout.write(`  ${pc.dim('recreate auth-service'.padEnd(34))}`);
+    const authResult = await recreateAuthService();
+    const authDur = `${(authResult.durationMs / 1000).toFixed(1)}s`;
+    process.stdout.write(`${authResult.ok ? pc.green('✓') : pc.red('✗')} ${authDur}\n`);
+    timings.push({ label: 'auth recreate', durationMs: authResult.durationMs });
+    if (!authResult.ok) {
+        printError(`Auth-service recreate failed: ${authResult.error ?? 'unknown'}\n\n` +
+            'Canonical tags advanced to new SHA; host-cp recreated but auth-service is broken.\n' +
+            'Recovery options:\n' +
+            '  • Run `olam upgrade --rollback` to restore the prior :olam-rollback set + working stack.\n' +
+            '  • Manually: `docker logs olam-auth` to diagnose; `olam auth up` to restart.');
         process.exitCode = 1;
         return;
     }
-    // Step h: wait for /health.
+    // Step h: wait for /health (host-cp readiness probe).
     process.stdout.write(`  ${pc.dim('waiting for /health'.padEnd(34))}`);
     const healthStart = Date.now();
     const healthy = await waitForHealth(10_000);
@@ -323,7 +1119,31 @@ async function handleUpgrade(opts) {
     process.stdout.write(`${healthy ? pc.green('✓') : pc.yellow('?')} ${healthDur}\n`);
     timings.push({ label: '/health', durationMs: healthDurationMs });
     if (!healthy) {
-        printWarning('Host CP started but /health did not respond within 10s. Check: docker logs olam-host-cp');
+        printWarning('Host CP started but /health did not respond within 10s.\n' +
+            '  • Check: docker logs olam-host-cp\n' +
+            '  • If the new SHA is broken: `olam upgrade --rollback` restores the prior set in <30s.');
+    }
+    // Phase 2a — A8: poll /api/version/status until all three SHAs match
+    // captured target. This is the success criterion for the entire upgrade —
+    // round-trips through Phase 1's detection path so the SPA banner clears
+    // automatically once the polling loop succeeds.
+    process.stdout.write(`  ${pc.dim('verify /version/status round-trip'.padEnd(34))}`);
+    const versionStart = Date.now();
+    const versionMatch = await waitForVersionMatch(_targetSha, 60_000);
+    const versionDurationMs = Date.now() - versionStart;
+    const versionDur = `${(versionDurationMs / 1000).toFixed(1)}s`;
+    process.stdout.write(`${versionMatch.matched ? pc.green('✓') : pc.yellow('?')} ${versionDur}\n`);
+    timings.push({ label: '/version/status round-trip', durationMs: versionDurationMs });
+    if (!versionMatch.matched) {
+        // Non-fatal — recreate succeeded; SHA propagation may be slow on cold
+        // host-cp boot. Operator gets diagnostic output + can decide whether to
+        // re-run, wait, or roll back.
+        printWarning(`Version round-trip incomplete after ${(versionDurationMs / 1000).toFixed(0)}s:\n` +
+            formatVersionMismatch(_targetSha, versionMatch.snapshot) + '\n' +
+            '  • Banner may still show UPDATE AVAILABLE until host-cp\'s next ' +
+            'poll cycle (~60s).\n' +
+            '  • If the mismatch persists, `olam upgrade --rollback` restores ' +
+            'the prior :olam-rollback set.');
     }
     // 5. Summary.
     process.stdout.write('\n');
@@ -345,12 +1165,22 @@ function printTimings(timings) {
 export function registerUpgrade(program) {
     program
         .command('upgrade')
-        .description('Self-upgrade the local Olam dev stack (pull + rebuild + restart host-cp)')
+        .description('Self-upgrade the local Olam dev stack (pull + rebuild + restart all three components)')
         .option('-y, --yes', 'Skip the confirmation prompt')
         .option('--skip-image', 'Skip docker image rebuild + container recreate (source rebuild only)')
         .option('--skip-install', 'Skip npm install entirely (use existing node_modules as-is). ' +
         'Useful when a native-module build failure blocks the normal upgrade path.')
         .option('--branch <name>', 'Switch to this branch before pulling (refuses if working tree is dirty)')
+        .option('--rollback', 'Restore canonical tags from the :olam-rollback set (created by the prior successful upgrade).\n' +
+        '                              No git pull, no build, no smoke — just retag + recreate.')
+        .option('--force', 'Bypass HEAD-drift refusal at the swap boundary. Swap advances canonical to the\n' +
+        '                              captured-at-pull SHA even if current HEAD differs.')
+        .option('--no-cache', 'Pass --no-cache to all three build scripts (DOCKER_BUILD_NO_CACHE=1).\n' +
+        '                              Useful when retrying after a cache-poisoning failure.')
+        .option('--history', 'Print the upgrade history (~/.olam/upgrade.log) and exit.\n' +
+        '                              No upgrade is performed.')
+        .option('-n <count>', 'Number of history rows to print (default 10)', '10')
+        .option('--json', 'Emit history as JSONL instead of a table')
         .action(async (opts) => {
         await handleUpgrade(parseUpgradeOpts(opts));
     });