@wopr-network/platform-core 1.22.0 → 1.23.0

package/dist/fleet/__tests__/rollout-orchestrator.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/fleet/__tests__/rollout-orchestrator.test.js

@@ -0,0 +1,262 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { RolloutOrchestrator } from "../rollout-orchestrator.js";
+function makeProfile(id, volumeName) {
+    return {
+        id,
+        tenantId: "tenant-1",
+        name: `bot-${id}`,
+        description: "",
+        image: "ghcr.io/wopr-network/paperclip:managed",
+        env: {},
+        restartPolicy: "unless-stopped",
+        releaseChannel: "stable",
+        updatePolicy: "nightly",
+        volumeName,
+    };
+}
+function makeResult(botId, success) {
+    return {
+        botId,
+        success,
+        previousImage: "old:latest",
+        newImage: "new:latest",
+        previousDigest: "sha256:old",
+        newDigest: "sha256:new",
+        rolledBack: !success,
+        error: success ? undefined : "Health check failed",
+    };
+}
+function mockUpdater(results) {
+    return {
+        updateBot: vi.fn(async (botId) => results.get(botId) ?? makeResult(botId, true)),
+    };
+}
+function mockSnapshotManager() {
+    return {
+        snapshot: vi.fn(async (volumeName) => ({
+            id: `${volumeName}-snap`,
+            volumeName,
+            archivePath: `/backup/${volumeName}-snap.tar`,
+            createdAt: new Date(),
+            sizeBytes: 1024,
+        })),
+        restore: vi.fn(async () => { }),
+        delete: vi.fn(async () => { }),
+    };
+}
+function mockStrategy(overrides = {}) {
+    return {
+        nextBatch: (remaining) => remaining.slice(0, 2),
+        pauseDuration: () => 0,
+        onBotFailure: () => "skip",
+        maxRetries: () => 2,
+        healthCheckTimeout: () => 120_000,
+        ...overrides,
+    };
+}
+describe("RolloutOrchestrator", () => {
+    let updater;
+    let snapMgr;
+    beforeEach(() => {
+        vi.clearAllMocks();
+        updater = mockUpdater(new Map());
+        snapMgr = mockSnapshotManager();
+    });
+    it("processes all bots in batches", async () => {
+        const profiles = [makeProfile("b1", "vol-1"), makeProfile("b2", "vol-2"), makeProfile("b3", "vol-3")];
+        const strategy = mockStrategy({ nextBatch: (r) => r.slice(0, 2) });
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy,
+            getUpdatableProfiles: async () => profiles,
+        });
+        const result = await orch.rollout();
+        expect(result.totalBots).toBe(3);
+        expect(result.succeeded).toBe(3);
+        expect(result.failed).toBe(0);
+        expect(result.aborted).toBe(false);
+        expect(updater.updateBot).toHaveBeenCalledTimes(3);
+    });
+    it("snapshots volumes before updating", async () => {
+        const profiles = [makeProfile("b1", "my-volume")];
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+        });
+        await orch.rollout();
+        expect(snapMgr.snapshot).toHaveBeenCalledWith("my-volume");
+        // On success, snapshot is cleaned up
+        expect(snapMgr.delete).toHaveBeenCalledWith("my-volume-snap");
+    });
+    it("skips snapshot for bots without volumes", async () => {
+        const profiles = [makeProfile("b1")]; // no volumeName
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+        });
+        await orch.rollout();
+        expect(snapMgr.snapshot).not.toHaveBeenCalled();
+        expect(updater.updateBot).toHaveBeenCalledWith("b1");
+    });
+    it("restores volumes on update failure", async () => {
+        const failResults = new Map([["b1", makeResult("b1", false)]]);
+        updater = mockUpdater(failResults);
+        const profiles = [makeProfile("b1", "my-volume")];
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+        });
+        const result = await orch.rollout();
+        expect(result.failed).toBe(1);
+        expect(result.results[0].volumeRestored).toBe(true);
+        expect(snapMgr.restore).toHaveBeenCalledWith("my-volume-snap");
+        // Snapshot NOT deleted on failure (restored instead)
+        expect(snapMgr.delete).not.toHaveBeenCalled();
+    });
+    it("aborts rollout when strategy says abort", async () => {
+        const failResults = new Map([["b1", makeResult("b1", false)]]);
+        updater = mockUpdater(failResults);
+        const profiles = [makeProfile("b1", "v1"), makeProfile("b2", "v2"), makeProfile("b3", "v3")];
+        const strategy = mockStrategy({
+            nextBatch: (r) => r.slice(0, 1),
+            onBotFailure: () => "abort",
+        });
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy,
+            getUpdatableProfiles: async () => profiles,
+        });
+        const result = await orch.rollout();
+        expect(result.aborted).toBe(true);
+        expect(result.succeeded).toBe(0);
+        expect(result.failed).toBe(1);
+        expect(result.skipped).toBe(2); // b2, b3 never processed
+        expect(updater.updateBot).toHaveBeenCalledTimes(1);
+    });
+    it("returns empty result when no bots to update", async () => {
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => [],
+        });
+        const result = await orch.rollout();
+        expect(result.totalBots).toBe(0);
+        expect(result.results).toHaveLength(0);
+    });
+    it("rejects concurrent rollouts", async () => {
+        const profiles = [makeProfile("b1")];
+        // Make updateBot slow
+        updater = {
+            updateBot: vi.fn(async (botId) => {
+                await new Promise((r) => setTimeout(r, 100));
+                return makeResult(botId, true);
+            }),
+        };
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+        });
+        const [r1, r2] = await Promise.all([orch.rollout(), orch.rollout()]);
+        // One succeeds, one is rejected as already running
+        const succeeded = [r1, r2].find((r) => r.totalBots > 0);
+        const rejected = [r1, r2].find((r) => r.alreadyRunning);
+        expect(succeeded).toBeDefined();
+        expect(rejected).toBeDefined();
+        expect(rejected?.alreadyRunning).toBe(true);
+        expect(rejected?.totalBots).toBe(0);
+    });
+    it("retries failed bots when strategy says retry", async () => {
+        let callCount = 0;
+        updater = {
+            updateBot: vi.fn(async (botId) => {
+                callCount++;
+                // Fail first attempt, succeed on retry
+                if (botId === "b1" && callCount === 1)
+                    return makeResult("b1", false);
+                return makeResult(botId, true);
+            }),
+        };
+        const profiles = [makeProfile("b1")];
+        const strategy = mockStrategy({
+            nextBatch: (r) => r.slice(0, 1),
+            onBotFailure: (_botId, _err, attempt) => (attempt < 2 ? "retry" : "skip"),
+        });
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy,
+            getUpdatableProfiles: async () => profiles,
+        });
+        const result = await orch.rollout();
+        // b1 failed once, retried, succeeded
+        expect(updater.updateBot).toHaveBeenCalledTimes(2);
+        expect(result.succeeded).toBe(1);
+        expect(result.failed).toBe(1); // first attempt counted as failed
+    });
+    it("calls onBotUpdated callback for each bot", async () => {
+        const profiles = [makeProfile("b1"), makeProfile("b2")];
+        const onBotUpdated = vi.fn();
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+            onBotUpdated,
+        });
+        await orch.rollout();
+        expect(onBotUpdated).toHaveBeenCalledTimes(2);
+    });
+    it("calls onRolloutComplete callback", async () => {
+        const profiles = [makeProfile("b1")];
+        const onRolloutComplete = vi.fn();
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+            onRolloutComplete,
+        });
+        await orch.rollout();
+        expect(onRolloutComplete).toHaveBeenCalledTimes(1);
+        expect(onRolloutComplete).toHaveBeenCalledWith(expect.objectContaining({ totalBots: 1, succeeded: 1, aborted: false }));
+    });
+    it("continues on snapshot failure (best-effort)", async () => {
+        const profiles = [makeProfile("b1", "my-volume")];
+        snapMgr.snapshot = vi.fn().mockRejectedValue(new Error("disk full"));
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+        });
+        const result = await orch.rollout();
+        // Update still proceeds despite snapshot failure
+        expect(result.succeeded).toBe(1);
+        expect(updater.updateBot).toHaveBeenCalledWith("b1");
+    });
+    it("isRolling reflects rollout state", async () => {
+        const profiles = [makeProfile("b1")];
+        const orch = new RolloutOrchestrator({
+            updater,
+            snapshotManager: snapMgr,
+            strategy: mockStrategy(),
+            getUpdatableProfiles: async () => profiles,
+        });
+        expect(orch.isRolling).toBe(false);
+        const promise = orch.rollout();
+        // isRolling is true during rollout (may already be done for sync mocks)
+        await promise;
+        expect(orch.isRolling).toBe(false);
+    });
+});

package/dist/fleet/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-orchestrator.js";
 export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";

package/dist/fleet/index.js

@@ -1,4 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-orchestrator.js";
 export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";

package/dist/fleet/rollout-orchestrator.d.ts

@@ -0,0 +1,69 @@
+/**
+ * RolloutOrchestrator — coordinates fleet-wide container updates using
+ * pluggable rollout strategies and volume snapshots for nuclear rollback.
+ *
+ * Sits between ImagePoller (detects new digests) and ContainerUpdater
+ * (handles per-bot pull/stop/recreate/health). Adds:
+ * - Strategy-driven batching (rolling wave, single bot, immediate)
+ * - Pre-update volume snapshots via VolumeSnapshotManager
+ * - Volume restore on health check failure (nuclear rollback)
+ * - Per-tenant update orchestration
+ */
+import type { IRolloutStrategy } from "./rollout-strategy.js";
+import type { BotProfile } from "./types.js";
+import type { ContainerUpdater, UpdateResult } from "./updater.js";
+import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
+export interface RolloutOrchestratorDeps {
+    updater: ContainerUpdater;
+    snapshotManager: VolumeSnapshotManager;
+    strategy: IRolloutStrategy;
+    /** Resolve running profiles that need updating for a given image digest */
+    getUpdatableProfiles: () => Promise<BotProfile[]>;
+    /** Optional callback after each bot update (success or failure) */
+    onBotUpdated?: (result: UpdateResult & {
+        volumeRestored: boolean;
+    }) => void;
+    /** Optional callback when a rollout completes */
+    onRolloutComplete?: (results: RolloutResult) => void;
+}
+export interface BotUpdateResult extends UpdateResult {
+    volumeRestored: boolean;
+}
+export interface RolloutResult {
+    totalBots: number;
+    succeeded: number;
+    failed: number;
+    skipped: number;
+    aborted: boolean;
+    /** True when a concurrent rollout was already in progress */
+    alreadyRunning: boolean;
+    results: BotUpdateResult[];
+}
+export declare class RolloutOrchestrator {
+    private readonly updater;
+    private readonly snapshotManager;
+    private readonly strategy;
+    private readonly getUpdatableProfiles;
+    private readonly onBotUpdated?;
+    private readonly onRolloutComplete?;
+    private rolling;
+    constructor(deps: RolloutOrchestratorDeps);
+    /** Whether a rollout is currently in progress. */
+    get isRolling(): boolean;
+    /**
+     * Execute a rollout across all updatable bots.
+     * Uses the configured strategy for batching, pausing, and failure handling.
+     */
+    rollout(): Promise<RolloutResult>;
+    /**
+     * Update a single bot with volume snapshot + nuclear rollback.
+     */
+    private updateBot;
+    /**
+     * Handle a bot failure using the strategy's failure policy.
+     * Retries the update up to maxRetries before escalating.
+     */
+    private handleFailure;
+    private restoreVolumes;
+    private cleanupSnapshots;
+}

package/dist/fleet/rollout-orchestrator.js

@@ -0,0 +1,204 @@
+/**
+ * RolloutOrchestrator — coordinates fleet-wide container updates using
+ * pluggable rollout strategies and volume snapshots for nuclear rollback.
+ *
+ * Sits between ImagePoller (detects new digests) and ContainerUpdater
+ * (handles per-bot pull/stop/recreate/health). Adds:
+ * - Strategy-driven batching (rolling wave, single bot, immediate)
+ * - Pre-update volume snapshots via VolumeSnapshotManager
+ * - Volume restore on health check failure (nuclear rollback)
+ * - Per-tenant update orchestration
+ */
+import { logger } from "../config/logger.js";
+export class RolloutOrchestrator {
+    updater;
+    snapshotManager;
+    strategy;
+    getUpdatableProfiles;
+    onBotUpdated;
+    onRolloutComplete;
+    rolling = false;
+    constructor(deps) {
+        this.updater = deps.updater;
+        this.snapshotManager = deps.snapshotManager;
+        this.strategy = deps.strategy;
+        this.getUpdatableProfiles = deps.getUpdatableProfiles;
+        this.onBotUpdated = deps.onBotUpdated;
+        this.onRolloutComplete = deps.onRolloutComplete;
+    }
+    /** Whether a rollout is currently in progress. */
+    get isRolling() {
+        return this.rolling;
+    }
+    /**
+     * Execute a rollout across all updatable bots.
+     * Uses the configured strategy for batching, pausing, and failure handling.
+     */
+    async rollout() {
+        if (this.rolling) {
+            logger.warn("Rollout already in progress — skipping");
+            return { totalBots: 0, succeeded: 0, failed: 0, skipped: 0, aborted: false, alreadyRunning: true, results: [] };
+        }
+        this.rolling = true;
+        const allResults = [];
+        let aborted = false;
+        try {
+            let remaining = await this.getUpdatableProfiles();
+            const totalBots = remaining.length;
+            if (totalBots === 0) {
+                logger.info("Rollout: no bots to update");
+                return {
+                    totalBots: 0,
+                    succeeded: 0,
+                    failed: 0,
+                    skipped: 0,
+                    aborted: false,
+                    alreadyRunning: false,
+                    results: [],
+                };
+            }
+            logger.info(`Rollout starting: ${totalBots} bots to update`);
+            while (remaining.length > 0 && !aborted) {
+                const batch = this.strategy.nextBatch(remaining);
+                if (batch.length === 0)
+                    break;
+                logger.info(`Rollout wave: ${batch.length} bots (${remaining.length} remaining)`);
+                // Process batch — each bot sequentially within a wave for safety
+                const retryProfiles = [];
+                for (const profile of batch) {
+                    if (aborted)
+                        break;
+                    const result = await this.updateBot(profile);
+                    allResults.push(result);
+                    this.onBotUpdated?.(result);
+                    if (!result.success) {
+                        const action = this.handleFailure(profile.id, result, allResults);
+                        if (action === "abort") {
+                            aborted = true;
+                            logger.warn(`Rollout aborted after bot ${profile.id} failure`);
+                        }
+                        else if (action === "retry") {
+                            retryProfiles.push(profile);
+                        }
+                        // "skip" → don't re-add, bot is dropped
+                    }
+                }
+                // Remove processed bots from remaining, but re-add retries
+                const processedIds = new Set(batch.map((b) => b.id));
+                const retryIds = new Set(retryProfiles.map((b) => b.id));
+                remaining = [
+                    ...remaining.filter((b) => !processedIds.has(b.id)),
+                    ...retryProfiles.filter((b) => retryIds.has(b.id)),
+                ];
+                // Pause between waves (unless aborted or done)
+                if (remaining.length > 0 && !aborted) {
+                    const pause = this.strategy.pauseDuration();
+                    if (pause > 0) {
+                        logger.info(`Rollout: pausing ${pause}ms before next wave`);
+                        await sleep(pause);
+                    }
+                }
+            }
+            const succeeded = allResults.filter((r) => r.success).length;
+            const failed = allResults.filter((r) => !r.success).length;
+            const skipped = totalBots - allResults.length;
+            const rolloutResult = {
+                totalBots,
+                succeeded,
+                failed,
+                skipped,
+                aborted,
+                alreadyRunning: false,
+                results: allResults,
+            };
+            logger.info(`Rollout complete: ${succeeded} succeeded, ${failed} failed, ${skipped} skipped, aborted=${aborted}`);
+            this.onRolloutComplete?.(rolloutResult);
+            return rolloutResult;
+        }
+        finally {
+            this.rolling = false;
+        }
+    }
+    /**
+     * Update a single bot with volume snapshot + nuclear rollback.
+     */
+    async updateBot(profile) {
+        const snapshotIds = [];
+        try {
+            // Step 1: Snapshot volumes before update
+            if (profile.volumeName) {
+                try {
+                    const snap = await this.snapshotManager.snapshot(profile.volumeName);
+                    snapshotIds.push(snap.id);
+                    logger.info(`Pre-update snapshot for ${profile.id}: ${snap.id}`);
+                }
+                catch (err) {
+                    logger.warn(`Volume snapshot failed for ${profile.id} — proceeding without backup`, { err });
+                }
+            }
+            // Step 2: Delegate to ContainerUpdater
+            const result = await this.updater.updateBot(profile.id);
+            if (result.success) {
+                // Clean up snapshots on success
+                await this.cleanupSnapshots(snapshotIds);
+                return { ...result, volumeRestored: false };
+            }
+            // Step 3: Nuclear rollback — restore volumes if update failed
+            const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
+            return { ...result, volumeRestored };
+        }
+        catch (err) {
+            logger.error(`Unexpected error updating bot ${profile.id}`, { err });
+            // Attempt volume restore on unexpected errors too
+            const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
+            return {
+                botId: profile.id,
+                success: false,
+                previousImage: profile.image,
+                newImage: profile.image,
+                previousDigest: null,
+                newDigest: null,
+                rolledBack: false,
+                volumeRestored,
+                error: err instanceof Error ? err.message : String(err),
+            };
+        }
+    }
+    /**
+     * Handle a bot failure using the strategy's failure policy.
+     * Retries the update up to maxRetries before escalating.
+     */
+    handleFailure(botId, result, allResults) {
+        const error = new Error(result.error ?? "Unknown error");
+        const failCount = allResults.filter((r) => r.botId === botId && !r.success).length;
+        return this.strategy.onBotFailure(botId, error, failCount);
+    }
+    async restoreVolumes(botId, snapshotIds) {
+        if (snapshotIds.length === 0)
+            return false;
+        for (const id of snapshotIds) {
+            try {
+                await this.snapshotManager.restore(id);
+                logger.info(`Volume restored for ${botId} from snapshot ${id}`);
+                return true;
+            }
+            catch (err) {
+                logger.error(`Volume restore failed for ${botId} snapshot ${id}`, { err });
+            }
+        }
+        return false;
+    }
+    async cleanupSnapshots(snapshotIds) {
+        for (const id of snapshotIds) {
+            try {
+                await this.snapshotManager.delete(id);
+            }
+            catch (err) {
+                logger.warn(`Failed to clean up snapshot ${id}`, { err });
+            }
+        }
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/fleet/services.d.ts

@@ -14,6 +14,8 @@ import type { IAffiliateRepository } from "../monetization/affiliate/drizzle-aff
 import type { IBotBilling } from "../monetization/credits/bot-billing.js";
 import type { IPhoneNumberRepository } from "../monetization/credits/drizzle-phone-number-repository.js";
 import { SystemResourceMonitor } from "../observability/system-resources.js";
+import type { RolloutOrchestrator } from "./rollout-orchestrator.js";
+import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
 import { AdminNotifier } from "./admin-notifier.js";
 import type { IBotInstanceRepository } from "./bot-instance-repository.js";
 import type { IBotProfileRepository } from "./bot-profile-repository.js";
@@ -79,6 +81,10 @@ export declare function getCapacityPolicy(configOverrides?: Partial<CapacityPoli
 export declare function getRestoreLogStore(): IRestoreLogStore;
 export declare function getBackupStatusStore(): IBackupStatusStore;
 export declare function getSnapshotManager(): SnapshotManager;
+export declare function getVolumeSnapshotManager(): VolumeSnapshotManager;
+export declare function setVolumeSnapshotManager(mgr: VolumeSnapshotManager): void;
+export declare function getRolloutOrchestrator(): RolloutOrchestrator;
+export declare function setRolloutOrchestrator(orch: RolloutOrchestrator): void;
 export declare function getRestoreService(): RestoreService;
 /** Call once at server startup to wire up fleet services. */
 export declare function initFleet(): void;

package/dist/fleet/services.js

@@ -105,6 +105,8 @@ let _restoreLogStore = null;
 let _restoreService = null;
 let _backupStatusStore = null;
 let _snapshotManager = null;
+let _volumeSnapshotManager = null;
+let _rolloutOrchestrator = null;
 const S3_BUCKET = process.env.S3_BUCKET || "wopr-backups";
 function envInt(key, fallback) {
     const raw = process.env[key];
@@ -427,6 +429,24 @@ export function getSnapshotManager() {
     }
     return _snapshotManager;
 }
+export function getVolumeSnapshotManager() {
+    if (!_volumeSnapshotManager) {
+        throw new Error("VolumeSnapshotManager not initialized — call setVolumeSnapshotManager() first");
+    }
+    return _volumeSnapshotManager;
+}
+export function setVolumeSnapshotManager(mgr) {
+    _volumeSnapshotManager = mgr;
+}
+export function getRolloutOrchestrator() {
+    if (!_rolloutOrchestrator) {
+        throw new Error("RolloutOrchestrator not initialized — call setRolloutOrchestrator() first");
+    }
+    return _rolloutOrchestrator;
+}
+export function setRolloutOrchestrator(orch) {
+    _rolloutOrchestrator = orch;
+}
 export function getRestoreService() {
     if (!_restoreService) {
         _restoreService = new RestoreService({
@@ -683,6 +703,8 @@ export function _resetForTest() {
     _restoreService = null;
     _backupStatusStore = null;
     _snapshotManager = null;
+    _volumeSnapshotManager = null;
+    _rolloutOrchestrator = null;
     _botBilling = null;
     _phoneNumberRepo = null;
     _affiliateRepo = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wopr-network/platform-core",
-  "version": "1.22.0",
+  "version": "1.23.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/fleet/__tests__/rollout-orchestrator.test.ts

@@ -0,0 +1,321 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { RolloutOrchestrator } from "../rollout-orchestrator.js";
+import type { IRolloutStrategy } from "../rollout-strategy.js";
+import type { BotProfile } from "../types.js";
+import type { ContainerUpdater, UpdateResult } from "../updater.js";
+import type { VolumeSnapshotManager } from "../volume-snapshot-manager.js";
+
+function makeProfile(id: string, volumeName?: string): BotProfile {
+  return {
+    id,
+    tenantId: "tenant-1",
+    name: `bot-${id}`,
+    description: "",
+    image: "ghcr.io/wopr-network/paperclip:managed",
+    env: {},
+    restartPolicy: "unless-stopped",
+    releaseChannel: "stable",
+    updatePolicy: "nightly",
+    volumeName,
+  } as BotProfile;
+}
+
+function makeResult(botId: string, success: boolean): UpdateResult {
+  return {
+    botId,
+    success,
+    previousImage: "old:latest",
+    newImage: "new:latest",
+    previousDigest: "sha256:old",
+    newDigest: "sha256:new",
+    rolledBack: !success,
+    error: success ? undefined : "Health check failed",
+  };
+}
+
+function mockUpdater(results: Map<string, UpdateResult>): ContainerUpdater {
+  return {
+    updateBot: vi.fn(async (botId: string) => results.get(botId) ?? makeResult(botId, true)),
+  } as unknown as ContainerUpdater;
+}
+
+function mockSnapshotManager(): VolumeSnapshotManager {
+  return {
+    snapshot: vi.fn(async (volumeName: string) => ({
+      id: `${volumeName}-snap`,
+      volumeName,
+      archivePath: `/backup/${volumeName}-snap.tar`,
+      createdAt: new Date(),
+      sizeBytes: 1024,
+    })),
+    restore: vi.fn(async () => {}),
+    delete: vi.fn(async () => {}),
+  } as unknown as VolumeSnapshotManager;
+}
+
+function mockStrategy(overrides: Partial<IRolloutStrategy> = {}): IRolloutStrategy {
+  return {
+    nextBatch: (remaining) => remaining.slice(0, 2),
+    pauseDuration: () => 0,
+    onBotFailure: () => "skip",
+    maxRetries: () => 2,
+    healthCheckTimeout: () => 120_000,
+    ...overrides,
+  };
+}
+
+describe("RolloutOrchestrator", () => {
+  let updater: ReturnType<typeof mockUpdater>;
+  let snapMgr: ReturnType<typeof mockSnapshotManager>;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    updater = mockUpdater(new Map());
+    snapMgr = mockSnapshotManager();
+  });
+
+  it("processes all bots in batches", async () => {
+    const profiles = [makeProfile("b1", "vol-1"), makeProfile("b2", "vol-2"), makeProfile("b3", "vol-3")];
+    const strategy = mockStrategy({ nextBatch: (r) => r.slice(0, 2) });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.totalBots).toBe(3);
+    expect(result.succeeded).toBe(3);
+    expect(result.failed).toBe(0);
+    expect(result.aborted).toBe(false);
+    expect(updater.updateBot).toHaveBeenCalledTimes(3);
+  });
+
+  it("snapshots volumes before updating", async () => {
+    const profiles = [makeProfile("b1", "my-volume")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    await orch.rollout();
+
+    expect(snapMgr.snapshot).toHaveBeenCalledWith("my-volume");
+    // On success, snapshot is cleaned up
+    expect(snapMgr.delete).toHaveBeenCalledWith("my-volume-snap");
+  });
+
+  it("skips snapshot for bots without volumes", async () => {
+    const profiles = [makeProfile("b1")]; // no volumeName
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    await orch.rollout();
+
+    expect(snapMgr.snapshot).not.toHaveBeenCalled();
+    expect(updater.updateBot).toHaveBeenCalledWith("b1");
+  });
+
+  it("restores volumes on update failure", async () => {
+    const failResults = new Map([["b1", makeResult("b1", false)]]);
+    updater = mockUpdater(failResults);
+    const profiles = [makeProfile("b1", "my-volume")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.failed).toBe(1);
+    expect(result.results[0].volumeRestored).toBe(true);
+    expect(snapMgr.restore).toHaveBeenCalledWith("my-volume-snap");
+    // Snapshot NOT deleted on failure (restored instead)
+    expect(snapMgr.delete).not.toHaveBeenCalled();
+  });
+
+  it("aborts rollout when strategy says abort", async () => {
+    const failResults = new Map([["b1", makeResult("b1", false)]]);
+    updater = mockUpdater(failResults);
+    const profiles = [makeProfile("b1", "v1"), makeProfile("b2", "v2"), makeProfile("b3", "v3")];
+    const strategy = mockStrategy({
+      nextBatch: (r) => r.slice(0, 1),
+      onBotFailure: () => "abort",
+    });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.aborted).toBe(true);
+    expect(result.succeeded).toBe(0);
+    expect(result.failed).toBe(1);
+    expect(result.skipped).toBe(2); // b2, b3 never processed
+    expect(updater.updateBot).toHaveBeenCalledTimes(1);
+  });
+
+  it("returns empty result when no bots to update", async () => {
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => [],
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.totalBots).toBe(0);
+    expect(result.results).toHaveLength(0);
+  });
+
+  it("rejects concurrent rollouts", async () => {
+    const profiles = [makeProfile("b1")];
+    // Make updateBot slow
+    updater = {
+      updateBot: vi.fn(async (botId: string) => {
+        await new Promise((r) => setTimeout(r, 100));
+        return makeResult(botId, true);
+      }),
+    } as unknown as ContainerUpdater;
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const [r1, r2] = await Promise.all([orch.rollout(), orch.rollout()]);
+
+    // One succeeds, one is rejected as already running
+    const succeeded = [r1, r2].find((r) => r.totalBots > 0);
+    const rejected = [r1, r2].find((r) => r.alreadyRunning);
+    expect(succeeded).toBeDefined();
+    expect(rejected).toBeDefined();
+    expect(rejected?.alreadyRunning).toBe(true);
+    expect(rejected?.totalBots).toBe(0);
+  });
+
+  it("retries failed bots when strategy says retry", async () => {
+    let callCount = 0;
+    updater = {
+      updateBot: vi.fn(async (botId: string) => {
+        callCount++;
+        // Fail first attempt, succeed on retry
+        if (botId === "b1" && callCount === 1) return makeResult("b1", false);
+        return makeResult(botId, true);
+      }),
+    } as unknown as ContainerUpdater;
+
+    const profiles = [makeProfile("b1")];
+    const strategy = mockStrategy({
+      nextBatch: (r) => r.slice(0, 1),
+      onBotFailure: (_botId, _err, attempt) => (attempt < 2 ? "retry" : "skip"),
+    });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    // b1 failed once, retried, succeeded
+    expect(updater.updateBot).toHaveBeenCalledTimes(2);
+    expect(result.succeeded).toBe(1);
+    expect(result.failed).toBe(1); // first attempt counted as failed
+  });
+
+  it("calls onBotUpdated callback for each bot", async () => {
+    const profiles = [makeProfile("b1"), makeProfile("b2")];
+    const onBotUpdated = vi.fn();
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+      onBotUpdated,
+    });
+
+    await orch.rollout();
+
+    expect(onBotUpdated).toHaveBeenCalledTimes(2);
+  });
+
+  it("calls onRolloutComplete callback", async () => {
+    const profiles = [makeProfile("b1")];
+    const onRolloutComplete = vi.fn();
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+      onRolloutComplete,
+    });
+
+    await orch.rollout();
+
+    expect(onRolloutComplete).toHaveBeenCalledTimes(1);
+    expect(onRolloutComplete).toHaveBeenCalledWith(
+      expect.objectContaining({ totalBots: 1, succeeded: 1, aborted: false }),
+    );
+  });
+
+  it("continues on snapshot failure (best-effort)", async () => {
+    const profiles = [makeProfile("b1", "my-volume")];
+    snapMgr.snapshot = vi.fn().mockRejectedValue(new Error("disk full"));
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    // Update still proceeds despite snapshot failure
+    expect(result.succeeded).toBe(1);
+    expect(updater.updateBot).toHaveBeenCalledWith("b1");
+  });
+
+  it("isRolling reflects rollout state", async () => {
+    const profiles = [makeProfile("b1")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    expect(orch.isRolling).toBe(false);
+    const promise = orch.rollout();
+    // isRolling is true during rollout (may already be done for sync mocks)
+    await promise;
+    expect(orch.isRolling).toBe(false);
+  });
+});

package/src/fleet/index.ts

@@ -1,4 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-orchestrator.js";
 export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";

package/src/fleet/rollout-orchestrator.ts

@@ -0,0 +1,262 @@
+/**
+ * RolloutOrchestrator — coordinates fleet-wide container updates using
+ * pluggable rollout strategies and volume snapshots for nuclear rollback.
+ *
+ * Sits between ImagePoller (detects new digests) and ContainerUpdater
+ * (handles per-bot pull/stop/recreate/health). Adds:
+ * - Strategy-driven batching (rolling wave, single bot, immediate)
+ * - Pre-update volume snapshots via VolumeSnapshotManager
+ * - Volume restore on health check failure (nuclear rollback)
+ * - Per-tenant update orchestration
+ */
+
+import { logger } from "../config/logger.js";
+import type { IRolloutStrategy } from "./rollout-strategy.js";
+import type { BotProfile } from "./types.js";
+import type { ContainerUpdater, UpdateResult } from "./updater.js";
+import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
+
+export interface RolloutOrchestratorDeps {
+  updater: ContainerUpdater;
+  snapshotManager: VolumeSnapshotManager;
+  strategy: IRolloutStrategy;
+  /** Resolve running profiles that need updating for a given image digest */
+  getUpdatableProfiles: () => Promise<BotProfile[]>;
+  /** Optional callback after each bot update (success or failure) */
+  onBotUpdated?: (result: UpdateResult & { volumeRestored: boolean }) => void;
+  /** Optional callback when a rollout completes */
+  onRolloutComplete?: (results: RolloutResult) => void;
+}
+
+export interface BotUpdateResult extends UpdateResult {
+  volumeRestored: boolean;
+}
+
+export interface RolloutResult {
+  totalBots: number;
+  succeeded: number;
+  failed: number;
+  skipped: number;
+  aborted: boolean;
+  /** True when a concurrent rollout was already in progress */
+  alreadyRunning: boolean;
+  results: BotUpdateResult[];
+}
+
+export class RolloutOrchestrator {
+  private readonly updater: ContainerUpdater;
+  private readonly snapshotManager: VolumeSnapshotManager;
+  private readonly strategy: IRolloutStrategy;
+  private readonly getUpdatableProfiles: () => Promise<BotProfile[]>;
+  private readonly onBotUpdated?: (result: BotUpdateResult) => void;
+  private readonly onRolloutComplete?: (results: RolloutResult) => void;
+  private rolling = false;
+
+  constructor(deps: RolloutOrchestratorDeps) {
+    this.updater = deps.updater;
+    this.snapshotManager = deps.snapshotManager;
+    this.strategy = deps.strategy;
+    this.getUpdatableProfiles = deps.getUpdatableProfiles;
+    this.onBotUpdated = deps.onBotUpdated;
+    this.onRolloutComplete = deps.onRolloutComplete;
+  }
+
+  /** Whether a rollout is currently in progress. */
+  get isRolling(): boolean {
+    return this.rolling;
+  }
+
+  /**
+   * Execute a rollout across all updatable bots.
+   * Uses the configured strategy for batching, pausing, and failure handling.
+   */
+  async rollout(): Promise<RolloutResult> {
+    if (this.rolling) {
+      logger.warn("Rollout already in progress — skipping");
+      return { totalBots: 0, succeeded: 0, failed: 0, skipped: 0, aborted: false, alreadyRunning: true, results: [] };
+    }
+
+    this.rolling = true;
+    const allResults: BotUpdateResult[] = [];
+    let aborted = false;
+
+    try {
+      let remaining = await this.getUpdatableProfiles();
+      const totalBots = remaining.length;
+
+      if (totalBots === 0) {
+        logger.info("Rollout: no bots to update");
+        return {
+          totalBots: 0,
+          succeeded: 0,
+          failed: 0,
+          skipped: 0,
+          aborted: false,
+          alreadyRunning: false,
+          results: [],
+        };
+      }
+
+      logger.info(`Rollout starting: ${totalBots} bots to update`);
+
+      while (remaining.length > 0 && !aborted) {
+        const batch = this.strategy.nextBatch(remaining);
+        if (batch.length === 0) break;
+
+        logger.info(`Rollout wave: ${batch.length} bots (${remaining.length} remaining)`);
+
+        // Process batch — each bot sequentially within a wave for safety
+        const retryProfiles: BotProfile[] = [];
+        for (const profile of batch) {
+          if (aborted) break;
+
+          const result = await this.updateBot(profile);
+          allResults.push(result);
+          this.onBotUpdated?.(result);
+
+          if (!result.success) {
+            const action = this.handleFailure(profile.id, result, allResults);
+            if (action === "abort") {
+              aborted = true;
+              logger.warn(`Rollout aborted after bot ${profile.id} failure`);
+            } else if (action === "retry") {
+              retryProfiles.push(profile);
+            }
+            // "skip" → don't re-add, bot is dropped
+          }
+        }
+
+        // Remove processed bots from remaining, but re-add retries
+        const processedIds = new Set(batch.map((b) => b.id));
+        const retryIds = new Set(retryProfiles.map((b) => b.id));
+        remaining = [
+          ...remaining.filter((b) => !processedIds.has(b.id)),
+          ...retryProfiles.filter((b) => retryIds.has(b.id)),
+        ];
+
+        // Pause between waves (unless aborted or done)
+        if (remaining.length > 0 && !aborted) {
+          const pause = this.strategy.pauseDuration();
+          if (pause > 0) {
+            logger.info(`Rollout: pausing ${pause}ms before next wave`);
+            await sleep(pause);
+          }
+        }
+      }
+
+      const succeeded = allResults.filter((r) => r.success).length;
+      const failed = allResults.filter((r) => !r.success).length;
+      const skipped = totalBots - allResults.length;
+
+      const rolloutResult: RolloutResult = {
+        totalBots,
+        succeeded,
+        failed,
+        skipped,
+        aborted,
+        alreadyRunning: false,
+        results: allResults,
+      };
+
+      logger.info(`Rollout complete: ${succeeded} succeeded, ${failed} failed, ${skipped} skipped, aborted=${aborted}`);
+      this.onRolloutComplete?.(rolloutResult);
+
+      return rolloutResult;
+    } finally {
+      this.rolling = false;
+    }
+  }
+
+  /**
+   * Update a single bot with volume snapshot + nuclear rollback.
+   */
+  private async updateBot(profile: BotProfile): Promise<BotUpdateResult> {
+    const snapshotIds: string[] = [];
+
+    try {
+      // Step 1: Snapshot volumes before update
+      if (profile.volumeName) {
+        try {
+          const snap = await this.snapshotManager.snapshot(profile.volumeName);
+          snapshotIds.push(snap.id);
+          logger.info(`Pre-update snapshot for ${profile.id}: ${snap.id}`);
+        } catch (err) {
+          logger.warn(`Volume snapshot failed for ${profile.id} — proceeding without backup`, { err });
+        }
+      }
+
+      // Step 2: Delegate to ContainerUpdater
+      const result = await this.updater.updateBot(profile.id);
+
+      if (result.success) {
+        // Clean up snapshots on success
+        await this.cleanupSnapshots(snapshotIds);
+        return { ...result, volumeRestored: false };
+      }
+
+      // Step 3: Nuclear rollback — restore volumes if update failed
+      const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
+      return { ...result, volumeRestored };
+    } catch (err) {
+      logger.error(`Unexpected error updating bot ${profile.id}`, { err });
+
+      // Attempt volume restore on unexpected errors too
+      const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
+
+      return {
+        botId: profile.id,
+        success: false,
+        previousImage: profile.image,
+        newImage: profile.image,
+        previousDigest: null,
+        newDigest: null,
+        rolledBack: false,
+        volumeRestored,
+        error: err instanceof Error ? err.message : String(err),
+      };
+    }
+  }
+
+  /**
+   * Handle a bot failure using the strategy's failure policy.
+   * Retries the update up to maxRetries before escalating.
+   */
+  private handleFailure(
+    botId: string,
+    result: BotUpdateResult,
+    allResults: BotUpdateResult[],
+  ): "abort" | "skip" | "retry" {
+    const error = new Error(result.error ?? "Unknown error");
+    const failCount = allResults.filter((r) => r.botId === botId && !r.success).length;
+    return this.strategy.onBotFailure(botId, error, failCount);
+  }
+
+  private async restoreVolumes(botId: string, snapshotIds: string[]): Promise<boolean> {
+    if (snapshotIds.length === 0) return false;
+
+    for (const id of snapshotIds) {
+      try {
+        await this.snapshotManager.restore(id);
+        logger.info(`Volume restored for ${botId} from snapshot ${id}`);
+        return true;
+      } catch (err) {
+        logger.error(`Volume restore failed for ${botId} snapshot ${id}`, { err });
+      }
+    }
+    return false;
+  }
+
+  private async cleanupSnapshots(snapshotIds: string[]): Promise<void> {
+    for (const id of snapshotIds) {
+      try {
+        await this.snapshotManager.delete(id);
+      } catch (err) {
+        logger.warn(`Failed to clean up snapshot ${id}`, { err });
+      }
+    }
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/fleet/services.ts

@@ -32,6 +32,8 @@ import { SystemResourceMonitor } from "../observability/system-resources.js";
 // Stub re-exports so existing references compile; consumers must call initPlatformServices().
 // TODO: Replace with proper DI / service-locator pattern in platform-core.
 import { DrizzleTwoFactorRepository } from "../security/two-factor-repository.js";
+import type { RolloutOrchestrator } from "./rollout-orchestrator.js";
+import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
 
 // Platform singletons (getAdminAuditLog, getCreditLedger, etc.) are wired by
 // the consuming application's own composition root (e.g. wopr-platform's
@@ -136,6 +138,8 @@ let _restoreLogStore: IRestoreLogStore | null = null;
 let _restoreService: RestoreService | null = null;
 let _backupStatusStore: IBackupStatusStore | null = null;
 let _snapshotManager: SnapshotManager | null = null;
+let _volumeSnapshotManager: VolumeSnapshotManager | null = null;
+let _rolloutOrchestrator: RolloutOrchestrator | null = null;
 
 const S3_BUCKET = process.env.S3_BUCKET || "wopr-backups";
 
@@ -537,6 +541,28 @@ export function getSnapshotManager(): SnapshotManager {
   return _snapshotManager;
 }
 
+export function getVolumeSnapshotManager(): VolumeSnapshotManager {
+  if (!_volumeSnapshotManager) {
+    throw new Error("VolumeSnapshotManager not initialized — call setVolumeSnapshotManager() first");
+  }
+  return _volumeSnapshotManager;
+}
+
+export function setVolumeSnapshotManager(mgr: VolumeSnapshotManager): void {
+  _volumeSnapshotManager = mgr;
+}
+
+export function getRolloutOrchestrator(): RolloutOrchestrator {
+  if (!_rolloutOrchestrator) {
+    throw new Error("RolloutOrchestrator not initialized — call setRolloutOrchestrator() first");
+  }
+  return _rolloutOrchestrator;
+}
+
+export function setRolloutOrchestrator(orch: RolloutOrchestrator): void {
+  _rolloutOrchestrator = orch;
+}
+
 export function getRestoreService(): RestoreService {
   if (!_restoreService) {
     _restoreService = new RestoreService({
@@ -877,6 +903,8 @@ export function _resetForTest(): void {
   _restoreService = null;
   _backupStatusStore = null;
   _snapshotManager = null;
+  _volumeSnapshotManager = null;
+  _rolloutOrchestrator = null;
   _botBilling = null;
   _phoneNumberRepo = null;
   _affiliateRepo = null;