@wopr-network/platform-core 1.21.0 → 1.23.0

package/src/billing/crypto/evm/settler.ts

@@ -28,7 +28,7 @@ export async function settleEvmPayment(deps: EvmSettlerDeps, event: EvmPaymentEv
 
   const charge = await chargeStore.getByDepositAddress(event.to.toLowerCase());
   if (!charge) {
-    return { handled: false, status: "Settled" };
+    return { handled: false, status: "Invalid" };
   }
 
   // Update charge status to Settled.

package/src/fleet/__tests__/rollout-orchestrator.test.ts

@@ -0,0 +1,321 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { RolloutOrchestrator } from "../rollout-orchestrator.js";
+import type { IRolloutStrategy } from "../rollout-strategy.js";
+import type { BotProfile } from "../types.js";
+import type { ContainerUpdater, UpdateResult } from "../updater.js";
+import type { VolumeSnapshotManager } from "../volume-snapshot-manager.js";
+
+function makeProfile(id: string, volumeName?: string): BotProfile {
+  return {
+    id,
+    tenantId: "tenant-1",
+    name: `bot-${id}`,
+    description: "",
+    image: "ghcr.io/wopr-network/paperclip:managed",
+    env: {},
+    restartPolicy: "unless-stopped",
+    releaseChannel: "stable",
+    updatePolicy: "nightly",
+    volumeName,
+  } as BotProfile;
+}
+
+function makeResult(botId: string, success: boolean): UpdateResult {
+  return {
+    botId,
+    success,
+    previousImage: "old:latest",
+    newImage: "new:latest",
+    previousDigest: "sha256:old",
+    newDigest: "sha256:new",
+    rolledBack: !success,
+    error: success ? undefined : "Health check failed",
+  };
+}
+
+function mockUpdater(results: Map<string, UpdateResult>): ContainerUpdater {
+  return {
+    updateBot: vi.fn(async (botId: string) => results.get(botId) ?? makeResult(botId, true)),
+  } as unknown as ContainerUpdater;
+}
+
+function mockSnapshotManager(): VolumeSnapshotManager {
+  return {
+    snapshot: vi.fn(async (volumeName: string) => ({
+      id: `${volumeName}-snap`,
+      volumeName,
+      archivePath: `/backup/${volumeName}-snap.tar`,
+      createdAt: new Date(),
+      sizeBytes: 1024,
+    })),
+    restore: vi.fn(async () => {}),
+    delete: vi.fn(async () => {}),
+  } as unknown as VolumeSnapshotManager;
+}
+
+function mockStrategy(overrides: Partial<IRolloutStrategy> = {}): IRolloutStrategy {
+  return {
+    nextBatch: (remaining) => remaining.slice(0, 2),
+    pauseDuration: () => 0,
+    onBotFailure: () => "skip",
+    maxRetries: () => 2,
+    healthCheckTimeout: () => 120_000,
+    ...overrides,
+  };
+}
+
+describe("RolloutOrchestrator", () => {
+  let updater: ReturnType<typeof mockUpdater>;
+  let snapMgr: ReturnType<typeof mockSnapshotManager>;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    updater = mockUpdater(new Map());
+    snapMgr = mockSnapshotManager();
+  });
+
+  it("processes all bots in batches", async () => {
+    const profiles = [makeProfile("b1", "vol-1"), makeProfile("b2", "vol-2"), makeProfile("b3", "vol-3")];
+    const strategy = mockStrategy({ nextBatch: (r) => r.slice(0, 2) });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.totalBots).toBe(3);
+    expect(result.succeeded).toBe(3);
+    expect(result.failed).toBe(0);
+    expect(result.aborted).toBe(false);
+    expect(updater.updateBot).toHaveBeenCalledTimes(3);
+  });
+
+  it("snapshots volumes before updating", async () => {
+    const profiles = [makeProfile("b1", "my-volume")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    await orch.rollout();
+
+    expect(snapMgr.snapshot).toHaveBeenCalledWith("my-volume");
+    // On success, snapshot is cleaned up
+    expect(snapMgr.delete).toHaveBeenCalledWith("my-volume-snap");
+  });
+
+  it("skips snapshot for bots without volumes", async () => {
+    const profiles = [makeProfile("b1")]; // no volumeName
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    await orch.rollout();
+
+    expect(snapMgr.snapshot).not.toHaveBeenCalled();
+    expect(updater.updateBot).toHaveBeenCalledWith("b1");
+  });
+
+  it("restores volumes on update failure", async () => {
+    const failResults = new Map([["b1", makeResult("b1", false)]]);
+    updater = mockUpdater(failResults);
+    const profiles = [makeProfile("b1", "my-volume")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.failed).toBe(1);
+    expect(result.results[0].volumeRestored).toBe(true);
+    expect(snapMgr.restore).toHaveBeenCalledWith("my-volume-snap");
+    // Snapshot NOT deleted on failure (restored instead)
+    expect(snapMgr.delete).not.toHaveBeenCalled();
+  });
+
+  it("aborts rollout when strategy says abort", async () => {
+    const failResults = new Map([["b1", makeResult("b1", false)]]);
+    updater = mockUpdater(failResults);
+    const profiles = [makeProfile("b1", "v1"), makeProfile("b2", "v2"), makeProfile("b3", "v3")];
+    const strategy = mockStrategy({
+      nextBatch: (r) => r.slice(0, 1),
+      onBotFailure: () => "abort",
+    });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.aborted).toBe(true);
+    expect(result.succeeded).toBe(0);
+    expect(result.failed).toBe(1);
+    expect(result.skipped).toBe(2); // b2, b3 never processed
+    expect(updater.updateBot).toHaveBeenCalledTimes(1);
+  });
+
+  it("returns empty result when no bots to update", async () => {
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => [],
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.totalBots).toBe(0);
+    expect(result.results).toHaveLength(0);
+  });
+
+  it("rejects concurrent rollouts", async () => {
+    const profiles = [makeProfile("b1")];
+    // Make updateBot slow
+    updater = {
+      updateBot: vi.fn(async (botId: string) => {
+        await new Promise((r) => setTimeout(r, 100));
+        return makeResult(botId, true);
+      }),
+    } as unknown as ContainerUpdater;
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const [r1, r2] = await Promise.all([orch.rollout(), orch.rollout()]);
+
+    // One succeeds, one is rejected as already running
+    const succeeded = [r1, r2].find((r) => r.totalBots > 0);
+    const rejected = [r1, r2].find((r) => r.alreadyRunning);
+    expect(succeeded).toBeDefined();
+    expect(rejected).toBeDefined();
+    expect(rejected?.alreadyRunning).toBe(true);
+    expect(rejected?.totalBots).toBe(0);
+  });
+
+  it("retries failed bots when strategy says retry", async () => {
+    let callCount = 0;
+    updater = {
+      updateBot: vi.fn(async (botId: string) => {
+        callCount++;
+        // Fail first attempt, succeed on retry
+        if (botId === "b1" && callCount === 1) return makeResult("b1", false);
+        return makeResult(botId, true);
+      }),
+    } as unknown as ContainerUpdater;
+
+    const profiles = [makeProfile("b1")];
+    const strategy = mockStrategy({
+      nextBatch: (r) => r.slice(0, 1),
+      onBotFailure: (_botId, _err, attempt) => (attempt < 2 ? "retry" : "skip"),
+    });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    // b1 failed once, retried, succeeded
+    expect(updater.updateBot).toHaveBeenCalledTimes(2);
+    expect(result.succeeded).toBe(1);
+    expect(result.failed).toBe(1); // first attempt counted as failed
+  });
+
+  it("calls onBotUpdated callback for each bot", async () => {
+    const profiles = [makeProfile("b1"), makeProfile("b2")];
+    const onBotUpdated = vi.fn();
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+      onBotUpdated,
+    });
+
+    await orch.rollout();
+
+    expect(onBotUpdated).toHaveBeenCalledTimes(2);
+  });
+
+  it("calls onRolloutComplete callback", async () => {
+    const profiles = [makeProfile("b1")];
+    const onRolloutComplete = vi.fn();
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+      onRolloutComplete,
+    });
+
+    await orch.rollout();
+
+    expect(onRolloutComplete).toHaveBeenCalledTimes(1);
+    expect(onRolloutComplete).toHaveBeenCalledWith(
+      expect.objectContaining({ totalBots: 1, succeeded: 1, aborted: false }),
+    );
+  });
+
+  it("continues on snapshot failure (best-effort)", async () => {
+    const profiles = [makeProfile("b1", "my-volume")];
+    snapMgr.snapshot = vi.fn().mockRejectedValue(new Error("disk full"));
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    // Update still proceeds despite snapshot failure
+    expect(result.succeeded).toBe(1);
+    expect(updater.updateBot).toHaveBeenCalledWith("b1");
+  });
+
+  it("isRolling reflects rollout state", async () => {
+    const profiles = [makeProfile("b1")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    expect(orch.isRolling).toBe(false);
+    const promise = orch.rollout();
+    // isRolling is true during rollout (may already be done for sync mocks)
+    await promise;
+    expect(orch.isRolling).toBe(false);
+  });
+});

package/src/fleet/index.ts

@@ -1,4 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-orchestrator.js";
 export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";

package/src/fleet/rollout-orchestrator.ts

@@ -0,0 +1,262 @@
+/**
+ * RolloutOrchestrator — coordinates fleet-wide container updates using
+ * pluggable rollout strategies and volume snapshots for nuclear rollback.
+ *
+ * Sits between ImagePoller (detects new digests) and ContainerUpdater
+ * (handles per-bot pull/stop/recreate/health). Adds:
+ * - Strategy-driven batching (rolling wave, single bot, immediate)
+ * - Pre-update volume snapshots via VolumeSnapshotManager
+ * - Volume restore on health check failure (nuclear rollback)
+ * - Per-tenant update orchestration
+ */
+
+import { logger } from "../config/logger.js";
+import type { IRolloutStrategy } from "./rollout-strategy.js";
+import type { BotProfile } from "./types.js";
+import type { ContainerUpdater, UpdateResult } from "./updater.js";
+import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
+
+export interface RolloutOrchestratorDeps {
+  updater: ContainerUpdater;
+  snapshotManager: VolumeSnapshotManager;
+  strategy: IRolloutStrategy;
+  /** Resolve running profiles that need updating for a given image digest */
+  getUpdatableProfiles: () => Promise<BotProfile[]>;
+  /** Optional callback after each bot update (success or failure) */
+  onBotUpdated?: (result: UpdateResult & { volumeRestored: boolean }) => void;
+  /** Optional callback when a rollout completes */
+  onRolloutComplete?: (results: RolloutResult) => void;
+}
+
+export interface BotUpdateResult extends UpdateResult {
+  volumeRestored: boolean;
+}
+
+export interface RolloutResult {
+  totalBots: number;
+  succeeded: number;
+  failed: number;
+  skipped: number;
+  aborted: boolean;
+  /** True when a concurrent rollout was already in progress */
+  alreadyRunning: boolean;
+  results: BotUpdateResult[];
+}
+
+export class RolloutOrchestrator {
+  private readonly updater: ContainerUpdater;
+  private readonly snapshotManager: VolumeSnapshotManager;
+  private readonly strategy: IRolloutStrategy;
+  private readonly getUpdatableProfiles: () => Promise<BotProfile[]>;
+  private readonly onBotUpdated?: (result: BotUpdateResult) => void;
+  private readonly onRolloutComplete?: (results: RolloutResult) => void;
+  private rolling = false;
+
+  constructor(deps: RolloutOrchestratorDeps) {
+    this.updater = deps.updater;
+    this.snapshotManager = deps.snapshotManager;
+    this.strategy = deps.strategy;
+    this.getUpdatableProfiles = deps.getUpdatableProfiles;
+    this.onBotUpdated = deps.onBotUpdated;
+    this.onRolloutComplete = deps.onRolloutComplete;
+  }
+
+  /** Whether a rollout is currently in progress. */
+  get isRolling(): boolean {
+    return this.rolling;
+  }
+
+  /**
+   * Execute a rollout across all updatable bots.
+   * Uses the configured strategy for batching, pausing, and failure handling.
+   */
+  async rollout(): Promise<RolloutResult> {
+    if (this.rolling) {
+      logger.warn("Rollout already in progress — skipping");
+      return { totalBots: 0, succeeded: 0, failed: 0, skipped: 0, aborted: false, alreadyRunning: true, results: [] };
+    }
+
+    this.rolling = true;
+    const allResults: BotUpdateResult[] = [];
+    let aborted = false;
+
+    try {
+      let remaining = await this.getUpdatableProfiles();
+      const totalBots = remaining.length;
+
+      if (totalBots === 0) {
+        logger.info("Rollout: no bots to update");
+        return {
+          totalBots: 0,
+          succeeded: 0,
+          failed: 0,
+          skipped: 0,
+          aborted: false,
+          alreadyRunning: false,
+          results: [],
+        };
+      }
+
+      logger.info(`Rollout starting: ${totalBots} bots to update`);
+
+      while (remaining.length > 0 && !aborted) {
+        const batch = this.strategy.nextBatch(remaining);
+        if (batch.length === 0) break;
+
+        logger.info(`Rollout wave: ${batch.length} bots (${remaining.length} remaining)`);
+
+        // Process batch — each bot sequentially within a wave for safety
+        const retryProfiles: BotProfile[] = [];
+        for (const profile of batch) {
+          if (aborted) break;
+
+          const result = await this.updateBot(profile);
+          allResults.push(result);
+          this.onBotUpdated?.(result);
+
+          if (!result.success) {
+            const action = this.handleFailure(profile.id, result, allResults);
+            if (action === "abort") {
+              aborted = true;
+              logger.warn(`Rollout aborted after bot ${profile.id} failure`);
+            } else if (action === "retry") {
+              retryProfiles.push(profile);
+            }
+            // "skip" → don't re-add, bot is dropped
+          }
+        }
+
+        // Remove processed bots from remaining, but re-add retries
+        const processedIds = new Set(batch.map((b) => b.id));
+        const retryIds = new Set(retryProfiles.map((b) => b.id));
+        remaining = [
+          ...remaining.filter((b) => !processedIds.has(b.id)),
+          ...retryProfiles.filter((b) => retryIds.has(b.id)),
+        ];
+
+        // Pause between waves (unless aborted or done)
+        if (remaining.length > 0 && !aborted) {
+          const pause = this.strategy.pauseDuration();
+          if (pause > 0) {
+            logger.info(`Rollout: pausing ${pause}ms before next wave`);
+            await sleep(pause);
+          }
+        }
+      }
+
+      const succeeded = allResults.filter((r) => r.success).length;
+      const failed = allResults.filter((r) => !r.success).length;
+      const skipped = totalBots - allResults.length;
+
+      const rolloutResult: RolloutResult = {
+        totalBots,
+        succeeded,
+        failed,
+        skipped,
+        aborted,
+        alreadyRunning: false,
+        results: allResults,
+      };
+
+      logger.info(`Rollout complete: ${succeeded} succeeded, ${failed} failed, ${skipped} skipped, aborted=${aborted}`);
+      this.onRolloutComplete?.(rolloutResult);
+
+      return rolloutResult;
+    } finally {
+      this.rolling = false;
+    }
+  }
+
+  /**
+   * Update a single bot with volume snapshot + nuclear rollback.
+   */
+  private async updateBot(profile: BotProfile): Promise<BotUpdateResult> {
+    const snapshotIds: string[] = [];
+
+    try {
+      // Step 1: Snapshot volumes before update
+      if (profile.volumeName) {
+        try {
+          const snap = await this.snapshotManager.snapshot(profile.volumeName);
+          snapshotIds.push(snap.id);
+          logger.info(`Pre-update snapshot for ${profile.id}: ${snap.id}`);
+        } catch (err) {
+          logger.warn(`Volume snapshot failed for ${profile.id} — proceeding without backup`, { err });
+        }
+      }
+
+      // Step 2: Delegate to ContainerUpdater
+      const result = await this.updater.updateBot(profile.id);
+
+      if (result.success) {
+        // Clean up snapshots on success
+        await this.cleanupSnapshots(snapshotIds);
+        return { ...result, volumeRestored: false };
+      }
+
+      // Step 3: Nuclear rollback — restore volumes if update failed
+      const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
+      return { ...result, volumeRestored };
+    } catch (err) {
+      logger.error(`Unexpected error updating bot ${profile.id}`, { err });
+
+      // Attempt volume restore on unexpected errors too
+      const volumeRestored = await this.restoreVolumes(profile.id, snapshotIds);
+
+      return {
+        botId: profile.id,
+        success: false,
+        previousImage: profile.image,
+        newImage: profile.image,
+        previousDigest: null,
+        newDigest: null,
+        rolledBack: false,
+        volumeRestored,
+        error: err instanceof Error ? err.message : String(err),
+      };
+    }
+  }
+
+  /**
+   * Handle a bot failure using the strategy's failure policy.
+   * Retries the update up to maxRetries before escalating.
+   */
+  private handleFailure(
+    botId: string,
+    result: BotUpdateResult,
+    allResults: BotUpdateResult[],
+  ): "abort" | "skip" | "retry" {
+    const error = new Error(result.error ?? "Unknown error");
+    const failCount = allResults.filter((r) => r.botId === botId && !r.success).length;
+    return this.strategy.onBotFailure(botId, error, failCount);
+  }
+
+  private async restoreVolumes(botId: string, snapshotIds: string[]): Promise<boolean> {
+    if (snapshotIds.length === 0) return false;
+
+    for (const id of snapshotIds) {
+      try {
+        await this.snapshotManager.restore(id);
+        logger.info(`Volume restored for ${botId} from snapshot ${id}`);
+        return true;
+      } catch (err) {
+        logger.error(`Volume restore failed for ${botId} snapshot ${id}`, { err });
+      }
+    }
+    return false;
+  }
+
+  private async cleanupSnapshots(snapshotIds: string[]): Promise<void> {
+    for (const id of snapshotIds) {
+      try {
+        await this.snapshotManager.delete(id);
+      } catch (err) {
+        logger.warn(`Failed to clean up snapshot ${id}`, { err });
+      }
+    }
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/fleet/services.ts

@@ -32,6 +32,8 @@ import { SystemResourceMonitor } from "../observability/system-resources.js";
 // Stub re-exports so existing references compile; consumers must call initPlatformServices().
 // TODO: Replace with proper DI / service-locator pattern in platform-core.
 import { DrizzleTwoFactorRepository } from "../security/two-factor-repository.js";
+import type { RolloutOrchestrator } from "./rollout-orchestrator.js";
+import type { VolumeSnapshotManager } from "./volume-snapshot-manager.js";
 
 // Platform singletons (getAdminAuditLog, getCreditLedger, etc.) are wired by
 // the consuming application's own composition root (e.g. wopr-platform's
@@ -136,6 +138,8 @@ let _restoreLogStore: IRestoreLogStore | null = null;
 let _restoreService: RestoreService | null = null;
 let _backupStatusStore: IBackupStatusStore | null = null;
 let _snapshotManager: SnapshotManager | null = null;
+let _volumeSnapshotManager: VolumeSnapshotManager | null = null;
+let _rolloutOrchestrator: RolloutOrchestrator | null = null;
 
 const S3_BUCKET = process.env.S3_BUCKET || "wopr-backups";
 
@@ -537,6 +541,28 @@ export function getSnapshotManager(): SnapshotManager {
   return _snapshotManager;
 }
 
+export function getVolumeSnapshotManager(): VolumeSnapshotManager {
+  if (!_volumeSnapshotManager) {
+    throw new Error("VolumeSnapshotManager not initialized — call setVolumeSnapshotManager() first");
+  }
+  return _volumeSnapshotManager;
+}
+
+export function setVolumeSnapshotManager(mgr: VolumeSnapshotManager): void {
+  _volumeSnapshotManager = mgr;
+}
+
+export function getRolloutOrchestrator(): RolloutOrchestrator {
+  if (!_rolloutOrchestrator) {
+    throw new Error("RolloutOrchestrator not initialized — call setRolloutOrchestrator() first");
+  }
+  return _rolloutOrchestrator;
+}
+
+export function setRolloutOrchestrator(orch: RolloutOrchestrator): void {
+  _rolloutOrchestrator = orch;
+}
+
 export function getRestoreService(): RestoreService {
   if (!_restoreService) {
     _restoreService = new RestoreService({
@@ -877,6 +903,8 @@ export function _resetForTest(): void {
   _restoreService = null;
   _backupStatusStore = null;
   _snapshotManager = null;
+  _volumeSnapshotManager = null;
+  _rolloutOrchestrator = null;
   _botBilling = null;
   _phoneNumberRepo = null;
   _affiliateRepo = null;