@wopr-network/platform-core 1.22.0 → 1.24.0

package/src/billing/crypto/btc/watcher.ts

@@ -1,3 +1,4 @@
+import type { IWatcherCursorStore } from "../cursor-store.js";
 import type { IPriceOracle } from "../oracle/types.js";
 import type { BitcoindConfig, BtcPaymentEvent } from "./types.js";
 
@@ -11,6 +12,8 @@ export interface BtcWatcherOpts {
   onPayment: (event: BtcPaymentEvent) => void | Promise<void>;
   /** Price oracle for BTC/USD conversion. */
   oracle: IPriceOracle;
+  /** Required — BTC has no block cursor, so txid dedup must be persisted. */
+  cursorStore: IWatcherCursorStore;
 }
 
 interface ReceivedByAddress {
@@ -26,7 +29,8 @@ export class BtcWatcher {
   private readonly onPayment: BtcWatcherOpts["onPayment"];
   private readonly minConfirmations: number;
   private readonly oracle: IPriceOracle;
-  private readonly processedTxids = new Set<string>();
+  private readonly cursorStore: IWatcherCursorStore;
+  private readonly watcherId: string;
 
   constructor(opts: BtcWatcherOpts) {
     this.rpc = opts.rpcCall;
@@ -34,6 +38,8 @@ export class BtcWatcher {
     this.onPayment = opts.onPayment;
     this.minConfirmations = opts.config.confirmations;
     this.oracle = opts.oracle;
+    this.cursorStore = opts.cursorStore;
+    this.watcherId = `btc:${opts.config.network}`;
   }
 
   /** Update the set of watched addresses. */
@@ -64,7 +70,8 @@ export class BtcWatcher {
       if (!this.addresses.has(entry.address)) continue;
 
       for (const txid of entry.txids) {
-        if (this.processedTxids.has(txid)) continue;
+        // Skip already-processed txids (persisted to DB, survives restart)
+        if (await this.cursorStore.hasProcessedTx(this.watcherId, txid)) continue;
 
         // Get transaction details for the exact amount sent to this address
         const tx = (await this.rpc("gettransaction", [txid, true])) as {
@@ -88,8 +95,8 @@ export class BtcWatcher {
         };
 
         await this.onPayment(event);
-        // Add AFTER successful onPayment to avoid skipping on failure
-        this.processedTxids.add(txid);
+        // Persist AFTER successful onPayment — survives restart, no unbounded memory
+        await this.cursorStore.markProcessedTx(this.watcherId, txid);
       }
     }
   }

package/src/billing/crypto/cursor-store.ts

@@ -0,0 +1,61 @@
+import { and, eq, sql } from "drizzle-orm";
+import type { PlatformDb } from "../../db/index.js";
+import { watcherCursors, watcherProcessed } from "../../db/schema/crypto.js";
+
+export interface IWatcherCursorStore {
+  /** Get persisted block cursor for a watcher. */
+  get(watcherId: string): Promise<number | null>;
+  /** Save block cursor after processing a range. */
+  save(watcherId: string, cursorBlock: number): Promise<void>;
+  /** Check if a specific tx has been processed (for watchers without block cursors). */
+  hasProcessedTx(watcherId: string, txId: string): Promise<boolean>;
+  /** Mark a tx as processed (for watchers without block cursors). */
+  markProcessedTx(watcherId: string, txId: string): Promise<void>;
+}
+
+/**
+ * Persists watcher state to PostgreSQL.
+ *
+ * Two patterns:
+ *   - Block cursor (EVM watchers): save/get cursor block number
+ *   - Processed txids (BTC watcher): hasProcessedTx/markProcessedTx
+ *
+ * Eliminates all in-memory watcher state. Clean restart recovery.
+ */
+export class DrizzleWatcherCursorStore implements IWatcherCursorStore {
+  constructor(private readonly db: PlatformDb) {}
+
+  async get(watcherId: string): Promise<number | null> {
+    const row = (
+      await this.db
+        .select({ cursorBlock: watcherCursors.cursorBlock })
+        .from(watcherCursors)
+        .where(eq(watcherCursors.watcherId, watcherId))
+    )[0];
+    return row?.cursorBlock ?? null;
+  }
+
+  async save(watcherId: string, cursorBlock: number): Promise<void> {
+    await this.db
+      .insert(watcherCursors)
+      .values({ watcherId, cursorBlock })
+      .onConflictDoUpdate({
+        target: watcherCursors.watcherId,
+        set: { cursorBlock, updatedAt: sql`(now())` },
+      });
+  }
+
+  async hasProcessedTx(watcherId: string, txId: string): Promise<boolean> {
+    const row = (
+      await this.db
+        .select({ txId: watcherProcessed.txId })
+        .from(watcherProcessed)
+        .where(and(eq(watcherProcessed.watcherId, watcherId), eq(watcherProcessed.txId, txId)))
+    )[0];
+    return row !== undefined;
+  }
+
+  async markProcessedTx(watcherId: string, txId: string): Promise<void> {
+    await this.db.insert(watcherProcessed).values({ watcherId, txId }).onConflictDoNothing();
+  }
+}

package/src/billing/crypto/evm/eth-watcher.ts

@@ -1,3 +1,4 @@
+import type { IWatcherCursorStore } from "../cursor-store.js";
 import { nativeToCents } from "../oracle/convert.js";
 import type { IPriceOracle } from "../oracle/types.js";
 import { getChainConfig } from "./config.js";
@@ -25,6 +26,7 @@ export interface EthWatcherOpts {
   fromBlock: number;
   onPayment: (event: EthPaymentEvent) => void | Promise<void>;
   watchedAddresses?: string[];
+  cursorStore?: IWatcherCursorStore;
 }
 
 interface RpcTransaction {
@@ -42,7 +44,9 @@ interface RpcTransaction {
  * this scans blocks for transactions where `to` matches a watched deposit
  * address and `value > 0`.
  *
- * Uses the price oracle to convert wei → USD cents at detection time.
+ * Processes one block at a time and persists cursor after each block.
+ * On restart, resumes from the last committed cursor — no replay, no
+ * unbounded in-memory state.
  */
 export class EthWatcher {
   private _cursor: number;
@@ -51,8 +55,9 @@ export class EthWatcher {
   private readonly oracle: IPriceOracle;
   private readonly onPayment: EthWatcherOpts["onPayment"];
   private readonly confirmations: number;
+  private readonly cursorStore?: IWatcherCursorStore;
+  private readonly watcherId: string;
   private _watchedAddresses: Set<string>;
-  private readonly processedTxids = new Set<string>();
 
   constructor(opts: EthWatcherOpts) {
     this.chain = opts.chain;
@@ -61,9 +66,18 @@ export class EthWatcher {
     this._cursor = opts.fromBlock;
     this.onPayment = opts.onPayment;
     this.confirmations = getChainConfig(opts.chain).confirmations;
+    this.cursorStore = opts.cursorStore;
+    this.watcherId = `eth:${opts.chain}`;
     this._watchedAddresses = new Set((opts.watchedAddresses ?? []).map((a) => a.toLowerCase()));
   }
 
+  /** Load cursor from DB. Call once at startup before first poll. */
+  async init(): Promise<void> {
+    if (!this.cursorStore) return;
+    const saved = await this.cursorStore.get(this.watcherId);
+    if (saved !== null) this._cursor = saved;
+  }
+
   setWatchedAddresses(addresses: string[]): void {
     this._watchedAddresses = new Set(addresses.map((a) => a.toLowerCase()));
   }
@@ -75,8 +89,9 @@ export class EthWatcher {
   /**
    * Poll for new native ETH transfers to watched addresses.
    *
-   * Scans each confirmed block's transactions. Only processes txs
-   * where `to` is in the watched set and `value > 0`.
+   * Processes one block at a time. After each block is fully processed,
+   * the cursor is persisted to the DB. If onPayment fails mid-block,
+   * the cursor hasn't advanced — the entire block is retried on next poll.
    */
   async poll(): Promise<void> {
     if (this._watchedAddresses.size === 0) return;
@@ -104,8 +119,6 @@ export class EthWatcher {
         const valueWei = BigInt(tx.value);
         if (valueWei === 0n) continue;
 
-        if (this.processedTxids.has(tx.hash)) continue;
-
         const amountUsdCents = nativeToCents(valueWei, priceCents, 18);
 
         const event: EthPaymentEvent = {
@@ -119,11 +132,13 @@ export class EthWatcher {
         };
 
         await this.onPayment(event);
-        // Add to processed AFTER successful onPayment to avoid skipping on failure
-        this.processedTxids.add(tx.hash);
       }
-    }
 
-    this._cursor = confirmed + 1;
+      // Block fully processed — persist cursor so we never re-scan it.
+      this._cursor = blockNum + 1;
+      if (this.cursorStore) {
+        await this.cursorStore.save(this.watcherId, this._cursor);
+      }
+    }
   }
 }

package/src/billing/crypto/evm/watcher.ts

@@ -1,3 +1,4 @@
+import type { IWatcherCursorStore } from "../cursor-store.js";
 import { centsFromTokenAmount, getChainConfig, getTokenConfig } from "./config.js";
 import type { EvmChain, EvmPaymentEvent, StablecoinToken } from "./types.js";
 
@@ -13,6 +14,7 @@ export interface EvmWatcherOpts {
   onPayment: (event: EvmPaymentEvent) => void | Promise<void>;
   /** Active deposit addresses to watch. Filters eth_getLogs by topic[2] (to address). */
   watchedAddresses?: string[];
+  cursorStore?: IWatcherCursorStore;
 }
 
 interface RpcLog {
@@ -33,6 +35,8 @@ export class EvmWatcher {
   private readonly confirmations: number;
   private readonly contractAddress: string;
   private readonly decimals: number;
+  private readonly cursorStore?: IWatcherCursorStore;
+  private readonly watcherId: string;
   private _watchedAddresses: string[];
 
   constructor(opts: EvmWatcherOpts) {
@@ -41,6 +45,8 @@ export class EvmWatcher {
     this.rpc = opts.rpcCall;
     this._cursor = opts.fromBlock;
     this.onPayment = opts.onPayment;
+    this.cursorStore = opts.cursorStore;
+    this.watcherId = `evm:${opts.chain}:${opts.token}`;
     this._watchedAddresses = (opts.watchedAddresses ?? []).map((a) => a.toLowerCase());
 
     const chainCfg = getChainConfig(opts.chain);
@@ -50,6 +56,13 @@ export class EvmWatcher {
     this.decimals = tokenCfg.decimals;
   }
 
+  /** Load cursor from DB. Call once at startup before first poll. */
+  async init(): Promise<void> {
+    if (!this.cursorStore) return;
+    const saved = await this.cursorStore.get(this.watcherId);
+    if (saved !== null) this._cursor = saved;
+  }
+
   /** Update the set of watched deposit addresses (e.g. after a new checkout). */
   setWatchedAddresses(addresses: string[]): void {
     this._watchedAddresses = addresses.map((a) => a.toLowerCase());
@@ -86,28 +99,53 @@ export class EvmWatcher {
       },
     ])) as RpcLog[];
 
+    // Group logs by block for incremental cursor checkpointing.
+    // If onPayment fails mid-batch, only the current block is replayed on next poll.
+    const logsByBlock = new Map<number, RpcLog[]>();
     for (const log of logs) {
-      const to = `0x${log.topics[2].slice(26)}`.toLowerCase();
-      const from = `0x${log.topics[1].slice(26)}`.toLowerCase();
-      const rawAmount = BigInt(log.data);
-      const amountUsdCents = centsFromTokenAmount(rawAmount, this.decimals);
-
-      const event: EvmPaymentEvent = {
-        chain: this.chain,
-        token: this.token,
-        from,
-        to,
-        rawAmount: rawAmount.toString(),
-        amountUsdCents,
-        txHash: log.transactionHash,
-        blockNumber: Number.parseInt(log.blockNumber, 16),
-        logIndex: Number.parseInt(log.logIndex, 16),
-      };
-
-      await this.onPayment(event);
+      const bn = Number.parseInt(log.blockNumber, 16);
+      const arr = logsByBlock.get(bn);
+      if (arr) arr.push(log);
+      else logsByBlock.set(bn, [log]);
     }
 
-    this._cursor = confirmed + 1;
+    // Process blocks in order, checkpoint after each.
+    const blockNums = [...logsByBlock.keys()].sort((a, b) => a - b);
+    for (const blockNum of blockNums) {
+      for (const log of logsByBlock.get(blockNum) ?? []) {
+        const to = `0x${log.topics[2].slice(26)}`.toLowerCase();
+        const from = `0x${log.topics[1].slice(26)}`.toLowerCase();
+        const rawAmount = BigInt(log.data);
+        const amountUsdCents = centsFromTokenAmount(rawAmount, this.decimals);
+
+        const event: EvmPaymentEvent = {
+          chain: this.chain,
+          token: this.token,
+          from,
+          to,
+          rawAmount: rawAmount.toString(),
+          amountUsdCents,
+          txHash: log.transactionHash,
+          blockNumber: blockNum,
+          logIndex: Number.parseInt(log.logIndex, 16),
+        };
+
+        await this.onPayment(event);
+      }
+
+      this._cursor = blockNum + 1;
+      if (this.cursorStore) {
+        await this.cursorStore.save(this.watcherId, this._cursor);
+      }
+    }
+
+    // Advance cursor even if no logs were found in the range.
+    if (blockNums.length === 0) {
+      this._cursor = confirmed + 1;
+      if (this.cursorStore) {
+        await this.cursorStore.save(this.watcherId, this._cursor);
+      }
+    }
   }
 }
 

package/src/billing/crypto/index.ts

@@ -4,6 +4,8 @@ export { CryptoChargeRepository, DrizzleCryptoChargeRepository } from "./charge-
 export { createCryptoCheckout, MIN_PAYMENT_USD } from "./checkout.js";
 export type { CryptoConfig } from "./client.js";
 export { BTCPayClient, loadCryptoConfig } from "./client.js";
+export type { IWatcherCursorStore } from "./cursor-store.js";
+export { DrizzleWatcherCursorStore } from "./cursor-store.js";
 export * from "./evm/index.js";
 export * from "./oracle/index.js";
 export type {

package/src/db/schema/crypto.ts

@@ -1,5 +1,5 @@
 import { sql } from "drizzle-orm";
-import { index, integer, pgTable, text } from "drizzle-orm/pg-core";
+import { index, integer, pgTable, primaryKey, text } from "drizzle-orm/pg-core";
 
 /**
  * Crypto payment charges — tracks the lifecycle of each BTCPay invoice.
@@ -35,3 +35,24 @@ export const cryptoCharges = pgTable(
     // Enforced via migration: CREATE UNIQUE INDEX.
   ],
 );
+
+/**
+ * Watcher cursor persistence — tracks the last processed block per watcher.
+ * Eliminates in-memory processedTxids and enables clean restart recovery.
+ */
+export const watcherCursors = pgTable("watcher_cursors", {
+  watcherId: text("watcher_id").primaryKey(),
+  cursorBlock: integer("cursor_block").notNull(),
+  updatedAt: text("updated_at").notNull().default(sql`(now())`),
+});
+
+/** Processed transaction IDs for watchers without block cursors (e.g. BTC). */
+export const watcherProcessed = pgTable(
+  "watcher_processed",
+  {
+    watcherId: text("watcher_id").notNull(),
+    txId: text("tx_id").notNull(),
+    processedAt: text("processed_at").notNull().default(sql`(now())`),
+  },
+  (table) => [primaryKey({ columns: [table.watcherId, table.txId] })],
+);

package/src/fleet/__tests__/rollout-orchestrator.test.ts

@@ -0,0 +1,321 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { RolloutOrchestrator } from "../rollout-orchestrator.js";
+import type { IRolloutStrategy } from "../rollout-strategy.js";
+import type { BotProfile } from "../types.js";
+import type { ContainerUpdater, UpdateResult } from "../updater.js";
+import type { VolumeSnapshotManager } from "../volume-snapshot-manager.js";
+
+function makeProfile(id: string, volumeName?: string): BotProfile {
+  return {
+    id,
+    tenantId: "tenant-1",
+    name: `bot-${id}`,
+    description: "",
+    image: "ghcr.io/wopr-network/paperclip:managed",
+    env: {},
+    restartPolicy: "unless-stopped",
+    releaseChannel: "stable",
+    updatePolicy: "nightly",
+    volumeName,
+  } as BotProfile;
+}
+
+function makeResult(botId: string, success: boolean): UpdateResult {
+  return {
+    botId,
+    success,
+    previousImage: "old:latest",
+    newImage: "new:latest",
+    previousDigest: "sha256:old",
+    newDigest: "sha256:new",
+    rolledBack: !success,
+    error: success ? undefined : "Health check failed",
+  };
+}
+
+function mockUpdater(results: Map<string, UpdateResult>): ContainerUpdater {
+  return {
+    updateBot: vi.fn(async (botId: string) => results.get(botId) ?? makeResult(botId, true)),
+  } as unknown as ContainerUpdater;
+}
+
+function mockSnapshotManager(): VolumeSnapshotManager {
+  return {
+    snapshot: vi.fn(async (volumeName: string) => ({
+      id: `${volumeName}-snap`,
+      volumeName,
+      archivePath: `/backup/${volumeName}-snap.tar`,
+      createdAt: new Date(),
+      sizeBytes: 1024,
+    })),
+    restore: vi.fn(async () => {}),
+    delete: vi.fn(async () => {}),
+  } as unknown as VolumeSnapshotManager;
+}
+
+function mockStrategy(overrides: Partial<IRolloutStrategy> = {}): IRolloutStrategy {
+  return {
+    nextBatch: (remaining) => remaining.slice(0, 2),
+    pauseDuration: () => 0,
+    onBotFailure: () => "skip",
+    maxRetries: () => 2,
+    healthCheckTimeout: () => 120_000,
+    ...overrides,
+  };
+}
+
+describe("RolloutOrchestrator", () => {
+  let updater: ReturnType<typeof mockUpdater>;
+  let snapMgr: ReturnType<typeof mockSnapshotManager>;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    updater = mockUpdater(new Map());
+    snapMgr = mockSnapshotManager();
+  });
+
+  it("processes all bots in batches", async () => {
+    const profiles = [makeProfile("b1", "vol-1"), makeProfile("b2", "vol-2"), makeProfile("b3", "vol-3")];
+    const strategy = mockStrategy({ nextBatch: (r) => r.slice(0, 2) });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.totalBots).toBe(3);
+    expect(result.succeeded).toBe(3);
+    expect(result.failed).toBe(0);
+    expect(result.aborted).toBe(false);
+    expect(updater.updateBot).toHaveBeenCalledTimes(3);
+  });
+
+  it("snapshots volumes before updating", async () => {
+    const profiles = [makeProfile("b1", "my-volume")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    await orch.rollout();
+
+    expect(snapMgr.snapshot).toHaveBeenCalledWith("my-volume");
+    // On success, snapshot is cleaned up
+    expect(snapMgr.delete).toHaveBeenCalledWith("my-volume-snap");
+  });
+
+  it("skips snapshot for bots without volumes", async () => {
+    const profiles = [makeProfile("b1")]; // no volumeName
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    await orch.rollout();
+
+    expect(snapMgr.snapshot).not.toHaveBeenCalled();
+    expect(updater.updateBot).toHaveBeenCalledWith("b1");
+  });
+
+  it("restores volumes on update failure", async () => {
+    const failResults = new Map([["b1", makeResult("b1", false)]]);
+    updater = mockUpdater(failResults);
+    const profiles = [makeProfile("b1", "my-volume")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.failed).toBe(1);
+    expect(result.results[0].volumeRestored).toBe(true);
+    expect(snapMgr.restore).toHaveBeenCalledWith("my-volume-snap");
+    // Snapshot NOT deleted on failure (restored instead)
+    expect(snapMgr.delete).not.toHaveBeenCalled();
+  });
+
+  it("aborts rollout when strategy says abort", async () => {
+    const failResults = new Map([["b1", makeResult("b1", false)]]);
+    updater = mockUpdater(failResults);
+    const profiles = [makeProfile("b1", "v1"), makeProfile("b2", "v2"), makeProfile("b3", "v3")];
+    const strategy = mockStrategy({
+      nextBatch: (r) => r.slice(0, 1),
+      onBotFailure: () => "abort",
+    });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.aborted).toBe(true);
+    expect(result.succeeded).toBe(0);
+    expect(result.failed).toBe(1);
+    expect(result.skipped).toBe(2); // b2, b3 never processed
+    expect(updater.updateBot).toHaveBeenCalledTimes(1);
+  });
+
+  it("returns empty result when no bots to update", async () => {
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => [],
+    });
+
+    const result = await orch.rollout();
+
+    expect(result.totalBots).toBe(0);
+    expect(result.results).toHaveLength(0);
+  });
+
+  it("rejects concurrent rollouts", async () => {
+    const profiles = [makeProfile("b1")];
+    // Make updateBot slow
+    updater = {
+      updateBot: vi.fn(async (botId: string) => {
+        await new Promise((r) => setTimeout(r, 100));
+        return makeResult(botId, true);
+      }),
+    } as unknown as ContainerUpdater;
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const [r1, r2] = await Promise.all([orch.rollout(), orch.rollout()]);
+
+    // One succeeds, one is rejected as already running
+    const succeeded = [r1, r2].find((r) => r.totalBots > 0);
+    const rejected = [r1, r2].find((r) => r.alreadyRunning);
+    expect(succeeded).toBeDefined();
+    expect(rejected).toBeDefined();
+    expect(rejected?.alreadyRunning).toBe(true);
+    expect(rejected?.totalBots).toBe(0);
+  });
+
+  it("retries failed bots when strategy says retry", async () => {
+    let callCount = 0;
+    updater = {
+      updateBot: vi.fn(async (botId: string) => {
+        callCount++;
+        // Fail first attempt, succeed on retry
+        if (botId === "b1" && callCount === 1) return makeResult("b1", false);
+        return makeResult(botId, true);
+      }),
+    } as unknown as ContainerUpdater;
+
+    const profiles = [makeProfile("b1")];
+    const strategy = mockStrategy({
+      nextBatch: (r) => r.slice(0, 1),
+      onBotFailure: (_botId, _err, attempt) => (attempt < 2 ? "retry" : "skip"),
+    });
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy,
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    // b1 failed once, retried, succeeded
+    expect(updater.updateBot).toHaveBeenCalledTimes(2);
+    expect(result.succeeded).toBe(1);
+    expect(result.failed).toBe(1); // first attempt counted as failed
+  });
+
+  it("calls onBotUpdated callback for each bot", async () => {
+    const profiles = [makeProfile("b1"), makeProfile("b2")];
+    const onBotUpdated = vi.fn();
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+      onBotUpdated,
+    });
+
+    await orch.rollout();
+
+    expect(onBotUpdated).toHaveBeenCalledTimes(2);
+  });
+
+  it("calls onRolloutComplete callback", async () => {
+    const profiles = [makeProfile("b1")];
+    const onRolloutComplete = vi.fn();
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+      onRolloutComplete,
+    });
+
+    await orch.rollout();
+
+    expect(onRolloutComplete).toHaveBeenCalledTimes(1);
+    expect(onRolloutComplete).toHaveBeenCalledWith(
+      expect.objectContaining({ totalBots: 1, succeeded: 1, aborted: false }),
+    );
+  });
+
+  it("continues on snapshot failure (best-effort)", async () => {
+    const profiles = [makeProfile("b1", "my-volume")];
+    snapMgr.snapshot = vi.fn().mockRejectedValue(new Error("disk full"));
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    const result = await orch.rollout();
+
+    // Update still proceeds despite snapshot failure
+    expect(result.succeeded).toBe(1);
+    expect(updater.updateBot).toHaveBeenCalledWith("b1");
+  });
+
+  it("isRolling reflects rollout state", async () => {
+    const profiles = [makeProfile("b1")];
+
+    const orch = new RolloutOrchestrator({
+      updater,
+      snapshotManager: snapMgr,
+      strategy: mockStrategy(),
+      getUpdatableProfiles: async () => profiles,
+    });
+
+    expect(orch.isRolling).toBe(false);
+    const promise = orch.rollout();
+    // isRolling is true during rollout (may already be done for sync mocks)
+    await promise;
+    expect(orch.isRolling).toBe(false);
+  });
+});

package/src/fleet/index.ts

@@ -1,4 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-orchestrator.js";
 export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";