@wopr-network/platform-core 1.18.0 → 1.20.0

package/src/fleet/__tests__/rollout-strategy.test.ts

@@ -0,0 +1,192 @@
+import { randomUUID } from "node:crypto";
+import { describe, expect, it } from "vitest";
+import {
+  createRolloutStrategy,
+  ImmediateStrategy,
+  RollingWaveStrategy,
+  SingleBotStrategy,
+} from "../rollout-strategy.js";
+import type { BotProfile } from "../types.js";
+
+function makeBots(count: number): BotProfile[] {
+  return Array.from({ length: count }, (_, i) => ({
+    id: randomUUID(),
+    tenantId: "tenant-1",
+    name: `bot-${i}`,
+    description: "",
+    image: "ghcr.io/wopr-network/test:latest",
+    env: {},
+    restartPolicy: "unless-stopped" as const,
+    releaseChannel: "stable" as const,
+    updatePolicy: "manual" as const,
+  }));
+}
+
+describe("RollingWaveStrategy", () => {
+  it("returns batchPercent of remaining bots", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 25 });
+    const bots = makeBots(10);
+    const batch = s.nextBatch(bots);
+    // 25% of 10 = 2.5, ceil = 3
+    expect(batch).toHaveLength(3);
+    expect(batch).toEqual(bots.slice(0, 3));
+  });
+
+  it("returns minimum 1 bot even with low percentage", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 1 });
+    const bots = makeBots(2);
+    expect(s.nextBatch(bots)).toHaveLength(1);
+  });
+
+  it("returns empty array for empty remaining", () => {
+    const s = new RollingWaveStrategy();
+    expect(s.nextBatch([])).toHaveLength(0);
+  });
+
+  it("handles 1 bot remaining", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 50 });
+    const bots = makeBots(1);
+    expect(s.nextBatch(bots)).toHaveLength(1);
+  });
+
+  it("handles batchPercent > 100", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 200 });
+    const bots = makeBots(5);
+    // 200% of 5 = 10, ceil = 10, but slice(0,10) on 5 items = 5
+    expect(s.nextBatch(bots)).toHaveLength(5);
+  });
+
+  it("returns correct pause duration", () => {
+    expect(new RollingWaveStrategy().pauseDuration()).toBe(60_000);
+    expect(new RollingWaveStrategy({ pauseMs: 30_000 }).pauseDuration()).toBe(30_000);
+  });
+
+  it("retries on failure until maxRetries", () => {
+    const s = new RollingWaveStrategy({ maxFailures: 2 });
+    const err = new Error("fail");
+    expect(s.onBotFailure("b1", err, 0)).toBe("retry");
+    expect(s.onBotFailure("b1", err, 1)).toBe("retry");
+  });
+
+  it("skips after maxRetries when under maxFailures", () => {
+    const s = new RollingWaveStrategy({ maxFailures: 3 });
+    const err = new Error("fail");
+    // attempt >= maxRetries (2), first total failure
+    expect(s.onBotFailure("b1", err, 2)).toBe("skip");
+  });
+
+  it("aborts when total failures reach maxFailures", () => {
+    const s = new RollingWaveStrategy({ maxFailures: 2 });
+    const err = new Error("fail");
+    // exhaust retries for bot1 → skip (totalFailures=1)
+    expect(s.onBotFailure("b1", err, 2)).toBe("skip");
+    // exhaust retries for bot2 → abort (totalFailures=2 >= maxFailures=2)
+    expect(s.onBotFailure("b2", err, 2)).toBe("abort");
+  });
+
+  it("has maxRetries of 2", () => {
+    expect(new RollingWaveStrategy().maxRetries()).toBe(2);
+  });
+
+  it("has healthCheckTimeout of 120_000", () => {
+    expect(new RollingWaveStrategy().healthCheckTimeout()).toBe(120_000);
+  });
+});
+
+describe("SingleBotStrategy", () => {
+  it("returns exactly 1 bot", () => {
+    const s = new SingleBotStrategy();
+    const bots = makeBots(10);
+    expect(s.nextBatch(bots)).toHaveLength(1);
+    expect(s.nextBatch(bots)[0]).toBe(bots[0]);
+  });
+
+  it("returns empty for empty remaining", () => {
+    expect(new SingleBotStrategy().nextBatch([])).toHaveLength(0);
+  });
+
+  it("has pauseDuration of 0", () => {
+    expect(new SingleBotStrategy().pauseDuration()).toBe(0);
+  });
+
+  it("always retries on failure", () => {
+    const s = new SingleBotStrategy();
+    const err = new Error("fail");
+    expect(s.onBotFailure("b1", err, 0)).toBe("retry");
+    expect(s.onBotFailure("b1", err, 1)).toBe("retry");
+    expect(s.onBotFailure("b1", err, 2)).toBe("retry");
+    // After maxRetries (3), aborts instead of retrying forever
+    expect(s.onBotFailure("b1", err, 99)).toBe("abort");
+  });
+
+  it("has maxRetries of 3", () => {
+    expect(new SingleBotStrategy().maxRetries()).toBe(3);
+  });
+
+  it("has healthCheckTimeout of 120_000", () => {
+    expect(new SingleBotStrategy().healthCheckTimeout()).toBe(120_000);
+  });
+});
+
+describe("ImmediateStrategy", () => {
+  it("returns all remaining bots", () => {
+    const s = new ImmediateStrategy();
+    const bots = makeBots(10);
+    expect(s.nextBatch(bots)).toHaveLength(10);
+    expect(s.nextBatch(bots)).toEqual(bots);
+  });
+
+  it("returns empty for empty remaining", () => {
+    expect(new ImmediateStrategy().nextBatch([])).toHaveLength(0);
+  });
+
+  it("does not mutate the input array", () => {
+    const s = new ImmediateStrategy();
+    const bots = makeBots(3);
+    const result = s.nextBatch(bots);
+    expect(result).not.toBe(bots);
+    expect(result).toEqual(bots);
+  });
+
+  it("has pauseDuration of 0", () => {
+    expect(new ImmediateStrategy().pauseDuration()).toBe(0);
+  });
+
+  it("always skips on failure", () => {
+    const s = new ImmediateStrategy();
+    const err = new Error("fail");
+    expect(s.onBotFailure("b1", err, 0)).toBe("skip");
+    expect(s.onBotFailure("b1", err, 5)).toBe("skip");
+  });
+
+  it("has maxRetries of 1", () => {
+    expect(new ImmediateStrategy().maxRetries()).toBe(1);
+  });
+
+  it("has healthCheckTimeout of 60_000", () => {
+    expect(new ImmediateStrategy().healthCheckTimeout()).toBe(60_000);
+  });
+});
+
+describe("createRolloutStrategy", () => {
+  it("creates RollingWaveStrategy", () => {
+    const s = createRolloutStrategy("rolling-wave");
+    expect(s).toBeInstanceOf(RollingWaveStrategy);
+  });
+
+  it("creates RollingWaveStrategy with options", () => {
+    const s = createRolloutStrategy("rolling-wave", { batchPercent: 50, pauseMs: 10_000 });
+    expect(s).toBeInstanceOf(RollingWaveStrategy);
+    expect(s.pauseDuration()).toBe(10_000);
+  });
+
+  it("creates SingleBotStrategy", () => {
+    const s = createRolloutStrategy("single-bot");
+    expect(s).toBeInstanceOf(SingleBotStrategy);
+  });
+
+  it("creates ImmediateStrategy", () => {
+    const s = createRolloutStrategy("immediate");
+    expect(s).toBeInstanceOf(ImmediateStrategy);
+  });
+});

package/src/fleet/__tests__/volume-snapshot-manager.test.ts

@@ -0,0 +1,218 @@
+import type Docker from "dockerode";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { VolumeSnapshotManager } from "../volume-snapshot-manager.js";
+
+// Mock fs/promises
+vi.mock("node:fs/promises", () => ({
+  mkdir: vi.fn().mockResolvedValue(undefined),
+  stat: vi.fn().mockResolvedValue({ size: 1024, mtime: new Date("2026-03-14T10:00:00Z") }),
+  readdir: vi.fn().mockResolvedValue([]),
+  rm: vi.fn().mockResolvedValue(undefined),
+}));
+
+// Mock logger
+vi.mock("../../config/logger.js", () => ({
+  logger: {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
+
+function mockContainer() {
+  return {
+    start: vi.fn().mockResolvedValue(undefined),
+    wait: vi.fn().mockResolvedValue({ StatusCode: 0 }),
+    remove: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+function mockDocker(): Docker {
+  return {
+    createContainer: vi.fn().mockResolvedValue(mockContainer()),
+  } as unknown as Docker;
+}
+
+describe("VolumeSnapshotManager", () => {
+  let docker: Docker;
+  let manager: VolumeSnapshotManager;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    docker = mockDocker();
+    manager = new VolumeSnapshotManager(docker, "/data/fleet/snapshots");
+  });
+
+  describe("snapshot()", () => {
+    it("creates container with correct binds and runs tar", async () => {
+      await manager.snapshot("my-volume");
+
+      expect(docker.createContainer).toHaveBeenCalledWith(
+        expect.objectContaining({
+          Image: "alpine:latest",
+          Cmd: expect.arrayContaining(["tar", "cf"]),
+          HostConfig: expect.objectContaining({
+            Binds: expect.arrayContaining(["my-volume:/source:ro", "/data/fleet/snapshots:/backup"]),
+            AutoRemove: true,
+          }),
+        }),
+      );
+
+      const container = await (docker.createContainer as ReturnType<typeof vi.fn>).mock.results[0].value;
+      expect(container.start).toHaveBeenCalled();
+      expect(container.wait).toHaveBeenCalled();
+    });
+
+    it("returns a VolumeSnapshot with correct fields", async () => {
+      const result = await manager.snapshot("my-volume");
+
+      expect(result.volumeName).toBe("my-volume");
+      expect(result.id).toMatch(/^my-volume-\d{4}-\d{2}-\d{2}T/);
+      expect(result.archivePath).toMatch(/^\/data\/fleet\/snapshots\/my-volume-.*\.tar$/);
+      expect(result.sizeBytes).toBe(1024);
+      expect(result.createdAt).toBeInstanceOf(Date);
+    });
+
+    it("ensures backup directory exists", async () => {
+      await manager.snapshot("my-volume");
+      expect(mkdir).toHaveBeenCalledWith("/data/fleet/snapshots", { recursive: true });
+    });
+
+    it("cleans up container on start failure", async () => {
+      const container = mockContainer();
+      container.start.mockRejectedValue(new Error("start failed"));
+      (docker.createContainer as ReturnType<typeof vi.fn>).mockResolvedValue(container);
+
+      await expect(manager.snapshot("my-volume")).rejects.toThrow("start failed");
+      expect(container.remove).toHaveBeenCalledWith({ force: true });
+    });
+  });
+
+  describe("restore()", () => {
+    it("creates container with correct binds and runs tar xf", async () => {
+      const snapshotId = "my-volume-2026-03-14T10-00-00-000Z";
+      await manager.restore(snapshotId);
+
+      expect(docker.createContainer).toHaveBeenCalledWith(
+        expect.objectContaining({
+          Image: "alpine:latest",
+          Cmd: ["sh", "-c", `cd /target && rm -rf ./* ./.??* && tar xf /backup/${snapshotId}.tar -C /target`],
+          HostConfig: expect.objectContaining({
+            Binds: expect.arrayContaining(["my-volume:/target", "/data/fleet/snapshots:/backup:ro"]),
+            AutoRemove: true,
+          }),
+        }),
+      );
+    });
+
+    it("starts and waits for container", async () => {
+      const container = mockContainer();
+      (docker.createContainer as ReturnType<typeof vi.fn>).mockResolvedValue(container);
+
+      await manager.restore("my-volume-2026-03-14T10-00-00-000Z");
+
+      expect(container.start).toHaveBeenCalled();
+      expect(container.wait).toHaveBeenCalled();
+    });
+
+    it("throws if archive does not exist", async () => {
+      (stat as ReturnType<typeof vi.fn>).mockRejectedValueOnce(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      await expect(manager.restore("nonexistent-2026-03-14T10-00-00-000Z")).rejects.toThrow("ENOENT");
+    });
+  });
+
+  describe("list()", () => {
+    it("returns snapshots sorted by date, newest first", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "my-volume-2026-03-12T08-00-00-000Z.tar",
+        "my-volume-2026-03-14T10-00-00-000Z.tar",
+        "my-volume-2026-03-13T09-00-00-000Z.tar",
+      ]);
+
+      const oldDate = new Date("2026-03-12T08:00:00Z");
+      const midDate = new Date("2026-03-13T09:00:00Z");
+      const newDate = new Date("2026-03-14T10:00:00Z");
+
+      (stat as ReturnType<typeof vi.fn>)
+        .mockResolvedValueOnce({ size: 100, mtime: oldDate })
+        .mockResolvedValueOnce({ size: 300, mtime: newDate })
+        .mockResolvedValueOnce({ size: 200, mtime: midDate });
+
+      const result = await manager.list("my-volume");
+
+      expect(result).toHaveLength(3);
+      expect(result[0].createdAt).toEqual(newDate);
+      expect(result[1].createdAt).toEqual(midDate);
+      expect(result[2].createdAt).toEqual(oldDate);
+    });
+
+    it("filters to only matching volume name", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "my-volume-2026-03-14T10-00-00-000Z.tar",
+        "other-volume-2026-03-14T10-00-00-000Z.tar",
+      ]);
+
+      const result = await manager.list("my-volume");
+      expect(result).toHaveLength(1);
+      expect(result[0].volumeName).toBe("my-volume");
+    });
+
+    it("returns empty array when backup dir does not exist", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      const result = await manager.list("my-volume");
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe("delete()", () => {
+    it("removes the archive file", async () => {
+      await manager.delete("my-volume-2026-03-14T10-00-00-000Z");
+
+      expect(rm).toHaveBeenCalledWith("/data/fleet/snapshots/my-volume-2026-03-14T10-00-00-000Z.tar", { force: true });
+    });
+  });
+
+  describe("cleanup()", () => {
+    it("removes old snapshots and keeps recent ones", async () => {
+      const now = Date.now();
+      const oldTime = new Date(now - 2 * 60 * 60 * 1000); // 2 hours ago
+      const recentTime = new Date(now - 10 * 60 * 1000); // 10 minutes ago
+
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "vol-2026-03-14T08-00-00-000Z.tar",
+        "vol-2026-03-14T09-50-00-000Z.tar",
+      ]);
+
+      (stat as ReturnType<typeof vi.fn>)
+        .mockResolvedValueOnce({ size: 100, mtime: oldTime })
+        .mockResolvedValueOnce({ size: 200, mtime: recentTime });
+
+      const maxAge = 60 * 60 * 1000; // 1 hour
+      const deleted = await manager.cleanup(maxAge);
+
+      expect(deleted).toBe(1);
+      expect(rm).toHaveBeenCalledTimes(1);
+      expect(rm).toHaveBeenCalledWith("/data/fleet/snapshots/vol-2026-03-14T08-00-00-000Z.tar", { force: true });
+    });
+
+    it("returns 0 when backup dir does not exist", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      const result = await manager.cleanup(60_000);
+      expect(result).toBe(0);
+    });
+
+    it("skips non-tar files", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue(["readme.txt", ".gitkeep"]);
+
+      const result = await manager.cleanup(60_000);
+      expect(result).toBe(0);
+      expect(stat).not.toHaveBeenCalled();
+    });
+  });
+});

package/src/fleet/index.ts

@@ -1,3 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";
+export * from "./volume-snapshot-manager.js";

package/src/fleet/rollout-strategy.ts

@@ -0,0 +1,128 @@
+import type { BotProfile } from "./types.js";
+
+export interface IRolloutStrategy {
+  /** Select next batch from remaining bots */
+  nextBatch(remaining: BotProfile[]): BotProfile[];
+  /** Milliseconds to wait between waves */
+  pauseDuration(): number;
+  /** What to do when a single bot update fails */
+  onBotFailure(botId: string, error: Error, attempt: number): "abort" | "skip" | "retry";
+  /** Max retries per bot before skip/abort */
+  maxRetries(): number;
+  /** Health check timeout per bot (ms) */
+  healthCheckTimeout(): number;
+}
+
+export interface RollingWaveOptions {
+  batchPercent?: number;
+  pauseMs?: number;
+  maxFailures?: number;
+}
+
+/**
+ * Rolling wave strategy — processes bots in configurable percentage batches.
+ * Create a new instance per rollout; totalFailures accumulates across waves
+ * within a single rollout. Call reset() if reusing across rollouts.
+ */
+export class RollingWaveStrategy implements IRolloutStrategy {
+  private readonly batchPercent: number;
+  private readonly pauseMs: number;
+  private readonly maxFailures: number;
+  private totalFailures = 0;
+
+  constructor(opts: RollingWaveOptions = {}) {
+    this.batchPercent = opts.batchPercent ?? 25;
+    this.pauseMs = opts.pauseMs ?? 60_000;
+    this.maxFailures = opts.maxFailures ?? 3;
+  }
+
+  nextBatch(remaining: BotProfile[]): BotProfile[] {
+    if (remaining.length === 0) return [];
+    const count = Math.max(1, Math.ceil((remaining.length * this.batchPercent) / 100));
+    return remaining.slice(0, count);
+  }
+
+  pauseDuration(): number {
+    return this.pauseMs;
+  }
+
+  onBotFailure(_botId: string, _error: Error, attempt: number): "abort" | "skip" | "retry" {
+    if (attempt < this.maxRetries()) return "retry";
+    this.totalFailures++;
+    if (this.totalFailures >= this.maxFailures) return "abort";
+    return "skip";
+  }
+
+  maxRetries(): number {
+    return 2;
+  }
+
+  healthCheckTimeout(): number {
+    return 120_000;
+  }
+
+  /** Reset failure counters for reuse across rollouts. */
+  reset(): void {
+    this.totalFailures = 0;
+  }
+}
+
+export class SingleBotStrategy implements IRolloutStrategy {
+  nextBatch(remaining: BotProfile[]): BotProfile[] {
+    if (remaining.length === 0) return [];
+    return remaining.slice(0, 1);
+  }
+
+  pauseDuration(): number {
+    return 0;
+  }
+
+  onBotFailure(_botId: string, _error: Error, attempt: number): "abort" | "skip" | "retry" {
+    if (attempt < this.maxRetries()) return "retry";
+    return "abort";
+  }
+
+  maxRetries(): number {
+    return 3;
+  }
+
+  healthCheckTimeout(): number {
+    return 120_000;
+  }
+}
+
+export class ImmediateStrategy implements IRolloutStrategy {
+  nextBatch(remaining: BotProfile[]): BotProfile[] {
+    return [...remaining];
+  }
+
+  pauseDuration(): number {
+    return 0;
+  }
+
+  onBotFailure(_botId: string, _error: Error, _attempt: number): "abort" | "skip" | "retry" {
+    return "skip";
+  }
+
+  maxRetries(): number {
+    return 1;
+  }
+
+  healthCheckTimeout(): number {
+    return 60_000;
+  }
+}
+
+export function createRolloutStrategy(
+  type: "rolling-wave" | "single-bot" | "immediate",
+  options?: RollingWaveOptions,
+): IRolloutStrategy {
+  switch (type) {
+    case "rolling-wave":
+      return new RollingWaveStrategy(options);
+    case "single-bot":
+      return new SingleBotStrategy();
+    case "immediate":
+      return new ImmediateStrategy();
+  }
+}