@wopr-network/platform-core 1.17.0 → 1.19.0

package/src/fleet/__tests__/volume-snapshot-manager.test.ts

@@ -0,0 +1,218 @@
+import type Docker from "dockerode";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { VolumeSnapshotManager } from "../volume-snapshot-manager.js";
+
+// Mock fs/promises
+vi.mock("node:fs/promises", () => ({
+  mkdir: vi.fn().mockResolvedValue(undefined),
+  stat: vi.fn().mockResolvedValue({ size: 1024, mtime: new Date("2026-03-14T10:00:00Z") }),
+  readdir: vi.fn().mockResolvedValue([]),
+  rm: vi.fn().mockResolvedValue(undefined),
+}));
+
+// Mock logger
+vi.mock("../../config/logger.js", () => ({
+  logger: {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
+
+function mockContainer() {
+  return {
+    start: vi.fn().mockResolvedValue(undefined),
+    wait: vi.fn().mockResolvedValue({ StatusCode: 0 }),
+    remove: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+function mockDocker(): Docker {
+  return {
+    createContainer: vi.fn().mockResolvedValue(mockContainer()),
+  } as unknown as Docker;
+}
+
+describe("VolumeSnapshotManager", () => {
+  let docker: Docker;
+  let manager: VolumeSnapshotManager;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    docker = mockDocker();
+    manager = new VolumeSnapshotManager(docker, "/data/fleet/snapshots");
+  });
+
+  describe("snapshot()", () => {
+    it("creates container with correct binds and runs tar", async () => {
+      await manager.snapshot("my-volume");
+
+      expect(docker.createContainer).toHaveBeenCalledWith(
+        expect.objectContaining({
+          Image: "alpine:latest",
+          Cmd: expect.arrayContaining(["tar", "cf"]),
+          HostConfig: expect.objectContaining({
+            Binds: expect.arrayContaining(["my-volume:/source:ro", "/data/fleet/snapshots:/backup"]),
+            AutoRemove: true,
+          }),
+        }),
+      );
+
+      const container = await (docker.createContainer as ReturnType<typeof vi.fn>).mock.results[0].value;
+      expect(container.start).toHaveBeenCalled();
+      expect(container.wait).toHaveBeenCalled();
+    });
+
+    it("returns a VolumeSnapshot with correct fields", async () => {
+      const result = await manager.snapshot("my-volume");
+
+      expect(result.volumeName).toBe("my-volume");
+      expect(result.id).toMatch(/^my-volume-\d{4}-\d{2}-\d{2}T/);
+      expect(result.archivePath).toMatch(/^\/data\/fleet\/snapshots\/my-volume-.*\.tar$/);
+      expect(result.sizeBytes).toBe(1024);
+      expect(result.createdAt).toBeInstanceOf(Date);
+    });
+
+    it("ensures backup directory exists", async () => {
+      await manager.snapshot("my-volume");
+      expect(mkdir).toHaveBeenCalledWith("/data/fleet/snapshots", { recursive: true });
+    });
+
+    it("cleans up container on start failure", async () => {
+      const container = mockContainer();
+      container.start.mockRejectedValue(new Error("start failed"));
+      (docker.createContainer as ReturnType<typeof vi.fn>).mockResolvedValue(container);
+
+      await expect(manager.snapshot("my-volume")).rejects.toThrow("start failed");
+      expect(container.remove).toHaveBeenCalledWith({ force: true });
+    });
+  });
+
+  describe("restore()", () => {
+    it("creates container with correct binds and runs tar xf", async () => {
+      const snapshotId = "my-volume-2026-03-14T10-00-00-000Z";
+      await manager.restore(snapshotId);
+
+      expect(docker.createContainer).toHaveBeenCalledWith(
+        expect.objectContaining({
+          Image: "alpine:latest",
+          Cmd: ["sh", "-c", `cd /target && rm -rf ./* ./.??* && tar xf /backup/${snapshotId}.tar -C /target`],
+          HostConfig: expect.objectContaining({
+            Binds: expect.arrayContaining(["my-volume:/target", "/data/fleet/snapshots:/backup:ro"]),
+            AutoRemove: true,
+          }),
+        }),
+      );
+    });
+
+    it("starts and waits for container", async () => {
+      const container = mockContainer();
+      (docker.createContainer as ReturnType<typeof vi.fn>).mockResolvedValue(container);
+
+      await manager.restore("my-volume-2026-03-14T10-00-00-000Z");
+
+      expect(container.start).toHaveBeenCalled();
+      expect(container.wait).toHaveBeenCalled();
+    });
+
+    it("throws if archive does not exist", async () => {
+      (stat as ReturnType<typeof vi.fn>).mockRejectedValueOnce(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      await expect(manager.restore("nonexistent-2026-03-14T10-00-00-000Z")).rejects.toThrow("ENOENT");
+    });
+  });
+
+  describe("list()", () => {
+    it("returns snapshots sorted by date, newest first", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "my-volume-2026-03-12T08-00-00-000Z.tar",
+        "my-volume-2026-03-14T10-00-00-000Z.tar",
+        "my-volume-2026-03-13T09-00-00-000Z.tar",
+      ]);
+
+      const oldDate = new Date("2026-03-12T08:00:00Z");
+      const midDate = new Date("2026-03-13T09:00:00Z");
+      const newDate = new Date("2026-03-14T10:00:00Z");
+
+      (stat as ReturnType<typeof vi.fn>)
+        .mockResolvedValueOnce({ size: 100, mtime: oldDate })
+        .mockResolvedValueOnce({ size: 300, mtime: newDate })
+        .mockResolvedValueOnce({ size: 200, mtime: midDate });
+
+      const result = await manager.list("my-volume");
+
+      expect(result).toHaveLength(3);
+      expect(result[0].createdAt).toEqual(newDate);
+      expect(result[1].createdAt).toEqual(midDate);
+      expect(result[2].createdAt).toEqual(oldDate);
+    });
+
+    it("filters to only matching volume name", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "my-volume-2026-03-14T10-00-00-000Z.tar",
+        "other-volume-2026-03-14T10-00-00-000Z.tar",
+      ]);
+
+      const result = await manager.list("my-volume");
+      expect(result).toHaveLength(1);
+      expect(result[0].volumeName).toBe("my-volume");
+    });
+
+    it("returns empty array when backup dir does not exist", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      const result = await manager.list("my-volume");
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe("delete()", () => {
+    it("removes the archive file", async () => {
+      await manager.delete("my-volume-2026-03-14T10-00-00-000Z");
+
+      expect(rm).toHaveBeenCalledWith("/data/fleet/snapshots/my-volume-2026-03-14T10-00-00-000Z.tar", { force: true });
+    });
+  });
+
+  describe("cleanup()", () => {
+    it("removes old snapshots and keeps recent ones", async () => {
+      const now = Date.now();
+      const oldTime = new Date(now - 2 * 60 * 60 * 1000); // 2 hours ago
+      const recentTime = new Date(now - 10 * 60 * 1000); // 10 minutes ago
+
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "vol-2026-03-14T08-00-00-000Z.tar",
+        "vol-2026-03-14T09-50-00-000Z.tar",
+      ]);
+
+      (stat as ReturnType<typeof vi.fn>)
+        .mockResolvedValueOnce({ size: 100, mtime: oldTime })
+        .mockResolvedValueOnce({ size: 200, mtime: recentTime });
+
+      const maxAge = 60 * 60 * 1000; // 1 hour
+      const deleted = await manager.cleanup(maxAge);
+
+      expect(deleted).toBe(1);
+      expect(rm).toHaveBeenCalledTimes(1);
+      expect(rm).toHaveBeenCalledWith("/data/fleet/snapshots/vol-2026-03-14T08-00-00-000Z.tar", { force: true });
+    });
+
+    it("returns 0 when backup dir does not exist", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      const result = await manager.cleanup(60_000);
+      expect(result).toBe(0);
+    });
+
+    it("skips non-tar files", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue(["readme.txt", ".gitkeep"]);
+
+      const result = await manager.cleanup(60_000);
+      expect(result).toBe(0);
+      expect(stat).not.toHaveBeenCalled();
+    });
+  });
+});

package/src/fleet/index.ts

@@ -1,3 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";
+export * from "./volume-snapshot-manager.js";

package/src/fleet/rollout-strategy.ts

@@ -0,0 +1,128 @@
+import type { BotProfile } from "./types.js";
+
+export interface IRolloutStrategy {
+  /** Select next batch from remaining bots */
+  nextBatch(remaining: BotProfile[]): BotProfile[];
+  /** Milliseconds to wait between waves */
+  pauseDuration(): number;
+  /** What to do when a single bot update fails */
+  onBotFailure(botId: string, error: Error, attempt: number): "abort" | "skip" | "retry";
+  /** Max retries per bot before skip/abort */
+  maxRetries(): number;
+  /** Health check timeout per bot (ms) */
+  healthCheckTimeout(): number;
+}
+
+export interface RollingWaveOptions {
+  batchPercent?: number;
+  pauseMs?: number;
+  maxFailures?: number;
+}
+
+/**
+ * Rolling wave strategy — processes bots in configurable percentage batches.
+ * Create a new instance per rollout; totalFailures accumulates across waves
+ * within a single rollout. Call reset() if reusing across rollouts.
+ */
+export class RollingWaveStrategy implements IRolloutStrategy {
+  private readonly batchPercent: number;
+  private readonly pauseMs: number;
+  private readonly maxFailures: number;
+  private totalFailures = 0;
+
+  constructor(opts: RollingWaveOptions = {}) {
+    this.batchPercent = opts.batchPercent ?? 25;
+    this.pauseMs = opts.pauseMs ?? 60_000;
+    this.maxFailures = opts.maxFailures ?? 3;
+  }
+
+  nextBatch(remaining: BotProfile[]): BotProfile[] {
+    if (remaining.length === 0) return [];
+    const count = Math.max(1, Math.ceil((remaining.length * this.batchPercent) / 100));
+    return remaining.slice(0, count);
+  }
+
+  pauseDuration(): number {
+    return this.pauseMs;
+  }
+
+  onBotFailure(_botId: string, _error: Error, attempt: number): "abort" | "skip" | "retry" {
+    if (attempt < this.maxRetries()) return "retry";
+    this.totalFailures++;
+    if (this.totalFailures >= this.maxFailures) return "abort";
+    return "skip";
+  }
+
+  maxRetries(): number {
+    return 2;
+  }
+
+  healthCheckTimeout(): number {
+    return 120_000;
+  }
+
+  /** Reset failure counters for reuse across rollouts. */
+  reset(): void {
+    this.totalFailures = 0;
+  }
+}
+
+export class SingleBotStrategy implements IRolloutStrategy {
+  nextBatch(remaining: BotProfile[]): BotProfile[] {
+    if (remaining.length === 0) return [];
+    return remaining.slice(0, 1);
+  }
+
+  pauseDuration(): number {
+    return 0;
+  }
+
+  onBotFailure(_botId: string, _error: Error, attempt: number): "abort" | "skip" | "retry" {
+    if (attempt < this.maxRetries()) return "retry";
+    return "abort";
+  }
+
+  maxRetries(): number {
+    return 3;
+  }
+
+  healthCheckTimeout(): number {
+    return 120_000;
+  }
+}
+
+export class ImmediateStrategy implements IRolloutStrategy {
+  nextBatch(remaining: BotProfile[]): BotProfile[] {
+    return [...remaining];
+  }
+
+  pauseDuration(): number {
+    return 0;
+  }
+
+  onBotFailure(_botId: string, _error: Error, _attempt: number): "abort" | "skip" | "retry" {
+    return "skip";
+  }
+
+  maxRetries(): number {
+    return 1;
+  }
+
+  healthCheckTimeout(): number {
+    return 60_000;
+  }
+}
+
+export function createRolloutStrategy(
+  type: "rolling-wave" | "single-bot" | "immediate",
+  options?: RollingWaveOptions,
+): IRolloutStrategy {
+  switch (type) {
+    case "rolling-wave":
+      return new RollingWaveStrategy(options);
+    case "single-bot":
+      return new SingleBotStrategy();
+    case "immediate":
+      return new ImmediateStrategy();
+  }
+}

package/src/fleet/volume-snapshot-manager.ts

@@ -0,0 +1,213 @@
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
+import { join } from "node:path";
+import type Docker from "dockerode";
+import { logger } from "../config/logger.js";
+
+export interface VolumeSnapshot {
+  id: string;
+  volumeName: string;
+  archivePath: string;
+  createdAt: Date;
+  sizeBytes: number;
+}
+
+const ALPINE_IMAGE = "alpine:latest";
+
+/** Strict validation for snapshot IDs — prevents path traversal and shell injection. */
+const SNAPSHOT_ID_RE = /^[A-Za-z0-9._-]+$/;
+
+function validateSnapshotId(snapshotId: string): void {
+  if (!SNAPSHOT_ID_RE.test(snapshotId)) {
+    throw new Error(`Invalid snapshot ID: ${snapshotId}`);
+  }
+}
+
+/**
+ * Snapshots and restores Docker named volumes using temporary alpine containers.
+ * Used for nuclear rollback during fleet updates — if a container update fails,
+ * we roll back both the image AND the data volumes.
+ */
+export class VolumeSnapshotManager {
+  private readonly docker: Docker;
+  private readonly backupDir: string;
+
+  constructor(docker: Docker, backupDir = "/data/fleet/snapshots") {
+    this.docker = docker;
+    this.backupDir = backupDir;
+  }
+
+  /** Create a snapshot of a Docker named volume */
+  async snapshot(volumeName: string): Promise<VolumeSnapshot> {
+    await mkdir(this.backupDir, { recursive: true });
+
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const id = `${volumeName}-${timestamp}`;
+    const archivePath = join(this.backupDir, `${id}.tar`);
+
+    const container = await this.docker.createContainer({
+      Image: ALPINE_IMAGE,
+      Cmd: ["tar", "cf", `/backup/${id}.tar`, "-C", "/source", "."],
+      HostConfig: {
+        Binds: [`${volumeName}:/source:ro`, `${this.backupDir}:/backup`],
+        AutoRemove: true,
+      },
+    });
+
+    try {
+      await container.start();
+      const result = await container.wait();
+      if (result.StatusCode !== 0) {
+        throw new Error(`Snapshot container exited with code ${result.StatusCode}`);
+      }
+    } catch (err) {
+      // AutoRemove handles cleanup, but if start failed the container may still exist
+      try {
+        await container.remove({ force: true });
+      } catch {
+        // already removed by AutoRemove
+      }
+      throw err;
+    }
+
+    const info = await stat(archivePath);
+
+    const snapshot: VolumeSnapshot = {
+      id,
+      volumeName,
+      archivePath,
+      createdAt: new Date(),
+      sizeBytes: info.size,
+    };
+
+    logger.info(`Volume snapshot created: ${id} (${info.size} bytes)`);
+    return snapshot;
+  }
+
+  /** Restore a volume from a snapshot */
+  async restore(snapshotId: string): Promise<void> {
+    validateSnapshotId(snapshotId);
+    const archivePath = join(this.backupDir, `${snapshotId}.tar`);
+
+    // Verify archive exists
+    await stat(archivePath);
+
+    // Extract volume name from snapshot ID (everything before the last ISO timestamp)
+    const volumeName = this.extractVolumeName(snapshotId);
+
+    const container = await this.docker.createContainer({
+      Image: ALPINE_IMAGE,
+      Cmd: ["sh", "-c", `cd /target && rm -rf ./* ./.??* && tar xf /backup/${snapshotId}.tar -C /target`],
+      HostConfig: {
+        Binds: [`${volumeName}:/target`, `${this.backupDir}:/backup:ro`],
+        AutoRemove: true,
+      },
+    });
+
+    try {
+      await container.start();
+      const result = await container.wait();
+      if (result.StatusCode !== 0) {
+        throw new Error(`Restore container exited with code ${result.StatusCode}`);
+      }
+    } catch (err) {
+      try {
+        await container.remove({ force: true });
+      } catch {
+        // already removed by AutoRemove
+      }
+      throw err;
+    }
+
+    logger.info(`Volume restored from snapshot: ${snapshotId}`);
+  }
+
+  /** List all snapshots for a volume */
+  async list(volumeName: string): Promise<VolumeSnapshot[]> {
+    let files: string[];
+    try {
+      files = await readdir(this.backupDir);
+    } catch {
+      return [];
+    }
+
+    const prefix = `${volumeName}-`;
+    const matching = files.filter((f) => f.startsWith(prefix) && f.endsWith(".tar"));
+
+    const snapshots: VolumeSnapshot[] = [];
+    for (const file of matching) {
+      const id = file.replace(/\.tar$/, "");
+      const archivePath = join(this.backupDir, file);
+      try {
+        const info = await stat(archivePath);
+        snapshots.push({
+          id,
+          volumeName,
+          archivePath,
+          createdAt: info.mtime,
+          sizeBytes: info.size,
+        });
+      } catch {
+        // File disappeared between readdir and stat — skip
+      }
+    }
+
+    // Sort newest first
+    snapshots.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
+    return snapshots;
+  }
+
+  /** Delete a snapshot archive */
+  async delete(snapshotId: string): Promise<void> {
+    validateSnapshotId(snapshotId);
+    const archivePath = join(this.backupDir, `${snapshotId}.tar`);
+    await rm(archivePath, { force: true });
+    logger.info(`Volume snapshot deleted: ${snapshotId}`);
+  }
+
+  /** Delete all snapshots older than maxAge ms */
+  async cleanup(maxAgeMs: number): Promise<number> {
+    let files: string[];
+    try {
+      files = await readdir(this.backupDir);
+    } catch {
+      return 0;
+    }
+
+    const cutoff = Date.now() - maxAgeMs;
+    let deleted = 0;
+
+    for (const file of files) {
+      if (!file.endsWith(".tar")) continue;
+      const archivePath = join(this.backupDir, file);
+      try {
+        const info = await stat(archivePath);
+        if (info.mtime.getTime() < cutoff) {
+          await rm(archivePath, { force: true });
+          deleted++;
+        }
+      } catch {
+        // File disappeared — skip
+      }
+    }
+
+    if (deleted > 0) {
+      logger.info(`Volume snapshot cleanup: removed ${deleted} old snapshots`);
+    }
+    return deleted;
+  }
+
+  /**
+   * Extract volume name from snapshot ID.
+   * Snapshot IDs are `${volumeName}-${ISO timestamp with colons/dots replaced}`.
+   * ISO timestamps start with 4 digits (year), so we find the last occurrence
+   * of `-YYYY` pattern to split.
+   */
+  private extractVolumeName(snapshotId: string): string {
+    // Match the timestamp part: -YYYY-MM-DDTHH-MM-SS-MMMZ
+    const match = snapshotId.match(/^(.+)-\d{4}-\d{2}-\d{2}T/);
+    if (!match) {
+      throw new Error(`Cannot extract volume name from snapshot ID: ${snapshotId}`);
+    }
+    return match[1];
+  }
+}

package/src/marketplace/volume-installer.test.ts

@@ -238,7 +238,10 @@ describe("rollbackPluginOnVolume", () => {
 });
 
 describe("npm package validation", () => {
-  const validExec = vi.fn((_cmd: unknown, _args: unknown, _opts: unknown, cb: Function) => cb(null, "", ""));
+  const validExec = vi.fn(
+    (_cmd: unknown, _args: unknown, _opts: unknown, cb: (err: unknown, stdout: string, stderr: string) => void) =>
+      cb(null, "", ""),
+  );
 
   beforeEach(() => {
     validExec.mockClear();
@@ -350,7 +353,10 @@ describe("npm package validation", () => {
   });
 
   it("passes -- separator before package name to prevent flag injection", async () => {
-    const execFn = vi.fn((_cmd: unknown, _args: unknown, _opts: unknown, cb: Function) => cb(null, "", ""));
+    const execFn = vi.fn(
+      (_cmd: unknown, _args: unknown, _opts: unknown, cb: (err: unknown, stdout: string, stderr: string) => void) =>
+        cb(null, "", ""),
+    );
     await installPluginToVolume({
       pluginId: "p1",
       npmPackage: "lodash",