@wopr-network/platform-core 1.18.0 → 1.19.0

package/dist/fleet/volume-snapshot-manager.js

@@ -0,0 +1,185 @@
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { logger } from "../config/logger.js";
+const ALPINE_IMAGE = "alpine:latest";
+/** Strict validation for snapshot IDs — prevents path traversal and shell injection. */
+const SNAPSHOT_ID_RE = /^[A-Za-z0-9._-]+$/;
+function validateSnapshotId(snapshotId) {
+    if (!SNAPSHOT_ID_RE.test(snapshotId)) {
+        throw new Error(`Invalid snapshot ID: ${snapshotId}`);
+    }
+}
+/**
+ * Snapshots and restores Docker named volumes using temporary alpine containers.
+ * Used for nuclear rollback during fleet updates — if a container update fails,
+ * we roll back both the image AND the data volumes.
+ */
+export class VolumeSnapshotManager {
+    docker;
+    backupDir;
+    constructor(docker, backupDir = "/data/fleet/snapshots") {
+        this.docker = docker;
+        this.backupDir = backupDir;
+    }
+    /** Create a snapshot of a Docker named volume */
+    async snapshot(volumeName) {
+        await mkdir(this.backupDir, { recursive: true });
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        const id = `${volumeName}-${timestamp}`;
+        const archivePath = join(this.backupDir, `${id}.tar`);
+        const container = await this.docker.createContainer({
+            Image: ALPINE_IMAGE,
+            Cmd: ["tar", "cf", `/backup/${id}.tar`, "-C", "/source", "."],
+            HostConfig: {
+                Binds: [`${volumeName}:/source:ro`, `${this.backupDir}:/backup`],
+                AutoRemove: true,
+            },
+        });
+        try {
+            await container.start();
+            const result = await container.wait();
+            if (result.StatusCode !== 0) {
+                throw new Error(`Snapshot container exited with code ${result.StatusCode}`);
+            }
+        }
+        catch (err) {
+            // AutoRemove handles cleanup, but if start failed the container may still exist
+            try {
+                await container.remove({ force: true });
+            }
+            catch {
+                // already removed by AutoRemove
+            }
+            throw err;
+        }
+        const info = await stat(archivePath);
+        const snapshot = {
+            id,
+            volumeName,
+            archivePath,
+            createdAt: new Date(),
+            sizeBytes: info.size,
+        };
+        logger.info(`Volume snapshot created: ${id} (${info.size} bytes)`);
+        return snapshot;
+    }
+    /** Restore a volume from a snapshot */
+    async restore(snapshotId) {
+        validateSnapshotId(snapshotId);
+        const archivePath = join(this.backupDir, `${snapshotId}.tar`);
+        // Verify archive exists
+        await stat(archivePath);
+        // Extract volume name from snapshot ID (everything before the last ISO timestamp)
+        const volumeName = this.extractVolumeName(snapshotId);
+        const container = await this.docker.createContainer({
+            Image: ALPINE_IMAGE,
+            Cmd: ["sh", "-c", `cd /target && rm -rf ./* ./.??* && tar xf /backup/${snapshotId}.tar -C /target`],
+            HostConfig: {
+                Binds: [`${volumeName}:/target`, `${this.backupDir}:/backup:ro`],
+                AutoRemove: true,
+            },
+        });
+        try {
+            await container.start();
+            const result = await container.wait();
+            if (result.StatusCode !== 0) {
+                throw new Error(`Restore container exited with code ${result.StatusCode}`);
+            }
+        }
+        catch (err) {
+            try {
+                await container.remove({ force: true });
+            }
+            catch {
+                // already removed by AutoRemove
+            }
+            throw err;
+        }
+        logger.info(`Volume restored from snapshot: ${snapshotId}`);
+    }
+    /** List all snapshots for a volume */
+    async list(volumeName) {
+        let files;
+        try {
+            files = await readdir(this.backupDir);
+        }
+        catch {
+            return [];
+        }
+        const prefix = `${volumeName}-`;
+        const matching = files.filter((f) => f.startsWith(prefix) && f.endsWith(".tar"));
+        const snapshots = [];
+        for (const file of matching) {
+            const id = file.replace(/\.tar$/, "");
+            const archivePath = join(this.backupDir, file);
+            try {
+                const info = await stat(archivePath);
+                snapshots.push({
+                    id,
+                    volumeName,
+                    archivePath,
+                    createdAt: info.mtime,
+                    sizeBytes: info.size,
+                });
+            }
+            catch {
+                // File disappeared between readdir and stat — skip
+            }
+        }
+        // Sort newest first
+        snapshots.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
+        return snapshots;
+    }
+    /** Delete a snapshot archive */
+    async delete(snapshotId) {
+        validateSnapshotId(snapshotId);
+        const archivePath = join(this.backupDir, `${snapshotId}.tar`);
+        await rm(archivePath, { force: true });
+        logger.info(`Volume snapshot deleted: ${snapshotId}`);
+    }
+    /** Delete all snapshots older than maxAge ms */
+    async cleanup(maxAgeMs) {
+        let files;
+        try {
+            files = await readdir(this.backupDir);
+        }
+        catch {
+            return 0;
+        }
+        const cutoff = Date.now() - maxAgeMs;
+        let deleted = 0;
+        for (const file of files) {
+            if (!file.endsWith(".tar"))
+                continue;
+            const archivePath = join(this.backupDir, file);
+            try {
+                const info = await stat(archivePath);
+                if (info.mtime.getTime() < cutoff) {
+                    await rm(archivePath, { force: true });
+                    deleted++;
+                }
+            }
+            catch {
+                // File disappeared — skip
+            }
+        }
+        if (deleted > 0) {
+            logger.info(`Volume snapshot cleanup: removed ${deleted} old snapshots`);
+        }
+        return deleted;
+    }
+    /**
+     * Extract volume name from snapshot ID.
+     * Snapshot IDs are `${volumeName}-${ISO timestamp with colons/dots replaced}`.
+     * ISO timestamps start with 4 digits (year), so we find the last occurrence
+     * of `-YYYY` pattern to split.
+     */
+    extractVolumeName(snapshotId) {
+        // Match the timestamp part: -YYYY-MM-DDTHH-MM-SS-MMMZ
+        const match = snapshotId.match(/^(.+)-\d{4}-\d{2}-\d{2}T/);
+        if (!match) {
+            throw new Error(`Cannot extract volume name from snapshot ID: ${snapshotId}`);
+        }
+        return match[1];
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wopr-network/platform-core",
-  "version": "1.18.0",
+  "version": "1.19.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/api/routes/admin-audit-helper.ts

@@ -2,7 +2,7 @@ import type { AuditEntry } from "../../admin/audit-log.js";
 
 /** Minimal interface for admin audit logging in route factories. */
 export interface AdminAuditLogger {
-  log(entry: AuditEntry): void | Promise<unknown>;
+  log(entry: AuditEntry): undefined | Promise<unknown>;
 }
 
 /** Safely log an admin audit entry — never throws. */

package/src/fleet/__tests__/rollout-strategy.test.ts

@@ -0,0 +1,192 @@
+import { randomUUID } from "node:crypto";
+import { describe, expect, it } from "vitest";
+import {
+  createRolloutStrategy,
+  ImmediateStrategy,
+  RollingWaveStrategy,
+  SingleBotStrategy,
+} from "../rollout-strategy.js";
+import type { BotProfile } from "../types.js";
+
+function makeBots(count: number): BotProfile[] {
+  return Array.from({ length: count }, (_, i) => ({
+    id: randomUUID(),
+    tenantId: "tenant-1",
+    name: `bot-${i}`,
+    description: "",
+    image: "ghcr.io/wopr-network/test:latest",
+    env: {},
+    restartPolicy: "unless-stopped" as const,
+    releaseChannel: "stable" as const,
+    updatePolicy: "manual" as const,
+  }));
+}
+
+describe("RollingWaveStrategy", () => {
+  it("returns batchPercent of remaining bots", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 25 });
+    const bots = makeBots(10);
+    const batch = s.nextBatch(bots);
+    // 25% of 10 = 2.5, ceil = 3
+    expect(batch).toHaveLength(3);
+    expect(batch).toEqual(bots.slice(0, 3));
+  });
+
+  it("returns minimum 1 bot even with low percentage", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 1 });
+    const bots = makeBots(2);
+    expect(s.nextBatch(bots)).toHaveLength(1);
+  });
+
+  it("returns empty array for empty remaining", () => {
+    const s = new RollingWaveStrategy();
+    expect(s.nextBatch([])).toHaveLength(0);
+  });
+
+  it("handles 1 bot remaining", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 50 });
+    const bots = makeBots(1);
+    expect(s.nextBatch(bots)).toHaveLength(1);
+  });
+
+  it("handles batchPercent > 100", () => {
+    const s = new RollingWaveStrategy({ batchPercent: 200 });
+    const bots = makeBots(5);
+    // 200% of 5 = 10, ceil = 10, but slice(0,10) on 5 items = 5
+    expect(s.nextBatch(bots)).toHaveLength(5);
+  });
+
+  it("returns correct pause duration", () => {
+    expect(new RollingWaveStrategy().pauseDuration()).toBe(60_000);
+    expect(new RollingWaveStrategy({ pauseMs: 30_000 }).pauseDuration()).toBe(30_000);
+  });
+
+  it("retries on failure until maxRetries", () => {
+    const s = new RollingWaveStrategy({ maxFailures: 2 });
+    const err = new Error("fail");
+    expect(s.onBotFailure("b1", err, 0)).toBe("retry");
+    expect(s.onBotFailure("b1", err, 1)).toBe("retry");
+  });
+
+  it("skips after maxRetries when under maxFailures", () => {
+    const s = new RollingWaveStrategy({ maxFailures: 3 });
+    const err = new Error("fail");
+    // attempt >= maxRetries (2), first total failure
+    expect(s.onBotFailure("b1", err, 2)).toBe("skip");
+  });
+
+  it("aborts when total failures reach maxFailures", () => {
+    const s = new RollingWaveStrategy({ maxFailures: 2 });
+    const err = new Error("fail");
+    // exhaust retries for bot1 → skip (totalFailures=1)
+    expect(s.onBotFailure("b1", err, 2)).toBe("skip");
+    // exhaust retries for bot2 → abort (totalFailures=2 >= maxFailures=2)
+    expect(s.onBotFailure("b2", err, 2)).toBe("abort");
+  });
+
+  it("has maxRetries of 2", () => {
+    expect(new RollingWaveStrategy().maxRetries()).toBe(2);
+  });
+
+  it("has healthCheckTimeout of 120_000", () => {
+    expect(new RollingWaveStrategy().healthCheckTimeout()).toBe(120_000);
+  });
+});
+
+describe("SingleBotStrategy", () => {
+  it("returns exactly 1 bot", () => {
+    const s = new SingleBotStrategy();
+    const bots = makeBots(10);
+    expect(s.nextBatch(bots)).toHaveLength(1);
+    expect(s.nextBatch(bots)[0]).toBe(bots[0]);
+  });
+
+  it("returns empty for empty remaining", () => {
+    expect(new SingleBotStrategy().nextBatch([])).toHaveLength(0);
+  });
+
+  it("has pauseDuration of 0", () => {
+    expect(new SingleBotStrategy().pauseDuration()).toBe(0);
+  });
+
+  it("always retries on failure", () => {
+    const s = new SingleBotStrategy();
+    const err = new Error("fail");
+    expect(s.onBotFailure("b1", err, 0)).toBe("retry");
+    expect(s.onBotFailure("b1", err, 1)).toBe("retry");
+    expect(s.onBotFailure("b1", err, 2)).toBe("retry");
+    // After maxRetries (3), aborts instead of retrying forever
+    expect(s.onBotFailure("b1", err, 99)).toBe("abort");
+  });
+
+  it("has maxRetries of 3", () => {
+    expect(new SingleBotStrategy().maxRetries()).toBe(3);
+  });
+
+  it("has healthCheckTimeout of 120_000", () => {
+    expect(new SingleBotStrategy().healthCheckTimeout()).toBe(120_000);
+  });
+});
+
+describe("ImmediateStrategy", () => {
+  it("returns all remaining bots", () => {
+    const s = new ImmediateStrategy();
+    const bots = makeBots(10);
+    expect(s.nextBatch(bots)).toHaveLength(10);
+    expect(s.nextBatch(bots)).toEqual(bots);
+  });
+
+  it("returns empty for empty remaining", () => {
+    expect(new ImmediateStrategy().nextBatch([])).toHaveLength(0);
+  });
+
+  it("does not mutate the input array", () => {
+    const s = new ImmediateStrategy();
+    const bots = makeBots(3);
+    const result = s.nextBatch(bots);
+    expect(result).not.toBe(bots);
+    expect(result).toEqual(bots);
+  });
+
+  it("has pauseDuration of 0", () => {
+    expect(new ImmediateStrategy().pauseDuration()).toBe(0);
+  });
+
+  it("always skips on failure", () => {
+    const s = new ImmediateStrategy();
+    const err = new Error("fail");
+    expect(s.onBotFailure("b1", err, 0)).toBe("skip");
+    expect(s.onBotFailure("b1", err, 5)).toBe("skip");
+  });
+
+  it("has maxRetries of 1", () => {
+    expect(new ImmediateStrategy().maxRetries()).toBe(1);
+  });
+
+  it("has healthCheckTimeout of 60_000", () => {
+    expect(new ImmediateStrategy().healthCheckTimeout()).toBe(60_000);
+  });
+});
+
+describe("createRolloutStrategy", () => {
+  it("creates RollingWaveStrategy", () => {
+    const s = createRolloutStrategy("rolling-wave");
+    expect(s).toBeInstanceOf(RollingWaveStrategy);
+  });
+
+  it("creates RollingWaveStrategy with options", () => {
+    const s = createRolloutStrategy("rolling-wave", { batchPercent: 50, pauseMs: 10_000 });
+    expect(s).toBeInstanceOf(RollingWaveStrategy);
+    expect(s.pauseDuration()).toBe(10_000);
+  });
+
+  it("creates SingleBotStrategy", () => {
+    const s = createRolloutStrategy("single-bot");
+    expect(s).toBeInstanceOf(SingleBotStrategy);
+  });
+
+  it("creates ImmediateStrategy", () => {
+    const s = createRolloutStrategy("immediate");
+    expect(s).toBeInstanceOf(ImmediateStrategy);
+  });
+});

package/src/fleet/__tests__/volume-snapshot-manager.test.ts

@@ -0,0 +1,218 @@
+import type Docker from "dockerode";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { VolumeSnapshotManager } from "../volume-snapshot-manager.js";
+
+// Mock fs/promises
+vi.mock("node:fs/promises", () => ({
+  mkdir: vi.fn().mockResolvedValue(undefined),
+  stat: vi.fn().mockResolvedValue({ size: 1024, mtime: new Date("2026-03-14T10:00:00Z") }),
+  readdir: vi.fn().mockResolvedValue([]),
+  rm: vi.fn().mockResolvedValue(undefined),
+}));
+
+// Mock logger
+vi.mock("../../config/logger.js", () => ({
+  logger: {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    debug: vi.fn(),
+  },
+}));
+
+import { mkdir, readdir, rm, stat } from "node:fs/promises";
+
+function mockContainer() {
+  return {
+    start: vi.fn().mockResolvedValue(undefined),
+    wait: vi.fn().mockResolvedValue({ StatusCode: 0 }),
+    remove: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+function mockDocker(): Docker {
+  return {
+    createContainer: vi.fn().mockResolvedValue(mockContainer()),
+  } as unknown as Docker;
+}
+
+describe("VolumeSnapshotManager", () => {
+  let docker: Docker;
+  let manager: VolumeSnapshotManager;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    docker = mockDocker();
+    manager = new VolumeSnapshotManager(docker, "/data/fleet/snapshots");
+  });
+
+  describe("snapshot()", () => {
+    it("creates container with correct binds and runs tar", async () => {
+      await manager.snapshot("my-volume");
+
+      expect(docker.createContainer).toHaveBeenCalledWith(
+        expect.objectContaining({
+          Image: "alpine:latest",
+          Cmd: expect.arrayContaining(["tar", "cf"]),
+          HostConfig: expect.objectContaining({
+            Binds: expect.arrayContaining(["my-volume:/source:ro", "/data/fleet/snapshots:/backup"]),
+            AutoRemove: true,
+          }),
+        }),
+      );
+
+      const container = await (docker.createContainer as ReturnType<typeof vi.fn>).mock.results[0].value;
+      expect(container.start).toHaveBeenCalled();
+      expect(container.wait).toHaveBeenCalled();
+    });
+
+    it("returns a VolumeSnapshot with correct fields", async () => {
+      const result = await manager.snapshot("my-volume");
+
+      expect(result.volumeName).toBe("my-volume");
+      expect(result.id).toMatch(/^my-volume-\d{4}-\d{2}-\d{2}T/);
+      expect(result.archivePath).toMatch(/^\/data\/fleet\/snapshots\/my-volume-.*\.tar$/);
+      expect(result.sizeBytes).toBe(1024);
+      expect(result.createdAt).toBeInstanceOf(Date);
+    });
+
+    it("ensures backup directory exists", async () => {
+      await manager.snapshot("my-volume");
+      expect(mkdir).toHaveBeenCalledWith("/data/fleet/snapshots", { recursive: true });
+    });
+
+    it("cleans up container on start failure", async () => {
+      const container = mockContainer();
+      container.start.mockRejectedValue(new Error("start failed"));
+      (docker.createContainer as ReturnType<typeof vi.fn>).mockResolvedValue(container);
+
+      await expect(manager.snapshot("my-volume")).rejects.toThrow("start failed");
+      expect(container.remove).toHaveBeenCalledWith({ force: true });
+    });
+  });
+
+  describe("restore()", () => {
+    it("creates container with correct binds and runs tar xf", async () => {
+      const snapshotId = "my-volume-2026-03-14T10-00-00-000Z";
+      await manager.restore(snapshotId);
+
+      expect(docker.createContainer).toHaveBeenCalledWith(
+        expect.objectContaining({
+          Image: "alpine:latest",
+          Cmd: ["sh", "-c", `cd /target && rm -rf ./* ./.??* && tar xf /backup/${snapshotId}.tar -C /target`],
+          HostConfig: expect.objectContaining({
+            Binds: expect.arrayContaining(["my-volume:/target", "/data/fleet/snapshots:/backup:ro"]),
+            AutoRemove: true,
+          }),
+        }),
+      );
+    });
+
+    it("starts and waits for container", async () => {
+      const container = mockContainer();
+      (docker.createContainer as ReturnType<typeof vi.fn>).mockResolvedValue(container);
+
+      await manager.restore("my-volume-2026-03-14T10-00-00-000Z");
+
+      expect(container.start).toHaveBeenCalled();
+      expect(container.wait).toHaveBeenCalled();
+    });
+
+    it("throws if archive does not exist", async () => {
+      (stat as ReturnType<typeof vi.fn>).mockRejectedValueOnce(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      await expect(manager.restore("nonexistent-2026-03-14T10-00-00-000Z")).rejects.toThrow("ENOENT");
+    });
+  });
+
+  describe("list()", () => {
+    it("returns snapshots sorted by date, newest first", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "my-volume-2026-03-12T08-00-00-000Z.tar",
+        "my-volume-2026-03-14T10-00-00-000Z.tar",
+        "my-volume-2026-03-13T09-00-00-000Z.tar",
+      ]);
+
+      const oldDate = new Date("2026-03-12T08:00:00Z");
+      const midDate = new Date("2026-03-13T09:00:00Z");
+      const newDate = new Date("2026-03-14T10:00:00Z");
+
+      (stat as ReturnType<typeof vi.fn>)
+        .mockResolvedValueOnce({ size: 100, mtime: oldDate })
+        .mockResolvedValueOnce({ size: 300, mtime: newDate })
+        .mockResolvedValueOnce({ size: 200, mtime: midDate });
+
+      const result = await manager.list("my-volume");
+
+      expect(result).toHaveLength(3);
+      expect(result[0].createdAt).toEqual(newDate);
+      expect(result[1].createdAt).toEqual(midDate);
+      expect(result[2].createdAt).toEqual(oldDate);
+    });
+
+    it("filters to only matching volume name", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "my-volume-2026-03-14T10-00-00-000Z.tar",
+        "other-volume-2026-03-14T10-00-00-000Z.tar",
+      ]);
+
+      const result = await manager.list("my-volume");
+      expect(result).toHaveLength(1);
+      expect(result[0].volumeName).toBe("my-volume");
+    });
+
+    it("returns empty array when backup dir does not exist", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      const result = await manager.list("my-volume");
+      expect(result).toEqual([]);
+    });
+  });
+
+  describe("delete()", () => {
+    it("removes the archive file", async () => {
+      await manager.delete("my-volume-2026-03-14T10-00-00-000Z");
+
+      expect(rm).toHaveBeenCalledWith("/data/fleet/snapshots/my-volume-2026-03-14T10-00-00-000Z.tar", { force: true });
+    });
+  });
+
+  describe("cleanup()", () => {
+    it("removes old snapshots and keeps recent ones", async () => {
+      const now = Date.now();
+      const oldTime = new Date(now - 2 * 60 * 60 * 1000); // 2 hours ago
+      const recentTime = new Date(now - 10 * 60 * 1000); // 10 minutes ago
+
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue([
+        "vol-2026-03-14T08-00-00-000Z.tar",
+        "vol-2026-03-14T09-50-00-000Z.tar",
+      ]);
+
+      (stat as ReturnType<typeof vi.fn>)
+        .mockResolvedValueOnce({ size: 100, mtime: oldTime })
+        .mockResolvedValueOnce({ size: 200, mtime: recentTime });
+
+      const maxAge = 60 * 60 * 1000; // 1 hour
+      const deleted = await manager.cleanup(maxAge);
+
+      expect(deleted).toBe(1);
+      expect(rm).toHaveBeenCalledTimes(1);
+      expect(rm).toHaveBeenCalledWith("/data/fleet/snapshots/vol-2026-03-14T08-00-00-000Z.tar", { force: true });
+    });
+
+    it("returns 0 when backup dir does not exist", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockRejectedValue(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
+
+      const result = await manager.cleanup(60_000);
+      expect(result).toBe(0);
+    });
+
+    it("skips non-tar files", async () => {
+      (readdir as ReturnType<typeof vi.fn>).mockResolvedValue(["readme.txt", ".gitkeep"]);
+
+      const result = await manager.cleanup(60_000);
+      expect(result).toBe(0);
+      expect(stat).not.toHaveBeenCalled();
+    });
+  });
+});

package/src/fleet/index.ts

@@ -1,3 +1,5 @@
 export * from "./repository-types.js";
+export * from "./rollout-strategy.js";
 export * from "./services.js";
 export * from "./types.js";
+export * from "./volume-snapshot-manager.js";