@wopr-network/platform-core 1.0.10 → 1.1.0

package/dist/fleet/inference-watchdog.js

@@ -0,0 +1,126 @@
+import { logger } from "../config/logger.js";
+/** Maps service name to its health-check port on the GPU node. */
+const SERVICE_PORTS = {
+    llama: 8080,
+    chatterbox: 8081,
+    whisper: 8082,
+    qwen: 8083,
+};
+export class InferenceWatchdog {
+    repo;
+    doClient;
+    notifier;
+    intervalMs;
+    healthTimeoutMs;
+    rebootThreshold;
+    failedTimeoutMs;
+    timer = null;
+    /** In-memory per-node failure tracking. Keyed by gpu node ID. */
+    nodeStates = new Map();
+    constructor(repo, doClient, notifier, options = {}) {
+        this.repo = repo;
+        this.doClient = doClient;
+        this.notifier = notifier;
+        this.intervalMs = options.intervalMs ?? 30_000;
+        this.healthTimeoutMs = options.healthTimeoutMs ?? 5_000;
+        this.rebootThreshold = options.rebootThreshold ?? 2;
+        this.failedTimeoutMs = options.failedTimeoutMs ?? 600_000;
+    }
+    start() {
+        if (this.timer) {
+            logger.warn("InferenceWatchdog already running");
+            return;
+        }
+        logger.info("Starting inference watchdog", {
+            intervalMs: this.intervalMs,
+            rebootThreshold: this.rebootThreshold,
+            failedTimeoutMs: this.failedTimeoutMs,
+        });
+        this.timer = setInterval(() => {
+            this.check().catch((err) => {
+                logger.error("InferenceWatchdog check failed", { err });
+            });
+        }, this.intervalMs);
+    }
+    stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+            logger.info("Inference watchdog stopped");
+        }
+    }
+    async check() {
+        const nodes = await this.repo.list(["active", "degraded"]);
+        for (const node of nodes) {
+            if (!node.host)
+                continue;
+            const health = await this.pollHealth(node.host);
+            const now = Math.floor(Date.now() / 1000);
+            await this.repo.updateServiceHealth(node.id, health, now);
+            const allDown = Object.values(health).every((s) => s === "down");
+            const anyUp = Object.values(health).some((s) => s === "ok");
+            let state = this.nodeStates.get(node.id);
+            if (!state) {
+                state = { consecutiveAllDown: 0, rebootedAt: null };
+                this.nodeStates.set(node.id, state);
+            }
+            if (anyUp) {
+                if (state.consecutiveAllDown > 0 || state.rebootedAt !== null) {
+                    state.consecutiveAllDown = 0;
+                    state.rebootedAt = null;
+                    if (node.status === "degraded") {
+                        await this.repo.updateStatus(node.id, "active");
+                    }
+                }
+                continue;
+            }
+            if (allDown) {
+                state.consecutiveAllDown++;
+                if (state.rebootedAt !== null) {
+                    const elapsed = Date.now() - state.rebootedAt;
+                    if (elapsed >= this.failedTimeoutMs) {
+                        await this.repo.updateStatus(node.id, "failed");
+                        this.notifier.gpuNodeFailed(node.id).catch((err) => {
+                            logger.error("Failed to send gpuNodeFailed notification", { nodeId: node.id, err });
+                        });
+                        this.nodeStates.delete(node.id);
+                        continue;
+                    }
+                    // Still within reboot window — wait
+                    continue;
+                }
+                if (state.consecutiveAllDown >= this.rebootThreshold) {
+                    await this.repo.updateStatus(node.id, "degraded");
+                    this.notifier.gpuNodeDegraded(node.id, health).catch((err) => {
+                        logger.error("Failed to send gpuNodeDegraded notification", { nodeId: node.id, err });
+                    });
+                    if (node.dropletId) {
+                        this.doClient.rebootDroplet(Number(node.dropletId)).catch((err) => {
+                            logger.error("Failed to reboot droplet", { nodeId: node.id, dropletId: node.dropletId, err });
+                        });
+                    }
+                    else {
+                        logger.error("Cannot reboot GPU node: no dropletId", { nodeId: node.id });
+                    }
+                    state.rebootedAt = Date.now();
+                }
+            }
+        }
+    }
+    async pollHealth(host) {
+        const results = {};
+        const checks = Object.entries(SERVICE_PORTS).map(async ([service, port]) => {
+            try {
+                const res = await fetch(`http://${host}:${port}/health`, {
+                    signal: AbortSignal.timeout(this.healthTimeoutMs),
+                });
+                results[service] = res.ok ? "ok" : "down";
+            }
+            catch {
+                results[service] = "down";
+            }
+        });
+        await Promise.all(checks);
+        return results;
+    }
+}

package/dist/fleet/inference-watchdog.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/fleet/inference-watchdog.test.js

@@ -0,0 +1,174 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { InferenceWatchdog } from "./inference-watchdog.js";
+function createMockRepo(nodes = []) {
+    return {
+        list: vi.fn().mockReturnValue(nodes),
+        updateServiceHealth: vi.fn(),
+        updateStatus: vi.fn(),
+        setError: vi.fn(),
+        insert: vi.fn(),
+        getById: vi.fn(),
+        updateStage: vi.fn(),
+        updateHost: vi.fn(),
+        delete: vi.fn(),
+    };
+}
+function createMockDoClient() {
+    return {
+        rebootDroplet: vi.fn().mockResolvedValue(undefined),
+    };
+}
+function createMockNotifier() {
+    return {
+        gpuNodeDegraded: vi.fn().mockResolvedValue(undefined),
+        gpuNodeFailed: vi.fn().mockResolvedValue(undefined),
+    };
+}
+function mockFetchHealthy() {
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true }));
+}
+function mockFetchAllDown() {
+    vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("connection refused")));
+}
+function mockFetchPartial(downServices) {
+    const downPorts = downServices.map((s) => {
+        const map = { llama: 8080, chatterbox: 8081, whisper: 8082, qwen: 8083 };
+        return map[s];
+    });
+    vi.stubGlobal("fetch", vi.fn().mockImplementation((url) => {
+        const port = Number(new URL(url).port);
+        if (downPorts.includes(port))
+            return Promise.reject(new Error("connection refused"));
+        return Promise.resolve({ ok: true });
+    }));
+}
+describe("InferenceWatchdog", () => {
+    let repo;
+    let doClient;
+    let notifier;
+    let watchdog;
+    const opts = { intervalMs: 1000, healthTimeoutMs: 100, rebootThreshold: 2, failedTimeoutMs: 5000 };
+    beforeEach(() => {
+        vi.useFakeTimers();
+        repo = createMockRepo();
+        doClient = createMockDoClient();
+        notifier = createMockNotifier();
+    });
+    afterEach(() => {
+        watchdog?.stop();
+        vi.useRealTimers();
+        vi.restoreAllMocks();
+        vi.unstubAllGlobals();
+    });
+    it("polls health endpoints and updates serviceHealth when all healthy", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: "10.0.0.1", dropletId: "12345", status: "active" }]);
+        mockFetchHealthy();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(repo.updateServiceHealth).toHaveBeenCalledWith("gpu-1", { llama: "ok", chatterbox: "ok", whisper: "ok", qwen: "ok" }, expect.any(Number));
+        expect(repo.updateStatus).not.toHaveBeenCalled();
+    });
+    it("marks partial failure in serviceHealth without triggering reboot", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: "10.0.0.1", dropletId: "12345", status: "active" }]);
+        mockFetchPartial(["whisper"]);
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(repo.updateServiceHealth).toHaveBeenCalledWith("gpu-1", { llama: "ok", chatterbox: "ok", whisper: "down", qwen: "ok" }, expect.any(Number));
+        expect(doClient.rebootDroplet).not.toHaveBeenCalled();
+    });
+    it("reboots after 2 consecutive all-down cycles", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: "10.0.0.1", dropletId: "12345", status: "active" }]);
+        mockFetchAllDown();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        // Cycle 1: all down, no reboot yet
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(doClient.rebootDroplet).not.toHaveBeenCalled();
+        // Cycle 2: all down again → reboot
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(doClient.rebootDroplet).toHaveBeenCalledWith(12345);
+        expect(repo.updateStatus).toHaveBeenCalledWith("gpu-1", "degraded");
+        expect(notifier.gpuNodeDegraded).toHaveBeenCalledWith("gpu-1", {
+            llama: "down",
+            chatterbox: "down",
+            whisper: "down",
+            qwen: "down",
+        });
+    });
+    it("resets failure count when a service recovers", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: "10.0.0.1", dropletId: "12345", status: "active" }]);
+        mockFetchAllDown();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        // Cycle 1: all down
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        // Cycle 2: one service recovers → counter resets
+        mockFetchPartial(["whisper", "qwen", "chatterbox"]); // llama is up
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(doClient.rebootDroplet).not.toHaveBeenCalled();
+    });
+    it("marks failed after 10min post-reboot still all-down", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: "10.0.0.1", dropletId: "12345", status: "active" }]);
+        mockFetchAllDown();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, { ...opts, failedTimeoutMs: 3000 });
+        watchdog.start();
+        // 2 cycles → reboot
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(doClient.rebootDroplet).toHaveBeenCalledTimes(1);
+        // Wait past failedTimeoutMs while still all-down
+        await vi.advanceTimersByTimeAsync(3000);
+        // The next cycle after timeout should mark failed
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(repo.updateStatus).toHaveBeenCalledWith("gpu-1", "failed");
+        expect(notifier.gpuNodeFailed).toHaveBeenCalledWith("gpu-1");
+    });
+    it("skips nodes with null host", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: null, dropletId: "12345", status: "active" }]);
+        mockFetchHealthy();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(repo.updateServiceHealth).not.toHaveBeenCalled();
+    });
+    it("skips reboot when dropletId is null", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: "10.0.0.1", dropletId: null, status: "active" }]);
+        mockFetchAllDown();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(doClient.rebootDroplet).not.toHaveBeenCalled();
+        expect(repo.updateStatus).toHaveBeenCalledWith("gpu-1", "degraded");
+    });
+    it("queries repo.list with active and degraded statuses", async () => {
+        mockFetchHealthy();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(repo.list).toHaveBeenCalledWith(["active", "degraded"]);
+    });
+    it("does not double-start", () => {
+        mockFetchHealthy();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        watchdog.start(); // second call is no-op
+        watchdog.stop();
+    });
+    it("clears reboot state when node recovers after reboot", async () => {
+        repo.list.mockReturnValue([{ id: "gpu-1", host: "10.0.0.1", dropletId: "12345", status: "degraded" }]);
+        mockFetchAllDown();
+        watchdog = new InferenceWatchdog(repo, doClient, notifier, opts);
+        watchdog.start();
+        // 2 cycles → reboot
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(doClient.rebootDroplet).toHaveBeenCalledTimes(1);
+        // Node recovers
+        mockFetchHealthy();
+        await vi.advanceTimersByTimeAsync(opts.intervalMs);
+        expect(repo.updateStatus).toHaveBeenCalledWith("gpu-1", "active");
+    });
+});

package/dist/fleet/migration-orchestrator.d.ts

@@ -0,0 +1,24 @@
+import type { IBotInstanceRepository } from "./bot-instance-repository.js";
+import type { INodeCommandBus } from "./node-command-bus.js";
+import type { INodeRepository } from "./node-repository.js";
+export interface MigrationResult {
+    success: boolean;
+    botId: string;
+    sourceNodeId: string;
+    targetNodeId: string;
+    downtimeMs: number;
+    error?: string;
+}
+/**
+ * Handles single-tenant live migration: export -> upload -> download -> stop -> import.
+ * Rollback to source on failure.
+ *
+ * CRITICAL: NEVER call docker rm on tenant containers.
+ */
+export declare class MigrationOrchestrator {
+    private readonly commandBus;
+    private readonly botInstanceRepo;
+    private readonly nodeRepo;
+    constructor(commandBus: INodeCommandBus, botInstanceRepo: IBotInstanceRepository, nodeRepo: INodeRepository);
+    migrate(botId: string, targetNodeId?: string, estimatedMb?: number): Promise<MigrationResult>;
+}

package/dist/fleet/migration-orchestrator.js

@@ -0,0 +1,177 @@
+import { logger } from "../config/logger.js";
+/**
+ * Handles single-tenant live migration: export -> upload -> download -> stop -> import.
+ * Rollback to source on failure.
+ *
+ * CRITICAL: NEVER call docker rm on tenant containers.
+ */
+export class MigrationOrchestrator {
+    commandBus;
+    botInstanceRepo;
+    nodeRepo;
+    constructor(commandBus, botInstanceRepo, nodeRepo) {
+        this.commandBus = commandBus;
+        this.botInstanceRepo = botInstanceRepo;
+        this.nodeRepo = nodeRepo;
+    }
+    async migrate(botId, targetNodeId, estimatedMb) {
+        const startTime = Date.now();
+        // 1. Look up bot instance
+        const instance = await this.botInstanceRepo.getById(botId);
+        if (!instance) {
+            return {
+                success: false,
+                botId,
+                sourceNodeId: "unknown",
+                targetNodeId: targetNodeId ?? "unknown",
+                downtimeMs: 0,
+                error: "Bot instance not found",
+            };
+        }
+        const sourceNodeId = instance.nodeId;
+        if (!sourceNodeId) {
+            return {
+                success: false,
+                botId,
+                sourceNodeId: "unassigned",
+                targetNodeId: targetNodeId ?? "unknown",
+                downtimeMs: 0,
+                error: "Bot has no assigned node",
+            };
+        }
+        // 2. Determine target node
+        let resolvedTarget = targetNodeId;
+        if (!resolvedTarget) {
+            const requiredMb = estimatedMb ?? 100;
+            const placement = await this.nodeRepo.findBestTarget(sourceNodeId, requiredMb);
+            if (!placement) {
+                return {
+                    success: false,
+                    botId,
+                    sourceNodeId,
+                    targetNodeId: "none",
+                    downtimeMs: 0,
+                    error: "No node with sufficient capacity",
+                };
+            }
+            resolvedTarget = placement.id;
+        }
+        if (!resolvedTarget) {
+            return {
+                success: false,
+                botId,
+                sourceNodeId,
+                targetNodeId: "unknown",
+                downtimeMs: 0,
+                error: "No target node resolved",
+            };
+        }
+        if (resolvedTarget === sourceNodeId) {
+            return {
+                success: false,
+                botId,
+                sourceNodeId,
+                targetNodeId: resolvedTarget,
+                downtimeMs: 0,
+                error: "Source and target are the same node",
+            };
+        }
+        const containerName = `tenant_${instance.tenantId}`;
+        logger.info(`Starting migration of bot ${botId} (${containerName}): ${sourceNodeId} -> ${resolvedTarget}`);
+        try {
+            // 3. Export container on source node
+            logger.info(`[migrate] Exporting ${containerName} on ${sourceNodeId}`);
+            await this.commandBus.send(sourceNodeId, {
+                type: "bot.export",
+                payload: { name: containerName },
+            });
+            // 4. Upload to DO Spaces
+            logger.info(`[migrate] Uploading ${containerName} backup to Spaces`);
+            await this.commandBus.send(sourceNodeId, {
+                type: "backup.upload",
+                payload: { filename: `${containerName}.tar.gz` },
+            });
+            // 5. Download on target node
+            logger.info(`[migrate] Downloading ${containerName} backup on ${resolvedTarget}`);
+            await this.commandBus.send(resolvedTarget, {
+                type: "backup.download",
+                payload: { filename: `${containerName}.tar.gz` },
+            });
+            // 6. Stop container on source — DOWNTIME STARTS
+            const downtimeStart = Date.now();
+            logger.info(`[migrate] Stopping ${containerName} on ${sourceNodeId}`);
+            await this.commandBus.send(sourceNodeId, {
+                type: "bot.stop",
+                payload: { name: containerName },
+            });
+            const image = process.env.WOPR_BOT_IMAGE ?? "ghcr.io/wopr-network/wopr:stable";
+            try {
+                // 7. Import + start on target
+                logger.info(`[migrate] Importing ${containerName} on ${resolvedTarget}`);
+                await this.commandBus.send(resolvedTarget, {
+                    type: "bot.import",
+                    payload: { name: containerName, image, env: {} },
+                });
+                // 8. Verify running on target
+                logger.info(`[migrate] Verifying ${containerName} on ${resolvedTarget}`);
+                await this.commandBus.send(resolvedTarget, {
+                    type: "bot.inspect",
+                    payload: { name: containerName },
+                });
+                // 9. Update routing — DOWNTIME ENDS
+                await this.botInstanceRepo.reassign(botId, resolvedTarget);
+            }
+            catch (migrationErr) {
+                // Rollback: stop any partially-imported container on target, then restart source
+                logger.error(`[migrate] Migration failed after stop for bot ${botId}, attempting rollback`, {
+                    err: migrationErr instanceof Error ? migrationErr.message : String(migrationErr),
+                });
+                // Best-effort cleanup on target to avoid container running on both nodes
+                try {
+                    await this.commandBus.send(resolvedTarget, {
+                        type: "bot.stop",
+                        payload: { name: containerName },
+                    });
+                    logger.info(`[migrate] Stopped orphaned container ${containerName} on target ${resolvedTarget}`);
+                }
+                catch {
+                    // Target container may not exist yet — that's fine
+                }
+                try {
+                    await this.commandBus.send(sourceNodeId, {
+                        type: "bot.start",
+                        payload: { name: containerName },
+                    });
+                    logger.info(`[migrate] Source container ${containerName} restarted on ${sourceNodeId}`);
+                }
+                catch (restartErr) {
+                    logger.error(`[migrate] Failed to restart source container ${containerName} on ${sourceNodeId}`, {
+                        err: restartErr instanceof Error ? restartErr.message : String(restartErr),
+                    });
+                }
+                throw migrationErr;
+            }
+            const downtimeMs = Date.now() - downtimeStart;
+            logger.info(`[migrate] Migration complete: ${botId} ${sourceNodeId} -> ${resolvedTarget} (downtime: ${downtimeMs}ms)`);
+            return {
+                success: true,
+                botId,
+                sourceNodeId,
+                targetNodeId: resolvedTarget,
+                downtimeMs,
+            };
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            logger.error(`[migrate] Migration failed for bot ${botId}`, { err: message });
+            return {
+                success: false,
+                botId,
+                sourceNodeId,
+                targetNodeId: resolvedTarget,
+                downtimeMs: Date.now() - startTime,
+                error: message,
+            };
+        }
+    }
+}

package/dist/fleet/migration-orchestrator.test.d.ts

@@ -0,0 +1 @@
+export {};