@wopr-network/platform-core 1.0.10 → 1.1.0

package/dist/fleet/migration-orchestrator.test.js

@@ -0,0 +1,261 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { MigrationOrchestrator } from "./migration-orchestrator.js";
+const BOT_ID = "bot-123";
+const SOURCE_NODE = "node-1";
+const TARGET_NODE = "node-2";
+const TENANT_ID = "tenant-abc";
+const defaultInstance = {
+    id: BOT_ID,
+    tenantId: TENANT_ID,
+    name: "My Bot",
+    nodeId: SOURCE_NODE,
+    billingState: "active",
+    suspendedAt: null,
+    destroyAfter: null,
+    createdAt: "2026-01-01T00:00:00Z",
+    updatedAt: "2026-01-01T00:00:00Z",
+};
+function makeCommandBus(overrides = {}) {
+    return {
+        send: vi.fn().mockResolvedValue({ id: "cmd-1", type: "command_result", command: "test", success: true }),
+        ...overrides,
+    };
+}
+function makeBotInstanceRepo(instance = defaultInstance) {
+    return {
+        getById: vi.fn().mockReturnValue(instance),
+        listByNode: vi.fn().mockReturnValue([]),
+        listByTenant: vi.fn().mockReturnValue([]),
+        create: vi.fn(),
+        reassign: vi.fn().mockReturnValue({ ...instance, nodeId: TARGET_NODE }),
+        setBillingState: vi.fn(),
+    };
+}
+function makeNodeRepo() {
+    return {
+        getById: vi.fn().mockReturnValue({ id: SOURCE_NODE, status: "active", capacityMb: 2048, usedMb: 100 }),
+        list: vi.fn().mockReturnValue([]),
+        register: vi.fn(),
+        transition: vi.fn().mockReturnValue({ id: SOURCE_NODE, status: "active" }),
+        updateHeartbeat: vi.fn(),
+        addCapacity: vi.fn(),
+        findBestTarget: vi.fn().mockReturnValue({ id: TARGET_NODE, host: "10.0.0.2", capacityMb: 1000, usedMb: 0 }),
+        listTransitions: vi.fn().mockReturnValue([]),
+    };
+}
+describe("MigrationOrchestrator.migrate", () => {
+    let commandBus;
+    let botInstanceRepo;
+    let nodeRepo;
+    beforeEach(() => {
+        commandBus = makeCommandBus();
+        botInstanceRepo = makeBotInstanceRepo();
+        nodeRepo = makeNodeRepo();
+        vi.clearAllMocks();
+    });
+    it("returns error when bot instance not found", async () => {
+        botInstanceRepo.getById.mockReturnValue(null);
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID, TARGET_NODE);
+        expect(result.success).toBe(false);
+        expect(result.error).toMatch(/not found/i);
+    });
+    it("returns error when bot has no assigned node", async () => {
+        botInstanceRepo.getById.mockReturnValue({ ...defaultInstance, nodeId: null });
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID, TARGET_NODE);
+        expect(result.success).toBe(false);
+        expect(result.error).toMatch(/no assigned node/i);
+    });
+    it("returns error when source and target are the same node", async () => {
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID, SOURCE_NODE);
+        expect(result.success).toBe(false);
+        expect(result.error).toMatch(/same node/i);
+    });
+    it("succeeds on happy path with explicit target", async () => {
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID, TARGET_NODE);
+        expect(result.success).toBe(true);
+        expect(result.botId).toBe(BOT_ID);
+        expect(result.sourceNodeId).toBe(SOURCE_NODE);
+        expect(result.targetNodeId).toBe(TARGET_NODE);
+        expect(result.downtimeMs).toBeGreaterThanOrEqual(0);
+    });
+    it("sends commands in correct order: export, upload, download, stop, import, inspect", async () => {
+        const send = vi.fn().mockResolvedValue({ id: "cmd-1", type: "command_result", command: "test", success: true });
+        commandBus = makeCommandBus({ send });
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        await orch.migrate(BOT_ID, TARGET_NODE);
+        const calls = send.mock.calls;
+        expect(calls[0]).toEqual([SOURCE_NODE, { type: "bot.export", payload: { name: `tenant_${TENANT_ID}` } }]);
+        expect(calls[1]).toEqual([
+            SOURCE_NODE,
+            { type: "backup.upload", payload: { filename: `tenant_${TENANT_ID}.tar.gz` } },
+        ]);
+        expect(calls[2]).toEqual([
+            TARGET_NODE,
+            { type: "backup.download", payload: { filename: `tenant_${TENANT_ID}.tar.gz` } },
+        ]);
+        expect(calls[3]).toEqual([SOURCE_NODE, { type: "bot.stop", payload: { name: `tenant_${TENANT_ID}` } }]);
+        expect(calls[4][0]).toBe(TARGET_NODE);
+        expect(calls[4][1].type).toBe("bot.import");
+        expect(calls[5]).toEqual([TARGET_NODE, { type: "bot.inspect", payload: { name: `tenant_${TENANT_ID}` } }]);
+    });
+    it("calls botInstanceRepo.reassign after successful migration", async () => {
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        await orch.migrate(BOT_ID, TARGET_NODE);
+        expect(botInstanceRepo.reassign).toHaveBeenCalledWith(BOT_ID, TARGET_NODE);
+    });
+    it("does not call reassign when migration fails early", async () => {
+        const send = vi.fn().mockRejectedValue(new Error("Export failed"));
+        commandBus = makeCommandBus({ send });
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID, TARGET_NODE);
+        expect(result.success).toBe(false);
+        expect(botInstanceRepo.reassign).not.toHaveBeenCalled();
+    });
+    it("restarts source container when import fails after stop (rollback)", async () => {
+        let callCount = 0;
+        const send = vi.fn().mockImplementation((_nodeId, cmd) => {
+            callCount++;
+            // Call 5 = bot.import on target → fail
+            if (callCount === 5)
+                return Promise.reject(new Error("Import failed"));
+            return Promise.resolve({ id: "cmd-1", type: "command_result", command: cmd.type, success: true });
+        });
+        commandBus = makeCommandBus({ send });
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID, TARGET_NODE);
+        expect(result.success).toBe(false);
+        expect(result.error).toMatch(/import failed/i);
+        // Should have attempted rollback: bot.start on source
+        const startCall = send.mock.calls.find((c) => c[1].type === "bot.start" && c[0] === SOURCE_NODE);
+        expect(startCall).not.toBeUndefined();
+    });
+    it("auto-selects target via nodeRepo.findBestTarget when no target specified", async () => {
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID);
+        expect(result.success).toBe(true);
+        expect(result.targetNodeId).toBe(TARGET_NODE);
+        expect(nodeRepo.findBestTarget).toHaveBeenCalledWith(SOURCE_NODE, 100);
+    });
+    it("returns error when no placement found and no explicit target", async () => {
+        nodeRepo.findBestTarget.mockReturnValue(null);
+        const orch = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        const result = await orch.migrate(BOT_ID);
+        expect(result.success).toBe(false);
+        expect(result.error).toMatch(/no node with sufficient capacity/i);
+    });
+});
+// ---------------------------------------------------------------------------
+// NodeDrainer tests
+// ---------------------------------------------------------------------------
+import { NodeDrainer } from "./node-drainer.js";
+function makeNotifier() {
+    return {
+        nodeRecoveryComplete: vi.fn().mockResolvedValue(undefined),
+        nodeStatusChange: vi.fn().mockResolvedValue(undefined),
+        capacityOverflow: vi.fn().mockResolvedValue(undefined),
+    };
+}
+describe("NodeDrainer.drain", () => {
+    let commandBus;
+    let botInstanceRepo;
+    let nodeRepo;
+    let notifier;
+    let orchestrator;
+    beforeEach(() => {
+        commandBus = makeCommandBus();
+        botInstanceRepo = makeBotInstanceRepo();
+        nodeRepo = makeNodeRepo();
+        notifier = makeNotifier();
+        orchestrator = new MigrationOrchestrator(commandBus, botInstanceRepo, nodeRepo);
+        vi.clearAllMocks();
+    });
+    it("transitions node to draining, then offline on full success", async () => {
+        botInstanceRepo.listByNode.mockReturnValue([
+            { ...defaultInstance, id: "bot-1", tenantId: "t1" },
+        ]);
+        vi.spyOn(orchestrator, "migrate").mockResolvedValue({
+            success: true,
+            botId: "bot-1",
+            sourceNodeId: SOURCE_NODE,
+            targetNodeId: TARGET_NODE,
+            downtimeMs: 10,
+        });
+        const drainer = new NodeDrainer(orchestrator, nodeRepo, botInstanceRepo, notifier);
+        const result = await drainer.drain(SOURCE_NODE);
+        expect(nodeRepo.transition).toHaveBeenCalledWith(SOURCE_NODE, "draining", "node_drain", "migration_orchestrator");
+        expect(result.migrated).toHaveLength(1);
+        expect(result.failed).toHaveLength(0);
+        expect(nodeRepo.transition).toHaveBeenCalledWith(SOURCE_NODE, "offline", "drain_complete", "migration_orchestrator");
+    });
+    it("stays draining when some tenants fail", async () => {
+        botInstanceRepo.listByNode.mockReturnValue([
+            { ...defaultInstance, id: "bot-1", tenantId: "t1" },
+            { ...defaultInstance, id: "bot-2", tenantId: "t2" },
+        ]);
+        let callIdx = 0;
+        vi.spyOn(orchestrator, "migrate").mockImplementation(async () => {
+            callIdx++;
+            if (callIdx === 1) {
+                return { success: true, botId: "bot-1", sourceNodeId: SOURCE_NODE, targetNodeId: TARGET_NODE, downtimeMs: 10 };
+            }
+            return {
+                success: false,
+                botId: "bot-2",
+                sourceNodeId: SOURCE_NODE,
+                targetNodeId: "none",
+                downtimeMs: 0,
+                error: "No capacity",
+            };
+        });
+        const drainer = new NodeDrainer(orchestrator, nodeRepo, botInstanceRepo, notifier);
+        const result = await drainer.drain(SOURCE_NODE);
+        expect(result.migrated).toHaveLength(1);
+        expect(result.failed).toHaveLength(1);
+        // Should NOT transition to offline -- stays draining
+        const offlineCalls = nodeRepo.transition.mock.calls.filter((c) => c[1] === "offline");
+        expect(offlineCalls).toHaveLength(0);
+    });
+    it("notifies admin when there are failures", async () => {
+        botInstanceRepo.listByNode.mockReturnValue([
+            { ...defaultInstance, id: "bot-1", tenantId: "t1" },
+        ]);
+        vi.spyOn(orchestrator, "migrate").mockResolvedValue({
+            success: false,
+            botId: "bot-1",
+            sourceNodeId: SOURCE_NODE,
+            targetNodeId: "none",
+            downtimeMs: 0,
+            error: "No capacity",
+        });
+        const drainer = new NodeDrainer(orchestrator, nodeRepo, botInstanceRepo, notifier);
+        await drainer.drain(SOURCE_NODE);
+        expect(notifier.capacityOverflow).toHaveBeenCalledWith(SOURCE_NODE, 1, 1);
+    });
+    it("does not notify admin when all tenants migrate", async () => {
+        botInstanceRepo.listByNode.mockReturnValue([
+            { ...defaultInstance, id: "bot-1", tenantId: "t1" },
+        ]);
+        vi.spyOn(orchestrator, "migrate").mockResolvedValue({
+            success: true,
+            botId: "bot-1",
+            sourceNodeId: SOURCE_NODE,
+            targetNodeId: TARGET_NODE,
+            downtimeMs: 10,
+        });
+        const drainer = new NodeDrainer(orchestrator, nodeRepo, botInstanceRepo, notifier);
+        await drainer.drain(SOURCE_NODE);
+        expect(notifier.capacityOverflow).not.toHaveBeenCalled();
+    });
+    it("handles empty node (no tenants)", async () => {
+        botInstanceRepo.listByNode.mockReturnValue([]);
+        const drainer = new NodeDrainer(orchestrator, nodeRepo, botInstanceRepo, notifier);
+        const result = await drainer.drain(SOURCE_NODE);
+        expect(result.migrated).toHaveLength(0);
+        expect(result.failed).toHaveLength(0);
+        expect(nodeRepo.transition).toHaveBeenCalledWith(SOURCE_NODE, "offline", "drain_complete", "migration_orchestrator");
+    });
+});

package/dist/fleet/node-command-bus.d.ts

@@ -0,0 +1,55 @@
+/** Command to send to a node agent */
+export interface Command {
+    id: string;
+    type: string;
+    payload: Record<string, unknown>;
+}
+/** Result from a command execution */
+export interface CommandResult {
+    id: string;
+    type: "command_result";
+    command: string;
+    success: boolean;
+    data?: unknown;
+    error?: string;
+}
+/** Minimal interface for socket lookup — satisfied by NodeConnectionRegistry (WOP-868) */
+export interface NodeConnectionRegistry {
+    getSocket(nodeId: string): {
+        send(data: string): void;
+        readyState: number;
+    } | null;
+}
+/** Interface for dependency injection — satisfied by NodeCommandBus */
+export interface INodeCommandBus {
+    send(nodeId: string, command: {
+        type: string;
+        payload: Record<string, unknown>;
+    }): Promise<CommandResult>;
+}
+/**
+ * Async command/response bus over WebSocket.
+ *
+ * Sends commands to node agents via NodeConnectionRegistry and matches
+ * responses via a pending Map with configurable timeout.
+ * No knowledge of what commands mean — pure transport.
+ */
+export declare class NodeCommandBus {
+    private readonly registry;
+    private readonly pending;
+    private readonly timeoutMs;
+    constructor(registry: NodeConnectionRegistry, options?: {
+        timeoutMs?: number;
+    });
+    /**
+     * Send a command to a node agent and return the result.
+     * Rejects if node is not connected or if response times out.
+     */
+    send(nodeId: string, command: Omit<Command, "id">): Promise<CommandResult>;
+    /**
+     * Handle an inbound command result from the WebSocket message handler.
+     * Resolves the matching pending promise on success, rejects on failure.
+     * Silently ignores unknown IDs.
+     */
+    handleResult(result: CommandResult): void;
+}

package/dist/fleet/node-command-bus.js

@@ -0,0 +1,55 @@
+import { randomUUID } from "node:crypto";
+/**
+ * Async command/response bus over WebSocket.
+ *
+ * Sends commands to node agents via NodeConnectionRegistry and matches
+ * responses via a pending Map with configurable timeout.
+ * No knowledge of what commands mean — pure transport.
+ */
+export class NodeCommandBus {
+    registry;
+    pending = new Map();
+    timeoutMs;
+    constructor(registry, options) {
+        this.registry = registry;
+        this.timeoutMs = options?.timeoutMs ?? 60_000;
+    }
+    /**
+     * Send a command to a node agent and return the result.
+     * Rejects if node is not connected or if response times out.
+     */
+    async send(nodeId, command) {
+        const ws = this.registry.getSocket(nodeId);
+        if (!ws || ws.readyState !== 1) {
+            throw new Error(`Node ${nodeId} is not connected`);
+        }
+        const id = randomUUID();
+        const fullCommand = { id, ...command };
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                this.pending.delete(id);
+                reject(new Error(`Command ${command.type} timed out on node ${nodeId}`));
+            }, this.timeoutMs);
+            this.pending.set(id, { resolve, reject, timeout });
+            ws.send(JSON.stringify(fullCommand));
+        });
+    }
+    /**
+     * Handle an inbound command result from the WebSocket message handler.
+     * Resolves the matching pending promise on success, rejects on failure.
+     * Silently ignores unknown IDs.
+     */
+    handleResult(result) {
+        const pending = this.pending.get(result.id);
+        if (!pending)
+            return;
+        clearTimeout(pending.timeout);
+        this.pending.delete(result.id);
+        if (!result.success) {
+            pending.reject(new Error(result.error ?? "command failed"));
+        }
+        else {
+            pending.resolve(result);
+        }
+    }
+}

package/dist/fleet/node-command-bus.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/fleet/node-command-bus.test.js

@@ -0,0 +1,85 @@
+import { describe, expect, it, vi } from "vitest";
+import { NodeCommandBus } from "./node-command-bus.js";
+function mockSocket(readyState = 1) {
+    const spy = vi.fn();
+    return { send: spy, readyState, _spy: spy };
+}
+function mockRegistry(sockets = {}) {
+    return {
+        getSocket(nodeId) {
+            const s = sockets[nodeId];
+            return s ?? null;
+        },
+    };
+}
+describe("NodeCommandBus", () => {
+    it("throws when node is not connected", async () => {
+        const bus = new NodeCommandBus(mockRegistry());
+        await expect(bus.send("node-unknown", { type: "bot.start", payload: {} })).rejects.toThrow("not connected");
+    });
+    it("resolves with command result on success", async () => {
+        const ws = mockSocket();
+        const bus = new NodeCommandBus(mockRegistry({ "node-1": ws }));
+        const promise = bus.send("node-1", { type: "bot.start", payload: { tenantId: "t1" } });
+        // Extract the command ID from what was sent over the socket
+        expect(ws._spy).toHaveBeenCalledOnce();
+        const sent = JSON.parse(ws._spy.mock.calls[0][0]);
+        expect(sent.id).toEqual(expect.any(String));
+        expect(sent.type).toBe("bot.start");
+        expect(sent.payload).toEqual({ tenantId: "t1" });
+        // Simulate the node agent responding
+        const result = {
+            id: sent.id,
+            type: "command_result",
+            command: "bot.start",
+            success: true,
+            data: { containerId: "abc123" },
+        };
+        bus.handleResult(result);
+        const resolved = await promise;
+        expect(resolved).toEqual(result);
+    });
+    it("rejects with error when handleResult is called with success: false", async () => {
+        const ws = mockSocket();
+        const bus = new NodeCommandBus(mockRegistry({ "node-1": ws }));
+        const promise = bus.send("node-1", { type: "bot.start", payload: { tenantId: "t1" } });
+        const sent = JSON.parse(ws._spy.mock.calls[0][0]);
+        const result = {
+            id: sent.id,
+            type: "command_result",
+            command: "bot.start",
+            success: false,
+            error: "container failed to start",
+        };
+        bus.handleResult(result);
+        await expect(promise).rejects.toThrow("container failed to start");
+    });
+    it("rejects with generic message when handleResult has success: false and no error", async () => {
+        const ws = mockSocket();
+        const bus = new NodeCommandBus(mockRegistry({ "node-1": ws }));
+        const promise = bus.send("node-1", { type: "bot.start", payload: {} });
+        const sent = JSON.parse(ws._spy.mock.calls[0][0]);
+        const result = {
+            id: sent.id,
+            type: "command_result",
+            command: "bot.start",
+            success: false,
+        };
+        bus.handleResult(result);
+        await expect(promise).rejects.toThrow("command failed");
+    });
+    it("rejects on timeout", async () => {
+        vi.useFakeTimers();
+        try {
+            const ws = mockSocket();
+            const bus = new NodeCommandBus(mockRegistry({ "node-1": ws }), { timeoutMs: 10 });
+            const promise = bus.send("node-1", { type: "bot.stop", payload: {} });
+            // Advance time past the timeout
+            vi.advanceTimersByTime(11);
+            await expect(promise).rejects.toThrow("timed out");
+        }
+        finally {
+            vi.useRealTimers();
+        }
+    });
+});

package/dist/fleet/node-connection-manager.d.ts

@@ -0,0 +1,118 @@
+import type { WebSocket } from "ws";
+import type { IBotInstanceRepository } from "./bot-instance-repository.js";
+import type { INodeRepository } from "./node-repository.js";
+import type { OrphanCleaner } from "./orphan-cleaner.js";
+import type { IRecoveryRepository } from "./recovery-repository.js";
+import type { Node } from "./repository-types.js";
+export type { Node as NodeInfo };
+/** Node registration request body */
+export interface NodeRegistration {
+    node_id: string;
+    host: string;
+    capacity_mb: number;
+    agent_version: string;
+}
+/** Tenant assignment to a specific node */
+export interface TenantAssignment {
+    id: string;
+    tenantId: string;
+    name: string;
+    containerName: string;
+    estimatedMb: number;
+}
+/** Command to send to a node agent */
+export interface Command {
+    id: string;
+    type: string;
+    payload: Record<string, unknown>;
+}
+/** Result from a command execution */
+export interface CommandResult {
+    id: string;
+    type: "command_result";
+    command: string;
+    success: boolean;
+    data?: unknown;
+    error?: string;
+}
+export declare class NodeConnectionManager {
+    private readonly nodeRepo;
+    private readonly botInstanceRepo;
+    private readonly recoveryRepo;
+    private readonly connections;
+    private readonly pending;
+    private orphanCleaner?;
+    private readonly cleanupInFlight;
+    private readonly onNodeRegistered?;
+    constructor(nodeRepo: INodeRepository, botInstanceRepo: IBotInstanceRepository, recoveryRepo: IRecoveryRepository, options?: {
+        onNodeRegistered?: () => void;
+    });
+    /** Inject OrphanCleaner after construction to break the circular dependency. */
+    setOrphanCleaner(cleaner: OrphanCleaner): void;
+    /**
+     * Register a new node (called from HTTP POST /internal/nodes/register)
+     */
+    registerNode(registration: NodeRegistration): Promise<void>;
+    /**
+     * Accept an incoming WebSocket for a node
+     */
+    handleWebSocket(nodeId: string, ws: WebSocket): void;
+    /**
+     * Handle an inbound WebSocket message from a node
+     */
+    private handleMessage;
+    /**
+     * Process a heartbeat message and update nodes table.
+     * For "returning" nodes, triggers OrphanCleaner (fire-and-forget) instead of
+     * immediately setting active — cleanup handles the transition.
+     */
+    private processHeartbeat;
+    /**
+     * Send a command to a node agent and return the result
+     */
+    sendCommand(nodeId: string, command: Omit<Command, "id">, timeoutMs?: number): Promise<CommandResult>;
+    /**
+     * Get current status of all nodes
+     */
+    listNodes(): Promise<Node[]>;
+    /**
+     * Get tenants assigned to a specific node
+     */
+    getNodeTenants(nodeId: string): Promise<TenantAssignment[]>;
+    /**
+     * Find the best target node for recovery (most free capacity)
+     */
+    findBestTarget(excludeNodeId: string, requiredMb: number): Promise<Node | null>;
+    /**
+     * Reassign a tenant to a new node (update bot_instances)
+     */
+    reassignTenant(botId: string, targetNodeId: string): Promise<void>;
+    /**
+     * Update a node's used capacity
+     */
+    addNodeCapacity(nodeId: string, deltaMb: number): Promise<void>;
+    /**
+     * Get a single node by ID
+     */
+    getNode(nodeId: string): Promise<Node | undefined>;
+    /**
+     * Check if a node is connected (has active WebSocket)
+     */
+    isConnected(nodeId: string): boolean;
+    /**
+     * Register a self-hosted node with owner and per-node secret hash.
+     */
+    registerSelfHostedNode(registration: NodeRegistration & {
+        ownerUserId: string;
+        label: string | null;
+        nodeSecretHash: string;
+    }): Promise<void>;
+    /**
+     * Look up a node by its persistent secret (hashed). Returns node or undefined.
+     */
+    getNodeBySecret(secret: string): Promise<Node | undefined>;
+    /**
+     * Remove a self-hosted node (deregister), closing any active WebSocket.
+     */
+    removeNode(nodeId: string): Promise<void>;
+}