@wopr-network/platform-core 1.37.0 → 1.39.0

package/src/fleet/image-poller.test.ts

@@ -530,38 +530,38 @@ describe("ContainerUpdater", () => {
   });
 
   it("performs update by pulling, stopping, and recreating", async () => {
-    // The fleet.update method will be called to recreate the container
-    // We need to make sure listContainers returns properly
+    // Trace through doUpdateBot("bot-1"):
+    // 1. getContainerDigest: listContainers + getContainer + inspect + getImage
+    // 2. wasRunning check: listContainers + getContainer + inspect
+    // 3. docker.pull + followProgress
+    // 4. fleet.update(botId, { image }):
+    //    a. findContainer: listContainers + getContainer
+    //    b. inspect (wasRunning check)
+    //    c. pullImage (updates.image set): docker.pull + followProgress
+    //    d. stop + remove
+    //    e. createContainer
+    //    f. findContainer after recreate (wasRunning): listContainers + getContainer
+    //    g. start
+    // 5. waitForHealthy: listContainers + getContainer + inspect
+    // 6. getContainerDigest (final): listContainers + getContainer + inspect + getImage
     docker.listContainers
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // getContainerDigest during updateBot
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // step 2: find container to stop
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // fleet.update -> findContainer
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // fleet.update -> findContainer after recreate
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // fleet.start -> findContainer
-      .mockResolvedValueOnce([{ Id: "container-123" }]); // waitForHealthy
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 1: getContainerDigest
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 2: doUpdateBot wasRunning
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 4a: fleet.update -> findContainer
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 4f: fleet.update -> findContainer after recreate
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 5: waitForHealthy
+      .mockResolvedValueOnce([{ Id: "container-123" }]); // 6: final getContainerDigest
 
     docker.getContainer.mockReturnValue(container);
 
-    // Sequence inspect calls: first two return "running" (for wasRunning checks in
-    // doUpdateBot and fleet.update), third returns "stopped" so fleet.start()'s
-    // assertValidState guard allows the start, subsequent calls use the default (running+healthy).
     const runningInspect = {
       Id: "container-123",
       Image: "sha256:abc123",
       Created: "2026-01-01T00:00:00Z",
       State: { Status: "running", Running: true, StartedAt: "2026-01-01T00:00:00Z", Health: { Status: "healthy" } },
     };
-    const stoppedInspect = {
-      Id: "container-123",
-      Image: "sha256:abc123",
-      Created: "2026-01-01T00:00:00Z",
-      State: { Status: "stopped", Running: false, StartedAt: "2026-01-01T00:00:00Z", Health: { Status: "healthy" } },
-    };
-    container.inspect
-      .mockResolvedValueOnce(runningInspect) // getContainerDigest: initial digest check
-      .mockResolvedValueOnce(runningInspect) // doUpdateBot: wasRunning check
-      .mockResolvedValueOnce(runningInspect) // fleet.update: wasRunning check
-      .mockResolvedValueOnce(stoppedInspect); // fleet.start: assertValidState (newly recreated container)
+    // All inspect calls return running+healthy (getContainerDigest, wasRunning checks, waitForHealthy, final digest)
+    container.inspect.mockResolvedValue(runningInspect);
 
     const result = await updater.updateBot("bot-1");
     expect(result.success).toBe(true);
@@ -576,36 +576,27 @@ describe("ContainerUpdater", () => {
       resolvePull = cb;
     });
 
+    // After the first followProgress (which hangs), subsequent calls use default (instant resolve)
+    // Trace: getContainerDigest, wasRunning, fleet.update->findContainer,
+    //   fleet.update->findContainer after recreate, waitForHealthy, final getContainerDigest
     docker.listContainers
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // first update: getContainerDigest
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // first update: step 2
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // fleet.update
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // fleet.start
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // waitForHealthy
-      .mockResolvedValueOnce([{ Id: "container-123" }]); // get new digest
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 1: getContainerDigest
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 2: doUpdateBot wasRunning
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 4a: fleet.update -> findContainer
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 4f: fleet.update -> findContainer after recreate
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 5: waitForHealthy
+      .mockResolvedValueOnce([{ Id: "container-123" }]); // 6: final getContainerDigest
 
     docker.getContainer.mockReturnValue(container);
 
-    // Sequence inspect calls: first two return "running" (for wasRunning checks in
-    // doUpdateBot and fleet.update), third returns "stopped" so fleet.start()'s
-    // assertValidState guard allows the start.
     const runningInspect = {
       Id: "container-123",
       Image: "sha256:abc123",
       Created: "2026-01-01T00:00:00Z",
       State: { Status: "running", Running: true, StartedAt: "2026-01-01T00:00:00Z", Health: { Status: "healthy" } },
     };
-    const stoppedInspect = {
-      Id: "container-123",
-      Image: "sha256:abc123",
-      Created: "2026-01-01T00:00:00Z",
-      State: { Status: "stopped", Running: false, StartedAt: "2026-01-01T00:00:00Z", Health: { Status: "healthy" } },
-    };
-    container.inspect
-      .mockResolvedValueOnce(runningInspect) // getContainerDigest: initial digest check
-      .mockResolvedValueOnce(runningInspect) // doUpdateBot: wasRunning check
-      .mockResolvedValueOnce(runningInspect) // fleet.update: wasRunning check
-      .mockResolvedValueOnce(stoppedInspect); // fleet.start: assertValidState (newly recreated container)
+    // All inspect calls return running+healthy
+    container.inspect.mockResolvedValue(runningInspect);
 
     // Start first update (will block on pull)
     const first = updater.updateBot("bot-1");
@@ -631,6 +622,7 @@ describe("ContainerUpdater", () => {
     const unhealthyContainer = mockContainer({
       inspect: vi.fn().mockResolvedValue({
         Id: "container-123",
+        Name: "/wopr-test-bot",
         Image: "sha256:new999",
         Created: "2026-01-01T00:00:00Z",
         State: {
@@ -639,25 +631,48 @@ describe("ContainerUpdater", () => {
           StartedAt: "2026-01-01T00:00:00Z",
           Health: { Status: "unhealthy" },
         },
+        NetworkSettings: { Ports: {} },
       }),
     });
 
-    // Make the new container unhealthy so rollback happens
+    // Trace through doUpdateBot("bot-1"):
+    // 1. getContainerDigest: listContainers + getContainer + inspect + getImage
+    // 2. wasRunning check: listContainers + getContainer + inspect
+    // 3. docker.pull + followProgress
+    // 4. fleet.update(botId, { image }):
+    //    a. findContainer: listContainers + getContainer
+    //    b. inspect (wasRunning check in update)
+    //    c. pullImage (updates.image is set): docker.pull + followProgress
+    //    d. stop + remove
+    //    e. createContainer
+    //    f. findContainer after recreate (wasRunning): listContainers + getContainer
+    //    g. start
+    // 5. waitForHealthy: listContainers + getContainer + inspect
+    // 6. unhealthy -> rollback via fleet.update:
+    //    a. findContainer: listContainers + getContainer
+    //    b. inspect (wasRunning)
+    //    c. pullImage: docker.pull + followProgress
+    //    d. stop + remove
+    //    e. createContainer
+    //    f. findContainer after recreate (wasRunning): listContainers + getContainer
+    //    g. start
     docker.listContainers
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // getContainerDigest
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // step 2: find container to stop
-      .mockResolvedValueOnce([]) // fleet.update -> findContainer (no existing)
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // fleet.start
-      .mockResolvedValueOnce([{ Id: "container-123" }]) // waitForHealthy
-      .mockResolvedValueOnce([]) // rollback fleet.update -> findContainer
-      .mockResolvedValueOnce([{ Id: "container-123" }]); // rollback fleet.start
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 1: getContainerDigest
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 2: doUpdateBot wasRunning
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 4a: fleet.update -> findContainer
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 4f: fleet.update -> findContainer after recreate
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 5: waitForHealthy
+      .mockResolvedValueOnce([{ Id: "container-123" }]) // 6a: rollback fleet.update -> findContainer
+      .mockResolvedValueOnce([{ Id: "container-123" }]); // 6f: rollback fleet.update -> findContainer after recreate
 
     docker.getContainer
-      .mockReturnValueOnce(container) // getContainerDigest inspect
-      .mockReturnValueOnce(container) // step 2: stop + remove
-      .mockReturnValueOnce(container) // fleet.start
-      .mockReturnValueOnce(unhealthyContainer) // waitForHealthy inspect
-      .mockReturnValueOnce(container); // rollback fleet.start
+      .mockReturnValueOnce(container) // 1: getContainerDigest
+      .mockReturnValueOnce(container) // 2: doUpdateBot wasRunning
+      .mockReturnValueOnce(container) // 4a: fleet.update -> findContainer (stop+remove this)
+      .mockReturnValueOnce(container) // 4f: fleet.update -> findContainer after recreate (start)
+      .mockReturnValueOnce(unhealthyContainer) // 5: waitForHealthy -> unhealthy
+      .mockReturnValueOnce(container) // 6a: rollback fleet.update -> findContainer (stop+remove)
+      .mockReturnValueOnce(container); // 6f: rollback fleet.update -> findContainer after recreate (start)
 
     const result = await updater.updateBot("bot-1");
     expect(result.success).toBe(false);

package/src/fleet/index.ts

@@ -3,6 +3,7 @@ export * from "./fleet-manager.js";
 export type { FleetNotificationListenerDeps } from "./fleet-notification-listener.js";
 export { initFleetNotificationListener } from "./fleet-notification-listener.js";
 export * from "./init-fleet-updater.js";
+export * from "./instance.js";
 export * from "./repository-types.js";
 export * from "./rollout-orchestrator.js";
 export * from "./rollout-strategy.js";

package/src/fleet/instance.ts

@@ -0,0 +1,171 @@
+import type Docker from "dockerode";
+import { logger } from "../config/logger.js";
+import type { ProxyManagerInterface } from "../proxy/types.js";
+import type { IBotInstanceRepository } from "./bot-instance-repository.js";
+import type { BotEventType, FleetEventEmitter } from "./fleet-event-emitter.js";
+import type { BotProfile } from "./types.js";
+
+/**
+ * Instance — a runtime handle to a container.
+ *
+ * FleetManager is the factory: pull image, create container, return Instance.
+ * Instance owns its lifecycle: start, stop, remove, setupBilling, setupProxy.
+ *
+ * Ephemeral instances (e.g., holyshippers) skip billing and proxy setup.
+ * They bill per-token at the gateway layer, not per-instance.
+ */
+
+export interface InstanceDeps {
+  docker: Docker;
+  profile: BotProfile;
+  containerId: string;
+  containerName: string;
+  url: string;
+  /** Optional — non-ephemeral instances use these for billing/proxy/events */
+  instanceRepo?: IBotInstanceRepository;
+  proxyManager?: ProxyManagerInterface;
+  eventEmitter?: FleetEventEmitter;
+}
+
+export class Instance {
+  readonly id: string;
+  readonly containerId: string;
+  readonly containerName: string;
+  readonly url: string;
+  readonly profile: BotProfile;
+
+  private readonly docker: Docker;
+  private readonly instanceRepo: IBotInstanceRepository | undefined;
+  private readonly proxyManager: ProxyManagerInterface | undefined;
+  private readonly eventEmitter: FleetEventEmitter | undefined;
+
+  constructor(deps: InstanceDeps) {
+    this.id = deps.profile.id;
+    this.containerId = deps.containerId;
+    this.containerName = deps.containerName;
+    this.url = deps.url;
+    this.profile = deps.profile;
+    this.docker = deps.docker;
+    this.instanceRepo = deps.instanceRepo;
+    this.proxyManager = deps.proxyManager;
+    this.eventEmitter = deps.eventEmitter;
+  }
+
+  /** Emit bot.created — call only from FleetManager.create(), not getInstance() */
+  emitCreated(): void {
+    this.emit("bot.created");
+  }
+
+  async start(): Promise<void> {
+    const container = this.docker.getContainer(this.containerId);
+    await container.start();
+    logger.info(`Instance started`, { id: this.id, containerName: this.containerName, url: this.url });
+    this.emit("bot.started");
+  }
+
+  async stop(): Promise<void> {
+    const container = this.docker.getContainer(this.containerId);
+    try {
+      await container.stop({ t: 10 });
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (!msg.includes("not running") && !msg.includes("already stopped")) {
+        throw err;
+      }
+    }
+    logger.info(`Instance stopped`, { id: this.id, containerName: this.containerName });
+    this.emit("bot.stopped");
+  }
+
+  async remove(): Promise<void> {
+    const container = this.docker.getContainer(this.containerId);
+    try {
+      await container.stop({ t: 5 }).catch(() => {});
+      await container.remove({ force: true });
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (!msg.includes("No such container")) {
+        throw err;
+      }
+    }
+
+    if (this.proxyManager) {
+      try {
+        await this.proxyManager.removeRoute(this.id);
+      } catch (err) {
+        logger.warn("Proxy route cleanup failed (non-fatal)", { id: this.id, err });
+      }
+    }
+
+    logger.info(`Instance removed`, { id: this.id, containerName: this.containerName });
+    this.emit("bot.removed");
+  }
+
+  async status(): Promise<"running" | "stopped" | "gone"> {
+    try {
+      const container = this.docker.getContainer(this.containerId);
+      const info = await container.inspect();
+      return info.State.Running ? "running" : "stopped";
+    } catch {
+      return "gone";
+    }
+  }
+
+  /**
+   * Register this instance in the billing system.
+   * Skip for ephemeral instances — they bill per-token, not per-instance.
+   */
+  async setupBilling(): Promise<void> {
+    if (this.profile.ephemeral) {
+      logger.info("Skipping billing setup (ephemeral)", { id: this.id });
+      return;
+    }
+    if (!this.instanceRepo) {
+      logger.warn("No instance repo — billing setup skipped", { id: this.id });
+      return;
+    }
+    await this.instanceRepo.register(this.id, this.profile.tenantId, this.profile.name);
+    logger.info("Billing registered", { id: this.id, tenantId: this.profile.tenantId });
+  }
+
+  /**
+   * Register a proxy route for tenant subdomain routing.
+   * Skip for ephemeral instances — they're accessed directly via Docker DNS.
+   */
+  async setupProxy(): Promise<void> {
+    if (this.profile.ephemeral) {
+      logger.info("Skipping proxy setup (ephemeral)", { id: this.id });
+      return;
+    }
+    if (!this.proxyManager) {
+      logger.warn("No proxy manager — proxy setup skipped", { id: this.id });
+      return;
+    }
+    try {
+      const subdomain = this.profile.name.toLowerCase().replace(/_/g, "-");
+      const envPort = this.profile.env?.PORT;
+      const upstreamPort = envPort ? Number.parseInt(envPort, 10) || 7437 : 7437;
+      await this.proxyManager.addRoute({
+        instanceId: this.id,
+        subdomain,
+        upstreamHost: this.containerName,
+        upstreamPort,
+        healthy: true,
+      });
+      logger.info("Proxy route registered", { id: this.id, subdomain });
+    } catch (err) {
+      logger.warn("Proxy route registration failed (non-fatal)", { id: this.id, err });
+    }
+  }
+
+  private emit(type: BotEventType): void {
+    if (this.eventEmitter) {
+      this.eventEmitter.emit({
+        type,
+        botId: this.id,
+        tenantId: this.profile.tenantId,
+        timestamp: new Date().toISOString(),
+      });
+    }
+  }
+}

package/src/fleet/types.ts

@@ -74,6 +74,10 @@ export const botProfileSchema = z.object({
   discovery: discoveryConfigSchema.optional(),
   /** Node this bot was placed on at creation time (optional, for future multi-node routing). */
   nodeId: z.string().uuid().optional(),
+  /** Docker network to attach the container to (bypasses NetworkPolicy). */
+  network: z.string().min(1).optional(),
+  /** When true, disables ReadonlyRootfs and CapDrop for containers that need write access (e.g., ephemeral workers). */
+  ephemeral: z.boolean().optional(),
 });
 
 export type BotProfile = z.infer<typeof botProfileSchema>;
@@ -96,6 +100,10 @@ export const createBotSchema = z.object({
   discovery: discoveryConfigSchema.optional(),
   /** Node this bot was placed on at creation time (optional, for future multi-node routing). */
   nodeId: z.string().uuid().optional(),
+  /** Docker network to attach the container to. */
+  network: z.string().min(1).optional(),
+  /** When true, disables ReadonlyRootfs and CapDrop for ephemeral workers. */
+  ephemeral: z.boolean().optional(),
 });
 
 /** Schema for updating a bot via the API */

package/src/fleet/updater.test.ts

@@ -77,7 +77,6 @@ function mockDocker(container: ReturnType<typeof mockContainer> | null = null) {
 function mockFleet() {
   return {
     update: vi.fn().mockResolvedValue(undefined),
-    start: vi.fn().mockResolvedValue(undefined),
   } as unknown as FleetManager;
 }
 
@@ -154,7 +153,6 @@ describe("ContainerUpdater", () => {
       // Verify the update pipeline was called
       expect(docker.pull).toHaveBeenCalledWith("ghcr.io/wopr-network/wopr:stable");
       expect(fleet.update).toHaveBeenCalledWith("bot-1", { image: "ghcr.io/wopr-network/wopr:stable" });
-      expect(fleet.start).toHaveBeenCalledWith("bot-1");
     });
 
     it("skips start and health check when container was not running", async () => {
@@ -182,7 +180,6 @@ describe("ContainerUpdater", () => {
 
       expect(result.success).toBe(true);
       expect(fleet.update).toHaveBeenCalled();
-      expect(fleet.start).not.toHaveBeenCalled();
     });
 
     it("considers container healthy when no HEALTHCHECK is configured", async () => {
@@ -259,8 +256,6 @@ describe("ContainerUpdater", () => {
       expect(fleet.update).toHaveBeenLastCalledWith("bot-1", {
         image: "ghcr.io/wopr-network/wopr@sha256:abc123",
       });
-      // Rollback starts the container since it was running
-      expect(fleet.start).toHaveBeenCalledTimes(2);
     });
 
     it("rolls back when health check times out (stays in starting)", async () => {
@@ -301,8 +296,8 @@ describe("ContainerUpdater", () => {
 
   // --- Rollback on startup failure ---
 
-  describe("rollback on startup failure", () => {
-    it("rolls back when fleet.start throws", async () => {
+  describe("rollback on update failure", () => {
+    it("rolls back when fleet.update throws (start embedded in update)", async () => {
       // getContainerDigest
       docker.listContainers.mockResolvedValueOnce([{ Id: "container-123" }]);
       docker.getContainer.mockReturnValueOnce(container);
@@ -310,8 +305,10 @@ describe("ContainerUpdater", () => {
       docker.listContainers.mockResolvedValueOnce([{ Id: "container-123" }]);
       docker.getContainer.mockReturnValueOnce(container);
 
-      // fleet.start fails after fleet.update succeeds
-      (fleet.start as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("Container start failed"));
+      // fleet.update fails (which now includes starting the container)
+      (fleet.update as ReturnType<typeof vi.fn>)
+        .mockRejectedValueOnce(new Error("Container start failed"))
+        .mockResolvedValueOnce(undefined); // rollback update succeeds
 
       const result = await updater.updateBot("bot-1");
 
@@ -331,12 +328,10 @@ describe("ContainerUpdater", () => {
       docker.listContainers.mockResolvedValueOnce([{ Id: "container-123" }]);
       docker.getContainer.mockReturnValueOnce(container);
 
-      // fleet.start fails
-      (fleet.start as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("Container start failed"));
-      // fleet.update (rollback) also fails
+      // fleet.update fails, then rollback fleet.update also fails
       (fleet.update as ReturnType<typeof vi.fn>)
-        .mockResolvedValueOnce(undefined) // initial update succeeds
-        .mockRejectedValueOnce(new Error("Rollback recreate failed")); // rollback fails
+        .mockRejectedValueOnce(new Error("Container start failed"))
+        .mockRejectedValueOnce(new Error("Rollback recreate failed"));
 
       const result = await updater.updateBot("bot-1");
 
@@ -666,8 +661,6 @@ describe("ContainerUpdater", () => {
 
       expect(result.success).toBe(false);
       expect(result.rolledBack).toBe(true);
-      // fleet.start should NOT be called at all — container was stopped
-      expect(fleet.start).not.toHaveBeenCalled();
     });
   });
 
@@ -687,8 +680,6 @@ describe("ContainerUpdater", () => {
       const result = await updater.updateBot("bot-1");
 
       expect(result.success).toBe(true);
-      // Since wasRunning defaults to false on error, start and health check are skipped
-      expect(fleet.start).not.toHaveBeenCalled();
     });
   });
 });

package/src/fleet/updater.ts

@@ -122,17 +122,12 @@ export class ContainerUpdater {
         });
       });
 
-      // Step 2: Delegate stop -> remove -> recreate to FleetManager.update().
-      // fleet.update() already implements: stop old -> remove old -> create new,
+      // Step 2: Delegate stop -> remove -> recreate -> start to FleetManager.update().
+      // fleet.update() already implements: stop old -> remove old -> create new -> start if was running,
       // with profile rollback if container creation fails.
       await this.fleet.update(botId, { image: profile.image });
 
-      // Step 3: Start the new container only if the old one was running
-      if (wasRunning) {
-        await this.fleet.start(botId);
-      }
-
-      // Step 4: Verify health (only meaningful if container is running)
+      // Step 3: Verify health (only meaningful if container is running)
       if (wasRunning) {
         const healthy = await this.waitForHealthy(botId);
 
@@ -229,7 +224,7 @@ export class ContainerUpdater {
     botId: string,
     previousImage: string,
     previousDigest: string | null,
-    wasRunning: boolean,
+    _wasRunning: boolean,
   ): Promise<UpdateResult> {
     // Use digest-pinned image reference when available to prevent rollback
     // from pulling a newer image that was pushed between the update and rollback.
@@ -237,15 +232,9 @@ export class ContainerUpdater {
     logger.info(`Rolling back bot ${botId} to ${rollbackImage}`);
 
     try {
+      // fleet.update() handles stop -> remove -> recreate -> start-if-was-running internally
       await this.fleet.update(botId, { image: rollbackImage });
 
-      // Only start on rollback if the container was running before the update
-      if (wasRunning) {
-        await this.fleet.start(botId).catch((err) => {
-          logger.warn(`Failed to start bot ${botId} during rollback (may already be running)`, { err });
-        });
-      }
-
       return {
         botId,
         success: false,