@shogo-ai/worker 2.5.3 → 2.5.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shogo-ai/worker",
-  "version": "2.5.3",
+  "version": "2.5.5",
   "description": "Shogo Cloud Agent Worker — run Shogo agents on your own machine (laptop, devbox, CI).",
   "license": "MIT",
   "author": "Shogo Technologies, Inc.",
@@ -20,6 +20,7 @@
     "./tunnel": "./src/lib/tunnel.ts",
     "./tunnel-host": "./src/lib/tunnel-host.ts",
     "./runtime-manager": "./src/lib/runtime-manager.ts",
+    "./port-pool": "./src/lib/port-pool.ts",
     "./runtime-activity": "./src/lib/runtime-activity.ts",
     "./resource-limits": "./src/lib/resource-limits.ts",
     "./cloud-sync-watcher": "./src/lib/cloud-sync-watcher.ts",
@@ -50,7 +51,7 @@
     "prepublishOnly": "bun run typecheck && bun run build:js"
   },
   "dependencies": {
-    "@shogo-ai/sdk": "^2.5.2",
+    "@shogo-ai/sdk": "^2.5.4",
     "commander": "^12.1.0",
     "picocolors": "^1.1.1"
   },

package/src/lib/__tests__/runtime-manager-coverage-gaps.test.ts

@@ -708,18 +708,10 @@ describe('doStart() via ensureRunning — makeSlot + proc handlers (L968-1072)',
 });
 
 // ──────────────────────────────────────────────────────────────────────
-// isPortListening() (L1356-1366)
+// Port occupancy now lives in port-pool.ts (SHO-335): the worker's HTTP HEAD
+// probe is gone, so its unit tests moved to runtime-manager-ports.test.ts.
 // ──────────────────────────────────────────────────────────────────────
 
-describe('isPortListening() (L1356-1366)', () => {
-  it('returns false when nothing is listening on the port', async () => {
-    const mgr = makeManager();
-    // Port 1 is always closed/refused
-    const result = await (mgr as any).isPortListening(1);
-    expect(result).toBe(false);
-  });
-});
-
 // ──────────────────────────────────────────────────────────────────────
 // armGraceTimer() callback body — covers L1135-1138
 // ──────────────────────────────────────────────────────────────────────

package/src/lib/__tests__/runtime-manager-ports.test.ts

@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: MIT
+// Copyright (C) 2026 Shogo Technologies, Inc.
+/**
+ * SHO-335 — port allocation must never hand out a busy port.
+ *
+ * The worker's `allocatePort` used to probe with an HTTP `HEAD`: a listener
+ * that accepted the connection but never answered (Vite mid-compile, a
+ * saturated event loop, any non-HTTP socket) read as FREE, and the agent
+ * runtime then died at `Bun.serve` with `EADDRINUSE` and crash-looped.
+ *
+ * These tests pin the two behaviours the fix depends on:
+ *   1. a busy-but-silent TCP listener is skipped (red on the HTTP probe), and
+ *   2. a probe that TIMES OUT counts as in use, never as free.
+ */
+import { afterEach, describe, expect, it } from 'bun:test';
+import { createServer, type Server, type Socket } from 'node:net';
+import {
+  DEFAULT_PORT_RANGE_END,
+  DEFAULT_PORT_RANGE_START,
+  PortPool,
+  probeTcpPort,
+} from '../port-pool.ts';
+import { WorkerRuntimeManager } from '../runtime-manager.ts';
+
+const RUNTIME_PORT_BLOCK = 16;
+
+/** A manager whose pool draws candidates from `random`; every level is logged. */
+function managerWithPool(pool: PortPool, logs: string[] = []): any {
+  return new WorkerRuntimeManager({
+    env: {} as NodeJS.ProcessEnv,
+    ports: pool,
+    logger: {
+      log: (line: string) => logs.push(line),
+      warn: (line: string) => logs.push(line),
+      error: (line: string) => logs.push(line),
+    },
+  }) as any;
+}
+
+/** Random value that makes `allocate` start its candidate search at `port`. */
+function randomTargeting(port: number, blockSize: number): () => number {
+  const candidates = DEFAULT_PORT_RANGE_END - DEFAULT_PORT_RANGE_START - blockSize + 2;
+  const index = port - DEFAULT_PORT_RANGE_START;
+  // +0.5 keeps the floor() away from float error at either edge of the bucket.
+  return () => (index + 0.5) / candidates;
+}
+
+const servers: Server[] = [];
+const sockets: Socket[] = [];
+
+afterEach(() => {
+  for (const socket of sockets.splice(0)) socket.destroy();
+  for (const server of servers.splice(0)) server.close();
+});
+
+/**
+ * Listen on `port`, accepting connections and never writing a byte — exactly
+ * what an HTTP HEAD probe misreads as free.
+ */
+async function silentListener(port: number): Promise<Server> {
+  const server = createServer((socket) => {
+    sockets.push(socket);
+  });
+  servers.push(server);
+  await new Promise<void>((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(port, '127.0.0.1', () => resolve());
+  });
+  return server;
+}
+
+/** True when `port` can be bound right now (the probe is released again). */
+async function portIsFree(port: number): Promise<boolean> {
+  const server = createServer();
+  try {
+    await new Promise<void>((resolve, reject) => {
+      server.once('error', reject);
+      server.listen(port, '127.0.0.1', () => resolve());
+    });
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** First port in the allocatable range that will accept a listener. */
+async function parkSilentListenerInRange(): Promise<number> {
+  for (let port = DEFAULT_PORT_RANGE_END - RUNTIME_PORT_BLOCK + 1; port > DEFAULT_PORT_RANGE_START; port--) {
+    try {
+      await silentListener(port);
+      return port;
+    } catch {
+      /* this one is taken by a real runtime (or another test) — try the next */
+    }
+  }
+  throw new Error('no free port in the allocatable range');
+}
+
+/** A candidate port outside `port`'s block whose eagerly-bound ports are free. */
+async function freeCandidateAwayFrom(port: number): Promise<number> {
+  for (let candidate = DEFAULT_PORT_RANGE_START; candidate <= DEFAULT_PORT_RANGE_END - RUNTIME_PORT_BLOCK + 1; candidate++) {
+    if (candidate >= port - RUNTIME_PORT_BLOCK && candidate <= port + RUNTIME_PORT_BLOCK) continue;
+    if ((await portIsFree(candidate)) && (await portIsFree(candidate + 1))) return candidate;
+  }
+  throw new Error('no free candidate port in the allocatable range');
+}
+
+describe('WorkerRuntimeManager.allocatePort — occupancy probe', () => {
+  it('skips a listener that accepts but never answers (HTTP HEAD reads it as free)', async () => {
+    const busyPort = await parkSilentListenerInRange();
+    const free = await freeCandidateAwayFrom(busyPort);
+
+    const pool = new PortPool();
+    // First candidate lands on the busy port; every later attempt lands on a
+    // port nothing is listening on.
+    const picks = [
+      randomTargeting(busyPort, RUNTIME_PORT_BLOCK),
+      randomTargeting(free, RUNTIME_PORT_BLOCK),
+    ];
+    (pool as any).random = () => (picks.length ? picks.shift()! : randomTargeting(free, RUNTIME_PORT_BLOCK))();
+
+    const block = await managerWithPool(pool).allocatePort();
+
+    expect(block).not.toBe(busyPort);
+    expect(block).toBe(free);
+    // The busy port is not claimed, in this block or any other.
+    expect(pool.isReserved(busyPort)).toBe(false);
+  });
+});
+
+describe('probeTcpPort — timeout semantics', () => {
+  function fakeSocket(emit: 'connect' | 'error' | 'timeout') {
+    const handlers = new Map<string, Array<(...args: any[]) => void>>();
+    const socket = {
+      setTimeout: () => socket,
+      destroy: () => socket,
+      once(event: string, listener: (...args: any[]) => void) {
+        handlers.set(event, [...(handlers.get(event) ?? []), listener]);
+        if (event === emit) queueMicrotask(() => listener(new Error('probe')));
+        return socket;
+      },
+    };
+    return socket as any;
+  }
+
+  it('treats a timed-out connect as in use', async () => {
+    await expect(
+      probeTcpPort(12345, { connect: (() => fakeSocket('timeout')) as any }),
+    ).resolves.toBe('in-use');
+  });
+
+  it('treats a completed connect as in use', async () => {
+    await expect(
+      probeTcpPort(12345, { connect: (() => fakeSocket('connect')) as any }),
+    ).resolves.toBe('in-use');
+  });
+
+  it('treats a refused connect as free', async () => {
+    await expect(
+      probeTcpPort(12345, { connect: (() => fakeSocket('error')) as any }),
+    ).resolves.toBe('free');
+  });
+
+  it('never allocates a port whose probe did not answer', async () => {
+    const timedOut = DEFAULT_PORT_RANGE_START + 10;
+    const free = DEFAULT_PORT_RANGE_START + 40;
+    const pool = new PortPool({
+      probe: async (port) => (port === timedOut ? 'in-use' : 'free'),
+    });
+    const picks = [randomTargeting(timedOut, RUNTIME_PORT_BLOCK), randomTargeting(free, RUNTIME_PORT_BLOCK)];
+    (pool as any).random = () => (picks.length ? picks.shift()! : randomTargeting(free, RUNTIME_PORT_BLOCK))();
+
+    const block = await pool.allocate({
+      start: DEFAULT_PORT_RANGE_START,
+      end: DEFAULT_PORT_RANGE_END,
+      blockSize: RUNTIME_PORT_BLOCK,
+      probeOffsets: [0, 1],
+    });
+
+    expect(block).toBe(free);
+    expect(pool.isReserved(timedOut)).toBe(false);
+  });
+});
+
+describe('WorkerRuntimeManager.allocatePort — shared pool', () => {
+  it('reserves a block in the pool it was given and releases it again', async () => {
+    const pool = new PortPool({ probe: async () => 'free' });
+    const logs: string[] = [];
+    const mgr = managerWithPool(pool, logs);
+
+    const block = await mgr.allocatePort();
+    expect(pool.isReserved(block)).toBe(true);
+    expect(pool.isReserved(block + RUNTIME_PORT_BLOCK - 1)).toBe(true);
+    // A host log must be able to attribute the next EADDRINUSE without a trace.
+    expect(logs.some((line) => line.includes(
+      `[WorkerRuntimeManager] Allocated agent block ${block}-${block + RUNTIME_PORT_BLOCK - 1}`,
+    ))).toBe(true);
+
+    mgr.releasePort(block);
+    for (let offset = 0; offset < RUNTIME_PORT_BLOCK; offset++) {
+      expect(pool.isReserved(block + offset)).toBe(false);
+    }
+  });
+});
+
+describe('WorkerRuntimeManager.handleExit — the dead port is not reused', () => {
+  /** A running slot holding `block`, with no live process behind it. */
+  function runningSlot(mgr: any, projectId: string, block: number): any {
+    const slot = mgr.makeSlot(projectId, {});
+    slot.agentPort = block;
+    slot.apiServerPort = block + 1;
+    slot.status = 'running';
+    slot.pid = null;
+    slot.startedAt = Date.now() - 1_000;
+    mgr.runtimes.set(projectId, slot);
+    return slot;
+  }
+
+  it('hands the block back on a non-clean exit so the restart must re-probe', async () => {
+    const pool = new PortPool({ probe: async () => 'free' });
+    const mgr = managerWithPool(pool);
+    const block = await mgr.allocatePort();
+    const slot = runningSlot(mgr, 'proj-crash', block);
+
+    mgr.handleExit(slot, 1, null);
+    if (slot.restartTimer) clearTimeout(slot.restartTimer);
+
+    // `doStart` only allocates when `agentPort` is 0 — keeping the block here
+    // is what re-booted the same stolen port for every attempt in the loop.
+    expect(slot.agentPort).toBe(0);
+    expect(slot.apiServerPort).toBe(0);
+    expect(pool.isReserved(block)).toBe(false);
+    expect(pool.isReserved(block + RUNTIME_PORT_BLOCK - 1)).toBe(false);
+  });
+
+  it('moves to a fresh block when the next boot finds the old one taken', async () => {
+    const taken = new Set<number>();
+    const pool = new PortPool({ probe: async () => 'free' });
+    const mgr = managerWithPool(pool);
+    const block = await mgr.allocatePort();
+    const slot = runningSlot(mgr, 'proj-stolen', block);
+
+    mgr.handleExit(slot, 1, null);
+    if (slot.restartTimer) clearTimeout(slot.restartTimer);
+
+    // Something outside the pool grabbed the block between probe and bind.
+    for (let offset = 0; offset < RUNTIME_PORT_BLOCK; offset++) taken.add(block + offset);
+    (pool as any).probe = async (port: number) => (taken.has(port) ? 'in-use' : 'free');
+
+    const next = await mgr.allocatePort();
+    expect(next).not.toBe(block);
+    expect(pool.isReserved(next)).toBe(true);
+  });
+});
+
+describe('WorkerRuntimeManager.logPortHolder — attributing an EADDRINUSE', () => {
+  it('names the process holding the port', async () => {
+    const logs: string[] = [];
+    const mgr = managerWithPool(new PortPool(), logs);
+    const busyPort = await parkSilentListenerInRange();
+    const slot = mgr.makeSlot('proj-held', {});
+    slot.agentPort = busyPort;
+
+    await mgr.logPortHolder(slot);
+
+    const holderLine = logs.find((line) => line.includes(`port ${busyPort}`));
+    expect(holderLine).toBeDefined();
+    expect(holderLine).toContain('is held by');
+    // The listener is this very test process — a real PID, not a placeholder.
+    expect(holderLine).toContain(String(process.pid));
+  });
+
+  it('says nothing holds the port when the bind failed for another reason', async () => {
+    const logs: string[] = [];
+    const mgr = managerWithPool(new PortPool(), logs);
+    const free = await freeCandidateAwayFrom(DEFAULT_PORT_RANGE_START - RUNTIME_PORT_BLOCK);
+    const slot = mgr.makeSlot('proj-free', {});
+    slot.agentPort = free;
+
+    await mgr.logPortHolder(slot);
+
+    expect(logs.some((line) => line.includes(`port ${free}`) && line.includes('no listener now'))).toBe(true);
+  });
+});

package/src/lib/port-pool.ts

@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: MIT
+// Copyright (C) 2026 Shogo Technologies, Inc.
+/**
+ * PortPool — the single in-process allocator for the runtime port range.
+ *
+ * Two managers hand out localhost ports inside one process: `apps/api`'s
+ * `RuntimeManager` (Vite / remote-forward port, plus `+1000`/`+1001` for the
+ * agent and its API server) and the `WorkerRuntimeManager` that api embeds
+ * (a 16-port block per agent runtime). Each used to keep its own `Set` over
+ * 37100–37900, so neither could see the other's reservations: api could hand
+ * 37566 to a Vite server while the worker picked the block containing 37566
+ * for an agent, and the child crash-looped on `EADDRINUSE` (SHO-335).
+ *
+ * The fix is one pool per process, passed into the worker by api. Disjoint
+ * sub-ranges were the alternative; they would have halved capacity and left
+ * the same bug for the next caller that picks a range.
+ *
+ * Occupancy probing is a kernel-level TCP connect, never an HTTP request:
+ * a listener that is up but slow to answer (Vite mid-compile, a saturated
+ * event loop, any non-HTTP socket) is still a busy port. A connect that is
+ * refused means free; a connect that TIMES OUT means in use — an accept
+ * queue that never completes is not a free port.
+ */
+import { connect, type Socket } from 'node:net';
+
+/** Port range for random allocation (mirrors the desktop manager). */
+export const DEFAULT_PORT_RANGE_START = 37100;
+export const DEFAULT_PORT_RANGE_END = 37900;
+
+/** Per-probe connect timeout. */
+export const DEFAULT_PROBE_TIMEOUT_MS = 500;
+
+/** How many random candidates one `allocate()` tries before giving up. */
+const DEFAULT_MAX_ATTEMPTS = 50;
+
+export type PortProbeResult = 'free' | 'in-use';
+
+export type ConnectFactory = (options: { host: string; port: number }) => Socket;
+
+export interface PortProbeOptions {
+  /** Connect timeout in ms. Defaults to {@link DEFAULT_PROBE_TIMEOUT_MS}. */
+  timeoutMs?: number;
+  /** Interface to dial. Defaults to the IPv4 loopback literal. */
+  host?: string;
+  /** Test seam: dial with something other than `net.connect`. */
+  connect?: ConnectFactory;
+}
+
+/**
+ * Probe a port with a TCP connect.
+ *
+ * - `connect` → `'in-use'`
+ * - `error` (ECONNREFUSED, EHOSTUNREACH, …) → `'free'`
+ * - `timeout` → `'in-use'` (a listener whose accept queue is full is busy)
+ */
+export async function probeTcpPort(
+  port: number,
+  options: PortProbeOptions = {},
+): Promise<PortProbeResult> {
+  const timeoutMs = options.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
+  const host = options.host ?? '127.0.0.1';
+  const dial: ConnectFactory = options.connect ?? ((target) => connect(target));
+
+  return await new Promise<PortProbeResult>((resolve) => {
+    let settled = false;
+    const finish = (result: PortProbeResult) => {
+      if (settled) return;
+      settled = true;
+      try {
+        socket.destroy();
+      } catch {
+        /* socket may already be torn down */
+      }
+      resolve(result);
+    };
+    const socket = dial({ host, port });
+    socket.setTimeout(timeoutMs);
+    socket.once('connect', () => finish('in-use'));
+    socket.once('error', () => finish('free'));
+    socket.once('timeout', () => finish('in-use'));
+  });
+}
+
+export interface PortPoolOptions {
+  /** Override the occupancy probe (tests). */
+  probe?: (port: number) => Promise<PortProbeResult>;
+  /** Connect timeout for the default TCP probe. */
+  probeTimeoutMs?: number;
+  /** Override randomness (tests). Defaults to `Math.random`. */
+  random?: () => number;
+}
+
+export interface PortBlockRequest {
+  /** First candidate port. Defaults to {@link DEFAULT_PORT_RANGE_START}. */
+  start?: number;
+  /** Last port the candidate BLOCK may reach. Defaults to {@link DEFAULT_PORT_RANGE_END}. */
+  end?: number;
+  /**
+   * Contiguous ports reserved at the candidate, starting with the candidate
+   * itself. Defaults to 1.
+   */
+  blockSize?: number;
+  /**
+   * Offsets from the candidate that are reserved AND liveness-probed. Unlike
+   * {@link blockSize} these are not bounded by `end`: the desktop manager's
+   * agent port lives at `+1000`, well past its Vite port range.
+   */
+  probeOffsets?: readonly number[];
+  maxAttempts?: number;
+}
+
+/**
+ * Reservation pool over the runtime port range. One instance per process;
+ * every allocator in that process must be handed the same one.
+ */
+export class PortPool {
+  /**
+   * Every port handed out and not yet released. Exposed so callers (and the
+   * managers' `usedPorts` accessor used by tests) share one source of truth
+   * instead of keeping a parallel set.
+   */
+  readonly reserved = new Set<number>();
+
+  private readonly probe: (port: number) => Promise<PortProbeResult>;
+  private readonly random: () => number;
+
+  constructor(options: PortPoolOptions = {}) {
+    this.probe =
+      options.probe ??
+      ((port) => probeTcpPort(port, { timeoutMs: options.probeTimeoutMs }));
+    this.random = options.random ?? Math.random;
+  }
+
+  /** Reserve ports that something outside this pool already bound. */
+  reserve(ports: Iterable<number>): void {
+    for (const port of ports) if (port > 0) this.reserved.add(port);
+  }
+
+  release(ports: Iterable<number>): void {
+    for (const port of ports) this.reserved.delete(port);
+  }
+
+  /** Release a contiguous block previously returned by {@link allocate}. */
+  releaseBlock(start: number, size: number): void {
+    if (!start) return;
+    for (let offset = 0; offset < size; offset++) this.reserved.delete(start + offset);
+  }
+
+  isReserved(port: number): boolean {
+    return this.reserved.has(port);
+  }
+
+  /** True when nothing is reserved in-process and nothing answers on the port. */
+  async isPortAvailable(port: number): Promise<boolean> {
+    if (port <= 0) return false;
+    if (this.reserved.has(port)) return false;
+    return await this.probeFree(port);
+  }
+
+  /**
+   * Pick a random candidate block and reserve it.
+   *
+   * Reserves BEFORE probing: the probe awaits, and two concurrent allocations
+   * must never both see the same block as free (the previous per-manager code
+   * check-then-set across an `await`, so a burst of starts could collide).
+   * The block is released again if any probed port turns out to be busy.
+   */
+  async allocate(request: PortBlockRequest = {}): Promise<number> {
+    const start = request.start ?? DEFAULT_PORT_RANGE_START;
+    const end = request.end ?? DEFAULT_PORT_RANGE_END;
+    const blockSize = request.blockSize ?? 1;
+    const probeOffsets = request.probeOffsets ?? [0];
+    const candidates = end - start - blockSize + 2;
+    if (candidates < 1) {
+      throw new Error(`Port range ${start}-${end} cannot hold a ${blockSize}-port block`);
+    }
+    const maxAttempts = request.maxAttempts ?? Math.min(candidates, DEFAULT_MAX_ATTEMPTS);
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      const candidate = start + Math.floor(this.random() * candidates);
+      const ports = this.candidatePorts(candidate, blockSize, probeOffsets);
+      if (ports.some((port) => this.reserved.has(port))) continue;
+
+      this.reserve(ports);
+      let free = true;
+      for (const offset of probeOffsets) {
+        if (!(await this.probeFree(candidate + offset))) {
+          free = false;
+          break;
+        }
+      }
+      if (!free) {
+        this.release(ports);
+        continue;
+      }
+      return candidate;
+    }
+
+    throw new Error(
+      `Cannot allocate port after ${maxAttempts} attempts in range ${start}-${end}`,
+    );
+  }
+
+  private candidatePorts(
+    start: number,
+    blockSize: number,
+    probeOffsets: readonly number[],
+  ): number[] {
+    const ports = new Set<number>();
+    for (let offset = 0; offset < blockSize; offset++) ports.add(start + offset);
+    for (const offset of probeOffsets) ports.add(start + offset);
+    return [...ports].filter((port) => port > 0);
+  }
+
+  private async probeFree(port: number): Promise<boolean> {
+    if (port <= 0) return false;
+    return (await this.probe(port)) === 'free';
+  }
+}