@shogo-ai/worker 2.5.2 → 2.5.4

package/src/lib/runtime-manager.ts

@@ -31,7 +31,7 @@
  *   - No Prisma reads. All policy comes from the spawn config.
  *   - No security policy build. Cloud signs the policy and sends it.
  */
-import { type ChildProcess, spawn, spawnSync } from './child-process.ts';
+import { type ChildProcess, execFile, spawn, spawnSync } from './child-process.ts';
 import { createHmac, randomBytes } from 'node:crypto';
 import { existsSync, mkdirSync, readdirSync, rmSync, unwatchFile, watch, watchFile } from 'node:fs';
 import { createConnection } from 'node:net';
@@ -65,10 +65,19 @@ import {
   type ActivitySample,
   type BusyDecision,
 } from './runtime-activity.ts';
+import {
+  DEFAULT_PORT_RANGE_END,
+  DEFAULT_PORT_RANGE_START,
+  PortPool,
+} from './port-pool.ts';
 
-/** Port range for random allocation (mirrors apps/api desktop manager). */
-const PORT_RANGE_START = 37100;
-const PORT_RANGE_END = 37900;
+/**
+ * Port range agents are allocated from. The same range is used by `apps/api`'s
+ * manager for its Vite ports, so both allocators in one process must share one
+ * {@link PortPool} — see port-pool.ts.
+ */
+const PORT_RANGE_START = DEFAULT_PORT_RANGE_START;
+const PORT_RANGE_END = DEFAULT_PORT_RANGE_END;
 const API_PORT_OFFSET = 1; // API server port = agentPort + 1.
 
 /**
@@ -117,6 +126,15 @@ const ACTIVITY_PROBE_TIMEOUT_MS = 3000;
  */
 const STDERR_TAIL_LINES = 8;
 
+/**
+ * The child's own words when it loses the bind race for its port. `Bun.serve`
+ * reports `Failed to start server. Is port 37404 in use?` followed by an
+ * `EADDRINUSE` code, and the runtime's health gate then reports
+ * `agent-runtime exited … before becoming healthy`. Matching either lets
+ * {@link WorkerRuntimeManager.logPortHolder} name the holder.
+ */
+const PORT_CONFLICT_RE = /EADDRINUSE|Is port \d+ in use/;
+
 /** Restart backoff bounds. */
 const RESTART_BACKOFF_BASE_MS = 1_000;
 const RESTART_BACKOFF_MAX_MS = 60_000;
@@ -424,6 +442,14 @@ export interface WorkerRuntimeManagerOptions {
    * callback are swallowed.
    */
   onRuntimeGone?: (projectId: string, info: RuntimeGoneInfo) => void;
+  /**
+   * Shared port reservation pool. Pass the SAME pool the host process uses for
+   * its own port allocations (the desktop API does) so two allocators in one
+   * process can never hand out the same port — the EADDRINUSE crash-loop in
+   * SHO-335. When absent the manager creates its own pool, which is correct for
+   * a standalone `shogo worker start` (it is the only allocator there).
+   */
+  ports?: PortPool;
 }
 
 export interface RuntimeGoneInfo {
@@ -559,6 +585,12 @@ interface InternalRuntime {
   activity?: ActivitySample | null;
   /** Last few stderr lines, logged when the process exits unrequested. */
   stderrTail?: string[];
+  /**
+   * Set once the current incarnation has reported an `EADDRINUSE` on its own
+   * port, so the holder lookup runs once per spawn instead of once per stderr
+   * line (the child repeats the error when it exits).
+   */
+  portConflictDiagnosed?: boolean;
 }
 
 /**
@@ -671,7 +703,14 @@ export class WorkerRuntimeManager implements RuntimeResolver {
   private readonly opts: WorkerRuntimeManagerOptions;
   private readonly log: Pick<Console, 'log' | 'warn' | 'error'>;
   private readonly runtimes = new Map<string, InternalRuntime>();
-  private readonly usedPorts = new Set<number>();
+  /**
+   * Port reservations, shared with the host when it passes one in (`apps/api`).
+   * Exposed to tests as `usedPorts` — the pool's set is the only one.
+   */
+  private readonly ports: PortPool;
+  private get usedPorts(): Set<number> {
+    return this.ports.reserved;
+  }
   private readonly spawnCommand: SpawnCommandFactory;
   private resolved: ResolvedRuntime | null = null;
   private stopped = false;
@@ -702,6 +741,7 @@ export class WorkerRuntimeManager implements RuntimeResolver {
   constructor(opts: WorkerRuntimeManagerOptions = {}) {
     this.opts = opts;
     this.log = opts.logger ?? console;
+    this.ports = opts.ports ?? new PortPool();
     this.spawnCommand = opts.spawnCommand ?? defaultSpawnCommand;
     this.sweepOrphanedViteWatches();
     // Activity is sampled on its own timer rather than probed at eviction
@@ -1492,6 +1532,7 @@ export class WorkerRuntimeManager implements RuntimeResolver {
     // one.
     slot.activity = null;
     slot.stderrTail = [];
+    slot.portConflictDiagnosed = false;
     const openAttemptId = slot.spawnConfig.extraEnv?.SHOGO_OPEN_ID;
     perfLog('agent-spawned', slot.projectId, openAttemptId, {
       pid: slot.pid,
@@ -1560,6 +1601,12 @@ export class WorkerRuntimeManager implements RuntimeResolver {
         this.log.error(`${prefix} ${line}`);
         tail.push(line);
         if (tail.length > STDERR_TAIL_LINES) tail.splice(0, tail.length - STDERR_TAIL_LINES);
+        // A child that cannot bind its port stays alive for seconds printing
+        // this, then exits — name the holder while it is still a question.
+        if (!slot.portConflictDiagnosed && PORT_CONFLICT_RE.test(line)) {
+          slot.portConflictDiagnosed = true;
+          void this.logPortHolder(slot);
+        }
       }
     });
 
@@ -2088,6 +2135,16 @@ export class WorkerRuntimeManager implements RuntimeResolver {
     this.reapChildLedgerFor(slot.projectId, slot.spawnConfig.projectDir);
     slot.pid = null;
 
+    // Hand the block back before the restart. `doStart` only allocates when
+    // `agentPort` is 0, so keeping it would re-boot the SAME port for every
+    // attempt in the loop — up to MAX_CONSECUTIVE_RESTARTS times on a port
+    // that is already somebody else's (SHO-335: block 37404-37419 allocated,
+    // EADDRINUSE 2s later, then eight boots of the same dead port). Releasing
+    // makes the next doStart() re-probe and move on.
+    this.releasePort(slot.agentPort);
+    slot.agentPort = 0;
+    slot.apiServerPort = 0;
+
     const now = Date.now();
     const withinWindow = now - slot.lastFailureAt <= RESTART_FAILURE_WINDOW_MS;
     slot.consecutiveFailures = withinWindow ? slot.consecutiveFailures + 1 : 1;
@@ -2379,50 +2436,71 @@ export class WorkerRuntimeManager implements RuntimeResolver {
   }
 
   private async allocatePort(): Promise<number> {
-    const range = PORT_RANGE_END - PORT_RANGE_START;
-    const maxAttempts = Math.min(range, 50);
-    for (let i = 0; i < maxAttempts; i++) {
-      const candidate = PORT_RANGE_START + Math.floor(Math.random() * range);
-      // Reserve a contiguous per-runtime block so the agent port, its API
-      // server AND every preview sidecar (WORKSPACE_API_PORT_BASE + idx) live
-      // in a range that no other warm runtime can overlap.
-      if (candidate + RUNTIME_PORT_BLOCK - 1 > PORT_RANGE_END) continue;
-      let blockFree = true;
-      for (let off = 0; off < RUNTIME_PORT_BLOCK; off++) {
-        if (this.usedPorts.has(candidate + off)) { blockFree = false; break; }
-      }
-      if (!blockFree) continue;
-      // Liveness-probe only the two ports we bind eagerly (agent + its API
-      // server). The sidecar ports are bound lazily by the agent-runtime and
-      // guarded by its own leaked-process force-kill, so probing the whole
-      // block here would just slow allocation down.
-      const agentInUse = await this.isPortListening(candidate);
-      const apiInUse = await this.isPortListening(candidate + API_PORT_OFFSET);
-      if (agentInUse || apiInUse) continue;
-      for (let off = 0; off < RUNTIME_PORT_BLOCK; off++) this.usedPorts.add(candidate + off);
-      return candidate;
-    }
-    throw new Error(
-      `Cannot allocate port in range ${PORT_RANGE_START}-${PORT_RANGE_END} after ${maxAttempts} attempts`,
+    // The agent port, its API server (+1) and every preview sidecar
+    // (+2 … +15) live in one contiguous block that no other runtime — and no
+    // other allocator in this process (api's Vite port, SHO-335) — can hand
+    // out. Only the two eagerly-bound ports are liveness-probed; the sidecar
+    // ports are bound lazily by the agent-runtime and guarded by its own
+    // leaked-process force-kill, so probing the whole block would just slow
+    // allocation down.
+    const agentPort = await this.ports.allocate({
+      start: PORT_RANGE_START,
+      end: PORT_RANGE_END,
+      blockSize: RUNTIME_PORT_BLOCK,
+      probeOffsets: [0, API_PORT_OFFSET],
+    });
+    this.log.log(
+      `[WorkerRuntimeManager] Allocated agent block ${agentPort}-${agentPort + RUNTIME_PORT_BLOCK - 1}`,
     );
+    return agentPort;
   }
 
   private releasePort(port: number): void {
     if (!port) return;
-    for (let off = 0; off < RUNTIME_PORT_BLOCK; off++) this.usedPorts.delete(port + off);
+    this.ports.releaseBlock(port, RUNTIME_PORT_BLOCK);
   }
 
-  private async isPortListening(port: number): Promise<boolean> {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), 250);
-    try {
-      await fetch(`http://127.0.0.1:${port}/`, { method: 'HEAD', signal: controller.signal });
-      clearTimeout(timer);
-      return true;
-    } catch {
-      clearTimeout(timer);
-      return false;
+  /**
+   * Name whoever holds the port the child failed to bind.
+   *
+   * The child's own `EADDRINUSE` says nothing about who won, and the block was
+   * reserved from a pool that no in-process allocator can duplicate, so the
+   * only remaining candidates are: a leaked child of an earlier incarnation, a
+   * neighbouring process on the host, or the runtime itself binding twice
+   * (SHO-335 QA round 1 could not tell these apart from the CI log alone).
+   * Best-effort: a missing `lsof`/`ss` is not worth a failure.
+   */
+  private async logPortHolder(slot: InternalRuntime): Promise<void> {
+    const port = slot.agentPort;
+    if (!port) return;
+    const label = `[WorkerRuntimeManager] port ${port} (${slot.projectId})`;
+    for (const [command, args] of this.portHolderCommands(port)) {
+      const result = await runDiagnostic(command, args);
+      if (!result) continue;
+      const holders = result.stdout
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.includes(String(port)));
+      if (holders.length) {
+        this.log.error(`${label} is held by:\n  ${holders.join('\n  ')}`);
+      } else {
+        this.log.error(
+          `${label} has no listener now — the bind failure came from this runtime's own tree ` +
+            `(or the holder exited with it), not from a neighbouring process`,
+        );
+      }
+      return;
     }
+    this.log.warn(`${label}: no port-listing tool available, holder unknown`);
+  }
+
+  /** Host tools that can name a TCP listener's owner, best first. */
+  private portHolderCommands(port: number): Array<[string, string[]]> {
+    if (process.platform === 'win32') return [['netstat', ['-ano', '-p', 'tcp']]];
+    return [
+      ['lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN']],
+      ['ss', ['-ltnp', `sport = :${port}`]],
+    ];
   }
 
   /**
@@ -2430,9 +2508,10 @@ export class WorkerRuntimeManager implements RuntimeResolver {
    * to `127.0.0.1:port` succeeds within {@link TCP_CONNECT_TIMEOUT_MS},
    * `false` for any failure mode (refused, timed out, host unreachable).
    *
-   * This is intentionally distinct from {@link isPortListening} (which
-   * fires an HTTP HEAD): we want to know whether `Bun.serve()` has
-   * bound the socket, NOT whether its request handler is responding.
+   * This is intentionally distinct from the occupancy probe in
+   * `port-pool.ts` (which a timeout counts as BUSY): we want to know whether
+   * `Bun.serve()` has bound the socket, NOT whether its request handler is
+   * responding. A timeout here means "not ready yet".
    * Once the kernel has the listener, the agent-runtime's
    * `export default { port, fetch }` has been evaluated — which is
    * sufficient evidence to clear the boot gate even if the event loop
@@ -2638,3 +2717,18 @@ export class WorkerRuntimeManager implements RuntimeResolver {
     };
   }
 }
+
+/**
+ * Run a short-lived listing command. `null` means the tool is not installed
+ * (ENOENT) — try the next one; `{ stdout: '' }` means it ran and found
+ * nothing, which is itself an answer. Never throws: this only runs on an
+ * already-failing path.
+ */
+function runDiagnostic(command: string, args: string[]): Promise<{ stdout: string } | null> {
+  return new Promise((resolve) => {
+    execFile(command, args, { timeout: 3_000 }, (err: any, stdout: string) => {
+      if (err?.code === 'ENOENT') return resolve(null);
+      resolve({ stdout: stdout ?? '' });
+    });
+  });
+}