@shogo-ai/worker 1.8.10 → 1.8.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shogo-ai/worker",
-  "version": "1.8.10",
+  "version": "1.8.11",
   "description": "Shogo Cloud Agent Worker — run Shogo agents on your own machine (laptop, devbox, CI).",
   "license": "MIT",
   "author": "Shogo Technologies, Inc.",

package/src/lib/__tests__/runtime-manager-wait-for-health.test.ts

@@ -1,34 +1,42 @@
 // SPDX-License-Identifier: MIT
 // Copyright (C) 2026 Shogo Technologies, Inc.
 /**
- * Pins {@link WorkerRuntimeManager.waitForHealth} against the three
- * readiness signals it now considers and the diagnostic behavior on
- * timeout. The historical bug this protects against: agent-runtime
- * cold boots on Windows with `bun --conditions=development run
- * packages/agent-runtime/src/server.ts` routinely take 30-45s just
- * to JIT-compile the TS dep graph (shared-runtime + generators +
- * tools + hooks + gateway). The kernel-level TCP listener is up
- * within ~1-2s of spawn (the moment `Bun.serve()` reads its default
- * export), but the event loop is still saturated for tens of seconds
- * after that, so the old HTTP-only /health gate timed out, SIGTERM'd
- * the still-booting child, and the restart-loop chewed through
- * `MAX_CONSECUTIVE_RESTARTS` before giving up — leaving chat stuck
- * with "Connection timed out — The agent runtime could not be
- * reached".
+ * Pins {@link WorkerRuntimeManager.waitForHealth} against the readiness
+ * contract every downstream caller of `status === 'running'` assumes:
+ * the agent runtime can actually serve HTTP, not just "the kernel
+ * accepted a TCP listener on the port". The historical context:
+ *
+ *   - First pass (pre-2026-05): HTTP /health was the only signal. Bun
+ *     on Windows takes 30-45s to JIT the runtime TS dep graph during
+ *     cold boot, and /health was starved on the busy event loop the
+ *     whole time, so the 30s timeout SIGTERM'd the still-booting
+ *     child and the restart loop chewed through `MAX_CONSECUTIVE_RESTARTS`.
+ *   - Second pass: added a TCP-listening + recent stdout fast path
+ *     that returned "ready" as soon as `Bun.serve()` bound the port
+ *     and the child kept emitting log lines. This stopped the SIGTERM
+ *     storm but tricked `/sandbox/url` into reporting ready before
+ *     HTTP actually worked, so the canvas iframe and AgentProxy
+ *     started chasing a black hole and surfaced as "Connection timed
+ *     out — The agent runtime could not be reached".
+ *   - Current pass (2026-05): keep waiting for a real /health 2xx with
+ *     HEALTH_BOOT_TIMEOUT_MS=30 s (now realistic on Windows because
+ *     LSP + IndexEngine were moved out of the critical-path boot
+ *     sequence in agent-runtime). TCP-listening + silent stdout >
+ *     STDOUT_PROGRESS_WINDOW_MS=25 s acts as a *wedge detector* —
+ *     a child that bound the port but stopped producing output for
+ *     25 s is genuinely stuck and the restart loop should recover
+ *     it instead of waiting for the hard timeout.
  *
  * The contracts pinned here:
  *
- *   1. HTTP /health 200 returns immediately — the happy path is
- *      unchanged for healthy runtimes.
- *   2. TCP-listening + recent stdout activity returns successfully
- *      even when /health never responds — handles the cold-boot
- *      saturation case.
+ *   1. HTTP /health 200 returns immediately — the happy path.
+ *   2. /health that never responds (with TCP up + stdout fresh) keeps
+ *      waiting until the timeout, instead of declaring ready early.
  *   3. TCP-listening but stdout silent past STDOUT_PROGRESS_WINDOW_MS
- *      does NOT short-circuit — a truly-wedged process still hits
- *      the hard timeout.
- *   4. Process death (exitCode/signalCode/killed) short-circuits
- *      immediately with the exit info, instead of waiting the full
- *      timeout window.
+ *      throws the "wedged" error so restart-with-backoff fires.
+ *   4. Neither TCP nor HTTP ever come up → clean timeout error.
+ *   5. Process death (exitCode/signalCode/killed) short-circuits
+ *      immediately with the exit info.
  */
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import { WorkerRuntimeManager } from '../runtime-manager.ts';
@@ -132,63 +140,73 @@ describe('WorkerRuntimeManager.waitForHealth (private)', () => {
   });
 
   it(
-    'accepts TCP-listening + recent stdout activity as ready when /health keeps failing',
+    'keeps waiting for /health even when TCP is up and stdout is fresh',
     async () => {
-      // Reproduces the Windows `--conditions=development` cold-boot:
-      // Bun.serve has bound the port (kernel-level listener up) and
-      // the child is still emitting log lines (LSP-TS init,
-      // preview-manager spawn, vite build), but /health is starved
-      // because the event loop is busy JIT-compiling. The wait
-      // should accept this as ready instead of timing out for 30s
-      // and SIGTERM'ing a still-booting process.
+      // Reproduces the Windows cold-boot saturation pattern: Bun.serve
+      // has bound the port and the child is still emitting log lines
+      // (LSP-TS init, preview-manager spawn, vite build), but /health
+      // never gets event-loop time. The wait must NOT declare the
+      // runtime ready on TCP+stdout alone any more — see the file
+      // docstring for the canvas-iframe "Connection timed out"
+      // regression that fast-path caused. It should keep polling
+      // /health and eventually hit the deadline.
       globalThis.fetch = (async () => {
         throw new Error('ECONNREFUSED');
       }) as typeof fetch;
       stubTcpProbe(mgr, true);
 
       const slot = makeSlot();
-      // Keep stdout "fresh" so the progress window stays open.
+      // Keep stdout "fresh" so the wedge detector stays quiet — we
+      // want to exercise the hard timeout, not the wedge bail-out.
       const bumper = setInterval(() => {
         slot.lastStdoutAt = Date.now();
       }, 200);
 
       const startedAt = Date.now();
       try {
-        await callWaitForHealth(mgr, slot, 30_000);
+        await expect(callWaitForHealth(mgr, slot, 1_500)).rejects.toThrow(
+          /Timeout waiting for agent-runtime \/health on port 41234.*tcpListening=true/s,
+        );
       } finally {
         clearInterval(bumper);
       }
       const elapsed = Date.now() - startedAt;
-      // Should return within the first iteration or two — TCP probe
-      // succeeds + stdout fresh = ready. Hard cap well below the 30s
-      // timeout to catch a regression to the legacy 30s spin.
-      expect(elapsed).toBeLessThan(5_000);
+      // Should have spent close to the full 1.5s window polling
+      // /health, not short-circuited early.
+      expect(elapsed).toBeGreaterThanOrEqual(1_400);
     },
-    15_000,
+    5_000,
   );
 
   it(
-    'does NOT accept TCP-listening alone when stdout is silent past the progress window',
+    'throws "wedged" when TCP is up and stdout is silent past the progress window',
     async () => {
       // The silent-but-bound case mimics a process that bound the
       // port and then wedged (infinite loop in top-level code, native
-      // crash mid-init that didn't propagate to the parent). The
-      // progress window correctly refuses to short-circuit here so
-      // the restart loop has a chance to recover.
+      // crash mid-init that didn't propagate to the parent). Bail
+      // out fast with a "wedged" error so the restart-with-backoff
+      // loop in handleExit() can SIGTERM and respawn, instead of
+      // spinning the full HEALTH_BOOT_TIMEOUT_MS for a process that
+      // will never recover.
       globalThis.fetch = (async () => {
         throw new Error('ECONNREFUSED');
       }) as typeof fetch;
       stubTcpProbe(mgr, true);
 
       const slot = makeSlot({
-        // Pre-age the stdout timestamp so the window is already
-        // closed before the first iteration.
-        lastStdoutAt: Date.now() - 60_000,
+        // Pre-age the stdout timestamp so the wedge detector trips
+        // on the first iteration. Must exceed STDOUT_PROGRESS_WINDOW_MS
+        // (25 s) to fire.
+        lastStdoutAt: Date.now() - 30_000,
       });
 
-      await expect(callWaitForHealth(mgr, slot, 1_200)).rejects.toThrow(
-        /Timeout waiting for agent-runtime \/health on port 41234.*tcpListening=true/s,
+      // Generous outer timeout — the wedge detector should fire
+      // immediately, well before the 30s timeout would.
+      const startedAt = Date.now();
+      await expect(callWaitForHealth(mgr, slot, 30_000)).rejects.toThrow(
+        /agent-runtime wedged on port 41234.*stdout silent for/s,
       );
+      expect(Date.now() - startedAt).toBeLessThan(2_500);
     },
     5_000,
   );

package/src/lib/runtime-manager.ts

@@ -87,30 +87,50 @@ const STARTUP_GRACE_MS = 60_000;
 const HEALTH_POLL_MS = 500;
 /**
  * Absolute ceiling on how long we'll wait for a freshly-spawned
- * agent-runtime to clear the boot gate (either a 200 /health response
- * or the TCP-listening + stdout-progress fallback below). Was 30s
- * before the Windows `--conditions=development` cold-boot
- * investigation surfaced that Bun routinely takes 30-45s just to JIT
- * the agent-runtime's TS dep graph (shared-runtime + generators +
- * tools + hooks + gateway) on a cold workspace, leaving zero budget
- * for /health to actually respond. 60s gives the slow path room
- * without giving a truly-hung process a free pass — the progress
- * detector below catches that case independently.
+ * agent-runtime to respond 2xx on /health. After the 2026-05 fix
+ * (`runtime-log-writer` async batched writes, deferred LSP +
+ * IndexEngine start, /health fast-path in the outer fetch handler),
+ * a Windows cold boot is ~8-12 s; macOS/Linux are ~3-5 s. 30 s gives
+ * a comfortable margin without hiding real hangs.
+ *
+ * Why we don't short-circuit on TCP-listening + stdout activity any
+ * more (the historical fast path that bypassed /health entirely):
+ * declaring "ready" while the event loop is saturated tricks the UI
+ * into starting to load the preview iframe and the API server into
+ * starting the agent-proxy, both of which then hit their own (much
+ * shorter) timeouts and surface as the "Connection timed out — The
+ * agent runtime could not be reached" toast and "[AgentProxy] PATCH
+ * /agent/config timeout" retries. Waiting for a real 2xx is slower
+ * but the resulting `'running'` status actually means "responsive to
+ * HTTP", which is what every caller of `status === 'running'` already
+ * assumes. The progress logger keeps devs informed during the slow
+ * path so the wait isn't a silent black-box.
+ */
+const HEALTH_BOOT_TIMEOUT_MS = 30_000;
+/**
+ * If `STDOUT_PROGRESS_WINDOW_MS` elapses without any new stdout AND
+ * TCP-listening is true AND /health still hasn't responded, the
+ * child is considered wedged and the wait short-circuits with the
+ * "stdout silent" error so the restart loop has a chance to recover.
+ * Note: this is the ONLY purpose of the window now — it does not
+ * declare the runtime ready on its own. See HEALTH_BOOT_TIMEOUT_MS.
+ *
+ * Sized at 25 s — short of the 30 s HEALTH_BOOT_TIMEOUT_MS so a
+ * wedged child surfaces with the more informative "stdout silent for
+ * Xms" message instead of the generic timeout. After the 2026-05
+ * cold-boot investigation we deferred LSP startup + IndexEngine
+ * pre-warm out of the critical path and got Windows cold boot down
+ * to ~8-12 s (with macOS at ~3-5 s), so a 25 s silent window is now
+ * a real anomaly worth respawning for.
  */
-const HEALTH_BOOT_TIMEOUT_MS = 60_000;
+const STDOUT_PROGRESS_WINDOW_MS = 25_000;
 /**
- * If the child's TCP listener is up AND its stdout has emitted a line
- * within this window, we accept it as ready even though /health
- * hasn't returned yet. Models "is the process making forward
- * progress?" — a Bun runtime mid `optimizeDeps` / LSP-spawn / hook
- * registration is unresponsive to HTTP but still emits `[LSP-TS]`,
- * `[preview-manager]`, `[AgentGateway]` etc. log lines as it walks
- * its boot sequence, so stdout activity is a reliable liveness
- * signal. A truly-hung child (event-loop wedged, zero log output)
- * stays silent past this window and falls through to the normal
- * /health timeout + SIGTERM + restart loop.
+ * Log a "still waiting for /health" progress line at most this
+ * often, so operators / devs watching `[dev:all]` see something
+ * happening during the slow Windows cold-boot path instead of a
+ * silent 30-90s gap.
  */
-const STDOUT_PROGRESS_WINDOW_MS = 10_000;
+const HEALTH_PROGRESS_LOG_MS = 5_000;
 /**
  * Per-attempt TCP connect budget for the kernel-level readiness
  * probe. Connect attempts only need a TCP SYN/SYN-ACK roundtrip on
@@ -1384,38 +1404,51 @@ export class WorkerRuntimeManager implements RuntimeResolver {
   }
 
   /**
-   * Wait for a freshly-spawned agent-runtime to clear the boot gate.
+   * Wait for a freshly-spawned agent-runtime to be HTTP-responsive on
+   * /health. This is the gate that drives the slot's `'starting'` →
+   * `'running'` transition; every consumer of `status === 'running'`
+   * (`/sandbox/url`, AgentProxy, the iframe preview readiness probe)
+   * assumes the runtime can actually serve requests, so this gate
+   * must mean exactly that — not "the kernel accepted a TCP listener
+   * on the port".
+   *
+   * Loop semantics:
+   *
+   *   1. **HTTP /health 2xx → return.** Happy path; the event loop
+   *      is responsive and the slot can transition to `'running'`.
    *
-   * Three independent readiness signals, in priority order:
+   *   2. **Child exits → throw.** Same as before — surface the
+   *      exit code/signal so restart-with-backoff can recover from
+   *      crashes-during-boot without burning the full timeout.
    *
-   *   1. **HTTP /health returns 2xx.** Happy path — the runtime is
-   *      fully booted and its event loop is responsive. We return
-   *      immediately and the caller transitions the slot to
-   *      `'running'`.
+   *   3. **TCP listening + stdout silent > {@link STDOUT_PROGRESS_WINDOW_MS}
+   *      → throw.** TCP up but no log lines for 30s is "wedged":
+   *      Bun is past `Bun.serve()` (so the kernel has the port) but
+   *      something inside the runtime spun the event loop to death
+   *      (infinite loop, deadlock). Bail so the restart loop can
+   *      SIGTERM and respawn instead of spinning the full timeoutMs.
    *
-   *   2. **TCP listener bound + recent stdout activity.** The kernel
-   *      has the port (so `Bun.serve()` evaluated its default export)
-   *      AND the child has printed a log line in the last
-   *      {@link STDOUT_PROGRESS_WINDOW_MS}. The process is alive,
-   *      bound, and making forward progress — just slow because Bun
-   *      is JIT-compiling the rest of the TS dep graph (the
-   *      `--conditions=development` cold-boot pattern) and /health
-   *      hasn't gotten a turn on the event loop yet. Accepting this
-   *      as ready unblocks the AgentProxy retry storm that would
-   *      otherwise compound the saturation. Once the child starts
-   *      handling requests for real, /health will respond and the
-   *      AgentProxy retries will succeed independently of this gate.
+   *   4. **`timeoutMs` elapsed → throw.** Final ceiling. After the
+   *      2026-05 fix that deferred LSP + IndexEngine out of the
+   *      critical-path boot sequence in agent-runtime, a healthy
+   *      Windows cold boot is ~8-12 s; the 30 s budget gives a
+   *      comfortable margin without hiding real hangs.
    *
-   *   3. **Hard timeout at `timeoutMs`.** Neither HTTP nor the
-   *      TCP-plus-progress fallback has cleared. Throw the
-   *      descriptive "Timeout waiting for agent-runtime /health" so
-   *      the restart-with-backoff loop in `handleExit()` SIGTERMs the
-   *      child and respawns.
+   * Progress logging: every {@link HEALTH_PROGRESS_LOG_MS} we emit a
+   * single line summarizing where the wait stands so the dev/operator
+   * watching `[dev:all]` sees something happen during the slow path
+   * instead of staring at a silent terminal for a minute.
    *
-   * Process-death short-circuit applies throughout: if the child exits
-   * mid-wait (e.g. spawn-time port conflict, missing native binding),
-   * we throw immediately with the exit code/signal rather than spinning
-   * for the full `timeoutMs`.
+   * What changed vs the pre-2026-05 fast path: that revision returned
+   * as soon as TCP-listening + recent stdout was true, treating "the
+   * child is making progress" as ready. In practice the child was
+   * still saturated for tens of seconds after that point, so the UI
+   * would start its preview iframe load and the API server its agent
+   * proxy, both of which then hit their own (4-15s) timeouts and
+   * surfaced as the "Connection timed out — The agent runtime could
+   * not be reached" toast plus "[AgentProxy] timeout, retrying" log
+   * spam. The new gate trades 20-60s of additional boot wait for a
+   * `'running'` signal the rest of the stack can actually trust.
    */
   private async waitForHealth(slot: InternalRuntime, timeoutMs: number): Promise<void> {
     const port = slot.agentPort;
@@ -1427,9 +1460,11 @@ export class WorkerRuntimeManager implements RuntimeResolver {
     const deadline = startedAt + timeoutMs;
     let lastError: string | null = null;
     let lastTcpListening = false;
+    let lastTcpAt = 0;
     let httpAttempts = 0;
     let tcpAttempts = 0;
     let iteration = 0;
+    let lastProgressLogAt = startedAt;
 
     while (Date.now() < deadline) {
       iteration++;
@@ -1439,7 +1474,9 @@ export class WorkerRuntimeManager implements RuntimeResolver {
         );
       }
 
-      // Signal 1: HTTP /health (happy path).
+      // Primary signal: HTTP /health. This is the only signal that
+      // declares the runtime ready — TCP-listening alone is not enough
+      // (see class docstring above for the rationale).
       httpAttempts++;
       const controller = new AbortController();
       const t = setTimeout(() => controller.abort(), 1500);
@@ -1457,8 +1494,7 @@ export class WorkerRuntimeManager implements RuntimeResolver {
           return;
         }
         // Non-2xx is recorded but doesn't short-circuit — the runtime
-        // may briefly serve 503 while initializing post-bind, and the
-        // TCP-progress fallback below covers that window too.
+        // may briefly serve 503 while initializing post-bind.
         lastError = `HTTP /health returned ${resp.status}`;
       } catch (err: any) {
         clearTimeout(t);
@@ -1467,23 +1503,37 @@ export class WorkerRuntimeManager implements RuntimeResolver {
         lastError = `HTTP /health failed: ${name}${code ? `(${code})` : ''}: ${err?.message ?? err}`;
       }
 
-      // Signal 2: TCP listener bound + recent stdout activity.
+      // Secondary signal: TCP listener + stdout activity. Now used ONLY
+      // as a wedge detector — if TCP is up but stdout has been silent
+      // for the progress window, abandon the wait so the restart loop
+      // can recover. Never declares the runtime ready on its own.
       tcpAttempts++;
       lastTcpListening = await this.tcpProbe(port);
       if (lastTcpListening) {
+        lastTcpAt = Date.now();
         const sinceStdoutMs = Date.now() - slot.lastStdoutAt;
-        if (sinceStdoutMs < STDOUT_PROGRESS_WINDOW_MS) {
-          this.log.log(
-            `[WorkerRuntimeManager] TCP-listening + stdout-active for ${slot.projectId} on ` +
-              `port ${port} (last stdout ${sinceStdoutMs}ms ago, ` +
-              `${Date.now() - startedAt}ms since spawn, ${httpAttempts} http, ${tcpAttempts} tcp). ` +
-              `Accepting as ready — /health still warming up but the child is bound and making progress.`,
+        if (sinceStdoutMs >= STDOUT_PROGRESS_WINDOW_MS) {
+          throw new Error(
+            `agent-runtime wedged on port ${port}: TCP listening but stdout silent for ` +
+              `${sinceStdoutMs}ms (> ${STDOUT_PROGRESS_WINDOW_MS}ms window); ` +
+              `${httpAttempts} /health attempts, last error: ${lastError ?? 'n/a'}`,
           );
-          return;
         }
-        lastError =
-          `TCP listening but stdout silent for ${sinceStdoutMs}ms ` +
-          `(> ${STDOUT_PROGRESS_WINDOW_MS}ms window); last http: ${lastError ?? 'n/a'}`;
+      }
+
+      const now = Date.now();
+      if (now - lastProgressLogAt >= HEALTH_PROGRESS_LOG_MS) {
+        const elapsedMs = now - startedAt;
+        const sinceStdoutMs = now - slot.lastStdoutAt;
+        const sinceTcpMs = lastTcpAt > 0 ? now - lastTcpAt : null;
+        this.log.log(
+          `[WorkerRuntimeManager] still waiting for /health on ${slot.projectId} ` +
+            `port ${port} (${(elapsedMs / 1000).toFixed(1)}s elapsed, ` +
+            `tcpListening=${lastTcpListening}${sinceTcpMs != null ? `(${sinceTcpMs}ms ago)` : ''}, ` +
+            `lastStdout=${sinceStdoutMs}ms ago, ${httpAttempts} http, ${tcpAttempts} tcp, ` +
+            `lastError=${lastError ?? 'n/a'})`,
+        );
+        lastProgressLogAt = now;
       }
 
       await new Promise((r) => setTimeout(r, HEALTH_POLL_MS));