@shogo-ai/worker 2.3.8 → 2.3.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shogo-ai/worker",
-  "version": "2.3.8",
+  "version": "2.3.10",
   "description": "Shogo Cloud Agent Worker — run Shogo agents on your own machine (laptop, devbox, CI).",
   "license": "MIT",
   "author": "Shogo Technologies, Inc.",
@@ -19,6 +19,7 @@
     ".": "./src/cli.ts",
     "./tunnel": "./src/lib/tunnel.ts",
     "./runtime-manager": "./src/lib/runtime-manager.ts",
+    "./runtime-activity": "./src/lib/runtime-activity.ts",
     "./resource-limits": "./src/lib/resource-limits.ts",
     "./cloud-sync-watcher": "./src/lib/cloud-sync-watcher.ts",
     "./git-cloner": "./src/lib/git-cloner.ts",

package/src/lib/__tests__/runtime-activity.test.ts

@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: MIT
+// Copyright (C) 2026 Shogo Technologies, Inc.
+/**
+ * The eviction verdict, behaviour-first (SHO-143).
+ *
+ * The rule these pin: a runtime is only evicted when the host can PROVE it is
+ * idle. Anything unverifiable — no sample, a failed probe, a sample older than
+ * the staleness window, a runtime still starting — keeps it. A Windows desktop
+ * evicted a background runtime 146 times in 24h with 0 keeps; every one of
+ * those came from "could not tell" being read as "idle".
+ */
+import { describe, expect, it } from 'bun:test';
+import {
+  ACTIVITY_STALE_MS,
+  DEAD_PROBE_MS,
+  STARTUP_GRACE_MS,
+  decideBusy,
+  type ActivitySample,
+} from '../runtime-activity.ts';
+
+const NOW = 1_800_000_000_000;
+const OLD_START = NOW - 30 * 60_000;
+
+function sample(over: Partial<ActivitySample> = {}, counts = { turns: 0, tasks: 0, streams: 0, children: 0 }): ActivitySample {
+  return { counts, at: NOW - 5_000, ok: true, ...over };
+}
+
+describe('decideBusy — unknown is busy', () => {
+  it('keeps a runtime with no sample yet', () => {
+    const d = decideBusy({ now: NOW, startedAt: OLD_START, status: 'running', sample: null });
+    expect(d.busy).toBe(true);
+    expect(d.dead).toBe(false);
+    expect(d.reason).toBe('unknown');
+  });
+
+  it('keeps a runtime whose last probe failed', () => {
+    const d = decideBusy({
+      now: NOW,
+      startedAt: OLD_START,
+      status: 'running',
+      sample: sample({ ok: false, error: 'no answer', failingSince: NOW - 5_000 }),
+    });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('unknown');
+  });
+
+  it('keeps a runtime whose sample has aged out of the staleness window', () => {
+    const d = decideBusy({
+      now: NOW,
+      startedAt: OLD_START,
+      status: 'running',
+      sample: sample({ at: NOW - ACTIVITY_STALE_MS - 1_000 }),
+    });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('stale');
+  });
+
+  it('treats a status it cannot vouch for as busy, not idle', () => {
+    const d = decideBusy({ now: NOW, startedAt: OLD_START, status: 'stopping', sample: sample() });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('unknown');
+  });
+});
+
+describe('decideBusy — real work is busy', () => {
+  it.each([
+    ['turns', { turns: 1, tasks: 0, streams: 0, children: 0 }],
+    ['tasks', { turns: 0, tasks: 2, streams: 0, children: 0 }],
+    ['streams', { turns: 0, tasks: 0, streams: 1, children: 0 }],
+    ['children', { turns: 0, tasks: 0, streams: 0, children: 1 }],
+  ])('keeps a runtime with %s in flight', (_label, counts) => {
+    const d = decideBusy({ now: NOW, startedAt: OLD_START, status: 'running', sample: sample({}, counts) });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('work');
+  });
+
+  it('keeps a runtime with an open terminal even when everything else is idle', () => {
+    const d = decideBusy({ now: NOW, startedAt: OLD_START, status: 'running', sample: sample(), terminalSessions: 1 });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('terminal');
+  });
+
+  it('keeps a runtime with a project call in flight', () => {
+    const d = decideBusy({ now: NOW, startedAt: OLD_START, status: 'running', sample: sample(), inflightCalls: 1 });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('inflight-call');
+  });
+
+  it('keeps a runtime that is still starting, whatever the sample says', () => {
+    const d = decideBusy({ now: NOW, startedAt: NOW - 1_000, status: 'starting', sample: sample() });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('grace');
+  });
+
+  it('keeps a freshly started runtime for the whole startup grace', () => {
+    const d = decideBusy({
+      now: NOW,
+      startedAt: NOW - (STARTUP_GRACE_MS - 1_000),
+      status: 'running',
+      sample: sample(),
+    });
+    expect(d.busy).toBe(true);
+    expect(d.reason).toBe('grace');
+  });
+
+  it('counts the log line for a busy runtime', () => {
+    const d = decideBusy({
+      now: NOW,
+      startedAt: OLD_START,
+      status: 'running',
+      sample: sample({}, { turns: 0, tasks: 2, streams: 0, children: 1 }),
+    });
+    expect(d.countsText).toBe('turns=0 tasks=2 streams=0 children=1');
+  });
+});
+
+describe('decideBusy — only proof of death reaps', () => {
+  it('calls an exited process dead, not busy', () => {
+    const d = decideBusy({ now: NOW, startedAt: OLD_START, status: 'running', processExited: true });
+    expect(d.dead).toBe(true);
+    expect(d.busy).toBe(false);
+  });
+
+  it('keeps a runtime that refuses connections briefly, then calls it dead', () => {
+    const refusedJustNow = decideBusy({
+      now: NOW,
+      startedAt: OLD_START,
+      status: 'running',
+      sample: sample({ ok: false, failingSince: NOW - 10_000 }),
+    });
+    expect(refusedJustNow.busy).toBe(true);
+    expect(refusedJustNow.dead).toBe(false);
+
+    const refusedForAges = decideBusy({
+      now: NOW,
+      startedAt: OLD_START,
+      status: 'running',
+      sample: sample({ ok: false, failingSince: NOW - DEAD_PROBE_MS - 1_000 }),
+    });
+    expect(refusedForAges.dead).toBe(true);
+  });
+});
+
+describe('decideBusy — idle is the only evictable answer', () => {
+  it('reports idle with zero counts and a fresh sample', () => {
+    const d = decideBusy({ now: NOW, startedAt: OLD_START, status: 'running', sample: sample() });
+    expect(d.busy).toBe(false);
+    expect(d.dead).toBe(false);
+    expect(d.reason).toBe('idle');
+    expect(d.countsText).toBe('turns=0 tasks=0 streams=0 children=0');
+  });
+});

package/src/lib/__tests__/runtime-manager-active-work.test.ts

@@ -54,7 +54,10 @@ function insertSlot(mgr: WorkerRuntimeManager, projectId: string, extra: Record<
     status: 'running',
     proc: fakeProc(),
     pid: null,
-    startedAt: Date.now(),
+    // Ten minutes up, i.e. past the 2-minute startup grace: a runtime that just
+    // started is never an eviction victim (SHO-143), and these tests are about
+    // what happens once it is settled.
+    startedAt: Date.now() - 10 * 60_000,
     lastUsedAt: Date.now(),
     restarts: 0,
     consecutiveFailures: 0,
@@ -125,14 +128,23 @@ describe('WorkerRuntimeManager active-work gating', () => {
     expect(slot.status).toBe('running');
     expect(slot.idleTimer).not.toBeNull();
 
-    activity = {};
+    // A current runtime always reports its counts; zeros are what "idle" means.
+    activity = { activeTurns: 0, activeTasks: 0, activeStreams: 0 };
+    // The reaper decides from the cached sample and only re-asks once that
+    // sample has aged out (SHO-143: never probe because we just decided to
+    // kill). Age it the way the sampler would on its next tick.
+    ;(slot as unknown as { activity?: { at: number } }).activity = {
+      counts: { turns: 1, tasks: 0, streams: 0, children: 0 },
+      at: Date.now() - 60_000,
+      ok: true,
+    };
     await sleep(200);
     expect(gone).toEqual(['busy']);
   });
 
   it('LRU cap skips busy runtimes and evicts the idle one', async () => {
     const busy = activityServer(() => ({ activeTasks: 1 }));
-    const idle = activityServer(() => ({}));
+    const idle = activityServer(() => ({ activeTurns: 0, activeTasks: 0, activeStreams: 0 }));
     const gone: string[] = [];
     const mgr = new WorkerRuntimeManager({
       logger: SILENT,

package/src/lib/__tests__/runtime-manager-busy-guard.test.ts

@@ -0,0 +1,363 @@
+// SPDX-License-Identifier: MIT
+// Copyright (C) 2026 Shogo Technologies, Inc.
+/**
+ * The cap must never be paid for with someone's work (SHO-143).
+ *
+ * Before this, the LRU asked the runtime "are you busy?" with a 3-second HTTP
+ * probe at the exact moment it wanted to kill it, and read a timeout as "no".
+ * On a host busy with builds that meant every victim looked idle: 146
+ * background evictions in 24h, 0 keeps. These tests make the probe hang, fail
+ * and lie, and require the runtime to survive each one.
+ */
+import { afterEach, describe, expect, it } from 'bun:test';
+import { WorkerRuntimeManager } from '../runtime-manager.ts';
+import { ZERO_COUNTS } from '../runtime-activity.ts';
+
+type Line = string;
+const servers: Array<{ stop: (force?: boolean) => void }> = [];
+afterEach(() => {
+  for (const s of servers.splice(0)) s.stop(true);
+});
+
+function logs() {
+  const lines: Line[] = [];
+  const logger = {
+    log: (m: string) => lines.push(m),
+    warn: (m: string) => lines.push(m),
+    error: (m: string) => lines.push(m),
+  };
+  return { lines, logger };
+}
+
+function fakeProc() {
+  const proc = {
+    exitCode: null as number | null,
+    signalCode: null,
+    killed: false,
+    pid: 4242,
+    kill() {
+      proc.killed = true;
+      return true;
+    },
+    once() {
+      return proc;
+    },
+    on() {
+      return proc;
+    },
+  };
+  return proc;
+}
+
+function insertSlot(mgr: WorkerRuntimeManager, projectId: string, extra: Record<string, unknown> = {}) {
+  const slot = {
+    projectId,
+    agentPort: 0,
+    apiServerPort: 0,
+    status: 'running',
+    proc: fakeProc(),
+    pid: 4242,
+    startedAt: Date.now() - 10 * 60_000,
+    lastUsedAt: Date.now() - 10 * 60_000,
+    restarts: 0,
+    consecutiveFailures: 0,
+    lastFailureAt: 0,
+    restartTimestamps: [] as number[],
+    breakerTrips: 0,
+    restartTimer: null as ReturnType<typeof setTimeout> | null,
+    idleTimer: null as ReturnType<typeof setTimeout> | null,
+    graceTimer: null,
+    limitWatchdog: null,
+    spawnConfig: {} as never,
+    startPromise: null,
+    runtimeToken: 'tok',
+    ...extra,
+  };
+  (mgr as unknown as { runtimes: Map<string, typeof slot> }).runtimes.set(projectId, slot);
+  return slot;
+}
+
+/** An activity endpoint that answers correctly, counts calls, or hangs. */
+function activityServer(opts: {
+  response?: () => Record<string, unknown>;
+  hangMs?: number;
+  status?: number;
+  token?: string;
+}) {
+  const seen: number[] = [];
+  const server = Bun.serve({
+    port: 0,
+    hostname: '127.0.0.1',
+    async fetch(req) {
+      seen.push(Date.now());
+      if ((req.headers.get('x-runtime-token') ?? '') !== (opts.token ?? 'tok')) {
+        return new Response('unauthorized', { status: 401 });
+      }
+      if (opts.hangMs) await new Promise((r) => setTimeout(r, opts.hangMs));
+      if (opts.status && opts.status >= 400) return new Response('boom', { status: opts.status });
+      return Response.json({
+        idleSeconds: 0,
+        activeSessions: 1,
+        activeTurns: 0,
+        activeTasks: 0,
+        activeStreams: 0,
+        activeChildren: 0,
+        ...(opts.response?.() ?? {}),
+      });
+    },
+  });
+  servers.push(server);
+  return { port: server.port as number, seen };
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+describe('WorkerRuntimeManager — a busy runtime is never the LRU victim', () => {
+  it('does not evict a runtime whose activity probe hangs (the Samsara case)', async () => {
+    // The probe never answers inside the 3s budget, exactly like a host
+    // saturated by Unreal builds. The runtime is running a task.
+    const hung = activityServer({ hangMs: 6_000 });
+    const idle = activityServer({ response: () => ({}) });
+    const { lines, logger } = logs();
+    const gone: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, maxRuntimes: 1, onRuntimeGone: (id) => gone.push(id) });
+
+    insertSlot(mgr, 'busy-hung-probe', { agentPort: hung.port, lastUsedAt: Date.now() - 20 * 60_000 });
+    insertSlot(mgr, 'settled-idle', { agentPort: idle.port, lastUsedAt: Date.now() - 10 * 60_000 });
+    insertSlot(mgr, 'just-started');
+
+    (mgr as unknown as { enforceMaxRuntimes: (k: string) => void }).enforceMaxRuntimes('just-started');
+    // The pass samples every candidate in parallel, so it settles when the
+    // hung probe hits its 3s budget.
+    await sleep(3_800);
+
+    // The runtime we could not question is kept; the one we could question and
+    // that answered "nothing running" is the one that goes.
+    expect(gone).toEqual(['settled-idle']);
+    expect(lines.some((l) => l.includes('keeping busy-hung-probe') && l.includes('unknown'))).toBe(true);
+    expect(lines.some((l) => l.includes('LRU-evicting settled-idle') && l.includes('turns=0'))).toBe(true);
+  }, 15_000);
+
+  it('does not evict a runtime whose activity endpoint reports no task counts (older runtime)', async () => {
+    // An older runtime answers liveness only. Zeros would make it look idle, so
+    // the answer is treated as unknown and the runtime is kept.
+    const legacy = activityServer({
+      response: () => ({
+        activeTurns: undefined,
+        activeTasks: undefined,
+        activeStreams: undefined,
+        activeChildren: undefined,
+      }),
+    });
+    const { lines, logger } = logs();
+    const gone: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, maxRuntimes: 1, onRuntimeGone: (id) => gone.push(id) });
+    insertSlot(mgr, 'legacy', { agentPort: legacy.port, lastUsedAt: Date.now() - 20 * 60_000 });
+    insertSlot(mgr, 'new');
+
+    (mgr as unknown as { enforceMaxRuntimes: (k: string) => void }).enforceMaxRuntimes('new');
+    await sleep(300);
+
+    expect(gone).toEqual([]);
+    expect(lines.some((l) => l.includes('keeping legacy') && l.includes('unknown'))).toBe(true);
+  }, 15_000);
+
+  it('does not evict a runtime whose activity endpoint answers 500', async () => {
+    const broken = activityServer({ status: 500 });
+    const { logger } = logs();
+    const gone: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, maxRuntimes: 1, onRuntimeGone: (id) => gone.push(id) });
+    insertSlot(mgr, 'broken', { agentPort: broken.port, lastUsedAt: Date.now() - 20 * 60_000 });
+    insertSlot(mgr, 'new');
+
+    (mgr as unknown as { enforceMaxRuntimes: (k: string) => void }).enforceMaxRuntimes('new');
+    await sleep(300);
+
+    expect(gone).toEqual([]);
+  });
+
+  it('keeps a runtime that is within its startup grace, before any sample exists', async () => {
+    const { logger } = logs();
+    const gone: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, onRuntimeGone: (id) => gone.push(id) });
+    const slot = insertSlot(mgr, 'booting', { status: 'starting', startedAt: Date.now(), agentPort: 0 });
+
+    const decision = mgr.activityDecision('booting');
+    expect(decision.busy).toBe(true);
+    expect(decision.reason).toBe('grace');
+    expect(gone).toEqual([]);
+    expect(slot.status).toBe('starting');
+  });
+
+  it('keeps a runtime mid-turn — the regression for 146 evictions / 0 keeps', async () => {
+    const midTurn = activityServer({ response: () => ({ activeTurns: 1 }) });
+    const { logger } = logs();
+    const gone: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, maxRuntimes: 1, onRuntimeGone: (id) => gone.push(id) });
+    insertSlot(mgr, 'mid-turn', { agentPort: midTurn.port, lastUsedAt: Date.now() - 20 * 60_000 });
+    insertSlot(mgr, 'new');
+
+    (mgr as unknown as { enforceMaxRuntimes: (k: string) => void }).enforceMaxRuntimes('new');
+    await sleep(300);
+
+    expect(gone).toEqual([]);
+  });
+});
+
+describe('WorkerRuntimeManager — the decision reads a cached sample', () => {
+  it('sampling on the timer, not at eviction time, is what the verdict uses', async () => {
+    const server = activityServer({ response: () => ({ activeTasks: 1, activeChildren: 1 }) });
+    const { logger } = logs();
+    const mgr = new WorkerRuntimeManager({ logger });
+    const slot = insertSlot(mgr, 'p', { agentPort: server.port });
+
+    await mgr.sampleAllActivity();
+    const afterSample = server.seen.length;
+
+    const decision = mgr.activityDecision('p');
+    expect(decision.busy).toBe(true);
+    expect(decision.reason).toBe('work');
+    expect(decision.countsText).toBe('turns=0 tasks=1 streams=0 children=1');
+    // No probe at decision time at all.
+    expect(server.seen.length).toBe(afterSample);
+    expect(slot.activity?.ok).toBe(true);
+  });
+
+  it('records a failed sample as unknown rather than as zero work', async () => {
+    const server = activityServer({ hangMs: 5_000 });
+    const { logger } = logs();
+    const mgr = new WorkerRuntimeManager({ logger });
+    const slot = insertSlot(mgr, 'p', { agentPort: server.port });
+
+    await mgr.sampleActivity(slot);
+
+    expect(slot.activity?.ok).toBe(false);
+    expect(mgr.activityDecision('p').busy).toBe(true);
+    expect(mgr.activityDecision('p').reason).toBe('unknown');
+  }, 10_000);
+
+  it('reaps a runtime whose process is gone as dead, not as an eviction', async () => {
+    const { logger } = logs();
+    const stopped: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, maxRuntimes: 1 });
+    insertSlot(mgr, 'dead', { proc: null, status: 'running' });
+    insertSlot(mgr, 'new');
+
+    const decision = mgr.activityDecision('dead');
+    expect(decision.dead).toBe(true);
+
+    const originalStop = (mgr as unknown as { stop: (id: string, signal?: string, reason?: string) => Promise<void> }).stop;
+    (mgr as unknown as { stop: (id: string, signal?: string, reason?: string) => Promise<void> }).stop = async (
+      id: string,
+      _signal?: string,
+      reason?: string,
+    ) => {
+      stopped.push(`${id}:${reason}`);
+      void originalStop;
+    };
+
+    (mgr as unknown as { enforceMaxRuntimes: (k: string) => void }).enforceMaxRuntimes('new');
+    await sleep(200);
+
+    expect(stopped).toEqual(['dead:dead']);
+  });
+});
+
+describe('WorkerRuntimeManager — a stale verdict is refreshed before it kills anything', () => {
+  it('re-probes a runtime whose cached "idle" aged out, and keeps it when the turn has started since', async () => {
+    // The timer sampled `turns=0` a minute ago; a turn started inside that gap.
+    // Evicting on the stale verdict is exactly the SHO-143 failure.
+    const server = activityServer({ response: () => ({ activeTurns: 1 }) });
+    const { lines, logger } = logs();
+    const gone: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, maxRuntimes: 1, onRuntimeGone: (id) => gone.push(id) });
+    insertSlot(mgr, 'stale-idle', {
+      agentPort: server.port,
+      lastUsedAt: Date.now() - 20 * 60_000,
+      activity: { counts: { ...ZERO_COUNTS }, at: Date.now() - 60_000, ok: true },
+    });
+    insertSlot(mgr, 'new');
+
+    (mgr as unknown as { enforceMaxRuntimes: (k: string) => void }).enforceMaxRuntimes('new');
+    await sleep(400);
+
+    expect(gone).toEqual([]);
+    expect(server.seen.length).toBe(1);
+    expect(lines.some((l) => l.includes('keeping stale-idle') && l.includes('turns=1'))).toBe(true);
+  }, 15_000);
+
+  it('does not probe a runtime whose verdict is fresh, so eviction never waits on the wire', async () => {
+    const server = activityServer({ response: () => ({}) });
+    const { logger } = logs();
+    const gone: string[] = [];
+    const mgr = new WorkerRuntimeManager({ logger, maxRuntimes: 1, onRuntimeGone: (id) => gone.push(id) });
+    insertSlot(mgr, 'fresh-idle', {
+      agentPort: server.port,
+      lastUsedAt: Date.now() - 20 * 60_000,
+      activity: { counts: { ...ZERO_COUNTS }, at: Date.now(), ok: true },
+    });
+    insertSlot(mgr, 'new');
+
+    (mgr as unknown as { enforceMaxRuntimes: (k: string) => void }).enforceMaxRuntimes('new');
+    await sleep(200);
+
+    expect(gone).toEqual(['fresh-idle']);
+    expect(server.seen.length).toBe(0);
+  });
+
+  it('refreshActivity re-reads only when the cached sample is stale', async () => {
+    const server = activityServer({ response: () => ({ activeTurns: 1 }) });
+    const { logger } = logs();
+    const mgr = new WorkerRuntimeManager({ logger });
+    const slot = insertSlot(mgr, 'p', {
+      agentPort: server.port,
+      activity: { counts: { ...ZERO_COUNTS }, at: Date.now(), ok: true },
+    });
+
+    await mgr.refreshActivity('p');
+    expect(server.seen.length).toBe(0);
+
+    slot.activity = { ...(slot.activity as { counts: typeof ZERO_COUNTS; ok: boolean }), at: Date.now() - 60_000 };
+    await mgr.refreshActivity('p');
+
+    expect(server.seen.length).toBe(1);
+    expect(mgr.activityDecision('p').reason).toBe('work');
+  }, 15_000);
+});
+
+describe('WorkerRuntimeManager — unexplained exits say why', () => {
+  it('logs the exit code, the uptime, the last activity and the stderr tail', async () => {
+    const { lines, logger } = logs();
+    const mgr = new WorkerRuntimeManager({ logger });
+    const slot = insertSlot(mgr, 'crasher');
+    (slot as unknown as { stderrTail: string[] }).stderrTail = ['FATAL heap out of memory', 'killed'];
+
+    (mgr as unknown as { handleExit: (s: unknown, c: number | null, sig: string | null) => void }).handleExit(
+      slot,
+      1,
+      null,
+    );
+
+    const line = lines.find((l) => l.includes('exited WITHOUT a stop request'));
+    expect(line).toBeDefined();
+    expect(line).toContain('code=1');
+    expect(line).toContain('last activity turns=0');
+    expect(line).toContain('FATAL heap out of memory');
+    expect(line).toContain('killed');
+  });
+
+  it('does not warn when the stop was ours', async () => {
+    const { lines, logger } = logs();
+    const mgr = new WorkerRuntimeManager({ logger });
+    const slot = insertSlot(mgr, 'stopping');
+    slot.status = 'stopping';
+
+    (mgr as unknown as { handleExit: (s: unknown, c: number | null, sig: string | null) => void }).handleExit(
+      slot,
+      0,
+      null,
+    );
+
+    expect(lines.some((l) => l.includes('exited WITHOUT a stop request'))).toBe(false);
+  });
+});

package/src/lib/__tests__/runtime-manager-idle-eviction.test.ts

@@ -71,8 +71,15 @@ function insertRunningSlot(mgr: WorkerRuntimeManager, projectId: string) {
     apiServerPort: 0,
     status: 'running' as const,
     proc,
-    startedAt: Date.now(),
+    // Settled, i.e. past the startup grace: a runtime that just booted is
+    // never an eviction victim (SHO-143).
+    startedAt: Date.now() - 10 * 60_000,
     lastUsedAt: Date.now(),
+    // The activity sampler has already run and last saw this runtime doing
+    // nothing — the only state the idle reaper may act on (SHO-143). Without a
+    // sample the verdict is `unknown`, which keeps the runtime.
+    activity: { counts: { turns: 0, tasks: 0, streams: 0, children: 0 }, at: Date.now(), ok: true },
+
     restarts: 0,
     restartTimer: null,
     idleTimer: null,

package/src/lib/__tests__/runtime-manager-on-runtime-gone.test.ts

@@ -32,8 +32,14 @@ function insertSlot(mgr: WorkerRuntimeManager, projectId: string, extra: Record<
     status: 'running',
     proc: makeFakeProc(),
     pid: null,
-    startedAt: Date.now(),
+    // Settled, i.e. past the startup grace: a runtime that just booted is
+    // never an eviction victim (SHO-143).
+    startedAt: Date.now() - 10 * 60_000,
     lastUsedAt: Date.now(),
+    // The activity sampler has already run and last saw this runtime doing
+    // nothing — the only state the idle reaper may act on (SHO-143). Without a
+    // sample the verdict is `unknown`, which keeps the runtime.
+    activity: { counts: { turns: 0, tasks: 0, streams: 0, children: 0 }, at: Date.now(), ok: true },
     restarts: 0,
     consecutiveFailures: 0,
     lastFailureAt: 0,

package/src/lib/__tests__/runtime-manager-vite-watch.e2e.test.ts

@@ -3,7 +3,8 @@
 
 import { afterEach, describe, expect, it } from 'bun:test'
 import { spawn, type ChildProcess } from 'node:child_process'
-import { mkdtempSync, rmSync, mkdirSync } from 'node:fs'
+import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs'
+import { execFileSync } from 'node:child_process'
 import { join } from 'node:path'
 import { WorkerRuntimeManager } from '../runtime-manager.ts'
 import { writeViteWatchPidfile } from '@shogo-ai/sdk/vite-watch'
@@ -60,4 +61,57 @@ describe.skipIf(!RUN_INTEGRATION)('WorkerRuntimeManager Vite watch cleanup e2e',
       new Promise((_, reject) => setTimeout(() => reject(new Error('watch process did not exit')), 2_000)),
     ])
   })
+
+  /**
+   * SHO-144: the pidfile only ever knew about vite. An exec shell and the
+   * preview API sidecar lead their own groups too, and a runtime that was
+   * SIGKILLed left them with no trail at all. The ledger is that trail; this
+   * asserts the boot sweep uses it, and that it refuses a reused pid.
+   */
+  it('reaps the child groups a dead runtime left in the ledger', async () => {
+    const projectsDir = mkdtempSync(join('/tmp', 'shogo-worker-ledger-e2e-'))
+    tempDirs.push(projectsDir)
+    const projectDir = join(projectsDir, 'project-ledger')
+    mkdirSync(join(projectDir, '.shogo', 'run'), { recursive: true })
+
+    const spawnSleeper = (label: string) =>
+      spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)', label], {
+        detached: true,
+        stdio: 'ignore',
+      })
+
+    const real = spawnSleeper('exec')
+    const reused = spawnSleeper('unrelated')
+    expect(real.pid).toBeDefined()
+    expect(reused.pid).toBeDefined()
+    real.unref()
+    reused.unref()
+    child = null
+    // Registered as group leaders so the ledger's `kill(-pgid)` has a group.
+    const realCommand = execFileSync('ps', ['-p', String(real.pid), '-o', 'command='], { encoding: 'utf8' }).trim()
+    writeFileSync(
+      join(projectDir, '.shogo', 'run', 'children.json'),
+      JSON.stringify([
+        // Ours: the command line still matches what was recorded.
+        { pgid: real.pid, pid: real.pid, label: 'exec', commandFingerprint: realCommand, startedAt: Date.now() },
+        // A pid that now belongs to something else: must never be signalled.
+        { pgid: reused.pid, pid: reused.pid, label: 'api-sidecar', commandFingerprint: 'bun run server.tsx', startedAt: Date.now() },
+      ]) + '\n',
+      'utf8',
+    )
+    const realExited = new Promise<void>((resolve) => real.once('exit', () => resolve()))
+
+    new WorkerRuntimeManager({
+      autoPull: { enabled: false, projectsDir },
+      logger: { log: () => {}, warn: () => {}, error: () => {} },
+    })
+
+    await Promise.race([
+      realExited,
+      new Promise((_, reject) => setTimeout(() => reject(new Error('ledger entry was not reaped')), 5_000)),
+    ])
+    // The unrelated process survives: a reused pid is dropped, never signalled.
+    expect(() => process.kill(reused.pid as number, 0)).not.toThrow()
+    reused.kill('SIGKILL')
+  })
 })