hub-runner 0.2.2 → 0.2.4

package/check.mjs

@@ -6,18 +6,18 @@ import { tmpdir } from "node:os";
 import { join as join3, resolve } from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 
-// ../../lib/clean-env.mjs
+// ../../hosted-hub/lib/clean-env.mjs
 var INHERITED_ENV = /^(EVE_|WORKFLOW_)/;
 function cleanEnv(env = process.env) {
   return Object.fromEntries(Object.entries(env).filter(([key]) => !INHERITED_ENV.test(key)));
 }
 
-// ../../lib/processes.mjs
-import { spawn, execFile, execFileSync } from "node:child_process";
-import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, readdirSync, statSync, writeFileSync, writeSync } from "node:fs";
+// ../../hosted-hub/lib/processes.mjs
+import { spawn } from "node:child_process";
+import { closeSync as closeSync2, existsSync, mkdirSync, openSync as openSync2, readFileSync, renameSync, rmSync, readdirSync, statSync as statSync2, writeFileSync, writeSync } from "node:fs";
 import { join } from "node:path";
 
-// ../../lib/fingerprint.mjs
+// ../../hosted-hub/lib/fingerprint.mjs
 var TREE_SKIP = /* @__PURE__ */ new Set(["node_modules", ".git", ".DS_Store"]);
 function fingerprintDiff(recorded, current) {
   if (!recorded?.hash) return "it was started without a fingerprint (by an older registry)";
@@ -36,22 +36,44 @@ function fingerprintDiff(recorded, current) {
 }
 var BUILD_SKIP = /* @__PURE__ */ new Set([...TREE_SKIP, ".eve", ".output"]);
 
-// ../../lib/processes.mjs
-var num = (key, fallback) => {
-  const raw = process.env[key];
-  return raw === void 0 || raw === "" || !Number.isFinite(Number(raw)) ? fallback : Number(raw);
-};
+// ../../hosted-hub/lib/log-tail.mjs
+import { closeSync, openSync, readSync, statSync } from "node:fs";
+function createLogTail(log) {
+  const tails = /* @__PURE__ */ new Map();
+  function follow(name, file, offset) {
+    tails.set(name, { file, offset, carry: "", prefix: `[${name}]`.padEnd(20) });
+  }
+  function pump() {
+    for (const tail of tails.values()) {
+      let size;
+      try {
+        size = statSync(tail.file).size;
+      } catch {
+        continue;
+      }
+      if (size < tail.offset) tail.offset = 0;
+      if (size === tail.offset) continue;
+      const length = Math.min(size - tail.offset, 1e6);
+      const buffer = Buffer.alloc(length);
+      const fd = openSync(tail.file, "r");
+      try {
+        readSync(fd, buffer, 0, length, tail.offset);
+      } finally {
+        closeSync(fd);
+      }
+      tail.offset += length;
+      const lines = (tail.carry + buffer.toString("utf8")).split("\n");
+      tail.carry = lines.pop();
+      for (const line of lines) if (line.trim() && !line.startsWith("Migration file of version")) log(`${tail.prefix} ${line}`);
+    }
+  }
+  return { follow, pump };
+}
+
+// ../../hosted-hub/lib/process-tree.mjs
+import { execFile, execFileSync } from "node:child_process";
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-var CRASH_LIMIT = num("ROSTER_CRASH_LIMIT", 5);
-var CRASH_WINDOW_MS = num("ROSTER_CRASH_WINDOW_MS", 10 * 6e4);
-var BACKOFF_BASE_MS = num("ROSTER_BACKOFF_BASE_MS", 1e3);
-var BACKOFF_MAX_MS = num("ROSTER_BACKOFF_MAX_MS", 6e4);
-var HEALTH_TIMEOUT_MS = num("ROSTER_START_HEALTH_TIMEOUT_MS", 12e4);
-var PORT_RECHECK_MS = num("ROSTER_PORT_RECHECK_MS", 5e3);
 var KILL_GRACE_MS = 3e3;
-var WATCH_MS = 1e3;
-var TREE_MS = 5e3;
-var LOG_ROTATE_BYTES = 2e7;
 function groupAlive(pgid) {
   if (!(pgid > 1)) return false;
   try {
@@ -74,7 +96,7 @@ var C = { ...process.env, LC_ALL: "C" };
 var SLOW_MS = 1e3;
 function slow(what, started) {
   const ms = Date.now() - started;
-  if (ms >= SLOW_MS) console.warn(`[registry] ${what} took ${ms}ms`);
+  if (ms >= SLOW_MS) console.warn(`[runner] ${what} took ${ms}ms`);
 }
 var LSTART = String.raw`(\w{3}\s+\w{3}\s+\d+\s+[\d:]+\s+\d{4})`;
 function table() {
@@ -137,6 +159,73 @@ async function healthy(port, ms = 1e3) {
   }
 }
 var merge = (...lists) => [...new Map(lists.flat().map((m) => [m.pid, m])).values()];
+var treeAlive = (pgid, tree) => groupAlive(pgid) || liveMembers(tree, table()).length > 0;
+async function untilTreeGone(pgid, tree, ms) {
+  const deadline = Date.now() + ms;
+  while (treeAlive(pgid, tree)) {
+    if (Date.now() >= deadline) return false;
+    await sleep(200);
+  }
+  return true;
+}
+function killTree(pgid, tree) {
+  signalGroup(pgid, "SIGKILL");
+  for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) signalGroup(g, "SIGKILL");
+}
+async function stopTree(pgid, tree, { name, timeoutMs, warn }) {
+  if (!treeAlive(pgid, tree)) return true;
+  let membersTermed = false;
+  const termMembers = () => {
+    membersTermed = true;
+    for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) if (g !== pgid) signalGroup(g, "SIGTERM");
+  };
+  if (groupAlive(pgid)) signalGroup(pgid, "SIGTERM");
+  else termMembers();
+  const deadline = Date.now() + timeoutMs;
+  let gone = true;
+  while (treeAlive(pgid, tree)) {
+    if (!membersTermed && !groupAlive(pgid)) termMembers();
+    if (Date.now() >= deadline) {
+      gone = false;
+      break;
+    }
+    await sleep(200);
+  }
+  if (gone) return true;
+  warn(`[runner] ${name} didn't stop within ${timeoutMs / 1e3}s of SIGTERM; sending SIGKILL`);
+  killTree(pgid, tree);
+  return untilTreeGone(pgid, tree, KILL_GRACE_MS);
+}
+function ownsLeader(record, rows, signature) {
+  if (!(record.pid > 1) || record.pgid !== record.pid) return false;
+  const row = rows.find((r) => r.pid === record.pid);
+  if (!row || row.pgid !== record.pgid) return false;
+  if (!` ${row.command} `.includes(signature)) return false;
+  const member = record.tree?.find((m) => m.pid === record.pid);
+  if (member) return member.startedAt === row.startedAt;
+  const recorded = Date.parse(record.startedAt);
+  return !(Number.isFinite(row.startedAt) && Number.isFinite(recorded) && Math.abs(row.startedAt - recorded) > 5e3);
+}
+function ourTree(record, rows, signature) {
+  const leader = ownsLeader(record, rows, signature);
+  return { leader, tree: merge(leader ? treeOf(record.pid, rows) : [], liveMembers(record.tree, rows)) };
+}
+
+// ../../hosted-hub/lib/processes.mjs
+var num = (key, fallback) => {
+  const raw = process.env[key];
+  return raw === void 0 || raw === "" || !Number.isFinite(Number(raw)) ? fallback : Number(raw);
+};
+var sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
+var CRASH_LIMIT = num("ROSTER_CRASH_LIMIT", 5);
+var CRASH_WINDOW_MS = num("ROSTER_CRASH_WINDOW_MS", 10 * 6e4);
+var BACKOFF_BASE_MS = num("ROSTER_BACKOFF_BASE_MS", 1e3);
+var BACKOFF_MAX_MS = num("ROSTER_BACKOFF_MAX_MS", 6e4);
+var HEALTH_TIMEOUT_MS = num("ROSTER_START_HEALTH_TIMEOUT_MS", 12e4);
+var PORT_RECHECK_MS = num("ROSTER_PORT_RECHECK_MS", 5e3);
+var WATCH_MS = 1e3;
+var TREE_MS = 5e3;
+var LOG_ROTATE_BYTES = 2e7;
 function createSupervisor({
   root,
   // the workspace: eve dev's cwd
@@ -149,18 +238,12 @@ function createSupervisor({
   isCurrent = () => true,
   // is this agent object still on the roster?
   // ({ agent, env, argv }) -> what a server spawned now runs with
-  // (lib/fingerprint.mjs), or null. A previous registry's server is
+  // (lib/fingerprint.mjs), or null. A previous runner's server is
   // adopted only if its recorded fingerprint matches.
   fingerprint = null,
   onChange = () => {
   },
   // status changed (broadcast)
-  onUsage = () => {
-  },
-  // a running tree's memory changed (status() reports rssKb)
-  onRecovered = () => {
-  },
-  // healthy again after a crash restart
   onUp = () => {
   },
   // a server is serving: adopted, spawned, or respawned ({ startedAt, adopted, recovered })
@@ -188,10 +271,7 @@ function createSupervisor({
       status: e.status,
       ...e.tree?.length ? { tree: e.tree } : {},
       ...e.lastExit ? { lastExit: e.lastExit } : {},
-      ...e.fingerprint ? { fingerprint: e.fingerprint } : {},
-      // The daemon token it was spawned with (HUB_INSTANCE): an adopted bot
-      // keeps its spawner's, and lib/daemon.mjs selfStopRefusal checks it here.
-      ...e.instance ? { instance: e.instance } : {}
+      ...e.fingerprint ? { fingerprint: e.fingerprint } : {}
     }));
     mkdirSync(stateDir, { recursive: true });
     const tmp = `${recordsPath}.${process.pid}.tmp`;
@@ -212,34 +292,7 @@ function createSupervisor({
     persist();
     onChange(entry.name);
   }
-  const tails = /* @__PURE__ */ new Map();
-  function follow(name, file, offset) {
-    tails.set(name, { file, offset, carry: "", prefix: `[${name}]`.padEnd(20) });
-  }
-  function pump() {
-    for (const tail of tails.values()) {
-      let size;
-      try {
-        size = statSync(tail.file).size;
-      } catch {
-        continue;
-      }
-      if (size < tail.offset) tail.offset = 0;
-      if (size === tail.offset) continue;
-      const length = Math.min(size - tail.offset, 1e6);
-      const buffer = Buffer.alloc(length);
-      const fd = openSync(tail.file, "r");
-      try {
-        readSync(fd, buffer, 0, length, tail.offset);
-      } finally {
-        closeSync(fd);
-      }
-      tail.offset += length;
-      const lines = (tail.carry + buffer.toString("utf8")).split("\n");
-      tail.carry = lines.pop();
-      for (const line of lines) if (line.trim() && !line.startsWith("Migration file of version")) log(`${tail.prefix} ${line}`);
-    }
-  }
+  const { follow, pump } = createLogTail(log);
   const pumpTimer = setInterval(pump, 250);
   pumpTimer.unref();
   function refreshTree(entry, rows = table()) {
@@ -247,10 +300,8 @@ function createSupervisor({
     const before = JSON.stringify(entry.tree ?? []);
     entry.tree = merge(treeOf(entry.pid, rows), liveMembers(entry.tree, rows));
     const own = rows.filter((r) => r.pid === entry.pid || r.ppid === entry.pid);
-    const rss = entry.rssKb;
     if (own.length) entry.rssKb = own.reduce((sum, r) => sum + (r.rssKb || 0), 0);
     if (JSON.stringify(entry.tree) !== before) persist();
-    if (entry.rssKb !== rss) onUsage(entry.name);
   }
   const treeTimer = setInterval(() => {
     const live = [...entries.values()].filter((e) => e.pid && !e.stopping);
@@ -259,45 +310,8 @@ function createSupervisor({
     for (const e of live) refreshTree(e, rows);
   }, TREE_MS);
   treeTimer.unref();
-  const treeAlive = (pgid, tree) => groupAlive(pgid) || liveMembers(tree, table()).length > 0;
-  async function untilTreeGone(pgid, tree, ms) {
-    const deadline = Date.now() + ms;
-    while (treeAlive(pgid, tree)) {
-      if (Date.now() >= deadline) return false;
-      await sleep(200);
-    }
-    return true;
-  }
-  function killTree(pgid, tree) {
-    signalGroup(pgid, "SIGKILL");
-    for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) signalGroup(g, "SIGKILL");
-  }
-  async function stopTree(pgid, tree, name) {
-    if (!treeAlive(pgid, tree)) return true;
-    let membersTermed = false;
-    const termMembers = () => {
-      membersTermed = true;
-      for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) if (g !== pgid) signalGroup(g, "SIGTERM");
-    };
-    if (groupAlive(pgid)) signalGroup(pgid, "SIGTERM");
-    else termMembers();
-    const deadline = Date.now() + stopTimeoutMs;
-    let gone = true;
-    while (treeAlive(pgid, tree)) {
-      if (!membersTermed && !groupAlive(pgid)) termMembers();
-      if (Date.now() >= deadline) {
-        gone = false;
-        break;
-      }
-      await sleep(200);
-    }
-    if (gone) return true;
-    warn(`[registry] ${name} didn't stop within ${stopTimeoutMs / 1e3}s of SIGTERM; sending SIGKILL`);
-    killTree(pgid, tree);
-    return untilTreeGone(pgid, tree, KILL_GRACE_MS);
-  }
   const logFile = (name) => join(logDir, `${name}.log`);
-  const sizeOf = (file) => existsSync(file) ? statSync(file).size : 0;
+  const sizeOf = (file) => existsSync(file) ? statSync2(file).size : 0;
   function entryFor(agent) {
     let entry = entries.get(agent.name);
     if (!entry) {
@@ -308,33 +322,19 @@ function createSupervisor({
     entry.port = agent.port;
     return entry;
   }
-  function ownsLeader(record, rows) {
-    if (!(record.pid > 1) || record.pgid !== record.pid) return false;
-    const row = rows.find((r) => r.pid === record.pid);
-    if (!row || row.pgid !== record.pgid) return false;
-    if (!` ${row.command} `.includes(signature(record.agent, record.port))) return false;
-    const member = record.tree?.find((m) => m.pid === record.pid);
-    if (member) return member.startedAt === row.startedAt;
-    const recorded = Date.parse(record.startedAt);
-    return !(Number.isFinite(row.startedAt) && Number.isFinite(recorded) && Math.abs(row.startedAt - recorded) > 5e3);
-  }
-  function ourTree(record, rows) {
-    const leader = ownsLeader(record, rows);
-    return { leader, tree: merge(leader ? treeOf(record.pid, rows) : [], liveMembers(record.tree, rows)) };
-  }
   async function killStale(record, why) {
     const rows = table();
-    const { leader, tree } = ourTree(record, rows);
+    const { leader, tree } = ourTree(record, rows, signature(record.agent, record.port));
     if (!leader && !tree.length) {
-      if (groupAlive(record.pgid)) warn(`[registry] ! ${record.agent}: process group ${record.pgid} from processes.json is no longer its server; leaving it alone`);
+      if (groupAlive(record.pgid)) warn(`[runner] ! ${record.agent}: process group ${record.pgid} from processes.json is no longer its server; leaving it alone`);
       return true;
     }
-    warn(`[registry] ${record.agent}: stopping its stale server (pid ${record.pid}): ${why}`);
-    return stopTree(leader ? record.pgid : null, tree, record.agent);
+    warn(`[runner] ${record.agent}: stopping its stale server (pid ${record.pid}): ${why}`);
+    return stopTree(leader ? record.pgid : null, tree, { name: record.agent, timeoutMs: stopTimeoutMs, warn });
   }
   async function vetAdoption(agent, record) {
     const rows = table();
-    const { leader, tree } = ourTree(record, rows);
+    const { leader, tree } = ourTree(record, rows, signature(record.agent, record.port));
     if (!leader && !tree.length) return { adopted: false };
     if (!leader) return { adopted: false, stale: true, why: "its leader is gone" };
     if (record.port !== agent.port) return { adopted: false, stale: true, why: `its port moved from ${record.port} to ${agent.port}` };
@@ -366,7 +366,7 @@ function createSupervisor({
     const { tree } = vetted;
     if (closed) return carry(agent, record, tree);
     const entry = entryFor(agent);
-    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: "up", adopted: true, child: null, stopping: false, error: void 0, tree, fingerprint: record.fingerprint, instance: record.instance, rssKb: void 0 });
+    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: "up", adopted: true, child: null, stopping: false, error: void 0, tree, fingerprint: record.fingerprint, rssKb: void 0 });
     const gen = ++entry.gen;
     follow(agent.name, logFile(agent.name), sizeOf(logFile(agent.name)));
     entry.watch = setInterval(() => {
@@ -377,14 +377,14 @@ function createSupervisor({
       }
     }, WATCH_MS);
     entry.watch.unref();
-    log(`[registry] adopted ${agent.name} (pid ${record.pid}, port ${agent.port})`);
+    log(`[runner] adopted ${agent.name} (pid ${record.pid}, port ${agent.port})`);
     changed(entry);
     onUp(agent, { startedAt: entry.startedAt, adopted: true, recovered: false });
     return { adopted: true };
   }
   function carry(agent, record, tree = record.tree ?? []) {
     const entry = entryFor(agent);
-    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: record.status ?? "up", adopted: true, child: null, stopping: false, tree, fingerprint: record.fingerprint, instance: record.instance });
+    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: record.status ?? "up", adopted: true, child: null, stopping: false, tree, fingerprint: record.fingerprint });
     entry.gen++;
     return { adopted: true, carried: true };
   }
@@ -410,7 +410,7 @@ function createSupervisor({
         return spawnAgent(entry, { recovered });
       }
       const error = `port ${port} is held by another process (${holders.map((pid) => describe(pid, rows)).join("; ")}), not by ${name}'s server; not touching it. ${name} starts once the port is free.`;
-      if (entry.error !== error || entry.status !== "blocked") warn(`[registry] ! ${error}`);
+      if (entry.error !== error || entry.status !== "blocked") warn(`[runner] ! ${error}`);
       Object.assign(entry, { status: "blocked", error, pid: null, pgid: null });
       entry.timer = setTimeout(() => spawnAgent(entry, { recovered }), PORT_RECHECK_MS);
       entry.timer.unref();
@@ -419,7 +419,7 @@ function createSupervisor({
     mkdirSync(logDir, { recursive: true });
     const file = logFile(name);
     if (sizeOf(file) > LOG_ROTATE_BYTES) renameSync(file, `${file}.1`);
-    const fd = openSync(file, "a");
+    const fd = openSync2(file, "a");
     writeSync(fd, `
 --- ${(/* @__PURE__ */ new Date()).toISOString()} hub: starting ${name} on port ${port}
 `);
@@ -429,7 +429,7 @@ function createSupervisor({
     try {
       fp = fingerprintFor(agent, spawnEnv);
     } catch (error) {
-      warn(`[registry] ! ${name}: couldn't fingerprint its inputs (${error.message}); the next registry respawns it instead of adopting it`);
+      warn(`[runner] ! ${name}: couldn't fingerprint its inputs (${error.message}); the next runner respawns it instead of adopting it`);
     }
     let child;
     try {
@@ -437,17 +437,17 @@ function createSupervisor({
         cwd: root,
         env: spawnEnv,
         detached: true,
-        // own process group; survives the registry
+        // own process group; survives the runner
         stdio: ["ignore", fd, fd]
       });
     } finally {
-      closeSync(fd);
+      closeSync2(fd);
     }
     const gen = ++entry.gen;
-    Object.assign(entry, { child, pid: child.pid ?? null, pgid: child.pid ?? null, startedAt: (/* @__PURE__ */ new Date()).toISOString(), status: "starting", adopted: false, stopping: false, error: void 0, nextRestartAt: void 0, tree: [], slowWarned: false, fingerprint: fp ?? void 0, instance: spawnEnv?.HUB_INSTANCE || void 0, rssKb: void 0 });
+    Object.assign(entry, { child, pid: child.pid ?? null, pgid: child.pid ?? null, startedAt: (/* @__PURE__ */ new Date()).toISOString(), status: "starting", adopted: false, stopping: false, error: void 0, nextRestartAt: void 0, tree: [], slowWarned: false, fingerprint: fp ?? void 0, rssKb: void 0 });
     follow(name, file, sizeOf(file));
     child.once("error", (error) => {
-      warn(`[registry] ! couldn't start ${name}: ${error.message}`);
+      warn(`[runner] ! couldn't start ${name}: ${error.message}`);
       exited(entry, gen, { code: null, signal: null, error: error.message });
     });
     child.once("exit", (code, signal) => exited(entry, gen, { code, signal }));
@@ -472,15 +472,14 @@ function createSupervisor({
           entry.status = "up";
           changed(entry);
           onUp(entry.agent, { startedAt: entry.startedAt, adopted: false, recovered });
-          if (recovered) onRecovered(entry.agent);
           return;
         }
       }
       if (Date.now() - started > HEALTH_TIMEOUT_MS && !entry.slowWarned) {
-        warn(`[registry] ! ${entry.name} isn't healthy ${HEALTH_TIMEOUT_MS / 1e3}s after starting; see ${logFile(entry.name)}`);
+        warn(`[runner] ! ${entry.name} isn't healthy ${HEALTH_TIMEOUT_MS / 1e3}s after starting; see ${logFile(entry.name)}`);
         entry.slowWarned = true;
       }
-      await sleep(500);
+      await sleep2(500);
     }
   }
   function exited(entry, gen, { code, signal, error }) {
@@ -506,13 +505,13 @@ function createSupervisor({
     if (entry.crashes.length >= CRASH_LIMIT) {
       entry.status = "crashed";
       entry.error = `exited ${entry.crashes.length} times within ${Math.round(CRASH_WINDOW_MS / 6e4)} min (last: ${how}); not restarting until restart_agent. Log: ${logFile(entry.name)}`;
-      warn(`[registry] ! ${entry.name} crashed: ${entry.error}`);
+      warn(`[runner] ! ${entry.name} crashed: ${entry.error}`);
       return changed(entry);
     }
     const delay2 = Math.min(BACKOFF_BASE_MS * 2 ** (entry.crashes.length - 1), BACKOFF_MAX_MS);
     entry.status = "down";
     entry.nextRestartAt = new Date(now + delay2).toISOString();
-    warn(`[registry] ${entry.name} exited unexpectedly (${how}); restarting in ${delay2 / 1e3}s`);
+    warn(`[runner] ${entry.name} exited unexpectedly (${how}); restarting in ${delay2 / 1e3}s`);
     entry.timer = setTimeout(async () => {
       await untilTreeGone(pgid, tree, KILL_GRACE_MS);
       if (entry.status !== "down" || entry.stopping) return;
@@ -524,7 +523,7 @@ function createSupervisor({
     changed(entry);
   }
   return {
-    // Startup: adopt what's still running from a previous registry, start the rest.
+    // Startup: adopt what's still running from a previous runner, start the rest.
     startAll(agents) {
       starting = (async () => {
         const records = readRecords();
@@ -542,7 +541,7 @@ function createSupervisor({
               const result = results.get(agent.name);
               if (result.adopted) return;
               if (closed && !result.superseded) {
-                if (ourTree(record, table()).tree.length) carry(agent, record);
+                if (ourTree(record, table(), signature(record.agent, record.port)).tree.length) carry(agent, record);
                 return;
               }
               await killStale(record, result.why ?? "it isn't serving");
@@ -586,7 +585,7 @@ function createSupervisor({
       }
       entry.status = "stopping";
       changed(entry);
-      const gone = await stopTree(pgid, tree, agent.name);
+      const gone = await stopTree(pgid, tree, { name: agent.name, timeoutMs: stopTimeoutMs, warn });
       if (gone) {
         if (entry.pgid === pgid) {
           entry.gen++;
@@ -608,42 +607,6 @@ function createSupervisor({
       pump();
       persist();
     },
-    // Exit without stopping any bot (a `hub restart` that keeps them): nothing
-    // new starts, pending (re)starts are cancelled, each tree is re-read and
-    // saved, and no signal is sent. The next registry vets every record
-    // (vetAdoption) and adopts what still matches. Returns the bots left running.
-    async detach() {
-      closed = true;
-      await starting.catch(() => {
-      });
-      clearInterval(pumpTimer);
-      clearInterval(treeTimer);
-      const rows = table();
-      for (const e of entries.values()) {
-        clearTimeout(e.timer);
-        e.timer = void 0;
-        e.claim = (e.claim ?? 0) + 1;
-        clearInterval(e.watch);
-        e.gen++;
-        if (e.pid) refreshTree(e, rows);
-      }
-      pump();
-      persist();
-      return [...entries.values()].filter((e) => e.pid).map((e) => ({ name: e.name, pid: e.pid }));
-    },
-    // Drop a removed bot's record (after its server is confirmed down).
-    forget(name) {
-      const entry = entries.get(name);
-      if (entry) {
-        clearTimeout(entry.timer);
-        clearInterval(entry.watch);
-        entry.gen++;
-        entry.claim = (entry.claim ?? 0) + 1;
-      }
-      entries.delete(name);
-      tails.delete(name);
-      persist();
-    },
     // Resolves true once the bot is up (spawned by us and healthy); false
     // once it can't be: it crashed, left the roster, or this supervisor closed.
     async waitUp(agent, timeoutMs) {
@@ -653,21 +616,10 @@ function createSupervisor({
         if (closed || !isCurrent(agent)) return false;
         if (entry?.agent === agent && entry.status === "up") return true;
         if (entry?.status === "crashed") return false;
-        await sleep(250);
+        await sleep2(250);
       }
       return false;
     },
-    // A running bot's memory as of the last tree refresh (eve dev + its
-    // server) and its uptime (an adopted one keeps its start). Kept out of
-    // status(): that's in every state snapshot, and these change constantly.
-    usage(name) {
-      const e = entries.get(name);
-      if (!e?.pid || e.status !== "up") return void 0;
-      return {
-        ...e.rssKb ? { rssMb: Math.round(e.rssKb / 1024) } : {},
-        ...e.startedAt ? { uptimeS: Math.max(0, Math.round((Date.now() - Date.parse(e.startedAt)) / 1e3)) } : {}
-      };
-    },
     status(name) {
       const e = entries.get(name);
       if (!e) return null;
@@ -742,7 +694,7 @@ async function createSession(fetchFn, eve, operationId) {
   return sessionId;
 }
 
-// ../../lib/v26-facts.mjs
+// ../../hosted-hub/lib/v26-facts.mjs
 function factsOfV26(event) {
   if (!event || typeof event.type !== "string") return { facts: [], progress: false };
   const data = isRecord(event.data) ? event.data : {};
@@ -962,7 +914,8 @@ function fact(type, data, source) {
 }
 var NO_RUN_SCOPE = /* @__PURE__ */ new Set(["context.started", "context.settled", "interaction.opened", "interaction.settled", "response.submitted", "response.settled"]);
 function content(data, kind, phase, value) {
-  return fact("content.completed", { partId: partIdOf(data, kind), runId: runIdOf(data), kind, phase, value }, data);
+  const run = typeof data.stepIndex === "number" ? { runId: runIdOf(data) } : {};
+  return fact("content.completed", { partId: partIdOf(data, kind), ...run, kind, phase, value }, data);
 }
 function turnSettled(data, settled) {
   const turnId = str(data.turnId);
@@ -1072,7 +1025,7 @@ var str = (value) => typeof value === "string" ? value : void 0;
 var num2 = (value) => typeof value === "number" && Number.isFinite(value) ? value : 0;
 var isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
 
-// ../../lib/session-stream.mjs
+// ../../hosted-hub/lib/session-stream.mjs
 async function* readSessionLines(body, startIndex) {
   let line = startIndex;
   let lastAt = null;
@@ -1155,9 +1108,19 @@ async function* streamLines(eve, sessionId, o) {
     }
     const idle = new AbortController(), signal = o.signal ? AbortSignal.any([o.signal, idle.signal]) : idle.signal;
     let timer = setTimeout(() => idle.abort(), idleMs);
+    const opened = Date.now();
+    let firstByte = false, lineAt = opened;
     const quiet = () => {
       clearTimeout(timer);
       timer = setTimeout(() => idle.abort(), idleMs);
+      if (!firstByte) {
+        firstByte = true;
+        console.log(JSON.stringify({ event: "hub.stream_first_byte", sessionId, startIndex: next, ms: Date.now() - opened }));
+      }
+      if (Date.now() - lineAt >= idleMs) {
+        console.log(JSON.stringify({ event: "hub.stream_no_lines", sessionId, next, ms: Date.now() - lineAt }));
+        lineAt = Date.now();
+      }
     };
     let renewed = false;
     try {
@@ -1185,6 +1148,7 @@ async function* streamLines(eve, sessionId, o) {
               break;
             }
             yield record;
+            lineAt = Date.now();
             next = record.line + 1;
             reopens = o.reopens ?? 3;
           }

package/main.mjs

@@ -496,14 +496,14 @@ var require_graceful_fs = __commonJS({
         return close;
       })(fs.close);
       fs.closeSync = (function(fs$closeSync) {
-        function closeSync3(fd) {
+        function closeSync4(fd) {
           fs$closeSync.apply(fs, arguments);
           resetQueue();
         }
-        Object.defineProperty(closeSync3, previousSymbol, {
+        Object.defineProperty(closeSync4, previousSymbol, {
           value: fs$closeSync
         });
-        return closeSync3;
+        return closeSync4;
       })(fs.closeSync);
       if (/\bgfs4\b/i.test(process.env.NODE_DEBUG || "")) {
         process.on("exit", function() {
@@ -1619,7 +1619,7 @@ import { randomUUID } from "node:crypto";
 import { existsSync as existsSync11, mkdirSync as mkdirSync7, readFileSync as readFileSync10, realpathSync as realpathSync4, writeFileSync as writeFileSync7 } from "node:fs";
 import { join as join16 } from "node:path";
 
-// ../../lib/clean-env.mjs
+// ../../hosted-hub/lib/clean-env.mjs
 var INHERITED_ENV = /^(EVE_|WORKFLOW_)/;
 function cleanEnv(env = process.env) {
   return Object.fromEntries(Object.entries(env).filter(([key]) => !INHERITED_ENV.test(key)));
@@ -1629,12 +1629,12 @@ function cleanGitEnv(env = process.env) {
   return Object.fromEntries(Object.entries(env).filter(([key]) => !GIT_REPO_ENV.test(key)));
 }
 
-// ../../lib/processes.mjs
-import { spawn, execFile, execFileSync } from "node:child_process";
-import { closeSync, existsSync as existsSync2, mkdirSync as mkdirSync2, openSync, readFileSync as readFileSync2, readSync, renameSync, rmSync, readdirSync as readdirSync2, statSync, writeFileSync as writeFileSync2, writeSync } from "node:fs";
+// ../../hosted-hub/lib/processes.mjs
+import { spawn } from "node:child_process";
+import { closeSync as closeSync2, existsSync as existsSync2, mkdirSync as mkdirSync2, openSync as openSync2, readFileSync as readFileSync2, renameSync, rmSync, readdirSync as readdirSync2, statSync as statSync2, writeFileSync as writeFileSync2, writeSync } from "node:fs";
 import { join as join2 } from "node:path";
 
-// ../../lib/fingerprint.mjs
+// ../../hosted-hub/lib/fingerprint.mjs
 import { createHash, createHmac, randomBytes } from "node:crypto";
 import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
 import { join, relative } from "node:path";
@@ -1701,22 +1701,44 @@ function fingerprintDiff(recorded, current) {
 }
 var BUILD_SKIP = /* @__PURE__ */ new Set([...TREE_SKIP, ".eve", ".output"]);
 
-// ../../lib/processes.mjs
-var num = (key, fallback) => {
-  const raw = process.env[key];
-  return raw === void 0 || raw === "" || !Number.isFinite(Number(raw)) ? fallback : Number(raw);
-};
+// ../../hosted-hub/lib/log-tail.mjs
+import { closeSync, openSync, readSync, statSync } from "node:fs";
+function createLogTail(log2) {
+  const tails = /* @__PURE__ */ new Map();
+  function follow(name, file, offset) {
+    tails.set(name, { file, offset, carry: "", prefix: `[${name}]`.padEnd(20) });
+  }
+  function pump() {
+    for (const tail of tails.values()) {
+      let size;
+      try {
+        size = statSync(tail.file).size;
+      } catch {
+        continue;
+      }
+      if (size < tail.offset) tail.offset = 0;
+      if (size === tail.offset) continue;
+      const length = Math.min(size - tail.offset, 1e6);
+      const buffer = Buffer.alloc(length);
+      const fd = openSync(tail.file, "r");
+      try {
+        readSync(fd, buffer, 0, length, tail.offset);
+      } finally {
+        closeSync(fd);
+      }
+      tail.offset += length;
+      const lines = (tail.carry + buffer.toString("utf8")).split("\n");
+      tail.carry = lines.pop();
+      for (const line of lines) if (line.trim() && !line.startsWith("Migration file of version")) log2(`${tail.prefix} ${line}`);
+    }
+  }
+  return { follow, pump };
+}
+
+// ../../hosted-hub/lib/process-tree.mjs
+import { execFile, execFileSync } from "node:child_process";
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-var CRASH_LIMIT = num("ROSTER_CRASH_LIMIT", 5);
-var CRASH_WINDOW_MS = num("ROSTER_CRASH_WINDOW_MS", 10 * 6e4);
-var BACKOFF_BASE_MS = num("ROSTER_BACKOFF_BASE_MS", 1e3);
-var BACKOFF_MAX_MS = num("ROSTER_BACKOFF_MAX_MS", 6e4);
-var HEALTH_TIMEOUT_MS = num("ROSTER_START_HEALTH_TIMEOUT_MS", 12e4);
-var PORT_RECHECK_MS = num("ROSTER_PORT_RECHECK_MS", 5e3);
 var KILL_GRACE_MS = 3e3;
-var WATCH_MS = 1e3;
-var TREE_MS = 5e3;
-var LOG_ROTATE_BYTES = 2e7;
 function groupAlive(pgid) {
   if (!(pgid > 1)) return false;
   try {
@@ -1739,7 +1761,7 @@ var C = { ...process.env, LC_ALL: "C" };
 var SLOW_MS = 1e3;
 function slow(what, started2) {
   const ms = Date.now() - started2;
-  if (ms >= SLOW_MS) console.warn(`[registry] ${what} took ${ms}ms`);
+  if (ms >= SLOW_MS) console.warn(`[runner] ${what} took ${ms}ms`);
 }
 var LSTART = String.raw`(\w{3}\s+\w{3}\s+\d+\s+[\d:]+\s+\d{4})`;
 function table() {
@@ -1802,6 +1824,73 @@ async function healthy(port, ms = 1e3) {
   }
 }
 var merge = (...lists) => [...new Map(lists.flat().map((m) => [m.pid, m])).values()];
+var treeAlive = (pgid, tree) => groupAlive(pgid) || liveMembers(tree, table()).length > 0;
+async function untilTreeGone(pgid, tree, ms) {
+  const deadline = Date.now() + ms;
+  while (treeAlive(pgid, tree)) {
+    if (Date.now() >= deadline) return false;
+    await sleep(200);
+  }
+  return true;
+}
+function killTree(pgid, tree) {
+  signalGroup(pgid, "SIGKILL");
+  for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) signalGroup(g, "SIGKILL");
+}
+async function stopTree(pgid, tree, { name, timeoutMs, warn }) {
+  if (!treeAlive(pgid, tree)) return true;
+  let membersTermed = false;
+  const termMembers = () => {
+    membersTermed = true;
+    for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) if (g !== pgid) signalGroup(g, "SIGTERM");
+  };
+  if (groupAlive(pgid)) signalGroup(pgid, "SIGTERM");
+  else termMembers();
+  const deadline = Date.now() + timeoutMs;
+  let gone = true;
+  while (treeAlive(pgid, tree)) {
+    if (!membersTermed && !groupAlive(pgid)) termMembers();
+    if (Date.now() >= deadline) {
+      gone = false;
+      break;
+    }
+    await sleep(200);
+  }
+  if (gone) return true;
+  warn(`[runner] ${name} didn't stop within ${timeoutMs / 1e3}s of SIGTERM; sending SIGKILL`);
+  killTree(pgid, tree);
+  return untilTreeGone(pgid, tree, KILL_GRACE_MS);
+}
+function ownsLeader(record, rows, signature) {
+  if (!(record.pid > 1) || record.pgid !== record.pid) return false;
+  const row = rows.find((r) => r.pid === record.pid);
+  if (!row || row.pgid !== record.pgid) return false;
+  if (!` ${row.command} `.includes(signature)) return false;
+  const member = record.tree?.find((m) => m.pid === record.pid);
+  if (member) return member.startedAt === row.startedAt;
+  const recorded = Date.parse(record.startedAt);
+  return !(Number.isFinite(row.startedAt) && Number.isFinite(recorded) && Math.abs(row.startedAt - recorded) > 5e3);
+}
+function ourTree(record, rows, signature) {
+  const leader = ownsLeader(record, rows, signature);
+  return { leader, tree: merge(leader ? treeOf(record.pid, rows) : [], liveMembers(record.tree, rows)) };
+}
+
+// ../../hosted-hub/lib/processes.mjs
+var num = (key, fallback) => {
+  const raw = process.env[key];
+  return raw === void 0 || raw === "" || !Number.isFinite(Number(raw)) ? fallback : Number(raw);
+};
+var sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
+var CRASH_LIMIT = num("ROSTER_CRASH_LIMIT", 5);
+var CRASH_WINDOW_MS = num("ROSTER_CRASH_WINDOW_MS", 10 * 6e4);
+var BACKOFF_BASE_MS = num("ROSTER_BACKOFF_BASE_MS", 1e3);
+var BACKOFF_MAX_MS = num("ROSTER_BACKOFF_MAX_MS", 6e4);
+var HEALTH_TIMEOUT_MS = num("ROSTER_START_HEALTH_TIMEOUT_MS", 12e4);
+var PORT_RECHECK_MS = num("ROSTER_PORT_RECHECK_MS", 5e3);
+var WATCH_MS = 1e3;
+var TREE_MS = 5e3;
+var LOG_ROTATE_BYTES = 2e7;
 function createSupervisor({
   root,
   // the workspace: eve dev's cwd
@@ -1814,18 +1903,12 @@ function createSupervisor({
   isCurrent = () => true,
   // is this agent object still on the roster?
   // ({ agent, env, argv }) -> what a server spawned now runs with
-  // (lib/fingerprint.mjs), or null. A previous registry's server is
+  // (lib/fingerprint.mjs), or null. A previous runner's server is
   // adopted only if its recorded fingerprint matches.
   fingerprint = null,
   onChange = () => {
   },
   // status changed (broadcast)
-  onUsage = () => {
-  },
-  // a running tree's memory changed (status() reports rssKb)
-  onRecovered = () => {
-  },
-  // healthy again after a crash restart
   onUp = () => {
   },
   // a server is serving: adopted, spawned, or respawned ({ startedAt, adopted, recovered })
@@ -1853,10 +1936,7 @@ function createSupervisor({
       status: e.status,
       ...e.tree?.length ? { tree: e.tree } : {},
       ...e.lastExit ? { lastExit: e.lastExit } : {},
-      ...e.fingerprint ? { fingerprint: e.fingerprint } : {},
-      // The daemon token it was spawned with (HUB_INSTANCE): an adopted bot
-      // keeps its spawner's, and lib/daemon.mjs selfStopRefusal checks it here.
-      ...e.instance ? { instance: e.instance } : {}
+      ...e.fingerprint ? { fingerprint: e.fingerprint } : {}
     }));
     mkdirSync2(stateDir, { recursive: true });
     const tmp = `${recordsPath}.${process.pid}.tmp`;
@@ -1877,34 +1957,7 @@ function createSupervisor({
     persist();
     onChange(entry.name);
   }
-  const tails = /* @__PURE__ */ new Map();
-  function follow(name, file, offset) {
-    tails.set(name, { file, offset, carry: "", prefix: `[${name}]`.padEnd(20) });
-  }
-  function pump() {
-    for (const tail of tails.values()) {
-      let size;
-      try {
-        size = statSync(tail.file).size;
-      } catch {
-        continue;
-      }
-      if (size < tail.offset) tail.offset = 0;
-      if (size === tail.offset) continue;
-      const length = Math.min(size - tail.offset, 1e6);
-      const buffer = Buffer.alloc(length);
-      const fd = openSync(tail.file, "r");
-      try {
-        readSync(fd, buffer, 0, length, tail.offset);
-      } finally {
-        closeSync(fd);
-      }
-      tail.offset += length;
-      const lines = (tail.carry + buffer.toString("utf8")).split("\n");
-      tail.carry = lines.pop();
-      for (const line of lines) if (line.trim() && !line.startsWith("Migration file of version")) log2(`${tail.prefix} ${line}`);
-    }
-  }
+  const { follow, pump } = createLogTail(log2);
   const pumpTimer = setInterval(pump, 250);
   pumpTimer.unref();
   function refreshTree(entry, rows = table()) {
@@ -1912,10 +1965,8 @@ function createSupervisor({
     const before = JSON.stringify(entry.tree ?? []);
     entry.tree = merge(treeOf(entry.pid, rows), liveMembers(entry.tree, rows));
     const own = rows.filter((r) => r.pid === entry.pid || r.ppid === entry.pid);
-    const rss = entry.rssKb;
     if (own.length) entry.rssKb = own.reduce((sum, r) => sum + (r.rssKb || 0), 0);
     if (JSON.stringify(entry.tree) !== before) persist();
-    if (entry.rssKb !== rss) onUsage(entry.name);
   }
   const treeTimer = setInterval(() => {
     const live = [...entries.values()].filter((e) => e.pid && !e.stopping);
@@ -1924,45 +1975,8 @@ function createSupervisor({
     for (const e of live) refreshTree(e, rows);
   }, TREE_MS);
   treeTimer.unref();
-  const treeAlive = (pgid, tree) => groupAlive(pgid) || liveMembers(tree, table()).length > 0;
-  async function untilTreeGone(pgid, tree, ms) {
-    const deadline = Date.now() + ms;
-    while (treeAlive(pgid, tree)) {
-      if (Date.now() >= deadline) return false;
-      await sleep(200);
-    }
-    return true;
-  }
-  function killTree(pgid, tree) {
-    signalGroup(pgid, "SIGKILL");
-    for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) signalGroup(g, "SIGKILL");
-  }
-  async function stopTree(pgid, tree, name) {
-    if (!treeAlive(pgid, tree)) return true;
-    let membersTermed = false;
-    const termMembers = () => {
-      membersTermed = true;
-      for (const g of new Set(liveMembers(tree, table()).map((m) => m.pgid))) if (g !== pgid) signalGroup(g, "SIGTERM");
-    };
-    if (groupAlive(pgid)) signalGroup(pgid, "SIGTERM");
-    else termMembers();
-    const deadline = Date.now() + stopTimeoutMs;
-    let gone = true;
-    while (treeAlive(pgid, tree)) {
-      if (!membersTermed && !groupAlive(pgid)) termMembers();
-      if (Date.now() >= deadline) {
-        gone = false;
-        break;
-      }
-      await sleep(200);
-    }
-    if (gone) return true;
-    warn(`[registry] ${name} didn't stop within ${stopTimeoutMs / 1e3}s of SIGTERM; sending SIGKILL`);
-    killTree(pgid, tree);
-    return untilTreeGone(pgid, tree, KILL_GRACE_MS);
-  }
   const logFile = (name) => join2(logDir, `${name}.log`);
-  const sizeOf = (file) => existsSync2(file) ? statSync(file).size : 0;
+  const sizeOf = (file) => existsSync2(file) ? statSync2(file).size : 0;
   function entryFor(agent) {
     let entry = entries.get(agent.name);
     if (!entry) {
@@ -1973,33 +1987,19 @@ function createSupervisor({
     entry.port = agent.port;
     return entry;
   }
-  function ownsLeader(record, rows) {
-    if (!(record.pid > 1) || record.pgid !== record.pid) return false;
-    const row = rows.find((r) => r.pid === record.pid);
-    if (!row || row.pgid !== record.pgid) return false;
-    if (!` ${row.command} `.includes(signature(record.agent, record.port))) return false;
-    const member = record.tree?.find((m) => m.pid === record.pid);
-    if (member) return member.startedAt === row.startedAt;
-    const recorded = Date.parse(record.startedAt);
-    return !(Number.isFinite(row.startedAt) && Number.isFinite(recorded) && Math.abs(row.startedAt - recorded) > 5e3);
-  }
-  function ourTree(record, rows) {
-    const leader = ownsLeader(record, rows);
-    return { leader, tree: merge(leader ? treeOf(record.pid, rows) : [], liveMembers(record.tree, rows)) };
-  }
   async function killStale(record, why) {
     const rows = table();
-    const { leader, tree } = ourTree(record, rows);
+    const { leader, tree } = ourTree(record, rows, signature(record.agent, record.port));
     if (!leader && !tree.length) {
-      if (groupAlive(record.pgid)) warn(`[registry] ! ${record.agent}: process group ${record.pgid} from processes.json is no longer its server; leaving it alone`);
+      if (groupAlive(record.pgid)) warn(`[runner] ! ${record.agent}: process group ${record.pgid} from processes.json is no longer its server; leaving it alone`);
       return true;
     }
-    warn(`[registry] ${record.agent}: stopping its stale server (pid ${record.pid}): ${why}`);
-    return stopTree(leader ? record.pgid : null, tree, record.agent);
+    warn(`[runner] ${record.agent}: stopping its stale server (pid ${record.pid}): ${why}`);
+    return stopTree(leader ? record.pgid : null, tree, { name: record.agent, timeoutMs: stopTimeoutMs, warn });
   }
   async function vetAdoption(agent, record) {
     const rows = table();
-    const { leader, tree } = ourTree(record, rows);
+    const { leader, tree } = ourTree(record, rows, signature(record.agent, record.port));
     if (!leader && !tree.length) return { adopted: false };
     if (!leader) return { adopted: false, stale: true, why: "its leader is gone" };
     if (record.port !== agent.port) return { adopted: false, stale: true, why: `its port moved from ${record.port} to ${agent.port}` };
@@ -2031,7 +2031,7 @@ function createSupervisor({
     const { tree } = vetted;
     if (closed) return carry(agent, record, tree);
     const entry = entryFor(agent);
-    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: "up", adopted: true, child: null, stopping: false, error: void 0, tree, fingerprint: record.fingerprint, instance: record.instance, rssKb: void 0 });
+    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: "up", adopted: true, child: null, stopping: false, error: void 0, tree, fingerprint: record.fingerprint, rssKb: void 0 });
     const gen = ++entry.gen;
     follow(agent.name, logFile(agent.name), sizeOf(logFile(agent.name)));
     entry.watch = setInterval(() => {
@@ -2042,14 +2042,14 @@ function createSupervisor({
       }
     }, WATCH_MS);
     entry.watch.unref();
-    log2(`[registry] adopted ${agent.name} (pid ${record.pid}, port ${agent.port})`);
+    log2(`[runner] adopted ${agent.name} (pid ${record.pid}, port ${agent.port})`);
     changed(entry);
     onUp(agent, { startedAt: entry.startedAt, adopted: true, recovered: false });
     return { adopted: true };
   }
   function carry(agent, record, tree = record.tree ?? []) {
     const entry = entryFor(agent);
-    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: record.status ?? "up", adopted: true, child: null, stopping: false, tree, fingerprint: record.fingerprint, instance: record.instance });
+    Object.assign(entry, { pid: record.pid, pgid: record.pgid, startedAt: record.startedAt, restarts: record.restarts ?? 0, status: record.status ?? "up", adopted: true, child: null, stopping: false, tree, fingerprint: record.fingerprint });
     entry.gen++;
     return { adopted: true, carried: true };
   }
@@ -2075,7 +2075,7 @@ function createSupervisor({
         return spawnAgent(entry, { recovered });
       }
       const error = `port ${port} is held by another process (${holders.map((pid) => describe(pid, rows)).join("; ")}), not by ${name}'s server; not touching it. ${name} starts once the port is free.`;
-      if (entry.error !== error || entry.status !== "blocked") warn(`[registry] ! ${error}`);
+      if (entry.error !== error || entry.status !== "blocked") warn(`[runner] ! ${error}`);
       Object.assign(entry, { status: "blocked", error, pid: null, pgid: null });
       entry.timer = setTimeout(() => spawnAgent(entry, { recovered }), PORT_RECHECK_MS);
       entry.timer.unref();
@@ -2084,7 +2084,7 @@ function createSupervisor({
     mkdirSync2(logDir, { recursive: true });
     const file = logFile(name);
     if (sizeOf(file) > LOG_ROTATE_BYTES) renameSync(file, `${file}.1`);
-    const fd = openSync(file, "a");
+    const fd = openSync2(file, "a");
     writeSync(fd, `
 --- ${(/* @__PURE__ */ new Date()).toISOString()} hub: starting ${name} on port ${port}
 `);
@@ -2094,7 +2094,7 @@ function createSupervisor({
     try {
       fp = fingerprintFor(agent, spawnEnv);
     } catch (error) {
-      warn(`[registry] ! ${name}: couldn't fingerprint its inputs (${error.message}); the next registry respawns it instead of adopting it`);
+      warn(`[runner] ! ${name}: couldn't fingerprint its inputs (${error.message}); the next runner respawns it instead of adopting it`);
     }
     let child;
     try {
@@ -2102,17 +2102,17 @@ function createSupervisor({
         cwd: root,
         env: spawnEnv,
         detached: true,
-        // own process group; survives the registry
+        // own process group; survives the runner
         stdio: ["ignore", fd, fd]
       });
     } finally {
-      closeSync(fd);
+      closeSync2(fd);
     }
     const gen = ++entry.gen;
-    Object.assign(entry, { child, pid: child.pid ?? null, pgid: child.pid ?? null, startedAt: (/* @__PURE__ */ new Date()).toISOString(), status: "starting", adopted: false, stopping: false, error: void 0, nextRestartAt: void 0, tree: [], slowWarned: false, fingerprint: fp ?? void 0, instance: spawnEnv?.HUB_INSTANCE || void 0, rssKb: void 0 });
+    Object.assign(entry, { child, pid: child.pid ?? null, pgid: child.pid ?? null, startedAt: (/* @__PURE__ */ new Date()).toISOString(), status: "starting", adopted: false, stopping: false, error: void 0, nextRestartAt: void 0, tree: [], slowWarned: false, fingerprint: fp ?? void 0, rssKb: void 0 });
     follow(name, file, sizeOf(file));
     child.once("error", (error) => {
-      warn(`[registry] ! couldn't start ${name}: ${error.message}`);
+      warn(`[runner] ! couldn't start ${name}: ${error.message}`);
       exited(entry, gen, { code: null, signal: null, error: error.message });
     });
     child.once("exit", (code, signal) => exited(entry, gen, { code, signal }));
@@ -2137,15 +2137,14 @@ function createSupervisor({
           entry.status = "up";
           changed(entry);
           onUp(entry.agent, { startedAt: entry.startedAt, adopted: false, recovered });
-          if (recovered) onRecovered(entry.agent);
           return;
         }
       }
       if (Date.now() - started2 > HEALTH_TIMEOUT_MS && !entry.slowWarned) {
-        warn(`[registry] ! ${entry.name} isn't healthy ${HEALTH_TIMEOUT_MS / 1e3}s after starting; see ${logFile(entry.name)}`);
+        warn(`[runner] ! ${entry.name} isn't healthy ${HEALTH_TIMEOUT_MS / 1e3}s after starting; see ${logFile(entry.name)}`);
         entry.slowWarned = true;
       }
-      await sleep(500);
+      await sleep2(500);
     }
   }
   function exited(entry, gen, { code, signal, error }) {
@@ -2171,25 +2170,25 @@ function createSupervisor({
     if (entry.crashes.length >= CRASH_LIMIT) {
       entry.status = "crashed";
       entry.error = `exited ${entry.crashes.length} times within ${Math.round(CRASH_WINDOW_MS / 6e4)} min (last: ${how}); not restarting until restart_agent. Log: ${logFile(entry.name)}`;
-      warn(`[registry] ! ${entry.name} crashed: ${entry.error}`);
+      warn(`[runner] ! ${entry.name} crashed: ${entry.error}`);
       return changed(entry);
     }
-    const delay2 = Math.min(BACKOFF_BASE_MS * 2 ** (entry.crashes.length - 1), BACKOFF_MAX_MS);
+    const delay3 = Math.min(BACKOFF_BASE_MS * 2 ** (entry.crashes.length - 1), BACKOFF_MAX_MS);
     entry.status = "down";
-    entry.nextRestartAt = new Date(now + delay2).toISOString();
-    warn(`[registry] ${entry.name} exited unexpectedly (${how}); restarting in ${delay2 / 1e3}s`);
+    entry.nextRestartAt = new Date(now + delay3).toISOString();
+    warn(`[runner] ${entry.name} exited unexpectedly (${how}); restarting in ${delay3 / 1e3}s`);
     entry.timer = setTimeout(async () => {
       await untilTreeGone(pgid, tree, KILL_GRACE_MS);
       if (entry.status !== "down" || entry.stopping) return;
       entry.restarts++;
       changed(entry);
       void spawnAgent(entry, { recovered: true });
-    }, delay2);
+    }, delay3);
     entry.timer.unref();
     changed(entry);
   }
   return {
-    // Startup: adopt what's still running from a previous registry, start the rest.
+    // Startup: adopt what's still running from a previous runner, start the rest.
     startAll(agents) {
       starting = (async () => {
         const records = readRecords();
@@ -2207,7 +2206,7 @@ function createSupervisor({
               const result = results.get(agent.name);
               if (result.adopted) return;
               if (closed && !result.superseded) {
-                if (ourTree(record, table()).tree.length) carry(agent, record);
+                if (ourTree(record, table(), signature(record.agent, record.port)).tree.length) carry(agent, record);
                 return;
               }
               await killStale(record, result.why ?? "it isn't serving");
@@ -2251,7 +2250,7 @@ function createSupervisor({
       }
       entry.status = "stopping";
       changed(entry);
-      const gone = await stopTree(pgid, tree, agent.name);
+      const gone = await stopTree(pgid, tree, { name: agent.name, timeoutMs: stopTimeoutMs, warn });
       if (gone) {
         if (entry.pgid === pgid) {
           entry.gen++;
@@ -2273,42 +2272,6 @@ function createSupervisor({
       pump();
       persist();
     },
-    // Exit without stopping any bot (a `hub restart` that keeps them): nothing
-    // new starts, pending (re)starts are cancelled, each tree is re-read and
-    // saved, and no signal is sent. The next registry vets every record
-    // (vetAdoption) and adopts what still matches. Returns the bots left running.
-    async detach() {
-      closed = true;
-      await starting.catch(() => {
-      });
-      clearInterval(pumpTimer);
-      clearInterval(treeTimer);
-      const rows = table();
-      for (const e of entries.values()) {
-        clearTimeout(e.timer);
-        e.timer = void 0;
-        e.claim = (e.claim ?? 0) + 1;
-        clearInterval(e.watch);
-        e.gen++;
-        if (e.pid) refreshTree(e, rows);
-      }
-      pump();
-      persist();
-      return [...entries.values()].filter((e) => e.pid).map((e) => ({ name: e.name, pid: e.pid }));
-    },
-    // Drop a removed bot's record (after its server is confirmed down).
-    forget(name) {
-      const entry = entries.get(name);
-      if (entry) {
-        clearTimeout(entry.timer);
-        clearInterval(entry.watch);
-        entry.gen++;
-        entry.claim = (entry.claim ?? 0) + 1;
-      }
-      entries.delete(name);
-      tails.delete(name);
-      persist();
-    },
     // Resolves true once the bot is up (spawned by us and healthy); false
     // once it can't be: it crashed, left the roster, or this supervisor closed.
     async waitUp(agent, timeoutMs) {
@@ -2318,21 +2281,10 @@ function createSupervisor({
         if (closed || !isCurrent(agent)) return false;
         if (entry?.agent === agent && entry.status === "up") return true;
         if (entry?.status === "crashed") return false;
-        await sleep(250);
+        await sleep2(250);
       }
       return false;
     },
-    // A running bot's memory as of the last tree refresh (eve dev + its
-    // server) and its uptime (an adopted one keeps its start). Kept out of
-    // status(): that's in every state snapshot, and these change constantly.
-    usage(name) {
-      const e = entries.get(name);
-      if (!e?.pid || e.status !== "up") return void 0;
-      return {
-        ...e.rssKb ? { rssMb: Math.round(e.rssKb / 1024) } : {},
-        ...e.startedAt ? { uptimeS: Math.max(0, Math.round((Date.now() - Date.parse(e.startedAt)) / 1e3)) } : {}
-      };
-    },
     status(name) {
       const e = entries.get(name);
       if (!e) return null;
@@ -2575,7 +2527,7 @@ async function sendToBot(fetchFn, eve, d, context, loadImage) {
 }
 
 // ../../hosted-hub/runner/hub-runner.ts
-import { setTimeout as delay } from "node:timers/promises";
+import { setTimeout as delay2 } from "node:timers/promises";
 
 // ../../hosted-hub/runner/status.ts
 import { renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs";
@@ -2606,7 +2558,7 @@ var status = (state, extra = {}) => {
   console.log(JSON.stringify({ state, ...extra }));
 };
 
-// ../../lib/v26-facts.mjs
+// ../../hosted-hub/lib/v26-facts.mjs
 function isV26Boundary(event) {
   const type = event?.type;
   return type === "session.waiting" || type === "session.completed" || type === "session.failed";
@@ -2830,7 +2782,8 @@ function fact(type, data, source) {
 }
 var NO_RUN_SCOPE = /* @__PURE__ */ new Set(["context.started", "context.settled", "interaction.opened", "interaction.settled", "response.submitted", "response.settled"]);
 function content(data, kind, phase, value) {
-  return fact("content.completed", { partId: partIdOf(data, kind), runId: runIdOf(data), kind, phase, value }, data);
+  const run4 = typeof data.stepIndex === "number" ? { runId: runIdOf(data) } : {};
+  return fact("content.completed", { partId: partIdOf(data, kind), ...run4, kind, phase, value }, data);
 }
 function turnSettled(data, settled) {
   const turnId = str(data.turnId);
@@ -2940,7 +2893,7 @@ var str = (value) => typeof value === "string" ? value : void 0;
 var num2 = (value) => typeof value === "number" && Number.isFinite(value) ? value : 0;
 var isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
 
-// ../../lib/session-stream.mjs
+// ../../hosted-hub/lib/session-stream.mjs
 async function* readSessionLines(body, startIndex) {
   let line = startIndex;
   let lastAt = null;
@@ -3023,9 +2976,19 @@ async function* streamLines(eve, sessionId, o) {
     }
     const idle = new AbortController(), signal = o.signal ? AbortSignal.any([o.signal, idle.signal]) : idle.signal;
     let timer = setTimeout(() => idle.abort(), idleMs);
+    const opened = Date.now();
+    let firstByte = false, lineAt = opened;
     const quiet = () => {
       clearTimeout(timer);
       timer = setTimeout(() => idle.abort(), idleMs);
+      if (!firstByte) {
+        firstByte = true;
+        console.log(JSON.stringify({ event: "hub.stream_first_byte", sessionId, startIndex: next, ms: Date.now() - opened }));
+      }
+      if (Date.now() - lineAt >= idleMs) {
+        console.log(JSON.stringify({ event: "hub.stream_no_lines", sessionId, next, ms: Date.now() - lineAt }));
+        lineAt = Date.now();
+      }
     };
     let renewed = false;
     try {
@@ -3053,6 +3016,7 @@ async function* streamLines(eve, sessionId, o) {
               break;
             }
             yield record;
+            lineAt = Date.now();
             next = record.line + 1;
             reopens = o.reopens ?? 3;
           }
@@ -3192,7 +3156,7 @@ async function downloadWorkspace(res, dir2) {
 }
 
 // ../../hosted-hub/runner/bot-state.ts
-import { closeSync as closeSync2, openSync as openSync2, readSync as readSync2, statSync as statSync2 } from "node:fs";
+import { closeSync as closeSync3, openSync as openSync3, readSync as readSync2, statSync as statSync3 } from "node:fs";
 var MARKER = "hub: starting";
 var WINDOW = 16 * 1024;
 var LINES = 5;
@@ -3200,13 +3164,13 @@ var CHARS = 500;
 function latestRun(file) {
   let text;
   try {
-    const size = statSync2(file).size, fd = openSync2(file, "r");
+    const size = statSync3(file).size, fd = openSync3(file, "r");
     try {
       const buf = Buffer.alloc(Math.min(size, WINDOW));
       readSync2(fd, buf, 0, buf.length, size - buf.length);
       text = buf.toString("utf8");
     } finally {
-      closeSync2(fd);
+      closeSync3(fd);
     }
   } catch {
     return void 0;
@@ -3363,7 +3327,7 @@ async function diskUsage(path) {
 
 // ../../workspaces/snapshot.mjs
 import { randomBytes as randomBytes3 } from "node:crypto";
-import { existsSync as existsSync5, lstatSync as lstatSync2, readdirSync as readdirSync4, readFileSync as readFileSync5, statSync as statSync3 } from "node:fs";
+import { existsSync as existsSync5, lstatSync as lstatSync2, readdirSync as readdirSync4, readFileSync as readFileSync5, statSync as statSync4 } from "node:fs";
 import { copyFile, mkdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname as dirname2, join as join6, relative as relative2, resolve as resolve2, sep } from "node:path";
@@ -3471,7 +3435,7 @@ async function saveCheckout(info, { threadId, name, snapshotRoot, root, label =
     const target = existsSync5(bundle) ? bundle.replace(/\.bundle$/, `-${sha2.slice(0, 8)}.bundle`) : bundle;
     await git(checkout, ["bundle", "create", "-q", target, tmpRef, "--branches", "--not", "--remotes"]);
     await git(checkout, ["bundle", "verify", "-q", target]);
-    return { checkout, kind: "bundle", bundle: target, bytes: statSync3(target).size, sha: sha2, dirty: info.dirty, unsaved: info.unsaved };
+    return { checkout, kind: "bundle", bundle: target, bytes: statSync4(target).size, sha: sha2, dirty: info.dirty, unsaved: info.unsaved };
   } finally {
     await git(checkout, ["update-ref", "-d", tmpRef], { allowFail: true });
   }
@@ -3938,11 +3902,11 @@ function fileRecords(dir2) {
 }
 
 // ../../workspaces/bare-guard.mjs
-import { existsSync as existsSync7, readFileSync as readFileSync6, statSync as statSync4 } from "node:fs";
+import { existsSync as existsSync7, readFileSync as readFileSync6, statSync as statSync5 } from "node:fs";
 import { join as join9 } from "node:path";
 async function repairBareFlag(repo) {
   const dotgit = join9(repo, ".git");
-  if (!existsSync7(dotgit) || !statSync4(dotgit).isDirectory()) return false;
+  if (!existsSync7(dotgit) || !statSync5(dotgit).isDirectory()) return false;
   const file = join9(dotgit, "config");
   try {
     const text = readFileSync6(file, "utf8");
@@ -4547,6 +4511,38 @@ function modelCredential(o) {
   } };
 }
 
+// ../../hosted-hub/runner/held.ts
+import { setTimeout as delay } from "node:timers/promises";
+function heldReports(send, failed) {
+  const told = /* @__PURE__ */ new Map();
+  const now = (bot) => told.has(bot) ? told.get(bot) : null;
+  let sending = Promise.resolve();
+  const tell = (bot, reason) => sending = sending.catch(() => {
+  }).then(async () => {
+    if (now(bot) === reason) return;
+    told.set(bot, void 0);
+    await send({ bot, reason });
+    if (reason) told.set(bot, reason);
+    else told.delete(bot);
+  });
+  return (waiting, why) => {
+    for (const bot of /* @__PURE__ */ new Set([...waiting, ...told.keys()])) {
+      const reason = why(bot);
+      if (now(bot) !== reason) tell(bot, reason).catch(failed);
+    }
+  };
+}
+async function stopWithin(stop, ms, signal) {
+  const stopped = new AbortController();
+  await Promise.race([
+    stop.finally(() => stopped.abort()),
+    delay(ms, void 0, { signal: AbortSignal.any([signal, stopped.signal]) }).then(() => {
+      throw new Error(`it did not stop in ${ms / 1e3}s`);
+    }, () => {
+    })
+  ]);
+}
+
 // ../../hosted-hub/runner/hub-runner.ts
 var realBots = { stage: stageLocal, supervise: createSupervisor };
 var HeldElsewhere = class extends Error {
@@ -4679,7 +4675,7 @@ async function run3(config, bots = realBots) {
     const open = work.sessions.filter((s) => s.bot === name);
     for (const s of open) {
       const res = await cancelTurn(net, eve(name, work.human), s.eveSessionId);
-      if (!res.ok) throw new Error(`cancel ${s.sessionId}: HTTP ${res.status}`);
+      if (!res.ok && !sessionEnded(res.status, await res.text())) throw new Error(`cancel ${s.sessionId}: HTTP ${res.status}`);
     }
     for (const s of open) await upload(s, name, work.human, s.foldedIndex + 1, { toTail: true });
     const sessionIds = open.map((s) => s.sessionId);
@@ -4824,7 +4820,7 @@ async function run3(config, bots = realBots) {
     if (restarted.get(name)?.requestedAt !== requestedAt) {
       if (agent) {
         try {
-          await supervisor.stop(agent);
+          await stopWithin(supervisor.stop(agent), config.stopTimeoutMs ?? 6e4, shutdown.signal);
           shutdown.signal.throwIfAborted();
           supervisor.start(agent);
           if (!await supervisor.waitUp(agent, 12e4)) throw new Error("it did not come back up");
@@ -4869,7 +4865,7 @@ async function run3(config, bots = realBots) {
     }
   }
   let syncing = null, again = false, drained = null;
-  const unsettled = /* @__PURE__ */ new Set();
+  const unsettled = /* @__PURE__ */ new Set(), held = heldReports((body) => hubJson("/api/local-runner/setup", { method: "POST", body: JSON.stringify(body), signal: AbortSignal.timeout(ACK_MS) }), (e) => failed("held report", e));
   async function sync() {
     const work = await hubJson("/api/local-runner/work");
     if (drained && !work.sessions.length && !work.children.length && !work.deliveries.some(([d]) => unsettled.has(d.id))) {
@@ -4897,6 +4893,7 @@ async function run3(config, bots = realBots) {
     for (const [id, abort] of following) if (![...sessions, ...work.children].some((s) => s.sessionId === id)) abort.abort();
     restart(work);
     compact2(work);
+    held(work.deliveries.map(([d]) => d.bot), (bot) => drained ? "draining" : restarting.has(bot) ? "restarting" : !running.has(bot) ? "stopped" : null);
     await Promise.all([
       ...sessions.filter((s) => s.cancel.length).map((s) => cancel(s, work.human).catch((e) => failed("cancel", e))),
       ...work.deliveries.filter(([d]) => (!drained || unsettled.has(d.id)) && running.has(d.bot) && !restarting.has(d.bot) && !dead.has(d.bot) && !retired.has(d.sessionId)).map((q) => deliver(q, work).catch((e) => failed("delivery", e)))
@@ -4953,7 +4950,7 @@ async function run3(config, bots = realBots) {
       } finally {
         clearTimeout(watchdog);
       }
-      await delay(RETRY_MS, void 0, { signal: shutdown.signal }).catch(() => {
+      await delay2(RETRY_MS, void 0, { signal: shutdown.signal }).catch(() => {
       });
     }
   })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hub-runner",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "license": "Apache-2.0",
   "description": "Runs a hub workspace's Mac bots on this Mac under launchd, with no app needed.",
   "type": "module",