hub-runner 0.2.2 → 0.2.3

package/main.mjs

@@ -2174,17 +2174,17 @@ function createSupervisor({
       warn(`[registry] ! ${entry.name} crashed: ${entry.error}`);
       return changed(entry);
     }
-    const delay2 = Math.min(BACKOFF_BASE_MS * 2 ** (entry.crashes.length - 1), BACKOFF_MAX_MS);
+    const delay3 = Math.min(BACKOFF_BASE_MS * 2 ** (entry.crashes.length - 1), BACKOFF_MAX_MS);
     entry.status = "down";
-    entry.nextRestartAt = new Date(now + delay2).toISOString();
-    warn(`[registry] ${entry.name} exited unexpectedly (${how}); restarting in ${delay2 / 1e3}s`);
+    entry.nextRestartAt = new Date(now + delay3).toISOString();
+    warn(`[registry] ${entry.name} exited unexpectedly (${how}); restarting in ${delay3 / 1e3}s`);
     entry.timer = setTimeout(async () => {
       await untilTreeGone(pgid, tree, KILL_GRACE_MS);
       if (entry.status !== "down" || entry.stopping) return;
       entry.restarts++;
       changed(entry);
       void spawnAgent(entry, { recovered: true });
-    }, delay2);
+    }, delay3);
     entry.timer.unref();
     changed(entry);
   }
@@ -2575,7 +2575,7 @@ async function sendToBot(fetchFn, eve, d, context, loadImage) {
 }
 
 // ../../hosted-hub/runner/hub-runner.ts
-import { setTimeout as delay } from "node:timers/promises";
+import { setTimeout as delay2 } from "node:timers/promises";
 
 // ../../hosted-hub/runner/status.ts
 import { renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs";
@@ -4547,6 +4547,38 @@ function modelCredential(o) {
   } };
 }
 
+// ../../hosted-hub/runner/held.ts
+import { setTimeout as delay } from "node:timers/promises";
+function heldReports(send, failed) {
+  const told = /* @__PURE__ */ new Map();
+  const now = (bot) => told.has(bot) ? told.get(bot) : null;
+  let sending = Promise.resolve();
+  const tell = (bot, reason) => sending = sending.catch(() => {
+  }).then(async () => {
+    if (now(bot) === reason) return;
+    told.set(bot, void 0);
+    await send({ bot, reason });
+    if (reason) told.set(bot, reason);
+    else told.delete(bot);
+  });
+  return (waiting, why) => {
+    for (const bot of /* @__PURE__ */ new Set([...waiting, ...told.keys()])) {
+      const reason = why(bot);
+      if (now(bot) !== reason) tell(bot, reason).catch(failed);
+    }
+  };
+}
+async function stopWithin(stop, ms, signal) {
+  const stopped = new AbortController();
+  await Promise.race([
+    stop.finally(() => stopped.abort()),
+    delay(ms, void 0, { signal: AbortSignal.any([signal, stopped.signal]) }).then(() => {
+      throw new Error(`it did not stop in ${ms / 1e3}s`);
+    }, () => {
+    })
+  ]);
+}
+
 // ../../hosted-hub/runner/hub-runner.ts
 var realBots = { stage: stageLocal, supervise: createSupervisor };
 var HeldElsewhere = class extends Error {
@@ -4824,7 +4856,7 @@ async function run3(config, bots = realBots) {
     if (restarted.get(name)?.requestedAt !== requestedAt) {
       if (agent) {
         try {
-          await supervisor.stop(agent);
+          await stopWithin(supervisor.stop(agent), config.stopTimeoutMs ?? 6e4, shutdown.signal);
           shutdown.signal.throwIfAborted();
           supervisor.start(agent);
           if (!await supervisor.waitUp(agent, 12e4)) throw new Error("it did not come back up");
@@ -4869,7 +4901,7 @@ async function run3(config, bots = realBots) {
     }
   }
   let syncing = null, again = false, drained = null;
-  const unsettled = /* @__PURE__ */ new Set();
+  const unsettled = /* @__PURE__ */ new Set(), held = heldReports((body) => hubJson("/api/local-runner/setup", { method: "POST", body: JSON.stringify(body), signal: AbortSignal.timeout(ACK_MS) }), (e) => failed("held report", e));
   async function sync() {
     const work = await hubJson("/api/local-runner/work");
     if (drained && !work.sessions.length && !work.children.length && !work.deliveries.some(([d]) => unsettled.has(d.id))) {
@@ -4897,6 +4929,7 @@ async function run3(config, bots = realBots) {
     for (const [id, abort] of following) if (![...sessions, ...work.children].some((s) => s.sessionId === id)) abort.abort();
     restart(work);
     compact2(work);
+    held(work.deliveries.map(([d]) => d.bot), (bot) => drained ? "draining" : restarting.has(bot) ? "restarting" : !running.has(bot) ? "stopped" : null);
     await Promise.all([
       ...sessions.filter((s) => s.cancel.length).map((s) => cancel(s, work.human).catch((e) => failed("cancel", e))),
       ...work.deliveries.filter(([d]) => (!drained || unsettled.has(d.id)) && running.has(d.bot) && !restarting.has(d.bot) && !dead.has(d.bot) && !retired.has(d.sessionId)).map((q) => deliver(q, work).catch((e) => failed("delivery", e)))
@@ -4953,7 +4986,7 @@ async function run3(config, bots = realBots) {
       } finally {
         clearTimeout(watchdog);
       }
-      await delay(RETRY_MS, void 0, { signal: shutdown.signal }).catch(() => {
+      await delay2(RETRY_MS, void 0, { signal: shutdown.signal }).catch(() => {
       });
     }
   })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hub-runner",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "license": "Apache-2.0",
   "description": "Runs a hub workspace's Mac bots on this Mac under launchd, with no app needed.",
   "type": "module",