@claw-link/gateway-host 0.3.6 → 0.3.8

package/bin/cli.js

@@ -104,8 +104,8 @@ async function main() {
         '  transport-test  Verify content-addressed Capsule relay (host ⇄ central sync + integrity)',
         '  p2p-test    Verify the native QUIC P2P transport end-to-end on this machine (loopback)',
         '  start       Start the background service',
-        '  stop        Stop the background service',
-        '  restart     Restart the background service (apply config changes)',
+        '  stop        Stop the background service (finishes the in-flight job first)',
+        '  restart     Restart the service — waits for in-flight jobs ([--force] [--timeout <s>])',
         '  update      Refresh the service to this CLI’s version — waits for in-flight jobs to finish',
         '              (clhost update [--force] [--timeout <seconds>])',
         '  setup       install + add-agent in one flow',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claw-link/gateway-host",
-  "version": "0.3.6",
+  "version": "0.3.8",
   "description": "ClawLink Host Gateway — a secure, outbound-only worker that bridges a local agent CLI (OpenClaw, Hermes, Claude, Codex, Cursor) to your ClawLink agents. No inbound ports; authenticated per-agent by a Host Token.",
   "homepage": "https://claw-link.co",
   "bin": {

package/scripts/install.js

@@ -139,9 +139,9 @@ async function run() {
       onWait: (s) => process.stdout.write(`\r     … still working: ${s.activeJobs} job(s) in flight      `),
     });
     if (live && live.busy) process.stdout.write('\n');
-    console.log(r.restarted
-      ? (r.hadWorker && !r.timedOut ? '  ✓ Finished in-flight work, then refreshed to the latest version.' : '  ✓ Service refreshed to the latest version.')
-      : '  ⚠ Could not refresh the service automatically — run:  clhost restart');
+    if (!r.restarted) console.log('  ⚠ Could not refresh the service automatically — run:  clhost restart');
+    else if (r.confirmed) console.log(`  ✓ Refreshed — now running v${r.version} (pid ${r.pid}).`);
+    else console.log(`  ⚠ Restart triggered but the new worker hasn't come up on v${VERSION} yet — check \`clhost status\` in a few seconds.`);
     reportCli();
     console.log('\n  Commands:  clhost status · clhost agents · clhost add-agent · clhost transport-test\n');
     return;
@@ -190,9 +190,10 @@ async function addAgentCmd() {
   rl.close();
   if (agent) {
     saveAgent(cfg, agent);
-    const restarted = serviceControl('restart');
-    console.log(`\n  ✓ Added agent ${agent.agent_id} (${agent.runtime}).` +
-      (restarted ? ' Service restarted.' : ' Run `clhost restart` (or start) to apply.') + '\n');
+    const r = await gracefulServiceRestartCLI();
+    const note = !r.restarted ? ' Run `clhost restart` (or start) to apply.'
+      : r.confirmed ? ' Service restarted.' : ' Restarting — check `clhost status`.';
+    console.log(`\n  ✓ Added agent ${agent.agent_id} (${agent.runtime}).` + note + '\n');
   }
 }
 
@@ -212,8 +213,10 @@ async function removeAgentCmd() {
   cfg.agents = cfg.agents.filter((a) => !(a.agent_id && (a.agent_id === target || a.agent_id.startsWith(target))));
   if (cfg.agents.length === before) { console.log(`  No agent matched '${target}'.`); return; }
   config.save(cfg);
-  const restarted = serviceControl('restart');
-  console.log(`  ✓ Removed agent '${target}'.` + (restarted ? ' Service restarted.' : ' Run `clhost restart` to apply.'));
+  const r = await gracefulServiceRestartCLI();
+  const note = !r.restarted ? ' Run `clhost restart` to apply.'
+    : r.confirmed ? ' Service restarted.' : ' Restarting — check `clhost status`.';
+  console.log(`  ✓ Removed agent '${target}'.` + note);
 }
 
 function printAgents(cfg) {
@@ -269,9 +272,11 @@ async function retokenCmd() {
       agent.runtime = resp.runtime;
     }
     config.save(cfg);
-    const restarted = serviceControl('restart');
-    console.log(`\n  ✓ Updated token for ${agent.agent_id} (…${token.slice(-4)}).` +
-      (restarted ? ' Service restarted.' : ' Run `clhost restart` to apply.') + '\n');
+    rl.close(); // release stdin before the (possibly long) graceful drain so Ctrl-C is handled by our cleanup
+    const r = await gracefulServiceRestartCLI();
+    const note = !r.restarted ? ' Run `clhost restart` to apply.'
+      : r.confirmed ? ' Service restarted.' : ' Restarting — check `clhost status`.';
+    console.log(`\n  ✓ Updated token for ${agent.agent_id} (…${token.slice(-4)}).` + note + '\n');
   } finally { rl.close(); }
 }
 
@@ -296,13 +301,33 @@ function serviceControl(action) {
     if (platform === 'win32') {
       if (action === 'stop')    return execSafe('schtasks /End /TN ClawLinkHost') !== null;
       if (action === 'start')   return execSafe('schtasks /Run /TN ClawLinkHost') !== null;
-      if (action === 'restart') { execSafe('schtasks /End /TN ClawLinkHost'); return execSafe('schtasks /Run /TN ClawLinkHost') !== null; }
+      if (action === 'restart') {
+        execSafe('schtasks /End /TN ClawLinkHost');
+        // /End is async, and the task is MultipleInstancesPolicy=IgnoreNew — so an immediate /Run can be
+        // SILENTLY DROPPED while the old instance is still stopping (the Windows analogue of the macOS
+        // bootout→bootstrap race). Give it a brief, LANGUAGE-NEUTRAL pause to stop before /Run: `ping -n
+        // 4 127.0.0.1` ≈ 3s (parsing schtasks' status would depend on the localized word "Running"; and
+        // `timeout` needs an interactive console). If /Run still doesn't take, restartAndConfirm's
+        // confirm-then-reinstall fallback recovers.
+        execSafe('ping -n 4 127.0.0.1');
+        return execSafe('schtasks /Run /TN ClawLinkHost') !== null;
+      }
     }
   } catch { /* fall through */ }
   return false;
 }
 
-function serviceControlCmd(action) {
+async function serviceControlCmd(action) {
+  // `restart` is graceful — drain in-flight jobs first (override with --force / --timeout <sec>).
+  // `stop` stays prompt: its SIGTERM already lets the worker finish in-flight work (2nd signal forces).
+  if (action === 'restart') {
+    const { force, timeoutMs } = parseDrainArgs();
+    const r = await gracefulServiceRestartCLI({ force, timeoutMs });
+    if (!r.restarted) console.log('  ⚠ Could not restart the service (is it installed? run `clhost install`).');
+    else if (r.confirmed) console.log(`  ✓ Service restarted — running v${r.version} (pid ${r.pid}).`);
+    else console.log('  ⚠ Restart triggered but the worker hasn’t reported yet — check `clhost status`.');
+    return;
+  }
   const ok = serviceControl(action);
   console.log(ok ? `  ✓ Service ${action}ed.` : `  ⚠ Could not ${action} the service (is it installed? run \`clhost install\`).`);
 }
@@ -538,6 +563,9 @@ async function drainToIdle({ timeoutMs = 15 * 60 * 1000, force = false, onWait }
   if (force) return { hadWorker: true, drained: false, forced: true };
 
   control.requestDrain();
+  // NOTE: the Ctrl-C cleanup that clears the drain flag on abort lives in withDrainGuard(), which
+  // wraps this call AND the restart — so there's no window (between drain finishing and the restart)
+  // where an interrupt could leave the worker drained. Don't install a handler here.
   const startedAt = Date.now();
   let waited = false;
   while (Date.now() - startedAt < timeoutMs) {
@@ -551,20 +579,92 @@ async function drainToIdle({ timeoutMs = 15 * 60 * 1000, force = false, onWait }
   return { hadWorker: true, drained: false, timedOut: true };
 }
 
-// Drain the running worker to idle (unless forced), then stage the new code + restart the service.
-// The fresh worker clears the drain flag on boot; if the restart fails we clear it here so the old
-// worker resumes claiming instead of sitting permanently drained.
+// Run an async drain+restart sequence under a Ctrl-C guard: if the operator interrupts at ANY point
+// (during the drain wait OR the restart), clear the drain flag FIRST so the still-running worker
+// resumes claiming — never left wedged — then exit. Spanning the whole sequence closes the gap that
+// a per-drainToIdle handler would leave open between draining and restarting.
+async function withDrainGuard(fn) {
+  const onInt = () => { try { control.clearDrain(); } catch { /* ignore */ } process.exit(130); };
+  process.on('SIGINT', onInt);
+  process.on('SIGTERM', onInt);
+  try { return await fn(); }
+  finally { process.off('SIGINT', onInt); process.off('SIGTERM', onInt); }
+}
+
+// After triggering a restart, wait until the NEW worker actually publishes status (a different pid
+// than before and — if given — the expected version), so a caller never reports success while the old
+// process is still winding down or a launchd unload/reload is mid-flight. Best-effort: confirmed=false
+// on timeout. This is what makes `clhost update`'s "runs vX" honest instead of premature.
+async function waitForWorkerUp(prevPid, wantVersion, timeoutMs = 25000) {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    await sleep(1000);
+    const s = control.liveStatus();
+    if (s && s.pid !== prevPid && (!wantVersion || String(s.version) === String(wantVersion))) {
+      return { confirmed: true, version: s.version, pid: s.pid };
+    }
+  }
+  const s = control.liveStatus();
+  return { confirmed: false, version: s && s.version, pid: s && s.pid };
+}
+
+// Restart the service and CONFIRM the new worker came up. Prefer an IN-PLACE restart
+// (kickstart -k / systemctl restart / schtasks) — it never unloads the job (no "Service: stopped"
+// window like bootout→bootstrap) and relaunches from the stable app dir. Only if that fails to bring
+// a worker up (service not loaded yet, or a stale Node path in the plist after an nvm/Node upgrade)
+// fall back to a full (re)install, which rewrites the plist. Clears the drain flag unless a fresh
+// worker is confirmed (which clears it on boot) so nothing is ever left drained.
+async function restartAndConfirm({ stageCode = false, wantVersion, onRestart } = {}) {
+  if (stageCode) syncAppDir(); // copy the new code into the stable app dir the service runs from
+  const prevPid = (control.liveStatus() || {}).pid;
+  let notified = false;
+  const notify = () => { if (!notified && typeof onRestart === 'function') { notified = true; onRestart(); } };
+  let restarted = serviceControl('restart');
+  if (restarted) notify();
+  let up = restarted ? await waitForWorkerUp(prevPid, wantVersion) : { confirmed: false };
+  if (!up.confirmed) {
+    const reinstalled = installService(); // rewrites plist/unit (handles a moved Node) + reload
+    if (reinstalled) { restarted = true; notify(); up = await waitForWorkerUp(prevPid, wantVersion); }
+  }
+  if (!up.confirmed) control.clearDrain();
+  return { restarted, ...up };
+}
+
+// `update` path — drain, then stage new code + restart, confirming the new version is actually live.
 async function gracefulRestart(opts = {}) {
-  const drain = await drainToIdle(opts);
-  const restarted = installService();  // syncAppDir (stage new code) + bootstrap/restart the service
-  if (!restarted) control.clearDrain();
-  return { ...drain, restarted };
+  return withDrainGuard(async () => {
+    const drain = await drainToIdle(opts);
+    const r = await restartAndConfirm({ stageCode: true, wantVersion: VERSION, onRestart: opts.onRestart });
+    return { ...drain, ...r };
+  });
 }
 
-// `update` — refresh the background service to the version of THIS CLI. GRACEFUL: it waits for any
-// in-flight job to finish before restarting (override with `--force`, cap the wait with
-// `--timeout <seconds>`). To update the CLI itself: npm i -g @claw-link/gateway-host@latest.
-async function update() {
+// Config-change restart (add/remove/retoken agent, `clhost restart`): drain first, then restart in
+// place (no code re-stage — same version), confirming the worker actually bounced (new pid).
+async function gracefulServiceRestart(opts = {}) {
+  return withDrainGuard(async () => {
+    const drain = await drainToIdle(opts);
+    const r = await restartAndConfirm({ stageCode: false, onRestart: opts.onRestart });
+    return { ...drain, ...r };
+  });
+}
+
+// Console wrapper around gracefulServiceRestart: prints a one-time "waiting" notice + a live progress
+// line while a job finishes, so config commands never silently hang. Returns the restart result.
+async function gracefulServiceRestartCLI(opts = {}) {
+  const live = control.liveStatus();
+  const willWait = live && live.busy && !opts.force;
+  if (willWait) console.log('  ⏳ An agent is mid-job — restarting once it finishes (Ctrl-C to cancel; it stays on the current version)…');
+  const r = await gracefulServiceRestart({
+    ...opts,
+    onWait: (s) => process.stdout.write(`\r     … still working: ${s.activeJobs} job(s) in flight      `),
+  });
+  if (willWait) process.stdout.write('\n');
+  return r;
+}
+
+// Shared `--force` / `--timeout <seconds>` parser for update() and the restart command.
+function parseDrainArgs() {
   const args = process.argv.slice(3);
   const force = args.includes('--force') || args.includes('-f');
   const ti = args.findIndex((a) => a === '--timeout' || a === '-t');
@@ -574,6 +674,14 @@ async function update() {
     if (Number.isFinite(secs) && secs > 0) timeoutMs = secs * 1000;
     else console.log(`  ⚠ Ignoring invalid --timeout '${args[ti + 1]}' (expected seconds) — using the default.`);
   }
+  return { force, timeoutMs };
+}
+
+// `update` — refresh the background service to the version of THIS CLI. GRACEFUL: it waits for any
+// in-flight job to finish before restarting (override with `--force`, cap the wait with
+// `--timeout <seconds>`). To update the CLI itself: npm i -g @claw-link/gateway-host@latest.
+async function update() {
+  const { force, timeoutMs } = parseDrainArgs();
 
   console.log(`  Refreshing the background service to v${VERSION}…`);
   const live = control.liveStatus();
@@ -588,10 +696,15 @@ async function update() {
   if (live && live.busy && !force) process.stdout.write('\n');
 
   if (!r.restarted) { console.log('  ⚠ Could not refresh automatically — run:  clhost restart'); return; }
-  if (r.forced) console.log(`  ✓ Restarted now (--force). Service runs v${VERSION} from ${appDir()}.`);
-  else if (r.timedOut) console.log(`  ⚠ A job was still running after the wait — restarted anyway (it may have been interrupted). Service runs v${VERSION}.`);
-  else if (r.hadWorker) console.log(`  ✓ Finished in-flight work, then refreshed. Service runs v${VERSION} from ${appDir()}.`);
-  else console.log(`  ✓ Service now runs v${VERSION} from ${appDir()}.`);
+  if (r.confirmed) {
+    // The success line now reflects what the new worker ACTUALLY reports (pid + version), not a guess.
+    const drained = r.hadWorker && !r.forced && !r.timedOut;
+    console.log(`  ✓ ${drained ? 'Finished in-flight work, then restarted' : 'Restarted'} — now running v${r.version} (pid ${r.pid}).`);
+    if (r.timedOut) console.log('  ⚠ (A job was still running after the wait — it may have been interrupted.)');
+  } else {
+    console.log(`  ⚠ Restart triggered but the new worker hasn't come up on v${VERSION} yet` +
+      (r.version ? ` (still seeing v${r.version})` : '') + ` — check \`clhost status\` in a few seconds (\`clhost logs\` if it doesn't appear).`);
+  }
   console.log('  (To upgrade the `clhost` CLI you type:  npm i -g @claw-link/gateway-host@latest)\n');
 }
 

package/src/worker.js

@@ -27,6 +27,8 @@ const TRANSPORT_MIN_BYTES = Number(process.env.CLAWHOST_TRANSPORT_MIN) || 1024;
 // forces immediate exit). The graceful-update path pre-drains to idle, so a restart's signal
 // finds the worker idle and exits at once; this only bounds a direct stop/restart/Ctrl-C.
 const SHUTDOWN_DRAIN_MS = Number(process.env.CLAWHOST_SHUTDOWN_DRAIN_MS) || JOB_MAX_MS;
+// Ceiling for the exponential backoff after consecutive claim() errors (transient Supabase 5xx).
+const CLAIM_BACKOFF_CAP_MS = Number(process.env.CLAWHOST_CLAIM_BACKOFF_CAP_MS) || 30 * 1000;
 
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 let stopping = false;
@@ -156,6 +158,7 @@ async function runAgentLoop(agentCfg, cfg, p2p) {
 
   let inFlight = 0;
   const starts = []; // job start timestamps for the rolling rate-limit window
+  let claimFails = 0; // consecutive claim() errors → exponential backoff w/ jitter (transient 5xx)
 
   while (!stopping) {
     const now = Date.now();
@@ -164,12 +167,13 @@ async function runAgentLoop(agentCfg, cfg, p2p) {
     // Draining (graceful update/shutdown): keep finishing in-flight jobs, but don't claim new ones.
     const draining = stopping || drainFlag;
     const canRun = !draining && inFlight < agentCfg.concurrency && starts.length < agentCfg.rate_limit_per_min;
+    let claimErrored = false;
     if (canRun) {
       let job = null;
       claimsInFlight++; // "busy" for the whole claim round-trip — closes the mid-claim drain race
       try {
-        try { const r = await bridge.claim(); job = r && r.job; }
-        catch (e) { logger.warn(`claim failed (${agentCfg.agent_id}): ${e.message}`); }
+        try { const r = await bridge.claim(); job = r && r.job; claimFails = 0; } // success (job or empty) resets backoff
+        catch (e) { claimErrored = true; claimFails++; logger.warn(`claim failed (${agentCfg.agent_id}) [x${claimFails}]: ${e.message}`); }
         if (job) {
           inFlight++; starts.push(Date.now()); activeJobIds.add(job.id);
           // Watchdog: guarantees the slot is reclaimed and the job is failed back even
@@ -191,7 +195,15 @@ async function runAgentLoop(agentCfg, cfg, p2p) {
       }
       if (job) continue; // filled a slot — try to fill remaining concurrency immediately
     }
-    await sleep(cfg.poll_interval_ms || 1500);
+    // Poll interval — but after repeated claim errors (a Supabase blip), back off exponentially with
+    // full jitter (capped) so N agents don't hammer the bridge in lockstep. Resets on the next success.
+    let waitMs = cfg.poll_interval_ms || 1500;
+    if (claimErrored && claimFails > 1) {
+      const base = cfg.poll_interval_ms || 1500;
+      const ceil = Math.min(base * 2 ** (claimFails - 1), CLAIM_BACKOFF_CAP_MS);
+      waitMs = base + Math.floor(Math.random() * Math.max(0, ceil - base));
+    }
+    await sleep(waitMs);
   }
 
   clearInterval(hb);
@@ -209,18 +221,14 @@ async function startWorker() {
 
   logger.info(`ClawLink Host Gateway v${VERSION} — ${agents.length} agent(s): ${agents.map((a) => `${(a.agent_id || 'pending').slice(0, 8)}…(${a.runtime || 'resolving'})`).join(', ')}`);
 
-  // H2H data plane: bring up the native QUIC transport once per host (shared by all agents). If no
-  // prebuilt native core exists for this platform, every transfer routes through the central relay.
   const p2p = new P2P(logger);
-  const p2pAddr = await p2p.start();
-  logger.info(p2pAddr
-    ? `H2H transport: native P2P active (${p2pAddr})`
-    : 'H2H transport: central relay (no native core for this platform — install acceleration later)');
 
   // Advertise liveness + in-flight count so `clhost update`/`restart` can wait for this host to go
   // idle before restarting, and pick up the CLI's drain flag (stop claiming, finish current work).
-  // Clear any stale flag first; and only ever honor a flag written AFTER this boot, so a leftover
-  // flag from a crashed update (or a clearDrain that failed) can never wedge a fresh worker.
+  // Do this BEFORE the (possibly slow) P2P init, so a restart is CONFIRMED fast — waitForWorkerUp sees
+  // the new pid/version within ~1s even while iroh is still binding. Clear any stale flag first; and
+  // only ever honor a flag written AFTER this boot, so a leftover flag from a crashed update (or a
+  // clearDrain that failed) can never wedge a fresh worker. (pushStatus never touches p2p.)
   control.clearDrain();      // drop any stale flag left by a previous life FIRST…
   const bootAt = Date.now(); // …then anchor "now": only a drain requested after this point is honored.
   const pushStatus = () => {
@@ -236,6 +244,13 @@ async function startWorker() {
   const statusTimer = setInterval(pushStatus, 2000);
   statusTimer.unref();
 
+  // H2H data plane: bring up the native QUIC transport once per host (shared by all agents). If no
+  // prebuilt native core exists for this platform, every transfer routes through the central relay.
+  const p2pAddr = await p2p.start();
+  logger.info(p2pAddr
+    ? `H2H transport: native P2P active (${p2pAddr})`
+    : 'H2H transport: central relay (no native core for this platform — install acceleration later)');
+
   // Graceful shutdown: stop claiming, let in-flight jobs finish (keeping P2P alive for them), then
   // exit. A second signal forces an immediate exit; SHUTDOWN_DRAIN_MS bounds the wait.
   let stopSignals = 0;