@claw-link/gateway-host 0.3.14 → 0.3.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claw-link/gateway-host",
-  "version": "0.3.14",
+  "version": "0.3.16",
   "description": "ClawLink Host Gateway — a secure, outbound-only worker that bridges a local agent CLI (OpenClaw, Hermes, Claude, Codex, Cursor) to your ClawLink agents. No inbound ports; authenticated per-agent by a Host Token.",
   "homepage": "https://claw-link.co",
   "bin": {

package/scripts/install.js

@@ -724,11 +724,25 @@ async function drainToIdle({ timeoutMs = 15 * 60 * 1000, force = false, onWait }
   // where an interrupt could leave the worker drained. Don't install a handler here.
   const startedAt = Date.now();
   let waited = false;
+  let claimOnlySince = 0; // busy with ZERO active jobs = just a poll round-trip, not real work
   while (Date.now() - startedAt < timeoutMs) {
     const s = control.liveStatus();
     if (!s) return { hadWorker: true, drained: true, workerGone: true };  // worker exited → safe
     // Safe to restart only once the worker has ACKED the drain (won't claim more) AND is idle.
     if (s.drain && !s.busy) return { hadWorker: true, drained: true, waited };
+    // Wedge escape: "busy" caused ONLY by an in-flight claim (no running job) that never clears —
+    // e.g. an older worker whose HTTP call to a dead backend hangs without a timeout. A claim
+    // round-trip is seconds; 90s of claim-only busy means it's stuck, and a restart can't lose
+    // work because no job is actually running.
+    if (s.drain && s.busy && Number(s.activeJobs) === 0) {
+      if (!claimOnlySince) claimOnlySince = Date.now();
+      if (Date.now() - claimOnlySince > 90_000) {
+        console.log('\n  ⚠ Worker reports busy with no running job for 90s (stuck network call) — proceeding with the restart.');
+        return { hadWorker: true, drained: true, waited, wedged: true };
+      }
+    } else {
+      claimOnlySince = 0;
+    }
     if (s.busy && typeof onWait === 'function') { onWait(s); waited = true; }
     await sleep(1500);
   }

package/src/bridge.js

@@ -15,15 +15,29 @@ class Bridge {
   }
 
   async _post(action, extra) {
-    const res = await fetch(this.url, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${this.token}`,
-      },
-      // agent_id is only a hint (the token resolves the agent); machine_id binds the host.
-      body: JSON.stringify({ action, agent_id: this.agentId || undefined, machine_id: this.machineId, ...extra }),
-    });
+    // Hard request timeout: a dead/paused backend must FAIL a call, never hang it — a hung claim
+    // pins claimsInFlight, which reports the worker "busy" forever and wedges graceful update's
+    // drain-wait. 45s covers slow uplinks posting large complete() payloads.
+    const ac = new AbortController();
+    const kill = setTimeout(() => ac.abort(), Number(process.env.CLAWHOST_HTTP_TIMEOUT_MS) || 45_000);
+    let res;
+    try {
+      res = await fetch(this.url, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${this.token}`,
+        },
+        // agent_id is only a hint (the token resolves the agent); machine_id binds the host.
+        body: JSON.stringify({ action, agent_id: this.agentId || undefined, machine_id: this.machineId, ...extra }),
+        signal: ac.signal,
+      });
+    } catch (e) {
+      if (ac.signal.aborted) { const err = new Error(`host-bridge ${action} timed out`); err.status = 0; throw err; }
+      throw e;
+    } finally {
+      clearTimeout(kill);
+    }
     let data = null;
     try { data = await res.json(); } catch { /* non-json */ }
     if (!res.ok) {
@@ -38,6 +52,9 @@ class Bridge {
   register(runtime, version) { return this._post('register', { runtime, version }); }
   heartbeat() { return this._post('heartbeat', {}); }
   claim() { return this._post('claim', { instance_id: this.instanceId }); }
+  // Multiplexed poll: ONE invocation authenticates all the host's agent tokens, refreshes their
+  // liveness (replacing per-agent heartbeats), and claims up to capacities[i] jobs per agent.
+  claimAll(tokens, capacities) { return this._post('claim_all', { tokens, capacities, instance_id: this.instanceId }); }
   stream(jobId, seq, type, content) { return this._post('stream', { job_id: jobId, seq, type, content }); }
   complete(jobId, finalContent, metadata) { return this._post('complete', { job_id: jobId, final_content: finalContent, metadata: metadata || {} }); }
   fail(jobId, error) { return this._post('fail', { job_id: jobId, error: String(error).slice(0, 1000) }); }

package/src/worker.js

@@ -127,11 +127,13 @@ async function processJob(bridge, agentCfg, job, p2p) {
   }
 }
 
-async function runAgentLoop(agentCfg, cfg, p2p) {
+// Register one agent and prepare its execution slot (its own Bridge for stream/complete/fail +
+// concurrency/rate counters). Returns null on a bad token / machine mismatch so the rest of the
+// host keeps working.
+async function prepareAgent(agentCfg, cfg) {
   const bridge = new Bridge(cfg.bridge_url, agentCfg, cfg.instance_id);
   logger.addSecret(agentCfg.host_token);
   if (agentCfg.local_gateway_token) logger.addSecret(agentCfg.local_gateway_token);
-
   try {
     // Token-only register: the server resolves agent_id + runtime (dashboard is
     // the source of truth) and binds this machine on first connect.
@@ -139,76 +141,132 @@ async function runAgentLoop(agentCfg, cfg, p2p) {
     agentCfg.agent_id = resp.agent_id || agentCfg.agent_id;
     if (resp.runtime) agentCfg.runtime = resp.runtime;
     bridge.agentId = agentCfg.agent_id;
-
-    // Resolve binary + work_dir now that the runtime is known.
     if (!agentCfg.binary && agentCfg.runtime !== 'openclaw') {
       const found = detectRuntimes()[agentCfg.runtime];
       if (found) agentCfg.binary = found;
       else logger.warn(`no '${agentCfg.runtime}' binary found on PATH — set "binary" in config for ${agentCfg.agent_id}`);
     }
     if (!agentCfg.work_dir) agentCfg.work_dir = config.defaultWorkDir(agentCfg.runtime, agentCfg.agent_id);
-
     logger.info(`registered ${agentCfg.agent_id} as '${agentCfg.runtime}'`);
+    return { bridge, agent: agentCfg, inFlight: 0, starts: [] };
   } catch (e) {
-    logger.error(`register failed — check the Host Token & machine binding (rotate the token in the dashboard to re-bind a new machine). ${e.message}`);
-    return; // bad token / machine mismatch: stop this agent's loop (don't spin)
+    logger.error(`register failed for ${String(agentCfg.agent_id || agentCfg.host_token || '?').slice(0, 8)}… — check the Host Token & machine binding (rotate in the dashboard to re-bind). ${e.message}`);
+    return null;
   }
+}
 
-  const hb = setInterval(() => { bridge.heartbeat().catch(() => {}); }, cfg.heartbeat_ms || 10000);
+// Start a claimed job with the watchdog + slot/liveness bookkeeping (shared by both poll modes).
+function dispatchJob(slot, job, p2p, jobMaxMs) {
+  slot.inFlight++; slot.starts.push(Date.now()); activeJobIds.add(job.id);
+  const watchdog = new Promise((resolve) => setTimeout(() => resolve('__watchdog__'), jobMaxMs).unref());
+  Promise.race([processJob(slot.bridge, slot.agent, job, p2p), watchdog])
+    .then(async (r) => {
+      if (r === '__watchdog__') {
+        logger.error(`job ${job.id} exceeded ${jobMaxMs}ms — reclaiming slot`);
+        try { await slot.bridge.fail(job.id, `host watchdog: job exceeded ${jobMaxMs}ms`); } catch { /* ignore */ }
+      }
+    })
+    .finally(() => { slot.inFlight--; activeJobIds.delete(job.id); });
+}
 
-  let inFlight = 0;
-  const starts = []; // job start timestamps for the rolling rate-limit window
-  let claimFails = 0; // consecutive claim() errors → exponential backoff w/ jitter (transient 5xx)
-  // Hard job ceiling: config `job_max_ms` overrides the env/default — long dev/build turns
-  // (harness agents running mvn/npm/docker) can legitimately exceed the 15-min default.
+// ── Multiplexed poll: ONE edge-function invocation per tick for the WHOLE host ─────────────────
+// claim_all authenticates every agent token, refreshes every agent's liveness (no separate
+// heartbeats), and returns jobs for all free slots. With adaptive idle backoff this cuts
+// invocations by ~2 orders of magnitude vs the old per-agent claim(1.5s) + heartbeat(10s) loops
+// (7 agents: ~460k/day → <10k/day idle) — the fix for the quota exhaustion that killed the
+// original Supabase project. Returns 'legacy' if the server doesn't know claim_all yet.
+async function multiplexLoop(slots, cfg, p2p) {
+  const base = cfg.poll_interval_ms || 1500;
+  // Idle ceiling: default 60s, configurable up to 5 min (`idle_poll_max_ms` in config). Higher =
+  // fewer invocations; the trade-off is worst-case pickup delay for the FIRST job after a long
+  // idle (cadence snaps back to `base` the moment any job arrives).
+  const idleMax = Math.min(Math.max(Number(cfg.idle_poll_max_ms) || 60_000, base), 300_000);
   const jobMaxMs = Number(cfg.job_max_ms) > 0 ? Number(cfg.job_max_ms) : JOB_MAX_MS;
+  const primary = slots[0].bridge;
+  let interval = base;
+  let claimFails = 0;
 
   while (!stopping) {
+    const draining = stopping || drainFlag;
     const now = Date.now();
-    while (starts.length && now - starts[0] > 60000) starts.shift();
+    for (const s of slots) { while (s.starts.length && now - s.starts[0] > 60000) s.starts.shift(); }
+    const tokens = slots.map((s) => s.agent.host_token);
+    // Capacity 0 still refreshes that agent's liveness — draining/busy agents stay "live".
+    const capacities = slots.map((s) => draining ? 0
+      : Math.max(0, Math.min(s.agent.concurrency - s.inFlight, s.agent.rate_limit_per_min - s.starts.length)));
+
+    let got = 0;
+    let errored = false;
+    claimsInFlight++; // busy for the whole round-trip — preserves the graceful-drain guarantee
+    try {
+      const r = await primary.claimAll(tokens, capacities);
+      claimFails = 0;
+      for (const job of r.jobs || []) {
+        const slot = slots.find((s) => s.agent.agent_id === job.agent_id);
+        if (!slot) continue;
+        got++;
+        dispatchJob(slot, job, p2p, jobMaxMs);
+      }
+      if (Array.isArray(r.denied) && r.denied.length) {
+        logger.warn(`claim_all: ${r.denied.length} token(s) rejected (rotated or bound to another machine)`);
+      }
+    } catch (e) {
+      errored = true;
+      claimFails++;
+      if (/unknown action/i.test(String(e.message))) return 'legacy'; // older server — fall back
+      logger.warn(`claim_all failed [x${claimFails}]: ${e.message}`);
+    } finally {
+      claimsInFlight--;
+    }
+
+    // Adaptive cadence: fast while work flows (a job arrived or any is running), decaying to
+    // idleMax when quiet; exponential backoff with full jitter on repeated errors.
+    const active = got > 0 || activeJobIds.size > 0;
+    if (errored && claimFails > 1) {
+      const ceil = Math.min(base * 2 ** (claimFails - 1), CLAIM_BACKOFF_CAP_MS);
+      interval = base + Math.floor(Math.random() * Math.max(0, ceil - base));
+    } else {
+      interval = active ? base : Math.min(Math.max(interval, base) * 1.6, idleMax);
+    }
+    await sleep(interval);
+  }
+  return 'stopped';
+}
+
+// Legacy per-agent loop — only used against a server that predates claim_all. Costs one claim
+// per agent per tick + a heartbeat timer, exactly like the original implementation.
+async function runAgentLoopLegacy(slot, cfg, p2p) {
+  const { bridge, agent: agentCfg } = slot;
+  const hb = setInterval(() => { bridge.heartbeat().catch(() => {}); }, cfg.heartbeat_ms || 10000);
+  let claimFails = 0;
+  const jobMaxMs = Number(cfg.job_max_ms) > 0 ? Number(cfg.job_max_ms) : JOB_MAX_MS;
 
-    // Draining (graceful update/shutdown): keep finishing in-flight jobs, but don't claim new ones.
+  while (!stopping) {
+    const now = Date.now();
+    while (slot.starts.length && now - slot.starts[0] > 60000) slot.starts.shift();
     const draining = stopping || drainFlag;
-    const canRun = !draining && inFlight < agentCfg.concurrency && starts.length < agentCfg.rate_limit_per_min;
+    const canRun = !draining && slot.inFlight < agentCfg.concurrency && slot.starts.length < agentCfg.rate_limit_per_min;
     let claimErrored = false;
     if (canRun) {
       let job = null;
-      claimsInFlight++; // "busy" for the whole claim round-trip — closes the mid-claim drain race
+      claimsInFlight++;
       try {
-        try { const r = await bridge.claim(); job = r && r.job; claimFails = 0; } // success (job or empty) resets backoff
+        try { const r = await bridge.claim(); job = r && r.job; claimFails = 0; }
         catch (e) { claimErrored = true; claimFails++; logger.warn(`claim failed (${agentCfg.agent_id}) [x${claimFails}]: ${e.message}`); }
-        if (job) {
-          inFlight++; starts.push(Date.now()); activeJobIds.add(job.id);
-          // Watchdog: guarantees the slot is reclaimed and the job is failed back even
-          // if the adapter never settles (no per-adapter timeout by default).
-          const watchdog = new Promise((resolve) => setTimeout(() => resolve('__watchdog__'), jobMaxMs).unref());
-          Promise.race([processJob(bridge, agentCfg, job, p2p), watchdog])
-            .then(async (r) => {
-              if (r === '__watchdog__') {
-                logger.error(`job ${job.id} exceeded ${jobMaxMs}ms — reclaiming slot`);
-                try { await bridge.fail(job.id, `host watchdog: job exceeded ${jobMaxMs}ms`); } catch { /* ignore */ }
-              }
-            })
-            .finally(() => { inFlight--; activeJobIds.delete(job.id); });
-        }
+        if (job) dispatchJob(slot, job, p2p, jobMaxMs);
       } finally {
-        // Decrement AFTER activeJobIds.add above (no await between them), so a claimed job hands off
-        // from claimsInFlight to activeJobIds without the busy signal ever dipping to false.
         claimsInFlight--;
       }
-      if (job) continue; // filled a slot — try to fill remaining concurrency immediately
+      if (job) continue;
     }
-    // Poll interval — but after repeated claim errors (a Supabase blip), back off exponentially with
-    // full jitter (capped) so N agents don't hammer the bridge in lockstep. Resets on the next success.
     let waitMs = cfg.poll_interval_ms || 1500;
     if (claimErrored && claimFails > 1) {
-      const base = cfg.poll_interval_ms || 1500;
-      const ceil = Math.min(base * 2 ** (claimFails - 1), CLAIM_BACKOFF_CAP_MS);
-      waitMs = base + Math.floor(Math.random() * Math.max(0, ceil - base));
+      const b = cfg.poll_interval_ms || 1500;
+      const ceil = Math.min(b * 2 ** (claimFails - 1), CLAIM_BACKOFF_CAP_MS);
+      waitMs = b + Math.floor(Math.random() * Math.max(0, ceil - b));
     }
     await sleep(waitMs);
   }
-
   clearInterval(hb);
 }
 
@@ -276,7 +334,34 @@ async function startWorker() {
   process.on('SIGINT', () => onStop('SIGINT'));
   process.on('SIGTERM', () => onStop('SIGTERM'));
 
-  await Promise.all(agents.map((a) => runAgentLoop(a, raw, p2p)));
+  // Register every agent up-front, then poll with ONE multiplexed claim_all per tick for the
+  // whole host (falls back to legacy per-agent loops against an older server).
+  let slots = [];
+  for (const a of agents) {
+    const s = await prepareAgent(a, raw);
+    if (s) slots.push(s);
+  }
+  // No agent registered (rotated/dead tokens, or the backend was migrated): DON'T exit — launchd
+  // would just crash-loop us and the status file would flap, confusing `clhost update`. Stay
+  // alive (status keeps publishing "live + idle") and retry registration every 60s; `clhost link`
+  // restarts the service anyway the moment new tokens arrive.
+  while (!slots.length && !stopping) {
+    logger.error('No agent could register — tokens rotated or invalid. Retrying in 60s (fix with: clhost link <code> · clhost retoken <id> · clhost doctor).');
+    const until = Date.now() + 60_000;
+    while (Date.now() < until && !stopping) await sleep(1000);
+    if (stopping) return;
+    slots = [];
+    for (const a of agents) {
+      const s = await prepareAgent(a, raw);
+      if (s) slots.push(s);
+    }
+  }
+  if (stopping) return;
+  const mode = await multiplexLoop(slots, raw, p2p);
+  if (mode === 'legacy') {
+    logger.warn('server has no claim_all yet — using per-agent polling (update the ClawLink backend to cut invocations)');
+    await Promise.all(slots.map((s) => runAgentLoopLegacy(s, raw, p2p)));
+  }
 }
 
 module.exports = { startWorker };