@claw-link/gateway-host 0.3.5 → 0.3.6

package/bin/cli.js

@@ -106,7 +106,8 @@ async function main() {
         '  start       Start the background service',
         '  stop        Stop the background service',
         '  restart     Restart the background service (apply config changes)',
-        '  update      Refresh the background service to this CLI’s version',
+        '  update      Refresh the service to this CLI’s version — waits for in-flight jobs to finish',
+        '              (clhost update [--force] [--timeout <seconds>])',
         '  setup       install + add-agent in one flow',
         '  rotate      How to rotate a Host Token / move to a new machine',
         '  uninstall   Stop + remove the background service',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@claw-link/gateway-host",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "ClawLink Host Gateway — a secure, outbound-only worker that bridges a local agent CLI (OpenClaw, Hermes, Claude, Codex, Cursor) to your ClawLink agents. No inbound ports; authenticated per-agent by a Host Token.",
   "homepage": "https://claw-link.co",
   "bin": {

package/scripts/install.js

@@ -12,6 +12,9 @@ const config = require('../src/config');
 const { detectRuntimes } = require('../src/detect');
 const { Bridge } = require('../src/bridge');
 const scaffold = require('../src/scaffold');
+const control = require('../src/control');
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
 const VERSION = (() => { try { return require('../package.json').version; } catch { return '0.0.0'; } })();
 const RUNTIMES = ['openclaw', 'hermes', 'claude', 'codex', 'cursor'];
@@ -129,9 +132,15 @@ async function run() {
   const existing = (cfg.agents || []).length;
   if (existing > 0) {
     console.log(`\n  ClawLink Host Gateway v${VERSION} — already set up (${existing} agent${existing > 1 ? 's' : ''}).`);
-    const installed = installService();
-    console.log(installed
-      ? '  ✓ Service refreshed to the latest version.'
+    // Graceful: let any in-flight job finish before the service restarts (same as `clhost update`).
+    const live = control.liveStatus();
+    if (live && live.busy) console.log('  ⏳ An agent is mid-job — finishing it before refreshing the service…');
+    const r = await gracefulRestart({
+      onWait: (s) => process.stdout.write(`\r     … still working: ${s.activeJobs} job(s) in flight      `),
+    });
+    if (live && live.busy) process.stdout.write('\n');
+    console.log(r.restarted
+      ? (r.hadWorker && !r.timedOut ? '  ✓ Finished in-flight work, then refreshed to the latest version.' : '  ✓ Service refreshed to the latest version.')
       : '  ⚠ Could not refresh the service automatically — run:  clhost restart');
     reportCli();
     console.log('\n  Commands:  clhost status · clhost agents · clhost add-agent · clhost transport-test\n');
@@ -371,7 +380,11 @@ WantedBy=default.target
 `;
       fs.writeFileSync(path.join(unitDir, 'clawlink-host.service'), unit, { mode: 0o644 });
       execSafe('systemctl --user daemon-reload');
-      return execSafe('systemctl --user enable --now clawlink-host') !== null;
+      execSafe('systemctl --user enable clawlink-host'); // start at login (idempotent)
+      // `restart` STARTS a stopped unit AND restarts a running one — so this both first-installs and
+      // UPDATES. (`enable --now` is a no-op on an already-running unit, which would leave the old
+      // code running and, mid-drain, wedge the worker with the drain flag never cleared.)
+      return execSafe('systemctl --user restart clawlink-host') !== null;
     }
     if (platform === 'win32') {
       return installServiceWindows();
@@ -446,6 +459,12 @@ function status() {
   if (!cfg) { console.log('Not configured. Run: clhost setup'); return; }
   console.log(`  Bridge:  ${cfg.bridge_url}`);
   console.log(`  Service: ${serviceRunning() ? '● running' : '○ stopped'}`);
+  // Live worker heartbeat (from the worker's status file) — shows whether an update would wait.
+  const live = control.liveStatus();
+  if (live) {
+    console.log(`  Worker:  ● live (v${live.version || '?'})` +
+      (live.busy ? ` — busy: ${live.activeJobs} job(s) in flight (\`clhost update\` will wait)` : ' — idle'));
+  }
   console.log('');
   printAgents(cfg);
 }
@@ -507,14 +526,72 @@ function logs() {
   console.log(`\n  ── last ${n} lines of ${logPath} ──`);
 }
 
-// `update` — refresh the background service to the version of THIS CLI (syncs the stable
-// app dir + restarts). To update the CLI itself: npm i -g @claw-link/gateway-host@latest.
-function update() {
+// Ask the running worker to finish its in-flight job(s) and stop claiming new ones, then wait until
+// it reports idle — so the coming restart never kills work in progress. Returns immediately when
+// there's no live worker to drain (first install, or a worker that predates drain support and thus
+// never writes a status file — in that case the restart is the old, non-graceful hard swap).
+async function drainToIdle({ timeoutMs = 15 * 60 * 1000, force = false, onWait } = {}) {
+  // Never let a bad/NaN timeout collapse the wait loop into an instant (silent) hard restart.
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) timeoutMs = 15 * 60 * 1000;
+  const live = control.liveStatus();
+  if (!live) return { hadWorker: false, drained: true };   // nothing running to drain
+  if (force) return { hadWorker: true, drained: false, forced: true };
+
+  control.requestDrain();
+  const startedAt = Date.now();
+  let waited = false;
+  while (Date.now() - startedAt < timeoutMs) {
+    const s = control.liveStatus();
+    if (!s) return { hadWorker: true, drained: true, workerGone: true };  // worker exited → safe
+    // Safe to restart only once the worker has ACKED the drain (won't claim more) AND is idle.
+    if (s.drain && !s.busy) return { hadWorker: true, drained: true, waited };
+    if (s.busy && typeof onWait === 'function') { onWait(s); waited = true; }
+    await sleep(1500);
+  }
+  return { hadWorker: true, drained: false, timedOut: true };
+}
+
+// Drain the running worker to idle (unless forced), then stage the new code + restart the service.
+// The fresh worker clears the drain flag on boot; if the restart fails we clear it here so the old
+// worker resumes claiming instead of sitting permanently drained.
+async function gracefulRestart(opts = {}) {
+  const drain = await drainToIdle(opts);
+  const restarted = installService();  // syncAppDir (stage new code) + bootstrap/restart the service
+  if (!restarted) control.clearDrain();
+  return { ...drain, restarted };
+}
+
+// `update` — refresh the background service to the version of THIS CLI. GRACEFUL: it waits for any
+// in-flight job to finish before restarting (override with `--force`, cap the wait with
+// `--timeout <seconds>`). To update the CLI itself: npm i -g @claw-link/gateway-host@latest.
+async function update() {
+  const args = process.argv.slice(3);
+  const force = args.includes('--force') || args.includes('-f');
+  const ti = args.findIndex((a) => a === '--timeout' || a === '-t');
+  let timeoutMs;
+  if (ti >= 0 && args[ti + 1]) {
+    const secs = Number(args[ti + 1]);
+    if (Number.isFinite(secs) && secs > 0) timeoutMs = secs * 1000;
+    else console.log(`  ⚠ Ignoring invalid --timeout '${args[ti + 1]}' (expected seconds) — using the default.`);
+  }
+
   console.log(`  Refreshing the background service to v${VERSION}…`);
-  const ok = installService();
-  console.log(ok
-    ? `  ✓ Service now runs v${VERSION} from ${appDir()}.`
-    : '  ⚠ Could not refresh automatically — run: clhost restart');
+  const live = control.liveStatus();
+  if (live && live.busy && !force) {
+    console.log('  ⏳ An agent is mid-job — waiting for it to finish before restarting.');
+    console.log('     (Ctrl-C to keep the current version · `clhost update --force` to restart now.)');
+  }
+  const r = await gracefulRestart({
+    force, timeoutMs,
+    onWait: (s) => process.stdout.write(`\r     … still working: ${s.activeJobs} job(s) in flight${s.currentJobId ? ` (${String(s.currentJobId).slice(0, 8)}…)` : ''}      `),
+  });
+  if (live && live.busy && !force) process.stdout.write('\n');
+
+  if (!r.restarted) { console.log('  ⚠ Could not refresh automatically — run:  clhost restart'); return; }
+  if (r.forced) console.log(`  ✓ Restarted now (--force). Service runs v${VERSION} from ${appDir()}.`);
+  else if (r.timedOut) console.log(`  ⚠ A job was still running after the wait — restarted anyway (it may have been interrupted). Service runs v${VERSION}.`);
+  else if (r.hadWorker) console.log(`  ✓ Finished in-flight work, then refreshed. Service runs v${VERSION} from ${appDir()}.`);
+  else console.log(`  ✓ Service now runs v${VERSION} from ${appDir()}.`);
   console.log('  (To upgrade the `clhost` CLI you type:  npm i -g @claw-link/gateway-host@latest)\n');
 }
 

package/src/control.js

@@ -0,0 +1,66 @@
+'use strict';
+// Local, file-based control channel between a running worker and the CLI (`clhost update`/
+// `restart`). Everything lives under ~/.clawlink-host so it works with zero network/DB access:
+//   • worker-status.json — the worker's heartbeat: pid, version, in-flight job count, drain ack.
+//   • drain               — a flag file; while present the worker STOPS claiming new jobs but keeps
+//                           finishing in-flight ones. The CLI uses it for graceful, no-job-killed
+//                           restarts: request drain → wait for the worker to report idle → restart.
+// The worker clears a stale `drain` flag on boot, so a crashed update can never wedge a fresh worker.
+
+const fs = require('fs');
+const path = require('path');
+const config = require('./config');
+
+const STATUS_PATH = path.join(config.HOME_DIR, 'worker-status.json');
+const DRAIN_PATH = path.join(config.HOME_DIR, 'drain');
+
+// A worker is considered "live" only if its status was written within this window — so the CLI
+// treats a crashed/stopped worker (stale file) as "nothing to drain" and proceeds immediately.
+const STATUS_FRESH_MS = 30 * 1000;
+
+function writeStatus(s) {
+  try {
+    config.ensureHomeDir();
+    fs.writeFileSync(STATUS_PATH, JSON.stringify({ ...s, ts: Date.now() }), { mode: 0o600 });
+  } catch { /* best effort — status is advisory */ }
+}
+
+function readStatus() {
+  try { return JSON.parse(fs.readFileSync(STATUS_PATH, 'utf8')); } catch { return null; }
+}
+
+// The last status IF a worker is actively writing it (fresh), else null.
+function liveStatus() {
+  const s = readStatus();
+  if (!s || typeof s.ts !== 'number') return null;
+  return Date.now() - s.ts < STATUS_FRESH_MS ? s : null;
+}
+
+function requestDrain() {
+  try { config.ensureHomeDir(); fs.writeFileSync(DRAIN_PATH, String(Date.now()), { mode: 0o600 }); return true; }
+  catch { return false; }
+}
+
+function clearDrain() {
+  try { fs.rmSync(DRAIN_PATH, { force: true }); } catch { /* ignore */ }
+}
+
+function drainRequested() {
+  try { return fs.existsSync(DRAIN_PATH); } catch { return false; }
+}
+
+// True only if a drain flag exists AND was written at/after `sinceMs`. The worker passes its boot
+// time here so a STALE flag left over from before it booted (e.g. a crashed update, or a clearDrain()
+// that failed) is IGNORED — the worker can never wedge itself refusing to claim. A present-but-
+// unparseable flag is honored (fail toward draining) since that's the job-safe direction.
+function drainRequestedSince(sinceMs) {
+  let raw;
+  try { raw = fs.readFileSync(DRAIN_PATH, 'utf8'); } catch { return false; } // no flag → not draining
+  const ts = Number(raw);
+  return Number.isFinite(ts) ? ts >= sinceMs : true;
+}
+
+module.exports = {
+  STATUS_PATH, DRAIN_PATH, STATUS_FRESH_MS,
+  writeStatus, readStatus, liveStatus, requestDrain, clearDrain, drainRequested, drainRequestedSince,
+};

package/src/worker.js

@@ -12,6 +12,7 @@ const { Bridge, withRetry } = require('./bridge');
 const { getAdapter } = require('./adapters');
 const { detectRuntimes } = require('./detect');
 const { P2P } = require('./transport/p2p');
+const control = require('./control');
 
 const VERSION = (() => { try { return require('../package.json').version; } catch { return '0.0.0'; } })();
 
@@ -22,9 +23,23 @@ const JOB_MAX_MS = Number(process.env.CLAWHOST_JOB_MAX_MS) || 15 * 60 * 1000;
 // Offer outputs larger than this over P2P (must match the engine's TRANSPORT_MIN_BYTES).
 // Lowered to 1 KiB for validation so a normal reply triggers P2P; raise for production.
 const TRANSPORT_MIN_BYTES = Number(process.env.CLAWHOST_TRANSPORT_MIN) || 1024;
+// On a stop signal, how long to let in-flight jobs finish before exiting anyway (a 2nd signal
+// forces immediate exit). The graceful-update path pre-drains to idle, so a restart's signal
+// finds the worker idle and exits at once; this only bounds a direct stop/restart/Ctrl-C.
+const SHUTDOWN_DRAIN_MS = Number(process.env.CLAWHOST_SHUTDOWN_DRAIN_MS) || JOB_MAX_MS;
 
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 let stopping = false;
+// In-flight job ids across ALL agent loops — the single source of truth for "is this host busy?".
+// Drives the status file (so `clhost update` can wait for idle) and the graceful-shutdown drain.
+const activeJobIds = new Set();
+// Claim RPCs currently outstanding (across all loops). Counted as "busy" for their whole round-trip
+// so a drain that lands MID-claim can't see a false-idle window and let the CLI restart on top of a
+// job that's about to start running. Kept in lockstep with activeJobIds (no await between add+decr).
+let claimsInFlight = 0;
+// When a drain is in effect (SIGTERM, or the CLI's drain flag) the loops stop claiming NEW jobs
+// but let in-flight ones finish. Cache the flag check so many agents don't stat the file every poll.
+let drainFlag = false;
 
 // Replace ⟦clawlink:transport:CID⟧ markers in an inbound job's content with the actual upstream
 // bytes — fetched P2P from the producing host, falling back to the inline DB copy. Mutates job.content.
@@ -146,26 +161,35 @@ async function runAgentLoop(agentCfg, cfg, p2p) {
     const now = Date.now();
     while (starts.length && now - starts[0] > 60000) starts.shift();
 
-    const canRun = inFlight < agentCfg.concurrency && starts.length < agentCfg.rate_limit_per_min;
+    // Draining (graceful update/shutdown): keep finishing in-flight jobs, but don't claim new ones.
+    const draining = stopping || drainFlag;
+    const canRun = !draining && inFlight < agentCfg.concurrency && starts.length < agentCfg.rate_limit_per_min;
     if (canRun) {
       let job = null;
-      try { const r = await bridge.claim(); job = r && r.job; }
-      catch (e) { logger.warn(`claim failed (${agentCfg.agent_id}): ${e.message}`); }
-      if (job) {
-        inFlight++; starts.push(Date.now());
-        // Watchdog: guarantees the slot is reclaimed and the job is failed back even
-        // if the adapter never settles (no per-adapter timeout by default).
-        const watchdog = new Promise((resolve) => setTimeout(() => resolve('__watchdog__'), JOB_MAX_MS).unref());
-        Promise.race([processJob(bridge, agentCfg, job, p2p), watchdog])
-          .then(async (r) => {
-            if (r === '__watchdog__') {
-              logger.error(`job ${job.id} exceeded ${JOB_MAX_MS}ms — reclaiming slot`);
-              try { await bridge.fail(job.id, `host watchdog: job exceeded ${JOB_MAX_MS}ms`); } catch { /* ignore */ }
-            }
-          })
-          .finally(() => { inFlight--; });
-        continue; // try to fill remaining concurrency immediately
+      claimsInFlight++; // "busy" for the whole claim round-trip — closes the mid-claim drain race
+      try {
+        try { const r = await bridge.claim(); job = r && r.job; }
+        catch (e) { logger.warn(`claim failed (${agentCfg.agent_id}): ${e.message}`); }
+        if (job) {
+          inFlight++; starts.push(Date.now()); activeJobIds.add(job.id);
+          // Watchdog: guarantees the slot is reclaimed and the job is failed back even
+          // if the adapter never settles (no per-adapter timeout by default).
+          const watchdog = new Promise((resolve) => setTimeout(() => resolve('__watchdog__'), JOB_MAX_MS).unref());
+          Promise.race([processJob(bridge, agentCfg, job, p2p), watchdog])
+            .then(async (r) => {
+              if (r === '__watchdog__') {
+                logger.error(`job ${job.id} exceeded ${JOB_MAX_MS}ms — reclaiming slot`);
+                try { await bridge.fail(job.id, `host watchdog: job exceeded ${JOB_MAX_MS}ms`); } catch { /* ignore */ }
+              }
+            })
+            .finally(() => { inFlight--; activeJobIds.delete(job.id); });
+        }
+      } finally {
+        // Decrement AFTER activeJobIds.add above (no await between them), so a claimed job hands off
+        // from claimsInFlight to activeJobIds without the busy signal ever dipping to false.
+        claimsInFlight--;
       }
+      if (job) continue; // filled a slot — try to fill remaining concurrency immediately
     }
     await sleep(cfg.poll_interval_ms || 1500);
   }
@@ -193,12 +217,43 @@ async function startWorker() {
     ? `H2H transport: native P2P active (${p2pAddr})`
     : 'H2H transport: central relay (no native core for this platform — install acceleration later)');
 
+  // Advertise liveness + in-flight count so `clhost update`/`restart` can wait for this host to go
+  // idle before restarting, and pick up the CLI's drain flag (stop claiming, finish current work).
+  // Clear any stale flag first; and only ever honor a flag written AFTER this boot, so a leftover
+  // flag from a crashed update (or a clearDrain that failed) can never wedge a fresh worker.
+  control.clearDrain();      // drop any stale flag left by a previous life FIRST…
+  const bootAt = Date.now(); // …then anchor "now": only a drain requested after this point is honored.
+  const pushStatus = () => {
+    drainFlag = control.drainRequestedSince(bootAt);
+    const busy = activeJobIds.size > 0 || claimsInFlight > 0;
+    control.writeStatus({
+      pid: process.pid, version: VERSION, agents: agents.length,
+      activeJobs: activeJobIds.size, busy, drain: drainFlag,
+      currentJobId: activeJobIds.size ? [...activeJobIds][0] : null,
+    });
+  };
+  pushStatus();
+  const statusTimer = setInterval(pushStatus, 2000);
+  statusTimer.unref();
+
+  // Graceful shutdown: stop claiming, let in-flight jobs finish (keeping P2P alive for them), then
+  // exit. A second signal forces an immediate exit; SHUTDOWN_DRAIN_MS bounds the wait.
+  let stopSignals = 0;
   const onStop = (sig) => {
-    if (stopping) return;
+    stopSignals += 1;
+    if (stopSignals >= 2) { logger.info(`${sig} again — exiting now (in-flight work abandoned).`); process.exit(0); }
     stopping = true;
-    logger.info(`${sig} received — draining in-flight jobs…`);
-    try { p2p.close(); } catch { /* ignore */ }
-    setTimeout(() => process.exit(0), 8000).unref();
+    const n = activeJobIds.size;
+    logger.info(n
+      ? `${sig} received — finishing ${n} in-flight job(s) before exit (send again to force)…`
+      : `${sig} received — exiting.`);
+    const startedAt = Date.now();
+    const finish = () => { try { p2p.close(); } catch { /* ignore */ } process.exit(0); };
+    const tick = setInterval(() => {
+      if (activeJobIds.size === 0) { clearInterval(tick); logger.info('drained — exiting.'); finish(); }
+      else if (Date.now() - startedAt > SHUTDOWN_DRAIN_MS) { clearInterval(tick); logger.info('drain timed out — exiting.'); finish(); }
+    }, 500);
+    tick.unref();
   };
   process.on('SIGINT', () => onStop('SIGINT'));
   process.on('SIGTERM', () => onStop('SIGTERM'));