@shipers-dev/multi 0.9.5 → 0.10.0

package/dist/index.js

@@ -5703,7 +5703,7 @@ import { join as join3, dirname as dirname2 } from "path";
 // package.json
 var package_default = {
   name: "@shipers-dev/multi",
-  version: "0.9.5",
+  version: "0.10.0",
   type: "module",
   bin: {
     "multi-agent": "./dist/index.js"
@@ -6052,6 +6052,9 @@ async function cmdConnect(apiUrl, config) {
             if (t.issue_id) {
               postStream(apiUrl, t.issue_id, "queued", { queue_position: pos });
             }
+            if (t.dispatch_id) {
+              ackDispatch(apiUrl, t.dispatch_id, config.dispatchSecret);
+            }
             queueMicrotask(() => schedule());
             return Response.json({ accepted: true, task_id: taskId }, { status: 202 });
           } catch (e) {
@@ -6144,24 +6147,25 @@ async function cmdConnect(apiUrl, config) {
   try {
     writeFileSync3(PORT_PATH, String(port));
   } catch {}
-  const cf = Bun.spawn(["cloudflared", "tunnel", "--no-autoupdate", "--url", `http://127.0.0.1:${port}`], {
-    stdout: "pipe",
-    stderr: "pipe",
-    stdin: "ignore"
-  });
-  const tunnelUrl = await parseTunnelUrl(cf.stderr);
-  if (!tunnelUrl) {
+  let tunnel = await startTunnel(port);
+  if (!tunnel) {
     log("\u274C cloudflared did not emit a tunnel URL \u2014 is `cloudflared` installed? (`brew install cloudflared`)");
-    try {
-      cf.kill();
-    } catch {}
     try {
       server.stop();
     } catch {}
     process.exit(1);
   }
-  log(`\u2601\uFE0F  Tunnel up: ${tunnelUrl}`);
-  await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "online", tunnel_url: tunnelUrl });
+  log(`\u2601\uFE0F  Tunnel up: ${tunnel.url}`);
+  const heartbeat = async () => {
+    const res = await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "online", tunnel_url: tunnel?.url });
+    return res.success && res.data?.pending_dispatches || 0;
+  };
+  {
+    const pending = await heartbeat();
+    if (pending > 0)
+      drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
+  }
+  drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
   let alive = true;
   const shutdown = async (reason) => {
     if (!alive)
@@ -6172,7 +6176,7 @@ async function cmdConnect(apiUrl, config) {
       server.stop();
     } catch {}
     try {
-      cf.kill();
+      tunnel?.child.kill();
     } catch {}
     try {
       await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "offline", tunnel_url: null });
@@ -6190,19 +6194,106 @@ async function cmdConnect(apiUrl, config) {
   process.on("SIGINT", () => shutdown("SIGINT"));
   process.on("SIGTERM", () => shutdown("SIGTERM"));
   schedule();
+  const restartTunnel = async (reason) => {
+    if (!alive)
+      return;
+    log(`\uD83D\uDD01 Restarting tunnel (${reason})`);
+    try {
+      tunnel?.child.kill();
+    } catch {}
+    for (let attempt = 1;alive; attempt++) {
+      const next = await startTunnel(port);
+      if (next) {
+        tunnel = next;
+        log(`\u2601\uFE0F  Tunnel up: ${tunnel.url}`);
+        try {
+          const pending = await heartbeat();
+          if (pending > 0)
+            drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
+        } catch (e) {
+          log(`heartbeat error after tunnel restart: ${String(e)}`);
+        }
+        return;
+      }
+      const wait = Math.min(30000, 2000 * attempt);
+      log(`tunnel restart failed, retry in ${wait}ms`);
+      await sleep(wait);
+    }
+  };
+  (async () => {
+    while (alive) {
+      const t = tunnel;
+      if (!t) {
+        await sleep(1000);
+        continue;
+      }
+      const code = await t.child.exited;
+      if (!alive)
+        return;
+      if (tunnel === t)
+        await restartTunnel(`cloudflared exited code=${code}`);
+    }
+  })();
+  let probeFailures = 0;
+  let tick = 0;
+  const PROBE_EVERY = 6;
   while (alive) {
     await sleep(20000);
     if (existsSync3(STOP_PATH)) {
       await shutdown("stop flag");
       break;
     }
+    tick++;
+    const currentUrl = tunnel?.url;
+    if (currentUrl && tick % PROBE_EVERY === 0) {
+      const ok = await probeTunnel(currentUrl);
+      if (!ok) {
+        probeFailures++;
+        log(`tunnel probe failed (${probeFailures}/2): ${currentUrl}`);
+        if (probeFailures >= 2) {
+          probeFailures = 0;
+          await restartTunnel("probe failed 2x");
+          continue;
+        }
+      } else {
+        probeFailures = 0;
+      }
+    }
     try {
-      await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "online", tunnel_url: tunnelUrl });
+      const pending = await heartbeat();
+      if (pending > 0)
+        drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
     } catch (e) {
       log(`heartbeat error: ${String(e)}`);
     }
   }
 }
+async function startTunnel(port) {
+  const child = Bun.spawn(["cloudflared", "tunnel", "--no-autoupdate", "--url", `http://127.0.0.1:${port}`], {
+    stdout: "pipe",
+    stderr: "pipe",
+    stdin: "ignore"
+  });
+  const url = await parseTunnelUrl(child.stderr);
+  if (!url) {
+    try {
+      child.kill();
+    } catch {}
+    return null;
+  }
+  return { child, url };
+}
+async function probeTunnel(url) {
+  try {
+    const ctrl = new AbortController;
+    const t = setTimeout(() => ctrl.abort(), 8000);
+    const res = await fetch(url, { method: "GET", signal: ctrl.signal });
+    clearTimeout(t);
+    return res.status > 0;
+  } catch {
+    return false;
+  }
+}
 async function cmdConnectDetached(apiUrl) {
   if (existsSync3(PID_PATH)) {
     const pid = Number(readFileSync3(PID_PATH, "utf8").trim());
@@ -6985,6 +7076,50 @@ async function resolveAcpAdapter(agentType, detectedPath) {
       return [c];
   return null;
 }
+async function ackDispatch(apiUrl, dispatchId, secret) {
+  try {
+    await fetch(`${apiUrl}/api/issues/dispatches/${dispatchId}/ack`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${secret}` }
+    });
+  } catch (e) {
+    log(`ack dispatch ${dispatchId} failed: ${String(e)}`);
+  }
+}
+async function drainOfflineDispatches(apiUrl, deviceId, secret, db, onEnqueued) {
+  let list;
+  try {
+    list = await apiClient.get(`${apiUrl}/api/devices/${deviceId}/dispatches/pending`);
+  } catch (e) {
+    log(`drain list failed: ${String(e)}`);
+    return;
+  }
+  const rows = list?.data?.results || list?.data || list?.results || list || [];
+  if (!Array.isArray(rows) || rows.length === 0)
+    return;
+  log(`\uD83E\uDEA3 draining ${rows.length} offline dispatch(es)`);
+  for (const r of rows) {
+    try {
+      const res = await fetch(`${apiUrl}/api/issues/dispatches/${r.id}/claim`, {
+        method: "POST",
+        headers: { authorization: `Bearer ${secret}` }
+      });
+      if (!res.ok) {
+        log(`claim ${r.id} skipped: ${res.status}`);
+        continue;
+      }
+      const { task } = await res.json();
+      const taskId = task.issue_id ? `${task.issue_id}-${Date.now()}` : crypto.randomUUID();
+      db.run("INSERT INTO tasks (id, status, payload, agent_id, issue_id) VALUES (?, 'queued', ?, ?, ?)", [taskId, JSON.stringify(task), task.agent_id ?? null, task.issue_id ?? null]);
+      if (task.issue_id)
+        postStream(apiUrl, task.issue_id, "queued", { drained: true });
+      ackDispatch(apiUrl, r.id, secret);
+    } catch (e) {
+      log(`drain ${r.id} failed: ${String(e)}`);
+    }
+  }
+  onEnqueued();
+}
 async function postStream(apiUrl, issueId, event_type, payload) {
   try {
     ensureDirs();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shipers-dev/multi",
-  "version": "0.9.5",
+  "version": "0.10.0",
   "type": "module",
   "bin": {
     "multi-agent": "./dist/index.js"

package/src/index.ts

@@ -355,6 +355,11 @@ async function cmdConnect(apiUrl: string, config: Config) {
             if (t.issue_id) {
               void postStream(apiUrl, t.issue_id, 'queued', { queue_position: pos });
             }
+            // Ack dispatch back to worker so UI flips from queued → acked. Fire-and-forget:
+            // missing/failed ack is fine, stall-sweep covers it.
+            if (t.dispatch_id) {
+              void ackDispatch(apiUrl, t.dispatch_id, config.dispatchSecret!);
+            }
             queueMicrotask(() => schedule());
             return Response.json({ accepted: true, task_id: taskId }, { status: 202 });
           } catch (e) {
@@ -432,20 +437,26 @@ async function cmdConnect(apiUrl: string, config: Config) {
   log(`🌐 Local server: http://127.0.0.1:${port}`);
   try { writeFileSync(PORT_PATH, String(port)); } catch {}
 
-  // Spawn cloudflared quick tunnel
-  const cf = Bun.spawn(['cloudflared', 'tunnel', '--no-autoupdate', '--url', `http://127.0.0.1:${port}`], {
-    stdout: 'pipe', stderr: 'pipe', stdin: 'ignore',
-  });
-  const tunnelUrl = await parseTunnelUrl(cf.stderr as ReadableStream<Uint8Array>);
-  if (!tunnelUrl) {
+  // Spawn cloudflared quick tunnel (with self-healing on death)
+  let tunnel = await startTunnel(port);
+  if (!tunnel) {
     log('❌ cloudflared did not emit a tunnel URL — is `cloudflared` installed? (`brew install cloudflared`)');
-    try { cf.kill(); } catch {}
     try { server.stop(); } catch {}
     process.exit(1);
   }
-  log(`☁️  Tunnel up: ${tunnelUrl}`);
+  log(`☁️  Tunnel up: ${tunnel.url}`);
 
-  await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'online', tunnel_url: tunnelUrl });
+  const heartbeat = async (): Promise<number> => {
+    const res = await apiClient.post<{ pending_dispatches?: number }>(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'online', tunnel_url: tunnel?.url });
+    return (res.success && res.data?.pending_dispatches) || 0;
+  };
+  {
+    const pending = await heartbeat();
+    if (pending > 0) void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
+  }
+  // Always attempt one drain on fresh startup, covering the case where worker
+  // couldn't reach the previous daemon instance right before restart.
+  void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
 
   let alive = true;
 
@@ -454,7 +465,7 @@ async function cmdConnect(apiUrl: string, config: Config) {
     alive = false;
     log(`🛑 Shutting down (${reason})`);
     try { server.stop(); } catch {}
-    try { cf.kill(); } catch {}
+    try { tunnel?.child.kill(); } catch {}
     try { await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'offline', tunnel_url: null }); } catch {}
     if (existsSync(PID_PATH)) unlinkSync(PID_PATH);
     if (existsSync(STOP_PATH)) unlinkSync(STOP_PATH);
@@ -469,18 +480,99 @@ async function cmdConnect(apiUrl: string, config: Config) {
   // Kick the scheduler on startup to drain any leftover queued rows.
   schedule();
 
-  // Heartbeat loop
+  // Tunnel self-heal: relaunch cloudflared if child exits or DNS stops resolving.
+  const restartTunnel = async (reason: string) => {
+    if (!alive) return;
+    log(`🔁 Restarting tunnel (${reason})`);
+    try { tunnel?.child.kill(); } catch {}
+    // Small backoff so we don't spam cloudflared edge under prolonged outage.
+    for (let attempt = 1; alive; attempt++) {
+      const next = await startTunnel(port);
+      if (next) {
+        tunnel = next;
+        log(`☁️  Tunnel up: ${tunnel.url}`);
+        try {
+          const pending = await heartbeat();
+          if (pending > 0) void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
+        } catch (e) {
+          log(`heartbeat error after tunnel restart: ${String(e)}`);
+        }
+        return;
+      }
+      const wait = Math.min(30000, 2000 * attempt);
+      log(`tunnel restart failed, retry in ${wait}ms`);
+      await sleep(wait);
+    }
+  };
+
+  // Watch for cloudflared process exit.
+  (async () => {
+    while (alive) {
+      const t = tunnel;
+      if (!t) { await sleep(1000); continue; }
+      const code = await t.child.exited;
+      if (!alive) return;
+      if (tunnel === t) await restartTunnel(`cloudflared exited code=${code}`);
+    }
+  })();
+
+  // Heartbeat (20s) + tunnel liveness probe (every 6 ticks = ~2min).
+  let probeFailures = 0;
+  let tick = 0;
+  const PROBE_EVERY = 6;
   while (alive) {
     await sleep(20000);
     if (existsSync(STOP_PATH)) { await shutdown('stop flag'); break; }
+    tick++;
+    const currentUrl = tunnel?.url;
+    if (currentUrl && tick % PROBE_EVERY === 0) {
+      const ok = await probeTunnel(currentUrl);
+      if (!ok) {
+        probeFailures++;
+        log(`tunnel probe failed (${probeFailures}/2): ${currentUrl}`);
+        if (probeFailures >= 2) {
+          probeFailures = 0;
+          await restartTunnel('probe failed 2x');
+          continue;
+        }
+      } else {
+        probeFailures = 0;
+      }
+    }
     try {
-      await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'online', tunnel_url: tunnelUrl });
+      const pending = await heartbeat();
+      if (pending > 0) void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
     } catch (e) {
       log(`heartbeat error: ${String(e)}`);
     }
   }
 }
 
+async function startTunnel(port: number): Promise<{ child: ReturnType<typeof Bun.spawn>; url: string } | null> {
+  const child = Bun.spawn(['cloudflared', 'tunnel', '--no-autoupdate', '--url', `http://127.0.0.1:${port}`], {
+    stdout: 'pipe', stderr: 'pipe', stdin: 'ignore',
+  });
+  const url = await parseTunnelUrl(child.stderr as ReadableStream<Uint8Array>);
+  if (!url) {
+    try { child.kill(); } catch {}
+    return null;
+  }
+  return { child, url };
+}
+
+async function probeTunnel(url: string): Promise<boolean> {
+  try {
+    const ctrl = new AbortController();
+    const t = setTimeout(() => ctrl.abort(), 8000);
+    // Any HTTP response — even 404 — proves the tunnel edge is routing.
+    const res = await fetch(url, { method: 'GET', signal: ctrl.signal });
+    clearTimeout(t);
+    return res.status > 0;
+  } catch {
+    return false;
+  }
+}
+
 async function cmdConnectDetached(apiUrl: string) {
   if (existsSync(PID_PATH)) {
     const pid = Number(readFileSync(PID_PATH, 'utf8').trim());
@@ -1172,6 +1264,55 @@ async function resolveAcpAdapter(agentType: string, detectedPath?: string): Prom
   return null;
 }
 
+// Ack a dispatch so the worker flips its status from dispatched → acked.
+// Uses dispatch_secret (Bearer) so the daemon can call an unauthenticated-by-user endpoint.
+async function ackDispatch(apiUrl: string, dispatchId: string, secret: string) {
+  try {
+    await fetch(`${apiUrl}/api/issues/dispatches/${dispatchId}/ack`, {
+      method: 'POST',
+      headers: { 'authorization': `Bearer ${secret}` },
+    });
+  } catch (e) {
+    log(`ack dispatch ${dispatchId} failed: ${String(e)}`);
+  }
+}
+
+// On reconnect, pull dispatches the worker couldn't deliver and enqueue them.
+// Each call claims the row (offline → dispatched) atomically so parallel daemons
+// don't double-run the same task.
+async function drainOfflineDispatches(apiUrl: string, deviceId: string, secret: string, db: Database, onEnqueued: () => void) {
+  let list: any;
+  try {
+    list = await apiClient.get<any>(`${apiUrl}/api/devices/${deviceId}/dispatches/pending`);
+  } catch (e) {
+    log(`drain list failed: ${String(e)}`);
+    return;
+  }
+  const rows = (list?.data?.results || list?.data || list?.results || list || []) as any[];
+  if (!Array.isArray(rows) || rows.length === 0) return;
+  log(`🪣 draining ${rows.length} offline dispatch(es)`);
+  for (const r of rows) {
+    try {
+      const res = await fetch(`${apiUrl}/api/issues/dispatches/${r.id}/claim`, {
+        method: 'POST',
+        headers: { 'authorization': `Bearer ${secret}` },
+      });
+      if (!res.ok) { log(`claim ${r.id} skipped: ${res.status}`); continue; }
+      const { task } = await res.json() as { task: any };
+      const taskId = task.issue_id ? `${task.issue_id}-${Date.now()}` : crypto.randomUUID();
+      db.run(
+        "INSERT INTO tasks (id, status, payload, agent_id, issue_id) VALUES (?, 'queued', ?, ?, ?)",
+        [taskId, JSON.stringify(task), task.agent_id ?? null, task.issue_id ?? null],
+      );
+      if (task.issue_id) void postStream(apiUrl, task.issue_id, 'queued', { drained: true });
+      void ackDispatch(apiUrl, r.id, secret);
+    } catch (e) {
+      log(`drain ${r.id} failed: ${String(e)}`);
+    }
+  }
+  onEnqueued();
+}
+
 async function postStream(apiUrl: string, issueId: string, event_type: string, payload: any) {
   // Local ndjson sink for tail -f debugging.
   try {