@shipers-dev/multi 0.9.6 → 0.10.1

package/dist/index.js

@@ -5703,7 +5703,7 @@ import { join as join3, dirname as dirname2 } from "path";
 // package.json
 var package_default = {
   name: "@shipers-dev/multi",
-  version: "0.9.6",
+  version: "0.10.1",
   type: "module",
   bin: {
     "multi-agent": "./dist/index.js"
@@ -6052,6 +6052,9 @@ async function cmdConnect(apiUrl, config) {
             if (t.issue_id) {
               postStream(apiUrl, t.issue_id, "queued", { queue_position: pos });
             }
+            if (t.dispatch_id) {
+              ackDispatch(apiUrl, t.dispatch_id, config.dispatchSecret);
+            }
             queueMicrotask(() => schedule());
             return Response.json({ accepted: true, task_id: taskId }, { status: 202 });
           } catch (e) {
@@ -6153,7 +6156,16 @@ async function cmdConnect(apiUrl, config) {
     process.exit(1);
   }
   log(`\u2601\uFE0F  Tunnel up: ${tunnel.url}`);
-  await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "online", tunnel_url: tunnel.url });
+  const heartbeat = async () => {
+    const res = await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "online", tunnel_url: tunnel?.url });
+    return res.success && res.data?.pending_dispatches || 0;
+  };
+  {
+    const pending = await heartbeat();
+    if (pending > 0)
+      drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
+  }
+  drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
   let alive = true;
   const shutdown = async (reason) => {
     if (!alive)
@@ -6182,28 +6194,38 @@ async function cmdConnect(apiUrl, config) {
   process.on("SIGINT", () => shutdown("SIGINT"));
   process.on("SIGTERM", () => shutdown("SIGTERM"));
   schedule();
+  let restarting = false;
   const restartTunnel = async (reason) => {
-    if (!alive)
+    if (!alive || restarting)
       return;
-    log(`\uD83D\uDD01 Restarting tunnel (${reason})`);
+    restarting = true;
     try {
-      tunnel?.child.kill();
-    } catch {}
-    for (let attempt = 1;alive; attempt++) {
-      const next = await startTunnel(port);
-      if (next) {
-        tunnel = next;
-        log(`\u2601\uFE0F  Tunnel up: ${tunnel.url}`);
-        try {
-          await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "online", tunnel_url: tunnel.url });
-        } catch (e) {
-          log(`heartbeat error after tunnel restart: ${String(e)}`);
+      log(`\uD83D\uDD01 Restarting tunnel (${reason})`);
+      const old = tunnel;
+      tunnel = null;
+      try {
+        old?.child.kill();
+      } catch {}
+      for (let attempt = 1;alive; attempt++) {
+        const next = await startTunnel(port);
+        if (next) {
+          tunnel = next;
+          log(`\u2601\uFE0F  Tunnel up: ${tunnel.url}`);
+          try {
+            const pending = await heartbeat();
+            if (pending > 0)
+              drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
+          } catch (e) {
+            log(`heartbeat error after tunnel restart: ${String(e)}`);
+          }
+          return;
         }
-        return;
+        const wait = Math.min(30000, 2000 * attempt);
+        log(`tunnel restart failed, retry in ${wait}ms`);
+        await sleep(wait);
       }
-      const wait = Math.min(30000, 2000 * attempt);
-      log(`tunnel restart failed, retry in ${wait}ms`);
-      await sleep(wait);
+    } finally {
+      restarting = false;
     }
   };
   (async () => {
@@ -6246,7 +6268,9 @@ async function cmdConnect(apiUrl, config) {
       }
     }
     try {
-      await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: "online", tunnel_url: tunnel?.url });
+      const pending = await heartbeat();
+      if (pending > 0)
+        drainOfflineDispatches(apiUrl, config.deviceId, config.dispatchSecret, db, () => schedule());
     } catch (e) {
       log(`heartbeat error: ${String(e)}`);
     }
@@ -7060,6 +7084,50 @@ async function resolveAcpAdapter(agentType, detectedPath) {
       return [c];
   return null;
 }
+async function ackDispatch(apiUrl, dispatchId, secret) {
+  try {
+    await fetch(`${apiUrl}/api/issues/dispatches/${dispatchId}/ack`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${secret}` }
+    });
+  } catch (e) {
+    log(`ack dispatch ${dispatchId} failed: ${String(e)}`);
+  }
+}
+async function drainOfflineDispatches(apiUrl, deviceId, secret, db, onEnqueued) {
+  let list;
+  try {
+    list = await apiClient.get(`${apiUrl}/api/devices/${deviceId}/dispatches/pending`);
+  } catch (e) {
+    log(`drain list failed: ${String(e)}`);
+    return;
+  }
+  const rows = list?.data?.results || list?.data || list?.results || list || [];
+  if (!Array.isArray(rows) || rows.length === 0)
+    return;
+  log(`\uD83E\uDEA3 draining ${rows.length} offline dispatch(es)`);
+  for (const r of rows) {
+    try {
+      const res = await fetch(`${apiUrl}/api/issues/dispatches/${r.id}/claim`, {
+        method: "POST",
+        headers: { authorization: `Bearer ${secret}` }
+      });
+      if (!res.ok) {
+        log(`claim ${r.id} skipped: ${res.status}`);
+        continue;
+      }
+      const { task } = await res.json();
+      const taskId = task.issue_id ? `${task.issue_id}-${Date.now()}` : crypto.randomUUID();
+      db.run("INSERT INTO tasks (id, status, payload, agent_id, issue_id) VALUES (?, 'queued', ?, ?, ?)", [taskId, JSON.stringify(task), task.agent_id ?? null, task.issue_id ?? null]);
+      if (task.issue_id)
+        postStream(apiUrl, task.issue_id, "queued", { drained: true });
+      ackDispatch(apiUrl, r.id, secret);
+    } catch (e) {
+      log(`drain ${r.id} failed: ${String(e)}`);
+    }
+  }
+  onEnqueued();
+}
 async function postStream(apiUrl, issueId, event_type, payload) {
   try {
     ensureDirs();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shipers-dev/multi",
-  "version": "0.9.6",
+  "version": "0.10.1",
   "type": "module",
   "bin": {
     "multi-agent": "./dist/index.js"

package/src/index.ts

@@ -355,6 +355,11 @@ async function cmdConnect(apiUrl: string, config: Config) {
             if (t.issue_id) {
               void postStream(apiUrl, t.issue_id, 'queued', { queue_position: pos });
             }
+            // Ack dispatch back to worker so UI flips from queued → acked. Fire-and-forget:
+            // missing/failed ack is fine, stall-sweep covers it.
+            if (t.dispatch_id) {
+              void ackDispatch(apiUrl, t.dispatch_id, config.dispatchSecret!);
+            }
             queueMicrotask(() => schedule());
             return Response.json({ accepted: true, task_id: taskId }, { status: 202 });
           } catch (e) {
@@ -441,7 +446,17 @@ async function cmdConnect(apiUrl: string, config: Config) {
   }
   log(`☁️  Tunnel up: ${tunnel.url}`);
 
-  await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'online', tunnel_url: tunnel.url });
+  const heartbeat = async (): Promise<number> => {
+    const res = await apiClient.post<{ pending_dispatches?: number }>(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'online', tunnel_url: tunnel?.url });
+    return (res.success && res.data?.pending_dispatches) || 0;
+  };
+  {
+    const pending = await heartbeat();
+    if (pending > 0) void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
+  }
+  // Always attempt one drain on fresh startup, covering the case where worker
+  // couldn't reach the previous daemon instance right before restart.
+  void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
 
   let alive = true;
 
@@ -466,26 +481,36 @@ async function cmdConnect(apiUrl: string, config: Config) {
   schedule();
 
   // Tunnel self-heal: relaunch cloudflared if child exits or DNS stops resolving.
+  // `restarting` guards against two entries racing (probe-failure + exit-watcher
+  // both firing when we kill the child as part of our own restart).
+  let restarting = false;
   const restartTunnel = async (reason: string) => {
-    if (!alive) return;
-    log(`🔁 Restarting tunnel (${reason})`);
-    try { tunnel?.child.kill(); } catch {}
-    // Small backoff so we don't spam cloudflared edge under prolonged outage.
-    for (let attempt = 1; alive; attempt++) {
-      const next = await startTunnel(port);
-      if (next) {
-        tunnel = next;
-        log(`☁️  Tunnel up: ${tunnel.url}`);
-        try {
-          await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'online', tunnel_url: tunnel.url });
-        } catch (e) {
-          log(`heartbeat error after tunnel restart: ${String(e)}`);
+    if (!alive || restarting) return;
+    restarting = true;
+    try {
+      log(`🔁 Restarting tunnel (${reason})`);
+      const old = tunnel;
+      tunnel = null; // null first so the exit-watcher's `tunnel === t` check skips our own kill
+      try { old?.child.kill(); } catch {}
+      for (let attempt = 1; alive; attempt++) {
+        const next = await startTunnel(port);
+        if (next) {
+          tunnel = next;
+          log(`☁️  Tunnel up: ${tunnel.url}`);
+          try {
+            const pending = await heartbeat();
+            if (pending > 0) void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
+          } catch (e) {
+            log(`heartbeat error after tunnel restart: ${String(e)}`);
+          }
+          return;
         }
-        return;
+        const wait = Math.min(30000, 2000 * attempt);
+        log(`tunnel restart failed, retry in ${wait}ms`);
+        await sleep(wait);
       }
-      const wait = Math.min(30000, 2000 * attempt);
-      log(`tunnel restart failed, retry in ${wait}ms`);
-      await sleep(wait);
+    } finally {
+      restarting = false;
     }
   };
 
@@ -496,6 +521,7 @@ async function cmdConnect(apiUrl: string, config: Config) {
       if (!t) { await sleep(1000); continue; }
       const code = await t.child.exited;
       if (!alive) return;
+      // If `tunnel` was nulled by restartTunnel, we killed it ourselves — skip.
       if (tunnel === t) await restartTunnel(`cloudflared exited code=${code}`);
     }
   })();
@@ -524,7 +550,8 @@ async function cmdConnect(apiUrl: string, config: Config) {
       }
     }
     try {
-      await apiClient.post(`${apiUrl}/api/devices/${config.deviceId}/heartbeat`, { status: 'online', tunnel_url: tunnel?.url });
+      const pending = await heartbeat();
+      if (pending > 0) void drainOfflineDispatches(apiUrl, config.deviceId!, config.dispatchSecret!, db, () => schedule());
     } catch (e) {
       log(`heartbeat error: ${String(e)}`);
     }
@@ -1247,6 +1274,55 @@ async function resolveAcpAdapter(agentType: string, detectedPath?: string): Prom
   return null;
 }
 
+// Ack a dispatch so the worker flips its status from dispatched → acked.
+// Uses dispatch_secret (Bearer) so the daemon can call an unauthenticated-by-user endpoint.
+async function ackDispatch(apiUrl: string, dispatchId: string, secret: string) {
+  try {
+    await fetch(`${apiUrl}/api/issues/dispatches/${dispatchId}/ack`, {
+      method: 'POST',
+      headers: { 'authorization': `Bearer ${secret}` },
+    });
+  } catch (e) {
+    log(`ack dispatch ${dispatchId} failed: ${String(e)}`);
+  }
+}
+
+// On reconnect, pull dispatches the worker couldn't deliver and enqueue them.
+// Each call claims the row (offline → dispatched) atomically so parallel daemons
+// don't double-run the same task.
+async function drainOfflineDispatches(apiUrl: string, deviceId: string, secret: string, db: Database, onEnqueued: () => void) {
+  let list: any;
+  try {
+    list = await apiClient.get<any>(`${apiUrl}/api/devices/${deviceId}/dispatches/pending`);
+  } catch (e) {
+    log(`drain list failed: ${String(e)}`);
+    return;
+  }
+  const rows = (list?.data?.results || list?.data || list?.results || list || []) as any[];
+  if (!Array.isArray(rows) || rows.length === 0) return;
+  log(`🪣 draining ${rows.length} offline dispatch(es)`);
+  for (const r of rows) {
+    try {
+      const res = await fetch(`${apiUrl}/api/issues/dispatches/${r.id}/claim`, {
+        method: 'POST',
+        headers: { 'authorization': `Bearer ${secret}` },
+      });
+      if (!res.ok) { log(`claim ${r.id} skipped: ${res.status}`); continue; }
+      const { task } = await res.json() as { task: any };
+      const taskId = task.issue_id ? `${task.issue_id}-${Date.now()}` : crypto.randomUUID();
+      db.run(
+        "INSERT INTO tasks (id, status, payload, agent_id, issue_id) VALUES (?, 'queued', ?, ?, ?)",
+        [taskId, JSON.stringify(task), task.agent_id ?? null, task.issue_id ?? null],
+      );
+      if (task.issue_id) void postStream(apiUrl, task.issue_id, 'queued', { drained: true });
+      void ackDispatch(apiUrl, r.id, secret);
+    } catch (e) {
+      log(`drain ${r.id} failed: ${String(e)}`);
+    }
+  }
+  onEnqueued();
+}
+
 async function postStream(apiUrl: string, issueId: string, event_type: string, payload: any) {
   // Local ndjson sink for tail -f debugging.
   try {