@tablation/crew 0.69.1 → 0.70.0

package/dist/cli.js

@@ -15265,6 +15265,37 @@ function syncPassengerSyncDaemons(plans, cfg, stateDir, deps = {}) {
   return endpoints;
 }
 
+// src/outage.ts
+var TRANSIENT_BACKOFF_MS = [15e3, 3e4, 6e4];
+var OUTAGE_ESCALATE_AFTER = 5;
+var TRACKER_OUTAGE_KEY = "tracker_unreachable";
+var NETWORK_CODES = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EAI_AGAIN"];
+function transientBackoffMs(n) {
+  return TRANSIENT_BACKOFF_MS[Math.min(Math.max(n, 1), TRANSIENT_BACKOFF_MS.length) - 1];
+}
+function codeOf(e) {
+  const code = e?.code;
+  return typeof code === "string" ? code : void 0;
+}
+function isTransientError(err) {
+  if (!(err instanceof Error)) return false;
+  const msg = err.message;
+  if (msg === "fetch failed" || msg.startsWith("fetch failed")) return true;
+  for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
+    const code = codeOf(e);
+    if (code && NETWORK_CODES.includes(code)) return true;
+  }
+  if (NETWORK_CODES.some((c) => msg.includes(c))) return true;
+  if (/\b(?:HTTP|status)\s*:?\s*(502|503|504)\b/i.test(msg) || /^(502|503|504)\b/.test(msg)) return true;
+  if (err instanceof SyntaxError && /Unexpected token '?<|is not valid JSON|Unexpected end of JSON/.test(msg)) return true;
+  return false;
+}
+function describeCause(err) {
+  const first = (err instanceof Error ? err.message : String(err)).split("\n")[0];
+  const code = codeOf(err?.cause);
+  return code && !first.includes(code) ? `${first} (${code})` : first;
+}
+
 // src/daemon.ts
 import { existsSync as existsSync32, mkdirSync as mkdirSync19, readFileSync as readFileSync26, readdirSync as readdirSync13, rmSync as rmSync13, statSync as statSync9, writeFileSync as writeFileSync20 } from "fs";
 import { join as join33 } from "path";
@@ -15364,12 +15395,32 @@ async function runDaemonLoop(o) {
   const sleep = o.sleep ?? defaultSleep;
   let backoff = floor;
   let passes = 0;
+  const now = o.now ?? Date.now;
+  let transientRun = 0;
+  let downSince = 0;
   for (; ; ) {
     if (o.signal?.aborted) return "aborted";
     let result;
     try {
       result = await o.runPass();
+      if (transientRun > 0) {
+        o.onRecovered?.({ downMs: now() - downSince });
+        transientRun = 0;
+      }
     } catch (err) {
+      if (isTransientError(err)) {
+        passes++;
+        if (transientRun === 0) downSince = now();
+        transientRun++;
+        const wait = transientBackoffMs(transientRun);
+        const downMs = now() - downSince;
+        o.onTransient?.({ err, consecutive: transientRun, backoffMs: wait, downMs });
+        if (transientRun === OUTAGE_ESCALATE_AFTER) o.onOutage?.({ err, consecutive: transientRun, downMs });
+        if (o.maxPasses !== void 0 && passes >= o.maxPasses) return "maxPasses";
+        if (o.signal?.aborted) return "aborted";
+        await sleep(wait);
+        continue;
+      }
       passes++;
       o.onPassError?.(err);
       if (o.maxPasses !== void 0 && passes >= o.maxPasses) return "maxPasses";
@@ -17466,6 +17517,23 @@ ${changed2.map((c) => `  - ${c}
 `)
           })
         }),
+        onTransient: ({ err, consecutive, backoffMs }) => emit.warn(
+          `tracker unavailable (${describeCause(err)}); attempt ${consecutive}, retrying in ${Math.round(backoffMs / 1e3)}s`
+        ),
+        onOutage: ({ err, downMs }) => {
+          const msg = `tracker unreachable for ${Math.max(1, Math.round(downMs / 6e4))} minutes: ${describeCause(err)}`;
+          emit.error(msg);
+          void raiseShipAttention({ state, emit, route, ship: cfg.ship }, {
+            kind: "tracker_unreachable",
+            key: TRACKER_OUTAGE_KEY,
+            message: msg,
+            since: (/* @__PURE__ */ new Date()).toISOString()
+          });
+        },
+        onRecovered: ({ downMs }) => {
+          clearShipAttention(state, TRACKER_OUTAGE_KEY);
+          emit.emit(`tracker is back after ${Math.max(1, Math.round(downMs / 1e3))}s`);
+        },
         onPassError: (err) => emit.error(
           `daemon pass failed, continuing: ${err instanceof Error ? err.stack ?? err.message : String(err)}`
         ),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tablation/crew",
-  "version": "0.69.1",
+  "version": "0.70.0",
   "description": "A standing team of headless agents that picks work off a Tablation board, does it on a machine you control, and reports back.",
   "type": "module",
   "license": "MIT",