@integrity-labs/agt-cli 0.20.3 → 0.20.6

package/dist/lib/manager-worker.js

@@ -22,7 +22,7 @@ import {
   resolveChannels,
   resolveDmTarget,
   wrapScheduledTaskPrompt
-} from "../chunk-INN5PXU3.js";
+} from "../chunk-SOYIZEZG.js";
 import {
   findTaskByTemplate,
   getProjectDir,
@@ -1549,6 +1549,56 @@ async function sendError(flag, opts, text) {
   await sendAck(flag, opts, text);
 }
 
+// src/lib/persistent-session-stuck-tracker.ts
+var DEFAULT_STUCK_WARN_THRESHOLD = 3;
+var PersistentSessionStuckTracker = class {
+  counts = /* @__PURE__ */ new Map();
+  warned = /* @__PURE__ */ new Set();
+  threshold;
+  constructor(threshold = DEFAULT_STUCK_WARN_THRESHOLD) {
+    if (threshold < 1) throw new Error("stuck-tracker threshold must be >= 1");
+    this.threshold = threshold;
+  }
+  /**
+   * Record one tick for one agent. Returns whether the caller should log
+   * a stuck-warning this tick. Re-warning only fires when the counter
+   * climbs above the threshold *again* after a recovery, so a stuck
+   * agent doesn't spam the log every 30s — operators get one WARN per
+   * stuck streak.
+   */
+  record(input) {
+    const { codeName, sessionHealthy, spawnAttempted } = input;
+    const wasStuck = (this.counts.get(codeName) ?? 0) >= this.threshold;
+    if (sessionHealthy || spawnAttempted) {
+      this.counts.delete(codeName);
+      const wasWarned = this.warned.delete(codeName);
+      return {
+        consecutiveTicks: 0,
+        shouldWarn: false,
+        recovered: wasStuck && wasWarned
+      };
+    }
+    const next = (this.counts.get(codeName) ?? 0) + 1;
+    this.counts.set(codeName, next);
+    const crossed = next >= this.threshold && !this.warned.has(codeName);
+    if (crossed) this.warned.add(codeName);
+    return {
+      consecutiveTicks: next,
+      shouldWarn: crossed,
+      recovered: false
+    };
+  }
+  /** Test/debug helper: peek at an agent's current streak. */
+  peek(codeName) {
+    return this.counts.get(codeName) ?? 0;
+  }
+  /** Test helper: clear all state. */
+  reset() {
+    this.counts.clear();
+    this.warned.clear();
+  }
+};
+
 // src/lib/realtime-chat.ts
 import { createClient } from "@supabase/supabase-js";
 var client = null;
@@ -2009,6 +2059,12 @@ function truncateForLog(s) {
   return lines.slice(-PANE_TAIL_PREVIEW_LINES).map((l) => `    | ${l}`).join("\n");
 }
 var KNOWN_SAFE_TAIL_SIGNATURES = /* @__PURE__ */ new Set(["session_id_in_use"]);
+function shouldSkipRevokedCleanup(previousKnownStatus) {
+  return previousKnownStatus === "revoked";
+}
+function hasRevokedResiduals(state2) {
+  return state2.gatewayRunning || state2.portAllocated || state2.provisionDirExists;
+}
 var knownVersions = /* @__PURE__ */ new Map();
 var knownStatuses = /* @__PURE__ */ new Map();
 var knownChannels = /* @__PURE__ */ new Map();
@@ -2144,7 +2200,7 @@ function clearAgentCaches(agentId, codeName) {
 var cachedFrameworkVersion = null;
 var lastVersionCheckAt = 0;
 var VERSION_CHECK_INTERVAL_MS = 5 * 60 * 1e3;
-var agtCliVersion = true ? "0.20.3" : "dev";
+var agtCliVersion = true ? "0.20.6" : "dev";
 function resolveBrewPath(execFileSync4) {
   try {
     const out = execFileSync4("which", ["brew"], { timeout: 5e3 }).toString().trim();
@@ -3308,7 +3364,7 @@ async function getOrCacheRegisteredAgents(adapter, profile) {
 }
 async function processAgent(agent, agentStates) {
   if (!config) return;
-  log(`==================== ${agent.display_name} (${agent.code_name}) ====================`);
+  const previousKnownStatus = knownStatuses.get(agent.agent_id);
   agentDisplayNames.set(agent.code_name, agent.display_name);
   codeNameToAgentId.set(agent.code_name, agent.agent_id);
   if (agent.framework) {
@@ -3318,7 +3374,9 @@ async function processAgent(agent, agentStates) {
   const adapter = resolveAgentFramework(agent.code_name);
   let agentDir = join4(adapter.getAgentDir(agent.code_name), "provision");
   if (agent.status === "draft" || agent.status === "paused") {
-    log(`Agent '${agent.code_name}' is ${agent.status}, skipping provisioning`);
+    if (previousKnownStatus !== agent.status) {
+      log(`Agent '${agent.code_name}' is ${agent.status}, skipping provisioning`);
+    }
     await stopGatewayIfRunning(agent.code_name, adapter);
     stopPersistentSessionAndForgetMcpBaseline(agent.code_name);
     try {
@@ -3343,9 +3401,41 @@ async function processAgent(agent, agentStates) {
       gatewayRunning: false,
       acpSessions: []
     });
+    knownStatuses.set(agent.agent_id, agent.status);
     return;
   }
   if (agent.status === "revoked") {
+    if (shouldSkipRevokedCleanup(previousKnownStatus)) {
+      const ports = loadGatewayPorts();
+      const gatewayLiveness = adapter.isGatewayRunning ? await adapter.isGatewayRunning(agent.code_name).catch(() => ({ running: false })) : { running: false };
+      const residuals = {
+        gatewayRunning: gatewayLiveness.running,
+        portAllocated: Object.prototype.hasOwnProperty.call(ports, agent.code_name),
+        provisionDirExists: existsSync3(agentDir)
+      };
+      if (!hasRevokedResiduals(residuals)) {
+        agentStates.push({
+          agentId: agent.agent_id,
+          codeName: agent.code_name,
+          status: agent.status,
+          charterVersion: "",
+          toolsVersion: "",
+          secretsHash: null,
+          lastRefreshAt: now,
+          lastProvisionAt: null,
+          lastDriftCheckAt: null,
+          lastSecretsProvisionAt: null,
+          gatewayPort: null,
+          gatewayPid: null,
+          gatewayRunning: false,
+          acpSessions: []
+        });
+        return;
+      }
+      log(
+        `[revoked] residuals for '${agent.code_name}': gateway=${residuals.gatewayRunning} dir=${residuals.provisionDirExists} port=${residuals.portAllocated ? ports[agent.code_name] : "none"} \u2014 retrying cleanup`
+      );
+    }
     log(`Agent '${agent.code_name}' is revoked, cleaning up`);
     await stopGatewayIfRunning(agent.code_name, adapter);
     stopPersistentSessionAndForgetMcpBaseline(agent.code_name);
@@ -4221,7 +4311,37 @@ async function processAgent(agent, agentStates) {
   const agentFw = agentFrameworkCache.get(agent.code_name) ?? "openclaw";
   const sessionMode = refreshData.agent.session_mode ?? "oneshot";
   if (agentFw === "claude-code" && sessionMode === "persistent") {
-    await ensurePersistentSession(agent, tasks, boardItems, refreshData);
+    const wasSessionHealthy = isSessionHealthy(agent.code_name);
+    let psResult;
+    try {
+      psResult = await ensurePersistentSession(agent, tasks, boardItems, refreshData);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      log(`[persistent-session] EXCEPTION evaluating '${agent.code_name}': ${msg}`);
+      psResult = {
+        decision: "skipped-auth-resolve-failed",
+        spawnAttempted: false,
+        sessionHealthyAfter: wasSessionHealthy,
+        detail: `exception: ${msg}`
+      };
+    }
+    const normalizedDetail = psResult.detail?.replace(/\s+/g, " ").trim();
+    const detailSuffix = normalizedDetail ? ` detail=${JSON.stringify(normalizedDetail)}` : "";
+    log(
+      `[persistent-session-decision] agent=${agent.code_name} decision=${psResult.decision} spawn_attempted=${psResult.spawnAttempted} session_healthy_after=${psResult.sessionHealthyAfter}${detailSuffix}`
+    );
+    const stuck = persistentSessionStuckTracker.record({
+      codeName: agent.code_name,
+      sessionHealthy: psResult.sessionHealthyAfter,
+      spawnAttempted: psResult.spawnAttempted
+    });
+    if (stuck.shouldWarn) {
+      log(
+        `[persistent-session-stuck] WARN: agent=${agent.code_name} unhealthy with no spawn attempt for ${stuck.consecutiveTicks} consecutive ticks (last_decision=${psResult.decision}) \u2014 manager-worker may need restart, or upstream guard is mis-firing (see ENG-5116)`
+      );
+    } else if (stuck.recovered) {
+      log(`[persistent-session-stuck] agent=${agent.code_name} recovered from stuck state`);
+    }
     try {
       checkMcpConfigDriftAndScheduleRestart(agent.code_name, getProjectDir2(agent.code_name));
     } catch (err) {
@@ -4954,6 +5074,7 @@ function fireClaudeWorkTrigger(codeName, agentId, boardItems) {
   });
 }
 var persistentSessionAgents = /* @__PURE__ */ new Set();
+var persistentSessionStuckTracker = new PersistentSessionStuckTracker();
 var claudeAuthTupleBySession = /* @__PURE__ */ new Map();
 async function ensurePersistentSession(agent, tasks, boardItems, refreshData) {
   const codeName = agent.code_name;
@@ -4986,7 +5107,11 @@ async function ensurePersistentSession(agent, tasks, boardItems, refreshData) {
     const apiKey = getApiKey();
     if (!apiKey) {
       log(`[persistent-session] Skipping '${codeName}' \u2014 AGT_API_KEY not set`);
-      return;
+      return {
+        decision: "skipped-no-api-key",
+        spawnAttempted: false,
+        sessionHealthyAfter: isSessionHealthy(codeName)
+      };
     }
     const exchange = await exchangeApiKey(apiKey, false, { forceRefresh: true });
     claudeAuthMode = exchange.claudeAuthMode;
@@ -5000,26 +5125,42 @@ async function ensurePersistentSession(agent, tasks, boardItems, refreshData) {
       persistentSessionAgents.delete(codeName);
       claudeAuthTupleBySession.delete(codeName);
     }
-    return;
+    return {
+      decision: "skipped-auth-resolve-failed",
+      spawnAttempted: false,
+      sessionHealthyAfter: isSessionHealthy(codeName),
+      detail: msg
+    };
   }
   if (claudeAuthMode === "subscription") {
     if (!agentRuntimeAuthenticated) {
       agentRuntimeAuthenticated = await checkClaudeAuth();
       if (!agentRuntimeAuthenticated) {
         log(`[persistent-session] Skipping '${codeName}' \u2014 Claude Code not authenticated (subscription mode)`);
-        return;
+        return {
+          decision: "skipped-not-authenticated",
+          spawnAttempted: false,
+          sessionHealthyAfter: isSessionHealthy(codeName)
+        };
       }
     }
   } else if (!anthropicApiKey) {
     log(`[persistent-session] Skipping '${codeName}' \u2014 api_key mode but no key returned from exchange`);
-    return;
+    return {
+      decision: "skipped-api-key-mode-no-key",
+      spawnAttempted: false,
+      sessionHealthyAfter: isSessionHealthy(codeName)
+    };
   }
+  let restartTrigger = null;
+  let dayRolloverDeferred = false;
   const currentAuthTuple = `${claudeAuthMode}:${anthropicApiKeyFingerprint ?? "none"}`;
   const recordedAuthTuple = claudeAuthTupleBySession.get(codeName);
   if (recordedAuthTuple && recordedAuthTuple !== currentAuthTuple && isSessionHealthy(codeName)) {
     log(`[persistent-session] Auth config changed for '${codeName}' (${recordedAuthTuple} \u2192 ${currentAuthTuple}) \u2014 restarting session`);
     stopPersistentSessionAndForgetMcpBaseline(codeName);
     persistentSessionAgents.delete(codeName);
+    restartTrigger = "auth-tuple";
   }
   if (isStaleForToday(codeName) && isSessionHealthy(codeName)) {
     const current = peekCurrentSession(codeName);
@@ -5031,10 +5172,12 @@ async function ensurePersistentSession(agent, tasks, boardItems, refreshData) {
         );
         stopPersistentSessionAndForgetMcpBaseline(codeName);
         persistentSessionAgents.delete(codeName);
+        restartTrigger = "day-rollover";
       } else {
         log(
           `[persistent-session] Day rollover for '${codeName}' deferred \u2014 agent still active on session ${current.sessionId} (will retry next tick)`
         );
+        dayRolloverDeferred = true;
       }
     }
   }
@@ -5081,7 +5224,12 @@ ${truncateForLog(ctx.tail)}` : `; pane_tail_hash=sha256:${createHash2("sha256").
     });
     persistentSessionAgents.add(codeName);
     claudeAuthTupleBySession.set(codeName, currentAuthTuple);
-    return;
+    return {
+      decision: "spawn",
+      spawnAttempted: true,
+      sessionHealthyAfter: isSessionHealthy(codeName),
+      detail: restartTrigger ? `trigger=${restartTrigger}` : void 0
+    };
   }
   resetRestartCount(codeName);
   if (!claudeAuthTupleBySession.has(codeName)) {
@@ -5163,6 +5311,12 @@ ${truncateForLog(ctx.tail)}` : `; pane_tail_hash=sha256:${createHash2("sha256").
       break;
     }
   }
+  const finalDecision = restartTrigger === "auth-tuple" ? "restart-pending" : restartTrigger === "day-rollover" ? "day-rollover-restart" : dayRolloverDeferred ? "day-rollover-deferred" : "healthy";
+  return {
+    decision: finalDecision,
+    spawnAttempted: false,
+    sessionHealthyAfter: isSessionHealthy(codeName)
+  };
 }
 var realtimeStarted = false;
 var realtimeDriftStarted = false;
@@ -7167,7 +7321,9 @@ export {
   applyRestartAcks,
   extractCharterSlackPeers,
   extractCharterTelegramPeers,
+  hasRevokedResiduals,
   markAgentForFreshMemorySync,
+  shouldSkipRevokedCleanup,
   startManager,
   stopManager
 };