@integrity-labs/agt-cli 0.20.9 → 0.21.1

package/dist/lib/manager-worker.js

@@ -22,7 +22,7 @@ import {
   resolveChannels,
   resolveDmTarget,
   wrapScheduledTaskPrompt
-} from "../chunk-UMGWKBRQ.js";
+} from "../chunk-NLEW5BPX.js";
 import {
   findTaskByTemplate,
   getProjectDir,
@@ -92,6 +92,25 @@ function computeIntegrationsHash(integrations) {
 
 // src/lib/stale-mcp-reaper.ts
 import { execFileSync } from "child_process";
+var rotationTimestamps = /* @__PURE__ */ new Map();
+var DEFAULT_ROTATION_GRACE_MS = 15e3;
+function rotationKey(codeName, serverKey) {
+  return `${codeName}\0${serverKey}`;
+}
+function recordStaleRotation(codeName, serverKeys, now = Date.now) {
+  const ts = now();
+  for (const key of serverKeys) {
+    rotationTimestamps.set(rotationKey(codeName, key), ts);
+  }
+}
+function wasRecentlyRotated(codeName, serverKey, withinMs, now = Date.now) {
+  const k = rotationKey(codeName, serverKey);
+  const ts = rotationTimestamps.get(k);
+  if (ts === void 0) return false;
+  if (now() - ts < withinMs) return true;
+  rotationTimestamps.delete(k);
+  return false;
+}
 function parseEnvIntegrationsEntries(content) {
   const entries = /* @__PURE__ */ new Map();
   for (const raw of content.split(/\r?\n/)) {
@@ -264,6 +283,16 @@ function reapStaleMcpChildren(args) {
   for (const pid of targets) {
     killProcess(pid, "SIGTERM");
   }
+  const killedKeys = /* @__PURE__ */ new Set();
+  for (const pid of targets) {
+    const argv = byPid.get(pid)?.args ?? "";
+    for (const key of serverKeys) {
+      const entry = mcpJson?.mcpServers?.[key];
+      const matchers = buildArgvMatchersForEntry(key, entry);
+      if (matchers.some((re) => re.test(argv))) killedKeys.add(key);
+    }
+  }
+  if (killedKeys.size > 0) recordStaleRotation(codeName, killedKeys);
   setTimeout(() => {
     try {
       let freshPsOutput;
@@ -299,6 +328,22 @@ function reapStaleMcpChildren(args) {
 // src/lib/mcp-presence-reaper.ts
 import { execFileSync as execFileSync2 } from "child_process";
 var DEFAULT_COLD_START_GRACE_MS = 9e4;
+var MAX_PRESENCE_RESTART_ATTEMPTS = 3;
+var presenceReaperState = /* @__PURE__ */ new Map();
+function stateKey(codeName, serverKey) {
+  return `${codeName}\0${serverKey}`;
+}
+function clearPresenceReaperState(codeName) {
+  const prefix = `${codeName}\0`;
+  for (const key of presenceReaperState.keys()) {
+    if (key.startsWith(prefix)) presenceReaperState.delete(key);
+  }
+}
+function clearPresenceReaperStateForKeys(codeName, keys) {
+  for (const key of keys) {
+    presenceReaperState.delete(stateKey(codeName, key));
+  }
+}
 function findMissingMcpServers(args) {
   const { rows, codeName, mcpJson } = args;
   const servers = mcpJson?.mcpServers ?? {};
@@ -326,7 +371,10 @@ function reapMissingMcpSessions(args) {
     mcpJson,
     sessionStartedAt,
     stopSession,
-    graceMs = DEFAULT_COLD_START_GRACE_MS
+    graceMs = DEFAULT_COLD_START_GRACE_MS,
+    rotationGraceMs = DEFAULT_ROTATION_GRACE_MS,
+    onGiveUp,
+    onRestart
   } = args;
   const now = args.now ?? Date.now;
   const runPs = args.runPs ?? (() => execFileSync2("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
@@ -351,15 +399,117 @@ function reapMissingMcpSessions(args) {
     return { missing: [], restarted: false, reason: "healthy" };
   }
   const rows = parsePsRows(psOutput);
-  const missing = findMissingMcpServers({ rows, codeName, mcpJson });
+  const missingRaw = findMissingMcpServers({ rows, codeName, mcpJson });
+  const rotationGraced = [];
+  const missing = [];
+  for (const key of missingRaw) {
+    if (wasRecentlyRotated(codeName, key, rotationGraceMs, now)) {
+      rotationGraced.push(key);
+    } else {
+      missing.push(key);
+    }
+  }
+  if (rotationGraced.length > 0) {
+    log2(
+      `[mcp-presence-reaper] '${codeName}': skipping [${rotationGraced.join(", ")}] within rotation grace window (${rotationGraceMs}ms) \u2014 child(ren) just SIGTERM'd by stale-mcp-reaper, awaiting respawn on next tool call (ENG-5344)`
+    );
+  }
+  const declared = mcpJson.mcpServers ?? {};
+  const liveKeys = /* @__PURE__ */ new Set();
+  for (const key of Object.keys(declared)) {
+    const entry = declared[key];
+    if (typeof entry?.command !== "string") continue;
+    if (!missingRaw.includes(key)) liveKeys.add(key);
+  }
+  for (const key of liveKeys) {
+    const state2 = presenceReaperState.get(stateKey(codeName, key));
+    if (state2) {
+      state2.attempts = 0;
+      state2.lastSeenLiveAt = now();
+      state2.lastAttemptedSessionStartedAt = null;
+      state2.gaveUpLogged = false;
+    }
+  }
   if (missing.length === 0) {
-    return { missing, restarted: false, reason: "healthy" };
+    return {
+      missing,
+      restarted: false,
+      reason: "healthy",
+      rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0
+    };
+  }
+  const givenUp = [];
+  const active = [];
+  for (const key of missing) {
+    const sk = stateKey(codeName, key);
+    const state2 = presenceReaperState.get(sk) ?? {
+      attempts: 0,
+      lastSeenLiveAt: null,
+      lastAttemptedSessionStartedAt: null,
+      gaveUpLogged: false
+    };
+    if (state2.lastAttemptedSessionStartedAt !== sessionStartedAt) {
+      state2.attempts += 1;
+      state2.lastAttemptedSessionStartedAt = sessionStartedAt;
+    }
+    presenceReaperState.set(sk, state2);
+    if (state2.attempts > MAX_PRESENCE_RESTART_ATTEMPTS) {
+      givenUp.push(key);
+      if (!state2.gaveUpLogged) {
+        log2(
+          `[mcp-presence-reaper] giving up on '${codeName}:${key}' after ${MAX_PRESENCE_RESTART_ATTEMPTS} consecutive failed restarts \u2014 declared but never recovers (likely orphaned config; see ENG-5279)`
+        );
+        state2.gaveUpLogged = true;
+        if (onGiveUp) {
+          try {
+            onGiveUp(codeName, key);
+          } catch (err) {
+            log2(
+              `[mcp-presence-reaper] onGiveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
+            );
+          }
+        }
+      }
+    } else {
+      active.push(key);
+    }
+  }
+  if (active.length === 0) {
+    return {
+      missing,
+      restarted: false,
+      reason: "all-keys-over-cap",
+      givenUp,
+      rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0
+    };
   }
+  const givenUpSuffix = givenUp.length > 0 ? ` (skipping over-cap: [${givenUp.join(", ")}])` : "";
   log2(
-    `[mcp-presence-reaper] '${codeName}': declared MCP(s) [${missing.join(", ")}] have no live children \u2014 restarting session`
+    `[mcp-presence-reaper] '${codeName}': declared MCP(s) [${active.join(", ")}] have no live children \u2014 restarting session${givenUpSuffix}`
   );
+  if (onRestart) {
+    try {
+      const maybePromise = onRestart(codeName, { activeKeys: active, givenUpKeys: givenUp });
+      if (maybePromise && typeof maybePromise.then === "function") {
+        void maybePromise.then(void 0, (err) => {
+          log2(
+            `[mcp-presence-reaper] onRestart callback rejected for '${codeName}' (suppressed; restart proceeds): ${err.message}`
+          );
+        });
+      }
+    } catch (err) {
+      log2(
+        `[mcp-presence-reaper] onRestart callback threw for '${codeName}' (suppressed; restart proceeds): ${err.message}`
+      );
+    }
+  }
   stopSession(codeName);
-  return { missing, restarted: true };
+  return {
+    missing,
+    restarted: true,
+    givenUp: givenUp.length > 0 ? givenUp : void 0,
+    rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0
+  };
 }
 
 // src/lib/channel-restart-decision.ts
@@ -2094,6 +2244,7 @@ function cancelPendingSessionRestart(codeName) {
 var writtenHashes = /* @__PURE__ */ new Map();
 var knownSecretsHashes = /* @__PURE__ */ new Map();
 var runningMcpHashes = /* @__PURE__ */ new Map();
+var runningMcpServerKeys = /* @__PURE__ */ new Map();
 function projectMcpHash(_codeName, projectDir) {
   try {
     const raw = readFileSync3(join4(projectDir, ".mcp.json"), "utf-8");
@@ -2102,10 +2253,22 @@ function projectMcpHash(_codeName, projectDir) {
     return null;
   }
 }
+function projectMcpKeys(_codeName, projectDir) {
+  try {
+    const raw = readFileSync3(join4(projectDir, ".mcp.json"), "utf-8");
+    const parsed = JSON.parse(raw);
+    const servers = parsed.mcpServers;
+    if (!servers || typeof servers !== "object") return /* @__PURE__ */ new Set();
+    return new Set(Object.keys(servers));
+  } catch {
+    return null;
+  }
+}
 function stopPersistentSessionAndForgetMcpBaseline(codeName) {
   cancelPendingSessionRestart(codeName);
   stopPersistentSession(codeName, log);
   runningMcpHashes.delete(codeName);
+  runningMcpServerKeys.delete(codeName);
 }
 function checkMcpConfigDriftAndScheduleRestart(codeName, projectDir) {
   const currentHash = projectMcpHash(codeName, projectDir);
@@ -2114,16 +2277,47 @@ function checkMcpConfigDriftAndScheduleRestart(codeName, projectDir) {
     case "no-config":
     case "no-drift":
       return;
-    case "baseline":
+    case "baseline": {
       runningMcpHashes.set(codeName, action.hash);
+      const baselineKeys = projectMcpKeys(codeName, projectDir);
+      if (baselineKeys) {
+        runningMcpServerKeys.set(codeName, baselineKeys);
+      } else {
+        runningMcpServerKeys.delete(codeName);
+      }
       return;
-    case "drift":
+    }
+    case "drift": {
       log(
         `[hot-reload] .mcp.json content changed for '${codeName}' (${action.previous.slice(0, 12)} \u2192 ${action.current.slice(0, 12)}); scheduling restart (ENG-4897)`
       );
+      const previousKeys = runningMcpServerKeys.get(codeName);
+      const currentKeys = projectMcpKeys(codeName, projectDir);
+      if (!previousKeys || !currentKeys) {
+        clearPresenceReaperState(codeName);
+        log(
+          `[hot-reload] .mcp.json key membership unavailable for '${codeName}' \u2014 clearing full presence-reaper state (ENG-5285)`
+        );
+      } else {
+        const addedOrRemoved = /* @__PURE__ */ new Set();
+        for (const k of currentKeys) if (!previousKeys.has(k)) addedOrRemoved.add(k);
+        for (const k of previousKeys) if (!currentKeys.has(k)) addedOrRemoved.add(k);
+        if (addedOrRemoved.size > 0) {
+          clearPresenceReaperStateForKeys(codeName, addedOrRemoved);
+          log(
+            `[hot-reload] .mcp.json membership changed for '${codeName}': ${addedOrRemoved.size} key(s) added/removed [${Array.from(addedOrRemoved).sort().join(", ")}] \u2014 reaper state cleared for those keys only (ENG-5285)`
+          );
+        } else {
+          log(
+            `[hot-reload] .mcp.json value-only drift for '${codeName}' \u2014 preserving presence-reaper state for the unchanged declared key set (ENG-5285)`
+          );
+        }
+      }
       scheduleSessionRestart(codeName, 0, ".mcp.json content change (ENG-4897)");
       runningMcpHashes.delete(codeName);
+      runningMcpServerKeys.delete(codeName);
       return;
+    }
   }
 }
 var knownChannelConfigHashes = /* @__PURE__ */ new Map();
@@ -2132,6 +2326,10 @@ var knownTasksHashes = /* @__PURE__ */ new Map();
 var knownIntegrationHashes = /* @__PURE__ */ new Map();
 var knownManagedMcpHashes = /* @__PURE__ */ new Map();
 var knownSkillHashes = /* @__PURE__ */ new Map();
+var managedToolkitIdByAgentAndServerId = /* @__PURE__ */ new Map();
+function managedToolkitMapKey(agentId, serverId) {
+  return `${agentId}\0${serverId}`;
+}
 var lastCronRunTs = /* @__PURE__ */ new Map();
 var lastWorkTriggerAt = /* @__PURE__ */ new Map();
 var alertedStaleItems = /* @__PURE__ */ new Set();
@@ -2200,7 +2398,7 @@ function clearAgentCaches(agentId, codeName) {
 var cachedFrameworkVersion = null;
 var lastVersionCheckAt = 0;
 var VERSION_CHECK_INTERVAL_MS = 5 * 60 * 1e3;
-var agtCliVersion = true ? "0.20.9" : "dev";
+var agtCliVersion = true ? "0.21.1" : "dev";
 function resolveBrewPath(execFileSync4) {
   try {
     const out = execFileSync4("which", ["brew"], { timeout: 5e3 }).toString().trim();
@@ -4011,6 +4209,10 @@ async function processAgent(agent, agentStates) {
         for (const tk of agentToolkits) {
           const serverId = tk.toolkit_id.replace(/[^a-z0-9]/gi, "_").toLowerCase();
           expectedServerIds.add(serverId);
+          managedToolkitIdByAgentAndServerId.set(
+            managedToolkitMapKey(agent.agent_id, serverId),
+            tk.toolkit_id
+          );
           const mcpUrl = tk.mcp_url;
           const mcpHeaders = tk.mcp_headers;
           const proxyUrl = tk.proxy_url.startsWith("/") ? `${requireHost()}${tk.proxy_url}` : tk.proxy_url;
@@ -4362,7 +4564,60 @@ async function processAgent(agent, agentStates) {
         codeName: agent.code_name,
         mcpJson: mcpJsonParsed,
         sessionStartedAt: sess?.startedAt ?? null,
-        stopSession: (codeName) => stopPersistentSessionAndForgetMcpBaseline(codeName)
+        stopSession: (codeName) => stopPersistentSessionAndForgetMcpBaseline(codeName),
+        // ENG-5292: when the reaper gives up on a managed MCP (cap from
+        // ENG-5279 + state-preservation from ENG-5285 both said "this
+        // MCP keeps failing after 3 restart cycles"), mark the matching
+        // integration unhealthy server-side. The next /host/managed-
+        // toolkits poll drops it from the response → next writeMcpServer
+        // omits it → broken MCP is cleanly gone instead of declared-but-
+        // dead.
+        //
+        // Async fire-and-forget — the reaper's correctness must not
+        // depend on the API call succeeding. Errors are caught inside
+        // the .then/.catch chain and logged.
+        onGiveUp: (codeName, serverKey) => {
+          const toolkitId = managedToolkitIdByAgentAndServerId.get(
+            managedToolkitMapKey(agent.agent_id, serverKey)
+          );
+          if (!toolkitId) {
+            log(
+              `[mcp-presence-reaper] give-up on '${codeName}:${serverKey}' \u2014 not a managed toolkit (no toolkit_id mapping), skipping integration-health API call`
+            );
+            return;
+          }
+          const agentId = agent.agent_id;
+          api.post("/host/integration-health", {
+            agent_id: agentId,
+            toolkit_id: toolkitId,
+            action: "mark_unhealthy",
+            reason: `mcp-presence-reaper cap (${3} consecutive failed restart cycles) tripped on '${serverKey}'`
+          }).then((res) => {
+            log(
+              `[mcp-presence-reaper] marked integration unhealthy for '${codeName}' toolkit '${toolkitId}' (ENG-5292): ${JSON.stringify(res).slice(0, 200)}`
+            );
+          }).catch((err) => {
+            log(
+              `[mcp-presence-reaper] failed to mark integration unhealthy for '${codeName}' toolkit '${toolkitId}' (ENG-5292): ${err.message} \u2014 local give-up state still recorded; will retry on next agent provisioning cycle if .mcp.json changes`
+            );
+          });
+        },
+        // ENG-5286: record the restart event with the API so the
+        // agent-heartbeat-monitor cron can aggregate per-agent counts
+        // into the McpReaperRestartsHourly CloudWatch metric. Fire-and-
+        // forget — the reaper's correctness must not depend on this.
+        onRestart: (codeName, { activeKeys, givenUpKeys }) => {
+          api.post("/host/mcp-reaper-restart", {
+            agent_id: agent.agent_id,
+            code_name: codeName,
+            server_keys: activeKeys,
+            given_up_keys: givenUpKeys
+          }).catch((err) => {
+            log(
+              `[mcp-presence-reaper] failed to record restart event for '${codeName}' (ENG-5286): ${err.message} \u2014 local restart still proceeded; CloudWatch metric will under-count this event`
+            );
+          });
+        }
       });
     } catch (err) {
       log(`[mcp-presence-reaper] presence check failed for '${agent.code_name}': ${err.message}`);