@integrity-labs/agt-cli 0.28.500 → 0.28.501

package/dist/lib/manager-worker.js

@@ -1,7 +1,9 @@
 import {
   ApiError,
   CHANNEL_SECRET_ENV_KEYS,
+  FIRST_CONNECT_WINDOW_MS,
   HostFlagStore,
+  RestartBreaker,
   SUPERVISOR_RESTART_EXIT_CODE,
   api,
   atomicWriteFileSync,
@@ -25,9 +27,11 @@ import {
   getCachedClaudeAuthMode,
   getHostId,
   givenUpMcpServerKeys,
+  isProvisioningReloadReason,
   isTransientAuthFailure,
   liveProxyExtraHeaderVars,
   liveProxyTokenVars,
+  markQuarantinedForProbe,
   migrateAgentDirToIdKeyed,
   pinAllowsUrgent,
   provision,
@@ -38,14 +42,17 @@ import {
   provisionPreCompactHook,
   provisionSessionStateHook,
   provisionStopHook,
+  readEnvNumber,
   reapMissingMcpSessions,
   reapStaleMcpChildren,
+  reaperRestartBreakerReason,
   reconstructQuarantinedMcpServerEntry,
   requireHost,
   resolveEffectivePinRaw,
   safeWriteJsonAtomic,
-  setConfigHash
-} from "../chunk-ICNRQ4L5.js";
+  setConfigHash,
+  tripClass
+} from "../chunk-N4Y5IBZR.js";
 import {
   getProjectDir as getProjectDir2,
   getReadyTasks,
@@ -56,7 +63,6 @@ import {
 import {
   ACCOUNT_ENFORCEMENT_MARKER_FILENAME,
   AnchorSessionClient,
-  BIND_FAILURE_QUARANTINE_THRESHOLD,
   CLAUDE_MD_MAX_CHARS,
   CONVERSATION_FAILURE_CATEGORIES,
   DEFAULT_FRAMEWORK,
@@ -67,9 +73,6 @@ import {
   KANBAN_CHECK_COMMAND,
   KANBAN_NUDGE_ACTIONABLE_STATUSES,
   MAX_AVATAR_ENV_URL_BYTES,
-  MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING,
-  RESTART_BREAKER_PROVISIONING_MAX,
-  RESTART_BREAKER_PROVISIONING_WINDOW_MS,
   SCHEDULED_TURN_MARKER_FILENAME,
   SUPPRESS_SENTINEL,
   StreamEncoder,
@@ -845,315 +848,8 @@ function logClaudeMdCeilingTransition(codeName, report, log2) {
   return line;
 }
 
-// src/lib/restart-breaker.ts
-function reaperRestartBreakerReason(activeKeys) {
-  return activeKeys.length >= 2 ? "mcp-presence-reaper" : void 0;
-}
-var KNOWN_REASON_CLASS_MAP = {
-  crash: true,
-  provisioning: true,
-  "credential-rotation": true,
-  // ENG-8215: the self-healing class (bind-remediation - the manager's OWN
-  // repair attempts, which must never count toward a trip). Adding it here is
-  // forced by the Record type, which is exactly why that shape was chosen over
-  // a Set literal (ENG-7577, CodeRabbit on #3861) - a new union member cannot
-  // be silently omitted from the rehydration allowlist.
-  "self-healing": true
-};
-var KNOWN_REASON_CLASSES = new Set(
-  Object.keys(KNOWN_REASON_CLASS_MAP)
-);
-var PROVISIONING_RELOAD_REASONS = /* @__PURE__ */ new Set([
-  "hot-reload-mcp",
-  "managed-mcp-churn",
-  // ENG-8215: 'bind-remediation' MOVED OUT of this set into SELF_HEALING_REASONS.
-  // It is the manager repairing itself, not a config reload, and counting it
-  // meant the harder the platform tried to fix an agent the more certain it
-  // became that it should give up. Deliberately NOT re-added here.
-  // ENG-7576: the dashboard integration-add stop itself - the first restart of
-  // the very burst this class exists for, previously uncounted entirely.
-  "integration-change",
-  // ENG-7771: a sender_policy delivery restart is a deliberate config-delivery
-  // reload (operator changed the policy, or the fail-closed first-poll verify
-  // after a manager restart with no persisted baseline), not a crash. On the
-  // tight crash tally, a host whose sender-policy-baseline.json write keeps
-  // failing (disk full, permissions) would convert the fail-closed restart
-  // into repeated crash tallies across a multi-deploy day and trip the
-  // breaker - taking the agent DOWN, which is strictly worse than the stale
-  // policy the restart exists to fix. The looser provisioning tally still
-  // trips + pages on sustained thrash.
-  "sender-policy-change"
-]);
-function isProvisioningReloadReason(reason) {
-  return PROVISIONING_RELOAD_REASONS.has(reason);
-}
-var CREDENTIAL_ROTATION_REASONS = /* @__PURE__ */ new Set([
-  "credential-rotation"
-]);
-function isCredentialRotationReason(reason) {
-  return CREDENTIAL_ROTATION_REASONS.has(reason);
-}
-var SELF_HEALING_REASONS = /* @__PURE__ */ new Set([
-  "bind-remediation"
-]);
-function isSelfHealingReason(reason) {
-  return SELF_HEALING_REASONS.has(reason);
-}
-function restartReasonClass(reason) {
-  if (isCredentialRotationReason(reason)) return "credential-rotation";
-  if (isSelfHealingReason(reason)) return "self-healing";
-  return isProvisioningReloadReason(reason) ? "provisioning" : "crash";
-}
-function countByClass(events) {
-  let crash = 0;
-  let provisioning = 0;
-  let credentialRotation = 0;
-  let selfHealing = 0;
-  for (const e of events) {
-    const klass = restartReasonClass(e.reason);
-    if (klass === "provisioning") provisioning += 1;
-    else if (klass === "credential-rotation") credentialRotation += 1;
-    else if (klass === "self-healing") selfHealing += 1;
-    else crash += 1;
-  }
-  return {
-    crash,
-    provisioning,
-    "credential-rotation": credentialRotation,
-    "self-healing": selfHealing
-  };
-}
-function maxProvisioningBucket(events) {
-  const buckets = /* @__PURE__ */ new Map();
-  for (const e of events) {
-    if (!isProvisioningReloadReason(e.reason)) continue;
-    const key = e.integrationKey ?? "";
-    buckets.set(key, (buckets.get(key) ?? 0) + 1);
-  }
-  let best = { count: 0, key: "" };
-  for (const [key, count] of buckets) {
-    if (count > best.count) best = { count, key };
-  }
-  return best;
-}
-function tripClass(trip) {
-  if (trip.trippedClass) return trip.trippedClass;
-  if (!Array.isArray(trip.eventsAtTrip) || trip.eventsAtTrip.length === 0) return "crash";
-  return trip.eventsAtTrip.every((e) => isProvisioningReloadReason(e.reason)) ? "provisioning" : "crash";
-}
-var DEFAULT_MAX = 2;
-var DEFAULT_WINDOW_MS = 6e5;
-var DEFAULT_PROVISIONING_MAX = RESTART_BREAKER_PROVISIONING_MAX;
-var DEFAULT_PROVISIONING_WINDOW_MS = RESTART_BREAKER_PROVISIONING_WINDOW_MS;
-function readEnvNumber(name, fallback) {
-  const raw = process.env[name];
-  if (!raw) return fallback;
-  const parsed = Number(raw);
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
-}
-var RestartBreaker = class {
-  max;
-  windowMs;
-  /**
-   * ENG-8215: NOT readonly - the constructor may clamp it upward to preserve the
-   * quarantine-fires-first ordering against a runtime override. Never mutated
-   * after construction.
-   */
-  provisioningMax;
-  provisioningWindowMs;
-  /** Longest window either tally needs — how long the events log must retain. */
-  retentionMs;
-  now;
-  events = /* @__PURE__ */ new Map();
-  trips = /* @__PURE__ */ new Map();
-  /**
-   * ENG-8215: set when a supplied `provisioningMax` was raised to keep MCP
-   * quarantine ahead of the breaker. Read once by the manager at construction so
-   * an overridden-and-clamped value is visible in manager.log rather than
-   * silently discarded.
-   */
-  quarantineOrderingClamped;
-  constructor(opts = {}) {
-    this.max = opts.max ?? readEnvNumber("AGT_RESTART_BREAKER_MAX", DEFAULT_MAX);
-    this.windowMs = opts.windowMs ?? readEnvNumber("AGT_RESTART_BREAKER_WINDOW_MS", DEFAULT_WINDOW_MS);
-    this.provisioningMax = opts.provisioningMax ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", DEFAULT_PROVISIONING_MAX);
-    this.provisioningWindowMs = opts.provisioningWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS", DEFAULT_PROVISIONING_WINDOW_MS);
-    this.now = opts.now ?? Date.now;
-    if (!Number.isFinite(this.max) || this.max < 1) {
-      throw new Error("restart-breaker max must be a finite number >= 1");
-    }
-    if (!Number.isFinite(this.windowMs) || this.windowMs < 1e3) {
-      throw new Error("restart-breaker windowMs must be a finite number >= 1000");
-    }
-    if (!Number.isFinite(this.provisioningMax) || this.provisioningMax < 1) {
-      throw new Error("restart-breaker provisioningMax must be a finite number >= 1");
-    }
-    if (this.provisioningMax < MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING) {
-      this.quarantineOrderingClamped = {
-        requested: this.provisioningMax,
-        applied: MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING,
-        quarantineThreshold: BIND_FAILURE_QUARANTINE_THRESHOLD
-      };
-      this.provisioningMax = MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING;
-    }
-    if (!Number.isFinite(this.provisioningWindowMs) || this.provisioningWindowMs < 1e3) {
-      throw new Error("restart-breaker provisioningWindowMs must be a finite number >= 1000");
-    }
-    this.retentionMs = Math.max(this.windowMs, this.provisioningWindowMs);
-  }
-  /**
-   * ENG-8215: the clamp record, if the configured provisioning max had to be
-   * raised to keep MCP quarantine ahead of the breaker. `undefined` when the
-   * configured value already satisfied the ordering.
-   */
-  getQuarantineOrderingClamp() {
-    return this.quarantineOrderingClamped;
-  }
-  /** True if this agent's breaker is currently tripped (manager must skip spawn). */
-  isTripped(codeName) {
-    return this.trips.has(codeName);
-  }
-  getTrip(codeName) {
-    return this.trips.get(codeName);
-  }
-  /**
-   * Record a restart event. If recording this event puts the count
-   * inside the window above `max`, the breaker trips and the call site
-   * should NOT respawn.
-   *
-   * Idempotent on already-tripped breakers: returns the existing trip
-   * without double-counting events. Callers may still record the reason
-   * via the decision-log for forensics.
-   */
-  record(codeName, reason, integrationKey) {
-    const existing = this.trips.get(codeName);
-    if (existing) {
-      const bucket = maxProvisioningBucket(existing.eventsAtTrip);
-      return {
-        tripped: false,
-        trip: existing,
-        windowCount: existing.eventsAtTrip.length,
-        classCounts: countByClass(existing.eventsAtTrip),
-        maxProvisioningBucket: bucket.count,
-        maxProvisioningBucketKey: bucket.key
-      };
-    }
-    const at = this.now();
-    const retentionCutoff = at - this.retentionMs;
-    const prior = (this.events.get(codeName) ?? []).filter((e) => e.at >= retentionCutoff);
-    prior.push(
-      isProvisioningReloadReason(reason) && integrationKey ? { reason, at, integrationKey } : { reason, at }
-    );
-    this.events.set(codeName, prior);
-    const crashCount = prior.filter(
-      (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
-    ).length;
-    const provEvents = prior.filter(
-      (e) => e.at >= at - this.provisioningWindowMs && isProvisioningReloadReason(e.reason)
-    );
-    const credRotCount = prior.filter(
-      (e) => e.at >= at - this.provisioningWindowMs && isCredentialRotationReason(e.reason)
-    ).length;
-    const selfHealCount = prior.filter(
-      (e) => e.at >= at - this.provisioningWindowMs && isSelfHealingReason(e.reason)
-    ).length;
-    const windowCount = prior.filter((e) => e.at >= at - this.windowMs).length;
-    const classCounts = {
-      crash: crashCount,
-      provisioning: provEvents.length,
-      "credential-rotation": credRotCount,
-      "self-healing": selfHealCount
-    };
-    const provBucket = maxProvisioningBucket(provEvents);
-    let trippedClass;
-    let tripEvents;
-    let tripWindowMs = this.windowMs;
-    if (crashCount > this.max) {
-      trippedClass = "crash";
-      tripEvents = prior.filter(
-        (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
-      );
-      tripWindowMs = this.windowMs;
-    } else if (provBucket.count > this.provisioningMax) {
-      trippedClass = "provisioning";
-      tripEvents = provEvents.filter((e) => (e.integrationKey ?? "") === provBucket.key);
-      tripWindowMs = this.provisioningWindowMs;
-    }
-    if (trippedClass && tripEvents) {
-      const trip = {
-        trippedAt: at,
-        eventsAtTrip: [...tripEvents],
-        statusMessage: formatStatusMessage(tripEvents, tripWindowMs, trippedClass, provBucket.key),
-        // ENG-7577: persist the tripping class so the recovery path can gate on
-        // it after a manager restart, without re-deriving it from prose.
-        trippedClass
-      };
-      this.trips.set(codeName, trip);
-      this.events.delete(codeName);
-      return {
-        tripped: true,
-        trip,
-        windowCount,
-        classCounts,
-        trippedClass,
-        maxProvisioningBucket: provBucket.count,
-        maxProvisioningBucketKey: provBucket.key
-      };
-    }
-    return {
-      tripped: false,
-      windowCount,
-      classCounts,
-      maxProvisioningBucket: provBucket.count,
-      maxProvisioningBucketKey: provBucket.key
-    };
-  }
-  /** Operator-initiated reset: drops the trip + the events log for this agent. */
-  clear(codeName) {
-    this.trips.delete(codeName);
-    this.events.delete(codeName);
-  }
-  /** Snapshot tripped agents for `manager-state.json`. */
-  serialize() {
-    return Object.fromEntries(this.trips.entries());
-  }
-  /**
-   * Rehydrate trip state from `manager-state.json`. Called once at
-   * worker startup. Window history is intentionally NOT persisted — only
-   * tripped state. A tripped breaker survives manager restart; an
-   * un-tripped one starts a fresh window in the new worker.
-   */
-  hydrate(saved) {
-    if (!saved) return;
-    for (const [codeName, trip] of Object.entries(saved)) {
-      if (trip && typeof trip.trippedAt === "number" && Array.isArray(trip.eventsAtTrip)) {
-        const klass = trip.trippedClass;
-        this.trips.set(
-          codeName,
-          klass === void 0 || KNOWN_REASON_CLASSES.has(klass) ? trip : { ...trip, trippedClass: void 0 }
-        );
-      }
-    }
-  }
-  /** Test helper — current in-window event count for `codeName`. */
-  windowCount(codeName) {
-    const cutoff = this.now() - this.windowMs;
-    return (this.events.get(codeName) ?? []).filter((e) => e.at >= cutoff).length;
-  }
-};
-function formatStatusMessage(events, windowMs, klass = "crash", integrationKey = "") {
-  const last = events[events.length - 1];
-  const windowLabel = windowMs < 6e4 ? `${Math.round(windowMs / 1e3)}s` : `${(windowMs / 6e4).toFixed(1).replace(/\.0$/, "")}min`;
-  const reasonCounts = /* @__PURE__ */ new Map();
-  for (const e of events) reasonCounts.set(e.reason, (reasonCounts.get(e.reason) ?? 0) + 1);
-  const breakdown = Array.from(reasonCounts.entries()).map(([r, n]) => `${r}=${n}`).join(", ");
-  const classLabel = klass === "provisioning" ? "provisioning-reload " : "";
-  const integrationLabel = klass === "provisioning" && integrationKey ? ` for integration '${integrationKey}'` : "";
-  return `Circuit breaker tripped: ${events.length} ${classLabel}restarts in ${windowLabel}${integrationLabel} (${breakdown}); most recent=${last.reason} at ${new Date(last.at).toISOString()}`;
-}
-
 // src/lib/mcp-flap-dampener.ts
-var DEFAULT_WINDOW_MS2 = 6e5;
+var DEFAULT_WINDOW_MS = 6e5;
 var DEFAULT_MIN_TRANSITIONS = 4;
 var DEFAULT_MIN_DISTINCT = 2;
 var DEFAULT_LONG_WINDOW_MS = 36e5;
@@ -1182,7 +878,7 @@ var McpFlapDampener = class {
   // see that method for the Kylie/dig (2026-06-29) rationale.
   lastFlapAt = /* @__PURE__ */ new Map();
   constructor(opts = {}) {
-    this.windowMs = opts.windowMs ?? readEnvNumber2("AGT_MCP_FLAP_WINDOW_MS", DEFAULT_WINDOW_MS2);
+    this.windowMs = opts.windowMs ?? readEnvNumber2("AGT_MCP_FLAP_WINDOW_MS", DEFAULT_WINDOW_MS);
     this.minTransitions = opts.minTransitions ?? readEnvNumber2("AGT_MCP_FLAP_MIN_TRANSITIONS", DEFAULT_MIN_TRANSITIONS);
     this.minDistinct = opts.minDistinct ?? readEnvNumber2("AGT_MCP_FLAP_MIN_DISTINCT", DEFAULT_MIN_DISTINCT);
     this.longWindowMs = opts.longWindowMs ?? readEnvNumber2("AGT_MCP_FLAP_LONG_WINDOW_MS", DEFAULT_LONG_WINDOW_MS);
@@ -1586,7 +1282,11 @@ async function runConnectivityProbes(integrations, options = {}) {
       cliBinary: integ.cli_binary,
       mcpServerKey: integ.mcp_server_key,
       connectivityTest: integ.connectivity_test ?? null,
-      inlineServerEntry: integ.inline_server_entry
+      inlineServerEntry: integ.inline_server_entry,
+      // ENG-8363: row age + quarantine status decide whether a missing
+      // `.mcp.json` key is "not written yet" (transient) or "not wired" (down).
+      createdAt: integ.created_at ?? null,
+      expectedAbsent: integ.quarantined ?? false
     };
     let outcome;
     try {
@@ -9209,8 +8909,8 @@ var restartBreaker = new RestartBreaker();
   }
 }
 var reportedTrips = /* @__PURE__ */ new Map();
-var RESTART_BREAKER_PROVISIONING_MAX2 = readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", 5);
-var RESTART_BREAKER_PROVISIONING_WINDOW_MS2 = readEnvNumber(
+var RESTART_BREAKER_PROVISIONING_MAX = readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", 5);
+var RESTART_BREAKER_PROVISIONING_WINDOW_MS = readEnvNumber(
   "AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS",
   18e5
 );
@@ -9255,13 +8955,13 @@ function recordRestartForBreaker(codeName, reason) {
   let integrationKey;
   if (isProvisioningReloadReason(reason)) {
     const attribution = agentState.provisioningAttribution.get(codeName);
-    if (attribution && Date.now() - attribution.at <= RESTART_BREAKER_PROVISIONING_WINDOW_MS2) {
+    if (attribution && Date.now() - attribution.at <= RESTART_BREAKER_PROVISIONING_WINDOW_MS) {
       integrationKey = attribution.key;
     }
   }
   const result = restartBreaker.record(codeName, reason, integrationKey);
   if (!result.tripped || !result.trip) {
-    if (isProvisioningReloadReason(reason) && result.maxProvisioningBucket >= RESTART_BREAKER_PROVISIONING_MAX2) {
+    if (isProvisioningReloadReason(reason) && result.maxProvisioningBucket >= RESTART_BREAKER_PROVISIONING_MAX) {
       const culprit = result.maxProvisioningBucketKey ? ` [${result.maxProvisioningBucketKey}]` : "";
       log(
         `[restart-breaker] agent=${codeName} provisioning-reload churn at the trip bar: ${result.maxProvisioningBucket} restarts for one integration${culprit} in window (reason=${reason}); one more will auto-pause the agent; that integration is likely churning (ENG-7560, ENG-7812)`
@@ -9650,11 +9350,7 @@ function bindRemediationEnabled() {
   return hostFlagStore().getBoolean("bind-remediation");
 }
 var BIND_REMEDIATION_JITTER_MS = 3e4;
-var BIND_REMEDIATION_FIRST_CONNECT_WINDOW_MS = readEnvNumber(
-  "AGT_BIND_REMEDIATION_FIRST_CONNECT_WINDOW_MS",
-  9e5
-  // 15 min
-);
+var BIND_REMEDIATION_FIRST_CONNECT_WINDOW_MS = FIRST_CONNECT_WINDOW_MS;
 var BIND_REMEDIATION_MAX_CONNECTIVITY_DEFERRALS = readEnvNumber(
   "AGT_BIND_REMEDIATION_MAX_CONNECTIVITY_DEFERRALS",
   3
@@ -9954,7 +9650,13 @@ async function runAgentConnectivityProbes(agent, integrations, projectDir) {
         // ENG-8055: forward the reconstructed inline server entry (managed
         // quarantined rows) so the MCP probe legs hit the real server instead of
         // the missing `.mcp.json` key. Absent on normal (active) rows.
-        inline_server_entry: i.inline_server_entry
+        inline_server_entry: i.inline_server_entry,
+        // ENG-8363: age + quarantine status, so an absent `.mcp.json` key is
+        // classified rather than guessed. Forwarding both is what makes the
+        // escalation to `down` reachable at all — an omitted `created_at` reads
+        // as "still settling" and never escalates (fail-open by design).
+        created_at: i.created_at ?? null,
+        quarantined: i.quarantined ?? false
       };
     }),
     { probeDeps, intervalMs: intervalSec * 1e3 }
@@ -10251,7 +9953,7 @@ var agentRestartTimezoneInputs = /* @__PURE__ */ new Map();
 var lastVersionCheckAt = 0;
 var VERSION_CHECK_INTERVAL_MS = 5 * 60 * 1e3;
 var lastResponsivenessProbeAt = 0;
-var agtCliVersion = true ? "0.28.500" : "dev";
+var agtCliVersion = true ? "0.28.501" : "dev";
 function resolveBrewPath(execFileSync2) {
   try {
     const out = execFileSync2("which", ["brew"], { timeout: 5e3 }).toString().trim();
@@ -13315,7 +13017,7 @@ async function processAgent(agent, agentStates) {
                 );
               }
             }
-            probeSet = [...probeSet, ...qRows];
+            probeSet = [...probeSet, ...markQuarantinedForProbe(qRows)];
             log(
               `Auto-resume: connectivity-probing ${qRows.length} quarantined integration(s) for '${agent.code_name}'`
             );