@integrity-labs/agt-cli 0.28.1015 → 0.28.1017

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ import {
15
15
  parseEnvFileEntries,
16
16
  parseEnvIntegrations,
17
17
  shellQuote
18
- } from "./chunk-MHDNKKEO.js";
18
+ } from "./chunk-CVQDMHOP.js";
19
19
  import {
20
20
  BIND_FAILURE_QUARANTINE_THRESHOLD,
21
21
  INTEGRATIONS_SECTION_END,
@@ -77,7 +77,7 @@ import {
77
77
  sessionTranscriptDir,
78
78
  worseConnectivityOutcome,
79
79
  wrapScheduledTaskPrompt
80
- } from "./chunk-RJ3SI5JG.js";
80
+ } from "./chunk-X7SX25N6.js";
81
81
  import {
82
82
  classifyUnanswerablePane,
83
83
  summarizeUnanswerablePane
@@ -5507,8 +5507,10 @@ ${sections}`
5507
5507
  const senderPolicyTeamId = options?.senderPolicy?.mode === "team_agents_only" || options?.senderPolicy?.mode === "manager_only" || options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_id : void 0;
5508
5508
  const slackPrincipalId = options?.senderPolicy?.mode === "manager_only" ? options.senderPolicy.principal?.slack_user_id : void 0;
5509
5509
  const teamsPrincipalId = options?.senderPolicy?.mode === "manager_only" ? options.senderPolicy.principal?.teams_aad_object_id : void 0;
5510
+ const telegramPrincipalId = options?.senderPolicy?.mode === "manager_only" ? options.senderPolicy.principal?.telegram_chat_id : void 0;
5510
5511
  const slackTeamPrincipalIds = options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_principals?.slack_user_ids?.join(",") : void 0;
5511
5512
  const teamsTeamPrincipalIds = options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_principals?.teams_aad_object_ids?.join(",") : void 0;
5513
+ const telegramTeamPrincipalIds = options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_principals?.telegram_chat_ids?.join(",") : void 0;
5512
5514
  const senderPolicyInternalOnly = options?.senderPolicy?.internal_only === true;
5513
5515
  const senderPolicyEnv = senderPolicyTeamId ? { AGT_TEAM_ID: senderPolicyTeamId } : {};
5514
5516
  const agentDir = getAgentDir(codeName);
@@ -5546,6 +5548,18 @@ ${sections}`
5546
5548
  if (allowedChats && allowedChats.length > 0) {
5547
5549
  telegramEnv.TELEGRAM_ALLOWED_CHATS = allowedChats.join(",");
5548
5550
  }
5551
+ if (senderPolicyMode) {
5552
+ telegramEnv.TELEGRAM_SENDER_POLICY = senderPolicyMode;
5553
+ }
5554
+ for (const [policyKey, policyValue] of Object.entries(senderPolicyEnv)) {
5555
+ telegramEnv[policyKey] = policyValue;
5556
+ }
5557
+ if (telegramPrincipalId) {
5558
+ telegramEnv.TELEGRAM_SENDER_POLICY_PRINCIPAL_ID = telegramPrincipalId;
5559
+ }
5560
+ if (telegramTeamPrincipalIds) {
5561
+ telegramEnv.TELEGRAM_SENDER_POLICY_TEAM_PRINCIPAL_IDS = telegramTeamPrincipalIds;
5562
+ }
5549
5563
  const telegramAckReaction = config["ack_reaction"];
5550
5564
  if (typeof telegramAckReaction === "string" && telegramAckReaction.trim().length > 0) {
5551
5565
  telegramEnv.TELEGRAM_ACK_REACTION = telegramAckReaction.trim();
@@ -6993,7 +7007,7 @@ function exchangeFailureKind(err) {
6993
7007
  }
6994
7008
 
6995
7009
  // src/lib/api-client.ts
6996
- var agtCliVersion = true ? "0.28.1015" : "dev";
7010
+ var agtCliVersion = true ? "0.28.1017" : "dev";
6997
7011
  var lastConfigHash = null;
6998
7012
  function setConfigHash(hash) {
6999
7013
  lastConfigHash = hash && hash.length > 0 ? hash : null;
@@ -7609,1463 +7623,1634 @@ var HostFlagStore = class {
7609
7623
  }
7610
7624
  };
7611
7625
 
7612
- // src/lib/stale-mcp-reaper.ts
7613
- import { execFileSync } from "child_process";
7614
- function livenessFromProbe(r, onNoAnswer) {
7615
- if (r.kind === "ok") return true;
7616
- if (r.kind === "failed" && r.status !== null) return false;
7617
- onNoAnswer?.(r.kind === "timed-out" ? "did not answer within its bound" : `could not run: ${r.message}`);
7618
- return true;
7626
+ // src/lib/restart-breaker.ts
7627
+ function reaperRestartBreakerReason(activeKeys) {
7628
+ return activeKeys.length >= 2 ? "mcp-presence-reaper" : void 0;
7619
7629
  }
7620
- var rotationTimestamps = /* @__PURE__ */ new Map();
7621
- var DEFAULT_ROTATION_GRACE_MS = 15e3;
7622
- function rotationKey(codeName, serverKey) {
7623
- return `${codeName}\0${serverKey}`;
7630
+ var KNOWN_REASON_CLASS_MAP = {
7631
+ crash: true,
7632
+ provisioning: true,
7633
+ "credential-rotation": true,
7634
+ // ENG-8388: provisioning reloads a human demonstrably asked for. Same
7635
+ // Record-not-Set reasoning as above — a new class cannot be silently omitted.
7636
+ "operator-config": true,
7637
+ // ENG-8215: the self-healing class (bind-remediation - the manager's OWN
7638
+ // repair attempts, which must never count toward a trip). Adding it here is
7639
+ // forced by the Record type, which is exactly why that shape was chosen over
7640
+ // a Set literal (ENG-7577, CodeRabbit on #3861) - a new union member cannot
7641
+ // be silently omitted from the rehydration allowlist.
7642
+ "self-healing": true,
7643
+ // ENG-9138: the platform-scheduled class. Same Record-not-Set reasoning: the
7644
+ // compiler forces every new class into this allowlist, so a `trippedClass`
7645
+ // rehydrated from manager-state.json can never name a class this build does
7646
+ // not know about.
7647
+ scheduled: true,
7648
+ // ENG-10066: the operator-command class. Same Record-not-Set reasoning again —
7649
+ // the compiler forces every new class into this allowlist.
7650
+ "operator-command": true
7651
+ };
7652
+ var KNOWN_REASON_CLASSES = new Set(
7653
+ Object.keys(KNOWN_REASON_CLASS_MAP)
7654
+ );
7655
+ var PROVISIONING_RELOAD_REASONS = /* @__PURE__ */ new Set([
7656
+ "hot-reload-mcp",
7657
+ "managed-mcp-churn",
7658
+ // ENG-8215: 'bind-remediation' MOVED OUT of this set into SELF_HEALING_REASONS.
7659
+ // It is the manager repairing itself, not a config reload, and counting it
7660
+ // meant the harder the platform tried to fix an agent the more certain it
7661
+ // became that it should give up. Deliberately NOT re-added here.
7662
+ // ENG-7576: the dashboard integration-add stop itself - the first restart of
7663
+ // the very burst this class exists for, previously uncounted entirely.
7664
+ "integration-change",
7665
+ // ENG-7771: a sender_policy delivery restart is a deliberate config-delivery
7666
+ // reload (operator changed the policy, or the fail-closed first-poll verify
7667
+ // after a manager restart with no persisted baseline), not a crash. On the
7668
+ // tight crash tally, a host whose sender-policy-baseline.json write keeps
7669
+ // failing (disk full, permissions) would convert the fail-closed restart
7670
+ // into repeated crash tallies across a multi-deploy day and trip the
7671
+ // breaker - taking the agent DOWN, which is strictly worse than the stale
7672
+ // policy the restart exists to fix. The looser provisioning tally still
7673
+ // trips + pages on sustained thrash.
7674
+ "sender-policy-change",
7675
+ // ENG-9802: same argument, same failure mode — a host whose baseline write
7676
+ // keeps failing must not convert a fail-closed policy restart into a breaker
7677
+ // trip that takes the agent down.
7678
+ "telegram-policy-change",
7679
+ // ENG-9861: and again for standing rules. The fail-closed first-poll restart
7680
+ // has exactly the same shape, so it needs the same protection from a host
7681
+ // whose sidecar write keeps failing.
7682
+ "agent-rules-change",
7683
+ // ENG-10510 (Obi-Wan on #6313): a channel-set ADD is a config reload, not a
7684
+ // crash, and it was landing on the crash tally purely by falling through.
7685
+ // It is 'integration-change's exact sibling - both are "a channel or server
7686
+ // was added, re-render .mcp.json and respawn" - and it was never argued for
7687
+ // the crash class, it simply was not in this set.
7688
+ //
7689
+ // This is the ENG-9138 / ENG-10066 species verbatim: a PLATFORM-MANUFACTURED
7690
+ // restart counted as though the agent had crashed. ENG-9138 cost katie 8h16m
7691
+ // when day-rollover fell through the same way; ENG-10066 cost an operator an
7692
+ // auto-pause when a channel /restart did. Both were fixed by naming the
7693
+ // reason rather than leaving it to the catch-all, and this is the third.
7694
+ //
7695
+ // WHY NOW. Until ENG-10510 this restart only ever followed a human adding a
7696
+ // channel, so the crash tally was wrong but rarely reached: a person adding
7697
+ // two channels inside ten minutes is uncommon and self-limiting. Channel
7698
+ // auto-resume is the first thing that fires it UNATTENDED, on a timer, on
7699
+ // any agent with a quarantined channel. An agent with a second crash-class
7700
+ // restart in the same ten minutes (the presence reaper's
7701
+ // 'mcp-presence-reaper', for one) would cross DEFAULT_MAX and auto-pause
7702
+ // with no human anywhere in the loop - which is precisely the ENG-8215 harm
7703
+ // that feature's 2h floor exists to prevent.
7704
+ //
7705
+ // Counted, not exempt: provisioning is a LOOSER tally, not an absent one, so
7706
+ // sustained channel-set thrash still trips and still pages.
7707
+ "channel-set-change"
7708
+ ]);
7709
+ function isProvisioningReloadReason(reason) {
7710
+ return PROVISIONING_RELOAD_REASONS.has(reason);
7624
7711
  }
7625
- function recordStaleRotation(codeName, serverKeys, now = Date.now) {
7626
- const ts = now();
7627
- for (const key of serverKeys) {
7628
- rotationTimestamps.set(rotationKey(codeName, key), ts);
7629
- }
7712
+ var CREDENTIAL_ROTATION_REASONS = /* @__PURE__ */ new Set([
7713
+ "credential-rotation"
7714
+ ]);
7715
+ function isCredentialRotationReason(reason) {
7716
+ return CREDENTIAL_ROTATION_REASONS.has(reason);
7630
7717
  }
7631
- var ROTATION_TRACKING_RETENTION_MS = 15 * 60 * 1e3;
7632
- function classifyRotation(codeName, serverKey, graceMs, now = Date.now, retentionMs = ROTATION_TRACKING_RETENTION_MS) {
7633
- const k = rotationKey(codeName, serverKey);
7634
- const ts = rotationTimestamps.get(k);
7635
- if (ts === void 0) return "none";
7636
- const age = now() - ts;
7637
- if (age < graceMs) return "within-grace";
7638
- if (age < retentionMs) return "expired";
7639
- rotationTimestamps.delete(k);
7640
- return "none";
7718
+ var SELF_HEALING_REASONS = /* @__PURE__ */ new Set([
7719
+ "bind-remediation"
7720
+ ]);
7721
+ function isSelfHealingReason(reason) {
7722
+ return SELF_HEALING_REASONS.has(reason);
7641
7723
  }
7642
- function getRotationRecordedAt(codeName, serverKey) {
7643
- return rotationTimestamps.get(rotationKey(codeName, serverKey));
7724
+ var SCHEDULED_REASONS = /* @__PURE__ */ new Set([
7725
+ "day-rollover"
7726
+ ]);
7727
+ function isScheduledRestartReason(reason) {
7728
+ return SCHEDULED_REASONS.has(reason);
7644
7729
  }
7645
- function clearStaleRotation(codeName, serverKeys) {
7646
- for (const key of serverKeys) {
7647
- rotationTimestamps.delete(rotationKey(codeName, key));
7648
- }
7730
+ var OPERATOR_COMMAND_REASONS = /* @__PURE__ */ new Set([
7731
+ "channel-restart-flag"
7732
+ ]);
7733
+ function isOperatorCommandReason(reason) {
7734
+ return OPERATOR_COMMAND_REASONS.has(reason);
7649
7735
  }
7650
- function parseEnvIntegrationsEntries(content) {
7651
- const entries = /* @__PURE__ */ new Map();
7652
- for (const raw of content.split(/\r?\n/)) {
7653
- const line = raw.trim();
7654
- if (!line || line.startsWith("#")) continue;
7655
- const eq = line.indexOf("=");
7656
- if (eq <= 0) continue;
7657
- const name = line.slice(0, eq).trim();
7658
- if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) continue;
7659
- const value = line.slice(eq + 1);
7660
- entries.set(name, value);
7661
- }
7662
- return entries;
7736
+ var OPERATOR_CONFIG_REASONS = /* @__PURE__ */ new Set([
7737
+ "operator-integration-change"
7738
+ ]);
7739
+ function isOperatorConfigReason(reason) {
7740
+ return OPERATOR_CONFIG_REASONS.has(reason);
7663
7741
  }
7664
- function diffEnvIntegrations(oldContent, newContent) {
7665
- const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
7666
- const newEntries = parseEnvIntegrationsEntries(newContent);
7667
- const changed = /* @__PURE__ */ new Set();
7668
- for (const [name, value] of newEntries) {
7669
- if (oldEntries.get(name) !== value) changed.add(name);
7670
- }
7671
- for (const name of oldEntries.keys()) {
7672
- if (!newEntries.has(name)) changed.add(name);
7673
- }
7674
- return [...changed];
7742
+ var NEVER_INTEGRATION_BUCKETED = /* @__PURE__ */ new Set([
7743
+ "channel-set-change"
7744
+ ]);
7745
+ function isIntegrationBucketedReason(reason) {
7746
+ if (NEVER_INTEGRATION_BUCKETED.has(reason)) return false;
7747
+ return isProvisioningReloadReason(reason) || isOperatorConfigReason(reason);
7675
7748
  }
7676
- function classifyEnvIntegrationsDiff(oldContent, newContent) {
7677
- const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
7678
- const newEntries = parseEnvIntegrationsEntries(newContent);
7679
- const rotated = [];
7680
- const added = [];
7681
- const removed = [];
7682
- for (const [name, value] of newEntries) {
7683
- if (!oldEntries.has(name)) added.push(name);
7684
- else if (oldEntries.get(name) !== value) rotated.push(name);
7685
- }
7686
- for (const name of oldEntries.keys()) {
7687
- if (!newEntries.has(name)) removed.push(name);
7688
- }
7689
- return { rotated, added, removed };
7749
+ function restartReasonClass(reason) {
7750
+ if (isCredentialRotationReason(reason)) return "credential-rotation";
7751
+ if (isSelfHealingReason(reason)) return "self-healing";
7752
+ if (isOperatorConfigReason(reason)) return "operator-config";
7753
+ if (isScheduledRestartReason(reason)) return "scheduled";
7754
+ if (isOperatorCommandReason(reason)) return "operator-command";
7755
+ return isProvisioningReloadReason(reason) ? "provisioning" : "crash";
7690
7756
  }
7691
- function findMcpServersUsingVars(mcp, changedVars) {
7692
- const changedSet = new Set(changedVars);
7693
- if (!mcp?.mcpServers || changedSet.size === 0) return [];
7694
- const result = [];
7695
- for (const [serverKey, entry] of Object.entries(mcp.mcpServers)) {
7696
- if (!entry || typeof entry !== "object") continue;
7697
- const env = entry.env;
7698
- if (!env || typeof env !== "object") continue;
7699
- let matches = false;
7700
- for (const value of Object.values(env)) {
7701
- if (typeof value !== "string") continue;
7702
- const placeholderMatches = value.match(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g);
7703
- if (placeholderMatches) {
7704
- for (const ph of placeholderMatches) {
7705
- const name = ph.slice(2, -1);
7706
- if (changedSet.has(name)) {
7707
- matches = true;
7708
- break;
7709
- }
7710
- }
7711
- }
7712
- if (matches) break;
7713
- }
7714
- if (matches) result.push(serverKey);
7757
+ function countByClass(events) {
7758
+ let crash = 0;
7759
+ let provisioning = 0;
7760
+ let credentialRotation = 0;
7761
+ let selfHealing = 0;
7762
+ let operatorConfig = 0;
7763
+ let scheduled = 0;
7764
+ let operatorCommand = 0;
7765
+ for (const e of events) {
7766
+ const klass = restartReasonClass(e.reason);
7767
+ if (klass === "provisioning") provisioning += 1;
7768
+ else if (klass === "credential-rotation") credentialRotation += 1;
7769
+ else if (klass === "self-healing") selfHealing += 1;
7770
+ else if (klass === "operator-config") operatorConfig += 1;
7771
+ else if (klass === "scheduled") scheduled += 1;
7772
+ else if (klass === "operator-command") operatorCommand += 1;
7773
+ else crash += 1;
7715
7774
  }
7716
- return result;
7775
+ return {
7776
+ crash,
7777
+ provisioning,
7778
+ "credential-rotation": credentialRotation,
7779
+ "self-healing": selfHealing,
7780
+ "operator-config": operatorConfig,
7781
+ scheduled,
7782
+ "operator-command": operatorCommand
7783
+ };
7717
7784
  }
7718
- function envOnlyRespawnVars(changedVars, mcp, channelSecretKeys) {
7719
- const channelSet = new Set(channelSecretKeys);
7720
- return [...changedVars].filter(
7721
- (v) => !channelSet.has(v) && findMcpServersUsingVars(mcp, [v]).length === 0
7722
- );
7785
+ function maxProvisioningBucket(events) {
7786
+ return maxBucketWhere(events, isProvisioningReloadReason);
7723
7787
  }
7724
- function liveProxyExtraHeaderVars(mcp) {
7725
- const out = /* @__PURE__ */ new Set();
7726
- if (!mcp?.mcpServers) return out;
7727
- for (const entry of Object.values(mcp.mcpServers)) {
7728
- const spec = entry?.env?.["AGT_REMOTE_MCP_EXTRA_HEADERS"];
7729
- if (typeof spec !== "string") continue;
7730
- for (const pair of spec.split(",")) {
7731
- const colon = pair.indexOf(":");
7732
- if (colon <= 0) continue;
7733
- const varName = pair.slice(colon + 1).trim();
7734
- if (varName) out.add(varName);
7735
- }
7788
+ function maxBucketWhere(events, matches) {
7789
+ const buckets = /* @__PURE__ */ new Map();
7790
+ for (const e of events) {
7791
+ if (!matches(e.reason)) continue;
7792
+ const key = e.integrationKey ?? "";
7793
+ buckets.set(key, (buckets.get(key) ?? 0) + 1);
7736
7794
  }
7737
- return out;
7738
- }
7739
- function liveProxyTokenVars(mcp) {
7740
- const out = /* @__PURE__ */ new Set();
7741
- if (!mcp?.mcpServers) return out;
7742
- for (const entry of Object.values(mcp.mcpServers)) {
7743
- const tokenVar = entry?.env?.["AGT_REMOTE_MCP_TOKEN_VAR"];
7744
- if (typeof tokenVar === "string" && tokenVar.trim()) {
7745
- out.add(tokenVar.trim());
7746
- }
7795
+ let best = { count: 0, key: "" };
7796
+ for (const [key, count] of buckets) {
7797
+ if (count > best.count) best = { count, key };
7747
7798
  }
7748
- return out;
7799
+ return best;
7749
7800
  }
7750
- function buildArgvMatchersForEntry(key, entry) {
7751
- const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
7752
- const patterns = [];
7753
- const args = entry?.args ?? [];
7754
- for (let i = args.length - 1; i >= 0; i--) {
7755
- const arg = args[i];
7756
- if (typeof arg !== "string" || !arg) continue;
7757
- if (arg.startsWith("-")) continue;
7758
- const stripped = arg.replace(
7759
- /@[^/@]*$/,
7760
- (m) => (
7761
- // @latest, @1.2.3, etc. — drop. But @scope at the START of a
7762
- // package spec must NOT be stripped, so we only strip a tail
7763
- // `@...` if it doesn't itself start with a slash inside.
7764
- // The regex above only matches a single trailing `@...`
7765
- // segment after the last `/`, so this is safe for `@scope/pkg`.
7766
- m.includes("/") ? m : ""
7767
- )
7801
+ function tripClass(trip) {
7802
+ if (trip.trippedClass) return trip.trippedClass;
7803
+ if (!Array.isArray(trip.eventsAtTrip) || trip.eventsAtTrip.length === 0) return "crash";
7804
+ return trip.eventsAtTrip.every((e) => isProvisioningReloadReason(e.reason)) ? "provisioning" : "crash";
7805
+ }
7806
+ var MAX_NEW_AGENT_GRACE_MS = 365 * 24 * 60 * 60 * 1e3;
7807
+ var DEFAULT_MAX = 2;
7808
+ var DEFAULT_WINDOW_MS = 6e5;
7809
+ var DEFAULT_PROVISIONING_MAX = RESTART_BREAKER_PROVISIONING_MAX;
7810
+ var DEFAULT_PROVISIONING_WINDOW_MS = RESTART_BREAKER_PROVISIONING_WINDOW_MS;
7811
+ var DEFAULT_OPERATOR_MAX = RESTART_BREAKER_OPERATOR_MAX;
7812
+ var DEFAULT_SCHEDULED_MAX = RESTART_BREAKER_SCHEDULED_MAX;
7813
+ var DEFAULT_SCHEDULED_WINDOW_MS = RESTART_BREAKER_SCHEDULED_WINDOW_MS;
7814
+ var DEFAULT_OPERATOR_COMMAND_MAX = RESTART_BREAKER_OPERATOR_COMMAND_MAX;
7815
+ var DEFAULT_OPERATOR_COMMAND_WINDOW_MS = RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS;
7816
+ function readEnvNumber(name, fallback) {
7817
+ const raw = process.env[name];
7818
+ if (!raw) return fallback;
7819
+ const parsed = Number(raw);
7820
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
7821
+ }
7822
+ function migrateMapKey(map, from, to) {
7823
+ if (map.has(to) || !map.has(from)) return;
7824
+ map.set(to, map.get(from));
7825
+ map.delete(from);
7826
+ }
7827
+ var RestartBreaker = class {
7828
+ max;
7829
+ windowMs;
7830
+ /**
7831
+ * ENG-8215: NOT readonly - the constructor may clamp it upward to preserve the
7832
+ * quarantine-fires-first ordering against a runtime override. Never mutated
7833
+ * after construction.
7834
+ */
7835
+ provisioningMax;
7836
+ provisioningWindowMs;
7837
+ /**
7838
+ * ENG-8388: the looser operator-config budget. Shares provisioningWindowMs.
7839
+ * NOT readonly - the constructor floors it at `provisioningMax` so setting one
7840
+ * env var without the other cannot invert the two tallies.
7841
+ */
7842
+ operatorMax;
7843
+ /**
7844
+ * ENG-9138: the scheduled class's LOOP bar. Readonly and deliberately NOT
7845
+ * floored against any other tally: it gates a REPORT, not a pause, so it
7846
+ * cannot invert anything the way operatorMax/provisioningMax can.
7847
+ */
7848
+ scheduledMax;
7849
+ /** ENG-9138: sliding window for the scheduled tally. */
7850
+ scheduledWindowMs;
7851
+ operatorCommandMax;
7852
+ operatorCommandWindowMs;
7853
+ /** Longest window any tally needs — how long the events log must retain. */
7854
+ retentionMs;
7855
+ now;
7856
+ /** ENG-7891 slice 5b: code_name → durable runtime key (agent_id when id-keyed). Identity on the legacy fleet. */
7857
+ runtimeKey;
7858
+ /** ENG-8690: agent creation time lookup; undefined result ⇒ no grace. */
7859
+ agentCreatedAt;
7860
+ /** ENG-8690: grace duration; 0 disables. */
7861
+ newAgentGraceMs;
7862
+ /** ENG-8690: the bound the grace does NOT lift. NOT readonly - floored below. */
7863
+ newAgentGraceCeiling;
7864
+ events = /* @__PURE__ */ new Map();
7865
+ trips = /* @__PURE__ */ new Map();
7866
+ /**
7867
+ * ENG-8215: set when a supplied `provisioningMax` was raised to keep MCP
7868
+ * quarantine ahead of the breaker. Read once by the manager at construction so
7869
+ * an overridden-and-clamped value is visible in manager.log rather than
7870
+ * silently discarded.
7871
+ */
7872
+ quarantineOrderingClamped;
7873
+ constructor(opts = {}) {
7874
+ this.max = opts.max ?? readEnvNumber("AGT_RESTART_BREAKER_MAX", DEFAULT_MAX);
7875
+ this.windowMs = opts.windowMs ?? readEnvNumber("AGT_RESTART_BREAKER_WINDOW_MS", DEFAULT_WINDOW_MS);
7876
+ this.provisioningMax = opts.provisioningMax ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", DEFAULT_PROVISIONING_MAX);
7877
+ this.provisioningWindowMs = opts.provisioningWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS", DEFAULT_PROVISIONING_WINDOW_MS);
7878
+ this.operatorMax = opts.operatorMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_MAX", DEFAULT_OPERATOR_MAX);
7879
+ this.scheduledMax = opts.scheduledMax ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_MAX", DEFAULT_SCHEDULED_MAX);
7880
+ this.scheduledWindowMs = opts.scheduledWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_WINDOW_MS", DEFAULT_SCHEDULED_WINDOW_MS);
7881
+ this.operatorCommandMax = opts.operatorCommandMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_COMMAND_MAX", DEFAULT_OPERATOR_COMMAND_MAX);
7882
+ this.operatorCommandWindowMs = opts.operatorCommandWindowMs ?? readEnvNumber(
7883
+ "AGT_RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS",
7884
+ DEFAULT_OPERATOR_COMMAND_WINDOW_MS
7768
7885
  );
7769
- const tail = "(?![A-Za-z0-9_-])";
7770
- if (/^@?[a-z0-9]([a-z0-9._-]*\/)?[a-z0-9._-]+$/i.test(stripped)) {
7771
- patterns.push(new RegExp(`${escapeRe(stripped)}${tail}`));
7772
- const basename = stripped.split("/").pop();
7773
- if (basename && basename !== stripped) {
7774
- patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
7775
- }
7776
- break;
7886
+ this.now = opts.now ?? Date.now;
7887
+ this.runtimeKey = opts.runtimeKey ?? ((codeName) => codeName);
7888
+ this.agentCreatedAt = opts.agentCreatedAt ?? (() => void 0);
7889
+ const graceMs = opts.newAgentGraceMs ?? readEnvNumber("AGT_RESTART_BREAKER_NEW_AGENT_GRACE_MS", RESTART_BREAKER_NEW_AGENT_GRACE_MS);
7890
+ const cappedGrace = Math.min(graceMs, MAX_NEW_AGENT_GRACE_MS);
7891
+ this.newAgentGraceMs = Number.isFinite(graceMs) && graceMs > 0 ? cappedGrace : 0;
7892
+ const ceiling = opts.newAgentGraceCeiling ?? readEnvNumber(
7893
+ "AGT_RESTART_BREAKER_NEW_AGENT_GRACE_CEILING",
7894
+ RESTART_BREAKER_NEW_AGENT_GRACE_CEILING
7895
+ );
7896
+ this.newAgentGraceCeiling = Number.isFinite(ceiling) && ceiling > 0 ? ceiling : RESTART_BREAKER_NEW_AGENT_GRACE_CEILING;
7897
+ if (!Number.isFinite(this.max) || this.max < 1) {
7898
+ throw new Error("restart-breaker max must be a finite number >= 1");
7777
7899
  }
7778
- if (stripped.includes("/")) {
7779
- const basename = stripped.split("/").pop();
7780
- if (basename) {
7781
- patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
7782
- break;
7783
- }
7900
+ if (!Number.isFinite(this.windowMs) || this.windowMs < 1e3) {
7901
+ throw new Error("restart-breaker windowMs must be a finite number >= 1000");
7784
7902
  }
7785
- }
7786
- if (patterns.length === 0) {
7787
- const safe = escapeRe(key);
7788
- patterns.push(new RegExp(`(?<![A-Za-z0-9_])${safe}(?![A-Za-z0-9_])`));
7789
- if (safe.includes("_")) {
7790
- const dashed = safe.replace(/_/g, "-");
7791
- patterns.push(new RegExp(`(?<![A-Za-z0-9_])${dashed}(?![A-Za-z0-9_])`));
7903
+ if (!Number.isFinite(this.provisioningMax) || this.provisioningMax < 1) {
7904
+ throw new Error("restart-breaker provisioningMax must be a finite number >= 1");
7792
7905
  }
7793
- }
7794
- return patterns;
7795
- }
7796
- function buildClaudeAgentMatcher(codeName) {
7797
- const safe = codeName.replace(/[^A-Za-z0-9_-]/g, "");
7798
- return new RegExp(`\\bclaude\\b.*--name\\s+agt-${safe}(?=\\s|$)`);
7799
- }
7800
- function findMcpChildrenForAgent(args) {
7801
- const { rows, codeName, serverKeys, mcpJson } = args;
7802
- const maxDepth = args.maxDepth ?? 8;
7803
- const inContainer = args.inContainer ?? false;
7804
- if (serverKeys.length === 0 || rows.length === 0) return [];
7805
- const claudeMatcher = inContainer ? /\bclaude\b/ : buildClaudeAgentMatcher(codeName);
7806
- const argvMatchers = [];
7807
- for (const key of serverKeys) {
7808
- const entry = mcpJson?.mcpServers?.[key];
7809
- argvMatchers.push(...buildArgvMatchersForEntry(key, entry));
7810
- }
7811
- const byPid = new Map(rows.map((r) => [r.pid, r]));
7812
- const matched = [];
7813
- for (const row of rows) {
7814
- if (!argvMatchers.some((re) => re.test(row.args))) continue;
7815
- if (/\bclaude\b/.test(row.args) && row.args.includes(`--name agt-${codeName}`)) continue;
7816
- let cur = byPid.get(row.ppid);
7817
- let belongs = false;
7818
- for (let depth = 0; depth < maxDepth && cur; depth++) {
7819
- if (claudeMatcher.test(cur.args)) {
7820
- belongs = true;
7821
- break;
7822
- }
7823
- if (cur.pid === 1) break;
7824
- cur = byPid.get(cur.ppid);
7906
+ if (!Number.isFinite(this.operatorMax) || this.operatorMax < 1) {
7907
+ throw new Error("restart-breaker operatorMax must be a finite number >= 1");
7825
7908
  }
7826
- if (belongs) matched.push(row.pid);
7827
- }
7828
- return matched;
7829
- }
7830
- function reapStaleMcpChildren(args) {
7831
- const { log: log2, codeName, serverKeys, mcpJson, graceMs = 5e3, isolated = false } = args;
7832
- if (serverKeys.length === 0) return [];
7833
- const runPs = args.runPs ?? (isolated ? (
7834
- // ENG-9781: hard-bounded. See bounded-exec.ts — `execFileSync`'s own
7835
- // `timeout` is not a bound, and this runs per isolated agent per cycle
7836
- // on the manager-worker's critical path. Throws on a non-answer so the
7837
- // caller's existing catch skips the agent rather than acting on an
7838
- // empty process list.
7839
- () => {
7840
- const r = hardBoundedExecFile(
7841
- "docker",
7842
- ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
7843
- { timeoutMs: 8e3, log: log2 }
7844
- );
7845
- if (r.kind === "ok") return r.stdout;
7846
- throw new Error(
7847
- r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
7848
- );
7909
+ if (this.operatorMax < this.provisioningMax) {
7910
+ this.operatorMax = this.provisioningMax;
7849
7911
  }
7850
- ) : () => execFileSync("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
7851
- const killProcess = args.killProcess ?? (isolated ? (pid, signal) => {
7852
- hardBoundedExecFile(
7853
- "docker",
7854
- ["exec", `agt-${codeName}`, "kill", signal === "SIGKILL" ? "-KILL" : "-TERM", String(pid)],
7855
- { timeoutMs: 8e3, log: log2 }
7856
- );
7857
- } : (pid, signal) => {
7858
- try {
7859
- process.kill(pid, signal);
7860
- } catch {
7912
+ if (this.provisioningMax < MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING) {
7913
+ this.quarantineOrderingClamped = {
7914
+ requested: this.provisioningMax,
7915
+ applied: MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING,
7916
+ quarantineThreshold: BIND_FAILURE_QUARANTINE_THRESHOLD
7917
+ };
7918
+ this.provisioningMax = MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING;
7861
7919
  }
7862
- });
7863
- const isAlive = args.isAlive ?? (isolated ? (pid) => {
7864
- const r = hardBoundedExecFile(
7865
- "docker",
7866
- ["exec", `agt-${codeName}`, "kill", "-0", String(pid)],
7867
- { timeoutMs: 8e3, log: log2 }
7868
- );
7869
- return livenessFromProbe(
7870
- r,
7871
- (why) => log2(
7872
- `[stale-mcp-reaper] liveness probe for pid ${pid} in 'agt-${codeName}' ${why} \u2014 treating as ALIVE (no reap) rather than guessing`
7873
- )
7874
- );
7875
- } : (pid) => {
7876
- try {
7877
- process.kill(pid, 0);
7878
- return true;
7879
- } catch {
7880
- return false;
7920
+ if (!Number.isFinite(this.provisioningWindowMs) || this.provisioningWindowMs < 1e3) {
7921
+ throw new Error("restart-breaker provisioningWindowMs must be a finite number >= 1000");
7881
7922
  }
7882
- });
7883
- let psOutput;
7884
- try {
7885
- psOutput = runPs();
7886
- } catch (err) {
7887
- log2(`[stale-mcp-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping reap`);
7888
- return [];
7889
- }
7890
- const rows = parsePsRows(psOutput);
7891
- const targets = findMcpChildrenForAgent({ rows, codeName, serverKeys, mcpJson, inContainer: isolated });
7892
- if (targets.length === 0) return [];
7893
- const byPid = new Map(rows.map((r) => [r.pid, r]));
7894
- const describe = (pid) => {
7895
- const argv = byPid.get(pid)?.args ?? "";
7896
- const pkgMatch = argv.match(/(@[a-z0-9_-]+\/[a-z0-9_-]+|[a-z0-9_-]+-mcp-server|[a-z0-9_-]+-mcp\b)/i);
7897
- return pkgMatch ? `${pkgMatch[1]} (pid ${pid})` : `pid ${pid}`;
7898
- };
7899
- log2(
7900
- `[stale-mcp-reaper] '${codeName}': rotating ${targets.length} stale MCP child(ren) for [${serverKeys.join(", ")}]: ${targets.map(describe).join(", ")}`
7901
- );
7902
- for (const pid of targets) {
7903
- killProcess(pid, "SIGTERM");
7904
- }
7905
- const killedKeys = /* @__PURE__ */ new Set();
7906
- for (const pid of targets) {
7907
- const argv = byPid.get(pid)?.args ?? "";
7908
- for (const key of serverKeys) {
7909
- const entry = mcpJson?.mcpServers?.[key];
7910
- const matchers = buildArgvMatchersForEntry(key, entry);
7911
- if (matchers.some((re) => re.test(argv))) killedKeys.add(key);
7923
+ if (!Number.isFinite(this.scheduledMax) || this.scheduledMax < 1) {
7924
+ throw new Error("restart-breaker scheduledMax must be a finite number >= 1");
7912
7925
  }
7913
- }
7914
- if (killedKeys.size > 0) recordStaleRotation(codeName, killedKeys);
7915
- setTimeout(() => {
7916
- try {
7917
- let freshPsOutput;
7918
- try {
7919
- freshPsOutput = runPs();
7920
- } catch (err) {
7921
- log2(`[stale-mcp-reaper] '${codeName}': fresh ps for SIGKILL re-verify failed: ${err.message} \u2014 skipping SIGKILL pass`);
7922
- return;
7923
- }
7924
- const stillOwned = new Set(
7925
- findMcpChildrenForAgent({
7926
- rows: parsePsRows(freshPsOutput),
7927
- codeName,
7928
- serverKeys,
7929
- mcpJson,
7930
- inContainer: isolated
7931
- })
7932
- );
7933
- const stragglers = targets.filter((pid) => isAlive(pid) && stillOwned.has(pid));
7934
- if (stragglers.length === 0) return;
7935
- log2(
7936
- `[stale-mcp-reaper] '${codeName}': ${stragglers.length} child(ren) survived SIGTERM; sending SIGKILL: ${stragglers.map(describe).join(", ")}`
7937
- );
7938
- for (const pid of stragglers) {
7939
- killProcess(pid, "SIGKILL");
7940
- }
7941
- } catch (err) {
7942
- log2(`[stale-mcp-reaper] '${codeName}': error in SIGKILL pass: ${err.message}`);
7926
+ if (!Number.isFinite(this.scheduledWindowMs) || this.scheduledWindowMs < 1e3) {
7927
+ throw new Error("restart-breaker scheduledWindowMs must be a finite number >= 1000");
7943
7928
  }
7944
- }, graceMs).unref();
7945
- return targets;
7946
- }
7947
-
7948
- // src/lib/mcp-presence-reaper.ts
7949
- import { execFileSync as execFileSync2 } from "child_process";
7950
- var DEFAULT_COLD_START_GRACE_MS = 9e4;
7951
- var MAX_PRESENCE_RESTART_ATTEMPTS = 3;
7952
- var BACKOFF_AFTER_RESTARTS = MAX_PRESENCE_RESTART_ATTEMPTS;
7953
- var BACKOFF_BASE_MS = 6e4;
7954
- var BACKOFF_MAX_MS = 30 * 6e4;
7955
- var BACKOFF_WINDOW_MS = 60 * 6e4;
7956
- var BACKOFF_GIVE_UP_RESTARTS = 2 * MAX_PRESENCE_RESTART_ATTEMPTS;
7957
- var BACKOFF_STABLE_RESET_MS = 5 * 6e4;
7958
- var DEAD_CHANNEL_MS = 15 * 6e4;
7959
- var presenceReaperState = /* @__PURE__ */ new Map();
7960
- var MAX_ROTATION_PROMPT_PROBES = 3;
7961
- var ROTATION_PROMPT_PROBE_INTERVAL_MS = 3e4;
7962
- var rotationStallState = /* @__PURE__ */ new Map();
7963
- function stateKey(codeName, serverKey) {
7964
- return `${codeName}\0${serverKey}`;
7965
- }
7966
- function clearPresenceReaperState(codeName) {
7967
- const prefix = `${codeName}\0`;
7968
- for (const key of presenceReaperState.keys()) {
7969
- if (key.startsWith(prefix)) presenceReaperState.delete(key);
7970
- }
7971
- }
7972
- function clearPresenceReaperStateForKeys(codeName, keys) {
7973
- for (const key of keys) {
7974
- presenceReaperState.delete(stateKey(codeName, key));
7975
- rotationStallState.delete(stateKey(codeName, key));
7976
- }
7977
- }
7978
- function givenUpMcpServerKeys(codeName) {
7979
- const prefix = `${codeName}\0`;
7980
- const out = /* @__PURE__ */ new Set();
7981
- for (const [key, st] of presenceReaperState) {
7982
- if (key.startsWith(prefix) && (st.gaveUp || st.attempts >= MAX_PRESENCE_RESTART_ATTEMPTS)) {
7983
- out.add(key.slice(prefix.length));
7929
+ if (!Number.isFinite(this.operatorCommandMax) || this.operatorCommandMax < 1) {
7930
+ throw new Error("restart-breaker operatorCommandMax must be a finite number >= 1");
7931
+ }
7932
+ if (!Number.isFinite(this.operatorCommandWindowMs) || this.operatorCommandWindowMs < 1e3) {
7933
+ throw new Error("restart-breaker operatorCommandWindowMs must be a finite number >= 1000");
7934
+ }
7935
+ const minimumGraceCeiling = Math.max(this.provisioningMax, this.operatorMax);
7936
+ if (this.newAgentGraceCeiling < minimumGraceCeiling) {
7937
+ this.newAgentGraceCeiling = minimumGraceCeiling;
7984
7938
  }
7939
+ this.retentionMs = Math.max(
7940
+ this.windowMs,
7941
+ this.provisioningWindowMs,
7942
+ this.scheduledWindowMs,
7943
+ this.operatorCommandWindowMs
7944
+ );
7985
7945
  }
7986
- return out;
7987
- }
7988
- function findMissingMcpServers(args) {
7989
- const { rows, codeName, mcpJson, inContainer } = args;
7990
- const servers = mcpJson?.mcpServers ?? {};
7991
- const declared = Object.keys(servers);
7992
- if (declared.length === 0) return [];
7993
- const missing = [];
7994
- for (const key of declared) {
7995
- const entry = servers[key];
7996
- const isStdio = typeof entry?.command === "string";
7997
- if (!isStdio) continue;
7998
- const live = findMcpChildrenForAgent({
7999
- rows,
8000
- codeName,
8001
- serverKeys: [key],
8002
- mcpJson,
8003
- inContainer
8004
- });
8005
- if (live.length === 0) missing.push(key);
7946
+ /**
7947
+ * ENG-8215: the clamp record, if the configured provisioning max had to be
7948
+ * raised to keep MCP quarantine ahead of the breaker. `undefined` when the
7949
+ * configured value already satisfied the ordering.
7950
+ */
7951
+ getQuarantineOrderingClamp() {
7952
+ return this.quarantineOrderingClamped;
8006
7953
  }
8007
- return missing;
8008
- }
8009
- function reapMissingMcpSessions(args) {
8010
- const {
8011
- log: log2,
8012
- codeName,
8013
- mcpJson,
8014
- sessionStartedAt,
8015
- restartPending = false,
8016
- stopSession,
8017
- graceMs = DEFAULT_COLD_START_GRACE_MS,
8018
- rotationGraceMs = DEFAULT_ROTATION_GRACE_MS,
8019
- onGiveUp,
8020
- onRestart,
8021
- // ENG-8225: unwired callbacks make the prompt-probe path a strict no-op —
8022
- // `onRotationRespawnStalled` absent means "not armed", which falls through to
8023
- // the pre-ENG-8225 restart. Legacy callers and tests are unchanged.
8024
- onRotationRespawnStalled,
8025
- onRotationRespawnGaveUp,
8026
- onRotationRespawnOutcome,
8027
- maxRotationPromptProbes = MAX_ROTATION_PROMPT_PROBES,
8028
- rotationPromptProbeIntervalMs = ROTATION_PROMPT_PROBE_INTERVAL_MS,
8029
- // ENG-5932: defaults make quarantine a strict no-op (classify everything
8030
- // essential, mode off) so legacy callers/tests are unaffected.
8031
- classifyKey = () => "essential",
8032
- quarantineDwellMs = 0,
8033
- deadChannelMs = DEAD_CHANNEL_MS,
8034
- quarantineMode = "off",
8035
- onQuarantine,
8036
- isolated = false,
8037
- // ENG-8013: default false → a "no live child" observation is only excluded
8038
- // from the give-up/quarantine budget when the caller positively identifies
8039
- // the current generation as externally-churned. Legacy callers/tests unchanged.
8040
- currentGenerationExternallyStarted = false,
8041
- // ENG-9932: unwired => strict no-op, restart proceeds exactly as before.
8042
- readPane,
8043
- onPaneBlocked
8044
- } = args;
8045
- const now = args.now ?? Date.now;
8046
- const runPs = args.runPs ?? (isolated ? (
8047
- // ENG-9781: HARD-bounded, and the distinction is the whole ticket. The
8048
- // previous `execFileSync(..., { timeout: 8_000 })` bounded nothing —
8049
- // Node fires SIGTERM at the deadline and then keeps waiting, so a
8050
- // docker CLI talking to a wedged daemon blocked the manager-worker's
8051
- // event loop and stopped scheduling for EVERY agent on the host
8052
- // (33m46s, twice, on 2026-09-01). This call runs once per isolated
8053
- // agent per poll cycle, so it is the single worst place on the manager
8054
- // for an unbounded subprocess.
8055
- //
8056
- // Throwing on a non-answer is deliberate rather than lazy: the caller
8057
- // already catches, logs and SKIPS the agent for the cycle, which is
8058
- // exactly right. A timeout must not become an empty process list —
8059
- // that reads as "every declared MCP server is missing" and restarts the
8060
- // session, turning a docker hiccup into a fleet-wide restart storm.
8061
- () => {
8062
- const r = hardBoundedExecFile(
8063
- "docker",
8064
- ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
8065
- { timeoutMs: 8e3, log: log2 }
8066
- );
8067
- if (r.kind === "ok") return r.stdout;
8068
- throw new Error(
8069
- r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
8070
- );
7954
+ /**
7955
+ * ENG-7891 slice 5b: the DURABLE key for this agent's events/trips entries.
7956
+ * Resolves the code_name to the runtime key (agent_id on id-keyed hosts) and,
7957
+ * when they differ, lazily migrates a legacy code_name-keyed entry onto it — so
7958
+ * a trip persisted under the old key by a pre-arming manager (or from before a
7959
+ * codename rename) is found under the new key instead of being lost. Identity on
7960
+ * the legacy fleet ⇒ from === to ⇒ no migration and no behaviour change. Both
7961
+ * maps migrate together so events and a trip for the same agent never split
7962
+ * across keys mid-generation (a migration can flip the resolver mid-process).
7963
+ *
7964
+ * BOUNDARY — a pure-LEGACY rename (a host that was never armed) still strands a
7965
+ * held trip, exactly as it did before slice 5b: with no symlink there is no
7966
+ * agent_id to bridge the old codename to the new, so the runtime key stays the
7967
+ * (new) codename and the trip persisted under the old codename is not found.
7968
+ * That is unchanged, pre-existing behaviour and it is benign — the fresh
7969
+ * generation re-evaluates the agent under its new name and a still-pathological
7970
+ * agent simply re-trips within the window. The rename-survival guarantee this
7971
+ * slice adds is an id-keyed-only, post-arming property (the symlink is the
7972
+ * bridge); it is deliberately NOT retrofitted onto the legacy path.
7973
+ */
7974
+ keyFor(codeName) {
7975
+ const key = this.runtimeKey(codeName);
7976
+ if (key !== codeName) {
7977
+ migrateMapKey(this.events, codeName, key);
7978
+ migrateMapKey(this.trips, codeName, key);
8071
7979
  }
8072
- ) : () => execFileSync2("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
8073
- if (!mcpJson?.mcpServers) {
8074
- return { missing: [], restarted: false, reason: "no-mcp-json" };
8075
- }
8076
- const declaredCount = Object.keys(mcpJson.mcpServers).length;
8077
- if (declaredCount === 0) {
8078
- return { missing: [], restarted: false, reason: "no-declared-servers" };
7980
+ return key;
8079
7981
  }
8080
- if (restartPending) {
8081
- return { missing: [], restarted: false, reason: "restart-pending" };
8082
- }
8083
- if (sessionStartedAt === null) {
8084
- return { missing: [], restarted: false, reason: "session-start-unknown" };
8085
- }
8086
- if (now() - sessionStartedAt < graceMs) {
8087
- return { missing: [], restarted: false, reason: "cold-start" };
7982
+ /** True if this agent's breaker is currently tripped (manager must skip spawn). */
7983
+ isTripped(codeName) {
7984
+ return this.trips.has(this.keyFor(codeName));
8088
7985
  }
8089
- let psOutput;
8090
- try {
8091
- psOutput = runPs();
8092
- } catch (err) {
8093
- log2(`[mcp-presence-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping`);
8094
- return { missing: [], restarted: false, reason: "healthy" };
7986
+ getTrip(codeName) {
7987
+ return this.trips.get(this.keyFor(codeName));
8095
7988
  }
8096
- const rows = parsePsRows(psOutput);
8097
- const missingRaw = findMissingMcpServers({ rows, codeName, mcpJson, inContainer: isolated });
8098
- const rotationGraced = [];
8099
- const rotationProbeArmed = [];
8100
- const rotationProbeWaiting = [];
8101
- const rotationProbeExhausted = [];
8102
- const rotationRespawnOutcomes = {};
8103
- const recordRotationOutcome = (key, outcome) => {
8104
- rotationRespawnOutcomes[key] = outcome;
8105
- if (!onRotationRespawnOutcome) return;
8106
- try {
8107
- onRotationRespawnOutcome(codeName, key, outcome);
8108
- } catch (err) {
8109
- log2(
8110
- `[mcp-presence-reaper] onRotationRespawnOutcome callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8111
- );
8112
- }
8113
- };
8114
- const missing = [];
8115
- for (const key of missingRaw) {
8116
- const rotation = classifyRotation(codeName, key, rotationGraceMs, now);
8117
- if (rotation === "within-grace") {
8118
- rotationGraced.push(key);
8119
- continue;
8120
- }
8121
- if (rotation === "none") {
8122
- rotationStallState.delete(stateKey(codeName, key));
8123
- missing.push(key);
8124
- continue;
8125
- }
8126
- const sk = stateKey(codeName, key);
8127
- const rotationRecordedAt = getRotationRecordedAt(codeName, key) ?? now();
8128
- const priorStall = rotationStallState.get(sk);
8129
- const stall = priorStall && priorStall.rotationRecordedAt === rotationRecordedAt ? priorStall : {
8130
- rotationRecordedAt,
8131
- probeAttempts: 0,
8132
- lastProbeAt: null,
8133
- escalated: false
8134
- };
8135
- if (priorStall && priorStall !== stall) {
8136
- log2(
8137
- `[mcp-presence-reaper] '${codeName}:${key}' discarding stall state from a previous rotation (rotation=${priorStall.rotationRecordedAt}, probes=${priorStall.probeAttempts}, escalated=${priorStall.escalated}) \u2014 this is a NEW rotation (rotation=${rotationRecordedAt}) and gets a full prompt-probe budget (ENG-8225)`
8138
- );
8139
- }
8140
- rotationStallState.set(sk, stall);
8141
- if (stall.escalated) {
8142
- missing.push(key);
8143
- continue;
8144
- }
8145
- const spacingElapsed = stall.lastProbeAt === null || now() - stall.lastProbeAt >= rotationPromptProbeIntervalMs;
8146
- if (!spacingElapsed) {
8147
- rotationProbeWaiting.push(key);
8148
- continue;
8149
- }
8150
- if (stall.probeAttempts >= maxRotationPromptProbes) {
8151
- stall.escalated = true;
8152
- rotationProbeExhausted.push(key);
8153
- recordRotationOutcome(key, "never-respawned");
8154
- log2(
8155
- `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=never-respawned \u2014 child SIGTERM'd by stale-mcp-reaper never returned after ${stall.probeAttempts} prompt bind-probe(s); escalating (integration marked unhealthy + reaper_action alert) and releasing to the normal restart path (ENG-8225)`
8156
- );
8157
- if (onRotationRespawnGaveUp) {
8158
- try {
8159
- onRotationRespawnGaveUp(codeName, key, { attempts: stall.probeAttempts });
8160
- } catch (err) {
8161
- log2(
8162
- `[mcp-presence-reaper] onRotationRespawnGaveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8163
- );
8164
- }
8165
- }
8166
- missing.push(key);
8167
- continue;
8168
- }
8169
- const attempt = stall.probeAttempts + 1;
8170
- let armed = false;
8171
- if (onRotationRespawnStalled) {
8172
- try {
8173
- armed = onRotationRespawnStalled(codeName, key, {
8174
- attempt,
8175
- maxAttempts: maxRotationPromptProbes
8176
- });
8177
- } catch (err) {
8178
- log2(
8179
- `[mcp-presence-reaper] onRotationRespawnStalled callback threw for '${codeName}:${key}' (suppressed; falling back to the pre-ENG-8225 restart path): ${err.message}`
8180
- );
8181
- armed = false;
8182
- }
8183
- }
8184
- if (!armed) {
8185
- missing.push(key);
8186
- continue;
7989
+ /**
7990
+ * Record a restart event. If recording this event puts the count
7991
+ * inside the window above `max`, the breaker trips and the call site
7992
+ * should NOT respawn.
7993
+ *
7994
+ * Idempotent on already-tripped breakers: returns the existing trip
7995
+ * without double-counting events. Callers may still record the reason
7996
+ * via the decision-log for forensics.
7997
+ */
7998
+ record(codeName, reason, integrationKey) {
7999
+ const key = this.keyFor(codeName);
8000
+ const existing = this.trips.get(key);
8001
+ if (existing) {
8002
+ const bucket = maxProvisioningBucket(existing.eventsAtTrip);
8003
+ return {
8004
+ tripped: false,
8005
+ trip: existing,
8006
+ windowCount: existing.eventsAtTrip.length,
8007
+ classCounts: countByClass(existing.eventsAtTrip),
8008
+ maxProvisioningBucket: bucket.count,
8009
+ maxProvisioningBucketKey: bucket.key
8010
+ };
8187
8011
  }
8188
- stall.probeAttempts = attempt;
8189
- stall.lastProbeAt = now();
8190
- rotationProbeArmed.push(key);
8191
- log2(
8192
- `[mcp-presence-reaper] '${codeName}:${key}' rotation grace expired (${rotationGraceMs}ms) with no live child and no tool call to respawn it \u2014 armed prompt bind-probe ${attempt}/${maxRotationPromptProbes} (ENG-7429) instead of restarting the session (ENG-8225)`
8193
- );
8194
- }
8195
- if (rotationGraced.length > 0) {
8196
- log2(
8197
- `[mcp-presence-reaper] '${codeName}': skipping [${rotationGraced.join(", ")}] within rotation grace window (${rotationGraceMs}ms) \u2014 child(ren) just SIGTERM'd by stale-mcp-reaper, awaiting respawn on next tool call (ENG-5344)`
8012
+ const at = this.now();
8013
+ const retentionCutoff = at - this.retentionMs;
8014
+ const prior = (this.events.get(key) ?? []).filter((e) => e.at >= retentionCutoff);
8015
+ const bucketed = isIntegrationBucketedReason(reason);
8016
+ prior.push(bucketed && integrationKey ? { reason, at, integrationKey } : { reason, at });
8017
+ this.events.set(key, prior);
8018
+ const crashCount = prior.filter(
8019
+ (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8020
+ ).length;
8021
+ const provEvents = prior.filter(
8022
+ (e) => e.at >= at - this.provisioningWindowMs && isProvisioningReloadReason(e.reason)
8198
8023
  );
8199
- }
8200
- if (rotationProbeWaiting.length > 0) {
8201
- log2(
8202
- `[mcp-presence-reaper] '${codeName}': skipping [${rotationProbeWaiting.join(", ")}] \u2014 prompt bind-probe already armed for the stalled rotation, waiting out the ${rotationPromptProbeIntervalMs}ms re-arm interval (ENG-8225)`
8024
+ const credRotCount = prior.filter(
8025
+ (e) => e.at >= at - this.provisioningWindowMs && isCredentialRotationReason(e.reason)
8026
+ ).length;
8027
+ const selfHealCount = prior.filter(
8028
+ (e) => e.at >= at - this.provisioningWindowMs && isSelfHealingReason(e.reason)
8029
+ ).length;
8030
+ const operatorEvents = prior.filter(
8031
+ (e) => e.at >= at - this.provisioningWindowMs && isOperatorConfigReason(e.reason)
8203
8032
  );
8204
- }
8205
- const declared = mcpJson.mcpServers ?? {};
8206
- const liveKeys = /* @__PURE__ */ new Set();
8207
- for (const key of Object.keys(declared)) {
8208
- const entry = declared[key];
8209
- if (typeof entry?.command !== "string") continue;
8210
- if (!missingRaw.includes(key)) liveKeys.add(key);
8211
- }
8212
- for (const key of liveKeys) {
8213
- const sk = stateKey(codeName, key);
8214
- if (classifyRotation(codeName, key, rotationGraceMs, now) === "none") {
8215
- rotationStallState.delete(sk);
8216
- continue;
8217
- }
8218
- const stall = rotationStallState.get(sk);
8219
- const viaProbe = (stall?.probeAttempts ?? 0) > 0;
8220
- recordRotationOutcome(
8221
- key,
8222
- viaProbe ? "respawned-after-prompt-probe" : "respawned-on-tool-call"
8033
+ const scheduledEvents = prior.filter(
8034
+ (e) => e.at >= at - this.scheduledWindowMs && isScheduledRestartReason(e.reason)
8223
8035
  );
8224
- log2(
8225
- viaProbe ? `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-after-prompt-probe \u2014 child is live again after ${stall?.probeAttempts} prompt bind-probe(s)${stall?.escalated ? " (recovered AFTER escalation \u2014 the earlier never-respawned verdict stands for that window)" : ""}, no session restart needed (ENG-8225)` : `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-on-tool-call \u2014 child returned on the agent's own next tool call, ENG-5344's assumption held (ENG-8225)`
8036
+ const operatorCommandEvents = prior.filter(
8037
+ (e) => e.at >= at - this.operatorCommandWindowMs && isOperatorCommandReason(e.reason)
8226
8038
  );
8227
- clearStaleRotation(codeName, [key]);
8228
- rotationStallState.delete(sk);
8229
- }
8230
- for (const key of liveKeys) {
8231
- const state = presenceReaperState.get(stateKey(codeName, key));
8232
- if (state) {
8233
- state.attempts = 0;
8234
- state.lastSeenLiveAt = now();
8235
- state.lastAttemptedSessionStartedAt = null;
8236
- state.gaveUpLogged = false;
8237
- state.firstMissingAt = null;
8238
- state.quarantineLogged = false;
8239
- state.continuouslyMissingSince = null;
8240
- state.deadChannelLogged = false;
8241
- state.quarantineWithheldLogged = false;
8242
- state.gaveUp = false;
8243
- if (state.stableLiveSince === null) state.stableLiveSince = now();
8244
- if (now() - state.stableLiveSince >= BACKOFF_STABLE_RESET_MS) {
8245
- state.restartHistory = [];
8246
- }
8247
- }
8248
- }
8249
- if (missing.length === 0) {
8250
- return {
8251
- missing,
8252
- restarted: false,
8253
- reason: "healthy",
8254
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8255
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8256
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8257
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8258
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
8039
+ const windowCount = prior.filter((e) => e.at >= at - this.windowMs).length;
8040
+ const classCounts = {
8041
+ crash: crashCount,
8042
+ provisioning: provEvents.length,
8043
+ "credential-rotation": credRotCount,
8044
+ "self-healing": selfHealCount,
8045
+ "operator-config": operatorEvents.length,
8046
+ scheduled: scheduledEvents.length,
8047
+ "operator-command": operatorCommandEvents.length
8259
8048
  };
8260
- }
8261
- const nowMs = now();
8262
- const givenUp = [];
8263
- const active = [];
8264
- const backingOff = [];
8265
- const newGenerationKeys = /* @__PURE__ */ new Set();
8266
- const quarantined = [];
8267
- const wouldQuarantine = [];
8268
- const quarantineWithheld = [];
8269
- const anyStdioServerLive = liveKeys.size > 0;
8270
- const genuinelyDead = [];
8271
- for (const key of missing) {
8272
- const sk = stateKey(codeName, key);
8273
- const state = presenceReaperState.get(sk) ?? {
8274
- attempts: 0,
8275
- lastSeenLiveAt: null,
8276
- lastAttemptedSessionStartedAt: null,
8277
- gaveUpLogged: false,
8278
- firstMissingAt: null,
8279
- quarantineLogged: false,
8280
- restartHistory: [],
8281
- stableLiveSince: null,
8282
- continuouslyMissingSince: null,
8283
- deadChannelLogged: false,
8284
- quarantineWithheldLogged: false,
8285
- gaveUp: false
8286
- };
8287
- if (state.continuouslyMissingSince === null) {
8288
- state.continuouslyMissingSince = nowMs;
8289
- }
8290
- const isGenuinelyDead = nowMs - state.continuouslyMissingSince >= deadChannelMs;
8291
- const suppressAsChurn = currentGenerationExternallyStarted && !isGenuinelyDead;
8292
- if (isGenuinelyDead) {
8293
- genuinelyDead.push(key);
8294
- if (currentGenerationExternallyStarted && !state.deadChannelLogged) {
8295
- state.deadChannelLogged = true;
8296
- log2(
8297
- `[mcp-presence-reaper] '${codeName}:${key}' continuously missing ${Math.round((nowMs - state.continuouslyMissingSince) / 6e4)}m (> ${Math.round(deadChannelMs / 6e4)}m) \u2014 genuinely dead, overriding storm-awareness so it can reach give-up/quarantine instead of churning (ENG-8062)`
8298
- );
8299
- }
8049
+ const scheduledLoop = scheduledEvents.length > this.scheduledMax ? {
8050
+ count: scheduledEvents.length,
8051
+ windowMs: this.scheduledWindowMs,
8052
+ reason: scheduledEvents[scheduledEvents.length - 1].reason
8053
+ } : void 0;
8054
+ const operatorCommandLoop = operatorCommandEvents.length > this.operatorCommandMax ? {
8055
+ count: operatorCommandEvents.length,
8056
+ windowMs: this.operatorCommandWindowMs,
8057
+ reason: operatorCommandEvents[operatorCommandEvents.length - 1].reason
8058
+ } : void 0;
8059
+ const provBucket = maxProvisioningBucket(provEvents);
8060
+ const operatorBucket = maxBucketWhere(operatorEvents, isOperatorConfigReason);
8061
+ const createdAt = this.newAgentGraceMs > 0 ? this.agentCreatedAt(codeName) : void 0;
8062
+ const age = typeof createdAt === "number" && Number.isFinite(createdAt) ? at - createdAt : void 0;
8063
+ const newAgentGraceActive = age !== void 0 && age >= 0 && age < this.newAgentGraceMs;
8064
+ const newAgentGraceExpiresAt = newAgentGraceActive && typeof createdAt === "number" ? createdAt + this.newAgentGraceMs : void 0;
8065
+ const graceCeilingEvents = newAgentGraceActive ? prior.filter(
8066
+ (e) => e.at >= at - this.provisioningWindowMs && (isProvisioningReloadReason(e.reason) || isOperatorConfigReason(e.reason))
8067
+ ) : [];
8068
+ const graceCeilingExceeded = newAgentGraceActive && graceCeilingEvents.length > this.newAgentGraceCeiling;
8069
+ let trippedClass;
8070
+ let tripEvents;
8071
+ let tripWindowMs = this.windowMs;
8072
+ let tripBucketKey = "";
8073
+ if (crashCount > this.max) {
8074
+ trippedClass = "crash";
8075
+ tripEvents = prior.filter(
8076
+ (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8077
+ );
8078
+ tripWindowMs = this.windowMs;
8079
+ } else if (newAgentGraceActive && graceCeilingExceeded) {
8080
+ trippedClass = "provisioning";
8081
+ tripEvents = graceCeilingEvents;
8082
+ tripWindowMs = this.provisioningWindowMs;
8083
+ tripBucketKey = "";
8084
+ } else if (newAgentGraceActive) {
8085
+ } else if (operatorBucket.count > this.operatorMax) {
8086
+ trippedClass = "operator-config";
8087
+ tripEvents = operatorEvents.filter((e) => (e.integrationKey ?? "") === operatorBucket.key);
8088
+ tripWindowMs = this.provisioningWindowMs;
8089
+ tripBucketKey = operatorBucket.key;
8090
+ } else if (provBucket.count > this.provisioningMax) {
8091
+ trippedClass = "provisioning";
8092
+ tripEvents = provEvents.filter((e) => (e.integrationKey ?? "") === provBucket.key);
8093
+ tripWindowMs = this.provisioningWindowMs;
8094
+ tripBucketKey = provBucket.key;
8300
8095
  }
8301
- if (state.firstMissingAt === null || suppressAsChurn) {
8302
- state.firstMissingAt = nowMs;
8096
+ if (trippedClass && tripEvents) {
8097
+ const baseMessage = graceCeilingExceeded ? formatStatusMessage(tripEvents, tripWindowMs) : formatStatusMessage(tripEvents, tripWindowMs, trippedClass, tripBucketKey);
8098
+ const graceHours = Math.round(this.newAgentGraceMs / 36e5 * 10) / 10;
8099
+ const statusMessage = graceCeilingExceeded ? `${baseMessage} \u2014 paused DESPITE the new-agent grace: ${graceCeilingEvents.length} provisioning-shaped restarts (provisioning + operator-config) exceeded the grace ceiling of ${this.newAgentGraceCeiling} (ENG-8690). The agent is inside its ${graceHours}h grace and is doing little except restarting.` : baseMessage;
8100
+ const trip = {
8101
+ trippedAt: at,
8102
+ eventsAtTrip: [...tripEvents],
8103
+ statusMessage,
8104
+ // ENG-7577: persist the tripping class so the recovery path can gate on
8105
+ // it after a manager restart, without re-deriving it from prose.
8106
+ trippedClass
8107
+ };
8108
+ this.trips.set(key, trip);
8109
+ this.events.delete(key);
8110
+ return {
8111
+ tripped: true,
8112
+ trip,
8113
+ windowCount,
8114
+ classCounts,
8115
+ trippedClass,
8116
+ maxProvisioningBucket: provBucket.count,
8117
+ maxProvisioningBucketKey: provBucket.key,
8118
+ ...graceCeilingExceeded ? { newAgentGraceCeilingExceeded: true } : {},
8119
+ // ENG-9138: a co-occurring rollover loop still reports on a real trip.
8120
+ ...scheduledLoop ? { scheduledLoop } : {},
8121
+ ...operatorCommandLoop ? { operatorCommandLoop } : {}
8122
+ };
8303
8123
  }
8304
- state.stableLiveSince = null;
8305
- if (state.lastAttemptedSessionStartedAt !== sessionStartedAt) {
8306
- state.lastAttemptedSessionStartedAt = sessionStartedAt;
8307
- if (!suppressAsChurn) {
8308
- state.attempts += 1;
8309
- newGenerationKeys.add(key);
8310
- }
8124
+ const suppressedOperator = operatorBucket.count > this.operatorMax;
8125
+ const suppressedProvisioning = !suppressedOperator && provBucket.count > this.provisioningMax;
8126
+ const wouldHaveTripped = suppressedOperator || suppressedProvisioning;
8127
+ return {
8128
+ tripped: false,
8129
+ windowCount,
8130
+ classCounts,
8131
+ maxProvisioningBucket: provBucket.count,
8132
+ maxProvisioningBucketKey: provBucket.key,
8133
+ ...scheduledLoop ? { scheduledLoop } : {},
8134
+ ...operatorCommandLoop ? { operatorCommandLoop } : {},
8135
+ ...newAgentGraceActive ? {
8136
+ newAgentGraceSuppressedTrip: wouldHaveTripped,
8137
+ ...newAgentGraceExpiresAt !== void 0 ? { newAgentGraceExpiresAt } : {},
8138
+ ...wouldHaveTripped ? {
8139
+ newAgentGraceSuppressed: {
8140
+ class: suppressedOperator ? "operator-config" : "provisioning",
8141
+ bucketKey: suppressedOperator ? operatorBucket.key : provBucket.key,
8142
+ bucketCount: suppressedOperator ? operatorBucket.count : provBucket.count,
8143
+ agentAgeMs: age ?? 0
8144
+ }
8145
+ } : {}
8146
+ } : {}
8147
+ };
8148
+ }
8149
+ /** Operator-initiated reset: drops the trip + the events log for this agent. */
8150
+ clear(codeName) {
8151
+ const key = this.keyFor(codeName);
8152
+ this.trips.delete(key);
8153
+ this.events.delete(key);
8154
+ if (key !== codeName) {
8155
+ this.trips.delete(codeName);
8156
+ this.events.delete(codeName);
8311
8157
  }
8312
- state.restartHistory = state.restartHistory.filter((t) => nowMs - t < BACKOFF_WINDOW_MS);
8313
- const restartsInWindow = state.restartHistory.length;
8314
- presenceReaperState.set(sk, state);
8315
- const overAttemptsCap = state.attempts > MAX_PRESENCE_RESTART_ATTEMPTS;
8316
- const overRateCap = restartsInWindow >= BACKOFF_GIVE_UP_RESTARTS;
8317
- if (overAttemptsCap || overRateCap) {
8318
- givenUp.push(key);
8319
- state.gaveUp = true;
8320
- if (!state.gaveUpLogged) {
8321
- const cause = overAttemptsCap ? `${MAX_PRESENCE_RESTART_ATTEMPTS} consecutive failed restarts \u2014 declared but never recovers (likely orphaned config; see ENG-5279)` : `${restartsInWindow} restarts within ${Math.round(BACKOFF_WINDOW_MS / 6e4)}m \u2014 persistently flapping (crash-loop; ENG-6480)`;
8322
- log2(
8323
- `[mcp-presence-reaper] giving up on '${codeName}:${key}' after ${cause}`
8158
+ }
8159
+ /** Snapshot tripped agents for `manager-state.json`. */
8160
+ serialize() {
8161
+ return Object.fromEntries(this.trips.entries());
8162
+ }
8163
+ /**
8164
+ * Rehydrate trip state from `manager-state.json`. Called once at
8165
+ * worker startup. Window history is intentionally NOT persisted — only
8166
+ * tripped state. A tripped breaker survives manager restart; an
8167
+ * un-tripped one starts a fresh window in the new worker.
8168
+ */
8169
+ hydrate(saved) {
8170
+ if (!saved) return;
8171
+ for (const [codeName, trip] of Object.entries(saved)) {
8172
+ if (trip && typeof trip.trippedAt === "number" && Array.isArray(trip.eventsAtTrip)) {
8173
+ const klass = trip.trippedClass;
8174
+ this.trips.set(
8175
+ codeName,
8176
+ klass === void 0 || KNOWN_REASON_CLASSES.has(klass) ? trip : { ...trip, trippedClass: void 0 }
8324
8177
  );
8325
- state.gaveUpLogged = true;
8326
- if (onGiveUp) {
8327
- try {
8328
- onGiveUp(codeName, key);
8329
- } catch (err) {
8330
- log2(
8331
- `[mcp-presence-reaper] onGiveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8332
- );
8333
- }
8334
- }
8335
- }
8336
- if (quarantineMode !== "off" && !state.quarantineLogged && classifyKey(key) === "optional") {
8337
- const dwellElapsed = quarantineDwellMs <= 0 || state.firstMissingAt !== null && nowMs - state.firstMissingAt >= quarantineDwellMs;
8338
- if (dwellElapsed && !anyStdioServerLive) {
8339
- quarantineWithheld.push(key);
8340
- if (!state.quarantineWithheldLogged) {
8341
- state.quarantineWithheldLogged = true;
8342
- log2(
8343
- `[mcp-presence-reaper] '${codeName}:${key}' given up but NOT quarantined - no declared stdio MCP server is live, so the restart budget was exhausted by a session that cannot spawn anything, not by this channel dying. Check the pane (tmux attach -t agt-${codeName}); the channel stays provisioned and will quarantine on a later tick if it is still missing once the session is up (ENG-10500)`
8344
- );
8345
- }
8346
- } else if (dwellElapsed) {
8347
- state.quarantineLogged = true;
8348
- if (quarantineMode === "enforce") {
8349
- quarantined.push(key);
8350
- log2(
8351
- `[mcp-presence-reaper] QUARANTINE '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; dropping from provisioned set so the dead channel stops restarting the whole session (ENG-5932)`
8352
- );
8353
- if (onQuarantine) {
8354
- try {
8355
- onQuarantine(codeName, key);
8356
- } catch (err) {
8357
- log2(
8358
- `[mcp-presence-reaper] onQuarantine callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8359
- );
8360
- }
8361
- }
8362
- } else {
8363
- wouldQuarantine.push(key);
8364
- log2(
8365
- `[mcp-presence-reaper] SHADOW would-quarantine '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; no action taken (ENG-5932 shadow mode)`
8366
- );
8367
- }
8368
- }
8369
- }
8370
- } else {
8371
- const backoffMs = restartsInWindow < BACKOFF_AFTER_RESTARTS ? 0 : Math.min(
8372
- BACKOFF_BASE_MS * 2 ** (restartsInWindow - BACKOFF_AFTER_RESTARTS),
8373
- BACKOFF_MAX_MS
8374
- );
8375
- const lastRestartAt = state.restartHistory[state.restartHistory.length - 1] ?? null;
8376
- if (lastRestartAt !== null && nowMs - lastRestartAt < backoffMs) {
8377
- backingOff.push(key);
8378
- } else {
8379
- active.push(key);
8380
8178
  }
8381
8179
  }
8382
8180
  }
8383
- if (active.length === 0) {
8384
- const reason = backingOff.length > 0 ? "backoff" : "all-keys-over-cap";
8385
- if (backingOff.length > 0) {
8386
- log2(
8387
- `[mcp-presence-reaper] '${codeName}': holding off restart for [${backingOff.join(", ")}] \u2014 escalating backoff interval not yet elapsed (ENG-6480)${givenUp.length > 0 ? ` (given up: [${givenUp.join(", ")}])` : ""}`
8388
- );
8389
- }
8390
- return {
8391
- missing,
8392
- restarted: false,
8393
- reason,
8394
- givenUp,
8395
- backingOff: backingOff.length > 0 ? backingOff : void 0,
8396
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8397
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8398
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8399
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8400
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
8401
- quarantined: quarantined.length > 0 ? quarantined : void 0,
8402
- wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
8403
- genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
8404
- quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
8405
- };
8181
+ /**
8182
+ * ENG-10878: how many CRASH-bar restarts `codeName` can still afford in the
8183
+ * current window.
8184
+ *
8185
+ * Exists so a caller can decline to spend the last unit of a budget it did
8186
+ * not fill. `mcp-presence-reaper` sits on this bar next to `auth-tuple-change`
8187
+ * and `model-change`, so a reaper restart landing beside an auth-tuple
8188
+ * rotation is the pair that trips it - and the agent is paused on the other
8189
+ * reason's account.
8190
+ *
8191
+ * THE ORIGINAL VERSION OF THIS PARAGRAPH NAMED THE WRONG REASONS (obi-wan
8192
+ * `486aa45fa3bd`), and it said so in three files. It claimed `day-rollover`
8193
+ * and `channel-set-change` share this bar and used "a reaper restart at 00:59
8194
+ * beside a day-rollover at 01:00" as the worked example. Neither is crash
8195
+ * class: `restartReasonClass('day-rollover')` is `'scheduled'` (ENG-9138) and
8196
+ * `'channel-set-change'` is `'provisioning'` (ENG-10510), so neither is
8197
+ * counted by the filter below and that pair cannot trip anything. The PR's
8198
+ * own new test asserts exactly that, which is the sort of contradiction that
8199
+ * should have been caught by writing the test - it was not, because the test
8200
+ * and the prose were written for different purposes an hour apart.
8201
+ *
8202
+ * The GUARD is unaffected: `auth-tuple-change` and `model-change` really are
8203
+ * on this bar, so the co-occurrence risk is real. Only the example was wrong,
8204
+ * and an argument resting on an impossible example is worth no more than the
8205
+ * example.
8206
+ *
8207
+ * The arithmetic lives HERE rather than at the call site on purpose: `max`
8208
+ * and the window are this class's business, they are env-tunable
8209
+ * (AGT_RESTART_BREAKER_MAX / _WINDOW_MS), and a caller doing
8210
+ * `max - windowCount()` itself would silently stop being correct the moment
8211
+ * either moves. Floored at 0 so an already-tripped agent reads as "nothing
8212
+ * left" rather than a negative number.
8213
+ *
8214
+ * CRASH BAR ONLY. The provisioning, operator, scheduled and operator-command
8215
+ * bars have their own maxima and windows; this answers for the default bar,
8216
+ * which is the one `mcp-presence-reaper` is counted against.
8217
+ */
8218
+ remainingCrashBudget(codeName) {
8219
+ const key = this.keyFor(codeName);
8220
+ if (this.trips.has(key)) return 0;
8221
+ const cutoff = this.now() - this.windowMs;
8222
+ const crashCount = (this.events.get(key) ?? []).filter(
8223
+ (e) => e.at >= cutoff && restartReasonClass(e.reason) === "crash"
8224
+ ).length;
8225
+ return Math.max(0, this.max - crashCount);
8406
8226
  }
8407
- if (readPane) {
8408
- let pane = null;
8409
- try {
8410
- pane = readPane(codeName);
8411
- } catch (err) {
8412
- log2(
8413
- `[mcp-presence-reaper] '${codeName}': pane read threw (${err.message}) \u2014 proceeding with restart (ENG-9932 fails open)`
8414
- );
8415
- }
8416
- const blocked = classifyUnanswerablePane(pane);
8417
- if (blocked) {
8418
- const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
8419
- log2(
8420
- `[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 session is blocked on an interactive prompt (${blocked}), so declared MCP(s) [${active.join(", ")}] were never spawned rather than having died. A restart re-presents the same prompt and burns the ENG-5441 breaker budget. Needs a human at the pane: tmux attach -t agt-${codeName}` + // Fingerprint, never pane text — see summarizeUnanswerablePane.
8421
- (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
8422
- );
8423
- if (onPaneBlocked) {
8424
- try {
8425
- onPaneBlocked(codeName, {
8426
- kind: blocked,
8427
- activeKeys: [...active],
8428
- paneHash: fingerprint?.hash ?? null,
8429
- paneLength: fingerprint?.length ?? null
8430
- });
8431
- } catch (err) {
8432
- log2(
8433
- `[mcp-presence-reaper] onPaneBlocked callback threw for '${codeName}' (suppressed): ${err.message}`
8434
- );
8435
- }
8436
- }
8437
- return {
8438
- missing,
8439
- restarted: false,
8440
- reason: "pane-blocked-on-dialog",
8441
- paneBlockedOn: blocked,
8442
- givenUp: givenUp.length > 0 ? givenUp : void 0,
8443
- backingOff: backingOff.length > 0 ? backingOff : void 0,
8444
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8445
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8446
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8447
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8448
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
8449
- quarantined: quarantined.length > 0 ? quarantined : void 0,
8450
- wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
8451
- genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
8452
- quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
8453
- };
8454
- }
8455
- }
8456
- const givenUpSuffix = givenUp.length > 0 ? ` (skipping over-cap: [${givenUp.join(", ")}])` : "";
8457
- log2(
8458
- `[mcp-presence-reaper] '${codeName}': declared MCP(s) [${active.join(", ")}] have no live children \u2014 restarting session${givenUpSuffix}`
8459
- );
8460
- if (onRestart) {
8461
- try {
8462
- const maybePromise = onRestart(codeName, { activeKeys: active, givenUpKeys: givenUp });
8463
- if (maybePromise && typeof maybePromise.then === "function") {
8464
- void maybePromise.then(void 0, (err) => {
8465
- log2(
8466
- `[mcp-presence-reaper] onRestart callback rejected for '${codeName}' (suppressed; restart proceeds): ${err.message}`
8467
- );
8468
- });
8469
- }
8470
- } catch (err) {
8471
- log2(
8472
- `[mcp-presence-reaper] onRestart callback threw for '${codeName}' (suppressed; restart proceeds): ${err.message}`
8473
- );
8474
- }
8475
- }
8476
- stopSession(codeName, { activeKeys: active });
8477
- for (const key of active) {
8478
- if (!newGenerationKeys.has(key)) continue;
8479
- const st = presenceReaperState.get(stateKey(codeName, key));
8480
- if (st) st.restartHistory.push(nowMs);
8227
+ /** Test helper — current in-window event count for `codeName`. */
8228
+ windowCount(codeName) {
8229
+ const cutoff = this.now() - this.windowMs;
8230
+ return (this.events.get(this.keyFor(codeName)) ?? []).filter((e) => e.at >= cutoff).length;
8481
8231
  }
8482
- return {
8483
- missing,
8484
- restarted: true,
8485
- givenUp: givenUp.length > 0 ? givenUp : void 0,
8486
- backingOff: backingOff.length > 0 ? backingOff : void 0,
8487
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8488
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8489
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8490
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8491
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
8492
- quarantined: quarantined.length > 0 ? quarantined : void 0,
8493
- wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
8494
- genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
8495
- quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
8496
- };
8232
+ };
8233
+ function formatStatusMessage(events, windowMs, klass = "crash", integrationKey = "") {
8234
+ const last = events[events.length - 1];
8235
+ const windowLabel = windowMs < 6e4 ? `${Math.round(windowMs / 1e3)}s` : `${(windowMs / 6e4).toFixed(1).replace(/\.0$/, "")}min`;
8236
+ const reasonCounts = /* @__PURE__ */ new Map();
8237
+ for (const e of events) reasonCounts.set(e.reason, (reasonCounts.get(e.reason) ?? 0) + 1);
8238
+ const breakdown = Array.from(reasonCounts.entries()).map(([r, n]) => `${r}=${n}`).join(", ");
8239
+ const classLabel = klass === "provisioning" ? "provisioning-reload " : klass === "operator-config" ? "operator-requested config-reload " : "";
8240
+ const integrationLabel = (klass === "provisioning" || klass === "operator-config") && integrationKey ? ` for integration '${integrationKey}'` : "";
8241
+ return `Circuit breaker tripped: ${events.length} ${classLabel}restarts in ${windowLabel}${integrationLabel} (${breakdown}); most recent=${last.reason} at ${new Date(last.at).toISOString()}`;
8497
8242
  }
8498
8243
 
8499
- // src/lib/restart-breaker.ts
8500
- function reaperRestartBreakerReason(activeKeys) {
8501
- return activeKeys.length >= 2 ? "mcp-presence-reaper" : void 0;
8244
+ // src/lib/stale-mcp-reaper.ts
8245
+ import { execFileSync } from "child_process";
8246
+ function livenessFromProbe(r, onNoAnswer) {
8247
+ if (r.kind === "ok") return true;
8248
+ if (r.kind === "failed" && r.status !== null) return false;
8249
+ onNoAnswer?.(r.kind === "timed-out" ? "did not answer within its bound" : `could not run: ${r.message}`);
8250
+ return true;
8502
8251
  }
8503
- var KNOWN_REASON_CLASS_MAP = {
8504
- crash: true,
8505
- provisioning: true,
8506
- "credential-rotation": true,
8507
- // ENG-8388: provisioning reloads a human demonstrably asked for. Same
8508
- // Record-not-Set reasoning as above — a new class cannot be silently omitted.
8509
- "operator-config": true,
8510
- // ENG-8215: the self-healing class (bind-remediation - the manager's OWN
8511
- // repair attempts, which must never count toward a trip). Adding it here is
8512
- // forced by the Record type, which is exactly why that shape was chosen over
8513
- // a Set literal (ENG-7577, CodeRabbit on #3861) - a new union member cannot
8514
- // be silently omitted from the rehydration allowlist.
8515
- "self-healing": true,
8516
- // ENG-9138: the platform-scheduled class. Same Record-not-Set reasoning: the
8517
- // compiler forces every new class into this allowlist, so a `trippedClass`
8518
- // rehydrated from manager-state.json can never name a class this build does
8519
- // not know about.
8520
- scheduled: true,
8521
- // ENG-10066: the operator-command class. Same Record-not-Set reasoning again —
8522
- // the compiler forces every new class into this allowlist.
8523
- "operator-command": true
8524
- };
8525
- var KNOWN_REASON_CLASSES = new Set(
8526
- Object.keys(KNOWN_REASON_CLASS_MAP)
8527
- );
8528
- var PROVISIONING_RELOAD_REASONS = /* @__PURE__ */ new Set([
8529
- "hot-reload-mcp",
8530
- "managed-mcp-churn",
8531
- // ENG-8215: 'bind-remediation' MOVED OUT of this set into SELF_HEALING_REASONS.
8532
- // It is the manager repairing itself, not a config reload, and counting it
8533
- // meant the harder the platform tried to fix an agent the more certain it
8534
- // became that it should give up. Deliberately NOT re-added here.
8535
- // ENG-7576: the dashboard integration-add stop itself - the first restart of
8536
- // the very burst this class exists for, previously uncounted entirely.
8537
- "integration-change",
8538
- // ENG-7771: a sender_policy delivery restart is a deliberate config-delivery
8539
- // reload (operator changed the policy, or the fail-closed first-poll verify
8540
- // after a manager restart with no persisted baseline), not a crash. On the
8541
- // tight crash tally, a host whose sender-policy-baseline.json write keeps
8542
- // failing (disk full, permissions) would convert the fail-closed restart
8543
- // into repeated crash tallies across a multi-deploy day and trip the
8544
- // breaker - taking the agent DOWN, which is strictly worse than the stale
8545
- // policy the restart exists to fix. The looser provisioning tally still
8546
- // trips + pages on sustained thrash.
8547
- "sender-policy-change",
8548
- // ENG-9802: same argument, same failure mode — a host whose baseline write
8549
- // keeps failing must not convert a fail-closed policy restart into a breaker
8550
- // trip that takes the agent down.
8551
- "telegram-policy-change",
8552
- // ENG-9861: and again for standing rules. The fail-closed first-poll restart
8553
- // has exactly the same shape, so it needs the same protection from a host
8554
- // whose sidecar write keeps failing.
8555
- "agent-rules-change",
8556
- // ENG-10510 (Obi-Wan on #6313): a channel-set ADD is a config reload, not a
8557
- // crash, and it was landing on the crash tally purely by falling through.
8558
- // It is 'integration-change's exact sibling - both are "a channel or server
8559
- // was added, re-render .mcp.json and respawn" - and it was never argued for
8560
- // the crash class, it simply was not in this set.
8561
- //
8562
- // This is the ENG-9138 / ENG-10066 species verbatim: a PLATFORM-MANUFACTURED
8563
- // restart counted as though the agent had crashed. ENG-9138 cost katie 8h16m
8564
- // when day-rollover fell through the same way; ENG-10066 cost an operator an
8565
- // auto-pause when a channel /restart did. Both were fixed by naming the
8566
- // reason rather than leaving it to the catch-all, and this is the third.
8567
- //
8568
- // WHY NOW. Until ENG-10510 this restart only ever followed a human adding a
8569
- // channel, so the crash tally was wrong but rarely reached: a person adding
8570
- // two channels inside ten minutes is uncommon and self-limiting. Channel
8571
- // auto-resume is the first thing that fires it UNATTENDED, on a timer, on
8572
- // any agent with a quarantined channel. An agent with a second crash-class
8573
- // restart in the same ten minutes (the presence reaper's
8574
- // 'mcp-presence-reaper', for one) would cross DEFAULT_MAX and auto-pause
8575
- // with no human anywhere in the loop - which is precisely the ENG-8215 harm
8576
- // that feature's 2h floor exists to prevent.
8577
- //
8578
- // Counted, not exempt: provisioning is a LOOSER tally, not an absent one, so
8579
- // sustained channel-set thrash still trips and still pages.
8580
- "channel-set-change"
8581
- ]);
8582
- function isProvisioningReloadReason(reason) {
8583
- return PROVISIONING_RELOAD_REASONS.has(reason);
8252
+ var rotationTimestamps = /* @__PURE__ */ new Map();
8253
+ var DEFAULT_ROTATION_GRACE_MS = 15e3;
8254
+ function rotationKey(codeName, serverKey) {
8255
+ return `${codeName}\0${serverKey}`;
8584
8256
  }
8585
- var CREDENTIAL_ROTATION_REASONS = /* @__PURE__ */ new Set([
8586
- "credential-rotation"
8587
- ]);
8588
- function isCredentialRotationReason(reason) {
8589
- return CREDENTIAL_ROTATION_REASONS.has(reason);
8257
+ function recordStaleRotation(codeName, serverKeys, now = Date.now) {
8258
+ const ts = now();
8259
+ for (const key of serverKeys) {
8260
+ rotationTimestamps.set(rotationKey(codeName, key), ts);
8261
+ }
8590
8262
  }
8591
- var SELF_HEALING_REASONS = /* @__PURE__ */ new Set([
8592
- "bind-remediation"
8593
- ]);
8594
- function isSelfHealingReason(reason) {
8595
- return SELF_HEALING_REASONS.has(reason);
8263
+ var ROTATION_TRACKING_RETENTION_MS = 15 * 60 * 1e3;
8264
+ function classifyRotation(codeName, serverKey, graceMs, now = Date.now, retentionMs = ROTATION_TRACKING_RETENTION_MS) {
8265
+ const k = rotationKey(codeName, serverKey);
8266
+ const ts = rotationTimestamps.get(k);
8267
+ if (ts === void 0) return "none";
8268
+ const age = now() - ts;
8269
+ if (age < graceMs) return "within-grace";
8270
+ if (age < retentionMs) return "expired";
8271
+ rotationTimestamps.delete(k);
8272
+ return "none";
8596
8273
  }
8597
- var SCHEDULED_REASONS = /* @__PURE__ */ new Set([
8598
- "day-rollover"
8599
- ]);
8600
- function isScheduledRestartReason(reason) {
8601
- return SCHEDULED_REASONS.has(reason);
8274
+ function getRotationRecordedAt(codeName, serverKey) {
8275
+ return rotationTimestamps.get(rotationKey(codeName, serverKey));
8602
8276
  }
8603
- var OPERATOR_COMMAND_REASONS = /* @__PURE__ */ new Set([
8604
- "channel-restart-flag"
8605
- ]);
8606
- function isOperatorCommandReason(reason) {
8607
- return OPERATOR_COMMAND_REASONS.has(reason);
8277
+ function clearStaleRotation(codeName, serverKeys) {
8278
+ for (const key of serverKeys) {
8279
+ rotationTimestamps.delete(rotationKey(codeName, key));
8280
+ }
8608
8281
  }
8609
- var OPERATOR_CONFIG_REASONS = /* @__PURE__ */ new Set([
8610
- "operator-integration-change"
8611
- ]);
8612
- function isOperatorConfigReason(reason) {
8613
- return OPERATOR_CONFIG_REASONS.has(reason);
8282
+ function parseEnvIntegrationsEntries(content) {
8283
+ const entries = /* @__PURE__ */ new Map();
8284
+ for (const raw of content.split(/\r?\n/)) {
8285
+ const line = raw.trim();
8286
+ if (!line || line.startsWith("#")) continue;
8287
+ const eq = line.indexOf("=");
8288
+ if (eq <= 0) continue;
8289
+ const name = line.slice(0, eq).trim();
8290
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) continue;
8291
+ const value = line.slice(eq + 1);
8292
+ entries.set(name, value);
8293
+ }
8294
+ return entries;
8614
8295
  }
8615
- var NEVER_INTEGRATION_BUCKETED = /* @__PURE__ */ new Set([
8616
- "channel-set-change"
8617
- ]);
8618
- function isIntegrationBucketedReason(reason) {
8619
- if (NEVER_INTEGRATION_BUCKETED.has(reason)) return false;
8620
- return isProvisioningReloadReason(reason) || isOperatorConfigReason(reason);
8296
+ function diffEnvIntegrations(oldContent, newContent) {
8297
+ const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
8298
+ const newEntries = parseEnvIntegrationsEntries(newContent);
8299
+ const changed = /* @__PURE__ */ new Set();
8300
+ for (const [name, value] of newEntries) {
8301
+ if (oldEntries.get(name) !== value) changed.add(name);
8302
+ }
8303
+ for (const name of oldEntries.keys()) {
8304
+ if (!newEntries.has(name)) changed.add(name);
8305
+ }
8306
+ return [...changed];
8621
8307
  }
8622
- function restartReasonClass(reason) {
8623
- if (isCredentialRotationReason(reason)) return "credential-rotation";
8624
- if (isSelfHealingReason(reason)) return "self-healing";
8625
- if (isOperatorConfigReason(reason)) return "operator-config";
8626
- if (isScheduledRestartReason(reason)) return "scheduled";
8627
- if (isOperatorCommandReason(reason)) return "operator-command";
8628
- return isProvisioningReloadReason(reason) ? "provisioning" : "crash";
8629
- }
8630
- function countByClass(events) {
8631
- let crash = 0;
8632
- let provisioning = 0;
8633
- let credentialRotation = 0;
8634
- let selfHealing = 0;
8635
- let operatorConfig = 0;
8636
- let scheduled = 0;
8637
- let operatorCommand = 0;
8638
- for (const e of events) {
8639
- const klass = restartReasonClass(e.reason);
8640
- if (klass === "provisioning") provisioning += 1;
8641
- else if (klass === "credential-rotation") credentialRotation += 1;
8642
- else if (klass === "self-healing") selfHealing += 1;
8643
- else if (klass === "operator-config") operatorConfig += 1;
8644
- else if (klass === "scheduled") scheduled += 1;
8645
- else if (klass === "operator-command") operatorCommand += 1;
8646
- else crash += 1;
8308
+ function classifyEnvIntegrationsDiff(oldContent, newContent) {
8309
+ const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
8310
+ const newEntries = parseEnvIntegrationsEntries(newContent);
8311
+ const rotated = [];
8312
+ const added = [];
8313
+ const removed = [];
8314
+ for (const [name, value] of newEntries) {
8315
+ if (!oldEntries.has(name)) added.push(name);
8316
+ else if (oldEntries.get(name) !== value) rotated.push(name);
8647
8317
  }
8648
- return {
8649
- crash,
8650
- provisioning,
8651
- "credential-rotation": credentialRotation,
8652
- "self-healing": selfHealing,
8653
- "operator-config": operatorConfig,
8654
- scheduled,
8655
- "operator-command": operatorCommand
8656
- };
8657
- }
8658
- function maxProvisioningBucket(events) {
8659
- return maxBucketWhere(events, isProvisioningReloadReason);
8660
- }
8661
- function maxBucketWhere(events, matches) {
8662
- const buckets = /* @__PURE__ */ new Map();
8663
- for (const e of events) {
8664
- if (!matches(e.reason)) continue;
8665
- const key = e.integrationKey ?? "";
8666
- buckets.set(key, (buckets.get(key) ?? 0) + 1);
8318
+ for (const name of oldEntries.keys()) {
8319
+ if (!newEntries.has(name)) removed.push(name);
8667
8320
  }
8668
- let best = { count: 0, key: "" };
8669
- for (const [key, count] of buckets) {
8670
- if (count > best.count) best = { count, key };
8321
+ return { rotated, added, removed };
8322
+ }
8323
+ function findMcpServersUsingVars(mcp, changedVars) {
8324
+ const changedSet = new Set(changedVars);
8325
+ if (!mcp?.mcpServers || changedSet.size === 0) return [];
8326
+ const result = [];
8327
+ for (const [serverKey, entry] of Object.entries(mcp.mcpServers)) {
8328
+ if (!entry || typeof entry !== "object") continue;
8329
+ const env = entry.env;
8330
+ if (!env || typeof env !== "object") continue;
8331
+ let matches = false;
8332
+ for (const value of Object.values(env)) {
8333
+ if (typeof value !== "string") continue;
8334
+ const placeholderMatches = value.match(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g);
8335
+ if (placeholderMatches) {
8336
+ for (const ph of placeholderMatches) {
8337
+ const name = ph.slice(2, -1);
8338
+ if (changedSet.has(name)) {
8339
+ matches = true;
8340
+ break;
8341
+ }
8342
+ }
8343
+ }
8344
+ if (matches) break;
8345
+ }
8346
+ if (matches) result.push(serverKey);
8671
8347
  }
8672
- return best;
8348
+ return result;
8673
8349
  }
8674
- function tripClass(trip) {
8675
- if (trip.trippedClass) return trip.trippedClass;
8676
- if (!Array.isArray(trip.eventsAtTrip) || trip.eventsAtTrip.length === 0) return "crash";
8677
- return trip.eventsAtTrip.every((e) => isProvisioningReloadReason(e.reason)) ? "provisioning" : "crash";
8350
+ function envOnlyRespawnVars(changedVars, mcp, channelSecretKeys) {
8351
+ const channelSet = new Set(channelSecretKeys);
8352
+ return [...changedVars].filter(
8353
+ (v) => !channelSet.has(v) && findMcpServersUsingVars(mcp, [v]).length === 0
8354
+ );
8678
8355
  }
8679
- var MAX_NEW_AGENT_GRACE_MS = 365 * 24 * 60 * 60 * 1e3;
8680
- var DEFAULT_MAX = 2;
8681
- var DEFAULT_WINDOW_MS = 6e5;
8682
- var DEFAULT_PROVISIONING_MAX = RESTART_BREAKER_PROVISIONING_MAX;
8683
- var DEFAULT_PROVISIONING_WINDOW_MS = RESTART_BREAKER_PROVISIONING_WINDOW_MS;
8684
- var DEFAULT_OPERATOR_MAX = RESTART_BREAKER_OPERATOR_MAX;
8685
- var DEFAULT_SCHEDULED_MAX = RESTART_BREAKER_SCHEDULED_MAX;
8686
- var DEFAULT_SCHEDULED_WINDOW_MS = RESTART_BREAKER_SCHEDULED_WINDOW_MS;
8687
- var DEFAULT_OPERATOR_COMMAND_MAX = RESTART_BREAKER_OPERATOR_COMMAND_MAX;
8688
- var DEFAULT_OPERATOR_COMMAND_WINDOW_MS = RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS;
8689
- function readEnvNumber(name, fallback) {
8690
- const raw = process.env[name];
8691
- if (!raw) return fallback;
8692
- const parsed = Number(raw);
8693
- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
8356
+ function liveProxyExtraHeaderVars(mcp) {
8357
+ const out = /* @__PURE__ */ new Set();
8358
+ if (!mcp?.mcpServers) return out;
8359
+ for (const entry of Object.values(mcp.mcpServers)) {
8360
+ const spec = entry?.env?.["AGT_REMOTE_MCP_EXTRA_HEADERS"];
8361
+ if (typeof spec !== "string") continue;
8362
+ for (const pair of spec.split(",")) {
8363
+ const colon = pair.indexOf(":");
8364
+ if (colon <= 0) continue;
8365
+ const varName = pair.slice(colon + 1).trim();
8366
+ if (varName) out.add(varName);
8367
+ }
8368
+ }
8369
+ return out;
8694
8370
  }
8695
- function migrateMapKey(map, from, to) {
8696
- if (map.has(to) || !map.has(from)) return;
8697
- map.set(to, map.get(from));
8698
- map.delete(from);
8371
+ function liveProxyTokenVars(mcp) {
8372
+ const out = /* @__PURE__ */ new Set();
8373
+ if (!mcp?.mcpServers) return out;
8374
+ for (const entry of Object.values(mcp.mcpServers)) {
8375
+ const tokenVar = entry?.env?.["AGT_REMOTE_MCP_TOKEN_VAR"];
8376
+ if (typeof tokenVar === "string" && tokenVar.trim()) {
8377
+ out.add(tokenVar.trim());
8378
+ }
8379
+ }
8380
+ return out;
8699
8381
  }
8700
- var RestartBreaker = class {
8701
- max;
8702
- windowMs;
8703
- /**
8704
- * ENG-8215: NOT readonly - the constructor may clamp it upward to preserve the
8705
- * quarantine-fires-first ordering against a runtime override. Never mutated
8706
- * after construction.
8707
- */
8708
- provisioningMax;
8709
- provisioningWindowMs;
8710
- /**
8711
- * ENG-8388: the looser operator-config budget. Shares provisioningWindowMs.
8712
- * NOT readonly - the constructor floors it at `provisioningMax` so setting one
8713
- * env var without the other cannot invert the two tallies.
8714
- */
8715
- operatorMax;
8716
- /**
8717
- * ENG-9138: the scheduled class's LOOP bar. Readonly and deliberately NOT
8718
- * floored against any other tally: it gates a REPORT, not a pause, so it
8719
- * cannot invert anything the way operatorMax/provisioningMax can.
8720
- */
8721
- scheduledMax;
8722
- /** ENG-9138: sliding window for the scheduled tally. */
8723
- scheduledWindowMs;
8724
- operatorCommandMax;
8725
- operatorCommandWindowMs;
8726
- /** Longest window any tally needs — how long the events log must retain. */
8727
- retentionMs;
8728
- now;
8729
- /** ENG-7891 slice 5b: code_name → durable runtime key (agent_id when id-keyed). Identity on the legacy fleet. */
8730
- runtimeKey;
8731
- /** ENG-8690: agent creation time lookup; undefined result ⇒ no grace. */
8732
- agentCreatedAt;
8733
- /** ENG-8690: grace duration; 0 disables. */
8734
- newAgentGraceMs;
8735
- /** ENG-8690: the bound the grace does NOT lift. NOT readonly - floored below. */
8736
- newAgentGraceCeiling;
8737
- events = /* @__PURE__ */ new Map();
8738
- trips = /* @__PURE__ */ new Map();
8739
- /**
8740
- * ENG-8215: set when a supplied `provisioningMax` was raised to keep MCP
8741
- * quarantine ahead of the breaker. Read once by the manager at construction so
8742
- * an overridden-and-clamped value is visible in manager.log rather than
8743
- * silently discarded.
8744
- */
8745
- quarantineOrderingClamped;
8746
- constructor(opts = {}) {
8747
- this.max = opts.max ?? readEnvNumber("AGT_RESTART_BREAKER_MAX", DEFAULT_MAX);
8748
- this.windowMs = opts.windowMs ?? readEnvNumber("AGT_RESTART_BREAKER_WINDOW_MS", DEFAULT_WINDOW_MS);
8749
- this.provisioningMax = opts.provisioningMax ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", DEFAULT_PROVISIONING_MAX);
8750
- this.provisioningWindowMs = opts.provisioningWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS", DEFAULT_PROVISIONING_WINDOW_MS);
8751
- this.operatorMax = opts.operatorMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_MAX", DEFAULT_OPERATOR_MAX);
8752
- this.scheduledMax = opts.scheduledMax ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_MAX", DEFAULT_SCHEDULED_MAX);
8753
- this.scheduledWindowMs = opts.scheduledWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_WINDOW_MS", DEFAULT_SCHEDULED_WINDOW_MS);
8754
- this.operatorCommandMax = opts.operatorCommandMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_COMMAND_MAX", DEFAULT_OPERATOR_COMMAND_MAX);
8755
- this.operatorCommandWindowMs = opts.operatorCommandWindowMs ?? readEnvNumber(
8756
- "AGT_RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS",
8757
- DEFAULT_OPERATOR_COMMAND_WINDOW_MS
8758
- );
8759
- this.now = opts.now ?? Date.now;
8760
- this.runtimeKey = opts.runtimeKey ?? ((codeName) => codeName);
8761
- this.agentCreatedAt = opts.agentCreatedAt ?? (() => void 0);
8762
- const graceMs = opts.newAgentGraceMs ?? readEnvNumber("AGT_RESTART_BREAKER_NEW_AGENT_GRACE_MS", RESTART_BREAKER_NEW_AGENT_GRACE_MS);
8763
- const cappedGrace = Math.min(graceMs, MAX_NEW_AGENT_GRACE_MS);
8764
- this.newAgentGraceMs = Number.isFinite(graceMs) && graceMs > 0 ? cappedGrace : 0;
8765
- const ceiling = opts.newAgentGraceCeiling ?? readEnvNumber(
8766
- "AGT_RESTART_BREAKER_NEW_AGENT_GRACE_CEILING",
8767
- RESTART_BREAKER_NEW_AGENT_GRACE_CEILING
8382
+ function buildArgvMatchersForEntry(key, entry) {
8383
+ const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
8384
+ const patterns = [];
8385
+ const args = entry?.args ?? [];
8386
+ for (let i = args.length - 1; i >= 0; i--) {
8387
+ const arg = args[i];
8388
+ if (typeof arg !== "string" || !arg) continue;
8389
+ if (arg.startsWith("-")) continue;
8390
+ const stripped = arg.replace(
8391
+ /@[^/@]*$/,
8392
+ (m) => (
8393
+ // @latest, @1.2.3, etc. — drop. But @scope at the START of a
8394
+ // package spec must NOT be stripped, so we only strip a tail
8395
+ // `@...` if it doesn't itself start with a slash inside.
8396
+ // The regex above only matches a single trailing `@...`
8397
+ // segment after the last `/`, so this is safe for `@scope/pkg`.
8398
+ m.includes("/") ? m : ""
8399
+ )
8768
8400
  );
8769
- this.newAgentGraceCeiling = Number.isFinite(ceiling) && ceiling > 0 ? ceiling : RESTART_BREAKER_NEW_AGENT_GRACE_CEILING;
8770
- if (!Number.isFinite(this.max) || this.max < 1) {
8771
- throw new Error("restart-breaker max must be a finite number >= 1");
8772
- }
8773
- if (!Number.isFinite(this.windowMs) || this.windowMs < 1e3) {
8774
- throw new Error("restart-breaker windowMs must be a finite number >= 1000");
8401
+ const tail = "(?![A-Za-z0-9_-])";
8402
+ if (/^@?[a-z0-9]([a-z0-9._-]*\/)?[a-z0-9._-]+$/i.test(stripped)) {
8403
+ patterns.push(new RegExp(`${escapeRe(stripped)}${tail}`));
8404
+ const basename = stripped.split("/").pop();
8405
+ if (basename && basename !== stripped) {
8406
+ patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
8407
+ }
8408
+ break;
8775
8409
  }
8776
- if (!Number.isFinite(this.provisioningMax) || this.provisioningMax < 1) {
8777
- throw new Error("restart-breaker provisioningMax must be a finite number >= 1");
8410
+ if (stripped.includes("/")) {
8411
+ const basename = stripped.split("/").pop();
8412
+ if (basename) {
8413
+ patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
8414
+ break;
8415
+ }
8778
8416
  }
8779
- if (!Number.isFinite(this.operatorMax) || this.operatorMax < 1) {
8780
- throw new Error("restart-breaker operatorMax must be a finite number >= 1");
8417
+ }
8418
+ if (patterns.length === 0) {
8419
+ const safe = escapeRe(key);
8420
+ patterns.push(new RegExp(`(?<![A-Za-z0-9_])${safe}(?![A-Za-z0-9_])`));
8421
+ if (safe.includes("_")) {
8422
+ const dashed = safe.replace(/_/g, "-");
8423
+ patterns.push(new RegExp(`(?<![A-Za-z0-9_])${dashed}(?![A-Za-z0-9_])`));
8781
8424
  }
8782
- if (this.operatorMax < this.provisioningMax) {
8783
- this.operatorMax = this.provisioningMax;
8784
- }
8785
- if (this.provisioningMax < MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING) {
8786
- this.quarantineOrderingClamped = {
8787
- requested: this.provisioningMax,
8788
- applied: MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING,
8789
- quarantineThreshold: BIND_FAILURE_QUARANTINE_THRESHOLD
8790
- };
8791
- this.provisioningMax = MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING;
8792
- }
8793
- if (!Number.isFinite(this.provisioningWindowMs) || this.provisioningWindowMs < 1e3) {
8794
- throw new Error("restart-breaker provisioningWindowMs must be a finite number >= 1000");
8795
- }
8796
- if (!Number.isFinite(this.scheduledMax) || this.scheduledMax < 1) {
8797
- throw new Error("restart-breaker scheduledMax must be a finite number >= 1");
8798
- }
8799
- if (!Number.isFinite(this.scheduledWindowMs) || this.scheduledWindowMs < 1e3) {
8800
- throw new Error("restart-breaker scheduledWindowMs must be a finite number >= 1000");
8801
- }
8802
- if (!Number.isFinite(this.operatorCommandMax) || this.operatorCommandMax < 1) {
8803
- throw new Error("restart-breaker operatorCommandMax must be a finite number >= 1");
8425
+ }
8426
+ return patterns;
8427
+ }
8428
+ function buildClaudeAgentMatcher(codeName) {
8429
+ const safe = codeName.replace(/[^A-Za-z0-9_-]/g, "");
8430
+ return new RegExp(`\\bclaude\\b.*--name\\s+agt-${safe}(?=\\s|$)`);
8431
+ }
8432
+ function findMcpChildrenForAgent(args) {
8433
+ const { rows, codeName, serverKeys, mcpJson } = args;
8434
+ const maxDepth = args.maxDepth ?? 8;
8435
+ const inContainer = args.inContainer ?? false;
8436
+ if (serverKeys.length === 0 || rows.length === 0) return [];
8437
+ const claudeMatcher = inContainer ? /\bclaude\b/ : buildClaudeAgentMatcher(codeName);
8438
+ const argvMatchers = [];
8439
+ for (const key of serverKeys) {
8440
+ const entry = mcpJson?.mcpServers?.[key];
8441
+ argvMatchers.push(...buildArgvMatchersForEntry(key, entry));
8442
+ }
8443
+ const byPid = new Map(rows.map((r) => [r.pid, r]));
8444
+ const matched = [];
8445
+ for (const row of rows) {
8446
+ if (!argvMatchers.some((re) => re.test(row.args))) continue;
8447
+ if (/\bclaude\b/.test(row.args) && row.args.includes(`--name agt-${codeName}`)) continue;
8448
+ let cur = byPid.get(row.ppid);
8449
+ let belongs = false;
8450
+ for (let depth = 0; depth < maxDepth && cur; depth++) {
8451
+ if (claudeMatcher.test(cur.args)) {
8452
+ belongs = true;
8453
+ break;
8454
+ }
8455
+ if (cur.pid === 1) break;
8456
+ cur = byPid.get(cur.ppid);
8804
8457
  }
8805
- if (!Number.isFinite(this.operatorCommandWindowMs) || this.operatorCommandWindowMs < 1e3) {
8806
- throw new Error("restart-breaker operatorCommandWindowMs must be a finite number >= 1000");
8458
+ if (belongs) matched.push(row.pid);
8459
+ }
8460
+ return matched;
8461
+ }
8462
+ function reapStaleMcpChildren(args) {
8463
+ const { log: log2, codeName, serverKeys, mcpJson, graceMs = 5e3, isolated = false } = args;
8464
+ if (serverKeys.length === 0) return [];
8465
+ const runPs = args.runPs ?? (isolated ? (
8466
+ // ENG-9781: hard-bounded. See bounded-exec.ts — `execFileSync`'s own
8467
+ // `timeout` is not a bound, and this runs per isolated agent per cycle
8468
+ // on the manager-worker's critical path. Throws on a non-answer so the
8469
+ // caller's existing catch skips the agent rather than acting on an
8470
+ // empty process list.
8471
+ () => {
8472
+ const r = hardBoundedExecFile(
8473
+ "docker",
8474
+ ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
8475
+ { timeoutMs: 8e3, log: log2 }
8476
+ );
8477
+ if (r.kind === "ok") return r.stdout;
8478
+ throw new Error(
8479
+ r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
8480
+ );
8807
8481
  }
8808
- const minimumGraceCeiling = Math.max(this.provisioningMax, this.operatorMax);
8809
- if (this.newAgentGraceCeiling < minimumGraceCeiling) {
8810
- this.newAgentGraceCeiling = minimumGraceCeiling;
8482
+ ) : () => execFileSync("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
8483
+ const killProcess = args.killProcess ?? (isolated ? (pid, signal) => {
8484
+ hardBoundedExecFile(
8485
+ "docker",
8486
+ ["exec", `agt-${codeName}`, "kill", signal === "SIGKILL" ? "-KILL" : "-TERM", String(pid)],
8487
+ { timeoutMs: 8e3, log: log2 }
8488
+ );
8489
+ } : (pid, signal) => {
8490
+ try {
8491
+ process.kill(pid, signal);
8492
+ } catch {
8811
8493
  }
8812
- this.retentionMs = Math.max(
8813
- this.windowMs,
8814
- this.provisioningWindowMs,
8815
- this.scheduledWindowMs,
8816
- this.operatorCommandWindowMs
8494
+ });
8495
+ const isAlive = args.isAlive ?? (isolated ? (pid) => {
8496
+ const r = hardBoundedExecFile(
8497
+ "docker",
8498
+ ["exec", `agt-${codeName}`, "kill", "-0", String(pid)],
8499
+ { timeoutMs: 8e3, log: log2 }
8500
+ );
8501
+ return livenessFromProbe(
8502
+ r,
8503
+ (why) => log2(
8504
+ `[stale-mcp-reaper] liveness probe for pid ${pid} in 'agt-${codeName}' ${why} \u2014 treating as ALIVE (no reap) rather than guessing`
8505
+ )
8817
8506
  );
8507
+ } : (pid) => {
8508
+ try {
8509
+ process.kill(pid, 0);
8510
+ return true;
8511
+ } catch {
8512
+ return false;
8513
+ }
8514
+ });
8515
+ let psOutput;
8516
+ try {
8517
+ psOutput = runPs();
8518
+ } catch (err) {
8519
+ log2(`[stale-mcp-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping reap`);
8520
+ return [];
8818
8521
  }
8819
- /**
8820
- * ENG-8215: the clamp record, if the configured provisioning max had to be
8821
- * raised to keep MCP quarantine ahead of the breaker. `undefined` when the
8822
- * configured value already satisfied the ordering.
8823
- */
8824
- getQuarantineOrderingClamp() {
8825
- return this.quarantineOrderingClamped;
8522
+ const rows = parsePsRows(psOutput);
8523
+ const targets = findMcpChildrenForAgent({ rows, codeName, serverKeys, mcpJson, inContainer: isolated });
8524
+ if (targets.length === 0) return [];
8525
+ const byPid = new Map(rows.map((r) => [r.pid, r]));
8526
+ const describe = (pid) => {
8527
+ const argv = byPid.get(pid)?.args ?? "";
8528
+ const pkgMatch = argv.match(/(@[a-z0-9_-]+\/[a-z0-9_-]+|[a-z0-9_-]+-mcp-server|[a-z0-9_-]+-mcp\b)/i);
8529
+ return pkgMatch ? `${pkgMatch[1]} (pid ${pid})` : `pid ${pid}`;
8530
+ };
8531
+ log2(
8532
+ `[stale-mcp-reaper] '${codeName}': rotating ${targets.length} stale MCP child(ren) for [${serverKeys.join(", ")}]: ${targets.map(describe).join(", ")}`
8533
+ );
8534
+ for (const pid of targets) {
8535
+ killProcess(pid, "SIGTERM");
8826
8536
  }
8827
- /**
8828
- * ENG-7891 slice 5b: the DURABLE key for this agent's events/trips entries.
8829
- * Resolves the code_name to the runtime key (agent_id on id-keyed hosts) and,
8830
- * when they differ, lazily migrates a legacy code_name-keyed entry onto it — so
8831
- * a trip persisted under the old key by a pre-arming manager (or from before a
8832
- * codename rename) is found under the new key instead of being lost. Identity on
8833
- * the legacy fleet ⇒ from === to ⇒ no migration and no behaviour change. Both
8834
- * maps migrate together so events and a trip for the same agent never split
8835
- * across keys mid-generation (a migration can flip the resolver mid-process).
8836
- *
8837
- * BOUNDARY — a pure-LEGACY rename (a host that was never armed) still strands a
8838
- * held trip, exactly as it did before slice 5b: with no symlink there is no
8839
- * agent_id to bridge the old codename to the new, so the runtime key stays the
8840
- * (new) codename and the trip persisted under the old codename is not found.
8841
- * That is unchanged, pre-existing behaviour and it is benign — the fresh
8842
- * generation re-evaluates the agent under its new name and a still-pathological
8843
- * agent simply re-trips within the window. The rename-survival guarantee this
8844
- * slice adds is an id-keyed-only, post-arming property (the symlink is the
8845
- * bridge); it is deliberately NOT retrofitted onto the legacy path.
8846
- */
8847
- keyFor(codeName) {
8848
- const key = this.runtimeKey(codeName);
8849
- if (key !== codeName) {
8850
- migrateMapKey(this.events, codeName, key);
8851
- migrateMapKey(this.trips, codeName, key);
8537
+ const killedKeys = /* @__PURE__ */ new Set();
8538
+ for (const pid of targets) {
8539
+ const argv = byPid.get(pid)?.args ?? "";
8540
+ for (const key of serverKeys) {
8541
+ const entry = mcpJson?.mcpServers?.[key];
8542
+ const matchers = buildArgvMatchersForEntry(key, entry);
8543
+ if (matchers.some((re) => re.test(argv))) killedKeys.add(key);
8852
8544
  }
8853
- return key;
8854
8545
  }
8855
- /** True if this agent's breaker is currently tripped (manager must skip spawn). */
8856
- isTripped(codeName) {
8857
- return this.trips.has(this.keyFor(codeName));
8546
+ if (killedKeys.size > 0) recordStaleRotation(codeName, killedKeys);
8547
+ setTimeout(() => {
8548
+ try {
8549
+ let freshPsOutput;
8550
+ try {
8551
+ freshPsOutput = runPs();
8552
+ } catch (err) {
8553
+ log2(`[stale-mcp-reaper] '${codeName}': fresh ps for SIGKILL re-verify failed: ${err.message} \u2014 skipping SIGKILL pass`);
8554
+ return;
8555
+ }
8556
+ const stillOwned = new Set(
8557
+ findMcpChildrenForAgent({
8558
+ rows: parsePsRows(freshPsOutput),
8559
+ codeName,
8560
+ serverKeys,
8561
+ mcpJson,
8562
+ inContainer: isolated
8563
+ })
8564
+ );
8565
+ const stragglers = targets.filter((pid) => isAlive(pid) && stillOwned.has(pid));
8566
+ if (stragglers.length === 0) return;
8567
+ log2(
8568
+ `[stale-mcp-reaper] '${codeName}': ${stragglers.length} child(ren) survived SIGTERM; sending SIGKILL: ${stragglers.map(describe).join(", ")}`
8569
+ );
8570
+ for (const pid of stragglers) {
8571
+ killProcess(pid, "SIGKILL");
8572
+ }
8573
+ } catch (err) {
8574
+ log2(`[stale-mcp-reaper] '${codeName}': error in SIGKILL pass: ${err.message}`);
8575
+ }
8576
+ }, graceMs).unref();
8577
+ return targets;
8578
+ }
8579
+
8580
+ // src/lib/mcp-presence-reaper.ts
8581
+ import { execFileSync as execFileSync2 } from "child_process";
8582
+ var DEFAULT_COLD_START_GRACE_MS = 9e4;
8583
+ var MAX_PRESENCE_RESTART_ATTEMPTS = 3;
8584
+ var BACKOFF_AFTER_RESTARTS = MAX_PRESENCE_RESTART_ATTEMPTS;
8585
+ var BACKOFF_BASE_MS = 6e4;
8586
+ var BACKOFF_MAX_MS = 30 * 6e4;
8587
+ var BACKOFF_WINDOW_MS = 60 * 6e4;
8588
+ var BACKOFF_GIVE_UP_RESTARTS = 2 * MAX_PRESENCE_RESTART_ATTEMPTS;
8589
+ var BACKOFF_STABLE_RESET_MS = 5 * 6e4;
8590
+ var DEAD_CHANNEL_MS = 15 * 6e4;
8591
+ var presenceReaperState = /* @__PURE__ */ new Map();
8592
+ var MAX_ROTATION_PROMPT_PROBES = 3;
8593
+ var ROTATION_PROMPT_PROBE_INTERVAL_MS = 3e4;
8594
+ var rotationStallState = /* @__PURE__ */ new Map();
8595
+ var blockedPromptEpisodes = /* @__PURE__ */ new Map();
8596
+ function stateKey(codeName, serverKey) {
8597
+ return `${codeName}\0${serverKey}`;
8598
+ }
8599
+ function clearPresenceReaperState(codeName) {
8600
+ const prefix = `${codeName}\0`;
8601
+ for (const key of presenceReaperState.keys()) {
8602
+ if (key.startsWith(prefix)) presenceReaperState.delete(key);
8858
8603
  }
8859
- getTrip(codeName) {
8860
- return this.trips.get(this.keyFor(codeName));
8604
+ }
8605
+ function clearPresenceReaperStateForKeys(codeName, keys) {
8606
+ for (const key of keys) {
8607
+ presenceReaperState.delete(stateKey(codeName, key));
8608
+ rotationStallState.delete(stateKey(codeName, key));
8861
8609
  }
8862
- /**
8863
- * Record a restart event. If recording this event puts the count
8864
- * inside the window above `max`, the breaker trips and the call site
8865
- * should NOT respawn.
8866
- *
8867
- * Idempotent on already-tripped breakers: returns the existing trip
8868
- * without double-counting events. Callers may still record the reason
8869
- * via the decision-log for forensics.
8870
- */
8871
- record(codeName, reason, integrationKey) {
8872
- const key = this.keyFor(codeName);
8873
- const existing = this.trips.get(key);
8874
- if (existing) {
8875
- const bucket = maxProvisioningBucket(existing.eventsAtTrip);
8876
- return {
8877
- tripped: false,
8878
- trip: existing,
8879
- windowCount: existing.eventsAtTrip.length,
8880
- classCounts: countByClass(existing.eventsAtTrip),
8881
- maxProvisioningBucket: bucket.count,
8882
- maxProvisioningBucketKey: bucket.key
8883
- };
8610
+ }
8611
+ function givenUpMcpServerKeys(codeName) {
8612
+ const prefix = `${codeName}\0`;
8613
+ const out = /* @__PURE__ */ new Set();
8614
+ for (const [key, st] of presenceReaperState) {
8615
+ if (key.startsWith(prefix) && (st.gaveUp || st.attempts >= MAX_PRESENCE_RESTART_ATTEMPTS)) {
8616
+ out.add(key.slice(prefix.length));
8884
8617
  }
8885
- const at = this.now();
8886
- const retentionCutoff = at - this.retentionMs;
8887
- const prior = (this.events.get(key) ?? []).filter((e) => e.at >= retentionCutoff);
8888
- const bucketed = isIntegrationBucketedReason(reason);
8889
- prior.push(bucketed && integrationKey ? { reason, at, integrationKey } : { reason, at });
8890
- this.events.set(key, prior);
8891
- const crashCount = prior.filter(
8892
- (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8893
- ).length;
8894
- const provEvents = prior.filter(
8895
- (e) => e.at >= at - this.provisioningWindowMs && isProvisioningReloadReason(e.reason)
8618
+ }
8619
+ return out;
8620
+ }
8621
+ function findMissingMcpServers(args) {
8622
+ const { rows, codeName, mcpJson, inContainer } = args;
8623
+ const servers = mcpJson?.mcpServers ?? {};
8624
+ const declared = Object.keys(servers);
8625
+ if (declared.length === 0) return [];
8626
+ const missing = [];
8627
+ for (const key of declared) {
8628
+ const entry = servers[key];
8629
+ const isStdio = typeof entry?.command === "string";
8630
+ if (!isStdio) continue;
8631
+ const live = findMcpChildrenForAgent({
8632
+ rows,
8633
+ codeName,
8634
+ serverKeys: [key],
8635
+ mcpJson,
8636
+ inContainer
8637
+ });
8638
+ if (live.length === 0) missing.push(key);
8639
+ }
8640
+ return missing;
8641
+ }
8642
+ function describeBlockedPromptHold(ctx) {
8643
+ const keys = `declared MCP(s) [${ctx.activeKeys.join(", ")}]`;
8644
+ if (ctx.restartDeferred) {
8645
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. The recovery restart was ATTEMPTED and the host deferred it (maintenance window or agent not idle), so nothing was torn down and this episode's one restart is still unspent - it retries when the gate opens. Needs a human if it persists`;
8646
+ }
8647
+ if (ctx.restartSpentThisEpisode) {
8648
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. This episode's ONE restart was already spent and the agent is STILL blocked, so a further restart is not the answer. Needs a human at the pane`;
8649
+ }
8650
+ if (ctx.restartBudgetRemaining === 0) {
8651
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. No restart was attempted: the crash breaker has NO budget left in its window, so a restart now would be the event that trips it and pauses this agent - attributed to mcp-presence-reaper, not to whatever filled the window. It may clear once the breaker window rolls; needs a human if it does not`;
8652
+ }
8653
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children and no restart was made. Needs a human at the pane`;
8654
+ }
8655
+ function reapMissingMcpSessions(args) {
8656
+ const {
8657
+ log: log2,
8658
+ codeName,
8659
+ mcpJson,
8660
+ sessionStartedAt,
8661
+ restartPending = false,
8662
+ stopSession,
8663
+ graceMs = DEFAULT_COLD_START_GRACE_MS,
8664
+ rotationGraceMs = DEFAULT_ROTATION_GRACE_MS,
8665
+ onGiveUp,
8666
+ onRestart,
8667
+ // ENG-8225: unwired callbacks make the prompt-probe path a strict no-op —
8668
+ // `onRotationRespawnStalled` absent means "not armed", which falls through to
8669
+ // the pre-ENG-8225 restart. Legacy callers and tests are unchanged.
8670
+ onRotationRespawnStalled,
8671
+ onRotationRespawnGaveUp,
8672
+ onRotationRespawnOutcome,
8673
+ maxRotationPromptProbes = MAX_ROTATION_PROMPT_PROBES,
8674
+ rotationPromptProbeIntervalMs = ROTATION_PROMPT_PROBE_INTERVAL_MS,
8675
+ // ENG-5932: defaults make quarantine a strict no-op (classify everything
8676
+ // essential, mode off) so legacy callers/tests are unaffected.
8677
+ classifyKey = () => "essential",
8678
+ quarantineDwellMs = 0,
8679
+ deadChannelMs = DEAD_CHANNEL_MS,
8680
+ quarantineMode = "off",
8681
+ onQuarantine,
8682
+ isolated = false,
8683
+ // ENG-8013: default false → a "no live child" observation is only excluded
8684
+ // from the give-up/quarantine budget when the caller positively identifies
8685
+ // the current generation as externally-churned. Legacy callers/tests unchanged.
8686
+ currentGenerationExternallyStarted = false,
8687
+ // ENG-9932: unwired => strict no-op, restart proceeds exactly as before.
8688
+ readPane,
8689
+ onPaneBlocked
8690
+ } = args;
8691
+ const now = args.now ?? Date.now;
8692
+ const runPs = args.runPs ?? (isolated ? (
8693
+ // ENG-9781: HARD-bounded, and the distinction is the whole ticket. The
8694
+ // previous `execFileSync(..., { timeout: 8_000 })` bounded nothing —
8695
+ // Node fires SIGTERM at the deadline and then keeps waiting, so a
8696
+ // docker CLI talking to a wedged daemon blocked the manager-worker's
8697
+ // event loop and stopped scheduling for EVERY agent on the host
8698
+ // (33m46s, twice, on 2026-09-01). This call runs once per isolated
8699
+ // agent per poll cycle, so it is the single worst place on the manager
8700
+ // for an unbounded subprocess.
8701
+ //
8702
+ // Throwing on a non-answer is deliberate rather than lazy: the caller
8703
+ // already catches, logs and SKIPS the agent for the cycle, which is
8704
+ // exactly right. A timeout must not become an empty process list —
8705
+ // that reads as "every declared MCP server is missing" and restarts the
8706
+ // session, turning a docker hiccup into a fleet-wide restart storm.
8707
+ () => {
8708
+ const r = hardBoundedExecFile(
8709
+ "docker",
8710
+ ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
8711
+ { timeoutMs: 8e3, log: log2 }
8712
+ );
8713
+ if (r.kind === "ok") return r.stdout;
8714
+ throw new Error(
8715
+ r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
8716
+ );
8717
+ }
8718
+ ) : () => execFileSync2("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
8719
+ if (!mcpJson?.mcpServers) {
8720
+ return { missing: [], restarted: false, reason: "no-mcp-json" };
8721
+ }
8722
+ const declaredCount = Object.keys(mcpJson.mcpServers).length;
8723
+ if (declaredCount === 0) {
8724
+ return { missing: [], restarted: false, reason: "no-declared-servers" };
8725
+ }
8726
+ if (restartPending) {
8727
+ return { missing: [], restarted: false, reason: "restart-pending" };
8728
+ }
8729
+ if (sessionStartedAt === null) {
8730
+ return { missing: [], restarted: false, reason: "session-start-unknown" };
8731
+ }
8732
+ if (now() - sessionStartedAt < graceMs) {
8733
+ return { missing: [], restarted: false, reason: "cold-start" };
8734
+ }
8735
+ let psOutput;
8736
+ try {
8737
+ psOutput = runPs();
8738
+ } catch (err) {
8739
+ log2(`[mcp-presence-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping`);
8740
+ return { missing: [], restarted: false, reason: "healthy" };
8741
+ }
8742
+ const rows = parsePsRows(psOutput);
8743
+ const missingRaw = findMissingMcpServers({ rows, codeName, mcpJson, inContainer: isolated });
8744
+ const rotationGraced = [];
8745
+ const rotationProbeArmed = [];
8746
+ const rotationProbeWaiting = [];
8747
+ const rotationProbeExhausted = [];
8748
+ const rotationRespawnOutcomes = {};
8749
+ const recordRotationOutcome = (key, outcome) => {
8750
+ rotationRespawnOutcomes[key] = outcome;
8751
+ if (!onRotationRespawnOutcome) return;
8752
+ try {
8753
+ onRotationRespawnOutcome(codeName, key, outcome);
8754
+ } catch (err) {
8755
+ log2(
8756
+ `[mcp-presence-reaper] onRotationRespawnOutcome callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8757
+ );
8758
+ }
8759
+ };
8760
+ const missing = [];
8761
+ for (const key of missingRaw) {
8762
+ const rotation = classifyRotation(codeName, key, rotationGraceMs, now);
8763
+ if (rotation === "within-grace") {
8764
+ rotationGraced.push(key);
8765
+ continue;
8766
+ }
8767
+ if (rotation === "none") {
8768
+ rotationStallState.delete(stateKey(codeName, key));
8769
+ missing.push(key);
8770
+ continue;
8771
+ }
8772
+ const sk = stateKey(codeName, key);
8773
+ const rotationRecordedAt = getRotationRecordedAt(codeName, key) ?? now();
8774
+ const priorStall = rotationStallState.get(sk);
8775
+ const stall = priorStall && priorStall.rotationRecordedAt === rotationRecordedAt ? priorStall : {
8776
+ rotationRecordedAt,
8777
+ probeAttempts: 0,
8778
+ lastProbeAt: null,
8779
+ escalated: false
8780
+ };
8781
+ if (priorStall && priorStall !== stall) {
8782
+ log2(
8783
+ `[mcp-presence-reaper] '${codeName}:${key}' discarding stall state from a previous rotation (rotation=${priorStall.rotationRecordedAt}, probes=${priorStall.probeAttempts}, escalated=${priorStall.escalated}) \u2014 this is a NEW rotation (rotation=${rotationRecordedAt}) and gets a full prompt-probe budget (ENG-8225)`
8784
+ );
8785
+ }
8786
+ rotationStallState.set(sk, stall);
8787
+ if (stall.escalated) {
8788
+ missing.push(key);
8789
+ continue;
8790
+ }
8791
+ const spacingElapsed = stall.lastProbeAt === null || now() - stall.lastProbeAt >= rotationPromptProbeIntervalMs;
8792
+ if (!spacingElapsed) {
8793
+ rotationProbeWaiting.push(key);
8794
+ continue;
8795
+ }
8796
+ if (stall.probeAttempts >= maxRotationPromptProbes) {
8797
+ stall.escalated = true;
8798
+ rotationProbeExhausted.push(key);
8799
+ recordRotationOutcome(key, "never-respawned");
8800
+ log2(
8801
+ `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=never-respawned \u2014 child SIGTERM'd by stale-mcp-reaper never returned after ${stall.probeAttempts} prompt bind-probe(s); escalating (integration marked unhealthy + reaper_action alert) and releasing to the normal restart path (ENG-8225)`
8802
+ );
8803
+ if (onRotationRespawnGaveUp) {
8804
+ try {
8805
+ onRotationRespawnGaveUp(codeName, key, { attempts: stall.probeAttempts });
8806
+ } catch (err) {
8807
+ log2(
8808
+ `[mcp-presence-reaper] onRotationRespawnGaveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8809
+ );
8810
+ }
8811
+ }
8812
+ missing.push(key);
8813
+ continue;
8814
+ }
8815
+ const attempt = stall.probeAttempts + 1;
8816
+ let armed = false;
8817
+ if (onRotationRespawnStalled) {
8818
+ try {
8819
+ armed = onRotationRespawnStalled(codeName, key, {
8820
+ attempt,
8821
+ maxAttempts: maxRotationPromptProbes
8822
+ });
8823
+ } catch (err) {
8824
+ log2(
8825
+ `[mcp-presence-reaper] onRotationRespawnStalled callback threw for '${codeName}:${key}' (suppressed; falling back to the pre-ENG-8225 restart path): ${err.message}`
8826
+ );
8827
+ armed = false;
8828
+ }
8829
+ }
8830
+ if (!armed) {
8831
+ missing.push(key);
8832
+ continue;
8833
+ }
8834
+ stall.probeAttempts = attempt;
8835
+ stall.lastProbeAt = now();
8836
+ rotationProbeArmed.push(key);
8837
+ log2(
8838
+ `[mcp-presence-reaper] '${codeName}:${key}' rotation grace expired (${rotationGraceMs}ms) with no live child and no tool call to respawn it \u2014 armed prompt bind-probe ${attempt}/${maxRotationPromptProbes} (ENG-7429) instead of restarting the session (ENG-8225)`
8896
8839
  );
8897
- const credRotCount = prior.filter(
8898
- (e) => e.at >= at - this.provisioningWindowMs && isCredentialRotationReason(e.reason)
8899
- ).length;
8900
- const selfHealCount = prior.filter(
8901
- (e) => e.at >= at - this.provisioningWindowMs && isSelfHealingReason(e.reason)
8902
- ).length;
8903
- const operatorEvents = prior.filter(
8904
- (e) => e.at >= at - this.provisioningWindowMs && isOperatorConfigReason(e.reason)
8840
+ }
8841
+ if (rotationGraced.length > 0) {
8842
+ log2(
8843
+ `[mcp-presence-reaper] '${codeName}': skipping [${rotationGraced.join(", ")}] within rotation grace window (${rotationGraceMs}ms) \u2014 child(ren) just SIGTERM'd by stale-mcp-reaper, awaiting respawn on next tool call (ENG-5344)`
8905
8844
  );
8906
- const scheduledEvents = prior.filter(
8907
- (e) => e.at >= at - this.scheduledWindowMs && isScheduledRestartReason(e.reason)
8845
+ }
8846
+ if (rotationProbeWaiting.length > 0) {
8847
+ log2(
8848
+ `[mcp-presence-reaper] '${codeName}': skipping [${rotationProbeWaiting.join(", ")}] \u2014 prompt bind-probe already armed for the stalled rotation, waiting out the ${rotationPromptProbeIntervalMs}ms re-arm interval (ENG-8225)`
8908
8849
  );
8909
- const operatorCommandEvents = prior.filter(
8910
- (e) => e.at >= at - this.operatorCommandWindowMs && isOperatorCommandReason(e.reason)
8850
+ }
8851
+ const declared = mcpJson.mcpServers ?? {};
8852
+ const liveKeys = /* @__PURE__ */ new Set();
8853
+ for (const key of Object.keys(declared)) {
8854
+ const entry = declared[key];
8855
+ if (typeof entry?.command !== "string") continue;
8856
+ if (!missingRaw.includes(key)) liveKeys.add(key);
8857
+ }
8858
+ for (const key of liveKeys) {
8859
+ const sk = stateKey(codeName, key);
8860
+ if (classifyRotation(codeName, key, rotationGraceMs, now) === "none") {
8861
+ rotationStallState.delete(sk);
8862
+ continue;
8863
+ }
8864
+ const stall = rotationStallState.get(sk);
8865
+ const viaProbe = (stall?.probeAttempts ?? 0) > 0;
8866
+ recordRotationOutcome(
8867
+ key,
8868
+ viaProbe ? "respawned-after-prompt-probe" : "respawned-on-tool-call"
8911
8869
  );
8912
- const windowCount = prior.filter((e) => e.at >= at - this.windowMs).length;
8913
- const classCounts = {
8914
- crash: crashCount,
8915
- provisioning: provEvents.length,
8916
- "credential-rotation": credRotCount,
8917
- "self-healing": selfHealCount,
8918
- "operator-config": operatorEvents.length,
8919
- scheduled: scheduledEvents.length,
8920
- "operator-command": operatorCommandEvents.length
8870
+ log2(
8871
+ viaProbe ? `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-after-prompt-probe \u2014 child is live again after ${stall?.probeAttempts} prompt bind-probe(s)${stall?.escalated ? " (recovered AFTER escalation \u2014 the earlier never-respawned verdict stands for that window)" : ""}, no session restart needed (ENG-8225)` : `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-on-tool-call \u2014 child returned on the agent's own next tool call, ENG-5344's assumption held (ENG-8225)`
8872
+ );
8873
+ clearStaleRotation(codeName, [key]);
8874
+ rotationStallState.delete(sk);
8875
+ }
8876
+ for (const key of liveKeys) {
8877
+ const state = presenceReaperState.get(stateKey(codeName, key));
8878
+ if (state) {
8879
+ state.attempts = 0;
8880
+ state.lastSeenLiveAt = now();
8881
+ state.lastAttemptedSessionStartedAt = null;
8882
+ state.gaveUpLogged = false;
8883
+ state.firstMissingAt = null;
8884
+ state.quarantineLogged = false;
8885
+ state.continuouslyMissingSince = null;
8886
+ state.deadChannelLogged = false;
8887
+ state.quarantineWithheldLogged = false;
8888
+ state.gaveUp = false;
8889
+ if (state.stableLiveSince === null) state.stableLiveSince = now();
8890
+ if (now() - state.stableLiveSince >= BACKOFF_STABLE_RESET_MS) {
8891
+ state.restartHistory = [];
8892
+ }
8893
+ }
8894
+ }
8895
+ if (missing.length === 0) {
8896
+ if (missingRaw.length > 0) {
8897
+ return {
8898
+ missing,
8899
+ restarted: false,
8900
+ reason: "healthy",
8901
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8902
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8903
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8904
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8905
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
8906
+ };
8907
+ }
8908
+ blockedPromptEpisodes.delete(codeName);
8909
+ return {
8910
+ missing,
8911
+ restarted: false,
8912
+ reason: "healthy",
8913
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8914
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8915
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8916
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8917
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
8921
8918
  };
8922
- const scheduledLoop = scheduledEvents.length > this.scheduledMax ? {
8923
- count: scheduledEvents.length,
8924
- windowMs: this.scheduledWindowMs,
8925
- reason: scheduledEvents[scheduledEvents.length - 1].reason
8926
- } : void 0;
8927
- const operatorCommandLoop = operatorCommandEvents.length > this.operatorCommandMax ? {
8928
- count: operatorCommandEvents.length,
8929
- windowMs: this.operatorCommandWindowMs,
8930
- reason: operatorCommandEvents[operatorCommandEvents.length - 1].reason
8931
- } : void 0;
8932
- const provBucket = maxProvisioningBucket(provEvents);
8933
- const operatorBucket = maxBucketWhere(operatorEvents, isOperatorConfigReason);
8934
- const createdAt = this.newAgentGraceMs > 0 ? this.agentCreatedAt(codeName) : void 0;
8935
- const age = typeof createdAt === "number" && Number.isFinite(createdAt) ? at - createdAt : void 0;
8936
- const newAgentGraceActive = age !== void 0 && age >= 0 && age < this.newAgentGraceMs;
8937
- const newAgentGraceExpiresAt = newAgentGraceActive && typeof createdAt === "number" ? createdAt + this.newAgentGraceMs : void 0;
8938
- const graceCeilingEvents = newAgentGraceActive ? prior.filter(
8939
- (e) => e.at >= at - this.provisioningWindowMs && (isProvisioningReloadReason(e.reason) || isOperatorConfigReason(e.reason))
8940
- ) : [];
8941
- const graceCeilingExceeded = newAgentGraceActive && graceCeilingEvents.length > this.newAgentGraceCeiling;
8942
- let trippedClass;
8943
- let tripEvents;
8944
- let tripWindowMs = this.windowMs;
8945
- let tripBucketKey = "";
8946
- if (crashCount > this.max) {
8947
- trippedClass = "crash";
8948
- tripEvents = prior.filter(
8949
- (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8950
- );
8951
- tripWindowMs = this.windowMs;
8952
- } else if (newAgentGraceActive && graceCeilingExceeded) {
8953
- trippedClass = "provisioning";
8954
- tripEvents = graceCeilingEvents;
8955
- tripWindowMs = this.provisioningWindowMs;
8956
- tripBucketKey = "";
8957
- } else if (newAgentGraceActive) {
8958
- } else if (operatorBucket.count > this.operatorMax) {
8959
- trippedClass = "operator-config";
8960
- tripEvents = operatorEvents.filter((e) => (e.integrationKey ?? "") === operatorBucket.key);
8961
- tripWindowMs = this.provisioningWindowMs;
8962
- tripBucketKey = operatorBucket.key;
8963
- } else if (provBucket.count > this.provisioningMax) {
8964
- trippedClass = "provisioning";
8965
- tripEvents = provEvents.filter((e) => (e.integrationKey ?? "") === provBucket.key);
8966
- tripWindowMs = this.provisioningWindowMs;
8967
- tripBucketKey = provBucket.key;
8919
+ }
8920
+ const nowMs = now();
8921
+ const givenUp = [];
8922
+ const active = [];
8923
+ const backingOff = [];
8924
+ const quarantined = [];
8925
+ const wouldQuarantine = [];
8926
+ const quarantineWithheld = [];
8927
+ const anyStdioServerLive = liveKeys.size > 0;
8928
+ const genuinelyDead = [];
8929
+ for (const key of missing) {
8930
+ const sk = stateKey(codeName, key);
8931
+ const state = presenceReaperState.get(sk) ?? {
8932
+ attempts: 0,
8933
+ lastSeenLiveAt: null,
8934
+ lastAttemptedSessionStartedAt: null,
8935
+ historyOwedForSessionStartedAt: null,
8936
+ gaveUpLogged: false,
8937
+ firstMissingAt: null,
8938
+ quarantineLogged: false,
8939
+ restartHistory: [],
8940
+ stableLiveSince: null,
8941
+ continuouslyMissingSince: null,
8942
+ deadChannelLogged: false,
8943
+ quarantineWithheldLogged: false,
8944
+ gaveUp: false
8945
+ };
8946
+ if (state.continuouslyMissingSince === null) {
8947
+ state.continuouslyMissingSince = nowMs;
8948
+ }
8949
+ const isGenuinelyDead = nowMs - state.continuouslyMissingSince >= deadChannelMs;
8950
+ const suppressAsChurn = currentGenerationExternallyStarted && !isGenuinelyDead;
8951
+ if (isGenuinelyDead) {
8952
+ genuinelyDead.push(key);
8953
+ if (currentGenerationExternallyStarted && !state.deadChannelLogged) {
8954
+ state.deadChannelLogged = true;
8955
+ log2(
8956
+ `[mcp-presence-reaper] '${codeName}:${key}' continuously missing ${Math.round((nowMs - state.continuouslyMissingSince) / 6e4)}m (> ${Math.round(deadChannelMs / 6e4)}m) \u2014 genuinely dead, overriding storm-awareness so it can reach give-up/quarantine instead of churning (ENG-8062)`
8957
+ );
8958
+ }
8959
+ }
8960
+ if (state.firstMissingAt === null || suppressAsChurn) {
8961
+ state.firstMissingAt = nowMs;
8962
+ }
8963
+ state.stableLiveSince = null;
8964
+ if (state.lastAttemptedSessionStartedAt !== sessionStartedAt) {
8965
+ state.lastAttemptedSessionStartedAt = sessionStartedAt;
8966
+ if (!suppressAsChurn) {
8967
+ state.attempts += 1;
8968
+ state.historyOwedForSessionStartedAt = sessionStartedAt;
8969
+ }
8970
+ }
8971
+ state.restartHistory = state.restartHistory.filter((t) => nowMs - t < BACKOFF_WINDOW_MS);
8972
+ const restartsInWindow = state.restartHistory.length;
8973
+ presenceReaperState.set(sk, state);
8974
+ const overAttemptsCap = state.attempts > MAX_PRESENCE_RESTART_ATTEMPTS;
8975
+ const overRateCap = restartsInWindow >= BACKOFF_GIVE_UP_RESTARTS;
8976
+ if (overAttemptsCap || overRateCap) {
8977
+ givenUp.push(key);
8978
+ state.gaveUp = true;
8979
+ if (!state.gaveUpLogged) {
8980
+ const cause = overAttemptsCap ? `${MAX_PRESENCE_RESTART_ATTEMPTS} consecutive failed restarts \u2014 declared but never recovers (likely orphaned config; see ENG-5279)` : `${restartsInWindow} restarts within ${Math.round(BACKOFF_WINDOW_MS / 6e4)}m \u2014 persistently flapping (crash-loop; ENG-6480)`;
8981
+ log2(
8982
+ `[mcp-presence-reaper] giving up on '${codeName}:${key}' after ${cause}`
8983
+ );
8984
+ state.gaveUpLogged = true;
8985
+ if (onGiveUp) {
8986
+ try {
8987
+ onGiveUp(codeName, key);
8988
+ } catch (err) {
8989
+ log2(
8990
+ `[mcp-presence-reaper] onGiveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8991
+ );
8992
+ }
8993
+ }
8994
+ }
8995
+ if (quarantineMode !== "off" && !state.quarantineLogged && classifyKey(key) === "optional") {
8996
+ const dwellElapsed = quarantineDwellMs <= 0 || state.firstMissingAt !== null && nowMs - state.firstMissingAt >= quarantineDwellMs;
8997
+ if (dwellElapsed && !anyStdioServerLive) {
8998
+ quarantineWithheld.push(key);
8999
+ if (!state.quarantineWithheldLogged) {
9000
+ state.quarantineWithheldLogged = true;
9001
+ log2(
9002
+ `[mcp-presence-reaper] '${codeName}:${key}' given up but NOT quarantined - no declared stdio MCP server is live, so the restart budget was exhausted by a session that cannot spawn anything, not by this channel dying. Check the pane (tmux attach -t agt-${codeName}); the channel stays provisioned and will quarantine on a later tick if it is still missing once the session is up (ENG-10500)`
9003
+ );
9004
+ }
9005
+ } else if (dwellElapsed) {
9006
+ state.quarantineLogged = true;
9007
+ if (quarantineMode === "enforce") {
9008
+ quarantined.push(key);
9009
+ log2(
9010
+ `[mcp-presence-reaper] QUARANTINE '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; dropping from provisioned set so the dead channel stops restarting the whole session (ENG-5932)`
9011
+ );
9012
+ if (onQuarantine) {
9013
+ try {
9014
+ onQuarantine(codeName, key);
9015
+ } catch (err) {
9016
+ log2(
9017
+ `[mcp-presence-reaper] onQuarantine callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
9018
+ );
9019
+ }
9020
+ }
9021
+ } else {
9022
+ wouldQuarantine.push(key);
9023
+ log2(
9024
+ `[mcp-presence-reaper] SHADOW would-quarantine '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; no action taken (ENG-5932 shadow mode)`
9025
+ );
9026
+ }
9027
+ }
9028
+ }
9029
+ } else {
9030
+ const backoffMs = restartsInWindow < BACKOFF_AFTER_RESTARTS ? 0 : Math.min(
9031
+ BACKOFF_BASE_MS * 2 ** (restartsInWindow - BACKOFF_AFTER_RESTARTS),
9032
+ BACKOFF_MAX_MS
9033
+ );
9034
+ const lastRestartAt = state.restartHistory[state.restartHistory.length - 1] ?? null;
9035
+ if (lastRestartAt !== null && nowMs - lastRestartAt < backoffMs) {
9036
+ backingOff.push(key);
9037
+ } else {
9038
+ active.push(key);
9039
+ }
8968
9040
  }
8969
- if (trippedClass && tripEvents) {
8970
- const baseMessage = graceCeilingExceeded ? formatStatusMessage(tripEvents, tripWindowMs) : formatStatusMessage(tripEvents, tripWindowMs, trippedClass, tripBucketKey);
8971
- const graceHours = Math.round(this.newAgentGraceMs / 36e5 * 10) / 10;
8972
- const statusMessage = graceCeilingExceeded ? `${baseMessage} \u2014 paused DESPITE the new-agent grace: ${graceCeilingEvents.length} provisioning-shaped restarts (provisioning + operator-config) exceeded the grace ceiling of ${this.newAgentGraceCeiling} (ENG-8690). The agent is inside its ${graceHours}h grace and is doing little except restarting.` : baseMessage;
8973
- const trip = {
8974
- trippedAt: at,
8975
- eventsAtTrip: [...tripEvents],
8976
- statusMessage,
8977
- // ENG-7577: persist the tripping class so the recovery path can gate on
8978
- // it after a manager restart, without re-deriving it from prose.
8979
- trippedClass
8980
- };
8981
- this.trips.set(key, trip);
8982
- this.events.delete(key);
8983
- return {
8984
- tripped: true,
8985
- trip,
8986
- windowCount,
8987
- classCounts,
8988
- trippedClass,
8989
- maxProvisioningBucket: provBucket.count,
8990
- maxProvisioningBucketKey: provBucket.key,
8991
- ...graceCeilingExceeded ? { newAgentGraceCeilingExceeded: true } : {},
8992
- // ENG-9138: a co-occurring rollover loop still reports on a real trip.
8993
- ...scheduledLoop ? { scheduledLoop } : {},
8994
- ...operatorCommandLoop ? { operatorCommandLoop } : {}
8995
- };
9041
+ }
9042
+ if (active.length === 0) {
9043
+ const reason = backingOff.length > 0 ? "backoff" : "all-keys-over-cap";
9044
+ if (backingOff.length > 0) {
9045
+ log2(
9046
+ `[mcp-presence-reaper] '${codeName}': holding off restart for [${backingOff.join(", ")}] \u2014 escalating backoff interval not yet elapsed (ENG-6480)${givenUp.length > 0 ? ` (given up: [${givenUp.join(", ")}])` : ""}`
9047
+ );
8996
9048
  }
8997
- const suppressedOperator = operatorBucket.count > this.operatorMax;
8998
- const suppressedProvisioning = !suppressedOperator && provBucket.count > this.provisioningMax;
8999
- const wouldHaveTripped = suppressedOperator || suppressedProvisioning;
9000
9049
  return {
9001
- tripped: false,
9002
- windowCount,
9003
- classCounts,
9004
- maxProvisioningBucket: provBucket.count,
9005
- maxProvisioningBucketKey: provBucket.key,
9006
- ...scheduledLoop ? { scheduledLoop } : {},
9007
- ...operatorCommandLoop ? { operatorCommandLoop } : {},
9008
- ...newAgentGraceActive ? {
9009
- newAgentGraceSuppressedTrip: wouldHaveTripped,
9010
- ...newAgentGraceExpiresAt !== void 0 ? { newAgentGraceExpiresAt } : {},
9011
- ...wouldHaveTripped ? {
9012
- newAgentGraceSuppressed: {
9013
- class: suppressedOperator ? "operator-config" : "provisioning",
9014
- bucketKey: suppressedOperator ? operatorBucket.key : provBucket.key,
9015
- bucketCount: suppressedOperator ? operatorBucket.count : provBucket.count,
9016
- agentAgeMs: age ?? 0
9017
- }
9018
- } : {}
9019
- } : {}
9050
+ missing,
9051
+ restarted: false,
9052
+ reason,
9053
+ givenUp,
9054
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9055
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9056
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9057
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9058
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9059
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
9060
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9061
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9062
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9063
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
9020
9064
  };
9021
9065
  }
9022
- /** Operator-initiated reset: drops the trip + the events log for this agent. */
9023
- clear(codeName) {
9024
- const key = this.keyFor(codeName);
9025
- this.trips.delete(key);
9026
- this.events.delete(key);
9027
- if (key !== codeName) {
9028
- this.trips.delete(codeName);
9029
- this.events.delete(codeName);
9066
+ const readRestartBudget = () => {
9067
+ try {
9068
+ return args.restartBudgetRemaining?.(codeName) ?? null;
9069
+ } catch (err) {
9070
+ log2(
9071
+ `[mcp-presence-reaper] restartBudgetRemaining threw for '${codeName}' (suppressed; treated as "cannot say"): ${err.message}`
9072
+ );
9073
+ return null;
9030
9074
  }
9031
- }
9032
- /** Snapshot tripped agents for `manager-state.json`. */
9033
- serialize() {
9034
- return Object.fromEntries(this.trips.entries());
9035
- }
9036
- /**
9037
- * Rehydrate trip state from `manager-state.json`. Called once at
9038
- * worker startup. Window history is intentionally NOT persisted — only
9039
- * tripped state. A tripped breaker survives manager restart; an
9040
- * un-tripped one starts a fresh window in the new worker.
9041
- */
9042
- hydrate(saved) {
9043
- if (!saved) return;
9044
- for (const [codeName, trip] of Object.entries(saved)) {
9045
- if (trip && typeof trip.trippedAt === "number" && Array.isArray(trip.eventsAtTrip)) {
9046
- const klass = trip.trippedClass;
9047
- this.trips.set(
9048
- codeName,
9049
- klass === void 0 || KNOWN_REASON_CLASSES.has(klass) ? trip : { ...trip, trippedClass: void 0 }
9075
+ };
9076
+ let blockedRestartPending = null;
9077
+ if (readPane) {
9078
+ let pane = null;
9079
+ try {
9080
+ pane = readPane(codeName);
9081
+ } catch (err) {
9082
+ log2(
9083
+ `[mcp-presence-reaper] '${codeName}': pane read threw (${err.message}) \u2014 proceeding with restart (ENG-9932 fails open)`
9084
+ );
9085
+ }
9086
+ const holdBlockedOnPrompt = (blocked2, opts = {}) => {
9087
+ const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
9088
+ if (onPaneBlocked) {
9089
+ try {
9090
+ onPaneBlocked(codeName, {
9091
+ kind: blocked2,
9092
+ activeKeys: [...active],
9093
+ paneHash: fingerprint?.hash ?? null,
9094
+ paneLength: fingerprint?.length ?? null,
9095
+ // ENG-10878: WHY the hold, not just that there was one. An
9096
+ // operator paged by this alert needs to know whether the one
9097
+ // restart was spent and failed, or was never affordable - those
9098
+ // want different next actions, and without this the two are
9099
+ // indistinguishable on the alert board.
9100
+ restartBudgetRemaining: readRestartBudget(),
9101
+ restartSpentThisEpisode: blockedPromptEpisodes.get(codeName)?.restarted ?? false,
9102
+ restartDeferred: opts.restartDeferred ?? false
9103
+ });
9104
+ } catch (err) {
9105
+ log2(
9106
+ `[mcp-presence-reaper] onPaneBlocked callback threw for '${codeName}' (suppressed): ${err.message}`
9107
+ );
9108
+ }
9109
+ }
9110
+ return {
9111
+ missing,
9112
+ restarted: false,
9113
+ reason: "pane-blocked-on-dialog",
9114
+ paneBlockedOn: blocked2,
9115
+ givenUp: givenUp.length > 0 ? givenUp : void 0,
9116
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9117
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9118
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9119
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9120
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9121
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
9122
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9123
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9124
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9125
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
9126
+ };
9127
+ };
9128
+ const blocked = classifyUnanswerablePane(pane);
9129
+ const openBlockedEpisode = blockedPromptEpisodes.get(codeName);
9130
+ if (pane !== null && !blocked && liveKeys.size > 0 && openBlockedEpisode) {
9131
+ blockedPromptEpisodes.delete(codeName);
9132
+ }
9133
+ if (blocked) {
9134
+ const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
9135
+ const budgetLeft = readRestartBudget();
9136
+ const episode = blockedPromptEpisodes.get(codeName);
9137
+ const restartWouldCount = reaperRestartBreakerReason(active) !== void 0;
9138
+ if (!episode && budgetLeft === 0 && restartWouldCount) {
9139
+ log2(
9140
+ `[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 blocked on an interactive prompt (${blocked}) and the crash breaker has NO budget left in its window, so the one restart ENG-10878 would spend would be the event that TRIPS it - remaining===0 means crashCount===max, and the trip test fires at max+1, attributed to mcp-presence-reaper itself. Needs a human at the pane: tmux attach -t agt-${codeName}` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
9141
+ );
9142
+ return holdBlockedOnPrompt(blocked);
9143
+ }
9144
+ if (!episode) {
9145
+ blockedRestartPending = {
9146
+ blocked,
9147
+ paneHash: fingerprint?.hash ?? null,
9148
+ hold: (opts) => holdBlockedOnPrompt(blocked, opts)
9149
+ };
9150
+ log2(
9151
+ `[mcp-presence-reaper] '${codeName}': blocked on an interactive prompt (${blocked}) \u2014 ATTEMPTING this episode's ONE restart (ENG-10878). A restart cured this on 2026-09-22 and did not on 2026-09-03, so the next poll decides which it is` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
9050
9152
  );
9153
+ } else {
9154
+ const comparable = fingerprint !== null && episode.paneHashAtRestart !== null;
9155
+ const modalDelta = !comparable ? "modal fingerprint unavailable on one or both polls, so same-or-different is UNKNOWN" : fingerprint.hash === episode.paneHashAtRestart ? "SAME modal as before the restart" : "a DIFFERENT modal than before the restart";
9156
+ log2(
9157
+ `[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 still blocked on an interactive prompt (${blocked}) after this episode's one restart, so the restart did not clear it (${modalDelta}). Needs a human at the pane: tmux attach -t agt-${codeName}` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
9158
+ );
9159
+ return holdBlockedOnPrompt(blocked);
9051
9160
  }
9052
9161
  }
9053
9162
  }
9054
- /** Test helper — current in-window event count for `codeName`. */
9055
- windowCount(codeName) {
9056
- const cutoff = this.now() - this.windowMs;
9057
- return (this.events.get(this.keyFor(codeName)) ?? []).filter((e) => e.at >= cutoff).length;
9163
+ const givenUpSuffix = givenUp.length > 0 ? ` (skipping over-cap: [${givenUp.join(", ")}])` : "";
9164
+ log2(
9165
+ `[mcp-presence-reaper] '${codeName}': declared MCP(s) [${active.join(", ")}] have no live children \u2014 restarting session${givenUpSuffix}`
9166
+ );
9167
+ const tornDown = stopSession(codeName, { activeKeys: active });
9168
+ if (onRestart && tornDown !== false) {
9169
+ try {
9170
+ const maybePromise = onRestart(codeName, { activeKeys: active, givenUpKeys: givenUp });
9171
+ if (maybePromise && typeof maybePromise.then === "function") {
9172
+ void maybePromise.then(void 0, (err) => {
9173
+ log2(
9174
+ `[mcp-presence-reaper] onRestart callback rejected for '${codeName}' (suppressed; restart proceeds): ${err.message}`
9175
+ );
9176
+ });
9177
+ }
9178
+ } catch (err) {
9179
+ log2(
9180
+ `[mcp-presence-reaper] onRestart callback threw for '${codeName}' (suppressed; restart proceeds): ${err.message}`
9181
+ );
9182
+ }
9058
9183
  }
9059
- };
9060
- function formatStatusMessage(events, windowMs, klass = "crash", integrationKey = "") {
9061
- const last = events[events.length - 1];
9062
- const windowLabel = windowMs < 6e4 ? `${Math.round(windowMs / 1e3)}s` : `${(windowMs / 6e4).toFixed(1).replace(/\.0$/, "")}min`;
9063
- const reasonCounts = /* @__PURE__ */ new Map();
9064
- for (const e of events) reasonCounts.set(e.reason, (reasonCounts.get(e.reason) ?? 0) + 1);
9065
- const breakdown = Array.from(reasonCounts.entries()).map(([r, n]) => `${r}=${n}`).join(", ");
9066
- const classLabel = klass === "provisioning" ? "provisioning-reload " : klass === "operator-config" ? "operator-requested config-reload " : "";
9067
- const integrationLabel = (klass === "provisioning" || klass === "operator-config") && integrationKey ? ` for integration '${integrationKey}'` : "";
9068
- return `Circuit breaker tripped: ${events.length} ${classLabel}restarts in ${windowLabel}${integrationLabel} (${breakdown}); most recent=${last.reason} at ${new Date(last.at).toISOString()}`;
9184
+ if (blockedRestartPending) {
9185
+ if (tornDown === false) {
9186
+ log2(
9187
+ `[mcp-presence-reaper] '${codeName}': restart DEFERRED by the host gate this poll (maintenance window / not idle), so nothing was torn down and this episode's one restart is still unspent \u2014 it will be re-attempted when the gate opens. Needs a human at the pane if it persists: tmux attach -t agt-${codeName}`
9188
+ );
9189
+ return blockedRestartPending.hold({ restartDeferred: true });
9190
+ }
9191
+ blockedPromptEpisodes.set(codeName, {
9192
+ restarted: true,
9193
+ paneHashAtRestart: blockedRestartPending.paneHash
9194
+ });
9195
+ log2(
9196
+ `[mcp-presence-reaper] '${codeName}': this episode's ONE restart is now SPENT (teardown confirmed) \u2014 the next poll decides whether it cleared the modal.`
9197
+ );
9198
+ }
9199
+ if (tornDown === false) {
9200
+ log2(
9201
+ `[mcp-presence-reaper] '${codeName}': restart DEFERRED by the host gate (maintenance window / not idle) \u2014 nothing was torn down, so this poll records no restart and the next poll re-detects and re-tries (ENG-10878)`
9202
+ );
9203
+ return {
9204
+ missing,
9205
+ restarted: false,
9206
+ reason: "restart-deferred",
9207
+ givenUp: givenUp.length > 0 ? givenUp : void 0,
9208
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9209
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9210
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9211
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9212
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0,
9213
+ // ROTATION STATE SURVIVES A DEFERRAL (CodeRabbit on #6399). This return
9214
+ // omitted all five, and a deferred poll is exactly a poll where rotation
9215
+ // work may already have happened: `recordRotationOutcome` runs during the
9216
+ // scan, well before the host gate declines the teardown. So a rotation
9217
+ // that escalated to `never-respawned` on this poll was recorded in the
9218
+ // map, dropped from the result, and unrecoverable by the caller -
9219
+ // probe-armed keys are held OUT of `missing` precisely so they are not
9220
+ // torn down, which also means a caller cannot rebuild them from it.
9221
+ //
9222
+ // `ReapMissingMcpSessionsResult` documents these as present whenever an
9223
+ // entry exists, so omitting them here made the result LIE rather than
9224
+ // merely under-report.
9225
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9226
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9227
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9228
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9229
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
9230
+ };
9231
+ }
9232
+ for (const key of active) {
9233
+ const st = presenceReaperState.get(stateKey(codeName, key));
9234
+ if (!st) continue;
9235
+ if (st.historyOwedForSessionStartedAt !== sessionStartedAt) continue;
9236
+ st.restartHistory.push(nowMs);
9237
+ st.historyOwedForSessionStartedAt = null;
9238
+ }
9239
+ return {
9240
+ missing,
9241
+ restarted: true,
9242
+ givenUp: givenUp.length > 0 ? givenUp : void 0,
9243
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9244
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9245
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9246
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9247
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9248
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
9249
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9250
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9251
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9252
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
9253
+ };
9069
9254
  }
9070
9255
 
9071
9256
  // src/lib/mcp-config-lookup.ts
@@ -11725,6 +11910,12 @@ export {
11725
11910
  resolveFlagFromLayers,
11726
11911
  resolveAllFlags,
11727
11912
  HostFlagStore,
11913
+ reaperRestartBreakerReason,
11914
+ isProvisioningReloadReason,
11915
+ isIntegrationBucketedReason,
11916
+ tripClass,
11917
+ readEnvNumber,
11918
+ RestartBreaker,
11728
11919
  diffEnvIntegrations,
11729
11920
  classifyEnvIntegrationsDiff,
11730
11921
  findMcpServersUsingVars,
@@ -11736,13 +11927,8 @@ export {
11736
11927
  clearPresenceReaperState,
11737
11928
  clearPresenceReaperStateForKeys,
11738
11929
  givenUpMcpServerKeys,
11930
+ describeBlockedPromptHold,
11739
11931
  reapMissingMcpSessions,
11740
- reaperRestartBreakerReason,
11741
- isProvisioningReloadReason,
11742
- isIntegrationBucketedReason,
11743
- tripClass,
11744
- readEnvNumber,
11745
- RestartBreaker,
11746
11932
  markQuarantinedForProbe,
11747
11933
  FIRST_CONNECT_WINDOW_MS,
11748
11934
  isWithinReconnectQuiesceWindow,
@@ -11775,4 +11961,4 @@ export {
11775
11961
  managerInstallSystemUnitCommand,
11776
11962
  managerUninstallSystemUnitCommand
11777
11963
  };
11778
- //# sourceMappingURL=chunk-GHHDFHKE.js.map
11964
+ //# sourceMappingURL=chunk-HIDB6U3D.js.map