@integrity-labs/agt-cli 0.28.1016 → 0.28.1017

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ import {
15
15
  parseEnvFileEntries,
16
16
  parseEnvIntegrations,
17
17
  shellQuote
18
- } from "./chunk-YPS4AUHN.js";
18
+ } from "./chunk-CVQDMHOP.js";
19
19
  import {
20
20
  BIND_FAILURE_QUARANTINE_THRESHOLD,
21
21
  INTEGRATIONS_SECTION_END,
@@ -77,7 +77,7 @@ import {
77
77
  sessionTranscriptDir,
78
78
  worseConnectivityOutcome,
79
79
  wrapScheduledTaskPrompt
80
- } from "./chunk-B7XGAC54.js";
80
+ } from "./chunk-X7SX25N6.js";
81
81
  import {
82
82
  classifyUnanswerablePane,
83
83
  summarizeUnanswerablePane
@@ -7007,7 +7007,7 @@ function exchangeFailureKind(err) {
7007
7007
  }
7008
7008
 
7009
7009
  // src/lib/api-client.ts
7010
- var agtCliVersion = true ? "0.28.1016" : "dev";
7010
+ var agtCliVersion = true ? "0.28.1017" : "dev";
7011
7011
  var lastConfigHash = null;
7012
7012
  function setConfigHash(hash) {
7013
7013
  lastConfigHash = hash && hash.length > 0 ? hash : null;
@@ -7623,1463 +7623,1634 @@ var HostFlagStore = class {
7623
7623
  }
7624
7624
  };
7625
7625
 
7626
- // src/lib/stale-mcp-reaper.ts
7627
- import { execFileSync } from "child_process";
7628
- function livenessFromProbe(r, onNoAnswer) {
7629
- if (r.kind === "ok") return true;
7630
- if (r.kind === "failed" && r.status !== null) return false;
7631
- onNoAnswer?.(r.kind === "timed-out" ? "did not answer within its bound" : `could not run: ${r.message}`);
7632
- return true;
7626
+ // src/lib/restart-breaker.ts
7627
+ function reaperRestartBreakerReason(activeKeys) {
7628
+ return activeKeys.length >= 2 ? "mcp-presence-reaper" : void 0;
7633
7629
  }
7634
- var rotationTimestamps = /* @__PURE__ */ new Map();
7635
- var DEFAULT_ROTATION_GRACE_MS = 15e3;
7636
- function rotationKey(codeName, serverKey) {
7637
- return `${codeName}\0${serverKey}`;
7630
+ var KNOWN_REASON_CLASS_MAP = {
7631
+ crash: true,
7632
+ provisioning: true,
7633
+ "credential-rotation": true,
7634
+ // ENG-8388: provisioning reloads a human demonstrably asked for. Same
7635
+ // Record-not-Set reasoning as above — a new class cannot be silently omitted.
7636
+ "operator-config": true,
7637
+ // ENG-8215: the self-healing class (bind-remediation - the manager's OWN
7638
+ // repair attempts, which must never count toward a trip). Adding it here is
7639
+ // forced by the Record type, which is exactly why that shape was chosen over
7640
+ // a Set literal (ENG-7577, CodeRabbit on #3861) - a new union member cannot
7641
+ // be silently omitted from the rehydration allowlist.
7642
+ "self-healing": true,
7643
+ // ENG-9138: the platform-scheduled class. Same Record-not-Set reasoning: the
7644
+ // compiler forces every new class into this allowlist, so a `trippedClass`
7645
+ // rehydrated from manager-state.json can never name a class this build does
7646
+ // not know about.
7647
+ scheduled: true,
7648
+ // ENG-10066: the operator-command class. Same Record-not-Set reasoning again —
7649
+ // the compiler forces every new class into this allowlist.
7650
+ "operator-command": true
7651
+ };
7652
+ var KNOWN_REASON_CLASSES = new Set(
7653
+ Object.keys(KNOWN_REASON_CLASS_MAP)
7654
+ );
7655
+ var PROVISIONING_RELOAD_REASONS = /* @__PURE__ */ new Set([
7656
+ "hot-reload-mcp",
7657
+ "managed-mcp-churn",
7658
+ // ENG-8215: 'bind-remediation' MOVED OUT of this set into SELF_HEALING_REASONS.
7659
+ // It is the manager repairing itself, not a config reload, and counting it
7660
+ // meant the harder the platform tried to fix an agent the more certain it
7661
+ // became that it should give up. Deliberately NOT re-added here.
7662
+ // ENG-7576: the dashboard integration-add stop itself - the first restart of
7663
+ // the very burst this class exists for, previously uncounted entirely.
7664
+ "integration-change",
7665
+ // ENG-7771: a sender_policy delivery restart is a deliberate config-delivery
7666
+ // reload (operator changed the policy, or the fail-closed first-poll verify
7667
+ // after a manager restart with no persisted baseline), not a crash. On the
7668
+ // tight crash tally, a host whose sender-policy-baseline.json write keeps
7669
+ // failing (disk full, permissions) would convert the fail-closed restart
7670
+ // into repeated crash tallies across a multi-deploy day and trip the
7671
+ // breaker - taking the agent DOWN, which is strictly worse than the stale
7672
+ // policy the restart exists to fix. The looser provisioning tally still
7673
+ // trips + pages on sustained thrash.
7674
+ "sender-policy-change",
7675
+ // ENG-9802: same argument, same failure mode — a host whose baseline write
7676
+ // keeps failing must not convert a fail-closed policy restart into a breaker
7677
+ // trip that takes the agent down.
7678
+ "telegram-policy-change",
7679
+ // ENG-9861: and again for standing rules. The fail-closed first-poll restart
7680
+ // has exactly the same shape, so it needs the same protection from a host
7681
+ // whose sidecar write keeps failing.
7682
+ "agent-rules-change",
7683
+ // ENG-10510 (Obi-Wan on #6313): a channel-set ADD is a config reload, not a
7684
+ // crash, and it was landing on the crash tally purely by falling through.
7685
+ // It is 'integration-change's exact sibling - both are "a channel or server
7686
+ // was added, re-render .mcp.json and respawn" - and it was never argued for
7687
+ // the crash class, it simply was not in this set.
7688
+ //
7689
+ // This is the ENG-9138 / ENG-10066 species verbatim: a PLATFORM-MANUFACTURED
7690
+ // restart counted as though the agent had crashed. ENG-9138 cost katie 8h16m
7691
+ // when day-rollover fell through the same way; ENG-10066 cost an operator an
7692
+ // auto-pause when a channel /restart did. Both were fixed by naming the
7693
+ // reason rather than leaving it to the catch-all, and this is the third.
7694
+ //
7695
+ // WHY NOW. Until ENG-10510 this restart only ever followed a human adding a
7696
+ // channel, so the crash tally was wrong but rarely reached: a person adding
7697
+ // two channels inside ten minutes is uncommon and self-limiting. Channel
7698
+ // auto-resume is the first thing that fires it UNATTENDED, on a timer, on
7699
+ // any agent with a quarantined channel. An agent with a second crash-class
7700
+ // restart in the same ten minutes (the presence reaper's
7701
+ // 'mcp-presence-reaper', for one) would cross DEFAULT_MAX and auto-pause
7702
+ // with no human anywhere in the loop - which is precisely the ENG-8215 harm
7703
+ // that feature's 2h floor exists to prevent.
7704
+ //
7705
+ // Counted, not exempt: provisioning is a LOOSER tally, not an absent one, so
7706
+ // sustained channel-set thrash still trips and still pages.
7707
+ "channel-set-change"
7708
+ ]);
7709
+ function isProvisioningReloadReason(reason) {
7710
+ return PROVISIONING_RELOAD_REASONS.has(reason);
7638
7711
  }
7639
- function recordStaleRotation(codeName, serverKeys, now = Date.now) {
7640
- const ts = now();
7641
- for (const key of serverKeys) {
7642
- rotationTimestamps.set(rotationKey(codeName, key), ts);
7643
- }
7712
+ var CREDENTIAL_ROTATION_REASONS = /* @__PURE__ */ new Set([
7713
+ "credential-rotation"
7714
+ ]);
7715
+ function isCredentialRotationReason(reason) {
7716
+ return CREDENTIAL_ROTATION_REASONS.has(reason);
7644
7717
  }
7645
- var ROTATION_TRACKING_RETENTION_MS = 15 * 60 * 1e3;
7646
- function classifyRotation(codeName, serverKey, graceMs, now = Date.now, retentionMs = ROTATION_TRACKING_RETENTION_MS) {
7647
- const k = rotationKey(codeName, serverKey);
7648
- const ts = rotationTimestamps.get(k);
7649
- if (ts === void 0) return "none";
7650
- const age = now() - ts;
7651
- if (age < graceMs) return "within-grace";
7652
- if (age < retentionMs) return "expired";
7653
- rotationTimestamps.delete(k);
7654
- return "none";
7718
+ var SELF_HEALING_REASONS = /* @__PURE__ */ new Set([
7719
+ "bind-remediation"
7720
+ ]);
7721
+ function isSelfHealingReason(reason) {
7722
+ return SELF_HEALING_REASONS.has(reason);
7655
7723
  }
7656
- function getRotationRecordedAt(codeName, serverKey) {
7657
- return rotationTimestamps.get(rotationKey(codeName, serverKey));
7724
+ var SCHEDULED_REASONS = /* @__PURE__ */ new Set([
7725
+ "day-rollover"
7726
+ ]);
7727
+ function isScheduledRestartReason(reason) {
7728
+ return SCHEDULED_REASONS.has(reason);
7658
7729
  }
7659
- function clearStaleRotation(codeName, serverKeys) {
7660
- for (const key of serverKeys) {
7661
- rotationTimestamps.delete(rotationKey(codeName, key));
7662
- }
7730
+ var OPERATOR_COMMAND_REASONS = /* @__PURE__ */ new Set([
7731
+ "channel-restart-flag"
7732
+ ]);
7733
+ function isOperatorCommandReason(reason) {
7734
+ return OPERATOR_COMMAND_REASONS.has(reason);
7663
7735
  }
7664
- function parseEnvIntegrationsEntries(content) {
7665
- const entries = /* @__PURE__ */ new Map();
7666
- for (const raw of content.split(/\r?\n/)) {
7667
- const line = raw.trim();
7668
- if (!line || line.startsWith("#")) continue;
7669
- const eq = line.indexOf("=");
7670
- if (eq <= 0) continue;
7671
- const name = line.slice(0, eq).trim();
7672
- if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) continue;
7673
- const value = line.slice(eq + 1);
7674
- entries.set(name, value);
7675
- }
7676
- return entries;
7736
+ var OPERATOR_CONFIG_REASONS = /* @__PURE__ */ new Set([
7737
+ "operator-integration-change"
7738
+ ]);
7739
+ function isOperatorConfigReason(reason) {
7740
+ return OPERATOR_CONFIG_REASONS.has(reason);
7677
7741
  }
7678
- function diffEnvIntegrations(oldContent, newContent) {
7679
- const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
7680
- const newEntries = parseEnvIntegrationsEntries(newContent);
7681
- const changed = /* @__PURE__ */ new Set();
7682
- for (const [name, value] of newEntries) {
7683
- if (oldEntries.get(name) !== value) changed.add(name);
7684
- }
7685
- for (const name of oldEntries.keys()) {
7686
- if (!newEntries.has(name)) changed.add(name);
7687
- }
7688
- return [...changed];
7742
+ var NEVER_INTEGRATION_BUCKETED = /* @__PURE__ */ new Set([
7743
+ "channel-set-change"
7744
+ ]);
7745
+ function isIntegrationBucketedReason(reason) {
7746
+ if (NEVER_INTEGRATION_BUCKETED.has(reason)) return false;
7747
+ return isProvisioningReloadReason(reason) || isOperatorConfigReason(reason);
7689
7748
  }
7690
- function classifyEnvIntegrationsDiff(oldContent, newContent) {
7691
- const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
7692
- const newEntries = parseEnvIntegrationsEntries(newContent);
7693
- const rotated = [];
7694
- const added = [];
7695
- const removed = [];
7696
- for (const [name, value] of newEntries) {
7697
- if (!oldEntries.has(name)) added.push(name);
7698
- else if (oldEntries.get(name) !== value) rotated.push(name);
7699
- }
7700
- for (const name of oldEntries.keys()) {
7701
- if (!newEntries.has(name)) removed.push(name);
7702
- }
7703
- return { rotated, added, removed };
7749
+ function restartReasonClass(reason) {
7750
+ if (isCredentialRotationReason(reason)) return "credential-rotation";
7751
+ if (isSelfHealingReason(reason)) return "self-healing";
7752
+ if (isOperatorConfigReason(reason)) return "operator-config";
7753
+ if (isScheduledRestartReason(reason)) return "scheduled";
7754
+ if (isOperatorCommandReason(reason)) return "operator-command";
7755
+ return isProvisioningReloadReason(reason) ? "provisioning" : "crash";
7704
7756
  }
7705
- function findMcpServersUsingVars(mcp, changedVars) {
7706
- const changedSet = new Set(changedVars);
7707
- if (!mcp?.mcpServers || changedSet.size === 0) return [];
7708
- const result = [];
7709
- for (const [serverKey, entry] of Object.entries(mcp.mcpServers)) {
7710
- if (!entry || typeof entry !== "object") continue;
7711
- const env = entry.env;
7712
- if (!env || typeof env !== "object") continue;
7713
- let matches = false;
7714
- for (const value of Object.values(env)) {
7715
- if (typeof value !== "string") continue;
7716
- const placeholderMatches = value.match(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g);
7717
- if (placeholderMatches) {
7718
- for (const ph of placeholderMatches) {
7719
- const name = ph.slice(2, -1);
7720
- if (changedSet.has(name)) {
7721
- matches = true;
7722
- break;
7723
- }
7724
- }
7725
- }
7726
- if (matches) break;
7727
- }
7728
- if (matches) result.push(serverKey);
7757
+ function countByClass(events) {
7758
+ let crash = 0;
7759
+ let provisioning = 0;
7760
+ let credentialRotation = 0;
7761
+ let selfHealing = 0;
7762
+ let operatorConfig = 0;
7763
+ let scheduled = 0;
7764
+ let operatorCommand = 0;
7765
+ for (const e of events) {
7766
+ const klass = restartReasonClass(e.reason);
7767
+ if (klass === "provisioning") provisioning += 1;
7768
+ else if (klass === "credential-rotation") credentialRotation += 1;
7769
+ else if (klass === "self-healing") selfHealing += 1;
7770
+ else if (klass === "operator-config") operatorConfig += 1;
7771
+ else if (klass === "scheduled") scheduled += 1;
7772
+ else if (klass === "operator-command") operatorCommand += 1;
7773
+ else crash += 1;
7729
7774
  }
7730
- return result;
7775
+ return {
7776
+ crash,
7777
+ provisioning,
7778
+ "credential-rotation": credentialRotation,
7779
+ "self-healing": selfHealing,
7780
+ "operator-config": operatorConfig,
7781
+ scheduled,
7782
+ "operator-command": operatorCommand
7783
+ };
7731
7784
  }
7732
- function envOnlyRespawnVars(changedVars, mcp, channelSecretKeys) {
7733
- const channelSet = new Set(channelSecretKeys);
7734
- return [...changedVars].filter(
7735
- (v) => !channelSet.has(v) && findMcpServersUsingVars(mcp, [v]).length === 0
7736
- );
7785
+ function maxProvisioningBucket(events) {
7786
+ return maxBucketWhere(events, isProvisioningReloadReason);
7737
7787
  }
7738
- function liveProxyExtraHeaderVars(mcp) {
7739
- const out = /* @__PURE__ */ new Set();
7740
- if (!mcp?.mcpServers) return out;
7741
- for (const entry of Object.values(mcp.mcpServers)) {
7742
- const spec = entry?.env?.["AGT_REMOTE_MCP_EXTRA_HEADERS"];
7743
- if (typeof spec !== "string") continue;
7744
- for (const pair of spec.split(",")) {
7745
- const colon = pair.indexOf(":");
7746
- if (colon <= 0) continue;
7747
- const varName = pair.slice(colon + 1).trim();
7748
- if (varName) out.add(varName);
7749
- }
7788
+ function maxBucketWhere(events, matches) {
7789
+ const buckets = /* @__PURE__ */ new Map();
7790
+ for (const e of events) {
7791
+ if (!matches(e.reason)) continue;
7792
+ const key = e.integrationKey ?? "";
7793
+ buckets.set(key, (buckets.get(key) ?? 0) + 1);
7750
7794
  }
7751
- return out;
7752
- }
7753
- function liveProxyTokenVars(mcp) {
7754
- const out = /* @__PURE__ */ new Set();
7755
- if (!mcp?.mcpServers) return out;
7756
- for (const entry of Object.values(mcp.mcpServers)) {
7757
- const tokenVar = entry?.env?.["AGT_REMOTE_MCP_TOKEN_VAR"];
7758
- if (typeof tokenVar === "string" && tokenVar.trim()) {
7759
- out.add(tokenVar.trim());
7760
- }
7795
+ let best = { count: 0, key: "" };
7796
+ for (const [key, count] of buckets) {
7797
+ if (count > best.count) best = { count, key };
7761
7798
  }
7762
- return out;
7799
+ return best;
7763
7800
  }
7764
- function buildArgvMatchersForEntry(key, entry) {
7765
- const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
7766
- const patterns = [];
7767
- const args = entry?.args ?? [];
7768
- for (let i = args.length - 1; i >= 0; i--) {
7769
- const arg = args[i];
7770
- if (typeof arg !== "string" || !arg) continue;
7771
- if (arg.startsWith("-")) continue;
7772
- const stripped = arg.replace(
7773
- /@[^/@]*$/,
7774
- (m) => (
7775
- // @latest, @1.2.3, etc. — drop. But @scope at the START of a
7776
- // package spec must NOT be stripped, so we only strip a tail
7777
- // `@...` if it doesn't itself start with a slash inside.
7778
- // The regex above only matches a single trailing `@...`
7779
- // segment after the last `/`, so this is safe for `@scope/pkg`.
7780
- m.includes("/") ? m : ""
7781
- )
7801
+ function tripClass(trip) {
7802
+ if (trip.trippedClass) return trip.trippedClass;
7803
+ if (!Array.isArray(trip.eventsAtTrip) || trip.eventsAtTrip.length === 0) return "crash";
7804
+ return trip.eventsAtTrip.every((e) => isProvisioningReloadReason(e.reason)) ? "provisioning" : "crash";
7805
+ }
7806
+ var MAX_NEW_AGENT_GRACE_MS = 365 * 24 * 60 * 60 * 1e3;
7807
+ var DEFAULT_MAX = 2;
7808
+ var DEFAULT_WINDOW_MS = 6e5;
7809
+ var DEFAULT_PROVISIONING_MAX = RESTART_BREAKER_PROVISIONING_MAX;
7810
+ var DEFAULT_PROVISIONING_WINDOW_MS = RESTART_BREAKER_PROVISIONING_WINDOW_MS;
7811
+ var DEFAULT_OPERATOR_MAX = RESTART_BREAKER_OPERATOR_MAX;
7812
+ var DEFAULT_SCHEDULED_MAX = RESTART_BREAKER_SCHEDULED_MAX;
7813
+ var DEFAULT_SCHEDULED_WINDOW_MS = RESTART_BREAKER_SCHEDULED_WINDOW_MS;
7814
+ var DEFAULT_OPERATOR_COMMAND_MAX = RESTART_BREAKER_OPERATOR_COMMAND_MAX;
7815
+ var DEFAULT_OPERATOR_COMMAND_WINDOW_MS = RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS;
7816
+ function readEnvNumber(name, fallback) {
7817
+ const raw = process.env[name];
7818
+ if (!raw) return fallback;
7819
+ const parsed = Number(raw);
7820
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
7821
+ }
7822
+ function migrateMapKey(map, from, to) {
7823
+ if (map.has(to) || !map.has(from)) return;
7824
+ map.set(to, map.get(from));
7825
+ map.delete(from);
7826
+ }
7827
+ var RestartBreaker = class {
7828
+ max;
7829
+ windowMs;
7830
+ /**
7831
+ * ENG-8215: NOT readonly - the constructor may clamp it upward to preserve the
7832
+ * quarantine-fires-first ordering against a runtime override. Never mutated
7833
+ * after construction.
7834
+ */
7835
+ provisioningMax;
7836
+ provisioningWindowMs;
7837
+ /**
7838
+ * ENG-8388: the looser operator-config budget. Shares provisioningWindowMs.
7839
+ * NOT readonly - the constructor floors it at `provisioningMax` so setting one
7840
+ * env var without the other cannot invert the two tallies.
7841
+ */
7842
+ operatorMax;
7843
+ /**
7844
+ * ENG-9138: the scheduled class's LOOP bar. Readonly and deliberately NOT
7845
+ * floored against any other tally: it gates a REPORT, not a pause, so it
7846
+ * cannot invert anything the way operatorMax/provisioningMax can.
7847
+ */
7848
+ scheduledMax;
7849
+ /** ENG-9138: sliding window for the scheduled tally. */
7850
+ scheduledWindowMs;
7851
+ operatorCommandMax;
7852
+ operatorCommandWindowMs;
7853
+ /** Longest window any tally needs — how long the events log must retain. */
7854
+ retentionMs;
7855
+ now;
7856
+ /** ENG-7891 slice 5b: code_name → durable runtime key (agent_id when id-keyed). Identity on the legacy fleet. */
7857
+ runtimeKey;
7858
+ /** ENG-8690: agent creation time lookup; undefined result ⇒ no grace. */
7859
+ agentCreatedAt;
7860
+ /** ENG-8690: grace duration; 0 disables. */
7861
+ newAgentGraceMs;
7862
+ /** ENG-8690: the bound the grace does NOT lift. NOT readonly - floored below. */
7863
+ newAgentGraceCeiling;
7864
+ events = /* @__PURE__ */ new Map();
7865
+ trips = /* @__PURE__ */ new Map();
7866
+ /**
7867
+ * ENG-8215: set when a supplied `provisioningMax` was raised to keep MCP
7868
+ * quarantine ahead of the breaker. Read once by the manager at construction so
7869
+ * an overridden-and-clamped value is visible in manager.log rather than
7870
+ * silently discarded.
7871
+ */
7872
+ quarantineOrderingClamped;
7873
+ constructor(opts = {}) {
7874
+ this.max = opts.max ?? readEnvNumber("AGT_RESTART_BREAKER_MAX", DEFAULT_MAX);
7875
+ this.windowMs = opts.windowMs ?? readEnvNumber("AGT_RESTART_BREAKER_WINDOW_MS", DEFAULT_WINDOW_MS);
7876
+ this.provisioningMax = opts.provisioningMax ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", DEFAULT_PROVISIONING_MAX);
7877
+ this.provisioningWindowMs = opts.provisioningWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS", DEFAULT_PROVISIONING_WINDOW_MS);
7878
+ this.operatorMax = opts.operatorMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_MAX", DEFAULT_OPERATOR_MAX);
7879
+ this.scheduledMax = opts.scheduledMax ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_MAX", DEFAULT_SCHEDULED_MAX);
7880
+ this.scheduledWindowMs = opts.scheduledWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_WINDOW_MS", DEFAULT_SCHEDULED_WINDOW_MS);
7881
+ this.operatorCommandMax = opts.operatorCommandMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_COMMAND_MAX", DEFAULT_OPERATOR_COMMAND_MAX);
7882
+ this.operatorCommandWindowMs = opts.operatorCommandWindowMs ?? readEnvNumber(
7883
+ "AGT_RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS",
7884
+ DEFAULT_OPERATOR_COMMAND_WINDOW_MS
7782
7885
  );
7783
- const tail = "(?![A-Za-z0-9_-])";
7784
- if (/^@?[a-z0-9]([a-z0-9._-]*\/)?[a-z0-9._-]+$/i.test(stripped)) {
7785
- patterns.push(new RegExp(`${escapeRe(stripped)}${tail}`));
7786
- const basename = stripped.split("/").pop();
7787
- if (basename && basename !== stripped) {
7788
- patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
7789
- }
7790
- break;
7886
+ this.now = opts.now ?? Date.now;
7887
+ this.runtimeKey = opts.runtimeKey ?? ((codeName) => codeName);
7888
+ this.agentCreatedAt = opts.agentCreatedAt ?? (() => void 0);
7889
+ const graceMs = opts.newAgentGraceMs ?? readEnvNumber("AGT_RESTART_BREAKER_NEW_AGENT_GRACE_MS", RESTART_BREAKER_NEW_AGENT_GRACE_MS);
7890
+ const cappedGrace = Math.min(graceMs, MAX_NEW_AGENT_GRACE_MS);
7891
+ this.newAgentGraceMs = Number.isFinite(graceMs) && graceMs > 0 ? cappedGrace : 0;
7892
+ const ceiling = opts.newAgentGraceCeiling ?? readEnvNumber(
7893
+ "AGT_RESTART_BREAKER_NEW_AGENT_GRACE_CEILING",
7894
+ RESTART_BREAKER_NEW_AGENT_GRACE_CEILING
7895
+ );
7896
+ this.newAgentGraceCeiling = Number.isFinite(ceiling) && ceiling > 0 ? ceiling : RESTART_BREAKER_NEW_AGENT_GRACE_CEILING;
7897
+ if (!Number.isFinite(this.max) || this.max < 1) {
7898
+ throw new Error("restart-breaker max must be a finite number >= 1");
7791
7899
  }
7792
- if (stripped.includes("/")) {
7793
- const basename = stripped.split("/").pop();
7794
- if (basename) {
7795
- patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
7796
- break;
7797
- }
7900
+ if (!Number.isFinite(this.windowMs) || this.windowMs < 1e3) {
7901
+ throw new Error("restart-breaker windowMs must be a finite number >= 1000");
7798
7902
  }
7799
- }
7800
- if (patterns.length === 0) {
7801
- const safe = escapeRe(key);
7802
- patterns.push(new RegExp(`(?<![A-Za-z0-9_])${safe}(?![A-Za-z0-9_])`));
7803
- if (safe.includes("_")) {
7804
- const dashed = safe.replace(/_/g, "-");
7805
- patterns.push(new RegExp(`(?<![A-Za-z0-9_])${dashed}(?![A-Za-z0-9_])`));
7903
+ if (!Number.isFinite(this.provisioningMax) || this.provisioningMax < 1) {
7904
+ throw new Error("restart-breaker provisioningMax must be a finite number >= 1");
7806
7905
  }
7807
- }
7808
- return patterns;
7809
- }
7810
- function buildClaudeAgentMatcher(codeName) {
7811
- const safe = codeName.replace(/[^A-Za-z0-9_-]/g, "");
7812
- return new RegExp(`\\bclaude\\b.*--name\\s+agt-${safe}(?=\\s|$)`);
7813
- }
7814
- function findMcpChildrenForAgent(args) {
7815
- const { rows, codeName, serverKeys, mcpJson } = args;
7816
- const maxDepth = args.maxDepth ?? 8;
7817
- const inContainer = args.inContainer ?? false;
7818
- if (serverKeys.length === 0 || rows.length === 0) return [];
7819
- const claudeMatcher = inContainer ? /\bclaude\b/ : buildClaudeAgentMatcher(codeName);
7820
- const argvMatchers = [];
7821
- for (const key of serverKeys) {
7822
- const entry = mcpJson?.mcpServers?.[key];
7823
- argvMatchers.push(...buildArgvMatchersForEntry(key, entry));
7824
- }
7825
- const byPid = new Map(rows.map((r) => [r.pid, r]));
7826
- const matched = [];
7827
- for (const row of rows) {
7828
- if (!argvMatchers.some((re) => re.test(row.args))) continue;
7829
- if (/\bclaude\b/.test(row.args) && row.args.includes(`--name agt-${codeName}`)) continue;
7830
- let cur = byPid.get(row.ppid);
7831
- let belongs = false;
7832
- for (let depth = 0; depth < maxDepth && cur; depth++) {
7833
- if (claudeMatcher.test(cur.args)) {
7834
- belongs = true;
7835
- break;
7836
- }
7837
- if (cur.pid === 1) break;
7838
- cur = byPid.get(cur.ppid);
7906
+ if (!Number.isFinite(this.operatorMax) || this.operatorMax < 1) {
7907
+ throw new Error("restart-breaker operatorMax must be a finite number >= 1");
7839
7908
  }
7840
- if (belongs) matched.push(row.pid);
7841
- }
7842
- return matched;
7843
- }
7844
- function reapStaleMcpChildren(args) {
7845
- const { log: log2, codeName, serverKeys, mcpJson, graceMs = 5e3, isolated = false } = args;
7846
- if (serverKeys.length === 0) return [];
7847
- const runPs = args.runPs ?? (isolated ? (
7848
- // ENG-9781: hard-bounded. See bounded-exec.ts — `execFileSync`'s own
7849
- // `timeout` is not a bound, and this runs per isolated agent per cycle
7850
- // on the manager-worker's critical path. Throws on a non-answer so the
7851
- // caller's existing catch skips the agent rather than acting on an
7852
- // empty process list.
7853
- () => {
7854
- const r = hardBoundedExecFile(
7855
- "docker",
7856
- ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
7857
- { timeoutMs: 8e3, log: log2 }
7858
- );
7859
- if (r.kind === "ok") return r.stdout;
7860
- throw new Error(
7861
- r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
7862
- );
7909
+ if (this.operatorMax < this.provisioningMax) {
7910
+ this.operatorMax = this.provisioningMax;
7863
7911
  }
7864
- ) : () => execFileSync("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
7865
- const killProcess = args.killProcess ?? (isolated ? (pid, signal) => {
7866
- hardBoundedExecFile(
7867
- "docker",
7868
- ["exec", `agt-${codeName}`, "kill", signal === "SIGKILL" ? "-KILL" : "-TERM", String(pid)],
7869
- { timeoutMs: 8e3, log: log2 }
7870
- );
7871
- } : (pid, signal) => {
7872
- try {
7873
- process.kill(pid, signal);
7874
- } catch {
7912
+ if (this.provisioningMax < MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING) {
7913
+ this.quarantineOrderingClamped = {
7914
+ requested: this.provisioningMax,
7915
+ applied: MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING,
7916
+ quarantineThreshold: BIND_FAILURE_QUARANTINE_THRESHOLD
7917
+ };
7918
+ this.provisioningMax = MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING;
7875
7919
  }
7876
- });
7877
- const isAlive = args.isAlive ?? (isolated ? (pid) => {
7878
- const r = hardBoundedExecFile(
7879
- "docker",
7880
- ["exec", `agt-${codeName}`, "kill", "-0", String(pid)],
7881
- { timeoutMs: 8e3, log: log2 }
7882
- );
7883
- return livenessFromProbe(
7884
- r,
7885
- (why) => log2(
7886
- `[stale-mcp-reaper] liveness probe for pid ${pid} in 'agt-${codeName}' ${why} \u2014 treating as ALIVE (no reap) rather than guessing`
7887
- )
7888
- );
7889
- } : (pid) => {
7890
- try {
7891
- process.kill(pid, 0);
7892
- return true;
7893
- } catch {
7894
- return false;
7920
+ if (!Number.isFinite(this.provisioningWindowMs) || this.provisioningWindowMs < 1e3) {
7921
+ throw new Error("restart-breaker provisioningWindowMs must be a finite number >= 1000");
7895
7922
  }
7896
- });
7897
- let psOutput;
7898
- try {
7899
- psOutput = runPs();
7900
- } catch (err) {
7901
- log2(`[stale-mcp-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping reap`);
7902
- return [];
7903
- }
7904
- const rows = parsePsRows(psOutput);
7905
- const targets = findMcpChildrenForAgent({ rows, codeName, serverKeys, mcpJson, inContainer: isolated });
7906
- if (targets.length === 0) return [];
7907
- const byPid = new Map(rows.map((r) => [r.pid, r]));
7908
- const describe = (pid) => {
7909
- const argv = byPid.get(pid)?.args ?? "";
7910
- const pkgMatch = argv.match(/(@[a-z0-9_-]+\/[a-z0-9_-]+|[a-z0-9_-]+-mcp-server|[a-z0-9_-]+-mcp\b)/i);
7911
- return pkgMatch ? `${pkgMatch[1]} (pid ${pid})` : `pid ${pid}`;
7912
- };
7913
- log2(
7914
- `[stale-mcp-reaper] '${codeName}': rotating ${targets.length} stale MCP child(ren) for [${serverKeys.join(", ")}]: ${targets.map(describe).join(", ")}`
7915
- );
7916
- for (const pid of targets) {
7917
- killProcess(pid, "SIGTERM");
7918
- }
7919
- const killedKeys = /* @__PURE__ */ new Set();
7920
- for (const pid of targets) {
7921
- const argv = byPid.get(pid)?.args ?? "";
7922
- for (const key of serverKeys) {
7923
- const entry = mcpJson?.mcpServers?.[key];
7924
- const matchers = buildArgvMatchersForEntry(key, entry);
7925
- if (matchers.some((re) => re.test(argv))) killedKeys.add(key);
7923
+ if (!Number.isFinite(this.scheduledMax) || this.scheduledMax < 1) {
7924
+ throw new Error("restart-breaker scheduledMax must be a finite number >= 1");
7926
7925
  }
7927
- }
7928
- if (killedKeys.size > 0) recordStaleRotation(codeName, killedKeys);
7929
- setTimeout(() => {
7930
- try {
7931
- let freshPsOutput;
7932
- try {
7933
- freshPsOutput = runPs();
7934
- } catch (err) {
7935
- log2(`[stale-mcp-reaper] '${codeName}': fresh ps for SIGKILL re-verify failed: ${err.message} \u2014 skipping SIGKILL pass`);
7936
- return;
7937
- }
7938
- const stillOwned = new Set(
7939
- findMcpChildrenForAgent({
7940
- rows: parsePsRows(freshPsOutput),
7941
- codeName,
7942
- serverKeys,
7943
- mcpJson,
7944
- inContainer: isolated
7945
- })
7946
- );
7947
- const stragglers = targets.filter((pid) => isAlive(pid) && stillOwned.has(pid));
7948
- if (stragglers.length === 0) return;
7949
- log2(
7950
- `[stale-mcp-reaper] '${codeName}': ${stragglers.length} child(ren) survived SIGTERM; sending SIGKILL: ${stragglers.map(describe).join(", ")}`
7951
- );
7952
- for (const pid of stragglers) {
7953
- killProcess(pid, "SIGKILL");
7954
- }
7955
- } catch (err) {
7956
- log2(`[stale-mcp-reaper] '${codeName}': error in SIGKILL pass: ${err.message}`);
7926
+ if (!Number.isFinite(this.scheduledWindowMs) || this.scheduledWindowMs < 1e3) {
7927
+ throw new Error("restart-breaker scheduledWindowMs must be a finite number >= 1000");
7957
7928
  }
7958
- }, graceMs).unref();
7959
- return targets;
7960
- }
7961
-
7962
- // src/lib/mcp-presence-reaper.ts
7963
- import { execFileSync as execFileSync2 } from "child_process";
7964
- var DEFAULT_COLD_START_GRACE_MS = 9e4;
7965
- var MAX_PRESENCE_RESTART_ATTEMPTS = 3;
7966
- var BACKOFF_AFTER_RESTARTS = MAX_PRESENCE_RESTART_ATTEMPTS;
7967
- var BACKOFF_BASE_MS = 6e4;
7968
- var BACKOFF_MAX_MS = 30 * 6e4;
7969
- var BACKOFF_WINDOW_MS = 60 * 6e4;
7970
- var BACKOFF_GIVE_UP_RESTARTS = 2 * MAX_PRESENCE_RESTART_ATTEMPTS;
7971
- var BACKOFF_STABLE_RESET_MS = 5 * 6e4;
7972
- var DEAD_CHANNEL_MS = 15 * 6e4;
7973
- var presenceReaperState = /* @__PURE__ */ new Map();
7974
- var MAX_ROTATION_PROMPT_PROBES = 3;
7975
- var ROTATION_PROMPT_PROBE_INTERVAL_MS = 3e4;
7976
- var rotationStallState = /* @__PURE__ */ new Map();
7977
- function stateKey(codeName, serverKey) {
7978
- return `${codeName}\0${serverKey}`;
7979
- }
7980
- function clearPresenceReaperState(codeName) {
7981
- const prefix = `${codeName}\0`;
7982
- for (const key of presenceReaperState.keys()) {
7983
- if (key.startsWith(prefix)) presenceReaperState.delete(key);
7984
- }
7985
- }
7986
- function clearPresenceReaperStateForKeys(codeName, keys) {
7987
- for (const key of keys) {
7988
- presenceReaperState.delete(stateKey(codeName, key));
7989
- rotationStallState.delete(stateKey(codeName, key));
7990
- }
7991
- }
7992
- function givenUpMcpServerKeys(codeName) {
7993
- const prefix = `${codeName}\0`;
7994
- const out = /* @__PURE__ */ new Set();
7995
- for (const [key, st] of presenceReaperState) {
7996
- if (key.startsWith(prefix) && (st.gaveUp || st.attempts >= MAX_PRESENCE_RESTART_ATTEMPTS)) {
7997
- out.add(key.slice(prefix.length));
7929
+ if (!Number.isFinite(this.operatorCommandMax) || this.operatorCommandMax < 1) {
7930
+ throw new Error("restart-breaker operatorCommandMax must be a finite number >= 1");
7931
+ }
7932
+ if (!Number.isFinite(this.operatorCommandWindowMs) || this.operatorCommandWindowMs < 1e3) {
7933
+ throw new Error("restart-breaker operatorCommandWindowMs must be a finite number >= 1000");
7934
+ }
7935
+ const minimumGraceCeiling = Math.max(this.provisioningMax, this.operatorMax);
7936
+ if (this.newAgentGraceCeiling < minimumGraceCeiling) {
7937
+ this.newAgentGraceCeiling = minimumGraceCeiling;
7998
7938
  }
7939
+ this.retentionMs = Math.max(
7940
+ this.windowMs,
7941
+ this.provisioningWindowMs,
7942
+ this.scheduledWindowMs,
7943
+ this.operatorCommandWindowMs
7944
+ );
7999
7945
  }
8000
- return out;
8001
- }
8002
- function findMissingMcpServers(args) {
8003
- const { rows, codeName, mcpJson, inContainer } = args;
8004
- const servers = mcpJson?.mcpServers ?? {};
8005
- const declared = Object.keys(servers);
8006
- if (declared.length === 0) return [];
8007
- const missing = [];
8008
- for (const key of declared) {
8009
- const entry = servers[key];
8010
- const isStdio = typeof entry?.command === "string";
8011
- if (!isStdio) continue;
8012
- const live = findMcpChildrenForAgent({
8013
- rows,
8014
- codeName,
8015
- serverKeys: [key],
8016
- mcpJson,
8017
- inContainer
8018
- });
8019
- if (live.length === 0) missing.push(key);
7946
+ /**
7947
+ * ENG-8215: the clamp record, if the configured provisioning max had to be
7948
+ * raised to keep MCP quarantine ahead of the breaker. `undefined` when the
7949
+ * configured value already satisfied the ordering.
7950
+ */
7951
+ getQuarantineOrderingClamp() {
7952
+ return this.quarantineOrderingClamped;
8020
7953
  }
8021
- return missing;
8022
- }
8023
- function reapMissingMcpSessions(args) {
8024
- const {
8025
- log: log2,
8026
- codeName,
8027
- mcpJson,
8028
- sessionStartedAt,
8029
- restartPending = false,
8030
- stopSession,
8031
- graceMs = DEFAULT_COLD_START_GRACE_MS,
8032
- rotationGraceMs = DEFAULT_ROTATION_GRACE_MS,
8033
- onGiveUp,
8034
- onRestart,
8035
- // ENG-8225: unwired callbacks make the prompt-probe path a strict no-op —
8036
- // `onRotationRespawnStalled` absent means "not armed", which falls through to
8037
- // the pre-ENG-8225 restart. Legacy callers and tests are unchanged.
8038
- onRotationRespawnStalled,
8039
- onRotationRespawnGaveUp,
8040
- onRotationRespawnOutcome,
8041
- maxRotationPromptProbes = MAX_ROTATION_PROMPT_PROBES,
8042
- rotationPromptProbeIntervalMs = ROTATION_PROMPT_PROBE_INTERVAL_MS,
8043
- // ENG-5932: defaults make quarantine a strict no-op (classify everything
8044
- // essential, mode off) so legacy callers/tests are unaffected.
8045
- classifyKey = () => "essential",
8046
- quarantineDwellMs = 0,
8047
- deadChannelMs = DEAD_CHANNEL_MS,
8048
- quarantineMode = "off",
8049
- onQuarantine,
8050
- isolated = false,
8051
- // ENG-8013: default false → a "no live child" observation is only excluded
8052
- // from the give-up/quarantine budget when the caller positively identifies
8053
- // the current generation as externally-churned. Legacy callers/tests unchanged.
8054
- currentGenerationExternallyStarted = false,
8055
- // ENG-9932: unwired => strict no-op, restart proceeds exactly as before.
8056
- readPane,
8057
- onPaneBlocked
8058
- } = args;
8059
- const now = args.now ?? Date.now;
8060
- const runPs = args.runPs ?? (isolated ? (
8061
- // ENG-9781: HARD-bounded, and the distinction is the whole ticket. The
8062
- // previous `execFileSync(..., { timeout: 8_000 })` bounded nothing —
8063
- // Node fires SIGTERM at the deadline and then keeps waiting, so a
8064
- // docker CLI talking to a wedged daemon blocked the manager-worker's
8065
- // event loop and stopped scheduling for EVERY agent on the host
8066
- // (33m46s, twice, on 2026-09-01). This call runs once per isolated
8067
- // agent per poll cycle, so it is the single worst place on the manager
8068
- // for an unbounded subprocess.
8069
- //
8070
- // Throwing on a non-answer is deliberate rather than lazy: the caller
8071
- // already catches, logs and SKIPS the agent for the cycle, which is
8072
- // exactly right. A timeout must not become an empty process list —
8073
- // that reads as "every declared MCP server is missing" and restarts the
8074
- // session, turning a docker hiccup into a fleet-wide restart storm.
8075
- () => {
8076
- const r = hardBoundedExecFile(
8077
- "docker",
8078
- ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
8079
- { timeoutMs: 8e3, log: log2 }
8080
- );
8081
- if (r.kind === "ok") return r.stdout;
8082
- throw new Error(
8083
- r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
8084
- );
7954
+ /**
7955
+ * ENG-7891 slice 5b: the DURABLE key for this agent's events/trips entries.
7956
+ * Resolves the code_name to the runtime key (agent_id on id-keyed hosts) and,
7957
+ * when they differ, lazily migrates a legacy code_name-keyed entry onto it — so
7958
+ * a trip persisted under the old key by a pre-arming manager (or from before a
7959
+ * codename rename) is found under the new key instead of being lost. Identity on
7960
+ * the legacy fleet ⇒ from === to ⇒ no migration and no behaviour change. Both
7961
+ * maps migrate together so events and a trip for the same agent never split
7962
+ * across keys mid-generation (a migration can flip the resolver mid-process).
7963
+ *
7964
+ * BOUNDARY — a pure-LEGACY rename (a host that was never armed) still strands a
7965
+ * held trip, exactly as it did before slice 5b: with no symlink there is no
7966
+ * agent_id to bridge the old codename to the new, so the runtime key stays the
7967
+ * (new) codename and the trip persisted under the old codename is not found.
7968
+ * That is unchanged, pre-existing behaviour and it is benign — the fresh
7969
+ * generation re-evaluates the agent under its new name and a still-pathological
7970
+ * agent simply re-trips within the window. The rename-survival guarantee this
7971
+ * slice adds is an id-keyed-only, post-arming property (the symlink is the
7972
+ * bridge); it is deliberately NOT retrofitted onto the legacy path.
7973
+ */
7974
+ keyFor(codeName) {
7975
+ const key = this.runtimeKey(codeName);
7976
+ if (key !== codeName) {
7977
+ migrateMapKey(this.events, codeName, key);
7978
+ migrateMapKey(this.trips, codeName, key);
8085
7979
  }
8086
- ) : () => execFileSync2("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
8087
- if (!mcpJson?.mcpServers) {
8088
- return { missing: [], restarted: false, reason: "no-mcp-json" };
7980
+ return key;
8089
7981
  }
8090
- const declaredCount = Object.keys(mcpJson.mcpServers).length;
8091
- if (declaredCount === 0) {
8092
- return { missing: [], restarted: false, reason: "no-declared-servers" };
7982
+ /** True if this agent's breaker is currently tripped (manager must skip spawn). */
7983
+ isTripped(codeName) {
7984
+ return this.trips.has(this.keyFor(codeName));
8093
7985
  }
8094
- if (restartPending) {
8095
- return { missing: [], restarted: false, reason: "restart-pending" };
8096
- }
8097
- if (sessionStartedAt === null) {
8098
- return { missing: [], restarted: false, reason: "session-start-unknown" };
8099
- }
8100
- if (now() - sessionStartedAt < graceMs) {
8101
- return { missing: [], restarted: false, reason: "cold-start" };
8102
- }
8103
- let psOutput;
8104
- try {
8105
- psOutput = runPs();
8106
- } catch (err) {
8107
- log2(`[mcp-presence-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping`);
8108
- return { missing: [], restarted: false, reason: "healthy" };
7986
+ getTrip(codeName) {
7987
+ return this.trips.get(this.keyFor(codeName));
8109
7988
  }
8110
- const rows = parsePsRows(psOutput);
8111
- const missingRaw = findMissingMcpServers({ rows, codeName, mcpJson, inContainer: isolated });
8112
- const rotationGraced = [];
8113
- const rotationProbeArmed = [];
8114
- const rotationProbeWaiting = [];
8115
- const rotationProbeExhausted = [];
8116
- const rotationRespawnOutcomes = {};
8117
- const recordRotationOutcome = (key, outcome) => {
8118
- rotationRespawnOutcomes[key] = outcome;
8119
- if (!onRotationRespawnOutcome) return;
8120
- try {
8121
- onRotationRespawnOutcome(codeName, key, outcome);
8122
- } catch (err) {
8123
- log2(
8124
- `[mcp-presence-reaper] onRotationRespawnOutcome callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8125
- );
8126
- }
8127
- };
8128
- const missing = [];
8129
- for (const key of missingRaw) {
8130
- const rotation = classifyRotation(codeName, key, rotationGraceMs, now);
8131
- if (rotation === "within-grace") {
8132
- rotationGraced.push(key);
8133
- continue;
8134
- }
8135
- if (rotation === "none") {
8136
- rotationStallState.delete(stateKey(codeName, key));
8137
- missing.push(key);
8138
- continue;
8139
- }
8140
- const sk = stateKey(codeName, key);
8141
- const rotationRecordedAt = getRotationRecordedAt(codeName, key) ?? now();
8142
- const priorStall = rotationStallState.get(sk);
8143
- const stall = priorStall && priorStall.rotationRecordedAt === rotationRecordedAt ? priorStall : {
8144
- rotationRecordedAt,
8145
- probeAttempts: 0,
8146
- lastProbeAt: null,
8147
- escalated: false
8148
- };
8149
- if (priorStall && priorStall !== stall) {
8150
- log2(
8151
- `[mcp-presence-reaper] '${codeName}:${key}' discarding stall state from a previous rotation (rotation=${priorStall.rotationRecordedAt}, probes=${priorStall.probeAttempts}, escalated=${priorStall.escalated}) \u2014 this is a NEW rotation (rotation=${rotationRecordedAt}) and gets a full prompt-probe budget (ENG-8225)`
8152
- );
8153
- }
8154
- rotationStallState.set(sk, stall);
8155
- if (stall.escalated) {
8156
- missing.push(key);
8157
- continue;
8158
- }
8159
- const spacingElapsed = stall.lastProbeAt === null || now() - stall.lastProbeAt >= rotationPromptProbeIntervalMs;
8160
- if (!spacingElapsed) {
8161
- rotationProbeWaiting.push(key);
8162
- continue;
8163
- }
8164
- if (stall.probeAttempts >= maxRotationPromptProbes) {
8165
- stall.escalated = true;
8166
- rotationProbeExhausted.push(key);
8167
- recordRotationOutcome(key, "never-respawned");
8168
- log2(
8169
- `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=never-respawned \u2014 child SIGTERM'd by stale-mcp-reaper never returned after ${stall.probeAttempts} prompt bind-probe(s); escalating (integration marked unhealthy + reaper_action alert) and releasing to the normal restart path (ENG-8225)`
8170
- );
8171
- if (onRotationRespawnGaveUp) {
8172
- try {
8173
- onRotationRespawnGaveUp(codeName, key, { attempts: stall.probeAttempts });
8174
- } catch (err) {
8175
- log2(
8176
- `[mcp-presence-reaper] onRotationRespawnGaveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8177
- );
8178
- }
8179
- }
8180
- missing.push(key);
8181
- continue;
8182
- }
8183
- const attempt = stall.probeAttempts + 1;
8184
- let armed = false;
8185
- if (onRotationRespawnStalled) {
8186
- try {
8187
- armed = onRotationRespawnStalled(codeName, key, {
8188
- attempt,
8189
- maxAttempts: maxRotationPromptProbes
8190
- });
8191
- } catch (err) {
8192
- log2(
8193
- `[mcp-presence-reaper] onRotationRespawnStalled callback threw for '${codeName}:${key}' (suppressed; falling back to the pre-ENG-8225 restart path): ${err.message}`
8194
- );
8195
- armed = false;
8196
- }
8197
- }
8198
- if (!armed) {
8199
- missing.push(key);
8200
- continue;
7989
+ /**
7990
+ * Record a restart event. If recording this event puts the count
7991
+ * inside the window above `max`, the breaker trips and the call site
7992
+ * should NOT respawn.
7993
+ *
7994
+ * Idempotent on already-tripped breakers: returns the existing trip
7995
+ * without double-counting events. Callers may still record the reason
7996
+ * via the decision-log for forensics.
7997
+ */
7998
+ record(codeName, reason, integrationKey) {
7999
+ const key = this.keyFor(codeName);
8000
+ const existing = this.trips.get(key);
8001
+ if (existing) {
8002
+ const bucket = maxProvisioningBucket(existing.eventsAtTrip);
8003
+ return {
8004
+ tripped: false,
8005
+ trip: existing,
8006
+ windowCount: existing.eventsAtTrip.length,
8007
+ classCounts: countByClass(existing.eventsAtTrip),
8008
+ maxProvisioningBucket: bucket.count,
8009
+ maxProvisioningBucketKey: bucket.key
8010
+ };
8201
8011
  }
8202
- stall.probeAttempts = attempt;
8203
- stall.lastProbeAt = now();
8204
- rotationProbeArmed.push(key);
8205
- log2(
8206
- `[mcp-presence-reaper] '${codeName}:${key}' rotation grace expired (${rotationGraceMs}ms) with no live child and no tool call to respawn it \u2014 armed prompt bind-probe ${attempt}/${maxRotationPromptProbes} (ENG-7429) instead of restarting the session (ENG-8225)`
8207
- );
8208
- }
8209
- if (rotationGraced.length > 0) {
8210
- log2(
8211
- `[mcp-presence-reaper] '${codeName}': skipping [${rotationGraced.join(", ")}] within rotation grace window (${rotationGraceMs}ms) \u2014 child(ren) just SIGTERM'd by stale-mcp-reaper, awaiting respawn on next tool call (ENG-5344)`
8012
+ const at = this.now();
8013
+ const retentionCutoff = at - this.retentionMs;
8014
+ const prior = (this.events.get(key) ?? []).filter((e) => e.at >= retentionCutoff);
8015
+ const bucketed = isIntegrationBucketedReason(reason);
8016
+ prior.push(bucketed && integrationKey ? { reason, at, integrationKey } : { reason, at });
8017
+ this.events.set(key, prior);
8018
+ const crashCount = prior.filter(
8019
+ (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8020
+ ).length;
8021
+ const provEvents = prior.filter(
8022
+ (e) => e.at >= at - this.provisioningWindowMs && isProvisioningReloadReason(e.reason)
8212
8023
  );
8213
- }
8214
- if (rotationProbeWaiting.length > 0) {
8215
- log2(
8216
- `[mcp-presence-reaper] '${codeName}': skipping [${rotationProbeWaiting.join(", ")}] \u2014 prompt bind-probe already armed for the stalled rotation, waiting out the ${rotationPromptProbeIntervalMs}ms re-arm interval (ENG-8225)`
8024
+ const credRotCount = prior.filter(
8025
+ (e) => e.at >= at - this.provisioningWindowMs && isCredentialRotationReason(e.reason)
8026
+ ).length;
8027
+ const selfHealCount = prior.filter(
8028
+ (e) => e.at >= at - this.provisioningWindowMs && isSelfHealingReason(e.reason)
8029
+ ).length;
8030
+ const operatorEvents = prior.filter(
8031
+ (e) => e.at >= at - this.provisioningWindowMs && isOperatorConfigReason(e.reason)
8217
8032
  );
8218
- }
8219
- const declared = mcpJson.mcpServers ?? {};
8220
- const liveKeys = /* @__PURE__ */ new Set();
8221
- for (const key of Object.keys(declared)) {
8222
- const entry = declared[key];
8223
- if (typeof entry?.command !== "string") continue;
8224
- if (!missingRaw.includes(key)) liveKeys.add(key);
8225
- }
8226
- for (const key of liveKeys) {
8227
- const sk = stateKey(codeName, key);
8228
- if (classifyRotation(codeName, key, rotationGraceMs, now) === "none") {
8229
- rotationStallState.delete(sk);
8230
- continue;
8231
- }
8232
- const stall = rotationStallState.get(sk);
8233
- const viaProbe = (stall?.probeAttempts ?? 0) > 0;
8234
- recordRotationOutcome(
8235
- key,
8236
- viaProbe ? "respawned-after-prompt-probe" : "respawned-on-tool-call"
8033
+ const scheduledEvents = prior.filter(
8034
+ (e) => e.at >= at - this.scheduledWindowMs && isScheduledRestartReason(e.reason)
8237
8035
  );
8238
- log2(
8239
- viaProbe ? `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-after-prompt-probe \u2014 child is live again after ${stall?.probeAttempts} prompt bind-probe(s)${stall?.escalated ? " (recovered AFTER escalation \u2014 the earlier never-respawned verdict stands for that window)" : ""}, no session restart needed (ENG-8225)` : `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-on-tool-call \u2014 child returned on the agent's own next tool call, ENG-5344's assumption held (ENG-8225)`
8036
+ const operatorCommandEvents = prior.filter(
8037
+ (e) => e.at >= at - this.operatorCommandWindowMs && isOperatorCommandReason(e.reason)
8240
8038
  );
8241
- clearStaleRotation(codeName, [key]);
8242
- rotationStallState.delete(sk);
8243
- }
8244
- for (const key of liveKeys) {
8245
- const state = presenceReaperState.get(stateKey(codeName, key));
8246
- if (state) {
8247
- state.attempts = 0;
8248
- state.lastSeenLiveAt = now();
8249
- state.lastAttemptedSessionStartedAt = null;
8250
- state.gaveUpLogged = false;
8251
- state.firstMissingAt = null;
8252
- state.quarantineLogged = false;
8253
- state.continuouslyMissingSince = null;
8254
- state.deadChannelLogged = false;
8255
- state.quarantineWithheldLogged = false;
8256
- state.gaveUp = false;
8257
- if (state.stableLiveSince === null) state.stableLiveSince = now();
8258
- if (now() - state.stableLiveSince >= BACKOFF_STABLE_RESET_MS) {
8259
- state.restartHistory = [];
8260
- }
8261
- }
8262
- }
8263
- if (missing.length === 0) {
8264
- return {
8265
- missing,
8266
- restarted: false,
8267
- reason: "healthy",
8268
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8269
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8270
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8271
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8272
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
8273
- };
8274
- }
8275
- const nowMs = now();
8276
- const givenUp = [];
8277
- const active = [];
8278
- const backingOff = [];
8279
- const newGenerationKeys = /* @__PURE__ */ new Set();
8280
- const quarantined = [];
8281
- const wouldQuarantine = [];
8282
- const quarantineWithheld = [];
8283
- const anyStdioServerLive = liveKeys.size > 0;
8284
- const genuinelyDead = [];
8285
- for (const key of missing) {
8286
- const sk = stateKey(codeName, key);
8287
- const state = presenceReaperState.get(sk) ?? {
8288
- attempts: 0,
8289
- lastSeenLiveAt: null,
8290
- lastAttemptedSessionStartedAt: null,
8291
- gaveUpLogged: false,
8292
- firstMissingAt: null,
8293
- quarantineLogged: false,
8294
- restartHistory: [],
8295
- stableLiveSince: null,
8296
- continuouslyMissingSince: null,
8297
- deadChannelLogged: false,
8298
- quarantineWithheldLogged: false,
8299
- gaveUp: false
8300
- };
8301
- if (state.continuouslyMissingSince === null) {
8302
- state.continuouslyMissingSince = nowMs;
8303
- }
8304
- const isGenuinelyDead = nowMs - state.continuouslyMissingSince >= deadChannelMs;
8305
- const suppressAsChurn = currentGenerationExternallyStarted && !isGenuinelyDead;
8306
- if (isGenuinelyDead) {
8307
- genuinelyDead.push(key);
8308
- if (currentGenerationExternallyStarted && !state.deadChannelLogged) {
8309
- state.deadChannelLogged = true;
8310
- log2(
8311
- `[mcp-presence-reaper] '${codeName}:${key}' continuously missing ${Math.round((nowMs - state.continuouslyMissingSince) / 6e4)}m (> ${Math.round(deadChannelMs / 6e4)}m) \u2014 genuinely dead, overriding storm-awareness so it can reach give-up/quarantine instead of churning (ENG-8062)`
8312
- );
8313
- }
8314
- }
8315
- if (state.firstMissingAt === null || suppressAsChurn) {
8316
- state.firstMissingAt = nowMs;
8317
- }
8318
- state.stableLiveSince = null;
8319
- if (state.lastAttemptedSessionStartedAt !== sessionStartedAt) {
8320
- state.lastAttemptedSessionStartedAt = sessionStartedAt;
8321
- if (!suppressAsChurn) {
8322
- state.attempts += 1;
8323
- newGenerationKeys.add(key);
8324
- }
8325
- }
8326
- state.restartHistory = state.restartHistory.filter((t) => nowMs - t < BACKOFF_WINDOW_MS);
8327
- const restartsInWindow = state.restartHistory.length;
8328
- presenceReaperState.set(sk, state);
8329
- const overAttemptsCap = state.attempts > MAX_PRESENCE_RESTART_ATTEMPTS;
8330
- const overRateCap = restartsInWindow >= BACKOFF_GIVE_UP_RESTARTS;
8331
- if (overAttemptsCap || overRateCap) {
8332
- givenUp.push(key);
8333
- state.gaveUp = true;
8334
- if (!state.gaveUpLogged) {
8335
- const cause = overAttemptsCap ? `${MAX_PRESENCE_RESTART_ATTEMPTS} consecutive failed restarts \u2014 declared but never recovers (likely orphaned config; see ENG-5279)` : `${restartsInWindow} restarts within ${Math.round(BACKOFF_WINDOW_MS / 6e4)}m \u2014 persistently flapping (crash-loop; ENG-6480)`;
8336
- log2(
8337
- `[mcp-presence-reaper] giving up on '${codeName}:${key}' after ${cause}`
8338
- );
8339
- state.gaveUpLogged = true;
8340
- if (onGiveUp) {
8341
- try {
8342
- onGiveUp(codeName, key);
8343
- } catch (err) {
8344
- log2(
8345
- `[mcp-presence-reaper] onGiveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8346
- );
8347
- }
8348
- }
8349
- }
8350
- if (quarantineMode !== "off" && !state.quarantineLogged && classifyKey(key) === "optional") {
8351
- const dwellElapsed = quarantineDwellMs <= 0 || state.firstMissingAt !== null && nowMs - state.firstMissingAt >= quarantineDwellMs;
8352
- if (dwellElapsed && !anyStdioServerLive) {
8353
- quarantineWithheld.push(key);
8354
- if (!state.quarantineWithheldLogged) {
8355
- state.quarantineWithheldLogged = true;
8356
- log2(
8357
- `[mcp-presence-reaper] '${codeName}:${key}' given up but NOT quarantined - no declared stdio MCP server is live, so the restart budget was exhausted by a session that cannot spawn anything, not by this channel dying. Check the pane (tmux attach -t agt-${codeName}); the channel stays provisioned and will quarantine on a later tick if it is still missing once the session is up (ENG-10500)`
8358
- );
8359
- }
8360
- } else if (dwellElapsed) {
8361
- state.quarantineLogged = true;
8362
- if (quarantineMode === "enforce") {
8363
- quarantined.push(key);
8364
- log2(
8365
- `[mcp-presence-reaper] QUARANTINE '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; dropping from provisioned set so the dead channel stops restarting the whole session (ENG-5932)`
8366
- );
8367
- if (onQuarantine) {
8368
- try {
8369
- onQuarantine(codeName, key);
8370
- } catch (err) {
8371
- log2(
8372
- `[mcp-presence-reaper] onQuarantine callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8373
- );
8374
- }
8375
- }
8376
- } else {
8377
- wouldQuarantine.push(key);
8378
- log2(
8379
- `[mcp-presence-reaper] SHADOW would-quarantine '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; no action taken (ENG-5932 shadow mode)`
8380
- );
8381
- }
8382
- }
8383
- }
8384
- } else {
8385
- const backoffMs = restartsInWindow < BACKOFF_AFTER_RESTARTS ? 0 : Math.min(
8386
- BACKOFF_BASE_MS * 2 ** (restartsInWindow - BACKOFF_AFTER_RESTARTS),
8387
- BACKOFF_MAX_MS
8388
- );
8389
- const lastRestartAt = state.restartHistory[state.restartHistory.length - 1] ?? null;
8390
- if (lastRestartAt !== null && nowMs - lastRestartAt < backoffMs) {
8391
- backingOff.push(key);
8392
- } else {
8393
- active.push(key);
8394
- }
8395
- }
8396
- }
8397
- if (active.length === 0) {
8398
- const reason = backingOff.length > 0 ? "backoff" : "all-keys-over-cap";
8399
- if (backingOff.length > 0) {
8400
- log2(
8401
- `[mcp-presence-reaper] '${codeName}': holding off restart for [${backingOff.join(", ")}] \u2014 escalating backoff interval not yet elapsed (ENG-6480)${givenUp.length > 0 ? ` (given up: [${givenUp.join(", ")}])` : ""}`
8402
- );
8403
- }
8404
- return {
8405
- missing,
8406
- restarted: false,
8407
- reason,
8408
- givenUp,
8409
- backingOff: backingOff.length > 0 ? backingOff : void 0,
8410
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8411
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8412
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8413
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8414
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
8415
- quarantined: quarantined.length > 0 ? quarantined : void 0,
8416
- wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
8417
- genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
8418
- quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
8039
+ const windowCount = prior.filter((e) => e.at >= at - this.windowMs).length;
8040
+ const classCounts = {
8041
+ crash: crashCount,
8042
+ provisioning: provEvents.length,
8043
+ "credential-rotation": credRotCount,
8044
+ "self-healing": selfHealCount,
8045
+ "operator-config": operatorEvents.length,
8046
+ scheduled: scheduledEvents.length,
8047
+ "operator-command": operatorCommandEvents.length
8419
8048
  };
8420
- }
8421
- if (readPane) {
8422
- let pane = null;
8423
- try {
8424
- pane = readPane(codeName);
8425
- } catch (err) {
8426
- log2(
8427
- `[mcp-presence-reaper] '${codeName}': pane read threw (${err.message}) \u2014 proceeding with restart (ENG-9932 fails open)`
8049
+ const scheduledLoop = scheduledEvents.length > this.scheduledMax ? {
8050
+ count: scheduledEvents.length,
8051
+ windowMs: this.scheduledWindowMs,
8052
+ reason: scheduledEvents[scheduledEvents.length - 1].reason
8053
+ } : void 0;
8054
+ const operatorCommandLoop = operatorCommandEvents.length > this.operatorCommandMax ? {
8055
+ count: operatorCommandEvents.length,
8056
+ windowMs: this.operatorCommandWindowMs,
8057
+ reason: operatorCommandEvents[operatorCommandEvents.length - 1].reason
8058
+ } : void 0;
8059
+ const provBucket = maxProvisioningBucket(provEvents);
8060
+ const operatorBucket = maxBucketWhere(operatorEvents, isOperatorConfigReason);
8061
+ const createdAt = this.newAgentGraceMs > 0 ? this.agentCreatedAt(codeName) : void 0;
8062
+ const age = typeof createdAt === "number" && Number.isFinite(createdAt) ? at - createdAt : void 0;
8063
+ const newAgentGraceActive = age !== void 0 && age >= 0 && age < this.newAgentGraceMs;
8064
+ const newAgentGraceExpiresAt = newAgentGraceActive && typeof createdAt === "number" ? createdAt + this.newAgentGraceMs : void 0;
8065
+ const graceCeilingEvents = newAgentGraceActive ? prior.filter(
8066
+ (e) => e.at >= at - this.provisioningWindowMs && (isProvisioningReloadReason(e.reason) || isOperatorConfigReason(e.reason))
8067
+ ) : [];
8068
+ const graceCeilingExceeded = newAgentGraceActive && graceCeilingEvents.length > this.newAgentGraceCeiling;
8069
+ let trippedClass;
8070
+ let tripEvents;
8071
+ let tripWindowMs = this.windowMs;
8072
+ let tripBucketKey = "";
8073
+ if (crashCount > this.max) {
8074
+ trippedClass = "crash";
8075
+ tripEvents = prior.filter(
8076
+ (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8428
8077
  );
8078
+ tripWindowMs = this.windowMs;
8079
+ } else if (newAgentGraceActive && graceCeilingExceeded) {
8080
+ trippedClass = "provisioning";
8081
+ tripEvents = graceCeilingEvents;
8082
+ tripWindowMs = this.provisioningWindowMs;
8083
+ tripBucketKey = "";
8084
+ } else if (newAgentGraceActive) {
8085
+ } else if (operatorBucket.count > this.operatorMax) {
8086
+ trippedClass = "operator-config";
8087
+ tripEvents = operatorEvents.filter((e) => (e.integrationKey ?? "") === operatorBucket.key);
8088
+ tripWindowMs = this.provisioningWindowMs;
8089
+ tripBucketKey = operatorBucket.key;
8090
+ } else if (provBucket.count > this.provisioningMax) {
8091
+ trippedClass = "provisioning";
8092
+ tripEvents = provEvents.filter((e) => (e.integrationKey ?? "") === provBucket.key);
8093
+ tripWindowMs = this.provisioningWindowMs;
8094
+ tripBucketKey = provBucket.key;
8429
8095
  }
8430
- const blocked = classifyUnanswerablePane(pane);
8431
- if (blocked) {
8432
- const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
8433
- log2(
8434
- `[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 session is blocked on an interactive prompt (${blocked}), so declared MCP(s) [${active.join(", ")}] were never spawned rather than having died. A restart re-presents the same prompt and burns the ENG-5441 breaker budget. Needs a human at the pane: tmux attach -t agt-${codeName}` + // Fingerprint, never pane text — see summarizeUnanswerablePane.
8435
- (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
8436
- );
8437
- if (onPaneBlocked) {
8438
- try {
8439
- onPaneBlocked(codeName, {
8440
- kind: blocked,
8441
- activeKeys: [...active],
8442
- paneHash: fingerprint?.hash ?? null,
8443
- paneLength: fingerprint?.length ?? null
8444
- });
8445
- } catch (err) {
8446
- log2(
8447
- `[mcp-presence-reaper] onPaneBlocked callback threw for '${codeName}' (suppressed): ${err.message}`
8448
- );
8449
- }
8450
- }
8451
- return {
8452
- missing,
8453
- restarted: false,
8454
- reason: "pane-blocked-on-dialog",
8455
- paneBlockedOn: blocked,
8456
- givenUp: givenUp.length > 0 ? givenUp : void 0,
8457
- backingOff: backingOff.length > 0 ? backingOff : void 0,
8458
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8459
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8460
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8461
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8462
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
8463
- quarantined: quarantined.length > 0 ? quarantined : void 0,
8464
- wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
8465
- genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
8466
- quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
8096
+ if (trippedClass && tripEvents) {
8097
+ const baseMessage = graceCeilingExceeded ? formatStatusMessage(tripEvents, tripWindowMs) : formatStatusMessage(tripEvents, tripWindowMs, trippedClass, tripBucketKey);
8098
+ const graceHours = Math.round(this.newAgentGraceMs / 36e5 * 10) / 10;
8099
+ const statusMessage = graceCeilingExceeded ? `${baseMessage} \u2014 paused DESPITE the new-agent grace: ${graceCeilingEvents.length} provisioning-shaped restarts (provisioning + operator-config) exceeded the grace ceiling of ${this.newAgentGraceCeiling} (ENG-8690). The agent is inside its ${graceHours}h grace and is doing little except restarting.` : baseMessage;
8100
+ const trip = {
8101
+ trippedAt: at,
8102
+ eventsAtTrip: [...tripEvents],
8103
+ statusMessage,
8104
+ // ENG-7577: persist the tripping class so the recovery path can gate on
8105
+ // it after a manager restart, without re-deriving it from prose.
8106
+ trippedClass
8467
8107
  };
8108
+ this.trips.set(key, trip);
8109
+ this.events.delete(key);
8110
+ return {
8111
+ tripped: true,
8112
+ trip,
8113
+ windowCount,
8114
+ classCounts,
8115
+ trippedClass,
8116
+ maxProvisioningBucket: provBucket.count,
8117
+ maxProvisioningBucketKey: provBucket.key,
8118
+ ...graceCeilingExceeded ? { newAgentGraceCeilingExceeded: true } : {},
8119
+ // ENG-9138: a co-occurring rollover loop still reports on a real trip.
8120
+ ...scheduledLoop ? { scheduledLoop } : {},
8121
+ ...operatorCommandLoop ? { operatorCommandLoop } : {}
8122
+ };
8123
+ }
8124
+ const suppressedOperator = operatorBucket.count > this.operatorMax;
8125
+ const suppressedProvisioning = !suppressedOperator && provBucket.count > this.provisioningMax;
8126
+ const wouldHaveTripped = suppressedOperator || suppressedProvisioning;
8127
+ return {
8128
+ tripped: false,
8129
+ windowCount,
8130
+ classCounts,
8131
+ maxProvisioningBucket: provBucket.count,
8132
+ maxProvisioningBucketKey: provBucket.key,
8133
+ ...scheduledLoop ? { scheduledLoop } : {},
8134
+ ...operatorCommandLoop ? { operatorCommandLoop } : {},
8135
+ ...newAgentGraceActive ? {
8136
+ newAgentGraceSuppressedTrip: wouldHaveTripped,
8137
+ ...newAgentGraceExpiresAt !== void 0 ? { newAgentGraceExpiresAt } : {},
8138
+ ...wouldHaveTripped ? {
8139
+ newAgentGraceSuppressed: {
8140
+ class: suppressedOperator ? "operator-config" : "provisioning",
8141
+ bucketKey: suppressedOperator ? operatorBucket.key : provBucket.key,
8142
+ bucketCount: suppressedOperator ? operatorBucket.count : provBucket.count,
8143
+ agentAgeMs: age ?? 0
8144
+ }
8145
+ } : {}
8146
+ } : {}
8147
+ };
8148
+ }
8149
+ /** Operator-initiated reset: drops the trip + the events log for this agent. */
8150
+ clear(codeName) {
8151
+ const key = this.keyFor(codeName);
8152
+ this.trips.delete(key);
8153
+ this.events.delete(key);
8154
+ if (key !== codeName) {
8155
+ this.trips.delete(codeName);
8156
+ this.events.delete(codeName);
8468
8157
  }
8469
8158
  }
8470
- const givenUpSuffix = givenUp.length > 0 ? ` (skipping over-cap: [${givenUp.join(", ")}])` : "";
8471
- log2(
8472
- `[mcp-presence-reaper] '${codeName}': declared MCP(s) [${active.join(", ")}] have no live children \u2014 restarting session${givenUpSuffix}`
8473
- );
8474
- if (onRestart) {
8475
- try {
8476
- const maybePromise = onRestart(codeName, { activeKeys: active, givenUpKeys: givenUp });
8477
- if (maybePromise && typeof maybePromise.then === "function") {
8478
- void maybePromise.then(void 0, (err) => {
8479
- log2(
8480
- `[mcp-presence-reaper] onRestart callback rejected for '${codeName}' (suppressed; restart proceeds): ${err.message}`
8481
- );
8482
- });
8159
+ /** Snapshot tripped agents for `manager-state.json`. */
8160
+ serialize() {
8161
+ return Object.fromEntries(this.trips.entries());
8162
+ }
8163
+ /**
8164
+ * Rehydrate trip state from `manager-state.json`. Called once at
8165
+ * worker startup. Window history is intentionally NOT persisted — only
8166
+ * tripped state. A tripped breaker survives manager restart; an
8167
+ * un-tripped one starts a fresh window in the new worker.
8168
+ */
8169
+ hydrate(saved) {
8170
+ if (!saved) return;
8171
+ for (const [codeName, trip] of Object.entries(saved)) {
8172
+ if (trip && typeof trip.trippedAt === "number" && Array.isArray(trip.eventsAtTrip)) {
8173
+ const klass = trip.trippedClass;
8174
+ this.trips.set(
8175
+ codeName,
8176
+ klass === void 0 || KNOWN_REASON_CLASSES.has(klass) ? trip : { ...trip, trippedClass: void 0 }
8177
+ );
8483
8178
  }
8484
- } catch (err) {
8485
- log2(
8486
- `[mcp-presence-reaper] onRestart callback threw for '${codeName}' (suppressed; restart proceeds): ${err.message}`
8487
- );
8488
8179
  }
8489
8180
  }
8490
- stopSession(codeName, { activeKeys: active });
8491
- for (const key of active) {
8492
- if (!newGenerationKeys.has(key)) continue;
8493
- const st = presenceReaperState.get(stateKey(codeName, key));
8494
- if (st) st.restartHistory.push(nowMs);
8181
+ /**
8182
+ * ENG-10878: how many CRASH-bar restarts `codeName` can still afford in the
8183
+ * current window.
8184
+ *
8185
+ * Exists so a caller can decline to spend the last unit of a budget it did
8186
+ * not fill. `mcp-presence-reaper` sits on this bar next to `auth-tuple-change`
8187
+ * and `model-change`, so a reaper restart landing beside an auth-tuple
8188
+ * rotation is the pair that trips it - and the agent is paused on the other
8189
+ * reason's account.
8190
+ *
8191
+ * THE ORIGINAL VERSION OF THIS PARAGRAPH NAMED THE WRONG REASONS (obi-wan
8192
+ * `486aa45fa3bd`), and it said so in three files. It claimed `day-rollover`
8193
+ * and `channel-set-change` share this bar and used "a reaper restart at 00:59
8194
+ * beside a day-rollover at 01:00" as the worked example. Neither is crash
8195
+ * class: `restartReasonClass('day-rollover')` is `'scheduled'` (ENG-9138) and
8196
+ * `'channel-set-change'` is `'provisioning'` (ENG-10510), so neither is
8197
+ * counted by the filter below and that pair cannot trip anything. The PR's
8198
+ * own new test asserts exactly that, which is the sort of contradiction that
8199
+ * should have been caught by writing the test - it was not, because the test
8200
+ * and the prose were written for different purposes an hour apart.
8201
+ *
8202
+ * The GUARD is unaffected: `auth-tuple-change` and `model-change` really are
8203
+ * on this bar, so the co-occurrence risk is real. Only the example was wrong,
8204
+ * and an argument resting on an impossible example is worth no more than the
8205
+ * example.
8206
+ *
8207
+ * The arithmetic lives HERE rather than at the call site on purpose: `max`
8208
+ * and the window are this class's business, they are env-tunable
8209
+ * (AGT_RESTART_BREAKER_MAX / _WINDOW_MS), and a caller doing
8210
+ * `max - windowCount()` itself would silently stop being correct the moment
8211
+ * either moves. Floored at 0 so an already-tripped agent reads as "nothing
8212
+ * left" rather than a negative number.
8213
+ *
8214
+ * CRASH BAR ONLY. The provisioning, operator, scheduled and operator-command
8215
+ * bars have their own maxima and windows; this answers for the default bar,
8216
+ * which is the one `mcp-presence-reaper` is counted against.
8217
+ */
8218
+ remainingCrashBudget(codeName) {
8219
+ const key = this.keyFor(codeName);
8220
+ if (this.trips.has(key)) return 0;
8221
+ const cutoff = this.now() - this.windowMs;
8222
+ const crashCount = (this.events.get(key) ?? []).filter(
8223
+ (e) => e.at >= cutoff && restartReasonClass(e.reason) === "crash"
8224
+ ).length;
8225
+ return Math.max(0, this.max - crashCount);
8495
8226
  }
8496
- return {
8497
- missing,
8498
- restarted: true,
8499
- givenUp: givenUp.length > 0 ? givenUp : void 0,
8500
- backingOff: backingOff.length > 0 ? backingOff : void 0,
8501
- rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8502
- rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8503
- rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8504
- rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8505
- rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
8506
- quarantined: quarantined.length > 0 ? quarantined : void 0,
8507
- wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
8508
- genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
8509
- quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
8510
- };
8227
+ /** Test helper — current in-window event count for `codeName`. */
8228
+ windowCount(codeName) {
8229
+ const cutoff = this.now() - this.windowMs;
8230
+ return (this.events.get(this.keyFor(codeName)) ?? []).filter((e) => e.at >= cutoff).length;
8231
+ }
8232
+ };
8233
+ function formatStatusMessage(events, windowMs, klass = "crash", integrationKey = "") {
8234
+ const last = events[events.length - 1];
8235
+ const windowLabel = windowMs < 6e4 ? `${Math.round(windowMs / 1e3)}s` : `${(windowMs / 6e4).toFixed(1).replace(/\.0$/, "")}min`;
8236
+ const reasonCounts = /* @__PURE__ */ new Map();
8237
+ for (const e of events) reasonCounts.set(e.reason, (reasonCounts.get(e.reason) ?? 0) + 1);
8238
+ const breakdown = Array.from(reasonCounts.entries()).map(([r, n]) => `${r}=${n}`).join(", ");
8239
+ const classLabel = klass === "provisioning" ? "provisioning-reload " : klass === "operator-config" ? "operator-requested config-reload " : "";
8240
+ const integrationLabel = (klass === "provisioning" || klass === "operator-config") && integrationKey ? ` for integration '${integrationKey}'` : "";
8241
+ return `Circuit breaker tripped: ${events.length} ${classLabel}restarts in ${windowLabel}${integrationLabel} (${breakdown}); most recent=${last.reason} at ${new Date(last.at).toISOString()}`;
8511
8242
  }
8512
8243
 
8513
- // src/lib/restart-breaker.ts
8514
- function reaperRestartBreakerReason(activeKeys) {
8515
- return activeKeys.length >= 2 ? "mcp-presence-reaper" : void 0;
8244
+ // src/lib/stale-mcp-reaper.ts
8245
+ import { execFileSync } from "child_process";
8246
+ function livenessFromProbe(r, onNoAnswer) {
8247
+ if (r.kind === "ok") return true;
8248
+ if (r.kind === "failed" && r.status !== null) return false;
8249
+ onNoAnswer?.(r.kind === "timed-out" ? "did not answer within its bound" : `could not run: ${r.message}`);
8250
+ return true;
8516
8251
  }
8517
- var KNOWN_REASON_CLASS_MAP = {
8518
- crash: true,
8519
- provisioning: true,
8520
- "credential-rotation": true,
8521
- // ENG-8388: provisioning reloads a human demonstrably asked for. Same
8522
- // Record-not-Set reasoning as above — a new class cannot be silently omitted.
8523
- "operator-config": true,
8524
- // ENG-8215: the self-healing class (bind-remediation - the manager's OWN
8525
- // repair attempts, which must never count toward a trip). Adding it here is
8526
- // forced by the Record type, which is exactly why that shape was chosen over
8527
- // a Set literal (ENG-7577, CodeRabbit on #3861) - a new union member cannot
8528
- // be silently omitted from the rehydration allowlist.
8529
- "self-healing": true,
8530
- // ENG-9138: the platform-scheduled class. Same Record-not-Set reasoning: the
8531
- // compiler forces every new class into this allowlist, so a `trippedClass`
8532
- // rehydrated from manager-state.json can never name a class this build does
8533
- // not know about.
8534
- scheduled: true,
8535
- // ENG-10066: the operator-command class. Same Record-not-Set reasoning again —
8536
- // the compiler forces every new class into this allowlist.
8537
- "operator-command": true
8538
- };
8539
- var KNOWN_REASON_CLASSES = new Set(
8540
- Object.keys(KNOWN_REASON_CLASS_MAP)
8541
- );
8542
- var PROVISIONING_RELOAD_REASONS = /* @__PURE__ */ new Set([
8543
- "hot-reload-mcp",
8544
- "managed-mcp-churn",
8545
- // ENG-8215: 'bind-remediation' MOVED OUT of this set into SELF_HEALING_REASONS.
8546
- // It is the manager repairing itself, not a config reload, and counting it
8547
- // meant the harder the platform tried to fix an agent the more certain it
8548
- // became that it should give up. Deliberately NOT re-added here.
8549
- // ENG-7576: the dashboard integration-add stop itself - the first restart of
8550
- // the very burst this class exists for, previously uncounted entirely.
8551
- "integration-change",
8552
- // ENG-7771: a sender_policy delivery restart is a deliberate config-delivery
8553
- // reload (operator changed the policy, or the fail-closed first-poll verify
8554
- // after a manager restart with no persisted baseline), not a crash. On the
8555
- // tight crash tally, a host whose sender-policy-baseline.json write keeps
8556
- // failing (disk full, permissions) would convert the fail-closed restart
8557
- // into repeated crash tallies across a multi-deploy day and trip the
8558
- // breaker - taking the agent DOWN, which is strictly worse than the stale
8559
- // policy the restart exists to fix. The looser provisioning tally still
8560
- // trips + pages on sustained thrash.
8561
- "sender-policy-change",
8562
- // ENG-9802: same argument, same failure mode — a host whose baseline write
8563
- // keeps failing must not convert a fail-closed policy restart into a breaker
8564
- // trip that takes the agent down.
8565
- "telegram-policy-change",
8566
- // ENG-9861: and again for standing rules. The fail-closed first-poll restart
8567
- // has exactly the same shape, so it needs the same protection from a host
8568
- // whose sidecar write keeps failing.
8569
- "agent-rules-change",
8570
- // ENG-10510 (Obi-Wan on #6313): a channel-set ADD is a config reload, not a
8571
- // crash, and it was landing on the crash tally purely by falling through.
8572
- // It is 'integration-change's exact sibling - both are "a channel or server
8573
- // was added, re-render .mcp.json and respawn" - and it was never argued for
8574
- // the crash class, it simply was not in this set.
8575
- //
8576
- // This is the ENG-9138 / ENG-10066 species verbatim: a PLATFORM-MANUFACTURED
8577
- // restart counted as though the agent had crashed. ENG-9138 cost katie 8h16m
8578
- // when day-rollover fell through the same way; ENG-10066 cost an operator an
8579
- // auto-pause when a channel /restart did. Both were fixed by naming the
8580
- // reason rather than leaving it to the catch-all, and this is the third.
8581
- //
8582
- // WHY NOW. Until ENG-10510 this restart only ever followed a human adding a
8583
- // channel, so the crash tally was wrong but rarely reached: a person adding
8584
- // two channels inside ten minutes is uncommon and self-limiting. Channel
8585
- // auto-resume is the first thing that fires it UNATTENDED, on a timer, on
8586
- // any agent with a quarantined channel. An agent with a second crash-class
8587
- // restart in the same ten minutes (the presence reaper's
8588
- // 'mcp-presence-reaper', for one) would cross DEFAULT_MAX and auto-pause
8589
- // with no human anywhere in the loop - which is precisely the ENG-8215 harm
8590
- // that feature's 2h floor exists to prevent.
8591
- //
8592
- // Counted, not exempt: provisioning is a LOOSER tally, not an absent one, so
8593
- // sustained channel-set thrash still trips and still pages.
8594
- "channel-set-change"
8595
- ]);
8596
- function isProvisioningReloadReason(reason) {
8597
- return PROVISIONING_RELOAD_REASONS.has(reason);
8252
+ var rotationTimestamps = /* @__PURE__ */ new Map();
8253
+ var DEFAULT_ROTATION_GRACE_MS = 15e3;
8254
+ function rotationKey(codeName, serverKey) {
8255
+ return `${codeName}\0${serverKey}`;
8256
+ }
8257
+ function recordStaleRotation(codeName, serverKeys, now = Date.now) {
8258
+ const ts = now();
8259
+ for (const key of serverKeys) {
8260
+ rotationTimestamps.set(rotationKey(codeName, key), ts);
8261
+ }
8262
+ }
8263
+ var ROTATION_TRACKING_RETENTION_MS = 15 * 60 * 1e3;
8264
+ function classifyRotation(codeName, serverKey, graceMs, now = Date.now, retentionMs = ROTATION_TRACKING_RETENTION_MS) {
8265
+ const k = rotationKey(codeName, serverKey);
8266
+ const ts = rotationTimestamps.get(k);
8267
+ if (ts === void 0) return "none";
8268
+ const age = now() - ts;
8269
+ if (age < graceMs) return "within-grace";
8270
+ if (age < retentionMs) return "expired";
8271
+ rotationTimestamps.delete(k);
8272
+ return "none";
8273
+ }
8274
+ function getRotationRecordedAt(codeName, serverKey) {
8275
+ return rotationTimestamps.get(rotationKey(codeName, serverKey));
8276
+ }
8277
+ function clearStaleRotation(codeName, serverKeys) {
8278
+ for (const key of serverKeys) {
8279
+ rotationTimestamps.delete(rotationKey(codeName, key));
8280
+ }
8281
+ }
8282
+ function parseEnvIntegrationsEntries(content) {
8283
+ const entries = /* @__PURE__ */ new Map();
8284
+ for (const raw of content.split(/\r?\n/)) {
8285
+ const line = raw.trim();
8286
+ if (!line || line.startsWith("#")) continue;
8287
+ const eq = line.indexOf("=");
8288
+ if (eq <= 0) continue;
8289
+ const name = line.slice(0, eq).trim();
8290
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) continue;
8291
+ const value = line.slice(eq + 1);
8292
+ entries.set(name, value);
8293
+ }
8294
+ return entries;
8598
8295
  }
8599
- var CREDENTIAL_ROTATION_REASONS = /* @__PURE__ */ new Set([
8600
- "credential-rotation"
8601
- ]);
8602
- function isCredentialRotationReason(reason) {
8603
- return CREDENTIAL_ROTATION_REASONS.has(reason);
8296
+ function diffEnvIntegrations(oldContent, newContent) {
8297
+ const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
8298
+ const newEntries = parseEnvIntegrationsEntries(newContent);
8299
+ const changed = /* @__PURE__ */ new Set();
8300
+ for (const [name, value] of newEntries) {
8301
+ if (oldEntries.get(name) !== value) changed.add(name);
8302
+ }
8303
+ for (const name of oldEntries.keys()) {
8304
+ if (!newEntries.has(name)) changed.add(name);
8305
+ }
8306
+ return [...changed];
8604
8307
  }
8605
- var SELF_HEALING_REASONS = /* @__PURE__ */ new Set([
8606
- "bind-remediation"
8607
- ]);
8608
- function isSelfHealingReason(reason) {
8609
- return SELF_HEALING_REASONS.has(reason);
8308
+ function classifyEnvIntegrationsDiff(oldContent, newContent) {
8309
+ const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
8310
+ const newEntries = parseEnvIntegrationsEntries(newContent);
8311
+ const rotated = [];
8312
+ const added = [];
8313
+ const removed = [];
8314
+ for (const [name, value] of newEntries) {
8315
+ if (!oldEntries.has(name)) added.push(name);
8316
+ else if (oldEntries.get(name) !== value) rotated.push(name);
8317
+ }
8318
+ for (const name of oldEntries.keys()) {
8319
+ if (!newEntries.has(name)) removed.push(name);
8320
+ }
8321
+ return { rotated, added, removed };
8610
8322
  }
8611
- var SCHEDULED_REASONS = /* @__PURE__ */ new Set([
8612
- "day-rollover"
8613
- ]);
8614
- function isScheduledRestartReason(reason) {
8615
- return SCHEDULED_REASONS.has(reason);
8323
+ function findMcpServersUsingVars(mcp, changedVars) {
8324
+ const changedSet = new Set(changedVars);
8325
+ if (!mcp?.mcpServers || changedSet.size === 0) return [];
8326
+ const result = [];
8327
+ for (const [serverKey, entry] of Object.entries(mcp.mcpServers)) {
8328
+ if (!entry || typeof entry !== "object") continue;
8329
+ const env = entry.env;
8330
+ if (!env || typeof env !== "object") continue;
8331
+ let matches = false;
8332
+ for (const value of Object.values(env)) {
8333
+ if (typeof value !== "string") continue;
8334
+ const placeholderMatches = value.match(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g);
8335
+ if (placeholderMatches) {
8336
+ for (const ph of placeholderMatches) {
8337
+ const name = ph.slice(2, -1);
8338
+ if (changedSet.has(name)) {
8339
+ matches = true;
8340
+ break;
8341
+ }
8342
+ }
8343
+ }
8344
+ if (matches) break;
8345
+ }
8346
+ if (matches) result.push(serverKey);
8347
+ }
8348
+ return result;
8616
8349
  }
8617
- var OPERATOR_COMMAND_REASONS = /* @__PURE__ */ new Set([
8618
- "channel-restart-flag"
8619
- ]);
8620
- function isOperatorCommandReason(reason) {
8621
- return OPERATOR_COMMAND_REASONS.has(reason);
8350
+ function envOnlyRespawnVars(changedVars, mcp, channelSecretKeys) {
8351
+ const channelSet = new Set(channelSecretKeys);
8352
+ return [...changedVars].filter(
8353
+ (v) => !channelSet.has(v) && findMcpServersUsingVars(mcp, [v]).length === 0
8354
+ );
8622
8355
  }
8623
- var OPERATOR_CONFIG_REASONS = /* @__PURE__ */ new Set([
8624
- "operator-integration-change"
8625
- ]);
8626
- function isOperatorConfigReason(reason) {
8627
- return OPERATOR_CONFIG_REASONS.has(reason);
8356
+ function liveProxyExtraHeaderVars(mcp) {
8357
+ const out = /* @__PURE__ */ new Set();
8358
+ if (!mcp?.mcpServers) return out;
8359
+ for (const entry of Object.values(mcp.mcpServers)) {
8360
+ const spec = entry?.env?.["AGT_REMOTE_MCP_EXTRA_HEADERS"];
8361
+ if (typeof spec !== "string") continue;
8362
+ for (const pair of spec.split(",")) {
8363
+ const colon = pair.indexOf(":");
8364
+ if (colon <= 0) continue;
8365
+ const varName = pair.slice(colon + 1).trim();
8366
+ if (varName) out.add(varName);
8367
+ }
8368
+ }
8369
+ return out;
8628
8370
  }
8629
- var NEVER_INTEGRATION_BUCKETED = /* @__PURE__ */ new Set([
8630
- "channel-set-change"
8631
- ]);
8632
- function isIntegrationBucketedReason(reason) {
8633
- if (NEVER_INTEGRATION_BUCKETED.has(reason)) return false;
8634
- return isProvisioningReloadReason(reason) || isOperatorConfigReason(reason);
8371
+ function liveProxyTokenVars(mcp) {
8372
+ const out = /* @__PURE__ */ new Set();
8373
+ if (!mcp?.mcpServers) return out;
8374
+ for (const entry of Object.values(mcp.mcpServers)) {
8375
+ const tokenVar = entry?.env?.["AGT_REMOTE_MCP_TOKEN_VAR"];
8376
+ if (typeof tokenVar === "string" && tokenVar.trim()) {
8377
+ out.add(tokenVar.trim());
8378
+ }
8379
+ }
8380
+ return out;
8635
8381
  }
8636
- function restartReasonClass(reason) {
8637
- if (isCredentialRotationReason(reason)) return "credential-rotation";
8638
- if (isSelfHealingReason(reason)) return "self-healing";
8639
- if (isOperatorConfigReason(reason)) return "operator-config";
8640
- if (isScheduledRestartReason(reason)) return "scheduled";
8641
- if (isOperatorCommandReason(reason)) return "operator-command";
8642
- return isProvisioningReloadReason(reason) ? "provisioning" : "crash";
8382
+ function buildArgvMatchersForEntry(key, entry) {
8383
+ const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
8384
+ const patterns = [];
8385
+ const args = entry?.args ?? [];
8386
+ for (let i = args.length - 1; i >= 0; i--) {
8387
+ const arg = args[i];
8388
+ if (typeof arg !== "string" || !arg) continue;
8389
+ if (arg.startsWith("-")) continue;
8390
+ const stripped = arg.replace(
8391
+ /@[^/@]*$/,
8392
+ (m) => (
8393
+ // @latest, @1.2.3, etc. — drop. But @scope at the START of a
8394
+ // package spec must NOT be stripped, so we only strip a tail
8395
+ // `@...` if it doesn't itself start with a slash inside.
8396
+ // The regex above only matches a single trailing `@...`
8397
+ // segment after the last `/`, so this is safe for `@scope/pkg`.
8398
+ m.includes("/") ? m : ""
8399
+ )
8400
+ );
8401
+ const tail = "(?![A-Za-z0-9_-])";
8402
+ if (/^@?[a-z0-9]([a-z0-9._-]*\/)?[a-z0-9._-]+$/i.test(stripped)) {
8403
+ patterns.push(new RegExp(`${escapeRe(stripped)}${tail}`));
8404
+ const basename = stripped.split("/").pop();
8405
+ if (basename && basename !== stripped) {
8406
+ patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
8407
+ }
8408
+ break;
8409
+ }
8410
+ if (stripped.includes("/")) {
8411
+ const basename = stripped.split("/").pop();
8412
+ if (basename) {
8413
+ patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
8414
+ break;
8415
+ }
8416
+ }
8417
+ }
8418
+ if (patterns.length === 0) {
8419
+ const safe = escapeRe(key);
8420
+ patterns.push(new RegExp(`(?<![A-Za-z0-9_])${safe}(?![A-Za-z0-9_])`));
8421
+ if (safe.includes("_")) {
8422
+ const dashed = safe.replace(/_/g, "-");
8423
+ patterns.push(new RegExp(`(?<![A-Za-z0-9_])${dashed}(?![A-Za-z0-9_])`));
8424
+ }
8425
+ }
8426
+ return patterns;
8643
8427
  }
8644
- function countByClass(events) {
8645
- let crash = 0;
8646
- let provisioning = 0;
8647
- let credentialRotation = 0;
8648
- let selfHealing = 0;
8649
- let operatorConfig = 0;
8650
- let scheduled = 0;
8651
- let operatorCommand = 0;
8652
- for (const e of events) {
8653
- const klass = restartReasonClass(e.reason);
8654
- if (klass === "provisioning") provisioning += 1;
8655
- else if (klass === "credential-rotation") credentialRotation += 1;
8656
- else if (klass === "self-healing") selfHealing += 1;
8657
- else if (klass === "operator-config") operatorConfig += 1;
8658
- else if (klass === "scheduled") scheduled += 1;
8659
- else if (klass === "operator-command") operatorCommand += 1;
8660
- else crash += 1;
8428
+ function buildClaudeAgentMatcher(codeName) {
8429
+ const safe = codeName.replace(/[^A-Za-z0-9_-]/g, "");
8430
+ return new RegExp(`\\bclaude\\b.*--name\\s+agt-${safe}(?=\\s|$)`);
8431
+ }
8432
+ function findMcpChildrenForAgent(args) {
8433
+ const { rows, codeName, serverKeys, mcpJson } = args;
8434
+ const maxDepth = args.maxDepth ?? 8;
8435
+ const inContainer = args.inContainer ?? false;
8436
+ if (serverKeys.length === 0 || rows.length === 0) return [];
8437
+ const claudeMatcher = inContainer ? /\bclaude\b/ : buildClaudeAgentMatcher(codeName);
8438
+ const argvMatchers = [];
8439
+ for (const key of serverKeys) {
8440
+ const entry = mcpJson?.mcpServers?.[key];
8441
+ argvMatchers.push(...buildArgvMatchersForEntry(key, entry));
8442
+ }
8443
+ const byPid = new Map(rows.map((r) => [r.pid, r]));
8444
+ const matched = [];
8445
+ for (const row of rows) {
8446
+ if (!argvMatchers.some((re) => re.test(row.args))) continue;
8447
+ if (/\bclaude\b/.test(row.args) && row.args.includes(`--name agt-${codeName}`)) continue;
8448
+ let cur = byPid.get(row.ppid);
8449
+ let belongs = false;
8450
+ for (let depth = 0; depth < maxDepth && cur; depth++) {
8451
+ if (claudeMatcher.test(cur.args)) {
8452
+ belongs = true;
8453
+ break;
8454
+ }
8455
+ if (cur.pid === 1) break;
8456
+ cur = byPid.get(cur.ppid);
8457
+ }
8458
+ if (belongs) matched.push(row.pid);
8459
+ }
8460
+ return matched;
8461
+ }
8462
+ function reapStaleMcpChildren(args) {
8463
+ const { log: log2, codeName, serverKeys, mcpJson, graceMs = 5e3, isolated = false } = args;
8464
+ if (serverKeys.length === 0) return [];
8465
+ const runPs = args.runPs ?? (isolated ? (
8466
+ // ENG-9781: hard-bounded. See bounded-exec.ts — `execFileSync`'s own
8467
+ // `timeout` is not a bound, and this runs per isolated agent per cycle
8468
+ // on the manager-worker's critical path. Throws on a non-answer so the
8469
+ // caller's existing catch skips the agent rather than acting on an
8470
+ // empty process list.
8471
+ () => {
8472
+ const r = hardBoundedExecFile(
8473
+ "docker",
8474
+ ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
8475
+ { timeoutMs: 8e3, log: log2 }
8476
+ );
8477
+ if (r.kind === "ok") return r.stdout;
8478
+ throw new Error(
8479
+ r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
8480
+ );
8481
+ }
8482
+ ) : () => execFileSync("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
8483
+ const killProcess = args.killProcess ?? (isolated ? (pid, signal) => {
8484
+ hardBoundedExecFile(
8485
+ "docker",
8486
+ ["exec", `agt-${codeName}`, "kill", signal === "SIGKILL" ? "-KILL" : "-TERM", String(pid)],
8487
+ { timeoutMs: 8e3, log: log2 }
8488
+ );
8489
+ } : (pid, signal) => {
8490
+ try {
8491
+ process.kill(pid, signal);
8492
+ } catch {
8493
+ }
8494
+ });
8495
+ const isAlive = args.isAlive ?? (isolated ? (pid) => {
8496
+ const r = hardBoundedExecFile(
8497
+ "docker",
8498
+ ["exec", `agt-${codeName}`, "kill", "-0", String(pid)],
8499
+ { timeoutMs: 8e3, log: log2 }
8500
+ );
8501
+ return livenessFromProbe(
8502
+ r,
8503
+ (why) => log2(
8504
+ `[stale-mcp-reaper] liveness probe for pid ${pid} in 'agt-${codeName}' ${why} \u2014 treating as ALIVE (no reap) rather than guessing`
8505
+ )
8506
+ );
8507
+ } : (pid) => {
8508
+ try {
8509
+ process.kill(pid, 0);
8510
+ return true;
8511
+ } catch {
8512
+ return false;
8513
+ }
8514
+ });
8515
+ let psOutput;
8516
+ try {
8517
+ psOutput = runPs();
8518
+ } catch (err) {
8519
+ log2(`[stale-mcp-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping reap`);
8520
+ return [];
8661
8521
  }
8662
- return {
8663
- crash,
8664
- provisioning,
8665
- "credential-rotation": credentialRotation,
8666
- "self-healing": selfHealing,
8667
- "operator-config": operatorConfig,
8668
- scheduled,
8669
- "operator-command": operatorCommand
8522
+ const rows = parsePsRows(psOutput);
8523
+ const targets = findMcpChildrenForAgent({ rows, codeName, serverKeys, mcpJson, inContainer: isolated });
8524
+ if (targets.length === 0) return [];
8525
+ const byPid = new Map(rows.map((r) => [r.pid, r]));
8526
+ const describe = (pid) => {
8527
+ const argv = byPid.get(pid)?.args ?? "";
8528
+ const pkgMatch = argv.match(/(@[a-z0-9_-]+\/[a-z0-9_-]+|[a-z0-9_-]+-mcp-server|[a-z0-9_-]+-mcp\b)/i);
8529
+ return pkgMatch ? `${pkgMatch[1]} (pid ${pid})` : `pid ${pid}`;
8670
8530
  };
8531
+ log2(
8532
+ `[stale-mcp-reaper] '${codeName}': rotating ${targets.length} stale MCP child(ren) for [${serverKeys.join(", ")}]: ${targets.map(describe).join(", ")}`
8533
+ );
8534
+ for (const pid of targets) {
8535
+ killProcess(pid, "SIGTERM");
8536
+ }
8537
+ const killedKeys = /* @__PURE__ */ new Set();
8538
+ for (const pid of targets) {
8539
+ const argv = byPid.get(pid)?.args ?? "";
8540
+ for (const key of serverKeys) {
8541
+ const entry = mcpJson?.mcpServers?.[key];
8542
+ const matchers = buildArgvMatchersForEntry(key, entry);
8543
+ if (matchers.some((re) => re.test(argv))) killedKeys.add(key);
8544
+ }
8545
+ }
8546
+ if (killedKeys.size > 0) recordStaleRotation(codeName, killedKeys);
8547
+ setTimeout(() => {
8548
+ try {
8549
+ let freshPsOutput;
8550
+ try {
8551
+ freshPsOutput = runPs();
8552
+ } catch (err) {
8553
+ log2(`[stale-mcp-reaper] '${codeName}': fresh ps for SIGKILL re-verify failed: ${err.message} \u2014 skipping SIGKILL pass`);
8554
+ return;
8555
+ }
8556
+ const stillOwned = new Set(
8557
+ findMcpChildrenForAgent({
8558
+ rows: parsePsRows(freshPsOutput),
8559
+ codeName,
8560
+ serverKeys,
8561
+ mcpJson,
8562
+ inContainer: isolated
8563
+ })
8564
+ );
8565
+ const stragglers = targets.filter((pid) => isAlive(pid) && stillOwned.has(pid));
8566
+ if (stragglers.length === 0) return;
8567
+ log2(
8568
+ `[stale-mcp-reaper] '${codeName}': ${stragglers.length} child(ren) survived SIGTERM; sending SIGKILL: ${stragglers.map(describe).join(", ")}`
8569
+ );
8570
+ for (const pid of stragglers) {
8571
+ killProcess(pid, "SIGKILL");
8572
+ }
8573
+ } catch (err) {
8574
+ log2(`[stale-mcp-reaper] '${codeName}': error in SIGKILL pass: ${err.message}`);
8575
+ }
8576
+ }, graceMs).unref();
8577
+ return targets;
8671
8578
  }
8672
- function maxProvisioningBucket(events) {
8673
- return maxBucketWhere(events, isProvisioningReloadReason);
8579
+
8580
+ // src/lib/mcp-presence-reaper.ts
8581
+ import { execFileSync as execFileSync2 } from "child_process";
8582
+ var DEFAULT_COLD_START_GRACE_MS = 9e4;
8583
+ var MAX_PRESENCE_RESTART_ATTEMPTS = 3;
8584
+ var BACKOFF_AFTER_RESTARTS = MAX_PRESENCE_RESTART_ATTEMPTS;
8585
+ var BACKOFF_BASE_MS = 6e4;
8586
+ var BACKOFF_MAX_MS = 30 * 6e4;
8587
+ var BACKOFF_WINDOW_MS = 60 * 6e4;
8588
+ var BACKOFF_GIVE_UP_RESTARTS = 2 * MAX_PRESENCE_RESTART_ATTEMPTS;
8589
+ var BACKOFF_STABLE_RESET_MS = 5 * 6e4;
8590
+ var DEAD_CHANNEL_MS = 15 * 6e4;
8591
+ var presenceReaperState = /* @__PURE__ */ new Map();
8592
+ var MAX_ROTATION_PROMPT_PROBES = 3;
8593
+ var ROTATION_PROMPT_PROBE_INTERVAL_MS = 3e4;
8594
+ var rotationStallState = /* @__PURE__ */ new Map();
8595
+ var blockedPromptEpisodes = /* @__PURE__ */ new Map();
8596
+ function stateKey(codeName, serverKey) {
8597
+ return `${codeName}\0${serverKey}`;
8674
8598
  }
8675
- function maxBucketWhere(events, matches) {
8676
- const buckets = /* @__PURE__ */ new Map();
8677
- for (const e of events) {
8678
- if (!matches(e.reason)) continue;
8679
- const key = e.integrationKey ?? "";
8680
- buckets.set(key, (buckets.get(key) ?? 0) + 1);
8599
+ function clearPresenceReaperState(codeName) {
8600
+ const prefix = `${codeName}\0`;
8601
+ for (const key of presenceReaperState.keys()) {
8602
+ if (key.startsWith(prefix)) presenceReaperState.delete(key);
8681
8603
  }
8682
- let best = { count: 0, key: "" };
8683
- for (const [key, count] of buckets) {
8684
- if (count > best.count) best = { count, key };
8604
+ }
8605
+ function clearPresenceReaperStateForKeys(codeName, keys) {
8606
+ for (const key of keys) {
8607
+ presenceReaperState.delete(stateKey(codeName, key));
8608
+ rotationStallState.delete(stateKey(codeName, key));
8685
8609
  }
8686
- return best;
8687
8610
  }
8688
- function tripClass(trip) {
8689
- if (trip.trippedClass) return trip.trippedClass;
8690
- if (!Array.isArray(trip.eventsAtTrip) || trip.eventsAtTrip.length === 0) return "crash";
8691
- return trip.eventsAtTrip.every((e) => isProvisioningReloadReason(e.reason)) ? "provisioning" : "crash";
8611
+ function givenUpMcpServerKeys(codeName) {
8612
+ const prefix = `${codeName}\0`;
8613
+ const out = /* @__PURE__ */ new Set();
8614
+ for (const [key, st] of presenceReaperState) {
8615
+ if (key.startsWith(prefix) && (st.gaveUp || st.attempts >= MAX_PRESENCE_RESTART_ATTEMPTS)) {
8616
+ out.add(key.slice(prefix.length));
8617
+ }
8618
+ }
8619
+ return out;
8692
8620
  }
8693
- var MAX_NEW_AGENT_GRACE_MS = 365 * 24 * 60 * 60 * 1e3;
8694
- var DEFAULT_MAX = 2;
8695
- var DEFAULT_WINDOW_MS = 6e5;
8696
- var DEFAULT_PROVISIONING_MAX = RESTART_BREAKER_PROVISIONING_MAX;
8697
- var DEFAULT_PROVISIONING_WINDOW_MS = RESTART_BREAKER_PROVISIONING_WINDOW_MS;
8698
- var DEFAULT_OPERATOR_MAX = RESTART_BREAKER_OPERATOR_MAX;
8699
- var DEFAULT_SCHEDULED_MAX = RESTART_BREAKER_SCHEDULED_MAX;
8700
- var DEFAULT_SCHEDULED_WINDOW_MS = RESTART_BREAKER_SCHEDULED_WINDOW_MS;
8701
- var DEFAULT_OPERATOR_COMMAND_MAX = RESTART_BREAKER_OPERATOR_COMMAND_MAX;
8702
- var DEFAULT_OPERATOR_COMMAND_WINDOW_MS = RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS;
8703
- function readEnvNumber(name, fallback) {
8704
- const raw = process.env[name];
8705
- if (!raw) return fallback;
8706
- const parsed = Number(raw);
8707
- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
8621
+ function findMissingMcpServers(args) {
8622
+ const { rows, codeName, mcpJson, inContainer } = args;
8623
+ const servers = mcpJson?.mcpServers ?? {};
8624
+ const declared = Object.keys(servers);
8625
+ if (declared.length === 0) return [];
8626
+ const missing = [];
8627
+ for (const key of declared) {
8628
+ const entry = servers[key];
8629
+ const isStdio = typeof entry?.command === "string";
8630
+ if (!isStdio) continue;
8631
+ const live = findMcpChildrenForAgent({
8632
+ rows,
8633
+ codeName,
8634
+ serverKeys: [key],
8635
+ mcpJson,
8636
+ inContainer
8637
+ });
8638
+ if (live.length === 0) missing.push(key);
8639
+ }
8640
+ return missing;
8708
8641
  }
8709
- function migrateMapKey(map, from, to) {
8710
- if (map.has(to) || !map.has(from)) return;
8711
- map.set(to, map.get(from));
8712
- map.delete(from);
8642
+ function describeBlockedPromptHold(ctx) {
8643
+ const keys = `declared MCP(s) [${ctx.activeKeys.join(", ")}]`;
8644
+ if (ctx.restartDeferred) {
8645
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. The recovery restart was ATTEMPTED and the host deferred it (maintenance window or agent not idle), so nothing was torn down and this episode's one restart is still unspent - it retries when the gate opens. Needs a human if it persists`;
8646
+ }
8647
+ if (ctx.restartSpentThisEpisode) {
8648
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. This episode's ONE restart was already spent and the agent is STILL blocked, so a further restart is not the answer. Needs a human at the pane`;
8649
+ }
8650
+ if (ctx.restartBudgetRemaining === 0) {
8651
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. No restart was attempted: the crash breaker has NO budget left in its window, so a restart now would be the event that trips it and pauses this agent - attributed to mcp-presence-reaper, not to whatever filled the window. It may clear once the breaker window rolls; needs a human if it does not`;
8652
+ }
8653
+ return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children and no restart was made. Needs a human at the pane`;
8713
8654
  }
8714
- var RestartBreaker = class {
8715
- max;
8716
- windowMs;
8717
- /**
8718
- * ENG-8215: NOT readonly - the constructor may clamp it upward to preserve the
8719
- * quarantine-fires-first ordering against a runtime override. Never mutated
8720
- * after construction.
8721
- */
8722
- provisioningMax;
8723
- provisioningWindowMs;
8724
- /**
8725
- * ENG-8388: the looser operator-config budget. Shares provisioningWindowMs.
8726
- * NOT readonly - the constructor floors it at `provisioningMax` so setting one
8727
- * env var without the other cannot invert the two tallies.
8728
- */
8729
- operatorMax;
8730
- /**
8731
- * ENG-9138: the scheduled class's LOOP bar. Readonly and deliberately NOT
8732
- * floored against any other tally: it gates a REPORT, not a pause, so it
8733
- * cannot invert anything the way operatorMax/provisioningMax can.
8734
- */
8735
- scheduledMax;
8736
- /** ENG-9138: sliding window for the scheduled tally. */
8737
- scheduledWindowMs;
8738
- operatorCommandMax;
8739
- operatorCommandWindowMs;
8740
- /** Longest window any tally needs — how long the events log must retain. */
8741
- retentionMs;
8742
- now;
8743
- /** ENG-7891 slice 5b: code_name → durable runtime key (agent_id when id-keyed). Identity on the legacy fleet. */
8744
- runtimeKey;
8745
- /** ENG-8690: agent creation time lookup; undefined result ⇒ no grace. */
8746
- agentCreatedAt;
8747
- /** ENG-8690: grace duration; 0 disables. */
8748
- newAgentGraceMs;
8749
- /** ENG-8690: the bound the grace does NOT lift. NOT readonly - floored below. */
8750
- newAgentGraceCeiling;
8751
- events = /* @__PURE__ */ new Map();
8752
- trips = /* @__PURE__ */ new Map();
8753
- /**
8754
- * ENG-8215: set when a supplied `provisioningMax` was raised to keep MCP
8755
- * quarantine ahead of the breaker. Read once by the manager at construction so
8756
- * an overridden-and-clamped value is visible in manager.log rather than
8757
- * silently discarded.
8758
- */
8759
- quarantineOrderingClamped;
8760
- constructor(opts = {}) {
8761
- this.max = opts.max ?? readEnvNumber("AGT_RESTART_BREAKER_MAX", DEFAULT_MAX);
8762
- this.windowMs = opts.windowMs ?? readEnvNumber("AGT_RESTART_BREAKER_WINDOW_MS", DEFAULT_WINDOW_MS);
8763
- this.provisioningMax = opts.provisioningMax ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", DEFAULT_PROVISIONING_MAX);
8764
- this.provisioningWindowMs = opts.provisioningWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS", DEFAULT_PROVISIONING_WINDOW_MS);
8765
- this.operatorMax = opts.operatorMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_MAX", DEFAULT_OPERATOR_MAX);
8766
- this.scheduledMax = opts.scheduledMax ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_MAX", DEFAULT_SCHEDULED_MAX);
8767
- this.scheduledWindowMs = opts.scheduledWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_WINDOW_MS", DEFAULT_SCHEDULED_WINDOW_MS);
8768
- this.operatorCommandMax = opts.operatorCommandMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_COMMAND_MAX", DEFAULT_OPERATOR_COMMAND_MAX);
8769
- this.operatorCommandWindowMs = opts.operatorCommandWindowMs ?? readEnvNumber(
8770
- "AGT_RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS",
8771
- DEFAULT_OPERATOR_COMMAND_WINDOW_MS
8772
- );
8773
- this.now = opts.now ?? Date.now;
8774
- this.runtimeKey = opts.runtimeKey ?? ((codeName) => codeName);
8775
- this.agentCreatedAt = opts.agentCreatedAt ?? (() => void 0);
8776
- const graceMs = opts.newAgentGraceMs ?? readEnvNumber("AGT_RESTART_BREAKER_NEW_AGENT_GRACE_MS", RESTART_BREAKER_NEW_AGENT_GRACE_MS);
8777
- const cappedGrace = Math.min(graceMs, MAX_NEW_AGENT_GRACE_MS);
8778
- this.newAgentGraceMs = Number.isFinite(graceMs) && graceMs > 0 ? cappedGrace : 0;
8779
- const ceiling = opts.newAgentGraceCeiling ?? readEnvNumber(
8780
- "AGT_RESTART_BREAKER_NEW_AGENT_GRACE_CEILING",
8781
- RESTART_BREAKER_NEW_AGENT_GRACE_CEILING
8782
- );
8783
- this.newAgentGraceCeiling = Number.isFinite(ceiling) && ceiling > 0 ? ceiling : RESTART_BREAKER_NEW_AGENT_GRACE_CEILING;
8784
- if (!Number.isFinite(this.max) || this.max < 1) {
8785
- throw new Error("restart-breaker max must be a finite number >= 1");
8786
- }
8787
- if (!Number.isFinite(this.windowMs) || this.windowMs < 1e3) {
8788
- throw new Error("restart-breaker windowMs must be a finite number >= 1000");
8789
- }
8790
- if (!Number.isFinite(this.provisioningMax) || this.provisioningMax < 1) {
8791
- throw new Error("restart-breaker provisioningMax must be a finite number >= 1");
8655
+ function reapMissingMcpSessions(args) {
8656
+ const {
8657
+ log: log2,
8658
+ codeName,
8659
+ mcpJson,
8660
+ sessionStartedAt,
8661
+ restartPending = false,
8662
+ stopSession,
8663
+ graceMs = DEFAULT_COLD_START_GRACE_MS,
8664
+ rotationGraceMs = DEFAULT_ROTATION_GRACE_MS,
8665
+ onGiveUp,
8666
+ onRestart,
8667
+ // ENG-8225: unwired callbacks make the prompt-probe path a strict no-op —
8668
+ // `onRotationRespawnStalled` absent means "not armed", which falls through to
8669
+ // the pre-ENG-8225 restart. Legacy callers and tests are unchanged.
8670
+ onRotationRespawnStalled,
8671
+ onRotationRespawnGaveUp,
8672
+ onRotationRespawnOutcome,
8673
+ maxRotationPromptProbes = MAX_ROTATION_PROMPT_PROBES,
8674
+ rotationPromptProbeIntervalMs = ROTATION_PROMPT_PROBE_INTERVAL_MS,
8675
+ // ENG-5932: defaults make quarantine a strict no-op (classify everything
8676
+ // essential, mode off) so legacy callers/tests are unaffected.
8677
+ classifyKey = () => "essential",
8678
+ quarantineDwellMs = 0,
8679
+ deadChannelMs = DEAD_CHANNEL_MS,
8680
+ quarantineMode = "off",
8681
+ onQuarantine,
8682
+ isolated = false,
8683
+ // ENG-8013: default false → a "no live child" observation is only excluded
8684
+ // from the give-up/quarantine budget when the caller positively identifies
8685
+ // the current generation as externally-churned. Legacy callers/tests unchanged.
8686
+ currentGenerationExternallyStarted = false,
8687
+ // ENG-9932: unwired => strict no-op, restart proceeds exactly as before.
8688
+ readPane,
8689
+ onPaneBlocked
8690
+ } = args;
8691
+ const now = args.now ?? Date.now;
8692
+ const runPs = args.runPs ?? (isolated ? (
8693
+ // ENG-9781: HARD-bounded, and the distinction is the whole ticket. The
8694
+ // previous `execFileSync(..., { timeout: 8_000 })` bounded nothing —
8695
+ // Node fires SIGTERM at the deadline and then keeps waiting, so a
8696
+ // docker CLI talking to a wedged daemon blocked the manager-worker's
8697
+ // event loop and stopped scheduling for EVERY agent on the host
8698
+ // (33m46s, twice, on 2026-09-01). This call runs once per isolated
8699
+ // agent per poll cycle, so it is the single worst place on the manager
8700
+ // for an unbounded subprocess.
8701
+ //
8702
+ // Throwing on a non-answer is deliberate rather than lazy: the caller
8703
+ // already catches, logs and SKIPS the agent for the cycle, which is
8704
+ // exactly right. A timeout must not become an empty process list —
8705
+ // that reads as "every declared MCP server is missing" and restarts the
8706
+ // session, turning a docker hiccup into a fleet-wide restart storm.
8707
+ () => {
8708
+ const r = hardBoundedExecFile(
8709
+ "docker",
8710
+ ["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
8711
+ { timeoutMs: 8e3, log: log2 }
8712
+ );
8713
+ if (r.kind === "ok") return r.stdout;
8714
+ throw new Error(
8715
+ r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
8716
+ );
8792
8717
  }
8793
- if (!Number.isFinite(this.operatorMax) || this.operatorMax < 1) {
8794
- throw new Error("restart-breaker operatorMax must be a finite number >= 1");
8718
+ ) : () => execFileSync2("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
8719
+ if (!mcpJson?.mcpServers) {
8720
+ return { missing: [], restarted: false, reason: "no-mcp-json" };
8721
+ }
8722
+ const declaredCount = Object.keys(mcpJson.mcpServers).length;
8723
+ if (declaredCount === 0) {
8724
+ return { missing: [], restarted: false, reason: "no-declared-servers" };
8725
+ }
8726
+ if (restartPending) {
8727
+ return { missing: [], restarted: false, reason: "restart-pending" };
8728
+ }
8729
+ if (sessionStartedAt === null) {
8730
+ return { missing: [], restarted: false, reason: "session-start-unknown" };
8731
+ }
8732
+ if (now() - sessionStartedAt < graceMs) {
8733
+ return { missing: [], restarted: false, reason: "cold-start" };
8734
+ }
8735
+ let psOutput;
8736
+ try {
8737
+ psOutput = runPs();
8738
+ } catch (err) {
8739
+ log2(`[mcp-presence-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping`);
8740
+ return { missing: [], restarted: false, reason: "healthy" };
8741
+ }
8742
+ const rows = parsePsRows(psOutput);
8743
+ const missingRaw = findMissingMcpServers({ rows, codeName, mcpJson, inContainer: isolated });
8744
+ const rotationGraced = [];
8745
+ const rotationProbeArmed = [];
8746
+ const rotationProbeWaiting = [];
8747
+ const rotationProbeExhausted = [];
8748
+ const rotationRespawnOutcomes = {};
8749
+ const recordRotationOutcome = (key, outcome) => {
8750
+ rotationRespawnOutcomes[key] = outcome;
8751
+ if (!onRotationRespawnOutcome) return;
8752
+ try {
8753
+ onRotationRespawnOutcome(codeName, key, outcome);
8754
+ } catch (err) {
8755
+ log2(
8756
+ `[mcp-presence-reaper] onRotationRespawnOutcome callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8757
+ );
8795
8758
  }
8796
- if (this.operatorMax < this.provisioningMax) {
8797
- this.operatorMax = this.provisioningMax;
8759
+ };
8760
+ const missing = [];
8761
+ for (const key of missingRaw) {
8762
+ const rotation = classifyRotation(codeName, key, rotationGraceMs, now);
8763
+ if (rotation === "within-grace") {
8764
+ rotationGraced.push(key);
8765
+ continue;
8798
8766
  }
8799
- if (this.provisioningMax < MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING) {
8800
- this.quarantineOrderingClamped = {
8801
- requested: this.provisioningMax,
8802
- applied: MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING,
8803
- quarantineThreshold: BIND_FAILURE_QUARANTINE_THRESHOLD
8804
- };
8805
- this.provisioningMax = MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING;
8767
+ if (rotation === "none") {
8768
+ rotationStallState.delete(stateKey(codeName, key));
8769
+ missing.push(key);
8770
+ continue;
8806
8771
  }
8807
- if (!Number.isFinite(this.provisioningWindowMs) || this.provisioningWindowMs < 1e3) {
8808
- throw new Error("restart-breaker provisioningWindowMs must be a finite number >= 1000");
8772
+ const sk = stateKey(codeName, key);
8773
+ const rotationRecordedAt = getRotationRecordedAt(codeName, key) ?? now();
8774
+ const priorStall = rotationStallState.get(sk);
8775
+ const stall = priorStall && priorStall.rotationRecordedAt === rotationRecordedAt ? priorStall : {
8776
+ rotationRecordedAt,
8777
+ probeAttempts: 0,
8778
+ lastProbeAt: null,
8779
+ escalated: false
8780
+ };
8781
+ if (priorStall && priorStall !== stall) {
8782
+ log2(
8783
+ `[mcp-presence-reaper] '${codeName}:${key}' discarding stall state from a previous rotation (rotation=${priorStall.rotationRecordedAt}, probes=${priorStall.probeAttempts}, escalated=${priorStall.escalated}) \u2014 this is a NEW rotation (rotation=${rotationRecordedAt}) and gets a full prompt-probe budget (ENG-8225)`
8784
+ );
8809
8785
  }
8810
- if (!Number.isFinite(this.scheduledMax) || this.scheduledMax < 1) {
8811
- throw new Error("restart-breaker scheduledMax must be a finite number >= 1");
8786
+ rotationStallState.set(sk, stall);
8787
+ if (stall.escalated) {
8788
+ missing.push(key);
8789
+ continue;
8812
8790
  }
8813
- if (!Number.isFinite(this.scheduledWindowMs) || this.scheduledWindowMs < 1e3) {
8814
- throw new Error("restart-breaker scheduledWindowMs must be a finite number >= 1000");
8791
+ const spacingElapsed = stall.lastProbeAt === null || now() - stall.lastProbeAt >= rotationPromptProbeIntervalMs;
8792
+ if (!spacingElapsed) {
8793
+ rotationProbeWaiting.push(key);
8794
+ continue;
8815
8795
  }
8816
- if (!Number.isFinite(this.operatorCommandMax) || this.operatorCommandMax < 1) {
8817
- throw new Error("restart-breaker operatorCommandMax must be a finite number >= 1");
8796
+ if (stall.probeAttempts >= maxRotationPromptProbes) {
8797
+ stall.escalated = true;
8798
+ rotationProbeExhausted.push(key);
8799
+ recordRotationOutcome(key, "never-respawned");
8800
+ log2(
8801
+ `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=never-respawned \u2014 child SIGTERM'd by stale-mcp-reaper never returned after ${stall.probeAttempts} prompt bind-probe(s); escalating (integration marked unhealthy + reaper_action alert) and releasing to the normal restart path (ENG-8225)`
8802
+ );
8803
+ if (onRotationRespawnGaveUp) {
8804
+ try {
8805
+ onRotationRespawnGaveUp(codeName, key, { attempts: stall.probeAttempts });
8806
+ } catch (err) {
8807
+ log2(
8808
+ `[mcp-presence-reaper] onRotationRespawnGaveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8809
+ );
8810
+ }
8811
+ }
8812
+ missing.push(key);
8813
+ continue;
8818
8814
  }
8819
- if (!Number.isFinite(this.operatorCommandWindowMs) || this.operatorCommandWindowMs < 1e3) {
8820
- throw new Error("restart-breaker operatorCommandWindowMs must be a finite number >= 1000");
8815
+ const attempt = stall.probeAttempts + 1;
8816
+ let armed = false;
8817
+ if (onRotationRespawnStalled) {
8818
+ try {
8819
+ armed = onRotationRespawnStalled(codeName, key, {
8820
+ attempt,
8821
+ maxAttempts: maxRotationPromptProbes
8822
+ });
8823
+ } catch (err) {
8824
+ log2(
8825
+ `[mcp-presence-reaper] onRotationRespawnStalled callback threw for '${codeName}:${key}' (suppressed; falling back to the pre-ENG-8225 restart path): ${err.message}`
8826
+ );
8827
+ armed = false;
8828
+ }
8821
8829
  }
8822
- const minimumGraceCeiling = Math.max(this.provisioningMax, this.operatorMax);
8823
- if (this.newAgentGraceCeiling < minimumGraceCeiling) {
8824
- this.newAgentGraceCeiling = minimumGraceCeiling;
8830
+ if (!armed) {
8831
+ missing.push(key);
8832
+ continue;
8825
8833
  }
8826
- this.retentionMs = Math.max(
8827
- this.windowMs,
8828
- this.provisioningWindowMs,
8829
- this.scheduledWindowMs,
8830
- this.operatorCommandWindowMs
8834
+ stall.probeAttempts = attempt;
8835
+ stall.lastProbeAt = now();
8836
+ rotationProbeArmed.push(key);
8837
+ log2(
8838
+ `[mcp-presence-reaper] '${codeName}:${key}' rotation grace expired (${rotationGraceMs}ms) with no live child and no tool call to respawn it \u2014 armed prompt bind-probe ${attempt}/${maxRotationPromptProbes} (ENG-7429) instead of restarting the session (ENG-8225)`
8831
8839
  );
8832
8840
  }
8833
- /**
8834
- * ENG-8215: the clamp record, if the configured provisioning max had to be
8835
- * raised to keep MCP quarantine ahead of the breaker. `undefined` when the
8836
- * configured value already satisfied the ordering.
8837
- */
8838
- getQuarantineOrderingClamp() {
8839
- return this.quarantineOrderingClamped;
8841
+ if (rotationGraced.length > 0) {
8842
+ log2(
8843
+ `[mcp-presence-reaper] '${codeName}': skipping [${rotationGraced.join(", ")}] within rotation grace window (${rotationGraceMs}ms) \u2014 child(ren) just SIGTERM'd by stale-mcp-reaper, awaiting respawn on next tool call (ENG-5344)`
8844
+ );
8840
8845
  }
8841
- /**
8842
- * ENG-7891 slice 5b: the DURABLE key for this agent's events/trips entries.
8843
- * Resolves the code_name to the runtime key (agent_id on id-keyed hosts) and,
8844
- * when they differ, lazily migrates a legacy code_name-keyed entry onto it — so
8845
- * a trip persisted under the old key by a pre-arming manager (or from before a
8846
- * codename rename) is found under the new key instead of being lost. Identity on
8847
- * the legacy fleet ⇒ from === to ⇒ no migration and no behaviour change. Both
8848
- * maps migrate together so events and a trip for the same agent never split
8849
- * across keys mid-generation (a migration can flip the resolver mid-process).
8850
- *
8851
- * BOUNDARY — a pure-LEGACY rename (a host that was never armed) still strands a
8852
- * held trip, exactly as it did before slice 5b: with no symlink there is no
8853
- * agent_id to bridge the old codename to the new, so the runtime key stays the
8854
- * (new) codename and the trip persisted under the old codename is not found.
8855
- * That is unchanged, pre-existing behaviour and it is benign — the fresh
8856
- * generation re-evaluates the agent under its new name and a still-pathological
8857
- * agent simply re-trips within the window. The rename-survival guarantee this
8858
- * slice adds is an id-keyed-only, post-arming property (the symlink is the
8859
- * bridge); it is deliberately NOT retrofitted onto the legacy path.
8860
- */
8861
- keyFor(codeName) {
8862
- const key = this.runtimeKey(codeName);
8863
- if (key !== codeName) {
8864
- migrateMapKey(this.events, codeName, key);
8865
- migrateMapKey(this.trips, codeName, key);
8846
+ if (rotationProbeWaiting.length > 0) {
8847
+ log2(
8848
+ `[mcp-presence-reaper] '${codeName}': skipping [${rotationProbeWaiting.join(", ")}] \u2014 prompt bind-probe already armed for the stalled rotation, waiting out the ${rotationPromptProbeIntervalMs}ms re-arm interval (ENG-8225)`
8849
+ );
8850
+ }
8851
+ const declared = mcpJson.mcpServers ?? {};
8852
+ const liveKeys = /* @__PURE__ */ new Set();
8853
+ for (const key of Object.keys(declared)) {
8854
+ const entry = declared[key];
8855
+ if (typeof entry?.command !== "string") continue;
8856
+ if (!missingRaw.includes(key)) liveKeys.add(key);
8857
+ }
8858
+ for (const key of liveKeys) {
8859
+ const sk = stateKey(codeName, key);
8860
+ if (classifyRotation(codeName, key, rotationGraceMs, now) === "none") {
8861
+ rotationStallState.delete(sk);
8862
+ continue;
8866
8863
  }
8867
- return key;
8868
- }
8869
- /** True if this agent's breaker is currently tripped (manager must skip spawn). */
8870
- isTripped(codeName) {
8871
- return this.trips.has(this.keyFor(codeName));
8864
+ const stall = rotationStallState.get(sk);
8865
+ const viaProbe = (stall?.probeAttempts ?? 0) > 0;
8866
+ recordRotationOutcome(
8867
+ key,
8868
+ viaProbe ? "respawned-after-prompt-probe" : "respawned-on-tool-call"
8869
+ );
8870
+ log2(
8871
+ viaProbe ? `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-after-prompt-probe \u2014 child is live again after ${stall?.probeAttempts} prompt bind-probe(s)${stall?.escalated ? " (recovered AFTER escalation \u2014 the earlier never-respawned verdict stands for that window)" : ""}, no session restart needed (ENG-8225)` : `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-on-tool-call \u2014 child returned on the agent's own next tool call, ENG-5344's assumption held (ENG-8225)`
8872
+ );
8873
+ clearStaleRotation(codeName, [key]);
8874
+ rotationStallState.delete(sk);
8872
8875
  }
8873
- getTrip(codeName) {
8874
- return this.trips.get(this.keyFor(codeName));
8876
+ for (const key of liveKeys) {
8877
+ const state = presenceReaperState.get(stateKey(codeName, key));
8878
+ if (state) {
8879
+ state.attempts = 0;
8880
+ state.lastSeenLiveAt = now();
8881
+ state.lastAttemptedSessionStartedAt = null;
8882
+ state.gaveUpLogged = false;
8883
+ state.firstMissingAt = null;
8884
+ state.quarantineLogged = false;
8885
+ state.continuouslyMissingSince = null;
8886
+ state.deadChannelLogged = false;
8887
+ state.quarantineWithheldLogged = false;
8888
+ state.gaveUp = false;
8889
+ if (state.stableLiveSince === null) state.stableLiveSince = now();
8890
+ if (now() - state.stableLiveSince >= BACKOFF_STABLE_RESET_MS) {
8891
+ state.restartHistory = [];
8892
+ }
8893
+ }
8875
8894
  }
8876
- /**
8877
- * Record a restart event. If recording this event puts the count
8878
- * inside the window above `max`, the breaker trips and the call site
8879
- * should NOT respawn.
8880
- *
8881
- * Idempotent on already-tripped breakers: returns the existing trip
8882
- * without double-counting events. Callers may still record the reason
8883
- * via the decision-log for forensics.
8884
- */
8885
- record(codeName, reason, integrationKey) {
8886
- const key = this.keyFor(codeName);
8887
- const existing = this.trips.get(key);
8888
- if (existing) {
8889
- const bucket = maxProvisioningBucket(existing.eventsAtTrip);
8895
+ if (missing.length === 0) {
8896
+ if (missingRaw.length > 0) {
8890
8897
  return {
8891
- tripped: false,
8892
- trip: existing,
8893
- windowCount: existing.eventsAtTrip.length,
8894
- classCounts: countByClass(existing.eventsAtTrip),
8895
- maxProvisioningBucket: bucket.count,
8896
- maxProvisioningBucketKey: bucket.key
8898
+ missing,
8899
+ restarted: false,
8900
+ reason: "healthy",
8901
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8902
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8903
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8904
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8905
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
8897
8906
  };
8898
8907
  }
8899
- const at = this.now();
8900
- const retentionCutoff = at - this.retentionMs;
8901
- const prior = (this.events.get(key) ?? []).filter((e) => e.at >= retentionCutoff);
8902
- const bucketed = isIntegrationBucketedReason(reason);
8903
- prior.push(bucketed && integrationKey ? { reason, at, integrationKey } : { reason, at });
8904
- this.events.set(key, prior);
8905
- const crashCount = prior.filter(
8906
- (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8907
- ).length;
8908
- const provEvents = prior.filter(
8909
- (e) => e.at >= at - this.provisioningWindowMs && isProvisioningReloadReason(e.reason)
8910
- );
8911
- const credRotCount = prior.filter(
8912
- (e) => e.at >= at - this.provisioningWindowMs && isCredentialRotationReason(e.reason)
8913
- ).length;
8914
- const selfHealCount = prior.filter(
8915
- (e) => e.at >= at - this.provisioningWindowMs && isSelfHealingReason(e.reason)
8916
- ).length;
8917
- const operatorEvents = prior.filter(
8918
- (e) => e.at >= at - this.provisioningWindowMs && isOperatorConfigReason(e.reason)
8919
- );
8920
- const scheduledEvents = prior.filter(
8921
- (e) => e.at >= at - this.scheduledWindowMs && isScheduledRestartReason(e.reason)
8922
- );
8923
- const operatorCommandEvents = prior.filter(
8924
- (e) => e.at >= at - this.operatorCommandWindowMs && isOperatorCommandReason(e.reason)
8925
- );
8926
- const windowCount = prior.filter((e) => e.at >= at - this.windowMs).length;
8927
- const classCounts = {
8928
- crash: crashCount,
8929
- provisioning: provEvents.length,
8930
- "credential-rotation": credRotCount,
8931
- "self-healing": selfHealCount,
8932
- "operator-config": operatorEvents.length,
8933
- scheduled: scheduledEvents.length,
8934
- "operator-command": operatorCommandEvents.length
8908
+ blockedPromptEpisodes.delete(codeName);
8909
+ return {
8910
+ missing,
8911
+ restarted: false,
8912
+ reason: "healthy",
8913
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
8914
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
8915
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
8916
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
8917
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
8935
8918
  };
8936
- const scheduledLoop = scheduledEvents.length > this.scheduledMax ? {
8937
- count: scheduledEvents.length,
8938
- windowMs: this.scheduledWindowMs,
8939
- reason: scheduledEvents[scheduledEvents.length - 1].reason
8940
- } : void 0;
8941
- const operatorCommandLoop = operatorCommandEvents.length > this.operatorCommandMax ? {
8942
- count: operatorCommandEvents.length,
8943
- windowMs: this.operatorCommandWindowMs,
8944
- reason: operatorCommandEvents[operatorCommandEvents.length - 1].reason
8945
- } : void 0;
8946
- const provBucket = maxProvisioningBucket(provEvents);
8947
- const operatorBucket = maxBucketWhere(operatorEvents, isOperatorConfigReason);
8948
- const createdAt = this.newAgentGraceMs > 0 ? this.agentCreatedAt(codeName) : void 0;
8949
- const age = typeof createdAt === "number" && Number.isFinite(createdAt) ? at - createdAt : void 0;
8950
- const newAgentGraceActive = age !== void 0 && age >= 0 && age < this.newAgentGraceMs;
8951
- const newAgentGraceExpiresAt = newAgentGraceActive && typeof createdAt === "number" ? createdAt + this.newAgentGraceMs : void 0;
8952
- const graceCeilingEvents = newAgentGraceActive ? prior.filter(
8953
- (e) => e.at >= at - this.provisioningWindowMs && (isProvisioningReloadReason(e.reason) || isOperatorConfigReason(e.reason))
8954
- ) : [];
8955
- const graceCeilingExceeded = newAgentGraceActive && graceCeilingEvents.length > this.newAgentGraceCeiling;
8956
- let trippedClass;
8957
- let tripEvents;
8958
- let tripWindowMs = this.windowMs;
8959
- let tripBucketKey = "";
8960
- if (crashCount > this.max) {
8961
- trippedClass = "crash";
8962
- tripEvents = prior.filter(
8963
- (e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
8964
- );
8965
- tripWindowMs = this.windowMs;
8966
- } else if (newAgentGraceActive && graceCeilingExceeded) {
8967
- trippedClass = "provisioning";
8968
- tripEvents = graceCeilingEvents;
8969
- tripWindowMs = this.provisioningWindowMs;
8970
- tripBucketKey = "";
8971
- } else if (newAgentGraceActive) {
8972
- } else if (operatorBucket.count > this.operatorMax) {
8973
- trippedClass = "operator-config";
8974
- tripEvents = operatorEvents.filter((e) => (e.integrationKey ?? "") === operatorBucket.key);
8975
- tripWindowMs = this.provisioningWindowMs;
8976
- tripBucketKey = operatorBucket.key;
8977
- } else if (provBucket.count > this.provisioningMax) {
8978
- trippedClass = "provisioning";
8979
- tripEvents = provEvents.filter((e) => (e.integrationKey ?? "") === provBucket.key);
8980
- tripWindowMs = this.provisioningWindowMs;
8981
- tripBucketKey = provBucket.key;
8919
+ }
8920
+ const nowMs = now();
8921
+ const givenUp = [];
8922
+ const active = [];
8923
+ const backingOff = [];
8924
+ const quarantined = [];
8925
+ const wouldQuarantine = [];
8926
+ const quarantineWithheld = [];
8927
+ const anyStdioServerLive = liveKeys.size > 0;
8928
+ const genuinelyDead = [];
8929
+ for (const key of missing) {
8930
+ const sk = stateKey(codeName, key);
8931
+ const state = presenceReaperState.get(sk) ?? {
8932
+ attempts: 0,
8933
+ lastSeenLiveAt: null,
8934
+ lastAttemptedSessionStartedAt: null,
8935
+ historyOwedForSessionStartedAt: null,
8936
+ gaveUpLogged: false,
8937
+ firstMissingAt: null,
8938
+ quarantineLogged: false,
8939
+ restartHistory: [],
8940
+ stableLiveSince: null,
8941
+ continuouslyMissingSince: null,
8942
+ deadChannelLogged: false,
8943
+ quarantineWithheldLogged: false,
8944
+ gaveUp: false
8945
+ };
8946
+ if (state.continuouslyMissingSince === null) {
8947
+ state.continuouslyMissingSince = nowMs;
8948
+ }
8949
+ const isGenuinelyDead = nowMs - state.continuouslyMissingSince >= deadChannelMs;
8950
+ const suppressAsChurn = currentGenerationExternallyStarted && !isGenuinelyDead;
8951
+ if (isGenuinelyDead) {
8952
+ genuinelyDead.push(key);
8953
+ if (currentGenerationExternallyStarted && !state.deadChannelLogged) {
8954
+ state.deadChannelLogged = true;
8955
+ log2(
8956
+ `[mcp-presence-reaper] '${codeName}:${key}' continuously missing ${Math.round((nowMs - state.continuouslyMissingSince) / 6e4)}m (> ${Math.round(deadChannelMs / 6e4)}m) \u2014 genuinely dead, overriding storm-awareness so it can reach give-up/quarantine instead of churning (ENG-8062)`
8957
+ );
8958
+ }
8959
+ }
8960
+ if (state.firstMissingAt === null || suppressAsChurn) {
8961
+ state.firstMissingAt = nowMs;
8962
+ }
8963
+ state.stableLiveSince = null;
8964
+ if (state.lastAttemptedSessionStartedAt !== sessionStartedAt) {
8965
+ state.lastAttemptedSessionStartedAt = sessionStartedAt;
8966
+ if (!suppressAsChurn) {
8967
+ state.attempts += 1;
8968
+ state.historyOwedForSessionStartedAt = sessionStartedAt;
8969
+ }
8970
+ }
8971
+ state.restartHistory = state.restartHistory.filter((t) => nowMs - t < BACKOFF_WINDOW_MS);
8972
+ const restartsInWindow = state.restartHistory.length;
8973
+ presenceReaperState.set(sk, state);
8974
+ const overAttemptsCap = state.attempts > MAX_PRESENCE_RESTART_ATTEMPTS;
8975
+ const overRateCap = restartsInWindow >= BACKOFF_GIVE_UP_RESTARTS;
8976
+ if (overAttemptsCap || overRateCap) {
8977
+ givenUp.push(key);
8978
+ state.gaveUp = true;
8979
+ if (!state.gaveUpLogged) {
8980
+ const cause = overAttemptsCap ? `${MAX_PRESENCE_RESTART_ATTEMPTS} consecutive failed restarts \u2014 declared but never recovers (likely orphaned config; see ENG-5279)` : `${restartsInWindow} restarts within ${Math.round(BACKOFF_WINDOW_MS / 6e4)}m \u2014 persistently flapping (crash-loop; ENG-6480)`;
8981
+ log2(
8982
+ `[mcp-presence-reaper] giving up on '${codeName}:${key}' after ${cause}`
8983
+ );
8984
+ state.gaveUpLogged = true;
8985
+ if (onGiveUp) {
8986
+ try {
8987
+ onGiveUp(codeName, key);
8988
+ } catch (err) {
8989
+ log2(
8990
+ `[mcp-presence-reaper] onGiveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
8991
+ );
8992
+ }
8993
+ }
8994
+ }
8995
+ if (quarantineMode !== "off" && !state.quarantineLogged && classifyKey(key) === "optional") {
8996
+ const dwellElapsed = quarantineDwellMs <= 0 || state.firstMissingAt !== null && nowMs - state.firstMissingAt >= quarantineDwellMs;
8997
+ if (dwellElapsed && !anyStdioServerLive) {
8998
+ quarantineWithheld.push(key);
8999
+ if (!state.quarantineWithheldLogged) {
9000
+ state.quarantineWithheldLogged = true;
9001
+ log2(
9002
+ `[mcp-presence-reaper] '${codeName}:${key}' given up but NOT quarantined - no declared stdio MCP server is live, so the restart budget was exhausted by a session that cannot spawn anything, not by this channel dying. Check the pane (tmux attach -t agt-${codeName}); the channel stays provisioned and will quarantine on a later tick if it is still missing once the session is up (ENG-10500)`
9003
+ );
9004
+ }
9005
+ } else if (dwellElapsed) {
9006
+ state.quarantineLogged = true;
9007
+ if (quarantineMode === "enforce") {
9008
+ quarantined.push(key);
9009
+ log2(
9010
+ `[mcp-presence-reaper] QUARANTINE '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; dropping from provisioned set so the dead channel stops restarting the whole session (ENG-5932)`
9011
+ );
9012
+ if (onQuarantine) {
9013
+ try {
9014
+ onQuarantine(codeName, key);
9015
+ } catch (err) {
9016
+ log2(
9017
+ `[mcp-presence-reaper] onQuarantine callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
9018
+ );
9019
+ }
9020
+ }
9021
+ } else {
9022
+ wouldQuarantine.push(key);
9023
+ log2(
9024
+ `[mcp-presence-reaper] SHADOW would-quarantine '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; no action taken (ENG-5932 shadow mode)`
9025
+ );
9026
+ }
9027
+ }
9028
+ }
9029
+ } else {
9030
+ const backoffMs = restartsInWindow < BACKOFF_AFTER_RESTARTS ? 0 : Math.min(
9031
+ BACKOFF_BASE_MS * 2 ** (restartsInWindow - BACKOFF_AFTER_RESTARTS),
9032
+ BACKOFF_MAX_MS
9033
+ );
9034
+ const lastRestartAt = state.restartHistory[state.restartHistory.length - 1] ?? null;
9035
+ if (lastRestartAt !== null && nowMs - lastRestartAt < backoffMs) {
9036
+ backingOff.push(key);
9037
+ } else {
9038
+ active.push(key);
9039
+ }
8982
9040
  }
8983
- if (trippedClass && tripEvents) {
8984
- const baseMessage = graceCeilingExceeded ? formatStatusMessage(tripEvents, tripWindowMs) : formatStatusMessage(tripEvents, tripWindowMs, trippedClass, tripBucketKey);
8985
- const graceHours = Math.round(this.newAgentGraceMs / 36e5 * 10) / 10;
8986
- const statusMessage = graceCeilingExceeded ? `${baseMessage} \u2014 paused DESPITE the new-agent grace: ${graceCeilingEvents.length} provisioning-shaped restarts (provisioning + operator-config) exceeded the grace ceiling of ${this.newAgentGraceCeiling} (ENG-8690). The agent is inside its ${graceHours}h grace and is doing little except restarting.` : baseMessage;
8987
- const trip = {
8988
- trippedAt: at,
8989
- eventsAtTrip: [...tripEvents],
8990
- statusMessage,
8991
- // ENG-7577: persist the tripping class so the recovery path can gate on
8992
- // it after a manager restart, without re-deriving it from prose.
8993
- trippedClass
8994
- };
8995
- this.trips.set(key, trip);
8996
- this.events.delete(key);
8997
- return {
8998
- tripped: true,
8999
- trip,
9000
- windowCount,
9001
- classCounts,
9002
- trippedClass,
9003
- maxProvisioningBucket: provBucket.count,
9004
- maxProvisioningBucketKey: provBucket.key,
9005
- ...graceCeilingExceeded ? { newAgentGraceCeilingExceeded: true } : {},
9006
- // ENG-9138: a co-occurring rollover loop still reports on a real trip.
9007
- ...scheduledLoop ? { scheduledLoop } : {},
9008
- ...operatorCommandLoop ? { operatorCommandLoop } : {}
9009
- };
9041
+ }
9042
+ if (active.length === 0) {
9043
+ const reason = backingOff.length > 0 ? "backoff" : "all-keys-over-cap";
9044
+ if (backingOff.length > 0) {
9045
+ log2(
9046
+ `[mcp-presence-reaper] '${codeName}': holding off restart for [${backingOff.join(", ")}] \u2014 escalating backoff interval not yet elapsed (ENG-6480)${givenUp.length > 0 ? ` (given up: [${givenUp.join(", ")}])` : ""}`
9047
+ );
9010
9048
  }
9011
- const suppressedOperator = operatorBucket.count > this.operatorMax;
9012
- const suppressedProvisioning = !suppressedOperator && provBucket.count > this.provisioningMax;
9013
- const wouldHaveTripped = suppressedOperator || suppressedProvisioning;
9014
9049
  return {
9015
- tripped: false,
9016
- windowCount,
9017
- classCounts,
9018
- maxProvisioningBucket: provBucket.count,
9019
- maxProvisioningBucketKey: provBucket.key,
9020
- ...scheduledLoop ? { scheduledLoop } : {},
9021
- ...operatorCommandLoop ? { operatorCommandLoop } : {},
9022
- ...newAgentGraceActive ? {
9023
- newAgentGraceSuppressedTrip: wouldHaveTripped,
9024
- ...newAgentGraceExpiresAt !== void 0 ? { newAgentGraceExpiresAt } : {},
9025
- ...wouldHaveTripped ? {
9026
- newAgentGraceSuppressed: {
9027
- class: suppressedOperator ? "operator-config" : "provisioning",
9028
- bucketKey: suppressedOperator ? operatorBucket.key : provBucket.key,
9029
- bucketCount: suppressedOperator ? operatorBucket.count : provBucket.count,
9030
- agentAgeMs: age ?? 0
9031
- }
9032
- } : {}
9033
- } : {}
9050
+ missing,
9051
+ restarted: false,
9052
+ reason,
9053
+ givenUp,
9054
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9055
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9056
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9057
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9058
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9059
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
9060
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9061
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9062
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9063
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
9034
9064
  };
9035
9065
  }
9036
- /** Operator-initiated reset: drops the trip + the events log for this agent. */
9037
- clear(codeName) {
9038
- const key = this.keyFor(codeName);
9039
- this.trips.delete(key);
9040
- this.events.delete(key);
9041
- if (key !== codeName) {
9042
- this.trips.delete(codeName);
9043
- this.events.delete(codeName);
9066
+ const readRestartBudget = () => {
9067
+ try {
9068
+ return args.restartBudgetRemaining?.(codeName) ?? null;
9069
+ } catch (err) {
9070
+ log2(
9071
+ `[mcp-presence-reaper] restartBudgetRemaining threw for '${codeName}' (suppressed; treated as "cannot say"): ${err.message}`
9072
+ );
9073
+ return null;
9044
9074
  }
9045
- }
9046
- /** Snapshot tripped agents for `manager-state.json`. */
9047
- serialize() {
9048
- return Object.fromEntries(this.trips.entries());
9049
- }
9050
- /**
9051
- * Rehydrate trip state from `manager-state.json`. Called once at
9052
- * worker startup. Window history is intentionally NOT persisted — only
9053
- * tripped state. A tripped breaker survives manager restart; an
9054
- * un-tripped one starts a fresh window in the new worker.
9055
- */
9056
- hydrate(saved) {
9057
- if (!saved) return;
9058
- for (const [codeName, trip] of Object.entries(saved)) {
9059
- if (trip && typeof trip.trippedAt === "number" && Array.isArray(trip.eventsAtTrip)) {
9060
- const klass = trip.trippedClass;
9061
- this.trips.set(
9062
- codeName,
9063
- klass === void 0 || KNOWN_REASON_CLASSES.has(klass) ? trip : { ...trip, trippedClass: void 0 }
9075
+ };
9076
+ let blockedRestartPending = null;
9077
+ if (readPane) {
9078
+ let pane = null;
9079
+ try {
9080
+ pane = readPane(codeName);
9081
+ } catch (err) {
9082
+ log2(
9083
+ `[mcp-presence-reaper] '${codeName}': pane read threw (${err.message}) \u2014 proceeding with restart (ENG-9932 fails open)`
9084
+ );
9085
+ }
9086
+ const holdBlockedOnPrompt = (blocked2, opts = {}) => {
9087
+ const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
9088
+ if (onPaneBlocked) {
9089
+ try {
9090
+ onPaneBlocked(codeName, {
9091
+ kind: blocked2,
9092
+ activeKeys: [...active],
9093
+ paneHash: fingerprint?.hash ?? null,
9094
+ paneLength: fingerprint?.length ?? null,
9095
+ // ENG-10878: WHY the hold, not just that there was one. An
9096
+ // operator paged by this alert needs to know whether the one
9097
+ // restart was spent and failed, or was never affordable - those
9098
+ // want different next actions, and without this the two are
9099
+ // indistinguishable on the alert board.
9100
+ restartBudgetRemaining: readRestartBudget(),
9101
+ restartSpentThisEpisode: blockedPromptEpisodes.get(codeName)?.restarted ?? false,
9102
+ restartDeferred: opts.restartDeferred ?? false
9103
+ });
9104
+ } catch (err) {
9105
+ log2(
9106
+ `[mcp-presence-reaper] onPaneBlocked callback threw for '${codeName}' (suppressed): ${err.message}`
9107
+ );
9108
+ }
9109
+ }
9110
+ return {
9111
+ missing,
9112
+ restarted: false,
9113
+ reason: "pane-blocked-on-dialog",
9114
+ paneBlockedOn: blocked2,
9115
+ givenUp: givenUp.length > 0 ? givenUp : void 0,
9116
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9117
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9118
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9119
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9120
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9121
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
9122
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9123
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9124
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9125
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
9126
+ };
9127
+ };
9128
+ const blocked = classifyUnanswerablePane(pane);
9129
+ const openBlockedEpisode = blockedPromptEpisodes.get(codeName);
9130
+ if (pane !== null && !blocked && liveKeys.size > 0 && openBlockedEpisode) {
9131
+ blockedPromptEpisodes.delete(codeName);
9132
+ }
9133
+ if (blocked) {
9134
+ const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
9135
+ const budgetLeft = readRestartBudget();
9136
+ const episode = blockedPromptEpisodes.get(codeName);
9137
+ const restartWouldCount = reaperRestartBreakerReason(active) !== void 0;
9138
+ if (!episode && budgetLeft === 0 && restartWouldCount) {
9139
+ log2(
9140
+ `[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 blocked on an interactive prompt (${blocked}) and the crash breaker has NO budget left in its window, so the one restart ENG-10878 would spend would be the event that TRIPS it - remaining===0 means crashCount===max, and the trip test fires at max+1, attributed to mcp-presence-reaper itself. Needs a human at the pane: tmux attach -t agt-${codeName}` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
9141
+ );
9142
+ return holdBlockedOnPrompt(blocked);
9143
+ }
9144
+ if (!episode) {
9145
+ blockedRestartPending = {
9146
+ blocked,
9147
+ paneHash: fingerprint?.hash ?? null,
9148
+ hold: (opts) => holdBlockedOnPrompt(blocked, opts)
9149
+ };
9150
+ log2(
9151
+ `[mcp-presence-reaper] '${codeName}': blocked on an interactive prompt (${blocked}) \u2014 ATTEMPTING this episode's ONE restart (ENG-10878). A restart cured this on 2026-09-22 and did not on 2026-09-03, so the next poll decides which it is` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
9064
9152
  );
9153
+ } else {
9154
+ const comparable = fingerprint !== null && episode.paneHashAtRestart !== null;
9155
+ const modalDelta = !comparable ? "modal fingerprint unavailable on one or both polls, so same-or-different is UNKNOWN" : fingerprint.hash === episode.paneHashAtRestart ? "SAME modal as before the restart" : "a DIFFERENT modal than before the restart";
9156
+ log2(
9157
+ `[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 still blocked on an interactive prompt (${blocked}) after this episode's one restart, so the restart did not clear it (${modalDelta}). Needs a human at the pane: tmux attach -t agt-${codeName}` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
9158
+ );
9159
+ return holdBlockedOnPrompt(blocked);
9065
9160
  }
9066
9161
  }
9067
9162
  }
9068
- /** Test helper — current in-window event count for `codeName`. */
9069
- windowCount(codeName) {
9070
- const cutoff = this.now() - this.windowMs;
9071
- return (this.events.get(this.keyFor(codeName)) ?? []).filter((e) => e.at >= cutoff).length;
9163
+ const givenUpSuffix = givenUp.length > 0 ? ` (skipping over-cap: [${givenUp.join(", ")}])` : "";
9164
+ log2(
9165
+ `[mcp-presence-reaper] '${codeName}': declared MCP(s) [${active.join(", ")}] have no live children \u2014 restarting session${givenUpSuffix}`
9166
+ );
9167
+ const tornDown = stopSession(codeName, { activeKeys: active });
9168
+ if (onRestart && tornDown !== false) {
9169
+ try {
9170
+ const maybePromise = onRestart(codeName, { activeKeys: active, givenUpKeys: givenUp });
9171
+ if (maybePromise && typeof maybePromise.then === "function") {
9172
+ void maybePromise.then(void 0, (err) => {
9173
+ log2(
9174
+ `[mcp-presence-reaper] onRestart callback rejected for '${codeName}' (suppressed; restart proceeds): ${err.message}`
9175
+ );
9176
+ });
9177
+ }
9178
+ } catch (err) {
9179
+ log2(
9180
+ `[mcp-presence-reaper] onRestart callback threw for '${codeName}' (suppressed; restart proceeds): ${err.message}`
9181
+ );
9182
+ }
9072
9183
  }
9073
- };
9074
- function formatStatusMessage(events, windowMs, klass = "crash", integrationKey = "") {
9075
- const last = events[events.length - 1];
9076
- const windowLabel = windowMs < 6e4 ? `${Math.round(windowMs / 1e3)}s` : `${(windowMs / 6e4).toFixed(1).replace(/\.0$/, "")}min`;
9077
- const reasonCounts = /* @__PURE__ */ new Map();
9078
- for (const e of events) reasonCounts.set(e.reason, (reasonCounts.get(e.reason) ?? 0) + 1);
9079
- const breakdown = Array.from(reasonCounts.entries()).map(([r, n]) => `${r}=${n}`).join(", ");
9080
- const classLabel = klass === "provisioning" ? "provisioning-reload " : klass === "operator-config" ? "operator-requested config-reload " : "";
9081
- const integrationLabel = (klass === "provisioning" || klass === "operator-config") && integrationKey ? ` for integration '${integrationKey}'` : "";
9082
- return `Circuit breaker tripped: ${events.length} ${classLabel}restarts in ${windowLabel}${integrationLabel} (${breakdown}); most recent=${last.reason} at ${new Date(last.at).toISOString()}`;
9184
+ if (blockedRestartPending) {
9185
+ if (tornDown === false) {
9186
+ log2(
9187
+ `[mcp-presence-reaper] '${codeName}': restart DEFERRED by the host gate this poll (maintenance window / not idle), so nothing was torn down and this episode's one restart is still unspent \u2014 it will be re-attempted when the gate opens. Needs a human at the pane if it persists: tmux attach -t agt-${codeName}`
9188
+ );
9189
+ return blockedRestartPending.hold({ restartDeferred: true });
9190
+ }
9191
+ blockedPromptEpisodes.set(codeName, {
9192
+ restarted: true,
9193
+ paneHashAtRestart: blockedRestartPending.paneHash
9194
+ });
9195
+ log2(
9196
+ `[mcp-presence-reaper] '${codeName}': this episode's ONE restart is now SPENT (teardown confirmed) \u2014 the next poll decides whether it cleared the modal.`
9197
+ );
9198
+ }
9199
+ if (tornDown === false) {
9200
+ log2(
9201
+ `[mcp-presence-reaper] '${codeName}': restart DEFERRED by the host gate (maintenance window / not idle) \u2014 nothing was torn down, so this poll records no restart and the next poll re-detects and re-tries (ENG-10878)`
9202
+ );
9203
+ return {
9204
+ missing,
9205
+ restarted: false,
9206
+ reason: "restart-deferred",
9207
+ givenUp: givenUp.length > 0 ? givenUp : void 0,
9208
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9209
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9210
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9211
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9212
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0,
9213
+ // ROTATION STATE SURVIVES A DEFERRAL (CodeRabbit on #6399). This return
9214
+ // omitted all five, and a deferred poll is exactly a poll where rotation
9215
+ // work may already have happened: `recordRotationOutcome` runs during the
9216
+ // scan, well before the host gate declines the teardown. So a rotation
9217
+ // that escalated to `never-respawned` on this poll was recorded in the
9218
+ // map, dropped from the result, and unrecoverable by the caller -
9219
+ // probe-armed keys are held OUT of `missing` precisely so they are not
9220
+ // torn down, which also means a caller cannot rebuild them from it.
9221
+ //
9222
+ // `ReapMissingMcpSessionsResult` documents these as present whenever an
9223
+ // entry exists, so omitting them here made the result LIE rather than
9224
+ // merely under-report.
9225
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9226
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9227
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9228
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9229
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
9230
+ };
9231
+ }
9232
+ for (const key of active) {
9233
+ const st = presenceReaperState.get(stateKey(codeName, key));
9234
+ if (!st) continue;
9235
+ if (st.historyOwedForSessionStartedAt !== sessionStartedAt) continue;
9236
+ st.restartHistory.push(nowMs);
9237
+ st.historyOwedForSessionStartedAt = null;
9238
+ }
9239
+ return {
9240
+ missing,
9241
+ restarted: true,
9242
+ givenUp: givenUp.length > 0 ? givenUp : void 0,
9243
+ backingOff: backingOff.length > 0 ? backingOff : void 0,
9244
+ rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
9245
+ rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
9246
+ rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
9247
+ rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
9248
+ rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
9249
+ quarantined: quarantined.length > 0 ? quarantined : void 0,
9250
+ wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
9251
+ genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
9252
+ quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
9253
+ };
9083
9254
  }
9084
9255
 
9085
9256
  // src/lib/mcp-config-lookup.ts
@@ -11739,6 +11910,12 @@ export {
11739
11910
  resolveFlagFromLayers,
11740
11911
  resolveAllFlags,
11741
11912
  HostFlagStore,
11913
+ reaperRestartBreakerReason,
11914
+ isProvisioningReloadReason,
11915
+ isIntegrationBucketedReason,
11916
+ tripClass,
11917
+ readEnvNumber,
11918
+ RestartBreaker,
11742
11919
  diffEnvIntegrations,
11743
11920
  classifyEnvIntegrationsDiff,
11744
11921
  findMcpServersUsingVars,
@@ -11750,13 +11927,8 @@ export {
11750
11927
  clearPresenceReaperState,
11751
11928
  clearPresenceReaperStateForKeys,
11752
11929
  givenUpMcpServerKeys,
11930
+ describeBlockedPromptHold,
11753
11931
  reapMissingMcpSessions,
11754
- reaperRestartBreakerReason,
11755
- isProvisioningReloadReason,
11756
- isIntegrationBucketedReason,
11757
- tripClass,
11758
- readEnvNumber,
11759
- RestartBreaker,
11760
11932
  markQuarantinedForProbe,
11761
11933
  FIRST_CONNECT_WINDOW_MS,
11762
11934
  isWithinReconnectQuiesceWindow,
@@ -11789,4 +11961,4 @@ export {
11789
11961
  managerInstallSystemUnitCommand,
11790
11962
  managerUninstallSystemUnitCommand
11791
11963
  };
11792
- //# sourceMappingURL=chunk-DFKQZGM7.js.map
11964
+ //# sourceMappingURL=chunk-HIDB6U3D.js.map