@integrity-labs/agt-cli 0.28.1015 → 0.28.1017
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/agt.js +5 -5
- package/dist/{chunk-MHDNKKEO.js → chunk-CVQDMHOP.js} +10 -2
- package/dist/{chunk-MHDNKKEO.js.map → chunk-CVQDMHOP.js.map} +1 -1
- package/dist/{chunk-GHHDFHKE.js → chunk-HIDB6U3D.js} +1558 -1372
- package/dist/chunk-HIDB6U3D.js.map +1 -0
- package/dist/{chunk-RJ3SI5JG.js → chunk-X7SX25N6.js} +20 -2
- package/dist/chunk-X7SX25N6.js.map +1 -0
- package/dist/{claude-pair-runtime-GSPFYCG4.js → claude-pair-runtime-JQRJXNSL.js} +2 -2
- package/dist/lib/manager-worker.js +178 -21
- package/dist/lib/manager-worker.js.map +1 -1
- package/dist/mcp/telegram-channel.js +194 -3
- package/dist/{persistent-session-YP2UTQGA.js → persistent-session-UZBB7EOS.js} +3 -3
- package/dist/{responsiveness-probe-YTFWHUTQ.js → responsiveness-probe-KLOUFBM5.js} +3 -3
- package/dist/{session-auth-dead-TX4YQUTJ.js → session-auth-dead-JJUKS75B.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-GHHDFHKE.js.map +0 -1
- package/dist/chunk-RJ3SI5JG.js.map +0 -1
- /package/dist/{claude-pair-runtime-GSPFYCG4.js.map → claude-pair-runtime-JQRJXNSL.js.map} +0 -0
- /package/dist/{persistent-session-YP2UTQGA.js.map → persistent-session-UZBB7EOS.js.map} +0 -0
- /package/dist/{responsiveness-probe-YTFWHUTQ.js.map → responsiveness-probe-KLOUFBM5.js.map} +0 -0
- /package/dist/{session-auth-dead-TX4YQUTJ.js.map → session-auth-dead-JJUKS75B.js.map} +0 -0
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
parseEnvFileEntries,
|
|
16
16
|
parseEnvIntegrations,
|
|
17
17
|
shellQuote
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-CVQDMHOP.js";
|
|
19
19
|
import {
|
|
20
20
|
BIND_FAILURE_QUARANTINE_THRESHOLD,
|
|
21
21
|
INTEGRATIONS_SECTION_END,
|
|
@@ -77,7 +77,7 @@ import {
|
|
|
77
77
|
sessionTranscriptDir,
|
|
78
78
|
worseConnectivityOutcome,
|
|
79
79
|
wrapScheduledTaskPrompt
|
|
80
|
-
} from "./chunk-
|
|
80
|
+
} from "./chunk-X7SX25N6.js";
|
|
81
81
|
import {
|
|
82
82
|
classifyUnanswerablePane,
|
|
83
83
|
summarizeUnanswerablePane
|
|
@@ -5507,8 +5507,10 @@ ${sections}`
|
|
|
5507
5507
|
const senderPolicyTeamId = options?.senderPolicy?.mode === "team_agents_only" || options?.senderPolicy?.mode === "manager_only" || options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_id : void 0;
|
|
5508
5508
|
const slackPrincipalId = options?.senderPolicy?.mode === "manager_only" ? options.senderPolicy.principal?.slack_user_id : void 0;
|
|
5509
5509
|
const teamsPrincipalId = options?.senderPolicy?.mode === "manager_only" ? options.senderPolicy.principal?.teams_aad_object_id : void 0;
|
|
5510
|
+
const telegramPrincipalId = options?.senderPolicy?.mode === "manager_only" ? options.senderPolicy.principal?.telegram_chat_id : void 0;
|
|
5510
5511
|
const slackTeamPrincipalIds = options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_principals?.slack_user_ids?.join(",") : void 0;
|
|
5511
5512
|
const teamsTeamPrincipalIds = options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_principals?.teams_aad_object_ids?.join(",") : void 0;
|
|
5513
|
+
const telegramTeamPrincipalIds = options?.senderPolicy?.mode === "team_only" ? options.senderPolicy.team_principals?.telegram_chat_ids?.join(",") : void 0;
|
|
5512
5514
|
const senderPolicyInternalOnly = options?.senderPolicy?.internal_only === true;
|
|
5513
5515
|
const senderPolicyEnv = senderPolicyTeamId ? { AGT_TEAM_ID: senderPolicyTeamId } : {};
|
|
5514
5516
|
const agentDir = getAgentDir(codeName);
|
|
@@ -5546,6 +5548,18 @@ ${sections}`
|
|
|
5546
5548
|
if (allowedChats && allowedChats.length > 0) {
|
|
5547
5549
|
telegramEnv.TELEGRAM_ALLOWED_CHATS = allowedChats.join(",");
|
|
5548
5550
|
}
|
|
5551
|
+
if (senderPolicyMode) {
|
|
5552
|
+
telegramEnv.TELEGRAM_SENDER_POLICY = senderPolicyMode;
|
|
5553
|
+
}
|
|
5554
|
+
for (const [policyKey, policyValue] of Object.entries(senderPolicyEnv)) {
|
|
5555
|
+
telegramEnv[policyKey] = policyValue;
|
|
5556
|
+
}
|
|
5557
|
+
if (telegramPrincipalId) {
|
|
5558
|
+
telegramEnv.TELEGRAM_SENDER_POLICY_PRINCIPAL_ID = telegramPrincipalId;
|
|
5559
|
+
}
|
|
5560
|
+
if (telegramTeamPrincipalIds) {
|
|
5561
|
+
telegramEnv.TELEGRAM_SENDER_POLICY_TEAM_PRINCIPAL_IDS = telegramTeamPrincipalIds;
|
|
5562
|
+
}
|
|
5549
5563
|
const telegramAckReaction = config["ack_reaction"];
|
|
5550
5564
|
if (typeof telegramAckReaction === "string" && telegramAckReaction.trim().length > 0) {
|
|
5551
5565
|
telegramEnv.TELEGRAM_ACK_REACTION = telegramAckReaction.trim();
|
|
@@ -6993,7 +7007,7 @@ function exchangeFailureKind(err) {
|
|
|
6993
7007
|
}
|
|
6994
7008
|
|
|
6995
7009
|
// src/lib/api-client.ts
|
|
6996
|
-
var agtCliVersion = true ? "0.28.
|
|
7010
|
+
var agtCliVersion = true ? "0.28.1017" : "dev";
|
|
6997
7011
|
var lastConfigHash = null;
|
|
6998
7012
|
function setConfigHash(hash) {
|
|
6999
7013
|
lastConfigHash = hash && hash.length > 0 ? hash : null;
|
|
@@ -7609,1463 +7623,1634 @@ var HostFlagStore = class {
|
|
|
7609
7623
|
}
|
|
7610
7624
|
};
|
|
7611
7625
|
|
|
7612
|
-
// src/lib/
|
|
7613
|
-
|
|
7614
|
-
|
|
7615
|
-
if (r.kind === "ok") return true;
|
|
7616
|
-
if (r.kind === "failed" && r.status !== null) return false;
|
|
7617
|
-
onNoAnswer?.(r.kind === "timed-out" ? "did not answer within its bound" : `could not run: ${r.message}`);
|
|
7618
|
-
return true;
|
|
7626
|
+
// src/lib/restart-breaker.ts
|
|
7627
|
+
function reaperRestartBreakerReason(activeKeys) {
|
|
7628
|
+
return activeKeys.length >= 2 ? "mcp-presence-reaper" : void 0;
|
|
7619
7629
|
}
|
|
7620
|
-
var
|
|
7621
|
-
|
|
7622
|
-
|
|
7623
|
-
|
|
7630
|
+
var KNOWN_REASON_CLASS_MAP = {
|
|
7631
|
+
crash: true,
|
|
7632
|
+
provisioning: true,
|
|
7633
|
+
"credential-rotation": true,
|
|
7634
|
+
// ENG-8388: provisioning reloads a human demonstrably asked for. Same
|
|
7635
|
+
// Record-not-Set reasoning as above — a new class cannot be silently omitted.
|
|
7636
|
+
"operator-config": true,
|
|
7637
|
+
// ENG-8215: the self-healing class (bind-remediation - the manager's OWN
|
|
7638
|
+
// repair attempts, which must never count toward a trip). Adding it here is
|
|
7639
|
+
// forced by the Record type, which is exactly why that shape was chosen over
|
|
7640
|
+
// a Set literal (ENG-7577, CodeRabbit on #3861) - a new union member cannot
|
|
7641
|
+
// be silently omitted from the rehydration allowlist.
|
|
7642
|
+
"self-healing": true,
|
|
7643
|
+
// ENG-9138: the platform-scheduled class. Same Record-not-Set reasoning: the
|
|
7644
|
+
// compiler forces every new class into this allowlist, so a `trippedClass`
|
|
7645
|
+
// rehydrated from manager-state.json can never name a class this build does
|
|
7646
|
+
// not know about.
|
|
7647
|
+
scheduled: true,
|
|
7648
|
+
// ENG-10066: the operator-command class. Same Record-not-Set reasoning again —
|
|
7649
|
+
// the compiler forces every new class into this allowlist.
|
|
7650
|
+
"operator-command": true
|
|
7651
|
+
};
|
|
7652
|
+
var KNOWN_REASON_CLASSES = new Set(
|
|
7653
|
+
Object.keys(KNOWN_REASON_CLASS_MAP)
|
|
7654
|
+
);
|
|
7655
|
+
var PROVISIONING_RELOAD_REASONS = /* @__PURE__ */ new Set([
|
|
7656
|
+
"hot-reload-mcp",
|
|
7657
|
+
"managed-mcp-churn",
|
|
7658
|
+
// ENG-8215: 'bind-remediation' MOVED OUT of this set into SELF_HEALING_REASONS.
|
|
7659
|
+
// It is the manager repairing itself, not a config reload, and counting it
|
|
7660
|
+
// meant the harder the platform tried to fix an agent the more certain it
|
|
7661
|
+
// became that it should give up. Deliberately NOT re-added here.
|
|
7662
|
+
// ENG-7576: the dashboard integration-add stop itself - the first restart of
|
|
7663
|
+
// the very burst this class exists for, previously uncounted entirely.
|
|
7664
|
+
"integration-change",
|
|
7665
|
+
// ENG-7771: a sender_policy delivery restart is a deliberate config-delivery
|
|
7666
|
+
// reload (operator changed the policy, or the fail-closed first-poll verify
|
|
7667
|
+
// after a manager restart with no persisted baseline), not a crash. On the
|
|
7668
|
+
// tight crash tally, a host whose sender-policy-baseline.json write keeps
|
|
7669
|
+
// failing (disk full, permissions) would convert the fail-closed restart
|
|
7670
|
+
// into repeated crash tallies across a multi-deploy day and trip the
|
|
7671
|
+
// breaker - taking the agent DOWN, which is strictly worse than the stale
|
|
7672
|
+
// policy the restart exists to fix. The looser provisioning tally still
|
|
7673
|
+
// trips + pages on sustained thrash.
|
|
7674
|
+
"sender-policy-change",
|
|
7675
|
+
// ENG-9802: same argument, same failure mode — a host whose baseline write
|
|
7676
|
+
// keeps failing must not convert a fail-closed policy restart into a breaker
|
|
7677
|
+
// trip that takes the agent down.
|
|
7678
|
+
"telegram-policy-change",
|
|
7679
|
+
// ENG-9861: and again for standing rules. The fail-closed first-poll restart
|
|
7680
|
+
// has exactly the same shape, so it needs the same protection from a host
|
|
7681
|
+
// whose sidecar write keeps failing.
|
|
7682
|
+
"agent-rules-change",
|
|
7683
|
+
// ENG-10510 (Obi-Wan on #6313): a channel-set ADD is a config reload, not a
|
|
7684
|
+
// crash, and it was landing on the crash tally purely by falling through.
|
|
7685
|
+
// It is 'integration-change's exact sibling - both are "a channel or server
|
|
7686
|
+
// was added, re-render .mcp.json and respawn" - and it was never argued for
|
|
7687
|
+
// the crash class, it simply was not in this set.
|
|
7688
|
+
//
|
|
7689
|
+
// This is the ENG-9138 / ENG-10066 species verbatim: a PLATFORM-MANUFACTURED
|
|
7690
|
+
// restart counted as though the agent had crashed. ENG-9138 cost katie 8h16m
|
|
7691
|
+
// when day-rollover fell through the same way; ENG-10066 cost an operator an
|
|
7692
|
+
// auto-pause when a channel /restart did. Both were fixed by naming the
|
|
7693
|
+
// reason rather than leaving it to the catch-all, and this is the third.
|
|
7694
|
+
//
|
|
7695
|
+
// WHY NOW. Until ENG-10510 this restart only ever followed a human adding a
|
|
7696
|
+
// channel, so the crash tally was wrong but rarely reached: a person adding
|
|
7697
|
+
// two channels inside ten minutes is uncommon and self-limiting. Channel
|
|
7698
|
+
// auto-resume is the first thing that fires it UNATTENDED, on a timer, on
|
|
7699
|
+
// any agent with a quarantined channel. An agent with a second crash-class
|
|
7700
|
+
// restart in the same ten minutes (the presence reaper's
|
|
7701
|
+
// 'mcp-presence-reaper', for one) would cross DEFAULT_MAX and auto-pause
|
|
7702
|
+
// with no human anywhere in the loop - which is precisely the ENG-8215 harm
|
|
7703
|
+
// that feature's 2h floor exists to prevent.
|
|
7704
|
+
//
|
|
7705
|
+
// Counted, not exempt: provisioning is a LOOSER tally, not an absent one, so
|
|
7706
|
+
// sustained channel-set thrash still trips and still pages.
|
|
7707
|
+
"channel-set-change"
|
|
7708
|
+
]);
|
|
7709
|
+
function isProvisioningReloadReason(reason) {
|
|
7710
|
+
return PROVISIONING_RELOAD_REASONS.has(reason);
|
|
7624
7711
|
}
|
|
7625
|
-
|
|
7626
|
-
|
|
7627
|
-
|
|
7628
|
-
|
|
7629
|
-
|
|
7712
|
+
var CREDENTIAL_ROTATION_REASONS = /* @__PURE__ */ new Set([
|
|
7713
|
+
"credential-rotation"
|
|
7714
|
+
]);
|
|
7715
|
+
function isCredentialRotationReason(reason) {
|
|
7716
|
+
return CREDENTIAL_ROTATION_REASONS.has(reason);
|
|
7630
7717
|
}
|
|
7631
|
-
var
|
|
7632
|
-
|
|
7633
|
-
|
|
7634
|
-
|
|
7635
|
-
|
|
7636
|
-
const age = now() - ts;
|
|
7637
|
-
if (age < graceMs) return "within-grace";
|
|
7638
|
-
if (age < retentionMs) return "expired";
|
|
7639
|
-
rotationTimestamps.delete(k);
|
|
7640
|
-
return "none";
|
|
7718
|
+
var SELF_HEALING_REASONS = /* @__PURE__ */ new Set([
|
|
7719
|
+
"bind-remediation"
|
|
7720
|
+
]);
|
|
7721
|
+
function isSelfHealingReason(reason) {
|
|
7722
|
+
return SELF_HEALING_REASONS.has(reason);
|
|
7641
7723
|
}
|
|
7642
|
-
|
|
7643
|
-
|
|
7724
|
+
var SCHEDULED_REASONS = /* @__PURE__ */ new Set([
|
|
7725
|
+
"day-rollover"
|
|
7726
|
+
]);
|
|
7727
|
+
function isScheduledRestartReason(reason) {
|
|
7728
|
+
return SCHEDULED_REASONS.has(reason);
|
|
7644
7729
|
}
|
|
7645
|
-
|
|
7646
|
-
|
|
7647
|
-
|
|
7648
|
-
|
|
7730
|
+
var OPERATOR_COMMAND_REASONS = /* @__PURE__ */ new Set([
|
|
7731
|
+
"channel-restart-flag"
|
|
7732
|
+
]);
|
|
7733
|
+
function isOperatorCommandReason(reason) {
|
|
7734
|
+
return OPERATOR_COMMAND_REASONS.has(reason);
|
|
7649
7735
|
}
|
|
7650
|
-
|
|
7651
|
-
|
|
7652
|
-
|
|
7653
|
-
|
|
7654
|
-
|
|
7655
|
-
const eq = line.indexOf("=");
|
|
7656
|
-
if (eq <= 0) continue;
|
|
7657
|
-
const name = line.slice(0, eq).trim();
|
|
7658
|
-
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) continue;
|
|
7659
|
-
const value = line.slice(eq + 1);
|
|
7660
|
-
entries.set(name, value);
|
|
7661
|
-
}
|
|
7662
|
-
return entries;
|
|
7736
|
+
var OPERATOR_CONFIG_REASONS = /* @__PURE__ */ new Set([
|
|
7737
|
+
"operator-integration-change"
|
|
7738
|
+
]);
|
|
7739
|
+
function isOperatorConfigReason(reason) {
|
|
7740
|
+
return OPERATOR_CONFIG_REASONS.has(reason);
|
|
7663
7741
|
}
|
|
7664
|
-
|
|
7665
|
-
|
|
7666
|
-
|
|
7667
|
-
|
|
7668
|
-
|
|
7669
|
-
|
|
7670
|
-
}
|
|
7671
|
-
for (const name of oldEntries.keys()) {
|
|
7672
|
-
if (!newEntries.has(name)) changed.add(name);
|
|
7673
|
-
}
|
|
7674
|
-
return [...changed];
|
|
7742
|
+
var NEVER_INTEGRATION_BUCKETED = /* @__PURE__ */ new Set([
|
|
7743
|
+
"channel-set-change"
|
|
7744
|
+
]);
|
|
7745
|
+
function isIntegrationBucketedReason(reason) {
|
|
7746
|
+
if (NEVER_INTEGRATION_BUCKETED.has(reason)) return false;
|
|
7747
|
+
return isProvisioningReloadReason(reason) || isOperatorConfigReason(reason);
|
|
7675
7748
|
}
|
|
7676
|
-
function
|
|
7677
|
-
|
|
7678
|
-
|
|
7679
|
-
|
|
7680
|
-
|
|
7681
|
-
|
|
7682
|
-
|
|
7683
|
-
if (!oldEntries.has(name)) added.push(name);
|
|
7684
|
-
else if (oldEntries.get(name) !== value) rotated.push(name);
|
|
7685
|
-
}
|
|
7686
|
-
for (const name of oldEntries.keys()) {
|
|
7687
|
-
if (!newEntries.has(name)) removed.push(name);
|
|
7688
|
-
}
|
|
7689
|
-
return { rotated, added, removed };
|
|
7749
|
+
function restartReasonClass(reason) {
|
|
7750
|
+
if (isCredentialRotationReason(reason)) return "credential-rotation";
|
|
7751
|
+
if (isSelfHealingReason(reason)) return "self-healing";
|
|
7752
|
+
if (isOperatorConfigReason(reason)) return "operator-config";
|
|
7753
|
+
if (isScheduledRestartReason(reason)) return "scheduled";
|
|
7754
|
+
if (isOperatorCommandReason(reason)) return "operator-command";
|
|
7755
|
+
return isProvisioningReloadReason(reason) ? "provisioning" : "crash";
|
|
7690
7756
|
}
|
|
7691
|
-
function
|
|
7692
|
-
|
|
7693
|
-
|
|
7694
|
-
|
|
7695
|
-
|
|
7696
|
-
|
|
7697
|
-
|
|
7698
|
-
|
|
7699
|
-
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
|
|
7704
|
-
|
|
7705
|
-
|
|
7706
|
-
|
|
7707
|
-
|
|
7708
|
-
break;
|
|
7709
|
-
}
|
|
7710
|
-
}
|
|
7711
|
-
}
|
|
7712
|
-
if (matches) break;
|
|
7713
|
-
}
|
|
7714
|
-
if (matches) result.push(serverKey);
|
|
7757
|
+
function countByClass(events) {
|
|
7758
|
+
let crash = 0;
|
|
7759
|
+
let provisioning = 0;
|
|
7760
|
+
let credentialRotation = 0;
|
|
7761
|
+
let selfHealing = 0;
|
|
7762
|
+
let operatorConfig = 0;
|
|
7763
|
+
let scheduled = 0;
|
|
7764
|
+
let operatorCommand = 0;
|
|
7765
|
+
for (const e of events) {
|
|
7766
|
+
const klass = restartReasonClass(e.reason);
|
|
7767
|
+
if (klass === "provisioning") provisioning += 1;
|
|
7768
|
+
else if (klass === "credential-rotation") credentialRotation += 1;
|
|
7769
|
+
else if (klass === "self-healing") selfHealing += 1;
|
|
7770
|
+
else if (klass === "operator-config") operatorConfig += 1;
|
|
7771
|
+
else if (klass === "scheduled") scheduled += 1;
|
|
7772
|
+
else if (klass === "operator-command") operatorCommand += 1;
|
|
7773
|
+
else crash += 1;
|
|
7715
7774
|
}
|
|
7716
|
-
return
|
|
7775
|
+
return {
|
|
7776
|
+
crash,
|
|
7777
|
+
provisioning,
|
|
7778
|
+
"credential-rotation": credentialRotation,
|
|
7779
|
+
"self-healing": selfHealing,
|
|
7780
|
+
"operator-config": operatorConfig,
|
|
7781
|
+
scheduled,
|
|
7782
|
+
"operator-command": operatorCommand
|
|
7783
|
+
};
|
|
7717
7784
|
}
|
|
7718
|
-
function
|
|
7719
|
-
|
|
7720
|
-
return [...changedVars].filter(
|
|
7721
|
-
(v) => !channelSet.has(v) && findMcpServersUsingVars(mcp, [v]).length === 0
|
|
7722
|
-
);
|
|
7785
|
+
function maxProvisioningBucket(events) {
|
|
7786
|
+
return maxBucketWhere(events, isProvisioningReloadReason);
|
|
7723
7787
|
}
|
|
7724
|
-
function
|
|
7725
|
-
const
|
|
7726
|
-
|
|
7727
|
-
|
|
7728
|
-
const
|
|
7729
|
-
|
|
7730
|
-
for (const pair of spec.split(",")) {
|
|
7731
|
-
const colon = pair.indexOf(":");
|
|
7732
|
-
if (colon <= 0) continue;
|
|
7733
|
-
const varName = pair.slice(colon + 1).trim();
|
|
7734
|
-
if (varName) out.add(varName);
|
|
7735
|
-
}
|
|
7788
|
+
function maxBucketWhere(events, matches) {
|
|
7789
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
7790
|
+
for (const e of events) {
|
|
7791
|
+
if (!matches(e.reason)) continue;
|
|
7792
|
+
const key = e.integrationKey ?? "";
|
|
7793
|
+
buckets.set(key, (buckets.get(key) ?? 0) + 1);
|
|
7736
7794
|
}
|
|
7737
|
-
|
|
7738
|
-
|
|
7739
|
-
|
|
7740
|
-
const out = /* @__PURE__ */ new Set();
|
|
7741
|
-
if (!mcp?.mcpServers) return out;
|
|
7742
|
-
for (const entry of Object.values(mcp.mcpServers)) {
|
|
7743
|
-
const tokenVar = entry?.env?.["AGT_REMOTE_MCP_TOKEN_VAR"];
|
|
7744
|
-
if (typeof tokenVar === "string" && tokenVar.trim()) {
|
|
7745
|
-
out.add(tokenVar.trim());
|
|
7746
|
-
}
|
|
7795
|
+
let best = { count: 0, key: "" };
|
|
7796
|
+
for (const [key, count] of buckets) {
|
|
7797
|
+
if (count > best.count) best = { count, key };
|
|
7747
7798
|
}
|
|
7748
|
-
return
|
|
7799
|
+
return best;
|
|
7749
7800
|
}
|
|
7750
|
-
function
|
|
7751
|
-
|
|
7752
|
-
|
|
7753
|
-
|
|
7754
|
-
|
|
7755
|
-
|
|
7756
|
-
|
|
7757
|
-
|
|
7758
|
-
|
|
7759
|
-
|
|
7760
|
-
|
|
7761
|
-
|
|
7762
|
-
|
|
7763
|
-
|
|
7764
|
-
|
|
7765
|
-
|
|
7766
|
-
|
|
7767
|
-
|
|
7801
|
+
function tripClass(trip) {
|
|
7802
|
+
if (trip.trippedClass) return trip.trippedClass;
|
|
7803
|
+
if (!Array.isArray(trip.eventsAtTrip) || trip.eventsAtTrip.length === 0) return "crash";
|
|
7804
|
+
return trip.eventsAtTrip.every((e) => isProvisioningReloadReason(e.reason)) ? "provisioning" : "crash";
|
|
7805
|
+
}
|
|
7806
|
+
var MAX_NEW_AGENT_GRACE_MS = 365 * 24 * 60 * 60 * 1e3;
|
|
7807
|
+
var DEFAULT_MAX = 2;
|
|
7808
|
+
var DEFAULT_WINDOW_MS = 6e5;
|
|
7809
|
+
var DEFAULT_PROVISIONING_MAX = RESTART_BREAKER_PROVISIONING_MAX;
|
|
7810
|
+
var DEFAULT_PROVISIONING_WINDOW_MS = RESTART_BREAKER_PROVISIONING_WINDOW_MS;
|
|
7811
|
+
var DEFAULT_OPERATOR_MAX = RESTART_BREAKER_OPERATOR_MAX;
|
|
7812
|
+
var DEFAULT_SCHEDULED_MAX = RESTART_BREAKER_SCHEDULED_MAX;
|
|
7813
|
+
var DEFAULT_SCHEDULED_WINDOW_MS = RESTART_BREAKER_SCHEDULED_WINDOW_MS;
|
|
7814
|
+
var DEFAULT_OPERATOR_COMMAND_MAX = RESTART_BREAKER_OPERATOR_COMMAND_MAX;
|
|
7815
|
+
var DEFAULT_OPERATOR_COMMAND_WINDOW_MS = RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS;
|
|
7816
|
+
function readEnvNumber(name, fallback) {
|
|
7817
|
+
const raw = process.env[name];
|
|
7818
|
+
if (!raw) return fallback;
|
|
7819
|
+
const parsed = Number(raw);
|
|
7820
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
7821
|
+
}
|
|
7822
|
+
function migrateMapKey(map, from, to) {
|
|
7823
|
+
if (map.has(to) || !map.has(from)) return;
|
|
7824
|
+
map.set(to, map.get(from));
|
|
7825
|
+
map.delete(from);
|
|
7826
|
+
}
|
|
7827
|
+
var RestartBreaker = class {
|
|
7828
|
+
max;
|
|
7829
|
+
windowMs;
|
|
7830
|
+
/**
|
|
7831
|
+
* ENG-8215: NOT readonly - the constructor may clamp it upward to preserve the
|
|
7832
|
+
* quarantine-fires-first ordering against a runtime override. Never mutated
|
|
7833
|
+
* after construction.
|
|
7834
|
+
*/
|
|
7835
|
+
provisioningMax;
|
|
7836
|
+
provisioningWindowMs;
|
|
7837
|
+
/**
|
|
7838
|
+
* ENG-8388: the looser operator-config budget. Shares provisioningWindowMs.
|
|
7839
|
+
* NOT readonly - the constructor floors it at `provisioningMax` so setting one
|
|
7840
|
+
* env var without the other cannot invert the two tallies.
|
|
7841
|
+
*/
|
|
7842
|
+
operatorMax;
|
|
7843
|
+
/**
|
|
7844
|
+
* ENG-9138: the scheduled class's LOOP bar. Readonly and deliberately NOT
|
|
7845
|
+
* floored against any other tally: it gates a REPORT, not a pause, so it
|
|
7846
|
+
* cannot invert anything the way operatorMax/provisioningMax can.
|
|
7847
|
+
*/
|
|
7848
|
+
scheduledMax;
|
|
7849
|
+
/** ENG-9138: sliding window for the scheduled tally. */
|
|
7850
|
+
scheduledWindowMs;
|
|
7851
|
+
operatorCommandMax;
|
|
7852
|
+
operatorCommandWindowMs;
|
|
7853
|
+
/** Longest window any tally needs — how long the events log must retain. */
|
|
7854
|
+
retentionMs;
|
|
7855
|
+
now;
|
|
7856
|
+
/** ENG-7891 slice 5b: code_name → durable runtime key (agent_id when id-keyed). Identity on the legacy fleet. */
|
|
7857
|
+
runtimeKey;
|
|
7858
|
+
/** ENG-8690: agent creation time lookup; undefined result ⇒ no grace. */
|
|
7859
|
+
agentCreatedAt;
|
|
7860
|
+
/** ENG-8690: grace duration; 0 disables. */
|
|
7861
|
+
newAgentGraceMs;
|
|
7862
|
+
/** ENG-8690: the bound the grace does NOT lift. NOT readonly - floored below. */
|
|
7863
|
+
newAgentGraceCeiling;
|
|
7864
|
+
events = /* @__PURE__ */ new Map();
|
|
7865
|
+
trips = /* @__PURE__ */ new Map();
|
|
7866
|
+
/**
|
|
7867
|
+
* ENG-8215: set when a supplied `provisioningMax` was raised to keep MCP
|
|
7868
|
+
* quarantine ahead of the breaker. Read once by the manager at construction so
|
|
7869
|
+
* an overridden-and-clamped value is visible in manager.log rather than
|
|
7870
|
+
* silently discarded.
|
|
7871
|
+
*/
|
|
7872
|
+
quarantineOrderingClamped;
|
|
7873
|
+
constructor(opts = {}) {
|
|
7874
|
+
this.max = opts.max ?? readEnvNumber("AGT_RESTART_BREAKER_MAX", DEFAULT_MAX);
|
|
7875
|
+
this.windowMs = opts.windowMs ?? readEnvNumber("AGT_RESTART_BREAKER_WINDOW_MS", DEFAULT_WINDOW_MS);
|
|
7876
|
+
this.provisioningMax = opts.provisioningMax ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", DEFAULT_PROVISIONING_MAX);
|
|
7877
|
+
this.provisioningWindowMs = opts.provisioningWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS", DEFAULT_PROVISIONING_WINDOW_MS);
|
|
7878
|
+
this.operatorMax = opts.operatorMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_MAX", DEFAULT_OPERATOR_MAX);
|
|
7879
|
+
this.scheduledMax = opts.scheduledMax ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_MAX", DEFAULT_SCHEDULED_MAX);
|
|
7880
|
+
this.scheduledWindowMs = opts.scheduledWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_WINDOW_MS", DEFAULT_SCHEDULED_WINDOW_MS);
|
|
7881
|
+
this.operatorCommandMax = opts.operatorCommandMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_COMMAND_MAX", DEFAULT_OPERATOR_COMMAND_MAX);
|
|
7882
|
+
this.operatorCommandWindowMs = opts.operatorCommandWindowMs ?? readEnvNumber(
|
|
7883
|
+
"AGT_RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS",
|
|
7884
|
+
DEFAULT_OPERATOR_COMMAND_WINDOW_MS
|
|
7768
7885
|
);
|
|
7769
|
-
|
|
7770
|
-
|
|
7771
|
-
|
|
7772
|
-
|
|
7773
|
-
|
|
7774
|
-
|
|
7775
|
-
|
|
7776
|
-
|
|
7886
|
+
this.now = opts.now ?? Date.now;
|
|
7887
|
+
this.runtimeKey = opts.runtimeKey ?? ((codeName) => codeName);
|
|
7888
|
+
this.agentCreatedAt = opts.agentCreatedAt ?? (() => void 0);
|
|
7889
|
+
const graceMs = opts.newAgentGraceMs ?? readEnvNumber("AGT_RESTART_BREAKER_NEW_AGENT_GRACE_MS", RESTART_BREAKER_NEW_AGENT_GRACE_MS);
|
|
7890
|
+
const cappedGrace = Math.min(graceMs, MAX_NEW_AGENT_GRACE_MS);
|
|
7891
|
+
this.newAgentGraceMs = Number.isFinite(graceMs) && graceMs > 0 ? cappedGrace : 0;
|
|
7892
|
+
const ceiling = opts.newAgentGraceCeiling ?? readEnvNumber(
|
|
7893
|
+
"AGT_RESTART_BREAKER_NEW_AGENT_GRACE_CEILING",
|
|
7894
|
+
RESTART_BREAKER_NEW_AGENT_GRACE_CEILING
|
|
7895
|
+
);
|
|
7896
|
+
this.newAgentGraceCeiling = Number.isFinite(ceiling) && ceiling > 0 ? ceiling : RESTART_BREAKER_NEW_AGENT_GRACE_CEILING;
|
|
7897
|
+
if (!Number.isFinite(this.max) || this.max < 1) {
|
|
7898
|
+
throw new Error("restart-breaker max must be a finite number >= 1");
|
|
7777
7899
|
}
|
|
7778
|
-
if (
|
|
7779
|
-
|
|
7780
|
-
if (basename) {
|
|
7781
|
-
patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
|
|
7782
|
-
break;
|
|
7783
|
-
}
|
|
7900
|
+
if (!Number.isFinite(this.windowMs) || this.windowMs < 1e3) {
|
|
7901
|
+
throw new Error("restart-breaker windowMs must be a finite number >= 1000");
|
|
7784
7902
|
}
|
|
7785
|
-
|
|
7786
|
-
|
|
7787
|
-
const safe = escapeRe(key);
|
|
7788
|
-
patterns.push(new RegExp(`(?<![A-Za-z0-9_])${safe}(?![A-Za-z0-9_])`));
|
|
7789
|
-
if (safe.includes("_")) {
|
|
7790
|
-
const dashed = safe.replace(/_/g, "-");
|
|
7791
|
-
patterns.push(new RegExp(`(?<![A-Za-z0-9_])${dashed}(?![A-Za-z0-9_])`));
|
|
7903
|
+
if (!Number.isFinite(this.provisioningMax) || this.provisioningMax < 1) {
|
|
7904
|
+
throw new Error("restart-breaker provisioningMax must be a finite number >= 1");
|
|
7792
7905
|
}
|
|
7793
|
-
|
|
7794
|
-
|
|
7795
|
-
}
|
|
7796
|
-
function buildClaudeAgentMatcher(codeName) {
|
|
7797
|
-
const safe = codeName.replace(/[^A-Za-z0-9_-]/g, "");
|
|
7798
|
-
return new RegExp(`\\bclaude\\b.*--name\\s+agt-${safe}(?=\\s|$)`);
|
|
7799
|
-
}
|
|
7800
|
-
function findMcpChildrenForAgent(args) {
|
|
7801
|
-
const { rows, codeName, serverKeys, mcpJson } = args;
|
|
7802
|
-
const maxDepth = args.maxDepth ?? 8;
|
|
7803
|
-
const inContainer = args.inContainer ?? false;
|
|
7804
|
-
if (serverKeys.length === 0 || rows.length === 0) return [];
|
|
7805
|
-
const claudeMatcher = inContainer ? /\bclaude\b/ : buildClaudeAgentMatcher(codeName);
|
|
7806
|
-
const argvMatchers = [];
|
|
7807
|
-
for (const key of serverKeys) {
|
|
7808
|
-
const entry = mcpJson?.mcpServers?.[key];
|
|
7809
|
-
argvMatchers.push(...buildArgvMatchersForEntry(key, entry));
|
|
7810
|
-
}
|
|
7811
|
-
const byPid = new Map(rows.map((r) => [r.pid, r]));
|
|
7812
|
-
const matched = [];
|
|
7813
|
-
for (const row of rows) {
|
|
7814
|
-
if (!argvMatchers.some((re) => re.test(row.args))) continue;
|
|
7815
|
-
if (/\bclaude\b/.test(row.args) && row.args.includes(`--name agt-${codeName}`)) continue;
|
|
7816
|
-
let cur = byPid.get(row.ppid);
|
|
7817
|
-
let belongs = false;
|
|
7818
|
-
for (let depth = 0; depth < maxDepth && cur; depth++) {
|
|
7819
|
-
if (claudeMatcher.test(cur.args)) {
|
|
7820
|
-
belongs = true;
|
|
7821
|
-
break;
|
|
7822
|
-
}
|
|
7823
|
-
if (cur.pid === 1) break;
|
|
7824
|
-
cur = byPid.get(cur.ppid);
|
|
7906
|
+
if (!Number.isFinite(this.operatorMax) || this.operatorMax < 1) {
|
|
7907
|
+
throw new Error("restart-breaker operatorMax must be a finite number >= 1");
|
|
7825
7908
|
}
|
|
7826
|
-
if (
|
|
7827
|
-
|
|
7828
|
-
return matched;
|
|
7829
|
-
}
|
|
7830
|
-
function reapStaleMcpChildren(args) {
|
|
7831
|
-
const { log: log2, codeName, serverKeys, mcpJson, graceMs = 5e3, isolated = false } = args;
|
|
7832
|
-
if (serverKeys.length === 0) return [];
|
|
7833
|
-
const runPs = args.runPs ?? (isolated ? (
|
|
7834
|
-
// ENG-9781: hard-bounded. See bounded-exec.ts — `execFileSync`'s own
|
|
7835
|
-
// `timeout` is not a bound, and this runs per isolated agent per cycle
|
|
7836
|
-
// on the manager-worker's critical path. Throws on a non-answer so the
|
|
7837
|
-
// caller's existing catch skips the agent rather than acting on an
|
|
7838
|
-
// empty process list.
|
|
7839
|
-
() => {
|
|
7840
|
-
const r = hardBoundedExecFile(
|
|
7841
|
-
"docker",
|
|
7842
|
-
["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
|
|
7843
|
-
{ timeoutMs: 8e3, log: log2 }
|
|
7844
|
-
);
|
|
7845
|
-
if (r.kind === "ok") return r.stdout;
|
|
7846
|
-
throw new Error(
|
|
7847
|
-
r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
|
|
7848
|
-
);
|
|
7909
|
+
if (this.operatorMax < this.provisioningMax) {
|
|
7910
|
+
this.operatorMax = this.provisioningMax;
|
|
7849
7911
|
}
|
|
7850
|
-
|
|
7851
|
-
|
|
7852
|
-
|
|
7853
|
-
|
|
7854
|
-
|
|
7855
|
-
|
|
7856
|
-
|
|
7857
|
-
} : (pid, signal) => {
|
|
7858
|
-
try {
|
|
7859
|
-
process.kill(pid, signal);
|
|
7860
|
-
} catch {
|
|
7912
|
+
if (this.provisioningMax < MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING) {
|
|
7913
|
+
this.quarantineOrderingClamped = {
|
|
7914
|
+
requested: this.provisioningMax,
|
|
7915
|
+
applied: MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING,
|
|
7916
|
+
quarantineThreshold: BIND_FAILURE_QUARANTINE_THRESHOLD
|
|
7917
|
+
};
|
|
7918
|
+
this.provisioningMax = MIN_PROVISIONING_MAX_FOR_QUARANTINE_ORDERING;
|
|
7861
7919
|
}
|
|
7862
|
-
|
|
7863
|
-
|
|
7864
|
-
const r = hardBoundedExecFile(
|
|
7865
|
-
"docker",
|
|
7866
|
-
["exec", `agt-${codeName}`, "kill", "-0", String(pid)],
|
|
7867
|
-
{ timeoutMs: 8e3, log: log2 }
|
|
7868
|
-
);
|
|
7869
|
-
return livenessFromProbe(
|
|
7870
|
-
r,
|
|
7871
|
-
(why) => log2(
|
|
7872
|
-
`[stale-mcp-reaper] liveness probe for pid ${pid} in 'agt-${codeName}' ${why} \u2014 treating as ALIVE (no reap) rather than guessing`
|
|
7873
|
-
)
|
|
7874
|
-
);
|
|
7875
|
-
} : (pid) => {
|
|
7876
|
-
try {
|
|
7877
|
-
process.kill(pid, 0);
|
|
7878
|
-
return true;
|
|
7879
|
-
} catch {
|
|
7880
|
-
return false;
|
|
7920
|
+
if (!Number.isFinite(this.provisioningWindowMs) || this.provisioningWindowMs < 1e3) {
|
|
7921
|
+
throw new Error("restart-breaker provisioningWindowMs must be a finite number >= 1000");
|
|
7881
7922
|
}
|
|
7882
|
-
|
|
7883
|
-
|
|
7884
|
-
try {
|
|
7885
|
-
psOutput = runPs();
|
|
7886
|
-
} catch (err) {
|
|
7887
|
-
log2(`[stale-mcp-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping reap`);
|
|
7888
|
-
return [];
|
|
7889
|
-
}
|
|
7890
|
-
const rows = parsePsRows(psOutput);
|
|
7891
|
-
const targets = findMcpChildrenForAgent({ rows, codeName, serverKeys, mcpJson, inContainer: isolated });
|
|
7892
|
-
if (targets.length === 0) return [];
|
|
7893
|
-
const byPid = new Map(rows.map((r) => [r.pid, r]));
|
|
7894
|
-
const describe = (pid) => {
|
|
7895
|
-
const argv = byPid.get(pid)?.args ?? "";
|
|
7896
|
-
const pkgMatch = argv.match(/(@[a-z0-9_-]+\/[a-z0-9_-]+|[a-z0-9_-]+-mcp-server|[a-z0-9_-]+-mcp\b)/i);
|
|
7897
|
-
return pkgMatch ? `${pkgMatch[1]} (pid ${pid})` : `pid ${pid}`;
|
|
7898
|
-
};
|
|
7899
|
-
log2(
|
|
7900
|
-
`[stale-mcp-reaper] '${codeName}': rotating ${targets.length} stale MCP child(ren) for [${serverKeys.join(", ")}]: ${targets.map(describe).join(", ")}`
|
|
7901
|
-
);
|
|
7902
|
-
for (const pid of targets) {
|
|
7903
|
-
killProcess(pid, "SIGTERM");
|
|
7904
|
-
}
|
|
7905
|
-
const killedKeys = /* @__PURE__ */ new Set();
|
|
7906
|
-
for (const pid of targets) {
|
|
7907
|
-
const argv = byPid.get(pid)?.args ?? "";
|
|
7908
|
-
for (const key of serverKeys) {
|
|
7909
|
-
const entry = mcpJson?.mcpServers?.[key];
|
|
7910
|
-
const matchers = buildArgvMatchersForEntry(key, entry);
|
|
7911
|
-
if (matchers.some((re) => re.test(argv))) killedKeys.add(key);
|
|
7923
|
+
if (!Number.isFinite(this.scheduledMax) || this.scheduledMax < 1) {
|
|
7924
|
+
throw new Error("restart-breaker scheduledMax must be a finite number >= 1");
|
|
7912
7925
|
}
|
|
7913
|
-
|
|
7914
|
-
|
|
7915
|
-
setTimeout(() => {
|
|
7916
|
-
try {
|
|
7917
|
-
let freshPsOutput;
|
|
7918
|
-
try {
|
|
7919
|
-
freshPsOutput = runPs();
|
|
7920
|
-
} catch (err) {
|
|
7921
|
-
log2(`[stale-mcp-reaper] '${codeName}': fresh ps for SIGKILL re-verify failed: ${err.message} \u2014 skipping SIGKILL pass`);
|
|
7922
|
-
return;
|
|
7923
|
-
}
|
|
7924
|
-
const stillOwned = new Set(
|
|
7925
|
-
findMcpChildrenForAgent({
|
|
7926
|
-
rows: parsePsRows(freshPsOutput),
|
|
7927
|
-
codeName,
|
|
7928
|
-
serverKeys,
|
|
7929
|
-
mcpJson,
|
|
7930
|
-
inContainer: isolated
|
|
7931
|
-
})
|
|
7932
|
-
);
|
|
7933
|
-
const stragglers = targets.filter((pid) => isAlive(pid) && stillOwned.has(pid));
|
|
7934
|
-
if (stragglers.length === 0) return;
|
|
7935
|
-
log2(
|
|
7936
|
-
`[stale-mcp-reaper] '${codeName}': ${stragglers.length} child(ren) survived SIGTERM; sending SIGKILL: ${stragglers.map(describe).join(", ")}`
|
|
7937
|
-
);
|
|
7938
|
-
for (const pid of stragglers) {
|
|
7939
|
-
killProcess(pid, "SIGKILL");
|
|
7940
|
-
}
|
|
7941
|
-
} catch (err) {
|
|
7942
|
-
log2(`[stale-mcp-reaper] '${codeName}': error in SIGKILL pass: ${err.message}`);
|
|
7926
|
+
if (!Number.isFinite(this.scheduledWindowMs) || this.scheduledWindowMs < 1e3) {
|
|
7927
|
+
throw new Error("restart-breaker scheduledWindowMs must be a finite number >= 1000");
|
|
7943
7928
|
}
|
|
7944
|
-
|
|
7945
|
-
|
|
7946
|
-
}
|
|
7947
|
-
|
|
7948
|
-
|
|
7949
|
-
|
|
7950
|
-
|
|
7951
|
-
|
|
7952
|
-
|
|
7953
|
-
var BACKOFF_BASE_MS = 6e4;
|
|
7954
|
-
var BACKOFF_MAX_MS = 30 * 6e4;
|
|
7955
|
-
var BACKOFF_WINDOW_MS = 60 * 6e4;
|
|
7956
|
-
var BACKOFF_GIVE_UP_RESTARTS = 2 * MAX_PRESENCE_RESTART_ATTEMPTS;
|
|
7957
|
-
var BACKOFF_STABLE_RESET_MS = 5 * 6e4;
|
|
7958
|
-
var DEAD_CHANNEL_MS = 15 * 6e4;
|
|
7959
|
-
var presenceReaperState = /* @__PURE__ */ new Map();
|
|
7960
|
-
var MAX_ROTATION_PROMPT_PROBES = 3;
|
|
7961
|
-
var ROTATION_PROMPT_PROBE_INTERVAL_MS = 3e4;
|
|
7962
|
-
var rotationStallState = /* @__PURE__ */ new Map();
|
|
7963
|
-
function stateKey(codeName, serverKey) {
|
|
7964
|
-
return `${codeName}\0${serverKey}`;
|
|
7965
|
-
}
|
|
7966
|
-
function clearPresenceReaperState(codeName) {
|
|
7967
|
-
const prefix = `${codeName}\0`;
|
|
7968
|
-
for (const key of presenceReaperState.keys()) {
|
|
7969
|
-
if (key.startsWith(prefix)) presenceReaperState.delete(key);
|
|
7970
|
-
}
|
|
7971
|
-
}
|
|
7972
|
-
function clearPresenceReaperStateForKeys(codeName, keys) {
|
|
7973
|
-
for (const key of keys) {
|
|
7974
|
-
presenceReaperState.delete(stateKey(codeName, key));
|
|
7975
|
-
rotationStallState.delete(stateKey(codeName, key));
|
|
7976
|
-
}
|
|
7977
|
-
}
|
|
7978
|
-
function givenUpMcpServerKeys(codeName) {
|
|
7979
|
-
const prefix = `${codeName}\0`;
|
|
7980
|
-
const out = /* @__PURE__ */ new Set();
|
|
7981
|
-
for (const [key, st] of presenceReaperState) {
|
|
7982
|
-
if (key.startsWith(prefix) && (st.gaveUp || st.attempts >= MAX_PRESENCE_RESTART_ATTEMPTS)) {
|
|
7983
|
-
out.add(key.slice(prefix.length));
|
|
7929
|
+
if (!Number.isFinite(this.operatorCommandMax) || this.operatorCommandMax < 1) {
|
|
7930
|
+
throw new Error("restart-breaker operatorCommandMax must be a finite number >= 1");
|
|
7931
|
+
}
|
|
7932
|
+
if (!Number.isFinite(this.operatorCommandWindowMs) || this.operatorCommandWindowMs < 1e3) {
|
|
7933
|
+
throw new Error("restart-breaker operatorCommandWindowMs must be a finite number >= 1000");
|
|
7934
|
+
}
|
|
7935
|
+
const minimumGraceCeiling = Math.max(this.provisioningMax, this.operatorMax);
|
|
7936
|
+
if (this.newAgentGraceCeiling < minimumGraceCeiling) {
|
|
7937
|
+
this.newAgentGraceCeiling = minimumGraceCeiling;
|
|
7984
7938
|
}
|
|
7939
|
+
this.retentionMs = Math.max(
|
|
7940
|
+
this.windowMs,
|
|
7941
|
+
this.provisioningWindowMs,
|
|
7942
|
+
this.scheduledWindowMs,
|
|
7943
|
+
this.operatorCommandWindowMs
|
|
7944
|
+
);
|
|
7985
7945
|
}
|
|
7986
|
-
|
|
7987
|
-
|
|
7988
|
-
|
|
7989
|
-
|
|
7990
|
-
|
|
7991
|
-
|
|
7992
|
-
|
|
7993
|
-
const missing = [];
|
|
7994
|
-
for (const key of declared) {
|
|
7995
|
-
const entry = servers[key];
|
|
7996
|
-
const isStdio = typeof entry?.command === "string";
|
|
7997
|
-
if (!isStdio) continue;
|
|
7998
|
-
const live = findMcpChildrenForAgent({
|
|
7999
|
-
rows,
|
|
8000
|
-
codeName,
|
|
8001
|
-
serverKeys: [key],
|
|
8002
|
-
mcpJson,
|
|
8003
|
-
inContainer
|
|
8004
|
-
});
|
|
8005
|
-
if (live.length === 0) missing.push(key);
|
|
7946
|
+
/**
|
|
7947
|
+
* ENG-8215: the clamp record, if the configured provisioning max had to be
|
|
7948
|
+
* raised to keep MCP quarantine ahead of the breaker. `undefined` when the
|
|
7949
|
+
* configured value already satisfied the ordering.
|
|
7950
|
+
*/
|
|
7951
|
+
getQuarantineOrderingClamp() {
|
|
7952
|
+
return this.quarantineOrderingClamped;
|
|
8006
7953
|
}
|
|
8007
|
-
|
|
8008
|
-
|
|
8009
|
-
|
|
8010
|
-
|
|
8011
|
-
|
|
8012
|
-
|
|
8013
|
-
|
|
8014
|
-
|
|
8015
|
-
|
|
8016
|
-
|
|
8017
|
-
|
|
8018
|
-
|
|
8019
|
-
|
|
8020
|
-
|
|
8021
|
-
|
|
8022
|
-
|
|
8023
|
-
|
|
8024
|
-
|
|
8025
|
-
|
|
8026
|
-
|
|
8027
|
-
|
|
8028
|
-
|
|
8029
|
-
|
|
8030
|
-
|
|
8031
|
-
|
|
8032
|
-
quarantineDwellMs = 0,
|
|
8033
|
-
deadChannelMs = DEAD_CHANNEL_MS,
|
|
8034
|
-
quarantineMode = "off",
|
|
8035
|
-
onQuarantine,
|
|
8036
|
-
isolated = false,
|
|
8037
|
-
// ENG-8013: default false → a "no live child" observation is only excluded
|
|
8038
|
-
// from the give-up/quarantine budget when the caller positively identifies
|
|
8039
|
-
// the current generation as externally-churned. Legacy callers/tests unchanged.
|
|
8040
|
-
currentGenerationExternallyStarted = false,
|
|
8041
|
-
// ENG-9932: unwired => strict no-op, restart proceeds exactly as before.
|
|
8042
|
-
readPane,
|
|
8043
|
-
onPaneBlocked
|
|
8044
|
-
} = args;
|
|
8045
|
-
const now = args.now ?? Date.now;
|
|
8046
|
-
const runPs = args.runPs ?? (isolated ? (
|
|
8047
|
-
// ENG-9781: HARD-bounded, and the distinction is the whole ticket. The
|
|
8048
|
-
// previous `execFileSync(..., { timeout: 8_000 })` bounded nothing —
|
|
8049
|
-
// Node fires SIGTERM at the deadline and then keeps waiting, so a
|
|
8050
|
-
// docker CLI talking to a wedged daemon blocked the manager-worker's
|
|
8051
|
-
// event loop and stopped scheduling for EVERY agent on the host
|
|
8052
|
-
// (33m46s, twice, on 2026-09-01). This call runs once per isolated
|
|
8053
|
-
// agent per poll cycle, so it is the single worst place on the manager
|
|
8054
|
-
// for an unbounded subprocess.
|
|
8055
|
-
//
|
|
8056
|
-
// Throwing on a non-answer is deliberate rather than lazy: the caller
|
|
8057
|
-
// already catches, logs and SKIPS the agent for the cycle, which is
|
|
8058
|
-
// exactly right. A timeout must not become an empty process list —
|
|
8059
|
-
// that reads as "every declared MCP server is missing" and restarts the
|
|
8060
|
-
// session, turning a docker hiccup into a fleet-wide restart storm.
|
|
8061
|
-
() => {
|
|
8062
|
-
const r = hardBoundedExecFile(
|
|
8063
|
-
"docker",
|
|
8064
|
-
["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
|
|
8065
|
-
{ timeoutMs: 8e3, log: log2 }
|
|
8066
|
-
);
|
|
8067
|
-
if (r.kind === "ok") return r.stdout;
|
|
8068
|
-
throw new Error(
|
|
8069
|
-
r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
|
|
8070
|
-
);
|
|
7954
|
+
/**
|
|
7955
|
+
* ENG-7891 slice 5b: the DURABLE key for this agent's events/trips entries.
|
|
7956
|
+
* Resolves the code_name to the runtime key (agent_id on id-keyed hosts) and,
|
|
7957
|
+
* when they differ, lazily migrates a legacy code_name-keyed entry onto it — so
|
|
7958
|
+
* a trip persisted under the old key by a pre-arming manager (or from before a
|
|
7959
|
+
* codename rename) is found under the new key instead of being lost. Identity on
|
|
7960
|
+
* the legacy fleet ⇒ from === to ⇒ no migration and no behaviour change. Both
|
|
7961
|
+
* maps migrate together so events and a trip for the same agent never split
|
|
7962
|
+
* across keys mid-generation (a migration can flip the resolver mid-process).
|
|
7963
|
+
*
|
|
7964
|
+
* BOUNDARY — a pure-LEGACY rename (a host that was never armed) still strands a
|
|
7965
|
+
* held trip, exactly as it did before slice 5b: with no symlink there is no
|
|
7966
|
+
* agent_id to bridge the old codename to the new, so the runtime key stays the
|
|
7967
|
+
* (new) codename and the trip persisted under the old codename is not found.
|
|
7968
|
+
* That is unchanged, pre-existing behaviour and it is benign — the fresh
|
|
7969
|
+
* generation re-evaluates the agent under its new name and a still-pathological
|
|
7970
|
+
* agent simply re-trips within the window. The rename-survival guarantee this
|
|
7971
|
+
* slice adds is an id-keyed-only, post-arming property (the symlink is the
|
|
7972
|
+
* bridge); it is deliberately NOT retrofitted onto the legacy path.
|
|
7973
|
+
*/
|
|
7974
|
+
keyFor(codeName) {
|
|
7975
|
+
const key = this.runtimeKey(codeName);
|
|
7976
|
+
if (key !== codeName) {
|
|
7977
|
+
migrateMapKey(this.events, codeName, key);
|
|
7978
|
+
migrateMapKey(this.trips, codeName, key);
|
|
8071
7979
|
}
|
|
8072
|
-
|
|
8073
|
-
if (!mcpJson?.mcpServers) {
|
|
8074
|
-
return { missing: [], restarted: false, reason: "no-mcp-json" };
|
|
8075
|
-
}
|
|
8076
|
-
const declaredCount = Object.keys(mcpJson.mcpServers).length;
|
|
8077
|
-
if (declaredCount === 0) {
|
|
8078
|
-
return { missing: [], restarted: false, reason: "no-declared-servers" };
|
|
7980
|
+
return key;
|
|
8079
7981
|
}
|
|
8080
|
-
if (
|
|
8081
|
-
|
|
8082
|
-
|
|
8083
|
-
if (sessionStartedAt === null) {
|
|
8084
|
-
return { missing: [], restarted: false, reason: "session-start-unknown" };
|
|
8085
|
-
}
|
|
8086
|
-
if (now() - sessionStartedAt < graceMs) {
|
|
8087
|
-
return { missing: [], restarted: false, reason: "cold-start" };
|
|
7982
|
+
/** True if this agent's breaker is currently tripped (manager must skip spawn). */
|
|
7983
|
+
isTripped(codeName) {
|
|
7984
|
+
return this.trips.has(this.keyFor(codeName));
|
|
8088
7985
|
}
|
|
8089
|
-
|
|
8090
|
-
|
|
8091
|
-
psOutput = runPs();
|
|
8092
|
-
} catch (err) {
|
|
8093
|
-
log2(`[mcp-presence-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping`);
|
|
8094
|
-
return { missing: [], restarted: false, reason: "healthy" };
|
|
7986
|
+
getTrip(codeName) {
|
|
7987
|
+
return this.trips.get(this.keyFor(codeName));
|
|
8095
7988
|
}
|
|
8096
|
-
|
|
8097
|
-
|
|
8098
|
-
|
|
8099
|
-
|
|
8100
|
-
|
|
8101
|
-
|
|
8102
|
-
|
|
8103
|
-
|
|
8104
|
-
|
|
8105
|
-
|
|
8106
|
-
|
|
8107
|
-
|
|
8108
|
-
|
|
8109
|
-
|
|
8110
|
-
|
|
8111
|
-
|
|
8112
|
-
|
|
8113
|
-
|
|
8114
|
-
|
|
8115
|
-
|
|
8116
|
-
|
|
8117
|
-
|
|
8118
|
-
rotationGraced.push(key);
|
|
8119
|
-
continue;
|
|
8120
|
-
}
|
|
8121
|
-
if (rotation === "none") {
|
|
8122
|
-
rotationStallState.delete(stateKey(codeName, key));
|
|
8123
|
-
missing.push(key);
|
|
8124
|
-
continue;
|
|
8125
|
-
}
|
|
8126
|
-
const sk = stateKey(codeName, key);
|
|
8127
|
-
const rotationRecordedAt = getRotationRecordedAt(codeName, key) ?? now();
|
|
8128
|
-
const priorStall = rotationStallState.get(sk);
|
|
8129
|
-
const stall = priorStall && priorStall.rotationRecordedAt === rotationRecordedAt ? priorStall : {
|
|
8130
|
-
rotationRecordedAt,
|
|
8131
|
-
probeAttempts: 0,
|
|
8132
|
-
lastProbeAt: null,
|
|
8133
|
-
escalated: false
|
|
8134
|
-
};
|
|
8135
|
-
if (priorStall && priorStall !== stall) {
|
|
8136
|
-
log2(
|
|
8137
|
-
`[mcp-presence-reaper] '${codeName}:${key}' discarding stall state from a previous rotation (rotation=${priorStall.rotationRecordedAt}, probes=${priorStall.probeAttempts}, escalated=${priorStall.escalated}) \u2014 this is a NEW rotation (rotation=${rotationRecordedAt}) and gets a full prompt-probe budget (ENG-8225)`
|
|
8138
|
-
);
|
|
8139
|
-
}
|
|
8140
|
-
rotationStallState.set(sk, stall);
|
|
8141
|
-
if (stall.escalated) {
|
|
8142
|
-
missing.push(key);
|
|
8143
|
-
continue;
|
|
8144
|
-
}
|
|
8145
|
-
const spacingElapsed = stall.lastProbeAt === null || now() - stall.lastProbeAt >= rotationPromptProbeIntervalMs;
|
|
8146
|
-
if (!spacingElapsed) {
|
|
8147
|
-
rotationProbeWaiting.push(key);
|
|
8148
|
-
continue;
|
|
8149
|
-
}
|
|
8150
|
-
if (stall.probeAttempts >= maxRotationPromptProbes) {
|
|
8151
|
-
stall.escalated = true;
|
|
8152
|
-
rotationProbeExhausted.push(key);
|
|
8153
|
-
recordRotationOutcome(key, "never-respawned");
|
|
8154
|
-
log2(
|
|
8155
|
-
`[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=never-respawned \u2014 child SIGTERM'd by stale-mcp-reaper never returned after ${stall.probeAttempts} prompt bind-probe(s); escalating (integration marked unhealthy + reaper_action alert) and releasing to the normal restart path (ENG-8225)`
|
|
8156
|
-
);
|
|
8157
|
-
if (onRotationRespawnGaveUp) {
|
|
8158
|
-
try {
|
|
8159
|
-
onRotationRespawnGaveUp(codeName, key, { attempts: stall.probeAttempts });
|
|
8160
|
-
} catch (err) {
|
|
8161
|
-
log2(
|
|
8162
|
-
`[mcp-presence-reaper] onRotationRespawnGaveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
|
|
8163
|
-
);
|
|
8164
|
-
}
|
|
8165
|
-
}
|
|
8166
|
-
missing.push(key);
|
|
8167
|
-
continue;
|
|
8168
|
-
}
|
|
8169
|
-
const attempt = stall.probeAttempts + 1;
|
|
8170
|
-
let armed = false;
|
|
8171
|
-
if (onRotationRespawnStalled) {
|
|
8172
|
-
try {
|
|
8173
|
-
armed = onRotationRespawnStalled(codeName, key, {
|
|
8174
|
-
attempt,
|
|
8175
|
-
maxAttempts: maxRotationPromptProbes
|
|
8176
|
-
});
|
|
8177
|
-
} catch (err) {
|
|
8178
|
-
log2(
|
|
8179
|
-
`[mcp-presence-reaper] onRotationRespawnStalled callback threw for '${codeName}:${key}' (suppressed; falling back to the pre-ENG-8225 restart path): ${err.message}`
|
|
8180
|
-
);
|
|
8181
|
-
armed = false;
|
|
8182
|
-
}
|
|
8183
|
-
}
|
|
8184
|
-
if (!armed) {
|
|
8185
|
-
missing.push(key);
|
|
8186
|
-
continue;
|
|
7989
|
+
/**
|
|
7990
|
+
* Record a restart event. If recording this event puts the count
|
|
7991
|
+
* inside the window above `max`, the breaker trips and the call site
|
|
7992
|
+
* should NOT respawn.
|
|
7993
|
+
*
|
|
7994
|
+
* Idempotent on already-tripped breakers: returns the existing trip
|
|
7995
|
+
* without double-counting events. Callers may still record the reason
|
|
7996
|
+
* via the decision-log for forensics.
|
|
7997
|
+
*/
|
|
7998
|
+
record(codeName, reason, integrationKey) {
|
|
7999
|
+
const key = this.keyFor(codeName);
|
|
8000
|
+
const existing = this.trips.get(key);
|
|
8001
|
+
if (existing) {
|
|
8002
|
+
const bucket = maxProvisioningBucket(existing.eventsAtTrip);
|
|
8003
|
+
return {
|
|
8004
|
+
tripped: false,
|
|
8005
|
+
trip: existing,
|
|
8006
|
+
windowCount: existing.eventsAtTrip.length,
|
|
8007
|
+
classCounts: countByClass(existing.eventsAtTrip),
|
|
8008
|
+
maxProvisioningBucket: bucket.count,
|
|
8009
|
+
maxProvisioningBucketKey: bucket.key
|
|
8010
|
+
};
|
|
8187
8011
|
}
|
|
8188
|
-
|
|
8189
|
-
|
|
8190
|
-
|
|
8191
|
-
|
|
8192
|
-
|
|
8193
|
-
);
|
|
8194
|
-
|
|
8195
|
-
|
|
8196
|
-
|
|
8197
|
-
|
|
8012
|
+
const at = this.now();
|
|
8013
|
+
const retentionCutoff = at - this.retentionMs;
|
|
8014
|
+
const prior = (this.events.get(key) ?? []).filter((e) => e.at >= retentionCutoff);
|
|
8015
|
+
const bucketed = isIntegrationBucketedReason(reason);
|
|
8016
|
+
prior.push(bucketed && integrationKey ? { reason, at, integrationKey } : { reason, at });
|
|
8017
|
+
this.events.set(key, prior);
|
|
8018
|
+
const crashCount = prior.filter(
|
|
8019
|
+
(e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
|
|
8020
|
+
).length;
|
|
8021
|
+
const provEvents = prior.filter(
|
|
8022
|
+
(e) => e.at >= at - this.provisioningWindowMs && isProvisioningReloadReason(e.reason)
|
|
8198
8023
|
);
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
|
|
8024
|
+
const credRotCount = prior.filter(
|
|
8025
|
+
(e) => e.at >= at - this.provisioningWindowMs && isCredentialRotationReason(e.reason)
|
|
8026
|
+
).length;
|
|
8027
|
+
const selfHealCount = prior.filter(
|
|
8028
|
+
(e) => e.at >= at - this.provisioningWindowMs && isSelfHealingReason(e.reason)
|
|
8029
|
+
).length;
|
|
8030
|
+
const operatorEvents = prior.filter(
|
|
8031
|
+
(e) => e.at >= at - this.provisioningWindowMs && isOperatorConfigReason(e.reason)
|
|
8203
8032
|
);
|
|
8204
|
-
|
|
8205
|
-
|
|
8206
|
-
const liveKeys = /* @__PURE__ */ new Set();
|
|
8207
|
-
for (const key of Object.keys(declared)) {
|
|
8208
|
-
const entry = declared[key];
|
|
8209
|
-
if (typeof entry?.command !== "string") continue;
|
|
8210
|
-
if (!missingRaw.includes(key)) liveKeys.add(key);
|
|
8211
|
-
}
|
|
8212
|
-
for (const key of liveKeys) {
|
|
8213
|
-
const sk = stateKey(codeName, key);
|
|
8214
|
-
if (classifyRotation(codeName, key, rotationGraceMs, now) === "none") {
|
|
8215
|
-
rotationStallState.delete(sk);
|
|
8216
|
-
continue;
|
|
8217
|
-
}
|
|
8218
|
-
const stall = rotationStallState.get(sk);
|
|
8219
|
-
const viaProbe = (stall?.probeAttempts ?? 0) > 0;
|
|
8220
|
-
recordRotationOutcome(
|
|
8221
|
-
key,
|
|
8222
|
-
viaProbe ? "respawned-after-prompt-probe" : "respawned-on-tool-call"
|
|
8033
|
+
const scheduledEvents = prior.filter(
|
|
8034
|
+
(e) => e.at >= at - this.scheduledWindowMs && isScheduledRestartReason(e.reason)
|
|
8223
8035
|
);
|
|
8224
|
-
|
|
8225
|
-
|
|
8036
|
+
const operatorCommandEvents = prior.filter(
|
|
8037
|
+
(e) => e.at >= at - this.operatorCommandWindowMs && isOperatorCommandReason(e.reason)
|
|
8226
8038
|
);
|
|
8227
|
-
|
|
8228
|
-
|
|
8229
|
-
|
|
8230
|
-
|
|
8231
|
-
|
|
8232
|
-
|
|
8233
|
-
|
|
8234
|
-
|
|
8235
|
-
|
|
8236
|
-
state.gaveUpLogged = false;
|
|
8237
|
-
state.firstMissingAt = null;
|
|
8238
|
-
state.quarantineLogged = false;
|
|
8239
|
-
state.continuouslyMissingSince = null;
|
|
8240
|
-
state.deadChannelLogged = false;
|
|
8241
|
-
state.quarantineWithheldLogged = false;
|
|
8242
|
-
state.gaveUp = false;
|
|
8243
|
-
if (state.stableLiveSince === null) state.stableLiveSince = now();
|
|
8244
|
-
if (now() - state.stableLiveSince >= BACKOFF_STABLE_RESET_MS) {
|
|
8245
|
-
state.restartHistory = [];
|
|
8246
|
-
}
|
|
8247
|
-
}
|
|
8248
|
-
}
|
|
8249
|
-
if (missing.length === 0) {
|
|
8250
|
-
return {
|
|
8251
|
-
missing,
|
|
8252
|
-
restarted: false,
|
|
8253
|
-
reason: "healthy",
|
|
8254
|
-
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
8255
|
-
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
8256
|
-
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
8257
|
-
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
8258
|
-
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
|
|
8039
|
+
const windowCount = prior.filter((e) => e.at >= at - this.windowMs).length;
|
|
8040
|
+
const classCounts = {
|
|
8041
|
+
crash: crashCount,
|
|
8042
|
+
provisioning: provEvents.length,
|
|
8043
|
+
"credential-rotation": credRotCount,
|
|
8044
|
+
"self-healing": selfHealCount,
|
|
8045
|
+
"operator-config": operatorEvents.length,
|
|
8046
|
+
scheduled: scheduledEvents.length,
|
|
8047
|
+
"operator-command": operatorCommandEvents.length
|
|
8259
8048
|
};
|
|
8260
|
-
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
const
|
|
8273
|
-
const
|
|
8274
|
-
|
|
8275
|
-
|
|
8276
|
-
|
|
8277
|
-
|
|
8278
|
-
|
|
8279
|
-
|
|
8280
|
-
|
|
8281
|
-
|
|
8282
|
-
|
|
8283
|
-
|
|
8284
|
-
|
|
8285
|
-
|
|
8286
|
-
|
|
8287
|
-
|
|
8288
|
-
|
|
8289
|
-
|
|
8290
|
-
|
|
8291
|
-
|
|
8292
|
-
|
|
8293
|
-
|
|
8294
|
-
|
|
8295
|
-
|
|
8296
|
-
|
|
8297
|
-
|
|
8298
|
-
|
|
8299
|
-
|
|
8049
|
+
const scheduledLoop = scheduledEvents.length > this.scheduledMax ? {
|
|
8050
|
+
count: scheduledEvents.length,
|
|
8051
|
+
windowMs: this.scheduledWindowMs,
|
|
8052
|
+
reason: scheduledEvents[scheduledEvents.length - 1].reason
|
|
8053
|
+
} : void 0;
|
|
8054
|
+
const operatorCommandLoop = operatorCommandEvents.length > this.operatorCommandMax ? {
|
|
8055
|
+
count: operatorCommandEvents.length,
|
|
8056
|
+
windowMs: this.operatorCommandWindowMs,
|
|
8057
|
+
reason: operatorCommandEvents[operatorCommandEvents.length - 1].reason
|
|
8058
|
+
} : void 0;
|
|
8059
|
+
const provBucket = maxProvisioningBucket(provEvents);
|
|
8060
|
+
const operatorBucket = maxBucketWhere(operatorEvents, isOperatorConfigReason);
|
|
8061
|
+
const createdAt = this.newAgentGraceMs > 0 ? this.agentCreatedAt(codeName) : void 0;
|
|
8062
|
+
const age = typeof createdAt === "number" && Number.isFinite(createdAt) ? at - createdAt : void 0;
|
|
8063
|
+
const newAgentGraceActive = age !== void 0 && age >= 0 && age < this.newAgentGraceMs;
|
|
8064
|
+
const newAgentGraceExpiresAt = newAgentGraceActive && typeof createdAt === "number" ? createdAt + this.newAgentGraceMs : void 0;
|
|
8065
|
+
const graceCeilingEvents = newAgentGraceActive ? prior.filter(
|
|
8066
|
+
(e) => e.at >= at - this.provisioningWindowMs && (isProvisioningReloadReason(e.reason) || isOperatorConfigReason(e.reason))
|
|
8067
|
+
) : [];
|
|
8068
|
+
const graceCeilingExceeded = newAgentGraceActive && graceCeilingEvents.length > this.newAgentGraceCeiling;
|
|
8069
|
+
let trippedClass;
|
|
8070
|
+
let tripEvents;
|
|
8071
|
+
let tripWindowMs = this.windowMs;
|
|
8072
|
+
let tripBucketKey = "";
|
|
8073
|
+
if (crashCount > this.max) {
|
|
8074
|
+
trippedClass = "crash";
|
|
8075
|
+
tripEvents = prior.filter(
|
|
8076
|
+
(e) => e.at >= at - this.windowMs && restartReasonClass(e.reason) === "crash"
|
|
8077
|
+
);
|
|
8078
|
+
tripWindowMs = this.windowMs;
|
|
8079
|
+
} else if (newAgentGraceActive && graceCeilingExceeded) {
|
|
8080
|
+
trippedClass = "provisioning";
|
|
8081
|
+
tripEvents = graceCeilingEvents;
|
|
8082
|
+
tripWindowMs = this.provisioningWindowMs;
|
|
8083
|
+
tripBucketKey = "";
|
|
8084
|
+
} else if (newAgentGraceActive) {
|
|
8085
|
+
} else if (operatorBucket.count > this.operatorMax) {
|
|
8086
|
+
trippedClass = "operator-config";
|
|
8087
|
+
tripEvents = operatorEvents.filter((e) => (e.integrationKey ?? "") === operatorBucket.key);
|
|
8088
|
+
tripWindowMs = this.provisioningWindowMs;
|
|
8089
|
+
tripBucketKey = operatorBucket.key;
|
|
8090
|
+
} else if (provBucket.count > this.provisioningMax) {
|
|
8091
|
+
trippedClass = "provisioning";
|
|
8092
|
+
tripEvents = provEvents.filter((e) => (e.integrationKey ?? "") === provBucket.key);
|
|
8093
|
+
tripWindowMs = this.provisioningWindowMs;
|
|
8094
|
+
tripBucketKey = provBucket.key;
|
|
8300
8095
|
}
|
|
8301
|
-
if (
|
|
8302
|
-
|
|
8096
|
+
if (trippedClass && tripEvents) {
|
|
8097
|
+
const baseMessage = graceCeilingExceeded ? formatStatusMessage(tripEvents, tripWindowMs) : formatStatusMessage(tripEvents, tripWindowMs, trippedClass, tripBucketKey);
|
|
8098
|
+
const graceHours = Math.round(this.newAgentGraceMs / 36e5 * 10) / 10;
|
|
8099
|
+
const statusMessage = graceCeilingExceeded ? `${baseMessage} \u2014 paused DESPITE the new-agent grace: ${graceCeilingEvents.length} provisioning-shaped restarts (provisioning + operator-config) exceeded the grace ceiling of ${this.newAgentGraceCeiling} (ENG-8690). The agent is inside its ${graceHours}h grace and is doing little except restarting.` : baseMessage;
|
|
8100
|
+
const trip = {
|
|
8101
|
+
trippedAt: at,
|
|
8102
|
+
eventsAtTrip: [...tripEvents],
|
|
8103
|
+
statusMessage,
|
|
8104
|
+
// ENG-7577: persist the tripping class so the recovery path can gate on
|
|
8105
|
+
// it after a manager restart, without re-deriving it from prose.
|
|
8106
|
+
trippedClass
|
|
8107
|
+
};
|
|
8108
|
+
this.trips.set(key, trip);
|
|
8109
|
+
this.events.delete(key);
|
|
8110
|
+
return {
|
|
8111
|
+
tripped: true,
|
|
8112
|
+
trip,
|
|
8113
|
+
windowCount,
|
|
8114
|
+
classCounts,
|
|
8115
|
+
trippedClass,
|
|
8116
|
+
maxProvisioningBucket: provBucket.count,
|
|
8117
|
+
maxProvisioningBucketKey: provBucket.key,
|
|
8118
|
+
...graceCeilingExceeded ? { newAgentGraceCeilingExceeded: true } : {},
|
|
8119
|
+
// ENG-9138: a co-occurring rollover loop still reports on a real trip.
|
|
8120
|
+
...scheduledLoop ? { scheduledLoop } : {},
|
|
8121
|
+
...operatorCommandLoop ? { operatorCommandLoop } : {}
|
|
8122
|
+
};
|
|
8303
8123
|
}
|
|
8304
|
-
|
|
8305
|
-
|
|
8306
|
-
|
|
8307
|
-
|
|
8308
|
-
|
|
8309
|
-
|
|
8310
|
-
|
|
8124
|
+
const suppressedOperator = operatorBucket.count > this.operatorMax;
|
|
8125
|
+
const suppressedProvisioning = !suppressedOperator && provBucket.count > this.provisioningMax;
|
|
8126
|
+
const wouldHaveTripped = suppressedOperator || suppressedProvisioning;
|
|
8127
|
+
return {
|
|
8128
|
+
tripped: false,
|
|
8129
|
+
windowCount,
|
|
8130
|
+
classCounts,
|
|
8131
|
+
maxProvisioningBucket: provBucket.count,
|
|
8132
|
+
maxProvisioningBucketKey: provBucket.key,
|
|
8133
|
+
...scheduledLoop ? { scheduledLoop } : {},
|
|
8134
|
+
...operatorCommandLoop ? { operatorCommandLoop } : {},
|
|
8135
|
+
...newAgentGraceActive ? {
|
|
8136
|
+
newAgentGraceSuppressedTrip: wouldHaveTripped,
|
|
8137
|
+
...newAgentGraceExpiresAt !== void 0 ? { newAgentGraceExpiresAt } : {},
|
|
8138
|
+
...wouldHaveTripped ? {
|
|
8139
|
+
newAgentGraceSuppressed: {
|
|
8140
|
+
class: suppressedOperator ? "operator-config" : "provisioning",
|
|
8141
|
+
bucketKey: suppressedOperator ? operatorBucket.key : provBucket.key,
|
|
8142
|
+
bucketCount: suppressedOperator ? operatorBucket.count : provBucket.count,
|
|
8143
|
+
agentAgeMs: age ?? 0
|
|
8144
|
+
}
|
|
8145
|
+
} : {}
|
|
8146
|
+
} : {}
|
|
8147
|
+
};
|
|
8148
|
+
}
|
|
8149
|
+
/** Operator-initiated reset: drops the trip + the events log for this agent. */
|
|
8150
|
+
clear(codeName) {
|
|
8151
|
+
const key = this.keyFor(codeName);
|
|
8152
|
+
this.trips.delete(key);
|
|
8153
|
+
this.events.delete(key);
|
|
8154
|
+
if (key !== codeName) {
|
|
8155
|
+
this.trips.delete(codeName);
|
|
8156
|
+
this.events.delete(codeName);
|
|
8311
8157
|
}
|
|
8312
|
-
|
|
8313
|
-
|
|
8314
|
-
|
|
8315
|
-
|
|
8316
|
-
|
|
8317
|
-
|
|
8318
|
-
|
|
8319
|
-
|
|
8320
|
-
|
|
8321
|
-
|
|
8322
|
-
|
|
8323
|
-
|
|
8158
|
+
}
|
|
8159
|
+
/** Snapshot tripped agents for `manager-state.json`. */
|
|
8160
|
+
serialize() {
|
|
8161
|
+
return Object.fromEntries(this.trips.entries());
|
|
8162
|
+
}
|
|
8163
|
+
/**
|
|
8164
|
+
* Rehydrate trip state from `manager-state.json`. Called once at
|
|
8165
|
+
* worker startup. Window history is intentionally NOT persisted — only
|
|
8166
|
+
* tripped state. A tripped breaker survives manager restart; an
|
|
8167
|
+
* un-tripped one starts a fresh window in the new worker.
|
|
8168
|
+
*/
|
|
8169
|
+
hydrate(saved) {
|
|
8170
|
+
if (!saved) return;
|
|
8171
|
+
for (const [codeName, trip] of Object.entries(saved)) {
|
|
8172
|
+
if (trip && typeof trip.trippedAt === "number" && Array.isArray(trip.eventsAtTrip)) {
|
|
8173
|
+
const klass = trip.trippedClass;
|
|
8174
|
+
this.trips.set(
|
|
8175
|
+
codeName,
|
|
8176
|
+
klass === void 0 || KNOWN_REASON_CLASSES.has(klass) ? trip : { ...trip, trippedClass: void 0 }
|
|
8324
8177
|
);
|
|
8325
|
-
state.gaveUpLogged = true;
|
|
8326
|
-
if (onGiveUp) {
|
|
8327
|
-
try {
|
|
8328
|
-
onGiveUp(codeName, key);
|
|
8329
|
-
} catch (err) {
|
|
8330
|
-
log2(
|
|
8331
|
-
`[mcp-presence-reaper] onGiveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
|
|
8332
|
-
);
|
|
8333
|
-
}
|
|
8334
|
-
}
|
|
8335
|
-
}
|
|
8336
|
-
if (quarantineMode !== "off" && !state.quarantineLogged && classifyKey(key) === "optional") {
|
|
8337
|
-
const dwellElapsed = quarantineDwellMs <= 0 || state.firstMissingAt !== null && nowMs - state.firstMissingAt >= quarantineDwellMs;
|
|
8338
|
-
if (dwellElapsed && !anyStdioServerLive) {
|
|
8339
|
-
quarantineWithheld.push(key);
|
|
8340
|
-
if (!state.quarantineWithheldLogged) {
|
|
8341
|
-
state.quarantineWithheldLogged = true;
|
|
8342
|
-
log2(
|
|
8343
|
-
`[mcp-presence-reaper] '${codeName}:${key}' given up but NOT quarantined - no declared stdio MCP server is live, so the restart budget was exhausted by a session that cannot spawn anything, not by this channel dying. Check the pane (tmux attach -t agt-${codeName}); the channel stays provisioned and will quarantine on a later tick if it is still missing once the session is up (ENG-10500)`
|
|
8344
|
-
);
|
|
8345
|
-
}
|
|
8346
|
-
} else if (dwellElapsed) {
|
|
8347
|
-
state.quarantineLogged = true;
|
|
8348
|
-
if (quarantineMode === "enforce") {
|
|
8349
|
-
quarantined.push(key);
|
|
8350
|
-
log2(
|
|
8351
|
-
`[mcp-presence-reaper] QUARANTINE '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; dropping from provisioned set so the dead channel stops restarting the whole session (ENG-5932)`
|
|
8352
|
-
);
|
|
8353
|
-
if (onQuarantine) {
|
|
8354
|
-
try {
|
|
8355
|
-
onQuarantine(codeName, key);
|
|
8356
|
-
} catch (err) {
|
|
8357
|
-
log2(
|
|
8358
|
-
`[mcp-presence-reaper] onQuarantine callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
|
|
8359
|
-
);
|
|
8360
|
-
}
|
|
8361
|
-
}
|
|
8362
|
-
} else {
|
|
8363
|
-
wouldQuarantine.push(key);
|
|
8364
|
-
log2(
|
|
8365
|
-
`[mcp-presence-reaper] SHADOW would-quarantine '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; no action taken (ENG-5932 shadow mode)`
|
|
8366
|
-
);
|
|
8367
|
-
}
|
|
8368
|
-
}
|
|
8369
|
-
}
|
|
8370
|
-
} else {
|
|
8371
|
-
const backoffMs = restartsInWindow < BACKOFF_AFTER_RESTARTS ? 0 : Math.min(
|
|
8372
|
-
BACKOFF_BASE_MS * 2 ** (restartsInWindow - BACKOFF_AFTER_RESTARTS),
|
|
8373
|
-
BACKOFF_MAX_MS
|
|
8374
|
-
);
|
|
8375
|
-
const lastRestartAt = state.restartHistory[state.restartHistory.length - 1] ?? null;
|
|
8376
|
-
if (lastRestartAt !== null && nowMs - lastRestartAt < backoffMs) {
|
|
8377
|
-
backingOff.push(key);
|
|
8378
|
-
} else {
|
|
8379
|
-
active.push(key);
|
|
8380
8178
|
}
|
|
8381
8179
|
}
|
|
8382
8180
|
}
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8388
|
-
|
|
8389
|
-
|
|
8390
|
-
|
|
8391
|
-
|
|
8392
|
-
|
|
8393
|
-
|
|
8394
|
-
|
|
8395
|
-
|
|
8396
|
-
|
|
8397
|
-
|
|
8398
|
-
|
|
8399
|
-
|
|
8400
|
-
|
|
8401
|
-
|
|
8402
|
-
|
|
8403
|
-
|
|
8404
|
-
|
|
8405
|
-
|
|
8181
|
+
/**
|
|
8182
|
+
* ENG-10878: how many CRASH-bar restarts `codeName` can still afford in the
|
|
8183
|
+
* current window.
|
|
8184
|
+
*
|
|
8185
|
+
* Exists so a caller can decline to spend the last unit of a budget it did
|
|
8186
|
+
* not fill. `mcp-presence-reaper` sits on this bar next to `auth-tuple-change`
|
|
8187
|
+
* and `model-change`, so a reaper restart landing beside an auth-tuple
|
|
8188
|
+
* rotation is the pair that trips it - and the agent is paused on the other
|
|
8189
|
+
* reason's account.
|
|
8190
|
+
*
|
|
8191
|
+
* THE ORIGINAL VERSION OF THIS PARAGRAPH NAMED THE WRONG REASONS (obi-wan
|
|
8192
|
+
* `486aa45fa3bd`), and it said so in three files. It claimed `day-rollover`
|
|
8193
|
+
* and `channel-set-change` share this bar and used "a reaper restart at 00:59
|
|
8194
|
+
* beside a day-rollover at 01:00" as the worked example. Neither is crash
|
|
8195
|
+
* class: `restartReasonClass('day-rollover')` is `'scheduled'` (ENG-9138) and
|
|
8196
|
+
* `'channel-set-change'` is `'provisioning'` (ENG-10510), so neither is
|
|
8197
|
+
* counted by the filter below and that pair cannot trip anything. The PR's
|
|
8198
|
+
* own new test asserts exactly that, which is the sort of contradiction that
|
|
8199
|
+
* should have been caught by writing the test - it was not, because the test
|
|
8200
|
+
* and the prose were written for different purposes an hour apart.
|
|
8201
|
+
*
|
|
8202
|
+
* The GUARD is unaffected: `auth-tuple-change` and `model-change` really are
|
|
8203
|
+
* on this bar, so the co-occurrence risk is real. Only the example was wrong,
|
|
8204
|
+
* and an argument resting on an impossible example is worth no more than the
|
|
8205
|
+
* example.
|
|
8206
|
+
*
|
|
8207
|
+
* The arithmetic lives HERE rather than at the call site on purpose: `max`
|
|
8208
|
+
* and the window are this class's business, they are env-tunable
|
|
8209
|
+
* (AGT_RESTART_BREAKER_MAX / _WINDOW_MS), and a caller doing
|
|
8210
|
+
* `max - windowCount()` itself would silently stop being correct the moment
|
|
8211
|
+
* either moves. Floored at 0 so an already-tripped agent reads as "nothing
|
|
8212
|
+
* left" rather than a negative number.
|
|
8213
|
+
*
|
|
8214
|
+
* CRASH BAR ONLY. The provisioning, operator, scheduled and operator-command
|
|
8215
|
+
* bars have their own maxima and windows; this answers for the default bar,
|
|
8216
|
+
* which is the one `mcp-presence-reaper` is counted against.
|
|
8217
|
+
*/
|
|
8218
|
+
remainingCrashBudget(codeName) {
|
|
8219
|
+
const key = this.keyFor(codeName);
|
|
8220
|
+
if (this.trips.has(key)) return 0;
|
|
8221
|
+
const cutoff = this.now() - this.windowMs;
|
|
8222
|
+
const crashCount = (this.events.get(key) ?? []).filter(
|
|
8223
|
+
(e) => e.at >= cutoff && restartReasonClass(e.reason) === "crash"
|
|
8224
|
+
).length;
|
|
8225
|
+
return Math.max(0, this.max - crashCount);
|
|
8406
8226
|
}
|
|
8407
|
-
|
|
8408
|
-
|
|
8409
|
-
|
|
8410
|
-
|
|
8411
|
-
} catch (err) {
|
|
8412
|
-
log2(
|
|
8413
|
-
`[mcp-presence-reaper] '${codeName}': pane read threw (${err.message}) \u2014 proceeding with restart (ENG-9932 fails open)`
|
|
8414
|
-
);
|
|
8415
|
-
}
|
|
8416
|
-
const blocked = classifyUnanswerablePane(pane);
|
|
8417
|
-
if (blocked) {
|
|
8418
|
-
const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
|
|
8419
|
-
log2(
|
|
8420
|
-
`[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 session is blocked on an interactive prompt (${blocked}), so declared MCP(s) [${active.join(", ")}] were never spawned rather than having died. A restart re-presents the same prompt and burns the ENG-5441 breaker budget. Needs a human at the pane: tmux attach -t agt-${codeName}` + // Fingerprint, never pane text — see summarizeUnanswerablePane.
|
|
8421
|
-
(fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
|
|
8422
|
-
);
|
|
8423
|
-
if (onPaneBlocked) {
|
|
8424
|
-
try {
|
|
8425
|
-
onPaneBlocked(codeName, {
|
|
8426
|
-
kind: blocked,
|
|
8427
|
-
activeKeys: [...active],
|
|
8428
|
-
paneHash: fingerprint?.hash ?? null,
|
|
8429
|
-
paneLength: fingerprint?.length ?? null
|
|
8430
|
-
});
|
|
8431
|
-
} catch (err) {
|
|
8432
|
-
log2(
|
|
8433
|
-
`[mcp-presence-reaper] onPaneBlocked callback threw for '${codeName}' (suppressed): ${err.message}`
|
|
8434
|
-
);
|
|
8435
|
-
}
|
|
8436
|
-
}
|
|
8437
|
-
return {
|
|
8438
|
-
missing,
|
|
8439
|
-
restarted: false,
|
|
8440
|
-
reason: "pane-blocked-on-dialog",
|
|
8441
|
-
paneBlockedOn: blocked,
|
|
8442
|
-
givenUp: givenUp.length > 0 ? givenUp : void 0,
|
|
8443
|
-
backingOff: backingOff.length > 0 ? backingOff : void 0,
|
|
8444
|
-
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
8445
|
-
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
8446
|
-
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
8447
|
-
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
8448
|
-
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
|
|
8449
|
-
quarantined: quarantined.length > 0 ? quarantined : void 0,
|
|
8450
|
-
wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
|
|
8451
|
-
genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
|
|
8452
|
-
quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
|
|
8453
|
-
};
|
|
8454
|
-
}
|
|
8455
|
-
}
|
|
8456
|
-
const givenUpSuffix = givenUp.length > 0 ? ` (skipping over-cap: [${givenUp.join(", ")}])` : "";
|
|
8457
|
-
log2(
|
|
8458
|
-
`[mcp-presence-reaper] '${codeName}': declared MCP(s) [${active.join(", ")}] have no live children \u2014 restarting session${givenUpSuffix}`
|
|
8459
|
-
);
|
|
8460
|
-
if (onRestart) {
|
|
8461
|
-
try {
|
|
8462
|
-
const maybePromise = onRestart(codeName, { activeKeys: active, givenUpKeys: givenUp });
|
|
8463
|
-
if (maybePromise && typeof maybePromise.then === "function") {
|
|
8464
|
-
void maybePromise.then(void 0, (err) => {
|
|
8465
|
-
log2(
|
|
8466
|
-
`[mcp-presence-reaper] onRestart callback rejected for '${codeName}' (suppressed; restart proceeds): ${err.message}`
|
|
8467
|
-
);
|
|
8468
|
-
});
|
|
8469
|
-
}
|
|
8470
|
-
} catch (err) {
|
|
8471
|
-
log2(
|
|
8472
|
-
`[mcp-presence-reaper] onRestart callback threw for '${codeName}' (suppressed; restart proceeds): ${err.message}`
|
|
8473
|
-
);
|
|
8474
|
-
}
|
|
8475
|
-
}
|
|
8476
|
-
stopSession(codeName, { activeKeys: active });
|
|
8477
|
-
for (const key of active) {
|
|
8478
|
-
if (!newGenerationKeys.has(key)) continue;
|
|
8479
|
-
const st = presenceReaperState.get(stateKey(codeName, key));
|
|
8480
|
-
if (st) st.restartHistory.push(nowMs);
|
|
8227
|
+
/** Test helper — current in-window event count for `codeName`. */
|
|
8228
|
+
windowCount(codeName) {
|
|
8229
|
+
const cutoff = this.now() - this.windowMs;
|
|
8230
|
+
return (this.events.get(this.keyFor(codeName)) ?? []).filter((e) => e.at >= cutoff).length;
|
|
8481
8231
|
}
|
|
8482
|
-
|
|
8483
|
-
|
|
8484
|
-
|
|
8485
|
-
|
|
8486
|
-
|
|
8487
|
-
|
|
8488
|
-
|
|
8489
|
-
|
|
8490
|
-
|
|
8491
|
-
|
|
8492
|
-
quarantined: quarantined.length > 0 ? quarantined : void 0,
|
|
8493
|
-
wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
|
|
8494
|
-
genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
|
|
8495
|
-
quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
|
|
8496
|
-
};
|
|
8232
|
+
};
|
|
8233
|
+
function formatStatusMessage(events, windowMs, klass = "crash", integrationKey = "") {
|
|
8234
|
+
const last = events[events.length - 1];
|
|
8235
|
+
const windowLabel = windowMs < 6e4 ? `${Math.round(windowMs / 1e3)}s` : `${(windowMs / 6e4).toFixed(1).replace(/\.0$/, "")}min`;
|
|
8236
|
+
const reasonCounts = /* @__PURE__ */ new Map();
|
|
8237
|
+
for (const e of events) reasonCounts.set(e.reason, (reasonCounts.get(e.reason) ?? 0) + 1);
|
|
8238
|
+
const breakdown = Array.from(reasonCounts.entries()).map(([r, n]) => `${r}=${n}`).join(", ");
|
|
8239
|
+
const classLabel = klass === "provisioning" ? "provisioning-reload " : klass === "operator-config" ? "operator-requested config-reload " : "";
|
|
8240
|
+
const integrationLabel = (klass === "provisioning" || klass === "operator-config") && integrationKey ? ` for integration '${integrationKey}'` : "";
|
|
8241
|
+
return `Circuit breaker tripped: ${events.length} ${classLabel}restarts in ${windowLabel}${integrationLabel} (${breakdown}); most recent=${last.reason} at ${new Date(last.at).toISOString()}`;
|
|
8497
8242
|
}
|
|
8498
8243
|
|
|
8499
|
-
// src/lib/
|
|
8500
|
-
|
|
8501
|
-
|
|
8244
|
+
// src/lib/stale-mcp-reaper.ts
|
|
8245
|
+
import { execFileSync } from "child_process";
|
|
8246
|
+
function livenessFromProbe(r, onNoAnswer) {
|
|
8247
|
+
if (r.kind === "ok") return true;
|
|
8248
|
+
if (r.kind === "failed" && r.status !== null) return false;
|
|
8249
|
+
onNoAnswer?.(r.kind === "timed-out" ? "did not answer within its bound" : `could not run: ${r.message}`);
|
|
8250
|
+
return true;
|
|
8502
8251
|
}
|
|
8503
|
-
var
|
|
8504
|
-
|
|
8505
|
-
|
|
8506
|
-
|
|
8507
|
-
// ENG-8388: provisioning reloads a human demonstrably asked for. Same
|
|
8508
|
-
// Record-not-Set reasoning as above — a new class cannot be silently omitted.
|
|
8509
|
-
"operator-config": true,
|
|
8510
|
-
// ENG-8215: the self-healing class (bind-remediation - the manager's OWN
|
|
8511
|
-
// repair attempts, which must never count toward a trip). Adding it here is
|
|
8512
|
-
// forced by the Record type, which is exactly why that shape was chosen over
|
|
8513
|
-
// a Set literal (ENG-7577, CodeRabbit on #3861) - a new union member cannot
|
|
8514
|
-
// be silently omitted from the rehydration allowlist.
|
|
8515
|
-
"self-healing": true,
|
|
8516
|
-
// ENG-9138: the platform-scheduled class. Same Record-not-Set reasoning: the
|
|
8517
|
-
// compiler forces every new class into this allowlist, so a `trippedClass`
|
|
8518
|
-
// rehydrated from manager-state.json can never name a class this build does
|
|
8519
|
-
// not know about.
|
|
8520
|
-
scheduled: true,
|
|
8521
|
-
// ENG-10066: the operator-command class. Same Record-not-Set reasoning again —
|
|
8522
|
-
// the compiler forces every new class into this allowlist.
|
|
8523
|
-
"operator-command": true
|
|
8524
|
-
};
|
|
8525
|
-
var KNOWN_REASON_CLASSES = new Set(
|
|
8526
|
-
Object.keys(KNOWN_REASON_CLASS_MAP)
|
|
8527
|
-
);
|
|
8528
|
-
var PROVISIONING_RELOAD_REASONS = /* @__PURE__ */ new Set([
|
|
8529
|
-
"hot-reload-mcp",
|
|
8530
|
-
"managed-mcp-churn",
|
|
8531
|
-
// ENG-8215: 'bind-remediation' MOVED OUT of this set into SELF_HEALING_REASONS.
|
|
8532
|
-
// It is the manager repairing itself, not a config reload, and counting it
|
|
8533
|
-
// meant the harder the platform tried to fix an agent the more certain it
|
|
8534
|
-
// became that it should give up. Deliberately NOT re-added here.
|
|
8535
|
-
// ENG-7576: the dashboard integration-add stop itself - the first restart of
|
|
8536
|
-
// the very burst this class exists for, previously uncounted entirely.
|
|
8537
|
-
"integration-change",
|
|
8538
|
-
// ENG-7771: a sender_policy delivery restart is a deliberate config-delivery
|
|
8539
|
-
// reload (operator changed the policy, or the fail-closed first-poll verify
|
|
8540
|
-
// after a manager restart with no persisted baseline), not a crash. On the
|
|
8541
|
-
// tight crash tally, a host whose sender-policy-baseline.json write keeps
|
|
8542
|
-
// failing (disk full, permissions) would convert the fail-closed restart
|
|
8543
|
-
// into repeated crash tallies across a multi-deploy day and trip the
|
|
8544
|
-
// breaker - taking the agent DOWN, which is strictly worse than the stale
|
|
8545
|
-
// policy the restart exists to fix. The looser provisioning tally still
|
|
8546
|
-
// trips + pages on sustained thrash.
|
|
8547
|
-
"sender-policy-change",
|
|
8548
|
-
// ENG-9802: same argument, same failure mode — a host whose baseline write
|
|
8549
|
-
// keeps failing must not convert a fail-closed policy restart into a breaker
|
|
8550
|
-
// trip that takes the agent down.
|
|
8551
|
-
"telegram-policy-change",
|
|
8552
|
-
// ENG-9861: and again for standing rules. The fail-closed first-poll restart
|
|
8553
|
-
// has exactly the same shape, so it needs the same protection from a host
|
|
8554
|
-
// whose sidecar write keeps failing.
|
|
8555
|
-
"agent-rules-change",
|
|
8556
|
-
// ENG-10510 (Obi-Wan on #6313): a channel-set ADD is a config reload, not a
|
|
8557
|
-
// crash, and it was landing on the crash tally purely by falling through.
|
|
8558
|
-
// It is 'integration-change's exact sibling - both are "a channel or server
|
|
8559
|
-
// was added, re-render .mcp.json and respawn" - and it was never argued for
|
|
8560
|
-
// the crash class, it simply was not in this set.
|
|
8561
|
-
//
|
|
8562
|
-
// This is the ENG-9138 / ENG-10066 species verbatim: a PLATFORM-MANUFACTURED
|
|
8563
|
-
// restart counted as though the agent had crashed. ENG-9138 cost katie 8h16m
|
|
8564
|
-
// when day-rollover fell through the same way; ENG-10066 cost an operator an
|
|
8565
|
-
// auto-pause when a channel /restart did. Both were fixed by naming the
|
|
8566
|
-
// reason rather than leaving it to the catch-all, and this is the third.
|
|
8567
|
-
//
|
|
8568
|
-
// WHY NOW. Until ENG-10510 this restart only ever followed a human adding a
|
|
8569
|
-
// channel, so the crash tally was wrong but rarely reached: a person adding
|
|
8570
|
-
// two channels inside ten minutes is uncommon and self-limiting. Channel
|
|
8571
|
-
// auto-resume is the first thing that fires it UNATTENDED, on a timer, on
|
|
8572
|
-
// any agent with a quarantined channel. An agent with a second crash-class
|
|
8573
|
-
// restart in the same ten minutes (the presence reaper's
|
|
8574
|
-
// 'mcp-presence-reaper', for one) would cross DEFAULT_MAX and auto-pause
|
|
8575
|
-
// with no human anywhere in the loop - which is precisely the ENG-8215 harm
|
|
8576
|
-
// that feature's 2h floor exists to prevent.
|
|
8577
|
-
//
|
|
8578
|
-
// Counted, not exempt: provisioning is a LOOSER tally, not an absent one, so
|
|
8579
|
-
// sustained channel-set thrash still trips and still pages.
|
|
8580
|
-
"channel-set-change"
|
|
8581
|
-
]);
|
|
8582
|
-
function isProvisioningReloadReason(reason) {
|
|
8583
|
-
return PROVISIONING_RELOAD_REASONS.has(reason);
|
|
8252
|
+
var rotationTimestamps = /* @__PURE__ */ new Map();
|
|
8253
|
+
var DEFAULT_ROTATION_GRACE_MS = 15e3;
|
|
8254
|
+
function rotationKey(codeName, serverKey) {
|
|
8255
|
+
return `${codeName}\0${serverKey}`;
|
|
8584
8256
|
}
|
|
8585
|
-
|
|
8586
|
-
|
|
8587
|
-
|
|
8588
|
-
|
|
8589
|
-
|
|
8257
|
+
function recordStaleRotation(codeName, serverKeys, now = Date.now) {
|
|
8258
|
+
const ts = now();
|
|
8259
|
+
for (const key of serverKeys) {
|
|
8260
|
+
rotationTimestamps.set(rotationKey(codeName, key), ts);
|
|
8261
|
+
}
|
|
8590
8262
|
}
|
|
8591
|
-
var
|
|
8592
|
-
|
|
8593
|
-
|
|
8594
|
-
|
|
8595
|
-
|
|
8263
|
+
var ROTATION_TRACKING_RETENTION_MS = 15 * 60 * 1e3;
|
|
8264
|
+
function classifyRotation(codeName, serverKey, graceMs, now = Date.now, retentionMs = ROTATION_TRACKING_RETENTION_MS) {
|
|
8265
|
+
const k = rotationKey(codeName, serverKey);
|
|
8266
|
+
const ts = rotationTimestamps.get(k);
|
|
8267
|
+
if (ts === void 0) return "none";
|
|
8268
|
+
const age = now() - ts;
|
|
8269
|
+
if (age < graceMs) return "within-grace";
|
|
8270
|
+
if (age < retentionMs) return "expired";
|
|
8271
|
+
rotationTimestamps.delete(k);
|
|
8272
|
+
return "none";
|
|
8596
8273
|
}
|
|
8597
|
-
|
|
8598
|
-
|
|
8599
|
-
]);
|
|
8600
|
-
function isScheduledRestartReason(reason) {
|
|
8601
|
-
return SCHEDULED_REASONS.has(reason);
|
|
8274
|
+
function getRotationRecordedAt(codeName, serverKey) {
|
|
8275
|
+
return rotationTimestamps.get(rotationKey(codeName, serverKey));
|
|
8602
8276
|
}
|
|
8603
|
-
|
|
8604
|
-
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
return OPERATOR_COMMAND_REASONS.has(reason);
|
|
8277
|
+
function clearStaleRotation(codeName, serverKeys) {
|
|
8278
|
+
for (const key of serverKeys) {
|
|
8279
|
+
rotationTimestamps.delete(rotationKey(codeName, key));
|
|
8280
|
+
}
|
|
8608
8281
|
}
|
|
8609
|
-
|
|
8610
|
-
|
|
8611
|
-
|
|
8612
|
-
|
|
8613
|
-
|
|
8282
|
+
function parseEnvIntegrationsEntries(content) {
|
|
8283
|
+
const entries = /* @__PURE__ */ new Map();
|
|
8284
|
+
for (const raw of content.split(/\r?\n/)) {
|
|
8285
|
+
const line = raw.trim();
|
|
8286
|
+
if (!line || line.startsWith("#")) continue;
|
|
8287
|
+
const eq = line.indexOf("=");
|
|
8288
|
+
if (eq <= 0) continue;
|
|
8289
|
+
const name = line.slice(0, eq).trim();
|
|
8290
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) continue;
|
|
8291
|
+
const value = line.slice(eq + 1);
|
|
8292
|
+
entries.set(name, value);
|
|
8293
|
+
}
|
|
8294
|
+
return entries;
|
|
8614
8295
|
}
|
|
8615
|
-
|
|
8616
|
-
|
|
8617
|
-
|
|
8618
|
-
|
|
8619
|
-
|
|
8620
|
-
|
|
8296
|
+
function diffEnvIntegrations(oldContent, newContent) {
|
|
8297
|
+
const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
|
|
8298
|
+
const newEntries = parseEnvIntegrationsEntries(newContent);
|
|
8299
|
+
const changed = /* @__PURE__ */ new Set();
|
|
8300
|
+
for (const [name, value] of newEntries) {
|
|
8301
|
+
if (oldEntries.get(name) !== value) changed.add(name);
|
|
8302
|
+
}
|
|
8303
|
+
for (const name of oldEntries.keys()) {
|
|
8304
|
+
if (!newEntries.has(name)) changed.add(name);
|
|
8305
|
+
}
|
|
8306
|
+
return [...changed];
|
|
8621
8307
|
}
|
|
8622
|
-
function
|
|
8623
|
-
|
|
8624
|
-
|
|
8625
|
-
|
|
8626
|
-
|
|
8627
|
-
|
|
8628
|
-
|
|
8629
|
-
|
|
8630
|
-
|
|
8631
|
-
let crash = 0;
|
|
8632
|
-
let provisioning = 0;
|
|
8633
|
-
let credentialRotation = 0;
|
|
8634
|
-
let selfHealing = 0;
|
|
8635
|
-
let operatorConfig = 0;
|
|
8636
|
-
let scheduled = 0;
|
|
8637
|
-
let operatorCommand = 0;
|
|
8638
|
-
for (const e of events) {
|
|
8639
|
-
const klass = restartReasonClass(e.reason);
|
|
8640
|
-
if (klass === "provisioning") provisioning += 1;
|
|
8641
|
-
else if (klass === "credential-rotation") credentialRotation += 1;
|
|
8642
|
-
else if (klass === "self-healing") selfHealing += 1;
|
|
8643
|
-
else if (klass === "operator-config") operatorConfig += 1;
|
|
8644
|
-
else if (klass === "scheduled") scheduled += 1;
|
|
8645
|
-
else if (klass === "operator-command") operatorCommand += 1;
|
|
8646
|
-
else crash += 1;
|
|
8308
|
+
function classifyEnvIntegrationsDiff(oldContent, newContent) {
|
|
8309
|
+
const oldEntries = oldContent === void 0 ? /* @__PURE__ */ new Map() : parseEnvIntegrationsEntries(oldContent);
|
|
8310
|
+
const newEntries = parseEnvIntegrationsEntries(newContent);
|
|
8311
|
+
const rotated = [];
|
|
8312
|
+
const added = [];
|
|
8313
|
+
const removed = [];
|
|
8314
|
+
for (const [name, value] of newEntries) {
|
|
8315
|
+
if (!oldEntries.has(name)) added.push(name);
|
|
8316
|
+
else if (oldEntries.get(name) !== value) rotated.push(name);
|
|
8647
8317
|
}
|
|
8648
|
-
|
|
8649
|
-
|
|
8650
|
-
provisioning,
|
|
8651
|
-
"credential-rotation": credentialRotation,
|
|
8652
|
-
"self-healing": selfHealing,
|
|
8653
|
-
"operator-config": operatorConfig,
|
|
8654
|
-
scheduled,
|
|
8655
|
-
"operator-command": operatorCommand
|
|
8656
|
-
};
|
|
8657
|
-
}
|
|
8658
|
-
function maxProvisioningBucket(events) {
|
|
8659
|
-
return maxBucketWhere(events, isProvisioningReloadReason);
|
|
8660
|
-
}
|
|
8661
|
-
function maxBucketWhere(events, matches) {
|
|
8662
|
-
const buckets = /* @__PURE__ */ new Map();
|
|
8663
|
-
for (const e of events) {
|
|
8664
|
-
if (!matches(e.reason)) continue;
|
|
8665
|
-
const key = e.integrationKey ?? "";
|
|
8666
|
-
buckets.set(key, (buckets.get(key) ?? 0) + 1);
|
|
8318
|
+
for (const name of oldEntries.keys()) {
|
|
8319
|
+
if (!newEntries.has(name)) removed.push(name);
|
|
8667
8320
|
}
|
|
8668
|
-
|
|
8669
|
-
|
|
8670
|
-
|
|
8321
|
+
return { rotated, added, removed };
|
|
8322
|
+
}
|
|
8323
|
+
function findMcpServersUsingVars(mcp, changedVars) {
|
|
8324
|
+
const changedSet = new Set(changedVars);
|
|
8325
|
+
if (!mcp?.mcpServers || changedSet.size === 0) return [];
|
|
8326
|
+
const result = [];
|
|
8327
|
+
for (const [serverKey, entry] of Object.entries(mcp.mcpServers)) {
|
|
8328
|
+
if (!entry || typeof entry !== "object") continue;
|
|
8329
|
+
const env = entry.env;
|
|
8330
|
+
if (!env || typeof env !== "object") continue;
|
|
8331
|
+
let matches = false;
|
|
8332
|
+
for (const value of Object.values(env)) {
|
|
8333
|
+
if (typeof value !== "string") continue;
|
|
8334
|
+
const placeholderMatches = value.match(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g);
|
|
8335
|
+
if (placeholderMatches) {
|
|
8336
|
+
for (const ph of placeholderMatches) {
|
|
8337
|
+
const name = ph.slice(2, -1);
|
|
8338
|
+
if (changedSet.has(name)) {
|
|
8339
|
+
matches = true;
|
|
8340
|
+
break;
|
|
8341
|
+
}
|
|
8342
|
+
}
|
|
8343
|
+
}
|
|
8344
|
+
if (matches) break;
|
|
8345
|
+
}
|
|
8346
|
+
if (matches) result.push(serverKey);
|
|
8671
8347
|
}
|
|
8672
|
-
return
|
|
8348
|
+
return result;
|
|
8673
8349
|
}
|
|
8674
|
-
function
|
|
8675
|
-
|
|
8676
|
-
|
|
8677
|
-
|
|
8350
|
+
function envOnlyRespawnVars(changedVars, mcp, channelSecretKeys) {
|
|
8351
|
+
const channelSet = new Set(channelSecretKeys);
|
|
8352
|
+
return [...changedVars].filter(
|
|
8353
|
+
(v) => !channelSet.has(v) && findMcpServersUsingVars(mcp, [v]).length === 0
|
|
8354
|
+
);
|
|
8678
8355
|
}
|
|
8679
|
-
|
|
8680
|
-
|
|
8681
|
-
|
|
8682
|
-
|
|
8683
|
-
|
|
8684
|
-
|
|
8685
|
-
|
|
8686
|
-
|
|
8687
|
-
|
|
8688
|
-
|
|
8689
|
-
|
|
8690
|
-
|
|
8691
|
-
|
|
8692
|
-
|
|
8693
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
8356
|
+
function liveProxyExtraHeaderVars(mcp) {
|
|
8357
|
+
const out = /* @__PURE__ */ new Set();
|
|
8358
|
+
if (!mcp?.mcpServers) return out;
|
|
8359
|
+
for (const entry of Object.values(mcp.mcpServers)) {
|
|
8360
|
+
const spec = entry?.env?.["AGT_REMOTE_MCP_EXTRA_HEADERS"];
|
|
8361
|
+
if (typeof spec !== "string") continue;
|
|
8362
|
+
for (const pair of spec.split(",")) {
|
|
8363
|
+
const colon = pair.indexOf(":");
|
|
8364
|
+
if (colon <= 0) continue;
|
|
8365
|
+
const varName = pair.slice(colon + 1).trim();
|
|
8366
|
+
if (varName) out.add(varName);
|
|
8367
|
+
}
|
|
8368
|
+
}
|
|
8369
|
+
return out;
|
|
8694
8370
|
}
|
|
8695
|
-
function
|
|
8696
|
-
|
|
8697
|
-
|
|
8698
|
-
|
|
8371
|
+
function liveProxyTokenVars(mcp) {
|
|
8372
|
+
const out = /* @__PURE__ */ new Set();
|
|
8373
|
+
if (!mcp?.mcpServers) return out;
|
|
8374
|
+
for (const entry of Object.values(mcp.mcpServers)) {
|
|
8375
|
+
const tokenVar = entry?.env?.["AGT_REMOTE_MCP_TOKEN_VAR"];
|
|
8376
|
+
if (typeof tokenVar === "string" && tokenVar.trim()) {
|
|
8377
|
+
out.add(tokenVar.trim());
|
|
8378
|
+
}
|
|
8379
|
+
}
|
|
8380
|
+
return out;
|
|
8699
8381
|
}
|
|
8700
|
-
|
|
8701
|
-
|
|
8702
|
-
|
|
8703
|
-
|
|
8704
|
-
|
|
8705
|
-
|
|
8706
|
-
|
|
8707
|
-
|
|
8708
|
-
|
|
8709
|
-
|
|
8710
|
-
|
|
8711
|
-
|
|
8712
|
-
|
|
8713
|
-
|
|
8714
|
-
|
|
8715
|
-
|
|
8716
|
-
|
|
8717
|
-
|
|
8718
|
-
* floored against any other tally: it gates a REPORT, not a pause, so it
|
|
8719
|
-
* cannot invert anything the way operatorMax/provisioningMax can.
|
|
8720
|
-
*/
|
|
8721
|
-
scheduledMax;
|
|
8722
|
-
/** ENG-9138: sliding window for the scheduled tally. */
|
|
8723
|
-
scheduledWindowMs;
|
|
8724
|
-
operatorCommandMax;
|
|
8725
|
-
operatorCommandWindowMs;
|
|
8726
|
-
/** Longest window any tally needs — how long the events log must retain. */
|
|
8727
|
-
retentionMs;
|
|
8728
|
-
now;
|
|
8729
|
-
/** ENG-7891 slice 5b: code_name → durable runtime key (agent_id when id-keyed). Identity on the legacy fleet. */
|
|
8730
|
-
runtimeKey;
|
|
8731
|
-
/** ENG-8690: agent creation time lookup; undefined result ⇒ no grace. */
|
|
8732
|
-
agentCreatedAt;
|
|
8733
|
-
/** ENG-8690: grace duration; 0 disables. */
|
|
8734
|
-
newAgentGraceMs;
|
|
8735
|
-
/** ENG-8690: the bound the grace does NOT lift. NOT readonly - floored below. */
|
|
8736
|
-
newAgentGraceCeiling;
|
|
8737
|
-
events = /* @__PURE__ */ new Map();
|
|
8738
|
-
trips = /* @__PURE__ */ new Map();
|
|
8739
|
-
/**
|
|
8740
|
-
* ENG-8215: set when a supplied `provisioningMax` was raised to keep MCP
|
|
8741
|
-
* quarantine ahead of the breaker. Read once by the manager at construction so
|
|
8742
|
-
* an overridden-and-clamped value is visible in manager.log rather than
|
|
8743
|
-
* silently discarded.
|
|
8744
|
-
*/
|
|
8745
|
-
quarantineOrderingClamped;
|
|
8746
|
-
constructor(opts = {}) {
|
|
8747
|
-
this.max = opts.max ?? readEnvNumber("AGT_RESTART_BREAKER_MAX", DEFAULT_MAX);
|
|
8748
|
-
this.windowMs = opts.windowMs ?? readEnvNumber("AGT_RESTART_BREAKER_WINDOW_MS", DEFAULT_WINDOW_MS);
|
|
8749
|
-
this.provisioningMax = opts.provisioningMax ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_MAX", DEFAULT_PROVISIONING_MAX);
|
|
8750
|
-
this.provisioningWindowMs = opts.provisioningWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_PROVISIONING_WINDOW_MS", DEFAULT_PROVISIONING_WINDOW_MS);
|
|
8751
|
-
this.operatorMax = opts.operatorMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_MAX", DEFAULT_OPERATOR_MAX);
|
|
8752
|
-
this.scheduledMax = opts.scheduledMax ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_MAX", DEFAULT_SCHEDULED_MAX);
|
|
8753
|
-
this.scheduledWindowMs = opts.scheduledWindowMs ?? readEnvNumber("AGT_RESTART_BREAKER_SCHEDULED_WINDOW_MS", DEFAULT_SCHEDULED_WINDOW_MS);
|
|
8754
|
-
this.operatorCommandMax = opts.operatorCommandMax ?? readEnvNumber("AGT_RESTART_BREAKER_OPERATOR_COMMAND_MAX", DEFAULT_OPERATOR_COMMAND_MAX);
|
|
8755
|
-
this.operatorCommandWindowMs = opts.operatorCommandWindowMs ?? readEnvNumber(
|
|
8756
|
-
"AGT_RESTART_BREAKER_OPERATOR_COMMAND_WINDOW_MS",
|
|
8757
|
-
DEFAULT_OPERATOR_COMMAND_WINDOW_MS
|
|
8758
|
-
);
|
|
8759
|
-
this.now = opts.now ?? Date.now;
|
|
8760
|
-
this.runtimeKey = opts.runtimeKey ?? ((codeName) => codeName);
|
|
8761
|
-
this.agentCreatedAt = opts.agentCreatedAt ?? (() => void 0);
|
|
8762
|
-
const graceMs = opts.newAgentGraceMs ?? readEnvNumber("AGT_RESTART_BREAKER_NEW_AGENT_GRACE_MS", RESTART_BREAKER_NEW_AGENT_GRACE_MS);
|
|
8763
|
-
const cappedGrace = Math.min(graceMs, MAX_NEW_AGENT_GRACE_MS);
|
|
8764
|
-
this.newAgentGraceMs = Number.isFinite(graceMs) && graceMs > 0 ? cappedGrace : 0;
|
|
8765
|
-
const ceiling = opts.newAgentGraceCeiling ?? readEnvNumber(
|
|
8766
|
-
"AGT_RESTART_BREAKER_NEW_AGENT_GRACE_CEILING",
|
|
8767
|
-
RESTART_BREAKER_NEW_AGENT_GRACE_CEILING
|
|
8382
|
+
function buildArgvMatchersForEntry(key, entry) {
|
|
8383
|
+
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
8384
|
+
const patterns = [];
|
|
8385
|
+
const args = entry?.args ?? [];
|
|
8386
|
+
for (let i = args.length - 1; i >= 0; i--) {
|
|
8387
|
+
const arg = args[i];
|
|
8388
|
+
if (typeof arg !== "string" || !arg) continue;
|
|
8389
|
+
if (arg.startsWith("-")) continue;
|
|
8390
|
+
const stripped = arg.replace(
|
|
8391
|
+
/@[^/@]*$/,
|
|
8392
|
+
(m) => (
|
|
8393
|
+
// @latest, @1.2.3, etc. — drop. But @scope at the START of a
|
|
8394
|
+
// package spec must NOT be stripped, so we only strip a tail
|
|
8395
|
+
// `@...` if it doesn't itself start with a slash inside.
|
|
8396
|
+
// The regex above only matches a single trailing `@...`
|
|
8397
|
+
// segment after the last `/`, so this is safe for `@scope/pkg`.
|
|
8398
|
+
m.includes("/") ? m : ""
|
|
8399
|
+
)
|
|
8768
8400
|
);
|
|
8769
|
-
|
|
8770
|
-
if (
|
|
8771
|
-
|
|
8772
|
-
|
|
8773
|
-
|
|
8774
|
-
|
|
8401
|
+
const tail = "(?![A-Za-z0-9_-])";
|
|
8402
|
+
if (/^@?[a-z0-9]([a-z0-9._-]*\/)?[a-z0-9._-]+$/i.test(stripped)) {
|
|
8403
|
+
patterns.push(new RegExp(`${escapeRe(stripped)}${tail}`));
|
|
8404
|
+
const basename = stripped.split("/").pop();
|
|
8405
|
+
if (basename && basename !== stripped) {
|
|
8406
|
+
patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
|
|
8407
|
+
}
|
|
8408
|
+
break;
|
|
8775
8409
|
}
|
|
8776
|
-
if (
|
|
8777
|
-
|
|
8410
|
+
if (stripped.includes("/")) {
|
|
8411
|
+
const basename = stripped.split("/").pop();
|
|
8412
|
+
if (basename) {
|
|
8413
|
+
patterns.push(new RegExp(`${escapeRe(basename)}${tail}`));
|
|
8414
|
+
break;
|
|
8415
|
+
}
|
|
8778
8416
|
}
|
|
8779
|
-
|
|
8780
|
-
|
|
8417
|
+
}
|
|
8418
|
+
if (patterns.length === 0) {
|
|
8419
|
+
const safe = escapeRe(key);
|
|
8420
|
+
patterns.push(new RegExp(`(?<![A-Za-z0-9_])${safe}(?![A-Za-z0-9_])`));
|
|
8421
|
+
if (safe.includes("_")) {
|
|
8422
|
+
const dashed = safe.replace(/_/g, "-");
|
|
8423
|
+
patterns.push(new RegExp(`(?<![A-Za-z0-9_])${dashed}(?![A-Za-z0-9_])`));
|
|
8781
8424
|
}
|
|
8782
|
-
|
|
8783
|
-
|
|
8784
|
-
|
|
8785
|
-
|
|
8786
|
-
|
|
8787
|
-
|
|
8788
|
-
|
|
8789
|
-
|
|
8790
|
-
|
|
8791
|
-
|
|
8792
|
-
|
|
8793
|
-
|
|
8794
|
-
|
|
8795
|
-
|
|
8796
|
-
|
|
8797
|
-
|
|
8798
|
-
|
|
8799
|
-
|
|
8800
|
-
|
|
8801
|
-
|
|
8802
|
-
|
|
8803
|
-
|
|
8425
|
+
}
|
|
8426
|
+
return patterns;
|
|
8427
|
+
}
|
|
8428
|
+
function buildClaudeAgentMatcher(codeName) {
|
|
8429
|
+
const safe = codeName.replace(/[^A-Za-z0-9_-]/g, "");
|
|
8430
|
+
return new RegExp(`\\bclaude\\b.*--name\\s+agt-${safe}(?=\\s|$)`);
|
|
8431
|
+
}
|
|
8432
|
+
function findMcpChildrenForAgent(args) {
|
|
8433
|
+
const { rows, codeName, serverKeys, mcpJson } = args;
|
|
8434
|
+
const maxDepth = args.maxDepth ?? 8;
|
|
8435
|
+
const inContainer = args.inContainer ?? false;
|
|
8436
|
+
if (serverKeys.length === 0 || rows.length === 0) return [];
|
|
8437
|
+
const claudeMatcher = inContainer ? /\bclaude\b/ : buildClaudeAgentMatcher(codeName);
|
|
8438
|
+
const argvMatchers = [];
|
|
8439
|
+
for (const key of serverKeys) {
|
|
8440
|
+
const entry = mcpJson?.mcpServers?.[key];
|
|
8441
|
+
argvMatchers.push(...buildArgvMatchersForEntry(key, entry));
|
|
8442
|
+
}
|
|
8443
|
+
const byPid = new Map(rows.map((r) => [r.pid, r]));
|
|
8444
|
+
const matched = [];
|
|
8445
|
+
for (const row of rows) {
|
|
8446
|
+
if (!argvMatchers.some((re) => re.test(row.args))) continue;
|
|
8447
|
+
if (/\bclaude\b/.test(row.args) && row.args.includes(`--name agt-${codeName}`)) continue;
|
|
8448
|
+
let cur = byPid.get(row.ppid);
|
|
8449
|
+
let belongs = false;
|
|
8450
|
+
for (let depth = 0; depth < maxDepth && cur; depth++) {
|
|
8451
|
+
if (claudeMatcher.test(cur.args)) {
|
|
8452
|
+
belongs = true;
|
|
8453
|
+
break;
|
|
8454
|
+
}
|
|
8455
|
+
if (cur.pid === 1) break;
|
|
8456
|
+
cur = byPid.get(cur.ppid);
|
|
8804
8457
|
}
|
|
8805
|
-
if (
|
|
8806
|
-
|
|
8458
|
+
if (belongs) matched.push(row.pid);
|
|
8459
|
+
}
|
|
8460
|
+
return matched;
|
|
8461
|
+
}
|
|
8462
|
+
function reapStaleMcpChildren(args) {
|
|
8463
|
+
const { log: log2, codeName, serverKeys, mcpJson, graceMs = 5e3, isolated = false } = args;
|
|
8464
|
+
if (serverKeys.length === 0) return [];
|
|
8465
|
+
const runPs = args.runPs ?? (isolated ? (
|
|
8466
|
+
// ENG-9781: hard-bounded. See bounded-exec.ts — `execFileSync`'s own
|
|
8467
|
+
// `timeout` is not a bound, and this runs per isolated agent per cycle
|
|
8468
|
+
// on the manager-worker's critical path. Throws on a non-answer so the
|
|
8469
|
+
// caller's existing catch skips the agent rather than acting on an
|
|
8470
|
+
// empty process list.
|
|
8471
|
+
() => {
|
|
8472
|
+
const r = hardBoundedExecFile(
|
|
8473
|
+
"docker",
|
|
8474
|
+
["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
|
|
8475
|
+
{ timeoutMs: 8e3, log: log2 }
|
|
8476
|
+
);
|
|
8477
|
+
if (r.kind === "ok") return r.stdout;
|
|
8478
|
+
throw new Error(
|
|
8479
|
+
r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
|
|
8480
|
+
);
|
|
8807
8481
|
}
|
|
8808
|
-
|
|
8809
|
-
|
|
8810
|
-
|
|
8482
|
+
) : () => execFileSync("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
|
|
8483
|
+
const killProcess = args.killProcess ?? (isolated ? (pid, signal) => {
|
|
8484
|
+
hardBoundedExecFile(
|
|
8485
|
+
"docker",
|
|
8486
|
+
["exec", `agt-${codeName}`, "kill", signal === "SIGKILL" ? "-KILL" : "-TERM", String(pid)],
|
|
8487
|
+
{ timeoutMs: 8e3, log: log2 }
|
|
8488
|
+
);
|
|
8489
|
+
} : (pid, signal) => {
|
|
8490
|
+
try {
|
|
8491
|
+
process.kill(pid, signal);
|
|
8492
|
+
} catch {
|
|
8811
8493
|
}
|
|
8812
|
-
|
|
8813
|
-
|
|
8814
|
-
|
|
8815
|
-
|
|
8816
|
-
|
|
8494
|
+
});
|
|
8495
|
+
const isAlive = args.isAlive ?? (isolated ? (pid) => {
|
|
8496
|
+
const r = hardBoundedExecFile(
|
|
8497
|
+
"docker",
|
|
8498
|
+
["exec", `agt-${codeName}`, "kill", "-0", String(pid)],
|
|
8499
|
+
{ timeoutMs: 8e3, log: log2 }
|
|
8500
|
+
);
|
|
8501
|
+
return livenessFromProbe(
|
|
8502
|
+
r,
|
|
8503
|
+
(why) => log2(
|
|
8504
|
+
`[stale-mcp-reaper] liveness probe for pid ${pid} in 'agt-${codeName}' ${why} \u2014 treating as ALIVE (no reap) rather than guessing`
|
|
8505
|
+
)
|
|
8817
8506
|
);
|
|
8507
|
+
} : (pid) => {
|
|
8508
|
+
try {
|
|
8509
|
+
process.kill(pid, 0);
|
|
8510
|
+
return true;
|
|
8511
|
+
} catch {
|
|
8512
|
+
return false;
|
|
8513
|
+
}
|
|
8514
|
+
});
|
|
8515
|
+
let psOutput;
|
|
8516
|
+
try {
|
|
8517
|
+
psOutput = runPs();
|
|
8518
|
+
} catch (err) {
|
|
8519
|
+
log2(`[stale-mcp-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping reap`);
|
|
8520
|
+
return [];
|
|
8818
8521
|
}
|
|
8819
|
-
|
|
8820
|
-
|
|
8821
|
-
|
|
8822
|
-
|
|
8823
|
-
|
|
8824
|
-
|
|
8825
|
-
|
|
8522
|
+
const rows = parsePsRows(psOutput);
|
|
8523
|
+
const targets = findMcpChildrenForAgent({ rows, codeName, serverKeys, mcpJson, inContainer: isolated });
|
|
8524
|
+
if (targets.length === 0) return [];
|
|
8525
|
+
const byPid = new Map(rows.map((r) => [r.pid, r]));
|
|
8526
|
+
const describe = (pid) => {
|
|
8527
|
+
const argv = byPid.get(pid)?.args ?? "";
|
|
8528
|
+
const pkgMatch = argv.match(/(@[a-z0-9_-]+\/[a-z0-9_-]+|[a-z0-9_-]+-mcp-server|[a-z0-9_-]+-mcp\b)/i);
|
|
8529
|
+
return pkgMatch ? `${pkgMatch[1]} (pid ${pid})` : `pid ${pid}`;
|
|
8530
|
+
};
|
|
8531
|
+
log2(
|
|
8532
|
+
`[stale-mcp-reaper] '${codeName}': rotating ${targets.length} stale MCP child(ren) for [${serverKeys.join(", ")}]: ${targets.map(describe).join(", ")}`
|
|
8533
|
+
);
|
|
8534
|
+
for (const pid of targets) {
|
|
8535
|
+
killProcess(pid, "SIGTERM");
|
|
8826
8536
|
}
|
|
8827
|
-
|
|
8828
|
-
|
|
8829
|
-
|
|
8830
|
-
|
|
8831
|
-
|
|
8832
|
-
|
|
8833
|
-
|
|
8834
|
-
* maps migrate together so events and a trip for the same agent never split
|
|
8835
|
-
* across keys mid-generation (a migration can flip the resolver mid-process).
|
|
8836
|
-
*
|
|
8837
|
-
* BOUNDARY — a pure-LEGACY rename (a host that was never armed) still strands a
|
|
8838
|
-
* held trip, exactly as it did before slice 5b: with no symlink there is no
|
|
8839
|
-
* agent_id to bridge the old codename to the new, so the runtime key stays the
|
|
8840
|
-
* (new) codename and the trip persisted under the old codename is not found.
|
|
8841
|
-
* That is unchanged, pre-existing behaviour and it is benign — the fresh
|
|
8842
|
-
* generation re-evaluates the agent under its new name and a still-pathological
|
|
8843
|
-
* agent simply re-trips within the window. The rename-survival guarantee this
|
|
8844
|
-
* slice adds is an id-keyed-only, post-arming property (the symlink is the
|
|
8845
|
-
* bridge); it is deliberately NOT retrofitted onto the legacy path.
|
|
8846
|
-
*/
|
|
8847
|
-
keyFor(codeName) {
|
|
8848
|
-
const key = this.runtimeKey(codeName);
|
|
8849
|
-
if (key !== codeName) {
|
|
8850
|
-
migrateMapKey(this.events, codeName, key);
|
|
8851
|
-
migrateMapKey(this.trips, codeName, key);
|
|
8537
|
+
const killedKeys = /* @__PURE__ */ new Set();
|
|
8538
|
+
for (const pid of targets) {
|
|
8539
|
+
const argv = byPid.get(pid)?.args ?? "";
|
|
8540
|
+
for (const key of serverKeys) {
|
|
8541
|
+
const entry = mcpJson?.mcpServers?.[key];
|
|
8542
|
+
const matchers = buildArgvMatchersForEntry(key, entry);
|
|
8543
|
+
if (matchers.some((re) => re.test(argv))) killedKeys.add(key);
|
|
8852
8544
|
}
|
|
8853
|
-
return key;
|
|
8854
8545
|
}
|
|
8855
|
-
|
|
8856
|
-
|
|
8857
|
-
|
|
8546
|
+
if (killedKeys.size > 0) recordStaleRotation(codeName, killedKeys);
|
|
8547
|
+
setTimeout(() => {
|
|
8548
|
+
try {
|
|
8549
|
+
let freshPsOutput;
|
|
8550
|
+
try {
|
|
8551
|
+
freshPsOutput = runPs();
|
|
8552
|
+
} catch (err) {
|
|
8553
|
+
log2(`[stale-mcp-reaper] '${codeName}': fresh ps for SIGKILL re-verify failed: ${err.message} \u2014 skipping SIGKILL pass`);
|
|
8554
|
+
return;
|
|
8555
|
+
}
|
|
8556
|
+
const stillOwned = new Set(
|
|
8557
|
+
findMcpChildrenForAgent({
|
|
8558
|
+
rows: parsePsRows(freshPsOutput),
|
|
8559
|
+
codeName,
|
|
8560
|
+
serverKeys,
|
|
8561
|
+
mcpJson,
|
|
8562
|
+
inContainer: isolated
|
|
8563
|
+
})
|
|
8564
|
+
);
|
|
8565
|
+
const stragglers = targets.filter((pid) => isAlive(pid) && stillOwned.has(pid));
|
|
8566
|
+
if (stragglers.length === 0) return;
|
|
8567
|
+
log2(
|
|
8568
|
+
`[stale-mcp-reaper] '${codeName}': ${stragglers.length} child(ren) survived SIGTERM; sending SIGKILL: ${stragglers.map(describe).join(", ")}`
|
|
8569
|
+
);
|
|
8570
|
+
for (const pid of stragglers) {
|
|
8571
|
+
killProcess(pid, "SIGKILL");
|
|
8572
|
+
}
|
|
8573
|
+
} catch (err) {
|
|
8574
|
+
log2(`[stale-mcp-reaper] '${codeName}': error in SIGKILL pass: ${err.message}`);
|
|
8575
|
+
}
|
|
8576
|
+
}, graceMs).unref();
|
|
8577
|
+
return targets;
|
|
8578
|
+
}
|
|
8579
|
+
|
|
8580
|
+
// src/lib/mcp-presence-reaper.ts
|
|
8581
|
+
import { execFileSync as execFileSync2 } from "child_process";
|
|
8582
|
+
var DEFAULT_COLD_START_GRACE_MS = 9e4;
|
|
8583
|
+
var MAX_PRESENCE_RESTART_ATTEMPTS = 3;
|
|
8584
|
+
var BACKOFF_AFTER_RESTARTS = MAX_PRESENCE_RESTART_ATTEMPTS;
|
|
8585
|
+
var BACKOFF_BASE_MS = 6e4;
|
|
8586
|
+
var BACKOFF_MAX_MS = 30 * 6e4;
|
|
8587
|
+
var BACKOFF_WINDOW_MS = 60 * 6e4;
|
|
8588
|
+
var BACKOFF_GIVE_UP_RESTARTS = 2 * MAX_PRESENCE_RESTART_ATTEMPTS;
|
|
8589
|
+
var BACKOFF_STABLE_RESET_MS = 5 * 6e4;
|
|
8590
|
+
var DEAD_CHANNEL_MS = 15 * 6e4;
|
|
8591
|
+
var presenceReaperState = /* @__PURE__ */ new Map();
|
|
8592
|
+
var MAX_ROTATION_PROMPT_PROBES = 3;
|
|
8593
|
+
var ROTATION_PROMPT_PROBE_INTERVAL_MS = 3e4;
|
|
8594
|
+
var rotationStallState = /* @__PURE__ */ new Map();
|
|
8595
|
+
var blockedPromptEpisodes = /* @__PURE__ */ new Map();
|
|
8596
|
+
function stateKey(codeName, serverKey) {
|
|
8597
|
+
return `${codeName}\0${serverKey}`;
|
|
8598
|
+
}
|
|
8599
|
+
function clearPresenceReaperState(codeName) {
|
|
8600
|
+
const prefix = `${codeName}\0`;
|
|
8601
|
+
for (const key of presenceReaperState.keys()) {
|
|
8602
|
+
if (key.startsWith(prefix)) presenceReaperState.delete(key);
|
|
8858
8603
|
}
|
|
8859
|
-
|
|
8860
|
-
|
|
8604
|
+
}
|
|
8605
|
+
function clearPresenceReaperStateForKeys(codeName, keys) {
|
|
8606
|
+
for (const key of keys) {
|
|
8607
|
+
presenceReaperState.delete(stateKey(codeName, key));
|
|
8608
|
+
rotationStallState.delete(stateKey(codeName, key));
|
|
8861
8609
|
}
|
|
8862
|
-
|
|
8863
|
-
|
|
8864
|
-
|
|
8865
|
-
|
|
8866
|
-
|
|
8867
|
-
|
|
8868
|
-
|
|
8869
|
-
* via the decision-log for forensics.
|
|
8870
|
-
*/
|
|
8871
|
-
record(codeName, reason, integrationKey) {
|
|
8872
|
-
const key = this.keyFor(codeName);
|
|
8873
|
-
const existing = this.trips.get(key);
|
|
8874
|
-
if (existing) {
|
|
8875
|
-
const bucket = maxProvisioningBucket(existing.eventsAtTrip);
|
|
8876
|
-
return {
|
|
8877
|
-
tripped: false,
|
|
8878
|
-
trip: existing,
|
|
8879
|
-
windowCount: existing.eventsAtTrip.length,
|
|
8880
|
-
classCounts: countByClass(existing.eventsAtTrip),
|
|
8881
|
-
maxProvisioningBucket: bucket.count,
|
|
8882
|
-
maxProvisioningBucketKey: bucket.key
|
|
8883
|
-
};
|
|
8610
|
+
}
|
|
8611
|
+
function givenUpMcpServerKeys(codeName) {
|
|
8612
|
+
const prefix = `${codeName}\0`;
|
|
8613
|
+
const out = /* @__PURE__ */ new Set();
|
|
8614
|
+
for (const [key, st] of presenceReaperState) {
|
|
8615
|
+
if (key.startsWith(prefix) && (st.gaveUp || st.attempts >= MAX_PRESENCE_RESTART_ATTEMPTS)) {
|
|
8616
|
+
out.add(key.slice(prefix.length));
|
|
8884
8617
|
}
|
|
8885
|
-
|
|
8886
|
-
|
|
8887
|
-
|
|
8888
|
-
|
|
8889
|
-
|
|
8890
|
-
|
|
8891
|
-
|
|
8892
|
-
|
|
8893
|
-
|
|
8894
|
-
|
|
8895
|
-
|
|
8618
|
+
}
|
|
8619
|
+
return out;
|
|
8620
|
+
}
|
|
8621
|
+
function findMissingMcpServers(args) {
|
|
8622
|
+
const { rows, codeName, mcpJson, inContainer } = args;
|
|
8623
|
+
const servers = mcpJson?.mcpServers ?? {};
|
|
8624
|
+
const declared = Object.keys(servers);
|
|
8625
|
+
if (declared.length === 0) return [];
|
|
8626
|
+
const missing = [];
|
|
8627
|
+
for (const key of declared) {
|
|
8628
|
+
const entry = servers[key];
|
|
8629
|
+
const isStdio = typeof entry?.command === "string";
|
|
8630
|
+
if (!isStdio) continue;
|
|
8631
|
+
const live = findMcpChildrenForAgent({
|
|
8632
|
+
rows,
|
|
8633
|
+
codeName,
|
|
8634
|
+
serverKeys: [key],
|
|
8635
|
+
mcpJson,
|
|
8636
|
+
inContainer
|
|
8637
|
+
});
|
|
8638
|
+
if (live.length === 0) missing.push(key);
|
|
8639
|
+
}
|
|
8640
|
+
return missing;
|
|
8641
|
+
}
|
|
8642
|
+
function describeBlockedPromptHold(ctx) {
|
|
8643
|
+
const keys = `declared MCP(s) [${ctx.activeKeys.join(", ")}]`;
|
|
8644
|
+
if (ctx.restartDeferred) {
|
|
8645
|
+
return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. The recovery restart was ATTEMPTED and the host deferred it (maintenance window or agent not idle), so nothing was torn down and this episode's one restart is still unspent - it retries when the gate opens. Needs a human if it persists`;
|
|
8646
|
+
}
|
|
8647
|
+
if (ctx.restartSpentThisEpisode) {
|
|
8648
|
+
return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. This episode's ONE restart was already spent and the agent is STILL blocked, so a further restart is not the answer. Needs a human at the pane`;
|
|
8649
|
+
}
|
|
8650
|
+
if (ctx.restartBudgetRemaining === 0) {
|
|
8651
|
+
return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children. No restart was attempted: the crash breaker has NO budget left in its window, so a restart now would be the event that trips it and pauses this agent - attributed to mcp-presence-reaper, not to whatever filled the window. It may clear once the breaker window rolls; needs a human if it does not`;
|
|
8652
|
+
}
|
|
8653
|
+
return `session blocked on an interactive prompt (${ctx.kind}); ${keys} have no live children and no restart was made. Needs a human at the pane`;
|
|
8654
|
+
}
|
|
8655
|
+
function reapMissingMcpSessions(args) {
|
|
8656
|
+
const {
|
|
8657
|
+
log: log2,
|
|
8658
|
+
codeName,
|
|
8659
|
+
mcpJson,
|
|
8660
|
+
sessionStartedAt,
|
|
8661
|
+
restartPending = false,
|
|
8662
|
+
stopSession,
|
|
8663
|
+
graceMs = DEFAULT_COLD_START_GRACE_MS,
|
|
8664
|
+
rotationGraceMs = DEFAULT_ROTATION_GRACE_MS,
|
|
8665
|
+
onGiveUp,
|
|
8666
|
+
onRestart,
|
|
8667
|
+
// ENG-8225: unwired callbacks make the prompt-probe path a strict no-op —
|
|
8668
|
+
// `onRotationRespawnStalled` absent means "not armed", which falls through to
|
|
8669
|
+
// the pre-ENG-8225 restart. Legacy callers and tests are unchanged.
|
|
8670
|
+
onRotationRespawnStalled,
|
|
8671
|
+
onRotationRespawnGaveUp,
|
|
8672
|
+
onRotationRespawnOutcome,
|
|
8673
|
+
maxRotationPromptProbes = MAX_ROTATION_PROMPT_PROBES,
|
|
8674
|
+
rotationPromptProbeIntervalMs = ROTATION_PROMPT_PROBE_INTERVAL_MS,
|
|
8675
|
+
// ENG-5932: defaults make quarantine a strict no-op (classify everything
|
|
8676
|
+
// essential, mode off) so legacy callers/tests are unaffected.
|
|
8677
|
+
classifyKey = () => "essential",
|
|
8678
|
+
quarantineDwellMs = 0,
|
|
8679
|
+
deadChannelMs = DEAD_CHANNEL_MS,
|
|
8680
|
+
quarantineMode = "off",
|
|
8681
|
+
onQuarantine,
|
|
8682
|
+
isolated = false,
|
|
8683
|
+
// ENG-8013: default false → a "no live child" observation is only excluded
|
|
8684
|
+
// from the give-up/quarantine budget when the caller positively identifies
|
|
8685
|
+
// the current generation as externally-churned. Legacy callers/tests unchanged.
|
|
8686
|
+
currentGenerationExternallyStarted = false,
|
|
8687
|
+
// ENG-9932: unwired => strict no-op, restart proceeds exactly as before.
|
|
8688
|
+
readPane,
|
|
8689
|
+
onPaneBlocked
|
|
8690
|
+
} = args;
|
|
8691
|
+
const now = args.now ?? Date.now;
|
|
8692
|
+
const runPs = args.runPs ?? (isolated ? (
|
|
8693
|
+
// ENG-9781: HARD-bounded, and the distinction is the whole ticket. The
|
|
8694
|
+
// previous `execFileSync(..., { timeout: 8_000 })` bounded nothing —
|
|
8695
|
+
// Node fires SIGTERM at the deadline and then keeps waiting, so a
|
|
8696
|
+
// docker CLI talking to a wedged daemon blocked the manager-worker's
|
|
8697
|
+
// event loop and stopped scheduling for EVERY agent on the host
|
|
8698
|
+
// (33m46s, twice, on 2026-09-01). This call runs once per isolated
|
|
8699
|
+
// agent per poll cycle, so it is the single worst place on the manager
|
|
8700
|
+
// for an unbounded subprocess.
|
|
8701
|
+
//
|
|
8702
|
+
// Throwing on a non-answer is deliberate rather than lazy: the caller
|
|
8703
|
+
// already catches, logs and SKIPS the agent for the cycle, which is
|
|
8704
|
+
// exactly right. A timeout must not become an empty process list —
|
|
8705
|
+
// that reads as "every declared MCP server is missing" and restarts the
|
|
8706
|
+
// session, turning a docker hiccup into a fleet-wide restart storm.
|
|
8707
|
+
() => {
|
|
8708
|
+
const r = hardBoundedExecFile(
|
|
8709
|
+
"docker",
|
|
8710
|
+
["exec", `agt-${codeName}`, "ps", "-eo", "pid,ppid,args"],
|
|
8711
|
+
{ timeoutMs: 8e3, log: log2 }
|
|
8712
|
+
);
|
|
8713
|
+
if (r.kind === "ok") return r.stdout;
|
|
8714
|
+
throw new Error(
|
|
8715
|
+
r.kind === "timed-out" ? `docker exec ps did not answer within its bound (bounded=${r.bounded})` : `docker exec ps failed (status=${r.status}): ${r.message}`
|
|
8716
|
+
);
|
|
8717
|
+
}
|
|
8718
|
+
) : () => execFileSync2("ps", ["-eo", "pid,ppid,args"], { encoding: "utf-8", timeout: 5e3 }));
|
|
8719
|
+
if (!mcpJson?.mcpServers) {
|
|
8720
|
+
return { missing: [], restarted: false, reason: "no-mcp-json" };
|
|
8721
|
+
}
|
|
8722
|
+
const declaredCount = Object.keys(mcpJson.mcpServers).length;
|
|
8723
|
+
if (declaredCount === 0) {
|
|
8724
|
+
return { missing: [], restarted: false, reason: "no-declared-servers" };
|
|
8725
|
+
}
|
|
8726
|
+
if (restartPending) {
|
|
8727
|
+
return { missing: [], restarted: false, reason: "restart-pending" };
|
|
8728
|
+
}
|
|
8729
|
+
if (sessionStartedAt === null) {
|
|
8730
|
+
return { missing: [], restarted: false, reason: "session-start-unknown" };
|
|
8731
|
+
}
|
|
8732
|
+
if (now() - sessionStartedAt < graceMs) {
|
|
8733
|
+
return { missing: [], restarted: false, reason: "cold-start" };
|
|
8734
|
+
}
|
|
8735
|
+
let psOutput;
|
|
8736
|
+
try {
|
|
8737
|
+
psOutput = runPs();
|
|
8738
|
+
} catch (err) {
|
|
8739
|
+
log2(`[mcp-presence-reaper] ps invocation failed for '${codeName}': ${err.message} \u2014 skipping`);
|
|
8740
|
+
return { missing: [], restarted: false, reason: "healthy" };
|
|
8741
|
+
}
|
|
8742
|
+
const rows = parsePsRows(psOutput);
|
|
8743
|
+
const missingRaw = findMissingMcpServers({ rows, codeName, mcpJson, inContainer: isolated });
|
|
8744
|
+
const rotationGraced = [];
|
|
8745
|
+
const rotationProbeArmed = [];
|
|
8746
|
+
const rotationProbeWaiting = [];
|
|
8747
|
+
const rotationProbeExhausted = [];
|
|
8748
|
+
const rotationRespawnOutcomes = {};
|
|
8749
|
+
const recordRotationOutcome = (key, outcome) => {
|
|
8750
|
+
rotationRespawnOutcomes[key] = outcome;
|
|
8751
|
+
if (!onRotationRespawnOutcome) return;
|
|
8752
|
+
try {
|
|
8753
|
+
onRotationRespawnOutcome(codeName, key, outcome);
|
|
8754
|
+
} catch (err) {
|
|
8755
|
+
log2(
|
|
8756
|
+
`[mcp-presence-reaper] onRotationRespawnOutcome callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
|
|
8757
|
+
);
|
|
8758
|
+
}
|
|
8759
|
+
};
|
|
8760
|
+
const missing = [];
|
|
8761
|
+
for (const key of missingRaw) {
|
|
8762
|
+
const rotation = classifyRotation(codeName, key, rotationGraceMs, now);
|
|
8763
|
+
if (rotation === "within-grace") {
|
|
8764
|
+
rotationGraced.push(key);
|
|
8765
|
+
continue;
|
|
8766
|
+
}
|
|
8767
|
+
if (rotation === "none") {
|
|
8768
|
+
rotationStallState.delete(stateKey(codeName, key));
|
|
8769
|
+
missing.push(key);
|
|
8770
|
+
continue;
|
|
8771
|
+
}
|
|
8772
|
+
const sk = stateKey(codeName, key);
|
|
8773
|
+
const rotationRecordedAt = getRotationRecordedAt(codeName, key) ?? now();
|
|
8774
|
+
const priorStall = rotationStallState.get(sk);
|
|
8775
|
+
const stall = priorStall && priorStall.rotationRecordedAt === rotationRecordedAt ? priorStall : {
|
|
8776
|
+
rotationRecordedAt,
|
|
8777
|
+
probeAttempts: 0,
|
|
8778
|
+
lastProbeAt: null,
|
|
8779
|
+
escalated: false
|
|
8780
|
+
};
|
|
8781
|
+
if (priorStall && priorStall !== stall) {
|
|
8782
|
+
log2(
|
|
8783
|
+
`[mcp-presence-reaper] '${codeName}:${key}' discarding stall state from a previous rotation (rotation=${priorStall.rotationRecordedAt}, probes=${priorStall.probeAttempts}, escalated=${priorStall.escalated}) \u2014 this is a NEW rotation (rotation=${rotationRecordedAt}) and gets a full prompt-probe budget (ENG-8225)`
|
|
8784
|
+
);
|
|
8785
|
+
}
|
|
8786
|
+
rotationStallState.set(sk, stall);
|
|
8787
|
+
if (stall.escalated) {
|
|
8788
|
+
missing.push(key);
|
|
8789
|
+
continue;
|
|
8790
|
+
}
|
|
8791
|
+
const spacingElapsed = stall.lastProbeAt === null || now() - stall.lastProbeAt >= rotationPromptProbeIntervalMs;
|
|
8792
|
+
if (!spacingElapsed) {
|
|
8793
|
+
rotationProbeWaiting.push(key);
|
|
8794
|
+
continue;
|
|
8795
|
+
}
|
|
8796
|
+
if (stall.probeAttempts >= maxRotationPromptProbes) {
|
|
8797
|
+
stall.escalated = true;
|
|
8798
|
+
rotationProbeExhausted.push(key);
|
|
8799
|
+
recordRotationOutcome(key, "never-respawned");
|
|
8800
|
+
log2(
|
|
8801
|
+
`[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=never-respawned \u2014 child SIGTERM'd by stale-mcp-reaper never returned after ${stall.probeAttempts} prompt bind-probe(s); escalating (integration marked unhealthy + reaper_action alert) and releasing to the normal restart path (ENG-8225)`
|
|
8802
|
+
);
|
|
8803
|
+
if (onRotationRespawnGaveUp) {
|
|
8804
|
+
try {
|
|
8805
|
+
onRotationRespawnGaveUp(codeName, key, { attempts: stall.probeAttempts });
|
|
8806
|
+
} catch (err) {
|
|
8807
|
+
log2(
|
|
8808
|
+
`[mcp-presence-reaper] onRotationRespawnGaveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
|
|
8809
|
+
);
|
|
8810
|
+
}
|
|
8811
|
+
}
|
|
8812
|
+
missing.push(key);
|
|
8813
|
+
continue;
|
|
8814
|
+
}
|
|
8815
|
+
const attempt = stall.probeAttempts + 1;
|
|
8816
|
+
let armed = false;
|
|
8817
|
+
if (onRotationRespawnStalled) {
|
|
8818
|
+
try {
|
|
8819
|
+
armed = onRotationRespawnStalled(codeName, key, {
|
|
8820
|
+
attempt,
|
|
8821
|
+
maxAttempts: maxRotationPromptProbes
|
|
8822
|
+
});
|
|
8823
|
+
} catch (err) {
|
|
8824
|
+
log2(
|
|
8825
|
+
`[mcp-presence-reaper] onRotationRespawnStalled callback threw for '${codeName}:${key}' (suppressed; falling back to the pre-ENG-8225 restart path): ${err.message}`
|
|
8826
|
+
);
|
|
8827
|
+
armed = false;
|
|
8828
|
+
}
|
|
8829
|
+
}
|
|
8830
|
+
if (!armed) {
|
|
8831
|
+
missing.push(key);
|
|
8832
|
+
continue;
|
|
8833
|
+
}
|
|
8834
|
+
stall.probeAttempts = attempt;
|
|
8835
|
+
stall.lastProbeAt = now();
|
|
8836
|
+
rotationProbeArmed.push(key);
|
|
8837
|
+
log2(
|
|
8838
|
+
`[mcp-presence-reaper] '${codeName}:${key}' rotation grace expired (${rotationGraceMs}ms) with no live child and no tool call to respawn it \u2014 armed prompt bind-probe ${attempt}/${maxRotationPromptProbes} (ENG-7429) instead of restarting the session (ENG-8225)`
|
|
8896
8839
|
);
|
|
8897
|
-
|
|
8898
|
-
|
|
8899
|
-
|
|
8900
|
-
|
|
8901
|
-
(e) => e.at >= at - this.provisioningWindowMs && isSelfHealingReason(e.reason)
|
|
8902
|
-
).length;
|
|
8903
|
-
const operatorEvents = prior.filter(
|
|
8904
|
-
(e) => e.at >= at - this.provisioningWindowMs && isOperatorConfigReason(e.reason)
|
|
8840
|
+
}
|
|
8841
|
+
if (rotationGraced.length > 0) {
|
|
8842
|
+
log2(
|
|
8843
|
+
`[mcp-presence-reaper] '${codeName}': skipping [${rotationGraced.join(", ")}] within rotation grace window (${rotationGraceMs}ms) \u2014 child(ren) just SIGTERM'd by stale-mcp-reaper, awaiting respawn on next tool call (ENG-5344)`
|
|
8905
8844
|
);
|
|
8906
|
-
|
|
8907
|
-
|
|
8845
|
+
}
|
|
8846
|
+
if (rotationProbeWaiting.length > 0) {
|
|
8847
|
+
log2(
|
|
8848
|
+
`[mcp-presence-reaper] '${codeName}': skipping [${rotationProbeWaiting.join(", ")}] \u2014 prompt bind-probe already armed for the stalled rotation, waiting out the ${rotationPromptProbeIntervalMs}ms re-arm interval (ENG-8225)`
|
|
8908
8849
|
);
|
|
8909
|
-
|
|
8910
|
-
|
|
8850
|
+
}
|
|
8851
|
+
const declared = mcpJson.mcpServers ?? {};
|
|
8852
|
+
const liveKeys = /* @__PURE__ */ new Set();
|
|
8853
|
+
for (const key of Object.keys(declared)) {
|
|
8854
|
+
const entry = declared[key];
|
|
8855
|
+
if (typeof entry?.command !== "string") continue;
|
|
8856
|
+
if (!missingRaw.includes(key)) liveKeys.add(key);
|
|
8857
|
+
}
|
|
8858
|
+
for (const key of liveKeys) {
|
|
8859
|
+
const sk = stateKey(codeName, key);
|
|
8860
|
+
if (classifyRotation(codeName, key, rotationGraceMs, now) === "none") {
|
|
8861
|
+
rotationStallState.delete(sk);
|
|
8862
|
+
continue;
|
|
8863
|
+
}
|
|
8864
|
+
const stall = rotationStallState.get(sk);
|
|
8865
|
+
const viaProbe = (stall?.probeAttempts ?? 0) > 0;
|
|
8866
|
+
recordRotationOutcome(
|
|
8867
|
+
key,
|
|
8868
|
+
viaProbe ? "respawned-after-prompt-probe" : "respawned-on-tool-call"
|
|
8911
8869
|
);
|
|
8912
|
-
|
|
8913
|
-
|
|
8914
|
-
|
|
8915
|
-
|
|
8916
|
-
|
|
8917
|
-
|
|
8918
|
-
|
|
8919
|
-
|
|
8920
|
-
|
|
8870
|
+
log2(
|
|
8871
|
+
viaProbe ? `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-after-prompt-probe \u2014 child is live again after ${stall?.probeAttempts} prompt bind-probe(s)${stall?.escalated ? " (recovered AFTER escalation \u2014 the earlier never-respawned verdict stands for that window)" : ""}, no session restart needed (ENG-8225)` : `[mcp-presence-reaper] '${codeName}:${key}' rotation outcome=respawned-on-tool-call \u2014 child returned on the agent's own next tool call, ENG-5344's assumption held (ENG-8225)`
|
|
8872
|
+
);
|
|
8873
|
+
clearStaleRotation(codeName, [key]);
|
|
8874
|
+
rotationStallState.delete(sk);
|
|
8875
|
+
}
|
|
8876
|
+
for (const key of liveKeys) {
|
|
8877
|
+
const state = presenceReaperState.get(stateKey(codeName, key));
|
|
8878
|
+
if (state) {
|
|
8879
|
+
state.attempts = 0;
|
|
8880
|
+
state.lastSeenLiveAt = now();
|
|
8881
|
+
state.lastAttemptedSessionStartedAt = null;
|
|
8882
|
+
state.gaveUpLogged = false;
|
|
8883
|
+
state.firstMissingAt = null;
|
|
8884
|
+
state.quarantineLogged = false;
|
|
8885
|
+
state.continuouslyMissingSince = null;
|
|
8886
|
+
state.deadChannelLogged = false;
|
|
8887
|
+
state.quarantineWithheldLogged = false;
|
|
8888
|
+
state.gaveUp = false;
|
|
8889
|
+
if (state.stableLiveSince === null) state.stableLiveSince = now();
|
|
8890
|
+
if (now() - state.stableLiveSince >= BACKOFF_STABLE_RESET_MS) {
|
|
8891
|
+
state.restartHistory = [];
|
|
8892
|
+
}
|
|
8893
|
+
}
|
|
8894
|
+
}
|
|
8895
|
+
if (missing.length === 0) {
|
|
8896
|
+
if (missingRaw.length > 0) {
|
|
8897
|
+
return {
|
|
8898
|
+
missing,
|
|
8899
|
+
restarted: false,
|
|
8900
|
+
reason: "healthy",
|
|
8901
|
+
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
8902
|
+
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
8903
|
+
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
8904
|
+
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
8905
|
+
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
|
|
8906
|
+
};
|
|
8907
|
+
}
|
|
8908
|
+
blockedPromptEpisodes.delete(codeName);
|
|
8909
|
+
return {
|
|
8910
|
+
missing,
|
|
8911
|
+
restarted: false,
|
|
8912
|
+
reason: "healthy",
|
|
8913
|
+
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
8914
|
+
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
8915
|
+
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
8916
|
+
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
8917
|
+
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
|
|
8921
8918
|
};
|
|
8922
|
-
|
|
8923
|
-
|
|
8924
|
-
|
|
8925
|
-
|
|
8926
|
-
|
|
8927
|
-
|
|
8928
|
-
|
|
8929
|
-
|
|
8930
|
-
|
|
8931
|
-
|
|
8932
|
-
|
|
8933
|
-
const
|
|
8934
|
-
const
|
|
8935
|
-
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
|
|
8940
|
-
|
|
8941
|
-
|
|
8942
|
-
|
|
8943
|
-
|
|
8944
|
-
|
|
8945
|
-
|
|
8946
|
-
|
|
8947
|
-
|
|
8948
|
-
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
|
|
8952
|
-
|
|
8953
|
-
|
|
8954
|
-
|
|
8955
|
-
|
|
8956
|
-
|
|
8957
|
-
|
|
8958
|
-
|
|
8959
|
-
|
|
8960
|
-
|
|
8961
|
-
|
|
8962
|
-
|
|
8963
|
-
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
8967
|
-
|
|
8919
|
+
}
|
|
8920
|
+
const nowMs = now();
|
|
8921
|
+
const givenUp = [];
|
|
8922
|
+
const active = [];
|
|
8923
|
+
const backingOff = [];
|
|
8924
|
+
const quarantined = [];
|
|
8925
|
+
const wouldQuarantine = [];
|
|
8926
|
+
const quarantineWithheld = [];
|
|
8927
|
+
const anyStdioServerLive = liveKeys.size > 0;
|
|
8928
|
+
const genuinelyDead = [];
|
|
8929
|
+
for (const key of missing) {
|
|
8930
|
+
const sk = stateKey(codeName, key);
|
|
8931
|
+
const state = presenceReaperState.get(sk) ?? {
|
|
8932
|
+
attempts: 0,
|
|
8933
|
+
lastSeenLiveAt: null,
|
|
8934
|
+
lastAttemptedSessionStartedAt: null,
|
|
8935
|
+
historyOwedForSessionStartedAt: null,
|
|
8936
|
+
gaveUpLogged: false,
|
|
8937
|
+
firstMissingAt: null,
|
|
8938
|
+
quarantineLogged: false,
|
|
8939
|
+
restartHistory: [],
|
|
8940
|
+
stableLiveSince: null,
|
|
8941
|
+
continuouslyMissingSince: null,
|
|
8942
|
+
deadChannelLogged: false,
|
|
8943
|
+
quarantineWithheldLogged: false,
|
|
8944
|
+
gaveUp: false
|
|
8945
|
+
};
|
|
8946
|
+
if (state.continuouslyMissingSince === null) {
|
|
8947
|
+
state.continuouslyMissingSince = nowMs;
|
|
8948
|
+
}
|
|
8949
|
+
const isGenuinelyDead = nowMs - state.continuouslyMissingSince >= deadChannelMs;
|
|
8950
|
+
const suppressAsChurn = currentGenerationExternallyStarted && !isGenuinelyDead;
|
|
8951
|
+
if (isGenuinelyDead) {
|
|
8952
|
+
genuinelyDead.push(key);
|
|
8953
|
+
if (currentGenerationExternallyStarted && !state.deadChannelLogged) {
|
|
8954
|
+
state.deadChannelLogged = true;
|
|
8955
|
+
log2(
|
|
8956
|
+
`[mcp-presence-reaper] '${codeName}:${key}' continuously missing ${Math.round((nowMs - state.continuouslyMissingSince) / 6e4)}m (> ${Math.round(deadChannelMs / 6e4)}m) \u2014 genuinely dead, overriding storm-awareness so it can reach give-up/quarantine instead of churning (ENG-8062)`
|
|
8957
|
+
);
|
|
8958
|
+
}
|
|
8959
|
+
}
|
|
8960
|
+
if (state.firstMissingAt === null || suppressAsChurn) {
|
|
8961
|
+
state.firstMissingAt = nowMs;
|
|
8962
|
+
}
|
|
8963
|
+
state.stableLiveSince = null;
|
|
8964
|
+
if (state.lastAttemptedSessionStartedAt !== sessionStartedAt) {
|
|
8965
|
+
state.lastAttemptedSessionStartedAt = sessionStartedAt;
|
|
8966
|
+
if (!suppressAsChurn) {
|
|
8967
|
+
state.attempts += 1;
|
|
8968
|
+
state.historyOwedForSessionStartedAt = sessionStartedAt;
|
|
8969
|
+
}
|
|
8970
|
+
}
|
|
8971
|
+
state.restartHistory = state.restartHistory.filter((t) => nowMs - t < BACKOFF_WINDOW_MS);
|
|
8972
|
+
const restartsInWindow = state.restartHistory.length;
|
|
8973
|
+
presenceReaperState.set(sk, state);
|
|
8974
|
+
const overAttemptsCap = state.attempts > MAX_PRESENCE_RESTART_ATTEMPTS;
|
|
8975
|
+
const overRateCap = restartsInWindow >= BACKOFF_GIVE_UP_RESTARTS;
|
|
8976
|
+
if (overAttemptsCap || overRateCap) {
|
|
8977
|
+
givenUp.push(key);
|
|
8978
|
+
state.gaveUp = true;
|
|
8979
|
+
if (!state.gaveUpLogged) {
|
|
8980
|
+
const cause = overAttemptsCap ? `${MAX_PRESENCE_RESTART_ATTEMPTS} consecutive failed restarts \u2014 declared but never recovers (likely orphaned config; see ENG-5279)` : `${restartsInWindow} restarts within ${Math.round(BACKOFF_WINDOW_MS / 6e4)}m \u2014 persistently flapping (crash-loop; ENG-6480)`;
|
|
8981
|
+
log2(
|
|
8982
|
+
`[mcp-presence-reaper] giving up on '${codeName}:${key}' after ${cause}`
|
|
8983
|
+
);
|
|
8984
|
+
state.gaveUpLogged = true;
|
|
8985
|
+
if (onGiveUp) {
|
|
8986
|
+
try {
|
|
8987
|
+
onGiveUp(codeName, key);
|
|
8988
|
+
} catch (err) {
|
|
8989
|
+
log2(
|
|
8990
|
+
`[mcp-presence-reaper] onGiveUp callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
|
|
8991
|
+
);
|
|
8992
|
+
}
|
|
8993
|
+
}
|
|
8994
|
+
}
|
|
8995
|
+
if (quarantineMode !== "off" && !state.quarantineLogged && classifyKey(key) === "optional") {
|
|
8996
|
+
const dwellElapsed = quarantineDwellMs <= 0 || state.firstMissingAt !== null && nowMs - state.firstMissingAt >= quarantineDwellMs;
|
|
8997
|
+
if (dwellElapsed && !anyStdioServerLive) {
|
|
8998
|
+
quarantineWithheld.push(key);
|
|
8999
|
+
if (!state.quarantineWithheldLogged) {
|
|
9000
|
+
state.quarantineWithheldLogged = true;
|
|
9001
|
+
log2(
|
|
9002
|
+
`[mcp-presence-reaper] '${codeName}:${key}' given up but NOT quarantined - no declared stdio MCP server is live, so the restart budget was exhausted by a session that cannot spawn anything, not by this channel dying. Check the pane (tmux attach -t agt-${codeName}); the channel stays provisioned and will quarantine on a later tick if it is still missing once the session is up (ENG-10500)`
|
|
9003
|
+
);
|
|
9004
|
+
}
|
|
9005
|
+
} else if (dwellElapsed) {
|
|
9006
|
+
state.quarantineLogged = true;
|
|
9007
|
+
if (quarantineMode === "enforce") {
|
|
9008
|
+
quarantined.push(key);
|
|
9009
|
+
log2(
|
|
9010
|
+
`[mcp-presence-reaper] QUARANTINE '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; dropping from provisioned set so the dead channel stops restarting the whole session (ENG-5932)`
|
|
9011
|
+
);
|
|
9012
|
+
if (onQuarantine) {
|
|
9013
|
+
try {
|
|
9014
|
+
onQuarantine(codeName, key);
|
|
9015
|
+
} catch (err) {
|
|
9016
|
+
log2(
|
|
9017
|
+
`[mcp-presence-reaper] onQuarantine callback threw for '${codeName}:${key}' (suppressed; reaper continues): ${err.message}`
|
|
9018
|
+
);
|
|
9019
|
+
}
|
|
9020
|
+
}
|
|
9021
|
+
} else {
|
|
9022
|
+
wouldQuarantine.push(key);
|
|
9023
|
+
log2(
|
|
9024
|
+
`[mcp-presence-reaper] SHADOW would-quarantine '${codeName}:${key}' \u2014 optional channel dead past restart budget + ${quarantineDwellMs}ms dwell; no action taken (ENG-5932 shadow mode)`
|
|
9025
|
+
);
|
|
9026
|
+
}
|
|
9027
|
+
}
|
|
9028
|
+
}
|
|
9029
|
+
} else {
|
|
9030
|
+
const backoffMs = restartsInWindow < BACKOFF_AFTER_RESTARTS ? 0 : Math.min(
|
|
9031
|
+
BACKOFF_BASE_MS * 2 ** (restartsInWindow - BACKOFF_AFTER_RESTARTS),
|
|
9032
|
+
BACKOFF_MAX_MS
|
|
9033
|
+
);
|
|
9034
|
+
const lastRestartAt = state.restartHistory[state.restartHistory.length - 1] ?? null;
|
|
9035
|
+
if (lastRestartAt !== null && nowMs - lastRestartAt < backoffMs) {
|
|
9036
|
+
backingOff.push(key);
|
|
9037
|
+
} else {
|
|
9038
|
+
active.push(key);
|
|
9039
|
+
}
|
|
8968
9040
|
}
|
|
8969
|
-
|
|
8970
|
-
|
|
8971
|
-
|
|
8972
|
-
|
|
8973
|
-
|
|
8974
|
-
|
|
8975
|
-
|
|
8976
|
-
statusMessage,
|
|
8977
|
-
// ENG-7577: persist the tripping class so the recovery path can gate on
|
|
8978
|
-
// it after a manager restart, without re-deriving it from prose.
|
|
8979
|
-
trippedClass
|
|
8980
|
-
};
|
|
8981
|
-
this.trips.set(key, trip);
|
|
8982
|
-
this.events.delete(key);
|
|
8983
|
-
return {
|
|
8984
|
-
tripped: true,
|
|
8985
|
-
trip,
|
|
8986
|
-
windowCount,
|
|
8987
|
-
classCounts,
|
|
8988
|
-
trippedClass,
|
|
8989
|
-
maxProvisioningBucket: provBucket.count,
|
|
8990
|
-
maxProvisioningBucketKey: provBucket.key,
|
|
8991
|
-
...graceCeilingExceeded ? { newAgentGraceCeilingExceeded: true } : {},
|
|
8992
|
-
// ENG-9138: a co-occurring rollover loop still reports on a real trip.
|
|
8993
|
-
...scheduledLoop ? { scheduledLoop } : {},
|
|
8994
|
-
...operatorCommandLoop ? { operatorCommandLoop } : {}
|
|
8995
|
-
};
|
|
9041
|
+
}
|
|
9042
|
+
if (active.length === 0) {
|
|
9043
|
+
const reason = backingOff.length > 0 ? "backoff" : "all-keys-over-cap";
|
|
9044
|
+
if (backingOff.length > 0) {
|
|
9045
|
+
log2(
|
|
9046
|
+
`[mcp-presence-reaper] '${codeName}': holding off restart for [${backingOff.join(", ")}] \u2014 escalating backoff interval not yet elapsed (ENG-6480)${givenUp.length > 0 ? ` (given up: [${givenUp.join(", ")}])` : ""}`
|
|
9047
|
+
);
|
|
8996
9048
|
}
|
|
8997
|
-
const suppressedOperator = operatorBucket.count > this.operatorMax;
|
|
8998
|
-
const suppressedProvisioning = !suppressedOperator && provBucket.count > this.provisioningMax;
|
|
8999
|
-
const wouldHaveTripped = suppressedOperator || suppressedProvisioning;
|
|
9000
9049
|
return {
|
|
9001
|
-
|
|
9002
|
-
|
|
9003
|
-
|
|
9004
|
-
|
|
9005
|
-
|
|
9006
|
-
|
|
9007
|
-
|
|
9008
|
-
|
|
9009
|
-
|
|
9010
|
-
|
|
9011
|
-
|
|
9012
|
-
|
|
9013
|
-
|
|
9014
|
-
|
|
9015
|
-
bucketCount: suppressedOperator ? operatorBucket.count : provBucket.count,
|
|
9016
|
-
agentAgeMs: age ?? 0
|
|
9017
|
-
}
|
|
9018
|
-
} : {}
|
|
9019
|
-
} : {}
|
|
9050
|
+
missing,
|
|
9051
|
+
restarted: false,
|
|
9052
|
+
reason,
|
|
9053
|
+
givenUp,
|
|
9054
|
+
backingOff: backingOff.length > 0 ? backingOff : void 0,
|
|
9055
|
+
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
9056
|
+
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
9057
|
+
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
9058
|
+
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
9059
|
+
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
|
|
9060
|
+
quarantined: quarantined.length > 0 ? quarantined : void 0,
|
|
9061
|
+
wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
|
|
9062
|
+
genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
|
|
9063
|
+
quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
|
|
9020
9064
|
};
|
|
9021
9065
|
}
|
|
9022
|
-
|
|
9023
|
-
|
|
9024
|
-
|
|
9025
|
-
|
|
9026
|
-
|
|
9027
|
-
|
|
9028
|
-
|
|
9029
|
-
|
|
9066
|
+
const readRestartBudget = () => {
|
|
9067
|
+
try {
|
|
9068
|
+
return args.restartBudgetRemaining?.(codeName) ?? null;
|
|
9069
|
+
} catch (err) {
|
|
9070
|
+
log2(
|
|
9071
|
+
`[mcp-presence-reaper] restartBudgetRemaining threw for '${codeName}' (suppressed; treated as "cannot say"): ${err.message}`
|
|
9072
|
+
);
|
|
9073
|
+
return null;
|
|
9030
9074
|
}
|
|
9031
|
-
}
|
|
9032
|
-
|
|
9033
|
-
|
|
9034
|
-
|
|
9035
|
-
|
|
9036
|
-
|
|
9037
|
-
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
|
|
9041
|
-
|
|
9042
|
-
|
|
9043
|
-
|
|
9044
|
-
|
|
9045
|
-
|
|
9046
|
-
|
|
9047
|
-
|
|
9048
|
-
|
|
9049
|
-
|
|
9075
|
+
};
|
|
9076
|
+
let blockedRestartPending = null;
|
|
9077
|
+
if (readPane) {
|
|
9078
|
+
let pane = null;
|
|
9079
|
+
try {
|
|
9080
|
+
pane = readPane(codeName);
|
|
9081
|
+
} catch (err) {
|
|
9082
|
+
log2(
|
|
9083
|
+
`[mcp-presence-reaper] '${codeName}': pane read threw (${err.message}) \u2014 proceeding with restart (ENG-9932 fails open)`
|
|
9084
|
+
);
|
|
9085
|
+
}
|
|
9086
|
+
const holdBlockedOnPrompt = (blocked2, opts = {}) => {
|
|
9087
|
+
const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
|
|
9088
|
+
if (onPaneBlocked) {
|
|
9089
|
+
try {
|
|
9090
|
+
onPaneBlocked(codeName, {
|
|
9091
|
+
kind: blocked2,
|
|
9092
|
+
activeKeys: [...active],
|
|
9093
|
+
paneHash: fingerprint?.hash ?? null,
|
|
9094
|
+
paneLength: fingerprint?.length ?? null,
|
|
9095
|
+
// ENG-10878: WHY the hold, not just that there was one. An
|
|
9096
|
+
// operator paged by this alert needs to know whether the one
|
|
9097
|
+
// restart was spent and failed, or was never affordable - those
|
|
9098
|
+
// want different next actions, and without this the two are
|
|
9099
|
+
// indistinguishable on the alert board.
|
|
9100
|
+
restartBudgetRemaining: readRestartBudget(),
|
|
9101
|
+
restartSpentThisEpisode: blockedPromptEpisodes.get(codeName)?.restarted ?? false,
|
|
9102
|
+
restartDeferred: opts.restartDeferred ?? false
|
|
9103
|
+
});
|
|
9104
|
+
} catch (err) {
|
|
9105
|
+
log2(
|
|
9106
|
+
`[mcp-presence-reaper] onPaneBlocked callback threw for '${codeName}' (suppressed): ${err.message}`
|
|
9107
|
+
);
|
|
9108
|
+
}
|
|
9109
|
+
}
|
|
9110
|
+
return {
|
|
9111
|
+
missing,
|
|
9112
|
+
restarted: false,
|
|
9113
|
+
reason: "pane-blocked-on-dialog",
|
|
9114
|
+
paneBlockedOn: blocked2,
|
|
9115
|
+
givenUp: givenUp.length > 0 ? givenUp : void 0,
|
|
9116
|
+
backingOff: backingOff.length > 0 ? backingOff : void 0,
|
|
9117
|
+
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
9118
|
+
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
9119
|
+
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
9120
|
+
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
9121
|
+
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
|
|
9122
|
+
quarantined: quarantined.length > 0 ? quarantined : void 0,
|
|
9123
|
+
wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
|
|
9124
|
+
genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
|
|
9125
|
+
quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
|
|
9126
|
+
};
|
|
9127
|
+
};
|
|
9128
|
+
const blocked = classifyUnanswerablePane(pane);
|
|
9129
|
+
const openBlockedEpisode = blockedPromptEpisodes.get(codeName);
|
|
9130
|
+
if (pane !== null && !blocked && liveKeys.size > 0 && openBlockedEpisode) {
|
|
9131
|
+
blockedPromptEpisodes.delete(codeName);
|
|
9132
|
+
}
|
|
9133
|
+
if (blocked) {
|
|
9134
|
+
const fingerprint = pane ? summarizeUnanswerablePane(pane) : null;
|
|
9135
|
+
const budgetLeft = readRestartBudget();
|
|
9136
|
+
const episode = blockedPromptEpisodes.get(codeName);
|
|
9137
|
+
const restartWouldCount = reaperRestartBreakerReason(active) !== void 0;
|
|
9138
|
+
if (!episode && budgetLeft === 0 && restartWouldCount) {
|
|
9139
|
+
log2(
|
|
9140
|
+
`[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 blocked on an interactive prompt (${blocked}) and the crash breaker has NO budget left in its window, so the one restart ENG-10878 would spend would be the event that TRIPS it - remaining===0 means crashCount===max, and the trip test fires at max+1, attributed to mcp-presence-reaper itself. Needs a human at the pane: tmux attach -t agt-${codeName}` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
|
|
9141
|
+
);
|
|
9142
|
+
return holdBlockedOnPrompt(blocked);
|
|
9143
|
+
}
|
|
9144
|
+
if (!episode) {
|
|
9145
|
+
blockedRestartPending = {
|
|
9146
|
+
blocked,
|
|
9147
|
+
paneHash: fingerprint?.hash ?? null,
|
|
9148
|
+
hold: (opts) => holdBlockedOnPrompt(blocked, opts)
|
|
9149
|
+
};
|
|
9150
|
+
log2(
|
|
9151
|
+
`[mcp-presence-reaper] '${codeName}': blocked on an interactive prompt (${blocked}) \u2014 ATTEMPTING this episode's ONE restart (ENG-10878). A restart cured this on 2026-09-22 and did not on 2026-09-03, so the next poll decides which it is` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
|
|
9050
9152
|
);
|
|
9153
|
+
} else {
|
|
9154
|
+
const comparable = fingerprint !== null && episode.paneHashAtRestart !== null;
|
|
9155
|
+
const modalDelta = !comparable ? "modal fingerprint unavailable on one or both polls, so same-or-different is UNKNOWN" : fingerprint.hash === episode.paneHashAtRestart ? "SAME modal as before the restart" : "a DIFFERENT modal than before the restart";
|
|
9156
|
+
log2(
|
|
9157
|
+
`[mcp-presence-reaper] '${codeName}': NOT restarting \u2014 still blocked on an interactive prompt (${blocked}) after this episode's one restart, so the restart did not clear it (${modalDelta}). Needs a human at the pane: tmux attach -t agt-${codeName}` + (fingerprint ? ` (modal fp=${fingerprint.hash} len=${fingerprint.length})` : "")
|
|
9158
|
+
);
|
|
9159
|
+
return holdBlockedOnPrompt(blocked);
|
|
9051
9160
|
}
|
|
9052
9161
|
}
|
|
9053
9162
|
}
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9163
|
+
const givenUpSuffix = givenUp.length > 0 ? ` (skipping over-cap: [${givenUp.join(", ")}])` : "";
|
|
9164
|
+
log2(
|
|
9165
|
+
`[mcp-presence-reaper] '${codeName}': declared MCP(s) [${active.join(", ")}] have no live children \u2014 restarting session${givenUpSuffix}`
|
|
9166
|
+
);
|
|
9167
|
+
const tornDown = stopSession(codeName, { activeKeys: active });
|
|
9168
|
+
if (onRestart && tornDown !== false) {
|
|
9169
|
+
try {
|
|
9170
|
+
const maybePromise = onRestart(codeName, { activeKeys: active, givenUpKeys: givenUp });
|
|
9171
|
+
if (maybePromise && typeof maybePromise.then === "function") {
|
|
9172
|
+
void maybePromise.then(void 0, (err) => {
|
|
9173
|
+
log2(
|
|
9174
|
+
`[mcp-presence-reaper] onRestart callback rejected for '${codeName}' (suppressed; restart proceeds): ${err.message}`
|
|
9175
|
+
);
|
|
9176
|
+
});
|
|
9177
|
+
}
|
|
9178
|
+
} catch (err) {
|
|
9179
|
+
log2(
|
|
9180
|
+
`[mcp-presence-reaper] onRestart callback threw for '${codeName}' (suppressed; restart proceeds): ${err.message}`
|
|
9181
|
+
);
|
|
9182
|
+
}
|
|
9058
9183
|
}
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
9064
|
-
|
|
9065
|
-
|
|
9066
|
-
|
|
9067
|
-
|
|
9068
|
-
|
|
9184
|
+
if (blockedRestartPending) {
|
|
9185
|
+
if (tornDown === false) {
|
|
9186
|
+
log2(
|
|
9187
|
+
`[mcp-presence-reaper] '${codeName}': restart DEFERRED by the host gate this poll (maintenance window / not idle), so nothing was torn down and this episode's one restart is still unspent \u2014 it will be re-attempted when the gate opens. Needs a human at the pane if it persists: tmux attach -t agt-${codeName}`
|
|
9188
|
+
);
|
|
9189
|
+
return blockedRestartPending.hold({ restartDeferred: true });
|
|
9190
|
+
}
|
|
9191
|
+
blockedPromptEpisodes.set(codeName, {
|
|
9192
|
+
restarted: true,
|
|
9193
|
+
paneHashAtRestart: blockedRestartPending.paneHash
|
|
9194
|
+
});
|
|
9195
|
+
log2(
|
|
9196
|
+
`[mcp-presence-reaper] '${codeName}': this episode's ONE restart is now SPENT (teardown confirmed) \u2014 the next poll decides whether it cleared the modal.`
|
|
9197
|
+
);
|
|
9198
|
+
}
|
|
9199
|
+
if (tornDown === false) {
|
|
9200
|
+
log2(
|
|
9201
|
+
`[mcp-presence-reaper] '${codeName}': restart DEFERRED by the host gate (maintenance window / not idle) \u2014 nothing was torn down, so this poll records no restart and the next poll re-detects and re-tries (ENG-10878)`
|
|
9202
|
+
);
|
|
9203
|
+
return {
|
|
9204
|
+
missing,
|
|
9205
|
+
restarted: false,
|
|
9206
|
+
reason: "restart-deferred",
|
|
9207
|
+
givenUp: givenUp.length > 0 ? givenUp : void 0,
|
|
9208
|
+
backingOff: backingOff.length > 0 ? backingOff : void 0,
|
|
9209
|
+
quarantined: quarantined.length > 0 ? quarantined : void 0,
|
|
9210
|
+
wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
|
|
9211
|
+
genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
|
|
9212
|
+
quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0,
|
|
9213
|
+
// ROTATION STATE SURVIVES A DEFERRAL (CodeRabbit on #6399). This return
|
|
9214
|
+
// omitted all five, and a deferred poll is exactly a poll where rotation
|
|
9215
|
+
// work may already have happened: `recordRotationOutcome` runs during the
|
|
9216
|
+
// scan, well before the host gate declines the teardown. So a rotation
|
|
9217
|
+
// that escalated to `never-respawned` on this poll was recorded in the
|
|
9218
|
+
// map, dropped from the result, and unrecoverable by the caller -
|
|
9219
|
+
// probe-armed keys are held OUT of `missing` precisely so they are not
|
|
9220
|
+
// torn down, which also means a caller cannot rebuild them from it.
|
|
9221
|
+
//
|
|
9222
|
+
// `ReapMissingMcpSessionsResult` documents these as present whenever an
|
|
9223
|
+
// entry exists, so omitting them here made the result LIE rather than
|
|
9224
|
+
// merely under-report.
|
|
9225
|
+
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
9226
|
+
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
9227
|
+
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
9228
|
+
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
9229
|
+
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0
|
|
9230
|
+
};
|
|
9231
|
+
}
|
|
9232
|
+
for (const key of active) {
|
|
9233
|
+
const st = presenceReaperState.get(stateKey(codeName, key));
|
|
9234
|
+
if (!st) continue;
|
|
9235
|
+
if (st.historyOwedForSessionStartedAt !== sessionStartedAt) continue;
|
|
9236
|
+
st.restartHistory.push(nowMs);
|
|
9237
|
+
st.historyOwedForSessionStartedAt = null;
|
|
9238
|
+
}
|
|
9239
|
+
return {
|
|
9240
|
+
missing,
|
|
9241
|
+
restarted: true,
|
|
9242
|
+
givenUp: givenUp.length > 0 ? givenUp : void 0,
|
|
9243
|
+
backingOff: backingOff.length > 0 ? backingOff : void 0,
|
|
9244
|
+
rotationGraced: rotationGraced.length > 0 ? rotationGraced : void 0,
|
|
9245
|
+
rotationProbeArmed: rotationProbeArmed.length > 0 ? rotationProbeArmed : void 0,
|
|
9246
|
+
rotationProbeWaiting: rotationProbeWaiting.length > 0 ? rotationProbeWaiting : void 0,
|
|
9247
|
+
rotationProbeExhausted: rotationProbeExhausted.length > 0 ? rotationProbeExhausted : void 0,
|
|
9248
|
+
rotationRespawnOutcomes: Object.keys(rotationRespawnOutcomes).length > 0 ? rotationRespawnOutcomes : void 0,
|
|
9249
|
+
quarantined: quarantined.length > 0 ? quarantined : void 0,
|
|
9250
|
+
wouldQuarantine: wouldQuarantine.length > 0 ? wouldQuarantine : void 0,
|
|
9251
|
+
genuinelyDead: genuinelyDead.length > 0 ? genuinelyDead : void 0,
|
|
9252
|
+
quarantineWithheld: quarantineWithheld.length > 0 ? quarantineWithheld : void 0
|
|
9253
|
+
};
|
|
9069
9254
|
}
|
|
9070
9255
|
|
|
9071
9256
|
// src/lib/mcp-config-lookup.ts
|
|
@@ -11725,6 +11910,12 @@ export {
|
|
|
11725
11910
|
resolveFlagFromLayers,
|
|
11726
11911
|
resolveAllFlags,
|
|
11727
11912
|
HostFlagStore,
|
|
11913
|
+
reaperRestartBreakerReason,
|
|
11914
|
+
isProvisioningReloadReason,
|
|
11915
|
+
isIntegrationBucketedReason,
|
|
11916
|
+
tripClass,
|
|
11917
|
+
readEnvNumber,
|
|
11918
|
+
RestartBreaker,
|
|
11728
11919
|
diffEnvIntegrations,
|
|
11729
11920
|
classifyEnvIntegrationsDiff,
|
|
11730
11921
|
findMcpServersUsingVars,
|
|
@@ -11736,13 +11927,8 @@ export {
|
|
|
11736
11927
|
clearPresenceReaperState,
|
|
11737
11928
|
clearPresenceReaperStateForKeys,
|
|
11738
11929
|
givenUpMcpServerKeys,
|
|
11930
|
+
describeBlockedPromptHold,
|
|
11739
11931
|
reapMissingMcpSessions,
|
|
11740
|
-
reaperRestartBreakerReason,
|
|
11741
|
-
isProvisioningReloadReason,
|
|
11742
|
-
isIntegrationBucketedReason,
|
|
11743
|
-
tripClass,
|
|
11744
|
-
readEnvNumber,
|
|
11745
|
-
RestartBreaker,
|
|
11746
11932
|
markQuarantinedForProbe,
|
|
11747
11933
|
FIRST_CONNECT_WINDOW_MS,
|
|
11748
11934
|
isWithinReconnectQuiesceWindow,
|
|
@@ -11775,4 +11961,4 @@ export {
|
|
|
11775
11961
|
managerInstallSystemUnitCommand,
|
|
11776
11962
|
managerUninstallSystemUnitCommand
|
|
11777
11963
|
};
|
|
11778
|
-
//# sourceMappingURL=chunk-
|
|
11964
|
+
//# sourceMappingURL=chunk-HIDB6U3D.js.map
|