@integrity-labs/agt-cli 0.28.341 → 0.28.342
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -46,7 +46,7 @@ import {
|
|
|
46
46
|
resolveEffectivePinRaw,
|
|
47
47
|
safeWriteJsonAtomic,
|
|
48
48
|
setConfigHash
|
|
49
|
-
} from "../chunk-
|
|
49
|
+
} from "../chunk-V2FVNWF2.js";
|
|
50
50
|
import {
|
|
51
51
|
getProjectDir as getProjectDir2,
|
|
52
52
|
getReadyTasks,
|
|
@@ -6354,11 +6354,15 @@ function resolveSlackIngestParamsFromEnv(env, ctx) {
|
|
|
6354
6354
|
const appToken = env["SLACK_APP_TOKEN"];
|
|
6355
6355
|
const botToken = env["SLACK_BOT_TOKEN"];
|
|
6356
6356
|
if (!appToken || !botToken) return null;
|
|
6357
|
+
const ackReaction = (env["SLACK_ACK_REACTION"] ?? "").trim() || void 0;
|
|
6358
|
+
const skipReaction = (env["SLACK_SKIP_REACTION"] ?? "").trim() || void 0;
|
|
6357
6359
|
return {
|
|
6358
6360
|
appToken,
|
|
6359
6361
|
botToken,
|
|
6360
6362
|
senderPolicy: buildSlackSenderPolicyFromEnv(env, ctx),
|
|
6361
|
-
peerConfig: buildSlackPeerClassifierConfigFromEnv(env)
|
|
6363
|
+
peerConfig: buildSlackPeerClassifierConfigFromEnv(env),
|
|
6364
|
+
...ackReaction ? { ackReaction } : {},
|
|
6365
|
+
...skipReaction ? { skipReaction } : {}
|
|
6362
6366
|
};
|
|
6363
6367
|
}
|
|
6364
6368
|
var BoundedSet = class {
|
|
@@ -6396,6 +6400,12 @@ async function dispatchSlackInbound(event, ctx) {
|
|
|
6396
6400
|
ctx.log(`[slack-ingest:${ctx.codeName}] dropped ${redact(event.channel)} reason=${decision.reason}`);
|
|
6397
6401
|
return "dropped_gate";
|
|
6398
6402
|
}
|
|
6403
|
+
let ackReactionInFlight = Promise.resolve();
|
|
6404
|
+
if (ctx.ackReaction && event.channel && event.ts) {
|
|
6405
|
+
ackReactionInFlight = ctx.addReaction({ channel: event.channel, timestamp: event.ts, name: ctx.ackReaction });
|
|
6406
|
+
ackReactionInFlight.catch(() => {
|
|
6407
|
+
});
|
|
6408
|
+
}
|
|
6399
6409
|
const conversationKey = `${event.channel ?? "nochannel"}:${event.thread_ts ?? event.ts ?? ""}`;
|
|
6400
6410
|
const meta = {
|
|
6401
6411
|
source: "slack",
|
|
@@ -6423,6 +6433,23 @@ async function dispatchSlackInbound(event, ctx) {
|
|
|
6423
6433
|
});
|
|
6424
6434
|
return "admitted_replied";
|
|
6425
6435
|
}
|
|
6436
|
+
if (ctx.skipReaction && event.channel && event.ts) {
|
|
6437
|
+
const channel = event.channel;
|
|
6438
|
+
const timestamp = event.ts;
|
|
6439
|
+
const { ackReaction, skipReaction } = ctx;
|
|
6440
|
+
if (ackReaction !== skipReaction) {
|
|
6441
|
+
void (async () => {
|
|
6442
|
+
try {
|
|
6443
|
+
if (ackReaction) {
|
|
6444
|
+
await ackReactionInFlight;
|
|
6445
|
+
await ctx.removeReaction({ channel, timestamp, name: ackReaction });
|
|
6446
|
+
}
|
|
6447
|
+
await ctx.addReaction({ channel, timestamp, name: skipReaction });
|
|
6448
|
+
} catch {
|
|
6449
|
+
}
|
|
6450
|
+
})();
|
|
6451
|
+
}
|
|
6452
|
+
}
|
|
6426
6453
|
return "admitted_no_reply";
|
|
6427
6454
|
}
|
|
6428
6455
|
function redact(id) {
|
|
@@ -6448,6 +6475,46 @@ function defaultSendReply(botToken, fetchImpl, log2) {
|
|
|
6448
6475
|
}
|
|
6449
6476
|
};
|
|
6450
6477
|
}
|
|
6478
|
+
function defaultAddReaction(botToken, fetchImpl, log2) {
|
|
6479
|
+
return async ({ channel, timestamp, name }) => {
|
|
6480
|
+
try {
|
|
6481
|
+
const res = await fetchImpl(`${SLACK_API}/reactions.add`, {
|
|
6482
|
+
method: "POST",
|
|
6483
|
+
headers: {
|
|
6484
|
+
Authorization: `Bearer ${botToken}`,
|
|
6485
|
+
"Content-Type": "application/json; charset=utf-8"
|
|
6486
|
+
},
|
|
6487
|
+
body: JSON.stringify({ channel, timestamp, name })
|
|
6488
|
+
});
|
|
6489
|
+
const data = await res.json();
|
|
6490
|
+
if (!data.ok && data.error !== "already_reacted") {
|
|
6491
|
+
log2(`[slack-ingest] reactions.add failed: ${data.error}`);
|
|
6492
|
+
}
|
|
6493
|
+
} catch (err) {
|
|
6494
|
+
log2(`[slack-ingest] reactions.add threw: ${err instanceof Error ? err.message : String(err)}`);
|
|
6495
|
+
}
|
|
6496
|
+
};
|
|
6497
|
+
}
|
|
6498
|
+
function defaultRemoveReaction(botToken, fetchImpl, log2) {
|
|
6499
|
+
return async ({ channel, timestamp, name }) => {
|
|
6500
|
+
try {
|
|
6501
|
+
const res = await fetchImpl(`${SLACK_API}/reactions.remove`, {
|
|
6502
|
+
method: "POST",
|
|
6503
|
+
headers: {
|
|
6504
|
+
Authorization: `Bearer ${botToken}`,
|
|
6505
|
+
"Content-Type": "application/json; charset=utf-8"
|
|
6506
|
+
},
|
|
6507
|
+
body: JSON.stringify({ channel, timestamp, name })
|
|
6508
|
+
});
|
|
6509
|
+
const data = await res.json();
|
|
6510
|
+
if (!data.ok && data.error !== "no_reaction") {
|
|
6511
|
+
log2(`[slack-ingest] reactions.remove failed: ${data.error}`);
|
|
6512
|
+
}
|
|
6513
|
+
} catch (err) {
|
|
6514
|
+
log2(`[slack-ingest] reactions.remove threw: ${err instanceof Error ? err.message : String(err)}`);
|
|
6515
|
+
}
|
|
6516
|
+
};
|
|
6517
|
+
}
|
|
6451
6518
|
function startSlackIngest(config2) {
|
|
6452
6519
|
const fetchImpl = config2.fetchImpl ?? globalThis.fetch;
|
|
6453
6520
|
const socketFactory = config2.socketFactory ?? ((url) => {
|
|
@@ -6456,6 +6523,8 @@ function startSlackIngest(config2) {
|
|
|
6456
6523
|
return new Ctor(url);
|
|
6457
6524
|
});
|
|
6458
6525
|
const sendReply = config2.sendReply ?? defaultSendReply(config2.botToken, fetchImpl, config2.log);
|
|
6526
|
+
const addReaction = config2.addReaction ?? defaultAddReaction(config2.botToken, fetchImpl, config2.log);
|
|
6527
|
+
const removeReaction = config2.removeReaction ?? defaultRemoveReaction(config2.botToken, fetchImpl, config2.log);
|
|
6459
6528
|
const seen = new BoundedSet();
|
|
6460
6529
|
let stopped = false;
|
|
6461
6530
|
let ws = null;
|
|
@@ -6466,6 +6535,10 @@ function startSlackIngest(config2) {
|
|
|
6466
6535
|
peerConfig: config2.peerConfig,
|
|
6467
6536
|
inject: config2.inject,
|
|
6468
6537
|
sendReply,
|
|
6538
|
+
addReaction,
|
|
6539
|
+
removeReaction,
|
|
6540
|
+
...config2.ackReaction ? { ackReaction: config2.ackReaction } : {},
|
|
6541
|
+
...config2.skipReaction ? { skipReaction: config2.skipReaction } : {},
|
|
6469
6542
|
log: config2.log,
|
|
6470
6543
|
seen
|
|
6471
6544
|
};
|
|
@@ -6564,7 +6637,9 @@ function fingerprint(params) {
|
|
|
6564
6637
|
appToken: params.appToken,
|
|
6565
6638
|
botToken: params.botToken,
|
|
6566
6639
|
senderPolicy: params.senderPolicy,
|
|
6567
|
-
peerConfig: params.peerConfig
|
|
6640
|
+
peerConfig: params.peerConfig,
|
|
6641
|
+
ackReaction: params.ackReaction ?? null,
|
|
6642
|
+
skipReaction: params.skipReaction ?? null
|
|
6568
6643
|
})
|
|
6569
6644
|
).digest("hex");
|
|
6570
6645
|
}
|
|
@@ -6618,6 +6693,8 @@ async function ensureOpencodeSlackIngest(opts) {
|
|
|
6618
6693
|
peerConfig: params.peerConfig,
|
|
6619
6694
|
inject: (msg) => injectOpencodeMessage(codeName, msg, { awaitReply: true }),
|
|
6620
6695
|
log: log2,
|
|
6696
|
+
...params.ackReaction ? { ackReaction: params.ackReaction } : {},
|
|
6697
|
+
...params.skipReaction ? { skipReaction: params.skipReaction } : {},
|
|
6621
6698
|
...opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {},
|
|
6622
6699
|
...opts.socketFactory ? { socketFactory: opts.socketFactory } : {}
|
|
6623
6700
|
});
|
|
@@ -8391,7 +8468,7 @@ var agentRestartTimezoneInputs = /* @__PURE__ */ new Map();
|
|
|
8391
8468
|
var lastVersionCheckAt = 0;
|
|
8392
8469
|
var VERSION_CHECK_INTERVAL_MS = 5 * 60 * 1e3;
|
|
8393
8470
|
var lastResponsivenessProbeAt = 0;
|
|
8394
|
-
var agtCliVersion = true ? "0.28.
|
|
8471
|
+
var agtCliVersion = true ? "0.28.342" : "dev";
|
|
8395
8472
|
function resolveBrewPath(execFileSync2) {
|
|
8396
8473
|
try {
|
|
8397
8474
|
const out = execFileSync2("which", ["brew"], { timeout: 5e3 }).toString().trim();
|
|
@@ -10094,6 +10171,15 @@ async function pollCycle() {
|
|
|
10094
10171
|
pollDirectChatMessages(agentStates).catch((err) => {
|
|
10095
10172
|
log(`Direct chat poll error: ${err.message}`);
|
|
10096
10173
|
});
|
|
10174
|
+
} else {
|
|
10175
|
+
const opencodeAgents = agentStates.filter(
|
|
10176
|
+
(a) => (agentFrameworkCache.get(a.codeName) ?? DEFAULT_FRAMEWORK) === "opencode"
|
|
10177
|
+
);
|
|
10178
|
+
if (opencodeAgents.length > 0) {
|
|
10179
|
+
pollDirectChatMessages(opencodeAgents).catch((err) => {
|
|
10180
|
+
log(`Direct chat poll error (opencode): ${err.message}`);
|
|
10181
|
+
});
|
|
10182
|
+
}
|
|
10097
10183
|
}
|
|
10098
10184
|
ensureRealtimeStarted(agentStates);
|
|
10099
10185
|
ensureRealtimeDriftStarted(agentStates);
|
|
@@ -11701,7 +11787,7 @@ async function processAgent(agent, agentStates) {
|
|
|
11701
11787
|
}
|
|
11702
11788
|
}
|
|
11703
11789
|
}
|
|
11704
|
-
if (agentFw === "claude-code" && sessionMode === "persistent") {
|
|
11790
|
+
if (agentFw === "claude-code" && sessionMode === "persistent" || agentFw === "opencode") {
|
|
11705
11791
|
const wasSessionHealthy = isSessionHealthy(agent.code_name);
|
|
11706
11792
|
let psResult;
|
|
11707
11793
|
try {
|
|
@@ -11733,167 +11819,169 @@ async function processAgent(agent, agentStates) {
|
|
|
11733
11819
|
} else if (stuck.recovered) {
|
|
11734
11820
|
log(`[persistent-session-stuck] agent=${agent.code_name} recovered from stuck state`);
|
|
11735
11821
|
}
|
|
11736
|
-
|
|
11737
|
-
checkMcpConfigDriftAndScheduleRestart(agent.code_name, getProjectDir(agent.code_name));
|
|
11738
|
-
} catch (err) {
|
|
11739
|
-
log(`[hot-reload] .mcp.json drift check failed for '${agent.code_name}': ${err.message}`);
|
|
11740
|
-
}
|
|
11741
|
-
if (channelConfigConverged) {
|
|
11822
|
+
if (agentFw === "claude-code") {
|
|
11742
11823
|
try {
|
|
11743
|
-
|
|
11824
|
+
checkMcpConfigDriftAndScheduleRestart(agent.code_name, getProjectDir(agent.code_name));
|
|
11744
11825
|
} catch (err) {
|
|
11745
|
-
log(`[hot-reload]
|
|
11826
|
+
log(`[hot-reload] .mcp.json drift check failed for '${agent.code_name}': ${err.message}`);
|
|
11746
11827
|
}
|
|
11747
|
-
|
|
11748
|
-
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
11752
|
-
|
|
11753
|
-
mcpJsonParsed = JSON.parse(readFileSync18(mcpPath, "utf-8"));
|
|
11754
|
-
} catch {
|
|
11828
|
+
if (channelConfigConverged) {
|
|
11829
|
+
try {
|
|
11830
|
+
checkChannelSecretDriftAndScheduleRestart(agent.code_name, getProjectDir(agent.code_name));
|
|
11831
|
+
} catch (err) {
|
|
11832
|
+
log(`[hot-reload] channel-secret drift check failed for '${agent.code_name}': ${err.message}`);
|
|
11833
|
+
}
|
|
11755
11834
|
}
|
|
11756
|
-
|
|
11757
|
-
|
|
11758
|
-
|
|
11759
|
-
|
|
11760
|
-
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
|
|
11771
|
-
|
|
11772
|
-
|
|
11773
|
-
|
|
11774
|
-
|
|
11775
|
-
|
|
11776
|
-
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
|
|
11781
|
-
|
|
11782
|
-
|
|
11783
|
-
|
|
11784
|
-
|
|
11785
|
-
|
|
11786
|
-
|
|
11787
|
-
|
|
11788
|
-
|
|
11789
|
-
|
|
11790
|
-
|
|
11791
|
-
|
|
11792
|
-
|
|
11793
|
-
|
|
11794
|
-
|
|
11795
|
-
|
|
11796
|
-
|
|
11797
|
-
|
|
11798
|
-
// toolkits poll drops it from the response → next writeMcpServer
|
|
11799
|
-
// omits it → broken MCP is cleanly gone instead of declared-but-
|
|
11800
|
-
// dead.
|
|
11801
|
-
//
|
|
11802
|
-
// Async fire-and-forget — the reaper's correctness must not
|
|
11803
|
-
// depend on the API call succeeding. Errors are caught inside
|
|
11804
|
-
// the .then/.catch chain and logged.
|
|
11805
|
-
onGiveUp: (codeName, serverKey) => {
|
|
11806
|
-
const mapping = managedToolkitIdByAgentAndServerId.get(
|
|
11807
|
-
managedToolkitMapKey(agent.agent_id, serverKey)
|
|
11808
|
-
);
|
|
11809
|
-
if (!mapping) {
|
|
11810
|
-
log(
|
|
11811
|
-
`[mcp-presence-reaper] give-up on '${codeName}:${serverKey}' \u2014 not a managed toolkit (no toolkit_id mapping), skipping integration-health API call`
|
|
11812
|
-
);
|
|
11813
|
-
return;
|
|
11814
|
-
}
|
|
11815
|
-
const { toolkitId, connectionKey } = mapping;
|
|
11816
|
-
const agentId = agent.agent_id;
|
|
11817
|
-
api.post("/host/integration-health", {
|
|
11818
|
-
agent_id: agentId,
|
|
11819
|
-
toolkit_id: toolkitId,
|
|
11820
|
-
// ENG-7543: scope the mark-unhealthy to the specific connection whose
|
|
11821
|
-
// MCP server gave up, so a dead 2nd mailbox doesn't disable the first.
|
|
11822
|
-
connection_key: connectionKey,
|
|
11823
|
-
action: "mark_unhealthy",
|
|
11824
|
-
reason: `mcp-presence-reaper cap (${3} consecutive failed restart cycles) tripped on '${serverKey}'`
|
|
11825
|
-
}).then((res) => {
|
|
11826
|
-
log(
|
|
11827
|
-
`[mcp-presence-reaper] marked integration unhealthy for '${codeName}' toolkit '${toolkitId}' (ENG-5292): ${JSON.stringify(res).slice(0, 200)}`
|
|
11828
|
-
);
|
|
11829
|
-
}).catch((err) => {
|
|
11830
|
-
log(
|
|
11831
|
-
`[mcp-presence-reaper] failed to mark integration unhealthy for '${codeName}' toolkit '${toolkitId}' (ENG-5292): ${err.message} \u2014 local give-up state still recorded; will retry on next agent provisioning cycle if .mcp.json changes`
|
|
11832
|
-
);
|
|
11833
|
-
});
|
|
11834
|
-
},
|
|
11835
|
-
// ENG-5286: record the restart event with the API so the
|
|
11836
|
-
// agent-heartbeat-monitor cron can aggregate per-agent counts
|
|
11837
|
-
// into the McpReaperRestartsHourly CloudWatch metric. Fire-and-
|
|
11838
|
-
// forget — the reaper's correctness must not depend on this.
|
|
11839
|
-
onRestart: (codeName, { activeKeys, givenUpKeys }) => {
|
|
11840
|
-
api.post("/host/mcp-reaper-restart", {
|
|
11841
|
-
agent_id: agent.agent_id,
|
|
11842
|
-
code_name: codeName,
|
|
11843
|
-
server_keys: activeKeys,
|
|
11844
|
-
given_up_keys: givenUpKeys
|
|
11845
|
-
}).catch((err) => {
|
|
11846
|
-
log(
|
|
11847
|
-
`[mcp-presence-reaper] failed to record restart event for '${codeName}' (ENG-5286): ${err.message} \u2014 local restart still proceeded; CloudWatch metric will under-count this event`
|
|
11848
|
-
);
|
|
11849
|
-
});
|
|
11850
|
-
void reportReaperEvent(
|
|
11851
|
-
agent.agent_id,
|
|
11852
|
-
"mcp-presence-reaper",
|
|
11853
|
-
"restart",
|
|
11854
|
-
givenUpKeys.length > 0 ? `restarted session; MCP server(s) still down after restart budget: ${givenUpKeys.join(", ")}` : `restarted session to recover MCP server(s): ${activeKeys.join(", ")}`
|
|
11855
|
-
);
|
|
11856
|
-
},
|
|
11857
|
-
// ENG-5932: classify each declared key for the quarantine path. Reads a
|
|
11858
|
-
// server-declared criticality field off the .mcp.json entry when
|
|
11859
|
-
// present (Slice 2 will stamp it); otherwise fails closed to ESSENTIAL
|
|
11860
|
-
// for everything except the known optional channel set. Tool servers
|
|
11861
|
-
// (composio_*/xero) classify ESSENTIAL here and stay on the existing
|
|
11862
|
-
// managed-toolkit onGiveUp path above.
|
|
11863
|
-
classifyKey: (serverKey) => classifyChannelCriticality(
|
|
11864
|
-
serverKey,
|
|
11865
|
-
mcpJsonParsed?.mcpServers?.[serverKey]
|
|
11866
|
-
),
|
|
11867
|
-
quarantineDwellMs: channelQuarantineDwellMs(),
|
|
11868
|
-
quarantineMode: channelQuarantineMode(),
|
|
11869
|
-
// ENG-5932: enforce-mode quarantine. Persist the local marker; the next
|
|
11870
|
-
// provisioning poll then omits the channel from .mcp.json + the launch
|
|
11871
|
-
// flags, and the channel-set-change / .mcp.json-drift restart paths are
|
|
11872
|
-
// taught to NOT bounce the session for a quarantine-driven removal.
|
|
11873
|
-
// Local-authoritative: no API call — with the control plane down, the
|
|
11874
|
-
// flap still stops.
|
|
11875
|
-
onQuarantine: (codeName, serverKey) => {
|
|
11876
|
-
const newly = channelQuarantineStore().quarantine(
|
|
11835
|
+
try {
|
|
11836
|
+
const sess = getSessionState(agent.code_name);
|
|
11837
|
+
let mcpJsonParsed = null;
|
|
11838
|
+
try {
|
|
11839
|
+
const mcpPath = join21(getProjectDir(agent.code_name), ".mcp.json");
|
|
11840
|
+
mcpJsonParsed = JSON.parse(readFileSync18(mcpPath, "utf-8"));
|
|
11841
|
+
} catch {
|
|
11842
|
+
}
|
|
11843
|
+
reapMissingMcpSessions({
|
|
11844
|
+
log,
|
|
11845
|
+
codeName: agent.code_name,
|
|
11846
|
+
mcpJson: mcpJsonParsed,
|
|
11847
|
+
sessionStartedAt: sess?.startedAt ?? null,
|
|
11848
|
+
// ENG-6660: under Docker the MCP children live in the agent's container
|
|
11849
|
+
// PID namespace - probe inside it (docker exec ps) instead of the host,
|
|
11850
|
+
// or every declared MCP looks "missing" -> restart loop -> flapping.
|
|
11851
|
+
isolated: isolationMode(agent.code_name) === "docker",
|
|
11852
|
+
// ENG-6486: a restart is already scheduled/deferred for this agent (e.g.
|
|
11853
|
+
// a freshly-added channel awaiting its convergence restart, deferred by
|
|
11854
|
+
// the maintenance window). The pending restart will spawn the declared-
|
|
11855
|
+
// but-not-yet-live MCP, so don't let the reaper re-fire + inflate the
|
|
11856
|
+
// McpReaperRestartsHourly metric into a false flapping alarm.
|
|
11857
|
+
restartPending: pendingSessionRestarts.has(agent.code_name),
|
|
11858
|
+
// ENG-5441 / ENG-5547: a mcp-presence-reaper restart counts against
|
|
11859
|
+
// the agent-wide breaker ONLY when several distinct MCP servers are
|
|
11860
|
+
// missing at once — a thrashing collection that rotates restarts of
|
|
11861
|
+
// the whole session, which the per-MCP cap can't see. A restart
|
|
11862
|
+
// driven by a SINGLE persistently-failing server is left to that
|
|
11863
|
+
// server's own ENG-5279 give-up cap (→ marked unhealthy via onGiveUp
|
|
11864
|
+
// → dropped from the next .mcp.json), so one bad integration is
|
|
11865
|
+
// isolated and the agent keeps running degraded instead of being
|
|
11866
|
+
// paused wholesale. reaperRestartBreakerReason() encodes that
|
|
11867
|
+
// single-vs-multi decision; undefined means "restart, don't count".
|
|
11868
|
+
//
|
|
11869
|
+
// ENG-6264: the breaker-count reason (above, undefined for a single dead
|
|
11870
|
+
// MCP) is decoupled from the GATE reason. Pre-6264 an undefined breaker
|
|
11871
|
+
// reason also made the restart non-gateable → 'bypass' → the session was
|
|
11872
|
+
// torn down mid-turn (the common single-MCP case interrupted busy
|
|
11873
|
+
// agents). Always pass 'mcp-presence-reaper' as the gate reason so the
|
|
11874
|
+
// restart defers-until-idle, while breakerReason still governs whether it
|
|
11875
|
+
// counts against the breaker.
|
|
11876
|
+
stopSession: (codeName, ctx) => stopPersistentSessionAndForgetMcpBaseline(
|
|
11877
11877
|
codeName,
|
|
11878
|
-
|
|
11879
|
-
|
|
11880
|
-
|
|
11881
|
-
|
|
11882
|
-
|
|
11883
|
-
|
|
11884
|
-
|
|
11878
|
+
reaperRestartBreakerReason(ctx.activeKeys),
|
|
11879
|
+
"mcp-presence-reaper"
|
|
11880
|
+
),
|
|
11881
|
+
// ENG-5292: when the reaper gives up on a managed MCP (cap from
|
|
11882
|
+
// ENG-5279 + state-preservation from ENG-5285 both said "this
|
|
11883
|
+
// MCP keeps failing after 3 restart cycles"), mark the matching
|
|
11884
|
+
// integration unhealthy server-side. The next /host/managed-
|
|
11885
|
+
// toolkits poll drops it from the response → next writeMcpServer
|
|
11886
|
+
// omits it → broken MCP is cleanly gone instead of declared-but-
|
|
11887
|
+
// dead.
|
|
11888
|
+
//
|
|
11889
|
+
// Async fire-and-forget — the reaper's correctness must not
|
|
11890
|
+
// depend on the API call succeeding. Errors are caught inside
|
|
11891
|
+
// the .then/.catch chain and logged.
|
|
11892
|
+
onGiveUp: (codeName, serverKey) => {
|
|
11893
|
+
const mapping = managedToolkitIdByAgentAndServerId.get(
|
|
11894
|
+
managedToolkitMapKey(agent.agent_id, serverKey)
|
|
11885
11895
|
);
|
|
11896
|
+
if (!mapping) {
|
|
11897
|
+
log(
|
|
11898
|
+
`[mcp-presence-reaper] give-up on '${codeName}:${serverKey}' \u2014 not a managed toolkit (no toolkit_id mapping), skipping integration-health API call`
|
|
11899
|
+
);
|
|
11900
|
+
return;
|
|
11901
|
+
}
|
|
11902
|
+
const { toolkitId, connectionKey } = mapping;
|
|
11903
|
+
const agentId = agent.agent_id;
|
|
11904
|
+
api.post("/host/integration-health", {
|
|
11905
|
+
agent_id: agentId,
|
|
11906
|
+
toolkit_id: toolkitId,
|
|
11907
|
+
// ENG-7543: scope the mark-unhealthy to the specific connection whose
|
|
11908
|
+
// MCP server gave up, so a dead 2nd mailbox doesn't disable the first.
|
|
11909
|
+
connection_key: connectionKey,
|
|
11910
|
+
action: "mark_unhealthy",
|
|
11911
|
+
reason: `mcp-presence-reaper cap (${3} consecutive failed restart cycles) tripped on '${serverKey}'`
|
|
11912
|
+
}).then((res) => {
|
|
11913
|
+
log(
|
|
11914
|
+
`[mcp-presence-reaper] marked integration unhealthy for '${codeName}' toolkit '${toolkitId}' (ENG-5292): ${JSON.stringify(res).slice(0, 200)}`
|
|
11915
|
+
);
|
|
11916
|
+
}).catch((err) => {
|
|
11917
|
+
log(
|
|
11918
|
+
`[mcp-presence-reaper] failed to mark integration unhealthy for '${codeName}' toolkit '${toolkitId}' (ENG-5292): ${err.message} \u2014 local give-up state still recorded; will retry on next agent provisioning cycle if .mcp.json changes`
|
|
11919
|
+
);
|
|
11920
|
+
});
|
|
11921
|
+
},
|
|
11922
|
+
// ENG-5286: record the restart event with the API so the
|
|
11923
|
+
// agent-heartbeat-monitor cron can aggregate per-agent counts
|
|
11924
|
+
// into the McpReaperRestartsHourly CloudWatch metric. Fire-and-
|
|
11925
|
+
// forget — the reaper's correctness must not depend on this.
|
|
11926
|
+
onRestart: (codeName, { activeKeys, givenUpKeys }) => {
|
|
11927
|
+
api.post("/host/mcp-reaper-restart", {
|
|
11928
|
+
agent_id: agent.agent_id,
|
|
11929
|
+
code_name: codeName,
|
|
11930
|
+
server_keys: activeKeys,
|
|
11931
|
+
given_up_keys: givenUpKeys
|
|
11932
|
+
}).catch((err) => {
|
|
11933
|
+
log(
|
|
11934
|
+
`[mcp-presence-reaper] failed to record restart event for '${codeName}' (ENG-5286): ${err.message} \u2014 local restart still proceeded; CloudWatch metric will under-count this event`
|
|
11935
|
+
);
|
|
11936
|
+
});
|
|
11886
11937
|
void reportReaperEvent(
|
|
11887
11938
|
agent.agent_id,
|
|
11888
11939
|
"mcp-presence-reaper",
|
|
11889
|
-
"
|
|
11890
|
-
`
|
|
11940
|
+
"restart",
|
|
11941
|
+
givenUpKeys.length > 0 ? `restarted session; MCP server(s) still down after restart budget: ${givenUpKeys.join(", ")}` : `restarted session to recover MCP server(s): ${activeKeys.join(", ")}`
|
|
11942
|
+
);
|
|
11943
|
+
},
|
|
11944
|
+
// ENG-5932: classify each declared key for the quarantine path. Reads a
|
|
11945
|
+
// server-declared criticality field off the .mcp.json entry when
|
|
11946
|
+
// present (Slice 2 will stamp it); otherwise fails closed to ESSENTIAL
|
|
11947
|
+
// for everything except the known optional channel set. Tool servers
|
|
11948
|
+
// (composio_*/xero) classify ESSENTIAL here and stay on the existing
|
|
11949
|
+
// managed-toolkit onGiveUp path above.
|
|
11950
|
+
classifyKey: (serverKey) => classifyChannelCriticality(
|
|
11951
|
+
serverKey,
|
|
11952
|
+
mcpJsonParsed?.mcpServers?.[serverKey]
|
|
11953
|
+
),
|
|
11954
|
+
quarantineDwellMs: channelQuarantineDwellMs(),
|
|
11955
|
+
quarantineMode: channelQuarantineMode(),
|
|
11956
|
+
// ENG-5932: enforce-mode quarantine. Persist the local marker; the next
|
|
11957
|
+
// provisioning poll then omits the channel from .mcp.json + the launch
|
|
11958
|
+
// flags, and the channel-set-change / .mcp.json-drift restart paths are
|
|
11959
|
+
// taught to NOT bounce the session for a quarantine-driven removal.
|
|
11960
|
+
// Local-authoritative: no API call — with the control plane down, the
|
|
11961
|
+
// flap still stops.
|
|
11962
|
+
onQuarantine: (codeName, serverKey) => {
|
|
11963
|
+
const newly = channelQuarantineStore().quarantine(
|
|
11964
|
+
codeName,
|
|
11965
|
+
serverKey,
|
|
11966
|
+
`mcp-presence-reaper: optional channel dead past restart budget + dwell`,
|
|
11967
|
+
Date.now()
|
|
11891
11968
|
);
|
|
11969
|
+
if (newly) {
|
|
11970
|
+
log(
|
|
11971
|
+
`[channel-quarantine] persisted quarantine marker for '${codeName}:${serverKey}' \u2014 will be dropped from .mcp.json on next provisioning poll (ENG-5932)`
|
|
11972
|
+
);
|
|
11973
|
+
void reportReaperEvent(
|
|
11974
|
+
agent.agent_id,
|
|
11975
|
+
"mcp-presence-reaper",
|
|
11976
|
+
"quarantine",
|
|
11977
|
+
`optional channel '${serverKey}' dead past restart budget + dwell \u2014 quarantined`
|
|
11978
|
+
);
|
|
11979
|
+
}
|
|
11892
11980
|
}
|
|
11893
|
-
}
|
|
11894
|
-
})
|
|
11895
|
-
|
|
11896
|
-
|
|
11981
|
+
});
|
|
11982
|
+
} catch (err) {
|
|
11983
|
+
log(`[mcp-presence-reaper] presence check failed for '${agent.code_name}': ${err.message}`);
|
|
11984
|
+
}
|
|
11897
11985
|
}
|
|
11898
11986
|
} else if (agentFw === "claude-code" && tasks.length > 0) {
|
|
11899
11987
|
await syncAndCheckClaudeScheduler(agent, tasks, boardItems, refreshData);
|
|
@@ -12161,7 +12249,7 @@ async function ensureOpencodeRuntime(agent, refreshData, agentTimezone) {
|
|
|
12161
12249
|
return { decision: "healthy", spawnAttempted: false, sessionHealthyAfter: true };
|
|
12162
12250
|
}
|
|
12163
12251
|
stopOpencodeSlackIngest(codeName, log);
|
|
12164
|
-
const opencodeProjectDir = join21(getFramework("opencode").getAgentDir(codeName), "
|
|
12252
|
+
const opencodeProjectDir = join21(getFramework("opencode").getAgentDir(codeName), "provision");
|
|
12165
12253
|
const openRouterRaw = refreshData.openrouter ?? null;
|
|
12166
12254
|
const serveEnv = {
|
|
12167
12255
|
AGT_HOST: requireHost(),
|