@integrity-labs/agt-cli 0.27.9-test.13 → 0.27.9-test.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/agt.js +4 -4
- package/dist/chunk-354FAVQR.js +173 -0
- package/dist/chunk-354FAVQR.js.map +1 -0
- package/dist/{chunk-MQJ5DMPT.js → chunk-7GKJZBTB.js} +111 -204
- package/dist/chunk-7GKJZBTB.js.map +1 -0
- package/dist/{chunk-HR5T2RQF.js → chunk-I3YS5WFV.js} +3 -1
- package/dist/chunk-I3YS5WFV.js.map +1 -0
- package/dist/{chunk-VNIHFGA2.js → chunk-R7NKCGWN.js} +341 -46
- package/dist/chunk-R7NKCGWN.js.map +1 -0
- package/dist/{chunk-I2OTQJH3.js → chunk-WOOYOAPG.js} +2644 -2142
- package/dist/chunk-WOOYOAPG.js.map +1 -0
- package/dist/{claude-pair-runtime-F6USL3EW.js → claude-pair-runtime-GIUCD7IG.js} +2 -2
- package/dist/{claude-scheduler-EM24LTGV.js → claude-scheduler-FATCLHDM.js} +2 -2
- package/dist/daily-session-PNQX5URX.js +27 -0
- package/dist/lib/manager-worker.js +1125 -149
- package/dist/lib/manager-worker.js.map +1 -1
- package/dist/mcp/augmented-admin.js +21335 -0
- package/dist/mcp/index.js +121 -4
- package/dist/mcp/slack-channel.js +845 -208
- package/dist/mcp/teams-channel.js +73 -29
- package/dist/mcp/telegram-channel.js +579 -157
- package/dist/{persistent-session-OL5EDYB4.js → persistent-session-35PWSTLO.js} +10 -3
- package/dist/persistent-session-35PWSTLO.js.map +1 -0
- package/dist/responsiveness-probe-MA4M2QM4.js +158 -0
- package/dist/responsiveness-probe-MA4M2QM4.js.map +1 -0
- package/package.json +3 -2
- package/dist/chunk-HR5T2RQF.js.map +0 -1
- package/dist/chunk-I2OTQJH3.js.map +0 -1
- package/dist/chunk-MQJ5DMPT.js.map +0 -1
- package/dist/chunk-VNIHFGA2.js.map +0 -1
- package/dist/responsiveness-probe-B6LJJRUD.js +0 -74
- package/dist/responsiveness-probe-B6LJJRUD.js.map +0 -1
- /package/dist/{claude-pair-runtime-F6USL3EW.js.map → claude-pair-runtime-GIUCD7IG.js.map} +0 -0
- /package/dist/{claude-scheduler-EM24LTGV.js.map → claude-scheduler-FATCLHDM.js.map} +0 -0
- /package/dist/{persistent-session-OL5EDYB4.js.map → daily-session-PNQX5URX.js.map} +0 -0
|
@@ -12,52 +12,52 @@ import {
|
|
|
12
12
|
getHostId,
|
|
13
13
|
getIntegration,
|
|
14
14
|
provision,
|
|
15
|
+
provisionAutoKanbanProgressHook,
|
|
15
16
|
provisionIsolationHook,
|
|
16
17
|
provisionOrientHook,
|
|
17
18
|
provisionStopHook,
|
|
18
19
|
requireHost,
|
|
19
20
|
safeWriteJsonAtomic
|
|
20
|
-
} from "../chunk-
|
|
21
|
+
} from "../chunk-R7NKCGWN.js";
|
|
21
22
|
import {
|
|
22
23
|
getProjectDir as getProjectDir2,
|
|
23
24
|
getReadyTasks,
|
|
24
25
|
loadSchedulerState,
|
|
25
26
|
markTaskFired,
|
|
26
27
|
syncTasksToScheduler
|
|
27
|
-
} from "../chunk-
|
|
28
|
+
} from "../chunk-I3YS5WFV.js";
|
|
28
29
|
import {
|
|
29
30
|
buildAllowedTools,
|
|
30
31
|
checkChannelInputs,
|
|
31
32
|
creditWatchdogGiveUpCount,
|
|
33
|
+
expandTemplateVars,
|
|
32
34
|
formatMissingVar,
|
|
33
35
|
getLastFailureContext,
|
|
34
36
|
getProjectDir,
|
|
35
37
|
getSessionState,
|
|
36
38
|
injectMessage,
|
|
37
39
|
injectMessageWithStatus,
|
|
38
|
-
isAgentIdle,
|
|
39
40
|
isAgentPromptReady,
|
|
40
41
|
isSessionHealthy,
|
|
41
|
-
isStaleForToday,
|
|
42
42
|
paneLogPath,
|
|
43
43
|
parseEnvIntegrations,
|
|
44
|
-
peekCurrentSession,
|
|
45
44
|
prepareForRespawn,
|
|
46
45
|
probeMcpEnvSubstitution,
|
|
47
46
|
readPaneLogTail,
|
|
48
47
|
resetRestartCount,
|
|
49
48
|
resolveClaudeBinary,
|
|
49
|
+
rotateSessionForWedge,
|
|
50
50
|
sanitizeMcpJson,
|
|
51
51
|
sendToAgent,
|
|
52
|
-
sessionTranscriptDir,
|
|
53
52
|
startPersistentSession,
|
|
54
53
|
stopAllSessionsAndWait,
|
|
55
54
|
stopPersistentSession,
|
|
56
55
|
takeWatchdogGiveUpCount,
|
|
57
56
|
takeZombieDetection
|
|
58
|
-
} from "../chunk-
|
|
57
|
+
} from "../chunk-7GKJZBTB.js";
|
|
59
58
|
import {
|
|
60
59
|
KANBAN_CHECK_COMMAND,
|
|
60
|
+
MAX_AVATAR_ENV_URL_BYTES,
|
|
61
61
|
SUPPRESS_SENTINEL,
|
|
62
62
|
appendDmFooter,
|
|
63
63
|
attributeTranscriptUsageByRun,
|
|
@@ -74,12 +74,22 @@ import {
|
|
|
74
74
|
parseDeliveryTarget,
|
|
75
75
|
parseTranscriptUsage,
|
|
76
76
|
parseUsageBanner,
|
|
77
|
+
probeComposioAccount,
|
|
78
|
+
probeComposioMcpToolCall,
|
|
77
79
|
probeHttpProvider,
|
|
80
|
+
resolveAvatarEnvUrl,
|
|
78
81
|
resolveChannels,
|
|
79
82
|
resolveConnectivityProbe,
|
|
80
83
|
resolveDmTarget,
|
|
84
|
+
worseConnectivityOutcome,
|
|
81
85
|
wrapScheduledTaskPrompt
|
|
82
|
-
} from "../chunk-
|
|
86
|
+
} from "../chunk-WOOYOAPG.js";
|
|
87
|
+
import {
|
|
88
|
+
isAgentIdle,
|
|
89
|
+
isStaleForToday,
|
|
90
|
+
peekCurrentSession,
|
|
91
|
+
sessionTranscriptDir
|
|
92
|
+
} from "../chunk-354FAVQR.js";
|
|
83
93
|
import {
|
|
84
94
|
parsePsRows,
|
|
85
95
|
reapOrphanChannelMcps
|
|
@@ -123,6 +133,25 @@ function channelSecretValueHash(envEntries, channelSecretKeys) {
|
|
|
123
133
|
return createHash("sha256").update(basis).digest("hex").slice(0, 16);
|
|
124
134
|
}
|
|
125
135
|
|
|
136
|
+
// src/lib/mcp-restart-verify.ts
|
|
137
|
+
var RESTART_VERIFY_DEADLINE_MS = 9e4;
|
|
138
|
+
var RESTART_VERIFY_MAX_ATTEMPTS = 3;
|
|
139
|
+
function decidePostRestartVerification(pending, sessionStartedAt, sessionHealthy, sessionRespawnId, now, opts = {}) {
|
|
140
|
+
const deadlineMs = opts.deadlineMs ?? RESTART_VERIFY_DEADLINE_MS;
|
|
141
|
+
const maxAttempts = opts.maxAttempts ?? RESTART_VERIFY_MAX_ATTEMPTS;
|
|
142
|
+
if (sessionStartedAt !== null && sessionStartedAt > pending.firedAt && sessionHealthy && sessionRespawnId !== null) {
|
|
143
|
+
return { kind: "verified" };
|
|
144
|
+
}
|
|
145
|
+
if (now - pending.firedAt < deadlineMs) {
|
|
146
|
+
return { kind: "waiting" };
|
|
147
|
+
}
|
|
148
|
+
const attempt = pending.attempts + 1;
|
|
149
|
+
return { kind: "unverified", attempt, final: attempt >= maxAttempts };
|
|
150
|
+
}
|
|
151
|
+
function shouldRemediateUnverifiedRestart(outcome, ctx) {
|
|
152
|
+
return ctx.enabled && !ctx.breakerTripped && outcome.kind === "unverified" && outcome.final;
|
|
153
|
+
}
|
|
154
|
+
|
|
126
155
|
// src/lib/integration-hash.ts
|
|
127
156
|
import { createHash as createHash2 } from "crypto";
|
|
128
157
|
function canonicalize(value) {
|
|
@@ -482,14 +511,14 @@ function reapMissingMcpSessions(args) {
|
|
|
482
511
|
if (!missingRaw.includes(key)) liveKeys.add(key);
|
|
483
512
|
}
|
|
484
513
|
for (const key of liveKeys) {
|
|
485
|
-
const
|
|
486
|
-
if (
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
514
|
+
const state7 = presenceReaperState.get(stateKey(codeName, key));
|
|
515
|
+
if (state7) {
|
|
516
|
+
state7.attempts = 0;
|
|
517
|
+
state7.lastSeenLiveAt = now();
|
|
518
|
+
state7.lastAttemptedSessionStartedAt = null;
|
|
519
|
+
state7.gaveUpLogged = false;
|
|
520
|
+
state7.firstMissingAt = null;
|
|
521
|
+
state7.quarantineLogged = false;
|
|
493
522
|
}
|
|
494
523
|
}
|
|
495
524
|
if (missing.length === 0) {
|
|
@@ -507,7 +536,7 @@ function reapMissingMcpSessions(args) {
|
|
|
507
536
|
const wouldQuarantine = [];
|
|
508
537
|
for (const key of missing) {
|
|
509
538
|
const sk = stateKey(codeName, key);
|
|
510
|
-
const
|
|
539
|
+
const state7 = presenceReaperState.get(sk) ?? {
|
|
511
540
|
attempts: 0,
|
|
512
541
|
lastSeenLiveAt: null,
|
|
513
542
|
lastAttemptedSessionStartedAt: null,
|
|
@@ -515,19 +544,19 @@ function reapMissingMcpSessions(args) {
|
|
|
515
544
|
firstMissingAt: null,
|
|
516
545
|
quarantineLogged: false
|
|
517
546
|
};
|
|
518
|
-
if (
|
|
519
|
-
if (
|
|
520
|
-
|
|
521
|
-
|
|
547
|
+
if (state7.firstMissingAt === null) state7.firstMissingAt = nowMs;
|
|
548
|
+
if (state7.lastAttemptedSessionStartedAt !== sessionStartedAt) {
|
|
549
|
+
state7.attempts += 1;
|
|
550
|
+
state7.lastAttemptedSessionStartedAt = sessionStartedAt;
|
|
522
551
|
}
|
|
523
|
-
presenceReaperState.set(sk,
|
|
524
|
-
if (
|
|
552
|
+
presenceReaperState.set(sk, state7);
|
|
553
|
+
if (state7.attempts > MAX_PRESENCE_RESTART_ATTEMPTS) {
|
|
525
554
|
givenUp.push(key);
|
|
526
|
-
if (!
|
|
555
|
+
if (!state7.gaveUpLogged) {
|
|
527
556
|
log2(
|
|
528
557
|
`[mcp-presence-reaper] giving up on '${codeName}:${key}' after ${MAX_PRESENCE_RESTART_ATTEMPTS} consecutive failed restarts \u2014 declared but never recovers (likely orphaned config; see ENG-5279)`
|
|
529
558
|
);
|
|
530
|
-
|
|
559
|
+
state7.gaveUpLogged = true;
|
|
531
560
|
if (onGiveUp) {
|
|
532
561
|
try {
|
|
533
562
|
onGiveUp(codeName, key);
|
|
@@ -538,10 +567,10 @@ function reapMissingMcpSessions(args) {
|
|
|
538
567
|
}
|
|
539
568
|
}
|
|
540
569
|
}
|
|
541
|
-
if (quarantineMode !== "off" && !
|
|
542
|
-
const dwellElapsed = quarantineDwellMs <= 0 ||
|
|
570
|
+
if (quarantineMode !== "off" && !state7.quarantineLogged && classifyKey(key) === "optional") {
|
|
571
|
+
const dwellElapsed = quarantineDwellMs <= 0 || state7.firstMissingAt !== null && nowMs - state7.firstMissingAt >= quarantineDwellMs;
|
|
543
572
|
if (dwellElapsed) {
|
|
544
|
-
|
|
573
|
+
state7.quarantineLogged = true;
|
|
545
574
|
if (quarantineMode === "enforce") {
|
|
546
575
|
quarantined.push(key);
|
|
547
576
|
log2(
|
|
@@ -706,12 +735,15 @@ function extractMsTeamsBehaviourSubset(config2) {
|
|
|
706
735
|
// src/lib/slack-behaviour-restart.ts
|
|
707
736
|
function extractSlackBehaviourSubset(config2) {
|
|
708
737
|
const rawAllowed = config2?.["allowed_users"];
|
|
738
|
+
const blockKitEnabled = config2?.["block_kit_enabled"] === true;
|
|
709
739
|
return {
|
|
710
740
|
thread_auto_follow: config2?.["thread_auto_follow"] ?? "off",
|
|
711
741
|
channel_response_mode: config2?.["channel_response_mode"] ?? "mention_only",
|
|
712
742
|
// Mirror the adapter's defensive filter exactly — a malformed entry the
|
|
713
743
|
// adapter would drop must not count as an env change.
|
|
714
|
-
allowed_users: Array.isArray(rawAllowed) ? rawAllowed.filter((v) => typeof v === "string" && v.trim().length > 0).map((v) => v.trim()) : []
|
|
744
|
+
allowed_users: Array.isArray(rawAllowed) ? rawAllowed.filter((v) => typeof v === "string" && v.trim().length > 0).map((v) => v.trim()) : [],
|
|
745
|
+
block_kit_enabled: blockKitEnabled,
|
|
746
|
+
block_kit_ask_user_enabled: blockKitEnabled && config2?.["block_kit_ask_user_enabled"] === true
|
|
715
747
|
};
|
|
716
748
|
}
|
|
717
749
|
|
|
@@ -965,6 +997,135 @@ function formatStatusMessage(events, windowMs) {
|
|
|
965
997
|
return `Circuit breaker tripped: ${events.length} restarts in ${windowLabel} (${breakdown}); most recent=${last.reason} at ${new Date(last.at).toISOString()}`;
|
|
966
998
|
}
|
|
967
999
|
|
|
1000
|
+
// src/lib/mcp-flap-dampener.ts
|
|
1001
|
+
var DEFAULT_WINDOW_MS2 = 6e5;
|
|
1002
|
+
var DEFAULT_MIN_TRANSITIONS = 4;
|
|
1003
|
+
var DEFAULT_MIN_DISTINCT = 2;
|
|
1004
|
+
function readEnvNumber2(name, fallback) {
|
|
1005
|
+
const raw = process.env[name];
|
|
1006
|
+
if (!raw) return fallback;
|
|
1007
|
+
const parsed = Number(raw);
|
|
1008
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
1009
|
+
}
|
|
1010
|
+
function membershipSignature(keys) {
|
|
1011
|
+
return [...new Set(keys)].sort().join(",");
|
|
1012
|
+
}
|
|
1013
|
+
var McpFlapDampener = class {
|
|
1014
|
+
windowMs;
|
|
1015
|
+
minTransitions;
|
|
1016
|
+
minDistinct;
|
|
1017
|
+
now;
|
|
1018
|
+
history = /* @__PURE__ */ new Map();
|
|
1019
|
+
flapping = /* @__PURE__ */ new Set();
|
|
1020
|
+
constructor(opts = {}) {
|
|
1021
|
+
this.windowMs = opts.windowMs ?? readEnvNumber2("AGT_MCP_FLAP_WINDOW_MS", DEFAULT_WINDOW_MS2);
|
|
1022
|
+
this.minTransitions = opts.minTransitions ?? readEnvNumber2("AGT_MCP_FLAP_MIN_TRANSITIONS", DEFAULT_MIN_TRANSITIONS);
|
|
1023
|
+
this.minDistinct = opts.minDistinct ?? readEnvNumber2("AGT_MCP_FLAP_MIN_DISTINCT", DEFAULT_MIN_DISTINCT);
|
|
1024
|
+
this.now = opts.now ?? Date.now;
|
|
1025
|
+
if (!Number.isFinite(this.windowMs) || this.windowMs < 1e3) {
|
|
1026
|
+
throw new Error("mcp-flap-dampener windowMs must be a finite number >= 1000");
|
|
1027
|
+
}
|
|
1028
|
+
if (!Number.isFinite(this.minTransitions) || this.minTransitions < 2) {
|
|
1029
|
+
throw new Error("mcp-flap-dampener minTransitions must be a finite number >= 2");
|
|
1030
|
+
}
|
|
1031
|
+
if (!Number.isFinite(this.minDistinct) || this.minDistinct < 2) {
|
|
1032
|
+
throw new Error("mcp-flap-dampener minDistinct must be a finite number >= 2");
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
key(codeName, channel) {
|
|
1036
|
+
return `${codeName}\0${channel}`;
|
|
1037
|
+
}
|
|
1038
|
+
/**
|
|
1039
|
+
* Record an observed membership signature for (codeName, channel) and return
|
|
1040
|
+
* the current flap decision. Call this on every poll where the manager has
|
|
1041
|
+
* computed the desired set — including unchanged polls — so a set that has
|
|
1042
|
+
* gone quiet ages out of the window and the channel recovers.
|
|
1043
|
+
*/
|
|
1044
|
+
record(codeName, channel, signature) {
|
|
1045
|
+
const k = this.key(codeName, channel);
|
|
1046
|
+
const at = this.now();
|
|
1047
|
+
const cutoff = at - this.windowMs;
|
|
1048
|
+
const prior = (this.history.get(k) ?? []).filter((o) => o.at >= cutoff);
|
|
1049
|
+
prior.push({ sig: signature, at });
|
|
1050
|
+
this.history.set(k, prior);
|
|
1051
|
+
let transitions = 0;
|
|
1052
|
+
for (let i = 1; i < prior.length; i++) {
|
|
1053
|
+
if (prior[i].sig !== prior[i - 1].sig) transitions++;
|
|
1054
|
+
}
|
|
1055
|
+
const distinctStates = new Set(prior.map((o) => o.sig)).size;
|
|
1056
|
+
const revisited = prior.slice(0, -1).some((o, i) => o.sig === signature && i !== prior.length - 2);
|
|
1057
|
+
const transitionNow = prior.length >= 2 && prior[prior.length - 1].sig !== prior[prior.length - 2].sig;
|
|
1058
|
+
const oscillating = transitions >= this.minTransitions && distinctStates >= this.minDistinct && revisited;
|
|
1059
|
+
const wasFlapping = this.flapping.has(k);
|
|
1060
|
+
let onset = false;
|
|
1061
|
+
if (wasFlapping) {
|
|
1062
|
+
if (!transitionNow) this.flapping.delete(k);
|
|
1063
|
+
} else if (transitionNow && oscillating) {
|
|
1064
|
+
this.flapping.add(k);
|
|
1065
|
+
onset = true;
|
|
1066
|
+
}
|
|
1067
|
+
return {
|
|
1068
|
+
flapping: this.flapping.has(k),
|
|
1069
|
+
distinctStates,
|
|
1070
|
+
transitions,
|
|
1071
|
+
windowCount: prior.length,
|
|
1072
|
+
onset
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
/** True if (codeName, channel) is currently dampened. */
|
|
1076
|
+
isFlapping(codeName, channel) {
|
|
1077
|
+
return this.flapping.has(this.key(codeName, channel));
|
|
1078
|
+
}
|
|
1079
|
+
/** Drop all state for an agent (called on deprovision / agent removal). */
|
|
1080
|
+
forget(codeName) {
|
|
1081
|
+
const prefix = `${codeName}\0`;
|
|
1082
|
+
for (const k of [...this.history.keys()]) {
|
|
1083
|
+
if (k.startsWith(prefix)) this.history.delete(k);
|
|
1084
|
+
}
|
|
1085
|
+
for (const k of [...this.flapping]) {
|
|
1086
|
+
if (k.startsWith(prefix)) this.flapping.delete(k);
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
};
|
|
1090
|
+
var FLAP_CHANNEL_INTEGRATIONS = "integrations";
|
|
1091
|
+
var FLAP_CHANNEL_MANAGED_MCP = "managed-mcp";
|
|
1092
|
+
|
|
1093
|
+
// src/lib/auto-resume.ts
|
|
1094
|
+
var DEFAULT_QUIET_MS = 18e5;
|
|
1095
|
+
var DEFAULT_BACKOFF_RESET_MS = 864e5;
|
|
1096
|
+
function readAutoResumeConfig(env = process.env) {
|
|
1097
|
+
const raw = (env["AGT_AUTO_RESUME_ENABLED"] ?? "").trim().toLowerCase();
|
|
1098
|
+
return {
|
|
1099
|
+
enabled: raw === "1" || raw === "true" || raw === "yes" || raw === "on",
|
|
1100
|
+
quietMs: readEnvNumber("AGT_AUTO_RESUME_QUIET_MS", DEFAULT_QUIET_MS),
|
|
1101
|
+
backoffResetMs: readEnvNumber("AGT_AUTO_RESUME_BACKOFF_RESET_MS", DEFAULT_BACKOFF_RESET_MS)
|
|
1102
|
+
};
|
|
1103
|
+
}
|
|
1104
|
+
function decideAutoResume(opts) {
|
|
1105
|
+
const { trip, marker, config: config2, now } = opts;
|
|
1106
|
+
if (!config2.enabled) return { action: "skip", reason: "disabled" };
|
|
1107
|
+
if (!trip) return { action: "skip", reason: "no-trip" };
|
|
1108
|
+
if (marker && now - marker.autoResumedAt < config2.backoffResetMs) {
|
|
1109
|
+
return { action: "skip", reason: "already-auto-resumed" };
|
|
1110
|
+
}
|
|
1111
|
+
const lastEventAt = trip.eventsAtTrip.length > 0 ? Math.max(trip.trippedAt, ...trip.eventsAtTrip.map((e) => e.at)) : trip.trippedAt;
|
|
1112
|
+
const quietFor = now - lastEventAt;
|
|
1113
|
+
if (quietFor < config2.quietMs) {
|
|
1114
|
+
return { action: "skip", reason: "not-quiet-yet", remainingMs: config2.quietMs - quietFor };
|
|
1115
|
+
}
|
|
1116
|
+
return { action: "resume", quietMs: quietFor };
|
|
1117
|
+
}
|
|
1118
|
+
function hydrateAutoResumeMarkers(saved) {
|
|
1119
|
+
const map = /* @__PURE__ */ new Map();
|
|
1120
|
+
if (!saved) return map;
|
|
1121
|
+
for (const [codeName, rec] of Object.entries(saved)) {
|
|
1122
|
+
if (rec && typeof rec.trippedAt === "number" && typeof rec.autoResumedAt === "number") {
|
|
1123
|
+
map.set(codeName, rec);
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
return map;
|
|
1127
|
+
}
|
|
1128
|
+
|
|
968
1129
|
// src/lib/model-change-respawn.ts
|
|
969
1130
|
function shouldRespawnForModelChange(input) {
|
|
970
1131
|
const { previousModel, primaryModel, framework, sessionHealthy } = input;
|
|
@@ -980,7 +1141,12 @@ async function executeConnectivityProbe(target, deps = {}) {
|
|
|
980
1141
|
const descriptor = resolveConnectivityProbe({
|
|
981
1142
|
definitionId: target.definitionId,
|
|
982
1143
|
sourceType: target.sourceType,
|
|
983
|
-
authType: target.authType
|
|
1144
|
+
authType: target.authType,
|
|
1145
|
+
// ENG-6242: carry the toolkit's connectivity_test override so a managed
|
|
1146
|
+
// descriptor surfaces `probeTool`/`probeArgs` (the prescribed read-only tool)
|
|
1147
|
+
// — without this the hourly probe auto-picked a different tool than the Test
|
|
1148
|
+
// path, the false-RED that flagged every managed Linear install "unreachable".
|
|
1149
|
+
connectivityTest: target.connectivityTest ?? null
|
|
984
1150
|
});
|
|
985
1151
|
if (!descriptor.readOnly) {
|
|
986
1152
|
throw new Error(`Refusing non-read-only probe for ${target.definitionId}`);
|
|
@@ -991,6 +1157,35 @@ async function executeConnectivityProbe(target, deps = {}) {
|
|
|
991
1157
|
case "composio_account":
|
|
992
1158
|
if (!deps.composioProbe) return null;
|
|
993
1159
|
return deps.composioProbe(target.definitionId, target.credentials);
|
|
1160
|
+
case "managed_composite": {
|
|
1161
|
+
const outcomes = [];
|
|
1162
|
+
if (deps.mcpProbe) {
|
|
1163
|
+
outcomes.push(
|
|
1164
|
+
await deps.mcpProbe({
|
|
1165
|
+
serverKey: target.mcpServerKey ?? target.definitionId,
|
|
1166
|
+
definitionId: target.definitionId
|
|
1167
|
+
})
|
|
1168
|
+
);
|
|
1169
|
+
}
|
|
1170
|
+
if (deps.composioProbe) {
|
|
1171
|
+
outcomes.push(await deps.composioProbe(target.definitionId, target.credentials));
|
|
1172
|
+
}
|
|
1173
|
+
if (deps.composioToolCallProbe) {
|
|
1174
|
+
const toolCall = await deps.composioToolCallProbe({
|
|
1175
|
+
serverKey: target.mcpServerKey ?? target.definitionId,
|
|
1176
|
+
definitionId: target.definitionId,
|
|
1177
|
+
// ENG-6242: thread the prescribed tool through to the live tool-call
|
|
1178
|
+
// leg. resolveConnectivityProbe only sets these for managed toolkits
|
|
1179
|
+
// with a stored override; the probe re-validates read-only and falls
|
|
1180
|
+
// back to auto-pick on drift, so a missing/invalid override is safe.
|
|
1181
|
+
toolName: descriptor.probeTool ?? null,
|
|
1182
|
+
toolArgs: descriptor.probeArgs ?? null
|
|
1183
|
+
});
|
|
1184
|
+
if (toolCall) outcomes.push(toolCall);
|
|
1185
|
+
}
|
|
1186
|
+
if (outcomes.length === 0) return null;
|
|
1187
|
+
return outcomes.reduce((acc, o) => worseConnectivityOutcome(acc, o));
|
|
1188
|
+
}
|
|
994
1189
|
case "mcp_tools_list":
|
|
995
1190
|
if (!deps.mcpProbe) return null;
|
|
996
1191
|
return deps.mcpProbe({
|
|
@@ -1038,7 +1233,8 @@ async function runConnectivityProbes(integrations, options = {}) {
|
|
|
1038
1233
|
authType: integ.auth_type ?? null,
|
|
1039
1234
|
credentials: integ.credentials ?? {},
|
|
1040
1235
|
cliBinary: integ.cli_binary,
|
|
1041
|
-
mcpServerKey: integ.mcp_server_key
|
|
1236
|
+
mcpServerKey: integ.mcp_server_key,
|
|
1237
|
+
connectivityTest: integ.connectivity_test ?? null
|
|
1042
1238
|
};
|
|
1043
1239
|
let outcome;
|
|
1044
1240
|
try {
|
|
@@ -1926,14 +2122,173 @@ function readRecentTurns(dir, nowMs) {
|
|
|
1926
2122
|
return turns;
|
|
1927
2123
|
}
|
|
1928
2124
|
|
|
2125
|
+
// src/lib/memory-extractor.ts
|
|
2126
|
+
var MIN_CHECK_INTERVAL_MS4 = 10 * 6e4;
|
|
2127
|
+
var WINDOW_PAD_MS2 = 5 * 6e4;
|
|
2128
|
+
var VALID_MEMORY_TYPES = /* @__PURE__ */ new Set(["user", "feedback", "project", "reference"]);
|
|
2129
|
+
var SECRET_PATTERNS = [
|
|
2130
|
+
// labelled secrets: `api_key=...`, `password: ...`, `bearer ...`
|
|
2131
|
+
/\b[\w-]*(?:secret|password|passwd|token|api[_-]?key|bearer)[\w-]*\s*[=:]\s*\S+/gi,
|
|
2132
|
+
// common credential prefixes (stripe, github, slack, augmented host keys, …)
|
|
2133
|
+
/\b(?:sk|pk|rk|tlk|ghp|gho|ghs|xox[baprs])[-_][A-Za-z0-9]{12,}\b/g,
|
|
2134
|
+
// long hex blobs (hashes / hex-encoded keys)
|
|
2135
|
+
/\b[A-Fa-f0-9]{32,}\b/g,
|
|
2136
|
+
// long base64 blobs (tokens / encoded secrets)
|
|
2137
|
+
/\b[A-Za-z0-9+/]{40,}={0,2}\b/g,
|
|
2138
|
+
// email addresses
|
|
2139
|
+
/\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g
|
|
2140
|
+
];
|
|
2141
|
+
function scrubSensitive(text) {
|
|
2142
|
+
let out = text;
|
|
2143
|
+
for (const re of SECRET_PATTERNS) out = out.replace(re, "[redacted]");
|
|
2144
|
+
return out;
|
|
2145
|
+
}
|
|
2146
|
+
var state4 = /* @__PURE__ */ new Map();
|
|
2147
|
+
function buildExtractionPrompt(channel, transcript) {
|
|
2148
|
+
return `You are extracting durable, reusable memories from one completed ${channel} conversation between an AI agent and an end-user. A good memory is a stable preference, fact, correction, or recurring work context that will help the agent in FUTURE conversations \u2014 not a one-off task detail or pleasantry.
|
|
2149
|
+
|
|
2150
|
+
Conversation transcript:
|
|
2151
|
+
"""
|
|
2152
|
+
${transcript}
|
|
2153
|
+
"""
|
|
2154
|
+
|
|
2155
|
+
Extract 0-5 candidate memories. For each:
|
|
2156
|
+
- name: short descriptive title (max 50 chars)
|
|
2157
|
+
- content: the memory itself, self-contained (max 200 chars). Do NOT include secrets or sensitive personal data.
|
|
2158
|
+
- type: "user" (a person's stable preference), "feedback" (a correction to how the agent should work), "project" (durable work context), or "reference" (a pointer to an external resource)
|
|
2159
|
+
- confidence: 0.0-1.0 \u2014 how sure you are this is worth remembering long-term
|
|
2160
|
+
- rationale: why this should persist (max 100 chars)
|
|
2161
|
+
|
|
2162
|
+
If there is nothing durable worth remembering, return an empty array. Do NOT invent memories to fill the list.
|
|
2163
|
+
|
|
2164
|
+
Respond with ONLY a JSON array, no other text:
|
|
2165
|
+
[{"name":"...","content":"...","type":"...","confidence":0.8,"rationale":"..."}]`;
|
|
2166
|
+
}
|
|
2167
|
+
function parseCandidates(raw) {
|
|
2168
|
+
const match = raw.match(/\[[\s\S]*\]/);
|
|
2169
|
+
if (!match) return null;
|
|
2170
|
+
let arr;
|
|
2171
|
+
try {
|
|
2172
|
+
arr = JSON.parse(match[0]);
|
|
2173
|
+
} catch {
|
|
2174
|
+
return null;
|
|
2175
|
+
}
|
|
2176
|
+
if (!Array.isArray(arr)) return null;
|
|
2177
|
+
const out = [];
|
|
2178
|
+
for (const entry of arr) {
|
|
2179
|
+
if (!entry || typeof entry !== "object") continue;
|
|
2180
|
+
const c = entry;
|
|
2181
|
+
const name = typeof c.name === "string" ? c.name.trim() : "";
|
|
2182
|
+
if (!name) continue;
|
|
2183
|
+
if (typeof c.type !== "string" || !VALID_MEMORY_TYPES.has(c.type)) continue;
|
|
2184
|
+
const confidence = typeof c.confidence === "number" && Number.isFinite(c.confidence) ? Math.max(0, Math.min(1, c.confidence)) : 0;
|
|
2185
|
+
if (confidence === 0) continue;
|
|
2186
|
+
out.push({
|
|
2187
|
+
name: name.slice(0, 80),
|
|
2188
|
+
content: (typeof c.content === "string" ? c.content : "").trim().slice(0, 500),
|
|
2189
|
+
type: c.type,
|
|
2190
|
+
confidence,
|
|
2191
|
+
rationale: (typeof c.rationale === "string" ? c.rationale : "").trim().slice(0, 200)
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
2194
|
+
return out;
|
|
2195
|
+
}
|
|
2196
|
+
async function maybeExtractMemories(args) {
|
|
2197
|
+
const { api: api2, backend, codeName, agentId, log: log2 } = args;
|
|
2198
|
+
const now = args.now ?? /* @__PURE__ */ new Date();
|
|
2199
|
+
const nowMs = now.getTime();
|
|
2200
|
+
const existing = state4.get(codeName);
|
|
2201
|
+
if (existing && nowMs - existing.lastCheckedAt < MIN_CHECK_INTERVAL_MS4) {
|
|
2202
|
+
return;
|
|
2203
|
+
}
|
|
2204
|
+
state4.set(codeName, { lastCheckedAt: nowMs });
|
|
2205
|
+
let pending;
|
|
2206
|
+
try {
|
|
2207
|
+
const resp = await api2.get(
|
|
2208
|
+
`/host/memories/pending-extraction?agent_id=${encodeURIComponent(agentId)}`
|
|
2209
|
+
);
|
|
2210
|
+
pending = resp?.conversations ?? [];
|
|
2211
|
+
} catch (err) {
|
|
2212
|
+
log2(`[memory-extract] ${codeName}: pending fetch failed: ${err.message}`);
|
|
2213
|
+
return;
|
|
2214
|
+
}
|
|
2215
|
+
if (pending.length === 0) return;
|
|
2216
|
+
const dir = args.transcriptDir ?? sessionTranscriptDir(getProjectDir(codeName));
|
|
2217
|
+
const allTurns = readRecentTurns(dir, nowMs);
|
|
2218
|
+
if (allTurns.length === 0) {
|
|
2219
|
+
for (const conv of pending) {
|
|
2220
|
+
await reportSkip2(api2, agentId, conv.conversation_id, log2, codeName);
|
|
2221
|
+
}
|
|
2222
|
+
return;
|
|
2223
|
+
}
|
|
2224
|
+
for (const conv of pending) {
|
|
2225
|
+
const tokens = channelRefTokens(conv.channel_ref);
|
|
2226
|
+
const windowStart = Date.parse(conv.started_at) - WINDOW_PAD_MS2;
|
|
2227
|
+
const windowEnd = Date.parse(conv.last_message_at) + WINDOW_PAD_MS2;
|
|
2228
|
+
const turns = reconstructConversation(allTurns, tokens, windowStart, windowEnd);
|
|
2229
|
+
if (turns.length === 0) {
|
|
2230
|
+
await reportSkip2(api2, agentId, conv.conversation_id, log2, codeName);
|
|
2231
|
+
continue;
|
|
2232
|
+
}
|
|
2233
|
+
const transcript = renderTranscript(turns);
|
|
2234
|
+
if (!transcript.trim()) {
|
|
2235
|
+
await reportSkip2(api2, agentId, conv.conversation_id, log2, codeName);
|
|
2236
|
+
continue;
|
|
2237
|
+
}
|
|
2238
|
+
let candidates;
|
|
2239
|
+
try {
|
|
2240
|
+
const out = await backend.run(buildExtractionPrompt(conv.channel, transcript));
|
|
2241
|
+
candidates = parseCandidates(out);
|
|
2242
|
+
} catch (err) {
|
|
2243
|
+
log2(`[memory-extract] ${codeName}: extraction failed: ${err.message}`);
|
|
2244
|
+
continue;
|
|
2245
|
+
}
|
|
2246
|
+
if (candidates === null) {
|
|
2247
|
+
log2(`[memory-extract] ${codeName}: unparseable output for ${conv.conversation_id.slice(0, 8)} \u2014 will retry`);
|
|
2248
|
+
continue;
|
|
2249
|
+
}
|
|
2250
|
+
const scrubbed = candidates.map((c) => ({
|
|
2251
|
+
...c,
|
|
2252
|
+
content: scrubSensitive(c.content),
|
|
2253
|
+
rationale: scrubSensitive(c.rationale)
|
|
2254
|
+
}));
|
|
2255
|
+
try {
|
|
2256
|
+
const res = await api2.post(
|
|
2257
|
+
"/host/memories/candidates",
|
|
2258
|
+
{
|
|
2259
|
+
agent_id: agentId,
|
|
2260
|
+
conversation_id: conv.conversation_id,
|
|
2261
|
+
candidates: scrubbed
|
|
2262
|
+
}
|
|
2263
|
+
);
|
|
2264
|
+
log2(
|
|
2265
|
+
`[memory-extract] ${codeName}: ${conv.conversation_id.slice(0, 8)} \u2192 ${candidates.length} candidate(s), ${res?.promoted ?? 0} promoted`
|
|
2266
|
+
);
|
|
2267
|
+
} catch (err) {
|
|
2268
|
+
log2(`[memory-extract] ${codeName}: report failed: ${err.message}`);
|
|
2269
|
+
}
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
async function reportSkip2(api2, agentId, conversationId, log2, codeName) {
|
|
2273
|
+
try {
|
|
2274
|
+
await api2.post("/host/memories/candidates", {
|
|
2275
|
+
agent_id: agentId,
|
|
2276
|
+
conversation_id: conversationId,
|
|
2277
|
+
skipped: true
|
|
2278
|
+
});
|
|
2279
|
+
} catch (err) {
|
|
2280
|
+
log2(`[memory-extract] ${codeName}: skip report failed: ${err.message}`);
|
|
2281
|
+
}
|
|
2282
|
+
}
|
|
2283
|
+
|
|
1929
2284
|
// src/lib/activity-cache-monitor.ts
|
|
1930
2285
|
import { existsSync as existsSync2, readFileSync as readFileSync6 } from "fs";
|
|
1931
2286
|
import { homedir } from "os";
|
|
1932
2287
|
import { join as join4 } from "path";
|
|
1933
|
-
var
|
|
2288
|
+
var MIN_CHECK_INTERVAL_MS5 = 6e4;
|
|
1934
2289
|
var STATS_CACHE_PATH = join4(homedir(), ".claude", "stats-cache.json");
|
|
1935
2290
|
var ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
|
1936
|
-
var
|
|
2291
|
+
var state5 = { lastObservedDate: null, lastCheckedAt: 0 };
|
|
1937
2292
|
function selectNewDailyRows(raw, lastObservedDate) {
|
|
1938
2293
|
let parsed;
|
|
1939
2294
|
try {
|
|
@@ -1972,8 +2327,8 @@ async function maybeReportActivityCache(args) {
|
|
|
1972
2327
|
const { api: api2, log: log2 } = args;
|
|
1973
2328
|
const now = args.now ?? /* @__PURE__ */ new Date();
|
|
1974
2329
|
const nowMs = now.getTime();
|
|
1975
|
-
if (nowMs -
|
|
1976
|
-
|
|
2330
|
+
if (nowMs - state5.lastCheckedAt < MIN_CHECK_INTERVAL_MS5) return;
|
|
2331
|
+
state5.lastCheckedAt = nowMs;
|
|
1977
2332
|
if (!existsSync2(STATS_CACHE_PATH)) {
|
|
1978
2333
|
return;
|
|
1979
2334
|
}
|
|
@@ -1984,12 +2339,12 @@ async function maybeReportActivityCache(args) {
|
|
|
1984
2339
|
log2(`[activity-cache] readFileSync failed: ${err.message}`);
|
|
1985
2340
|
return;
|
|
1986
2341
|
}
|
|
1987
|
-
const rows = selectNewDailyRows(raw,
|
|
2342
|
+
const rows = selectNewDailyRows(raw, state5.lastObservedDate);
|
|
1988
2343
|
if (rows.length === 0) return;
|
|
1989
2344
|
for (const row of rows) {
|
|
1990
2345
|
try {
|
|
1991
2346
|
await api2.post("/host/activity-observations", row);
|
|
1992
|
-
|
|
2347
|
+
state5.lastObservedDate = row.date;
|
|
1993
2348
|
} catch (err) {
|
|
1994
2349
|
log2(
|
|
1995
2350
|
`[activity-cache] POST /host/activity-observations failed for date=${row.date}: ${err.message}`
|
|
@@ -3019,6 +3374,128 @@ function clearAgentState(agentId, codeName) {
|
|
|
3019
3374
|
return channelCacheMutated;
|
|
3020
3375
|
}
|
|
3021
3376
|
|
|
3377
|
+
// src/lib/wedge-detection.ts
|
|
3378
|
+
var DEFAULTS = {
|
|
3379
|
+
inboundWaitSeconds: 120,
|
|
3380
|
+
// ENG-6238: the hard cap is now an ABSOLUTE BACKSTOP, not the primary
|
|
3381
|
+
// discriminator — a still-producing session (e.g. a runaway loop) is only
|
|
3382
|
+
// reaped here. Raised 300→1200 because the soft path now reliably catches
|
|
3383
|
+
// the frozen-turn wedge via the transcript signal, so the hard cap no longer
|
|
3384
|
+
// needs to fire early (which is exactly what false-killed kylie's long
|
|
3385
|
+
// legitimate turns, ENG-6238).
|
|
3386
|
+
inboundHardWaitSeconds: 1200,
|
|
3387
|
+
paneStaleSeconds: 120,
|
|
3388
|
+
transcriptStaleSeconds: 60,
|
|
3389
|
+
minCycles: 3
|
|
3390
|
+
};
|
|
3391
|
+
function parseMode(raw) {
|
|
3392
|
+
const v = (raw ?? "").trim().toLowerCase();
|
|
3393
|
+
return v === "shadow" || v === "enforce" ? v : "off";
|
|
3394
|
+
}
|
|
3395
|
+
function parsePositiveInt(raw, fallback, floor) {
|
|
3396
|
+
const n = raw ? Number.parseInt(raw, 10) : NaN;
|
|
3397
|
+
return Number.isInteger(n) && n >= floor ? n : fallback;
|
|
3398
|
+
}
|
|
3399
|
+
function resolveWedgeConfig(env = process.env) {
|
|
3400
|
+
const inboundWaitSeconds = parsePositiveInt(
|
|
3401
|
+
env.AGT_WEDGE_INBOUND_WAIT_SECONDS,
|
|
3402
|
+
DEFAULTS.inboundWaitSeconds,
|
|
3403
|
+
30
|
|
3404
|
+
);
|
|
3405
|
+
const inboundHardWaitSeconds = Math.max(
|
|
3406
|
+
inboundWaitSeconds,
|
|
3407
|
+
parsePositiveInt(env.AGT_WEDGE_INBOUND_HARD_WAIT_SECONDS, DEFAULTS.inboundHardWaitSeconds, 30)
|
|
3408
|
+
);
|
|
3409
|
+
return {
|
|
3410
|
+
mode: parseMode(env.AGT_WEDGE_RESTART_MODE),
|
|
3411
|
+
inboundWaitSeconds,
|
|
3412
|
+
inboundHardWaitSeconds,
|
|
3413
|
+
paneStaleSeconds: parsePositiveInt(env.AGT_WEDGE_PANE_STALE_SECONDS, DEFAULTS.paneStaleSeconds, 30),
|
|
3414
|
+
transcriptStaleSeconds: parsePositiveInt(
|
|
3415
|
+
env.AGT_WEDGE_TRANSCRIPT_STALE_SECONDS,
|
|
3416
|
+
DEFAULTS.transcriptStaleSeconds,
|
|
3417
|
+
15
|
|
3418
|
+
),
|
|
3419
|
+
minCycles: parsePositiveInt(env.AGT_WEDGE_MIN_CYCLES, DEFAULTS.minCycles, 2)
|
|
3420
|
+
};
|
|
3421
|
+
}
|
|
3422
|
+
function isSessionProducing(signals, config2) {
|
|
3423
|
+
const transcriptAge = signals.transcriptActivityAgeSeconds;
|
|
3424
|
+
if (transcriptAge !== null) return transcriptAge < config2.transcriptStaleSeconds;
|
|
3425
|
+
const paneAge = signals.paneActivityAgeSeconds;
|
|
3426
|
+
return paneAge !== null && paneAge < config2.paneStaleSeconds;
|
|
3427
|
+
}
|
|
3428
|
+
function isWedgeCandidateCycle(signals, config2) {
|
|
3429
|
+
const inboundAge = signals.pendingInboundOldestAgeSeconds;
|
|
3430
|
+
if (inboundAge === null) return false;
|
|
3431
|
+
if (inboundAge < config2.inboundWaitSeconds) return false;
|
|
3432
|
+
if (isSessionProducing(signals, config2)) {
|
|
3433
|
+
return inboundAge >= config2.inboundHardWaitSeconds;
|
|
3434
|
+
}
|
|
3435
|
+
return true;
|
|
3436
|
+
}
|
|
3437
|
+
function decideWedgeRestart(input, config2) {
|
|
3438
|
+
if (!isWedgeCandidateCycle(input, config2)) return "none";
|
|
3439
|
+
return input.consecutiveWedgeCycles >= config2.minCycles ? "wedged" : "none";
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
// src/lib/wedge-poison-card.ts
|
|
3443
|
+
var DEFAULTS2 = {
|
|
3444
|
+
threshold: 3,
|
|
3445
|
+
cooldownSeconds: 1800
|
|
3446
|
+
// 30 min — matches the kanban stale-auto-fail window
|
|
3447
|
+
};
|
|
3448
|
+
function parsePositiveInt2(raw, fallback, floor) {
|
|
3449
|
+
const n = raw ? Number.parseInt(raw, 10) : NaN;
|
|
3450
|
+
return Number.isInteger(n) && n >= floor ? n : fallback;
|
|
3451
|
+
}
|
|
3452
|
+
function resolvePoisonCardConfig(env = process.env) {
|
|
3453
|
+
return {
|
|
3454
|
+
threshold: parsePositiveInt2(env.AGT_WEDGE_POISON_CARD_THRESHOLD, DEFAULTS2.threshold, 2),
|
|
3455
|
+
cooldownMs: parsePositiveInt2(
|
|
3456
|
+
env.AGT_WEDGE_POISON_CARD_COOLDOWN_SECONDS,
|
|
3457
|
+
DEFAULTS2.cooldownSeconds,
|
|
3458
|
+
300
|
|
3459
|
+
) * 1e3
|
|
3460
|
+
};
|
|
3461
|
+
}
|
|
3462
|
+
function recordWedgeForCards(states, inProgressCardIds, nowMs, config2) {
|
|
3463
|
+
if (inProgressCardIds.length === 0) {
|
|
3464
|
+
return { next: new Map(states), newlyPoisoned: [] };
|
|
3465
|
+
}
|
|
3466
|
+
const next = /* @__PURE__ */ new Map();
|
|
3467
|
+
const newlyPoisoned = [];
|
|
3468
|
+
for (const id of inProgressCardIds) {
|
|
3469
|
+
if (next.has(id)) continue;
|
|
3470
|
+
const count = (states.get(id)?.count ?? 0) + 1;
|
|
3471
|
+
next.set(id, { count, lastWedgeAtMs: nowMs });
|
|
3472
|
+
if (count === config2.threshold) newlyPoisoned.push(id);
|
|
3473
|
+
}
|
|
3474
|
+
return { next, newlyPoisoned };
|
|
3475
|
+
}
|
|
3476
|
+
function pruneCardStates(states, liveInProgressCardIds, nowMs, config2) {
|
|
3477
|
+
const live = liveInProgressCardIds instanceof Set ? liveInProgressCardIds : new Set(liveInProgressCardIds);
|
|
3478
|
+
const next = /* @__PURE__ */ new Map();
|
|
3479
|
+
for (const [id, state7] of states) {
|
|
3480
|
+
if (!live.has(id)) continue;
|
|
3481
|
+
if (nowMs - state7.lastWedgeAtMs > config2.cooldownMs) continue;
|
|
3482
|
+
next.set(id, state7);
|
|
3483
|
+
}
|
|
3484
|
+
return next;
|
|
3485
|
+
}
|
|
3486
|
+
function isCardPoisoned(states, cardId, config2) {
|
|
3487
|
+
return (states.get(cardId)?.count ?? 0) >= config2.threshold;
|
|
3488
|
+
}
|
|
3489
|
+
function partitionActionableByPoison(actionable, states, config2) {
|
|
3490
|
+
const allowed = [];
|
|
3491
|
+
const suppressed = [];
|
|
3492
|
+
for (const item of actionable) {
|
|
3493
|
+
if (isCardPoisoned(states, item.id, config2)) suppressed.push(item);
|
|
3494
|
+
else allowed.push(item);
|
|
3495
|
+
}
|
|
3496
|
+
return { allowed, suppressed };
|
|
3497
|
+
}
|
|
3498
|
+
|
|
3022
3499
|
// src/lib/restart-flags.ts
|
|
3023
3500
|
import { existsSync as existsSync4, mkdirSync as mkdirSync3, readdirSync as readdirSync3, readFileSync as readFileSync8, renameSync as renameSync2, rmSync, writeFileSync as writeFileSync3 } from "fs";
|
|
3024
3501
|
import { homedir as homedir3 } from "os";
|
|
@@ -3095,8 +3572,9 @@ async function processOne(flag, opts) {
|
|
|
3095
3572
|
await sendError(flag, opts, `\`/restart\` is only supported for Claude Code agents (this agent runs on \`${framework}\`).`);
|
|
3096
3573
|
return;
|
|
3097
3574
|
}
|
|
3098
|
-
|
|
3099
|
-
opts.
|
|
3575
|
+
const reasonSuffix = flag.source === "agent" && flag.reason ? ` reason="${flag.reason}"` : "";
|
|
3576
|
+
opts.log(`[restart-handler] Restarting tmux session for '${flag.codeName}' (source: ${flag.source})${reasonSuffix}`);
|
|
3577
|
+
opts.stopSession(flag.codeName, flag);
|
|
3100
3578
|
await sendAck(flag, opts, `\u{1F504} Restart initiated for \`${flag.codeName}\` \u2014 the replacement session is being created.`);
|
|
3101
3579
|
}
|
|
3102
3580
|
async function sendAck(flag, opts, text) {
|
|
@@ -3692,12 +4170,40 @@ var KNOWN_SAFE_TAIL_SIGNATURES = /* @__PURE__ */ new Set(["session_id_in_use"]);
|
|
|
3692
4170
|
function shouldSkipRevokedCleanup(previousKnownStatus) {
|
|
3693
4171
|
return previousKnownStatus === "revoked";
|
|
3694
4172
|
}
|
|
3695
|
-
function hasRevokedResiduals(
|
|
3696
|
-
return
|
|
4173
|
+
function hasRevokedResiduals(state7) {
|
|
4174
|
+
return state7.gatewayRunning || state7.portAllocated || state7.provisionDirExists;
|
|
3697
4175
|
}
|
|
3698
4176
|
var pendingSessionRestarts = /* @__PURE__ */ new Map();
|
|
4177
|
+
var pendingRestartVerifications = /* @__PURE__ */ new Map();
|
|
3699
4178
|
var restartBreaker = new RestartBreaker();
|
|
3700
4179
|
var reportedTrips = /* @__PURE__ */ new Map();
|
|
4180
|
+
var mcpFlapDampener = new McpFlapDampener();
|
|
4181
|
+
function recordConfigChurnEvent(agentId, codeName, channel, signature) {
|
|
4182
|
+
api.post("/host/mcp-config-churn", {
|
|
4183
|
+
agent_id: agentId,
|
|
4184
|
+
code_name: codeName,
|
|
4185
|
+
channel,
|
|
4186
|
+
signature
|
|
4187
|
+
}).catch((err) => {
|
|
4188
|
+
log(
|
|
4189
|
+
`[mcp-flap-dampener] failed to record config-churn event for '${codeName}' (${channel}) (ENG-6123): ${err.message} \u2014 local suppression still active; CloudWatch metric will under-count`
|
|
4190
|
+
);
|
|
4191
|
+
});
|
|
4192
|
+
}
|
|
4193
|
+
async function reportReaperEvent(agentId, reaper, action, reason) {
|
|
4194
|
+
try {
|
|
4195
|
+
await api.post("/host/reaper-event", {
|
|
4196
|
+
agent_id: agentId,
|
|
4197
|
+
reaper,
|
|
4198
|
+
action,
|
|
4199
|
+
reason
|
|
4200
|
+
});
|
|
4201
|
+
} catch (err) {
|
|
4202
|
+
log(
|
|
4203
|
+
`[${reaper}] failed to report reaper-event (ENG-6184): ${err.message} \u2014 local reap still proceeded; the alert may be missed`
|
|
4204
|
+
);
|
|
4205
|
+
}
|
|
4206
|
+
}
|
|
3701
4207
|
function recordRestartForBreaker(codeName, reason) {
|
|
3702
4208
|
const result = restartBreaker.record(codeName, reason);
|
|
3703
4209
|
if (!result.tripped || !result.trip) return;
|
|
@@ -3727,6 +4233,65 @@ function maybeReportCurrentTrip(codeName) {
|
|
|
3727
4233
|
log(`[circuit-breaker] Failed to report trip for '${codeName}' to API: ${err.message}`);
|
|
3728
4234
|
});
|
|
3729
4235
|
}
|
|
4236
|
+
var autoResumeMarkers = /* @__PURE__ */ new Map();
|
|
4237
|
+
var autoResumeInFlight = /* @__PURE__ */ new Set();
|
|
4238
|
+
var AUTO_RESUME_SELF_WINDOW_MS = 12e4;
|
|
4239
|
+
var autoResumeLoggedSkips = /* @__PURE__ */ new Map();
|
|
4240
|
+
var autoResumeStandDowns = /* @__PURE__ */ new Set();
|
|
4241
|
+
var killPausedCodeNames = /* @__PURE__ */ new Set();
|
|
4242
|
+
function maybeAutoResume(agent) {
|
|
4243
|
+
const codeName = agent.code_name;
|
|
4244
|
+
if (autoResumeInFlight.has(codeName)) return;
|
|
4245
|
+
const trip = restartBreaker.getTrip(codeName);
|
|
4246
|
+
if (trip && autoResumeStandDowns.has(`${codeName}:${trip.trippedAt}`)) return;
|
|
4247
|
+
const decision = decideAutoResume({
|
|
4248
|
+
trip,
|
|
4249
|
+
marker: autoResumeMarkers.get(codeName),
|
|
4250
|
+
config: readAutoResumeConfig(),
|
|
4251
|
+
now: Date.now()
|
|
4252
|
+
});
|
|
4253
|
+
if (decision.action === "skip") {
|
|
4254
|
+
if ((decision.reason === "already-auto-resumed" || decision.reason === "not-quiet-yet") && trip) {
|
|
4255
|
+
const key = `${codeName}:${trip.trippedAt}`;
|
|
4256
|
+
if (autoResumeLoggedSkips.get(key) !== decision.reason) {
|
|
4257
|
+
autoResumeLoggedSkips.set(key, decision.reason);
|
|
4258
|
+
const detail = decision.reason === "not-quiet-yet" ? ` (~${Math.ceil((decision.remainingMs ?? 0) / 6e4)}min remaining)` : " \u2014 credit spent for this cycle; operator resume re-arms it";
|
|
4259
|
+
log(`[auto-resume] agent=${codeName} decision=skip reason=${decision.reason}${detail} (ENG-6088)`);
|
|
4260
|
+
}
|
|
4261
|
+
}
|
|
4262
|
+
return;
|
|
4263
|
+
}
|
|
4264
|
+
const trippedAt = trip.trippedAt;
|
|
4265
|
+
log(
|
|
4266
|
+
`[auto-resume] agent=${codeName} decision=resume quiet=${Math.round(decision.quietMs / 6e4)}min trippedAt=${new Date(trippedAt).toISOString()} (ENG-6088)`
|
|
4267
|
+
);
|
|
4268
|
+
autoResumeInFlight.add(codeName);
|
|
4269
|
+
api.post("/host/circuit-breaker/auto-resume", {
|
|
4270
|
+
agent_id: agent.agent_id,
|
|
4271
|
+
quiet_ms: decision.quietMs,
|
|
4272
|
+
tripped_at: new Date(trippedAt).toISOString()
|
|
4273
|
+
}).then((res) => {
|
|
4274
|
+
autoResumeInFlight.delete(codeName);
|
|
4275
|
+
if (res.resumed) {
|
|
4276
|
+
autoResumeMarkers.set(codeName, { trippedAt, autoResumedAt: Date.now() });
|
|
4277
|
+
restartBreaker.clear(codeName);
|
|
4278
|
+
reportedTrips.delete(codeName);
|
|
4279
|
+
state6 = {
|
|
4280
|
+
...state6,
|
|
4281
|
+
circuitBreakerTrips: restartBreaker.serialize(),
|
|
4282
|
+
circuitBreakerAutoResumes: Object.fromEntries(autoResumeMarkers.entries())
|
|
4283
|
+
};
|
|
4284
|
+
send({ type: "state-update", state: state6 });
|
|
4285
|
+
log(`[auto-resume] agent=${codeName} resumed \u2014 re-trip within backoff window will stay paused (ENG-6088)`);
|
|
4286
|
+
} else {
|
|
4287
|
+
autoResumeStandDowns.add(`${codeName}:${trippedAt}`);
|
|
4288
|
+
log(`[auto-resume] agent=${codeName} not applied (reason=${res.reason ?? "unknown"}) \u2014 standing down for this trip (ENG-6088)`);
|
|
4289
|
+
}
|
|
4290
|
+
}).catch((err) => {
|
|
4291
|
+
autoResumeInFlight.delete(codeName);
|
|
4292
|
+
log(`[auto-resume] agent=${codeName} POST failed (will retry next poll): ${err.message}`);
|
|
4293
|
+
});
|
|
4294
|
+
}
|
|
3730
4295
|
function scheduleSessionRestart(codeName, delayMs, reason, breakerReason = "hot-reload-mcp") {
|
|
3731
4296
|
const existing = pendingSessionRestarts.get(codeName);
|
|
3732
4297
|
if (existing) {
|
|
@@ -3737,29 +4302,104 @@ function scheduleSessionRestart(codeName, delayMs, reason, breakerReason = "hot-
|
|
|
3737
4302
|
pendingSessionRestarts.delete(codeName);
|
|
3738
4303
|
const gate = restartGateFor(codeName, breakerReason);
|
|
3739
4304
|
if (gate !== "bypass" && gate !== "proceed") {
|
|
4305
|
+
const lastLogged = deferLogThrottle.get(codeName);
|
|
4306
|
+
if (lastLogged === void 0 || Date.now() - lastLogged >= DEFER_LOG_THROTTLE_MS) {
|
|
4307
|
+
log(
|
|
4308
|
+
`[maintenance-window] Deferring '${reason}' restart for '${codeName}' (${gate}) \u2014 re-checking every ${RESTART_DEFER_RECHECK_MS / 1e3}s until the window opens`
|
|
4309
|
+
);
|
|
4310
|
+
deferLogThrottle.set(codeName, Date.now());
|
|
4311
|
+
}
|
|
4312
|
+
scheduleSessionRestart(codeName, RESTART_DEFER_RECHECK_MS, reason, breakerReason);
|
|
4313
|
+
return;
|
|
4314
|
+
}
|
|
4315
|
+
deferLogThrottle.delete(codeName);
|
|
4316
|
+
if (breakerReason === "bind-remediation" && restartBreaker.isTripped(codeName)) {
|
|
3740
4317
|
log(
|
|
3741
|
-
`[
|
|
4318
|
+
`[bind-remediation] '${codeName}' breaker tripped before the re-respawn fired \u2014 aborting to avoid stopping a session the breaker won't let respawn (ENG-6203)`
|
|
3742
4319
|
);
|
|
3743
|
-
scheduleSessionRestart(codeName, RESTART_DEFER_RECHECK_MS, reason, breakerReason);
|
|
3744
4320
|
return;
|
|
3745
4321
|
}
|
|
3746
4322
|
stopPersistentSession(codeName, log);
|
|
3747
4323
|
runningMcpHashes.delete(codeName);
|
|
3748
4324
|
recordRestartForBreaker(codeName, breakerReason);
|
|
3749
4325
|
log(`[hot-reload] Session stopped for '${codeName}' \u2014 will respawn with ${reason}`);
|
|
4326
|
+
if (breakerReason === "hot-reload-mcp" || breakerReason === "bind-remediation") {
|
|
4327
|
+
const prior = pendingRestartVerifications.get(codeName);
|
|
4328
|
+
pendingRestartVerifications.set(codeName, {
|
|
4329
|
+
firedAt: Date.now(),
|
|
4330
|
+
attempts: prior?.attempts ?? 0
|
|
4331
|
+
});
|
|
4332
|
+
}
|
|
3750
4333
|
}, delayMs);
|
|
3751
4334
|
timer.unref?.();
|
|
3752
4335
|
pendingSessionRestarts.set(codeName, timer);
|
|
3753
4336
|
}
|
|
3754
4337
|
function cancelPendingSessionRestart(codeName) {
|
|
4338
|
+
pendingRestartVerifications.delete(codeName);
|
|
3755
4339
|
const existing = pendingSessionRestarts.get(codeName);
|
|
3756
4340
|
if (!existing) return;
|
|
3757
4341
|
clearTimeout(existing);
|
|
3758
4342
|
pendingSessionRestarts.delete(codeName);
|
|
4343
|
+
deferLogThrottle.delete(codeName);
|
|
3759
4344
|
log(`[hot-reload] Cancelled pending restart timer for '${codeName}' (another teardown path is handling it)`);
|
|
3760
4345
|
}
|
|
4346
|
+
function bindRemediationEnabled() {
|
|
4347
|
+
const v = (process.env["AGT_BIND_REMEDIATION_ENABLED"] ?? "").trim().toLowerCase();
|
|
4348
|
+
return v === "true" || v === "1";
|
|
4349
|
+
}
|
|
4350
|
+
var BIND_REMEDIATION_JITTER_MS = 3e4;
|
|
4351
|
+
function verifyPendingRestarts(now) {
|
|
4352
|
+
if (pendingRestartVerifications.size === 0) return;
|
|
4353
|
+
for (const [codeName, pending] of pendingRestartVerifications) {
|
|
4354
|
+
const healthy = isSessionHealthy(codeName);
|
|
4355
|
+
const session = getSessionState(codeName);
|
|
4356
|
+
const startedAt = session?.startedAt ?? null;
|
|
4357
|
+
const respawnId = session?.currentSessionId ?? null;
|
|
4358
|
+
const outcome = decidePostRestartVerification(pending, startedAt, healthy, respawnId, now);
|
|
4359
|
+
switch (outcome.kind) {
|
|
4360
|
+
case "verified":
|
|
4361
|
+
pendingRestartVerifications.delete(codeName);
|
|
4362
|
+
log(`[restart-verify] '${codeName}' respawned healthy after MCP change \u2014 tools bound (ENG-6174)`);
|
|
4363
|
+
break;
|
|
4364
|
+
case "waiting":
|
|
4365
|
+
break;
|
|
4366
|
+
case "unverified":
|
|
4367
|
+
if (outcome.final) {
|
|
4368
|
+
pendingRestartVerifications.delete(codeName);
|
|
4369
|
+
log(
|
|
4370
|
+
`[restart-verify] ERROR '${codeName}' did NOT respawn healthy after ${outcome.attempt} attempts following an MCP change \u2014 the live session may be stuck on its pre-restart tools. Check session health / manager.log; a manual session restart may be required (ENG-6174)`
|
|
4371
|
+
);
|
|
4372
|
+
if (shouldRemediateUnverifiedRestart(outcome, {
|
|
4373
|
+
enabled: bindRemediationEnabled(),
|
|
4374
|
+
breakerTripped: restartBreaker.isTripped(codeName)
|
|
4375
|
+
})) {
|
|
4376
|
+
const jitterMs = Math.floor(Math.random() * BIND_REMEDIATION_JITTER_MS);
|
|
4377
|
+
log(
|
|
4378
|
+
`[bind-remediation] '${codeName}' restart never bound MCP tools \u2014 forcing one gated re-respawn in ${Math.round(jitterMs / 1e3)}s (breaker-capped, ENG-6203)`
|
|
4379
|
+
);
|
|
4380
|
+
scheduleSessionRestart(
|
|
4381
|
+
codeName,
|
|
4382
|
+
jitterMs,
|
|
4383
|
+
"bind-remediation: prior restart left MCP tools unbound",
|
|
4384
|
+
"bind-remediation"
|
|
4385
|
+
);
|
|
4386
|
+
}
|
|
4387
|
+
} else {
|
|
4388
|
+
pendingRestartVerifications.set(codeName, { firedAt: now, attempts: outcome.attempt });
|
|
4389
|
+
log(
|
|
4390
|
+
`[restart-verify] WARN '${codeName}' not yet respawned healthy after MCP change (attempt ${outcome.attempt}) \u2014 still watching (ENG-6174)`
|
|
4391
|
+
);
|
|
4392
|
+
}
|
|
4393
|
+
break;
|
|
4394
|
+
}
|
|
4395
|
+
}
|
|
4396
|
+
}
|
|
3761
4397
|
var RESTART_DEFER_RECHECK_MS = 6e4;
|
|
4398
|
+
var DEFER_LOG_THROTTLE_MS = 6e5;
|
|
4399
|
+
var deferLogThrottle = /* @__PURE__ */ new Map();
|
|
3762
4400
|
var lastInboundMs = /* @__PURE__ */ new Map();
|
|
4401
|
+
var consecutiveWedgeCycles = /* @__PURE__ */ new Map();
|
|
4402
|
+
var wedgeRestartsByCard = /* @__PURE__ */ new Map();
|
|
3763
4403
|
function noteInbound(codeName) {
|
|
3764
4404
|
lastInboundMs.set(codeName, Date.now());
|
|
3765
4405
|
}
|
|
@@ -3808,13 +4448,26 @@ function projectMcpKeys(_codeName, projectDir) {
|
|
|
3808
4448
|
return null;
|
|
3809
4449
|
}
|
|
3810
4450
|
}
|
|
3811
|
-
function readMcpHttpServerConfig(projectDir, serverKey) {
|
|
4451
|
+
function readMcpHttpServerConfig(projectDir, serverKey, env) {
|
|
3812
4452
|
try {
|
|
3813
4453
|
const raw = readFileSync9(join8(projectDir, ".mcp.json"), "utf-8");
|
|
3814
4454
|
const servers = JSON.parse(raw).mcpServers ?? {};
|
|
3815
4455
|
const entry = servers[serverKey];
|
|
3816
4456
|
if (entry && typeof entry.url === "string" && (entry.type === "http" || entry.type === void 0)) {
|
|
3817
|
-
|
|
4457
|
+
const unresolved = /* @__PURE__ */ new Set();
|
|
4458
|
+
const sub = (value) => {
|
|
4459
|
+
if (!env) return value;
|
|
4460
|
+
const r = expandTemplateVars(value, env);
|
|
4461
|
+
for (const name of r.unresolved) unresolved.add(name);
|
|
4462
|
+
return r.value;
|
|
4463
|
+
};
|
|
4464
|
+
const url = sub(entry.url);
|
|
4465
|
+
let headers;
|
|
4466
|
+
if (entry.headers) {
|
|
4467
|
+
headers = {};
|
|
4468
|
+
for (const [k, v] of Object.entries(entry.headers)) headers[k] = sub(v);
|
|
4469
|
+
}
|
|
4470
|
+
return { url, ...headers ? { headers } : {}, unresolved: [...unresolved] };
|
|
3818
4471
|
}
|
|
3819
4472
|
return null;
|
|
3820
4473
|
} catch {
|
|
@@ -3823,15 +4476,76 @@ function readMcpHttpServerConfig(projectDir, serverKey) {
|
|
|
3823
4476
|
}
|
|
3824
4477
|
async function runAgentConnectivityProbes(agent, integrations, projectDir) {
|
|
3825
4478
|
if (integrations.length === 0) return;
|
|
4479
|
+
const probeEnv = { ...process.env };
|
|
4480
|
+
try {
|
|
4481
|
+
const envIntPath = join8(projectDir, ".env.integrations");
|
|
4482
|
+
if (existsSync5(envIntPath)) {
|
|
4483
|
+
Object.assign(probeEnv, parseEnvIntegrations(readFileSync9(envIntPath, "utf-8")));
|
|
4484
|
+
}
|
|
4485
|
+
} catch {
|
|
4486
|
+
}
|
|
3826
4487
|
const probeDeps = {
|
|
3827
4488
|
fetchImpl: fetch,
|
|
3828
|
-
runCli: (binary, args) => runCliProbe(binary, args),
|
|
4489
|
+
runCli: (binary, args) => runCliProbe(binary, args, { env: probeEnv }),
|
|
3829
4490
|
mcpProbe: async (target) => {
|
|
3830
|
-
const cfg = readMcpHttpServerConfig(projectDir, target.serverKey);
|
|
4491
|
+
const cfg = readMcpHttpServerConfig(projectDir, target.serverKey, probeEnv);
|
|
3831
4492
|
if (!cfg) {
|
|
3832
4493
|
return { status: "transient_error", message: `MCP server '${target.serverKey}' not resolvable from .mcp.json` };
|
|
3833
4494
|
}
|
|
4495
|
+
if (cfg.unresolved.length > 0) {
|
|
4496
|
+
return {
|
|
4497
|
+
status: "transient_error",
|
|
4498
|
+
message: `MCP '${target.serverKey}' auth unresolved: ${cfg.unresolved.join(", ")}`
|
|
4499
|
+
};
|
|
4500
|
+
}
|
|
3834
4501
|
return probeMcpHttp(cfg);
|
|
4502
|
+
},
|
|
4503
|
+
// ENG-6139: connected-account binding check for managed (Composio) toolkits.
|
|
4504
|
+
// The MCP handshake reads green on a dead/mis-bound account, so the managed
|
|
4505
|
+
// probe also verifies the account is ACTIVE + bound to the entity the agent
|
|
4506
|
+
// queries with. Inputs come from the agent's OWN wired MCP server: the
|
|
4507
|
+
// `x-api-key` header and the `user_id` query param (the agent already
|
|
4508
|
+
// authenticates with these), plus the recorded connected_account_id.
|
|
4509
|
+
composioProbe: async (definitionId, credentials) => {
|
|
4510
|
+
const serverKey = definitionId.replace(/[^a-z0-9]/gi, "_").toLowerCase();
|
|
4511
|
+
const cfg = readMcpHttpServerConfig(projectDir, serverKey, probeEnv);
|
|
4512
|
+
if (!cfg) {
|
|
4513
|
+
return { status: "transient_error", message: `MCP server '${serverKey}' not resolvable from .mcp.json` };
|
|
4514
|
+
}
|
|
4515
|
+
if (cfg.unresolved.length > 0) {
|
|
4516
|
+
return {
|
|
4517
|
+
status: "transient_error",
|
|
4518
|
+
message: `MCP '${serverKey}' auth unresolved: ${cfg.unresolved.join(", ")}`
|
|
4519
|
+
};
|
|
4520
|
+
}
|
|
4521
|
+
const apiKey = cfg.headers?.["x-api-key"] ?? cfg.headers?.["X-API-Key"] ?? "";
|
|
4522
|
+
let expectedUserId = "";
|
|
4523
|
+
let serverId;
|
|
4524
|
+
try {
|
|
4525
|
+
const u = new URL(cfg.url);
|
|
4526
|
+
expectedUserId = u.searchParams.get("user_id") ?? "";
|
|
4527
|
+
const m = u.pathname.match(/\/v3\/mcp\/([^/]+)\/mcp/);
|
|
4528
|
+
serverId = m?.[1] ? decodeURIComponent(m[1]) : void 0;
|
|
4529
|
+
} catch {
|
|
4530
|
+
expectedUserId = "";
|
|
4531
|
+
}
|
|
4532
|
+
const connectedAccountId = typeof credentials?.["connected_account_id"] === "string" ? credentials["connected_account_id"] : "";
|
|
4533
|
+
return probeComposioAccount({ connectedAccountId, apiKey, expectedUserId, serverId });
|
|
4534
|
+
},
|
|
4535
|
+
// ENG-6157 (Phase 2): the live tool-call leg. Uses the agent's OWN wired MCP
|
|
4536
|
+
// URL + headers (the exact path tool calls take), so a broken auth_config
|
|
4537
|
+
// linkage surfaces as a real `No connected account found` instead of a
|
|
4538
|
+
// green handshake. Skips (`null`) when no safe read-only tool is callable.
|
|
4539
|
+
composioToolCallProbe: async (target) => {
|
|
4540
|
+
const cfg = readMcpHttpServerConfig(projectDir, target.serverKey, probeEnv);
|
|
4541
|
+
if (!cfg) return null;
|
|
4542
|
+
if (cfg.unresolved.length > 0) return null;
|
|
4543
|
+
return probeComposioMcpToolCall({
|
|
4544
|
+
url: cfg.url,
|
|
4545
|
+
headers: cfg.headers,
|
|
4546
|
+
toolName: target.toolName,
|
|
4547
|
+
toolArgs: target.toolArgs
|
|
4548
|
+
});
|
|
3835
4549
|
}
|
|
3836
4550
|
};
|
|
3837
4551
|
const intervalSec = Number(process.env.AGT_CONNECTIVITY_PROBE_INTERVAL_SECONDS) || 3600;
|
|
@@ -3850,7 +4564,9 @@ async function runAgentConnectivityProbes(agent, integrations, projectDir) {
|
|
|
3850
4564
|
// (definition_id === toolkit_id, sanitised: non-alnum → '_', lowercased —
|
|
3851
4565
|
// e.g. 'composio_outlook'). The executor looks this up in .mcp.json; an
|
|
3852
4566
|
// unresolvable key degrades to transient_error, never a false 'down'.
|
|
3853
|
-
mcp_server_key: i.source_type === "managed" || i.source_type === "mcp_server" ? i.definition_id.replace(/[^a-z0-9]/gi, "_").toLowerCase() : void 0
|
|
4567
|
+
mcp_server_key: i.source_type === "managed" || i.source_type === "mcp_server" ? i.definition_id.replace(/[^a-z0-9]/gi, "_").toLowerCase() : void 0,
|
|
4568
|
+
// ENG-6242: forward the prescribed connectivity-test tool to the executor.
|
|
4569
|
+
connectivity_test: i.connectivity_test ?? null
|
|
3854
4570
|
})),
|
|
3855
4571
|
{ probeDeps, intervalMs: intervalSec * 1e3 }
|
|
3856
4572
|
);
|
|
@@ -3912,6 +4628,7 @@ function checkMcpConfigDriftAndScheduleRestart(codeName, projectDir) {
|
|
|
3912
4628
|
const removed = decision.addedOrRemoved.filter((k) => !currentKeys.has(k));
|
|
3913
4629
|
quarantineOnlyDrift = added.length === 0 && removed.length > 0 && removed.every((k) => quarantined.has(k));
|
|
3914
4630
|
}
|
|
4631
|
+
const flapSuppressed = decision.restart && !quarantineOnlyDrift && mcpFlapDampener.isFlapping(codeName, FLAP_CHANNEL_MANAGED_MCP);
|
|
3915
4632
|
if (decision.membershipUnknown) {
|
|
3916
4633
|
clearPresenceReaperState(codeName);
|
|
3917
4634
|
log(
|
|
@@ -3921,6 +4638,10 @@ function checkMcpConfigDriftAndScheduleRestart(codeName, projectDir) {
|
|
|
3921
4638
|
log(
|
|
3922
4639
|
`[channel-quarantine] .mcp.json drift for '${codeName}' is quarantined-channel removal only [${decision.addedOrRemoved.join(", ")}] \u2014 adopting new baseline WITHOUT restart (0-restart-on-removal, ENG-5932)`
|
|
3923
4640
|
);
|
|
4641
|
+
} else if (flapSuppressed) {
|
|
4642
|
+
log(
|
|
4643
|
+
`[mcp-flap-dampener] .mcp.json membership drift for '${codeName}' [${decision.addedOrRemoved.join(", ")}] held \u2014 managed-MCP set is flapping, adopting baseline WITHOUT restart until it settles (ENG-6123)`
|
|
4644
|
+
);
|
|
3924
4645
|
} else if (decision.restart) {
|
|
3925
4646
|
clearPresenceReaperStateForKeys(codeName, new Set(decision.addedOrRemoved));
|
|
3926
4647
|
log(
|
|
@@ -3931,10 +4652,11 @@ function checkMcpConfigDriftAndScheduleRestart(codeName, projectDir) {
|
|
|
3931
4652
|
`[hot-reload] .mcp.json value-only drift for '${codeName}' \u2014 preserving presence-reaper state and NOT restarting (ENG-5285/ENG-5537)`
|
|
3932
4653
|
);
|
|
3933
4654
|
}
|
|
3934
|
-
if (decision.restart && !quarantineOnlyDrift) {
|
|
4655
|
+
if (decision.restart && !quarantineOnlyDrift && !flapSuppressed) {
|
|
3935
4656
|
scheduleSessionRestart(codeName, 0, ".mcp.json content change (ENG-4897)");
|
|
3936
4657
|
runningMcpHashes.delete(codeName);
|
|
3937
4658
|
runningMcpServerKeys.delete(codeName);
|
|
4659
|
+
} else if (flapSuppressed) {
|
|
3938
4660
|
} else {
|
|
3939
4661
|
runningMcpHashes.set(codeName, action.current);
|
|
3940
4662
|
if (currentKeys) runningMcpServerKeys.set(codeName, currentKeys);
|
|
@@ -3996,7 +4718,7 @@ function __setAgentChannelTokensForTest(codeName, tokens) {
|
|
|
3996
4718
|
}
|
|
3997
4719
|
var activeChannels = /* @__PURE__ */ new Map();
|
|
3998
4720
|
var gatewaysStartedThisCycle = /* @__PURE__ */ new Set();
|
|
3999
|
-
var
|
|
4721
|
+
var state6 = {
|
|
4000
4722
|
pid: process.pid,
|
|
4001
4723
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4002
4724
|
lastPollAt: null,
|
|
@@ -4029,6 +4751,8 @@ function clearAgentCaches(agentId, codeName) {
|
|
|
4029
4751
|
for (const key of taskDisplayInfo.keys()) {
|
|
4030
4752
|
if (key.startsWith(`${codeName}:`)) taskDisplayInfo.delete(key);
|
|
4031
4753
|
}
|
|
4754
|
+
mcpFlapDampener.forget(codeName);
|
|
4755
|
+
deferLogThrottle.delete(codeName);
|
|
4032
4756
|
if (channelCacheMutated) saveChannelHashCache2();
|
|
4033
4757
|
}
|
|
4034
4758
|
var cachedFrameworkVersion = null;
|
|
@@ -4036,7 +4760,7 @@ var cachedMaintenanceWindow = null;
|
|
|
4036
4760
|
var lastVersionCheckAt = 0;
|
|
4037
4761
|
var VERSION_CHECK_INTERVAL_MS = 5 * 60 * 1e3;
|
|
4038
4762
|
var lastResponsivenessProbeAt = 0;
|
|
4039
|
-
var agtCliVersion = true ? "0.27.9-test.
|
|
4763
|
+
var agtCliVersion = true ? "0.27.9-test.14" : "dev";
|
|
4040
4764
|
function resolveBrewPath(execFileSync4) {
|
|
4041
4765
|
try {
|
|
4042
4766
|
const out = execFileSync4("which", ["brew"], { timeout: 5e3 }).toString().trim();
|
|
@@ -4688,6 +5412,10 @@ async function runEvalClaude(prompt, model) {
|
|
|
4688
5412
|
});
|
|
4689
5413
|
return stdout;
|
|
4690
5414
|
}
|
|
5415
|
+
function memoryExtractionEnabled() {
|
|
5416
|
+
const v = (process.env["AGT_MEMORY_EXTRACTION_ENABLED"] ?? "").trim().toLowerCase();
|
|
5417
|
+
return v === "true" || v === "1" || v === "yes";
|
|
5418
|
+
}
|
|
4691
5419
|
var conversationEvalBackend = null;
|
|
4692
5420
|
function resolveConversationEvalBackend() {
|
|
4693
5421
|
if (conversationEvalBackend) return conversationEvalBackend;
|
|
@@ -5014,10 +5742,10 @@ function isGatewayHung(codeName) {
|
|
|
5014
5742
|
if (!Array.isArray(jobs)) return false;
|
|
5015
5743
|
const now = Date.now();
|
|
5016
5744
|
for (const job of jobs) {
|
|
5017
|
-
const
|
|
5018
|
-
if (!
|
|
5019
|
-
const runStartedAt =
|
|
5020
|
-
const isRunning =
|
|
5745
|
+
const state7 = job.state;
|
|
5746
|
+
if (!state7) continue;
|
|
5747
|
+
const runStartedAt = state7.runStartedAtMs;
|
|
5748
|
+
const isRunning = state7.status === "running" || state7.running === true;
|
|
5021
5749
|
if (isRunning && runStartedAt && now - runStartedAt > GATEWAY_HUNG_TIMEOUT_MS) {
|
|
5022
5750
|
return true;
|
|
5023
5751
|
}
|
|
@@ -5167,8 +5895,9 @@ async function pollCycle() {
|
|
|
5167
5895
|
await processRestartFlags({
|
|
5168
5896
|
log,
|
|
5169
5897
|
resolveFramework: (codeName) => agentFrameworkCache.get(codeName) ?? null,
|
|
5170
|
-
stopSession: (codeName) => {
|
|
5171
|
-
|
|
5898
|
+
stopSession: (codeName, flag) => {
|
|
5899
|
+
const breakerReason = flag.source === "agent" ? "agent-requested" : "channel-restart-flag";
|
|
5900
|
+
stopPersistentSessionAndForgetMcpBaseline(codeName, breakerReason);
|
|
5172
5901
|
agentState.persistentSessionAgents.delete(codeName);
|
|
5173
5902
|
claudeAuthTupleBySession.delete(codeName);
|
|
5174
5903
|
},
|
|
@@ -5218,7 +5947,7 @@ async function pollCycle() {
|
|
|
5218
5947
|
const now = Date.now();
|
|
5219
5948
|
if (now - lastVersionCheckAt > VERSION_CHECK_INTERVAL_MS) {
|
|
5220
5949
|
try {
|
|
5221
|
-
const firstAgent =
|
|
5950
|
+
const firstAgent = state6.agents[0];
|
|
5222
5951
|
const versionAdapter = firstAgent ? resolveAgentFramework(firstAgent.codeName) : getFramework("openclaw");
|
|
5223
5952
|
if (versionAdapter.getVersion) {
|
|
5224
5953
|
cachedFrameworkVersion = await versionAdapter.getVersion();
|
|
@@ -5229,7 +5958,7 @@ async function pollCycle() {
|
|
|
5229
5958
|
}
|
|
5230
5959
|
try {
|
|
5231
5960
|
const { detectHostSecurity } = await import("../host-security-6PDFG7F5.js");
|
|
5232
|
-
const { collectDiagnostics } = await import("../persistent-session-
|
|
5961
|
+
const { collectDiagnostics } = await import("../persistent-session-35PWSTLO.js");
|
|
5233
5962
|
const diagCodeNames = [...agentState.persistentSessionAgents];
|
|
5234
5963
|
const agentDiagnostics = diagCodeNames.length > 0 ? collectDiagnostics(diagCodeNames) : void 0;
|
|
5235
5964
|
let tailscaleHostname;
|
|
@@ -5262,7 +5991,7 @@ async function pollCycle() {
|
|
|
5262
5991
|
const errId = createHash3("sha256").update(errText).digest("hex").slice(0, 12);
|
|
5263
5992
|
log(`Claude auth detection failed (error_id=${errId})`);
|
|
5264
5993
|
}
|
|
5265
|
-
const hostHasClaudeCode =
|
|
5994
|
+
const hostHasClaudeCode = state6.agents.some(
|
|
5266
5995
|
(a) => agentFrameworkCache.get(a.codeName) === "claude-code"
|
|
5267
5996
|
);
|
|
5268
5997
|
if (hostHasClaudeCode) {
|
|
@@ -5316,23 +6045,30 @@ async function pollCycle() {
|
|
|
5316
6045
|
const {
|
|
5317
6046
|
collectResponsivenessProbes,
|
|
5318
6047
|
getResponsivenessIntervalMs
|
|
5319
|
-
} = await import("../responsiveness-probe-
|
|
6048
|
+
} = await import("../responsiveness-probe-MA4M2QM4.js");
|
|
5320
6049
|
const probeIntervalMs = getResponsivenessIntervalMs();
|
|
5321
6050
|
if (now - lastResponsivenessProbeAt > probeIntervalMs) {
|
|
5322
6051
|
const probeCodeNames = [...agentState.persistentSessionAgents];
|
|
5323
6052
|
if (probeCodeNames.length > 0) {
|
|
6053
|
+
const { takeAcpxExecFailureCount, creditAcpxExecFailureCount } = await import("../persistent-session-35PWSTLO.js");
|
|
5324
6054
|
const drainedGiveUps = /* @__PURE__ */ new Map();
|
|
6055
|
+
const drainedAcpxFailures = /* @__PURE__ */ new Map();
|
|
5325
6056
|
const probes = collectResponsivenessProbes(probeCodeNames).map((p) => {
|
|
5326
6057
|
const giveUps = takeWatchdogGiveUpCount(p.code_name);
|
|
5327
6058
|
if (giveUps > 0) drainedGiveUps.set(p.code_name, giveUps);
|
|
5328
|
-
|
|
6059
|
+
const acpxFailures = takeAcpxExecFailureCount(p.code_name);
|
|
6060
|
+
if (acpxFailures > 0) drainedAcpxFailures.set(p.code_name, acpxFailures);
|
|
6061
|
+
return { ...p, input_stuck_give_ups: giveUps, acpx_exec_failures: acpxFailures };
|
|
5329
6062
|
});
|
|
5330
6063
|
if (probes.length > 0) {
|
|
5331
6064
|
void api.post("/host/responsiveness-probe", { host_id: hostId, probes }).catch((err) => {
|
|
5332
6065
|
for (const [codeName, count] of drainedGiveUps) {
|
|
5333
6066
|
creditWatchdogGiveUpCount(codeName, count);
|
|
5334
6067
|
}
|
|
5335
|
-
|
|
6068
|
+
for (const [codeName, count] of drainedAcpxFailures) {
|
|
6069
|
+
creditAcpxExecFailureCount(codeName, count);
|
|
6070
|
+
}
|
|
6071
|
+
log(`[responsiveness-probe] post failed (give-up + acpx-failure counts re-credited): ${err.message}`);
|
|
5336
6072
|
});
|
|
5337
6073
|
}
|
|
5338
6074
|
}
|
|
@@ -5341,6 +6077,100 @@ async function pollCycle() {
|
|
|
5341
6077
|
} catch (err) {
|
|
5342
6078
|
log(`[responsiveness-probe] collection failed: ${err.message}`);
|
|
5343
6079
|
}
|
|
6080
|
+
try {
|
|
6081
|
+
const wedgeConfig = resolveWedgeConfig();
|
|
6082
|
+
if (wedgeConfig.mode !== "off") {
|
|
6083
|
+
const {
|
|
6084
|
+
collectResponsivenessProbes,
|
|
6085
|
+
livePendingInboundOldestAgeSeconds,
|
|
6086
|
+
deadLetterPendingInbound
|
|
6087
|
+
} = await import("../responsiveness-probe-MA4M2QM4.js");
|
|
6088
|
+
const { transcriptActivityAgeSeconds } = await import("../daily-session-PNQX5URX.js");
|
|
6089
|
+
const { getProjectDir: wedgeProjectDir } = await import("../claude-scheduler-FATCLHDM.js");
|
|
6090
|
+
const wedgeNow = /* @__PURE__ */ new Date();
|
|
6091
|
+
const liveAgents = agentState.persistentSessionAgents;
|
|
6092
|
+
for (const tracked of consecutiveWedgeCycles.keys()) {
|
|
6093
|
+
if (!liveAgents.has(tracked)) consecutiveWedgeCycles.delete(tracked);
|
|
6094
|
+
}
|
|
6095
|
+
for (const tracked of wedgeRestartsByCard.keys()) {
|
|
6096
|
+
if (!liveAgents.has(tracked)) wedgeRestartsByCard.delete(tracked);
|
|
6097
|
+
}
|
|
6098
|
+
const poisonConfig = resolvePoisonCardConfig();
|
|
6099
|
+
for (const probe of collectResponsivenessProbes([...liveAgents])) {
|
|
6100
|
+
const codeName = probe.code_name;
|
|
6101
|
+
if (!isSessionHealthy(codeName)) {
|
|
6102
|
+
consecutiveWedgeCycles.delete(codeName);
|
|
6103
|
+
continue;
|
|
6104
|
+
}
|
|
6105
|
+
const wedgeSession = getSessionState(codeName);
|
|
6106
|
+
const sessionStartMs = wedgeSession?.startedAt ?? null;
|
|
6107
|
+
const transcriptAge = transcriptActivityAgeSeconds(
|
|
6108
|
+
wedgeProjectDir(codeName),
|
|
6109
|
+
wedgeSession?.currentSessionId ?? null,
|
|
6110
|
+
wedgeNow
|
|
6111
|
+
);
|
|
6112
|
+
const signals = {
|
|
6113
|
+
paneActivityAgeSeconds: probe.pane_activity_age_seconds,
|
|
6114
|
+
transcriptActivityAgeSeconds: transcriptAge,
|
|
6115
|
+
pendingInboundOldestAgeSeconds: livePendingInboundOldestAgeSeconds(
|
|
6116
|
+
codeName,
|
|
6117
|
+
sessionStartMs,
|
|
6118
|
+
wedgeNow
|
|
6119
|
+
)
|
|
6120
|
+
};
|
|
6121
|
+
if (!isWedgeCandidateCycle(signals, wedgeConfig)) {
|
|
6122
|
+
consecutiveWedgeCycles.delete(codeName);
|
|
6123
|
+
continue;
|
|
6124
|
+
}
|
|
6125
|
+
const streak = (consecutiveWedgeCycles.get(codeName) ?? 0) + 1;
|
|
6126
|
+
consecutiveWedgeCycles.set(codeName, streak);
|
|
6127
|
+
if (decideWedgeRestart({ ...signals, consecutiveWedgeCycles: streak }, wedgeConfig) !== "wedged") {
|
|
6128
|
+
continue;
|
|
6129
|
+
}
|
|
6130
|
+
const transcriptAgeLabel = transcriptAge === null ? "n/a" : `${transcriptAge}s`;
|
|
6131
|
+
const detail = `agent=${codeName} paneAge=${signals.paneActivityAgeSeconds}s transcriptAge=${transcriptAgeLabel} inboundAge=${signals.pendingInboundOldestAgeSeconds}s cycles=${streak}`;
|
|
6132
|
+
if (wedgeConfig.mode === "shadow") {
|
|
6133
|
+
if (streak === wedgeConfig.minCycles) {
|
|
6134
|
+
log(`[wedge] SHADOW would force-fresh respawn ${detail}`);
|
|
6135
|
+
}
|
|
6136
|
+
continue;
|
|
6137
|
+
}
|
|
6138
|
+
try {
|
|
6139
|
+
const newId = rotateSessionForWedge(codeName);
|
|
6140
|
+
try {
|
|
6141
|
+
const { execSync: es } = await import("child_process");
|
|
6142
|
+
es(`tmux kill-session -t agt-${codeName} 2>/dev/null`, { stdio: "ignore" });
|
|
6143
|
+
} catch {
|
|
6144
|
+
}
|
|
6145
|
+
stopPersistentSessionAndForgetMcpBaseline(codeName);
|
|
6146
|
+
consecutiveWedgeCycles.delete(codeName);
|
|
6147
|
+
const deadLettered = deadLetterPendingInbound(codeName, wedgeNow);
|
|
6148
|
+
const deadNote = deadLettered > 0 ? `, ${deadLettered} stale inbound dead-lettered` : "";
|
|
6149
|
+
log(
|
|
6150
|
+
`[wedge] forced fresh respawn ${detail} \u2192 new session ${newId} (transcript preserved${deadNote})`
|
|
6151
|
+
);
|
|
6152
|
+
const inProgressCardIds = (kanbanBoardCache.get(codeName) ?? []).filter((item) => item.status === "in_progress" && isHybridActionable(item)).map((item) => item.id);
|
|
6153
|
+
const cardStates = wedgeRestartsByCard.get(codeName) ?? /* @__PURE__ */ new Map();
|
|
6154
|
+
const { next, newlyPoisoned } = recordWedgeForCards(
|
|
6155
|
+
cardStates,
|
|
6156
|
+
inProgressCardIds,
|
|
6157
|
+
wedgeNow.getTime(),
|
|
6158
|
+
poisonConfig
|
|
6159
|
+
);
|
|
6160
|
+
wedgeRestartsByCard.set(codeName, next);
|
|
6161
|
+
for (const cardId of newlyPoisoned) {
|
|
6162
|
+
log(
|
|
6163
|
+
`[wedge] poison-card suspected agent=${codeName} card=${cardId} consecutiveWedgeRespawns=${poisonConfig.threshold} \u2014 suppressing re-nudge (falls to stale-auto-fail). See ENG-6171.`
|
|
6164
|
+
);
|
|
6165
|
+
}
|
|
6166
|
+
} catch (err) {
|
|
6167
|
+
log(`[wedge] force-fresh respawn failed for ${codeName}: ${err.message}`);
|
|
6168
|
+
}
|
|
6169
|
+
}
|
|
6170
|
+
}
|
|
6171
|
+
} catch (err) {
|
|
6172
|
+
log(`[wedge] detection failed: ${err.message}`);
|
|
6173
|
+
}
|
|
5344
6174
|
try {
|
|
5345
6175
|
const { scrapeMcpFailedBannerCount } = await import("../pane-mcp-banner-scraper-JA437JIB.js");
|
|
5346
6176
|
const observations = [];
|
|
@@ -5384,7 +6214,7 @@ async function pollCycle() {
|
|
|
5384
6214
|
for (const agent of agents) {
|
|
5385
6215
|
const requested = agent.restart_requested_at ?? null;
|
|
5386
6216
|
if (!requested) continue;
|
|
5387
|
-
const prev =
|
|
6217
|
+
const prev = state6.agents.find((a) => a.agentId === agent.agent_id);
|
|
5388
6218
|
const lastProcessed = prev?.lastRestartProcessedAt ?? null;
|
|
5389
6219
|
if (lastProcessed && Date.parse(lastProcessed) >= Date.parse(requested)) continue;
|
|
5390
6220
|
log(`[restart] Dashboard requested restart for '${agent.code_name}' at ${requested}`);
|
|
@@ -5407,7 +6237,7 @@ async function pollCycle() {
|
|
|
5407
6237
|
await processAgent(agent, agentStates);
|
|
5408
6238
|
} catch (err) {
|
|
5409
6239
|
log(`Error processing agent '${agent.code_name}': ${err.message}`);
|
|
5410
|
-
const existing =
|
|
6240
|
+
const existing = state6.agents.find((a) => a.agentId === agent.agent_id);
|
|
5411
6241
|
if (existing) {
|
|
5412
6242
|
agentStates.push(existing);
|
|
5413
6243
|
} else {
|
|
@@ -5433,12 +6263,12 @@ async function pollCycle() {
|
|
|
5433
6263
|
void maybeReportActivityCache({ api, log });
|
|
5434
6264
|
const restartAckStateChanged = applyRestartAcks({
|
|
5435
6265
|
agentStates,
|
|
5436
|
-
priorAgents:
|
|
6266
|
+
priorAgents: state6.agents,
|
|
5437
6267
|
restartAcks
|
|
5438
6268
|
});
|
|
5439
6269
|
if (restartAckStateChanged) {
|
|
5440
6270
|
try {
|
|
5441
|
-
const ackedState = { ...
|
|
6271
|
+
const ackedState = { ...state6, agents: agentStates };
|
|
5442
6272
|
atomicWriteFileSync(getStateFile(), JSON.stringify(ackedState, null, 2));
|
|
5443
6273
|
} catch (err) {
|
|
5444
6274
|
log(`[restart] failed to persist ack immediately: ${err.message}`);
|
|
@@ -5456,7 +6286,7 @@ async function pollCycle() {
|
|
|
5456
6286
|
} catch {
|
|
5457
6287
|
}
|
|
5458
6288
|
const currentIds = new Set(agents.map((a) => a.agent_id));
|
|
5459
|
-
for (const prev of
|
|
6289
|
+
for (const prev of state6.agents) {
|
|
5460
6290
|
if (!currentIds.has(prev.agentId)) {
|
|
5461
6291
|
log(`Agent '${prev.codeName}' removed from host (deleted or unassigned)`);
|
|
5462
6292
|
const adapter = resolveAgentFramework(prev.codeName);
|
|
@@ -5597,23 +6427,26 @@ async function pollCycle() {
|
|
|
5597
6427
|
}
|
|
5598
6428
|
} catch {
|
|
5599
6429
|
}
|
|
5600
|
-
|
|
5601
|
-
...
|
|
6430
|
+
state6 = {
|
|
6431
|
+
...state6,
|
|
5602
6432
|
lastPollAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5603
|
-
pollCount:
|
|
6433
|
+
pollCount: state6.pollCount + 1,
|
|
5604
6434
|
agents: agentStates,
|
|
5605
6435
|
// ENG-5441: serialise trip state on every poll so manager restarts
|
|
5606
6436
|
// never silently clear a tripped breaker. Cheap — only tripped
|
|
5607
6437
|
// entries are stored, and the typical state is zero entries.
|
|
5608
|
-
circuitBreakerTrips: restartBreaker.serialize()
|
|
6438
|
+
circuitBreakerTrips: restartBreaker.serialize(),
|
|
6439
|
+
// ENG-6088: spent auto-resume credits survive manager restarts too.
|
|
6440
|
+
circuitBreakerAutoResumes: Object.fromEntries(autoResumeMarkers.entries())
|
|
5609
6441
|
};
|
|
5610
6442
|
if (consecutivePollFailures > 0) {
|
|
5611
6443
|
log(`[poll-backoff] recovered after ${consecutivePollFailures} failure(s), resuming normal interval`);
|
|
5612
6444
|
consecutivePollFailures = 0;
|
|
5613
6445
|
}
|
|
5614
|
-
|
|
6446
|
+
verifyPendingRestarts(Date.now());
|
|
6447
|
+
send({ type: "state-update", state: state6 });
|
|
5615
6448
|
} catch (err) {
|
|
5616
|
-
|
|
6449
|
+
state6.errorCount++;
|
|
5617
6450
|
const message = err.message;
|
|
5618
6451
|
log(`Poll error: ${message}`);
|
|
5619
6452
|
send({ type: "error", message });
|
|
@@ -5664,6 +6497,15 @@ async function processAgent(agent, agentStates) {
|
|
|
5664
6497
|
agentId: agent.agent_id,
|
|
5665
6498
|
log
|
|
5666
6499
|
});
|
|
6500
|
+
if (memoryExtractionEnabled()) {
|
|
6501
|
+
void maybeExtractMemories({
|
|
6502
|
+
api,
|
|
6503
|
+
backend: resolveConversationEvalBackend(),
|
|
6504
|
+
codeName: agent.code_name,
|
|
6505
|
+
agentId: agent.agent_id,
|
|
6506
|
+
log
|
|
6507
|
+
});
|
|
6508
|
+
}
|
|
5667
6509
|
}
|
|
5668
6510
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5669
6511
|
const adapter = resolveAgentFramework(agent.code_name);
|
|
@@ -5672,6 +6514,16 @@ async function processAgent(agent, agentStates) {
|
|
|
5672
6514
|
if (previousKnownStatus !== agent.status) {
|
|
5673
6515
|
log(`Agent '${agent.code_name}' is ${agent.status}, skipping provisioning`);
|
|
5674
6516
|
}
|
|
6517
|
+
if (agent.kill_switch) {
|
|
6518
|
+
killPausedCodeNames.add(agent.code_name);
|
|
6519
|
+
if (previousKnownStatus !== agent.status) {
|
|
6520
|
+
log(
|
|
6521
|
+
`[kill-switch] '${agent.code_name}' halted by ${agent.kill_switch.scope ?? "unknown"} kill switch` + (agent.kill_switch.reason ? ` (reason: ${agent.kill_switch.reason})` : "") + ` source=${agent.kill_switch.source ?? "unknown"}`
|
|
6522
|
+
);
|
|
6523
|
+
}
|
|
6524
|
+
} else if (killPausedCodeNames.delete(agent.code_name)) {
|
|
6525
|
+
log(`[kill-switch] '${agent.code_name}' kill switch cleared while still ${agent.status} (no resume \u2014 paused for another reason)`);
|
|
6526
|
+
}
|
|
5675
6527
|
await stopGatewayIfRunning(agent.code_name, adapter);
|
|
5676
6528
|
stopPersistentSessionAndForgetMcpBaseline(agent.code_name);
|
|
5677
6529
|
try {
|
|
@@ -5680,6 +6532,9 @@ async function processAgent(agent, agentStates) {
|
|
|
5680
6532
|
log(`Killed tmux session for paused agent '${agent.code_name}'`);
|
|
5681
6533
|
} catch {
|
|
5682
6534
|
}
|
|
6535
|
+
if (agent.status === "paused" && !agent.kill_switch) {
|
|
6536
|
+
maybeAutoResume(agent);
|
|
6537
|
+
}
|
|
5683
6538
|
agentStates.push({
|
|
5684
6539
|
agentId: agent.agent_id,
|
|
5685
6540
|
codeName: agent.code_name,
|
|
@@ -5771,6 +6626,28 @@ async function processAgent(agent, agentStates) {
|
|
|
5771
6626
|
reportedTrips.delete(agent.code_name);
|
|
5772
6627
|
log(`[circuit-breaker] Cleared trip for '${agent.code_name}' on operator resume (paused \u2192 active)`);
|
|
5773
6628
|
}
|
|
6629
|
+
if (previousStatus === "paused" && agent.status === "active") {
|
|
6630
|
+
const marker = autoResumeMarkers.get(agent.code_name);
|
|
6631
|
+
const isSelfResume = marker !== void 0 && Date.now() - marker.autoResumedAt < AUTO_RESUME_SELF_WINDOW_MS;
|
|
6632
|
+
if (!isSelfResume && autoResumeMarkers.delete(agent.code_name)) {
|
|
6633
|
+
state6 = {
|
|
6634
|
+
...state6,
|
|
6635
|
+
circuitBreakerAutoResumes: Object.fromEntries(autoResumeMarkers.entries())
|
|
6636
|
+
};
|
|
6637
|
+
send({ type: "state-update", state: state6 });
|
|
6638
|
+
log(`[auto-resume] Cleared auto-resume marker for '${agent.code_name}' on operator resume \u2014 credit re-armed (ENG-6088)`);
|
|
6639
|
+
}
|
|
6640
|
+
}
|
|
6641
|
+
if (previousStatus === "paused" && agent.status === "active" && killPausedCodeNames.delete(agent.code_name)) {
|
|
6642
|
+
log(`[kill-switch] '${agent.code_name}' kill switch cleared \u2014 resuming session`);
|
|
6643
|
+
void injectMessage(
|
|
6644
|
+
agent.code_name,
|
|
6645
|
+
"system",
|
|
6646
|
+
"The kill switch that halted you has been cleared. Your session is resuming.",
|
|
6647
|
+
{ task_name: "kill-switch-resume" },
|
|
6648
|
+
log
|
|
6649
|
+
).catch(() => false);
|
|
6650
|
+
}
|
|
5774
6651
|
let channelCacheMutated = false;
|
|
5775
6652
|
for (const key of agentState.knownChannelConfigHashes.keys()) {
|
|
5776
6653
|
if (key.startsWith(`${agent.agent_id}:`)) {
|
|
@@ -5788,7 +6665,7 @@ async function processAgent(agent, agentStates) {
|
|
|
5788
6665
|
});
|
|
5789
6666
|
} catch (err) {
|
|
5790
6667
|
log(`Refresh failed for '${agent.code_name}': ${err.message}`);
|
|
5791
|
-
const existing =
|
|
6668
|
+
const existing = state6.agents.find((a) => a.agentId === agent.agent_id);
|
|
5792
6669
|
agentStates.push(existing ?? {
|
|
5793
6670
|
agentId: agent.agent_id,
|
|
5794
6671
|
codeName: agent.code_name,
|
|
@@ -5852,7 +6729,7 @@ async function processAgent(agent, agentStates) {
|
|
|
5852
6729
|
const charterVersion = refreshData.charter.version;
|
|
5853
6730
|
const toolsVersion = refreshData.tools.version;
|
|
5854
6731
|
const known = agentState.knownVersions.get(agent.agent_id);
|
|
5855
|
-
let lastProvisionAt =
|
|
6732
|
+
let lastProvisionAt = state6.agents.find((a) => a.agentId === agent.agent_id)?.lastProvisionAt ?? null;
|
|
5856
6733
|
const quarantinedChannels = channelQuarantineStore().getQuarantinedKeys(agent.code_name);
|
|
5857
6734
|
const currentChannelIds = setWithout(
|
|
5858
6735
|
launchableChannelIds(refreshData.channel_configs),
|
|
@@ -6158,6 +7035,17 @@ async function processAgent(agent, agentStates) {
|
|
|
6158
7035
|
const telegramPeers = channelId === "telegram" ? extractCharterTelegramPeers(refreshData.charter?.raw_content ?? "", gateContext) : void 0;
|
|
6159
7036
|
const slackPeers = channelId === "slack" ? extractCharterSlackPeers(refreshData.charter?.raw_content ?? "", gateContext) : void 0;
|
|
6160
7037
|
const senderPolicyForCall = channelId === "slack" || channelId === "msteams" ? refreshData.sender_policy ?? void 0 : void 0;
|
|
7038
|
+
let agentAvatarUrl;
|
|
7039
|
+
if (channelId === "slack") {
|
|
7040
|
+
const rawAvatarUrl = refreshData.agent.avatar_url ?? void 0;
|
|
7041
|
+
const avatarResolution = resolveAvatarEnvUrl(rawAvatarUrl);
|
|
7042
|
+
agentAvatarUrl = avatarResolution.url ?? void 0;
|
|
7043
|
+
if (avatarResolution.skipReason && avatarResolution.skipReason !== "empty") {
|
|
7044
|
+
log(
|
|
7045
|
+
`WARN: skipped Slack bot avatar for '${agent.code_name}' (reason=${avatarResolution.skipReason}${avatarResolution.bytes !== void 0 ? `, bytes=${avatarResolution.bytes}` : ""}) \u2014 not a hosted URL / exceeds ${MAX_AVATAR_ENV_URL_BYTES}B cap; Slack runs without the custom photo. Upload the avatar to storage and store the hosted URL to restore it (ENG-6245).`
|
|
7046
|
+
);
|
|
7047
|
+
}
|
|
7048
|
+
}
|
|
6161
7049
|
frameworkAdapter.writeChannelCredentials(
|
|
6162
7050
|
agent.code_name,
|
|
6163
7051
|
channelId,
|
|
@@ -6170,7 +7058,8 @@ async function processAgent(agent, agentStates) {
|
|
|
6170
7058
|
telegramPeers,
|
|
6171
7059
|
slackPeers,
|
|
6172
7060
|
agentTimezone,
|
|
6173
|
-
senderPolicy: senderPolicyForCall
|
|
7061
|
+
senderPolicy: senderPolicyForCall,
|
|
7062
|
+
agentAvatarUrl
|
|
6174
7063
|
}
|
|
6175
7064
|
);
|
|
6176
7065
|
agentState.knownChannelConfigHashes.set(cacheKey, configHash);
|
|
@@ -6414,7 +7303,7 @@ async function processAgent(agent, agentStates) {
|
|
|
6414
7303
|
log(`Failed to provision direct-chat channel for '${agent.code_name}': ${err.message}`);
|
|
6415
7304
|
}
|
|
6416
7305
|
}
|
|
6417
|
-
let lastSecretsProvisionAt =
|
|
7306
|
+
let lastSecretsProvisionAt = state6.agents.find((a) => a.agentId === agent.agent_id)?.lastSecretsProvisionAt ?? null;
|
|
6418
7307
|
let secretsHash = agentState.knownSecretsHashes.get(agent.agent_id) ?? null;
|
|
6419
7308
|
try {
|
|
6420
7309
|
const secretsData = await api.post("/host/secrets", { agent_id: agent.agent_id });
|
|
@@ -6477,6 +7366,14 @@ async function processAgent(agent, agentStates) {
|
|
|
6477
7366
|
if (integrations.length > 0) {
|
|
6478
7367
|
const intHash = computeIntegrationsHash(integrations);
|
|
6479
7368
|
const prevIntHash = agentState.knownIntegrationHashes.get(agent.agent_id);
|
|
7369
|
+
const intMembership = membershipSignature(integrations.map((i) => i.definition_id));
|
|
7370
|
+
const intFlap = mcpFlapDampener.record(agent.code_name, FLAP_CHANNEL_INTEGRATIONS, intMembership);
|
|
7371
|
+
if (intFlap.onset) {
|
|
7372
|
+
log(
|
|
7373
|
+
`[mcp-flap-dampener] integration set for '${agent.code_name}' is FLAPPING (${intFlap.transitions} transitions / ${intFlap.distinctStates} states in window) \u2014 suppressing integration-update notices until it settles (ENG-6123)`
|
|
7374
|
+
);
|
|
7375
|
+
recordConfigChurnEvent(agent.agent_id, agent.code_name, FLAP_CHANNEL_INTEGRATIONS, intMembership);
|
|
7376
|
+
}
|
|
6480
7377
|
if (intHash !== prevIntHash) {
|
|
6481
7378
|
const projectDir = join8(homedir4(), ".augmented", agent.code_name, "project");
|
|
6482
7379
|
const envIntPath = join8(projectDir, ".env.integrations");
|
|
@@ -6510,11 +7407,15 @@ async function processAgent(agent, agentStates) {
|
|
|
6510
7407
|
}
|
|
6511
7408
|
const names = integrations.map((i) => i.display_name || i.definition_id).join(", ");
|
|
6512
7409
|
const reapNote = affectedServerKeys.length > 0 ? ` The MCP servers that depend on rotating credentials (${affectedServerKeys.join(", ")}) have been signalled to reconnect.` : "";
|
|
6513
|
-
|
|
6514
|
-
|
|
6515
|
-
}
|
|
6516
|
-
|
|
6517
|
-
|
|
7410
|
+
if (intFlap.flapping) {
|
|
7411
|
+
log(`[mcp-flap-dampener] suppressed integration-update notice for '${agent.code_name}' (set flapping, ENG-6123): ${names}`);
|
|
7412
|
+
} else {
|
|
7413
|
+
injectMessage(agent.code_name, "system", `Your integrations have been refreshed. You have access to: ${names}.${reapNote}`, {
|
|
7414
|
+
task_name: "integration-update"
|
|
7415
|
+
}, log).catch(() => {
|
|
7416
|
+
});
|
|
7417
|
+
log(`[hot-reload] Notified '${agent.code_name}' about integration update: ${names} (reaped ${affectedServerKeys.length} stale MCP server(s))`);
|
|
7418
|
+
}
|
|
6518
7419
|
} catch (err) {
|
|
6519
7420
|
rotationHandled = false;
|
|
6520
7421
|
log(`[hot-reload] Failed to compute / reap affected MCP servers for '${agent.code_name}': ${err.message} \u2014 will retry next tick`);
|
|
@@ -6554,6 +7455,14 @@ async function processAgent(agent, agentStates) {
|
|
|
6554
7455
|
const prevMcpHash = agentState.knownManagedMcpHashes.get(agent.agent_id);
|
|
6555
7456
|
const structureHash = managedMcpStructureHash(desiredEntries);
|
|
6556
7457
|
const prevStructureHash = agentState.knownManagedMcpStructure.get(agent.agent_id);
|
|
7458
|
+
const mcpMembership = membershipSignature(expectedServerIds);
|
|
7459
|
+
const mcpFlap = mcpFlapDampener.record(agent.code_name, FLAP_CHANNEL_MANAGED_MCP, mcpMembership);
|
|
7460
|
+
if (mcpFlap.onset) {
|
|
7461
|
+
log(
|
|
7462
|
+
`[mcp-flap-dampener] managed-MCP set for '${agent.code_name}' is FLAPPING (${mcpFlap.transitions} transitions / ${mcpFlap.distinctStates} states in window) \u2014 suppressing mcp-update notice + restart until it settles (ENG-6123)`
|
|
7463
|
+
);
|
|
7464
|
+
recordConfigChurnEvent(agent.agent_id, agent.code_name, FLAP_CHANNEL_MANAGED_MCP, mcpMembership);
|
|
7465
|
+
}
|
|
6557
7466
|
if (mcpHash !== prevMcpHash) {
|
|
6558
7467
|
for (const e of desiredEntries) {
|
|
6559
7468
|
frameworkAdapter.writeMcpServer(agent.code_name, e.serverId, { url: e.url, headers: e.headers });
|
|
@@ -6592,7 +7501,12 @@ async function processAgent(agent, agentStates) {
|
|
|
6592
7501
|
}
|
|
6593
7502
|
agentState.knownManagedMcpHashes.set(agent.agent_id, mcpHash);
|
|
6594
7503
|
agentState.knownManagedMcpStructure.set(agent.agent_id, structureHash);
|
|
6595
|
-
if (prevStructureHash !== void 0 && structureHash !== prevStructureHash && fwForMcp === "claude-code" && isSessionHealthy(agent.code_name)
|
|
7504
|
+
if (prevStructureHash !== void 0 && structureHash !== prevStructureHash && fwForMcp === "claude-code" && isSessionHealthy(agent.code_name) && // ENG-6123: hold the restart + inject-notice while the managed-MCP
|
|
7505
|
+
// membership is oscillating. The .mcp.json above was still written
|
|
7506
|
+
// (the file converges to the live set); we only withhold the
|
|
7507
|
+
// opus-facing notice + the session reload — the two per-flip burns.
|
|
7508
|
+
// When the set settles, the next structural change restarts once.
|
|
7509
|
+
!mcpFlap.flapping) {
|
|
6596
7510
|
const mcpNames = agentToolkits.map((tk) => tk.toolkit_name).join(", ") || "none (all removed)";
|
|
6597
7511
|
log(`[hot-reload] MCP servers changed for '${agent.code_name}': ${mcpNames} \u2014 restarting session`);
|
|
6598
7512
|
const restartNotice = agentToolkits.length > 0 ? `New MCP tool servers have been configured: ${mcpNames}. Note: MCP servers require a session restart to connect. Your manager will restart your session shortly.` : "Managed MCP tool servers were removed. Your manager will restart your session shortly so the session drops those tools.";
|
|
@@ -6868,11 +7782,22 @@ async function processAgent(agent, agentStates) {
|
|
|
6868
7782
|
}
|
|
6869
7783
|
void reconcileScheduledRuns(agent.code_name, agent.agent_id, hybridBoard);
|
|
6870
7784
|
if (wantsHybridInject) {
|
|
6871
|
-
|
|
6872
|
-
|
|
6873
|
-
|
|
7785
|
+
const poisonCfg = resolvePoisonCardConfig();
|
|
7786
|
+
const existingStates = wedgeRestartsByCard.get(agent.code_name);
|
|
7787
|
+
if (existingStates && existingStates.size > 0) {
|
|
7788
|
+
const liveInProgress = hybridBoard.filter((item) => item.status === "in_progress" && isHybridActionable(item)).map((item) => item.id);
|
|
7789
|
+
const pruned = pruneCardStates(existingStates, liveInProgress, Date.now(), poisonCfg);
|
|
7790
|
+
if (pruned.size > 0) wedgeRestartsByCard.set(agent.code_name, pruned);
|
|
7791
|
+
else wedgeRestartsByCard.delete(agent.code_name);
|
|
7792
|
+
}
|
|
7793
|
+
const cardStates = wedgeRestartsByCard.get(agent.code_name);
|
|
7794
|
+
const actionable = hybridBoard.filter(isHybridActionable);
|
|
7795
|
+
const { allowed, suppressed } = cardStates ? partitionActionableByPoison(actionable, cardStates, poisonCfg) : { allowed: actionable, suppressed: [] };
|
|
7796
|
+
if (allowed.length > 0) {
|
|
7797
|
+
void maybeInjectKanbanCheck(agent.code_name, agent.agent_id, allowed.length);
|
|
6874
7798
|
} else {
|
|
6875
|
-
|
|
7799
|
+
const reason = suppressed.length > 0 ? "kanban inject suppressed \u2014 suspected poison card(s) (ENG-6171)" : "kanban board drained";
|
|
7800
|
+
closeInjectedRunIfOpen(agent.code_name, "completed", reason);
|
|
6876
7801
|
}
|
|
6877
7802
|
}
|
|
6878
7803
|
}
|
|
@@ -6996,6 +7921,12 @@ async function processAgent(agent, agentStates) {
|
|
|
6996
7921
|
`[mcp-presence-reaper] failed to record restart event for '${codeName}' (ENG-5286): ${err.message} \u2014 local restart still proceeded; CloudWatch metric will under-count this event`
|
|
6997
7922
|
);
|
|
6998
7923
|
});
|
|
7924
|
+
void reportReaperEvent(
|
|
7925
|
+
agent.agent_id,
|
|
7926
|
+
"mcp-presence-reaper",
|
|
7927
|
+
"restart",
|
|
7928
|
+
givenUpKeys.length > 0 ? `restarted session; MCP server(s) still down after restart budget: ${givenUpKeys.join(", ")}` : `restarted session to recover MCP server(s): ${activeKeys.join(", ")}`
|
|
7929
|
+
);
|
|
6999
7930
|
},
|
|
7000
7931
|
// ENG-5932: classify each declared key for the quarantine path. Reads a
|
|
7001
7932
|
// server-declared criticality field off the .mcp.json entry when
|
|
@@ -7026,6 +7957,12 @@ async function processAgent(agent, agentStates) {
|
|
|
7026
7957
|
log(
|
|
7027
7958
|
`[channel-quarantine] persisted quarantine marker for '${codeName}:${serverKey}' \u2014 will be dropped from .mcp.json on next provisioning poll (ENG-5932)`
|
|
7028
7959
|
);
|
|
7960
|
+
void reportReaperEvent(
|
|
7961
|
+
agent.agent_id,
|
|
7962
|
+
"mcp-presence-reaper",
|
|
7963
|
+
"quarantine",
|
|
7964
|
+
`optional channel '${serverKey}' dead past restart budget + dwell \u2014 quarantined`
|
|
7965
|
+
);
|
|
7029
7966
|
}
|
|
7030
7967
|
}
|
|
7031
7968
|
});
|
|
@@ -7359,19 +8296,19 @@ function clearStaleCronRunState(jobsPath) {
|
|
|
7359
8296
|
let changed = false;
|
|
7360
8297
|
const now = Date.now();
|
|
7361
8298
|
for (const job of jobs) {
|
|
7362
|
-
const
|
|
7363
|
-
if (!
|
|
7364
|
-
const runStartedAt =
|
|
7365
|
-
const isRunning =
|
|
8299
|
+
const state7 = job.state;
|
|
8300
|
+
if (!state7) continue;
|
|
8301
|
+
const runStartedAt = state7.runningAtMs ?? state7.runStartedAtMs;
|
|
8302
|
+
const isRunning = state7.running === true || state7.status === "running";
|
|
7366
8303
|
if (isRunning && runStartedAt && now - runStartedAt > STALE_RUN_TIMEOUT_MS) {
|
|
7367
|
-
|
|
7368
|
-
delete
|
|
7369
|
-
delete
|
|
7370
|
-
delete
|
|
8304
|
+
state7.running = false;
|
|
8305
|
+
delete state7.status;
|
|
8306
|
+
delete state7.runStartedAtMs;
|
|
8307
|
+
delete state7.currentRunId;
|
|
7371
8308
|
changed = true;
|
|
7372
8309
|
log(`Cleared stale running state for cron job '${job.name}' (stuck for ${Math.round((now - runStartedAt) / 6e4)}min)`);
|
|
7373
8310
|
} else if (isRunning && !runStartedAt) {
|
|
7374
|
-
|
|
8311
|
+
state7.running = false;
|
|
7375
8312
|
changed = true;
|
|
7376
8313
|
log(`Cleared stale running state for cron job '${job.name}' (no start timestamp)`);
|
|
7377
8314
|
}
|
|
@@ -7405,6 +8342,26 @@ function cleanupOldFiles(dir, maxAgeDays, ext) {
|
|
|
7405
8342
|
} catch {
|
|
7406
8343
|
}
|
|
7407
8344
|
}
|
|
8345
|
+
function buildSchedulerTaskInput(t) {
|
|
8346
|
+
return {
|
|
8347
|
+
id: t.id,
|
|
8348
|
+
template_id: t.template_id,
|
|
8349
|
+
name: t.name,
|
|
8350
|
+
schedule_kind: t.schedule_kind,
|
|
8351
|
+
schedule_expr: t.schedule_expr ?? null,
|
|
8352
|
+
schedule_every: t.schedule_every ?? null,
|
|
8353
|
+
schedule_at: t.schedule_at ?? null,
|
|
8354
|
+
timezone: t.timezone ?? "UTC",
|
|
8355
|
+
prompt: t.prompt ?? "",
|
|
8356
|
+
session_target: t.session_target ?? "isolated",
|
|
8357
|
+
delivery_mode: t.delivery_mode ?? "none",
|
|
8358
|
+
delivery_policy: t.delivery_policy ?? "always",
|
|
8359
|
+
delivery_channel: t.delivery_channel ?? null,
|
|
8360
|
+
delivery_to: t.delivery_to ?? null,
|
|
8361
|
+
enabled: t.enabled ?? true,
|
|
8362
|
+
triggered_at: t.triggered_at ?? null
|
|
8363
|
+
};
|
|
8364
|
+
}
|
|
7408
8365
|
var inFlightClaudeTasks = /* @__PURE__ */ new Set();
|
|
7409
8366
|
var claudeTaskConcurrency = /* @__PURE__ */ new Map();
|
|
7410
8367
|
var MAX_CLAUDE_CONCURRENCY = 2;
|
|
@@ -7437,33 +8394,17 @@ async function syncAndCheckClaudeScheduler(agent, tasks, boardItems, refreshData
|
|
|
7437
8394
|
const combinedHash = `${stableTasksHash}:${boardHash}:${modelsHash}`;
|
|
7438
8395
|
const prevHash = agentState.knownTasksHashes.get(agent.agent_id);
|
|
7439
8396
|
if (combinedHash !== prevHash) {
|
|
7440
|
-
const taskInputs = tasks.map((t) => (
|
|
7441
|
-
|
|
7442
|
-
|
|
7443
|
-
name: t.name,
|
|
7444
|
-
schedule_kind: t.schedule_kind,
|
|
7445
|
-
schedule_expr: t.schedule_expr ?? null,
|
|
7446
|
-
schedule_every: t.schedule_every ?? null,
|
|
7447
|
-
schedule_at: t.schedule_at ?? null,
|
|
7448
|
-
timezone: t.timezone ?? "UTC",
|
|
7449
|
-
prompt: t.prompt ?? "",
|
|
7450
|
-
session_target: t.session_target ?? "isolated",
|
|
7451
|
-
delivery_mode: t.delivery_mode ?? "none",
|
|
7452
|
-
delivery_channel: t.delivery_channel ?? null,
|
|
7453
|
-
delivery_to: t.delivery_to ?? null,
|
|
7454
|
-
enabled: t.enabled ?? true,
|
|
7455
|
-
triggered_at: t.triggered_at ?? null
|
|
7456
|
-
}));
|
|
7457
|
-
const state7 = syncTasksToScheduler(codeName, agent.agent_id, taskInputs);
|
|
7458
|
-
claudeSchedulerStates.set(codeName, state7);
|
|
8397
|
+
const taskInputs = tasks.map((t) => buildSchedulerTaskInput(t));
|
|
8398
|
+
const state8 = syncTasksToScheduler(codeName, agent.agent_id, taskInputs);
|
|
8399
|
+
claudeSchedulerStates.set(codeName, state8);
|
|
7459
8400
|
agentState.knownTasksHashes.set(agent.agent_id, combinedHash);
|
|
7460
8401
|
log(`[claude-scheduler] Tasks synced for '${codeName}' (${taskInputs.length} task(s))`);
|
|
7461
8402
|
}
|
|
7462
8403
|
if (!claudeSchedulerStates.has(codeName)) {
|
|
7463
8404
|
claudeSchedulerStates.set(codeName, loadSchedulerState(codeName));
|
|
7464
8405
|
}
|
|
7465
|
-
const
|
|
7466
|
-
const ready = getReadyTasks(
|
|
8406
|
+
const state7 = claudeSchedulerStates.get(codeName);
|
|
8407
|
+
const ready = getReadyTasks(state7, inFlightClaudeTasks);
|
|
7467
8408
|
if (ready.length === 0) return;
|
|
7468
8409
|
for (const task of ready) {
|
|
7469
8410
|
if ((claudeTaskConcurrency.get(codeName) ?? 0) >= MAX_CLAUDE_CONCURRENCY) break;
|
|
@@ -7623,8 +8564,8 @@ async function deliverScheduledCardResult(codeName, agentId, cardId) {
|
|
|
7623
8564
|
markScheduledCardDeliveryComplete(cardId);
|
|
7624
8565
|
return "terminal";
|
|
7625
8566
|
}
|
|
7626
|
-
const
|
|
7627
|
-
const task =
|
|
8567
|
+
const state7 = claudeSchedulerStates.get(codeName) ?? loadSchedulerState(codeName);
|
|
8568
|
+
const task = state7.tasks[card.source_ref];
|
|
7628
8569
|
if (!task) {
|
|
7629
8570
|
log(`[scheduled-kanban] delivery: no scheduler task for source_ref=${card.source_ref} on '${codeName}' \u2014 skipping`);
|
|
7630
8571
|
markScheduledCardDeliveryComplete(cardId);
|
|
@@ -7635,6 +8576,31 @@ async function deliverScheduledCardResult(codeName, agentId, cardId) {
|
|
|
7635
8576
|
markScheduledCardDeliveryComplete(cardId);
|
|
7636
8577
|
return "terminal";
|
|
7637
8578
|
}
|
|
8579
|
+
const deliveryPolicy = task.deliveryPolicy ?? "always";
|
|
8580
|
+
if (deliveryPolicy === "never" || deliveryPolicy === "conditional" && card.suppress_delivery !== false) {
|
|
8581
|
+
log(
|
|
8582
|
+
`[scheduled-kanban] Suppressed by delivery_policy=${deliveryPolicy} for card ${cardId} (task '${task.name}') on '${codeName}' \u2014 result recorded on the card only (tombstone)`
|
|
8583
|
+
);
|
|
8584
|
+
if (task.deliveryMode === "announce" && task.deliveryTo) {
|
|
8585
|
+
await reportDeliveryStatus(agentId, task.taskId, {
|
|
8586
|
+
status: "skipped",
|
|
8587
|
+
error_code: "SUPPRESSED_BY_POLICY"
|
|
8588
|
+
});
|
|
8589
|
+
}
|
|
8590
|
+
markScheduledCardDeliveryComplete(cardId);
|
|
8591
|
+
return "delivered";
|
|
8592
|
+
}
|
|
8593
|
+
if (card.suppress_delivery === true) {
|
|
8594
|
+
log(`[scheduled-kanban] Suppressing delivery (structured flag) for card ${cardId} (task '${task.name}') on '${codeName}' \u2014 result recorded on the card only`);
|
|
8595
|
+
if (task.deliveryMode === "announce" && task.deliveryTo) {
|
|
8596
|
+
await reportDeliveryStatus(agentId, task.taskId, {
|
|
8597
|
+
status: "skipped",
|
|
8598
|
+
error_code: "NO_CONTENT"
|
|
8599
|
+
});
|
|
8600
|
+
}
|
|
8601
|
+
markScheduledCardDeliveryComplete(cardId);
|
|
8602
|
+
return "delivered";
|
|
8603
|
+
}
|
|
7638
8604
|
await processClaudeTaskResult(codeName, agentId, task.templateId, card.result ?? "", {
|
|
7639
8605
|
mode: task.deliveryMode,
|
|
7640
8606
|
channel: task.deliveryChannel,
|
|
@@ -7664,6 +8630,12 @@ async function reconcileScheduledRuns(codeName, agentId, board) {
|
|
|
7664
8630
|
}
|
|
7665
8631
|
if (m.size === 0) scheduledRunsByCode.delete(codeName);
|
|
7666
8632
|
}
|
|
8633
|
+
var SCHEDULED_CARD_DELIVERY_CONTRACT = `
|
|
8634
|
+
|
|
8635
|
+
---
|
|
8636
|
+
[delivery contract \u2014 system]
|
|
8637
|
+
The result you write on this card IS the message that will be delivered to the user. Delivery happens automatically from the card \u2014 never send, post, or message it yourself.
|
|
8638
|
+
If \u2014 and ONLY if \u2014 the task above contains explicit opt-out wording the user typed ("DO NOT notify me unless \u2026", "only if X", "stay silent unless \u2026") and that condition is NOT met this run, call kanban_done with suppress_delivery: true \u2014 your result is still recorded on the card but will NOT be messaged to the user. That is how you honor the user's do-not-notify instruction; a "nothing urgent" / "all quiet" result WITHOUT the flag would still be DELIVERED as a message. (Legacy fallback: writing exactly ${SUPPRESS_SENTINEL} alone as the result also suppresses.) If the task asks for a digest/report WITHOUT opt-out wording, a zero-item report is a valid deliverable \u2014 deliver it.`;
|
|
7667
8639
|
async function routeScheduledTaskViaKanban(codeName, agentId, task, prompt) {
|
|
7668
8640
|
const { run_id, kanban_item_id } = await startRun({
|
|
7669
8641
|
agent_id: agentId,
|
|
@@ -7672,7 +8644,7 @@ async function routeScheduledTaskViaKanban(codeName, agentId, task, prompt) {
|
|
|
7672
8644
|
metadata: { template_id: task.templateId, name: task.name, via: "kanban-inject" },
|
|
7673
8645
|
materialize_kanban: {
|
|
7674
8646
|
title: task.name,
|
|
7675
|
-
description: prompt,
|
|
8647
|
+
description: prompt + SCHEDULED_CARD_DELIVERY_CONTRACT,
|
|
7676
8648
|
priority: 2,
|
|
7677
8649
|
initial_status: "todo",
|
|
7678
8650
|
...deriveScheduledTaskNotify(task)
|
|
@@ -7683,9 +8655,11 @@ async function routeScheduledTaskViaKanban(codeName, agentId, task, prompt) {
|
|
|
7683
8655
|
if (run_id) void finishRun(run_id, "failed", { outcomeMessage: "kanban materialise incomplete", completeKanbanItemId: kanban_item_id });
|
|
7684
8656
|
return false;
|
|
7685
8657
|
}
|
|
7686
|
-
const
|
|
7687
|
-
If \u2014 and ONLY if \u2014 the task description contains explicit opt-out wording the user typed (e.g. "DO NOT notify me unless urgent", "only if X", "stay silent unless \u2026") and that condition is NOT met this run,
|
|
7688
|
-
|
|
8658
|
+
const suppressionTeaching = (task.deliveryPolicy ?? "always") === "conditional" ? `THIS task is CONDITIONAL: quiet runs stay SILENT automatically. Write your findings on the card and mark it done \u2014 if nothing met the user's criteria, NOTHING will be sent (no flag needed). If something DID meet the criteria and the user must be told, call kanban_done with suppress_delivery: false \u2014 that explicit assert is the ONLY way this result reaches the user.
|
|
8659
|
+
` : `If \u2014 and ONLY if \u2014 the task description contains explicit opt-out wording the user typed (e.g. "DO NOT notify me unless urgent", "only if X", "stay silent unless \u2026") and that condition is NOT met this run, call kanban_done with suppress_delivery: true \u2014 your result stays on the card but will NOT be messaged to the user. That is how you honor the user's do-not-notify instruction: a "nothing urgent" result without the flag would still be DELIVERED as a message. A report with zero items is only a valid deliverable when the task asks for a digest WITHOUT opt-out wording \u2014 when in doubt, deliver.
|
|
8660
|
+
`;
|
|
8661
|
+
const nudge = `You have a new scheduled task on your kanban board: id=${kanban_item_id} title=${JSON.stringify(task.name)}. Call kanban_move("${kanban_item_id}", "in_progress"), do the work described on the card, then write the finished result onto the card and mark it done. Do NOT send, post, or message the result to anyone yourself \u2014 the result you write on the card IS the message that will be delivered to the user; delivery happens automatically from the card.
|
|
8662
|
+
` + suppressionTeaching + formatRunMarker(run_id);
|
|
7689
8663
|
let injectStatus;
|
|
7690
8664
|
try {
|
|
7691
8665
|
injectStatus = await injectMessageWithStatus(codeName, "task", nudge, { task_name: "scheduled-task" }, log);
|
|
@@ -8108,6 +9082,11 @@ ${truncateForLog(ctx.tail)}` : `; pane_tail_hash=sha256:${createHash3("sha256").
|
|
|
8108
9082
|
} catch (err) {
|
|
8109
9083
|
log(`[persistent-session] Failed to provision orient hook for '${codeName}': ${err.message}`);
|
|
8110
9084
|
}
|
|
9085
|
+
try {
|
|
9086
|
+
provisionAutoKanbanProgressHook(codeName);
|
|
9087
|
+
} catch (err) {
|
|
9088
|
+
log(`[persistent-session] Failed to provision auto-progress hook for '${codeName}': ${err.message}`);
|
|
9089
|
+
}
|
|
8111
9090
|
const sessionRunResult = await startRun({
|
|
8112
9091
|
agent_id: agent.agent_id,
|
|
8113
9092
|
source_type: "system",
|
|
@@ -8146,23 +9125,7 @@ ${truncateForLog(ctx.tail)}` : `; pane_tail_hash=sha256:${createHash3("sha256").
|
|
|
8146
9125
|
const stableTasksHash = createHash3("sha256").update(JSON.stringify(tasks)).digest("hex").slice(0, 16);
|
|
8147
9126
|
const prevHash = agentState.knownTasksHashes.get(agent.agent_id);
|
|
8148
9127
|
if (stableTasksHash !== prevHash) {
|
|
8149
|
-
const taskInputs = tasks.map((t) => (
|
|
8150
|
-
id: t.id,
|
|
8151
|
-
template_id: t.template_id,
|
|
8152
|
-
name: t.name,
|
|
8153
|
-
schedule_kind: t.schedule_kind,
|
|
8154
|
-
schedule_expr: t.schedule_expr ?? null,
|
|
8155
|
-
schedule_every: t.schedule_every ?? null,
|
|
8156
|
-
schedule_at: t.schedule_at ?? null,
|
|
8157
|
-
timezone: t.timezone ?? "UTC",
|
|
8158
|
-
prompt: t.prompt ?? "",
|
|
8159
|
-
session_target: t.session_target ?? "isolated",
|
|
8160
|
-
delivery_mode: t.delivery_mode ?? "none",
|
|
8161
|
-
delivery_channel: t.delivery_channel ?? null,
|
|
8162
|
-
delivery_to: t.delivery_to ?? null,
|
|
8163
|
-
enabled: t.enabled ?? true,
|
|
8164
|
-
triggered_at: t.triggered_at ?? null
|
|
8165
|
-
}));
|
|
9128
|
+
const taskInputs = tasks.map((t) => buildSchedulerTaskInput(t));
|
|
8166
9129
|
const schedulerState = syncTasksToScheduler(codeName, agent.agent_id, taskInputs);
|
|
8167
9130
|
claudeSchedulerStates.set(codeName, schedulerState);
|
|
8168
9131
|
agentState.knownTasksHashes.set(agent.agent_id, stableTasksHash);
|
|
@@ -8170,9 +9133,9 @@ ${truncateForLog(ctx.tail)}` : `; pane_tail_hash=sha256:${createHash3("sha256").
|
|
|
8170
9133
|
} else if (!claudeSchedulerStates.has(codeName)) {
|
|
8171
9134
|
claudeSchedulerStates.set(codeName, loadSchedulerState(codeName));
|
|
8172
9135
|
}
|
|
8173
|
-
const
|
|
8174
|
-
if (
|
|
8175
|
-
const ready = getReadyTasks(
|
|
9136
|
+
const state7 = claudeSchedulerStates.get(codeName);
|
|
9137
|
+
if (state7) {
|
|
9138
|
+
const ready = getReadyTasks(state7, inFlightClaudeTasks);
|
|
8176
9139
|
if (ready.length > 0) {
|
|
8177
9140
|
log(`[persistent-session] ${ready.length} ready task(s) for '${codeName}': ${ready.map((t) => `${t.name}(next=${t.nextFireAt ? new Date(t.nextFireAt).toISOString() : "null"})`).join(", ")}`);
|
|
8178
9141
|
}
|
|
@@ -8581,7 +9544,7 @@ ${escapeXml(msg.content)}
|
|
|
8581
9544
|
return;
|
|
8582
9545
|
}
|
|
8583
9546
|
log(`[direct-chat] One-shot spawn for '${agent.codeName}' (msg=${msg.id}; host in-flight=${directChatSpawnGate.hostInFlight}, queued=${directChatSpawnGate.queuedCount})`);
|
|
8584
|
-
const { getProjectDir: ccProjectDir } = await import("../claude-scheduler-
|
|
9547
|
+
const { getProjectDir: ccProjectDir } = await import("../claude-scheduler-FATCLHDM.js");
|
|
8585
9548
|
const projDir = ccProjectDir(agent.codeName);
|
|
8586
9549
|
const mcpConfigPath = join8(projDir, ".mcp.json");
|
|
8587
9550
|
const serverNames = [];
|
|
@@ -9652,7 +10615,7 @@ async function processClaudePairSessions(agents) {
|
|
|
9652
10615
|
killPairSession,
|
|
9653
10616
|
pairTmuxSession,
|
|
9654
10617
|
finalizeClaudePairOnboarding
|
|
9655
|
-
} = await import("../claude-pair-runtime-
|
|
10618
|
+
} = await import("../claude-pair-runtime-GIUCD7IG.js");
|
|
9656
10619
|
for (const pairId of pendingResp.cancelled_pair_ids ?? []) {
|
|
9657
10620
|
log(`[claude-pair] sweeping orphan tmux session for pair ${pairId.slice(0, 8)}`);
|
|
9658
10621
|
const killed = await killPairSession(pairTmuxSession(pairId));
|
|
@@ -10270,14 +11233,19 @@ function startManager(opts) {
|
|
|
10270
11233
|
const raw = readFileSync9(stateFile, "utf-8");
|
|
10271
11234
|
const parsed = JSON.parse(raw);
|
|
10272
11235
|
if (Array.isArray(parsed.agents)) {
|
|
10273
|
-
|
|
10274
|
-
log(`[startup] rehydrated ${
|
|
11236
|
+
state6.agents = parsed.agents;
|
|
11237
|
+
log(`[startup] rehydrated ${state6.agents.length} agent state(s) from ${stateFile}`);
|
|
10275
11238
|
}
|
|
10276
11239
|
if (parsed.circuitBreakerTrips && typeof parsed.circuitBreakerTrips === "object") {
|
|
10277
11240
|
restartBreaker.hydrate(parsed.circuitBreakerTrips);
|
|
10278
11241
|
const n = Object.keys(parsed.circuitBreakerTrips).length;
|
|
10279
11242
|
if (n > 0) log(`[startup] rehydrated ${n} circuit-breaker trip(s) \u2014 agents will remain paused until manually resumed`);
|
|
10280
11243
|
}
|
|
11244
|
+
if (parsed.circuitBreakerAutoResumes && typeof parsed.circuitBreakerAutoResumes === "object") {
|
|
11245
|
+
autoResumeMarkers = hydrateAutoResumeMarkers(parsed.circuitBreakerAutoResumes);
|
|
11246
|
+
const n = autoResumeMarkers.size;
|
|
11247
|
+
if (n > 0) log(`[startup] rehydrated ${n} auto-resume marker(s) (ENG-6088)`);
|
|
11248
|
+
}
|
|
10281
11249
|
}
|
|
10282
11250
|
} catch (err) {
|
|
10283
11251
|
log(`[startup] state rehydration failed (continuing with empty state): ${err.message}`);
|
|
@@ -10455,8 +11423,14 @@ function deployMcpAssets() {
|
|
|
10455
11423
|
"slack-channel.js",
|
|
10456
11424
|
"direct-chat-channel.js",
|
|
10457
11425
|
"telegram-channel.js",
|
|
10458
|
-
"teams-channel.js"
|
|
11426
|
+
"teams-channel.js",
|
|
10459
11427
|
// ENG-6019
|
|
11428
|
+
// ENG-6195: the staff-only admin-debug broker, bundled with the CLI (never
|
|
11429
|
+
// published to npm). NOT in RESTARTABLE_CHANNEL_FILES — it's not a channel,
|
|
11430
|
+
// so it doesn't drive the channel-restart path; the mcp-presence-reaper
|
|
11431
|
+
// (re)starts it with the session, and a bundle update applies on the next
|
|
11432
|
+
// natural session restart.
|
|
11433
|
+
"augmented-admin.js"
|
|
10460
11434
|
]) {
|
|
10461
11435
|
const src = join8(mcpSourceDir, file);
|
|
10462
11436
|
const dst = join8(targetDir, file);
|
|
@@ -10529,10 +11503,12 @@ process.on("disconnect", () => {
|
|
|
10529
11503
|
});
|
|
10530
11504
|
export {
|
|
10531
11505
|
ChildProcessError,
|
|
11506
|
+
SCHEDULED_CARD_DELIVERY_CONTRACT,
|
|
10532
11507
|
__resetScheduledDeliveryDedupeForTest,
|
|
10533
11508
|
__setAgentChannelTokensForTest,
|
|
10534
11509
|
applyRestartAcks,
|
|
10535
11510
|
buildDoneCardNotification,
|
|
11511
|
+
buildSchedulerTaskInput,
|
|
10536
11512
|
claudeCodeUpgradeMarkerPath,
|
|
10537
11513
|
claudeCodeUpgradeThrottled,
|
|
10538
11514
|
claudeManagedSettingsPath,
|