@0xmaxma/claude-gateway 1.3.25 → 1.3.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -1
- package/config.template.json +6 -2
- package/dist/agent/incident-store.d.ts +89 -0
- package/dist/agent/incident-store.d.ts.map +1 -0
- package/dist/agent/incident-store.js +299 -0
- package/dist/agent/incident-store.js.map +1 -0
- package/dist/agent/incident.d.ts +156 -0
- package/dist/agent/incident.d.ts.map +1 -0
- package/dist/agent/incident.js +177 -0
- package/dist/agent/incident.js.map +1 -0
- package/dist/agent/recovery-executor.d.ts +117 -0
- package/dist/agent/recovery-executor.d.ts.map +1 -0
- package/dist/agent/recovery-executor.js +168 -0
- package/dist/agent/recovery-executor.js.map +1 -0
- package/dist/agent/recovery-policy.d.ts +97 -0
- package/dist/agent/recovery-policy.d.ts.map +1 -0
- package/dist/agent/recovery-policy.js +164 -0
- package/dist/agent/recovery-policy.js.map +1 -0
- package/dist/agent/runner.d.ts +44 -0
- package/dist/agent/runner.d.ts.map +1 -1
- package/dist/agent/runner.js +272 -2
- package/dist/agent/runner.js.map +1 -1
- package/dist/agent/safe-mode.d.ts +61 -0
- package/dist/agent/safe-mode.d.ts.map +1 -0
- package/dist/agent/safe-mode.js +102 -0
- package/dist/agent/safe-mode.js.map +1 -0
- package/dist/agent/triage.d.ts +94 -0
- package/dist/agent/triage.d.ts.map +1 -0
- package/dist/agent/triage.js +209 -0
- package/dist/agent/triage.js.map +1 -0
- package/dist/agent/turn-trace.d.ts +120 -0
- package/dist/agent/turn-trace.d.ts.map +1 -0
- package/dist/agent/turn-trace.js +122 -0
- package/dist/agent/turn-trace.js.map +1 -0
- package/dist/api/gateway-router.d.ts +21 -0
- package/dist/api/gateway-router.d.ts.map +1 -1
- package/dist/api/gateway-router.js +58 -17
- package/dist/api/gateway-router.js.map +1 -1
- package/dist/config/migrator.d.ts +4 -0
- package/dist/config/migrator.d.ts.map +1 -1
- package/dist/config/migrator.js +60 -3
- package/dist/config/migrator.js.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/session/process.d.ts +18 -0
- package/dist/session/process.d.ts.map +1 -1
- package/dist/session/process.js +61 -1
- package/dist/session/process.js.map +1 -1
- package/dist/shell/claude-pty-shell.js +59 -0
- package/dist/shell/claude-pty-shell.js.map +1 -1
- package/dist/shell/control-channel.d.ts +74 -0
- package/dist/shell/control-channel.d.ts.map +1 -0
- package/dist/shell/control-channel.js +114 -0
- package/dist/shell/control-channel.js.map +1 -0
- package/dist/types.d.ts +21 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/ui/web-ui.d.ts.map +1 -1
- package/dist/ui/web-ui.js +103 -2
- package/dist/ui/web-ui.js.map +1 -1
- package/mcp/tools/skills/handlers.ts +4 -2
- package/mcp/tools/telegram/receiver-server.ts +163 -0
- package/mcp/tools/telegram/typing.ts +124 -0
- package/package.json +3 -1
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Recovery policy (Epic #195, Phase 3).
|
|
4
|
+
*
|
|
5
|
+
* Turns a (schema-validated) triage verdict into an actual recovery action —
|
|
6
|
+
* but only after enforcing three independent guards, so a compromised or
|
|
7
|
+
* mistaken triage step can never drive an unsafe intervention:
|
|
8
|
+
*
|
|
9
|
+
* 1. Per-stage whitelist. Each pipeline stage permits only a small set of
|
|
10
|
+
* actions (a `dispatch` stall may redeliver or restart the receiver; a
|
|
11
|
+
* `progress` PTY wedge may press esc or restart the session). An action
|
|
12
|
+
* the stage does not permit is clamped to `notify-only`, regardless of
|
|
13
|
+
* what triage proposed.
|
|
14
|
+
* 2. Budget + cooldown. Interventions are capped per turn and spaced by a
|
|
15
|
+
* cooldown, so a flapping stall cannot trigger an unbounded action loop.
|
|
16
|
+
* 3. Safe-mode escalation. Repeated PTY-stage failures recommend flipping the
|
|
17
|
+
* agent to the headless backend instead of retrying the same wedge.
|
|
18
|
+
*
|
|
19
|
+
* All functions are pure — no IO, no clock reads (the caller passes `now`).
|
|
20
|
+
* The executor that actually presses keys / restarts / flips backend lives in
|
|
21
|
+
* the runner; this module decides, it never acts.
|
|
22
|
+
*/
|
|
23
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
24
|
+
exports.DEFAULT_SAFE_MODE_THRESHOLD = exports.DEFAULT_BUDGET = exports.DEFAULT_COOLDOWN_MS = exports.DEFAULT_MAX_INTERVENTIONS_PER_TURN = exports.STAGE_ALLOWED_ACTIONS = exports.NOTIFY_ONLY_PLAN = void 0;
|
|
25
|
+
exports.selectRecoveryAction = selectRecoveryAction;
|
|
26
|
+
exports.defaultActionForStage = defaultActionForStage;
|
|
27
|
+
exports.initialBudget = initialBudget;
|
|
28
|
+
exports.checkBudget = checkBudget;
|
|
29
|
+
exports.shouldEnterSafeMode = shouldEnterSafeMode;
|
|
30
|
+
exports.shouldExitSafeMode = shouldExitSafeMode;
|
|
31
|
+
/** The safe no-op plan. */
|
|
32
|
+
exports.NOTIFY_ONLY_PLAN = {
|
|
33
|
+
action: 'notify-only',
|
|
34
|
+
reason: 'no permitted action',
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Actions permitted per stalled stage. Deliberately conservative: destructive
|
|
38
|
+
* or cross-cutting actions (restart-receiver, fallback-headless) are only
|
|
39
|
+
* allowed where they make sense. `notify-only` is always implicitly allowed.
|
|
40
|
+
*/
|
|
41
|
+
exports.STAGE_ALLOWED_ACTIONS = {
|
|
42
|
+
// Receiver/channel ingress — nothing to press; escalate the transport.
|
|
43
|
+
inbound: ['restart-receiver', 'notify-only'],
|
|
44
|
+
// Runner failed to inject — restart the session.
|
|
45
|
+
inject: ['restart-session', 'notify-only'],
|
|
46
|
+
// Session/CLI startup wedged — restart, or fall back to headless.
|
|
47
|
+
startup: ['restart-session', 'fallback-headless', 'notify-only'],
|
|
48
|
+
// Claude producing no output — a TUI overlay is the usual cause: dismiss it,
|
|
49
|
+
// pick a menu option, or restart; repeated failures escalate to headless.
|
|
50
|
+
progress: [
|
|
51
|
+
'esc',
|
|
52
|
+
'esc-esc',
|
|
53
|
+
'enter',
|
|
54
|
+
'select-option',
|
|
55
|
+
'bridge-menu',
|
|
56
|
+
'restart-session',
|
|
57
|
+
'fallback-headless',
|
|
58
|
+
'notify-only',
|
|
59
|
+
],
|
|
60
|
+
// Turn ended, no artifact — redeliver or restart the session.
|
|
61
|
+
delivery: ['redeliver-forward', 'restart-session', 'notify-only'],
|
|
62
|
+
// Artifact written, not sent — redeliver, or restart the receiver.
|
|
63
|
+
dispatch: ['redeliver-forward', 'restart-receiver', 'notify-only'],
|
|
64
|
+
};
|
|
65
|
+
/**
|
|
66
|
+
* Decide the recovery plan for a stalled stage given an optional triage verdict.
|
|
67
|
+
* If a verdict is supplied, its action is honoured ONLY if the stage permits it;
|
|
68
|
+
* otherwise the plan clamps to notify-only. With no verdict, a conservative
|
|
69
|
+
* default action for the stage is used (still whitelist-checked).
|
|
70
|
+
*/
|
|
71
|
+
function selectRecoveryAction(input) {
|
|
72
|
+
const allowed = exports.STAGE_ALLOWED_ACTIONS[input.stage] ?? ['notify-only'];
|
|
73
|
+
if (input.verdict) {
|
|
74
|
+
const a = input.verdict.action;
|
|
75
|
+
if (a === 'notify-only' || a === 'none') {
|
|
76
|
+
return { action: 'notify-only', reason: `triage:${a}` };
|
|
77
|
+
}
|
|
78
|
+
if (allowed.includes(a)) {
|
|
79
|
+
const plan = { action: a, reason: `triage:${input.verdict.state}` };
|
|
80
|
+
if (a === 'select-option' && typeof input.verdict.option === 'number') {
|
|
81
|
+
plan.option = input.verdict.option;
|
|
82
|
+
}
|
|
83
|
+
// select-option with no option index is not actionable → clamp.
|
|
84
|
+
if (a === 'select-option' && plan.option === undefined) {
|
|
85
|
+
return { action: 'notify-only', reason: 'select-option missing option index' };
|
|
86
|
+
}
|
|
87
|
+
return plan;
|
|
88
|
+
}
|
|
89
|
+
// Triage proposed an action this stage does not permit → refuse it.
|
|
90
|
+
return { action: 'notify-only', reason: `clamped: ${a} not allowed at ${input.stage}` };
|
|
91
|
+
}
|
|
92
|
+
// No triage available → deterministic default per stage.
|
|
93
|
+
const fallback = defaultActionForStage(input.stage);
|
|
94
|
+
if (allowed.includes(fallback)) {
|
|
95
|
+
return { action: fallback, reason: `default:${input.stage}` };
|
|
96
|
+
}
|
|
97
|
+
return exports.NOTIFY_ONLY_PLAN;
|
|
98
|
+
}
|
|
99
|
+
/** Conservative default action when no triage verdict is available. */
|
|
100
|
+
function defaultActionForStage(stage) {
|
|
101
|
+
switch (stage) {
|
|
102
|
+
case 'dispatch':
|
|
103
|
+
return 'redeliver-forward';
|
|
104
|
+
case 'delivery':
|
|
105
|
+
return 'redeliver-forward';
|
|
106
|
+
case 'inject':
|
|
107
|
+
case 'startup':
|
|
108
|
+
return 'restart-session';
|
|
109
|
+
case 'inbound':
|
|
110
|
+
return 'restart-receiver';
|
|
111
|
+
case 'progress':
|
|
112
|
+
// A progress wedge is ambiguous without triage; do not blindly press keys.
|
|
113
|
+
return 'notify-only';
|
|
114
|
+
default:
|
|
115
|
+
return 'notify-only';
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// ─── Budget + cooldown ──────────────────────────────────────────────────────
|
|
119
|
+
exports.DEFAULT_MAX_INTERVENTIONS_PER_TURN = 3;
|
|
120
|
+
exports.DEFAULT_COOLDOWN_MS = 30000;
|
|
121
|
+
exports.DEFAULT_BUDGET = {
|
|
122
|
+
maxPerTurn: exports.DEFAULT_MAX_INTERVENTIONS_PER_TURN,
|
|
123
|
+
cooldownMs: exports.DEFAULT_COOLDOWN_MS,
|
|
124
|
+
};
|
|
125
|
+
function initialBudget(turnKey) {
|
|
126
|
+
return { turnKey, attempts: 0, lastAt: 0 };
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Decide whether an intervention is allowed now, and return the state to keep.
|
|
130
|
+
* Resets the counter when the turn changes; enforces the per-turn cap and the
|
|
131
|
+
* cooldown between attempts. Pure — the caller supplies `now`.
|
|
132
|
+
*/
|
|
133
|
+
function checkBudget(state, turnKey, now, cfg = exports.DEFAULT_BUDGET) {
|
|
134
|
+
// New turn → fresh budget.
|
|
135
|
+
const base = state.turnKey === turnKey ? state : initialBudget(turnKey);
|
|
136
|
+
if (base.attempts >= cfg.maxPerTurn) {
|
|
137
|
+
return { allowed: false, reason: 'budget exhausted for this turn', next: base };
|
|
138
|
+
}
|
|
139
|
+
if (base.attempts > 0 && now - base.lastAt < cfg.cooldownMs) {
|
|
140
|
+
return { allowed: false, reason: 'cooldown active', next: base };
|
|
141
|
+
}
|
|
142
|
+
return {
|
|
143
|
+
allowed: true,
|
|
144
|
+
reason: 'ok',
|
|
145
|
+
next: { turnKey, attempts: base.attempts + 1, lastAt: now },
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
// ─── Safe-mode escalation ─────────────────────────────────────────────────
|
|
149
|
+
exports.DEFAULT_SAFE_MODE_THRESHOLD = 3;
|
|
150
|
+
/**
|
|
151
|
+
* Whether repeated PTY-stage failures should escalate to safe mode (flip the
|
|
152
|
+
* agent to the headless backend). Pure.
|
|
153
|
+
*/
|
|
154
|
+
function shouldEnterSafeMode(consecutivePtyFailures, threshold = exports.DEFAULT_SAFE_MODE_THRESHOLD) {
|
|
155
|
+
return consecutivePtyFailures >= threshold;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Whether safe mode should be lifted. Safe mode is a reactive fallback; it is
|
|
159
|
+
* cleared explicitly — on a successful PTY turn after a fix, or on user command.
|
|
160
|
+
*/
|
|
161
|
+
function shouldExitSafeMode(input) {
|
|
162
|
+
return Boolean(input.manualRestore || input.healthyPtyTurnObserved);
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=recovery-policy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recovery-policy.js","sourceRoot":"","sources":["../../src/agent/recovery-policy.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;;;AA0DH,oDAiCC;AAGD,sDAiBC;AA2BD,sCAEC;AAcD,kCAqBC;AAUD,kDAKC;AAMD,gDAKC;AAzLD,2BAA2B;AACd,QAAA,gBAAgB,GAAiB;IAC5C,MAAM,EAAE,aAAa;IACrB,MAAM,EAAE,qBAAqB;CAC9B,CAAA;AAED;;;;GAIG;AACU,QAAA,qBAAqB,GAAkD;IAClF,uEAAuE;IACvE,OAAO,EAAE,CAAC,kBAAkB,EAAE,aAAa,CAAC;IAC5C,iDAAiD;IACjD,MAAM,EAAE,CAAC,iBAAiB,EAAE,aAAa,CAAC;IAC1C,kEAAkE;IAClE,OAAO,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,aAAa,CAAC;IAChE,6EAA6E;IAC7E,0EAA0E;IAC1E,QAAQ,EAAE;QACR,KAAK;QACL,SAAS;QACT,OAAO;QACP,eAAe;QACf,aAAa;QACb,iBAAiB;QACjB,mBAAmB;QACnB,aAAa;KACd;IACD,8DAA8D;IAC9D,QAAQ,EAAE,CAAC,mBAAmB,EAAE,iBAAiB,EAAE,aAAa,CAAC;IACjE,mEAAmE;IACnE,QAAQ,EAAE,CAAC,mBAAmB,EAAE,kBAAkB,EAAE,aAAa,CAAC;CACnE,CAAA;AAED;;;;;GAKG;AACH,SAAgB,oBAAoB,CAAC,KAIpC;IACC,MAAM,OAAO,GAAG,6BAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAA;QAC9B,IAAI,CAAC,KAAK,aAAa,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;YACxC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,EAAE,CAAA;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAiB,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAA;YACjF,IAAI,CAAC,KAAK,eAAe,IAAI,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACtE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAA;YACpC,CAAC;YACD,gEAAgE;YAChE,IAAI,CAAC,KAAK,eAAe,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvD,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,oCAAoC,EAAE,CAAA;YAChF,CAAC;YACD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,oEAAoE;QACpE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,CAAC,mBAAmB,KAAK,CAAC,KAAK,EAAE,EAAE,CAAA;IACzF,CAAC;IAED,yDAAyD;IACzD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACnD,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,KAAK,CAAC,KAAK,EAAE,EAAE,CAAA;IAC/D,CAAC;IACD,OAAO,wBAAgB,CAAA;AACzB,CAAC;AAED,uEAAuE;AACvE,SAAgB,qBAAqB,CAAC,KAAa;IACjD,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,UAAU;YACb,OAAO,mBAAmB,CAAA;QAC5B,KAAK,UAAU;YACb,OAAO,mBAAmB,CAAA;QAC5B,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,iBAAiB,CAAA;QAC1B,KAAK,SAAS;YACZ,OAAO,kBAAkB,CAAA;QAC3B,KAAK,UAAU;YACb,2EAA2E;YAC3E,OAAO,aAAa,CAAA;QACtB;YACE,OAAO,aAAa,CAAA;IACxB,CAAC;AACH,CAAC;AAED,+EAA+E;AAElE,QAAA,kCAAkC,GAAG,CAAC,CAAA;AACtC,QAAA,mBAAmB,GAAG,KAAM,CAAA;AAO5B,QAAA,cAAc,GAAiB;IAC1C,UAAU,EAAE,0CAAkC;IAC9C,UAAU,EAAE,2BAAmB;CAChC,CAAA;AAYD,SAAgB,aAAa,CAAC,OAAe;IAC3C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAA;AAC5C,CAAC;AASD;;;;GAIG;AACH,SAAgB,WAAW,CACzB,KAAkB,EAClB,OAAe,EACf,GAAW,EACX,MAAoB,sBAAc;IAElC,2BAA2B;IAC3B,MAAM,IAAI,GACR,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;IAE5D,IAAI,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,gCAAgC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IACjF,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QAC5D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAClE,CAAC;IACD,OAAO;QACL,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,IAAI;QACZ,IAAI,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE;KAC5D,CAAA;AACH,CAAC;AAED,6EAA6E;AAEhE,QAAA,2BAA2B,GAAG,CAAC,CAAA;AAE5C;;;GAGG;AACH,SAAgB,mBAAmB,CACjC,sBAA8B,EAC9B,YAAoB,mCAA2B;IAE/C,OAAO,sBAAsB,IAAI,SAAS,CAAA;AAC5C,CAAC;AAED;;;GAGG;AACH,SAAgB,kBAAkB,CAAC,KAGlC;IACC,OAAO,OAAO,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;AACrE,CAAC"}
|
package/dist/agent/runner.d.ts
CHANGED
|
@@ -31,6 +31,9 @@ export declare class AgentRunner extends EventEmitter {
|
|
|
31
31
|
private readonly sessionSpawnLocks;
|
|
32
32
|
private readonly tooLargeRecoveries;
|
|
33
33
|
private readonly tooLargeExhausted;
|
|
34
|
+
private readonly safeMode;
|
|
35
|
+
private readonly recoveryBudgets;
|
|
36
|
+
private readonly lastTurn;
|
|
34
37
|
private readonly pendingImagePaths;
|
|
35
38
|
private readonly channelCoalesce;
|
|
36
39
|
private readonly coalesceWindowMs;
|
|
@@ -55,6 +58,47 @@ export declare class AgentRunner extends EventEmitter {
|
|
|
55
58
|
* Each payload is routed to the appropriate SessionProcess by chat_id.
|
|
56
59
|
*/
|
|
57
60
|
private startCallbackServer;
|
|
61
|
+
/**
|
|
62
|
+
* Handle POST /recover from the receiver process (Epic #195, Phase 3b).
|
|
63
|
+
* The watchdog detects a stall in the receiver, but recovery must run here in
|
|
64
|
+
* the runner, where the live control surfaces live. Runs the pure executor
|
|
65
|
+
* with effects bound to the affected session and returns the outcome so the
|
|
66
|
+
* receiver can persist it to the incident bundle.
|
|
67
|
+
*/
|
|
68
|
+
private handleRecoverRequest;
|
|
69
|
+
/**
|
|
70
|
+
* Route raw interactive-terminal input to the session with this actual
|
|
71
|
+
* sessionId (Issue #201). The dashboard's Terminal Viewer input mode
|
|
72
|
+
* streams keystrokes here via the pty-stream WebSocket. Sessions are keyed by
|
|
73
|
+
* chatId internally, so we match on the process's own sessionId. Returns true
|
|
74
|
+
* only when a live pty-shell session accepted the bytes (headless / missing /
|
|
75
|
+
* not-writable → false), so the caller can surface an accurate result.
|
|
76
|
+
*/
|
|
77
|
+
sendInputToSession(sessionId: string, data: string): boolean;
|
|
78
|
+
/**
|
|
79
|
+
* Build the recovery effects bound to one chat's session (Phase 3b). Each
|
|
80
|
+
* effect resolves the session fresh so it survives a restart mid-recovery.
|
|
81
|
+
* Keystroke effects go through the wrapper control channel; restart/backend
|
|
82
|
+
* effects act on the session and safe-mode manager. Effects intentionally
|
|
83
|
+
* omitted (redeliver-forward, restart-receiver, bridge-menu) are transport /
|
|
84
|
+
* wrapper concerns not bridged to the runner — the executor reports them as
|
|
85
|
+
* unsupported rather than guessing.
|
|
86
|
+
*/
|
|
87
|
+
private buildRecoveryEffects;
|
|
88
|
+
/**
|
|
89
|
+
* Evict finished-turn budget entries (Epic #195, Phase 3b). Called on each
|
|
90
|
+
* budget write so the per-turn map cannot grow without bound on a long-lived
|
|
91
|
+
* runner. `nowMs` is the timestamp of the write just made; any entry whose last
|
|
92
|
+
* attempt predates the TTL belongs to a turn that is no longer being recovered.
|
|
93
|
+
*/
|
|
94
|
+
private pruneRecoveryBudgets;
|
|
95
|
+
/**
|
|
96
|
+
* One-shot local `claude -p` triage runner (Phase 3b). Feeds the closed
|
|
97
|
+
* classification prompt on stdin and returns raw stdout. Hard timeout →
|
|
98
|
+
* timedOut, which the executor collapses to a safe notify-only verdict. Only
|
|
99
|
+
* invoked when autoRecover is enabled.
|
|
100
|
+
*/
|
|
101
|
+
private recoveryTriageSpawn;
|
|
58
102
|
/**
|
|
59
103
|
* Handle POST /command requests from the receiver process.
|
|
60
104
|
* Supports: get_model, set_model, restart.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/agent/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAMtC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAW,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAShH,OAAO,EAA0C,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/agent/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAMtC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAW,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAShH,OAAO,EAA0C,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAQvF,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAmB1C,eAAO,MAAM,oBAAoB,QAAmB,CAAC;AAErD,eAAO,MAAM,cAAc,EAAE,WAAW,EAUvC,CAAC;AAqBF,eAAO,MAAM,0BAA0B,OAAO,CAAC;AA6C/C,qBAAa,WAAY,SAAQ,YAAY;IAC3C,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA6B;IAC9D,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAqB;IACrD,OAAO,CAAC,SAAS,CAAoC;IAErD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IACjC,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAEvC,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqC;IAC9D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAsD;IACvF,OAAO,CAAC,QAAQ,CAAiC;IACjD,OAAO,CAAC,eAAe,CAAgC;IAEvD,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,gBAAgB,CAA+C;IAGvE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAqB;IAGxD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA8C;IAMhF,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA6B;IAKhE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAqB;IAOvD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAQtB;IAIH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkC;IAKlE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4E;IAGrG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA+B;IAOjE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAO5B;IAIJ,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAG1C,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAA+B;IAGrE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoH;IAKnJ,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAGxD,OAAO,CAAC,aAAa,CAAwC;IAG7D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAGpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IAGtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAGlC,OAAO,CAAC,aAAa,CAA6B;gBAEtC,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,MAAM;IAwBnF;;OAEG;IACH,gBAAgB,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI;IAI/C,gBAAgB,IAAI,aAAa;IAIjC,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED;;;OAGG;YACW,mBAAmB;IAwHjC;;;;;;OAMG;YACW,oBAAoB;IA2ElC;;;;;;;OAOG;IACH,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO;IAU5D;;;;;;;;OAQG;IACH,OAAO,CAAC,oBAAoB;IAgC5B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB,CA8BzB;IAEF;;;OAGG;YACW,oBAAoB;IAqLlC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAY5B,OAAO,CAAC,MAAM,CAAC,aAAa;IAI5B,OAAO,CAAC,iBAAiB;IAYzB;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IA2GxB,OAAO,CAAC,MAAM,CAAC,eAAe;YA0ChB,iBAAiB;YAgDjB,YAAY;IAiZ1B,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAIpD;;OAEG;YACW,oBAAoB;IAsBlC;;OAEG;YACW,qBAAqB;IAiBnC;;OAEG;YACW,wBAAwB;IA+CtC;;OAEG;YACW,gBAAgB;IAM9B;;OAEG;YACW,mBAAmB;IAUjC;;;OAGG;YACW,iBAAiB;IAM/B;;;OAGG;YACW,kBAAkB;IA8BhC;;OAEG;YACW,oBAAoB;IA4ClC;;;OAGG;YACW,aAAa;IAS3B;;;OAGG;YACW,cAAc;IAS5B;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,qBAAqB;IAyC7B;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,cAAc,CAAC,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BlE;;OAEG;IACH,OAAO,CAAC,SAAS;IAWjB;;;;OAIG;IACD,OAAO,CAAC,YAAY;IAMtB,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,eAAe;YAST,wBAAwB;IA2BtC,OAAO,CAAC,gBAAgB;IAexB;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAexB,OAAO,CAAC,gBAAgB;IAalB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B5B,iBAAiB,CAAC,SAAS,EAAE,WAAW,GAAG,IAAI;IAQ/C,cAAc,IAAI,IAAI;IAetB,aAAa,IAAI,IAAI;IAOrB,cAAc,IAAI,WAAW;IAI7B,qBAAqB,IAAI,IAAI;IAY7B,oBAAoB,IAAI,IAAI;IAO5B,oBAAoB,IAAI,IAAI;IAY5B,mBAAmB,IAAI,IAAI;IAOrB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB3B;;;;OAIG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAK5F,OAAO,CAAC,sBAAsB;IAqBxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAM9B,SAAS,IAAI,OAAO;IAIpB,kBAAkB,IAAI,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,OAAO,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA4BzL;;;;;;;OAOG;IACG,cAAc,CAClB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,OAAO,CAAA;KAAE,GAClH,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC;IA4L1D;;;;;OAKG;IACG,oBAAoB,CACxB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,SAAS,EAAE;QACT,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,IAAI,CAAC;QACjE,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;KAC/B,EACD,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,OAAO,CAAA;KAAE,GAClH,OAAO,CAAC,MAAM,IAAI,CAAC;IAyNtB;;OAEG;IACH,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAI/C;;;;OAIG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI;IAK/D;;;OAGG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,EAAE;IAcrD,OAAO,CAAC,sBAAsB;IAc9B,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,YAAY,IAAI,SAAS;IAIzB,iBAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAIrD,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,CAAC,OAAO,UAAU,EAAE,YAAY,CAAC;IAIvH,iBAAiB,CACrB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,GAC/B,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAsK/D,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASzC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,UAAU,EAAE,YAAY,CAAC;IAIzE,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,UAAU,EAAE,WAAW,CAAC;IAc/G,OAAO,CAAC,+BAA+B;IAsBjC,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAmB7F,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAW5I,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWxE;;;;OAIG;IACG,oBAAoB,CACxB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,UAAU,GAAG,SAAS,GAAG,MAAM,EACxC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,SAAS,EAAE;QACT,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;QACnC,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,CAAC;KAC/B,EACD,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAC1B,OAAO,CAAC,MAAM,IAAI,CAAC;IAiItB,OAAO,CAAC,iBAAiB;IA2CzB;;;OAGG;IACH,eAAe,IAAI,MAAM;IAIzB;;;;;;OAMG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;CAoBnC"}
|
package/dist/agent/runner.js
CHANGED
|
@@ -49,6 +49,11 @@ const line_reply_manager_1 = require("./line-reply-manager");
|
|
|
49
49
|
const markdown_1 = require("../telegram/markdown");
|
|
50
50
|
const skills_1 = require("../skills");
|
|
51
51
|
const builtin_commands_1 = require("./builtin-commands");
|
|
52
|
+
const safe_mode_1 = require("./safe-mode");
|
|
53
|
+
const recovery_executor_1 = require("./recovery-executor");
|
|
54
|
+
const recovery_policy_1 = require("./recovery-policy");
|
|
55
|
+
const incident_1 = require("./incident");
|
|
56
|
+
const pty_stream_registry_1 = require("../shell/pty-stream-registry");
|
|
52
57
|
const screen_1 = require("../shell/screen");
|
|
53
58
|
const db_1 = require("../history/db");
|
|
54
59
|
const media_store_1 = require("../history/media-store");
|
|
@@ -82,6 +87,14 @@ const PROTECTED_WORKSPACE_FILES = [
|
|
|
82
87
|
'IDENTITY.md', 'USER.md', 'HEARTBEAT.md',
|
|
83
88
|
];
|
|
84
89
|
const MAX_API_IMAGES = 5;
|
|
90
|
+
// Hard timeout for the one-shot local `claude -p` triage during recovery
|
|
91
|
+
// (Epic #195, Phase 3b). A slow/hung triage collapses to a safe notify-only.
|
|
92
|
+
const RECOVERY_TRIAGE_TIMEOUT_MS = 15000;
|
|
93
|
+
// TTL after which a per-turn recovery budget entry is evicted. A turn's budget
|
|
94
|
+
// only matters during its active stall window (a few interventions spaced by a
|
|
95
|
+
// 30s cooldown), so anything older is a finished turn — pruned to keep the
|
|
96
|
+
// budget map from growing unbounded over a long-lived runner (Epic #195, 3b).
|
|
97
|
+
const RECOVERY_BUDGET_TTL_MS = 10 * 60000;
|
|
85
98
|
// Trailing-edge window for coalescing channel messages that carry an image (or that
|
|
86
99
|
// arrive while an image is already buffered) into a single turn. Reset on every new
|
|
87
100
|
// related update, so an album burst or a client-split photo+caption is gathered
|
|
@@ -157,6 +170,27 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
157
170
|
// restarting would just churn a context that cannot shrink. Cleared together
|
|
158
171
|
// with the counter on a successful result and on /clear.
|
|
159
172
|
this.tooLargeExhausted = new Set();
|
|
173
|
+
// Safe-mode manager (Epic #195, Phase 3): tracks repeated PTY-backend failures
|
|
174
|
+
// and, past a threshold, forces this agent to the headless backend so it keeps
|
|
175
|
+
// serving turns without a gateway restart. In-memory only — a restart clears it
|
|
176
|
+
// and re-reads the user's real config. spawnSession reads isActive() to set
|
|
177
|
+
// SessionProcess.forceHeadless.
|
|
178
|
+
this.safeMode = new safe_mode_1.SafeModeManager({
|
|
179
|
+
audit: (e) => this.logger.info('Safe-mode transition', {
|
|
180
|
+
agentId: e.agentId,
|
|
181
|
+
action: e.action,
|
|
182
|
+
reason: e.reason,
|
|
183
|
+
failures: e.failures,
|
|
184
|
+
}),
|
|
185
|
+
});
|
|
186
|
+
// Recovery executor state (Epic #195, Phase 3b). Per-turn intervention budget,
|
|
187
|
+
// keyed by turnKey so it resets each turn. In-memory only.
|
|
188
|
+
this.recoveryBudgets = new Map();
|
|
189
|
+
// Last injected turn per chat, for the C1 guarded resend: `delivered` flips
|
|
190
|
+
// true once assistant output reaches the channel, so a resend after recovery
|
|
191
|
+
// only fires when the stalled turn produced nothing (no double-submit). `resent`
|
|
192
|
+
// guards against resending more than once per turn.
|
|
193
|
+
this.lastTurn = new Map();
|
|
160
194
|
// Tracks pending Telegram image paths per chatId (queue) for size accumulation after each turn.
|
|
161
195
|
this.pendingImagePaths = new Map();
|
|
162
196
|
// Coalescing buffer: collects channel messages that arrive close together so a
|
|
@@ -177,6 +211,46 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
177
211
|
this.skillRegistry = { skills: new Map() };
|
|
178
212
|
// Cancel function for the daily history cleanup timer
|
|
179
213
|
this.cancelCleanup = null;
|
|
214
|
+
/**
|
|
215
|
+
* One-shot local `claude -p` triage runner (Phase 3b). Feeds the closed
|
|
216
|
+
* classification prompt on stdin and returns raw stdout. Hard timeout →
|
|
217
|
+
* timedOut, which the executor collapses to a safe notify-only verdict. Only
|
|
218
|
+
* invoked when autoRecover is enabled.
|
|
219
|
+
*/
|
|
220
|
+
this.recoveryTriageSpawn = async (prompt) => {
|
|
221
|
+
const { spawn } = await Promise.resolve().then(() => __importStar(require('child_process')));
|
|
222
|
+
return new Promise((resolve) => {
|
|
223
|
+
let done = false;
|
|
224
|
+
let out = '';
|
|
225
|
+
const finish = (r) => {
|
|
226
|
+
if (done)
|
|
227
|
+
return;
|
|
228
|
+
done = true;
|
|
229
|
+
clearTimeout(timer);
|
|
230
|
+
resolve(r);
|
|
231
|
+
};
|
|
232
|
+
const timer = setTimeout(() => {
|
|
233
|
+
try {
|
|
234
|
+
child.kill('SIGKILL');
|
|
235
|
+
}
|
|
236
|
+
catch { /* already gone */ }
|
|
237
|
+
finish({ stdout: '', timedOut: true });
|
|
238
|
+
}, RECOVERY_TRIAGE_TIMEOUT_MS);
|
|
239
|
+
const child = spawn('claude', ['-p', '--output-format', 'text'], {
|
|
240
|
+
stdio: ['pipe', 'pipe', 'ignore'],
|
|
241
|
+
});
|
|
242
|
+
child.stdout?.on('data', (d) => { out += String(d); });
|
|
243
|
+
child.on('error', () => finish({ stdout: '', timedOut: true }));
|
|
244
|
+
child.on('close', () => finish({ stdout: out }));
|
|
245
|
+
try {
|
|
246
|
+
child.stdin?.write(prompt);
|
|
247
|
+
child.stdin?.end();
|
|
248
|
+
}
|
|
249
|
+
catch {
|
|
250
|
+
finish({ stdout: '', timedOut: true });
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
};
|
|
180
254
|
this.agentConfig = agentConfig;
|
|
181
255
|
this.gatewayConfig = gatewayConfig;
|
|
182
256
|
this.logger = logger ?? (0, logger_1.createLogger)(agentConfig.id, gatewayConfig.gateway.logDir);
|
|
@@ -229,6 +303,13 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
229
303
|
this.handleCommandRequest(raw, res);
|
|
230
304
|
return;
|
|
231
305
|
}
|
|
306
|
+
if (url.pathname === '/recover') {
|
|
307
|
+
// Cross-process recovery bridge (Epic #195, Phase 3b): the receiver
|
|
308
|
+
// process detects a stall but recovery must run here, where the live
|
|
309
|
+
// control surfaces (session stdin, restart, safe-mode) live.
|
|
310
|
+
this.handleRecoverRequest(raw, res);
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
232
313
|
// Default: /channel — existing channel message handler
|
|
233
314
|
// Connection: close prevents keep-alive pool reuse: after the OS's
|
|
234
315
|
// keepAliveTimeout expires the server closes the TCP connection, and the
|
|
@@ -315,6 +396,160 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
315
396
|
});
|
|
316
397
|
this.logger.info('Channel callback server listening', { port: this.callbackPort });
|
|
317
398
|
}
|
|
399
|
+
/**
|
|
400
|
+
* Handle POST /recover from the receiver process (Epic #195, Phase 3b).
|
|
401
|
+
* The watchdog detects a stall in the receiver, but recovery must run here in
|
|
402
|
+
* the runner, where the live control surfaces live. Runs the pure executor
|
|
403
|
+
* with effects bound to the affected session and returns the outcome so the
|
|
404
|
+
* receiver can persist it to the incident bundle.
|
|
405
|
+
*/
|
|
406
|
+
async handleRecoverRequest(raw, res) {
|
|
407
|
+
const respond = (data, status = 200) => {
|
|
408
|
+
res.writeHead(status, { 'Content-Type': 'application/json' });
|
|
409
|
+
res.end(JSON.stringify(data));
|
|
410
|
+
};
|
|
411
|
+
let body;
|
|
412
|
+
try {
|
|
413
|
+
body = JSON.parse(raw);
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
respond({ ok: false, error: 'Invalid JSON' }, 400);
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
const chatId = body.chatId ?? '';
|
|
420
|
+
const stage = body.stage ?? '';
|
|
421
|
+
if (!chatId || !stage) {
|
|
422
|
+
respond({ ok: false, error: 'missing chatId/stage' }, 400);
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
const session = this.sessions.get(chatId);
|
|
426
|
+
const autoRecover = this.gatewayConfig.gateway.selfHealing?.autoRecover === true;
|
|
427
|
+
const req = {
|
|
428
|
+
incidentId: body.incidentId ?? '',
|
|
429
|
+
agentId: this.agentConfig.id,
|
|
430
|
+
chatId,
|
|
431
|
+
// Prefer the live session's id (survives a request that names a stale one).
|
|
432
|
+
sessionId: session?.sessionId ?? body.sessionId ?? '',
|
|
433
|
+
stage,
|
|
434
|
+
failureClass: body.failureClass ?? null,
|
|
435
|
+
turnKey: body.turnKey ?? chatId,
|
|
436
|
+
};
|
|
437
|
+
try {
|
|
438
|
+
const result = await (0, recovery_executor_1.runRecovery)(req, {
|
|
439
|
+
autoRecover,
|
|
440
|
+
effects: this.buildRecoveryEffects(chatId),
|
|
441
|
+
now: () => Date.now(),
|
|
442
|
+
budget: {
|
|
443
|
+
get: (turnKey) => this.recoveryBudgets.get(turnKey) ?? (0, recovery_policy_1.initialBudget)(turnKey),
|
|
444
|
+
set: (s) => {
|
|
445
|
+
this.recoveryBudgets.set(s.turnKey, s);
|
|
446
|
+
this.pruneRecoveryBudgets(s.lastAt);
|
|
447
|
+
},
|
|
448
|
+
},
|
|
449
|
+
// Live triage is the most sensitive surface — only ever run when the
|
|
450
|
+
// operator has opted in. When off, the executor uses the deterministic
|
|
451
|
+
// per-stage default action (still whitelist-clamped).
|
|
452
|
+
triageSpawn: autoRecover ? this.recoveryTriageSpawn : undefined,
|
|
453
|
+
gatherEvidence: async () => {
|
|
454
|
+
const s = this.sessions.get(chatId);
|
|
455
|
+
if (!s)
|
|
456
|
+
return null;
|
|
457
|
+
try {
|
|
458
|
+
const snap = await pty_stream_registry_1.ptyStreamRegistry.screenText(s.sessionId);
|
|
459
|
+
if (!snap)
|
|
460
|
+
return null;
|
|
461
|
+
// Scrub the untrusted screen before it leaves the process / reaches
|
|
462
|
+
// the triage model. Redact the chat id explicitly on top of the
|
|
463
|
+
// pattern-based secret scrub.
|
|
464
|
+
return { screenText: (0, incident_1.scrubText)(snap.text, [chatId]) };
|
|
465
|
+
}
|
|
466
|
+
catch {
|
|
467
|
+
return null;
|
|
468
|
+
}
|
|
469
|
+
},
|
|
470
|
+
resendAfterRecover: autoRecover,
|
|
471
|
+
log: (msg, meta) => this.logger.info(msg, meta),
|
|
472
|
+
});
|
|
473
|
+
respond({ ok: true, outcome: (0, recovery_executor_1.toRecoveryOutcome)(result), result });
|
|
474
|
+
}
|
|
475
|
+
catch (err) {
|
|
476
|
+
this.logger.error('Recovery attempt failed', { chatId, stage, error: err.message });
|
|
477
|
+
respond({ ok: false, error: err.message }, 500);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Route raw interactive-terminal input to the session with this actual
|
|
482
|
+
* sessionId (Issue #201). The dashboard's Terminal Viewer input mode
|
|
483
|
+
* streams keystrokes here via the pty-stream WebSocket. Sessions are keyed by
|
|
484
|
+
* chatId internally, so we match on the process's own sessionId. Returns true
|
|
485
|
+
* only when a live pty-shell session accepted the bytes (headless / missing /
|
|
486
|
+
* not-writable → false), so the caller can surface an accurate result.
|
|
487
|
+
*/
|
|
488
|
+
sendInputToSession(sessionId, data) {
|
|
489
|
+
if (!sessionId || typeof data !== 'string' || data.length === 0)
|
|
490
|
+
return false;
|
|
491
|
+
for (const proc of this.sessions.values()) {
|
|
492
|
+
if (proc.sessionId === sessionId) {
|
|
493
|
+
return proc.sendInput(data);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
return false;
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* Build the recovery effects bound to one chat's session (Phase 3b). Each
|
|
500
|
+
* effect resolves the session fresh so it survives a restart mid-recovery.
|
|
501
|
+
* Keystroke effects go through the wrapper control channel; restart/backend
|
|
502
|
+
* effects act on the session and safe-mode manager. Effects intentionally
|
|
503
|
+
* omitted (redeliver-forward, restart-receiver, bridge-menu) are transport /
|
|
504
|
+
* wrapper concerns not bridged to the runner — the executor reports them as
|
|
505
|
+
* unsupported rather than guessing.
|
|
506
|
+
*/
|
|
507
|
+
buildRecoveryEffects(chatId) {
|
|
508
|
+
const control = (key, option) => {
|
|
509
|
+
this.sessions.get(chatId)?.sendControl(key, option);
|
|
510
|
+
};
|
|
511
|
+
return {
|
|
512
|
+
esc: () => control('esc'),
|
|
513
|
+
escEsc: () => control('esc-esc'),
|
|
514
|
+
enter: () => control('enter'),
|
|
515
|
+
selectOption: (option) => control('select-option', option),
|
|
516
|
+
restartSession: () => this.restartProcess(chatId),
|
|
517
|
+
fallbackHeadless: async () => {
|
|
518
|
+
// Flip to the headless backend, then restart so the new session respawns
|
|
519
|
+
// headless (spawnSession reads safeMode.isActive → forceHeadless).
|
|
520
|
+
this.safeMode.enter(this.agentConfig.id, 'recovery: fallback-headless');
|
|
521
|
+
await this.restartProcess(chatId);
|
|
522
|
+
},
|
|
523
|
+
resendLast: () => {
|
|
524
|
+
const lt = this.lastTurn.get(chatId);
|
|
525
|
+
// Guard: only resend a turn that produced no output and was not already
|
|
526
|
+
// resent — never double-submit.
|
|
527
|
+
if (!lt || lt.delivered || lt.resent)
|
|
528
|
+
return false;
|
|
529
|
+
const s = this.sessions.get(chatId);
|
|
530
|
+
if (!s)
|
|
531
|
+
return false;
|
|
532
|
+
lt.resent = true;
|
|
533
|
+
s.setProcessing(true);
|
|
534
|
+
s.sendMessage(lt.text);
|
|
535
|
+
s.touch();
|
|
536
|
+
return true;
|
|
537
|
+
},
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
/**
|
|
541
|
+
* Evict finished-turn budget entries (Epic #195, Phase 3b). Called on each
|
|
542
|
+
* budget write so the per-turn map cannot grow without bound on a long-lived
|
|
543
|
+
* runner. `nowMs` is the timestamp of the write just made; any entry whose last
|
|
544
|
+
* attempt predates the TTL belongs to a turn that is no longer being recovered.
|
|
545
|
+
*/
|
|
546
|
+
pruneRecoveryBudgets(nowMs) {
|
|
547
|
+
const cutoff = nowMs - RECOVERY_BUDGET_TTL_MS;
|
|
548
|
+
for (const [key, state] of this.recoveryBudgets) {
|
|
549
|
+
if (state.lastAt < cutoff)
|
|
550
|
+
this.recoveryBudgets.delete(key);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
318
553
|
/**
|
|
319
554
|
* Handle POST /command requests from the receiver process.
|
|
320
555
|
* Supports: get_model, set_model, restart.
|
|
@@ -599,7 +834,12 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
599
834
|
}
|
|
600
835
|
}
|
|
601
836
|
session.setProcessing(true);
|
|
602
|
-
|
|
837
|
+
const turnText = blocks.join('\n');
|
|
838
|
+
session.sendMessage(turnText);
|
|
839
|
+
// Remember this turn for the C1 guarded resend (Phase 3b): reset the
|
|
840
|
+
// delivered/resent flags so a resend can only fire if this turn stalls
|
|
841
|
+
// before producing output.
|
|
842
|
+
this.lastTurn.set(chatId, { text: turnText, delivered: false, resent: false });
|
|
603
843
|
session.touch();
|
|
604
844
|
this.logger.debug('Injected channel turn into session', {
|
|
605
845
|
chatId,
|
|
@@ -729,6 +969,15 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
729
969
|
mapKey, recoveryCount, historyLimit: proc.historyLimit,
|
|
730
970
|
});
|
|
731
971
|
}
|
|
972
|
+
// Safe mode (Epic #195, Phase 3): if the PTY backend has repeatedly failed
|
|
973
|
+
// for this agent, force the headless backend for this spawn. Cleared
|
|
974
|
+
// automatically once safe mode exits (a later healthy turn / user restore).
|
|
975
|
+
if (this.safeMode.isActive(this.agentConfig.id)) {
|
|
976
|
+
proc.forceHeadless = true;
|
|
977
|
+
this.logger.info('Spawning in safe mode (headless backend forced)', {
|
|
978
|
+
mapKey, agentId: this.agentConfig.id,
|
|
979
|
+
});
|
|
980
|
+
}
|
|
732
981
|
await proc.start();
|
|
733
982
|
// Forward all session output lines so listeners on AgentRunner (GatewayRouter,
|
|
734
983
|
// CronScheduler, tests) receive them without needing individual session references.
|
|
@@ -736,7 +985,16 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
736
985
|
// Notify typing indicator when session permanently fails (max restarts exceeded)
|
|
737
986
|
if (source !== 'api') {
|
|
738
987
|
proc.once('failed', () => {
|
|
739
|
-
|
|
988
|
+
// Safe mode (Epic #195, Phase 3): a hard failure of the PTY (interactive
|
|
989
|
+
// TUI) backend counts toward the safe-mode threshold. Once crossed, the
|
|
990
|
+
// agent auto-flips to headless on the next spawn so the user's next
|
|
991
|
+
// message is served instead of re-wedging. Headless failures don't count
|
|
992
|
+
// (there is no PTY wrapper to blame / fall back from). When we just
|
|
993
|
+
// entered safe mode, tell the user that specifically instead of the
|
|
994
|
+
// generic "stopped" notice (one .error file, so pick the better code).
|
|
995
|
+
const enteredSafeMode = proc.backend === 'pty-shell' &&
|
|
996
|
+
this.safeMode.recordPtyFailure(this.agentConfig.id);
|
|
997
|
+
this.writeTypingError(mapKey, enteredSafeMode ? 'SAFE_MODE_ENABLED' : 'PROCESS_FAILED');
|
|
740
998
|
this.sessions.delete(mapKey);
|
|
741
999
|
// LINE: surface an error for any outstanding postback button so a tap
|
|
742
1000
|
// returns the interrupted notice instead of a stale "still thinking".
|
|
@@ -899,6 +1157,13 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
899
1157
|
// the next 32MB (if any) starts fresh at the top of the ladder.
|
|
900
1158
|
this.tooLargeRecoveries.delete(mapKey);
|
|
901
1159
|
this.tooLargeExhausted.delete(mapKey);
|
|
1160
|
+
// Safe mode (Epic #195, Phase 3): a healthy PTY turn resets the
|
|
1161
|
+
// consecutive-failure counter and lifts safe mode if it was active
|
|
1162
|
+
// (the interactive backend recovered). Headless successes don't
|
|
1163
|
+
// touch it — that's the fallback doing its job, not the PTY healing.
|
|
1164
|
+
if (proc.backend === 'pty-shell') {
|
|
1165
|
+
this.safeMode.recordSuccess(this.agentConfig.id);
|
|
1166
|
+
}
|
|
902
1167
|
}
|
|
903
1168
|
// When a menu was rendered to buttons this turn, the wrapper appends
|
|
904
1169
|
// the same menu text to the turn's result — strip that suffix so it
|
|
@@ -941,6 +1206,11 @@ class AgentRunner extends events_1.EventEmitter {
|
|
|
941
1206
|
else {
|
|
942
1207
|
this.writeAutoForward(mapKey, text);
|
|
943
1208
|
}
|
|
1209
|
+
// Assistant output reached the channel — mark the turn delivered so
|
|
1210
|
+
// a later recovery does not resend a message that was answered (C1).
|
|
1211
|
+
const lt = this.lastTurn.get(mapKey);
|
|
1212
|
+
if (lt)
|
|
1213
|
+
lt.delivered = true;
|
|
944
1214
|
}
|
|
945
1215
|
replyCalled = false; // reset for next turn
|
|
946
1216
|
replyToolUseId = null;
|