instar 1.3.211 → 1.3.213

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/commands/server.d.ts.map +1 -1
  2. package/dist/commands/server.js +13 -0
  3. package/dist/commands/server.js.map +1 -1
  4. package/dist/core/SessionManager.d.ts +24 -0
  5. package/dist/core/SessionManager.d.ts.map +1 -1
  6. package/dist/core/SessionManager.js +79 -0
  7. package/dist/core/SessionManager.js.map +1 -1
  8. package/dist/core/SessionRecoveryChannel.d.ts +103 -0
  9. package/dist/core/SessionRecoveryChannel.d.ts.map +1 -0
  10. package/dist/core/SessionRecoveryChannel.js +158 -0
  11. package/dist/core/SessionRecoveryChannel.js.map +1 -0
  12. package/dist/core/SessionRecoveryConsumer.d.ts +60 -0
  13. package/dist/core/SessionRecoveryConsumer.d.ts.map +1 -0
  14. package/dist/core/SessionRecoveryConsumer.js +110 -0
  15. package/dist/core/SessionRecoveryConsumer.js.map +1 -0
  16. package/dist/core/StuckInputSentinel.d.ts +59 -2
  17. package/dist/core/StuckInputSentinel.d.ts.map +1 -1
  18. package/dist/core/StuckInputSentinel.js +127 -1
  19. package/dist/core/StuckInputSentinel.js.map +1 -1
  20. package/dist/core/types.d.ts +23 -0
  21. package/dist/core/types.d.ts.map +1 -1
  22. package/dist/core/types.js.map +1 -1
  23. package/dist/lifeline/TelegramLifeline.d.ts +1 -0
  24. package/dist/lifeline/TelegramLifeline.d.ts.map +1 -1
  25. package/dist/lifeline/TelegramLifeline.js +28 -0
  26. package/dist/lifeline/TelegramLifeline.js.map +1 -1
  27. package/dist/monitoring/StaleSessionBackstop.d.ts +15 -0
  28. package/dist/monitoring/StaleSessionBackstop.d.ts.map +1 -1
  29. package/dist/monitoring/StaleSessionBackstop.js +16 -4
  30. package/dist/monitoring/StaleSessionBackstop.js.map +1 -1
  31. package/package.json +1 -1
  32. package/src/data/builtin-manifest.json +4 -4
  33. package/upgrades/1.3.212.md +65 -0
  34. package/upgrades/1.3.213.md +50 -0
  35. package/upgrades/side-effects/codex-wedge-self-recovery.md +162 -0
  36. package/upgrades/side-effects/codex-wedged-job-cpu-detection.md +118 -0
@@ -0,0 +1,103 @@
1
+ /**
2
+ * SessionRecoveryChannel — the cross-process request/ack channel for codex
3
+ * session-wedge SELF-recovery.
4
+ *
5
+ * The constraint that shapes this (grounded 2026-06-03):
6
+ * - The DETECTOR (`StuckInputSentinel`) runs in the SERVER process.
7
+ * - The RESTART AUTHORITY (`ServerSupervisor` + `TelegramLifeline.replayQueue`)
8
+ * runs in the LIFELINE process.
9
+ * So the sentinel cannot restart anything directly — it must cross the process
10
+ * boundary. This channel is that boundary, modelled on the existing
11
+ * version-skew signal-file pattern (`state/lifeline-restart-requested.json`)
12
+ * but dedicated to per-session recovery and SAFE for two processes to share:
13
+ *
14
+ * - `state/session-recovery-requested.json` — SOLE writer is the SERVER
15
+ * (sentinel requests recovery). The lifeline only READS it.
16
+ * - `state/session-recovery-acked.json` — SOLE writer is the LIFELINE
17
+ * (it acks progress/outcome). The server only READS it.
18
+ *
19
+ * Single-writer-per-file means there is never a cross-process write race; both
20
+ * writes are atomic (temp+rename via SafeFsExecutor). Requests/acks are keyed by
21
+ * sessionId and carry an `attemptId` so a stale ack from a prior escalation
22
+ * episode can't be mistaken for the current one.
23
+ *
24
+ * Signal vs Authority: the request is a SIGNAL the server emits; the lifeline
25
+ * holds the AUTHORITY to act (or decline). This module is pure I/O — it performs
26
+ * NO restart and makes NO decision. Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
27
+ */
28
+ /** Escalation tier the sentinel is asking the lifeline to perform.
29
+ * - `redeliver`: re-deliver the pending/queued message to the wedged session
30
+ * (the targeted primitive — this is what cleared the live wedge).
31
+ * - `server-restart-replay`: graceful server restart + queue replay (the heavy
32
+ * fallback; highest blast radius, hard-gated by the caller). */
33
+ export type RecoveryTier = 'redeliver' | 'server-restart-replay';
34
+ /** Lifecycle of a recovery attempt as reported back by the lifeline. */
35
+ export type RecoveryOutcome = 'in-progress' | 'recovered' | 'failed';
36
+ export interface RecoveryRequest {
37
+ /** tmux session name of the wedged session. */
38
+ sessionId: string;
39
+ tier: RecoveryTier;
40
+ /** Human-readable reason (e.g. "input present ≥N ticks, no turn progress"). */
41
+ reason: string;
42
+ /** ISO timestamp the stall was observed. */
43
+ observedAt: string;
44
+ /** Unique per escalation episode — distinguishes a fresh request from a stale
45
+ * one for the same session. */
46
+ attemptId: string;
47
+ /** Who emitted it (e.g. 'StuckInputSentinel'). */
48
+ requestedBy: string;
49
+ }
50
+ export interface RecoveryAck {
51
+ sessionId: string;
52
+ /** Echoes the request's attemptId so the server can match ack→request. */
53
+ attemptId: string;
54
+ tier: RecoveryTier;
55
+ outcome: RecoveryOutcome;
56
+ /** Optional detail (e.g. "Replay complete: 1 delivered"). */
57
+ detail?: string;
58
+ /** ISO timestamp of this ack. */
59
+ updatedAt: string;
60
+ }
61
+ /**
62
+ * Cross-process recovery request/ack channel. Construct on BOTH sides with the
63
+ * same stateDir; the server uses the request* / readAck methods, the lifeline
64
+ * uses the readPendingRequests / ack* methods. The class enforces the
65
+ * single-writer-per-file invariant by never writing the file the other side owns.
66
+ */
67
+ export declare class SessionRecoveryChannel {
68
+ private readonly requestPath;
69
+ private readonly ackPath;
70
+ private readonly cooldownPath;
71
+ constructor(stateDir: string);
72
+ /** Emit/refresh a recovery request for a session. Idempotent per
73
+ * (sessionId, attemptId): re-requesting the SAME attempt is a no-op write of
74
+ * identical content; a NEW attemptId replaces the prior request for that
75
+ * session. Returns true if the file changed. */
76
+ requestRecovery(req: RecoveryRequest): boolean;
77
+ /** Read the lifeline's ack for a session (null if none). The server matches
78
+ * `attemptId` to confirm the ack is for the CURRENT attempt. */
79
+ readAck(sessionId: string): RecoveryAck | null;
80
+ /** Clear a session's request after recovery is verified (or abandoned). */
81
+ clearRequest(sessionId: string): boolean;
82
+ /** All pending recovery requests (server-emitted). Empty array if none. */
83
+ readPendingRequests(): RecoveryRequest[];
84
+ /** Record/refresh the lifeline's ack for a session. Sole writer of the ack
85
+ * file; overwrites the prior ack for that session (the latest outcome wins). */
86
+ ackRecovery(ack: RecoveryAck): void;
87
+ /** Drop a session's ack once the server has consumed it (keeps the file small). */
88
+ clearAck(sessionId: string): boolean;
89
+ /** Record that the lifeline performed a tier-C restart for a session. DURABLE:
90
+ * survives the server restart that wipes the sentinel's in-memory bound, so it
91
+ * is the only thing standing between a restart-can't-fix wedge and an infinite
92
+ * restart loop. `atEpochMs` is supplied by the caller (Date.now()). */
93
+ recordRestart(sessionId: string, atEpochMs: number): void;
94
+ /** Epoch-ms of the last tier-C restart for a session, or null if none recorded. */
95
+ lastRestartAt(sessionId: string): number | null;
96
+ /** True if a tier-C restart for this session happened within `cooldownMs` of
97
+ * `nowMs` — the lifeline MUST NOT restart again while this is true (loop guard). */
98
+ isInCooldown(sessionId: string, nowMs: number, cooldownMs: number): boolean;
99
+ private ensureDir;
100
+ private writeRequests;
101
+ private writeAcks;
102
+ }
103
+ //# sourceMappingURL=SessionRecoveryChannel.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"SessionRecoveryChannel.d.ts","sourceRoot":"","sources":["../../src/core/SessionRecoveryChannel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAMH;;;;mEAImE;AACnE,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,uBAAuB,CAAC;AAEjE,wEAAwE;AACxE,MAAM,MAAM,eAAe,GAAG,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;AAErE,MAAM,WAAW,eAAe;IAC9B,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,YAAY,CAAC;IACnB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB;oCACgC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,YAAY,CAAC;IACnB,OAAO,EAAE,eAAe,CAAC;IACzB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;CACnB;AAoCD;;;;;GAKG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,QAAQ,EAAE,MAAM;IAU5B;;;qDAGiD;IACjD,eAAe,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO;IAW9C;qEACiE;IACjE,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAK9C,2EAA2E;IAC3E,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAUxC,2EAA2E;IAC3E,mBAAmB,IAAI,eAAe,EAAE;IAMxC;qFACiF;IACjF,WAAW,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI;IAMnC,mFAAmF;IACnF,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAUpC;;;4EAGwE;IACxE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAOzD,mFAAmF;IACnF,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAM/C;yFACqF;IACrF,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO;IAO3E,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,SAAS;CAIlB"}
@@ -0,0 +1,158 @@
1
+ /**
2
+ * SessionRecoveryChannel — the cross-process request/ack channel for codex
3
+ * session-wedge SELF-recovery.
4
+ *
5
+ * The constraint that shapes this (grounded 2026-06-03):
6
+ * - The DETECTOR (`StuckInputSentinel`) runs in the SERVER process.
7
+ * - The RESTART AUTHORITY (`ServerSupervisor` + `TelegramLifeline.replayQueue`)
8
+ * runs in the LIFELINE process.
9
+ * So the sentinel cannot restart anything directly — it must cross the process
10
+ * boundary. This channel is that boundary, modelled on the existing
11
+ * version-skew signal-file pattern (`state/lifeline-restart-requested.json`)
12
+ * but dedicated to per-session recovery and SAFE for two processes to share:
13
+ *
14
+ * - `state/session-recovery-requested.json` — SOLE writer is the SERVER
15
+ * (sentinel requests recovery). The lifeline only READS it.
16
+ * - `state/session-recovery-acked.json` — SOLE writer is the LIFELINE
17
+ * (it acks progress/outcome). The server only READS it.
18
+ *
19
+ * Single-writer-per-file means there is never a cross-process write race; both
20
+ * writes are atomic (temp+rename via SafeFsExecutor). Requests/acks are keyed by
21
+ * sessionId and carry an `attemptId` so a stale ack from a prior escalation
22
+ * episode can't be mistaken for the current one.
23
+ *
24
+ * Signal vs Authority: the request is a SIGNAL the server emits; the lifeline
25
+ * holds the AUTHORITY to act (or decline). This module is pure I/O — it performs
26
+ * NO restart and makes NO decision. Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
27
+ */
28
+ import * as path from 'node:path';
29
+ import * as fs from 'node:fs';
30
+ import { SafeFsExecutor } from './SafeFsExecutor.js';
31
+ const REQUEST_RELPATH = path.join('state', 'session-recovery-requested.json');
32
+ const ACK_RELPATH = path.join('state', 'session-recovery-acked.json');
33
+ const COOLDOWN_RELPATH = path.join('state', 'session-recovery-cooldown.json');
34
+ function readJsonOrNull(filePath) {
35
+ try {
36
+ if (!fs.existsSync(filePath))
37
+ return null;
38
+ const raw = fs.readFileSync(filePath, 'utf8');
39
+ if (!raw.trim())
40
+ return null;
41
+ return JSON.parse(raw);
42
+ }
43
+ catch {
44
+ // Corrupt/torn file → treat as empty rather than throw. A subsequent write
45
+ // overwrites it cleanly.
46
+ return null;
47
+ }
48
+ }
49
+ /**
50
+ * Cross-process recovery request/ack channel. Construct on BOTH sides with the
51
+ * same stateDir; the server uses the request* / readAck methods, the lifeline
52
+ * uses the readPendingRequests / ack* methods. The class enforces the
53
+ * single-writer-per-file invariant by never writing the file the other side owns.
54
+ */
55
+ export class SessionRecoveryChannel {
56
+ requestPath;
57
+ ackPath;
58
+ cooldownPath;
59
+ constructor(stateDir) {
60
+ // stateDir is the agent's .instar dir; the signal files live under
61
+ // <stateDir>/state/ alongside the version-skew signal.
62
+ this.requestPath = path.join(stateDir, REQUEST_RELPATH);
63
+ this.ackPath = path.join(stateDir, ACK_RELPATH);
64
+ this.cooldownPath = path.join(stateDir, COOLDOWN_RELPATH);
65
+ }
66
+ // ---- SERVER (sentinel) side — sole writer of the request file ----
67
+ /** Emit/refresh a recovery request for a session. Idempotent per
68
+ * (sessionId, attemptId): re-requesting the SAME attempt is a no-op write of
69
+ * identical content; a NEW attemptId replaces the prior request for that
70
+ * session. Returns true if the file changed. */
71
+ requestRecovery(req) {
72
+ const file = readJsonOrNull(this.requestPath) ?? { version: 1, requests: {} };
73
+ const existing = file.requests[req.sessionId];
74
+ if (existing && existing.attemptId === req.attemptId && existing.tier === req.tier) {
75
+ return false; // identical in-flight request — nothing to do
76
+ }
77
+ file.requests[req.sessionId] = req;
78
+ this.writeRequests(file);
79
+ return true;
80
+ }
81
+ /** Read the lifeline's ack for a session (null if none). The server matches
82
+ * `attemptId` to confirm the ack is for the CURRENT attempt. */
83
+ readAck(sessionId) {
84
+ const file = readJsonOrNull(this.ackPath);
85
+ return file?.acks?.[sessionId] ?? null;
86
+ }
87
+ /** Clear a session's request after recovery is verified (or abandoned). */
88
+ clearRequest(sessionId) {
89
+ const file = readJsonOrNull(this.requestPath);
90
+ if (!file || !file.requests[sessionId])
91
+ return false;
92
+ delete file.requests[sessionId];
93
+ this.writeRequests(file);
94
+ return true;
95
+ }
96
+ // ---- LIFELINE side — sole writer of the ack file ----
97
+ /** All pending recovery requests (server-emitted). Empty array if none. */
98
+ readPendingRequests() {
99
+ const file = readJsonOrNull(this.requestPath);
100
+ if (!file?.requests)
101
+ return [];
102
+ return Object.values(file.requests);
103
+ }
104
+ /** Record/refresh the lifeline's ack for a session. Sole writer of the ack
105
+ * file; overwrites the prior ack for that session (the latest outcome wins). */
106
+ ackRecovery(ack) {
107
+ const file = readJsonOrNull(this.ackPath) ?? { version: 1, acks: {} };
108
+ file.acks[ack.sessionId] = ack;
109
+ this.writeAcks(file);
110
+ }
111
+ /** Drop a session's ack once the server has consumed it (keeps the file small). */
112
+ clearAck(sessionId) {
113
+ const file = readJsonOrNull(this.ackPath);
114
+ if (!file || !file.acks[sessionId])
115
+ return false;
116
+ delete file.acks[sessionId];
117
+ this.writeAcks(file);
118
+ return true;
119
+ }
120
+ // ---- LIFELINE side — durable restart cooldown (sole writer of the cooldown file) ----
121
+ /** Record that the lifeline performed a tier-C restart for a session. DURABLE:
122
+ * survives the server restart that wipes the sentinel's in-memory bound, so it
123
+ * is the only thing standing between a restart-can't-fix wedge and an infinite
124
+ * restart loop. `atEpochMs` is supplied by the caller (Date.now()). */
125
+ recordRestart(sessionId, atEpochMs) {
126
+ const file = readJsonOrNull(this.cooldownPath) ?? { version: 1, restarts: {} };
127
+ file.restarts[sessionId] = atEpochMs;
128
+ this.ensureDir(this.cooldownPath);
129
+ SafeFsExecutor.atomicWriteJsonSync(this.cooldownPath, file, { operation: 'SessionRecoveryChannel.recordRestart' });
130
+ }
131
+ /** Epoch-ms of the last tier-C restart for a session, or null if none recorded. */
132
+ lastRestartAt(sessionId) {
133
+ const file = readJsonOrNull(this.cooldownPath);
134
+ const v = file?.restarts?.[sessionId];
135
+ return typeof v === 'number' ? v : null;
136
+ }
137
+ /** True if a tier-C restart for this session happened within `cooldownMs` of
138
+ * `nowMs` — the lifeline MUST NOT restart again while this is true (loop guard). */
139
+ isInCooldown(sessionId, nowMs, cooldownMs) {
140
+ const last = this.lastRestartAt(sessionId);
141
+ return last !== null && nowMs - last < cooldownMs;
142
+ }
143
+ // ---- internals ----
144
+ ensureDir(filePath) {
145
+ const dir = path.dirname(filePath);
146
+ if (!fs.existsSync(dir))
147
+ fs.mkdirSync(dir, { recursive: true });
148
+ }
149
+ writeRequests(file) {
150
+ this.ensureDir(this.requestPath);
151
+ SafeFsExecutor.atomicWriteJsonSync(this.requestPath, file, { operation: 'SessionRecoveryChannel.writeRequests' });
152
+ }
153
+ writeAcks(file) {
154
+ this.ensureDir(this.ackPath);
155
+ SafeFsExecutor.atomicWriteJsonSync(this.ackPath, file, { operation: 'SessionRecoveryChannel.writeAcks' });
156
+ }
157
+ }
158
+ //# sourceMappingURL=SessionRecoveryChannel.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"SessionRecoveryChannel.js","sourceRoot":"","sources":["../../src/core/SessionRecoveryChannel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAuCrD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iCAAiC,CAAC,CAAC;AAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,6BAA6B,CAAC,CAAC;AACtE,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,gCAAgC,CAAC,CAAC;AAmB9E,SAAS,cAAc,CAAI,QAAgB;IACzC,IAAI,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,2EAA2E;QAC3E,yBAAyB;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,sBAAsB;IAChB,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,YAAY,CAAS;IAEtC,YAAY,QAAgB;QAC1B,mEAAmE;QACnE,uDAAuD;QACvD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAED,qEAAqE;IAErE;;;qDAGiD;IACjD,eAAe,CAAC,GAAoB;QAClC,MAAM,IAAI,GAAG,cAAc,CAAc,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;YACnF,OAAO,KAAK,CAAC,CAAC,8CAA8C;QAC9D,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;QACnC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED;qEACiE;IACjE,OAAO,CAAC,SAAiB;QACvB,MAAM,IAAI,GAAG,cAAc,CAAU,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,OAAO,IAAI,EAAE,IAAI,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IACzC,CAAC;IAED,2EAA2E;IAC3E,YAAY,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,cAAc,CAAc,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAC;QACrD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,wDAAwD;IAExD,2EAA2E;IAC3E,mBAAmB;QACjB,MAAM,IAAI,GAAG,cAAc,CAAc,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,EAAE,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED;qFACiF;IACjF,WAAW,CAAC,GAAgB;QAC1B,MAAM,IAAI,GAAG,cAAc,CAAU,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;QAC/E,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,mFAAmF;IACnF,QAAQ,CAAC,SAAiB;QACxB,MAAM,IAAI,GAAG,cAAc,CAAU,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAC;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,wFAAwF;IAExF;;;4EAGwE;IACxE,aAAa,CAAC,SAAiB,EAAE,SAAiB;QAChD,MAAM,IAAI,GAAG,cAAc,CAAe,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAC7F,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;QACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAClC,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,sCAAsC,EAAE,CAAC,CAAC;IACrH,CAAC;IAED,mFAAmF;IACnF,aAAa,CAAC,SAAiB;QAC7B,MAAM,IAAI,GAAG,cAAc,CAAe,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1C,CAAC;IAED;yFACqF;IACrF,YAAY,CAAC,SAAiB,EAAE,KAAa,EAAE,UAAkB;QAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC3C,OAAO,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,IAAI,GAAG,UAAU,CAAC;IACpD,CAAC;IAED,sBAAsB;IAEd,SAAS,CAAC,QAAgB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IAEO,aAAa,CAAC,IAAiB;QACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,sCAAsC,EAAE,CAAC,CAAC;IACpH,CAAC;IAEO,SAAS,CAAC,IAAa;QAC7B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,kCAAkC,EAAE,CAAC,CAAC;IAC5G,CAAC;CACF"}
@@ -0,0 +1,60 @@
1
+ /**
2
+ * SessionRecoveryConsumer — the LIFELINE-side executor for codex session-wedge
3
+ * self-recovery (tier C). It is the AUTHORITY half of the Signal-vs-Authority
4
+ * split: the server-process sentinel emits requests via SessionRecoveryChannel;
5
+ * this consumer (run from TelegramLifeline, where ServerSupervisor lives) reads
6
+ * them and actually performs the server restart + queue replay.
7
+ *
8
+ * The logic is deliberately separated from TelegramLifeline and takes its
9
+ * effects as injected callbacks (`restart`, `replay`) + an injectable clock, so
10
+ * it is unit-testable without standing up a lifeline. TelegramLifeline wires the
11
+ * real `supervisor.performGracefulRestart` and `replayQueue` and calls `tick()`
12
+ * on an interval.
13
+ *
14
+ * Safety properties (this is the highest-blast-radius path — it restarts the
15
+ * whole agent server):
16
+ * - DRY-RUN first: `dryRun:true` logs what it WOULD do and acks recovered
17
+ * without restarting. Ships in this mode.
18
+ * - DURABLE cooldown: before any restart it checks `channel.isInCooldown`. A
19
+ * tier-C restart wipes the sentinel's in-memory bound, so this durable guard
20
+ * is the only thing preventing a restart-can't-fix wedge from looping.
21
+ * - Records the restart BEFORE acting, so a crash mid-restart still counts
22
+ * against the cooldown.
23
+ * - Dedups on (sessionId, attemptId) so a lingering request (the sentinel was
24
+ * restarted before it could clear it) is not re-executed.
25
+ * - Never writes the request file (the server owns it) — ack + cooldown only.
26
+ *
27
+ * Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
28
+ */
29
+ import type { SessionRecoveryChannel } from './SessionRecoveryChannel.js';
30
+ export interface SessionRecoveryConsumerOptions {
31
+ channel: SessionRecoveryChannel;
32
+ /** Perform the server restart (TelegramLifeline → supervisor.performGracefulRestart). */
33
+ restart: (reason: string) => Promise<boolean>;
34
+ /** Replay the queued messages after the server is back (TelegramLifeline.replayQueue). */
35
+ replay: () => void;
36
+ /** When true, log only — never actually restart. Ships true. */
37
+ dryRun: boolean;
38
+ /** Minimum gap between tier-C restarts for the same session (loop guard). */
39
+ cooldownMs: number;
40
+ /** Injectable clock (epoch ms). Defaults to Date.now. */
41
+ now?: () => number;
42
+ /** Injectable logger. Defaults to console.log. */
43
+ log?: (msg: string) => void;
44
+ }
45
+ export declare class SessionRecoveryConsumer {
46
+ private readonly channel;
47
+ private readonly restart;
48
+ private readonly replay;
49
+ private readonly dryRun;
50
+ private readonly cooldownMs;
51
+ private readonly now;
52
+ private readonly log;
53
+ constructor(opts: SessionRecoveryConsumerOptions);
54
+ /** One pass: handle every pending tier-C recovery request. Best-effort — a
55
+ * read/exec failure on one request never throws out of tick(). */
56
+ tick(): Promise<void>;
57
+ private handle;
58
+ private ack;
59
+ }
60
+ //# sourceMappingURL=SessionRecoveryConsumer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"SessionRecoveryConsumer.d.ts","sourceRoot":"","sources":["../../src/core/SessionRecoveryConsumer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAmB,MAAM,6BAA6B,CAAC;AAE3F,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,sBAAsB,CAAC;IAChC,yFAAyF;IACzF,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9C,0FAA0F;IAC1F,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,gEAAgE;IAChE,MAAM,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,kDAAkD;IAClD,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;IACjD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAwB;gBAEhC,IAAI,EAAE,8BAA8B;IAUhD;uEACmE;IAC7D,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAiBb,MAAM;IAqCpB,OAAO,CAAC,GAAG;CAUZ"}
@@ -0,0 +1,110 @@
1
+ /**
2
+ * SessionRecoveryConsumer — the LIFELINE-side executor for codex session-wedge
3
+ * self-recovery (tier C). It is the AUTHORITY half of the Signal-vs-Authority
4
+ * split: the server-process sentinel emits requests via SessionRecoveryChannel;
5
+ * this consumer (run from TelegramLifeline, where ServerSupervisor lives) reads
6
+ * them and actually performs the server restart + queue replay.
7
+ *
8
+ * The logic is deliberately separated from TelegramLifeline and takes its
9
+ * effects as injected callbacks (`restart`, `replay`) + an injectable clock, so
10
+ * it is unit-testable without standing up a lifeline. TelegramLifeline wires the
11
+ * real `supervisor.performGracefulRestart` and `replayQueue` and calls `tick()`
12
+ * on an interval.
13
+ *
14
+ * Safety properties (this is the highest-blast-radius path — it restarts the
15
+ * whole agent server):
16
+ * - DRY-RUN first: `dryRun:true` logs what it WOULD do and acks recovered
17
+ * without restarting. Ships in this mode.
18
+ * - DURABLE cooldown: before any restart it checks `channel.isInCooldown`. A
19
+ * tier-C restart wipes the sentinel's in-memory bound, so this durable guard
20
+ * is the only thing preventing a restart-can't-fix wedge from looping.
21
+ * - Records the restart BEFORE acting, so a crash mid-restart still counts
22
+ * against the cooldown.
23
+ * - Dedups on (sessionId, attemptId) so a lingering request (the sentinel was
24
+ * restarted before it could clear it) is not re-executed.
25
+ * - Never writes the request file (the server owns it) — ack + cooldown only.
26
+ *
27
+ * Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
28
+ */
29
+ export class SessionRecoveryConsumer {
30
+ channel;
31
+ restart;
32
+ replay;
33
+ dryRun;
34
+ cooldownMs;
35
+ now;
36
+ log;
37
+ constructor(opts) {
38
+ this.channel = opts.channel;
39
+ this.restart = opts.restart;
40
+ this.replay = opts.replay;
41
+ this.dryRun = opts.dryRun;
42
+ this.cooldownMs = opts.cooldownMs;
43
+ this.now = opts.now ?? (() => Date.now());
44
+ this.log = opts.log ?? ((m) => console.log(m));
45
+ }
46
+ /** One pass: handle every pending tier-C recovery request. Best-effort — a
47
+ * read/exec failure on one request never throws out of tick(). */
48
+ async tick() {
49
+ let pending;
50
+ try {
51
+ pending = this.channel.readPendingRequests();
52
+ }
53
+ catch {
54
+ return;
55
+ }
56
+ for (const req of pending) {
57
+ if (req.tier !== 'server-restart-replay')
58
+ continue; // lifeline only owns tier C
59
+ try {
60
+ await this.handle(req);
61
+ }
62
+ catch (err) {
63
+ this.ack(req, 'failed', `executor error: ${err instanceof Error ? err.message : String(err)}`);
64
+ }
65
+ }
66
+ }
67
+ async handle(req) {
68
+ const now = this.now();
69
+ // Dedup: this exact attempt already reached a terminal ack → done.
70
+ const prior = this.channel.readAck(req.sessionId);
71
+ if (prior && prior.attemptId === req.attemptId && prior.outcome !== 'in-progress') {
72
+ return;
73
+ }
74
+ // Durable cooldown — the cross-restart loop guard.
75
+ if (this.channel.isInCooldown(req.sessionId, now, this.cooldownMs)) {
76
+ this.ack(req, 'failed', 'restart cooldown active (loop guard)');
77
+ this.log(`[SessionRecovery] cooldown active for ${req.sessionId} — refusing restart`);
78
+ return;
79
+ }
80
+ // Mark in-progress and record the restart BEFORE acting (so a crash mid-restart
81
+ // still counts against the cooldown).
82
+ this.ack(req, 'in-progress', this.dryRun ? 'dry-run' : 'restarting server + replay');
83
+ this.channel.recordRestart(req.sessionId, now);
84
+ if (this.dryRun) {
85
+ this.log(`[SessionRecovery] [DRY-RUN] would restart server + replay for wedged session ${req.sessionId} (${req.reason})`);
86
+ this.ack(req, 'recovered', 'dry-run: no restart performed');
87
+ return;
88
+ }
89
+ this.log(`[SessionRecovery] restarting server + replay for wedged session ${req.sessionId} (${req.reason})`);
90
+ const ok = await this.restart(`codex-wedge-recovery: ${req.sessionId}`);
91
+ if (ok) {
92
+ this.replay();
93
+ this.ack(req, 'recovered', 'server restarted + queue replayed');
94
+ }
95
+ else {
96
+ this.ack(req, 'failed', 'graceful restart returned false');
97
+ }
98
+ }
99
+ ack(req, outcome, detail) {
100
+ this.channel.ackRecovery({
101
+ sessionId: req.sessionId,
102
+ attemptId: req.attemptId,
103
+ tier: req.tier,
104
+ outcome,
105
+ detail,
106
+ updatedAt: new Date(this.now()).toISOString(),
107
+ });
108
+ }
109
+ }
110
+ //# sourceMappingURL=SessionRecoveryConsumer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"SessionRecoveryConsumer.js","sourceRoot":"","sources":["../../src/core/SessionRecoveryConsumer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAoBH,MAAM,OAAO,uBAAuB;IACjB,OAAO,CAAyB;IAChC,OAAO,CAAuC;IAC9C,MAAM,CAAa;IACnB,MAAM,CAAU;IAChB,UAAU,CAAS;IACnB,GAAG,CAAe;IAClB,GAAG,CAAwB;IAE5C,YAAY,IAAoC;QAC9C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;uEACmE;IACnE,KAAK,CAAC,IAAI;QACR,IAAI,OAA0B,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,uBAAuB;gBAAE,SAAS,CAAC,4BAA4B;YAChF,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAoB;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,mEAAmE;QACnE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,aAAa,EAAE,CAAC;YAClF,OAAO;QACT,CAAC;QAED,mDAAmD;QACnD,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACnE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,sCAAsC,CAAC,CAAC;YAChE,IAAI,CAAC,GAAG,CAAC,yCAAyC,GAAG,CAAC,SAAS,qBAAqB,CAAC,CAAC;YACtF,OAAO;QACT,CAAC;QAED,gFAAgF;QAChF,sCAAsC;QACtC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC;QACrF,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,gFAAgF,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;YAC1H,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,+BAA+B,CAAC,CAAC;YAC5D,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,mEAAmE,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7G,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,yBAAyB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACxE,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,mCAAmC,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,iCAAiC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,GAAG,CAAC,GAAoB,EAAE,OAA+C,EAAE,MAAc;QAC/F,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YACvB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,OAAO;YACP,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE;SAC9C,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -49,6 +49,7 @@
49
49
  * silently disappear after a server crash."
50
50
  */
51
51
  import type { SessionManager } from './SessionManager.js';
52
+ import { SessionRecoveryChannel, type RecoveryTier } from './SessionRecoveryChannel.js';
52
53
  export interface StuckInputSentinelOptions {
53
54
  /** Poll interval. Default 10s. */
54
55
  tickMs?: number;
@@ -63,6 +64,18 @@ export interface StuckInputSentinelOptions {
63
64
  stateDir: string;
64
65
  /** Disable persistence to disk (used by unit tests). */
65
66
  noPersist?: boolean;
67
+ /** Cross-process recovery channel. When provided AND escalationEnabled, the
68
+ * sentinel escalates PAST the keypress ladder (tier C: it requests a server
69
+ * restart + replay from the lifeline, which owns ServerSupervisor). When
70
+ * absent the sentinel keeps its legacy behavior (exhaust → stop). */
71
+ recoveryChannel?: SessionRecoveryChannel;
72
+ /** Master gate for the deeper-tier escalation. Default false (dark): the
73
+ * sentinel marks the record exhausted after the keypress ladder, exactly as
74
+ * before. Set true (via config) to enable tier-C recovery requests. */
75
+ escalationEnabled?: boolean;
76
+ /** How many ticks to wait for the lifeline to ack a tier-C request before
77
+ * giving up (bounded — no restart loop). Default 6. */
78
+ escalationTimeoutTicks?: number;
66
79
  }
67
80
  /** In-memory record of what the sentinel has seen for a given session. */
68
81
  interface SessionStuckRecord {
@@ -77,6 +90,16 @@ interface SessionStuckRecord {
77
90
  /** Whether this record is exhausted (max attempts reached, won't fire
78
91
  * again until the prompt text changes). */
79
92
  exhausted: boolean;
93
+ /** Deeper-tier escalation phase (after the keypress ladder exhausts).
94
+ * - 'none': not escalating (keypress ladder still running or escalation off).
95
+ * - 'requested': a tier-C recovery request is in flight to the lifeline;
96
+ * subsequent ticks read the ack.
97
+ * - 'recovered'/'gave-up': terminal. */
98
+ recoveryPhase: 'none' | 'requested' | 'recovered' | 'gave-up';
99
+ /** attemptId of the in-flight recovery request (matches the channel ack). */
100
+ recoveryAttemptId?: string;
101
+ /** tickCounter value when the request was emitted (for the bounded wait). */
102
+ recoveryRequestedTick?: number;
80
103
  }
81
104
  /** Event row written to stuck-input-events.jsonl on each fire. */
82
105
  export interface StuckInputEvent {
@@ -84,8 +107,10 @@ export interface StuckInputEvent {
84
107
  session: string;
85
108
  promptText: string;
86
109
  attempt: number;
87
- action: 'Enter' | 'C-m' | 'Enter-sleep-Enter';
88
- outcome: 'fired' | 'fire-error';
110
+ action: 'Enter' | 'C-m' | 'Enter-sleep-Enter' | 'escalate-request' | 'escalate-recovered' | 'escalate-failed' | 'escalate-timeout';
111
+ outcome: 'fired' | 'fire-error' | 'escalated' | 'recovered' | 'gave-up';
112
+ /** Recovery tier for escalation events (absent for keypress events). */
113
+ tier?: RecoveryTier;
89
114
  error?: string;
90
115
  }
91
116
  export declare class StuckInputSentinel {
@@ -96,11 +121,16 @@ export declare class StuckInputSentinel {
96
121
  private readonly stateDir;
97
122
  private readonly noPersist;
98
123
  private readonly eventsLogPath;
124
+ private readonly recoveryChannel;
125
+ private readonly escalationEnabled;
126
+ private readonly escalationTimeoutTicks;
99
127
  /** Per-session sticky state. Keyed by tmuxSession. */
100
128
  private readonly records;
101
129
  private intervalHandle;
102
130
  private tickInProgress;
103
131
  private degradationReportedOnce;
132
+ /** Monotonic tick counter — drives the bounded escalation wait. */
133
+ private tickCounter;
104
134
  constructor(sessionManager: SessionManager, opts: StuckInputSentinelOptions);
105
135
  start(): void;
106
136
  stop(): void;
@@ -149,6 +179,33 @@ export declare class StuckInputSentinel {
149
179
  /** Map attempt index → human-readable action name (matches
150
180
  * SessionManager.fireStuckInputRecovery's escalation). */
151
181
  static actionForAttempt(attempt: number): StuckInputEvent['action'];
182
+ /**
183
+ * Clear a session's in-flight tier-C recovery request when the session has
184
+ * recovered out-of-band — it's now actively working (the wedge drained) or it
185
+ * is gone. The sentinel is the sole writer of the request file, so it (not the
186
+ * lifeline) does this cleanup. No-op unless a request is actually in flight.
187
+ * This closes the "request lingers after a successful drain" path that the
188
+ * stuck-detection branches would otherwise skip (they early-return before
189
+ * handleEscalation ever reads the ack).
190
+ */
191
+ private finalizePendingRecovery;
192
+ /**
193
+ * Deeper-tier escalation, invoked once the keypress ladder (tier A) is
194
+ * exhausted but the prompt is still stuck. This is the codex session-wedge
195
+ * SELF-recovery path: the sentinel (server process) cannot restart the server
196
+ * itself — ServerSupervisor lives in the lifeline process — so it REQUESTS a
197
+ * tier-C recovery (server restart + queue replay) through SessionRecoveryChannel
198
+ * and the lifeline executes + acks. The sentinel here only signals, polls the
199
+ * ack, verifies, and bounds the wait (no restart loop). Spec:
200
+ * docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md.
201
+ *
202
+ * Dark by default: with escalation disabled or no channel, this reproduces the
203
+ * legacy behavior exactly (mark the record exhausted and stop).
204
+ *
205
+ * NOTE: a gentler server-side tier B (re-inject the full pending message before
206
+ * requesting a restart) is a planned refinement; this first cut goes A→C.
207
+ */
208
+ private handleEscalation;
152
209
  /** Append one event row to the JSONL log. Best-effort. */
153
210
  private recordEvent;
154
211
  /** Test helper: read the in-memory record for a session. */
@@ -1 +1 @@
1
- {"version":3,"file":"StuckInputSentinel.d.ts","sourceRoot":"","sources":["../../src/core/StuckInputSentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1D,MAAM,WAAW,yBAAyB;IACxC,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;0CAEsC;IACtC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;wEACoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,0EAA0E;AAC1E,UAAU,kBAAkB;IAC1B,kEAAkE;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,CAAC;IACzB,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;IACjB;gDAC4C;IAC5C,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,kEAAkE;AAClE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,GAAG,KAAK,GAAG,mBAAmB,CAAC;IAC9C,OAAO,EAAE,OAAO,GAAG,YAAY,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAeD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC,sDAAsD;IACtD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8C;IAEtE,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,uBAAuB,CAAS;gBAE5B,cAAc,EAAE,cAAc,EAAE,IAAI,EAAE,yBAAyB;IAU3E,KAAK,IAAI,IAAI;IAiBb,IAAI,IAAI,IAAI;IAOZ;;;;;;OAMG;IACH,IAAI,IAAI,IAAI;IA0BZ,+DAA+D;IAC/D,OAAO,CAAC,eAAe;IA4GvB;;;6EAGyE;IACzE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAoB9C;;;;;;;;;;;;;;;;;;;;;iCAqB6B;IAC7B,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAK5C;;oDAEgD;IAChD,OAAO,CAAC,YAAY;IAUpB;+DAC2D;IAC3D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC;IAMnE,0DAA0D;IAC1D,OAAO,CAAC,WAAW;IAUnB,4DAA4D;IAC5D,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAIrE,8CAA8C;IAC9C,YAAY,IAAI,IAAI;CAIrB"}
1
+ {"version":3,"file":"StuckInputSentinel.d.ts","sourceRoot":"","sources":["../../src/core/StuckInputSentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAG1D,OAAO,EAAE,sBAAsB,EAAE,KAAK,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAExF,MAAM,WAAW,yBAAyB;IACxC,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;0CAEsC;IACtC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;wEACoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;0EAGsE;IACtE,eAAe,CAAC,EAAE,sBAAsB,CAAC;IACzC;;4EAEwE;IACxE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;4DACwD;IACxD,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,0EAA0E;AAC1E,UAAU,kBAAkB;IAC1B,kEAAkE;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,CAAC;IACzB,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;IACjB;gDAC4C;IAC5C,SAAS,EAAE,OAAO,CAAC;IACnB;;;;8CAI0C;IAC1C,aAAa,EAAE,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC;IAC9D,6EAA6E;IAC7E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,kEAAkE;AAClE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,GAAG,KAAK,GAAG,mBAAmB,GACzC,kBAAkB,GAAG,oBAAoB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;IACvF,OAAO,EAAE,OAAO,GAAG,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC;IACxE,wEAAwE;IACxE,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAeD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAChE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAU;IAC5C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,sDAAsD;IACtD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8C;IAEtE,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,uBAAuB,CAAS;IACxC,mEAAmE;IACnE,OAAO,CAAC,WAAW,CAAK;gBAEZ,cAAc,EAAE,cAAc,EAAE,IAAI,EAAE,yBAAyB;IAa3E,KAAK,IAAI,IAAI;IAiBb,IAAI,IAAI,IAAI;IAOZ;;;;;;OAMG;IACH,IAAI,IAAI,IAAI;IA2BZ,+DAA+D;IAC/D,OAAO,CAAC,eAAe;IAsHvB;;;6EAGyE;IACzE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAoB9C;;;;;;;;;;;;;;;;;;;;;iCAqB6B;IAC7B,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAK5C;;oDAEgD;IAChD,OAAO,CAAC,YAAY;IAUpB;+DAC2D;IAC3D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC;IAMnE;;;;;;;;OAQG;IACH,OAAO,CAAC,uBAAuB;IAkB/B;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,gBAAgB;IAwExB,0DAA0D;IAC1D,OAAO,CAAC,WAAW;IAUnB,4DAA4D;IAC5D,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAIrE,8CAA8C;IAC9C,YAAY,IAAI,IAAI;CAIrB"}