instar 1.3.211 → 1.3.213
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +13 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/core/SessionManager.d.ts +24 -0
- package/dist/core/SessionManager.d.ts.map +1 -1
- package/dist/core/SessionManager.js +79 -0
- package/dist/core/SessionManager.js.map +1 -1
- package/dist/core/SessionRecoveryChannel.d.ts +103 -0
- package/dist/core/SessionRecoveryChannel.d.ts.map +1 -0
- package/dist/core/SessionRecoveryChannel.js +158 -0
- package/dist/core/SessionRecoveryChannel.js.map +1 -0
- package/dist/core/SessionRecoveryConsumer.d.ts +60 -0
- package/dist/core/SessionRecoveryConsumer.d.ts.map +1 -0
- package/dist/core/SessionRecoveryConsumer.js +110 -0
- package/dist/core/SessionRecoveryConsumer.js.map +1 -0
- package/dist/core/StuckInputSentinel.d.ts +59 -2
- package/dist/core/StuckInputSentinel.d.ts.map +1 -1
- package/dist/core/StuckInputSentinel.js +127 -1
- package/dist/core/StuckInputSentinel.js.map +1 -1
- package/dist/core/types.d.ts +23 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/lifeline/TelegramLifeline.d.ts +1 -0
- package/dist/lifeline/TelegramLifeline.d.ts.map +1 -1
- package/dist/lifeline/TelegramLifeline.js +28 -0
- package/dist/lifeline/TelegramLifeline.js.map +1 -1
- package/dist/monitoring/StaleSessionBackstop.d.ts +15 -0
- package/dist/monitoring/StaleSessionBackstop.d.ts.map +1 -1
- package/dist/monitoring/StaleSessionBackstop.js +16 -4
- package/dist/monitoring/StaleSessionBackstop.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +4 -4
- package/upgrades/1.3.212.md +65 -0
- package/upgrades/1.3.213.md +50 -0
- package/upgrades/side-effects/codex-wedge-self-recovery.md +162 -0
- package/upgrades/side-effects/codex-wedged-job-cpu-detection.md +118 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionRecoveryChannel — the cross-process request/ack channel for codex
|
|
3
|
+
* session-wedge SELF-recovery.
|
|
4
|
+
*
|
|
5
|
+
* The constraint that shapes this (grounded 2026-06-03):
|
|
6
|
+
* - The DETECTOR (`StuckInputSentinel`) runs in the SERVER process.
|
|
7
|
+
* - The RESTART AUTHORITY (`ServerSupervisor` + `TelegramLifeline.replayQueue`)
|
|
8
|
+
* runs in the LIFELINE process.
|
|
9
|
+
* So the sentinel cannot restart anything directly — it must cross the process
|
|
10
|
+
* boundary. This channel is that boundary, modelled on the existing
|
|
11
|
+
* version-skew signal-file pattern (`state/lifeline-restart-requested.json`)
|
|
12
|
+
* but dedicated to per-session recovery and SAFE for two processes to share:
|
|
13
|
+
*
|
|
14
|
+
* - `state/session-recovery-requested.json` — SOLE writer is the SERVER
|
|
15
|
+
* (sentinel requests recovery). The lifeline only READS it.
|
|
16
|
+
* - `state/session-recovery-acked.json` — SOLE writer is the LIFELINE
|
|
17
|
+
* (it acks progress/outcome). The server only READS it.
|
|
18
|
+
*
|
|
19
|
+
* Single-writer-per-file means there is never a cross-process write race; both
|
|
20
|
+
* writes are atomic (temp+rename via SafeFsExecutor). Requests/acks are keyed by
|
|
21
|
+
* sessionId and carry an `attemptId` so a stale ack from a prior escalation
|
|
22
|
+
* episode can't be mistaken for the current one.
|
|
23
|
+
*
|
|
24
|
+
* Signal vs Authority: the request is a SIGNAL the server emits; the lifeline
|
|
25
|
+
* holds the AUTHORITY to act (or decline). This module is pure I/O — it performs
|
|
26
|
+
* NO restart and makes NO decision. Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
|
|
27
|
+
*/
|
|
28
|
+
/** Escalation tier the sentinel is asking the lifeline to perform.
|
|
29
|
+
* - `redeliver`: re-deliver the pending/queued message to the wedged session
|
|
30
|
+
* (the targeted primitive — this is what cleared the live wedge).
|
|
31
|
+
* - `server-restart-replay`: graceful server restart + queue replay (the heavy
|
|
32
|
+
* fallback; highest blast radius, hard-gated by the caller). */
|
|
33
|
+
export type RecoveryTier = 'redeliver' | 'server-restart-replay';
|
|
34
|
+
/** Lifecycle of a recovery attempt as reported back by the lifeline. */
|
|
35
|
+
export type RecoveryOutcome = 'in-progress' | 'recovered' | 'failed';
|
|
36
|
+
export interface RecoveryRequest {
|
|
37
|
+
/** tmux session name of the wedged session. */
|
|
38
|
+
sessionId: string;
|
|
39
|
+
tier: RecoveryTier;
|
|
40
|
+
/** Human-readable reason (e.g. "input present ≥N ticks, no turn progress"). */
|
|
41
|
+
reason: string;
|
|
42
|
+
/** ISO timestamp the stall was observed. */
|
|
43
|
+
observedAt: string;
|
|
44
|
+
/** Unique per escalation episode — distinguishes a fresh request from a stale
|
|
45
|
+
* one for the same session. */
|
|
46
|
+
attemptId: string;
|
|
47
|
+
/** Who emitted it (e.g. 'StuckInputSentinel'). */
|
|
48
|
+
requestedBy: string;
|
|
49
|
+
}
|
|
50
|
+
export interface RecoveryAck {
|
|
51
|
+
sessionId: string;
|
|
52
|
+
/** Echoes the request's attemptId so the server can match ack→request. */
|
|
53
|
+
attemptId: string;
|
|
54
|
+
tier: RecoveryTier;
|
|
55
|
+
outcome: RecoveryOutcome;
|
|
56
|
+
/** Optional detail (e.g. "Replay complete: 1 delivered"). */
|
|
57
|
+
detail?: string;
|
|
58
|
+
/** ISO timestamp of this ack. */
|
|
59
|
+
updatedAt: string;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Cross-process recovery request/ack channel. Construct on BOTH sides with the
|
|
63
|
+
* same stateDir; the server uses the request* / readAck methods, the lifeline
|
|
64
|
+
* uses the readPendingRequests / ack* methods. The class enforces the
|
|
65
|
+
* single-writer-per-file invariant by never writing the file the other side owns.
|
|
66
|
+
*/
|
|
67
|
+
export declare class SessionRecoveryChannel {
|
|
68
|
+
private readonly requestPath;
|
|
69
|
+
private readonly ackPath;
|
|
70
|
+
private readonly cooldownPath;
|
|
71
|
+
constructor(stateDir: string);
|
|
72
|
+
/** Emit/refresh a recovery request for a session. Idempotent per
|
|
73
|
+
* (sessionId, attemptId): re-requesting the SAME attempt is a no-op write of
|
|
74
|
+
* identical content; a NEW attemptId replaces the prior request for that
|
|
75
|
+
* session. Returns true if the file changed. */
|
|
76
|
+
requestRecovery(req: RecoveryRequest): boolean;
|
|
77
|
+
/** Read the lifeline's ack for a session (null if none). The server matches
|
|
78
|
+
* `attemptId` to confirm the ack is for the CURRENT attempt. */
|
|
79
|
+
readAck(sessionId: string): RecoveryAck | null;
|
|
80
|
+
/** Clear a session's request after recovery is verified (or abandoned). */
|
|
81
|
+
clearRequest(sessionId: string): boolean;
|
|
82
|
+
/** All pending recovery requests (server-emitted). Empty array if none. */
|
|
83
|
+
readPendingRequests(): RecoveryRequest[];
|
|
84
|
+
/** Record/refresh the lifeline's ack for a session. Sole writer of the ack
|
|
85
|
+
* file; overwrites the prior ack for that session (the latest outcome wins). */
|
|
86
|
+
ackRecovery(ack: RecoveryAck): void;
|
|
87
|
+
/** Drop a session's ack once the server has consumed it (keeps the file small). */
|
|
88
|
+
clearAck(sessionId: string): boolean;
|
|
89
|
+
/** Record that the lifeline performed a tier-C restart for a session. DURABLE:
|
|
90
|
+
* survives the server restart that wipes the sentinel's in-memory bound, so it
|
|
91
|
+
* is the only thing standing between a restart-can't-fix wedge and an infinite
|
|
92
|
+
* restart loop. `atEpochMs` is supplied by the caller (Date.now()). */
|
|
93
|
+
recordRestart(sessionId: string, atEpochMs: number): void;
|
|
94
|
+
/** Epoch-ms of the last tier-C restart for a session, or null if none recorded. */
|
|
95
|
+
lastRestartAt(sessionId: string): number | null;
|
|
96
|
+
/** True if a tier-C restart for this session happened within `cooldownMs` of
|
|
97
|
+
* `nowMs` — the lifeline MUST NOT restart again while this is true (loop guard). */
|
|
98
|
+
isInCooldown(sessionId: string, nowMs: number, cooldownMs: number): boolean;
|
|
99
|
+
private ensureDir;
|
|
100
|
+
private writeRequests;
|
|
101
|
+
private writeAcks;
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=SessionRecoveryChannel.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SessionRecoveryChannel.d.ts","sourceRoot":"","sources":["../../src/core/SessionRecoveryChannel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAMH;;;;mEAImE;AACnE,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,uBAAuB,CAAC;AAEjE,wEAAwE;AACxE,MAAM,MAAM,eAAe,GAAG,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;AAErE,MAAM,WAAW,eAAe;IAC9B,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,YAAY,CAAC;IACnB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB;oCACgC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,YAAY,CAAC;IACnB,OAAO,EAAE,eAAe,CAAC;IACzB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;CACnB;AAoCD;;;;;GAKG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,QAAQ,EAAE,MAAM;IAU5B;;;qDAGiD;IACjD,eAAe,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO;IAW9C;qEACiE;IACjE,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAK9C,2EAA2E;IAC3E,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAUxC,2EAA2E;IAC3E,mBAAmB,IAAI,eAAe,EAAE;IAMxC;qFACiF;IACjF,WAAW,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI;IAMnC,mFAAmF;IACnF,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAUpC;;;4EAGwE;IACxE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAOzD,mFAAmF;IACnF,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAM/C;yFACqF;IACrF,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO;IAO3E,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,SAAS;CAIlB"}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionRecoveryChannel — the cross-process request/ack channel for codex
|
|
3
|
+
* session-wedge SELF-recovery.
|
|
4
|
+
*
|
|
5
|
+
* The constraint that shapes this (grounded 2026-06-03):
|
|
6
|
+
* - The DETECTOR (`StuckInputSentinel`) runs in the SERVER process.
|
|
7
|
+
* - The RESTART AUTHORITY (`ServerSupervisor` + `TelegramLifeline.replayQueue`)
|
|
8
|
+
* runs in the LIFELINE process.
|
|
9
|
+
* So the sentinel cannot restart anything directly — it must cross the process
|
|
10
|
+
* boundary. This channel is that boundary, modelled on the existing
|
|
11
|
+
* version-skew signal-file pattern (`state/lifeline-restart-requested.json`)
|
|
12
|
+
* but dedicated to per-session recovery and SAFE for two processes to share:
|
|
13
|
+
*
|
|
14
|
+
* - `state/session-recovery-requested.json` — SOLE writer is the SERVER
|
|
15
|
+
* (sentinel requests recovery). The lifeline only READS it.
|
|
16
|
+
* - `state/session-recovery-acked.json` — SOLE writer is the LIFELINE
|
|
17
|
+
* (it acks progress/outcome). The server only READS it.
|
|
18
|
+
*
|
|
19
|
+
* Single-writer-per-file means there is never a cross-process write race; both
|
|
20
|
+
* writes are atomic (temp+rename via SafeFsExecutor). Requests/acks are keyed by
|
|
21
|
+
* sessionId and carry an `attemptId` so a stale ack from a prior escalation
|
|
22
|
+
* episode can't be mistaken for the current one.
|
|
23
|
+
*
|
|
24
|
+
* Signal vs Authority: the request is a SIGNAL the server emits; the lifeline
|
|
25
|
+
* holds the AUTHORITY to act (or decline). This module is pure I/O — it performs
|
|
26
|
+
* NO restart and makes NO decision. Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
|
|
27
|
+
*/
|
|
28
|
+
import * as path from 'node:path';
|
|
29
|
+
import * as fs from 'node:fs';
|
|
30
|
+
import { SafeFsExecutor } from './SafeFsExecutor.js';
|
|
31
|
+
const REQUEST_RELPATH = path.join('state', 'session-recovery-requested.json');
|
|
32
|
+
const ACK_RELPATH = path.join('state', 'session-recovery-acked.json');
|
|
33
|
+
const COOLDOWN_RELPATH = path.join('state', 'session-recovery-cooldown.json');
|
|
34
|
+
function readJsonOrNull(filePath) {
|
|
35
|
+
try {
|
|
36
|
+
if (!fs.existsSync(filePath))
|
|
37
|
+
return null;
|
|
38
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
39
|
+
if (!raw.trim())
|
|
40
|
+
return null;
|
|
41
|
+
return JSON.parse(raw);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// Corrupt/torn file → treat as empty rather than throw. A subsequent write
|
|
45
|
+
// overwrites it cleanly.
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Cross-process recovery request/ack channel. Construct on BOTH sides with the
|
|
51
|
+
* same stateDir; the server uses the request* / readAck methods, the lifeline
|
|
52
|
+
* uses the readPendingRequests / ack* methods. The class enforces the
|
|
53
|
+
* single-writer-per-file invariant by never writing the file the other side owns.
|
|
54
|
+
*/
|
|
55
|
+
export class SessionRecoveryChannel {
|
|
56
|
+
requestPath;
|
|
57
|
+
ackPath;
|
|
58
|
+
cooldownPath;
|
|
59
|
+
constructor(stateDir) {
|
|
60
|
+
// stateDir is the agent's .instar dir; the signal files live under
|
|
61
|
+
// <stateDir>/state/ alongside the version-skew signal.
|
|
62
|
+
this.requestPath = path.join(stateDir, REQUEST_RELPATH);
|
|
63
|
+
this.ackPath = path.join(stateDir, ACK_RELPATH);
|
|
64
|
+
this.cooldownPath = path.join(stateDir, COOLDOWN_RELPATH);
|
|
65
|
+
}
|
|
66
|
+
// ---- SERVER (sentinel) side — sole writer of the request file ----
|
|
67
|
+
/** Emit/refresh a recovery request for a session. Idempotent per
|
|
68
|
+
* (sessionId, attemptId): re-requesting the SAME attempt is a no-op write of
|
|
69
|
+
* identical content; a NEW attemptId replaces the prior request for that
|
|
70
|
+
* session. Returns true if the file changed. */
|
|
71
|
+
requestRecovery(req) {
|
|
72
|
+
const file = readJsonOrNull(this.requestPath) ?? { version: 1, requests: {} };
|
|
73
|
+
const existing = file.requests[req.sessionId];
|
|
74
|
+
if (existing && existing.attemptId === req.attemptId && existing.tier === req.tier) {
|
|
75
|
+
return false; // identical in-flight request — nothing to do
|
|
76
|
+
}
|
|
77
|
+
file.requests[req.sessionId] = req;
|
|
78
|
+
this.writeRequests(file);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
/** Read the lifeline's ack for a session (null if none). The server matches
|
|
82
|
+
* `attemptId` to confirm the ack is for the CURRENT attempt. */
|
|
83
|
+
readAck(sessionId) {
|
|
84
|
+
const file = readJsonOrNull(this.ackPath);
|
|
85
|
+
return file?.acks?.[sessionId] ?? null;
|
|
86
|
+
}
|
|
87
|
+
/** Clear a session's request after recovery is verified (or abandoned). */
|
|
88
|
+
clearRequest(sessionId) {
|
|
89
|
+
const file = readJsonOrNull(this.requestPath);
|
|
90
|
+
if (!file || !file.requests[sessionId])
|
|
91
|
+
return false;
|
|
92
|
+
delete file.requests[sessionId];
|
|
93
|
+
this.writeRequests(file);
|
|
94
|
+
return true;
|
|
95
|
+
}
|
|
96
|
+
// ---- LIFELINE side — sole writer of the ack file ----
|
|
97
|
+
/** All pending recovery requests (server-emitted). Empty array if none. */
|
|
98
|
+
readPendingRequests() {
|
|
99
|
+
const file = readJsonOrNull(this.requestPath);
|
|
100
|
+
if (!file?.requests)
|
|
101
|
+
return [];
|
|
102
|
+
return Object.values(file.requests);
|
|
103
|
+
}
|
|
104
|
+
/** Record/refresh the lifeline's ack for a session. Sole writer of the ack
|
|
105
|
+
* file; overwrites the prior ack for that session (the latest outcome wins). */
|
|
106
|
+
ackRecovery(ack) {
|
|
107
|
+
const file = readJsonOrNull(this.ackPath) ?? { version: 1, acks: {} };
|
|
108
|
+
file.acks[ack.sessionId] = ack;
|
|
109
|
+
this.writeAcks(file);
|
|
110
|
+
}
|
|
111
|
+
/** Drop a session's ack once the server has consumed it (keeps the file small). */
|
|
112
|
+
clearAck(sessionId) {
|
|
113
|
+
const file = readJsonOrNull(this.ackPath);
|
|
114
|
+
if (!file || !file.acks[sessionId])
|
|
115
|
+
return false;
|
|
116
|
+
delete file.acks[sessionId];
|
|
117
|
+
this.writeAcks(file);
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
// ---- LIFELINE side — durable restart cooldown (sole writer of the cooldown file) ----
|
|
121
|
+
/** Record that the lifeline performed a tier-C restart for a session. DURABLE:
|
|
122
|
+
* survives the server restart that wipes the sentinel's in-memory bound, so it
|
|
123
|
+
* is the only thing standing between a restart-can't-fix wedge and an infinite
|
|
124
|
+
* restart loop. `atEpochMs` is supplied by the caller (Date.now()). */
|
|
125
|
+
recordRestart(sessionId, atEpochMs) {
|
|
126
|
+
const file = readJsonOrNull(this.cooldownPath) ?? { version: 1, restarts: {} };
|
|
127
|
+
file.restarts[sessionId] = atEpochMs;
|
|
128
|
+
this.ensureDir(this.cooldownPath);
|
|
129
|
+
SafeFsExecutor.atomicWriteJsonSync(this.cooldownPath, file, { operation: 'SessionRecoveryChannel.recordRestart' });
|
|
130
|
+
}
|
|
131
|
+
/** Epoch-ms of the last tier-C restart for a session, or null if none recorded. */
|
|
132
|
+
lastRestartAt(sessionId) {
|
|
133
|
+
const file = readJsonOrNull(this.cooldownPath);
|
|
134
|
+
const v = file?.restarts?.[sessionId];
|
|
135
|
+
return typeof v === 'number' ? v : null;
|
|
136
|
+
}
|
|
137
|
+
/** True if a tier-C restart for this session happened within `cooldownMs` of
|
|
138
|
+
* `nowMs` — the lifeline MUST NOT restart again while this is true (loop guard). */
|
|
139
|
+
isInCooldown(sessionId, nowMs, cooldownMs) {
|
|
140
|
+
const last = this.lastRestartAt(sessionId);
|
|
141
|
+
return last !== null && nowMs - last < cooldownMs;
|
|
142
|
+
}
|
|
143
|
+
// ---- internals ----
|
|
144
|
+
ensureDir(filePath) {
|
|
145
|
+
const dir = path.dirname(filePath);
|
|
146
|
+
if (!fs.existsSync(dir))
|
|
147
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
148
|
+
}
|
|
149
|
+
writeRequests(file) {
|
|
150
|
+
this.ensureDir(this.requestPath);
|
|
151
|
+
SafeFsExecutor.atomicWriteJsonSync(this.requestPath, file, { operation: 'SessionRecoveryChannel.writeRequests' });
|
|
152
|
+
}
|
|
153
|
+
writeAcks(file) {
|
|
154
|
+
this.ensureDir(this.ackPath);
|
|
155
|
+
SafeFsExecutor.atomicWriteJsonSync(this.ackPath, file, { operation: 'SessionRecoveryChannel.writeAcks' });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=SessionRecoveryChannel.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SessionRecoveryChannel.js","sourceRoot":"","sources":["../../src/core/SessionRecoveryChannel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAuCrD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iCAAiC,CAAC,CAAC;AAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,6BAA6B,CAAC,CAAC;AACtE,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,gCAAgC,CAAC,CAAC;AAmB9E,SAAS,cAAc,CAAI,QAAgB;IACzC,IAAI,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,2EAA2E;QAC3E,yBAAyB;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,sBAAsB;IAChB,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,YAAY,CAAS;IAEtC,YAAY,QAAgB;QAC1B,mEAAmE;QACnE,uDAAuD;QACvD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAED,qEAAqE;IAErE;;;qDAGiD;IACjD,eAAe,CAAC,GAAoB;QAClC,MAAM,IAAI,GAAG,cAAc,CAAc,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC;YACnF,OAAO,KAAK,CAAC,CAAC,8CAA8C;QAC9D,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;QACnC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED;qEACiE;IACjE,OAAO,CAAC,SAAiB;QACvB,MAAM,IAAI,GAAG,cAAc,CAAU,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,OAAO,IAAI,EAAE,IAAI,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;IACzC,CAAC;IAED,2EAA2E;IAC3E,YAAY,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,cAAc,CAAc,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAC;QACrD,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,wDAAwD;IAExD,2EAA2E;IAC3E,mBAAmB;QACjB,MAAM,IAAI,GAAG,cAAc,CAAc,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC,IAAI,EAAE,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED;qFACiF;IACjF,WAAW,CAAC,GAAgB;QAC1B,MAAM,IAAI,GAAG,cAAc,CAAU,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;QAC/E,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,mFAAmF;IACnF,QAAQ,CAAC,SAAiB;QACxB,MAAM,IAAI,GAAG,cAAc,CAAU,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAC;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,wFAAwF;IAExF;;;4EAGwE;IACxE,aAAa,CAAC,SAAiB,EAAE,SAAiB;QAChD,MAAM,IAAI,GAAG,cAAc,CAAe,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAC7F,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;QACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAClC,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,sCAAsC,EAAE,CAAC,CAAC;IACrH,CAAC;IAED,mFAAmF;IACnF,aAAa,CAAC,SAAiB;QAC7B,MAAM,IAAI,GAAG,cAAc,CAAe,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,MAAM,CAAC,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,SAAS,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1C,CAAC;IAED;yFACqF;IACrF,YAAY,CAAC,SAAiB,EAAE,KAAa,EAAE,UAAkB;QAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC3C,OAAO,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,IAAI,GAAG,UAAU,CAAC;IACpD,CAAC;IAED,sBAAsB;IAEd,SAAS,CAAC,QAAgB;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IAEO,aAAa,CAAC,IAAiB;QACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACjC,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,sCAAsC,EAAE,CAAC,CAAC;IACpH,CAAC;IAEO,SAAS,CAAC,IAAa;QAC7B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,kCAAkC,EAAE,CAAC,CAAC;IAC5G,CAAC;CACF"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionRecoveryConsumer — the LIFELINE-side executor for codex session-wedge
|
|
3
|
+
* self-recovery (tier C). It is the AUTHORITY half of the Signal-vs-Authority
|
|
4
|
+
* split: the server-process sentinel emits requests via SessionRecoveryChannel;
|
|
5
|
+
* this consumer (run from TelegramLifeline, where ServerSupervisor lives) reads
|
|
6
|
+
* them and actually performs the server restart + queue replay.
|
|
7
|
+
*
|
|
8
|
+
* The logic is deliberately separated from TelegramLifeline and takes its
|
|
9
|
+
* effects as injected callbacks (`restart`, `replay`) + an injectable clock, so
|
|
10
|
+
* it is unit-testable without standing up a lifeline. TelegramLifeline wires the
|
|
11
|
+
* real `supervisor.performGracefulRestart` and `replayQueue` and calls `tick()`
|
|
12
|
+
* on an interval.
|
|
13
|
+
*
|
|
14
|
+
* Safety properties (this is the highest-blast-radius path — it restarts the
|
|
15
|
+
* whole agent server):
|
|
16
|
+
* - DRY-RUN first: `dryRun:true` logs what it WOULD do and acks recovered
|
|
17
|
+
* without restarting. Ships in this mode.
|
|
18
|
+
* - DURABLE cooldown: before any restart it checks `channel.isInCooldown`. A
|
|
19
|
+
* tier-C restart wipes the sentinel's in-memory bound, so this durable guard
|
|
20
|
+
* is the only thing preventing a restart-can't-fix wedge from looping.
|
|
21
|
+
* - Records the restart BEFORE acting, so a crash mid-restart still counts
|
|
22
|
+
* against the cooldown.
|
|
23
|
+
* - Dedups on (sessionId, attemptId) so a lingering request (the sentinel was
|
|
24
|
+
* restarted before it could clear it) is not re-executed.
|
|
25
|
+
* - Never writes the request file (the server owns it) — ack + cooldown only.
|
|
26
|
+
*
|
|
27
|
+
* Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
|
|
28
|
+
*/
|
|
29
|
+
import type { SessionRecoveryChannel } from './SessionRecoveryChannel.js';
|
|
30
|
+
export interface SessionRecoveryConsumerOptions {
|
|
31
|
+
channel: SessionRecoveryChannel;
|
|
32
|
+
/** Perform the server restart (TelegramLifeline → supervisor.performGracefulRestart). */
|
|
33
|
+
restart: (reason: string) => Promise<boolean>;
|
|
34
|
+
/** Replay the queued messages after the server is back (TelegramLifeline.replayQueue). */
|
|
35
|
+
replay: () => void;
|
|
36
|
+
/** When true, log only — never actually restart. Ships true. */
|
|
37
|
+
dryRun: boolean;
|
|
38
|
+
/** Minimum gap between tier-C restarts for the same session (loop guard). */
|
|
39
|
+
cooldownMs: number;
|
|
40
|
+
/** Injectable clock (epoch ms). Defaults to Date.now. */
|
|
41
|
+
now?: () => number;
|
|
42
|
+
/** Injectable logger. Defaults to console.log. */
|
|
43
|
+
log?: (msg: string) => void;
|
|
44
|
+
}
|
|
45
|
+
export declare class SessionRecoveryConsumer {
|
|
46
|
+
private readonly channel;
|
|
47
|
+
private readonly restart;
|
|
48
|
+
private readonly replay;
|
|
49
|
+
private readonly dryRun;
|
|
50
|
+
private readonly cooldownMs;
|
|
51
|
+
private readonly now;
|
|
52
|
+
private readonly log;
|
|
53
|
+
constructor(opts: SessionRecoveryConsumerOptions);
|
|
54
|
+
/** One pass: handle every pending tier-C recovery request. Best-effort — a
|
|
55
|
+
* read/exec failure on one request never throws out of tick(). */
|
|
56
|
+
tick(): Promise<void>;
|
|
57
|
+
private handle;
|
|
58
|
+
private ack;
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=SessionRecoveryConsumer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SessionRecoveryConsumer.d.ts","sourceRoot":"","sources":["../../src/core/SessionRecoveryConsumer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,sBAAsB,EAAmB,MAAM,6BAA6B,CAAC;AAE3F,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,sBAAsB,CAAC;IAChC,yFAAyF;IACzF,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9C,0FAA0F;IAC1F,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,gEAAgE;IAChE,MAAM,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,kDAAkD;IAClD,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;IACjD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAwB;gBAEhC,IAAI,EAAE,8BAA8B;IAUhD;uEACmE;IAC7D,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAiBb,MAAM;IAqCpB,OAAO,CAAC,GAAG;CAUZ"}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionRecoveryConsumer — the LIFELINE-side executor for codex session-wedge
|
|
3
|
+
* self-recovery (tier C). It is the AUTHORITY half of the Signal-vs-Authority
|
|
4
|
+
* split: the server-process sentinel emits requests via SessionRecoveryChannel;
|
|
5
|
+
* this consumer (run from TelegramLifeline, where ServerSupervisor lives) reads
|
|
6
|
+
* them and actually performs the server restart + queue replay.
|
|
7
|
+
*
|
|
8
|
+
* The logic is deliberately separated from TelegramLifeline and takes its
|
|
9
|
+
* effects as injected callbacks (`restart`, `replay`) + an injectable clock, so
|
|
10
|
+
* it is unit-testable without standing up a lifeline. TelegramLifeline wires the
|
|
11
|
+
* real `supervisor.performGracefulRestart` and `replayQueue` and calls `tick()`
|
|
12
|
+
* on an interval.
|
|
13
|
+
*
|
|
14
|
+
* Safety properties (this is the highest-blast-radius path — it restarts the
|
|
15
|
+
* whole agent server):
|
|
16
|
+
* - DRY-RUN first: `dryRun:true` logs what it WOULD do and acks recovered
|
|
17
|
+
* without restarting. Ships in this mode.
|
|
18
|
+
* - DURABLE cooldown: before any restart it checks `channel.isInCooldown`. A
|
|
19
|
+
* tier-C restart wipes the sentinel's in-memory bound, so this durable guard
|
|
20
|
+
* is the only thing preventing a restart-can't-fix wedge from looping.
|
|
21
|
+
* - Records the restart BEFORE acting, so a crash mid-restart still counts
|
|
22
|
+
* against the cooldown.
|
|
23
|
+
* - Dedups on (sessionId, attemptId) so a lingering request (the sentinel was
|
|
24
|
+
* restarted before it could clear it) is not re-executed.
|
|
25
|
+
* - Never writes the request file (the server owns it) — ack + cooldown only.
|
|
26
|
+
*
|
|
27
|
+
* Spec: docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md
|
|
28
|
+
*/
|
|
29
|
+
export class SessionRecoveryConsumer {
|
|
30
|
+
channel;
|
|
31
|
+
restart;
|
|
32
|
+
replay;
|
|
33
|
+
dryRun;
|
|
34
|
+
cooldownMs;
|
|
35
|
+
now;
|
|
36
|
+
log;
|
|
37
|
+
constructor(opts) {
|
|
38
|
+
this.channel = opts.channel;
|
|
39
|
+
this.restart = opts.restart;
|
|
40
|
+
this.replay = opts.replay;
|
|
41
|
+
this.dryRun = opts.dryRun;
|
|
42
|
+
this.cooldownMs = opts.cooldownMs;
|
|
43
|
+
this.now = opts.now ?? (() => Date.now());
|
|
44
|
+
this.log = opts.log ?? ((m) => console.log(m));
|
|
45
|
+
}
|
|
46
|
+
/** One pass: handle every pending tier-C recovery request. Best-effort — a
|
|
47
|
+
* read/exec failure on one request never throws out of tick(). */
|
|
48
|
+
async tick() {
|
|
49
|
+
let pending;
|
|
50
|
+
try {
|
|
51
|
+
pending = this.channel.readPendingRequests();
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
for (const req of pending) {
|
|
57
|
+
if (req.tier !== 'server-restart-replay')
|
|
58
|
+
continue; // lifeline only owns tier C
|
|
59
|
+
try {
|
|
60
|
+
await this.handle(req);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
this.ack(req, 'failed', `executor error: ${err instanceof Error ? err.message : String(err)}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async handle(req) {
|
|
68
|
+
const now = this.now();
|
|
69
|
+
// Dedup: this exact attempt already reached a terminal ack → done.
|
|
70
|
+
const prior = this.channel.readAck(req.sessionId);
|
|
71
|
+
if (prior && prior.attemptId === req.attemptId && prior.outcome !== 'in-progress') {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
// Durable cooldown — the cross-restart loop guard.
|
|
75
|
+
if (this.channel.isInCooldown(req.sessionId, now, this.cooldownMs)) {
|
|
76
|
+
this.ack(req, 'failed', 'restart cooldown active (loop guard)');
|
|
77
|
+
this.log(`[SessionRecovery] cooldown active for ${req.sessionId} — refusing restart`);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
// Mark in-progress and record the restart BEFORE acting (so a crash mid-restart
|
|
81
|
+
// still counts against the cooldown).
|
|
82
|
+
this.ack(req, 'in-progress', this.dryRun ? 'dry-run' : 'restarting server + replay');
|
|
83
|
+
this.channel.recordRestart(req.sessionId, now);
|
|
84
|
+
if (this.dryRun) {
|
|
85
|
+
this.log(`[SessionRecovery] [DRY-RUN] would restart server + replay for wedged session ${req.sessionId} (${req.reason})`);
|
|
86
|
+
this.ack(req, 'recovered', 'dry-run: no restart performed');
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
this.log(`[SessionRecovery] restarting server + replay for wedged session ${req.sessionId} (${req.reason})`);
|
|
90
|
+
const ok = await this.restart(`codex-wedge-recovery: ${req.sessionId}`);
|
|
91
|
+
if (ok) {
|
|
92
|
+
this.replay();
|
|
93
|
+
this.ack(req, 'recovered', 'server restarted + queue replayed');
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
this.ack(req, 'failed', 'graceful restart returned false');
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
ack(req, outcome, detail) {
|
|
100
|
+
this.channel.ackRecovery({
|
|
101
|
+
sessionId: req.sessionId,
|
|
102
|
+
attemptId: req.attemptId,
|
|
103
|
+
tier: req.tier,
|
|
104
|
+
outcome,
|
|
105
|
+
detail,
|
|
106
|
+
updatedAt: new Date(this.now()).toISOString(),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
//# sourceMappingURL=SessionRecoveryConsumer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SessionRecoveryConsumer.js","sourceRoot":"","sources":["../../src/core/SessionRecoveryConsumer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAoBH,MAAM,OAAO,uBAAuB;IACjB,OAAO,CAAyB;IAChC,OAAO,CAAuC;IAC9C,MAAM,CAAa;IACnB,MAAM,CAAU;IAChB,UAAU,CAAS;IACnB,GAAG,CAAe;IAClB,GAAG,CAAwB;IAE5C,YAAY,IAAoC;QAC9C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;uEACmE;IACnE,KAAK,CAAC,IAAI;QACR,IAAI,OAA0B,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,uBAAuB;gBAAE,SAAS,CAAC,4BAA4B;YAChF,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAoB;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,mEAAmE;QACnE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,aAAa,EAAE,CAAC;YAClF,OAAO;QACT,CAAC;QAED,mDAAmD;QACnD,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACnE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,sCAAsC,CAAC,CAAC;YAChE,IAAI,CAAC,GAAG,CAAC,yCAAyC,GAAG,CAAC,SAAS,qBAAqB,CAAC,CAAC;YACtF,OAAO;QACT,CAAC;QAED,gFAAgF;QAChF,sCAAsC;QACtC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC;QACrF,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,gFAAgF,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;YAC1H,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,+BAA+B,CAAC,CAAC;YAC5D,OAAO;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,mEAAmE,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7G,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,yBAAyB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACxE,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,mCAAmC,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,iCAAiC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,GAAG,CAAC,GAAoB,EAAE,OAA+C,EAAE,MAAc;QAC/F,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YACvB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,OAAO;YACP,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE;SAC9C,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -49,6 +49,7 @@
|
|
|
49
49
|
* silently disappear after a server crash."
|
|
50
50
|
*/
|
|
51
51
|
import type { SessionManager } from './SessionManager.js';
|
|
52
|
+
import { SessionRecoveryChannel, type RecoveryTier } from './SessionRecoveryChannel.js';
|
|
52
53
|
export interface StuckInputSentinelOptions {
|
|
53
54
|
/** Poll interval. Default 10s. */
|
|
54
55
|
tickMs?: number;
|
|
@@ -63,6 +64,18 @@ export interface StuckInputSentinelOptions {
|
|
|
63
64
|
stateDir: string;
|
|
64
65
|
/** Disable persistence to disk (used by unit tests). */
|
|
65
66
|
noPersist?: boolean;
|
|
67
|
+
/** Cross-process recovery channel. When provided AND escalationEnabled, the
|
|
68
|
+
* sentinel escalates PAST the keypress ladder (tier C: it requests a server
|
|
69
|
+
* restart + replay from the lifeline, which owns ServerSupervisor). When
|
|
70
|
+
* absent the sentinel keeps its legacy behavior (exhaust → stop). */
|
|
71
|
+
recoveryChannel?: SessionRecoveryChannel;
|
|
72
|
+
/** Master gate for the deeper-tier escalation. Default false (dark): the
|
|
73
|
+
* sentinel marks the record exhausted after the keypress ladder, exactly as
|
|
74
|
+
* before. Set true (via config) to enable tier-C recovery requests. */
|
|
75
|
+
escalationEnabled?: boolean;
|
|
76
|
+
/** How many ticks to wait for the lifeline to ack a tier-C request before
|
|
77
|
+
* giving up (bounded — no restart loop). Default 6. */
|
|
78
|
+
escalationTimeoutTicks?: number;
|
|
66
79
|
}
|
|
67
80
|
/** In-memory record of what the sentinel has seen for a given session. */
|
|
68
81
|
interface SessionStuckRecord {
|
|
@@ -77,6 +90,16 @@ interface SessionStuckRecord {
|
|
|
77
90
|
/** Whether this record is exhausted (max attempts reached, won't fire
|
|
78
91
|
* again until the prompt text changes). */
|
|
79
92
|
exhausted: boolean;
|
|
93
|
+
/** Deeper-tier escalation phase (after the keypress ladder exhausts).
|
|
94
|
+
* - 'none': not escalating (keypress ladder still running or escalation off).
|
|
95
|
+
* - 'requested': a tier-C recovery request is in flight to the lifeline;
|
|
96
|
+
* subsequent ticks read the ack.
|
|
97
|
+
* - 'recovered'/'gave-up': terminal. */
|
|
98
|
+
recoveryPhase: 'none' | 'requested' | 'recovered' | 'gave-up';
|
|
99
|
+
/** attemptId of the in-flight recovery request (matches the channel ack). */
|
|
100
|
+
recoveryAttemptId?: string;
|
|
101
|
+
/** tickCounter value when the request was emitted (for the bounded wait). */
|
|
102
|
+
recoveryRequestedTick?: number;
|
|
80
103
|
}
|
|
81
104
|
/** Event row written to stuck-input-events.jsonl on each fire. */
|
|
82
105
|
export interface StuckInputEvent {
|
|
@@ -84,8 +107,10 @@ export interface StuckInputEvent {
|
|
|
84
107
|
session: string;
|
|
85
108
|
promptText: string;
|
|
86
109
|
attempt: number;
|
|
87
|
-
action: 'Enter' | 'C-m' | 'Enter-sleep-Enter';
|
|
88
|
-
outcome: 'fired' | 'fire-error';
|
|
110
|
+
action: 'Enter' | 'C-m' | 'Enter-sleep-Enter' | 'escalate-request' | 'escalate-recovered' | 'escalate-failed' | 'escalate-timeout';
|
|
111
|
+
outcome: 'fired' | 'fire-error' | 'escalated' | 'recovered' | 'gave-up';
|
|
112
|
+
/** Recovery tier for escalation events (absent for keypress events). */
|
|
113
|
+
tier?: RecoveryTier;
|
|
89
114
|
error?: string;
|
|
90
115
|
}
|
|
91
116
|
export declare class StuckInputSentinel {
|
|
@@ -96,11 +121,16 @@ export declare class StuckInputSentinel {
|
|
|
96
121
|
private readonly stateDir;
|
|
97
122
|
private readonly noPersist;
|
|
98
123
|
private readonly eventsLogPath;
|
|
124
|
+
private readonly recoveryChannel;
|
|
125
|
+
private readonly escalationEnabled;
|
|
126
|
+
private readonly escalationTimeoutTicks;
|
|
99
127
|
/** Per-session sticky state. Keyed by tmuxSession. */
|
|
100
128
|
private readonly records;
|
|
101
129
|
private intervalHandle;
|
|
102
130
|
private tickInProgress;
|
|
103
131
|
private degradationReportedOnce;
|
|
132
|
+
/** Monotonic tick counter — drives the bounded escalation wait. */
|
|
133
|
+
private tickCounter;
|
|
104
134
|
constructor(sessionManager: SessionManager, opts: StuckInputSentinelOptions);
|
|
105
135
|
start(): void;
|
|
106
136
|
stop(): void;
|
|
@@ -149,6 +179,33 @@ export declare class StuckInputSentinel {
|
|
|
149
179
|
/** Map attempt index → human-readable action name (matches
|
|
150
180
|
* SessionManager.fireStuckInputRecovery's escalation). */
|
|
151
181
|
static actionForAttempt(attempt: number): StuckInputEvent['action'];
|
|
182
|
+
/**
|
|
183
|
+
* Clear a session's in-flight tier-C recovery request when the session has
|
|
184
|
+
* recovered out-of-band — it's now actively working (the wedge drained) or it
|
|
185
|
+
* is gone. The sentinel is the sole writer of the request file, so it (not the
|
|
186
|
+
* lifeline) does this cleanup. No-op unless a request is actually in flight.
|
|
187
|
+
* This closes the "request lingers after a successful drain" path that the
|
|
188
|
+
* stuck-detection branches would otherwise skip (they early-return before
|
|
189
|
+
* handleEscalation ever reads the ack).
|
|
190
|
+
*/
|
|
191
|
+
private finalizePendingRecovery;
|
|
192
|
+
/**
|
|
193
|
+
* Deeper-tier escalation, invoked once the keypress ladder (tier A) is
|
|
194
|
+
* exhausted but the prompt is still stuck. This is the codex session-wedge
|
|
195
|
+
* SELF-recovery path: the sentinel (server process) cannot restart the server
|
|
196
|
+
* itself — ServerSupervisor lives in the lifeline process — so it REQUESTS a
|
|
197
|
+
* tier-C recovery (server restart + queue replay) through SessionRecoveryChannel
|
|
198
|
+
* and the lifeline executes + acks. The sentinel here only signals, polls the
|
|
199
|
+
* ack, verifies, and bounds the wait (no restart loop). Spec:
|
|
200
|
+
* docs/specs/CODEX-SESSION-WEDGE-SELF-RECOVERY.md.
|
|
201
|
+
*
|
|
202
|
+
* Dark by default: with escalation disabled or no channel, this reproduces the
|
|
203
|
+
* legacy behavior exactly (mark the record exhausted and stop).
|
|
204
|
+
*
|
|
205
|
+
* NOTE: a gentler server-side tier B (re-inject the full pending message before
|
|
206
|
+
* requesting a restart) is a planned refinement; this first cut goes A→C.
|
|
207
|
+
*/
|
|
208
|
+
private handleEscalation;
|
|
152
209
|
/** Append one event row to the JSONL log. Best-effort. */
|
|
153
210
|
private recordEvent;
|
|
154
211
|
/** Test helper: read the in-memory record for a session. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StuckInputSentinel.d.ts","sourceRoot":"","sources":["../../src/core/StuckInputSentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"StuckInputSentinel.d.ts","sourceRoot":"","sources":["../../src/core/StuckInputSentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAG1D,OAAO,EAAE,sBAAsB,EAAE,KAAK,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAExF,MAAM,WAAW,yBAAyB;IACxC,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;0CAEsC;IACtC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;wEACoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;0EAGsE;IACtE,eAAe,CAAC,EAAE,sBAAsB,CAAC;IACzC;;4EAEwE;IACxE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;4DACwD;IACxD,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,0EAA0E;AAC1E,UAAU,kBAAkB;IAC1B,kEAAkE;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,CAAC;IACzB,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC;IACjB;gDAC4C;IAC5C,SAAS,EAAE,OAAO,CAAC;IACnB;;;;8CAI0C;IAC1C,aAAa,EAAE,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC;IAC9D,6EAA6E;IAC7E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,kEAAkE;AAClE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,GAAG,KAAK,GAAG,mBAAmB,GACzC,kBAAkB,GAAG,oBAAoB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;IACvF,OAAO,EAAE,OAAO,GAAG,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC;IACxE,wEAAwE;IACxE,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAeD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAChE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAU;IAC5C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAEhD,sDAAsD;IACtD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8C;IAEtE,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,uBAAuB,CAAS;IACxC,mEAAmE;IACnE,OAAO,CAAC,WAAW,CAAK;gBAEZ,cAAc,EAAE,cAAc,EAAE,IAAI,EAAE,yBAAyB;IAa3E,KAAK,IAAI,IAAI;IAiBb,IAAI,IAAI,IAAI;IAOZ;;;;;;OAMG;IACH,IAAI,IAAI,IAAI;IA2BZ,+DAA+D;IAC/D,OAAO,CAAC,eAAe;IAsHvB;;;6EAGyE;IACzE,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAoB9C;;;;;;;;;;;;;;;;;;;;;iCAqB6B;IAC7B,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAK5C;;oDAEgD;IAChD,OAAO,CAAC,YAAY;IAUpB;+DAC2D;IAC3D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC;IAMnE;;;;;;;;OAQG;IACH,OAAO,CAAC,uBAAuB;IAkB/B;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,gBAAgB;IAwExB,0DAA0D;IAC1D,OAAO,CAAC,WAAW;IAUnB,4DAA4D;IAC5D,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAIrE,8CAA8C;IAC9C,YAAY,IAAI,IAAI;CAIrB"}
|