instar 1.2.40 → 1.2.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +32 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +10 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/SessionManager.d.ts +5 -0
- package/dist/core/SessionManager.d.ts.map +1 -1
- package/dist/core/SessionManager.js +7 -0
- package/dist/core/SessionManager.js.map +1 -1
- package/dist/core/WorktreeManager.d.ts +11 -0
- package/dist/core/WorktreeManager.d.ts.map +1 -1
- package/dist/core/WorktreeManager.js +69 -0
- package/dist/core/WorktreeManager.js.map +1 -1
- package/dist/core/types.d.ts +35 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/monitoring/ActiveWorkSilenceSentinel.d.ts +88 -0
- package/dist/monitoring/ActiveWorkSilenceSentinel.d.ts.map +1 -0
- package/dist/monitoring/ActiveWorkSilenceSentinel.js +181 -0
- package/dist/monitoring/ActiveWorkSilenceSentinel.js.map +1 -0
- package/dist/monitoring/SocketDisconnectSentinel.d.ts +113 -0
- package/dist/monitoring/SocketDisconnectSentinel.d.ts.map +1 -0
- package/dist/monitoring/SocketDisconnectSentinel.js +235 -0
- package/dist/monitoring/SocketDisconnectSentinel.js.map +1 -0
- package/dist/monitoring/sentinelWiring.d.ts +82 -0
- package/dist/monitoring/sentinelWiring.d.ts.map +1 -0
- package/dist/monitoring/sentinelWiring.js +154 -0
- package/dist/monitoring/sentinelWiring.js.map +1 -0
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +3 -3
- package/upgrades/1.2.40.md +15 -0
- package/upgrades/1.2.41.md +34 -0
- package/upgrades/1.2.42.md +34 -0
- package/upgrades/side-effects/silently-stopped-trio-wiring.md +185 -0
- package/upgrades/side-effects/silently-stopped-trio.md +115 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ActiveWorkSilenceSentinel — detects sessions in the registry that were
|
|
3
|
+
* actively producing output and then went silent for an extended period,
|
|
4
|
+
* independent of topic binding.
|
|
5
|
+
*
|
|
6
|
+
* Closes the watchdog gap surfaced 2026-05-22: a gsd-style sub-spawned
|
|
7
|
+
* worktree session went silent for 1h16m and three existing watchdogs all
|
|
8
|
+
* missed it:
|
|
9
|
+
* - SessionWatchdog requires a long-running child (this had none).
|
|
10
|
+
* - SessionMonitor only inspects topic-bound sessions (this wasn't).
|
|
11
|
+
* - PresenceProxy wakes on inbound user messages (none arrived).
|
|
12
|
+
*
|
|
13
|
+
* This sentinel sits one layer below those: it walks the SessionRegistry
|
|
14
|
+
* directly, looking for "had output recently, now hasn't for N minutes."
|
|
15
|
+
* On match it tries one gentle nudge; if that doesn't unstick the session
|
|
16
|
+
* within the verify window, it escalates via the tone-gated /attention
|
|
17
|
+
* path.
|
|
18
|
+
*
|
|
19
|
+
* Spec: docs/specs/silently-stopped-trio.md
|
|
20
|
+
*
|
|
21
|
+
* Signal-vs-authority: the threshold check is a detector. The nudge is a
|
|
22
|
+
* bounded recovery primitive. The escalation goes through MessagingToneGate
|
|
23
|
+
* via the notify path. No new blocking authority.
|
|
24
|
+
*/
|
|
25
|
+
import { EventEmitter } from 'node:events';
|
|
26
|
+
const DEFAULT_CONFIG = {
|
|
27
|
+
enabled: true,
|
|
28
|
+
tickIntervalMs: 60_000,
|
|
29
|
+
silenceThresholdMs: 15 * 60_000,
|
|
30
|
+
verifyWindowMs: 30_000,
|
|
31
|
+
};
|
|
32
|
+
export class ActiveWorkSilenceSentinel extends EventEmitter {
|
|
33
|
+
deps;
|
|
34
|
+
cfg;
|
|
35
|
+
states = new Map();
|
|
36
|
+
verifyTimers = new Map();
|
|
37
|
+
tickHandle = null;
|
|
38
|
+
constructor(deps, cfg = {}) {
|
|
39
|
+
super();
|
|
40
|
+
this.deps = deps;
|
|
41
|
+
this.cfg = { ...DEFAULT_CONFIG, ...cfg };
|
|
42
|
+
}
|
|
43
|
+
start() {
|
|
44
|
+
if (!this.cfg.enabled || this.tickHandle)
|
|
45
|
+
return;
|
|
46
|
+
this.tickHandle = setInterval(() => this.tick(), this.cfg.tickIntervalMs);
|
|
47
|
+
// Unref so this doesn't keep the process alive on shutdown.
|
|
48
|
+
if (typeof this.tickHandle.unref === 'function')
|
|
49
|
+
this.tickHandle.unref();
|
|
50
|
+
}
|
|
51
|
+
stop() {
|
|
52
|
+
if (this.tickHandle) {
|
|
53
|
+
clearInterval(this.tickHandle);
|
|
54
|
+
this.tickHandle = null;
|
|
55
|
+
}
|
|
56
|
+
for (const t of this.verifyTimers.values())
|
|
57
|
+
this.clearTimer(t);
|
|
58
|
+
this.verifyTimers.clear();
|
|
59
|
+
this.states.clear();
|
|
60
|
+
}
|
|
61
|
+
/** Public for tests. Walk the registry and act on silence findings. */
|
|
62
|
+
tick() {
|
|
63
|
+
const now = (this.deps.now ?? Date.now)();
|
|
64
|
+
const sessions = this.deps.listSessions();
|
|
65
|
+
for (const s of sessions) {
|
|
66
|
+
if (s.paused || s.recoveryInFlight)
|
|
67
|
+
continue;
|
|
68
|
+
if (!s.lastOutputAt || s.lastOutputAt <= 0)
|
|
69
|
+
continue;
|
|
70
|
+
// Session is in the registry but never produced output → not "actively working then stopped"; skip.
|
|
71
|
+
const idleMs = now - s.lastOutputAt;
|
|
72
|
+
if (idleMs < this.cfg.silenceThresholdMs)
|
|
73
|
+
continue;
|
|
74
|
+
const existing = this.states.get(s.sessionName);
|
|
75
|
+
if (existing)
|
|
76
|
+
continue; // already handling
|
|
77
|
+
this.report(s.sessionName, s.lastOutputAt);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** Public entry: report a silence finding. Idempotent. */
|
|
81
|
+
report(sessionName, lastOutputAt) {
|
|
82
|
+
if (!this.cfg.enabled)
|
|
83
|
+
return;
|
|
84
|
+
if (this.states.has(sessionName))
|
|
85
|
+
return;
|
|
86
|
+
const now = (this.deps.now ?? Date.now)();
|
|
87
|
+
const state = {
|
|
88
|
+
sessionName,
|
|
89
|
+
detectedAt: now,
|
|
90
|
+
lastOutputAtAtDetection: lastOutputAt,
|
|
91
|
+
nudgedAt: 0,
|
|
92
|
+
status: 'detected',
|
|
93
|
+
};
|
|
94
|
+
this.states.set(sessionName, state);
|
|
95
|
+
this.emit('silence', { sessionName, idleMs: now - lastOutputAt });
|
|
96
|
+
void this.runNudge(sessionName);
|
|
97
|
+
}
|
|
98
|
+
isRecoveryActive(sessionName) {
|
|
99
|
+
const s = this.states.get(sessionName);
|
|
100
|
+
return !!s && s.status !== 'recovered' && s.status !== 'escalated';
|
|
101
|
+
}
|
|
102
|
+
listActive() {
|
|
103
|
+
return Array.from(this.states.values());
|
|
104
|
+
}
|
|
105
|
+
clear(sessionName) {
|
|
106
|
+
const t = this.verifyTimers.get(sessionName);
|
|
107
|
+
if (t)
|
|
108
|
+
this.clearTimer(t);
|
|
109
|
+
this.verifyTimers.delete(sessionName);
|
|
110
|
+
this.states.delete(sessionName);
|
|
111
|
+
}
|
|
112
|
+
async runNudge(sessionName) {
|
|
113
|
+
const state = this.states.get(sessionName);
|
|
114
|
+
if (!state)
|
|
115
|
+
return;
|
|
116
|
+
state.status = 'nudged';
|
|
117
|
+
state.nudgedAt = (this.deps.now ?? Date.now)();
|
|
118
|
+
let accepted = false;
|
|
119
|
+
try {
|
|
120
|
+
accepted = await this.deps.nudgeFn(sessionName);
|
|
121
|
+
}
|
|
122
|
+
catch (err) {
|
|
123
|
+
this.emit('nudge-error', { sessionName, err });
|
|
124
|
+
accepted = false;
|
|
125
|
+
}
|
|
126
|
+
if (!accepted) {
|
|
127
|
+
// Couldn't even nudge — escalate immediately.
|
|
128
|
+
this.escalate(sessionName);
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
const handle = this.setTimer(() => this.verifyNudge(sessionName), this.cfg.verifyWindowMs);
|
|
132
|
+
this.verifyTimers.set(sessionName, handle);
|
|
133
|
+
}
|
|
134
|
+
verifyNudge(sessionName) {
|
|
135
|
+
const state = this.states.get(sessionName);
|
|
136
|
+
if (!state)
|
|
137
|
+
return;
|
|
138
|
+
// Re-poll the registry — has lastOutputAt advanced past detection point?
|
|
139
|
+
const fresh = this.deps.listSessions().find(s => s.sessionName === sessionName);
|
|
140
|
+
if (!fresh) {
|
|
141
|
+
// Session vanished from registry — treat as recovered (probably ended cleanly).
|
|
142
|
+
state.status = 'recovered';
|
|
143
|
+
this.clear(sessionName);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
if (fresh.lastOutputAt > state.lastOutputAtAtDetection) {
|
|
147
|
+
state.status = 'recovered';
|
|
148
|
+
this.emit('recovered', sessionName);
|
|
149
|
+
this.clear(sessionName);
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
this.escalate(sessionName);
|
|
153
|
+
}
|
|
154
|
+
escalate(sessionName) {
|
|
155
|
+
const state = this.states.get(sessionName);
|
|
156
|
+
if (!state)
|
|
157
|
+
return;
|
|
158
|
+
state.status = 'escalated';
|
|
159
|
+
const minutes = Math.max(1, Math.round(((this.deps.now ?? Date.now)() - state.lastOutputAtAtDetection) / 60_000));
|
|
160
|
+
void this.notify(sessionName, `${friendlyName(sessionName)} was working and went quiet about ${minutes} minutes ago. I tried a gentle nudge and nothing came back. Want me to dig in?`);
|
|
161
|
+
this.emit('escalated', sessionName);
|
|
162
|
+
}
|
|
163
|
+
async notify(sessionName, text) {
|
|
164
|
+
try {
|
|
165
|
+
await this.deps.notifyFn(sessionName, text);
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
this.emit('notify-error', { sessionName, err });
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
setTimer(fn, ms) {
|
|
172
|
+
return (this.deps.setTimer ?? setTimeout)(fn, ms);
|
|
173
|
+
}
|
|
174
|
+
clearTimer(handle) {
|
|
175
|
+
(this.deps.clearTimer ?? clearTimeout)(handle);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function friendlyName(sessionName) {
|
|
179
|
+
return sessionName.replace(/^ai\.instar\./, '').replace(/-server$/, '').replace(/-lifeline$/, '');
|
|
180
|
+
}
|
|
181
|
+
//# sourceMappingURL=ActiveWorkSilenceSentinel.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ActiveWorkSilenceSentinel.js","sourceRoot":"","sources":["../../src/monitoring/ActiveWorkSilenceSentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA8C3C,MAAM,cAAc,GAA8C;IAChE,OAAO,EAAE,IAAI;IACb,cAAc,EAAE,MAAM;IACtB,kBAAkB,EAAE,EAAE,GAAG,MAAM;IAC/B,cAAc,EAAE,MAAM;CACvB,CAAC;AAEF,MAAM,OAAO,yBAA0B,SAAQ,YAAY;IAM5B;IALZ,GAAG,CAA4C;IAC/C,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IACzC,YAAY,GAAG,IAAI,GAAG,EAAyC,CAAC;IACzE,UAAU,GAA0C,IAAI,CAAC;IAEjE,YAA6B,IAAmC,EAAE,MAAuC,EAAE;QACzG,KAAK,EAAE,CAAC;QADmB,SAAI,GAAJ,IAAI,CAA+B;QAE9D,IAAI,CAAC,GAAG,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,GAAG,EAAE,CAAC;IAC3C,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QACjD,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC1E,4DAA4D;QAC5D,IAAI,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,KAAK,UAAU;YAAE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC3E,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAED,uEAAuE;IACvE,IAAI;QACF,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,gBAAgB;gBAAE,SAAS;YAC7C,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC;gBAAE,SAAS;YACrD,oGAAoG;YACpG,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC;YACpC,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,kBAAkB;gBAAE,SAAS;YACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YAChD,IAAI,QAAQ;gBAAE,SAAS,CAAC,mBAAmB;YAC3C,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,MAAM,CAAC,WAAmB,EAAE,YAAoB;QAC9C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,OAAO;QAC9B,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YAAE,OAAO;QACzC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAiB;YAC1B,WAAW;YACX,UAAU,EAAE,GAAG;YACf,uBAAuB,EAAE,YAAY;YACrC,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,UAAU;SACnB,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,GAAG,YAAY,EAAE,CAAC,CAAC;QAClE,KAAK,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClC,CAAC;IAED,gBAAgB,CAAC,WAAmB;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;IACrE,CAAC;IAED,UAAU;QACR,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,WAAmB;QACvB,MAAM,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC7C,IAAI,CAAC;YAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,WAAmB;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QACxB,KAAK,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAE/C,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YAC/C,QAAQ,GAAG,KAAK,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,8CAA8C;YAC9C,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC3F,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAEO,WAAW,CAAC,WAAmB;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,yEAAyE;QACzE,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,WAAW,CAAC,CAAC;QAChF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,gFAAgF;YAChF,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,uBAAuB,EAAE,CAAC;YACvD,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC7B,CAAC;IAEO,QAAQ,CAAC,WAAmB;QAClC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,uBAAuB,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;QAClH,KAAK,IAAI,CAAC,MAAM,CACd,WAAW,EACX,GAAG,YAAY,CAAC,WAAW,CAAC,qCAAqC,OAAO,gFAAgF,CACzJ,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,WAAmB,EAAE,IAAY;QACpD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,EAAc,EAAE,EAAU;QACzC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACpD,CAAC;IAEO,UAAU,CAAC,MAAqC;QACtD,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;CACF;AAED,SAAS,YAAY,CAAC,WAAmB;IACvC,OAAO,WAAW,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AACpG,CAAC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SocketDisconnectSentinel — detects Claude Code's "socket connection closed
|
|
3
|
+
* unexpectedly" message in tracked sessions and runs a bounded recovery loop.
|
|
4
|
+
*
|
|
5
|
+
* Closes the gap surfaced by the joint diagnosis on 2026-05-22: the instar
|
|
6
|
+
* repo had detectors for rate limits, quota, hundreds of patterns, but zero
|
|
7
|
+
* detectors for Claude Code's own connection-drop string. When it fires, the
|
|
8
|
+
* session freezes; nothing classifies it; no recovery runs; no user-visible
|
|
9
|
+
* alert reaches the user.
|
|
10
|
+
*
|
|
11
|
+
* Pattern mirrors `RateLimitSentinel.report` → backoff → `resumeFn` →
|
|
12
|
+
* verification → escalate, but the failure shape is simpler (no Claude
|
|
13
|
+
* Code internal retries to wait out — disconnect is immediate) so the
|
|
14
|
+
* lifecycle is shorter.
|
|
15
|
+
*
|
|
16
|
+
* Spec: docs/specs/silently-stopped-trio.md
|
|
17
|
+
*
|
|
18
|
+
* Signal-vs-authority: the regex + state machine are detectors. The
|
|
19
|
+
* notifyFn path is the existing MessagingToneGate authority (called through
|
|
20
|
+
* /attention). No new blocking authority introduced.
|
|
21
|
+
*/
|
|
22
|
+
import { EventEmitter } from 'node:events';
|
|
23
|
+
export type SocketDisconnectStatus = 'detected' | 'recovering' | 'recovered' | 'escalated';
|
|
24
|
+
export interface SocketDisconnectState {
|
|
25
|
+
sessionName: string;
|
|
26
|
+
detectedAt: number;
|
|
27
|
+
attempts: number;
|
|
28
|
+
lastInjectAt: number;
|
|
29
|
+
status: SocketDisconnectStatus;
|
|
30
|
+
}
|
|
31
|
+
export interface SocketDisconnectSentinelDeps {
|
|
32
|
+
/**
|
|
33
|
+
* Inject a recovery nudge into the session: typically Ctrl+C then Enter.
|
|
34
|
+
* Returns whether the injection was accepted.
|
|
35
|
+
*/
|
|
36
|
+
resumeFn: (sessionName: string) => Promise<boolean>;
|
|
37
|
+
/** Route a user-facing message; server.ts owns topic routing. */
|
|
38
|
+
notifyFn: (sessionName: string, text: string) => Promise<void>;
|
|
39
|
+
/** Peek at the session's most recent tmux output. Returns a string that
|
|
40
|
+
* may be empty if the session has no recent output. */
|
|
41
|
+
getRecentOutput: (sessionName: string) => string;
|
|
42
|
+
/**
|
|
43
|
+
* Optional: list the session names to scan on each tick. Required for the
|
|
44
|
+
* self-driving `start()` loop; without it the sentinel is event-driven only
|
|
45
|
+
* (caller invokes `scanSession`/`report`). The server wiring supplies this.
|
|
46
|
+
*/
|
|
47
|
+
listSessionNames?: () => string[];
|
|
48
|
+
/** Override Date.now (tests). */
|
|
49
|
+
now?: () => number;
|
|
50
|
+
/** Override timer setters (tests). */
|
|
51
|
+
setTimer?: (fn: () => void, ms: number) => ReturnType<typeof setTimeout>;
|
|
52
|
+
clearTimer?: (handle: ReturnType<typeof setTimeout>) => void;
|
|
53
|
+
}
|
|
54
|
+
export interface SocketDisconnectSentinelConfig {
|
|
55
|
+
enabled?: boolean;
|
|
56
|
+
/** Backoff staircase in ms between recovery attempts. Last value repeats. */
|
|
57
|
+
backoffScheduleMs?: number[];
|
|
58
|
+
/** Max recovery attempts before escalating. Default 4. */
|
|
59
|
+
maxAttempts?: number;
|
|
60
|
+
/** How long to wait after a nudge before declaring recovery (ms). Default 60s. */
|
|
61
|
+
verifyWindowMs?: number;
|
|
62
|
+
/** How often the self-driving loop scans every session (ms). Default 15s. */
|
|
63
|
+
tickIntervalMs?: number;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Patterns Claude Code uses when its connection to Anthropic drops. Broad
|
|
67
|
+
* intentionally — false-positive cost is one harmless nudge; false-negative
|
|
68
|
+
* cost is the silently-stopped failure class this sentinel exists to close.
|
|
69
|
+
*/
|
|
70
|
+
export declare const SOCKET_DISCONNECT_PATTERNS: readonly RegExp[];
|
|
71
|
+
/**
|
|
72
|
+
* Detector — true if `text` contains any known disconnect pattern.
|
|
73
|
+
*/
|
|
74
|
+
export declare function detectSocketDisconnect(text: string): boolean;
|
|
75
|
+
export declare class SocketDisconnectSentinel extends EventEmitter {
|
|
76
|
+
private readonly deps;
|
|
77
|
+
private readonly cfg;
|
|
78
|
+
private readonly states;
|
|
79
|
+
private readonly timers;
|
|
80
|
+
private tickHandle;
|
|
81
|
+
constructor(deps: SocketDisconnectSentinelDeps, cfg?: SocketDisconnectSentinelConfig);
|
|
82
|
+
/**
|
|
83
|
+
* Begin the self-driving scan loop. No-op unless `listSessionNames` was
|
|
84
|
+
* provided (otherwise the sentinel is purely event-driven). The interval is
|
|
85
|
+
* unref'd so it never holds the process open at shutdown.
|
|
86
|
+
*/
|
|
87
|
+
start(): void;
|
|
88
|
+
/** Stop the scan loop and cancel all in-flight recovery. */
|
|
89
|
+
stop(): void;
|
|
90
|
+
/** One scan pass over every session the registry currently knows about. */
|
|
91
|
+
tick(): void;
|
|
92
|
+
/** Called by the tick loop (or by an event trigger) for each tracked session. */
|
|
93
|
+
scanSession(sessionName: string): void;
|
|
94
|
+
/**
|
|
95
|
+
* Public entry: report a detected disconnect. Idempotent — second call
|
|
96
|
+
* for the same session within an active recovery is a no-op.
|
|
97
|
+
*/
|
|
98
|
+
report(sessionName: string): void;
|
|
99
|
+
isRecoveryActive(sessionName: string): boolean;
|
|
100
|
+
listActive(): SocketDisconnectState[];
|
|
101
|
+
/** Cancel any in-flight recovery for this session (test + shutdown helper). */
|
|
102
|
+
clear(sessionName: string): void;
|
|
103
|
+
/** Cancel all in-flight recovery (server shutdown). */
|
|
104
|
+
shutdown(): void;
|
|
105
|
+
private scheduleNextAttempt;
|
|
106
|
+
private runAttempt;
|
|
107
|
+
private verifyAttempt;
|
|
108
|
+
private escalate;
|
|
109
|
+
private notify;
|
|
110
|
+
private setTimer;
|
|
111
|
+
private clearTimer;
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=SocketDisconnectSentinel.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SocketDisconnectSentinel.d.ts","sourceRoot":"","sources":["../../src/monitoring/SocketDisconnectSentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,MAAM,sBAAsB,GAC9B,UAAU,GACV,YAAY,GACZ,WAAW,GACX,WAAW,CAAC;AAEhB,MAAM,WAAW,qBAAqB;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,sBAAsB,CAAC;CAChC;AAED,MAAM,WAAW,4BAA4B;IAC3C;;;OAGG;IACH,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,iEAAiE;IACjE,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D;2DACuD;IACvD,eAAe,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,MAAM,CAAC;IACjD;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,MAAM,EAAE,CAAC;IAClC,iCAAiC;IACjC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,sCAAsC;IACtC,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,IAAI,EAAE,EAAE,EAAE,MAAM,KAAK,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;IACzE,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,CAAC;CAC9D;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,6EAA6E;IAC7E,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kFAAkF;IAClF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAUD;;;;GAIG;AACH,eAAO,MAAM,0BAA0B,EAAE,SAAS,MAAM,EAKvD,CAAC;AAEF;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAM5D;AAED,qBAAa,wBAAyB,SAAQ,YAAY;IAM5C,OAAO,CAAC,QAAQ,CAAC,IAAI;IALjC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA2C;IAC/D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA4C;IACnE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoD;IAC3E,OAAO,CAAC,UAAU,CAA+C;gBAEpC,IAAI,EAAE,4BAA4B,EAAE,GAAG,GAAE,8BAAmC;IAKzG;;;;OAIG;IACH,KAAK,IAAI,IAAI;IAOb,4DAA4D;IAC5D,IAAI,IAAI,IAAI;IAQZ,2EAA2E;IAC3E,IAAI,IAAI,IAAI;IAMZ,iFAAiF;IACjF,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAQtC;;;OAGG;IACH,MAAM,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAsBjC,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAK9C,UAAU,IAAI,qBAAqB,EAAE;IAIrC,+EAA+E;IAC/E,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAOhC,uDAAuD;IACvD,QAAQ,IAAI,IAAI;IAQhB,OAAO,CAAC,mBAAmB;YAcb,UAAU;IA2BxB,OAAO,CAAC,aAAa;IAoBrB,OAAO,CAAC,QAAQ;YAaF,MAAM;IAQpB,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,UAAU;CAGnB"}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SocketDisconnectSentinel — detects Claude Code's "socket connection closed
|
|
3
|
+
* unexpectedly" message in tracked sessions and runs a bounded recovery loop.
|
|
4
|
+
*
|
|
5
|
+
* Closes the gap surfaced by the joint diagnosis on 2026-05-22: the instar
|
|
6
|
+
* repo had detectors for rate limits, quota, hundreds of patterns, but zero
|
|
7
|
+
* detectors for Claude Code's own connection-drop string. When it fires, the
|
|
8
|
+
* session freezes; nothing classifies it; no recovery runs; no user-visible
|
|
9
|
+
* alert reaches the user.
|
|
10
|
+
*
|
|
11
|
+
* Pattern mirrors `RateLimitSentinel.report` → backoff → `resumeFn` →
|
|
12
|
+
* verification → escalate, but the failure shape is simpler (no Claude
|
|
13
|
+
* Code internal retries to wait out — disconnect is immediate) so the
|
|
14
|
+
* lifecycle is shorter.
|
|
15
|
+
*
|
|
16
|
+
* Spec: docs/specs/silently-stopped-trio.md
|
|
17
|
+
*
|
|
18
|
+
* Signal-vs-authority: the regex + state machine are detectors. The
|
|
19
|
+
* notifyFn path is the existing MessagingToneGate authority (called through
|
|
20
|
+
* /attention). No new blocking authority introduced.
|
|
21
|
+
*/
|
|
22
|
+
import { EventEmitter } from 'node:events';
|
|
23
|
+
const DEFAULT_CONFIG = {
|
|
24
|
+
enabled: true,
|
|
25
|
+
backoffScheduleMs: [60_000, 5 * 60_000, 15 * 60_000, 30 * 60_000],
|
|
26
|
+
maxAttempts: 4,
|
|
27
|
+
verifyWindowMs: 60_000,
|
|
28
|
+
tickIntervalMs: 15_000,
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Patterns Claude Code uses when its connection to Anthropic drops. Broad
|
|
32
|
+
* intentionally — false-positive cost is one harmless nudge; false-negative
|
|
33
|
+
* cost is the silently-stopped failure class this sentinel exists to close.
|
|
34
|
+
*/
|
|
35
|
+
export const SOCKET_DISCONNECT_PATTERNS = [
|
|
36
|
+
/socket connection closed unexpectedly/i,
|
|
37
|
+
/websocket.*reset by peer/i,
|
|
38
|
+
/ECONNRESET.*claude/i,
|
|
39
|
+
/connection.*closed.*unexpectedly/i,
|
|
40
|
+
];
|
|
41
|
+
/**
|
|
42
|
+
* Detector — true if `text` contains any known disconnect pattern.
|
|
43
|
+
*/
|
|
44
|
+
export function detectSocketDisconnect(text) {
|
|
45
|
+
if (!text)
|
|
46
|
+
return false;
|
|
47
|
+
for (const pat of SOCKET_DISCONNECT_PATTERNS) {
|
|
48
|
+
if (pat.test(text))
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
export class SocketDisconnectSentinel extends EventEmitter {
|
|
54
|
+
deps;
|
|
55
|
+
cfg;
|
|
56
|
+
states = new Map();
|
|
57
|
+
timers = new Map();
|
|
58
|
+
tickHandle = null;
|
|
59
|
+
constructor(deps, cfg = {}) {
|
|
60
|
+
super();
|
|
61
|
+
this.deps = deps;
|
|
62
|
+
this.cfg = { ...DEFAULT_CONFIG, ...cfg };
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Begin the self-driving scan loop. No-op unless `listSessionNames` was
|
|
66
|
+
* provided (otherwise the sentinel is purely event-driven). The interval is
|
|
67
|
+
* unref'd so it never holds the process open at shutdown.
|
|
68
|
+
*/
|
|
69
|
+
start() {
|
|
70
|
+
if (!this.cfg.enabled || this.tickHandle)
|
|
71
|
+
return;
|
|
72
|
+
if (!this.deps.listSessionNames)
|
|
73
|
+
return;
|
|
74
|
+
this.tickHandle = setInterval(() => this.tick(), this.cfg.tickIntervalMs);
|
|
75
|
+
if (typeof this.tickHandle.unref === 'function')
|
|
76
|
+
this.tickHandle.unref();
|
|
77
|
+
}
|
|
78
|
+
/** Stop the scan loop and cancel all in-flight recovery. */
|
|
79
|
+
stop() {
|
|
80
|
+
if (this.tickHandle) {
|
|
81
|
+
clearInterval(this.tickHandle);
|
|
82
|
+
this.tickHandle = null;
|
|
83
|
+
}
|
|
84
|
+
this.shutdown();
|
|
85
|
+
}
|
|
86
|
+
/** One scan pass over every session the registry currently knows about. */
|
|
87
|
+
tick() {
|
|
88
|
+
if (!this.cfg.enabled)
|
|
89
|
+
return;
|
|
90
|
+
const names = this.deps.listSessionNames?.() ?? [];
|
|
91
|
+
for (const name of names)
|
|
92
|
+
this.scanSession(name);
|
|
93
|
+
}
|
|
94
|
+
/** Called by the tick loop (or by an event trigger) for each tracked session. */
|
|
95
|
+
scanSession(sessionName) {
|
|
96
|
+
if (!this.cfg.enabled)
|
|
97
|
+
return;
|
|
98
|
+
if (this.states.has(sessionName))
|
|
99
|
+
return; // already handling
|
|
100
|
+
const output = this.deps.getRecentOutput(sessionName);
|
|
101
|
+
if (!detectSocketDisconnect(output))
|
|
102
|
+
return;
|
|
103
|
+
this.report(sessionName);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Public entry: report a detected disconnect. Idempotent — second call
|
|
107
|
+
* for the same session within an active recovery is a no-op.
|
|
108
|
+
*/
|
|
109
|
+
report(sessionName) {
|
|
110
|
+
if (!this.cfg.enabled)
|
|
111
|
+
return;
|
|
112
|
+
if (this.states.has(sessionName))
|
|
113
|
+
return;
|
|
114
|
+
const now = (this.deps.now ?? Date.now)();
|
|
115
|
+
const state = {
|
|
116
|
+
sessionName,
|
|
117
|
+
detectedAt: now,
|
|
118
|
+
attempts: 0,
|
|
119
|
+
lastInjectAt: 0,
|
|
120
|
+
status: 'detected',
|
|
121
|
+
};
|
|
122
|
+
this.states.set(sessionName, state);
|
|
123
|
+
// First user notice — tone-gated path via notifyFn.
|
|
124
|
+
void this.notify(sessionName, `${friendlyName(sessionName)} lost its connection to Claude Code. Trying to recover; will let you know if it can't.`);
|
|
125
|
+
this.scheduleNextAttempt(sessionName);
|
|
126
|
+
}
|
|
127
|
+
isRecoveryActive(sessionName) {
|
|
128
|
+
const s = this.states.get(sessionName);
|
|
129
|
+
return !!s && s.status !== 'recovered' && s.status !== 'escalated';
|
|
130
|
+
}
|
|
131
|
+
listActive() {
|
|
132
|
+
return Array.from(this.states.values());
|
|
133
|
+
}
|
|
134
|
+
/** Cancel any in-flight recovery for this session (test + shutdown helper). */
|
|
135
|
+
clear(sessionName) {
|
|
136
|
+
const t = this.timers.get(sessionName);
|
|
137
|
+
if (t)
|
|
138
|
+
this.clearTimer(t);
|
|
139
|
+
this.timers.delete(sessionName);
|
|
140
|
+
this.states.delete(sessionName);
|
|
141
|
+
}
|
|
142
|
+
/** Cancel all in-flight recovery (server shutdown). */
|
|
143
|
+
shutdown() {
|
|
144
|
+
for (const t of this.timers.values())
|
|
145
|
+
this.clearTimer(t);
|
|
146
|
+
this.timers.clear();
|
|
147
|
+
this.states.clear();
|
|
148
|
+
}
|
|
149
|
+
// ── Recovery loop ──────────────────────────────────────────────────
|
|
150
|
+
scheduleNextAttempt(sessionName) {
|
|
151
|
+
const state = this.states.get(sessionName);
|
|
152
|
+
if (!state)
|
|
153
|
+
return;
|
|
154
|
+
if (state.attempts >= this.cfg.maxAttempts) {
|
|
155
|
+
this.escalate(sessionName);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const idx = Math.min(state.attempts, this.cfg.backoffScheduleMs.length - 1);
|
|
159
|
+
const wait = this.cfg.backoffScheduleMs[idx];
|
|
160
|
+
state.status = 'recovering';
|
|
161
|
+
const handle = this.setTimer(() => this.runAttempt(sessionName), wait);
|
|
162
|
+
this.timers.set(sessionName, handle);
|
|
163
|
+
}
|
|
164
|
+
async runAttempt(sessionName) {
|
|
165
|
+
const state = this.states.get(sessionName);
|
|
166
|
+
if (!state)
|
|
167
|
+
return;
|
|
168
|
+
const now = (this.deps.now ?? Date.now)();
|
|
169
|
+
state.attempts += 1;
|
|
170
|
+
state.lastInjectAt = now;
|
|
171
|
+
let injected = false;
|
|
172
|
+
try {
|
|
173
|
+
injected = await this.deps.resumeFn(sessionName);
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
this.emit('recovery-error', { sessionName, err });
|
|
177
|
+
injected = false;
|
|
178
|
+
}
|
|
179
|
+
if (!injected) {
|
|
180
|
+
// Nudge couldn't be delivered — escalate immediately, this isn't
|
|
181
|
+
// something a backoff staircase can fix.
|
|
182
|
+
this.escalate(sessionName);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
// Schedule a verify check.
|
|
186
|
+
const handle = this.setTimer(() => this.verifyAttempt(sessionName), this.cfg.verifyWindowMs);
|
|
187
|
+
this.timers.set(sessionName, handle);
|
|
188
|
+
}
|
|
189
|
+
verifyAttempt(sessionName) {
|
|
190
|
+
const state = this.states.get(sessionName);
|
|
191
|
+
if (!state)
|
|
192
|
+
return;
|
|
193
|
+
const output = this.deps.getRecentOutput(sessionName);
|
|
194
|
+
// Heuristic: if the disconnect pattern no longer appears in the recent
|
|
195
|
+
// output, the session has progressed past the freeze.
|
|
196
|
+
if (!detectSocketDisconnect(output)) {
|
|
197
|
+
state.status = 'recovered';
|
|
198
|
+
void this.notify(sessionName, `${friendlyName(sessionName)} is reconnected and back to work.`);
|
|
199
|
+
this.emit('recovered', sessionName);
|
|
200
|
+
this.clear(sessionName);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
// Still stuck → next attempt.
|
|
204
|
+
this.scheduleNextAttempt(sessionName);
|
|
205
|
+
}
|
|
206
|
+
escalate(sessionName) {
|
|
207
|
+
const state = this.states.get(sessionName);
|
|
208
|
+
if (!state)
|
|
209
|
+
return;
|
|
210
|
+
state.status = 'escalated';
|
|
211
|
+
const minutes = Math.max(1, Math.round(((this.deps.now ?? Date.now)() - state.detectedAt) / 60_000));
|
|
212
|
+
void this.notify(sessionName, `${friendlyName(sessionName)} has been disconnected for about ${minutes} minutes and my recovery attempts haven't worked. Want me to dig in?`);
|
|
213
|
+
this.emit('escalated', sessionName);
|
|
214
|
+
// Keep state so a subsequent scanSession doesn't re-report the same outage.
|
|
215
|
+
}
|
|
216
|
+
async notify(sessionName, text) {
|
|
217
|
+
try {
|
|
218
|
+
await this.deps.notifyFn(sessionName, text);
|
|
219
|
+
}
|
|
220
|
+
catch (err) {
|
|
221
|
+
this.emit('notify-error', { sessionName, err });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
setTimer(fn, ms) {
|
|
225
|
+
return (this.deps.setTimer ?? setTimeout)(fn, ms);
|
|
226
|
+
}
|
|
227
|
+
clearTimer(handle) {
|
|
228
|
+
(this.deps.clearTimer ?? clearTimeout)(handle);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/** Strip prefix conventions for user-facing copy. Keeps B12 simple. */
|
|
232
|
+
function friendlyName(sessionName) {
|
|
233
|
+
return sessionName.replace(/^ai\.instar\./, '').replace(/-server$/, '').replace(/-lifeline$/, '');
|
|
234
|
+
}
|
|
235
|
+
//# sourceMappingURL=SocketDisconnectSentinel.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SocketDisconnectSentinel.js","sourceRoot":"","sources":["../../src/monitoring/SocketDisconnectSentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAoD3C,MAAM,cAAc,GAA6C;IAC/D,OAAO,EAAE,IAAI;IACb,iBAAiB,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC;IACjE,WAAW,EAAE,CAAC;IACd,cAAc,EAAE,MAAM;IACtB,cAAc,EAAE,MAAM;CACvB,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAsB;IAC3D,wCAAwC;IACxC,2BAA2B;IAC3B,qBAAqB;IACrB,mCAAmC;CACpC,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,0BAA0B,EAAE,CAAC;QAC7C,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,wBAAyB,SAAQ,YAAY;IAM3B;IALZ,GAAG,CAA2C;IAC9C,MAAM,GAAG,IAAI,GAAG,EAAiC,CAAC;IAClD,MAAM,GAAG,IAAI,GAAG,EAAyC,CAAC;IACnE,UAAU,GAA0C,IAAI,CAAC;IAEjE,YAA6B,IAAkC,EAAE,MAAsC,EAAE;QACvG,KAAK,EAAE,CAAC;QADmB,SAAI,GAAJ,IAAI,CAA8B;QAE7D,IAAI,CAAC,GAAG,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,GAAG,EAAE,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QACjD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO;QACxC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC1E,IAAI,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,KAAK,UAAU;YAAE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC3E,CAAC;IAED,4DAA4D;IAC5D,IAAI;QACF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;IAED,2EAA2E;IAC3E,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,OAAO;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,IAAI,EAAE,CAAC;QACnD,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IAED,iFAAiF;IACjF,WAAW,CAAC,WAAmB;QAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,OAAO;QAC9B,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YAAE,OAAO,CAAC,mBAAmB;QAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC;YAAE,OAAO;QAC5C,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,WAAmB;QACxB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,OAAO;QAC9B,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YAAE,OAAO;QACzC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,GAA0B;YACnC,WAAW;YACX,UAAU,EAAE,GAAG;YACf,QAAQ,EAAE,CAAC;YACX,YAAY,EAAE,CAAC;YACf,MAAM,EAAE,UAAU;SACnB,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAEpC,oDAAoD;QACpD,KAAK,IAAI,CAAC,MAAM,CACd,WAAW,EACX,GAAG,YAAY,CAAC,WAAW,CAAC,wFAAwF,CACrH,CAAC;QAEF,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IAED,gBAAgB,CAAC,WAAmB;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;IACrE,CAAC;IAED,UAAU;QACR,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,+EAA+E;IAC/E,KAAK,CAAC,WAAmB;QACvB,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACvC,IAAI,CAAC;YAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAClC,CAAC;IAED,uDAAuD;IACvD,QAAQ;QACN,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAED,sEAAsE;IAE9D,mBAAmB,CAAC,WAAmB;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,IAAI,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO;QACT,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAC7C,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC,CAAC;QACvE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,WAAmB;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1C,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;QACpB,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;QAEzB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YAClD,QAAQ,GAAG,KAAK,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,iEAAiE;YACjE,yCAAyC;YACzC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,2BAA2B;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC7F,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IAEO,aAAa,CAAC,WAAmB;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QACtD,uEAAuE;QACvE,sDAAsD;QACtD,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;YAC3B,KAAK,IAAI,CAAC,MAAM,CACd,WAAW,EACX,GAAG,YAAY,CAAC,WAAW,CAAC,mCAAmC,CAChE,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QACD,8BAA8B;QAC9B,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IAEO,QAAQ,CAAC,WAAmB;QAClC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;QACrG,KAAK,IAAI,CAAC,MAAM,CACd,WAAW,EACX,GAAG,YAAY,CAAC,WAAW,CAAC,oCAAoC,OAAO,sEAAsE,CAC9I,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACpC,4EAA4E;IAC9E,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,WAAmB,EAAE,IAAY;QACpD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,EAAc,EAAE,EAAU;QACzC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACpD,CAAC;IAEO,UAAU,CAAC,MAAqC;QACtD,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;CACF;AAED,uEAAuE;AACvE,SAAS,YAAY,CAAC,WAAmB;IACvC,OAAO,WAAW,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AACpG,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* sentinelWiring — constructs the runtime dependencies for the two
|
|
3
|
+
* "silently-stopped" sentinels (SocketDisconnectSentinel and
|
|
4
|
+
* ActiveWorkSilenceSentinel) from a SessionManager surface + config.
|
|
5
|
+
*
|
|
6
|
+
* Why this module exists: PR #334 merged both sentinels as standalone,
|
|
7
|
+
* unit-tested detectors but never instantiated them in the server. The
|
|
8
|
+
* release notes claimed "wired into server startup" — it wasn't true. This
|
|
9
|
+
* module is the missing wire-up, extracted from server.ts so the dependency
|
|
10
|
+
* construction is itself unit-testable (Testing Integrity Standard: every
|
|
11
|
+
* dependency-injected component needs wiring-integrity tests proving the deps
|
|
12
|
+
* are real functions that delegate to the underlying primitives, not nulls or
|
|
13
|
+
* silent no-ops).
|
|
14
|
+
*
|
|
15
|
+
* Signal-vs-authority: everything here is a detector or a bounded recovery
|
|
16
|
+
* primitive. `looksActivelyWorking` is a structural detector. All user-facing
|
|
17
|
+
* escalation routes through the existing MessagingToneGate via the `/attention`
|
|
18
|
+
* endpoint (category: degradation) — no new blocking authority is introduced.
|
|
19
|
+
*
|
|
20
|
+
* Spec: docs/specs/silently-stopped-trio.md
|
|
21
|
+
*/
|
|
22
|
+
import type { SocketDisconnectSentinelDeps } from './SocketDisconnectSentinel.js';
|
|
23
|
+
import type { ActiveWorkSilenceSentinelDeps, SessionRegistryEntry } from './ActiveWorkSilenceSentinel.js';
|
|
24
|
+
import type { IntelligenceFramework } from '../core/intelligenceProviderFactory.js';
|
|
25
|
+
/** Minimal SessionManager surface the wiring depends on. */
|
|
26
|
+
export interface SentinelSessionSurface {
|
|
27
|
+
captureOutput(tmuxSession: string, lines?: number): string | null;
|
|
28
|
+
isSessionAlive(tmuxSession: string): boolean;
|
|
29
|
+
sendKey(tmuxSession: string, key: string): boolean;
|
|
30
|
+
listRunningSessions(): Array<{
|
|
31
|
+
tmuxSession: string;
|
|
32
|
+
framework?: IntelligenceFramework;
|
|
33
|
+
}>;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Posts a user-facing escalation through the tone-gated `/attention` route.
|
|
37
|
+
* Returns true on 201 (delivered), false otherwise — including a 422 tone-gate
|
|
38
|
+
* block (which is the gate doing its job, not an error to retry).
|
|
39
|
+
*/
|
|
40
|
+
export type AttentionPoster = (item: {
|
|
41
|
+
id: string;
|
|
42
|
+
title: string;
|
|
43
|
+
summary: string;
|
|
44
|
+
category?: string;
|
|
45
|
+
priority?: string;
|
|
46
|
+
}) => Promise<boolean>;
|
|
47
|
+
export declare function makeAttentionPoster(opts: {
|
|
48
|
+
port: number;
|
|
49
|
+
authToken: string;
|
|
50
|
+
fetchImpl?: typeof fetch;
|
|
51
|
+
}): AttentionPoster;
|
|
52
|
+
export declare function buildSocketDisconnectDeps(opts: {
|
|
53
|
+
sessions: SentinelSessionSurface;
|
|
54
|
+
notify: AttentionPoster;
|
|
55
|
+
}): SocketDisconnectSentinelDeps;
|
|
56
|
+
/**
|
|
57
|
+
* Detector: does this captured frame show the framework actively working
|
|
58
|
+
* (spinner, tool call, "esc to interrupt", "(running)") rather than sitting
|
|
59
|
+
* idle at its prompt? Used to filter the silence sentinel's candidate set so a
|
|
60
|
+
* session that is simply waiting for the user is never flagged as "stopped".
|
|
61
|
+
*/
|
|
62
|
+
export declare function looksActivelyWorking(output: string, framework?: IntelligenceFramework): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Tracks per-session output-change time so the silence sentinel can tell
|
|
65
|
+
* "frozen mid-task" from "still producing output". Only sessions whose most
|
|
66
|
+
* recent frame shows active-work signatures are surfaced as candidates; an
|
|
67
|
+
* idle-at-prompt session is marked `paused` so the sentinel skips it — it is
|
|
68
|
+
* not "actively working then stopped".
|
|
69
|
+
*/
|
|
70
|
+
export declare class OutputActivityTracker {
|
|
71
|
+
private readonly sessions;
|
|
72
|
+
private readonly now;
|
|
73
|
+
private readonly last;
|
|
74
|
+
constructor(sessions: SentinelSessionSurface, now?: () => number);
|
|
75
|
+
snapshot(): SessionRegistryEntry[];
|
|
76
|
+
}
|
|
77
|
+
export declare function buildActiveWorkSilenceDeps(opts: {
|
|
78
|
+
tracker: OutputActivityTracker;
|
|
79
|
+
sessions: SentinelSessionSurface;
|
|
80
|
+
notify: AttentionPoster;
|
|
81
|
+
}): ActiveWorkSilenceSentinelDeps;
|
|
82
|
+
//# sourceMappingURL=sentinelWiring.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sentinelWiring.d.ts","sourceRoot":"","sources":["../../src/monitoring/sentinelWiring.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAClF,OAAO,KAAK,EACV,6BAA6B,EAC7B,oBAAoB,EACrB,MAAM,gCAAgC,CAAC;AAExC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,wCAAwC,CAAC;AAEpF,4DAA4D;AAC5D,MAAM,WAAW,sBAAsB;IACrC,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAClE,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;IAC7C,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACnD,mBAAmB,IAAI,KAAK,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,qBAAqB,CAAA;KAAE,CAAC,CAAC;CAC1F;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,IAAI,EAAE;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEvB,wBAAgB,mBAAmB,CAAC,IAAI,EAAE;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B,GAAG,eAAe,CAoBlB;AAKD,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC9C,QAAQ,EAAE,sBAAsB,CAAC;IACjC,MAAM,EAAE,eAAe,CAAC;CACzB,GAAG,4BAA4B,CAqB/B;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,qBAAqB,GAChC,OAAO,CAQT;AAED;;;;;;GAMG;AACH,qBAAa,qBAAqB;IAI9B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,GAAG;IAJtB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmD;gBAGrD,QAAQ,EAAE,sBAAsB,EAChC,GAAG,GAAE,MAAM,MAAiB;IAG/C,QAAQ,IAAI,oBAAoB,EAAE;CAuBnC;AAED,wBAAgB,0BAA0B,CAAC,IAAI,EAAE;IAC/C,OAAO,EAAE,qBAAqB,CAAC;IAC/B,QAAQ,EAAE,sBAAsB,CAAC;IACjC,MAAM,EAAE,eAAe,CAAC;CACzB,GAAG,6BAA6B,CAehC"}
|