@saluzi/saluzi-edu 0.4.6 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@saluzi/saluzi-edu",
3
- "version": "0.4.6",
3
+ "version": "0.4.7",
4
4
  "description": "Saluzi CLI - interactive AI coding assistant in the terminal",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -17,6 +17,8 @@ export function createMockConfig(overrides?: Partial<Record<string, unknown>>) {
17
17
  singleUser: false,
18
18
  wsIdleTimeout: 30,
19
19
  wsKeepaliveInterval: 20,
20
+ wsClientInactivityTimeout: 300,
21
+ stuckTurnTimeout: 900,
20
22
  artifactsPublicAccess: true,
21
23
  ...overrides,
22
24
  }
@@ -0,0 +1,453 @@
1
+ /**
2
+ * Regression: `slz rcs one` session stuck on `running` forever after the
3
+ * worker's turn-end result event is lost in transit.
4
+ *
5
+ * Reproduction of the reported symptom: task half-done, a tool call succeeds
6
+ * (its tool-result user event reaches the server), then silence — the web
7
+ * input shows the black stop square (isLoading) and every new message lands
8
+ * in the queue forever.
9
+ *
10
+ * Root cause chain (worker side, see src/bridge/outboundRetry.ts +
11
+ * replBridge.ts): sendResult()/sendControlRequest() are fire-and-forget —
12
+ * a result written during a transport swap window ("DROP sendResult: no
13
+ * transport") or dropped by the uploader (maxConsecutiveFailures / close
14
+ * grace) is never re-sent. The server-side status derivation only returns
15
+ * to idle on an inbound result, so the session stays `running` forever and
16
+ * NOTHING self-heals it: the worker write watchdog is quiet (POSTs are
17
+ * healthy again), the WS inactivity clock is kept fresh by keepalive POSTs,
18
+ * and the control-request watchdog only tracks RPCs the server itself sent.
19
+ *
20
+ * This test pins the server-side self-heal: the stuck-turn watchdog sweep
21
+ * flips stale `running` sessions back to idle (publishing session_status so
22
+ * live SSE viewers unlock the input) and fires the recovery work re-dispatch.
23
+ * `requires_action` is deliberately out of scope — a visible permission card
24
+ * is actionable, not stuck.
25
+ */
26
+ import { describe, test, expect, beforeEach, mock } from 'bun:test'
27
+ import { mockConfigModule } from './helpers/mock-config'
28
+
29
+ mock.module('../config', () =>
30
+ mockConfigModule({ pollTimeout: 1, stuckTurnTimeout: 60 }),
31
+ )
32
+
33
+ import { Hono } from 'hono'
34
+ import {
35
+ storeReset,
36
+ storeCreateEnvironment,
37
+ storeGetPendingWorkItem,
38
+ } from '../store'
39
+ import {
40
+ getAllEventBuses,
41
+ removeEventBus,
42
+ getEventBus,
43
+ type SessionEvent,
44
+ } from '../transport/event-bus'
45
+
46
+ import v1SessionIngress from '../routes/v1/session-ingress'
47
+ import webSessions from '../routes/web/sessions'
48
+ import webControl from '../routes/web/control'
49
+ import { issueToken } from '../auth/token'
50
+ import { getSession } from '../services/session'
51
+ import { sweepStuckTurnSessions } from '../services/stuck-turn-watchdog'
52
+
53
+ /** Matches the mocked config's stuckTurnTimeout (60s). */
54
+ const THRESHOLD_MS = 60 * 1000
55
+
56
+ function createApp() {
57
+ const app = new Hono()
58
+ app.route('/v2/session_ingress', v1SessionIngress)
59
+ app.route('/web', webSessions)
60
+ app.route('/web', webControl)
61
+ return app
62
+ }
63
+
64
+ const AUTH_HEADERS = {
65
+ Authorization: 'Bearer test-api-key',
66
+ 'X-Username': 'testuser',
67
+ }
68
+
69
+ function auth(user: string): string {
70
+ return `Bearer ${issueToken(user).token}`
71
+ }
72
+
73
+ /** Drive the session exactly like the real worker does: the Web UI starts
74
+ * the turn (outbound user), HybridTransport POSTs the worker's events via
75
+ * session-ingress (inbound assistant / tool-result user). */
76
+ async function driveTurn(app: Hono, sessionId: string): Promise<void> {
77
+ const sendRes = await app.request(`/web/sessions/${sessionId}/events`, {
78
+ method: 'POST',
79
+ headers: {
80
+ Authorization: auth('user-1'),
81
+ 'Content-Type': 'application/json',
82
+ },
83
+ body: JSON.stringify({
84
+ type: 'user',
85
+ uuid: 'web-uuid-1',
86
+ content: 'run the task',
87
+ message: { content: 'run the task' },
88
+ }),
89
+ })
90
+ expect(sendRes.status).toBe(200)
91
+
92
+ // Worker: assistant streams (running), then the tool executes successfully
93
+ // and its tool-result user message is mirrored — the last events the
94
+ // server ever sees.
95
+ const ingestRes = await app.request(
96
+ `/v2/session_ingress/session/${sessionId}/events`,
97
+ {
98
+ method: 'POST',
99
+ headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
100
+ body: JSON.stringify({
101
+ events: [
102
+ {
103
+ type: 'assistant',
104
+ uuid: 'asst-uuid-1',
105
+ message: {
106
+ role: 'assistant',
107
+ content: [
108
+ { type: 'tool_use', id: 't1', name: 'Bash', input: {} },
109
+ ],
110
+ },
111
+ },
112
+ {
113
+ type: 'user',
114
+ uuid: 'tool-result-1',
115
+ message: {
116
+ role: 'user',
117
+ content: [
118
+ { type: 'tool_result', tool_use_id: 't1', content: 'ok' },
119
+ ],
120
+ },
121
+ },
122
+ ],
123
+ }),
124
+ },
125
+ )
126
+ expect(ingestRes.status).toBe(200)
127
+ }
128
+
129
+ describe('stuck-turn watchdog (lost turn-end result regression)', () => {
130
+ let app: Hono
131
+ let sessionId: string
132
+ let envId: string
133
+
134
+ beforeEach(async () => {
135
+ storeReset()
136
+ for (const [key] of getAllEventBuses()) {
137
+ removeEventBus(key)
138
+ }
139
+ app = createApp()
140
+
141
+ const env = storeCreateEnvironment({ secret: 's' })
142
+ envId = env.id
143
+
144
+ const res = await app.request('/web/sessions', {
145
+ method: 'POST',
146
+ headers: {
147
+ Authorization: auth('user-1'),
148
+ 'Content-Type': 'application/json',
149
+ },
150
+ body: JSON.stringify({ environment_id: envId }),
151
+ })
152
+ const body = (await res.json()) as { id: string }
153
+ sessionId = body.id
154
+ })
155
+
156
+ test('lost turn-end result: session stuck running, sweep heals to idle', async () => {
157
+ await driveTurn(app, sessionId)
158
+
159
+ // ── The reproduction: the turn-end result event never arrives ──
160
+ // (dropped by the worker's uploader / transport-swap window). The
161
+ // session is now stuck on `running` — the web input locks and every
162
+ // new message queues forever.
163
+ expect(getSession(sessionId)?.status).toBe('running')
164
+
165
+ // Time passes; the worker's keepalive POSTs keep the WS alive but no
166
+ // conversation events arrive. Nothing on the server heals the status…
167
+ const healed = sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
168
+ expect(healed).toBe(1)
169
+ expect(getSession(sessionId)?.status).toBe('idle')
170
+
171
+ // A session_status event must be broadcast so live SSE viewers
172
+ // (RCSChatAdapter onStatusChange) unlock the input immediately.
173
+ const statuses = getEventBus(sessionId)
174
+ .getEventsSince(0)
175
+ .filter(e => e.type === 'session_status')
176
+ .map(e => (e.payload as Record<string, unknown>).status)
177
+ expect(statuses[statuses.length - 1]).toBe('idle')
178
+
179
+ // The sweep also queues the recovery work re-dispatch (zombie-transport
180
+ // rebuild) — same recovery the WS inactivity close uses.
181
+ expect(storeGetPendingWorkItem(envId)?.sessionId).toBe(sessionId)
182
+ })
183
+
184
+ test('genuinely progressing turn is NOT healed (recent inbound events)', async () => {
185
+ await driveTurn(app, sessionId)
186
+
187
+ // Worker still streaming — the sweep with fresh clock must not flip.
188
+ expect(sweepStuckTurnSessions()).toBe(0)
189
+ expect(getSession(sessionId)?.status).toBe('running')
190
+
191
+ // A fresh inbound assistant resets the silence clock — sweeping before
192
+ // the full threshold elapses since that event must not heal.
193
+ const ingestRes = await app.request(
194
+ `/v2/session_ingress/session/${sessionId}/events`,
195
+ {
196
+ method: 'POST',
197
+ headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
198
+ body: JSON.stringify({
199
+ events: [
200
+ {
201
+ type: 'assistant',
202
+ uuid: 'asst-uuid-2',
203
+ message: {
204
+ role: 'assistant',
205
+ content: [{ type: 'text', text: 'still working' }],
206
+ },
207
+ },
208
+ ],
209
+ }),
210
+ },
211
+ )
212
+ expect(ingestRes.status).toBe(200)
213
+
214
+ // The assistant arrived moments ago — sweeping at threshold-1s (i.e. the
215
+ // last signal is younger than the threshold) must not heal.
216
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
217
+ expect(getSession(sessionId)?.status).toBe('running')
218
+ })
219
+
220
+ test('turn just started (outbound user, no worker reply yet) is NOT healed', async () => {
221
+ const res = await app.request(`/web/sessions/${sessionId}/events`, {
222
+ method: 'POST',
223
+ headers: {
224
+ Authorization: auth('user-1'),
225
+ 'Content-Type': 'application/json',
226
+ },
227
+ body: JSON.stringify({
228
+ type: 'user',
229
+ uuid: 'fresh-user-1',
230
+ content: 'hello',
231
+ message: { content: 'hello' },
232
+ }),
233
+ })
234
+ expect(res.status).toBe(200)
235
+ expect(getSession(sessionId)?.status).toBe('running')
236
+
237
+ // The outbound user event marks the turn start — the worker gets the
238
+ // full threshold to produce its first inbound event.
239
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
240
+ expect(getSession(sessionId)?.status).toBe('running')
241
+
242
+ // …but a turn whose prompt is older than the threshold with zero worker
243
+ // output (worker never picked it up) does heal.
244
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
245
+ expect(getSession(sessionId)?.status).toBe('idle')
246
+ })
247
+
248
+ test('60s control RPC answers do NOT count as turn progress', async () => {
249
+ await driveTurn(app, sessionId)
250
+
251
+ // The Web UI polls get_context_usage every 60s while the page is open —
252
+ // the worker's control_response answers are RPC liveness, not turn
253
+ // progress. They must not keep a wedged turn looking alive.
254
+ const pollRes = await app.request(`/web/sessions/${sessionId}/control`, {
255
+ method: 'POST',
256
+ headers: {
257
+ Authorization: auth('user-1'),
258
+ 'Content-Type': 'application/json',
259
+ },
260
+ body: JSON.stringify({
261
+ type: 'control_request',
262
+ request_id: 'rpc-1',
263
+ request: { subtype: 'get_context_usage' },
264
+ }),
265
+ })
266
+ expect(pollRes.status).toBe(200)
267
+
268
+ const answerRes = await app.request(
269
+ `/v2/session_ingress/session/${sessionId}/events`,
270
+ {
271
+ method: 'POST',
272
+ headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
273
+ body: JSON.stringify({
274
+ events: [
275
+ {
276
+ type: 'control_response',
277
+ response: {
278
+ subtype: 'success',
279
+ request_id: 'rpc-1',
280
+ response: {},
281
+ },
282
+ },
283
+ ],
284
+ }),
285
+ },
286
+ )
287
+ expect(answerRes.status).toBe(200)
288
+
289
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
290
+ expect(getSession(sessionId)?.status).toBe('idle')
291
+ })
292
+
293
+ test('connect-time snapshots and background task_state do not feed the signal clock', async () => {
294
+ await driveTurn(app, sessionId)
295
+
296
+ // system/init (re-ssent on every reconnect), models_state, and
297
+ // task_state (background task churn outside any turn) are transport /
298
+ // task-list liveness, not turn progress — a wedged session must still
299
+ // heal even while these keep arriving.
300
+ const chatterRes = await app.request(
301
+ `/v2/session_ingress/session/${sessionId}/events`,
302
+ {
303
+ method: 'POST',
304
+ headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
305
+ body: JSON.stringify({
306
+ events: [
307
+ {
308
+ type: 'system',
309
+ subtype: 'init',
310
+ uuid: 'init-1',
311
+ message: {},
312
+ },
313
+ {
314
+ type: 'models_state',
315
+ uuid: 'models-1',
316
+ message: {},
317
+ },
318
+ {
319
+ type: 'task_state',
320
+ uuid: 'tasks-1',
321
+ message: {},
322
+ },
323
+ ],
324
+ }),
325
+ },
326
+ )
327
+ expect(chatterRes.status).toBe(200)
328
+
329
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
330
+ expect(getSession(sessionId)?.status).toBe('idle')
331
+ })
332
+
333
+ test('subagent task events DO feed the signal clock (genuine turn progress)', async () => {
334
+ await driveTurn(app, sessionId)
335
+
336
+ const progressRes = await app.request(
337
+ `/v2/session_ingress/session/${sessionId}/events`,
338
+ {
339
+ method: 'POST',
340
+ headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
341
+ body: JSON.stringify({
342
+ events: [
343
+ {
344
+ type: 'system',
345
+ subtype: 'task_progress',
346
+ uuid: 'task-progress-1',
347
+ task_id: 'task-1',
348
+ message: {},
349
+ },
350
+ ],
351
+ }),
352
+ },
353
+ )
354
+ expect(progressRes.status).toBe(200)
355
+
356
+ // The task_progress event just arrived — sweeping before the full
357
+ // threshold since it must not heal.
358
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
359
+ expect(getSession(sessionId)?.status).toBe('running')
360
+ })
361
+
362
+ test('pending permission ask (requires_action) is out of scope — never healed', async () => {
363
+ await driveTurn(app, sessionId)
364
+
365
+ // The worker's next tool needs permission — the can_use_tool ask is
366
+ // visible and actionable in the Web UI (card with approve/deny), so a
367
+ // requires_action session is NOT stuck: the user is the one blocking.
368
+ // The sweep must never touch it, no matter how long the user waits.
369
+ const askRes = await app.request(
370
+ `/v2/session_ingress/session/${sessionId}/events`,
371
+ {
372
+ method: 'POST',
373
+ headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
374
+ body: JSON.stringify({
375
+ events: [
376
+ {
377
+ type: 'control_request',
378
+ request_id: 'perm-1',
379
+ request: {
380
+ subtype: 'can_use_tool',
381
+ tool_name: 'Bash',
382
+ input: {},
383
+ tool_use_id: 't2',
384
+ },
385
+ },
386
+ ],
387
+ }),
388
+ },
389
+ )
390
+ expect(askRes.status).toBe(200)
391
+ expect(getSession(sessionId)?.status).toBe('requires_action')
392
+
393
+ expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0)
394
+ expect(getSession(sessionId)?.status).toBe('requires_action')
395
+ })
396
+
397
+ test('lost permission ask (stuck on running, card never shown) heals', async () => {
398
+ await driveTurn(app, sessionId)
399
+ // The ask itself was lost in transit — status never left running and no
400
+ // card is shown. Same sweep heals it (input unlock; the worker-side
401
+ // retry re-delivers the ask and flips it to requires_action).
402
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
403
+ expect(getSession(sessionId)?.status).toBe('idle')
404
+ })
405
+
406
+ test('idle sessions and fresh buses are untouched', async () => {
407
+ await driveTurn(app, sessionId)
408
+ sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
409
+ expect(getSession(sessionId)?.status).toBe('idle')
410
+
411
+ // Already idle — the sweep must not publish anything new.
412
+ const before = getEventBus(sessionId)
413
+ .getEventsSince(0)
414
+ .filter(e => e.type === 'session_status').length
415
+ expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0)
416
+ const after = getEventBus(sessionId)
417
+ .getEventsSince(0)
418
+ .filter(e => e.type === 'session_status').length
419
+ expect(after).toBe(before)
420
+ })
421
+
422
+ test('server-restart legacy: running session with an empty bus heals', async () => {
423
+ // A session persisted as `running` whose bus is fresh (server restart)
424
+ // has no turn signal at all — the sweep falls back to the session's
425
+ // creation time so restart-legacy stuck sessions still heal.
426
+ const { storeUpdateSession } = await import('../store')
427
+ storeUpdateSession(sessionId, { status: 'running' })
428
+ removeEventBus(sessionId) // simulate fresh bus after restart
429
+
430
+ const swept = sweepStuckTurnSessions(Date.now() + 1000)
431
+ // Session was created milliseconds ago — creation-time fallback gives
432
+ // the same grace window as a fresh turn.
433
+ expect(swept).toBe(0)
434
+
435
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
436
+ expect(getSession(sessionId)?.status).toBe('idle')
437
+ })
438
+
439
+ test("watchdog's own session_status event does not count as turn progress", async () => {
440
+ await driveTurn(app, sessionId)
441
+ sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
442
+ expect(getSession(sessionId)?.status).toBe('idle')
443
+
444
+ // Flip it back to running (e.g. a late assistant with turn signal) and
445
+ // verify the sweep's previously published session_status (inbound) did
446
+ // not arm the signal clock — a sweep right past the threshold still
447
+ // fires, and repeated sweeps keep healing.
448
+ const { storeUpdateSession } = await import('../store')
449
+ storeUpdateSession(sessionId, { status: 'running' })
450
+ expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 10_000)).toBe(1)
451
+ expect(getSession(sessionId)?.status).toBe('idle')
452
+ })
453
+ })
@@ -39,6 +39,14 @@ export interface RcsConfig {
39
39
  * (session_keepalive_interval_v2_ms, default 120s) plus jitter, otherwise
40
40
  * healthy idle sessions get torn down between keepalives. Default 300s. */
41
41
  wsClientInactivityTimeout: number
42
+ /** Stuck-turn watchdog (seconds). A session whose status is `running` with
43
+ * no inbound worker events (and no outbound user message) for this long is
44
+ * flipped back to `idle` + gets a recovery work re-dispatch — the turn-end
45
+ * result event was lost in transit (worker-side drop) and nothing else
46
+ * would ever release the Web UI input. Generous vs. long silent tools:
47
+ * a false flip only unlocks the input early (messages queue in the REPL);
48
+ * the next worker event re-derives `running`. 0 disables. Default 900s. */
49
+ stuckTurnTimeout: number
42
50
  /** Whether artifact content (GET /v1/artifacts/:id/content) is served
43
51
  * without authentication. Default true — the unguessable artifact id is
44
52
  * the secret, same trust model as the upstream cloud-artifacts service.
@@ -84,6 +92,7 @@ export const config: RcsConfig = {
84
92
  process.env.RCS_WS_CLIENT_INACTIVITY_TIMEOUT || '300',
85
93
  10,
86
94
  ),
95
+ stuckTurnTimeout: parseInt(process.env.RCS_STUCK_TURN_TIMEOUT || '900', 10),
87
96
  artifactsPublicAccess: process.env.RCS_ARTIFACTS_PUBLIC_ACCESS !== 'false',
88
97
  }
89
98
 
@@ -7,6 +7,7 @@ import {
7
7
  } from '../store'
8
8
  import { config } from '../config'
9
9
  import { sweepUnansweredControlRequests } from './transport'
10
+ import { sweepStuckTurnSessions } from './stuck-turn-watchdog'
10
11
 
11
12
  /** ACP env 离线后留给 acp-link 重连复用的宽限时间。超过后才将 env 置为
12
13
  * disconnected。会话不再自动归档——由用户手动删除。 */
@@ -62,5 +63,9 @@ export function startDisconnectMonitor(): ReturnType<typeof setInterval> {
62
63
  // path (WS healthy, POST path dead) from unanswered RPCs and fires the
63
64
  // recovery work re-dispatch. See services/transport.ts.
64
65
  sweepUnansweredControlRequests()
66
+ // Stuck-turn watchdog — heals sessions whose turn-end result event was
67
+ // lost in transit (status stuck `running`, Web UI input locked forever).
68
+ // See services/stuck-turn-watchdog.ts.
69
+ sweepStuckTurnSessions()
65
70
  }, 60_000) // Check every minute
66
71
  }
@@ -0,0 +1,144 @@
1
+ import { log } from '../logger'
2
+ import { storeListSessions } from '../store'
3
+ import { config } from '../config'
4
+ import type { SessionEvent } from '../transport/event-bus'
5
+ import { getEventBus } from '../transport/event-bus'
6
+ import { updateSessionStatus } from './session'
7
+ import { redispatchWorkForSession } from './work-dispatch'
8
+
9
+ /**
10
+ * Stuck-turn watchdog — the server-side self-heal for sessions whose
11
+ * turn-end signal was lost in transit.
12
+ *
13
+ * A session's status only returns from `running` to `idle` when the worker's
14
+ * `result`/`result_success` event arrives (ws-handler.ts ingestBridgeMessage)
15
+ * or the user interrupts. The worker sends that result fire-and-forget —
16
+ * when the write is dropped (transport-swap window, uploader batch drop
17
+ * after maxConsecutiveFailures, close-grace expiry), it is never re-sent and
18
+ * the session stays `running` forever. The Web UI locks the input (black
19
+ * stop square) and every new message lands in a queue that never drains.
20
+ *
21
+ * Nothing else detects this state:
22
+ * - the worker's write-path watchdog only fires when POSTs stop completing
23
+ * entirely (a transient drop followed by healthy POSTs is invisible);
24
+ * - the WS inactivity clock is kept fresh by keepalive POSTs;
25
+ * - the control-request watchdog only tracks RPCs the server itself sent.
26
+ *
27
+ * The sweep flips a `running` session with no turn signal for
28
+ * `config.stuckTurnTimeout` back to `idle` (updateSessionStatus publishes
29
+ * `session_status`, unlocking live SSE viewers immediately — a genuinely
30
+ * still-running turn re-derives `running` on its next event) and fires the
31
+ * recovery work re-dispatch (zombie-transport rebuild, same as the WS
32
+ * inactivity close).
33
+ *
34
+ * `requires_action` is deliberately out of scope: a permission ask that
35
+ * arrived is visible and actionable in the Web UI — the user is the one
36
+ * blocking, not a lost signal.
37
+ */
38
+
39
+ /**
40
+ * Inbound event types that count as turn progress — an ALLOWLIST, so future
41
+ * chatter types default to not feeding the signal clock:
42
+ * - `user` (worker-mirrored tool results), `assistant`, `result`/
43
+ * `result_success`, `error` — the conversation itself;
44
+ * - `system` — subagent task events (task_started/task_progress/…).
45
+ * Exception: subtype `init` is re-sent on every transport (re)connect —
46
+ * transport liveness, not turn progress.
47
+ *
48
+ * Deliberately NOT counted:
49
+ * - `control_request`/`control_response` — RPC traffic. The Web UI polls
50
+ * get_context_usage every 60s while the page is open, so the worker's
51
+ * control_response answers flow constantly even for a wedged session —
52
+ * counting them would make the watchdog permanently blind exactly when
53
+ * the user is watching the stuck session.
54
+ * - `models_state`, `system/init` — connect-time snapshots.
55
+ * - `task_state` — background task-list churn; a background task running
56
+ * outside any turn must not starve the watchdog.
57
+ * - `session_status` — server-synthesized (including this watchdog's own
58
+ * idle flip; counting it would make the sweep single-shot).
59
+ */
60
+ const SIGNAL_EVENT_TYPES = new Set([
61
+ 'user',
62
+ 'assistant',
63
+ 'partial_assistant',
64
+ 'result',
65
+ 'result_success',
66
+ 'error',
67
+ 'system',
68
+ ])
69
+
70
+ function isInboundSignal(event: SessionEvent): boolean {
71
+ if (event.direction !== 'inbound') return false
72
+ if (!SIGNAL_EVENT_TYPES.has(event.type)) return false
73
+ if (event.type === 'system') {
74
+ // system/init is a connect-time snapshot, re-sent on every
75
+ // (re)connect — not turn progress.
76
+ const subtype = (event.payload as Record<string, unknown> | null)?.subtype
77
+ if (subtype === 'init') return false
78
+ }
79
+ return true
80
+ }
81
+
82
+ /**
83
+ * Wall-clock of the latest event on the bus that proves the session's turn
84
+ * is (or was) progressing: any inbound worker conversation event (assistant,
85
+ * tool-result user, result, subagent task events), or the outbound user
86
+ * message that started the turn. Returns 0 when the bus carries no signal
87
+ * at all (fresh bus after a server restart — callers fall back to the
88
+ * persisted session record).
89
+ */
90
+ export function lastTurnSignalAt(events: SessionEvent[]): number {
91
+ for (let i = events.length - 1; i >= 0; i--) {
92
+ const event = events[i]
93
+ if (isInboundSignal(event)) {
94
+ return event.createdAt
95
+ }
96
+ if (event.direction === 'outbound' && event.type === 'user') {
97
+ return event.createdAt
98
+ }
99
+ }
100
+ return 0
101
+ }
102
+
103
+ /**
104
+ * Sweep all sessions for stuck turns. Returns the number of sessions healed.
105
+ * `now` is injectable for tests.
106
+ */
107
+ export function sweepStuckTurnSessions(now: number = Date.now()): number {
108
+ const thresholdMs = config.stuckTurnTimeout * 1000
109
+ if (!(thresholdMs > 0)) return 0
110
+
111
+ let swept = 0
112
+ for (const session of storeListSessions()) {
113
+ // Only `running` locks the input with no recovery path; see module docs
114
+ // for why requires_action (visible, actionable card) and archived
115
+ // sessions are excluded.
116
+ if (session.status !== 'running') continue
117
+ // ACP sessions carry their turn traffic on the per-channel-group ACP
118
+ // bus (getAcpEventBus), not the session bus this sweep scans — healing
119
+ // them from the wrong bus would be a false positive.
120
+ if (session.source === 'acp') continue
121
+
122
+ const bus = getEventBus(session.id)
123
+ const lastSignal = lastTurnSignalAt(bus.getEventsSince(0))
124
+ // Empty bus (server restart) or a signal older than the session record
125
+ // (persisted legacy) — fall back to the persisted creation time so
126
+ // restart-legacy stuck sessions still heal after the same grace window.
127
+ const reference = Math.max(lastSignal, session.createdAt.getTime())
128
+ if (now - reference <= thresholdMs) continue
129
+
130
+ const silenceSec = Math.round((now - reference) / 1000)
131
+ log(
132
+ `[RCS] Stuck-turn watchdog: session=${session.id} running with no turn signal for ${silenceSec}s` +
133
+ ' — flipping to idle (turn-end result was lost) and redispatching work',
134
+ )
135
+ // Recovery re-dispatch first: when the worker's transport is actually
136
+ // dead this rebuilds it (fresh uploader → the write path recovers).
137
+ // No-op for healthy transports with an unchanged token.
138
+ void redispatchWorkForSession(session.id)
139
+ // Publishes `session_status` → live SSE viewers unlock the input.
140
+ updateSessionStatus(session.id, 'idle')
141
+ swept++
142
+ }
143
+ return swept
144
+ }