@saluzi/saluzi-edu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +47 -47
- package/dist/guide/guide-data.json +175 -175
- package/package.json +1 -1
- package/packages/remote-control-server/src/__tests__/helpers/mock-config.ts +2 -0
- package/packages/remote-control-server/src/__tests__/stuck-turn-watchdog.test.ts +453 -0
- package/packages/remote-control-server/src/config.ts +9 -0
- package/packages/remote-control-server/src/services/disconnect-monitor.ts +5 -0
- package/packages/remote-control-server/src/services/stuck-turn-watchdog.ts +144 -0
package/package.json
CHANGED
|
@@ -17,6 +17,8 @@ export function createMockConfig(overrides?: Partial<Record<string, unknown>>) {
|
|
|
17
17
|
singleUser: false,
|
|
18
18
|
wsIdleTimeout: 30,
|
|
19
19
|
wsKeepaliveInterval: 20,
|
|
20
|
+
wsClientInactivityTimeout: 300,
|
|
21
|
+
stuckTurnTimeout: 900,
|
|
20
22
|
artifactsPublicAccess: true,
|
|
21
23
|
...overrides,
|
|
22
24
|
}
|
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regression: `slz rcs one` session stuck on `running` forever after the
|
|
3
|
+
* worker's turn-end result event is lost in transit.
|
|
4
|
+
*
|
|
5
|
+
* Reproduction of the reported symptom: task half-done, a tool call succeeds
|
|
6
|
+
* (its tool-result user event reaches the server), then silence — the web
|
|
7
|
+
* input shows the black stop square (isLoading) and every new message lands
|
|
8
|
+
* in the queue forever.
|
|
9
|
+
*
|
|
10
|
+
* Root cause chain (worker side, see src/bridge/outboundRetry.ts +
|
|
11
|
+
* replBridge.ts): sendResult()/sendControlRequest() are fire-and-forget —
|
|
12
|
+
* a result written during a transport swap window ("DROP sendResult: no
|
|
13
|
+
* transport") or dropped by the uploader (maxConsecutiveFailures / close
|
|
14
|
+
* grace) is never re-sent. The server-side status derivation only returns
|
|
15
|
+
* to idle on an inbound result, so the session stays `running` forever and
|
|
16
|
+
* NOTHING self-heals it: the worker write watchdog is quiet (POSTs are
|
|
17
|
+
* healthy again), the WS inactivity clock is kept fresh by keepalive POSTs,
|
|
18
|
+
* and the control-request watchdog only tracks RPCs the server itself sent.
|
|
19
|
+
*
|
|
20
|
+
* This test pins the server-side self-heal: the stuck-turn watchdog sweep
|
|
21
|
+
* flips stale `running` sessions back to idle (publishing session_status so
|
|
22
|
+
* live SSE viewers unlock the input) and fires the recovery work re-dispatch.
|
|
23
|
+
* `requires_action` is deliberately out of scope — a visible permission card
|
|
24
|
+
* is actionable, not stuck.
|
|
25
|
+
*/
|
|
26
|
+
import { describe, test, expect, beforeEach, mock } from 'bun:test'
|
|
27
|
+
import { mockConfigModule } from './helpers/mock-config'
|
|
28
|
+
|
|
29
|
+
mock.module('../config', () =>
|
|
30
|
+
mockConfigModule({ pollTimeout: 1, stuckTurnTimeout: 60 }),
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
import { Hono } from 'hono'
|
|
34
|
+
import {
|
|
35
|
+
storeReset,
|
|
36
|
+
storeCreateEnvironment,
|
|
37
|
+
storeGetPendingWorkItem,
|
|
38
|
+
} from '../store'
|
|
39
|
+
import {
|
|
40
|
+
getAllEventBuses,
|
|
41
|
+
removeEventBus,
|
|
42
|
+
getEventBus,
|
|
43
|
+
type SessionEvent,
|
|
44
|
+
} from '../transport/event-bus'
|
|
45
|
+
|
|
46
|
+
import v1SessionIngress from '../routes/v1/session-ingress'
|
|
47
|
+
import webSessions from '../routes/web/sessions'
|
|
48
|
+
import webControl from '../routes/web/control'
|
|
49
|
+
import { issueToken } from '../auth/token'
|
|
50
|
+
import { getSession } from '../services/session'
|
|
51
|
+
import { sweepStuckTurnSessions } from '../services/stuck-turn-watchdog'
|
|
52
|
+
|
|
53
|
+
/** Matches the mocked config's stuckTurnTimeout (60s). */
|
|
54
|
+
const THRESHOLD_MS = 60 * 1000
|
|
55
|
+
|
|
56
|
+
function createApp() {
|
|
57
|
+
const app = new Hono()
|
|
58
|
+
app.route('/v2/session_ingress', v1SessionIngress)
|
|
59
|
+
app.route('/web', webSessions)
|
|
60
|
+
app.route('/web', webControl)
|
|
61
|
+
return app
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const AUTH_HEADERS = {
|
|
65
|
+
Authorization: 'Bearer test-api-key',
|
|
66
|
+
'X-Username': 'testuser',
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function auth(user: string): string {
|
|
70
|
+
return `Bearer ${issueToken(user).token}`
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Drive the session exactly like the real worker does: the Web UI starts
|
|
74
|
+
* the turn (outbound user), HybridTransport POSTs the worker's events via
|
|
75
|
+
* session-ingress (inbound assistant / tool-result user). */
|
|
76
|
+
async function driveTurn(app: Hono, sessionId: string): Promise<void> {
|
|
77
|
+
const sendRes = await app.request(`/web/sessions/${sessionId}/events`, {
|
|
78
|
+
method: 'POST',
|
|
79
|
+
headers: {
|
|
80
|
+
Authorization: auth('user-1'),
|
|
81
|
+
'Content-Type': 'application/json',
|
|
82
|
+
},
|
|
83
|
+
body: JSON.stringify({
|
|
84
|
+
type: 'user',
|
|
85
|
+
uuid: 'web-uuid-1',
|
|
86
|
+
content: 'run the task',
|
|
87
|
+
message: { content: 'run the task' },
|
|
88
|
+
}),
|
|
89
|
+
})
|
|
90
|
+
expect(sendRes.status).toBe(200)
|
|
91
|
+
|
|
92
|
+
// Worker: assistant streams (running), then the tool executes successfully
|
|
93
|
+
// and its tool-result user message is mirrored — the last events the
|
|
94
|
+
// server ever sees.
|
|
95
|
+
const ingestRes = await app.request(
|
|
96
|
+
`/v2/session_ingress/session/${sessionId}/events`,
|
|
97
|
+
{
|
|
98
|
+
method: 'POST',
|
|
99
|
+
headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
|
|
100
|
+
body: JSON.stringify({
|
|
101
|
+
events: [
|
|
102
|
+
{
|
|
103
|
+
type: 'assistant',
|
|
104
|
+
uuid: 'asst-uuid-1',
|
|
105
|
+
message: {
|
|
106
|
+
role: 'assistant',
|
|
107
|
+
content: [
|
|
108
|
+
{ type: 'tool_use', id: 't1', name: 'Bash', input: {} },
|
|
109
|
+
],
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
type: 'user',
|
|
114
|
+
uuid: 'tool-result-1',
|
|
115
|
+
message: {
|
|
116
|
+
role: 'user',
|
|
117
|
+
content: [
|
|
118
|
+
{ type: 'tool_result', tool_use_id: 't1', content: 'ok' },
|
|
119
|
+
],
|
|
120
|
+
},
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
}),
|
|
124
|
+
},
|
|
125
|
+
)
|
|
126
|
+
expect(ingestRes.status).toBe(200)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
describe('stuck-turn watchdog (lost turn-end result regression)', () => {
|
|
130
|
+
let app: Hono
|
|
131
|
+
let sessionId: string
|
|
132
|
+
let envId: string
|
|
133
|
+
|
|
134
|
+
beforeEach(async () => {
|
|
135
|
+
storeReset()
|
|
136
|
+
for (const [key] of getAllEventBuses()) {
|
|
137
|
+
removeEventBus(key)
|
|
138
|
+
}
|
|
139
|
+
app = createApp()
|
|
140
|
+
|
|
141
|
+
const env = storeCreateEnvironment({ secret: 's' })
|
|
142
|
+
envId = env.id
|
|
143
|
+
|
|
144
|
+
const res = await app.request('/web/sessions', {
|
|
145
|
+
method: 'POST',
|
|
146
|
+
headers: {
|
|
147
|
+
Authorization: auth('user-1'),
|
|
148
|
+
'Content-Type': 'application/json',
|
|
149
|
+
},
|
|
150
|
+
body: JSON.stringify({ environment_id: envId }),
|
|
151
|
+
})
|
|
152
|
+
const body = (await res.json()) as { id: string }
|
|
153
|
+
sessionId = body.id
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
test('lost turn-end result: session stuck running, sweep heals to idle', async () => {
|
|
157
|
+
await driveTurn(app, sessionId)
|
|
158
|
+
|
|
159
|
+
// ── The reproduction: the turn-end result event never arrives ──
|
|
160
|
+
// (dropped by the worker's uploader / transport-swap window). The
|
|
161
|
+
// session is now stuck on `running` — the web input locks and every
|
|
162
|
+
// new message queues forever.
|
|
163
|
+
expect(getSession(sessionId)?.status).toBe('running')
|
|
164
|
+
|
|
165
|
+
// Time passes; the worker's keepalive POSTs keep the WS alive but no
|
|
166
|
+
// conversation events arrive. Nothing on the server heals the status…
|
|
167
|
+
const healed = sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
|
|
168
|
+
expect(healed).toBe(1)
|
|
169
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
170
|
+
|
|
171
|
+
// A session_status event must be broadcast so live SSE viewers
|
|
172
|
+
// (RCSChatAdapter onStatusChange) unlock the input immediately.
|
|
173
|
+
const statuses = getEventBus(sessionId)
|
|
174
|
+
.getEventsSince(0)
|
|
175
|
+
.filter(e => e.type === 'session_status')
|
|
176
|
+
.map(e => (e.payload as Record<string, unknown>).status)
|
|
177
|
+
expect(statuses[statuses.length - 1]).toBe('idle')
|
|
178
|
+
|
|
179
|
+
// The sweep also queues the recovery work re-dispatch (zombie-transport
|
|
180
|
+
// rebuild) — same recovery the WS inactivity close uses.
|
|
181
|
+
expect(storeGetPendingWorkItem(envId)?.sessionId).toBe(sessionId)
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
test('genuinely progressing turn is NOT healed (recent inbound events)', async () => {
|
|
185
|
+
await driveTurn(app, sessionId)
|
|
186
|
+
|
|
187
|
+
// Worker still streaming — the sweep with fresh clock must not flip.
|
|
188
|
+
expect(sweepStuckTurnSessions()).toBe(0)
|
|
189
|
+
expect(getSession(sessionId)?.status).toBe('running')
|
|
190
|
+
|
|
191
|
+
// A fresh inbound assistant resets the silence clock — sweeping before
|
|
192
|
+
// the full threshold elapses since that event must not heal.
|
|
193
|
+
const ingestRes = await app.request(
|
|
194
|
+
`/v2/session_ingress/session/${sessionId}/events`,
|
|
195
|
+
{
|
|
196
|
+
method: 'POST',
|
|
197
|
+
headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
|
|
198
|
+
body: JSON.stringify({
|
|
199
|
+
events: [
|
|
200
|
+
{
|
|
201
|
+
type: 'assistant',
|
|
202
|
+
uuid: 'asst-uuid-2',
|
|
203
|
+
message: {
|
|
204
|
+
role: 'assistant',
|
|
205
|
+
content: [{ type: 'text', text: 'still working' }],
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
],
|
|
209
|
+
}),
|
|
210
|
+
},
|
|
211
|
+
)
|
|
212
|
+
expect(ingestRes.status).toBe(200)
|
|
213
|
+
|
|
214
|
+
// The assistant arrived moments ago — sweeping at threshold-1s (i.e. the
|
|
215
|
+
// last signal is younger than the threshold) must not heal.
|
|
216
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
|
|
217
|
+
expect(getSession(sessionId)?.status).toBe('running')
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
test('turn just started (outbound user, no worker reply yet) is NOT healed', async () => {
|
|
221
|
+
const res = await app.request(`/web/sessions/${sessionId}/events`, {
|
|
222
|
+
method: 'POST',
|
|
223
|
+
headers: {
|
|
224
|
+
Authorization: auth('user-1'),
|
|
225
|
+
'Content-Type': 'application/json',
|
|
226
|
+
},
|
|
227
|
+
body: JSON.stringify({
|
|
228
|
+
type: 'user',
|
|
229
|
+
uuid: 'fresh-user-1',
|
|
230
|
+
content: 'hello',
|
|
231
|
+
message: { content: 'hello' },
|
|
232
|
+
}),
|
|
233
|
+
})
|
|
234
|
+
expect(res.status).toBe(200)
|
|
235
|
+
expect(getSession(sessionId)?.status).toBe('running')
|
|
236
|
+
|
|
237
|
+
// The outbound user event marks the turn start — the worker gets the
|
|
238
|
+
// full threshold to produce its first inbound event.
|
|
239
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
|
|
240
|
+
expect(getSession(sessionId)?.status).toBe('running')
|
|
241
|
+
|
|
242
|
+
// …but a turn whose prompt is older than the threshold with zero worker
|
|
243
|
+
// output (worker never picked it up) does heal.
|
|
244
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
|
|
245
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
246
|
+
})
|
|
247
|
+
|
|
248
|
+
test('60s control RPC answers do NOT count as turn progress', async () => {
|
|
249
|
+
await driveTurn(app, sessionId)
|
|
250
|
+
|
|
251
|
+
// The Web UI polls get_context_usage every 60s while the page is open —
|
|
252
|
+
// the worker's control_response answers are RPC liveness, not turn
|
|
253
|
+
// progress. They must not keep a wedged turn looking alive.
|
|
254
|
+
const pollRes = await app.request(`/web/sessions/${sessionId}/control`, {
|
|
255
|
+
method: 'POST',
|
|
256
|
+
headers: {
|
|
257
|
+
Authorization: auth('user-1'),
|
|
258
|
+
'Content-Type': 'application/json',
|
|
259
|
+
},
|
|
260
|
+
body: JSON.stringify({
|
|
261
|
+
type: 'control_request',
|
|
262
|
+
request_id: 'rpc-1',
|
|
263
|
+
request: { subtype: 'get_context_usage' },
|
|
264
|
+
}),
|
|
265
|
+
})
|
|
266
|
+
expect(pollRes.status).toBe(200)
|
|
267
|
+
|
|
268
|
+
const answerRes = await app.request(
|
|
269
|
+
`/v2/session_ingress/session/${sessionId}/events`,
|
|
270
|
+
{
|
|
271
|
+
method: 'POST',
|
|
272
|
+
headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
|
|
273
|
+
body: JSON.stringify({
|
|
274
|
+
events: [
|
|
275
|
+
{
|
|
276
|
+
type: 'control_response',
|
|
277
|
+
response: {
|
|
278
|
+
subtype: 'success',
|
|
279
|
+
request_id: 'rpc-1',
|
|
280
|
+
response: {},
|
|
281
|
+
},
|
|
282
|
+
},
|
|
283
|
+
],
|
|
284
|
+
}),
|
|
285
|
+
},
|
|
286
|
+
)
|
|
287
|
+
expect(answerRes.status).toBe(200)
|
|
288
|
+
|
|
289
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
|
|
290
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
291
|
+
})
|
|
292
|
+
|
|
293
|
+
test('connect-time snapshots and background task_state do not feed the signal clock', async () => {
|
|
294
|
+
await driveTurn(app, sessionId)
|
|
295
|
+
|
|
296
|
+
// system/init (re-ssent on every reconnect), models_state, and
|
|
297
|
+
// task_state (background task churn outside any turn) are transport /
|
|
298
|
+
// task-list liveness, not turn progress — a wedged session must still
|
|
299
|
+
// heal even while these keep arriving.
|
|
300
|
+
const chatterRes = await app.request(
|
|
301
|
+
`/v2/session_ingress/session/${sessionId}/events`,
|
|
302
|
+
{
|
|
303
|
+
method: 'POST',
|
|
304
|
+
headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
|
|
305
|
+
body: JSON.stringify({
|
|
306
|
+
events: [
|
|
307
|
+
{
|
|
308
|
+
type: 'system',
|
|
309
|
+
subtype: 'init',
|
|
310
|
+
uuid: 'init-1',
|
|
311
|
+
message: {},
|
|
312
|
+
},
|
|
313
|
+
{
|
|
314
|
+
type: 'models_state',
|
|
315
|
+
uuid: 'models-1',
|
|
316
|
+
message: {},
|
|
317
|
+
},
|
|
318
|
+
{
|
|
319
|
+
type: 'task_state',
|
|
320
|
+
uuid: 'tasks-1',
|
|
321
|
+
message: {},
|
|
322
|
+
},
|
|
323
|
+
],
|
|
324
|
+
}),
|
|
325
|
+
},
|
|
326
|
+
)
|
|
327
|
+
expect(chatterRes.status).toBe(200)
|
|
328
|
+
|
|
329
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
|
|
330
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
331
|
+
})
|
|
332
|
+
|
|
333
|
+
test('subagent task events DO feed the signal clock (genuine turn progress)', async () => {
|
|
334
|
+
await driveTurn(app, sessionId)
|
|
335
|
+
|
|
336
|
+
const progressRes = await app.request(
|
|
337
|
+
`/v2/session_ingress/session/${sessionId}/events`,
|
|
338
|
+
{
|
|
339
|
+
method: 'POST',
|
|
340
|
+
headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
|
|
341
|
+
body: JSON.stringify({
|
|
342
|
+
events: [
|
|
343
|
+
{
|
|
344
|
+
type: 'system',
|
|
345
|
+
subtype: 'task_progress',
|
|
346
|
+
uuid: 'task-progress-1',
|
|
347
|
+
task_id: 'task-1',
|
|
348
|
+
message: {},
|
|
349
|
+
},
|
|
350
|
+
],
|
|
351
|
+
}),
|
|
352
|
+
},
|
|
353
|
+
)
|
|
354
|
+
expect(progressRes.status).toBe(200)
|
|
355
|
+
|
|
356
|
+
// The task_progress event just arrived — sweeping before the full
|
|
357
|
+
// threshold since it must not heal.
|
|
358
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
|
|
359
|
+
expect(getSession(sessionId)?.status).toBe('running')
|
|
360
|
+
})
|
|
361
|
+
|
|
362
|
+
test('pending permission ask (requires_action) is out of scope — never healed', async () => {
|
|
363
|
+
await driveTurn(app, sessionId)
|
|
364
|
+
|
|
365
|
+
// The worker's next tool needs permission — the can_use_tool ask is
|
|
366
|
+
// visible and actionable in the Web UI (card with approve/deny), so a
|
|
367
|
+
// requires_action session is NOT stuck: the user is the one blocking.
|
|
368
|
+
// The sweep must never touch it, no matter how long the user waits.
|
|
369
|
+
const askRes = await app.request(
|
|
370
|
+
`/v2/session_ingress/session/${sessionId}/events`,
|
|
371
|
+
{
|
|
372
|
+
method: 'POST',
|
|
373
|
+
headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
|
|
374
|
+
body: JSON.stringify({
|
|
375
|
+
events: [
|
|
376
|
+
{
|
|
377
|
+
type: 'control_request',
|
|
378
|
+
request_id: 'perm-1',
|
|
379
|
+
request: {
|
|
380
|
+
subtype: 'can_use_tool',
|
|
381
|
+
tool_name: 'Bash',
|
|
382
|
+
input: {},
|
|
383
|
+
tool_use_id: 't2',
|
|
384
|
+
},
|
|
385
|
+
},
|
|
386
|
+
],
|
|
387
|
+
}),
|
|
388
|
+
},
|
|
389
|
+
)
|
|
390
|
+
expect(askRes.status).toBe(200)
|
|
391
|
+
expect(getSession(sessionId)?.status).toBe('requires_action')
|
|
392
|
+
|
|
393
|
+
expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0)
|
|
394
|
+
expect(getSession(sessionId)?.status).toBe('requires_action')
|
|
395
|
+
})
|
|
396
|
+
|
|
397
|
+
test('lost permission ask (stuck on running, card never shown) heals', async () => {
|
|
398
|
+
await driveTurn(app, sessionId)
|
|
399
|
+
// The ask itself was lost in transit — status never left running and no
|
|
400
|
+
// card is shown. Same sweep heals it (input unlock; the worker-side
|
|
401
|
+
// retry re-delivers the ask and flips it to requires_action).
|
|
402
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
|
|
403
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
404
|
+
})
|
|
405
|
+
|
|
406
|
+
test('idle sessions and fresh buses are untouched', async () => {
|
|
407
|
+
await driveTurn(app, sessionId)
|
|
408
|
+
sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
|
|
409
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
410
|
+
|
|
411
|
+
// Already idle — the sweep must not publish anything new.
|
|
412
|
+
const before = getEventBus(sessionId)
|
|
413
|
+
.getEventsSince(0)
|
|
414
|
+
.filter(e => e.type === 'session_status').length
|
|
415
|
+
expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0)
|
|
416
|
+
const after = getEventBus(sessionId)
|
|
417
|
+
.getEventsSince(0)
|
|
418
|
+
.filter(e => e.type === 'session_status').length
|
|
419
|
+
expect(after).toBe(before)
|
|
420
|
+
})
|
|
421
|
+
|
|
422
|
+
test('server-restart legacy: running session with an empty bus heals', async () => {
|
|
423
|
+
// A session persisted as `running` whose bus is fresh (server restart)
|
|
424
|
+
// has no turn signal at all — the sweep falls back to the session's
|
|
425
|
+
// creation time so restart-legacy stuck sessions still heal.
|
|
426
|
+
const { storeUpdateSession } = await import('../store')
|
|
427
|
+
storeUpdateSession(sessionId, { status: 'running' })
|
|
428
|
+
removeEventBus(sessionId) // simulate fresh bus after restart
|
|
429
|
+
|
|
430
|
+
const swept = sweepStuckTurnSessions(Date.now() + 1000)
|
|
431
|
+
// Session was created milliseconds ago — creation-time fallback gives
|
|
432
|
+
// the same grace window as a fresh turn.
|
|
433
|
+
expect(swept).toBe(0)
|
|
434
|
+
|
|
435
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
|
|
436
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
437
|
+
})
|
|
438
|
+
|
|
439
|
+
test("watchdog's own session_status event does not count as turn progress", async () => {
|
|
440
|
+
await driveTurn(app, sessionId)
|
|
441
|
+
sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
|
|
442
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
443
|
+
|
|
444
|
+
// Flip it back to running (e.g. a late assistant with turn signal) and
|
|
445
|
+
// verify the sweep's previously published session_status (inbound) did
|
|
446
|
+
// not arm the signal clock — a sweep right past the threshold still
|
|
447
|
+
// fires, and repeated sweeps keep healing.
|
|
448
|
+
const { storeUpdateSession } = await import('../store')
|
|
449
|
+
storeUpdateSession(sessionId, { status: 'running' })
|
|
450
|
+
expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 10_000)).toBe(1)
|
|
451
|
+
expect(getSession(sessionId)?.status).toBe('idle')
|
|
452
|
+
})
|
|
453
|
+
})
|
|
@@ -39,6 +39,14 @@ export interface RcsConfig {
|
|
|
39
39
|
* (session_keepalive_interval_v2_ms, default 120s) plus jitter, otherwise
|
|
40
40
|
* healthy idle sessions get torn down between keepalives. Default 300s. */
|
|
41
41
|
wsClientInactivityTimeout: number
|
|
42
|
+
/** Stuck-turn watchdog (seconds). A session whose status is `running` with
|
|
43
|
+
* no inbound worker events (and no outbound user message) for this long is
|
|
44
|
+
* flipped back to `idle` + gets a recovery work re-dispatch — the turn-end
|
|
45
|
+
* result event was lost in transit (worker-side drop) and nothing else
|
|
46
|
+
* would ever release the Web UI input. Generous vs. long silent tools:
|
|
47
|
+
* a false flip only unlocks the input early (messages queue in the REPL);
|
|
48
|
+
* the next worker event re-derives `running`. 0 disables. Default 900s. */
|
|
49
|
+
stuckTurnTimeout: number
|
|
42
50
|
/** Whether artifact content (GET /v1/artifacts/:id/content) is served
|
|
43
51
|
* without authentication. Default true — the unguessable artifact id is
|
|
44
52
|
* the secret, same trust model as the upstream cloud-artifacts service.
|
|
@@ -84,6 +92,7 @@ export const config: RcsConfig = {
|
|
|
84
92
|
process.env.RCS_WS_CLIENT_INACTIVITY_TIMEOUT || '300',
|
|
85
93
|
10,
|
|
86
94
|
),
|
|
95
|
+
stuckTurnTimeout: parseInt(process.env.RCS_STUCK_TURN_TIMEOUT || '900', 10),
|
|
87
96
|
artifactsPublicAccess: process.env.RCS_ARTIFACTS_PUBLIC_ACCESS !== 'false',
|
|
88
97
|
}
|
|
89
98
|
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
} from '../store'
|
|
8
8
|
import { config } from '../config'
|
|
9
9
|
import { sweepUnansweredControlRequests } from './transport'
|
|
10
|
+
import { sweepStuckTurnSessions } from './stuck-turn-watchdog'
|
|
10
11
|
|
|
11
12
|
/** ACP env 离线后留给 acp-link 重连复用的宽限时间。超过后才将 env 置为
|
|
12
13
|
* disconnected。会话不再自动归档——由用户手动删除。 */
|
|
@@ -62,5 +63,9 @@ export function startDisconnectMonitor(): ReturnType<typeof setInterval> {
|
|
|
62
63
|
// path (WS healthy, POST path dead) from unanswered RPCs and fires the
|
|
63
64
|
// recovery work re-dispatch. See services/transport.ts.
|
|
64
65
|
sweepUnansweredControlRequests()
|
|
66
|
+
// Stuck-turn watchdog — heals sessions whose turn-end result event was
|
|
67
|
+
// lost in transit (status stuck `running`, Web UI input locked forever).
|
|
68
|
+
// See services/stuck-turn-watchdog.ts.
|
|
69
|
+
sweepStuckTurnSessions()
|
|
65
70
|
}, 60_000) // Check every minute
|
|
66
71
|
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { log } from '../logger'
|
|
2
|
+
import { storeListSessions } from '../store'
|
|
3
|
+
import { config } from '../config'
|
|
4
|
+
import type { SessionEvent } from '../transport/event-bus'
|
|
5
|
+
import { getEventBus } from '../transport/event-bus'
|
|
6
|
+
import { updateSessionStatus } from './session'
|
|
7
|
+
import { redispatchWorkForSession } from './work-dispatch'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Stuck-turn watchdog — the server-side self-heal for sessions whose
|
|
11
|
+
* turn-end signal was lost in transit.
|
|
12
|
+
*
|
|
13
|
+
* A session's status only returns from `running` to `idle` when the worker's
|
|
14
|
+
* `result`/`result_success` event arrives (ws-handler.ts ingestBridgeMessage)
|
|
15
|
+
* or the user interrupts. The worker sends that result fire-and-forget —
|
|
16
|
+
* when the write is dropped (transport-swap window, uploader batch drop
|
|
17
|
+
* after maxConsecutiveFailures, close-grace expiry), it is never re-sent and
|
|
18
|
+
* the session stays `running` forever. The Web UI locks the input (black
|
|
19
|
+
* stop square) and every new message lands in a queue that never drains.
|
|
20
|
+
*
|
|
21
|
+
* Nothing else detects this state:
|
|
22
|
+
* - the worker's write-path watchdog only fires when POSTs stop completing
|
|
23
|
+
* entirely (a transient drop followed by healthy POSTs is invisible);
|
|
24
|
+
* - the WS inactivity clock is kept fresh by keepalive POSTs;
|
|
25
|
+
* - the control-request watchdog only tracks RPCs the server itself sent.
|
|
26
|
+
*
|
|
27
|
+
* The sweep flips a `running` session with no turn signal for
|
|
28
|
+
* `config.stuckTurnTimeout` back to `idle` (updateSessionStatus publishes
|
|
29
|
+
* `session_status`, unlocking live SSE viewers immediately — a genuinely
|
|
30
|
+
* still-running turn re-derives `running` on its next event) and fires the
|
|
31
|
+
* recovery work re-dispatch (zombie-transport rebuild, same as the WS
|
|
32
|
+
* inactivity close).
|
|
33
|
+
*
|
|
34
|
+
* `requires_action` is deliberately out of scope: a permission ask that
|
|
35
|
+
* arrived is visible and actionable in the Web UI — the user is the one
|
|
36
|
+
* blocking, not a lost signal.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Inbound event types that count as turn progress — an ALLOWLIST, so future
|
|
41
|
+
* chatter types default to not feeding the signal clock:
|
|
42
|
+
* - `user` (worker-mirrored tool results), `assistant`, `result`/
|
|
43
|
+
* `result_success`, `error` — the conversation itself;
|
|
44
|
+
* - `system` — subagent task events (task_started/task_progress/…).
|
|
45
|
+
* Exception: subtype `init` is re-sent on every transport (re)connect —
|
|
46
|
+
* transport liveness, not turn progress.
|
|
47
|
+
*
|
|
48
|
+
* Deliberately NOT counted:
|
|
49
|
+
* - `control_request`/`control_response` — RPC traffic. The Web UI polls
|
|
50
|
+
* get_context_usage every 60s while the page is open, so the worker's
|
|
51
|
+
* control_response answers flow constantly even for a wedged session —
|
|
52
|
+
* counting them would make the watchdog permanently blind exactly when
|
|
53
|
+
* the user is watching the stuck session.
|
|
54
|
+
* - `models_state`, `system/init` — connect-time snapshots.
|
|
55
|
+
* - `task_state` — background task-list churn; a background task running
|
|
56
|
+
* outside any turn must not starve the watchdog.
|
|
57
|
+
* - `session_status` — server-synthesized (including this watchdog's own
|
|
58
|
+
* idle flip; counting it would make the sweep single-shot).
|
|
59
|
+
*/
|
|
60
|
+
const SIGNAL_EVENT_TYPES = new Set([
|
|
61
|
+
'user',
|
|
62
|
+
'assistant',
|
|
63
|
+
'partial_assistant',
|
|
64
|
+
'result',
|
|
65
|
+
'result_success',
|
|
66
|
+
'error',
|
|
67
|
+
'system',
|
|
68
|
+
])
|
|
69
|
+
|
|
70
|
+
function isInboundSignal(event: SessionEvent): boolean {
|
|
71
|
+
if (event.direction !== 'inbound') return false
|
|
72
|
+
if (!SIGNAL_EVENT_TYPES.has(event.type)) return false
|
|
73
|
+
if (event.type === 'system') {
|
|
74
|
+
// system/init is a connect-time snapshot, re-sent on every
|
|
75
|
+
// (re)connect — not turn progress.
|
|
76
|
+
const subtype = (event.payload as Record<string, unknown> | null)?.subtype
|
|
77
|
+
if (subtype === 'init') return false
|
|
78
|
+
}
|
|
79
|
+
return true
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Wall-clock of the latest event on the bus that proves the session's turn
|
|
84
|
+
* is (or was) progressing: any inbound worker conversation event (assistant,
|
|
85
|
+
* tool-result user, result, subagent task events), or the outbound user
|
|
86
|
+
* message that started the turn. Returns 0 when the bus carries no signal
|
|
87
|
+
* at all (fresh bus after a server restart — callers fall back to the
|
|
88
|
+
* persisted session record).
|
|
89
|
+
*/
|
|
90
|
+
export function lastTurnSignalAt(events: SessionEvent[]): number {
|
|
91
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
92
|
+
const event = events[i]
|
|
93
|
+
if (isInboundSignal(event)) {
|
|
94
|
+
return event.createdAt
|
|
95
|
+
}
|
|
96
|
+
if (event.direction === 'outbound' && event.type === 'user') {
|
|
97
|
+
return event.createdAt
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return 0
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Sweep all sessions for stuck turns. Returns the number of sessions healed.
|
|
105
|
+
* `now` is injectable for tests.
|
|
106
|
+
*/
|
|
107
|
+
export function sweepStuckTurnSessions(now: number = Date.now()): number {
|
|
108
|
+
const thresholdMs = config.stuckTurnTimeout * 1000
|
|
109
|
+
if (!(thresholdMs > 0)) return 0
|
|
110
|
+
|
|
111
|
+
let swept = 0
|
|
112
|
+
for (const session of storeListSessions()) {
|
|
113
|
+
// Only `running` locks the input with no recovery path; see module docs
|
|
114
|
+
// for why requires_action (visible, actionable card) and archived
|
|
115
|
+
// sessions are excluded.
|
|
116
|
+
if (session.status !== 'running') continue
|
|
117
|
+
// ACP sessions carry their turn traffic on the per-channel-group ACP
|
|
118
|
+
// bus (getAcpEventBus), not the session bus this sweep scans — healing
|
|
119
|
+
// them from the wrong bus would be a false positive.
|
|
120
|
+
if (session.source === 'acp') continue
|
|
121
|
+
|
|
122
|
+
const bus = getEventBus(session.id)
|
|
123
|
+
const lastSignal = lastTurnSignalAt(bus.getEventsSince(0))
|
|
124
|
+
// Empty bus (server restart) or a signal older than the session record
|
|
125
|
+
// (persisted legacy) — fall back to the persisted creation time so
|
|
126
|
+
// restart-legacy stuck sessions still heal after the same grace window.
|
|
127
|
+
const reference = Math.max(lastSignal, session.createdAt.getTime())
|
|
128
|
+
if (now - reference <= thresholdMs) continue
|
|
129
|
+
|
|
130
|
+
const silenceSec = Math.round((now - reference) / 1000)
|
|
131
|
+
log(
|
|
132
|
+
`[RCS] Stuck-turn watchdog: session=${session.id} running with no turn signal for ${silenceSec}s` +
|
|
133
|
+
' — flipping to idle (turn-end result was lost) and redispatching work',
|
|
134
|
+
)
|
|
135
|
+
// Recovery re-dispatch first: when the worker's transport is actually
|
|
136
|
+
// dead this rebuilds it (fresh uploader → the write path recovers).
|
|
137
|
+
// No-op for healthy transports with an unchanged token.
|
|
138
|
+
void redispatchWorkForSession(session.id)
|
|
139
|
+
// Publishes `session_status` → live SSE viewers unlock the input.
|
|
140
|
+
updateSessionStatus(session.id, 'idle')
|
|
141
|
+
swept++
|
|
142
|
+
}
|
|
143
|
+
return swept
|
|
144
|
+
}
|