@saluzi/saluzi-edu 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +55 -55
- package/dist/guide/guide-data.json +175 -175
- package/package.json +1 -1
- package/packages/remote-control-server/src/__tests__/control-watchdog.test.ts +129 -0
- package/packages/remote-control-server/src/__tests__/helpers/mock-config.ts +2 -0
- package/packages/remote-control-server/src/__tests__/stuck-turn-watchdog.test.ts +453 -0
- package/packages/remote-control-server/src/__tests__/work-dispatch.test.ts +44 -0
- package/packages/remote-control-server/src/__tests__/ws-handler.test.ts +185 -1
- package/packages/remote-control-server/src/config.ts +9 -0
- package/packages/remote-control-server/src/services/disconnect-monitor.ts +10 -0
- package/packages/remote-control-server/src/services/stuck-turn-watchdog.ts +144 -0
- package/packages/remote-control-server/src/services/transport.ts +89 -0
- package/packages/remote-control-server/src/services/work-dispatch.ts +56 -1
- package/packages/remote-control-server/src/transport/ws-handler.ts +85 -20
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
import { log, error as logError } from '../logger'
|
|
11
11
|
import { toClientPayload } from './client-payload'
|
|
12
12
|
import { wipeSessionConversation } from '../services/session-clear'
|
|
13
|
+
import { redispatchWorkForSession } from '../services/work-dispatch'
|
|
13
14
|
import { config } from '../config'
|
|
14
15
|
|
|
15
16
|
// Per-connection cleanup, keyed by sessionId (only one WS per session)
|
|
@@ -25,6 +26,25 @@ const cleanupBySession = new Map<string, CleanupEntry>()
|
|
|
25
26
|
// Track all active WS connections for graceful shutdown
|
|
26
27
|
const activeConnections = new Set<WSContext>()
|
|
27
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Event types excluded from the WS-reconnect history replay. These are
|
|
31
|
+
* point-in-time RPC traffic, not conversation state: replaying them to the
|
|
32
|
+
* bridge would re-execute them as fresh requests — a replayed `interrupt`
|
|
33
|
+
* aborts whatever turn is running AFTER the reconnect, replayed
|
|
34
|
+
* `get_context_usage` fires spurious responses, and replayed
|
|
35
|
+
* permission asks/responses resurrect stale dialogs. The replay exists so
|
|
36
|
+
* the bridge can reconstruct the conversation transcript (user/assistant/
|
|
37
|
+
* system messages) — control traffic has no reconstruction value.
|
|
38
|
+
*/
|
|
39
|
+
const NON_REPLAYABLE_EVENT_TYPES = new Set([
|
|
40
|
+
'control_request',
|
|
41
|
+
'control_response',
|
|
42
|
+
'control_cancel_request',
|
|
43
|
+
'permission_request',
|
|
44
|
+
'permission_response',
|
|
45
|
+
'interrupt',
|
|
46
|
+
])
|
|
47
|
+
|
|
28
48
|
// Server-side keepalive interval (configurable via RCS_WS_KEEPALIVE_INTERVAL).
|
|
29
49
|
// Sends data frames to keep reverse proxies from closing idle connections.
|
|
30
50
|
const SERVER_KEEPALIVE_INTERVAL_MS = (config.wsKeepaliveInterval || 20) * 1000
|
|
@@ -64,14 +84,20 @@ export function handleWebSocketOpen(ws: WSContext, sessionId: string) {
|
|
|
64
84
|
|
|
65
85
|
const bus = getEventBus(sessionId)
|
|
66
86
|
|
|
67
|
-
// Replay
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
//
|
|
87
|
+
// Replay conversation events so the bridge can reconstruct the full
|
|
88
|
+
// history — assistant replies are inbound events. Transient streaming
|
|
89
|
+
// deltas are skipped (display-only, would flood the child's stdin parser),
|
|
90
|
+
// and so is point-in-time control traffic (see NON_REPLAYABLE_EVENT_TYPES —
|
|
91
|
+
// a replayed interrupt would abort the post-reconnect turn).
|
|
71
92
|
const missed = bus.getEventsSince(0)
|
|
72
93
|
if (missed.length > 0) {
|
|
73
|
-
|
|
74
|
-
|
|
94
|
+
const replayable = missed.filter(
|
|
95
|
+
event => !NON_REPLAYABLE_EVENT_TYPES.has(event.type),
|
|
96
|
+
)
|
|
97
|
+
log(
|
|
98
|
+
`[WS] Replaying ${replayable.length} of ${missed.length} retained event(s)`,
|
|
99
|
+
)
|
|
100
|
+
for (const event of replayable) {
|
|
75
101
|
if (ws.readyState !== 1) break
|
|
76
102
|
if (event.type === 'stream_event' || event.type === 'partial_assistant')
|
|
77
103
|
continue
|
|
@@ -102,17 +128,10 @@ export function handleWebSocketOpen(ws: WSContext, sessionId: string) {
|
|
|
102
128
|
clearInterval(keepalive)
|
|
103
129
|
return
|
|
104
130
|
}
|
|
105
|
-
// Check if client is still alive — close if no data received for too
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
`[WS] Client inactive for ${Math.round(silenceMs / 1000)}s on session=${sessionId}, closing dead connection`,
|
|
110
|
-
)
|
|
111
|
-
try {
|
|
112
|
-
ws.close(1000, 'client inactive')
|
|
113
|
-
} catch {
|
|
114
|
-
clearInterval(keepalive)
|
|
115
|
-
}
|
|
131
|
+
// Check if client is still alive — close if no data received for too
|
|
132
|
+
// long (see closeInactiveClient: the close doubles as the write-path
|
|
133
|
+
// recovery trigger via work re-dispatch).
|
|
134
|
+
if (closeInactiveClient(sessionId)) {
|
|
116
135
|
return
|
|
117
136
|
}
|
|
118
137
|
try {
|
|
@@ -133,18 +152,29 @@ export function handleWebSocketOpen(ws: WSContext, sessionId: string) {
|
|
|
133
152
|
|
|
134
153
|
/**
|
|
135
154
|
* Called from onMessage — bridge sends newline-delimited JSON.
|
|
155
|
+
*
|
|
156
|
+
* Activity tracking skips pure keep_alive frames: with HybridTransport the
|
|
157
|
+
* worker's WS-layer keepalives (WebSocketTransport.startKeepaliveInterval,
|
|
158
|
+
* direct socket writes) keep flowing even when the POST write path is dead —
|
|
159
|
+
* counting them would make the inactivity close (and its write-path
|
|
160
|
+
* recovery re-dispatch) unreachable for exactly the zombie state it exists
|
|
161
|
+
* to detect. Real WS messages and ALL POST ingress still refresh the clock.
|
|
136
162
|
*/
|
|
137
163
|
export function handleWebSocketMessage(
|
|
138
164
|
ws: WSContext,
|
|
139
165
|
sessionId: string,
|
|
140
166
|
data: string,
|
|
141
167
|
) {
|
|
142
|
-
// Track client activity for dead-connection detection
|
|
143
|
-
touchClientActivity(sessionId)
|
|
144
168
|
const lines = data.split('\n').filter(l => l.trim())
|
|
145
169
|
for (const line of lines) {
|
|
146
170
|
try {
|
|
147
|
-
|
|
171
|
+
const parsed = JSON.parse(line) as Record<string, unknown>
|
|
172
|
+
// Track client activity for dead-connection detection — non-keepalive
|
|
173
|
+
// frames only (see function comment).
|
|
174
|
+
if (parsed.type !== 'keep_alive') {
|
|
175
|
+
touchClientActivity(sessionId)
|
|
176
|
+
}
|
|
177
|
+
ingestBridgeMessage(sessionId, parsed)
|
|
148
178
|
} catch (err) {
|
|
149
179
|
logError('[WS] parse error:', err)
|
|
150
180
|
}
|
|
@@ -164,6 +194,41 @@ export function touchClientActivity(sessionId: string): void {
|
|
|
164
194
|
}
|
|
165
195
|
}
|
|
166
196
|
|
|
197
|
+
/**
|
|
198
|
+
* Inactivity sweep for one session's WS connection. Returns true when the
|
|
199
|
+
* connection was closed as dead.
|
|
200
|
+
*
|
|
201
|
+
* With HybridTransport the inactivity clock ONLY advances via HTTP POST
|
|
202
|
+
* ingress (writes go through POST; the WS carries no app-level
|
|
203
|
+
* client→server frames). 300s of silence with a live WS therefore means the
|
|
204
|
+
* worker's event uploader is wedged, not that the WS is dead — the WS will
|
|
205
|
+
* auto-reconnect within seconds. Closing it alone changes nothing (the
|
|
206
|
+
* zombie uploader is per-transport and survives WS reconnects), so this
|
|
207
|
+
* also queues a recovery work item: the worker's next at-capacity poll
|
|
208
|
+
* (≤10 min) fires onWorkReceived → fresh transport (fresh uploader) → the
|
|
209
|
+
* write path recovers. See redispatchWorkForSession.
|
|
210
|
+
*/
|
|
211
|
+
export function closeInactiveClient(
|
|
212
|
+
sessionId: string,
|
|
213
|
+
now: number = Date.now(),
|
|
214
|
+
): boolean {
|
|
215
|
+
const entry = cleanupBySession.get(sessionId)
|
|
216
|
+
if (!entry) return false
|
|
217
|
+
if (entry.ws.readyState !== 1) return false
|
|
218
|
+
const silenceMs = now - entry.lastClientActivity
|
|
219
|
+
if (silenceMs <= CLIENT_ACTIVITY_TIMEOUT_MS) return false
|
|
220
|
+
log(
|
|
221
|
+
`[WS] Client inactive for ${Math.round(silenceMs / 1000)}s on session=${sessionId}, closing dead connection`,
|
|
222
|
+
)
|
|
223
|
+
try {
|
|
224
|
+
entry.ws.close(1000, 'client inactive')
|
|
225
|
+
} catch {
|
|
226
|
+
// already closed
|
|
227
|
+
}
|
|
228
|
+
void redispatchWorkForSession(sessionId)
|
|
229
|
+
return true
|
|
230
|
+
}
|
|
231
|
+
|
|
167
232
|
/** Called from onClose — unsubscribes from event bus */
|
|
168
233
|
export function handleWebSocketClose(
|
|
169
234
|
ws: WSContext,
|