@saluzi/saluzi-edu 0.4.5 → 0.4.7

package/packages/remote-control-server/src/__tests__/ws-handler.test.ts

@@ -4,7 +4,11 @@ import { mockConfigModule } from './helpers/mock-config'
 // Mock config before imports
 mock.module('../config', () => mockConfigModule())
 
-import { storeReset } from '../store'
+import {
+  storeReset,
+  storeCreateEnvironment,
+  storeCreateSession,
+} from '../store'
 import {
   getEventBus,
   removeEventBus,
@@ -17,6 +21,7 @@ import {
   handleWebSocketClose,
   closeAllConnections,
   touchClientActivity,
+  closeInactiveClient,
 } from '../transport/ws-handler'
 
 // Minimal WSContext mock
@@ -451,6 +456,55 @@ describe('ws-handler', () => {
       expect(msg.isSynthetic).toBe(true)
     })
 
+    test('replay skips point-in-time control traffic (interrupt, permission RPCs)', () => {
+      // Regression: the replay used to forward EVERY retained event,
+      // including one-shot RPCs. A replayed `interrupt` control_request was
+      // re-executed by the bridge — aborting whatever turn was running
+      // AFTER the reconnect.
+      const bus = getEventBus('replay_ctrl')
+      bus.publish({
+        id: 'e1',
+        sessionId: 'replay_ctrl',
+        type: 'user',
+        payload: { content: 'task prompt' },
+        direction: 'outbound',
+      })
+      bus.publish({
+        id: 'e2',
+        sessionId: 'replay_ctrl',
+        type: 'control_request',
+        payload: { request_id: 'r1', request: { subtype: 'interrupt' } },
+        direction: 'outbound',
+      })
+      bus.publish({
+        id: 'e3',
+        sessionId: 'replay_ctrl',
+        type: 'permission_response',
+        payload: { request_id: 'r2', approved: true },
+        direction: 'outbound',
+      })
+      bus.publish({
+        id: 'e4',
+        sessionId: 'replay_ctrl',
+        type: 'assistant',
+        payload: { content: 'done' },
+        direction: 'inbound',
+      })
+
+      const ws = createMockWs()
+      handleWebSocketOpen(ws, 'replay_ctrl')
+
+      const sent = ws.getSentData()
+      const types = sent.map(
+        (line: string) => (JSON.parse(line) as { type: string }).type,
+      )
+      expect(types).toContain('user')
+      expect(types).toContain('assistant')
+      expect(types).not.toContain('control_request')
+      expect(types).not.toContain('permission_response')
+      expect(types).not.toContain('interrupt')
+    })
+
     test('replaces existing connection for same session', () => {
       const ws1 = createMockWs()
       const ws2 = createMockWs()
@@ -546,6 +600,136 @@ describe('ws-handler', () => {
     })
   })
 
+  describe('closeInactiveClient', () => {
+    // Regression: the original keepalive interval read a const closure
+    // variable captured at open time — touchClientActivity mutated a
+    // different copy on the cleanup entry, so EVERY connection was closed
+    // exactly CLIENT_ACTIVITY_TIMEOUT after open ("client inactive" 误报,
+    // observed as duration=300s closes on actively-POSTing workers).
+    test('client activity via touchClientActivity resets the inactivity clock', () => {
+      const closed: Array<{ code?: number; reason?: string }> = []
+      const ws = {
+        readyState: 1,
+        send: () => {},
+        close: (code?: number, reason?: string) =>
+          closed.push({ code, reason }),
+      } as any
+      handleWebSocketOpen(ws, 'inact1')
+
+      // Connection opened at T0; the worker POSTs (touch) at T0+10min.
+      // Measured 100s after the touch → within the 300s budget → NOT closed.
+      const touchedAt = Date.now() + 600_000
+      const originalNow = Date.now
+      Date.now = () => touchedAt
+      try {
+        touchClientActivity('inact1')
+      } finally {
+        Date.now = originalNow
+      }
+      expect(closeInactiveClient('inact1', touchedAt + 100_000)).toBe(false)
+      expect(closed).toHaveLength(0)
+
+      handleWebSocketClose(ws, 'inact1', 1000, 'done')
+    })
+
+    test('closes a truly silent client and queues a recovery work item', async () => {
+      const env = storeCreateEnvironment({ secret: 's' })
+      const session = storeCreateSession({ environmentId: env.id })
+
+      const closed: Array<{ code?: number; reason?: string }> = []
+      const ws = {
+        readyState: 1,
+        send: () => {},
+        close: (code?: number, reason?: string) =>
+          closed.push({ code, reason }),
+      } as any
+      handleWebSocketOpen(ws, session.id)
+
+      // Opened at ~now; nothing touches for 400s → closed as dead.
+      const openedAt = Date.now()
+      expect(closeInactiveClient(session.id, openedAt + 400_000)).toBe(true)
+      expect(closed).toHaveLength(1)
+      expect(closed[0]?.code).toBe(1000)
+
+      // Redispatch is fire-and-forget — let the microtask settle.
+      await new Promise(resolve => setTimeout(resolve, 0))
+      const { storeGetPendingWorkItem } = await import('../store')
+      expect(storeGetPendingWorkItem(env.id)?.sessionId).toBe(session.id)
+
+      handleWebSocketClose(ws, session.id, 1000, 'done')
+    })
+
+    test('does not close within the inactivity budget', () => {
+      const ws = createMockWs()
+      handleWebSocketOpen(ws, 'inact3')
+      expect(closeInactiveClient('inact3', Date.now() + 100_000)).toBe(false)
+      handleWebSocketClose(ws, 'inact3', 1000, 'done')
+    })
+
+    test('is a no-op for unknown sessions and dead sockets', () => {
+      expect(closeInactiveClient('never-opened')).toBe(false)
+      const ws = createMockWs(3) // CLOSED
+      handleWebSocketOpen(ws, 'inact4')
+      expect(closeInactiveClient('inact4', Date.now() + 400_000)).toBe(false)
+      handleWebSocketClose(ws, 'inact4', 1000, 'done')
+    })
+
+    // HybridTransport workers send WS-level keep_alive frames (direct
+    // socket writes) even when their POST write path is dead — counting
+    // them as activity would make the inactivity close unreachable for
+    // exactly the zombie state it exists to detect.
+    test('WS keep_alive frames do NOT refresh the inactivity clock', () => {
+      const closed: Array<{ code?: number; reason?: string }> = []
+      const ws = {
+        readyState: 1,
+        send: () => {},
+        close: (code?: number, reason?: string) =>
+          closed.push({ code, reason }),
+      } as any
+      handleWebSocketOpen(ws, 'ka1')
+
+      // keep_alive 帧到达 +250s（预算内）— 不刷新时钟
+      const at250 = Date.now() + 250_000
+      const originalNow = Date.now
+      Date.now = () => at250
+      try {
+        handleWebSocketMessage(ws, 'ka1', '{"type":"keep_alive"}\n')
+      } finally {
+        Date.now = originalNow
+      }
+
+      // +350s（自打开起超预算）→ 尽管 keepalive 一直在发，仍然关闭
+      expect(closeInactiveClient('ka1', Date.now() + 350_000)).toBe(true)
+      expect(closed).toHaveLength(1)
+
+      handleWebSocketClose(ws, 'ka1', 1000, 'done')
+    })
+
+    test('real WS messages still refresh the inactivity clock', () => {
+      const ws = createMockWs()
+      handleWebSocketOpen(ws, 'ka2')
+
+      // 真实消息（非 keep_alive）到达 +250s — 刷新时钟
+      const at250 = Date.now() + 250_000
+      const originalNow = Date.now
+      Date.now = () => at250
+      try {
+        handleWebSocketMessage(
+          ws,
+          'ka2',
+          '{"type":"user","message":{"role":"user","content":"hi"},"uuid":"u-ka"}\n',
+        )
+      } finally {
+        Date.now = originalNow
+      }
+
+      // +350s 距打开超预算，但距上次活动（+250s）仅 100s → 不关闭
+      expect(closeInactiveClient('ka2', at250 + 100_000)).toBe(false)
+
+      handleWebSocketClose(ws, 'ka2', 1000, 'done')
+    })
+  })
+
   describe('handleWebSocketClose', () => {
     test('cleans up on close', () => {
       const ws = createMockWs()

package/packages/remote-control-server/src/config.ts

@@ -39,6 +39,14 @@ export interface RcsConfig {
    *  (session_keepalive_interval_v2_ms, default 120s) plus jitter, otherwise
    *  healthy idle sessions get torn down between keepalives. Default 300s. */
   wsClientInactivityTimeout: number
+  /** Stuck-turn watchdog (seconds). A session whose status is `running` with
+   *  no inbound worker events (and no outbound user message) for this long is
+   *  flipped back to `idle` + gets a recovery work re-dispatch — the turn-end
+   *  result event was lost in transit (worker-side drop) and nothing else
+   *  would ever release the Web UI input. Generous vs. long silent tools:
+   *  a false flip only unlocks the input early (messages queue in the REPL);
+   *  the next worker event re-derives `running`. 0 disables. Default 900s. */
+  stuckTurnTimeout: number
   /** Whether artifact content (GET /v1/artifacts/:id/content) is served
    *  without authentication. Default true — the unguessable artifact id is
    *  the secret, same trust model as the upstream cloud-artifacts service.
@@ -84,6 +92,7 @@ export const config: RcsConfig = {
     process.env.RCS_WS_CLIENT_INACTIVITY_TIMEOUT || '300',
     10,
   ),
+  stuckTurnTimeout: parseInt(process.env.RCS_STUCK_TURN_TIMEOUT || '900', 10),
   artifactsPublicAccess: process.env.RCS_ARTIFACTS_PUBLIC_ACCESS !== 'false',
 }
 

package/packages/remote-control-server/src/services/disconnect-monitor.ts

@@ -6,6 +6,8 @@ import {
   storeMarkAcpAgentOffline,
 } from '../store'
 import { config } from '../config'
+import { sweepUnansweredControlRequests } from './transport'
+import { sweepStuckTurnSessions } from './stuck-turn-watchdog'
 
 /** ACP env 离线后留给 acp-link 重连复用的宽限时间。超过后才将 env 置为
  *  disconnected。会话不再自动归档——由用户手动删除。 */
@@ -57,5 +59,13 @@ export function runDisconnectMonitorSweep(now = Date.now()) {
 export function startDisconnectMonitor(): ReturnType<typeof setInterval> {
   return setInterval(() => {
     runDisconnectMonitorSweep()
+    // Control-request response watchdog — detects a wedged worker write
+    // path (WS healthy, POST path dead) from unanswered RPCs and fires the
+    // recovery work re-dispatch. See services/transport.ts.
+    sweepUnansweredControlRequests()
+    // Stuck-turn watchdog — heals sessions whose turn-end result event was
+    // lost in transit (status stuck `running`, Web UI input locked forever).
+    // See services/stuck-turn-watchdog.ts.
+    sweepStuckTurnSessions()
   }, 60_000) // Check every minute
 }

package/packages/remote-control-server/src/services/stuck-turn-watchdog.ts

@@ -0,0 +1,144 @@
+import { log } from '../logger'
+import { storeListSessions } from '../store'
+import { config } from '../config'
+import type { SessionEvent } from '../transport/event-bus'
+import { getEventBus } from '../transport/event-bus'
+import { updateSessionStatus } from './session'
+import { redispatchWorkForSession } from './work-dispatch'
+
+/**
+ * Stuck-turn watchdog — the server-side self-heal for sessions whose
+ * turn-end signal was lost in transit.
+ *
+ * A session's status only returns from `running` to `idle` when the worker's
+ * `result`/`result_success` event arrives (ws-handler.ts ingestBridgeMessage)
+ * or the user interrupts. The worker sends that result fire-and-forget —
+ * when the write is dropped (transport-swap window, uploader batch drop
+ * after maxConsecutiveFailures, close-grace expiry), it is never re-sent and
+ * the session stays `running` forever. The Web UI locks the input (black
+ * stop square) and every new message lands in a queue that never drains.
+ *
+ * Nothing else detects this state:
+ *  - the worker's write-path watchdog only fires when POSTs stop completing
+ *    entirely (a transient drop followed by healthy POSTs is invisible);
+ *  - the WS inactivity clock is kept fresh by keepalive POSTs;
+ *  - the control-request watchdog only tracks RPCs the server itself sent.
+ *
+ * The sweep flips a `running` session with no turn signal for
+ * `config.stuckTurnTimeout` back to `idle` (updateSessionStatus publishes
+ * `session_status`, unlocking live SSE viewers immediately — a genuinely
+ * still-running turn re-derives `running` on its next event) and fires the
+ * recovery work re-dispatch (zombie-transport rebuild, same as the WS
+ * inactivity close).
+ *
+ * `requires_action` is deliberately out of scope: a permission ask that
+ * arrived is visible and actionable in the Web UI — the user is the one
+ * blocking, not a lost signal.
+ */
+
+/**
+ * Inbound event types that count as turn progress — an ALLOWLIST, so future
+ * chatter types default to not feeding the signal clock:
+ *  - `user` (worker-mirrored tool results), `assistant`, `result`/
+ *    `result_success`, `error` — the conversation itself;
+ *  - `system` — subagent task events (task_started/task_progress/…).
+ *    Exception: subtype `init` is re-sent on every transport (re)connect —
+ *    transport liveness, not turn progress.
+ *
+ * Deliberately NOT counted:
+ *  - `control_request`/`control_response` — RPC traffic. The Web UI polls
+ *    get_context_usage every 60s while the page is open, so the worker's
+ *    control_response answers flow constantly even for a wedged session —
+ *    counting them would make the watchdog permanently blind exactly when
+ *    the user is watching the stuck session.
+ *  - `models_state`, `system/init` — connect-time snapshots.
+ *  - `task_state` — background task-list churn; a background task running
+ *    outside any turn must not starve the watchdog.
+ *  - `session_status` — server-synthesized (including this watchdog's own
+ *    idle flip; counting it would make the sweep single-shot).
+ */
+const SIGNAL_EVENT_TYPES = new Set([
+  'user',
+  'assistant',
+  'partial_assistant',
+  'result',
+  'result_success',
+  'error',
+  'system',
+])
+
+function isInboundSignal(event: SessionEvent): boolean {
+  if (event.direction !== 'inbound') return false
+  if (!SIGNAL_EVENT_TYPES.has(event.type)) return false
+  if (event.type === 'system') {
+    // system/init is a connect-time snapshot, re-sent on every
+    // (re)connect — not turn progress.
+    const subtype = (event.payload as Record<string, unknown> | null)?.subtype
+    if (subtype === 'init') return false
+  }
+  return true
+}
+
+/**
+ * Wall-clock of the latest event on the bus that proves the session's turn
+ * is (or was) progressing: any inbound worker conversation event (assistant,
+ * tool-result user, result, subagent task events), or the outbound user
+ * message that started the turn. Returns 0 when the bus carries no signal
+ * at all (fresh bus after a server restart — callers fall back to the
+ * persisted session record).
+ */
+export function lastTurnSignalAt(events: SessionEvent[]): number {
+  for (let i = events.length - 1; i >= 0; i--) {
+    const event = events[i]
+    if (isInboundSignal(event)) {
+      return event.createdAt
+    }
+    if (event.direction === 'outbound' && event.type === 'user') {
+      return event.createdAt
+    }
+  }
+  return 0
+}
+
+/**
+ * Sweep all sessions for stuck turns. Returns the number of sessions healed.
+ * `now` is injectable for tests.
+ */
+export function sweepStuckTurnSessions(now: number = Date.now()): number {
+  const thresholdMs = config.stuckTurnTimeout * 1000
+  if (!(thresholdMs > 0)) return 0
+
+  let swept = 0
+  for (const session of storeListSessions()) {
+    // Only `running` locks the input with no recovery path; see module docs
+    // for why requires_action (visible, actionable card) and archived
+    // sessions are excluded.
+    if (session.status !== 'running') continue
+    // ACP sessions carry their turn traffic on the per-channel-group ACP
+    // bus (getAcpEventBus), not the session bus this sweep scans — healing
+    // them from the wrong bus would be a false positive.
+    if (session.source === 'acp') continue
+
+    const bus = getEventBus(session.id)
+    const lastSignal = lastTurnSignalAt(bus.getEventsSince(0))
+    // Empty bus (server restart) or a signal older than the session record
+    // (persisted legacy) — fall back to the persisted creation time so
+    // restart-legacy stuck sessions still heal after the same grace window.
+    const reference = Math.max(lastSignal, session.createdAt.getTime())
+    if (now - reference <= thresholdMs) continue
+
+    const silenceSec = Math.round((now - reference) / 1000)
+    log(
+      `[RCS] Stuck-turn watchdog: session=${session.id} running with no turn signal for ${silenceSec}s` +
+        ' — flipping to idle (turn-end result was lost) and redispatching work',
+    )
+    // Recovery re-dispatch first: when the worker's transport is actually
+    // dead this rebuilds it (fresh uploader → the write path recovers).
+    // No-op for healthy transports with an unchanged token.
+    void redispatchWorkForSession(session.id)
+    // Publishes `session_status` → live SSE viewers unlock the input.
+    updateSessionStatus(session.id, 'idle')
+    swept++
+  }
+  return swept
+}

package/packages/remote-control-server/src/services/transport.ts

@@ -1,6 +1,8 @@
 import { randomUUID } from 'node:crypto'
 import { getEventBus } from '../transport/event-bus'
 import { persistSessionEvent } from './event-persistence'
+import { log } from '../logger'
+import { redispatchWorkForSession } from './work-dispatch'
 
 /**
  * Extract plain text from various message payload formats.
@@ -207,6 +209,85 @@ export function normalizePayload(
  *  carries the full content). */
 const TRANSIENT_EVENT_TYPES = new Set(['stream_event', 'partial_assistant'])
 
+// ─── Control-request response watchdog ─────────────────────────────────────
+//
+// Web→worker control_requests (get_context_usage / interrupt / set_model / …)
+// are programmatic RPCs that a healthy worker answers in well under a
+// second (every subtype branch of handleServerControlRequest writes a
+// response). A pending request older than the threshold therefore means the
+// worker's write path is wedged — responses are queued behind a stalled
+// uploader and never reach the server, while its WS keeps reading (the
+// zombie state observed as "session stuck running, new messages get no
+// response"). The sweep fires the same recovery the ws-handler inactivity
+// close uses: a fresh work item → worker poll → brand-new transport.
+
+interface PendingControlRequest {
+  sessionId: string
+  subtype: string
+  sentAt: number
+  /** Recovery already dispatched for this wedge episode. */
+  recoveryFired: boolean
+}
+
+const pendingControlRequests = new Map<string, PendingControlRequest>()
+
+/** Web→worker control_request has 90s to be answered before recovery fires.
+ *  Generous vs. the web UI's own 8s control round-trip timeout — only a
+ *  genuinely dead write path should trip it. */
+const CONTROL_RESPONSE_TIMEOUT_MS = 90_000
+/** Reap unanswerable entries (worker gone mid-RPC) so the map stays bounded. */
+const PENDING_REAP_MS = CONTROL_RESPONSE_TIMEOUT_MS * 4
+
+function recordOutboundControlRequest(
+  sessionId: string,
+  payload: unknown,
+): void {
+  const p = payload as {
+    request_id?: unknown
+    request?: { subtype?: unknown }
+  } | null
+  if (!p || typeof p.request_id !== 'string') return
+  pendingControlRequests.set(p.request_id, {
+    sessionId,
+    subtype:
+      typeof p.request?.subtype === 'string' ? p.request.subtype : '(unknown)',
+    sentAt: Date.now(),
+    recoveryFired: false,
+  })
+}
+
+function settleControlResponse(payload: unknown): void {
+  const resp = (payload as { response?: { request_id?: unknown } } | null)
+    ?.response
+  if (resp && typeof resp.request_id === 'string') {
+    pendingControlRequests.delete(resp.request_id)
+  }
+}
+
+/**
+ * Sweep pending web→worker control requests. For requests unanswered past
+ * the timeout, log the wedge and fire a recovery work re-dispatch (once
+ * per episode — redispatchWorkForSession itself also no-ops when an open
+ * item already exists). Called from the disconnect monitor's 60s interval.
+ */
+export function sweepUnansweredControlRequests(now = Date.now()): void {
+  for (const [requestId, pending] of pendingControlRequests) {
+    const ageMs = now - pending.sentAt
+    if (ageMs <= CONTROL_RESPONSE_TIMEOUT_MS) continue
+    if (!pending.recoveryFired) {
+      log(
+        `[RCS] control_request ${pending.subtype} request_id=${requestId} unanswered for ${Math.round(ageMs / 1000)}s on session=${pending.sessionId}` +
+          ' — worker write path may be wedged, redispatching work',
+      )
+      pending.recoveryFired = true
+      void redispatchWorkForSession(pending.sessionId)
+    }
+    if (ageMs > PENDING_REAP_MS) {
+      pendingControlRequests.delete(requestId)
+    }
+  }
+}
+
 /** Publish an event to a session's bus and persist it durably. */
 export function publishSessionEvent(
   sessionId: string,
@@ -232,6 +313,14 @@ export function publishSessionEvent(
   const normalized = normalizePayload(type, payload)
   const transient = TRANSIENT_EVENT_TYPES.has(type)
 
+  // Control-response watchdog bookkeeping (see above). The raw payload is
+  // used — normalizePayload preserves request_id/response for these types.
+  if (direction === 'outbound' && type === 'control_request') {
+    recordOutboundControlRequest(sessionId, payload)
+  } else if (direction === 'inbound' && type === 'control_response') {
+    settleControlResponse(payload)
+  }
+
   const event = bus.publish(
     {
       id: eventId,

package/packages/remote-control-server/src/services/work-dispatch.ts

@@ -7,10 +7,11 @@ import {
   storeListSessionsByEnvironment,
   storeGetEnvironment,
   storeListWorkItems,
+  storeGetSession,
 } from '../store'
 import { config } from '../config'
 import { getBaseUrl } from '../config'
-import { touchSession } from './session'
+import { touchSession, isSessionClosedStatus } from './session'
 import { touchEnvironmentActivity } from './environment'
 import type { WorkResponse } from '../types/api'
 
@@ -171,3 +172,57 @@ export function reconnectWorkForEnvironment(envId: string) {
   const promises = activeSessions.map(s => createWorkItem(envId, s.id))
   return Promise.all(promises)
 }
+
+/**
+ * Server-initiated recovery dispatch for a single session.
+ *
+ * Fired when the session's ingress WS is closed for client inactivity
+ * (ws-handler): with HybridTransport the inactivity clock only advances via
+ * HTTP POST ingress (all writes go through POST — the WS carries no
+ * app-level client→server frames), so 300s of POST silence with a live WS
+ * means the worker's event uploader is wedged (e.g. one hung POST stalled
+ * the serialized drain loop). The worker's WS auto-reconnects within
+ * seconds — from the server's side everything looks healthy again, so
+ * WITHOUT this re-dispatch the zombie transport is never replaced: the
+ * at-capacity poll (10 min interval) returns no work and the worker's
+ * write path stays dead forever (observed: session stuck 'running', all
+ * worker events and control_responses silently swallowed).
+ *
+ * A fresh work item makes the worker's next poll fire onWorkReceived →
+ * brand-new transport (fresh WS + fresh uploader) → recovery. Deliberately
+ * NOT filtered by session status — a wedged worker's session is typically
+ * stuck in 'running' (the turn-end result event was eaten), which is
+ * exactly the case that needs the rebuild.
+ */
+export async function redispatchWorkForSession(
+  sessionId: string,
+): Promise<void> {
+  const session = storeGetSession(sessionId)
+  if (!session) return
+  if (isSessionClosedStatus(session.status)) return
+  const envId = session.environmentId
+  if (!envId) return
+
+  // Don't stack items: if the env already has a pending or in-flight
+  // dispatch for this cycle, the recovery is already queued.
+  const hasOpenItem = storeListWorkItems().some(
+    item =>
+      item.environmentId === envId &&
+      item.sessionId === sessionId &&
+      (item.state === 'pending' || item.state === 'dispatched'),
+  )
+  if (hasOpenItem) return
+
+  try {
+    await createWorkItem(envId, sessionId)
+    log(
+      `[RCS] Redispatched work item after WS inactivity close: session=${sessionId} env=${envId}`,
+    )
+  } catch (err) {
+    // Env inactive (worker gone — disconnect monitor will handle it) or
+    // race with deletion. Not an error worth surfacing.
+    log(
+      `[RCS] Redispatch after inactivity close skipped (session=${sessionId}): ${err instanceof Error ? err.message : String(err)}`,
+    )
+  }
+}