@saluzi/saluzi-edu 0.4.5 → 0.4.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saluzi/saluzi-edu",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "Saluzi CLI - interactive AI coding assistant in the terminal",
   "license": "MIT",
   "type": "module",

package/packages/remote-control-server/src/__tests__/control-watchdog.test.ts

@@ -0,0 +1,129 @@
+import { describe, test, expect, beforeEach, mock } from 'bun:test'
+import { mockConfigModule } from './helpers/mock-config'
+
+// Mock config
+mock.module('../config', () => mockConfigModule({ pollTimeout: 1 }))
+
+import {
+  storeReset,
+  storeCreateEnvironment,
+  storeCreateSession,
+  storeGetPendingWorkItem,
+} from '../store'
+import {
+  getEventBus,
+  removeEventBus,
+  getAllEventBuses,
+} from '../transport/event-bus'
+import {
+  publishSessionEvent,
+  sweepUnansweredControlRequests,
+} from '../services/transport'
+
+/**
+ * Control-request response watchdog（services/transport.ts）。
+ *
+ * Web→worker control_request 是程序化 RPC（健康 worker 亚秒响应）。
+ * pending 超过 90s 未回 control_response = worker 写路径楔死（响应排在
+ * 停滞的 uploader 后面永远到不了服务端，WS 却照常读入）— sweep 触发
+ * 与 ws-handler 不活跃关闭相同的恢复：重派发 work item → worker 重建
+ * 全新 transport。
+ */
+describe('control-request response watchdog', () => {
+  let envId: string
+  let sessionId: string
+
+  beforeEach(() => {
+    storeReset()
+    for (const [key] of getAllEventBuses()) {
+      removeEventBus(key)
+    }
+    const env = storeCreateEnvironment({ secret: 's' })
+    envId = env.id
+    const session = storeCreateSession({ environmentId: envId })
+    sessionId = session.id
+  })
+
+  test('unanswered control_request past timeout fires recovery redispatch', async () => {
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      { request_id: 'r1', request: { subtype: 'get_context_usage' } },
+      'outbound',
+    )
+
+    // 未超时 — 不触发
+    sweepUnansweredControlRequests(Date.now() + 10_000)
+    expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+
+    // 超过 90s 阈值 — 触发恢复性重派发
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0)) // redispatch 异步落库
+    expect(storeGetPendingWorkItem(envId)?.sessionId).toBe(sessionId)
+  })
+
+  test('inbound control_response settles the pending request', async () => {
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      { request_id: 'r2', request: { subtype: 'get_context_usage' } },
+      'outbound',
+    )
+    publishSessionEvent(
+      sessionId,
+      'control_response',
+      { response: { subtype: 'success', request_id: 'r2' } },
+      'inbound',
+    )
+
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+  })
+
+  test('worker→web control_requests are not tracked (permission dialogs wait for humans)', async () => {
+    // 方向为 inbound 的 control_request（worker 发起的 can_use_tool 权限
+    // 询问）不进 pending 表 — 那些等待人类响应，分钟级是正常的。
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      {
+        request_id: 'r3',
+        request: { subtype: 'can_use_tool', tool_name: 'Bash' },
+      },
+      'inbound',
+    )
+
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+  })
+
+  test('recovery fires once per episode and stale entries are reaped', async () => {
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      { request_id: 'r4', request: { subtype: 'interrupt' } },
+      'outbound',
+    )
+
+    // 第一次超时 sweep — 触发；再 sweep 一次（仍未响应）— hasOpenItem
+    // 守卫阻止堆叠，只保留一个 work item。
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    sweepUnansweredControlRequests(Date.now() + 120_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    const items = (await import('../store'))
+      .storeListWorkItems()
+      .filter(item => item.sessionId === sessionId && item.state === 'pending')
+    expect(items).toHaveLength(1)
+
+    // 远超 reap 阈值（90s×4）— pending 表条目被清理（worker 彻底消失
+    // 的场景下保证 Map 有界）。已创建的 work item 不受影响。
+    sweepUnansweredControlRequests(Date.now() + 500_000)
+    const itemsAfterReap = (await import('../store'))
+      .storeListWorkItems()
+      .filter(item => item.sessionId === sessionId && item.state === 'pending')
+    expect(itemsAfterReap).toHaveLength(1)
+  })
+})

package/packages/remote-control-server/src/__tests__/work-dispatch.test.ts

@@ -20,6 +20,7 @@ import {
   stopWork,
   heartbeatWork,
   reconnectWorkForEnvironment,
+  redispatchWorkForSession,
 } from '../services/work-dispatch'
 
 describe('Work Dispatch', () => {
@@ -195,4 +196,47 @@ describe('Work Dispatch', () => {
       expect(workIds).toHaveLength(0)
     })
   })
+
+  describe('redispatchWorkForSession', () => {
+    test('creates a pending work item for a live session regardless of status', async () => {
+      // 楔死 worker 的会话通常卡在 running（turn-end result 被吞）—
+      // 重新派发不能像 reconnectWorkForEnvironment 那样按 idle 过滤。
+      const { storeUpdateSession } = await import('../store')
+      storeUpdateSession(sessionId, { status: 'running' })
+      await redispatchWorkForSession(sessionId)
+      const item = storeGetPendingWorkItem(envId)
+      expect(item).toBeDefined()
+      expect(item?.sessionId).toBe(sessionId)
+      expect(item?.state).toBe('pending')
+    })
+
+    test('skips when an open (pending/dispatched) item already exists', async () => {
+      await createWorkItem(envId, sessionId)
+      await redispatchWorkForSession(sessionId)
+      const items = (await import('../store'))
+        .storeListWorkItems()
+        .filter(item => item.sessionId === sessionId)
+      expect(items).toHaveLength(1)
+    })
+
+    test('skips closed (archived) sessions', async () => {
+      const { storeUpdateSession } = await import('../store')
+      storeUpdateSession(sessionId, { status: 'archived' })
+      await redispatchWorkForSession(sessionId)
+      expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+    })
+
+    test('skips sessions with no environment binding', async () => {
+      const orphan = storeCreateSession({})
+      await redispatchWorkForSession(orphan.id)
+      expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+    })
+
+    test('swallows errors for inactive environments', async () => {
+      storeUpdateEnvironment(envId, { status: 'disconnected' })
+      // 不抛错 — 断连 env 由 disconnect monitor 处理
+      await redispatchWorkForSession(sessionId)
+      expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+    })
+  })
 })

package/packages/remote-control-server/src/__tests__/ws-handler.test.ts

@@ -4,7 +4,11 @@ import { mockConfigModule } from './helpers/mock-config'
 // Mock config before imports
 mock.module('../config', () => mockConfigModule())
 
-import { storeReset } from '../store'
+import {
+  storeReset,
+  storeCreateEnvironment,
+  storeCreateSession,
+} from '../store'
 import {
   getEventBus,
   removeEventBus,
@@ -17,6 +21,7 @@ import {
   handleWebSocketClose,
   closeAllConnections,
   touchClientActivity,
+  closeInactiveClient,
 } from '../transport/ws-handler'
 
 // Minimal WSContext mock
@@ -451,6 +456,55 @@ describe('ws-handler', () => {
       expect(msg.isSynthetic).toBe(true)
     })
 
+    test('replay skips point-in-time control traffic (interrupt, permission RPCs)', () => {
+      // Regression: the replay used to forward EVERY retained event,
+      // including one-shot RPCs. A replayed `interrupt` control_request was
+      // re-executed by the bridge — aborting whatever turn was running
+      // AFTER the reconnect.
+      const bus = getEventBus('replay_ctrl')
+      bus.publish({
+        id: 'e1',
+        sessionId: 'replay_ctrl',
+        type: 'user',
+        payload: { content: 'task prompt' },
+        direction: 'outbound',
+      })
+      bus.publish({
+        id: 'e2',
+        sessionId: 'replay_ctrl',
+        type: 'control_request',
+        payload: { request_id: 'r1', request: { subtype: 'interrupt' } },
+        direction: 'outbound',
+      })
+      bus.publish({
+        id: 'e3',
+        sessionId: 'replay_ctrl',
+        type: 'permission_response',
+        payload: { request_id: 'r2', approved: true },
+        direction: 'outbound',
+      })
+      bus.publish({
+        id: 'e4',
+        sessionId: 'replay_ctrl',
+        type: 'assistant',
+        payload: { content: 'done' },
+        direction: 'inbound',
+      })
+
+      const ws = createMockWs()
+      handleWebSocketOpen(ws, 'replay_ctrl')
+
+      const sent = ws.getSentData()
+      const types = sent.map(
+        (line: string) => (JSON.parse(line) as { type: string }).type,
+      )
+      expect(types).toContain('user')
+      expect(types).toContain('assistant')
+      expect(types).not.toContain('control_request')
+      expect(types).not.toContain('permission_response')
+      expect(types).not.toContain('interrupt')
+    })
+
     test('replaces existing connection for same session', () => {
       const ws1 = createMockWs()
       const ws2 = createMockWs()
@@ -546,6 +600,136 @@ describe('ws-handler', () => {
     })
   })
 
+  describe('closeInactiveClient', () => {
+    // Regression: the original keepalive interval read a const closure
+    // variable captured at open time — touchClientActivity mutated a
+    // different copy on the cleanup entry, so EVERY connection was closed
+    // exactly CLIENT_ACTIVITY_TIMEOUT after open ("client inactive" 误报,
+    // observed as duration=300s closes on actively-POSTing workers).
+    test('client activity via touchClientActivity resets the inactivity clock', () => {
+      const closed: Array<{ code?: number; reason?: string }> = []
+      const ws = {
+        readyState: 1,
+        send: () => {},
+        close: (code?: number, reason?: string) =>
+          closed.push({ code, reason }),
+      } as any
+      handleWebSocketOpen(ws, 'inact1')
+
+      // Connection opened at T0; the worker POSTs (touch) at T0+10min.
+      // Measured 100s after the touch → within the 300s budget → NOT closed.
+      const touchedAt = Date.now() + 600_000
+      const originalNow = Date.now
+      Date.now = () => touchedAt
+      try {
+        touchClientActivity('inact1')
+      } finally {
+        Date.now = originalNow
+      }
+      expect(closeInactiveClient('inact1', touchedAt + 100_000)).toBe(false)
+      expect(closed).toHaveLength(0)
+
+      handleWebSocketClose(ws, 'inact1', 1000, 'done')
+    })
+
+    test('closes a truly silent client and queues a recovery work item', async () => {
+      const env = storeCreateEnvironment({ secret: 's' })
+      const session = storeCreateSession({ environmentId: env.id })
+
+      const closed: Array<{ code?: number; reason?: string }> = []
+      const ws = {
+        readyState: 1,
+        send: () => {},
+        close: (code?: number, reason?: string) =>
+          closed.push({ code, reason }),
+      } as any
+      handleWebSocketOpen(ws, session.id)
+
+      // Opened at ~now; nothing touches for 400s → closed as dead.
+      const openedAt = Date.now()
+      expect(closeInactiveClient(session.id, openedAt + 400_000)).toBe(true)
+      expect(closed).toHaveLength(1)
+      expect(closed[0]?.code).toBe(1000)
+
+      // Redispatch is fire-and-forget — let the microtask settle.
+      await new Promise(resolve => setTimeout(resolve, 0))
+      const { storeGetPendingWorkItem } = await import('../store')
+      expect(storeGetPendingWorkItem(env.id)?.sessionId).toBe(session.id)
+
+      handleWebSocketClose(ws, session.id, 1000, 'done')
+    })
+
+    test('does not close within the inactivity budget', () => {
+      const ws = createMockWs()
+      handleWebSocketOpen(ws, 'inact3')
+      expect(closeInactiveClient('inact3', Date.now() + 100_000)).toBe(false)
+      handleWebSocketClose(ws, 'inact3', 1000, 'done')
+    })
+
+    test('is a no-op for unknown sessions and dead sockets', () => {
+      expect(closeInactiveClient('never-opened')).toBe(false)
+      const ws = createMockWs(3) // CLOSED
+      handleWebSocketOpen(ws, 'inact4')
+      expect(closeInactiveClient('inact4', Date.now() + 400_000)).toBe(false)
+      handleWebSocketClose(ws, 'inact4', 1000, 'done')
+    })
+
+    // HybridTransport workers send WS-level keep_alive frames (direct
+    // socket writes) even when their POST write path is dead — counting
+    // them as activity would make the inactivity close unreachable for
+    // exactly the zombie state it exists to detect.
+    test('WS keep_alive frames do NOT refresh the inactivity clock', () => {
+      const closed: Array<{ code?: number; reason?: string }> = []
+      const ws = {
+        readyState: 1,
+        send: () => {},
+        close: (code?: number, reason?: string) =>
+          closed.push({ code, reason }),
+      } as any
+      handleWebSocketOpen(ws, 'ka1')
+
+      // keep_alive 帧到达 +250s（预算内）— 不刷新时钟
+      const at250 = Date.now() + 250_000
+      const originalNow = Date.now
+      Date.now = () => at250
+      try {
+        handleWebSocketMessage(ws, 'ka1', '{"type":"keep_alive"}\n')
+      } finally {
+        Date.now = originalNow
+      }
+
+      // +350s（自打开起超预算）→ 尽管 keepalive 一直在发，仍然关闭
+      expect(closeInactiveClient('ka1', Date.now() + 350_000)).toBe(true)
+      expect(closed).toHaveLength(1)
+
+      handleWebSocketClose(ws, 'ka1', 1000, 'done')
+    })
+
+    test('real WS messages still refresh the inactivity clock', () => {
+      const ws = createMockWs()
+      handleWebSocketOpen(ws, 'ka2')
+
+      // 真实消息（非 keep_alive）到达 +250s — 刷新时钟
+      const at250 = Date.now() + 250_000
+      const originalNow = Date.now
+      Date.now = () => at250
+      try {
+        handleWebSocketMessage(
+          ws,
+          'ka2',
+          '{"type":"user","message":{"role":"user","content":"hi"},"uuid":"u-ka"}\n',
+        )
+      } finally {
+        Date.now = originalNow
+      }
+
+      // +350s 距打开超预算，但距上次活动（+250s）仅 100s → 不关闭
+      expect(closeInactiveClient('ka2', at250 + 100_000)).toBe(false)
+
+      handleWebSocketClose(ws, 'ka2', 1000, 'done')
+    })
+  })
+
   describe('handleWebSocketClose', () => {
     test('cleans up on close', () => {
       const ws = createMockWs()

package/packages/remote-control-server/src/services/disconnect-monitor.ts

@@ -6,6 +6,7 @@ import {
   storeMarkAcpAgentOffline,
 } from '../store'
 import { config } from '../config'
+import { sweepUnansweredControlRequests } from './transport'
 
 /** ACP env 离线后留给 acp-link 重连复用的宽限时间。超过后才将 env 置为
  *  disconnected。会话不再自动归档——由用户手动删除。 */
@@ -57,5 +58,9 @@ export function runDisconnectMonitorSweep(now = Date.now()) {
 export function startDisconnectMonitor(): ReturnType<typeof setInterval> {
   return setInterval(() => {
     runDisconnectMonitorSweep()
+    // Control-request response watchdog — detects a wedged worker write
+    // path (WS healthy, POST path dead) from unanswered RPCs and fires the
+    // recovery work re-dispatch. See services/transport.ts.
+    sweepUnansweredControlRequests()
   }, 60_000) // Check every minute
 }

package/packages/remote-control-server/src/services/transport.ts

@@ -1,6 +1,8 @@
 import { randomUUID } from 'node:crypto'
 import { getEventBus } from '../transport/event-bus'
 import { persistSessionEvent } from './event-persistence'
+import { log } from '../logger'
+import { redispatchWorkForSession } from './work-dispatch'
 
 /**
  * Extract plain text from various message payload formats.
@@ -207,6 +209,85 @@ export function normalizePayload(
  *  carries the full content). */
 const TRANSIENT_EVENT_TYPES = new Set(['stream_event', 'partial_assistant'])
 
+// ─── Control-request response watchdog ─────────────────────────────────────
+//
+// Web→worker control_requests (get_context_usage / interrupt / set_model / …)
+// are programmatic RPCs that a healthy worker answers in well under a
+// second (every subtype branch of handleServerControlRequest writes a
+// response). A pending request older than the threshold therefore means the
+// worker's write path is wedged — responses are queued behind a stalled
+// uploader and never reach the server, while its WS keeps reading (the
+// zombie state observed as "session stuck running, new messages get no
+// response"). The sweep fires the same recovery the ws-handler inactivity
+// close uses: a fresh work item → worker poll → brand-new transport.
+
+interface PendingControlRequest {
+  sessionId: string
+  subtype: string
+  sentAt: number
+  /** Recovery already dispatched for this wedge episode. */
+  recoveryFired: boolean
+}
+
+const pendingControlRequests = new Map<string, PendingControlRequest>()
+
+/** Web→worker control_request has 90s to be answered before recovery fires.
+ *  Generous vs. the web UI's own 8s control round-trip timeout — only a
+ *  genuinely dead write path should trip it. */
+const CONTROL_RESPONSE_TIMEOUT_MS = 90_000
+/** Reap unanswerable entries (worker gone mid-RPC) so the map stays bounded. */
+const PENDING_REAP_MS = CONTROL_RESPONSE_TIMEOUT_MS * 4
+
+function recordOutboundControlRequest(
+  sessionId: string,
+  payload: unknown,
+): void {
+  const p = payload as {
+    request_id?: unknown
+    request?: { subtype?: unknown }
+  } | null
+  if (!p || typeof p.request_id !== 'string') return
+  pendingControlRequests.set(p.request_id, {
+    sessionId,
+    subtype:
+      typeof p.request?.subtype === 'string' ? p.request.subtype : '(unknown)',
+    sentAt: Date.now(),
+    recoveryFired: false,
+  })
+}
+
+function settleControlResponse(payload: unknown): void {
+  const resp = (payload as { response?: { request_id?: unknown } } | null)
+    ?.response
+  if (resp && typeof resp.request_id === 'string') {
+    pendingControlRequests.delete(resp.request_id)
+  }
+}
+
+/**
+ * Sweep pending web→worker control requests. For requests unanswered past
+ * the timeout, log the wedge and fire a recovery work re-dispatch (once
+ * per episode — redispatchWorkForSession itself also no-ops when an open
+ * item already exists). Called from the disconnect monitor's 60s interval.
+ */
+export function sweepUnansweredControlRequests(now = Date.now()): void {
+  for (const [requestId, pending] of pendingControlRequests) {
+    const ageMs = now - pending.sentAt
+    if (ageMs <= CONTROL_RESPONSE_TIMEOUT_MS) continue
+    if (!pending.recoveryFired) {
+      log(
+        `[RCS] control_request ${pending.subtype} request_id=${requestId} unanswered for ${Math.round(ageMs / 1000)}s on session=${pending.sessionId}` +
+          ' — worker write path may be wedged, redispatching work',
+      )
+      pending.recoveryFired = true
+      void redispatchWorkForSession(pending.sessionId)
+    }
+    if (ageMs > PENDING_REAP_MS) {
+      pendingControlRequests.delete(requestId)
+    }
+  }
+}
+
 /** Publish an event to a session's bus and persist it durably. */
 export function publishSessionEvent(
   sessionId: string,
@@ -232,6 +313,14 @@ export function publishSessionEvent(
   const normalized = normalizePayload(type, payload)
   const transient = TRANSIENT_EVENT_TYPES.has(type)
 
+  // Control-response watchdog bookkeeping (see above). The raw payload is
+  // used — normalizePayload preserves request_id/response for these types.
+  if (direction === 'outbound' && type === 'control_request') {
+    recordOutboundControlRequest(sessionId, payload)
+  } else if (direction === 'inbound' && type === 'control_response') {
+    settleControlResponse(payload)
+  }
+
   const event = bus.publish(
     {
       id: eventId,

package/packages/remote-control-server/src/services/work-dispatch.ts

@@ -7,10 +7,11 @@ import {
   storeListSessionsByEnvironment,
   storeGetEnvironment,
   storeListWorkItems,
+  storeGetSession,
 } from '../store'
 import { config } from '../config'
 import { getBaseUrl } from '../config'
-import { touchSession } from './session'
+import { touchSession, isSessionClosedStatus } from './session'
 import { touchEnvironmentActivity } from './environment'
 import type { WorkResponse } from '../types/api'
 
@@ -171,3 +172,57 @@ export function reconnectWorkForEnvironment(envId: string) {
   const promises = activeSessions.map(s => createWorkItem(envId, s.id))
   return Promise.all(promises)
 }
+
+/**
+ * Server-initiated recovery dispatch for a single session.
+ *
+ * Fired when the session's ingress WS is closed for client inactivity
+ * (ws-handler): with HybridTransport the inactivity clock only advances via
+ * HTTP POST ingress (all writes go through POST — the WS carries no
+ * app-level client→server frames), so 300s of POST silence with a live WS
+ * means the worker's event uploader is wedged (e.g. one hung POST stalled
+ * the serialized drain loop). The worker's WS auto-reconnects within
+ * seconds — from the server's side everything looks healthy again, so
+ * WITHOUT this re-dispatch the zombie transport is never replaced: the
+ * at-capacity poll (10 min interval) returns no work and the worker's
+ * write path stays dead forever (observed: session stuck 'running', all
+ * worker events and control_responses silently swallowed).
+ *
+ * A fresh work item makes the worker's next poll fire onWorkReceived →
+ * brand-new transport (fresh WS + fresh uploader) → recovery. Deliberately
+ * NOT filtered by session status — a wedged worker's session is typically
+ * stuck in 'running' (the turn-end result event was eaten), which is
+ * exactly the case that needs the rebuild.
+ */
+export async function redispatchWorkForSession(
+  sessionId: string,
+): Promise<void> {
+  const session = storeGetSession(sessionId)
+  if (!session) return
+  if (isSessionClosedStatus(session.status)) return
+  const envId = session.environmentId
+  if (!envId) return
+
+  // Don't stack items: if the env already has a pending or in-flight
+  // dispatch for this cycle, the recovery is already queued.
+  const hasOpenItem = storeListWorkItems().some(
+    item =>
+      item.environmentId === envId &&
+      item.sessionId === sessionId &&
+      (item.state === 'pending' || item.state === 'dispatched'),
+  )
+  if (hasOpenItem) return
+
+  try {
+    await createWorkItem(envId, sessionId)
+    log(
+      `[RCS] Redispatched work item after WS inactivity close: session=${sessionId} env=${envId}`,
+    )
+  } catch (err) {
+    // Env inactive (worker gone — disconnect monitor will handle it) or
+    // race with deletion. Not an error worth surfacing.
+    log(
+      `[RCS] Redispatch after inactivity close skipped (session=${sessionId}): ${err instanceof Error ? err.message : String(err)}`,
+    )
+  }
+}