@saluzi/saluzi-edu 0.4.5 → 0.4.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saluzi/saluzi-edu",
-  "version": "0.4.5",
+  "version": "0.4.7",
   "description": "Saluzi CLI - interactive AI coding assistant in the terminal",
   "license": "MIT",
   "type": "module",

package/packages/remote-control-server/src/__tests__/control-watchdog.test.ts

@@ -0,0 +1,129 @@
+import { describe, test, expect, beforeEach, mock } from 'bun:test'
+import { mockConfigModule } from './helpers/mock-config'
+
+// Mock config
+mock.module('../config', () => mockConfigModule({ pollTimeout: 1 }))
+
+import {
+  storeReset,
+  storeCreateEnvironment,
+  storeCreateSession,
+  storeGetPendingWorkItem,
+} from '../store'
+import {
+  getEventBus,
+  removeEventBus,
+  getAllEventBuses,
+} from '../transport/event-bus'
+import {
+  publishSessionEvent,
+  sweepUnansweredControlRequests,
+} from '../services/transport'
+
+/**
+ * Control-request response watchdog（services/transport.ts）。
+ *
+ * Web→worker control_request 是程序化 RPC（健康 worker 亚秒响应）。
+ * pending 超过 90s 未回 control_response = worker 写路径楔死（响应排在
+ * 停滞的 uploader 后面永远到不了服务端，WS 却照常读入）— sweep 触发
+ * 与 ws-handler 不活跃关闭相同的恢复：重派发 work item → worker 重建
+ * 全新 transport。
+ */
+describe('control-request response watchdog', () => {
+  let envId: string
+  let sessionId: string
+
+  beforeEach(() => {
+    storeReset()
+    for (const [key] of getAllEventBuses()) {
+      removeEventBus(key)
+    }
+    const env = storeCreateEnvironment({ secret: 's' })
+    envId = env.id
+    const session = storeCreateSession({ environmentId: envId })
+    sessionId = session.id
+  })
+
+  test('unanswered control_request past timeout fires recovery redispatch', async () => {
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      { request_id: 'r1', request: { subtype: 'get_context_usage' } },
+      'outbound',
+    )
+
+    // 未超时 — 不触发
+    sweepUnansweredControlRequests(Date.now() + 10_000)
+    expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+
+    // 超过 90s 阈值 — 触发恢复性重派发
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0)) // redispatch 异步落库
+    expect(storeGetPendingWorkItem(envId)?.sessionId).toBe(sessionId)
+  })
+
+  test('inbound control_response settles the pending request', async () => {
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      { request_id: 'r2', request: { subtype: 'get_context_usage' } },
+      'outbound',
+    )
+    publishSessionEvent(
+      sessionId,
+      'control_response',
+      { response: { subtype: 'success', request_id: 'r2' } },
+      'inbound',
+    )
+
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+  })
+
+  test('worker→web control_requests are not tracked (permission dialogs wait for humans)', async () => {
+    // 方向为 inbound 的 control_request（worker 发起的 can_use_tool 权限
+    // 询问）不进 pending 表 — 那些等待人类响应，分钟级是正常的。
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      {
+        request_id: 'r3',
+        request: { subtype: 'can_use_tool', tool_name: 'Bash' },
+      },
+      'inbound',
+    )
+
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+  })
+
+  test('recovery fires once per episode and stale entries are reaped', async () => {
+    publishSessionEvent(
+      sessionId,
+      'control_request',
+      { request_id: 'r4', request: { subtype: 'interrupt' } },
+      'outbound',
+    )
+
+    // 第一次超时 sweep — 触发；再 sweep 一次（仍未响应）— hasOpenItem
+    // 守卫阻止堆叠，只保留一个 work item。
+    sweepUnansweredControlRequests(Date.now() + 100_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    sweepUnansweredControlRequests(Date.now() + 120_000)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    const items = (await import('../store'))
+      .storeListWorkItems()
+      .filter(item => item.sessionId === sessionId && item.state === 'pending')
+    expect(items).toHaveLength(1)
+
+    // 远超 reap 阈值（90s×4）— pending 表条目被清理（worker 彻底消失
+    // 的场景下保证 Map 有界）。已创建的 work item 不受影响。
+    sweepUnansweredControlRequests(Date.now() + 500_000)
+    const itemsAfterReap = (await import('../store'))
+      .storeListWorkItems()
+      .filter(item => item.sessionId === sessionId && item.state === 'pending')
+    expect(itemsAfterReap).toHaveLength(1)
+  })
+})

package/packages/remote-control-server/src/__tests__/helpers/mock-config.ts

@@ -17,6 +17,8 @@ export function createMockConfig(overrides?: Partial<Record<string, unknown>>) {
     singleUser: false,
     wsIdleTimeout: 30,
     wsKeepaliveInterval: 20,
+    wsClientInactivityTimeout: 300,
+    stuckTurnTimeout: 900,
     artifactsPublicAccess: true,
     ...overrides,
   }

package/packages/remote-control-server/src/__tests__/stuck-turn-watchdog.test.ts

@@ -0,0 +1,453 @@
+/**
+ * Regression: `slz rcs one` session stuck on `running` forever after the
+ * worker's turn-end result event is lost in transit.
+ *
+ * Reproduction of the reported symptom: task half-done, a tool call succeeds
+ * (its tool-result user event reaches the server), then silence — the web
+ * input shows the black stop square (isLoading) and every new message lands
+ * in the queue forever.
+ *
+ * Root cause chain (worker side, see src/bridge/outboundRetry.ts +
+ * replBridge.ts): sendResult()/sendControlRequest() are fire-and-forget —
+ * a result written during a transport swap window ("DROP sendResult: no
+ * transport") or dropped by the uploader (maxConsecutiveFailures / close
+ * grace) is never re-sent. The server-side status derivation only returns
+ * to idle on an inbound result, so the session stays `running` forever and
+ * NOTHING self-heals it: the worker write watchdog is quiet (POSTs are
+ * healthy again), the WS inactivity clock is kept fresh by keepalive POSTs,
+ * and the control-request watchdog only tracks RPCs the server itself sent.
+ *
+ * This test pins the server-side self-heal: the stuck-turn watchdog sweep
+ * flips stale `running` sessions back to idle (publishing session_status so
+ * live SSE viewers unlock the input) and fires the recovery work re-dispatch.
+ * `requires_action` is deliberately out of scope — a visible permission card
+ * is actionable, not stuck.
+ */
+import { describe, test, expect, beforeEach, mock } from 'bun:test'
+import { mockConfigModule } from './helpers/mock-config'
+
+mock.module('../config', () =>
+  mockConfigModule({ pollTimeout: 1, stuckTurnTimeout: 60 }),
+)
+
+import { Hono } from 'hono'
+import {
+  storeReset,
+  storeCreateEnvironment,
+  storeGetPendingWorkItem,
+} from '../store'
+import {
+  getAllEventBuses,
+  removeEventBus,
+  getEventBus,
+  type SessionEvent,
+} from '../transport/event-bus'
+
+import v1SessionIngress from '../routes/v1/session-ingress'
+import webSessions from '../routes/web/sessions'
+import webControl from '../routes/web/control'
+import { issueToken } from '../auth/token'
+import { getSession } from '../services/session'
+import { sweepStuckTurnSessions } from '../services/stuck-turn-watchdog'
+
+/** Matches the mocked config's stuckTurnTimeout (60s). */
+const THRESHOLD_MS = 60 * 1000
+
+function createApp() {
+  const app = new Hono()
+  app.route('/v2/session_ingress', v1SessionIngress)
+  app.route('/web', webSessions)
+  app.route('/web', webControl)
+  return app
+}
+
+const AUTH_HEADERS = {
+  Authorization: 'Bearer test-api-key',
+  'X-Username': 'testuser',
+}
+
+function auth(user: string): string {
+  return `Bearer ${issueToken(user).token}`
+}
+
+/** Drive the session exactly like the real worker does: the Web UI starts
+ *  the turn (outbound user), HybridTransport POSTs the worker's events via
+ *  session-ingress (inbound assistant / tool-result user). */
+async function driveTurn(app: Hono, sessionId: string): Promise<void> {
+  const sendRes = await app.request(`/web/sessions/${sessionId}/events`, {
+    method: 'POST',
+    headers: {
+      Authorization: auth('user-1'),
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      type: 'user',
+      uuid: 'web-uuid-1',
+      content: 'run the task',
+      message: { content: 'run the task' },
+    }),
+  })
+  expect(sendRes.status).toBe(200)
+
+  // Worker: assistant streams (running), then the tool executes successfully
+  // and its tool-result user message is mirrored — the last events the
+  // server ever sees.
+  const ingestRes = await app.request(
+    `/v2/session_ingress/session/${sessionId}/events`,
+    {
+      method: 'POST',
+      headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        events: [
+          {
+            type: 'assistant',
+            uuid: 'asst-uuid-1',
+            message: {
+              role: 'assistant',
+              content: [
+                { type: 'tool_use', id: 't1', name: 'Bash', input: {} },
+              ],
+            },
+          },
+          {
+            type: 'user',
+            uuid: 'tool-result-1',
+            message: {
+              role: 'user',
+              content: [
+                { type: 'tool_result', tool_use_id: 't1', content: 'ok' },
+              ],
+            },
+          },
+        ],
+      }),
+    },
+  )
+  expect(ingestRes.status).toBe(200)
+}
+
+describe('stuck-turn watchdog (lost turn-end result regression)', () => {
+  let app: Hono
+  let sessionId: string
+  let envId: string
+
+  beforeEach(async () => {
+    storeReset()
+    for (const [key] of getAllEventBuses()) {
+      removeEventBus(key)
+    }
+    app = createApp()
+
+    const env = storeCreateEnvironment({ secret: 's' })
+    envId = env.id
+
+    const res = await app.request('/web/sessions', {
+      method: 'POST',
+      headers: {
+        Authorization: auth('user-1'),
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ environment_id: envId }),
+    })
+    const body = (await res.json()) as { id: string }
+    sessionId = body.id
+  })
+
+  test('lost turn-end result: session stuck running, sweep heals to idle', async () => {
+    await driveTurn(app, sessionId)
+
+    // ── The reproduction: the turn-end result event never arrives ──
+    // (dropped by the worker's uploader / transport-swap window). The
+    // session is now stuck on `running` — the web input locks and every
+    // new message queues forever.
+    expect(getSession(sessionId)?.status).toBe('running')
+
+    // Time passes; the worker's keepalive POSTs keep the WS alive but no
+    // conversation events arrive. Nothing on the server heals the status…
+    const healed = sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
+    expect(healed).toBe(1)
+    expect(getSession(sessionId)?.status).toBe('idle')
+
+    // A session_status event must be broadcast so live SSE viewers
+    // (RCSChatAdapter onStatusChange) unlock the input immediately.
+    const statuses = getEventBus(sessionId)
+      .getEventsSince(0)
+      .filter(e => e.type === 'session_status')
+      .map(e => (e.payload as Record<string, unknown>).status)
+    expect(statuses[statuses.length - 1]).toBe('idle')
+
+    // The sweep also queues the recovery work re-dispatch (zombie-transport
+    // rebuild) — same recovery the WS inactivity close uses.
+    expect(storeGetPendingWorkItem(envId)?.sessionId).toBe(sessionId)
+  })
+
+  test('genuinely progressing turn is NOT healed (recent inbound events)', async () => {
+    await driveTurn(app, sessionId)
+
+    // Worker still streaming — the sweep with fresh clock must not flip.
+    expect(sweepStuckTurnSessions()).toBe(0)
+    expect(getSession(sessionId)?.status).toBe('running')
+
+    // A fresh inbound assistant resets the silence clock — sweeping before
+    // the full threshold elapses since that event must not heal.
+    const ingestRes = await app.request(
+      `/v2/session_ingress/session/${sessionId}/events`,
+      {
+        method: 'POST',
+        headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          events: [
+            {
+              type: 'assistant',
+              uuid: 'asst-uuid-2',
+              message: {
+                role: 'assistant',
+                content: [{ type: 'text', text: 'still working' }],
+              },
+            },
+          ],
+        }),
+      },
+    )
+    expect(ingestRes.status).toBe(200)
+
+    // The assistant arrived moments ago — sweeping at threshold-1s (i.e. the
+    // last signal is younger than the threshold) must not heal.
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
+    expect(getSession(sessionId)?.status).toBe('running')
+  })
+
+  test('turn just started (outbound user, no worker reply yet) is NOT healed', async () => {
+    const res = await app.request(`/web/sessions/${sessionId}/events`, {
+      method: 'POST',
+      headers: {
+        Authorization: auth('user-1'),
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        type: 'user',
+        uuid: 'fresh-user-1',
+        content: 'hello',
+        message: { content: 'hello' },
+      }),
+    })
+    expect(res.status).toBe(200)
+    expect(getSession(sessionId)?.status).toBe('running')
+
+    // The outbound user event marks the turn start — the worker gets the
+    // full threshold to produce its first inbound event.
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
+    expect(getSession(sessionId)?.status).toBe('running')
+
+    // …but a turn whose prompt is older than the threshold with zero worker
+    // output (worker never picked it up) does heal.
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
+    expect(getSession(sessionId)?.status).toBe('idle')
+  })
+
+  test('60s control RPC answers do NOT count as turn progress', async () => {
+    await driveTurn(app, sessionId)
+
+    // The Web UI polls get_context_usage every 60s while the page is open —
+    // the worker's control_response answers are RPC liveness, not turn
+    // progress. They must not keep a wedged turn looking alive.
+    const pollRes = await app.request(`/web/sessions/${sessionId}/control`, {
+      method: 'POST',
+      headers: {
+        Authorization: auth('user-1'),
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        type: 'control_request',
+        request_id: 'rpc-1',
+        request: { subtype: 'get_context_usage' },
+      }),
+    })
+    expect(pollRes.status).toBe(200)
+
+    const answerRes = await app.request(
+      `/v2/session_ingress/session/${sessionId}/events`,
+      {
+        method: 'POST',
+        headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          events: [
+            {
+              type: 'control_response',
+              response: {
+                subtype: 'success',
+                request_id: 'rpc-1',
+                response: {},
+              },
+            },
+          ],
+        }),
+      },
+    )
+    expect(answerRes.status).toBe(200)
+
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
+    expect(getSession(sessionId)?.status).toBe('idle')
+  })
+
+  test('connect-time snapshots and background task_state do not feed the signal clock', async () => {
+    await driveTurn(app, sessionId)
+
+    // system/init (re-ssent on every reconnect), models_state, and
+    // task_state (background task churn outside any turn) are transport /
+    // task-list liveness, not turn progress — a wedged session must still
+    // heal even while these keep arriving.
+    const chatterRes = await app.request(
+      `/v2/session_ingress/session/${sessionId}/events`,
+      {
+        method: 'POST',
+        headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          events: [
+            {
+              type: 'system',
+              subtype: 'init',
+              uuid: 'init-1',
+              message: {},
+            },
+            {
+              type: 'models_state',
+              uuid: 'models-1',
+              message: {},
+            },
+            {
+              type: 'task_state',
+              uuid: 'tasks-1',
+              message: {},
+            },
+          ],
+        }),
+      },
+    )
+    expect(chatterRes.status).toBe(200)
+
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
+    expect(getSession(sessionId)?.status).toBe('idle')
+  })
+
+  test('subagent task events DO feed the signal clock (genuine turn progress)', async () => {
+    await driveTurn(app, sessionId)
+
+    const progressRes = await app.request(
+      `/v2/session_ingress/session/${sessionId}/events`,
+      {
+        method: 'POST',
+        headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          events: [
+            {
+              type: 'system',
+              subtype: 'task_progress',
+              uuid: 'task-progress-1',
+              task_id: 'task-1',
+              message: {},
+            },
+          ],
+        }),
+      },
+    )
+    expect(progressRes.status).toBe(200)
+
+    // The task_progress event just arrived — sweeping before the full
+    // threshold since it must not heal.
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS - 1000)).toBe(0)
+    expect(getSession(sessionId)?.status).toBe('running')
+  })
+
+  test('pending permission ask (requires_action) is out of scope — never healed', async () => {
+    await driveTurn(app, sessionId)
+
+    // The worker's next tool needs permission — the can_use_tool ask is
+    // visible and actionable in the Web UI (card with approve/deny), so a
+    // requires_action session is NOT stuck: the user is the one blocking.
+    // The sweep must never touch it, no matter how long the user waits.
+    const askRes = await app.request(
+      `/v2/session_ingress/session/${sessionId}/events`,
+      {
+        method: 'POST',
+        headers: { ...AUTH_HEADERS, 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+          events: [
+            {
+              type: 'control_request',
+              request_id: 'perm-1',
+              request: {
+                subtype: 'can_use_tool',
+                tool_name: 'Bash',
+                input: {},
+                tool_use_id: 't2',
+              },
+            },
+          ],
+        }),
+      },
+    )
+    expect(askRes.status).toBe(200)
+    expect(getSession(sessionId)?.status).toBe('requires_action')
+
+    expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0)
+    expect(getSession(sessionId)?.status).toBe('requires_action')
+  })
+
+  test('lost permission ask (stuck on running, card never shown) heals', async () => {
+    await driveTurn(app, sessionId)
+    // The ask itself was lost in transit — status never left running and no
+    // card is shown. Same sweep heals it (input unlock; the worker-side
+    // retry re-delivers the ask and flips it to requires_action).
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
+    expect(getSession(sessionId)?.status).toBe('idle')
+  })
+
+  test('idle sessions and fresh buses are untouched', async () => {
+    await driveTurn(app, sessionId)
+    sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
+    expect(getSession(sessionId)?.status).toBe('idle')
+
+    // Already idle — the sweep must not publish anything new.
+    const before = getEventBus(sessionId)
+      .getEventsSince(0)
+      .filter(e => e.type === 'session_status').length
+    expect(sweepStuckTurnSessions(Date.now() + 10 * THRESHOLD_MS)).toBe(0)
+    const after = getEventBus(sessionId)
+      .getEventsSince(0)
+      .filter(e => e.type === 'session_status').length
+    expect(after).toBe(before)
+  })
+
+  test('server-restart legacy: running session with an empty bus heals', async () => {
+    // A session persisted as `running` whose bus is fresh (server restart)
+    // has no turn signal at all — the sweep falls back to the session's
+    // creation time so restart-legacy stuck sessions still heal.
+    const { storeUpdateSession } = await import('../store')
+    storeUpdateSession(sessionId, { status: 'running' })
+    removeEventBus(sessionId) // simulate fresh bus after restart
+
+    const swept = sweepStuckTurnSessions(Date.now() + 1000)
+    // Session was created milliseconds ago — creation-time fallback gives
+    // the same grace window as a fresh turn.
+    expect(swept).toBe(0)
+
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)).toBe(1)
+    expect(getSession(sessionId)?.status).toBe('idle')
+  })
+
+  test("watchdog's own session_status event does not count as turn progress", async () => {
+    await driveTurn(app, sessionId)
+    sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 5000)
+    expect(getSession(sessionId)?.status).toBe('idle')
+
+    // Flip it back to running (e.g. a late assistant with turn signal) and
+    // verify the sweep's previously published session_status (inbound) did
+    // not arm the signal clock — a sweep right past the threshold still
+    // fires, and repeated sweeps keep healing.
+    const { storeUpdateSession } = await import('../store')
+    storeUpdateSession(sessionId, { status: 'running' })
+    expect(sweepStuckTurnSessions(Date.now() + THRESHOLD_MS + 10_000)).toBe(1)
+    expect(getSession(sessionId)?.status).toBe('idle')
+  })
+})

package/packages/remote-control-server/src/__tests__/work-dispatch.test.ts

@@ -20,6 +20,7 @@ import {
   stopWork,
   heartbeatWork,
   reconnectWorkForEnvironment,
+  redispatchWorkForSession,
 } from '../services/work-dispatch'
 
 describe('Work Dispatch', () => {
@@ -195,4 +196,47 @@ describe('Work Dispatch', () => {
       expect(workIds).toHaveLength(0)
     })
   })
+
+  describe('redispatchWorkForSession', () => {
+    test('creates a pending work item for a live session regardless of status', async () => {
+      // 楔死 worker 的会话通常卡在 running（turn-end result 被吞）—
+      // 重新派发不能像 reconnectWorkForEnvironment 那样按 idle 过滤。
+      const { storeUpdateSession } = await import('../store')
+      storeUpdateSession(sessionId, { status: 'running' })
+      await redispatchWorkForSession(sessionId)
+      const item = storeGetPendingWorkItem(envId)
+      expect(item).toBeDefined()
+      expect(item?.sessionId).toBe(sessionId)
+      expect(item?.state).toBe('pending')
+    })
+
+    test('skips when an open (pending/dispatched) item already exists', async () => {
+      await createWorkItem(envId, sessionId)
+      await redispatchWorkForSession(sessionId)
+      const items = (await import('../store'))
+        .storeListWorkItems()
+        .filter(item => item.sessionId === sessionId)
+      expect(items).toHaveLength(1)
+    })
+
+    test('skips closed (archived) sessions', async () => {
+      const { storeUpdateSession } = await import('../store')
+      storeUpdateSession(sessionId, { status: 'archived' })
+      await redispatchWorkForSession(sessionId)
+      expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+    })
+
+    test('skips sessions with no environment binding', async () => {
+      const orphan = storeCreateSession({})
+      await redispatchWorkForSession(orphan.id)
+      expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+    })
+
+    test('swallows errors for inactive environments', async () => {
+      storeUpdateEnvironment(envId, { status: 'disconnected' })
+      // 不抛错 — 断连 env 由 disconnect monitor 处理
+      await redispatchWorkForSession(sessionId)
+      expect(storeGetPendingWorkItem(envId)).toBeFalsy()
+    })
+  })
 })