@workclaw/openclaw-workclaw 1.0.20 → 1.0.25

package/src/gateway/reconnect.ts

@@ -0,0 +1,260 @@
+/**
+ * Reconnect Scheduler - manages WebSocket reconnection with exponential backoff
+ */
+
+import type { WorkClawBaseConfig } from '../types.js'
+import { Agent, WebSocket } from 'undici'
+import { clearWorkclawTokenCache, openWorkclawConnection } from '../connection/workclaw-client.js'
+
+import { clearWorkclawWsConnection, setWorkclawWsConnection } from '../runtime.js'
+
+export interface ReconnectSchedulerOptions {
+  /** Key for WebSocket storage (appKey) */
+  key: string
+  config: WorkClawBaseConfig
+  log?: {
+    info?: (msg: string) => void
+    warn?: (msg: string) => void
+    error?: (msg: string) => void
+    debug?: (msg: string) => void
+  }
+  onMessage: (data: string) => void
+  onReconnectSuccess?: () => void
+  /** Called when connection closes (after initial connect), to trigger reconnect */
+  onClose?: () => void
+  /** Called after WebSocket is registered (before onopen), useful for lock release */
+  onConnectingStarted?: (ws: WebSocket) => void
+}
+
+export function createReconnectScheduler(options: ReconnectSchedulerOptions): {
+  connect: () => Promise<void>
+  scheduleReconnect: () => void
+  stopReconnect: () => void
+} {
+  const { key, config, log, onMessage, onReconnectSuccess, onClose, onConnectingStarted } = options
+
+  let reconnectBackoff = 5000
+  let shouldStopReconnect = false
+  let pendingReconnectTimer: NodeJS.Timeout | null = null
+  /** 正在进行的 doConnect() 的 promise，所有 caller 共享同一个结果 */
+  let pendingDoConnectPromise: Promise<void> | null = null
+
+  /**
+   * 核心建连逻辑，首次连接和重连复用同一套代码
+   * 返回 Promise，连接建立后 resolve，失败后决定是否重连
+   * 关键：重试时复用 pendingDoConnectPromise，所有 caller 共享同一个结果
+   *  - 失败时：不删除 promise，让后续 connect() 调用者等待重试结果
+   *  - 重试定时器触发时：复用同一 promise，不创建新的
+   */
+  async function doConnect(): Promise<void> {
+    // 如果已有进行中的 doConnect（失败后重试期间），复用同一 promise
+    if (pendingDoConnectPromise !== null) {
+      return pendingDoConnectPromise
+    }
+
+    let resolveThis: (() => void) | undefined
+    let rejectThis: ((err: unknown) => void) | undefined
+    const p = new Promise<void>((resolve, reject) => {
+      resolveThis = resolve
+      rejectThis = reject
+    })
+    // 设置后再调用 connection，后续 connect() 调用能立即看到它
+    pendingDoConnectPromise = p
+
+    const tokenCacheKey = config.appKey || key
+    clearWorkclawTokenCache(tokenCacheKey)
+
+    const connConfig = {
+      baseUrl: config.baseUrl || '',
+      websocketUrl: config.websocketUrl,
+      appKey: config.appKey,
+      appSecret: config.appSecret,
+      localIp: config.localIp,
+      allowInsecureTls: config.allowInsecureTls,
+      requestTimeout: config.requestTimeout,
+    }
+
+    let endpoint: string
+    let ticket: string
+    try {
+      const result = await openWorkclawConnection(tokenCacheKey, connConfig)
+      endpoint = result.endpoint
+      ticket = result.ticket
+    }
+    catch (err) {
+      // openWorkclawConnection 失败时（如 AbortError），由 handleConnectError 决定是否重试
+      // 如果不重试则 reject promise；如果重试则保持 pending（后续 timer 会触发重试）
+      const willRetry = handleConnectError(String(err))
+      if (!willRetry) {
+        pendingDoConnectPromise = null
+        rejectThis!(err)
+      }
+      throw err
+    }
+    const url = `${endpoint}?ticket=${encodeURIComponent(ticket)}`
+
+    const dispatcher = connConfig.allowInsecureTls
+      ? new Agent({ connect: { rejectUnauthorized: false } })
+      : undefined
+    const wsOptions = dispatcher ? { dispatcher } : undefined
+
+    const ws = new WebSocket(url, wsOptions)
+    setWorkclawWsConnection(key, ws)
+
+    // 在 onopen 之前调用，让 waitForConnect() 释放锁
+    onConnectingStarted?.(ws)
+
+    // 统一用 resolveThis/rejectThis，ws.onopen 调用 resolveThis 让 await Promise 完成
+    ws.onopen = () => {
+      log?.info?.(`Gateway WebSocket connected endpoint=${endpoint}`)
+      reconnectBackoff = 5000
+      onReconnectSuccess?.()
+      resolveThis!()
+    }
+
+    ws.onerror = (event: any) => {
+      log?.error?.(`Gateway WebSocket error: ${String(event?.message ?? event)}`)
+      rejectThis!(new Error(String(event?.message ?? event)))
+    }
+
+    ws.onclose = (event: any) => {
+      const code = String(event?.code ?? '')
+      const reason = String(event?.reason ?? '')
+      log?.warn?.(`Gateway WebSocket closed code=${code} reason=${reason}`)
+      clearWorkclawWsConnection(key)
+      if (onClose) {
+        onClose()
+      }
+      else {
+        scheduleReconnect()
+      }
+      rejectThis!(new Error(`WebSocket closed code=${code} reason=${reason}`))
+    }
+
+    ws.onmessage = (event: any) => {
+      const data = typeof event.data === 'string' ? event.data : String(event.data ?? '')
+      log?.debug?.(`Gateway WebSocket message received bytes=${data.length}`)
+      onMessage(data)
+    }
+
+    // 等待 ws.onopen 完成（resolveThis）或 ws.onerror/ws.onclose 失败（rejectThis）
+    await new Promise<void>((resolve, reject) => {
+      const timeout = setTimeout(() => reject(new Error('websocket connection timeout')), 30000)
+      ws.addEventListener('open', () => { clearTimeout(timeout); resolve() }, { once: true })
+      ws.addEventListener('error', (event: any) => { clearTimeout(timeout); reject(new Error(String(event?.message ?? event))) }, { once: true })
+      ws.addEventListener('close', (event: any) => { clearTimeout(timeout); reject(new Error(`websocket closed before ready code=${event?.code ?? ''} reason=${event?.reason ?? ''}`)) }, { once: true })
+    })
+  }
+
+  const scheduleReconnect = () => {
+    if (shouldStopReconnect) {
+      log?.info?.(`Reconnect: Stopped due to previous auth failure`)
+      return
+    }
+    if (pendingReconnectTimer !== null) {
+      log?.info?.(`Reconnect: Already scheduled, skipping duplicate`)
+      return
+    }
+
+    const jitter = Math.floor(Math.random() * 3000)
+    const delayMs = reconnectBackoff + jitter
+    log?.info?.(`Reconnect: Scheduling in ${delayMs}ms (base=${reconnectBackoff} + jitter=${jitter}) key=${key}`)
+
+    pendingReconnectTimer = setTimeout(async () => {
+      pendingReconnectTimer = null
+      // 清除旧的 pendingDoConnectPromise，确保 doConnect() 真正建立新连接
+      // 否则 doConnect() 会直接返回已 resolved 的旧 promise（第一次失败后 promise 没被清除）
+      pendingDoConnectPromise = null
+      try {
+        log?.info?.(`Reconnect: Attempting... key=${key}`)
+        await doConnect()
+        log?.info?.(`Reconnect: Success`)
+      }
+      catch (err) {
+        const errorStr = String(err)
+        log?.error?.(`Reconnect: Failed: ${errorStr}`)
+        const willRetry = handleConnectError(errorStr)
+        if (!willRetry) {
+          // 永久失败，清除 pending promise
+          pendingDoConnectPromise = null
+        }
+        else {
+          // 错误不代表永久失败，调度下一次重连
+          scheduleReconnect()
+        }
+      }
+    }, delayMs)
+  }
+
+  /** 返回 true=会重试（promise 保持 pending），false=不重试（promise 应被 reject） */
+  function handleConnectError(errorStr: string): boolean {
+    if (errorStr.includes('访问过于频繁')) {
+      log?.warn?.(`Reconnect: Rate limited, retrying with minimal backoff`)
+      if (pendingReconnectTimer !== null) {
+        clearTimeout(pendingReconnectTimer)
+        pendingReconnectTimer = null
+      }
+      reconnectBackoff = 1000
+      scheduleReconnect()
+      return true // 会重试
+    }
+
+    const isAuthError
+      = errorStr.includes('"errCode":2001')
+        || errorStr.includes('"code":2001')
+        || errorStr.includes('鉴权失败')
+        || errorStr.includes('invalid credentials')
+        || errorStr.includes('unauthorized')
+
+    if (isAuthError) {
+      log?.error?.(`Reconnect: Authentication failed, stopping`)
+      shouldStopReconnect = true
+      if (pendingReconnectTimer !== null) {
+        clearTimeout(pendingReconnectTimer)
+        pendingReconnectTimer = null
+      }
+      return false // 不重试
+    }
+
+    // 其他错误：重试
+    return true
+  }
+
+  /**
+   * 首次连接或重连：调用 doConnect()，返回其 promise
+   * 如果 doConnect() 因失败正在重试（promise 仍 pending），返回同一 promise
+   * 调用方 await 此方法，连接建立后 resolve，失败且不重试时 reject
+   */
+  async function connect(): Promise<void> {
+    // 如果 doConnect() 正在进行中（失败后重试期间），返回同一 promise 让 caller 等待
+    if (pendingDoConnectPromise !== null) {
+      return pendingDoConnectPromise
+    }
+    try {
+      return await doConnect()
+    }
+    catch (err) {
+      const errorStr = String(err)
+      log?.error?.(`Gateway connect failed: ${errorStr}`)
+      const willRetry = handleConnectError(errorStr)
+      if (!willRetry) {
+        // 不重试时，清除 pending promise，后续 connect() 可创建新的
+        pendingDoConnectPromise = null
+        throw err
+      }
+      // handleConnectError 返回 true 时，安排后续重试
+      // 注意：pendingDoConnectPromise 仍 pending，下次定时器触发时会清除它再真正重连
+      scheduleReconnect()
+    }
+  }
+
+  const stopReconnect = () => {
+    shouldStopReconnect = true
+    if (pendingReconnectTimer !== null) {
+      clearTimeout(pendingReconnectTimer)
+      pendingReconnectTimer = null
+    }
+  }
+
+  return { connect, scheduleReconnect, stopReconnect }
+}