@workclaw/openclaw-workclaw 1.0.17 → 1.0.18

package/src/gateway/reconnect.ts

@@ -3,147 +3,251 @@
  * Supports both per-account and per-appKey strategies
  */
 
-import type { WorkClawBaseConfig } from '../types.js'
-import { clearOpenclawWorkclawTokenCache } from '../connection/workclaw-client.js'
-import { clearOpenclawWorkclawWsConnection, setOpenclawWorkclawWsConnection } from '../runtime.js'
+import { clearWorkclawTokenCache } from "../connection/workclaw-client.js";
+import { openWorkclawConnection } from "../connection/workclaw-client.js";
+import { Agent, WebSocket } from "undici";
+import { setWorkclawWsConnection, clearWorkclawWsConnection } from "../runtime.js";
+import type { WorkClawBaseConfig } from "../types.js";
 
 export interface ReconnectSchedulerOptions {
-  /** Key for WebSocket storage: accountId (per-account) or appKey (per-appKey) */
-  key: string
-  config: WorkClawBaseConfig
-  log?: {
-    info?: (msg: string) => void
-    warn?: (msg: string) => void
-    error?: (msg: string) => void
-    debug?: (msg: string) => void
-  }
-  onMessage: (data: string) => void
-  onReconnectSuccess?: () => void
+    /** Key for WebSocket storage: accountId (per-account) or appKey (per-appKey) */
+    key: string;
+    config: WorkClawBaseConfig;
+    log?: {
+        info?: (msg: string) => void;
+        warn?: (msg: string) => void;
+        error?: (msg: string) => void;
+        debug?: (msg: string) => void;
+    };
+    onMessage: (data: string) => void;
+    onReconnectSuccess?: () => void;
+    /** Called when connection closes (after initial connect), to trigger reconnect */
+    onClose?: () => void;
+    /** Called after WebSocket is registered (before onopen), useful for per-appKey lock release */
+    onConnectingStarted?: (ws: WebSocket) => void;
 }
 
 export function createReconnectScheduler(options: ReconnectSchedulerOptions): {
-  scheduleReconnect: () => void
-  stopReconnect: () => void
+    connect: () => Promise<void>;
+    scheduleReconnect: () => void;
+    stopReconnect: () => void;
 } {
-  const { key, config, log, onMessage, onReconnectSuccess } = options
-
-  let reconnectBackoff = 5000
-  let shouldStopReconnect = false
-  let pendingReconnectTimer: NodeJS.Timeout | null = null
-
-  const scheduleReconnect = (): void => {
-    if (shouldStopReconnect) {
-      log?.info?.(`Reconnect: Stopped due to previous auth failure`)
-      return
-    }
-    if (pendingReconnectTimer !== null) {
-      log?.info?.(`Reconnect: Already scheduled, skipping duplicate`)
-      return
-    }
-
-    const jitter = Math.floor(Math.random() * 3000)
-    const delayMs = reconnectBackoff + jitter
-    log?.info?.(`Reconnect: Scheduling in ${delayMs}ms (base=${reconnectBackoff} + jitter=${jitter}) key=${key}`)
+    const { key, config, log, onMessage, onReconnectSuccess, onClose, onConnectingStarted } = options;
+
+    let reconnectBackoff = 5000;
+    let shouldStopReconnect = false;
+    let pendingReconnectTimer: NodeJS.Timeout | null = null;
+    /** 正在进行的 doConnect() 的 promise，所有 caller 共享同一个结果 */
+    let pendingDoConnectPromise: Promise<void> | null = null;
+
+    /**
+     * 核心建连逻辑，首次连接和重连复用同一套代码
+     * 返回 Promise，连接建立后 resolve，失败后决定是否重连
+     * 关键：重试时复用 pendingDoConnectPromise，所有 caller 共享同一个结果
+     *  - 失败时：不删除 promise，让后续 connect() 调用者等待重试结果
+     *  - 重试定时器触发时：复用同一 promise，不创建新的
+     */
+    async function doConnect(): Promise<void> {
+        // 如果已有进行中的 doConnect（失败后重试期间），复用同一 promise
+        if (pendingDoConnectPromise !== null) {
+            return pendingDoConnectPromise;
+        }
 
-    pendingReconnectTimer = setTimeout(async () => {
-      pendingReconnectTimer = null
-      try {
-        log?.info?.(`Reconnect: Attempting... key=${key}`)
+        let resolveThis: (() => void) | undefined;
+        let rejectThis: ((err: unknown) => void) | undefined;
+        const p = new Promise<void>((resolve, reject) => {
+            resolveThis = resolve;
+            rejectThis = reject;
+        });
+        // 设置后再调用 connection，后续 connect() 调用能立即看到它
+        pendingDoConnectPromise = p;
 
-        const tokenCacheKey = config.appKey || key
-        clearOpenclawWorkclawTokenCache(tokenCacheKey)
+        const tokenCacheKey = config.appKey || key;
+        clearWorkclawTokenCache(tokenCacheKey);
 
         const connConfig = {
-          baseUrl: config.baseUrl || '',
-          websocketUrl: config.websocketUrl,
-          appKey: config.appKey,
-          appSecret: config.appSecret,
-          localIp: config.localIp,
-          allowInsecureTls: config.allowInsecureTls,
-          requestTimeout: config.requestTimeout,
+            baseUrl: config.baseUrl || "",
+            websocketUrl: config.websocketUrl,
+            appKey: config.appKey,
+            appSecret: config.appSecret,
+            localIp: config.localIp,
+            allowInsecureTls: config.allowInsecureTls,
+            requestTimeout: config.requestTimeout,
+        };
+
+        let endpoint: string;
+        let ticket: string;
+        try {
+            const result = await openWorkclawConnection(tokenCacheKey, connConfig);
+            endpoint = result.endpoint;
+            ticket = result.ticket;
+        } catch (err) {
+            // openWorkclawConnection 失败时（如 AbortError），由 handleConnectError 决定是否重试
+            // 如果不重试则 reject promise；如果重试则保持 pending（后续 timer 会触发重试）
+            const willRetry = handleConnectError(String(err));
+            if (!willRetry) {
+                pendingDoConnectPromise = null;
+                rejectThis!(err);
+            }
+            throw err;
         }
+        const url = `${endpoint}?ticket=${encodeURIComponent(ticket)}`;
 
-        const { openOpenclawWorkclawConnection } = await import('../connection/workclaw-client.js')
-        const { endpoint, ticket } = await openOpenclawWorkclawConnection(tokenCacheKey, connConfig)
-        const url = `${endpoint}?ticket=${encodeURIComponent(ticket)}`
-
-        const { Agent } = await import('undici')
         const dispatcher = connConfig.allowInsecureTls
-          ? new Agent({ connect: { rejectUnauthorized: false } })
-          : undefined
-        const wsOptions = dispatcher ? { dispatcher } : undefined
+            ? new Agent({ connect: { rejectUnauthorized: false } })
+            : undefined;
+        const wsOptions = dispatcher ? { dispatcher } : undefined;
+
+        const ws = new WebSocket(url, wsOptions);
+        setWorkclawWsConnection(key, ws);
 
-        const ws = new (await import('undici')).WebSocket(url, wsOptions)
-        setOpenclawWorkclawWsConnection(key, ws)
+        // 在 onopen 之前调用，让 waitForConnect() 释放锁
+        onConnectingStarted?.(ws);
 
+        // 统一用 resolveThis/rejectThis，ws.onopen 调用 resolveThis 让 await Promise 完成
         ws.onopen = () => {
-          log?.info?.(`Reconnect: Success`)
-          reconnectBackoff = 5000
-          onReconnectSuccess?.()
-        }
+            log?.info?.(`Gateway WebSocket connected endpoint=${endpoint}`);
+            reconnectBackoff = 5000;
+            onReconnectSuccess?.();
+            resolveThis!();
+        };
 
-        ws.onmessage = (event: any) => {
-          const data = typeof event.data === 'string' ? event.data : String(event.data ?? '')
-          log?.info?.(`Reconnect: Message bytes=${data.length}`)
-          onMessage(data)
-        }
+        ws.onerror = (event: any) => {
+            log?.error?.(`Gateway WebSocket error: ${String(event?.message ?? event)}`);
+            rejectThis!(new Error(String(event?.message ?? event)));
+        };
 
         ws.onclose = (event: any) => {
-          const code = String(event?.code ?? '')
-          const reason = String(event?.reason ?? '')
-          log?.warn?.(`Reconnect: Closed code=${code} reason=${reason}`)
-          clearOpenclawWorkclawWsConnection(key)
-          scheduleReconnect()
-        }
+            const code = String(event?.code ?? "");
+            const reason = String(event?.reason ?? "");
+            log?.warn?.(`Gateway WebSocket closed code=${code} reason=${reason}`);
+            clearWorkclawWsConnection(key);
+            if (onClose) {
+                onClose();
+            } else {
+                scheduleReconnect();
+            }
+            rejectThis!(new Error(`WebSocket closed code=${code} reason=${reason}`));
+        };
 
-        ws.onerror = (event: any) => {
-          log?.error?.(`Reconnect: Error: ${String(event?.message ?? event)}`)
+        ws.onmessage = (event: any) => {
+            const data = typeof event.data === "string" ? event.data : String(event.data ?? "");
+            log?.info?.(`Gateway WebSocket message received bytes=${data.length}`);
+            onMessage(data);
+        };
+
+        // 等待 ws.onopen 完成（resolveThis）或 ws.onerror/ws.onclose 失败（rejectThis）
+        await new Promise<void>((resolve, reject) => {
+            const timeout = setTimeout(() => reject(new Error("websocket connection timeout")), 30000);
+            ws.addEventListener("open", () => { clearTimeout(timeout); resolve(); }, { once: true });
+            ws.addEventListener("error", (event: any) => { clearTimeout(timeout); reject(new Error(String(event?.message ?? event))); }, { once: true });
+            ws.addEventListener("close", (event: any) => { clearTimeout(timeout); reject(new Error(`websocket closed before ready code=${event?.code ?? ""} reason=${event?.reason ?? ""}`)); }, { once: true });
+        });
+    }
+
+    const scheduleReconnect = () => {
+        if (shouldStopReconnect) {
+            log?.info?.(`Reconnect: Stopped due to previous auth failure`);
+            return;
+        }
+        if (pendingReconnectTimer !== null) {
+            log?.info?.(`Reconnect: Already scheduled, skipping duplicate`);
+            return;
         }
-      }
-      catch (err) {
-        const errorStr = String(err)
-        log?.error?.(`Reconnect: Failed: ${errorStr}`)
-
-        if (errorStr.includes('访问过于频繁')) {
-          log?.warn?.(`Reconnect: Rate limited, retrying with minimal backoff`)
-          if (pendingReconnectTimer !== null) {
-            clearTimeout(pendingReconnectTimer)
-            pendingReconnectTimer = null
-          }
-          reconnectBackoff = 1000
-          scheduleReconnect()
-          return
+
+        const jitter = Math.floor(Math.random() * 3000);
+        const delayMs = reconnectBackoff + jitter;
+        log?.info?.(`Reconnect: Scheduling in ${delayMs}ms (base=${reconnectBackoff} + jitter=${jitter}) key=${key}`);
+
+        pendingReconnectTimer = setTimeout(async () => {
+            pendingReconnectTimer = null;
+            // 清除旧的 pendingDoConnectPromise，确保 doConnect() 真正建立新连接
+            // 否则 doConnect() 会直接返回已 resolved 的旧 promise（第一次失败后 promise 没被清除）
+            pendingDoConnectPromise = null;
+            try {
+                log?.info?.(`Reconnect: Attempting... key=${key}`);
+                await doConnect();
+                log?.info?.(`Reconnect: Success`);
+            } catch (err) {
+                const errorStr = String(err);
+                log?.error?.(`Reconnect: Failed: ${errorStr}`);
+                const willRetry = handleConnectError(errorStr);
+                if (!willRetry) {
+                    // 永久失败，清除 pending promise
+                    pendingDoConnectPromise = null;
+                }
+            }
+        }, delayMs);
+    };
+
+    /** 返回 true=会重试（promise 保持 pending），false=不重试（promise 应被 reject） */
+    function handleConnectError(errorStr: string): boolean {
+        if (errorStr.includes("访问过于频繁")) {
+            log?.warn?.(`Reconnect: Rate limited, retrying with minimal backoff`);
+            if (pendingReconnectTimer !== null) {
+                clearTimeout(pendingReconnectTimer);
+                pendingReconnectTimer = null;
+            }
+            reconnectBackoff = 1000;
+            scheduleReconnect();
+            return true; // 会重试
         }
 
-        const isAuthError
-          = errorStr.includes('"errCode":2001')
-            || errorStr.includes('"code":2001')
-            || errorStr.includes('鉴权失败')
-            || errorStr.includes('invalid credentials')
-            || errorStr.includes('unauthorized')
+        const isAuthError =
+            errorStr.includes('"errCode":2001') ||
+            errorStr.includes('"code":2001') ||
+            errorStr.includes("鉴权失败") ||
+            errorStr.includes("invalid credentials") ||
+            errorStr.includes("unauthorized");
 
         if (isAuthError) {
-          log?.error?.(`Reconnect: Authentication failed, stopping`)
-          shouldStopReconnect = true
-          if (pendingReconnectTimer !== null) {
-            clearTimeout(pendingReconnectTimer)
-            pendingReconnectTimer = null
-          }
-          return
+            log?.error?.(`Reconnect: Authentication failed, stopping`);
+            shouldStopReconnect = true;
+            if (pendingReconnectTimer !== null) {
+                clearTimeout(pendingReconnectTimer);
+                pendingReconnectTimer = null;
+            }
+            return false; // 不重试
         }
 
-        scheduleReconnect()
-      }
-    }, delayMs)
-  }
+        // 其他错误：重试
+        return true;
+    }
 
-  const stopReconnect = (): void => {
-    shouldStopReconnect = true
-    if (pendingReconnectTimer !== null) {
-      clearTimeout(pendingReconnectTimer)
-      pendingReconnectTimer = null
+    /**
+     * 首次连接或重连：调用 doConnect()，返回其 promise
+     * 如果 doConnect() 因失败正在重试（promise 仍 pending），返回同一 promise
+     * 调用方 await 此方法，连接建立后 resolve，失败且不重试时 reject
+     */
+    async function connect(): Promise<void> {
+        // 如果 doConnect() 正在进行中（失败后重试期间），返回同一 promise 让 caller 等待
+        if (pendingDoConnectPromise !== null) {
+            return pendingDoConnectPromise;
+        }
+        try {
+            return await doConnect();
+        } catch (err) {
+            const errorStr = String(err);
+            log?.error?.(`Gateway connect failed: ${errorStr}`);
+            const willRetry = handleConnectError(errorStr);
+            if (!willRetry) {
+                // 不重试时，清除 pending promise，后续 connect() 可创建新的
+                pendingDoConnectPromise = null;
+                throw err;
+            }
+            // handleConnectError 返回 true 时，安排后续重试
+            // 注意：pendingDoConnectPromise 仍 pending，下次定时器触发时会清除它再真正重连
+            scheduleReconnect();
+        }
     }
-  }
 
-  return { scheduleReconnect, stopReconnect }
+    const stopReconnect = () => {
+        shouldStopReconnect = true;
+        if (pendingReconnectTimer !== null) {
+            clearTimeout(pendingReconnectTimer);
+            pendingReconnectTimer = null;
+        }
+    };
+
+    return { connect, scheduleReconnect, stopReconnect };
 }