@jeik/dingtalk-connector 0.8.27 → 0.8.29

package/src/core/connection.ts

@@ -3,12 +3,14 @@
  *
  * 职责：
  * - 管理单个钉钉账号的 WebSocket 连接
- * - 实现应用层心跳检测（10 秒间隔，90 秒超时）
+ * - 实现应用层心跳检测（10 秒间隔，20 秒无 pong 超时）
  * - 处理连接重连逻辑，带指数退避
  * - 消息去重（内置 Map，5 分钟 TTL）
  *
  * 核心特性：
  * - 关闭 SDK 内置 keepAlive，使用自定义心跳
+ * - connect 后等 OPEN + SYSTEM/REGISTERED 再报 ready（防首条消息丢失）
+ * - 重连后先挂 pong/message/close listener 再等就绪（防幽灵重连）
  * - 详细的消息接收日志（三阶段：接收、解析、处理）
  * - 连接统计和监控（每分钟输出）
  */
@@ -18,6 +20,10 @@ import type { ResolvedDingtalkAccount } from "../types/index.ts";
 import {
   checkAndMarkDingtalkMessage,
 } from "../utils/utils-legacy.ts";
+import {
+  getActiveBackgroundWorkCount,
+  onBackgroundWorkCountChange,
+} from "../utils/background-work.ts";
 
 // ============ 类型定义 ============
 
@@ -57,12 +63,61 @@ export type MonitorDingtalkAccountOpts = {
 
 /** 心跳间隔（毫秒） */
 const HEARTBEAT_INTERVAL = 10 * 1000; // 10 秒
-/** 超时阈值（毫秒） */
-const TIMEOUT_THRESHOLD = 20 * 1000; // 20 秒（2 次心跳未响应）
+/**
+ * 无 pong 软超时：仅计「连续未确认」次数，不立刻 disconnect。
+ * 过去这里直接 doReconnect()，会在 socket 仍 OPEN 时拆掉活连接，
+ * 造成数秒空窗 → 用户连发 3~7 条消息钉钉推不到本进程（网关 UI 完全收不到）。
+ */
+const SOFT_STALE_MS = 20 * 1000;
+/** 连续软超时次数达到该值，且 socket 非 OPEN 或 ping 失败，才硬重连 */
+const HARD_RECONNECT_AFTER_MISSES = 3;
+/** 两次硬重连之间的最小间隔（除非 socket 已死） */
+const MIN_RECONNECT_GAP_MS = 30 * 1000;
+/**
+ * 消息处理期间刷新 lastSocketAvailableTime 的间隔。
+ * 必须明显小于 SOFT_STALE_MS。
+ */
+const MESSAGE_PROCESSING_KEEPALIVE_MS = 15 * 1000; // 15 秒
 /** 基础退避时间（毫秒） */
 const BASE_BACKOFF_DELAY = 1000; // 1 秒
 /** 最大退避时间（毫秒） */
 const MAX_BACKOFF_DELAY = 30 * 1000; // 30 秒
+/** 单次等待 OPEN + REGISTERED 的超时 */
+const STREAM_READY_TIMEOUT_MS = 15_000;
+/**
+ * REGISTERED 未就绪时最多完整重连次数（含首次）。
+ * 超时禁止 OPEN-only 假 ready，避免「显示已连接但收不到 CALLBACK」。
+ */
+const REGISTERED_CONNECT_MAX_ATTEMPTS = 5;
+
+// ============ 上游 SDK 噪音抑制（借鉴官方 #571/#536/#573）============
+// dingtalk-stream 在 connect/disconnect 时直接 console.info 两条固定串，
+// 绕过插件 logger，频繁重连时会刷屏造成「故障」误判。只过滤这两条精确匹配。
+let _streamNoiseSilenced = false;
+function silenceDingtalkStreamConsoleNoise(): void {
+  if (_streamNoiseSilenced) return;
+  _streamNoiseSilenced = true;
+  const origConsoleInfo = console.info.bind(console);
+  console.info = (...args: any[]) => {
+    const first = args[0];
+    if (typeof first === "string") {
+      if (first === "Disconnecting.") return;
+      if (/^\[[^\]]+\] connect success$/.test(first)) return;
+    }
+    return origConsoleInfo(...args);
+  };
+}
+
+let _connectionNoticePrinted = false;
+function printConnectionNoticeOnce(): void {
+  if (_connectionNoticePrinted) return;
+  _connectionNoticePrinted = true;
+  console.log(
+    "[dingtalk-connector] ℹ️  上游 dingtalk-stream 噪音已过滤；" +
+      "OPEN 成功即可上线，REGISTERED 尽量等待；若环境不推 REGISTERED，首条消息会确认订阅。" +
+      "正常运行不应出现 ≤30s 周期性硬重连。",
+  );
+}
 
 // ============ 监控账号 ============
 
@@ -72,12 +127,17 @@ export async function monitorSingleAccount(
   const { cfg, account, runtime, abortSignal, messageHandler, onStatusChange } = opts;
   const { accountId } = account;
 
+  // 在动态 import dingtalk-stream 之前抑制其 console.info 噪音
+  silenceDingtalkStreamConsoleNoise();
+
   // 保存 cfg 以便传递给 messageHandler
   const clawdbotConfig = cfg;
   const log = runtime?.log;
   
   // 创建 debug logger（仅在 debug 模式下输出 info/debug 日志）
-  const { createLoggerFromConfig } = await import('../utils/logger');
+  // 配置 debug: true 时同步打开图片/引用/队列详细诊断（无需环境变量）
+  const { createLoggerFromConfig, setDingtalkDebug } = await import('../utils/logger');
+  if (account.config?.debug) setDingtalkDebug(true);
   const logger = createLoggerFromConfig(account.config, `DingTalk:${accountId}`);
 
   // 验证凭据是否存在
@@ -158,53 +218,68 @@ export async function monitorSingleAccount(
   let reconnectAttempts = 0;
   let keepAliveTimer: NodeJS.Timeout | null = null;
   let isStopped = false;
+  /** 连续 keepAlive 周期内未收到 pong / 未刷新的次数 */
+  let consecutiveStaleMisses = 0;
+  let lastHardReconnectAt = 0;
   
   // ============ 消息处理活跃标记 ============
-  // 用于在消息处理期间防止心跳超时触发重连
+  // 覆盖「WS 回调短暂入队」+「sessionQueues 后台 AI 全长」（见 background-work.ts）
+  // 仅覆盖 WS 回调会导致：入队后立刻 End → 503/长任务期间幽灵重连 → 后续消息无反应
+  let wsCallbackActive = false;
   let activeMessageProcessing = false;
   let messageProcessingKeepAliveTimer: NodeJS.Timeout | null = null;
-  
-  /**
-   * 标记消息处理开始，启动定期更新机制
-   * 在消息处理期间，每 30 秒更新一次 lastSocketAvailableTime
-   * 防止长时间处理（如复杂的 AI 任务）触发心跳超时
-   */
-  function markMessageProcessingStart() {
-    activeMessageProcessing = true;
-    lastSocketAvailableTime = Date.now();
-    
-    // 清理旧的定时器（如果存在）
-    if (messageProcessingKeepAliveTimer) {
-      clearInterval(messageProcessingKeepAliveTimer);
-    }
-    
-    // 每 30 秒更新一次，确保不会触发 90 秒超时
+  let unsubscribeBackgroundWork: (() => void) | null = null;
+
+  function ensureProcessingKeepAliveTimer(): void {
+    if (messageProcessingKeepAliveTimer) return;
     messageProcessingKeepAliveTimer = setInterval(() => {
       if (activeMessageProcessing) {
         lastSocketAvailableTime = Date.now();
-        logger.debug(`📝 消息处理中，更新 socket 可用时间`);
+        logger.debug(
+          `📝 消息处理中，更新 socket 可用时间 (ws=${wsCallbackActive} bg=${getActiveBackgroundWorkCount()})`,
+        );
       }
-    }, 30 * 1000); // 30 秒间隔
-    
-    logger.debug(`📝 消息处理开始，启动活跃标记定时器`);
+    }, MESSAGE_PROCESSING_KEEPALIVE_MS);
   }
-  
-  /**
-   * 标记消息处理结束，停止定期更新机制
-   */
-  function markMessageProcessingEnd() {
-    activeMessageProcessing = false;
-    
+
+  function clearProcessingKeepAliveTimer(): void {
     if (messageProcessingKeepAliveTimer) {
       clearInterval(messageProcessingKeepAliveTimer);
       messageProcessingKeepAliveTimer = null;
     }
-    
-    // 最后更新一次时间
+  }
+
+  /** 根据 WS 入队中 / 后台任务数 合成 activeMessageProcessing */
+  function refreshProcessingActive(reason: string): void {
+    const bg = getActiveBackgroundWorkCount();
+    const next = wsCallbackActive || bg > 0;
+    activeMessageProcessing = next;
     lastSocketAvailableTime = Date.now();
-    logger.debug(`✅ 消息处理结束，清理活跃标记定时器`);
+    if (next) {
+      ensureProcessingKeepAliveTimer();
+      logger.debug(
+        `📝 处理活跃 reason=${reason} ws=${wsCallbackActive} bg=${bg}`,
+      );
+    } else {
+      clearProcessingKeepAliveTimer();
+      logger.debug(`✅ 处理空闲 reason=${reason}`);
+    }
+  }
+
+  function markMessageProcessingStart() {
+    wsCallbackActive = true;
+    refreshProcessingActive("ws-start");
   }
 
+  function markMessageProcessingEnd() {
+    wsCallbackActive = false;
+    refreshProcessingActive("ws-end");
+  }
+
+  unsubscribeBackgroundWork = onBackgroundWorkCountChange(() => {
+    refreshProcessingActive("bg-count");
+  });
+
   // ============ 辅助函数 ============
 
   /** 计算指数退避延迟（带抖动） */
@@ -214,6 +289,198 @@ export async function monitorSingleAccount(
     return Math.min(exponentialDelay + jitter, MAX_BACKOFF_DELAY);
   }
 
+  /**
+   * 等待钉钉 Stream 可服务。
+   *
+   * - OPEN：必须成功，否则 throw
+   * - REGISTERED：尽量等待；超时则 **降级 OPEN 上线**（不 throw）
+   *
+   * 原因（生产日志）：部分环境 socket 已 OPEN 且 CALLBACK 能收消息，
+   * 但长期收不到 SYSTEM/REGISTERED 或 client.registered 不被置 true。
+   * 若硬性失败会触发网关 auto-restart 风暴，而旧连接/半连接其实仍在处理消息。
+   */
+  async function waitForStreamReady(timeoutMs = STREAM_READY_TIMEOUT_MS): Promise<{
+    registered: boolean;
+  }> {
+    const started = Date.now();
+
+    // 1) 等 WebSocket OPEN
+    if ((client as any).socket?.readyState !== 1) {
+      const opened = await new Promise<boolean>((resolve) => {
+        const socket = (client as any).socket;
+        if (!socket) {
+          resolve(false);
+          return;
+        }
+        if (socket.readyState === 1) {
+          resolve(true);
+          return;
+        }
+        const remain = Math.max(1_000, timeoutMs - (Date.now() - started));
+        let settled = false;
+        const finish = (ok: boolean) => {
+          if (settled) return;
+          settled = true;
+          clearTimeout(timer);
+          socket.removeListener?.("open", onOpen);
+          socket.removeListener?.("error", onError);
+          resolve(ok);
+        };
+        const timer = setTimeout(() => finish(false), remain);
+        const onOpen = () => finish(true);
+        const onError = () => finish(false);
+        socket.once("open", onOpen);
+        socket.once("error", onError);
+      });
+      if (!opened) {
+        throw new Error(`WebSocket OPEN 超时（${timeoutMs}ms）`);
+      }
+    }
+
+    // 2) 等服务端 REGISTERED（可选；超时降级）
+    if ((client as any).registered === true) {
+      logger.info(
+        `✅ Stream 已就绪（OPEN + REGISTERED），耗时 ${Date.now() - started}ms`,
+      );
+      return { registered: true };
+    }
+
+    const registered = await new Promise<boolean>((resolve) => {
+      const socket = (client as any).socket;
+      if (!socket) {
+        resolve(false);
+        return;
+      }
+      const remain = Math.max(1_000, timeoutMs - (Date.now() - started));
+      let settled = false;
+      const finish = (ok: boolean) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        clearInterval(poll);
+        socket.removeListener?.("message", onMessage);
+        resolve(ok);
+      };
+      const timer = setTimeout(() => finish(false), remain);
+
+      const onMessage = (data: any) => {
+        try {
+          const raw = typeof data === "string" ? data : data?.toString?.() ?? String(data);
+          const msg = JSON.parse(raw);
+          if (msg?.type === "SYSTEM" && msg?.headers?.topic === "REGISTERED") {
+            finish(true);
+            return;
+          }
+        } catch {
+          // ignore
+        }
+        if ((client as any).registered === true) {
+          finish(true);
+        }
+      };
+
+      const poll = setInterval(() => {
+        if ((client as any).registered === true) {
+          finish(true);
+        }
+      }, 100);
+
+      socket.on("message", onMessage);
+    });
+
+    if (registered || (client as any).registered === true) {
+      logger.info(
+        `✅ Stream 已就绪（OPEN + REGISTERED），耗时 ${Date.now() - started}ms`,
+      );
+      return { registered: true };
+    }
+
+    // 降级：OPEN 成功即可上线。许多环境不推 REGISTERED 或检测不到，但仍可收 CALLBACK。
+    logger.warn(
+      `⚠️ 未观察到 SYSTEM/REGISTERED（${timeoutMs}ms），socket 已 OPEN → 降级上线。` +
+        ` 收到首条机器人消息后会记为「订阅 empirically 生效」。`,
+    );
+    return { registered: false };
+  }
+
+  /**
+   * connect 之后、wait ready 之前挂上 pong/message/close。
+   * 官方 #566：若在 connect 前 setup，socket 为 undefined，listener 静默 no-op →
+   * pong 无人接 → lastSocketAvailableTime 不刷新 → TIMEOUT 幽灵重连。
+   * 必须在 wait OPEN 之前挂好，否则等待窗内的 pong 也会丢。
+   */
+  function attachSocketLifecycleListeners(): void {
+    setupPongListener();
+    setupMessageListener();
+    setupCloseListener();
+  }
+
+  /**
+   * 单次：connect + 挂 listener + 等 OPEN（REGISTERED 可选）。
+   * 仅 OPEN 失败 throw；REGISTERED 超时则降级成功。
+   */
+  async function connectAndWaitReady(): Promise<{ registered: boolean }> {
+    await client.connect();
+    attachSocketLifecycleListeners();
+    return await waitForStreamReady(STREAM_READY_TIMEOUT_MS);
+  }
+
+  /**
+   * 建立可服务连接。
+   * - 优先拿到 REGISTERED
+   * - OPEN 成功即可上线（避免环境不推 REGISTERED 时 restart 风暴）
+   * - 仅在 OPEN 都失败时多轮重试
+   */
+  async function ensureRegisteredConnection(
+    maxAttempts = REGISTERED_CONNECT_MAX_ATTEMPTS,
+  ): Promise<void> {
+    let lastError: Error | undefined;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      if (isStopped) {
+        throw new Error("连接已停止，中止 Stream 建立");
+      }
+      try {
+        if (attempt > 1) {
+          logger.warn(
+            `🔄 Stream 建立重试 ${attempt}/${maxAttempts}…`,
+          );
+          try {
+            if ((client as any).socket) {
+              await client.disconnect();
+            }
+          } catch (discErr: any) {
+            logger.debug(`断开旧连接: ${discErr?.message || discErr}`);
+          }
+          const delay = Math.min(1000 * attempt, 5_000);
+          await new Promise((r) => setTimeout(r, delay));
+        } else {
+          logger.info(
+            `⏳ 建立 Stream（OPEN 必须；REGISTERED 尽量等待 ${STREAM_READY_TIMEOUT_MS}ms）…`,
+          );
+        }
+
+        const ready = await connectAndWaitReady();
+        noteSocketAlive(ready.registered ? "registered-ok" : "open-degraded");
+        connectionEstablishedTime = Date.now();
+        logger.info(
+          `✅ Stream 可服务（attempt ${attempt}/${maxAttempts}, ` +
+            `registered=${ready.registered || Boolean((client as any).registered)}, ` +
+            `socket=${(client as any).socket?.readyState}）`,
+        );
+        return;
+      } catch (err: any) {
+        lastError = err instanceof Error ? err : new Error(String(err));
+        logger.warn(
+          `⚠️ Stream 建立失败 attempt=${attempt}/${maxAttempts}: ${lastError.message}`,
+        );
+      }
+    }
+    throw new Error(
+      `钉钉 Stream 在 ${maxAttempts} 次尝试后仍无法 OPEN。` +
+        ` 最后错误: ${lastError?.message || "unknown"}`,
+    );
+  }
+
   /** 统一重连函数，带指数退避（无限重连） */
   async function doReconnect(immediate = false) {
     if (isReconnecting || isStopped) {
@@ -233,68 +500,31 @@ export async function monitorSingleAccount(
     }
 
     try {
-      // 1. 先断开旧连接（检查 WebSocket 状态）
+      // 1. 先断开旧连接
       if ((client as any).socket?.readyState === 1 || (client as any).socket?.readyState === 3) {
         await client.disconnect();
         logger.info(`已断开旧连接`);
       }
 
-      // 2. 重新建立连接
-      await client.connect();
-
-      // 3. 等待连接真正建立（监听 open 事件，最多等待 10 秒）
-      const connectionEstablished = await new Promise<boolean>((resolve) => {
-        const timeout = setTimeout(() => {
-          resolve(false);
-        }, 10_000); // 10 秒超时
+      // 2. 重连直到 REGISTERED（内部含 connect + listener + wait，失败会多轮）
+      await ensureRegisteredConnection(REGISTERED_CONNECT_MAX_ATTEMPTS);
 
-        // 如果已经是 OPEN 状态，直接返回
-        if ((client as any).socket?.readyState === 1) {
-          clearTimeout(timeout);
-          resolve(true);
-          return;
-        }
-
-        // 否则监听 open 事件
-        const onOpen = () => {
-          clearTimeout(timeout);
-          (client as any).socket?.removeListener('open', onOpen);
-          (client as any).socket?.removeListener('error', onError);
-          resolve(true);
-        };
+      // 3. 重置计时与状态
+      noteSocketAlive("reconnect-ok");
+      connectionEstablishedTime = Date.now();
+      reconnectAttempts = 0;
+      lastHardReconnectAt = Date.now();
 
-        const onError = (err: any) => {
-          clearTimeout(timeout);
-          (client as any).socket?.removeListener('open', onOpen);
-          (client as any).socket?.removeListener('error', onError);
-          logger.warn(`连接建立失败: ${err.message}`);
-          resolve(false);
-        };
-
-        (client as any).socket?.once('open', onOpen);
-        (client as any).socket?.once('error', onError);
-      });
-
-      if (!connectionEstablished) {
-        throw new Error(`连接建立超时或失败`);
-      }
-
-      // 4. 重置 socket 可用时间、连接建立时间和重连计数
-      lastSocketAvailableTime = Date.now();
-      connectionEstablishedTime = Date.now(); // 重置连接建立时间
-      reconnectAttempts = 0; // 重连成功，重置计数
-
-      // 重连成功，向框架报告 connected: true
+      // 4. 真正可收 CALLBACK 后再报 connected
       onStatusChange?.({ connected: true, lastConnectedAt: Date.now() });
 
-      // 重新注册 socket 事件监听器（新 socket 需要新的 listener）
-      setupPongListener();
-      setupMessageListener();
-      setupCloseListener();
-
-      logger.info(`✅ 重连成功 (socket 状态=${(client as any).socket?.readyState})`);
+      logger.info(
+        `✅ 重连成功 (socket=${(client as any).socket?.readyState}, registered=${Boolean((client as any).registered)})`,
+      );
     } catch (err: any) {
       reconnectAttempts++;
+      // 未 REGISTERED：保持 connected=false，避免 UI 假在线
+      onStatusChange?.({ connected: false });
       logger.error(
         `重连失败：${err.message} (尝试 ${reconnectAttempts})`,
       );
@@ -308,10 +538,17 @@ export async function monitorSingleAccount(
   function setupPongListener() {
     (client as any).socket?.on("pong", () => {
       lastSocketAvailableTime = Date.now();
+      consecutiveStaleMisses = 0;
       logger.debug(`收到 PONG 响应`);
     });
   }
 
+  function noteSocketAlive(reason: string): void {
+    lastSocketAvailableTime = Date.now();
+    consecutiveStaleMisses = 0;
+    logger.debug(`socket alive reason=${reason}`);
+  }
+
   /** 监听 WebSocket message 事件，收到 disconnect 消息时立即触发重连 */
   function setupMessageListener() {
     (client as any).socket?.on("message", (data: any) => {
@@ -355,12 +592,54 @@ export async function monitorSingleAccount(
   }
 
   /**
-   * 启动 keepAlive 机制（单定时器 + 指数退避）
-   *
-   * 业界最佳实践：
-   * - 单定时器：每 10 秒检查一次，同时完成心跳和超时检测
-   * - 使用 WebSocket 原生 Ping
-   * - 指数退避重连：避免雪崩效应
+   * 请求硬重连（拆 socket）。
+   * - socket 仍 OPEN：先补 PING，连续 misses 未达阈值则绝不拆连接（防空窗吞消息）
+   * - socket 已死：立即重连
+   * - 冷却：非死连接时距上次硬重连 < MIN_RECONNECT_GAP_MS 则跳过
+   */
+  async function requestHardReconnect(reason: string, socketDead = false): Promise<void> {
+    if (isReconnecting || isStopped) return;
+
+    const socketState = (client as any).socket?.readyState;
+
+    // 活连接：优先补 ping，未达连续 miss 阈值不拆
+    if (!socketDead && socketState === 1) {
+      try {
+        (client as any).socket?.ping();
+        logger.warn(
+          `⚠️ 软超时但 socket 仍 OPEN (reason=${reason}, misses=${consecutiveStaleMisses}/${HARD_RECONNECT_AFTER_MISSES})，仅补 PING`,
+        );
+      } catch (err: any) {
+        logger.warn(`补 PING 失败: ${err.message}`);
+        consecutiveStaleMisses += 1;
+      }
+      if (consecutiveStaleMisses < HARD_RECONNECT_AFTER_MISSES) {
+        return;
+      }
+    }
+
+    // 冷却（socket 已死时跳过冷却）
+    if (!socketDead) {
+      const gap = Date.now() - lastHardReconnectAt;
+      if (lastHardReconnectAt > 0 && gap < MIN_RECONNECT_GAP_MS) {
+        logger.warn(
+          `⚠️ 跳过硬重连：距上次仅 ${Math.round(gap / 1000)}s < ${MIN_RECONNECT_GAP_MS / 1000}s 冷却 (reason=${reason})`,
+        );
+        return;
+      }
+    }
+
+    lastHardReconnectAt = Date.now();
+    consecutiveStaleMisses = 0;
+    logger.info(`🔄 硬重连 reason=${reason} socket=${socketState}`);
+    await doReconnect(socketDead);
+  }
+
+  /**
+   * 启动 keepAlive：
+   * - 10s 发 ping；pong 刷新 lastSocketAvailableTime
+   * - 软超时只计数 + 补 ping，达到 HARD_RECONNECT_AFTER_MISSES 且 socket 异常才拆连接
+   * - 处理中任务期间绝不硬重连
    */
   function startKeepAlive(): () => void {
     logger.debug(
@@ -375,24 +654,16 @@ export async function monitorSingleAccount(
 
       try {
         const elapsed = Date.now() - lastSocketAvailableTime;
-
-        // 【超时检测】超过 90 秒未确认 socket 可用，触发重连
-        if (elapsed > TIMEOUT_THRESHOLD) {
-          logger.info(
-            `⚠️ 超时检测：已 ${Math.round(elapsed / 1000)} 秒未确认 socket 可用，触发重连...`,
-          );
-          await doReconnect();
-          return;
-        }
-
-        // 【心跳检测】检查 socket 状态
         const socketState = (client as any).socket?.readyState;
         const timeSinceConnection = Date.now() - connectionEstablishedTime;
+
         logger.debug(
-          `心跳检测：socket 状态=${socketState}, elapsed=${Math.round(elapsed / 1000)}s, 连接已建立=${Math.round(timeSinceConnection / 1000)}s`,
+          `心跳检测：socket=${socketState}, elapsed=${Math.round(elapsed / 1000)}s, ` +
+            `misses=${consecutiveStaleMisses}, processing=${activeMessageProcessing}, ` +
+            `connectedFor=${Math.round(timeSinceConnection / 1000)}s`,
         );
 
-        // 给新建立的连接 15 秒宽限期，避免在连接建立初期就触发重连
+        // socket 非 OPEN：宽限期后硬重连
         if (socketState !== 1) {
           if (timeSinceConnection < 15_000) {
             logger.debug(
@@ -400,31 +671,55 @@ export async function monitorSingleAccount(
             );
             return;
           }
-          
-          logger.info(
-            `⚠️ 心跳检测：socket 状态=${socketState}，触发重连...`,
+          if (activeMessageProcessing) {
+            logger.warn(
+              `⚠️ socket 非 OPEN 但消息处理中，暂缓硬重连 state=${socketState}`,
+            );
+            return;
+          }
+          await requestHardReconnect(`socket-dead state=${socketState}`, true);
+          return;
+        }
+
+        // 软超时：无 pong/刷新超过 SOFT_STALE_MS
+        if (elapsed > SOFT_STALE_MS) {
+          consecutiveStaleMisses += 1;
+          if (activeMessageProcessing) {
+            noteSocketAlive("processing-hold");
+            logger.warn(
+              `⚠️ 软超时但处理中，刷新活跃时间不重连 (elapsed=${Math.round(elapsed / 1000)}s)`,
+            );
+            return;
+          }
+          logger.warn(
+            `⚠️ 软超时 misses=${consecutiveStaleMisses}/${HARD_RECONNECT_AFTER_MISSES} ` +
+              `elapsed=${Math.round(elapsed / 1000)}s socket=OPEN`,
           );
-          await doReconnect(true); // 立即重连，不退避
+          // OPEN 时优先补 ping；达到次数再考虑硬重连
+          await requestHardReconnect("stale-pong", false);
           return;
         }
 
-        // 【发送原生 Ping】仅发送，不刷新时间戳；
-        // 只有收到 pong 响应时才更新 lastSocketAvailableTime（见 setupPongListener）
+        consecutiveStaleMisses = 0;
+
+        // 正常心跳 ping（pong 到了才刷新时间戳）
         try {
           (client as any).socket?.ping();
           logger.debug(`💓 发送 PING 心跳成功`);
         } catch (err: any) {
           logger.warn(`发送 PING 失败：${err.message}`);
-          // 发送失败也计入超时
+          consecutiveStaleMisses += 1;
+          if (consecutiveStaleMisses >= HARD_RECONNECT_AFTER_MISSES) {
+            await requestHardReconnect("ping-failed", true);
+          }
         }
       } catch (err: any) {
         logger.error(`keepAlive 检测失败：${err.message}`);
       }
-    }, HEARTBEAT_INTERVAL); // 每 10 秒检测一次
+    }, HEARTBEAT_INTERVAL);
 
     logger.debug(`✅ keepAlive 定时器已启动`);
 
-    // 返回清理函数
     return () => {
       if (keepAliveTimer) clearInterval(keepAliveTimer);
       keepAliveTimer = null;
@@ -441,10 +736,11 @@ export async function monitorSingleAccount(
     keepAliveTimer = null;
 
     // 清理消息处理活跃标记定时器
-    if (messageProcessingKeepAliveTimer) {
-      clearInterval(messageProcessingKeepAliveTimer);
-      messageProcessingKeepAliveTimer = null;
-    }
+    clearProcessingKeepAliveTimer();
+    wsCallbackActive = false;
+    activeMessageProcessing = false;
+    unsubscribeBackgroundWork?.();
+    unsubscribeBackgroundWork = null;
 
     // 清理事件监听器
     if ((client as any).socket) {
@@ -492,113 +788,92 @@ export async function monitorSingleAccount(
     client.registerCallbackListener(TOPIC_ROBOT, async (res: any) => {
       receivedCount++;
       lastMessageTime = Date.now();
+      // 能收到 CALLBACK 说明链路活着——立刻刷新，打断「软超时→硬重连」误判
+      noteSocketAlive("inbound-callback");
+      // 部分环境收不到 SYSTEM/REGISTERED，但 CALLBACK 已通：记为 empirically 订阅成功
+      if ((client as any).registered !== true) {
+        (client as any).registered = true;
+        logger.info(
+          `✅ 收到机器人 CALLBACK，视为订阅已生效（此前可能未观察到 SYSTEM/REGISTERED）`,
+        );
+      }
 
       // 收到消息时，向框架报告 lastInboundAt（用于 UI 显示 "Last inbound"）
       onStatusChange?.({ lastInboundAt: Date.now() });
 
       const messageId = res.headers?.messageId;
-      const timestamp = new Date().toISOString();
-
-      // ===== 第一步：记录原始消息接收 =====
-      logger.info(`\n========== 收到新消息 ==========`);
-      logger.info(`时间：${timestamp}`);
-      logger.info(`MessageId: ${messageId || "N/A"}`);
-      logger.info(`Headers: ${JSON.stringify(res.headers || {})}`);
-      logger.info(`Data 长度：${res.data?.length || 0} 字符`);
-
-      // 立即确认回调
-      if (messageId) {
-        (client as any).socketCallBackResponse(messageId, { success: true });
-        logger.info(`✅ 已立即确认回调：messageId=${messageId}`);
-      } else {
-        logger.warn(`⚠️ 警告：消息没有 messageId`);
-      }
 
-      // 协议层去重（headers.messageId）：拦截同一次投递的重复回调
-      // 注意：业务层去重（data.msgId）在 JSON 解析后执行，两层合并在 checkAndMarkDingtalkMessage 中
-      // 此处仅做协议层的快速预检，避免不必要的 JSON 解析
-      if (messageId && checkAndMarkDingtalkMessage(accountId, messageId, undefined)) {
-        processedCount++;
-        logger.warn(`⚠️ 检测到重复消息（协议层），跳过处理：messageId=${messageId} (${processedCount}/${receivedCount})`);
-        logger.info(`========== 消息处理结束（重复） ==========\n`);
-        return;
-      }
+      // 入队成功后再 ACK；失败不 ACK 以便钉钉重投
+      let acked = false;
+      const ackMessage = (why: string) => {
+        if (acked || !messageId) return;
+        try {
+          (client as any).socketCallBackResponse(messageId, { success: true });
+          acked = true;
+          logger.debug(`ACK messageId=${messageId} (${why})`);
+        } catch (ackErr: any) {
+          logger.warn(`确认回调失败: ${ackErr?.message || ackErr}`);
+        }
+      };
 
-      // 异步处理消息
-      // ✅ 标记消息处理开始，防止长时间处理触发心跳超时
       markMessageProcessingStart();
-      
+
       try {
-        // 解析消息数据
         let data;
         try {
           data = JSON.parse(res.data);
         } catch (parseError: any) {
-          logger.error('Failed to parse response data as JSON:', {
-            error: parseError instanceof Error ? parseError.message : String(parseError),
-            rawData: typeof res.data === 'string' 
-              ? res.data.substring(0, 500) // 只记录前 500 字符
-              : res.data,
-            dataType: typeof res.data,
-          });
+          logger.error(
+            `JSON 解析失败: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
+          );
           throw new Error(
-            `Invalid JSON response from DingTalk API. ` +
-            `Error: ${parseError instanceof Error ? parseError.message : String(parseError)}. ` +
-            `Raw data (first 100 chars): ${String(res.data).substring(0, 100)}`
+            `Invalid JSON from DingTalk: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
           );
         }
 
-        // ===== 第二步：记录解析后的消息详情 =====
-        logger.info(`\n----- 消息详情 -----`);
-        logger.info(`消息类型：${data.msgtype || "unknown"}`);
-        logger.info(
-          `会话类型：${data.conversationType === "1" ? "DM (单聊)" : data.conversationType === "2" ? "Group (群聊)" : data.conversationType}`,
-        );
-        logger.info(
-          `发送者：${data.senderNick || "unknown"} (${data.senderStaffId || data.senderId || "unknown"})`,
-        );
-        logger.info(`会话 ID: ${data.conversationId || "N/A"}`);
-        logger.info(`消息 ID: ${data.msgId || "N/A"}`);
-        logger.info(
-          `SessionWebhook: ${data.sessionWebhook ? "已提供" : "未提供"}`,
-        );
+        const businessMsgId = data.msgId;
+        const convType =
+          data.conversationType === "1"
+            ? "DM"
+            : data.conversationType === "2"
+              ? "Group"
+              : String(data.conversationType ?? "?");
+        const preview =
+          (data.text?.content || "").trim().slice(0, 80) ||
+          data.msgtype ||
+          "(no text)";
+
+        // 一行摘要（debug 模式才打 Headers/全字段）
         logger.info(
-          `RobotCode: ${data.robotCode || account.config?.clientId || "N/A"}`,
+          `📩 入站 ${convType} from=${data.senderNick || data.senderStaffId || data.senderId || "?"} ` +
+            `msg=${businessMsgId || messageId || "-"} preview=${JSON.stringify(preview)}`,
         );
-        // 暴露当前机器人的加密身份（chatbotUserId / chatbotCorpId）
-        // 用途：多机器人协作时，把这两个值配进 openclaw.json 对应 account 的元数据下，
-        // 让其他 agent 能在群消息里通过 atDingtalkIds 写上对方的 chatbotUserId 触发 @ UI。
-        // 注意：这里**不**经过 logger.info（受 debug 开关控制），直接 console.log，
-        // 否则首次配置的用户在未开 debug 时永远看不到这两个 ID，无法完成多机器人协作配置。
+        logger.debug(`Headers: ${JSON.stringify(res.headers || {})}`);
+        logger.debug(`字段: ${Object.keys(data).join(",")}`);
+
+        // BotIdentity 每个进程每个账号只打一次（配置多机器人时用）
         if (data.chatbotUserId || data.chatbotCorpId) {
-          console.log(
-            `[DingTalk:${accountId}] [BotIdentity] accountId=${accountId} chatbotUserId=${data.chatbotUserId || "N/A"} chatbotCorpId=${data.chatbotCorpId || "N/A"}`,
-          );
+          const key = `${accountId}:${data.chatbotUserId || ""}:${data.chatbotCorpId || ""}`;
+          if (!(globalThis as any).__dingtalkBotIdentityLogged) {
+            (globalThis as any).__dingtalkBotIdentityLogged = new Set<string>();
+          }
+          const seen = (globalThis as any).__dingtalkBotIdentityLogged as Set<string>;
+          if (!seen.has(key)) {
+            seen.add(key);
+            console.log(
+              `[DingTalk:${accountId}] [BotIdentity] chatbotUserId=${data.chatbotUserId || "N/A"} chatbotCorpId=${data.chatbotCorpId || "N/A"}`,
+            );
+          }
         }
 
-        // ===== 业务层去重：补充 data.msgId，防止钉钉服务端重发穿透 =====
-        // 协议层已标记了 headers.messageId，此处再补充标记 data.msgId。
-        // 钉钉重发时 headers.messageId 是新值，但 data.msgId 不变，
-        // checkAndMarkDingtalkMessage 会命中 data.msgId 并返回 true 拦截重发。
-        const businessMsgId = data.msgId;
-
-        // 记录消息内容（简化版，避免过长）
-        let contentPreview = "N/A";
-        if (data.text?.content) {
-          contentPreview =
-            data.text.content.length > 100
-              ? data.text.content.substring(0, 100) + "..."
-              : data.text.content;
-        } else if (data.content) {
-          contentPreview =
-            JSON.stringify(data.content).substring(0, 100) + "...";
+        if (checkAndMarkDingtalkMessage(accountId, messageId, businessMsgId)) {
+          processedCount++;
+          ackMessage("duplicate");
+          logger.warn(
+            `⚠️ 重复消息已跳过 protocol=${messageId || "-"} business=${businessMsgId || "-"}`,
+          );
+          return;
         }
-        logger.info(`消息内容预览：${contentPreview}`);
-        logger.info(`完整数据字段：${Object.keys(data).join(", ")}`);
-        logger.info(`----- 消息详情结束 -----\n`);
-
-        // ===== 第三步：开始处理消息 =====
-        logger.info(`🚀 开始处理消息...`);
 
         await messageHandler({
           accountId,
@@ -610,21 +885,23 @@ export async function monitorSingleAccount(
           cfg: clawdbotConfig,
         });
 
+        ackMessage("enqueued");
+        noteSocketAlive("after-enqueue");
+
         processedCount++;
-        logger.info(`✅ 消息处理完成 (${processedCount}/${receivedCount})`);
-        logger.info(`========== 消息处理结束（成功） ==========\n`);
+        logger.info(`✅ 入队完成 (${processedCount}/${receivedCount})`);
       } catch (error: any) {
         processedCount++;
-        const errorMsg = `❌ 处理消息异常 (${processedCount}/${receivedCount}): ${error?.message || "未知错误"}`;
-        const errorStack = error?.stack || "无堆栈信息";
-        
-        // 使用 logger 记录错误信息
-        logger.error(errorMsg);
-        logger.error(`错误堆栈:\n${errorStack}`);
-        
-        logger.info(`========== 消息处理结束（失败） ==========\n`);
+        logger.error(
+          `❌ 入站处理失败 (${processedCount}/${receivedCount}): ${error?.message || "未知错误"}`,
+        );
+        if (error?.stack) logger.debug(error.stack);
+        if (!acked) {
+          logger.warn(
+            `⚠️ 未 ACK messageId=${messageId || "N/A"}，等待钉钉重投`,
+          );
+        }
       } finally {
-        // ✅ 无论成功或失败，都要标记消息处理结束
         markMessageProcessingEnd();
       }
     });
@@ -637,20 +914,27 @@ export async function monitorSingleAccount(
 
     // Connect to DingTalk Stream
     try {
-      await client.connect();
+      // 注意：registerCallbackListener 必须在 connect 之前（已在上方完成），
+      // 这样 getEndpoint 会把 ROBOT 回调 topic 写进 subscriptions。
+      //
+      // 关键：禁止 OPEN-only 假 ready。
+      // 未收到 SYSTEM/REGISTERED 时会强制多轮 disconnect+connect，
+      // 直到 registered=true 才 onStatusChange(connected)；否则启动失败。
+      await ensureRegisteredConnection(REGISTERED_CONNECT_MAX_ATTEMPTS);
 
-      // 注册 socket 事件监听器（必须在 connect 后，此时 (client as any).socket 已创建）
-      setupPongListener();
-      setupMessageListener();
-      setupCloseListener();
+      noteSocketAlive("initial-ready");
+      connectionEstablishedTime = Date.now();
 
       logger.info(`Connected to DingTalk Stream successfully`);
       logger.info(`PID: ${process.pid}`);
       logger.info(
-        `✅ 自定义 keepAlive: true (10 秒心跳，90 秒超时), 指数退避重连`,
+        `✅ keepAlive: 心跳 ${HEARTBEAT_INTERVAL / 1000}s / 软超时 ${SOFT_STALE_MS / 1000}s×${HARD_RECONNECT_AFTER_MISSES} / ` +
+          `处理中刷新 ${MESSAGE_PROCESSING_KEEPALIVE_MS / 1000}s / 重连冷却 ${MIN_RECONNECT_GAP_MS / 1000}s, ` +
+          `registered=${Boolean((client as any).registered)}`,
       );
+      printConnectionNoticeOnce();
 
-      // 初次连接成功，向框架报告 connected: true
+      // 仅 REGISTERED 成功后才报 connected
       onStatusChange?.({ connected: true, lastConnectedAt: Date.now() });
 
       // 启动自定义心跳检测