@soimy/dingtalk 3.1.4 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +808 -40
- package/index.ts +62 -0
- package/package.json +4 -2
- package/src/access-control.ts +18 -0
- package/src/ack-reaction-classifier.ts +62 -0
- package/src/ack-reaction-service.ts +135 -0
- package/src/attachment-text-extractor.ts +147 -0
- package/src/card-callback-service.ts +119 -0
- package/src/card-draft-controller.ts +114 -0
- package/src/card-service.ts +794 -62
- package/src/channel.ts +675 -179
- package/src/config-schema.ts +49 -5
- package/src/config.ts +136 -4
- package/src/connection-manager.ts +356 -36
- package/src/dedup.ts +1 -0
- package/src/docs-service.ts +198 -0
- package/src/draft-stream-loop.ts +119 -0
- package/src/feedback-learning-service.ts +643 -0
- package/src/feedback-learning-store.ts +543 -0
- package/src/group-members-store.ts +48 -14
- package/src/inbound-handler.ts +1191 -206
- package/src/learning-command-service.ts +339 -0
- package/src/media-utils.ts +566 -8
- package/src/message-utils.ts +301 -39
- package/src/onboarding.ts +85 -3
- package/src/peer-id-registry.ts +102 -0
- package/src/persistence-store.ts +131 -0
- package/src/quote-journal.ts +242 -0
- package/src/quoted-file-service.ts +385 -0
- package/src/quoted-msg-cache.ts +226 -0
- package/src/send-service.ts +239 -59
- package/src/session-command-service.ts +147 -0
- package/src/session-lock.ts +34 -0
- package/src/session-peer-store.ts +77 -0
- package/src/session-routing.ts +33 -0
- package/src/types.ts +165 -22
- package/src/utils.ts +231 -12
|
@@ -15,9 +15,21 @@ import type {
|
|
|
15
15
|
ConnectionManagerConfig,
|
|
16
16
|
ConnectionAttemptResult,
|
|
17
17
|
Logger,
|
|
18
|
+
StreamClientFactory,
|
|
18
19
|
} from "./types";
|
|
19
20
|
import { ConnectionState as ConnectionStateEnum } from "./types";
|
|
20
21
|
|
|
22
|
+
/**
|
|
23
|
+
* Thrown when a runtime reconnection cycle exceeds the configured deadline.
|
|
24
|
+
* Identified via instanceof in reconnect() to skip cycle counting.
|
|
25
|
+
*/
|
|
26
|
+
export class ReconnectDeadlineError extends Error {
|
|
27
|
+
constructor(options?: ErrorOptions) {
|
|
28
|
+
super("Reconnect deadline exceeded", options);
|
|
29
|
+
this.name = "ReconnectDeadlineError";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
21
33
|
/**
|
|
22
34
|
* ConnectionManager handles the robust connection lifecycle for DWClient
|
|
23
35
|
*/
|
|
@@ -34,14 +46,26 @@ export class ConnectionManager {
|
|
|
34
46
|
private connectedAt?: number;
|
|
35
47
|
private consecutiveUnhealthyChecks: number = 0;
|
|
36
48
|
|
|
37
|
-
private static readonly HEALTH_CHECK_INTERVAL_MS =
|
|
38
|
-
private static readonly HEALTH_CHECK_GRACE_MS =
|
|
49
|
+
private static readonly HEALTH_CHECK_INTERVAL_MS = 60000;
|
|
50
|
+
private static readonly HEALTH_CHECK_GRACE_MS = 30000;
|
|
39
51
|
private static readonly HEALTH_CHECK_UNHEALTHY_THRESHOLD = 2;
|
|
40
52
|
private static readonly DEFAULT_MAX_RECONNECT_CYCLES = 10;
|
|
53
|
+
private static readonly MAX_CYCLE_BACKOFF_MS = 5000;
|
|
54
|
+
private static readonly MAX_CONSECUTIVE_DEADLINE_TIMEOUTS = 5;
|
|
55
|
+
private static readonly HEARTBEAT_INTERVAL_MS = 20_000;
|
|
56
|
+
private static readonly HEARTBEAT_MISS_THRESHOLD = 2;
|
|
41
57
|
private runtimeReconnectCycles: number = 0;
|
|
58
|
+
private reconnectDeadline?: number;
|
|
59
|
+
private consecutiveDeadlineTimeouts: number = 0;
|
|
60
|
+
private lastSocketActivityAt?: number;
|
|
61
|
+
private lastHeartbeatPingAt?: number;
|
|
62
|
+
private consecutiveHeartbeatMisses: number = 0;
|
|
42
63
|
private runtimeCounters = {
|
|
43
64
|
healthUnhealthyChecks: 0,
|
|
44
65
|
healthTriggeredReconnects: 0,
|
|
66
|
+
heartbeatMisses: 0,
|
|
67
|
+
heartbeatTriggeredReconnects: 0,
|
|
68
|
+
serverDisconnectMessages: 0,
|
|
45
69
|
socketCloseEvents: 0,
|
|
46
70
|
runtimeDisconnects: 0,
|
|
47
71
|
reconnectAttempts: 0,
|
|
@@ -51,9 +75,12 @@ export class ConnectionManager {
|
|
|
51
75
|
|
|
52
76
|
// Runtime monitoring resources
|
|
53
77
|
private healthCheckInterval?: NodeJS.Timeout;
|
|
78
|
+
private heartbeatInterval?: NodeJS.Timeout;
|
|
54
79
|
private socketCloseHandler?: (code: number, reason: string) => void;
|
|
55
80
|
private socketErrorHandler?: (error: Error) => void;
|
|
56
|
-
private
|
|
81
|
+
private socketMessageHandler?: (data: any) => void;
|
|
82
|
+
private socketPongHandler?: () => void;
|
|
83
|
+
private monitoredSocket?: any;
|
|
57
84
|
|
|
58
85
|
// Sleep abort control
|
|
59
86
|
private sleepTimeout?: NodeJS.Timeout;
|
|
@@ -65,11 +92,18 @@ export class ConnectionManager {
|
|
|
65
92
|
// Client reference
|
|
66
93
|
private client: DWClient;
|
|
67
94
|
|
|
68
|
-
|
|
95
|
+
// Warm-reconnect: factory to create fresh DWClient instances with listeners
|
|
96
|
+
// already registered so the new socket starts receiving immediately.
|
|
97
|
+
private clientFactory?: StreamClientFactory;
|
|
98
|
+
// Old client pending cleanup after a warm reconnect swap.
|
|
99
|
+
private pendingOldClient?: DWClient;
|
|
100
|
+
|
|
101
|
+
constructor(client: DWClient, accountId: string, config: ConnectionManagerConfig, log?: Logger, clientFactory?: StreamClientFactory) {
|
|
69
102
|
this.client = client;
|
|
70
103
|
this.accountId = accountId;
|
|
71
104
|
this.config = config;
|
|
72
105
|
this.log = log;
|
|
106
|
+
this.clientFactory = clientFactory;
|
|
73
107
|
}
|
|
74
108
|
|
|
75
109
|
private notifyStateChange(error?: string): void {
|
|
@@ -81,10 +115,114 @@ export class ConnectionManager {
|
|
|
81
115
|
private logRuntimeCounters(reason: string): void {
|
|
82
116
|
const c = this.runtimeCounters;
|
|
83
117
|
this.log?.info?.(
|
|
84
|
-
`[${this.accountId}] Runtime counters (${reason}): healthUnhealthyChecks=${c.healthUnhealthyChecks}, healthTriggeredReconnects=${c.healthTriggeredReconnects}, socketCloseEvents=${c.socketCloseEvents}, runtimeDisconnects=${c.runtimeDisconnects}, reconnectAttempts=${c.reconnectAttempts}, reconnectSuccess=${c.reconnectSuccess}, reconnectFailures=${c.reconnectFailures}`,
|
|
118
|
+
`[${this.accountId}] Runtime counters (${reason}): healthUnhealthyChecks=${c.healthUnhealthyChecks}, healthTriggeredReconnects=${c.healthTriggeredReconnects}, heartbeatMisses=${c.heartbeatMisses}, heartbeatTriggeredReconnects=${c.heartbeatTriggeredReconnects}, serverDisconnectMessages=${c.serverDisconnectMessages}, socketCloseEvents=${c.socketCloseEvents}, runtimeDisconnects=${c.runtimeDisconnects}, reconnectAttempts=${c.reconnectAttempts}, reconnectSuccess=${c.reconnectSuccess}, reconnectFailures=${c.reconnectFailures}`,
|
|
85
119
|
);
|
|
86
120
|
}
|
|
87
121
|
|
|
122
|
+
private recordSocketActivity(now: number = Date.now()): void {
|
|
123
|
+
this.lastSocketActivityAt = now;
|
|
124
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private setupHeartbeat(socket: any): void {
|
|
128
|
+
this.cleanupHeartbeatInterval();
|
|
129
|
+
this.heartbeatInterval = setInterval(() => {
|
|
130
|
+
if (this.stopped || this.state !== ConnectionStateEnum.CONNECTED) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (socket.readyState !== 1) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const now = Date.now();
|
|
139
|
+
const idleMs = this.lastSocketActivityAt !== undefined
|
|
140
|
+
? now - this.lastSocketActivityAt
|
|
141
|
+
: ConnectionManager.HEARTBEAT_INTERVAL_MS;
|
|
142
|
+
if (idleMs >= ConnectionManager.HEARTBEAT_INTERVAL_MS) {
|
|
143
|
+
this.consecutiveHeartbeatMisses += 1;
|
|
144
|
+
this.runtimeCounters.heartbeatMisses += 1;
|
|
145
|
+
|
|
146
|
+
if (this.consecutiveHeartbeatMisses >= ConnectionManager.HEARTBEAT_MISS_THRESHOLD) {
|
|
147
|
+
const lastPingAgoMs = this.lastHeartbeatPingAt !== undefined
|
|
148
|
+
? now - this.lastHeartbeatPingAt
|
|
149
|
+
: undefined;
|
|
150
|
+
this.log?.warn?.(
|
|
151
|
+
`[${this.accountId}] Connection heartbeat missed ${this.consecutiveHeartbeatMisses}/${ConnectionManager.HEARTBEAT_MISS_THRESHOLD} checks, triggering reconnection (lastPingAgoMs=${lastPingAgoMs ?? "n/a"})`,
|
|
152
|
+
);
|
|
153
|
+
this.runtimeCounters.heartbeatTriggeredReconnects += 1;
|
|
154
|
+
this.logRuntimeCounters("heartbeat-triggered-reconnect");
|
|
155
|
+
this.cleanupHeartbeatInterval();
|
|
156
|
+
this.handleRuntimeDisconnection();
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
this.log?.debug?.(
|
|
161
|
+
`[${this.accountId}] Connection heartbeat missed (${this.consecutiveHeartbeatMisses}/${ConnectionManager.HEARTBEAT_MISS_THRESHOLD})`,
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
this.lastHeartbeatPingAt = now;
|
|
166
|
+
try {
|
|
167
|
+
socket.ping("", true);
|
|
168
|
+
} catch (err: any) {
|
|
169
|
+
this.log?.warn?.(
|
|
170
|
+
`[${this.accountId}] Failed to send heartbeat ping: ${err.message}`,
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
}, ConnectionManager.HEARTBEAT_INTERVAL_MS);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private cleanupHeartbeatInterval(): void {
|
|
177
|
+
if (this.heartbeatInterval) {
|
|
178
|
+
clearInterval(this.heartbeatInterval);
|
|
179
|
+
this.heartbeatInterval = undefined;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private clearClientHeartbeatInterval(client: DWClient): void {
|
|
184
|
+
const clientAny = client as any;
|
|
185
|
+
if (clientAny.heartbeatIntervallId !== undefined) {
|
|
186
|
+
clearInterval(clientAny.heartbeatIntervallId);
|
|
187
|
+
clientAny.heartbeatIntervallId = undefined;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private async waitForSocketOpen(clientAny: any): Promise<void> {
|
|
192
|
+
const socket = clientAny.socket;
|
|
193
|
+
if (!socket) {
|
|
194
|
+
throw new Error("Socket unavailable after connect");
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (socket.readyState === 1 || clientAny.connected) {
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const defaultOpenTimeout = 10_000;
|
|
202
|
+
const openTimeout = this.reconnectDeadline !== undefined
|
|
203
|
+
? Math.min(defaultOpenTimeout, Math.max(1000, this.reconnectDeadline - Date.now()))
|
|
204
|
+
: defaultOpenTimeout;
|
|
205
|
+
|
|
206
|
+
await new Promise<void>((resolve, reject) => {
|
|
207
|
+
const timeout = setTimeout(() => {
|
|
208
|
+
cleanup();
|
|
209
|
+
reject(new Error("Socket open timeout"));
|
|
210
|
+
}, openTimeout);
|
|
211
|
+
const cleanup = () => {
|
|
212
|
+
clearTimeout(timeout);
|
|
213
|
+
socket.removeListener("open", onOpen);
|
|
214
|
+
socket.removeListener("error", onError);
|
|
215
|
+
socket.removeListener("close", onClose);
|
|
216
|
+
};
|
|
217
|
+
const onOpen = () => { cleanup(); resolve(); };
|
|
218
|
+
const onError = (err: Error) => { cleanup(); reject(err); };
|
|
219
|
+
const onClose = () => { cleanup(); reject(new Error("Socket closed before open")); };
|
|
220
|
+
socket.once("open", onOpen);
|
|
221
|
+
socket.once("error", onError);
|
|
222
|
+
socket.once("close", onClose);
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
|
|
88
226
|
/**
|
|
89
227
|
* Calculate next reconnection delay with exponential backoff and jitter
|
|
90
228
|
* Formula: delay = min(initialDelay * 2^attempt, maxDelay) * (1 ± jitter)
|
|
@@ -129,11 +267,52 @@ export class ConnectionManager {
|
|
|
129
267
|
);
|
|
130
268
|
|
|
131
269
|
try {
|
|
132
|
-
//
|
|
270
|
+
// Warm-reconnect path: create a fresh DWClient so the new WebSocket
|
|
271
|
+
// can start receiving messages while the old zombie socket is cleaned
|
|
272
|
+
// up asynchronously. This minimizes the message-loss window during
|
|
273
|
+
// server-initiated disconnects (DingTalk robot msgs are fire-and-forget).
|
|
274
|
+
if (this.clientFactory) {
|
|
275
|
+
const oldClient = this.client;
|
|
276
|
+
this.pendingOldClient = oldClient;
|
|
277
|
+
try {
|
|
278
|
+
this.client = this.clientFactory();
|
|
279
|
+
this.log?.info?.(
|
|
280
|
+
`[${this.accountId}] Warm reconnect: created fresh DWClient, connecting new socket while old socket is cleaned up`,
|
|
281
|
+
);
|
|
282
|
+
} catch (factoryErr: any) {
|
|
283
|
+
this.log?.warn?.(
|
|
284
|
+
`[${this.accountId}] Client factory failed, falling back to same-client reconnect: ${factoryErr.message}`,
|
|
285
|
+
);
|
|
286
|
+
this.client = oldClient;
|
|
287
|
+
this.pendingOldClient = undefined;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if (!this.pendingOldClient) {
|
|
292
|
+
// Legacy single-client reconnect: disconnect before reconnecting.
|
|
293
|
+
try {
|
|
294
|
+
this.client.disconnect();
|
|
295
|
+
} catch (disconnectErr: any) {
|
|
296
|
+
this.log?.debug?.(
|
|
297
|
+
`[${this.accountId}] pre-connect cleanup disconnect failed: ${disconnectErr.message}`,
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// SDK _connect() resolves before socket "open" fires. If disconnect() runs
|
|
303
|
+
// before "open", heartbeatIntervallId is still undefined and clearInterval
|
|
304
|
+
// is a no-op. The deferred "open" handler then creates an interval that
|
|
305
|
+
// outlives the old socket and can terminate the next connection.
|
|
306
|
+
// For the race where interval is created AFTER this cleanup, the socket
|
|
307
|
+
// open timeout below serves as the final safety net.
|
|
308
|
+
// Field name "heartbeatIntervallId" (double-l typo) from dingtalk-stream
|
|
309
|
+
// SDK DWClient._connect() open handler (verified in SDK v1.x).
|
|
310
|
+
const clientAny = this.client as any;
|
|
311
|
+
this.clearClientHeartbeatInterval(this.client);
|
|
312
|
+
|
|
133
313
|
await this.client.connect();
|
|
314
|
+
await this.waitForSocketOpen(clientAny);
|
|
134
315
|
|
|
135
|
-
// Re-check stopped flag after async connect() completes
|
|
136
|
-
// This prevents race condition where stop() is called during connection
|
|
137
316
|
if (this.stopped) {
|
|
138
317
|
this.log?.warn?.(
|
|
139
318
|
`[${this.accountId}] Connection succeeded but manager was stopped during connect - disconnecting`,
|
|
@@ -152,21 +331,41 @@ export class ConnectionManager {
|
|
|
152
331
|
};
|
|
153
332
|
}
|
|
154
333
|
|
|
155
|
-
// Connection successful
|
|
156
334
|
this.state = ConnectionStateEnum.CONNECTED;
|
|
157
335
|
this.connectedAt = Date.now();
|
|
336
|
+
this.recordSocketActivity(this.connectedAt);
|
|
337
|
+
this.lastHeartbeatPingAt = undefined;
|
|
338
|
+
this.reconnectDeadline = undefined;
|
|
158
339
|
this.consecutiveUnhealthyChecks = 0;
|
|
340
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
159
341
|
this.notifyStateChange();
|
|
160
342
|
const successfulAttempt = this.attemptCount;
|
|
161
|
-
this.attemptCount = 0;
|
|
343
|
+
this.attemptCount = 0;
|
|
162
344
|
|
|
163
345
|
this.log?.info?.(`[${this.accountId}] DingTalk Stream client connected successfully`);
|
|
164
346
|
|
|
165
|
-
// Reset runtime reconnect cycle counter on successful connection
|
|
166
347
|
this.runtimeReconnectCycles = 0;
|
|
348
|
+
this.consecutiveDeadlineTimeouts = 0;
|
|
349
|
+
// Setup monitoring BEFORE cleaning up old client: setupRuntimeReconnection
|
|
350
|
+
// calls cleanupRuntimeMonitoring() which removes the old socket's event
|
|
351
|
+
// listeners, preventing the old socket's close event (triggered by
|
|
352
|
+
// disconnect below) from erroneously firing handleRuntimeDisconnection.
|
|
353
|
+
this.setupRuntimeReconnection();
|
|
354
|
+
this.cleanupPendingOldClient();
|
|
167
355
|
|
|
168
356
|
return { success: true, attempt: successfulAttempt };
|
|
169
357
|
} catch (err: any) {
|
|
358
|
+
// Warm-reconnect failed with new client: revert to old client so
|
|
359
|
+
// subsequent retry attempts don't keep creating throwaway instances.
|
|
360
|
+
if (this.pendingOldClient) {
|
|
361
|
+
try { this.client.disconnect(); } catch { /* best-effort cleanup of failed new client */ }
|
|
362
|
+
this.client = this.pendingOldClient;
|
|
363
|
+
this.pendingOldClient = undefined;
|
|
364
|
+
this.log?.debug?.(
|
|
365
|
+
`[${this.accountId}] Warm reconnect failed, reverted to previous client for next retry`,
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
|
|
170
369
|
this.log?.error?.(
|
|
171
370
|
`[${this.accountId}] Connection attempt ${this.attemptCount} failed: ${err.message}`,
|
|
172
371
|
);
|
|
@@ -208,17 +407,18 @@ export class ConnectionManager {
|
|
|
208
407
|
`[${this.accountId}] Starting DingTalk Stream client with robust connection...`,
|
|
209
408
|
);
|
|
210
409
|
|
|
211
|
-
// Keep trying until success or max attempts reached
|
|
212
410
|
while (!this.stopped && this.state !== ConnectionStateEnum.CONNECTED) {
|
|
411
|
+
if (this.reconnectDeadline !== undefined && Date.now() >= this.reconnectDeadline) {
|
|
412
|
+
this.reconnectDeadline = undefined;
|
|
413
|
+
throw new ReconnectDeadlineError();
|
|
414
|
+
}
|
|
415
|
+
|
|
213
416
|
const result = await this.attemptConnection();
|
|
214
417
|
|
|
215
418
|
if (result.success) {
|
|
216
|
-
// Connection successful
|
|
217
|
-
this.setupRuntimeReconnection();
|
|
218
419
|
return;
|
|
219
420
|
}
|
|
220
421
|
|
|
221
|
-
// Check if connection was stopped during connect
|
|
222
422
|
if (result.error?.message === "Connection manager stopped during connect") {
|
|
223
423
|
this.log?.info?.(
|
|
224
424
|
`[${this.accountId}] Connection cancelled: manager stopped during connect`,
|
|
@@ -227,12 +427,17 @@ export class ConnectionManager {
|
|
|
227
427
|
}
|
|
228
428
|
|
|
229
429
|
if (!result.nextDelay || this.attemptCount >= this.config.maxAttempts) {
|
|
230
|
-
// No more retries
|
|
231
430
|
throw new Error(`Failed to connect after ${this.attemptCount} attempts`);
|
|
232
431
|
}
|
|
233
432
|
|
|
234
|
-
//
|
|
235
|
-
|
|
433
|
+
// Truncate sleep to remaining deadline budget
|
|
434
|
+
let actualDelay = result.nextDelay;
|
|
435
|
+
if (this.reconnectDeadline !== undefined) {
|
|
436
|
+
const remaining = Math.max(0, this.reconnectDeadline - Date.now());
|
|
437
|
+
actualDelay = Math.min(actualDelay, remaining);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
await this.sleep(actualDelay);
|
|
236
441
|
}
|
|
237
442
|
}
|
|
238
443
|
|
|
@@ -275,7 +480,15 @@ export class ConnectionManager {
|
|
|
275
480
|
|
|
276
481
|
const socketReadyState = (client.socket as { readyState?: number } | undefined)?.readyState;
|
|
277
482
|
const socketOpen = socketReadyState === 1;
|
|
278
|
-
const
|
|
483
|
+
const registered = client.registered as boolean | undefined;
|
|
484
|
+
|
|
485
|
+
// Unhealthy if:
|
|
486
|
+
// 1. Not connected AND socket not open (full disconnect)
|
|
487
|
+
// 2. Socket is open, client.connected is false, AND not registered
|
|
488
|
+
// (server sent logical "disconnect" and the socket is now a zombie)
|
|
489
|
+
const unhealthy =
|
|
490
|
+
(!client.connected && !socketOpen) ||
|
|
491
|
+
(socketOpen && !client.connected && registered === false);
|
|
279
492
|
|
|
280
493
|
if (!unhealthy) {
|
|
281
494
|
this.consecutiveUnhealthyChecks = 0;
|
|
@@ -289,7 +502,7 @@ export class ConnectionManager {
|
|
|
289
502
|
ConnectionManager.HEALTH_CHECK_UNHEALTHY_THRESHOLD
|
|
290
503
|
) {
|
|
291
504
|
this.log?.debug?.(
|
|
292
|
-
`[${this.accountId}] Connection health check unhealthy (${this.consecutiveUnhealthyChecks}/${ConnectionManager.HEALTH_CHECK_UNHEALTHY_THRESHOLD}) connected=${String(client.connected)} socketReadyState=${socketReadyState ?? "unknown"}`,
|
|
505
|
+
`[${this.accountId}] Connection health check unhealthy (${this.consecutiveUnhealthyChecks}/${ConnectionManager.HEALTH_CHECK_UNHEALTHY_THRESHOLD}) connected=${String(client.connected)} registered=${String(registered)} socketReadyState=${socketReadyState ?? "unknown"}`,
|
|
293
506
|
);
|
|
294
507
|
return;
|
|
295
508
|
}
|
|
@@ -311,6 +524,7 @@ export class ConnectionManager {
|
|
|
311
524
|
const socket = client.socket;
|
|
312
525
|
// Store the socket instance we're attaching listeners to
|
|
313
526
|
this.monitoredSocket = socket;
|
|
527
|
+
this.setupHeartbeat(socket);
|
|
314
528
|
|
|
315
529
|
// Handler for socket close event
|
|
316
530
|
this.socketCloseHandler = (code: number, reason: string) => {
|
|
@@ -336,11 +550,61 @@ export class ConnectionManager {
|
|
|
336
550
|
);
|
|
337
551
|
};
|
|
338
552
|
|
|
339
|
-
// Listen to socket events
|
|
340
|
-
// Use 'once' for close to avoid duplicate reconnection triggers
|
|
341
553
|
socket.once("close", this.socketCloseHandler);
|
|
342
|
-
// Use 'once' for error as well to prevent accumulation across reconnects
|
|
343
554
|
socket.once("error", this.socketErrorHandler);
|
|
555
|
+
|
|
556
|
+
// Monitor for server-side disconnect system messages and passive socket
|
|
557
|
+
// activity.
|
|
558
|
+
this.socketMessageHandler = (data: any) => {
|
|
559
|
+
this.recordSocketActivity();
|
|
560
|
+
try {
|
|
561
|
+
const msg = JSON.parse(typeof data === "string" ? data : data.toString());
|
|
562
|
+
if (msg?.type === "SYSTEM" && msg?.headers?.topic === "disconnect") {
|
|
563
|
+
this.runtimeCounters.serverDisconnectMessages += 1;
|
|
564
|
+
this.log?.warn?.(
|
|
565
|
+
`[${this.accountId}] Server disconnect system message received, triggering immediate reconnection`,
|
|
566
|
+
);
|
|
567
|
+
this.logRuntimeCounters("server-disconnect");
|
|
568
|
+
if (this.healthCheckInterval) {
|
|
569
|
+
clearInterval(this.healthCheckInterval);
|
|
570
|
+
}
|
|
571
|
+
this.handleRuntimeDisconnection();
|
|
572
|
+
}
|
|
573
|
+
} catch {
|
|
574
|
+
// Ignore parse errors — other handlers will process the message
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
socket.on("message", this.socketMessageHandler);
|
|
578
|
+
|
|
579
|
+
this.socketPongHandler = () => {
|
|
580
|
+
this.recordSocketActivity();
|
|
581
|
+
};
|
|
582
|
+
socket.on("pong", this.socketPongHandler);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Disconnect and release the old client left over from a warm-reconnect swap.
|
|
588
|
+
*/
|
|
589
|
+
private cleanupPendingOldClient(): void {
|
|
590
|
+
if (!this.pendingOldClient) {
|
|
591
|
+
return;
|
|
592
|
+
}
|
|
593
|
+
const old = this.pendingOldClient;
|
|
594
|
+
this.pendingOldClient = undefined;
|
|
595
|
+
try {
|
|
596
|
+
// Clear stale heartbeat interval from old SDK instance.
|
|
597
|
+
const oldAny = old as any;
|
|
598
|
+
if (oldAny.heartbeatIntervallId !== undefined) {
|
|
599
|
+
clearInterval(oldAny.heartbeatIntervallId);
|
|
600
|
+
oldAny.heartbeatIntervallId = undefined;
|
|
601
|
+
}
|
|
602
|
+
old.disconnect();
|
|
603
|
+
this.log?.debug?.(`[${this.accountId}] Warm reconnect: old client disconnected`);
|
|
604
|
+
} catch (err: any) {
|
|
605
|
+
this.log?.debug?.(
|
|
606
|
+
`[${this.accountId}] Warm reconnect: old client cleanup failed: ${err.message}`,
|
|
607
|
+
);
|
|
344
608
|
}
|
|
345
609
|
}
|
|
346
610
|
|
|
@@ -354,6 +618,7 @@ export class ConnectionManager {
|
|
|
354
618
|
this.healthCheckInterval = undefined;
|
|
355
619
|
this.log?.debug?.(`[${this.accountId}] Health check interval cleared`);
|
|
356
620
|
}
|
|
621
|
+
this.cleanupHeartbeatInterval();
|
|
357
622
|
|
|
358
623
|
// Remove socket event listeners from the stored socket instance
|
|
359
624
|
if (this.monitoredSocket) {
|
|
@@ -367,6 +632,14 @@ export class ConnectionManager {
|
|
|
367
632
|
socket.removeListener("error", this.socketErrorHandler);
|
|
368
633
|
this.socketErrorHandler = undefined;
|
|
369
634
|
}
|
|
635
|
+
if (this.socketMessageHandler) {
|
|
636
|
+
socket.removeListener("message", this.socketMessageHandler);
|
|
637
|
+
this.socketMessageHandler = undefined;
|
|
638
|
+
}
|
|
639
|
+
if (this.socketPongHandler) {
|
|
640
|
+
socket.removeListener("pong", this.socketPongHandler);
|
|
641
|
+
this.socketPongHandler = undefined;
|
|
642
|
+
}
|
|
370
643
|
|
|
371
644
|
this.log?.debug?.(`[${this.accountId}] Socket event listeners removed from monitored socket`);
|
|
372
645
|
this.monitoredSocket = undefined;
|
|
@@ -377,35 +650,37 @@ export class ConnectionManager {
|
|
|
377
650
|
* Handle runtime disconnection and trigger reconnection
|
|
378
651
|
*/
|
|
379
652
|
private handleRuntimeDisconnection(): void {
|
|
380
|
-
if (this.stopped) {
|
|
653
|
+
if (this.stopped || this.state !== ConnectionStateEnum.CONNECTED) {
|
|
381
654
|
return;
|
|
382
655
|
}
|
|
383
656
|
|
|
657
|
+
this.state = ConnectionStateEnum.DISCONNECTED;
|
|
658
|
+
|
|
384
659
|
this.log?.warn?.(
|
|
385
660
|
`[${this.accountId}] Runtime disconnection detected, initiating reconnection...`,
|
|
386
661
|
);
|
|
387
662
|
this.runtimeCounters.runtimeDisconnects += 1;
|
|
388
663
|
|
|
389
|
-
this.state = ConnectionStateEnum.DISCONNECTED;
|
|
390
664
|
this.notifyStateChange("Runtime disconnection detected");
|
|
391
|
-
this.attemptCount = 0;
|
|
665
|
+
this.attemptCount = 0;
|
|
392
666
|
this.connectedAt = undefined;
|
|
667
|
+
this.lastSocketActivityAt = undefined;
|
|
668
|
+
this.lastHeartbeatPingAt = undefined;
|
|
393
669
|
this.consecutiveUnhealthyChecks = 0;
|
|
670
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
671
|
+
|
|
672
|
+
const deadlineMs = this.config.reconnectDeadlineMs ?? 50000;
|
|
673
|
+
this.reconnectDeadline = Date.now() + deadlineMs;
|
|
394
674
|
|
|
395
|
-
// Clear any existing timer
|
|
396
675
|
this.clearReconnectTimer();
|
|
397
676
|
|
|
398
|
-
|
|
399
|
-
const delay = this.calculateNextDelay(0);
|
|
400
|
-
this.log?.info?.(
|
|
401
|
-
`[${this.accountId}] Scheduling reconnection in ${(delay / 1000).toFixed(2)}s`,
|
|
402
|
-
);
|
|
677
|
+
this.log?.info?.(`[${this.accountId}] Scheduling immediate reconnection`);
|
|
403
678
|
|
|
404
679
|
this.reconnectTimer = setTimeout(() => {
|
|
405
680
|
this.reconnect().catch((err) => {
|
|
406
681
|
this.log?.error?.(`[${this.accountId}] Reconnection failed: ${err.message}`);
|
|
407
682
|
});
|
|
408
|
-
},
|
|
683
|
+
}, 0);
|
|
409
684
|
}
|
|
410
685
|
|
|
411
686
|
/**
|
|
@@ -433,7 +708,40 @@ export class ConnectionManager {
|
|
|
433
708
|
this.runtimeCounters.reconnectFailures += 1;
|
|
434
709
|
this.logRuntimeCounters("reconnect-failed");
|
|
435
710
|
|
|
436
|
-
|
|
711
|
+
if (err instanceof ReconnectDeadlineError) {
|
|
712
|
+
this.consecutiveDeadlineTimeouts += 1;
|
|
713
|
+
|
|
714
|
+
if (this.consecutiveDeadlineTimeouts >= ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS) {
|
|
715
|
+
this.log?.error?.(
|
|
716
|
+
`[${this.accountId}] Max consecutive deadline timeouts (${ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS}) reached. Giving up.`,
|
|
717
|
+
);
|
|
718
|
+
this.state = ConnectionStateEnum.FAILED;
|
|
719
|
+
this.connectedAt = undefined;
|
|
720
|
+
this.consecutiveUnhealthyChecks = 0;
|
|
721
|
+
this.reconnectDeadline = undefined;
|
|
722
|
+
this.notifyStateChange(
|
|
723
|
+
`Max consecutive deadline timeouts (${ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS}) reached`,
|
|
724
|
+
);
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
const deadlineMs = this.config.reconnectDeadlineMs ?? 50000;
|
|
729
|
+
this.reconnectDeadline = Date.now() + deadlineMs;
|
|
730
|
+
const delay = Math.min(
|
|
731
|
+
this.calculateNextDelay(0),
|
|
732
|
+
ConnectionManager.MAX_CYCLE_BACKOFF_MS,
|
|
733
|
+
);
|
|
734
|
+
this.attemptCount = 0;
|
|
735
|
+
this.clearReconnectTimer();
|
|
736
|
+
this.log?.warn?.(
|
|
737
|
+
`[${this.accountId}] Reconnect deadline exceeded (${this.consecutiveDeadlineTimeouts}/${ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS}); scheduling next cycle in ${(delay / 1000).toFixed(2)}s`,
|
|
738
|
+
);
|
|
739
|
+
this.reconnectTimer = setTimeout(() => {
|
|
740
|
+
void this.reconnect();
|
|
741
|
+
}, delay);
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
|
|
437
745
|
this.runtimeReconnectCycles += 1;
|
|
438
746
|
const maxCycles = this.config.maxReconnectCycles ?? ConnectionManager.DEFAULT_MAX_RECONNECT_CYCLES;
|
|
439
747
|
|
|
@@ -445,6 +753,7 @@ export class ConnectionManager {
|
|
|
445
753
|
this.state = ConnectionStateEnum.FAILED;
|
|
446
754
|
this.connectedAt = undefined;
|
|
447
755
|
this.consecutiveUnhealthyChecks = 0;
|
|
756
|
+
this.reconnectDeadline = undefined;
|
|
448
757
|
this.notifyStateChange(`Max runtime reconnect cycles (${maxCycles}) reached`);
|
|
449
758
|
return;
|
|
450
759
|
}
|
|
@@ -454,8 +763,11 @@ export class ConnectionManager {
|
|
|
454
763
|
this.consecutiveUnhealthyChecks = 0;
|
|
455
764
|
this.notifyStateChange(err.message);
|
|
456
765
|
|
|
457
|
-
|
|
458
|
-
const delay =
|
|
766
|
+
const rawDelay = this.calculateNextDelay(Math.min(this.runtimeReconnectCycles - 1, 6));
|
|
767
|
+
const delay = Math.min(rawDelay, ConnectionManager.MAX_CYCLE_BACKOFF_MS);
|
|
768
|
+
// Each cycle gets its own deadline so long-running retries don't block indefinitely
|
|
769
|
+
const deadlineMs = this.config.reconnectDeadlineMs ?? 50000;
|
|
770
|
+
this.reconnectDeadline = Date.now() + deadlineMs;
|
|
459
771
|
this.attemptCount = 0;
|
|
460
772
|
this.clearReconnectTimer();
|
|
461
773
|
this.log?.warn?.(
|
|
@@ -480,7 +792,12 @@ export class ConnectionManager {
|
|
|
480
792
|
this.stopped = true;
|
|
481
793
|
this.state = ConnectionStateEnum.DISCONNECTING;
|
|
482
794
|
this.connectedAt = undefined;
|
|
795
|
+
this.reconnectDeadline = undefined;
|
|
796
|
+
this.consecutiveDeadlineTimeouts = 0;
|
|
483
797
|
this.consecutiveUnhealthyChecks = 0;
|
|
798
|
+
this.lastSocketActivityAt = undefined;
|
|
799
|
+
this.lastHeartbeatPingAt = undefined;
|
|
800
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
484
801
|
|
|
485
802
|
// Clear reconnect timer
|
|
486
803
|
this.clearReconnectTimer();
|
|
@@ -491,6 +808,9 @@ export class ConnectionManager {
|
|
|
491
808
|
// Clean up runtime monitoring resources
|
|
492
809
|
this.cleanupRuntimeMonitoring();
|
|
493
810
|
|
|
811
|
+
// Clean up any pending old client from warm-reconnect swap.
|
|
812
|
+
this.cleanupPendingOldClient();
|
|
813
|
+
|
|
494
814
|
// Disconnect client
|
|
495
815
|
try {
|
|
496
816
|
this.client.disconnect();
|
package/src/dedup.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// ============ Message Deduplication ============
|
|
2
2
|
// Prevent duplicate processing when DingTalk retries delivery.
|
|
3
3
|
// In-memory TTL map + lazy cleanup keeps overhead small.
|
|
4
|
+
export const DEDUP_NAMESPACE_POLICY = "memory-only" as const;
|
|
4
5
|
|
|
5
6
|
const processedMessages = new Map<string, number>();
|
|
6
7
|
const MESSAGE_DEDUP_TTL = 60000;
|