@soimy/dingtalk 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +657 -32
- package/index.ts +62 -0
- package/package.json +3 -1
- package/src/access-control.ts +18 -0
- package/src/ack-reaction-classifier.ts +62 -0
- package/src/ack-reaction-service.ts +135 -0
- package/src/attachment-text-extractor.ts +147 -0
- package/src/card-callback-service.ts +119 -0
- package/src/card-draft-controller.ts +114 -0
- package/src/card-service.ts +776 -24
- package/src/channel.ts +408 -135
- package/src/config-schema.ts +40 -4
- package/src/config.ts +136 -4
- package/src/connection-manager.ts +354 -47
- package/src/dedup.ts +1 -0
- package/src/docs-service.ts +198 -0
- package/src/draft-stream-loop.ts +119 -0
- package/src/feedback-learning-service.ts +643 -0
- package/src/feedback-learning-store.ts +543 -0
- package/src/group-members-store.ts +48 -14
- package/src/inbound-handler.ts +1100 -159
- package/src/learning-command-service.ts +339 -0
- package/src/media-utils.ts +94 -50
- package/src/message-utils.ts +301 -39
- package/src/onboarding.ts +38 -0
- package/src/peer-id-registry.ts +102 -0
- package/src/persistence-store.ts +131 -0
- package/src/quote-journal.ts +242 -0
- package/src/quoted-file-service.ts +385 -0
- package/src/quoted-msg-cache.ts +226 -0
- package/src/send-service.ts +177 -43
- package/src/session-command-service.ts +147 -0
- package/src/session-lock.ts +2 -0
- package/src/session-peer-store.ts +77 -0
- package/src/session-routing.ts +33 -0
- package/src/types.ts +152 -24
- package/src/utils.ts +231 -12
|
@@ -15,9 +15,21 @@ import type {
|
|
|
15
15
|
ConnectionManagerConfig,
|
|
16
16
|
ConnectionAttemptResult,
|
|
17
17
|
Logger,
|
|
18
|
+
StreamClientFactory,
|
|
18
19
|
} from "./types";
|
|
19
20
|
import { ConnectionState as ConnectionStateEnum } from "./types";
|
|
20
21
|
|
|
22
|
+
/**
|
|
23
|
+
* Thrown when a runtime reconnection cycle exceeds the configured deadline.
|
|
24
|
+
* Identified via instanceof in reconnect() to skip cycle counting.
|
|
25
|
+
*/
|
|
26
|
+
export class ReconnectDeadlineError extends Error {
|
|
27
|
+
constructor(options?: ErrorOptions) {
|
|
28
|
+
super("Reconnect deadline exceeded", options);
|
|
29
|
+
this.name = "ReconnectDeadlineError";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
21
33
|
/**
|
|
22
34
|
* ConnectionManager handles the robust connection lifecycle for DWClient
|
|
23
35
|
*/
|
|
@@ -34,14 +46,26 @@ export class ConnectionManager {
|
|
|
34
46
|
private connectedAt?: number;
|
|
35
47
|
private consecutiveUnhealthyChecks: number = 0;
|
|
36
48
|
|
|
37
|
-
private static readonly HEALTH_CHECK_INTERVAL_MS =
|
|
38
|
-
private static readonly HEALTH_CHECK_GRACE_MS =
|
|
49
|
+
private static readonly HEALTH_CHECK_INTERVAL_MS = 60000;
|
|
50
|
+
private static readonly HEALTH_CHECK_GRACE_MS = 30000;
|
|
39
51
|
private static readonly HEALTH_CHECK_UNHEALTHY_THRESHOLD = 2;
|
|
40
52
|
private static readonly DEFAULT_MAX_RECONNECT_CYCLES = 10;
|
|
53
|
+
private static readonly MAX_CYCLE_BACKOFF_MS = 5000;
|
|
54
|
+
private static readonly MAX_CONSECUTIVE_DEADLINE_TIMEOUTS = 5;
|
|
55
|
+
private static readonly HEARTBEAT_INTERVAL_MS = 20_000;
|
|
56
|
+
private static readonly HEARTBEAT_MISS_THRESHOLD = 2;
|
|
41
57
|
private runtimeReconnectCycles: number = 0;
|
|
58
|
+
private reconnectDeadline?: number;
|
|
59
|
+
private consecutiveDeadlineTimeouts: number = 0;
|
|
60
|
+
private lastSocketActivityAt?: number;
|
|
61
|
+
private lastHeartbeatPingAt?: number;
|
|
62
|
+
private consecutiveHeartbeatMisses: number = 0;
|
|
42
63
|
private runtimeCounters = {
|
|
43
64
|
healthUnhealthyChecks: 0,
|
|
44
65
|
healthTriggeredReconnects: 0,
|
|
66
|
+
heartbeatMisses: 0,
|
|
67
|
+
heartbeatTriggeredReconnects: 0,
|
|
68
|
+
serverDisconnectMessages: 0,
|
|
45
69
|
socketCloseEvents: 0,
|
|
46
70
|
runtimeDisconnects: 0,
|
|
47
71
|
reconnectAttempts: 0,
|
|
@@ -51,9 +75,12 @@ export class ConnectionManager {
|
|
|
51
75
|
|
|
52
76
|
// Runtime monitoring resources
|
|
53
77
|
private healthCheckInterval?: NodeJS.Timeout;
|
|
78
|
+
private heartbeatInterval?: NodeJS.Timeout;
|
|
54
79
|
private socketCloseHandler?: (code: number, reason: string) => void;
|
|
55
80
|
private socketErrorHandler?: (error: Error) => void;
|
|
56
|
-
private
|
|
81
|
+
private socketMessageHandler?: (data: any) => void;
|
|
82
|
+
private socketPongHandler?: () => void;
|
|
83
|
+
private monitoredSocket?: any;
|
|
57
84
|
|
|
58
85
|
// Sleep abort control
|
|
59
86
|
private sleepTimeout?: NodeJS.Timeout;
|
|
@@ -65,11 +92,18 @@ export class ConnectionManager {
|
|
|
65
92
|
// Client reference
|
|
66
93
|
private client: DWClient;
|
|
67
94
|
|
|
68
|
-
|
|
95
|
+
// Warm-reconnect: factory to create fresh DWClient instances with listeners
|
|
96
|
+
// already registered so the new socket starts receiving immediately.
|
|
97
|
+
private clientFactory?: StreamClientFactory;
|
|
98
|
+
// Old client pending cleanup after a warm reconnect swap.
|
|
99
|
+
private pendingOldClient?: DWClient;
|
|
100
|
+
|
|
101
|
+
constructor(client: DWClient, accountId: string, config: ConnectionManagerConfig, log?: Logger, clientFactory?: StreamClientFactory) {
|
|
69
102
|
this.client = client;
|
|
70
103
|
this.accountId = accountId;
|
|
71
104
|
this.config = config;
|
|
72
105
|
this.log = log;
|
|
106
|
+
this.clientFactory = clientFactory;
|
|
73
107
|
}
|
|
74
108
|
|
|
75
109
|
private notifyStateChange(error?: string): void {
|
|
@@ -81,10 +115,114 @@ export class ConnectionManager {
|
|
|
81
115
|
private logRuntimeCounters(reason: string): void {
|
|
82
116
|
const c = this.runtimeCounters;
|
|
83
117
|
this.log?.info?.(
|
|
84
|
-
`[${this.accountId}] Runtime counters (${reason}): healthUnhealthyChecks=${c.healthUnhealthyChecks}, healthTriggeredReconnects=${c.healthTriggeredReconnects}, socketCloseEvents=${c.socketCloseEvents}, runtimeDisconnects=${c.runtimeDisconnects}, reconnectAttempts=${c.reconnectAttempts}, reconnectSuccess=${c.reconnectSuccess}, reconnectFailures=${c.reconnectFailures}`,
|
|
118
|
+
`[${this.accountId}] Runtime counters (${reason}): healthUnhealthyChecks=${c.healthUnhealthyChecks}, healthTriggeredReconnects=${c.healthTriggeredReconnects}, heartbeatMisses=${c.heartbeatMisses}, heartbeatTriggeredReconnects=${c.heartbeatTriggeredReconnects}, serverDisconnectMessages=${c.serverDisconnectMessages}, socketCloseEvents=${c.socketCloseEvents}, runtimeDisconnects=${c.runtimeDisconnects}, reconnectAttempts=${c.reconnectAttempts}, reconnectSuccess=${c.reconnectSuccess}, reconnectFailures=${c.reconnectFailures}`,
|
|
85
119
|
);
|
|
86
120
|
}
|
|
87
121
|
|
|
122
|
+
private recordSocketActivity(now: number = Date.now()): void {
|
|
123
|
+
this.lastSocketActivityAt = now;
|
|
124
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private setupHeartbeat(socket: any): void {
|
|
128
|
+
this.cleanupHeartbeatInterval();
|
|
129
|
+
this.heartbeatInterval = setInterval(() => {
|
|
130
|
+
if (this.stopped || this.state !== ConnectionStateEnum.CONNECTED) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (socket.readyState !== 1) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const now = Date.now();
|
|
139
|
+
const idleMs = this.lastSocketActivityAt !== undefined
|
|
140
|
+
? now - this.lastSocketActivityAt
|
|
141
|
+
: ConnectionManager.HEARTBEAT_INTERVAL_MS;
|
|
142
|
+
if (idleMs >= ConnectionManager.HEARTBEAT_INTERVAL_MS) {
|
|
143
|
+
this.consecutiveHeartbeatMisses += 1;
|
|
144
|
+
this.runtimeCounters.heartbeatMisses += 1;
|
|
145
|
+
|
|
146
|
+
if (this.consecutiveHeartbeatMisses >= ConnectionManager.HEARTBEAT_MISS_THRESHOLD) {
|
|
147
|
+
const lastPingAgoMs = this.lastHeartbeatPingAt !== undefined
|
|
148
|
+
? now - this.lastHeartbeatPingAt
|
|
149
|
+
: undefined;
|
|
150
|
+
this.log?.warn?.(
|
|
151
|
+
`[${this.accountId}] Connection heartbeat missed ${this.consecutiveHeartbeatMisses}/${ConnectionManager.HEARTBEAT_MISS_THRESHOLD} checks, triggering reconnection (lastPingAgoMs=${lastPingAgoMs ?? "n/a"})`,
|
|
152
|
+
);
|
|
153
|
+
this.runtimeCounters.heartbeatTriggeredReconnects += 1;
|
|
154
|
+
this.logRuntimeCounters("heartbeat-triggered-reconnect");
|
|
155
|
+
this.cleanupHeartbeatInterval();
|
|
156
|
+
this.handleRuntimeDisconnection();
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
this.log?.debug?.(
|
|
161
|
+
`[${this.accountId}] Connection heartbeat missed (${this.consecutiveHeartbeatMisses}/${ConnectionManager.HEARTBEAT_MISS_THRESHOLD})`,
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
this.lastHeartbeatPingAt = now;
|
|
166
|
+
try {
|
|
167
|
+
socket.ping("", true);
|
|
168
|
+
} catch (err: any) {
|
|
169
|
+
this.log?.warn?.(
|
|
170
|
+
`[${this.accountId}] Failed to send heartbeat ping: ${err.message}`,
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
}, ConnectionManager.HEARTBEAT_INTERVAL_MS);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private cleanupHeartbeatInterval(): void {
|
|
177
|
+
if (this.heartbeatInterval) {
|
|
178
|
+
clearInterval(this.heartbeatInterval);
|
|
179
|
+
this.heartbeatInterval = undefined;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private clearClientHeartbeatInterval(client: DWClient): void {
|
|
184
|
+
const clientAny = client as any;
|
|
185
|
+
if (clientAny.heartbeatIntervallId !== undefined) {
|
|
186
|
+
clearInterval(clientAny.heartbeatIntervallId);
|
|
187
|
+
clientAny.heartbeatIntervallId = undefined;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private async waitForSocketOpen(clientAny: any): Promise<void> {
|
|
192
|
+
const socket = clientAny.socket;
|
|
193
|
+
if (!socket) {
|
|
194
|
+
throw new Error("Socket unavailable after connect");
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (socket.readyState === 1 || clientAny.connected) {
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const defaultOpenTimeout = 10_000;
|
|
202
|
+
const openTimeout = this.reconnectDeadline !== undefined
|
|
203
|
+
? Math.min(defaultOpenTimeout, Math.max(1000, this.reconnectDeadline - Date.now()))
|
|
204
|
+
: defaultOpenTimeout;
|
|
205
|
+
|
|
206
|
+
await new Promise<void>((resolve, reject) => {
|
|
207
|
+
const timeout = setTimeout(() => {
|
|
208
|
+
cleanup();
|
|
209
|
+
reject(new Error("Socket open timeout"));
|
|
210
|
+
}, openTimeout);
|
|
211
|
+
const cleanup = () => {
|
|
212
|
+
clearTimeout(timeout);
|
|
213
|
+
socket.removeListener("open", onOpen);
|
|
214
|
+
socket.removeListener("error", onError);
|
|
215
|
+
socket.removeListener("close", onClose);
|
|
216
|
+
};
|
|
217
|
+
const onOpen = () => { cleanup(); resolve(); };
|
|
218
|
+
const onError = (err: Error) => { cleanup(); reject(err); };
|
|
219
|
+
const onClose = () => { cleanup(); reject(new Error("Socket closed before open")); };
|
|
220
|
+
socket.once("open", onOpen);
|
|
221
|
+
socket.once("error", onError);
|
|
222
|
+
socket.once("close", onClose);
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
|
|
88
226
|
/**
|
|
89
227
|
* Calculate next reconnection delay with exponential backoff and jitter
|
|
90
228
|
* Formula: delay = min(initialDelay * 2^attempt, maxDelay) * (1 ± jitter)
|
|
@@ -129,24 +267,52 @@ export class ConnectionManager {
|
|
|
129
267
|
);
|
|
130
268
|
|
|
131
269
|
try {
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
//
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
270
|
+
// Warm-reconnect path: create a fresh DWClient so the new WebSocket
|
|
271
|
+
// can start receiving messages while the old zombie socket is cleaned
|
|
272
|
+
// up asynchronously. This minimizes the message-loss window during
|
|
273
|
+
// server-initiated disconnects (DingTalk robot msgs are fire-and-forget).
|
|
274
|
+
if (this.clientFactory) {
|
|
275
|
+
const oldClient = this.client;
|
|
276
|
+
this.pendingOldClient = oldClient;
|
|
277
|
+
try {
|
|
278
|
+
this.client = this.clientFactory();
|
|
279
|
+
this.log?.info?.(
|
|
280
|
+
`[${this.accountId}] Warm reconnect: created fresh DWClient, connecting new socket while old socket is cleaned up`,
|
|
281
|
+
);
|
|
282
|
+
} catch (factoryErr: any) {
|
|
283
|
+
this.log?.warn?.(
|
|
284
|
+
`[${this.accountId}] Client factory failed, falling back to same-client reconnect: ${factoryErr.message}`,
|
|
285
|
+
);
|
|
286
|
+
this.client = oldClient;
|
|
287
|
+
this.pendingOldClient = undefined;
|
|
288
|
+
}
|
|
144
289
|
}
|
|
145
290
|
|
|
291
|
+
if (!this.pendingOldClient) {
|
|
292
|
+
// Legacy single-client reconnect: disconnect before reconnecting.
|
|
293
|
+
try {
|
|
294
|
+
this.client.disconnect();
|
|
295
|
+
} catch (disconnectErr: any) {
|
|
296
|
+
this.log?.debug?.(
|
|
297
|
+
`[${this.accountId}] pre-connect cleanup disconnect failed: ${disconnectErr.message}`,
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// SDK _connect() resolves before socket "open" fires. If disconnect() runs
|
|
303
|
+
// before "open", heartbeatIntervallId is still undefined and clearInterval
|
|
304
|
+
// is a no-op. The deferred "open" handler then creates an interval that
|
|
305
|
+
// outlives the old socket and can terminate the next connection.
|
|
306
|
+
// For the race where interval is created AFTER this cleanup, the socket
|
|
307
|
+
// open timeout below serves as the final safety net.
|
|
308
|
+
// Field name "heartbeatIntervallId" (double-l typo) from dingtalk-stream
|
|
309
|
+
// SDK DWClient._connect() open handler (verified in SDK v1.x).
|
|
310
|
+
const clientAny = this.client as any;
|
|
311
|
+
this.clearClientHeartbeatInterval(this.client);
|
|
312
|
+
|
|
146
313
|
await this.client.connect();
|
|
314
|
+
await this.waitForSocketOpen(clientAny);
|
|
147
315
|
|
|
148
|
-
// Re-check stopped flag after async connect() completes
|
|
149
|
-
// This prevents race condition where stop() is called during connection
|
|
150
316
|
if (this.stopped) {
|
|
151
317
|
this.log?.warn?.(
|
|
152
318
|
`[${this.accountId}] Connection succeeded but manager was stopped during connect - disconnecting`,
|
|
@@ -165,21 +331,41 @@ export class ConnectionManager {
|
|
|
165
331
|
};
|
|
166
332
|
}
|
|
167
333
|
|
|
168
|
-
// Connection successful
|
|
169
334
|
this.state = ConnectionStateEnum.CONNECTED;
|
|
170
335
|
this.connectedAt = Date.now();
|
|
336
|
+
this.recordSocketActivity(this.connectedAt);
|
|
337
|
+
this.lastHeartbeatPingAt = undefined;
|
|
338
|
+
this.reconnectDeadline = undefined;
|
|
171
339
|
this.consecutiveUnhealthyChecks = 0;
|
|
340
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
172
341
|
this.notifyStateChange();
|
|
173
342
|
const successfulAttempt = this.attemptCount;
|
|
174
|
-
this.attemptCount = 0;
|
|
343
|
+
this.attemptCount = 0;
|
|
175
344
|
|
|
176
345
|
this.log?.info?.(`[${this.accountId}] DingTalk Stream client connected successfully`);
|
|
177
346
|
|
|
178
|
-
// Reset runtime reconnect cycle counter on successful connection
|
|
179
347
|
this.runtimeReconnectCycles = 0;
|
|
348
|
+
this.consecutiveDeadlineTimeouts = 0;
|
|
349
|
+
// Setup monitoring BEFORE cleaning up old client: setupRuntimeReconnection
|
|
350
|
+
// calls cleanupRuntimeMonitoring() which removes the old socket's event
|
|
351
|
+
// listeners, preventing the old socket's close event (triggered by
|
|
352
|
+
// disconnect below) from erroneously firing handleRuntimeDisconnection.
|
|
353
|
+
this.setupRuntimeReconnection();
|
|
354
|
+
this.cleanupPendingOldClient();
|
|
180
355
|
|
|
181
356
|
return { success: true, attempt: successfulAttempt };
|
|
182
357
|
} catch (err: any) {
|
|
358
|
+
// Warm-reconnect failed with new client: revert to old client so
|
|
359
|
+
// subsequent retry attempts don't keep creating throwaway instances.
|
|
360
|
+
if (this.pendingOldClient) {
|
|
361
|
+
try { this.client.disconnect(); } catch { /* best-effort cleanup of failed new client */ }
|
|
362
|
+
this.client = this.pendingOldClient;
|
|
363
|
+
this.pendingOldClient = undefined;
|
|
364
|
+
this.log?.debug?.(
|
|
365
|
+
`[${this.accountId}] Warm reconnect failed, reverted to previous client for next retry`,
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
|
|
183
369
|
this.log?.error?.(
|
|
184
370
|
`[${this.accountId}] Connection attempt ${this.attemptCount} failed: ${err.message}`,
|
|
185
371
|
);
|
|
@@ -221,17 +407,18 @@ export class ConnectionManager {
|
|
|
221
407
|
`[${this.accountId}] Starting DingTalk Stream client with robust connection...`,
|
|
222
408
|
);
|
|
223
409
|
|
|
224
|
-
// Keep trying until success or max attempts reached
|
|
225
410
|
while (!this.stopped && this.state !== ConnectionStateEnum.CONNECTED) {
|
|
411
|
+
if (this.reconnectDeadline !== undefined && Date.now() >= this.reconnectDeadline) {
|
|
412
|
+
this.reconnectDeadline = undefined;
|
|
413
|
+
throw new ReconnectDeadlineError();
|
|
414
|
+
}
|
|
415
|
+
|
|
226
416
|
const result = await this.attemptConnection();
|
|
227
417
|
|
|
228
418
|
if (result.success) {
|
|
229
|
-
// Connection successful
|
|
230
|
-
this.setupRuntimeReconnection();
|
|
231
419
|
return;
|
|
232
420
|
}
|
|
233
421
|
|
|
234
|
-
// Check if connection was stopped during connect
|
|
235
422
|
if (result.error?.message === "Connection manager stopped during connect") {
|
|
236
423
|
this.log?.info?.(
|
|
237
424
|
`[${this.accountId}] Connection cancelled: manager stopped during connect`,
|
|
@@ -240,12 +427,17 @@ export class ConnectionManager {
|
|
|
240
427
|
}
|
|
241
428
|
|
|
242
429
|
if (!result.nextDelay || this.attemptCount >= this.config.maxAttempts) {
|
|
243
|
-
// No more retries
|
|
244
430
|
throw new Error(`Failed to connect after ${this.attemptCount} attempts`);
|
|
245
431
|
}
|
|
246
432
|
|
|
247
|
-
//
|
|
248
|
-
|
|
433
|
+
// Truncate sleep to remaining deadline budget
|
|
434
|
+
let actualDelay = result.nextDelay;
|
|
435
|
+
if (this.reconnectDeadline !== undefined) {
|
|
436
|
+
const remaining = Math.max(0, this.reconnectDeadline - Date.now());
|
|
437
|
+
actualDelay = Math.min(actualDelay, remaining);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
await this.sleep(actualDelay);
|
|
249
441
|
}
|
|
250
442
|
}
|
|
251
443
|
|
|
@@ -288,7 +480,15 @@ export class ConnectionManager {
|
|
|
288
480
|
|
|
289
481
|
const socketReadyState = (client.socket as { readyState?: number } | undefined)?.readyState;
|
|
290
482
|
const socketOpen = socketReadyState === 1;
|
|
291
|
-
const
|
|
483
|
+
const registered = client.registered as boolean | undefined;
|
|
484
|
+
|
|
485
|
+
// Unhealthy if:
|
|
486
|
+
// 1. Not connected AND socket not open (full disconnect)
|
|
487
|
+
// 2. Socket is open, client.connected is false, AND not registered
|
|
488
|
+
// (server sent logical "disconnect" and the socket is now a zombie)
|
|
489
|
+
const unhealthy =
|
|
490
|
+
(!client.connected && !socketOpen) ||
|
|
491
|
+
(socketOpen && !client.connected && registered === false);
|
|
292
492
|
|
|
293
493
|
if (!unhealthy) {
|
|
294
494
|
this.consecutiveUnhealthyChecks = 0;
|
|
@@ -302,7 +502,7 @@ export class ConnectionManager {
|
|
|
302
502
|
ConnectionManager.HEALTH_CHECK_UNHEALTHY_THRESHOLD
|
|
303
503
|
) {
|
|
304
504
|
this.log?.debug?.(
|
|
305
|
-
`[${this.accountId}] Connection health check unhealthy (${this.consecutiveUnhealthyChecks}/${ConnectionManager.HEALTH_CHECK_UNHEALTHY_THRESHOLD}) connected=${String(client.connected)} socketReadyState=${socketReadyState ?? "unknown"}`,
|
|
505
|
+
`[${this.accountId}] Connection health check unhealthy (${this.consecutiveUnhealthyChecks}/${ConnectionManager.HEALTH_CHECK_UNHEALTHY_THRESHOLD}) connected=${String(client.connected)} registered=${String(registered)} socketReadyState=${socketReadyState ?? "unknown"}`,
|
|
306
506
|
);
|
|
307
507
|
return;
|
|
308
508
|
}
|
|
@@ -324,6 +524,7 @@ export class ConnectionManager {
|
|
|
324
524
|
const socket = client.socket;
|
|
325
525
|
// Store the socket instance we're attaching listeners to
|
|
326
526
|
this.monitoredSocket = socket;
|
|
527
|
+
this.setupHeartbeat(socket);
|
|
327
528
|
|
|
328
529
|
// Handler for socket close event
|
|
329
530
|
this.socketCloseHandler = (code: number, reason: string) => {
|
|
@@ -349,11 +550,61 @@ export class ConnectionManager {
|
|
|
349
550
|
);
|
|
350
551
|
};
|
|
351
552
|
|
|
352
|
-
// Listen to socket events
|
|
353
|
-
// Use 'once' for close to avoid duplicate reconnection triggers
|
|
354
553
|
socket.once("close", this.socketCloseHandler);
|
|
355
|
-
// Use 'once' for error as well to prevent accumulation across reconnects
|
|
356
554
|
socket.once("error", this.socketErrorHandler);
|
|
555
|
+
|
|
556
|
+
// Monitor for server-side disconnect system messages and passive socket
|
|
557
|
+
// activity.
|
|
558
|
+
this.socketMessageHandler = (data: any) => {
|
|
559
|
+
this.recordSocketActivity();
|
|
560
|
+
try {
|
|
561
|
+
const msg = JSON.parse(typeof data === "string" ? data : data.toString());
|
|
562
|
+
if (msg?.type === "SYSTEM" && msg?.headers?.topic === "disconnect") {
|
|
563
|
+
this.runtimeCounters.serverDisconnectMessages += 1;
|
|
564
|
+
this.log?.warn?.(
|
|
565
|
+
`[${this.accountId}] Server disconnect system message received, triggering immediate reconnection`,
|
|
566
|
+
);
|
|
567
|
+
this.logRuntimeCounters("server-disconnect");
|
|
568
|
+
if (this.healthCheckInterval) {
|
|
569
|
+
clearInterval(this.healthCheckInterval);
|
|
570
|
+
}
|
|
571
|
+
this.handleRuntimeDisconnection();
|
|
572
|
+
}
|
|
573
|
+
} catch {
|
|
574
|
+
// Ignore parse errors — other handlers will process the message
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
socket.on("message", this.socketMessageHandler);
|
|
578
|
+
|
|
579
|
+
this.socketPongHandler = () => {
|
|
580
|
+
this.recordSocketActivity();
|
|
581
|
+
};
|
|
582
|
+
socket.on("pong", this.socketPongHandler);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Disconnect and release the old client left over from a warm-reconnect swap.
|
|
588
|
+
*/
|
|
589
|
+
private cleanupPendingOldClient(): void {
|
|
590
|
+
if (!this.pendingOldClient) {
|
|
591
|
+
return;
|
|
592
|
+
}
|
|
593
|
+
const old = this.pendingOldClient;
|
|
594
|
+
this.pendingOldClient = undefined;
|
|
595
|
+
try {
|
|
596
|
+
// Clear stale heartbeat interval from old SDK instance.
|
|
597
|
+
const oldAny = old as any;
|
|
598
|
+
if (oldAny.heartbeatIntervallId !== undefined) {
|
|
599
|
+
clearInterval(oldAny.heartbeatIntervallId);
|
|
600
|
+
oldAny.heartbeatIntervallId = undefined;
|
|
601
|
+
}
|
|
602
|
+
old.disconnect();
|
|
603
|
+
this.log?.debug?.(`[${this.accountId}] Warm reconnect: old client disconnected`);
|
|
604
|
+
} catch (err: any) {
|
|
605
|
+
this.log?.debug?.(
|
|
606
|
+
`[${this.accountId}] Warm reconnect: old client cleanup failed: ${err.message}`,
|
|
607
|
+
);
|
|
357
608
|
}
|
|
358
609
|
}
|
|
359
610
|
|
|
@@ -367,6 +618,7 @@ export class ConnectionManager {
|
|
|
367
618
|
this.healthCheckInterval = undefined;
|
|
368
619
|
this.log?.debug?.(`[${this.accountId}] Health check interval cleared`);
|
|
369
620
|
}
|
|
621
|
+
this.cleanupHeartbeatInterval();
|
|
370
622
|
|
|
371
623
|
// Remove socket event listeners from the stored socket instance
|
|
372
624
|
if (this.monitoredSocket) {
|
|
@@ -380,6 +632,14 @@ export class ConnectionManager {
|
|
|
380
632
|
socket.removeListener("error", this.socketErrorHandler);
|
|
381
633
|
this.socketErrorHandler = undefined;
|
|
382
634
|
}
|
|
635
|
+
if (this.socketMessageHandler) {
|
|
636
|
+
socket.removeListener("message", this.socketMessageHandler);
|
|
637
|
+
this.socketMessageHandler = undefined;
|
|
638
|
+
}
|
|
639
|
+
if (this.socketPongHandler) {
|
|
640
|
+
socket.removeListener("pong", this.socketPongHandler);
|
|
641
|
+
this.socketPongHandler = undefined;
|
|
642
|
+
}
|
|
383
643
|
|
|
384
644
|
this.log?.debug?.(`[${this.accountId}] Socket event listeners removed from monitored socket`);
|
|
385
645
|
this.monitoredSocket = undefined;
|
|
@@ -390,35 +650,37 @@ export class ConnectionManager {
|
|
|
390
650
|
* Handle runtime disconnection and trigger reconnection
|
|
391
651
|
*/
|
|
392
652
|
private handleRuntimeDisconnection(): void {
|
|
393
|
-
if (this.stopped) {
|
|
653
|
+
if (this.stopped || this.state !== ConnectionStateEnum.CONNECTED) {
|
|
394
654
|
return;
|
|
395
655
|
}
|
|
396
656
|
|
|
657
|
+
this.state = ConnectionStateEnum.DISCONNECTED;
|
|
658
|
+
|
|
397
659
|
this.log?.warn?.(
|
|
398
660
|
`[${this.accountId}] Runtime disconnection detected, initiating reconnection...`,
|
|
399
661
|
);
|
|
400
662
|
this.runtimeCounters.runtimeDisconnects += 1;
|
|
401
663
|
|
|
402
|
-
this.state = ConnectionStateEnum.DISCONNECTED;
|
|
403
664
|
this.notifyStateChange("Runtime disconnection detected");
|
|
404
|
-
this.attemptCount = 0;
|
|
665
|
+
this.attemptCount = 0;
|
|
405
666
|
this.connectedAt = undefined;
|
|
667
|
+
this.lastSocketActivityAt = undefined;
|
|
668
|
+
this.lastHeartbeatPingAt = undefined;
|
|
406
669
|
this.consecutiveUnhealthyChecks = 0;
|
|
670
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
671
|
+
|
|
672
|
+
const deadlineMs = this.config.reconnectDeadlineMs ?? 50000;
|
|
673
|
+
this.reconnectDeadline = Date.now() + deadlineMs;
|
|
407
674
|
|
|
408
|
-
// Clear any existing timer
|
|
409
675
|
this.clearReconnectTimer();
|
|
410
676
|
|
|
411
|
-
|
|
412
|
-
const delay = this.calculateNextDelay(0);
|
|
413
|
-
this.log?.info?.(
|
|
414
|
-
`[${this.accountId}] Scheduling reconnection in ${(delay / 1000).toFixed(2)}s`,
|
|
415
|
-
);
|
|
677
|
+
this.log?.info?.(`[${this.accountId}] Scheduling immediate reconnection`);
|
|
416
678
|
|
|
417
679
|
this.reconnectTimer = setTimeout(() => {
|
|
418
680
|
this.reconnect().catch((err) => {
|
|
419
681
|
this.log?.error?.(`[${this.accountId}] Reconnection failed: ${err.message}`);
|
|
420
682
|
});
|
|
421
|
-
},
|
|
683
|
+
}, 0);
|
|
422
684
|
}
|
|
423
685
|
|
|
424
686
|
/**
|
|
@@ -446,7 +708,40 @@ export class ConnectionManager {
|
|
|
446
708
|
this.runtimeCounters.reconnectFailures += 1;
|
|
447
709
|
this.logRuntimeCounters("reconnect-failed");
|
|
448
710
|
|
|
449
|
-
|
|
711
|
+
if (err instanceof ReconnectDeadlineError) {
|
|
712
|
+
this.consecutiveDeadlineTimeouts += 1;
|
|
713
|
+
|
|
714
|
+
if (this.consecutiveDeadlineTimeouts >= ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS) {
|
|
715
|
+
this.log?.error?.(
|
|
716
|
+
`[${this.accountId}] Max consecutive deadline timeouts (${ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS}) reached. Giving up.`,
|
|
717
|
+
);
|
|
718
|
+
this.state = ConnectionStateEnum.FAILED;
|
|
719
|
+
this.connectedAt = undefined;
|
|
720
|
+
this.consecutiveUnhealthyChecks = 0;
|
|
721
|
+
this.reconnectDeadline = undefined;
|
|
722
|
+
this.notifyStateChange(
|
|
723
|
+
`Max consecutive deadline timeouts (${ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS}) reached`,
|
|
724
|
+
);
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
const deadlineMs = this.config.reconnectDeadlineMs ?? 50000;
|
|
729
|
+
this.reconnectDeadline = Date.now() + deadlineMs;
|
|
730
|
+
const delay = Math.min(
|
|
731
|
+
this.calculateNextDelay(0),
|
|
732
|
+
ConnectionManager.MAX_CYCLE_BACKOFF_MS,
|
|
733
|
+
);
|
|
734
|
+
this.attemptCount = 0;
|
|
735
|
+
this.clearReconnectTimer();
|
|
736
|
+
this.log?.warn?.(
|
|
737
|
+
`[${this.accountId}] Reconnect deadline exceeded (${this.consecutiveDeadlineTimeouts}/${ConnectionManager.MAX_CONSECUTIVE_DEADLINE_TIMEOUTS}); scheduling next cycle in ${(delay / 1000).toFixed(2)}s`,
|
|
738
|
+
);
|
|
739
|
+
this.reconnectTimer = setTimeout(() => {
|
|
740
|
+
void this.reconnect();
|
|
741
|
+
}, delay);
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
|
|
450
745
|
this.runtimeReconnectCycles += 1;
|
|
451
746
|
const maxCycles = this.config.maxReconnectCycles ?? ConnectionManager.DEFAULT_MAX_RECONNECT_CYCLES;
|
|
452
747
|
|
|
@@ -458,6 +753,7 @@ export class ConnectionManager {
|
|
|
458
753
|
this.state = ConnectionStateEnum.FAILED;
|
|
459
754
|
this.connectedAt = undefined;
|
|
460
755
|
this.consecutiveUnhealthyChecks = 0;
|
|
756
|
+
this.reconnectDeadline = undefined;
|
|
461
757
|
this.notifyStateChange(`Max runtime reconnect cycles (${maxCycles}) reached`);
|
|
462
758
|
return;
|
|
463
759
|
}
|
|
@@ -467,8 +763,11 @@ export class ConnectionManager {
|
|
|
467
763
|
this.consecutiveUnhealthyChecks = 0;
|
|
468
764
|
this.notifyStateChange(err.message);
|
|
469
765
|
|
|
470
|
-
|
|
471
|
-
const delay =
|
|
766
|
+
const rawDelay = this.calculateNextDelay(Math.min(this.runtimeReconnectCycles - 1, 6));
|
|
767
|
+
const delay = Math.min(rawDelay, ConnectionManager.MAX_CYCLE_BACKOFF_MS);
|
|
768
|
+
// Each cycle gets its own deadline so long-running retries don't block indefinitely
|
|
769
|
+
const deadlineMs = this.config.reconnectDeadlineMs ?? 50000;
|
|
770
|
+
this.reconnectDeadline = Date.now() + deadlineMs;
|
|
472
771
|
this.attemptCount = 0;
|
|
473
772
|
this.clearReconnectTimer();
|
|
474
773
|
this.log?.warn?.(
|
|
@@ -493,7 +792,12 @@ export class ConnectionManager {
|
|
|
493
792
|
this.stopped = true;
|
|
494
793
|
this.state = ConnectionStateEnum.DISCONNECTING;
|
|
495
794
|
this.connectedAt = undefined;
|
|
795
|
+
this.reconnectDeadline = undefined;
|
|
796
|
+
this.consecutiveDeadlineTimeouts = 0;
|
|
496
797
|
this.consecutiveUnhealthyChecks = 0;
|
|
798
|
+
this.lastSocketActivityAt = undefined;
|
|
799
|
+
this.lastHeartbeatPingAt = undefined;
|
|
800
|
+
this.consecutiveHeartbeatMisses = 0;
|
|
497
801
|
|
|
498
802
|
// Clear reconnect timer
|
|
499
803
|
this.clearReconnectTimer();
|
|
@@ -504,6 +808,9 @@ export class ConnectionManager {
|
|
|
504
808
|
// Clean up runtime monitoring resources
|
|
505
809
|
this.cleanupRuntimeMonitoring();
|
|
506
810
|
|
|
811
|
+
// Clean up any pending old client from warm-reconnect swap.
|
|
812
|
+
this.cleanupPendingOldClient();
|
|
813
|
+
|
|
507
814
|
// Disconnect client
|
|
508
815
|
try {
|
|
509
816
|
this.client.disconnect();
|
package/src/dedup.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// ============ Message Deduplication ============
|
|
2
2
|
// Prevent duplicate processing when DingTalk retries delivery.
|
|
3
3
|
// In-memory TTL map + lazy cleanup keeps overhead small.
|
|
4
|
+
export const DEDUP_NAMESPACE_POLICY = "memory-only" as const;
|
|
4
5
|
|
|
5
6
|
const processedMessages = new Map<string, number>();
|
|
6
7
|
const MESSAGE_DEDUP_TTL = 60000;
|