@xmoxmo/bncr 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/channel.ts +404 -174
- package/src/messaging/outbound/diagnostics.ts +128 -6
package/package.json
CHANGED
package/src/channel.ts
CHANGED
|
@@ -20,18 +20,33 @@ import {
|
|
|
20
20
|
clearOutboundCapability,
|
|
21
21
|
findCapabilityConnection,
|
|
22
22
|
} from './core/connection-capability.ts';
|
|
23
|
+
import {
|
|
24
|
+
getRevalidatedAttemptReason,
|
|
25
|
+
hasAlternativeLiveConnection as hasAlternativeLiveConnectionFromRuntime,
|
|
26
|
+
hasRecentInboundReachability as hasRecentInboundReachabilityFromRuntime,
|
|
27
|
+
isRecentlyReachableConn as isRecentlyReachableConnFromRuntime,
|
|
28
|
+
resolveRecentInboundConnIds as resolveRecentInboundConnIdsFromRuntime,
|
|
29
|
+
} from './core/connection-reachability.ts';
|
|
30
|
+
import { buildDiagnosticsPayload } from './core/diagnostics.ts';
|
|
31
|
+
import { buildDownlinkHealth as buildDownlinkHealthFromRuntime } from './core/downlink-health.ts';
|
|
32
|
+
import { buildExtendedDiagnostics as buildExtendedDiagnosticsFromRuntime } from './core/extended-diagnostics.ts';
|
|
33
|
+
import { buildFileAckKey } from './core/file-ack.ts';
|
|
34
|
+
import {
|
|
35
|
+
buildFileTransferAbortPayload,
|
|
36
|
+
buildFileTransferChunkPayload,
|
|
37
|
+
buildFileTransferCompletePayload,
|
|
38
|
+
buildFileTransferInitPayload,
|
|
39
|
+
} from './core/file-transfer-payloads.ts';
|
|
40
|
+
import {
|
|
41
|
+
matchesTransferOwner as matchesTransferOwnerFromRuntime,
|
|
42
|
+
observeLeaseState,
|
|
43
|
+
} from './core/lease-state.ts';
|
|
44
|
+
import { emitBncrLog, emitBncrLogLine } from './core/logging.ts';
|
|
45
|
+
import { buildOutboxEnqueueDebugInfo } from './core/outbox-enqueue.ts';
|
|
23
46
|
import {
|
|
24
47
|
buildFileTransferOutboxEntry as buildFileTransferOutboxEntryFromRuntime,
|
|
25
48
|
buildTextOutboxEntry as buildTextOutboxEntryFromRuntime,
|
|
26
49
|
} from './core/outbox-entry-builders.ts';
|
|
27
|
-
import { buildOutboxEnqueueDebugInfo } from './core/outbox-enqueue.ts';
|
|
28
|
-
import {
|
|
29
|
-
appendDeadLetter,
|
|
30
|
-
buildDeadLetterEntry,
|
|
31
|
-
collectDueOutboxEntries,
|
|
32
|
-
} from './core/outbox-queue.ts';
|
|
33
|
-
import { resolveFileTransferGuard } from './core/outbox-file-transfer-guards.ts';
|
|
34
|
-
import { prepareFileTransferRouteSelection } from './core/outbox-file-transfer-prep.ts';
|
|
35
50
|
import {
|
|
36
51
|
buildFileTransferPushOkArgs,
|
|
37
52
|
buildFileTransferPushSuccessArgs,
|
|
@@ -40,58 +55,28 @@ import {
|
|
|
40
55
|
buildFileTransferPushFailureArgs,
|
|
41
56
|
resolveFileTransferFailureState,
|
|
42
57
|
} from './core/outbox-file-transfer-failure.ts';
|
|
58
|
+
import { resolveFileTransferGuard } from './core/outbox-file-transfer-guards.ts';
|
|
59
|
+
import { prepareFileTransferRouteSelection } from './core/outbox-file-transfer-prep.ts';
|
|
43
60
|
import {
|
|
44
61
|
buildFileTransferBroadcastPayload,
|
|
45
62
|
buildFileTransferRouteSelectArgs,
|
|
46
63
|
} from './core/outbox-file-transfer-success.ts';
|
|
64
|
+
import {
|
|
65
|
+
appendDeadLetter,
|
|
66
|
+
buildDeadLetterEntry,
|
|
67
|
+
collectDueOutboxEntries,
|
|
68
|
+
} from './core/outbox-queue.ts';
|
|
47
69
|
import { summarizeOutboxEntry } from './core/outbox-summary.ts';
|
|
70
|
+
import { buildTextPushFailureArgs } from './core/outbox-text-push-failure.ts';
|
|
48
71
|
import { resolveTextPushGuard } from './core/outbox-text-push-guards.ts';
|
|
49
72
|
import { prepareTextPushRouteSelection } from './core/outbox-text-push-prep.ts';
|
|
50
|
-
import { buildTextPushFailureArgs } from './core/outbox-text-push-failure.ts';
|
|
51
73
|
import {
|
|
52
74
|
buildTextPushBroadcastPayload,
|
|
53
75
|
buildTextPushOkArgs,
|
|
54
76
|
buildTextPushRouteSelectArgs,
|
|
55
77
|
buildTextPushSuccessArgs,
|
|
56
78
|
} from './core/outbox-text-push-success.ts';
|
|
57
|
-
import {
|
|
58
|
-
getRevalidatedAttemptReason,
|
|
59
|
-
hasAlternativeLiveConnection as hasAlternativeLiveConnectionFromRuntime,
|
|
60
|
-
hasRecentInboundReachability as hasRecentInboundReachabilityFromRuntime,
|
|
61
|
-
isRecentlyReachableConn as isRecentlyReachableConnFromRuntime,
|
|
62
|
-
resolveRecentInboundConnIds as resolveRecentInboundConnIdsFromRuntime,
|
|
63
|
-
} from './core/connection-reachability.ts';
|
|
64
|
-
import { buildDiagnosticsPayload } from './core/diagnostics.ts';
|
|
65
|
-
import { buildDownlinkHealth as buildDownlinkHealthFromRuntime } from './core/downlink-health.ts';
|
|
66
|
-
import { buildExtendedDiagnostics as buildExtendedDiagnosticsFromRuntime } from './core/extended-diagnostics.ts';
|
|
67
|
-
import { observeLeaseState, matchesTransferOwner as matchesTransferOwnerFromRuntime } from './core/lease-state.ts';
|
|
68
|
-
import { emitBncrLog, emitBncrLogLine } from './core/logging.ts';
|
|
69
|
-
import { buildFileAckKey } from './core/file-ack.ts';
|
|
70
|
-
import {
|
|
71
|
-
buildFileTransferAbortPayload,
|
|
72
|
-
buildFileTransferChunkPayload,
|
|
73
|
-
buildFileTransferCompletePayload,
|
|
74
|
-
buildFileTransferInitPayload,
|
|
75
|
-
} from './core/file-transfer-payloads.ts';
|
|
76
79
|
import { resolveBncrChannelPolicy, resolveBncrConfigWarnings } from './core/policy.ts';
|
|
77
|
-
import {
|
|
78
|
-
getOpenClawRuntimeConfig,
|
|
79
|
-
getOpenClawRuntimeConfigOrDefault,
|
|
80
|
-
} from './openclaw/config-runtime.ts';
|
|
81
|
-
import {
|
|
82
|
-
loadOpenClawWebMedia,
|
|
83
|
-
saveOpenClawChannelMediaBuffer,
|
|
84
|
-
type OpenClawLoadedMedia,
|
|
85
|
-
} from './openclaw/media-runtime.ts';
|
|
86
|
-
import { resolveOpenClawAgentRoute } from './openclaw/routing-runtime.ts';
|
|
87
|
-
import {
|
|
88
|
-
extractOpenClawToolSend,
|
|
89
|
-
openClawJsonResult,
|
|
90
|
-
readOpenClawBooleanParam,
|
|
91
|
-
readOpenClawJsonFileWithFallback,
|
|
92
|
-
readOpenClawStringParam,
|
|
93
|
-
writeOpenClawJsonFileAtomically,
|
|
94
|
-
} from './openclaw/sdk-helpers.ts';
|
|
95
80
|
import {
|
|
96
81
|
appendBoundedRegisterTrace,
|
|
97
82
|
buildRegisterDriftSnapshot,
|
|
@@ -129,14 +114,12 @@ import {
|
|
|
129
114
|
reactBncrMessageAction,
|
|
130
115
|
sendBncrReplyAction,
|
|
131
116
|
} from './messaging/outbound/actions.ts';
|
|
132
|
-
import {
|
|
133
|
-
buildBncrMediaOutboundFrame,
|
|
134
|
-
resolveBncrOutboundMessageType,
|
|
135
|
-
} from './messaging/outbound/media.ts';
|
|
136
117
|
import {
|
|
137
118
|
buildEnqueueFromReplyDebugInfo,
|
|
138
119
|
buildFlushDebugInfo,
|
|
139
120
|
buildOutboxAckDebugInfo,
|
|
121
|
+
buildOutboxDrainSkipDebugInfo,
|
|
122
|
+
buildOutboxDrainStuckDebugInfo,
|
|
140
123
|
buildOutboxPushOkDebugInfo,
|
|
141
124
|
buildOutboxPushSkipDebugInfo,
|
|
142
125
|
buildOutboxRouteSelectDebugInfo,
|
|
@@ -145,6 +128,28 @@ import {
|
|
|
145
128
|
buildReplyMediaFallbackDebugInfo,
|
|
146
129
|
buildRetryRerouteDebugInfo,
|
|
147
130
|
} from './messaging/outbound/diagnostics.ts';
|
|
131
|
+
import {
|
|
132
|
+
buildBncrMediaOutboundFrame,
|
|
133
|
+
resolveBncrOutboundMessageType,
|
|
134
|
+
} from './messaging/outbound/media.ts';
|
|
135
|
+
import {
|
|
136
|
+
getOpenClawRuntimeConfig,
|
|
137
|
+
getOpenClawRuntimeConfigOrDefault,
|
|
138
|
+
} from './openclaw/config-runtime.ts';
|
|
139
|
+
import {
|
|
140
|
+
type OpenClawLoadedMedia,
|
|
141
|
+
loadOpenClawWebMedia,
|
|
142
|
+
saveOpenClawChannelMediaBuffer,
|
|
143
|
+
} from './openclaw/media-runtime.ts';
|
|
144
|
+
import { resolveOpenClawAgentRoute } from './openclaw/routing-runtime.ts';
|
|
145
|
+
import {
|
|
146
|
+
extractOpenClawToolSend,
|
|
147
|
+
openClawJsonResult,
|
|
148
|
+
readOpenClawBooleanParam,
|
|
149
|
+
readOpenClawJsonFileWithFallback,
|
|
150
|
+
readOpenClawStringParam,
|
|
151
|
+
writeOpenClawJsonFileAtomically,
|
|
152
|
+
} from './openclaw/sdk-helpers.ts';
|
|
148
153
|
|
|
149
154
|
function buildInboundAcceptedLifecycleDebugInfo(args: {
|
|
150
155
|
stage: 'accepted';
|
|
@@ -278,33 +283,13 @@ function buildInboundResponsePayload(
|
|
|
278
283
|
};
|
|
279
284
|
}
|
|
280
285
|
}
|
|
286
|
+
import { buildBncrDurableQueuedResult } from './messaging/outbound/durable-queue-adapter.ts';
|
|
281
287
|
import {
|
|
282
|
-
buildMediaTextFallback,
|
|
283
288
|
type MediaDedupeCacheEntry,
|
|
289
|
+
buildMediaTextFallback,
|
|
284
290
|
normalizeMessageText,
|
|
285
291
|
normalizeReplyToId,
|
|
286
292
|
} from './messaging/outbound/media-dedupe.ts';
|
|
287
|
-
import {
|
|
288
|
-
buildReplyTextOutboxEntry,
|
|
289
|
-
enqueueNormalizedReplyPayload,
|
|
290
|
-
enqueueReplyMediaFallbackTextEntry,
|
|
291
|
-
enqueueReplyMediaFileTransferEntry,
|
|
292
|
-
enqueueSingleReplyMediaEntry,
|
|
293
|
-
enqueueReplyTextEntry,
|
|
294
|
-
hasReplyMediaEntries,
|
|
295
|
-
normalizeReplyPayload,
|
|
296
|
-
type NormalizedReplyPayload,
|
|
297
|
-
type ReplyMediaEntriesParams,
|
|
298
|
-
type ReplyMediaFileTransferParams,
|
|
299
|
-
type ReplyPayloadInput,
|
|
300
|
-
} from './messaging/outbound/reply-enqueue.ts';
|
|
301
|
-
import {
|
|
302
|
-
OUTBOUND_DEGRADE_REASON,
|
|
303
|
-
OUTBOUND_FLUSH_REASON,
|
|
304
|
-
OUTBOUND_FLUSH_TRIGGER,
|
|
305
|
-
OUTBOUND_SCHEDULE_SOURCE,
|
|
306
|
-
OUTBOUND_TERMINAL_REASON,
|
|
307
|
-
} from './messaging/outbound/reasons.ts';
|
|
308
293
|
import {
|
|
309
294
|
buildOutboxOnlineDebugInfo,
|
|
310
295
|
clampOutboxDrainDelay,
|
|
@@ -317,30 +302,47 @@ import {
|
|
|
317
302
|
selectOutboxTargetAccounts,
|
|
318
303
|
updateMinOutboxDelay,
|
|
319
304
|
} from './messaging/outbound/queue-selectors.ts';
|
|
305
|
+
import {
|
|
306
|
+
OUTBOUND_DEGRADE_REASON,
|
|
307
|
+
OUTBOUND_FLUSH_REASON,
|
|
308
|
+
OUTBOUND_FLUSH_TRIGGER,
|
|
309
|
+
OUTBOUND_SCHEDULE_SOURCE,
|
|
310
|
+
OUTBOUND_TERMINAL_REASON,
|
|
311
|
+
} from './messaging/outbound/reasons.ts';
|
|
312
|
+
import {
|
|
313
|
+
type NormalizedReplyPayload,
|
|
314
|
+
type ReplyMediaEntriesParams,
|
|
315
|
+
type ReplyMediaFileTransferParams,
|
|
316
|
+
type ReplyPayloadInput,
|
|
317
|
+
buildReplyTextOutboxEntry,
|
|
318
|
+
enqueueNormalizedReplyPayload,
|
|
319
|
+
enqueueReplyMediaFallbackTextEntry,
|
|
320
|
+
enqueueReplyMediaFileTransferEntry,
|
|
321
|
+
enqueueReplyTextEntry,
|
|
322
|
+
enqueueSingleReplyMediaEntry,
|
|
323
|
+
hasReplyMediaEntries,
|
|
324
|
+
normalizeReplyPayload,
|
|
325
|
+
} from './messaging/outbound/reply-enqueue.ts';
|
|
320
326
|
import {
|
|
321
327
|
computePushFailureDecision,
|
|
322
328
|
computeRetryRerouteDecision,
|
|
323
329
|
} from './messaging/outbound/retry-policy.ts';
|
|
324
330
|
import { sendBncrMedia, sendBncrText } from './messaging/outbound/send.ts';
|
|
325
|
-
import { buildBncrDurableQueuedResult } from './messaging/outbound/durable-queue-adapter.ts';
|
|
326
331
|
import { BNCR_CHANNEL_CAPABILITIES } from './plugin/capabilities.ts';
|
|
327
332
|
import { BNCR_CONFIG_SURFACE } from './plugin/config.ts';
|
|
328
|
-
import { createBncrGatewayRuntime } from './plugin/gateway-runtime.ts';
|
|
329
333
|
import { BNCR_GATEWAY_METHODS } from './plugin/gateway-methods.ts';
|
|
330
|
-
import {
|
|
331
|
-
import { createBncrMessagingSurface } from './plugin/messaging.ts';
|
|
332
|
-
import { createBncrMessageSend } from './plugin/message-send.ts';
|
|
334
|
+
import { createBncrGatewayRuntime } from './plugin/gateway-runtime.ts';
|
|
333
335
|
import { BNCR_MESSAGE_RECEIVE_POLICY } from './plugin/message-policy.ts';
|
|
336
|
+
import { createBncrMessageSend } from './plugin/message-send.ts';
|
|
337
|
+
import { createBncrMessagingSurface } from './plugin/messaging.ts';
|
|
338
|
+
import { BNCR_CHANNEL_META } from './plugin/meta.ts';
|
|
334
339
|
import { createBncrOutboundRuntime } from './plugin/outbound.ts';
|
|
335
340
|
import { BNCR_SETUP_SURFACE } from './plugin/setup.ts';
|
|
336
341
|
import { createBncrStatusSurface } from './plugin/status.ts';
|
|
337
342
|
import {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
stopBncrStatusWorker,
|
|
342
|
-
type ChannelAccountWorkerHandle,
|
|
343
|
-
} from './runtime/status-worker.ts';
|
|
343
|
+
pruneLogDedupeState as pruneLogDedupeStateFromRuntime,
|
|
344
|
+
shouldEmitDedupLog as shouldEmitDedupLogFromRuntime,
|
|
345
|
+
} from './runtime/log-dedupe.ts';
|
|
344
346
|
import {
|
|
345
347
|
buildBncrRuntimeAckStrategy,
|
|
346
348
|
computeBncrRecommendedAckTimeoutMs,
|
|
@@ -351,7 +353,6 @@ import {
|
|
|
351
353
|
buildBncrRuntimeStatusInput,
|
|
352
354
|
resolveBncrOutboundAckRequired,
|
|
353
355
|
} from './runtime/outbound-flags.ts';
|
|
354
|
-
import { buildRuntimeStatusSnapshots } from './runtime/status-snapshots.ts';
|
|
355
356
|
import {
|
|
356
357
|
applyBncrPushFailureDecisionToEntry,
|
|
357
358
|
applyBncrRetryRerouteDecisionToEntry,
|
|
@@ -360,10 +361,14 @@ import {
|
|
|
360
361
|
buildBncrOutboxFailureEntryPatch,
|
|
361
362
|
buildBncrOutboxPushSuccessEntryPatch,
|
|
362
363
|
} from './runtime/outbox-transitions.ts';
|
|
364
|
+
import { buildRuntimeStatusSnapshots } from './runtime/status-snapshots.ts';
|
|
363
365
|
import {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
366
|
+
type ChannelAccountWorkerHandle,
|
|
367
|
+
clearAllBncrStatusWorkers,
|
|
368
|
+
clearBncrStatusWorker,
|
|
369
|
+
startBncrStatusWorker,
|
|
370
|
+
stopBncrStatusWorker,
|
|
371
|
+
} from './runtime/status-worker.ts';
|
|
367
372
|
const BRIDGE_VERSION = 2;
|
|
368
373
|
const BNCR_PUSH_EVENT = 'plugin.bncr.push';
|
|
369
374
|
const BNCR_FILE_INIT_EVENT = 'plugin.bncr.file.init';
|
|
@@ -379,6 +384,9 @@ const MAX_ACCOUNT_ACTIVITY_ENTRIES = 1000;
|
|
|
379
384
|
const PUSH_DRAIN_INTERVAL_MS = 500;
|
|
380
385
|
const PUSH_DRAIN_ACCOUNT_BUDGET = 5;
|
|
381
386
|
const PUSH_DRAIN_ACCOUNT_TIME_BUDGET_MS = 2_000;
|
|
387
|
+
const PUSH_DRAIN_EXCEPTION_RETRY_LIMIT = 3;
|
|
388
|
+
const PUSH_DRAIN_EXCEPTION_RETRY_DELAY_MS = 1_000;
|
|
389
|
+
const PUSH_DRAIN_STUCK_WARN_MS = 30_000;
|
|
382
390
|
const PUSH_ACK_TIMEOUT_MS = 30_000;
|
|
383
391
|
const ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED = true;
|
|
384
392
|
const RECOMMENDED_ACK_TIMEOUT_MIN_MS = PUSH_ACK_TIMEOUT_MS;
|
|
@@ -392,7 +400,8 @@ const FILE_FORCE_CHUNK = true; // 统一走 WS 分块,保留 base64 仅作兜
|
|
|
392
400
|
const FILE_INLINE_THRESHOLD = 5 * 1024 * 1024; // fallback 阈值(仅 FILE_FORCE_CHUNK=false 时生效)
|
|
393
401
|
const FILE_CHUNK_SIZE = 256 * 1024; // 256KB
|
|
394
402
|
const INBOUND_FILE_TRANSFER_MAX_BYTES = 50 * 1024 * 1024;
|
|
395
|
-
const INBOUND_FILE_TRANSFER_MAX_CHUNKS =
|
|
403
|
+
const INBOUND_FILE_TRANSFER_MAX_CHUNKS =
|
|
404
|
+
Math.ceil(INBOUND_FILE_TRANSFER_MAX_BYTES / FILE_CHUNK_SIZE) + 1;
|
|
396
405
|
const FILE_CHUNK_RETRY = 3;
|
|
397
406
|
const FILE_ACK_TIMEOUT_MS = 30_000;
|
|
398
407
|
const FILE_TRANSFER_ACK_TTL_MS = 30_000;
|
|
@@ -558,7 +567,6 @@ function normalizeBncrSendParams(input: {
|
|
|
558
567
|
};
|
|
559
568
|
}
|
|
560
569
|
|
|
561
|
-
|
|
562
570
|
function now() {
|
|
563
571
|
return Date.now();
|
|
564
572
|
}
|
|
@@ -755,7 +763,10 @@ class BncrBridgeRuntime {
|
|
|
755
763
|
private lastLateAckQueueLatencyMsByAccount = new Map<string, number>();
|
|
756
764
|
private lastLateAckPushLatencyMsByAccount = new Map<string, number>();
|
|
757
765
|
private adaptiveAckRecoveryOkCountByAccount = new Map<string, number>();
|
|
758
|
-
private adaptiveAckTimeoutLogStateByAccount = new Map<
|
|
766
|
+
private adaptiveAckTimeoutLogStateByAccount = new Map<
|
|
767
|
+
string,
|
|
768
|
+
{ at: number; timeoutMs: number; reason: string }
|
|
769
|
+
>();
|
|
759
770
|
private channelAccountWorkers = new Map<string, ChannelAccountWorkerHandle>();
|
|
760
771
|
private logDedupeState = new Map<string, { at: number; sig: string }>();
|
|
761
772
|
private canonicalAgentId: string | null = null;
|
|
@@ -773,6 +784,9 @@ class BncrBridgeRuntime {
|
|
|
773
784
|
private saveTimer: NodeJS.Timeout | null = null;
|
|
774
785
|
private pushTimer: NodeJS.Timeout | null = null;
|
|
775
786
|
private pushDrainRunningAccounts = new Set<string>();
|
|
787
|
+
private pushDrainRunningSinceByAccount = new Map<string, number>();
|
|
788
|
+
private pushDrainStuckWarnedAtByAccount = new Map<string, number>();
|
|
789
|
+
private pushDrainExceptionRetryCount = 0;
|
|
776
790
|
private messageAckWaiters = new Map<
|
|
777
791
|
// Refactor boundary note (message ACK runtime):
|
|
778
792
|
// These waiters are part of the outbound message-ack lifecycle, not just a utility map.
|
|
@@ -1803,7 +1817,9 @@ class BncrBridgeRuntime {
|
|
|
1803
1817
|
const primaryKey = this.activeConnectionByAccount.get(acc);
|
|
1804
1818
|
const primary = primaryKey ? this.connections.get(primaryKey) : null;
|
|
1805
1819
|
|
|
1806
|
-
const isEligible = (
|
|
1820
|
+
const isEligible = (
|
|
1821
|
+
conn: BncrConnection | null | undefined,
|
|
1822
|
+
): conn is BncrConnection & {
|
|
1807
1823
|
outboundReadyUntil?: number;
|
|
1808
1824
|
preferredForOutboundUntil?: number;
|
|
1809
1825
|
inboundOnly?: boolean;
|
|
@@ -1848,10 +1864,13 @@ class BncrBridgeRuntime {
|
|
|
1848
1864
|
const sb = candidateScore(b);
|
|
1849
1865
|
if (sb.preferred !== sa.preferred) return sb.preferred - sa.preferred;
|
|
1850
1866
|
if (sb.ready !== sa.ready) return sb.ready - sa.ready;
|
|
1851
|
-
if (sa.recentTimeoutPenalty !== sb.recentTimeoutPenalty)
|
|
1852
|
-
|
|
1867
|
+
if (sa.recentTimeoutPenalty !== sb.recentTimeoutPenalty)
|
|
1868
|
+
return sa.recentTimeoutPenalty - sb.recentTimeoutPenalty;
|
|
1869
|
+
if (sa.pushFailureScore !== sb.pushFailureScore)
|
|
1870
|
+
return sa.pushFailureScore - sb.pushFailureScore;
|
|
1853
1871
|
if (sb.lastAckOkAt !== sa.lastAckOkAt) return sb.lastAckOkAt - sa.lastAckOkAt;
|
|
1854
|
-
if (sa.lastPushTimeoutAt !== sb.lastPushTimeoutAt)
|
|
1872
|
+
if (sa.lastPushTimeoutAt !== sb.lastPushTimeoutAt)
|
|
1873
|
+
return sa.lastPushTimeoutAt - sb.lastPushTimeoutAt;
|
|
1855
1874
|
if (sb.recentInbound !== sa.recentInbound) return sb.recentInbound - sa.recentInbound;
|
|
1856
1875
|
if (sb.lastSeenAt !== sa.lastSeenAt) return sb.lastSeenAt - sa.lastSeenAt;
|
|
1857
1876
|
return sb.connectedAt - sa.connectedAt;
|
|
@@ -1899,7 +1918,9 @@ class BncrBridgeRuntime {
|
|
|
1899
1918
|
const t = now();
|
|
1900
1919
|
const connIds = new Set<string>();
|
|
1901
1920
|
|
|
1902
|
-
const isEligible = (
|
|
1921
|
+
const isEligible = (
|
|
1922
|
+
conn: BncrConnection | null | undefined,
|
|
1923
|
+
): conn is BncrConnection & {
|
|
1903
1924
|
outboundReadyUntil?: number;
|
|
1904
1925
|
preferredForOutboundUntil?: number;
|
|
1905
1926
|
inboundOnly?: boolean;
|
|
@@ -1947,10 +1968,13 @@ class BncrBridgeRuntime {
|
|
|
1947
1968
|
const sb = candidateScore(b);
|
|
1948
1969
|
if (sb.preferred !== sa.preferred) return sb.preferred - sa.preferred;
|
|
1949
1970
|
if (sb.ready !== sa.ready) return sb.ready - sa.ready;
|
|
1950
|
-
if (sa.recentTimeoutPenalty !== sb.recentTimeoutPenalty)
|
|
1951
|
-
|
|
1971
|
+
if (sa.recentTimeoutPenalty !== sb.recentTimeoutPenalty)
|
|
1972
|
+
return sa.recentTimeoutPenalty - sb.recentTimeoutPenalty;
|
|
1973
|
+
if (sa.pushFailureScore !== sb.pushFailureScore)
|
|
1974
|
+
return sa.pushFailureScore - sb.pushFailureScore;
|
|
1952
1975
|
if (sb.lastAckOkAt !== sa.lastAckOkAt) return sb.lastAckOkAt - sa.lastAckOkAt;
|
|
1953
|
-
if (sa.lastPushTimeoutAt !== sb.lastPushTimeoutAt)
|
|
1976
|
+
if (sa.lastPushTimeoutAt !== sb.lastPushTimeoutAt)
|
|
1977
|
+
return sa.lastPushTimeoutAt - sb.lastPushTimeoutAt;
|
|
1954
1978
|
if (sb.recentInbound !== sa.recentInbound) return sb.recentInbound - sa.recentInbound;
|
|
1955
1979
|
if (sb.lastSeenAt !== sa.lastSeenAt) return sb.lastSeenAt - sa.lastSeenAt;
|
|
1956
1980
|
return sb.connectedAt - sa.connectedAt;
|
|
@@ -2034,10 +2058,7 @@ class BncrBridgeRuntime {
|
|
|
2034
2058
|
|
|
2035
2059
|
private tryAdoptTransferOwner(args: {
|
|
2036
2060
|
accountId: string;
|
|
2037
|
-
transfer:
|
|
2038
|
-
| FileSendTransferState
|
|
2039
|
-
| FileRecvTransferState
|
|
2040
|
-
| undefined;
|
|
2061
|
+
transfer: FileSendTransferState | FileRecvTransferState | undefined;
|
|
2041
2062
|
connId: string;
|
|
2042
2063
|
clientId?: string;
|
|
2043
2064
|
}): boolean {
|
|
@@ -2169,6 +2190,7 @@ class BncrBridgeRuntime {
|
|
|
2169
2190
|
this.recordOutboxPrePushFailure({
|
|
2170
2191
|
entry: args.entry,
|
|
2171
2192
|
lastError: args.guard.lastError,
|
|
2193
|
+
persist: true,
|
|
2172
2194
|
});
|
|
2173
2195
|
if (args.guard.reason === 'media-url-missing') {
|
|
2174
2196
|
this.logOutboxPushFailure({
|
|
@@ -2184,9 +2206,12 @@ class BncrBridgeRuntime {
|
|
|
2184
2206
|
messageId: args.entry.messageId,
|
|
2185
2207
|
accountId: args.entry.accountId,
|
|
2186
2208
|
kind: 'file-transfer',
|
|
2187
|
-
reason:
|
|
2209
|
+
reason:
|
|
2210
|
+
args.guard.reason === 'no-gateway-context' ? 'no-gateway-context' : 'no-active-connection',
|
|
2188
2211
|
recentInboundReachable:
|
|
2189
|
-
args.guard.reason === 'no-active-connection'
|
|
2212
|
+
args.guard.reason === 'no-active-connection'
|
|
2213
|
+
? args.guard.recentInboundReachable
|
|
2214
|
+
: undefined,
|
|
2190
2215
|
});
|
|
2191
2216
|
}
|
|
2192
2217
|
|
|
@@ -2479,12 +2504,24 @@ class BncrBridgeRuntime {
|
|
|
2479
2504
|
routeSelection: selection,
|
|
2480
2505
|
});
|
|
2481
2506
|
if (!guard.ok) {
|
|
2507
|
+
this.recordOutboxPrePushFailure({
|
|
2508
|
+
entry,
|
|
2509
|
+
lastError:
|
|
2510
|
+
guard.reason === 'no-gateway-context'
|
|
2511
|
+
? 'gateway context unavailable'
|
|
2512
|
+
: 'no active bncr client',
|
|
2513
|
+
persist: true,
|
|
2514
|
+
});
|
|
2482
2515
|
this.logOutboxPushSkip({
|
|
2483
2516
|
messageId: entry.messageId,
|
|
2484
2517
|
accountId: entry.accountId,
|
|
2485
2518
|
reason: guard.reason,
|
|
2486
2519
|
recentInboundReachable:
|
|
2487
2520
|
guard.reason === 'no-active-connection' ? guard.recentInboundReachable : undefined,
|
|
2521
|
+
routeReason: selection.routeReason,
|
|
2522
|
+
connIds: selection.connIds,
|
|
2523
|
+
ownerConnId: selection.ownerConnId,
|
|
2524
|
+
ownerClientId: owner?.clientId,
|
|
2488
2525
|
});
|
|
2489
2526
|
return false;
|
|
2490
2527
|
}
|
|
@@ -2516,10 +2553,24 @@ class BncrBridgeRuntime {
|
|
|
2516
2553
|
kind?: 'file-transfer';
|
|
2517
2554
|
reason: string;
|
|
2518
2555
|
recentInboundReachable?: boolean;
|
|
2556
|
+
routeReason?: string;
|
|
2557
|
+
connIds?: Iterable<string>;
|
|
2558
|
+
ownerConnId?: string;
|
|
2559
|
+
ownerClientId?: string;
|
|
2519
2560
|
}) {
|
|
2561
|
+
this.logInfo(
|
|
2562
|
+
'outbox push skip',
|
|
2563
|
+
`mid=${args.messageId}|q=${this.outbox.size}|reason=${args.reason}${args.kind ? `|kind=${args.kind}` : ''}`,
|
|
2564
|
+
);
|
|
2520
2565
|
this.logInfo(
|
|
2521
2566
|
'outbox',
|
|
2522
|
-
`push-skip ${JSON.stringify(
|
|
2567
|
+
`push-skip ${JSON.stringify(
|
|
2568
|
+
buildOutboxPushSkipDebugInfo({
|
|
2569
|
+
...args,
|
|
2570
|
+
activeConnectionCount: this.activeConnectionCount(args.accountId),
|
|
2571
|
+
connections: this.connections.values(),
|
|
2572
|
+
}),
|
|
2573
|
+
)}`,
|
|
2523
2574
|
{ debugOnly: true },
|
|
2524
2575
|
);
|
|
2525
2576
|
}
|
|
@@ -2550,11 +2601,9 @@ class BncrBridgeRuntime {
|
|
|
2550
2601
|
retryable?: boolean;
|
|
2551
2602
|
lastError?: string;
|
|
2552
2603
|
}) {
|
|
2553
|
-
this.logInfo(
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
{ debugOnly: true },
|
|
2557
|
-
);
|
|
2604
|
+
this.logInfo('outbox', `push-fail ${JSON.stringify(buildPushFailureDebugInfo(args))}`, {
|
|
2605
|
+
debugOnly: true,
|
|
2606
|
+
});
|
|
2558
2607
|
}
|
|
2559
2608
|
|
|
2560
2609
|
private logOutboxPushOkSummary(messageId: string) {
|
|
@@ -2607,14 +2656,18 @@ class BncrBridgeRuntime {
|
|
|
2607
2656
|
sessionKey: args.entry.sessionKey,
|
|
2608
2657
|
to: formatDisplayScope(args.entry.route),
|
|
2609
2658
|
kind:
|
|
2610
|
-
isPlainObject(args.entry.payload?._meta) &&
|
|
2659
|
+
isPlainObject(args.entry.payload?._meta) &&
|
|
2660
|
+
args.entry.payload?._meta?.kind === 'file-transfer'
|
|
2611
2661
|
? 'file-transfer'
|
|
2612
2662
|
: undefined,
|
|
2613
2663
|
requireAck: args.requireAck,
|
|
2614
2664
|
ackResult: args.ackResult,
|
|
2615
2665
|
ackStage: 'message',
|
|
2616
2666
|
ackOutcome: args.ackResult,
|
|
2617
|
-
reason:
|
|
2667
|
+
reason:
|
|
2668
|
+
args.ackResult === 'timeout'
|
|
2669
|
+
? OUTBOUND_TERMINAL_REASON.PUSH_ACK_TIMEOUT
|
|
2670
|
+
: 'message-acked',
|
|
2618
2671
|
ackTimeoutMs: typeof args.ackTimeoutMs === 'number' ? args.ackTimeoutMs : undefined,
|
|
2619
2672
|
adaptiveAckTimeoutEnabled: ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED,
|
|
2620
2673
|
onlineNow: args.onlineNow,
|
|
@@ -2639,16 +2692,13 @@ class BncrBridgeRuntime {
|
|
|
2639
2692
|
localNextDelay: number | null;
|
|
2640
2693
|
ackTimeoutMs?: number | null;
|
|
2641
2694
|
}) {
|
|
2642
|
-
this.logOutboxAckSummary(
|
|
2643
|
-
args.
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
waitMs: args.requireAck ? args.ackTimeoutMs : undefined,
|
|
2650
|
-
},
|
|
2651
|
-
);
|
|
2695
|
+
this.logOutboxAckSummary(args.requireAck ? 'outbox ack timeout' : 'outbox ack retry', {
|
|
2696
|
+
messageId: args.entry.messageId,
|
|
2697
|
+
connId: args.entry.lastPushConnId,
|
|
2698
|
+
clientId: args.entry.lastPushClientId,
|
|
2699
|
+
err: args.requireAck ? undefined : args.entry.lastError,
|
|
2700
|
+
waitMs: args.requireAck ? args.ackTimeoutMs : undefined,
|
|
2701
|
+
});
|
|
2652
2702
|
this.logInfo(
|
|
2653
2703
|
'outbox',
|
|
2654
2704
|
`retry-reroute ${JSON.stringify(
|
|
@@ -2684,12 +2734,7 @@ class BncrBridgeRuntime {
|
|
|
2684
2734
|
stale: boolean,
|
|
2685
2735
|
result: { ok: true; movedToDeadLetter?: true; willRetry?: true },
|
|
2686
2736
|
) {
|
|
2687
|
-
respond(
|
|
2688
|
-
true,
|
|
2689
|
-
stale
|
|
2690
|
-
? { ...result, stale: true, staleAccepted: true }
|
|
2691
|
-
: result,
|
|
2692
|
-
);
|
|
2737
|
+
respond(true, stale ? { ...result, stale: true, staleAccepted: true } : result);
|
|
2693
2738
|
}
|
|
2694
2739
|
|
|
2695
2740
|
private prepareAckHandling(args: {
|
|
@@ -2896,7 +2941,7 @@ class BncrBridgeRuntime {
|
|
|
2896
2941
|
entry,
|
|
2897
2942
|
});
|
|
2898
2943
|
this.respondAckResult(respond, staleObserved.stale, { ok: true });
|
|
2899
|
-
this.
|
|
2944
|
+
this.flushPushQueueBestEffort({
|
|
2900
2945
|
accountId,
|
|
2901
2946
|
trigger: OUTBOUND_FLUSH_TRIGGER.ACK_OK,
|
|
2902
2947
|
reason: OUTBOUND_FLUSH_REASON.MESSAGE_ACKED,
|
|
@@ -3037,7 +3082,15 @@ class BncrBridgeRuntime {
|
|
|
3037
3082
|
inboundOnly: boolean;
|
|
3038
3083
|
context: GatewayRequestHandlerOptions['context'];
|
|
3039
3084
|
}) {
|
|
3040
|
-
const {
|
|
3085
|
+
const {
|
|
3086
|
+
accountId,
|
|
3087
|
+
connId,
|
|
3088
|
+
clientId,
|
|
3089
|
+
outboundReady,
|
|
3090
|
+
preferredForOutbound,
|
|
3091
|
+
inboundOnly,
|
|
3092
|
+
context,
|
|
3093
|
+
} = args;
|
|
3041
3094
|
this.refreshAcceptedFileTransferLiveState({
|
|
3042
3095
|
accountId,
|
|
3043
3096
|
connId,
|
|
@@ -3076,16 +3129,15 @@ class BncrBridgeRuntime {
|
|
|
3076
3129
|
recentInboundReachable: boolean;
|
|
3077
3130
|
event: string;
|
|
3078
3131
|
}) {
|
|
3079
|
-
this.logInfo(
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
{ debugOnly: true },
|
|
3083
|
-
);
|
|
3132
|
+
this.logInfo('outbox', `push ${JSON.stringify(buildOutboxPushOkDebugInfo(args))}`, {
|
|
3133
|
+
debugOnly: true,
|
|
3134
|
+
});
|
|
3084
3135
|
}
|
|
3085
3136
|
|
|
3086
3137
|
private recordOutboxPrePushFailure(args: {
|
|
3087
3138
|
entry: OutboxEntry;
|
|
3088
3139
|
lastError: string;
|
|
3140
|
+
persist?: boolean;
|
|
3089
3141
|
}) {
|
|
3090
3142
|
const nextEntry = buildBncrOutboxFailureEntryPatch({
|
|
3091
3143
|
entry: args.entry,
|
|
@@ -3093,6 +3145,7 @@ class BncrBridgeRuntime {
|
|
|
3093
3145
|
});
|
|
3094
3146
|
Object.assign(args.entry, nextEntry);
|
|
3095
3147
|
this.outbox.set(nextEntry.messageId, args.entry);
|
|
3148
|
+
if (args.persist) this.scheduleSave();
|
|
3096
3149
|
}
|
|
3097
3150
|
|
|
3098
3151
|
private recordOutboxPushFailure(args: {
|
|
@@ -3156,13 +3209,39 @@ class BncrBridgeRuntime {
|
|
|
3156
3209
|
this.pushTimer = setTimeout(() => {
|
|
3157
3210
|
this.pushTimer = null;
|
|
3158
3211
|
if (this.stopped) return;
|
|
3159
|
-
|
|
3212
|
+
this.flushPushQueueBestEffort({
|
|
3160
3213
|
trigger: OUTBOUND_FLUSH_TRIGGER.TIMER,
|
|
3161
3214
|
reason: OUTBOUND_FLUSH_REASON.SCHEDULED_DRAIN,
|
|
3162
3215
|
});
|
|
3163
3216
|
}, delay);
|
|
3164
3217
|
}
|
|
3165
3218
|
|
|
3219
|
+
private flushPushQueueBestEffort(args?: {
|
|
3220
|
+
accountId?: string;
|
|
3221
|
+
trigger?: string;
|
|
3222
|
+
reason?: string;
|
|
3223
|
+
}) {
|
|
3224
|
+
void this.flushPushQueue(args)
|
|
3225
|
+
.then(() => {
|
|
3226
|
+
this.pushDrainExceptionRetryCount = 0;
|
|
3227
|
+
})
|
|
3228
|
+
.catch((error) => {
|
|
3229
|
+
const accountId = args?.accountId ? normalizeAccountId(args.accountId) : '';
|
|
3230
|
+
const reason = asString(args?.reason || args?.trigger || 'flush-error');
|
|
3231
|
+
const err = asString((error as any)?.message || error || 'flush-error');
|
|
3232
|
+
const nextRetryCount = this.pushDrainExceptionRetryCount + 1;
|
|
3233
|
+
const willRetry = nextRetryCount <= PUSH_DRAIN_EXCEPTION_RETRY_LIMIT;
|
|
3234
|
+
this.pushDrainExceptionRetryCount = nextRetryCount;
|
|
3235
|
+
this.logError(
|
|
3236
|
+
'outbox drain fail',
|
|
3237
|
+
`accountId=${accountId || '-'}|reason=${reason}|err=${err}|retry=${willRetry ? nextRetryCount : 'false'}|limit=${PUSH_DRAIN_EXCEPTION_RETRY_LIMIT}`,
|
|
3238
|
+
);
|
|
3239
|
+
if (willRetry) {
|
|
3240
|
+
this.schedulePushDrain(PUSH_DRAIN_EXCEPTION_RETRY_DELAY_MS);
|
|
3241
|
+
}
|
|
3242
|
+
});
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3166
3245
|
private isOutboundAckRequired(accountId?: string) {
|
|
3167
3246
|
return resolveBncrOutboundAckRequired({ api: this.api, accountId });
|
|
3168
3247
|
}
|
|
@@ -3179,6 +3258,60 @@ class BncrBridgeRuntime {
|
|
|
3179
3258
|
});
|
|
3180
3259
|
}
|
|
3181
3260
|
|
|
3261
|
+
private getAccountPendingOutboxEntries(accountId: string) {
|
|
3262
|
+
const acc = normalizeAccountId(accountId);
|
|
3263
|
+
return Array.from(this.outbox.values()).filter((entry) => entry.accountId === acc);
|
|
3264
|
+
}
|
|
3265
|
+
|
|
3266
|
+
private maybeLogOutboxDrainStuck(args: {
|
|
3267
|
+
accountId: string;
|
|
3268
|
+
trigger: string;
|
|
3269
|
+
reason: string;
|
|
3270
|
+
}) {
|
|
3271
|
+
const acc = normalizeAccountId(args.accountId);
|
|
3272
|
+
const startedAt = this.pushDrainRunningSinceByAccount.get(acc) || 0;
|
|
3273
|
+
if (!startedAt) return;
|
|
3274
|
+
|
|
3275
|
+
const t = now();
|
|
3276
|
+
const runningMs = Math.max(0, t - startedAt);
|
|
3277
|
+
if (runningMs < PUSH_DRAIN_STUCK_WARN_MS) return;
|
|
3278
|
+
|
|
3279
|
+
const lastWarnedAt = this.pushDrainStuckWarnedAtByAccount.get(acc) || 0;
|
|
3280
|
+
if (lastWarnedAt && t - lastWarnedAt < PUSH_DRAIN_STUCK_WARN_MS) return;
|
|
3281
|
+
|
|
3282
|
+
const pendingEntries = this.getAccountPendingOutboxEntries(acc);
|
|
3283
|
+
const pending = pendingEntries.length;
|
|
3284
|
+
if (!pending) return;
|
|
3285
|
+
|
|
3286
|
+
this.pushDrainStuckWarnedAtByAccount.set(acc, t);
|
|
3287
|
+
this.logWarn(
|
|
3288
|
+
'outbox drain stuck',
|
|
3289
|
+
`accountId=${acc}|pending=${pending}|runningMs=${runningMs}|waiters=${this.messageAckWaiters.size}/${this.fileAckWaiters.size}`,
|
|
3290
|
+
);
|
|
3291
|
+
this.logInfo(
|
|
3292
|
+
'outbox',
|
|
3293
|
+
`drain-stuck ${JSON.stringify(
|
|
3294
|
+
buildOutboxDrainStuckDebugInfo({
|
|
3295
|
+
bridgeId: this.bridgeId,
|
|
3296
|
+
accountId: acc,
|
|
3297
|
+
reason: args.reason,
|
|
3298
|
+
trigger: args.trigger,
|
|
3299
|
+
outboxSize: this.outbox.size,
|
|
3300
|
+
pending,
|
|
3301
|
+
runningMs,
|
|
3302
|
+
runningSince: startedAt,
|
|
3303
|
+
hasGatewayContext: Boolean(this.gatewayContext),
|
|
3304
|
+
activeConnectionCount: this.activeConnectionCount(acc),
|
|
3305
|
+
messageAckWaiters: this.messageAckWaiters.size,
|
|
3306
|
+
fileAckWaiters: this.fileAckWaiters.size,
|
|
3307
|
+
pendingEntries,
|
|
3308
|
+
connections: this.connections.values(),
|
|
3309
|
+
}),
|
|
3310
|
+
)}`,
|
|
3311
|
+
{ debugOnly: true },
|
|
3312
|
+
);
|
|
3313
|
+
}
|
|
3314
|
+
|
|
3182
3315
|
private async flushPushQueue(args?: {
|
|
3183
3316
|
accountId?: string;
|
|
3184
3317
|
trigger?: string;
|
|
@@ -3236,7 +3369,28 @@ class BncrBridgeRuntime {
|
|
|
3236
3369
|
let globalNextDelay: number | null = null;
|
|
3237
3370
|
|
|
3238
3371
|
for (const acc of targetAccounts) {
|
|
3239
|
-
if (!acc
|
|
3372
|
+
if (!acc) continue;
|
|
3373
|
+
if (this.pushDrainRunningAccounts.has(acc)) {
|
|
3374
|
+
this.logInfo(
|
|
3375
|
+
'outbox',
|
|
3376
|
+
`drain-skip ${JSON.stringify(
|
|
3377
|
+
buildOutboxDrainSkipDebugInfo({
|
|
3378
|
+
bridgeId: this.bridgeId,
|
|
3379
|
+
accountId: acc,
|
|
3380
|
+
reason: 'already-running',
|
|
3381
|
+
outboxSize: this.outbox.size,
|
|
3382
|
+
trigger,
|
|
3383
|
+
}),
|
|
3384
|
+
)}`,
|
|
3385
|
+
{ debugOnly: true },
|
|
3386
|
+
);
|
|
3387
|
+
this.maybeLogOutboxDrainStuck({
|
|
3388
|
+
accountId: acc,
|
|
3389
|
+
trigger,
|
|
3390
|
+
reason: reason || 'already-running',
|
|
3391
|
+
});
|
|
3392
|
+
continue;
|
|
3393
|
+
}
|
|
3240
3394
|
const online = this.isOnline(acc);
|
|
3241
3395
|
const recentInboundReachable = this.hasRecentInboundReachability(acc);
|
|
3242
3396
|
this.logInfo(
|
|
@@ -3253,6 +3407,8 @@ class BncrBridgeRuntime {
|
|
|
3253
3407
|
{ debugOnly: true },
|
|
3254
3408
|
);
|
|
3255
3409
|
this.pushDrainRunningAccounts.add(acc);
|
|
3410
|
+
this.pushDrainRunningSinceByAccount.set(acc, now());
|
|
3411
|
+
this.pushDrainStuckWarnedAtByAccount.delete(acc);
|
|
3256
3412
|
try {
|
|
3257
3413
|
let localNextDelay: number | null = null;
|
|
3258
3414
|
let processedThisRun = 0;
|
|
@@ -3260,7 +3416,10 @@ class BncrBridgeRuntime {
|
|
|
3260
3416
|
|
|
3261
3417
|
while (true) {
|
|
3262
3418
|
if (this.stopped) break;
|
|
3263
|
-
if (
|
|
3419
|
+
if (
|
|
3420
|
+
processedThisRun > 0 &&
|
|
3421
|
+
now() - accountDrainStartedAt >= PUSH_DRAIN_ACCOUNT_TIME_BUDGET_MS
|
|
3422
|
+
) {
|
|
3264
3423
|
localNextDelay = updateMinOutboxDelay(localNextDelay, 0);
|
|
3265
3424
|
this.logInfo(
|
|
3266
3425
|
'outbox',
|
|
@@ -3327,14 +3486,63 @@ class BncrBridgeRuntime {
|
|
|
3327
3486
|
|
|
3328
3487
|
const onlineNow = this.isOnline(acc);
|
|
3329
3488
|
const recentInboundReachable = this.hasRecentInboundReachability(acc);
|
|
3330
|
-
|
|
3489
|
+
let pushed = false;
|
|
3490
|
+
try {
|
|
3491
|
+
pushed = await this.tryPushEntry(entry);
|
|
3492
|
+
} catch (error) {
|
|
3493
|
+
const meta = isPlainObject(entry.payload?._meta) ? entry.payload._meta : null;
|
|
3494
|
+
if (meta?.kind === 'file-transfer') {
|
|
3495
|
+
this.handleFileTransferPushFailure({
|
|
3496
|
+
entry,
|
|
3497
|
+
error,
|
|
3498
|
+
});
|
|
3499
|
+
} else {
|
|
3500
|
+
this.handleTextPushFailure({
|
|
3501
|
+
entry,
|
|
3502
|
+
error,
|
|
3503
|
+
});
|
|
3504
|
+
}
|
|
3505
|
+
pushed = false;
|
|
3506
|
+
}
|
|
3331
3507
|
processedThisRun += 1;
|
|
3332
3508
|
if (pushed) {
|
|
3333
3509
|
const requireAck = this.isOutboundAckRequired(acc);
|
|
3334
3510
|
const ackTimeoutMs = requireAck ? this.resolveMessageAckTimeoutMs(acc) : null;
|
|
3335
3511
|
let ackResult: 'acked' | 'timeout' = requireAck ? 'timeout' : 'acked';
|
|
3336
3512
|
if (onlineNow && requireAck) {
|
|
3337
|
-
|
|
3513
|
+
this.logInfo(
|
|
3514
|
+
'outbox',
|
|
3515
|
+
`ack wait-start ${JSON.stringify(
|
|
3516
|
+
buildOutboxAckDebugInfo({
|
|
3517
|
+
messageId: entry.messageId,
|
|
3518
|
+
accountId: entry.accountId,
|
|
3519
|
+
sessionKey: entry.sessionKey,
|
|
3520
|
+
to: formatDisplayScope(entry.route),
|
|
3521
|
+
kind:
|
|
3522
|
+
isPlainObject(entry.payload?._meta) &&
|
|
3523
|
+
entry.payload?._meta?.kind === 'file-transfer'
|
|
3524
|
+
? 'file-transfer'
|
|
3525
|
+
: undefined,
|
|
3526
|
+
requireAck,
|
|
3527
|
+
ackResult: 'timeout',
|
|
3528
|
+
ackStage: 'message',
|
|
3529
|
+
ackOutcome: 'waiting',
|
|
3530
|
+
ackTimeoutMs: ackTimeoutMs || PUSH_ACK_TIMEOUT_MS,
|
|
3531
|
+
adaptiveAckTimeoutEnabled: ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED,
|
|
3532
|
+
onlineNow,
|
|
3533
|
+
recentInboundReachable,
|
|
3534
|
+
connIds: entry.lastPushConnId ? [entry.lastPushConnId] : [],
|
|
3535
|
+
ownerConnId: entry.lastPushConnId,
|
|
3536
|
+
ownerClientId: entry.lastPushClientId,
|
|
3537
|
+
event: BNCR_PUSH_EVENT,
|
|
3538
|
+
}),
|
|
3539
|
+
)}`,
|
|
3540
|
+
{ debugOnly: true },
|
|
3541
|
+
);
|
|
3542
|
+
ackResult = await this.waitForMessageAck(
|
|
3543
|
+
entry.messageId,
|
|
3544
|
+
ackTimeoutMs || PUSH_ACK_TIMEOUT_MS,
|
|
3545
|
+
);
|
|
3338
3546
|
}
|
|
3339
3547
|
|
|
3340
3548
|
this.logOutboxAckWait({
|
|
@@ -3484,6 +3692,8 @@ class BncrBridgeRuntime {
|
|
|
3484
3692
|
}
|
|
3485
3693
|
} finally {
|
|
3486
3694
|
this.pushDrainRunningAccounts.delete(acc);
|
|
3695
|
+
this.pushDrainRunningSinceByAccount.delete(acc);
|
|
3696
|
+
this.pushDrainStuckWarnedAtByAccount.delete(acc);
|
|
3487
3697
|
}
|
|
3488
3698
|
}
|
|
3489
3699
|
|
|
@@ -3506,12 +3716,7 @@ class BncrBridgeRuntime {
|
|
|
3506
3716
|
|
|
3507
3717
|
private async waitForMessageAck(messageId: string, waitMs: number): Promise<'acked' | 'timeout'> {
|
|
3508
3718
|
const key = asString(messageId).trim();
|
|
3509
|
-
const timeoutMs = clampFiniteNumber(
|
|
3510
|
-
waitMs,
|
|
3511
|
-
0,
|
|
3512
|
-
0,
|
|
3513
|
-
RECOMMENDED_ACK_TIMEOUT_MAX_MS,
|
|
3514
|
-
);
|
|
3719
|
+
const timeoutMs = clampFiniteNumber(waitMs, 0, 0, RECOMMENDED_ACK_TIMEOUT_MAX_MS);
|
|
3515
3720
|
if (!key || !timeoutMs) return 'timeout';
|
|
3516
3721
|
|
|
3517
3722
|
const existing = this.messageAckWaiters.get(key);
|
|
@@ -3613,7 +3818,9 @@ class BncrBridgeRuntime {
|
|
|
3613
3818
|
const t = now();
|
|
3614
3819
|
const prev = this.connections.get(key);
|
|
3615
3820
|
const previousActiveKey = this.activeConnectionByAccount.get(acc) || null;
|
|
3616
|
-
const previousActiveConn = previousActiveKey
|
|
3821
|
+
const previousActiveConn = previousActiveKey
|
|
3822
|
+
? this.connections.get(previousActiveKey) || null
|
|
3823
|
+
: null;
|
|
3617
3824
|
|
|
3618
3825
|
const nextConn = {
|
|
3619
3826
|
accountId: acc,
|
|
@@ -4033,8 +4240,9 @@ class BncrBridgeRuntime {
|
|
|
4033
4240
|
ackStage: stage,
|
|
4034
4241
|
ackOutcome: cached.ok ? 'acked' : 'failed',
|
|
4035
4242
|
waiterReused: false,
|
|
4036
|
-
chunkIndex:
|
|
4037
|
-
|
|
4243
|
+
chunkIndex: Number.isFinite(Number(params.chunkIndex))
|
|
4244
|
+
? Number(params.chunkIndex)
|
|
4245
|
+
: undefined,
|
|
4038
4246
|
key,
|
|
4039
4247
|
...ownerInfo,
|
|
4040
4248
|
ok: cached.ok,
|
|
@@ -4061,8 +4269,9 @@ class BncrBridgeRuntime {
|
|
|
4061
4269
|
ackStage: stage,
|
|
4062
4270
|
ackOutcome: 'waiter-reused',
|
|
4063
4271
|
waiterReused: true,
|
|
4064
|
-
chunkIndex:
|
|
4065
|
-
|
|
4272
|
+
chunkIndex: Number.isFinite(Number(params.chunkIndex))
|
|
4273
|
+
? Number(params.chunkIndex)
|
|
4274
|
+
: undefined,
|
|
4066
4275
|
key,
|
|
4067
4276
|
...ownerInfo,
|
|
4068
4277
|
}),
|
|
@@ -4080,8 +4289,9 @@ class BncrBridgeRuntime {
|
|
|
4080
4289
|
ackStage: stage,
|
|
4081
4290
|
ackOutcome: 'waiting',
|
|
4082
4291
|
waiterReused: false,
|
|
4083
|
-
chunkIndex:
|
|
4084
|
-
|
|
4292
|
+
chunkIndex: Number.isFinite(Number(params.chunkIndex))
|
|
4293
|
+
? Number(params.chunkIndex)
|
|
4294
|
+
: undefined,
|
|
4085
4295
|
key,
|
|
4086
4296
|
...ownerInfo,
|
|
4087
4297
|
timeoutMs,
|
|
@@ -4106,8 +4316,9 @@ class BncrBridgeRuntime {
|
|
|
4106
4316
|
ackStage: stage,
|
|
4107
4317
|
ackOutcome: 'timeout',
|
|
4108
4318
|
waiterReused: false,
|
|
4109
|
-
chunkIndex:
|
|
4110
|
-
|
|
4319
|
+
chunkIndex: Number.isFinite(Number(params.chunkIndex))
|
|
4320
|
+
? Number(params.chunkIndex)
|
|
4321
|
+
: undefined,
|
|
4111
4322
|
key,
|
|
4112
4323
|
...ownerInfo,
|
|
4113
4324
|
timeoutMs,
|
|
@@ -4153,8 +4364,9 @@ class BncrBridgeRuntime {
|
|
|
4153
4364
|
ackStage: stage,
|
|
4154
4365
|
ackOutcome: params.ok ? 'early-acked' : 'early-failed',
|
|
4155
4366
|
waiterReused: false,
|
|
4156
|
-
chunkIndex:
|
|
4157
|
-
|
|
4367
|
+
chunkIndex: Number.isFinite(Number(params.chunkIndex))
|
|
4368
|
+
? Number(params.chunkIndex)
|
|
4369
|
+
: undefined,
|
|
4158
4370
|
key,
|
|
4159
4371
|
...ownerInfo,
|
|
4160
4372
|
ok: params.ok,
|
|
@@ -4176,8 +4388,9 @@ class BncrBridgeRuntime {
|
|
|
4176
4388
|
ackStage: stage,
|
|
4177
4389
|
ackOutcome: params.ok ? 'acked' : 'failed',
|
|
4178
4390
|
waiterReused: false,
|
|
4179
|
-
chunkIndex:
|
|
4180
|
-
|
|
4391
|
+
chunkIndex: Number.isFinite(Number(params.chunkIndex))
|
|
4392
|
+
? Number(params.chunkIndex)
|
|
4393
|
+
: undefined,
|
|
4181
4394
|
key,
|
|
4182
4395
|
...ownerInfo,
|
|
4183
4396
|
ok: params.ok,
|
|
@@ -4226,7 +4439,6 @@ class BncrBridgeRuntime {
|
|
|
4226
4439
|
return mt || 'file';
|
|
4227
4440
|
}
|
|
4228
4441
|
|
|
4229
|
-
|
|
4230
4442
|
private computeRecommendedAckTimeoutReason(args: {
|
|
4231
4443
|
lateAckOkCount: number;
|
|
4232
4444
|
recentAckTimeoutCount: number;
|
|
@@ -4308,7 +4520,10 @@ class BncrBridgeRuntime {
|
|
|
4308
4520
|
const recentAckTimeoutCount = this.getCounter(this.ackTimeoutCountByAccount, acc);
|
|
4309
4521
|
const lastLateAckPushLatencyMs = this.lastLateAckPushLatencyMsByAccount.get(acc) || null;
|
|
4310
4522
|
const lastLateAckOkAt = this.lastLateAckOkByAccount.get(acc) || null;
|
|
4311
|
-
const adaptiveAckRecoveryOkCount = this.getCounter(
|
|
4523
|
+
const adaptiveAckRecoveryOkCount = this.getCounter(
|
|
4524
|
+
this.adaptiveAckRecoveryOkCountByAccount,
|
|
4525
|
+
acc,
|
|
4526
|
+
);
|
|
4312
4527
|
const nowMs = now();
|
|
4313
4528
|
const timeoutMs = this.computeRecommendedAckTimeoutMs({
|
|
4314
4529
|
lateAckOkCount,
|
|
@@ -4345,12 +4560,18 @@ class BncrBridgeRuntime {
|
|
|
4345
4560
|
const lastLateAckOkAt = this.lastLateAckOkByAccount.get(acc) || null;
|
|
4346
4561
|
const nowMs = now();
|
|
4347
4562
|
const lastLateAckAgeMs =
|
|
4348
|
-
typeof lastLateAckOkAt === 'number' && lastLateAckOkAt > 0
|
|
4563
|
+
typeof lastLateAckOkAt === 'number' && lastLateAckOkAt > 0
|
|
4564
|
+
? Math.max(0, nowMs - lastLateAckOkAt)
|
|
4565
|
+
: null;
|
|
4349
4566
|
const lateAckObservationTtlMs = ADAPTIVE_ACK_TIMEOUT_OBSERVATION_TTL_MS;
|
|
4350
4567
|
const lateAckObservationExpired =
|
|
4351
4568
|
typeof lastLateAckAgeMs === 'number' && lastLateAckAgeMs > lateAckObservationTtlMs;
|
|
4352
|
-
const adaptiveAckRecoveryOkCount = this.getCounter(
|
|
4353
|
-
|
|
4569
|
+
const adaptiveAckRecoveryOkCount = this.getCounter(
|
|
4570
|
+
this.adaptiveAckRecoveryOkCountByAccount,
|
|
4571
|
+
acc,
|
|
4572
|
+
);
|
|
4573
|
+
const adaptiveAckRecovered =
|
|
4574
|
+
adaptiveAckRecoveryOkCount >= ADAPTIVE_ACK_TIMEOUT_RECOVERY_OK_THRESHOLD;
|
|
4354
4575
|
const recommendedAckTimeoutMs = this.computeRecommendedAckTimeoutMs({
|
|
4355
4576
|
lateAckOkCount,
|
|
4356
4577
|
recentAckTimeoutCount,
|
|
@@ -4409,7 +4630,8 @@ class BncrBridgeRuntime {
|
|
|
4409
4630
|
sessionRouteEntries: this.sessionRoutes.values(),
|
|
4410
4631
|
countInvalidOutboxSessionKeys: (snapshotAccountId) =>
|
|
4411
4632
|
this.countInvalidOutboxSessionKeys(snapshotAccountId),
|
|
4412
|
-
countLegacyAccountResidue: (snapshotAccountId) =>
|
|
4633
|
+
countLegacyAccountResidue: (snapshotAccountId) =>
|
|
4634
|
+
this.countLegacyAccountResidue(snapshotAccountId),
|
|
4413
4635
|
connectEventsByAccount: this.connectEventsByAccount,
|
|
4414
4636
|
inboundEventsByAccount: this.inboundEventsByAccount,
|
|
4415
4637
|
activityEventsByAccount: this.activityEventsByAccount,
|
|
@@ -4435,7 +4657,9 @@ class BncrBridgeRuntime {
|
|
|
4435
4657
|
}
|
|
4436
4658
|
|
|
4437
4659
|
getAccountRuntimeSnapshot(accountId: string) {
|
|
4438
|
-
const snapshot = buildAccountRuntimeSnapshot(
|
|
4660
|
+
const snapshot = buildAccountRuntimeSnapshot(
|
|
4661
|
+
this.buildRuntimeStatusInput(accountId, { running: true }),
|
|
4662
|
+
);
|
|
4439
4663
|
const ackObservability = this.buildRuntimeAckObservability(accountId);
|
|
4440
4664
|
const ackStrategy = this.buildRuntimeAckStrategy(ackObservability);
|
|
4441
4665
|
return {
|
|
@@ -4509,7 +4733,7 @@ class BncrBridgeRuntime {
|
|
|
4509
4733
|
this.logOutboundSummary(entry);
|
|
4510
4734
|
this.outbox.set(entry.messageId, entry);
|
|
4511
4735
|
this.scheduleSave();
|
|
4512
|
-
this.
|
|
4736
|
+
this.flushPushQueueBestEffort({ accountId: entry.accountId });
|
|
4513
4737
|
}
|
|
4514
4738
|
|
|
4515
4739
|
private moveToDeadLetter(entry: OutboxEntry, reason: string) {
|
|
@@ -5281,7 +5505,7 @@ class BncrBridgeRuntime {
|
|
|
5281
5505
|
});
|
|
5282
5506
|
|
|
5283
5507
|
// WS 一旦在线,立即尝试把离线期间积压队列直推出去
|
|
5284
|
-
this.
|
|
5508
|
+
this.flushPushQueueBestEffort({
|
|
5285
5509
|
accountId,
|
|
5286
5510
|
trigger: OUTBOUND_FLUSH_TRIGGER.CONNECT,
|
|
5287
5511
|
reason: OUTBOUND_FLUSH_REASON.WS_ONLINE,
|
|
@@ -5363,7 +5587,7 @@ class BncrBridgeRuntime {
|
|
|
5363
5587
|
deadLetter: this.deadLetter.filter((v) => v.accountId === accountId).length,
|
|
5364
5588
|
now: now(),
|
|
5365
5589
|
});
|
|
5366
|
-
this.
|
|
5590
|
+
this.flushPushQueueBestEffort({
|
|
5367
5591
|
accountId,
|
|
5368
5592
|
trigger: OUTBOUND_FLUSH_TRIGGER.ACTIVITY,
|
|
5369
5593
|
reason: OUTBOUND_FLUSH_REASON.ACTIVITY_HEARTBEAT,
|
|
@@ -5538,7 +5762,9 @@ class BncrBridgeRuntime {
|
|
|
5538
5762
|
return;
|
|
5539
5763
|
}
|
|
5540
5764
|
if (chunkIndex >= st.totalChunks) {
|
|
5541
|
-
respond(false, {
|
|
5765
|
+
respond(false, {
|
|
5766
|
+
error: `chunkIndex out of range index=${chunkIndex} total=${st.totalChunks}`,
|
|
5767
|
+
});
|
|
5542
5768
|
return;
|
|
5543
5769
|
}
|
|
5544
5770
|
|
|
@@ -6097,7 +6323,7 @@ class BncrBridgeRuntime {
|
|
|
6097
6323
|
taskKey: extracted.taskKey ?? null,
|
|
6098
6324
|
}),
|
|
6099
6325
|
);
|
|
6100
|
-
this.
|
|
6326
|
+
this.flushPushQueueBestEffort({
|
|
6101
6327
|
accountId,
|
|
6102
6328
|
trigger: OUTBOUND_FLUSH_TRIGGER.INBOUND,
|
|
6103
6329
|
reason: OUTBOUND_FLUSH_REASON.INBOUND_ACCEPTED,
|
|
@@ -6262,7 +6488,9 @@ class BncrBridgeRuntime {
|
|
|
6262
6488
|
payload,
|
|
6263
6489
|
mediaLocalRoots: ctx.mediaLocalRoots,
|
|
6264
6490
|
});
|
|
6265
|
-
const entries = Array.from(this.outbox.values()).filter(
|
|
6491
|
+
const entries = Array.from(this.outbox.values()).filter(
|
|
6492
|
+
(entry) => !before.has(entry.messageId),
|
|
6493
|
+
);
|
|
6266
6494
|
if (!entries.length) {
|
|
6267
6495
|
throw new Error('bncr channel.message handoff did not enqueue an outbox entry');
|
|
6268
6496
|
}
|
|
@@ -6303,7 +6531,9 @@ class BncrBridgeRuntime {
|
|
|
6303
6531
|
asVoice: payload.asVoice === true,
|
|
6304
6532
|
audioAsVoice: payload.audioAsVoice === true,
|
|
6305
6533
|
kind: payload.kind,
|
|
6306
|
-
replyToId:
|
|
6534
|
+
replyToId:
|
|
6535
|
+
asString(payload.replyToId || ctx?.replyToId || ctx?.replyToMessageId || '').trim() ||
|
|
6536
|
+
undefined,
|
|
6307
6537
|
});
|
|
6308
6538
|
return buildBncrDurableQueuedResult({ entry });
|
|
6309
6539
|
};
|
|
@@ -6342,7 +6572,7 @@ export function createBncrChannelPlugin(getBridge: () => BncrBridgeRuntime) {
|
|
|
6342
6572
|
};
|
|
6343
6573
|
},
|
|
6344
6574
|
supportsAction: ({ action }) => action === 'send',
|
|
6345
|
-
extractToolSend: ({ args })
|
|
6575
|
+
extractToolSend: ({ args }) => extractOpenClawToolSend(args, 'sendMessage'),
|
|
6346
6576
|
handleAction: async ({ action, params, accountId, mediaLocalRoots }) => {
|
|
6347
6577
|
if (action !== 'send')
|
|
6348
6578
|
throw new Error(`Action ${action} is not supported for provider ${CHANNEL_ID}.`);
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
OUTBOUND_TERMINAL_REASON,
|
|
4
|
-
} from './reasons.ts';
|
|
1
|
+
import type { BncrConnection } from '../../core/types.ts';
|
|
2
|
+
import { OUTBOUND_TERMINAL_REASON, type OutboundScheduleSource } from './reasons.ts';
|
|
5
3
|
import type { RetryRerouteDecision } from './retry-policy.ts';
|
|
6
4
|
|
|
7
5
|
export function buildOutboxScheduleDebugInfo(args: {
|
|
@@ -33,12 +31,43 @@ export function buildOutboxPushSkipDebugInfo(args: {
|
|
|
33
31
|
reason: string;
|
|
34
32
|
recentInboundReachable?: boolean;
|
|
35
33
|
kind?: string;
|
|
34
|
+
routeReason?: string;
|
|
35
|
+
connIds?: Iterable<string>;
|
|
36
|
+
ownerConnId?: string;
|
|
37
|
+
ownerClientId?: string;
|
|
38
|
+
activeConnectionCount?: number;
|
|
39
|
+
connections?: Iterable<BncrConnection>;
|
|
36
40
|
}) {
|
|
37
41
|
return {
|
|
38
42
|
messageId: args.messageId,
|
|
39
43
|
accountId: args.accountId,
|
|
40
44
|
...(args.kind ? { kind: args.kind } : {}),
|
|
41
45
|
reason: args.reason,
|
|
46
|
+
...(args.routeReason ? { routeReason: args.routeReason } : {}),
|
|
47
|
+
...(args.connIds ? { connIds: Array.from(args.connIds) } : {}),
|
|
48
|
+
...(args.ownerConnId ? { ownerConnId: args.ownerConnId } : {}),
|
|
49
|
+
...(args.ownerClientId ? { ownerClientId: args.ownerClientId } : {}),
|
|
50
|
+
...(typeof args.activeConnectionCount === 'number'
|
|
51
|
+
? { activeConnectionCount: args.activeConnectionCount }
|
|
52
|
+
: {}),
|
|
53
|
+
...(args.connections
|
|
54
|
+
? {
|
|
55
|
+
connections: Array.from(args.connections)
|
|
56
|
+
.filter((c) => c.accountId === args.accountId)
|
|
57
|
+
.slice(0, 8)
|
|
58
|
+
.map((c) => ({
|
|
59
|
+
connId: c.connId,
|
|
60
|
+
clientId: c.clientId,
|
|
61
|
+
lastSeenAt: c.lastSeenAt,
|
|
62
|
+
outboundReadyUntil: (c as any).outboundReadyUntil,
|
|
63
|
+
preferredForOutboundUntil: (c as any).preferredForOutboundUntil,
|
|
64
|
+
inboundOnly: (c as any).inboundOnly,
|
|
65
|
+
lastAckOkAt: (c as any).lastAckOkAt,
|
|
66
|
+
lastPushTimeoutAt: (c as any).lastPushTimeoutAt,
|
|
67
|
+
pushFailureScore: (c as any).pushFailureScore,
|
|
68
|
+
})),
|
|
69
|
+
}
|
|
70
|
+
: {}),
|
|
42
71
|
...(typeof args.recentInboundReachable === 'boolean'
|
|
43
72
|
? { recentInboundReachable: args.recentInboundReachable }
|
|
44
73
|
: {}),
|
|
@@ -109,6 +138,100 @@ export function buildFlushDebugInfo(args: {
|
|
|
109
138
|
};
|
|
110
139
|
}
|
|
111
140
|
|
|
141
|
+
export function buildOutboxDrainSkipDebugInfo(args: {
|
|
142
|
+
bridgeId: string;
|
|
143
|
+
accountId: string;
|
|
144
|
+
reason: string;
|
|
145
|
+
outboxSize: number;
|
|
146
|
+
trigger: string;
|
|
147
|
+
}) {
|
|
148
|
+
return {
|
|
149
|
+
bridge: args.bridgeId,
|
|
150
|
+
accountId: args.accountId,
|
|
151
|
+
reason: args.reason,
|
|
152
|
+
outboxSize: args.outboxSize,
|
|
153
|
+
trigger: args.trigger,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function buildOutboxDrainStuckDebugInfo(args: {
|
|
158
|
+
bridgeId: string;
|
|
159
|
+
accountId: string;
|
|
160
|
+
reason: string;
|
|
161
|
+
trigger: string;
|
|
162
|
+
outboxSize: number;
|
|
163
|
+
pending: number;
|
|
164
|
+
runningMs: number;
|
|
165
|
+
runningSince?: number | null;
|
|
166
|
+
hasGatewayContext: boolean;
|
|
167
|
+
activeConnectionCount: number;
|
|
168
|
+
messageAckWaiters: number;
|
|
169
|
+
fileAckWaiters: number;
|
|
170
|
+
pendingEntries?: Iterable<{
|
|
171
|
+
messageId?: string;
|
|
172
|
+
retryCount?: number;
|
|
173
|
+
nextAttemptAt?: number;
|
|
174
|
+
lastAttemptAt?: number;
|
|
175
|
+
lastError?: string;
|
|
176
|
+
lastPushAt?: number;
|
|
177
|
+
lastPushConnId?: string;
|
|
178
|
+
routeAttemptConnIds?: string[];
|
|
179
|
+
}>;
|
|
180
|
+
connections?: Iterable<BncrConnection>;
|
|
181
|
+
}) {
|
|
182
|
+
return {
|
|
183
|
+
bridge: args.bridgeId,
|
|
184
|
+
accountId: args.accountId,
|
|
185
|
+
reason: args.reason,
|
|
186
|
+
trigger: args.trigger,
|
|
187
|
+
outboxSize: args.outboxSize,
|
|
188
|
+
pending: args.pending,
|
|
189
|
+
runningMs: args.runningMs,
|
|
190
|
+
runningSince: args.runningSince || null,
|
|
191
|
+
hasGatewayContext: args.hasGatewayContext,
|
|
192
|
+
activeConnectionCount: args.activeConnectionCount,
|
|
193
|
+
waiters: {
|
|
194
|
+
messageAck: args.messageAckWaiters,
|
|
195
|
+
fileAck: args.fileAckWaiters,
|
|
196
|
+
},
|
|
197
|
+
...(args.pendingEntries
|
|
198
|
+
? {
|
|
199
|
+
pendingEntries: Array.from(args.pendingEntries)
|
|
200
|
+
.slice(0, 8)
|
|
201
|
+
.map((entry) => ({
|
|
202
|
+
messageId: entry.messageId || '',
|
|
203
|
+
retryCount: entry.retryCount,
|
|
204
|
+
nextAttemptAt: entry.nextAttemptAt,
|
|
205
|
+
lastAttemptAt: entry.lastAttemptAt,
|
|
206
|
+
lastError: entry.lastError,
|
|
207
|
+
lastPushAt: entry.lastPushAt,
|
|
208
|
+
lastPushConnId: entry.lastPushConnId,
|
|
209
|
+
routeAttemptConnIds: entry.routeAttemptConnIds,
|
|
210
|
+
})),
|
|
211
|
+
}
|
|
212
|
+
: {}),
|
|
213
|
+
...(args.connections
|
|
214
|
+
? {
|
|
215
|
+
connections: Array.from(args.connections)
|
|
216
|
+
.filter((c) => c.accountId === args.accountId)
|
|
217
|
+
.slice(0, 8)
|
|
218
|
+
.map((c) => ({
|
|
219
|
+
connId: c.connId,
|
|
220
|
+
clientId: c.clientId,
|
|
221
|
+
connectedAt: c.connectedAt,
|
|
222
|
+
lastSeenAt: c.lastSeenAt,
|
|
223
|
+
outboundReadyUntil: (c as any).outboundReadyUntil,
|
|
224
|
+
preferredForOutboundUntil: (c as any).preferredForOutboundUntil,
|
|
225
|
+
inboundOnly: (c as any).inboundOnly,
|
|
226
|
+
lastAckOkAt: (c as any).lastAckOkAt,
|
|
227
|
+
lastPushTimeoutAt: (c as any).lastPushTimeoutAt,
|
|
228
|
+
pushFailureScore: (c as any).pushFailureScore,
|
|
229
|
+
})),
|
|
230
|
+
}
|
|
231
|
+
: {}),
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
112
235
|
export function buildOutboxAckDebugInfo(args: {
|
|
113
236
|
messageId: string;
|
|
114
237
|
accountId: string;
|
|
@@ -207,8 +330,7 @@ export function buildPushFailureDebugInfo(args: {
|
|
|
207
330
|
...(typeof args.retryable === 'boolean' ? { retryable: args.retryable } : {}),
|
|
208
331
|
retryCount: args.retryCount,
|
|
209
332
|
error:
|
|
210
|
-
(typeof args.lastError === 'string' && args.lastError) ||
|
|
211
|
-
OUTBOUND_TERMINAL_REASON.PUSH_RETRY,
|
|
333
|
+
(typeof args.lastError === 'string' && args.lastError) || OUTBOUND_TERMINAL_REASON.PUSH_RETRY,
|
|
212
334
|
};
|
|
213
335
|
}
|
|
214
336
|
|