@xmoxmo/bncr 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/index.ts +30 -15
- package/package.json +4 -3
- package/scripts/check-pack.mjs +61 -0
- package/scripts/selfcheck.mjs +10 -0
- package/src/channel.ts +892 -255
- package/src/core/connection-reachability.ts +41 -14
- package/src/core/diagnostics.ts +7 -2
- package/src/core/downlink-health.ts +7 -2
- package/src/core/outbox-entry-builders.ts +3 -2
- package/src/core/policy.ts +9 -0
- package/src/core/register-trace.ts +6 -1
- package/src/core/status.ts +7 -2
- package/src/core/targets.ts +10 -1
- package/src/core/types.ts +1 -0
- package/src/messaging/inbound/commands.ts +330 -77
- package/src/messaging/inbound/context-facts.ts +200 -0
- package/src/messaging/inbound/dispatch.ts +429 -119
- package/src/messaging/inbound/gate.ts +66 -26
- package/src/messaging/inbound/parse.ts +8 -0
- package/src/messaging/inbound/runtime-compat.ts +39 -0
- package/src/messaging/inbound/session-label.ts +115 -0
- package/src/messaging/outbound/diagnostics.ts +16 -0
- package/src/messaging/outbound/durable-message-adapter.ts +107 -0
- package/src/messaging/outbound/durable-queue-adapter.ts +157 -0
- package/src/messaging/outbound/media.ts +3 -1
- package/src/messaging/outbound/queue-selectors.ts +7 -2
- package/src/messaging/outbound/reasons.ts +4 -0
- package/src/messaging/outbound/reply-enqueue.ts +2 -2
- package/src/messaging/outbound/reply-target-policy.ts +13 -0
- package/src/messaging/outbound/retry-policy.ts +12 -3
- package/src/messaging/outbound/send.ts +6 -0
- package/src/messaging/outbound/session-route.ts +2 -2
- package/src/openclaw/config-runtime.ts +52 -0
- package/src/openclaw/inbound-session-runtime.ts +94 -0
- package/src/openclaw/ingress-runtime.ts +35 -0
- package/src/openclaw/media-runtime.ts +73 -0
- package/src/openclaw/reply-runtime.ts +104 -0
- package/src/openclaw/routing-runtime.ts +48 -0
- package/src/openclaw/sdk-helpers.ts +20 -0
- package/src/openclaw/session-route-runtime.ts +15 -0
package/src/channel.ts
CHANGED
|
@@ -1,21 +1,11 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
-
import { readBooleanParam } from 'openclaw/plugin-sdk/boolean-param';
|
|
5
4
|
import type {
|
|
6
5
|
GatewayRequestHandlerOptions,
|
|
7
6
|
OpenClawPluginApi,
|
|
8
7
|
OpenClawPluginServiceContext,
|
|
9
8
|
} from 'openclaw/plugin-sdk/core';
|
|
10
|
-
import {
|
|
11
|
-
applyAccountNameToChannelSection,
|
|
12
|
-
jsonResult,
|
|
13
|
-
setAccountEnabledInConfigSection,
|
|
14
|
-
} from 'openclaw/plugin-sdk/core';
|
|
15
|
-
import { readJsonFileWithFallback, writeJsonFileAtomically } from 'openclaw/plugin-sdk/json-store';
|
|
16
|
-
import { readStringParam } from 'openclaw/plugin-sdk/param-readers';
|
|
17
|
-
import { createDefaultChannelRuntimeState } from 'openclaw/plugin-sdk/status-helpers';
|
|
18
|
-
import { extractToolSend } from 'openclaw/plugin-sdk/tool-send';
|
|
19
9
|
import {
|
|
20
10
|
BNCR_DEFAULT_ACCOUNT_ID,
|
|
21
11
|
CHANNEL_ID,
|
|
@@ -77,7 +67,28 @@ import { buildDownlinkHealth as buildDownlinkHealthFromRuntime } from './core/do
|
|
|
77
67
|
import { buildExtendedDiagnostics as buildExtendedDiagnosticsFromRuntime } from './core/extended-diagnostics.ts';
|
|
78
68
|
import { observeLeaseState, matchesTransferOwner as matchesTransferOwnerFromRuntime } from './core/lease-state.ts';
|
|
79
69
|
import { emitBncrLog, emitBncrLogLine } from './core/logging.ts';
|
|
80
|
-
import { resolveBncrChannelPolicy } from './core/policy.ts';
|
|
70
|
+
import { resolveBncrChannelPolicy, resolveBncrConfigWarnings } from './core/policy.ts';
|
|
71
|
+
import {
|
|
72
|
+
getOpenClawRuntimeConfig,
|
|
73
|
+
getOpenClawRuntimeConfigOrDefault,
|
|
74
|
+
} from './openclaw/config-runtime.ts';
|
|
75
|
+
import {
|
|
76
|
+
loadOpenClawWebMedia,
|
|
77
|
+
saveOpenClawChannelMediaBuffer,
|
|
78
|
+
type OpenClawLoadedMedia,
|
|
79
|
+
} from './openclaw/media-runtime.ts';
|
|
80
|
+
import { resolveOpenClawAgentRoute } from './openclaw/routing-runtime.ts';
|
|
81
|
+
import {
|
|
82
|
+
applyOpenClawAccountNameToChannelSection,
|
|
83
|
+
createOpenClawDefaultChannelRuntimeState,
|
|
84
|
+
extractOpenClawToolSend,
|
|
85
|
+
openClawJsonResult,
|
|
86
|
+
readOpenClawBooleanParam,
|
|
87
|
+
readOpenClawJsonFileWithFallback,
|
|
88
|
+
readOpenClawStringParam,
|
|
89
|
+
setOpenClawAccountEnabledInConfigSection,
|
|
90
|
+
writeOpenClawJsonFileAtomically,
|
|
91
|
+
} from './openclaw/sdk-helpers.ts';
|
|
81
92
|
import {
|
|
82
93
|
buildRegisterTraceSummary as buildRegisterTraceSummaryFromEntries,
|
|
83
94
|
classifyRegisterTrace as classifyRegisterTraceFromStack,
|
|
@@ -309,6 +320,7 @@ import {
|
|
|
309
320
|
computeRetryRerouteDecision,
|
|
310
321
|
} from './messaging/outbound/retry-policy.ts';
|
|
311
322
|
import { sendBncrMedia, sendBncrText } from './messaging/outbound/send.ts';
|
|
323
|
+
import { buildBncrDurableQueuedResult } from './messaging/outbound/durable-queue-adapter.ts';
|
|
312
324
|
import { resolveBncrOutboundSessionRoute } from './messaging/outbound/session-route.ts';
|
|
313
325
|
import {
|
|
314
326
|
looksLikeBncrExplicitTarget,
|
|
@@ -323,17 +335,35 @@ const BNCR_FILE_ABORT_EVENT = 'plugin.bncr.file.abort';
|
|
|
323
335
|
const CONNECT_TTL_MS = 120_000;
|
|
324
336
|
const RECENT_INBOUND_SEND_WINDOW_MS = 60_000;
|
|
325
337
|
const MAX_RETRY = 10;
|
|
338
|
+
const MAX_DEAD_LETTER_ENTRIES = 1000;
|
|
339
|
+
const MAX_SESSION_ROUTE_ENTRIES = 1000;
|
|
340
|
+
const MAX_ACCOUNT_ACTIVITY_ENTRIES = 1000;
|
|
326
341
|
const PUSH_DRAIN_INTERVAL_MS = 500;
|
|
342
|
+
const PUSH_DRAIN_ACCOUNT_BUDGET = 5;
|
|
343
|
+
const PUSH_DRAIN_ACCOUNT_TIME_BUDGET_MS = 2_000;
|
|
327
344
|
const PUSH_ACK_TIMEOUT_MS = 30_000;
|
|
345
|
+
const ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED = true;
|
|
346
|
+
const RECOMMENDED_ACK_TIMEOUT_MIN_MS = PUSH_ACK_TIMEOUT_MS;
|
|
347
|
+
const RECOMMENDED_ACK_TIMEOUT_MAX_MS = 90_000;
|
|
348
|
+
const ADAPTIVE_ACK_TIMEOUT_OBSERVATION_TTL_MS = 60 * 60 * 1000;
|
|
349
|
+
const ADAPTIVE_ACK_TIMEOUT_RECOVERY_OK_THRESHOLD = 3;
|
|
350
|
+
const ADAPTIVE_ACK_TIMEOUT_LOG_THROTTLE_MS = 5 * 60 * 1000;
|
|
328
351
|
const OUTBOUND_READY_TTL_MS = 30_000;
|
|
329
352
|
const PREFERRED_OUTBOUND_TTL_MS = 12_000;
|
|
330
353
|
const FILE_FORCE_CHUNK = true; // 统一走 WS 分块,保留 base64 仅作兜底
|
|
354
|
+
const LOG_DEDUPE_STATE_TTL_MS = 10 * 60 * 1000;
|
|
355
|
+
const LOG_DEDUPE_STATE_MAX_ENTRIES = 1_000;
|
|
331
356
|
const FILE_INLINE_THRESHOLD = 5 * 1024 * 1024; // fallback 阈值(仅 FILE_FORCE_CHUNK=false 时生效)
|
|
332
357
|
const FILE_CHUNK_SIZE = 256 * 1024; // 256KB
|
|
358
|
+
const INBOUND_FILE_TRANSFER_MAX_BYTES = 50 * 1024 * 1024;
|
|
359
|
+
const INBOUND_FILE_TRANSFER_MAX_CHUNKS = Math.ceil(INBOUND_FILE_TRANSFER_MAX_BYTES / FILE_CHUNK_SIZE) + 1;
|
|
333
360
|
const FILE_CHUNK_RETRY = 3;
|
|
334
361
|
const FILE_ACK_TIMEOUT_MS = 30_000;
|
|
335
362
|
const FILE_TRANSFER_ACK_TTL_MS = 30_000;
|
|
363
|
+
const MAX_EARLY_FILE_ACKS = 1000;
|
|
364
|
+
const INTERNAL_SLEEP_MAX_MS = 120_000;
|
|
336
365
|
const FILE_TRANSFER_KEEP_MS = 6 * 60 * 60 * 1000;
|
|
366
|
+
const FILE_TRANSFER_TERMINAL_KEEP_MS = 10 * 60 * 1000;
|
|
337
367
|
const REGISTER_WARMUP_WINDOW_MS = 30_000;
|
|
338
368
|
let BNCR_DEBUG_VERBOSE = false; // 全局调试日志开关(默认关闭)
|
|
339
369
|
|
|
@@ -355,9 +385,16 @@ type FileSendTransferState = {
|
|
|
355
385
|
ownerConnId?: string;
|
|
356
386
|
ownerClientId?: string;
|
|
357
387
|
completedPath?: string;
|
|
388
|
+
terminalAt?: number;
|
|
358
389
|
error?: string;
|
|
359
390
|
};
|
|
360
391
|
|
|
392
|
+
type ChannelAccountWorkerHandle = {
|
|
393
|
+
timer: NodeJS.Timeout;
|
|
394
|
+
finish: (reason: string) => void;
|
|
395
|
+
cleanupAbortListener?: () => void;
|
|
396
|
+
};
|
|
397
|
+
|
|
361
398
|
type FileRecvTransferState = {
|
|
362
399
|
transferId: string;
|
|
363
400
|
accountId: string;
|
|
@@ -376,6 +413,7 @@ type FileRecvTransferState = {
|
|
|
376
413
|
ownerConnId?: string;
|
|
377
414
|
ownerClientId?: string;
|
|
378
415
|
completedPath?: string;
|
|
416
|
+
terminalAt?: number;
|
|
379
417
|
error?: string;
|
|
380
418
|
};
|
|
381
419
|
|
|
@@ -455,20 +493,20 @@ function normalizeBncrSendParams(input: {
|
|
|
455
493
|
accountId: string;
|
|
456
494
|
}): NormalizedBncrSendParams {
|
|
457
495
|
const paramsObj = isPlainObject(input.params) ? input.params : {};
|
|
458
|
-
const to =
|
|
496
|
+
const to = readOpenClawStringParam(paramsObj, 'to', { required: true });
|
|
459
497
|
const resolvedAccountId = normalizeAccountId(
|
|
460
|
-
|
|
498
|
+
readOpenClawStringParam(paramsObj, 'accountId') ?? input.accountId,
|
|
461
499
|
);
|
|
462
500
|
|
|
463
|
-
const message =
|
|
464
|
-
const caption =
|
|
501
|
+
const message = readOpenClawStringParam(paramsObj, 'message', { allowEmpty: true }) ?? '';
|
|
502
|
+
const caption = readOpenClawStringParam(paramsObj, 'caption', { allowEmpty: true }) ?? '';
|
|
465
503
|
const mediaUrl =
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
const asVoice =
|
|
471
|
-
const audioAsVoice =
|
|
504
|
+
readOpenClawStringParam(paramsObj, 'media', { trim: false }) ??
|
|
505
|
+
readOpenClawStringParam(paramsObj, 'path', { trim: false }) ??
|
|
506
|
+
readOpenClawStringParam(paramsObj, 'filePath', { trim: false }) ??
|
|
507
|
+
readOpenClawStringParam(paramsObj, 'mediaUrl', { trim: false });
|
|
508
|
+
const asVoice = readOpenClawBooleanParam(paramsObj, 'asVoice') ?? false;
|
|
509
|
+
const audioAsVoice = readOpenClawBooleanParam(paramsObj, 'audioAsVoice') ?? false;
|
|
472
510
|
|
|
473
511
|
if (asVoice && !mediaUrl) throw new Error('send voice requires media path');
|
|
474
512
|
|
|
@@ -501,6 +539,28 @@ function asString(v: unknown, fallback = ''): string {
|
|
|
501
539
|
return String(v);
|
|
502
540
|
}
|
|
503
541
|
|
|
542
|
+
function finiteNumberOr(value: unknown, fallback: number): number {
|
|
543
|
+
const n = Number(value);
|
|
544
|
+
return Number.isFinite(n) ? n : fallback;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function optionalFiniteNumber(value: unknown): number | undefined {
|
|
548
|
+
if (value == null || value === '') return undefined;
|
|
549
|
+
const n = Number(value);
|
|
550
|
+
return Number.isFinite(n) ? n : undefined;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
function finiteNonNegativeNumberOrNull(value: unknown): number | null {
|
|
554
|
+
const n = Number(value);
|
|
555
|
+
return Number.isFinite(n) && n >= 0 ? n : null;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function clampFiniteNumber(value: unknown, fallback: number, min: number, max: number): number {
|
|
559
|
+
const n = Number(value);
|
|
560
|
+
const finite = Number.isFinite(n) ? n : fallback;
|
|
561
|
+
return Math.max(min, Math.min(finite, max));
|
|
562
|
+
}
|
|
563
|
+
|
|
504
564
|
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
505
565
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
506
566
|
}
|
|
@@ -658,7 +718,15 @@ class BncrBridgeRuntime {
|
|
|
658
718
|
private lastAckOkByAccount = new Map<string, number>();
|
|
659
719
|
private lastAckTimeoutByAccount = new Map<string, number>();
|
|
660
720
|
private ackTimeoutCountByAccount = new Map<string, number>();
|
|
661
|
-
private
|
|
721
|
+
private lateAckOkCountByAccount = new Map<string, number>();
|
|
722
|
+
private lastLateAckOkByAccount = new Map<string, number>();
|
|
723
|
+
private lastAckQueueLatencyMsByAccount = new Map<string, number>();
|
|
724
|
+
private lastAckPushLatencyMsByAccount = new Map<string, number>();
|
|
725
|
+
private lastLateAckQueueLatencyMsByAccount = new Map<string, number>();
|
|
726
|
+
private lastLateAckPushLatencyMsByAccount = new Map<string, number>();
|
|
727
|
+
private adaptiveAckRecoveryOkCountByAccount = new Map<string, number>();
|
|
728
|
+
private adaptiveAckTimeoutLogStateByAccount = new Map<string, { at: number; timeoutMs: number; reason: string }>();
|
|
729
|
+
private channelAccountWorkers = new Map<string, ChannelAccountWorkerHandle>();
|
|
662
730
|
private logDedupeState = new Map<string, { at: number; sig: string }>();
|
|
663
731
|
private canonicalAgentId: string | null = null;
|
|
664
732
|
private canonicalAgentSource: 'startup' | 'runtime' | 'fallback-main' | null = null;
|
|
@@ -666,6 +734,7 @@ class BncrBridgeRuntime {
|
|
|
666
734
|
|
|
667
735
|
// 内置健康/回归计数(替代独立脚本)
|
|
668
736
|
private startedAt = now();
|
|
737
|
+
private stopped = false;
|
|
669
738
|
private connectEventsByAccount = new Map<string, number>();
|
|
670
739
|
private inboundEventsByAccount = new Map<string, number>();
|
|
671
740
|
private activityEventsByAccount = new Map<string, number>();
|
|
@@ -682,6 +751,7 @@ class BncrBridgeRuntime {
|
|
|
682
751
|
// then move storage + resolver/wait APIs together rather than partially splitting the map only.
|
|
683
752
|
string,
|
|
684
753
|
{
|
|
754
|
+
promise: Promise<'acked' | 'timeout'>;
|
|
685
755
|
resolve: (result: 'acked' | 'timeout') => void;
|
|
686
756
|
timer: NodeJS.Timeout;
|
|
687
757
|
}
|
|
@@ -694,6 +764,7 @@ class BncrBridgeRuntime {
|
|
|
694
764
|
private fileAckWaiters = new Map<
|
|
695
765
|
string,
|
|
696
766
|
{
|
|
767
|
+
promise: Promise<Record<string, unknown>>;
|
|
697
768
|
resolve: (payload: Record<string, unknown>) => void;
|
|
698
769
|
reject: (err: Error) => void;
|
|
699
770
|
timer: NodeJS.Timeout;
|
|
@@ -701,6 +772,15 @@ class BncrBridgeRuntime {
|
|
|
701
772
|
>();
|
|
702
773
|
private earlyFileAcks = new Map<string, FileAckPayloadState>();
|
|
703
774
|
|
|
775
|
+
private rememberEarlyFileAck(key: string, state: FileAckPayloadState) {
|
|
776
|
+
this.earlyFileAcks.set(key, state);
|
|
777
|
+
while (this.earlyFileAcks.size > MAX_EARLY_FILE_ACKS) {
|
|
778
|
+
const oldestKey = this.earlyFileAcks.keys().next().value;
|
|
779
|
+
if (!oldestKey) break;
|
|
780
|
+
this.earlyFileAcks.delete(oldestKey);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
704
784
|
constructor(api: OpenClawPluginApi) {
|
|
705
785
|
this.api = api;
|
|
706
786
|
}
|
|
@@ -756,11 +836,27 @@ class BncrBridgeRuntime {
|
|
|
756
836
|
this.logError(scope, this.buildDebugJsonMessage(event, payload), options);
|
|
757
837
|
}
|
|
758
838
|
|
|
839
|
+
private pruneLogDedupeState(currentTime = now()) {
|
|
840
|
+
for (const [key, entry] of this.logDedupeState.entries()) {
|
|
841
|
+
if (currentTime - entry.at > LOG_DEDUPE_STATE_TTL_MS) {
|
|
842
|
+
this.logDedupeState.delete(key);
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
while (this.logDedupeState.size > LOG_DEDUPE_STATE_MAX_ENTRIES) {
|
|
847
|
+
const oldestKey = this.logDedupeState.keys().next().value;
|
|
848
|
+
if (!oldestKey) break;
|
|
849
|
+
this.logDedupeState.delete(oldestKey);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
759
853
|
private shouldEmitDedupLog(key: string, sig: string, windowMs = 5 * 60 * 1000) {
|
|
760
854
|
const t = now();
|
|
855
|
+
this.pruneLogDedupeState(t);
|
|
761
856
|
const prev = this.logDedupeState.get(key) || null;
|
|
762
857
|
if (prev && prev.sig === sig && t - prev.at < windowMs) return false;
|
|
763
858
|
this.logDedupeState.set(key, { at: t, sig });
|
|
859
|
+
this.pruneLogDedupeState(t);
|
|
764
860
|
return true;
|
|
765
861
|
}
|
|
766
862
|
|
|
@@ -859,10 +955,9 @@ class BncrBridgeRuntime {
|
|
|
859
955
|
}
|
|
860
956
|
|
|
861
957
|
private clearChannelAccountWorker(accountId: string, reason: string) {
|
|
862
|
-
const
|
|
863
|
-
if (!
|
|
864
|
-
|
|
865
|
-
this.channelAccountTimers.delete(accountId);
|
|
958
|
+
const worker = this.channelAccountWorkers.get(accountId);
|
|
959
|
+
if (!worker) return false;
|
|
960
|
+
worker.finish(reason);
|
|
866
961
|
this.logInfo(
|
|
867
962
|
'health',
|
|
868
963
|
`status-worker cleared ${JSON.stringify({ bridge: this.bridgeId, accountId, reason })}`,
|
|
@@ -871,6 +966,12 @@ class BncrBridgeRuntime {
|
|
|
871
966
|
return true;
|
|
872
967
|
}
|
|
873
968
|
|
|
969
|
+
private clearAllChannelAccountWorkers(reason: string) {
|
|
970
|
+
for (const accountId of Array.from(this.channelAccountWorkers.keys())) {
|
|
971
|
+
this.clearChannelAccountWorker(accountId, reason);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
874
975
|
private captureDriftSnapshot(
|
|
875
976
|
summary: ReturnType<BncrBridgeRuntime['buildRegisterTraceSummary']>,
|
|
876
977
|
) {
|
|
@@ -1106,14 +1207,18 @@ class BncrBridgeRuntime {
|
|
|
1106
1207
|
}
|
|
1107
1208
|
|
|
1108
1209
|
startService = async (ctx: OpenClawPluginServiceContext, debug?: boolean) => {
|
|
1210
|
+
this.stopped = false;
|
|
1109
1211
|
this.statePath = path.join(ctx.stateDir, 'bncr-bridge-state.json');
|
|
1110
|
-
await this.loadState();
|
|
1111
1212
|
try {
|
|
1112
|
-
const cfg = this.api
|
|
1213
|
+
const cfg = getOpenClawRuntimeConfig(this.api);
|
|
1113
1214
|
this.initializeCanonicalAgentId(cfg);
|
|
1215
|
+
for (const warning of resolveBncrConfigWarnings(cfg?.channels?.[CHANNEL_ID] || {})) {
|
|
1216
|
+
this.logWarn('config', warning);
|
|
1217
|
+
}
|
|
1114
1218
|
} catch {
|
|
1115
1219
|
// ignore startup canonical agent initialization errors
|
|
1116
1220
|
}
|
|
1221
|
+
await this.loadState();
|
|
1117
1222
|
if (typeof debug === 'boolean') BNCR_DEBUG_VERBOSE = debug;
|
|
1118
1223
|
await this.refreshDebugFlagFromConfig({ forceLog: true });
|
|
1119
1224
|
const bootDiag = this.buildIntegratedDiagnostics(BNCR_DEFAULT_ACCOUNT_ID);
|
|
@@ -1129,15 +1234,34 @@ class BncrBridgeRuntime {
|
|
|
1129
1234
|
};
|
|
1130
1235
|
|
|
1131
1236
|
stopService = async () => {
|
|
1132
|
-
|
|
1133
|
-
clearTimeout(this.pushTimer);
|
|
1134
|
-
this.pushTimer = null;
|
|
1135
|
-
}
|
|
1237
|
+
this.cleanupRuntimeWaitersAndTimers('service stopped');
|
|
1136
1238
|
await this.flushState();
|
|
1137
1239
|
this.logInfo('debug', 'service stopped', { debugOnly: true });
|
|
1138
1240
|
};
|
|
1139
1241
|
|
|
1140
1242
|
shutdown() {
|
|
1243
|
+
this.cleanupRuntimeWaitersAndTimers('shutdown');
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
private cleanupRuntimeWaitersAndTimers(reason: string) {
|
|
1247
|
+
this.logInfo(
|
|
1248
|
+
'lifecycle',
|
|
1249
|
+
`cleanup ${JSON.stringify({
|
|
1250
|
+
bridge: this.bridgeId,
|
|
1251
|
+
reason,
|
|
1252
|
+
messageAckWaiters: this.messageAckWaiters.size,
|
|
1253
|
+
fileAckWaiters: this.fileAckWaiters.size,
|
|
1254
|
+
earlyFileAcks: this.earlyFileAcks.size,
|
|
1255
|
+
outbox: this.outbox.size,
|
|
1256
|
+
runningDrainAccounts: this.pushDrainRunningAccounts.size,
|
|
1257
|
+
channelAccountWorkers: this.channelAccountWorkers.size,
|
|
1258
|
+
hasSaveTimer: !!this.saveTimer,
|
|
1259
|
+
hasPushTimer: !!this.pushTimer,
|
|
1260
|
+
})}`,
|
|
1261
|
+
{ debugOnly: true },
|
|
1262
|
+
);
|
|
1263
|
+
this.stopped = true;
|
|
1264
|
+
this.clearAllChannelAccountWorkers(reason);
|
|
1141
1265
|
if (this.saveTimer) {
|
|
1142
1266
|
clearTimeout(this.saveTimer);
|
|
1143
1267
|
this.saveTimer = null;
|
|
@@ -1148,19 +1272,23 @@ class BncrBridgeRuntime {
|
|
|
1148
1272
|
}
|
|
1149
1273
|
for (const waiter of this.messageAckWaiters.values()) {
|
|
1150
1274
|
clearTimeout(waiter.timer);
|
|
1275
|
+
waiter.resolve('timeout');
|
|
1151
1276
|
}
|
|
1152
1277
|
this.messageAckWaiters.clear();
|
|
1153
1278
|
for (const waiter of this.fileAckWaiters.values()) {
|
|
1154
1279
|
clearTimeout(waiter.timer);
|
|
1280
|
+
waiter.reject(new Error(reason));
|
|
1155
1281
|
}
|
|
1156
1282
|
this.fileAckWaiters.clear();
|
|
1157
1283
|
this.earlyFileAcks.clear();
|
|
1158
1284
|
}
|
|
1159
1285
|
|
|
1160
1286
|
private scheduleSave() {
|
|
1287
|
+
if (this.stopped) return;
|
|
1161
1288
|
if (this.saveTimer) return;
|
|
1162
1289
|
this.saveTimer = setTimeout(() => {
|
|
1163
1290
|
this.saveTimer = null;
|
|
1291
|
+
if (this.stopped) return;
|
|
1164
1292
|
void this.flushState();
|
|
1165
1293
|
}, 300);
|
|
1166
1294
|
}
|
|
@@ -1176,7 +1304,7 @@ class BncrBridgeRuntime {
|
|
|
1176
1304
|
|
|
1177
1305
|
private async refreshDebugFlagFromConfig(options?: { forceLog?: boolean }) {
|
|
1178
1306
|
try {
|
|
1179
|
-
const cfg = this.api
|
|
1307
|
+
const cfg = getOpenClawRuntimeConfig(this.api);
|
|
1180
1308
|
const raw = (cfg as any)?.channels?.[CHANNEL_ID]?.debug?.verbose;
|
|
1181
1309
|
const next = typeof raw === 'boolean' ? raw : false;
|
|
1182
1310
|
const changed = next !== BNCR_DEBUG_VERBOSE;
|
|
@@ -1200,7 +1328,7 @@ class BncrBridgeRuntime {
|
|
|
1200
1328
|
channelId?: string;
|
|
1201
1329
|
}): string | null {
|
|
1202
1330
|
try {
|
|
1203
|
-
const resolved = this.api
|
|
1331
|
+
const resolved = resolveOpenClawAgentRoute(this.api, {
|
|
1204
1332
|
cfg: args.cfg,
|
|
1205
1333
|
channel: args.channelId || CHANNEL_ID,
|
|
1206
1334
|
accountId: normalizeAccountId(args.accountId),
|
|
@@ -1297,7 +1425,13 @@ class BncrBridgeRuntime {
|
|
|
1297
1425
|
}
|
|
1298
1426
|
|
|
1299
1427
|
private buildIntegratedDiagnostics(accountId: string) {
|
|
1300
|
-
|
|
1428
|
+
const ackObservability = this.buildRuntimeAckObservability(accountId);
|
|
1429
|
+
const ackStrategy = this.buildRuntimeAckStrategy(ackObservability);
|
|
1430
|
+
return {
|
|
1431
|
+
...buildIntegratedDiagnosticsFromRuntime(this.buildRuntimeStatusInput(accountId)),
|
|
1432
|
+
ackObservability,
|
|
1433
|
+
ackStrategy,
|
|
1434
|
+
};
|
|
1301
1435
|
}
|
|
1302
1436
|
|
|
1303
1437
|
private buildDownlinkHealth(accountId: string) {
|
|
@@ -1318,7 +1452,7 @@ class BncrBridgeRuntime {
|
|
|
1318
1452
|
|
|
1319
1453
|
private async loadState() {
|
|
1320
1454
|
if (!this.statePath) return;
|
|
1321
|
-
const loaded = await
|
|
1455
|
+
const loaded = await readOpenClawJsonFileWithFallback(this.statePath, {
|
|
1322
1456
|
outbox: [],
|
|
1323
1457
|
deadLetter: [],
|
|
1324
1458
|
sessionRoutes: [],
|
|
@@ -1347,10 +1481,10 @@ class BncrBridgeRuntime {
|
|
|
1347
1481
|
sessionKey: normalized.sessionKey,
|
|
1348
1482
|
route,
|
|
1349
1483
|
payload,
|
|
1350
|
-
createdAt:
|
|
1351
|
-
retryCount:
|
|
1352
|
-
nextAttemptAt:
|
|
1353
|
-
lastAttemptAt:
|
|
1484
|
+
createdAt: finiteNumberOr(entry.createdAt, now()),
|
|
1485
|
+
retryCount: finiteNumberOr(entry.retryCount, 0),
|
|
1486
|
+
nextAttemptAt: finiteNumberOr(entry.nextAttemptAt, now()),
|
|
1487
|
+
lastAttemptAt: optionalFiniteNumber(entry.lastAttemptAt),
|
|
1354
1488
|
lastError: entry.lastError ? asString(entry.lastError) : undefined,
|
|
1355
1489
|
};
|
|
1356
1490
|
|
|
@@ -1358,7 +1492,10 @@ class BncrBridgeRuntime {
|
|
|
1358
1492
|
}
|
|
1359
1493
|
|
|
1360
1494
|
this.deadLetter = [];
|
|
1361
|
-
|
|
1495
|
+
const persistedDeadLetter = Array.isArray(data.deadLetter)
|
|
1496
|
+
? data.deadLetter.slice(-MAX_DEAD_LETTER_ENTRIES)
|
|
1497
|
+
: [];
|
|
1498
|
+
for (const entry of persistedDeadLetter) {
|
|
1362
1499
|
if (!entry?.messageId) continue;
|
|
1363
1500
|
const accountId = normalizeAccountId(entry.accountId);
|
|
1364
1501
|
const sessionKey = asString(entry.sessionKey || '').trim();
|
|
@@ -1379,17 +1516,20 @@ class BncrBridgeRuntime {
|
|
|
1379
1516
|
sessionKey: normalized.sessionKey,
|
|
1380
1517
|
route,
|
|
1381
1518
|
payload,
|
|
1382
|
-
createdAt:
|
|
1383
|
-
retryCount:
|
|
1384
|
-
nextAttemptAt:
|
|
1385
|
-
lastAttemptAt:
|
|
1519
|
+
createdAt: finiteNumberOr(entry.createdAt, now()),
|
|
1520
|
+
retryCount: finiteNumberOr(entry.retryCount, 0),
|
|
1521
|
+
nextAttemptAt: finiteNumberOr(entry.nextAttemptAt, now()),
|
|
1522
|
+
lastAttemptAt: optionalFiniteNumber(entry.lastAttemptAt),
|
|
1386
1523
|
lastError: entry.lastError ? asString(entry.lastError) : undefined,
|
|
1387
1524
|
});
|
|
1388
1525
|
}
|
|
1389
1526
|
|
|
1390
1527
|
this.sessionRoutes.clear();
|
|
1391
1528
|
this.routeAliases.clear();
|
|
1392
|
-
|
|
1529
|
+
const persistedSessionRoutes = Array.isArray(data.sessionRoutes)
|
|
1530
|
+
? data.sessionRoutes.slice(-MAX_SESSION_ROUTE_ENTRIES)
|
|
1531
|
+
: [];
|
|
1532
|
+
for (const item of persistedSessionRoutes) {
|
|
1393
1533
|
const normalized = normalizeStoredSessionKey(
|
|
1394
1534
|
asString(item?.sessionKey || ''),
|
|
1395
1535
|
this.canonicalAgentId,
|
|
@@ -1398,7 +1538,7 @@ class BncrBridgeRuntime {
|
|
|
1398
1538
|
|
|
1399
1539
|
const route = parseRouteLike(item?.route) || normalized.route;
|
|
1400
1540
|
const accountId = normalizeAccountId(item?.accountId);
|
|
1401
|
-
const updatedAt =
|
|
1541
|
+
const updatedAt = finiteNumberOr(item?.updatedAt, now());
|
|
1402
1542
|
|
|
1403
1543
|
const info = {
|
|
1404
1544
|
accountId,
|
|
@@ -1411,14 +1551,17 @@ class BncrBridgeRuntime {
|
|
|
1411
1551
|
}
|
|
1412
1552
|
|
|
1413
1553
|
this.lastSessionByAccount.clear();
|
|
1414
|
-
|
|
1554
|
+
const persistedLastSessionByAccount = Array.isArray(data.lastSessionByAccount)
|
|
1555
|
+
? data.lastSessionByAccount.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES)
|
|
1556
|
+
: [];
|
|
1557
|
+
for (const item of persistedLastSessionByAccount) {
|
|
1415
1558
|
const accountId = normalizeAccountId(item?.accountId);
|
|
1416
1559
|
const normalized = normalizeStoredSessionKey(
|
|
1417
1560
|
asString(item?.sessionKey || ''),
|
|
1418
1561
|
this.canonicalAgentId,
|
|
1419
1562
|
);
|
|
1420
|
-
const updatedAt =
|
|
1421
|
-
if (!normalized ||
|
|
1563
|
+
const updatedAt = finiteNumberOr(item?.updatedAt, 0);
|
|
1564
|
+
if (!normalized || updatedAt <= 0) continue;
|
|
1422
1565
|
|
|
1423
1566
|
this.lastSessionByAccount.set(accountId, {
|
|
1424
1567
|
sessionKey: normalized.sessionKey,
|
|
@@ -1429,33 +1572,42 @@ class BncrBridgeRuntime {
|
|
|
1429
1572
|
}
|
|
1430
1573
|
|
|
1431
1574
|
this.lastActivityByAccount.clear();
|
|
1432
|
-
|
|
1575
|
+
const persistedLastActivityByAccount = Array.isArray(data.lastActivityByAccount)
|
|
1576
|
+
? data.lastActivityByAccount.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES)
|
|
1577
|
+
: [];
|
|
1578
|
+
for (const item of persistedLastActivityByAccount) {
|
|
1433
1579
|
const accountId = normalizeAccountId(item?.accountId);
|
|
1434
|
-
const updatedAt =
|
|
1435
|
-
if (
|
|
1580
|
+
const updatedAt = finiteNumberOr(item?.updatedAt, 0);
|
|
1581
|
+
if (updatedAt <= 0) continue;
|
|
1436
1582
|
this.lastActivityByAccount.set(accountId, updatedAt);
|
|
1437
1583
|
}
|
|
1438
1584
|
|
|
1439
1585
|
this.lastInboundByAccount.clear();
|
|
1440
|
-
|
|
1586
|
+
const persistedLastInboundByAccount = Array.isArray(data.lastInboundByAccount)
|
|
1587
|
+
? data.lastInboundByAccount.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES)
|
|
1588
|
+
: [];
|
|
1589
|
+
for (const item of persistedLastInboundByAccount) {
|
|
1441
1590
|
const accountId = normalizeAccountId(item?.accountId);
|
|
1442
|
-
const updatedAt =
|
|
1443
|
-
if (
|
|
1591
|
+
const updatedAt = finiteNumberOr(item?.updatedAt, 0);
|
|
1592
|
+
if (updatedAt <= 0) continue;
|
|
1444
1593
|
this.lastInboundByAccount.set(accountId, updatedAt);
|
|
1445
1594
|
}
|
|
1446
1595
|
|
|
1447
1596
|
this.lastOutboundByAccount.clear();
|
|
1448
|
-
|
|
1597
|
+
const persistedLastOutboundByAccount = Array.isArray(data.lastOutboundByAccount)
|
|
1598
|
+
? data.lastOutboundByAccount.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES)
|
|
1599
|
+
: [];
|
|
1600
|
+
for (const item of persistedLastOutboundByAccount) {
|
|
1449
1601
|
const accountId = normalizeAccountId(item?.accountId);
|
|
1450
|
-
const updatedAt =
|
|
1451
|
-
if (
|
|
1602
|
+
const updatedAt = finiteNumberOr(item?.updatedAt, 0);
|
|
1603
|
+
if (updatedAt <= 0) continue;
|
|
1452
1604
|
this.lastOutboundByAccount.set(accountId, updatedAt);
|
|
1453
1605
|
}
|
|
1454
1606
|
|
|
1455
1607
|
this.lastDriftSnapshot =
|
|
1456
1608
|
data.lastDriftSnapshot && typeof data.lastDriftSnapshot === 'object'
|
|
1457
1609
|
? {
|
|
1458
|
-
capturedAt:
|
|
1610
|
+
capturedAt: finiteNumberOr((data.lastDriftSnapshot as any).capturedAt, 0),
|
|
1459
1611
|
registerCount: Number.isFinite(Number((data.lastDriftSnapshot as any).registerCount))
|
|
1460
1612
|
? Number((data.lastDriftSnapshot as any).registerCount)
|
|
1461
1613
|
: null,
|
|
@@ -1478,7 +1630,7 @@ class BncrBridgeRuntime {
|
|
|
1478
1630
|
typeof (data.lastDriftSnapshot as any).sourceBuckets === 'object'
|
|
1479
1631
|
? { ...((data.lastDriftSnapshot as any).sourceBuckets as Record<string, number>) }
|
|
1480
1632
|
: {},
|
|
1481
|
-
traceWindowSize:
|
|
1633
|
+
traceWindowSize: finiteNumberOr((data.lastDriftSnapshot as any).traceWindowSize, 0),
|
|
1482
1634
|
traceRecent: Array.isArray((data.lastDriftSnapshot as any).traceRecent)
|
|
1483
1635
|
? [...((data.lastDriftSnapshot as any).traceRecent as Array<Record<string, unknown>>)]
|
|
1484
1636
|
: [],
|
|
@@ -1489,8 +1641,8 @@ class BncrBridgeRuntime {
|
|
|
1489
1641
|
if (this.lastSessionByAccount.size === 0 && this.sessionRoutes.size > 0) {
|
|
1490
1642
|
for (const [sessionKey, info] of this.sessionRoutes.entries()) {
|
|
1491
1643
|
const acc = normalizeAccountId(info.accountId);
|
|
1492
|
-
const updatedAt =
|
|
1493
|
-
if (
|
|
1644
|
+
const updatedAt = finiteNumberOr(info.updatedAt, 0);
|
|
1645
|
+
if (updatedAt <= 0) continue;
|
|
1494
1646
|
|
|
1495
1647
|
const current = this.lastSessionByAccount.get(acc);
|
|
1496
1648
|
if (!current || updatedAt >= current.updatedAt) {
|
|
@@ -1521,38 +1673,38 @@ class BncrBridgeRuntime {
|
|
|
1521
1673
|
route: v.route,
|
|
1522
1674
|
updatedAt: v.updatedAt,
|
|
1523
1675
|
}))
|
|
1524
|
-
.slice(-
|
|
1676
|
+
.slice(-MAX_SESSION_ROUTE_ENTRIES);
|
|
1525
1677
|
|
|
1526
1678
|
const data: PersistedState = {
|
|
1527
1679
|
outbox: Array.from(this.outbox.values()),
|
|
1528
|
-
deadLetter: this.deadLetter.slice(-
|
|
1680
|
+
deadLetter: this.deadLetter.slice(-MAX_DEAD_LETTER_ENTRIES),
|
|
1529
1681
|
sessionRoutes,
|
|
1530
|
-
lastSessionByAccount: Array.from(this.lastSessionByAccount.entries())
|
|
1531
|
-
([accountId, v]) => ({
|
|
1682
|
+
lastSessionByAccount: Array.from(this.lastSessionByAccount.entries())
|
|
1683
|
+
.map(([accountId, v]) => ({
|
|
1532
1684
|
accountId,
|
|
1533
1685
|
sessionKey: v.sessionKey,
|
|
1534
1686
|
scope: v.scope,
|
|
1535
1687
|
updatedAt: v.updatedAt,
|
|
1536
|
-
})
|
|
1537
|
-
|
|
1538
|
-
lastActivityByAccount: Array.from(this.lastActivityByAccount.entries())
|
|
1539
|
-
([accountId, updatedAt]) => ({
|
|
1688
|
+
}))
|
|
1689
|
+
.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES),
|
|
1690
|
+
lastActivityByAccount: Array.from(this.lastActivityByAccount.entries())
|
|
1691
|
+
.map(([accountId, updatedAt]) => ({
|
|
1540
1692
|
accountId,
|
|
1541
1693
|
updatedAt,
|
|
1542
|
-
})
|
|
1543
|
-
|
|
1544
|
-
lastInboundByAccount: Array.from(this.lastInboundByAccount.entries())
|
|
1545
|
-
([accountId, updatedAt]) => ({
|
|
1694
|
+
}))
|
|
1695
|
+
.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES),
|
|
1696
|
+
lastInboundByAccount: Array.from(this.lastInboundByAccount.entries())
|
|
1697
|
+
.map(([accountId, updatedAt]) => ({
|
|
1546
1698
|
accountId,
|
|
1547
1699
|
updatedAt,
|
|
1548
|
-
})
|
|
1549
|
-
|
|
1550
|
-
lastOutboundByAccount: Array.from(this.lastOutboundByAccount.entries())
|
|
1551
|
-
([accountId, updatedAt]) => ({
|
|
1700
|
+
}))
|
|
1701
|
+
.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES),
|
|
1702
|
+
lastOutboundByAccount: Array.from(this.lastOutboundByAccount.entries())
|
|
1703
|
+
.map(([accountId, updatedAt]) => ({
|
|
1552
1704
|
accountId,
|
|
1553
1705
|
updatedAt,
|
|
1554
|
-
})
|
|
1555
|
-
|
|
1706
|
+
}))
|
|
1707
|
+
.slice(-MAX_ACCOUNT_ACTIVITY_ENTRIES),
|
|
1556
1708
|
lastDriftSnapshot: this.lastDriftSnapshot
|
|
1557
1709
|
? {
|
|
1558
1710
|
capturedAt: this.lastDriftSnapshot.capturedAt,
|
|
@@ -1569,7 +1721,7 @@ class BncrBridgeRuntime {
|
|
|
1569
1721
|
: null,
|
|
1570
1722
|
};
|
|
1571
1723
|
|
|
1572
|
-
await
|
|
1724
|
+
await writeOpenClawJsonFileAtomically(this.statePath, data);
|
|
1573
1725
|
}
|
|
1574
1726
|
|
|
1575
1727
|
private resolveMessageAck(messageId: string, result: 'acked' | 'timeout' = 'acked') {
|
|
@@ -1606,11 +1758,11 @@ class BncrBridgeRuntime {
|
|
|
1606
1758
|
|
|
1607
1759
|
const recentInboundConnIds = this.resolveRecentInboundConnIds(acc);
|
|
1608
1760
|
const candidateScore = (conn: BncrConnection) => {
|
|
1609
|
-
const preferredForOutboundUntil =
|
|
1610
|
-
const outboundReadyUntil =
|
|
1611
|
-
const lastPushTimeoutAt =
|
|
1612
|
-
const lastAckOkAt =
|
|
1613
|
-
const pushFailureScore =
|
|
1761
|
+
const preferredForOutboundUntil = finiteNumberOr((conn as any).preferredForOutboundUntil, 0);
|
|
1762
|
+
const outboundReadyUntil = finiteNumberOr((conn as any).outboundReadyUntil, 0);
|
|
1763
|
+
const lastPushTimeoutAt = finiteNumberOr((conn as any).lastPushTimeoutAt, 0);
|
|
1764
|
+
const lastAckOkAt = finiteNumberOr((conn as any).lastAckOkAt, 0);
|
|
1765
|
+
const pushFailureScore = finiteNumberOr((conn as any).pushFailureScore, 0);
|
|
1614
1766
|
const recentTimeoutPenalty = lastPushTimeoutAt > 0 && t - lastPushTimeoutAt <= 30_000 ? 1 : 0;
|
|
1615
1767
|
return {
|
|
1616
1768
|
preferred: preferredForOutboundUntil > t ? 1 : 0,
|
|
@@ -1702,11 +1854,11 @@ class BncrBridgeRuntime {
|
|
|
1702
1854
|
|
|
1703
1855
|
const recentInboundConnIds = this.resolveRecentInboundConnIds(acc);
|
|
1704
1856
|
const candidateScore = (conn: BncrConnection) => {
|
|
1705
|
-
const preferredForOutboundUntil =
|
|
1706
|
-
const outboundReadyUntil =
|
|
1707
|
-
const lastPushTimeoutAt =
|
|
1708
|
-
const lastAckOkAt =
|
|
1709
|
-
const pushFailureScore =
|
|
1857
|
+
const preferredForOutboundUntil = finiteNumberOr((conn as any).preferredForOutboundUntil, 0);
|
|
1858
|
+
const outboundReadyUntil = finiteNumberOr((conn as any).outboundReadyUntil, 0);
|
|
1859
|
+
const lastPushTimeoutAt = finiteNumberOr((conn as any).lastPushTimeoutAt, 0);
|
|
1860
|
+
const lastAckOkAt = finiteNumberOr((conn as any).lastAckOkAt, 0);
|
|
1861
|
+
const pushFailureScore = finiteNumberOr((conn as any).pushFailureScore, 0);
|
|
1710
1862
|
const recentTimeoutPenalty = lastPushTimeoutAt > 0 && t - lastPushTimeoutAt <= 30_000 ? 1 : 0;
|
|
1711
1863
|
return {
|
|
1712
1864
|
preferred: preferredForOutboundUntil > t ? 1 : 0,
|
|
@@ -2356,17 +2508,26 @@ class BncrBridgeRuntime {
|
|
|
2356
2508
|
}
|
|
2357
2509
|
|
|
2358
2510
|
private logOutboxAckSummary(
|
|
2359
|
-
scope:
|
|
2511
|
+
scope:
|
|
2512
|
+
| 'outbox ack ok'
|
|
2513
|
+
| 'outbox ack ok late'
|
|
2514
|
+
| 'outbox ack retry'
|
|
2515
|
+
| 'outbox ack timeout'
|
|
2516
|
+
| 'outbox ack fatal',
|
|
2360
2517
|
args: {
|
|
2361
2518
|
messageId: string;
|
|
2362
2519
|
connId?: string;
|
|
2363
2520
|
clientId?: string;
|
|
2364
2521
|
err?: string;
|
|
2522
|
+
queueMs?: number | null;
|
|
2523
|
+
pushMs?: number | null;
|
|
2524
|
+
waitMs?: number | null;
|
|
2365
2525
|
},
|
|
2366
2526
|
) {
|
|
2367
2527
|
const parts = [`mid=${args.messageId}`, `q=${this.outbox.size}`];
|
|
2368
|
-
if (args.
|
|
2369
|
-
if (args.
|
|
2528
|
+
if (typeof args.queueMs === 'number') parts.push(`queueMs=${args.queueMs}`);
|
|
2529
|
+
if (typeof args.pushMs === 'number') parts.push(`pushMs=${args.pushMs}`);
|
|
2530
|
+
if (typeof args.waitMs === 'number') parts.push(`waitMs=${args.waitMs}`);
|
|
2370
2531
|
if (args.err) parts.push(`err=${args.err}`);
|
|
2371
2532
|
this.logInfo(scope, parts.join('|'));
|
|
2372
2533
|
}
|
|
@@ -2377,6 +2538,7 @@ class BncrBridgeRuntime {
|
|
|
2377
2538
|
ackResult: 'acked' | 'timeout';
|
|
2378
2539
|
onlineNow: boolean;
|
|
2379
2540
|
recentInboundReachable: boolean;
|
|
2541
|
+
ackTimeoutMs?: number | null;
|
|
2380
2542
|
}) {
|
|
2381
2543
|
this.logInfo(
|
|
2382
2544
|
'outbox',
|
|
@@ -2384,12 +2546,19 @@ class BncrBridgeRuntime {
|
|
|
2384
2546
|
buildOutboxAckDebugInfo({
|
|
2385
2547
|
messageId: args.entry.messageId,
|
|
2386
2548
|
accountId: args.entry.accountId,
|
|
2549
|
+
sessionKey: args.entry.sessionKey,
|
|
2550
|
+
to: formatDisplayScope(args.entry.route),
|
|
2387
2551
|
kind:
|
|
2388
2552
|
isPlainObject(args.entry.payload?._meta) && args.entry.payload?._meta?.kind === 'file-transfer'
|
|
2389
2553
|
? 'file-transfer'
|
|
2390
2554
|
: undefined,
|
|
2391
2555
|
requireAck: args.requireAck,
|
|
2392
2556
|
ackResult: args.ackResult,
|
|
2557
|
+
ackStage: 'message',
|
|
2558
|
+
ackOutcome: args.ackResult,
|
|
2559
|
+
reason: args.ackResult === 'timeout' ? OUTBOUND_TERMINAL_REASON.PUSH_ACK_TIMEOUT : 'message-acked',
|
|
2560
|
+
ackTimeoutMs: typeof args.ackTimeoutMs === 'number' ? args.ackTimeoutMs : undefined,
|
|
2561
|
+
adaptiveAckTimeoutEnabled: ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED,
|
|
2393
2562
|
onlineNow: args.onlineNow,
|
|
2394
2563
|
recentInboundReachable: args.recentInboundReachable,
|
|
2395
2564
|
connIds: args.entry.lastPushConnId ? [args.entry.lastPushConnId] : [],
|
|
@@ -2410,6 +2579,7 @@ class BncrBridgeRuntime {
|
|
|
2410
2579
|
availableConnIds: string[];
|
|
2411
2580
|
decision: ReturnType<typeof computeRetryRerouteDecision>;
|
|
2412
2581
|
localNextDelay: number | null;
|
|
2582
|
+
ackTimeoutMs?: number | null;
|
|
2413
2583
|
}) {
|
|
2414
2584
|
this.logOutboxAckSummary(
|
|
2415
2585
|
args.requireAck ? 'outbox ack timeout' : 'outbox ack retry',
|
|
@@ -2418,6 +2588,7 @@ class BncrBridgeRuntime {
|
|
|
2418
2588
|
connId: args.entry.lastPushConnId,
|
|
2419
2589
|
clientId: args.entry.lastPushClientId,
|
|
2420
2590
|
err: args.requireAck ? undefined : args.entry.lastError,
|
|
2591
|
+
waitMs: args.requireAck ? args.ackTimeoutMs : undefined,
|
|
2421
2592
|
},
|
|
2422
2593
|
);
|
|
2423
2594
|
this.logInfo(
|
|
@@ -2500,6 +2671,11 @@ class BncrBridgeRuntime {
|
|
|
2500
2671
|
return null;
|
|
2501
2672
|
}
|
|
2502
2673
|
|
|
2674
|
+
if (this.stopped) {
|
|
2675
|
+
respond(true, { ok: true, ignored: true, reason: 'service-stopped' });
|
|
2676
|
+
return null;
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2503
2679
|
const entry = this.outbox.get(messageId);
|
|
2504
2680
|
if (!entry) {
|
|
2505
2681
|
respond(true, { ok: true, message: 'already-acked-or-missing', stale: staleObserved.stale });
|
|
@@ -2548,6 +2724,7 @@ class BncrBridgeRuntime {
|
|
|
2548
2724
|
connId: string;
|
|
2549
2725
|
clientId?: string;
|
|
2550
2726
|
stale: boolean;
|
|
2727
|
+
entry: OutboxEntry;
|
|
2551
2728
|
}) {
|
|
2552
2729
|
this.markOutboundCapability({
|
|
2553
2730
|
accountId: args.accountId,
|
|
@@ -2556,14 +2733,47 @@ class BncrBridgeRuntime {
|
|
|
2556
2733
|
outboundReady: true,
|
|
2557
2734
|
preferredForOutbound: true,
|
|
2558
2735
|
});
|
|
2559
|
-
|
|
2736
|
+
const ackAt = now();
|
|
2737
|
+
this.lastAckOkByAccount.set(args.accountId, ackAt);
|
|
2738
|
+
const ackQueueLatencyMs = Math.max(0, ackAt - finiteNumberOr(args.entry.createdAt, ackAt));
|
|
2739
|
+
const ackPushLatencyMs =
|
|
2740
|
+
typeof args.entry.lastPushAt === 'number'
|
|
2741
|
+
? Math.max(0, ackAt - args.entry.lastPushAt)
|
|
2742
|
+
: null;
|
|
2743
|
+
this.lastAckQueueLatencyMsByAccount.set(args.accountId, ackQueueLatencyMs);
|
|
2744
|
+
if (typeof ackPushLatencyMs === 'number') {
|
|
2745
|
+
this.lastAckPushLatencyMsByAccount.set(args.accountId, ackPushLatencyMs);
|
|
2746
|
+
}
|
|
2747
|
+
const lateAccepted = args.entry.awaitingRetryPush === true;
|
|
2748
|
+
if (lateAccepted) {
|
|
2749
|
+
this.adaptiveAckRecoveryOkCountByAccount.set(args.accountId, 0);
|
|
2750
|
+
this.lateAckOkCountByAccount.set(
|
|
2751
|
+
args.accountId,
|
|
2752
|
+
this.getCounter(this.lateAckOkCountByAccount, args.accountId) + 1,
|
|
2753
|
+
);
|
|
2754
|
+
this.lastLateAckOkByAccount.set(args.accountId, ackAt);
|
|
2755
|
+
this.lastLateAckQueueLatencyMsByAccount.set(args.accountId, ackQueueLatencyMs);
|
|
2756
|
+
if (typeof ackPushLatencyMs === 'number') {
|
|
2757
|
+
this.lastLateAckPushLatencyMsByAccount.set(args.accountId, ackPushLatencyMs);
|
|
2758
|
+
}
|
|
2759
|
+
args.entry.awaitingRetryPush = false;
|
|
2760
|
+
args.entry.lastError = undefined;
|
|
2761
|
+
} else if (typeof ackPushLatencyMs === 'number' && ackPushLatencyMs <= PUSH_ACK_TIMEOUT_MS) {
|
|
2762
|
+
this.adaptiveAckRecoveryOkCountByAccount.set(
|
|
2763
|
+
args.accountId,
|
|
2764
|
+
this.getCounter(this.adaptiveAckRecoveryOkCountByAccount, args.accountId) + 1,
|
|
2765
|
+
);
|
|
2766
|
+
}
|
|
2560
2767
|
this.outbox.delete(args.messageId);
|
|
2561
2768
|
this.scheduleSave();
|
|
2562
2769
|
this.resolveMessageAck(args.messageId, 'acked');
|
|
2563
|
-
this.logOutboxAckSummary('outbox ack ok', {
|
|
2770
|
+
this.logOutboxAckSummary(lateAccepted ? 'outbox ack ok late' : 'outbox ack ok', {
|
|
2564
2771
|
messageId: args.messageId,
|
|
2565
2772
|
connId: args.connId,
|
|
2566
2773
|
clientId: args.clientId,
|
|
2774
|
+
queueMs: ackQueueLatencyMs,
|
|
2775
|
+
pushMs: ackPushLatencyMs,
|
|
2776
|
+
err: lateAccepted ? 'accepted-after-timeout' : undefined,
|
|
2567
2777
|
});
|
|
2568
2778
|
}
|
|
2569
2779
|
|
|
@@ -2592,6 +2802,7 @@ class BncrBridgeRuntime {
|
|
|
2592
2802
|
}) {
|
|
2593
2803
|
args.entry.nextAttemptAt = now() + 1_000;
|
|
2594
2804
|
args.entry.lastError = args.error;
|
|
2805
|
+
args.entry.awaitingRetryPush = true;
|
|
2595
2806
|
this.outbox.set(args.messageId, args.entry);
|
|
2596
2807
|
this.scheduleSave();
|
|
2597
2808
|
this.logOutboxAckSummary('outbox ack retry', {
|
|
@@ -2623,6 +2834,7 @@ class BncrBridgeRuntime {
|
|
|
2623
2834
|
connId,
|
|
2624
2835
|
clientId,
|
|
2625
2836
|
stale: staleObserved.stale,
|
|
2837
|
+
entry,
|
|
2626
2838
|
});
|
|
2627
2839
|
this.respondAckResult(respond, staleObserved.stale, { ok: true });
|
|
2628
2840
|
this.flushPushQueue({
|
|
@@ -2663,10 +2875,10 @@ class BncrBridgeRuntime {
|
|
|
2663
2875
|
});
|
|
2664
2876
|
}
|
|
2665
2877
|
|
|
2666
|
-
private prepareInboundAcceptance(args: {
|
|
2878
|
+
private async prepareInboundAcceptance(args: {
|
|
2667
2879
|
parsed: ReturnType<typeof parseBncrInboundParams>;
|
|
2668
2880
|
canonicalAgentId: string;
|
|
2669
|
-
}):
|
|
2881
|
+
}): Promise<
|
|
2670
2882
|
| {
|
|
2671
2883
|
ok: true;
|
|
2672
2884
|
accountId: string;
|
|
@@ -2678,7 +2890,8 @@ class BncrBridgeRuntime {
|
|
|
2678
2890
|
ok: false;
|
|
2679
2891
|
status: boolean;
|
|
2680
2892
|
payload: ReturnType<typeof buildInboundResponsePayload>;
|
|
2681
|
-
}
|
|
2893
|
+
}
|
|
2894
|
+
> {
|
|
2682
2895
|
const { parsed, canonicalAgentId } = args;
|
|
2683
2896
|
const {
|
|
2684
2897
|
accountId,
|
|
@@ -2715,8 +2928,8 @@ class BncrBridgeRuntime {
|
|
|
2715
2928
|
};
|
|
2716
2929
|
}
|
|
2717
2930
|
|
|
2718
|
-
const cfg = this.api
|
|
2719
|
-
const gate = checkBncrMessageGate({
|
|
2931
|
+
const cfg = getOpenClawRuntimeConfig(this.api);
|
|
2932
|
+
const gate = await checkBncrMessageGate({
|
|
2720
2933
|
parsed,
|
|
2721
2934
|
cfg,
|
|
2722
2935
|
account: resolveAccount(cfg, accountId),
|
|
@@ -2744,7 +2957,7 @@ class BncrBridgeRuntime {
|
|
|
2744
2957
|
taskKey: extracted.taskKey,
|
|
2745
2958
|
text,
|
|
2746
2959
|
extractedText: extracted.text,
|
|
2747
|
-
resolveAgentRoute: (params) => this.api
|
|
2960
|
+
resolveAgentRoute: (params) => resolveOpenClawAgentRoute(this.api, params),
|
|
2748
2961
|
});
|
|
2749
2962
|
|
|
2750
2963
|
return {
|
|
@@ -2842,6 +3055,7 @@ class BncrBridgeRuntime {
|
|
|
2842
3055
|
args.entry.lastPushConnId =
|
|
2843
3056
|
args.ownerConnId || (connIds.length === 1 ? connIds[0] : undefined);
|
|
2844
3057
|
args.entry.lastPushClientId = args.ownerClientId;
|
|
3058
|
+
args.entry.awaitingRetryPush = false;
|
|
2845
3059
|
if (!Array.isArray(args.entry.routeAttemptConnIds)) args.entry.routeAttemptConnIds = [];
|
|
2846
3060
|
if (
|
|
2847
3061
|
args.entry.lastPushConnId &&
|
|
@@ -2857,6 +3071,7 @@ class BncrBridgeRuntime {
|
|
|
2857
3071
|
}
|
|
2858
3072
|
|
|
2859
3073
|
private schedulePushDrain(delayMs = 0) {
|
|
3074
|
+
if (this.stopped) return;
|
|
2860
3075
|
// Structure note (drain scheduler):
|
|
2861
3076
|
// This is the single-timer gate for outbound retry scheduling. It intentionally coalesces
|
|
2862
3077
|
// multiple nudges into one pending timer and delegates all actual decision-making to
|
|
@@ -2877,6 +3092,7 @@ class BncrBridgeRuntime {
|
|
|
2877
3092
|
);
|
|
2878
3093
|
this.pushTimer = setTimeout(() => {
|
|
2879
3094
|
this.pushTimer = null;
|
|
3095
|
+
if (this.stopped) return;
|
|
2880
3096
|
void this.flushPushQueue({
|
|
2881
3097
|
trigger: OUTBOUND_FLUSH_TRIGGER.TIMER,
|
|
2882
3098
|
reason: OUTBOUND_FLUSH_REASON.SCHEDULED_DRAIN,
|
|
@@ -2886,7 +3102,7 @@ class BncrBridgeRuntime {
|
|
|
2886
3102
|
|
|
2887
3103
|
private isOutboundAckRequired(accountId?: string) {
|
|
2888
3104
|
try {
|
|
2889
|
-
const cfg = this.api
|
|
3105
|
+
const cfg = getOpenClawRuntimeConfig(this.api);
|
|
2890
3106
|
const channelCfg = (cfg as any)?.channels?.[CHANNEL_ID];
|
|
2891
3107
|
const accountCfg =
|
|
2892
3108
|
accountId && channelCfg?.accounts && typeof channelCfg.accounts === 'object'
|
|
@@ -2905,7 +3121,7 @@ class BncrBridgeRuntime {
|
|
|
2905
3121
|
private buildRuntimeFlags(accountId?: string) {
|
|
2906
3122
|
let ackPolicySource: 'channel' | 'default' = 'default';
|
|
2907
3123
|
try {
|
|
2908
|
-
const cfg = this.api
|
|
3124
|
+
const cfg = getOpenClawRuntimeConfig(this.api);
|
|
2909
3125
|
const global = (cfg as any)?.channels?.[CHANNEL_ID]?.outboundRequireAck;
|
|
2910
3126
|
if (typeof global === 'boolean') ackPolicySource = 'channel';
|
|
2911
3127
|
} catch {
|
|
@@ -2914,7 +3130,9 @@ class BncrBridgeRuntime {
|
|
|
2914
3130
|
return {
|
|
2915
3131
|
outboundRequireAck: this.isOutboundAckRequired(accountId),
|
|
2916
3132
|
ackPolicySource,
|
|
2917
|
-
messageAckTimeoutMs:
|
|
3133
|
+
messageAckTimeoutMs: this.resolveMessageAckTimeoutMs(accountId),
|
|
3134
|
+
adaptiveAckTimeoutEnabled: ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED,
|
|
3135
|
+
defaultMessageAckTimeoutMs: PUSH_ACK_TIMEOUT_MS,
|
|
2918
3136
|
fileAckTimeoutMs: FILE_ACK_TIMEOUT_MS,
|
|
2919
3137
|
debugVerbose: BNCR_DEBUG_VERBOSE,
|
|
2920
3138
|
};
|
|
@@ -2925,6 +3143,7 @@ class BncrBridgeRuntime {
|
|
|
2925
3143
|
trigger?: string;
|
|
2926
3144
|
reason?: string;
|
|
2927
3145
|
}): Promise<void> {
|
|
3146
|
+
if (this.stopped) return;
|
|
2928
3147
|
// Structure guide for future safe extraction:
|
|
2929
3148
|
// - pre-check: choose target accounts, skip accounts already draining, emit flush context logs
|
|
2930
3149
|
// - tryPush: pick one due entry per account and attempt actual outbound delivery
|
|
@@ -2995,8 +3214,45 @@ class BncrBridgeRuntime {
|
|
|
2995
3214
|
this.pushDrainRunningAccounts.add(acc);
|
|
2996
3215
|
try {
|
|
2997
3216
|
let localNextDelay: number | null = null;
|
|
3217
|
+
let processedThisRun = 0;
|
|
3218
|
+
const accountDrainStartedAt = now();
|
|
2998
3219
|
|
|
2999
3220
|
while (true) {
|
|
3221
|
+
if (this.stopped) break;
|
|
3222
|
+
if (processedThisRun > 0 && now() - accountDrainStartedAt >= PUSH_DRAIN_ACCOUNT_TIME_BUDGET_MS) {
|
|
3223
|
+
localNextDelay = updateMinOutboxDelay(localNextDelay, 0);
|
|
3224
|
+
this.logInfo(
|
|
3225
|
+
'outbox',
|
|
3226
|
+
`schedule ${JSON.stringify(
|
|
3227
|
+
buildOutboxScheduleDebugInfo({
|
|
3228
|
+
bridgeId: this.bridgeId,
|
|
3229
|
+
accountId: acc,
|
|
3230
|
+
source: OUTBOUND_SCHEDULE_SOURCE.ACCOUNT_TIME_BUDGET_YIELD,
|
|
3231
|
+
wait: 0,
|
|
3232
|
+
localNextDelay,
|
|
3233
|
+
}),
|
|
3234
|
+
)}`,
|
|
3235
|
+
{ debugOnly: true },
|
|
3236
|
+
);
|
|
3237
|
+
break;
|
|
3238
|
+
}
|
|
3239
|
+
if (processedThisRun >= PUSH_DRAIN_ACCOUNT_BUDGET) {
|
|
3240
|
+
localNextDelay = updateMinOutboxDelay(localNextDelay, 0);
|
|
3241
|
+
this.logInfo(
|
|
3242
|
+
'outbox',
|
|
3243
|
+
`schedule ${JSON.stringify(
|
|
3244
|
+
buildOutboxScheduleDebugInfo({
|
|
3245
|
+
bridgeId: this.bridgeId,
|
|
3246
|
+
accountId: acc,
|
|
3247
|
+
source: OUTBOUND_SCHEDULE_SOURCE.ACCOUNT_BUDGET_YIELD,
|
|
3248
|
+
wait: 0,
|
|
3249
|
+
localNextDelay,
|
|
3250
|
+
}),
|
|
3251
|
+
)}`,
|
|
3252
|
+
{ debugOnly: true },
|
|
3253
|
+
);
|
|
3254
|
+
break;
|
|
3255
|
+
}
|
|
3000
3256
|
const t = now();
|
|
3001
3257
|
const entries = listAccountOutboxEntries({
|
|
3002
3258
|
accountId: acc,
|
|
@@ -3031,11 +3287,13 @@ class BncrBridgeRuntime {
|
|
|
3031
3287
|
const onlineNow = this.isOnline(acc);
|
|
3032
3288
|
const recentInboundReachable = this.hasRecentInboundReachability(acc);
|
|
3033
3289
|
const pushed = await this.tryPushEntry(entry);
|
|
3290
|
+
processedThisRun += 1;
|
|
3034
3291
|
if (pushed) {
|
|
3035
3292
|
const requireAck = this.isOutboundAckRequired(acc);
|
|
3293
|
+
const ackTimeoutMs = requireAck ? this.resolveMessageAckTimeoutMs(acc) : null;
|
|
3036
3294
|
let ackResult: 'acked' | 'timeout' = requireAck ? 'timeout' : 'acked';
|
|
3037
3295
|
if (onlineNow && requireAck) {
|
|
3038
|
-
ackResult = await this.waitForMessageAck(entry.messageId, PUSH_ACK_TIMEOUT_MS);
|
|
3296
|
+
ackResult = await this.waitForMessageAck(entry.messageId, ackTimeoutMs || PUSH_ACK_TIMEOUT_MS);
|
|
3039
3297
|
}
|
|
3040
3298
|
|
|
3041
3299
|
this.logOutboxAckWait({
|
|
@@ -3044,6 +3302,7 @@ class BncrBridgeRuntime {
|
|
|
3044
3302
|
ackResult,
|
|
3045
3303
|
onlineNow,
|
|
3046
3304
|
recentInboundReachable,
|
|
3305
|
+
ackTimeoutMs,
|
|
3047
3306
|
});
|
|
3048
3307
|
|
|
3049
3308
|
if (!this.outbox.has(entry.messageId)) {
|
|
@@ -3112,6 +3371,7 @@ class BncrBridgeRuntime {
|
|
|
3112
3371
|
acc,
|
|
3113
3372
|
this.getCounter(this.ackTimeoutCountByAccount, acc) + 1,
|
|
3114
3373
|
);
|
|
3374
|
+
this.adaptiveAckRecoveryOkCountByAccount.set(acc, 0);
|
|
3115
3375
|
}
|
|
3116
3376
|
const wait = computeOutboxRetryWait(decision.nextAttemptAt, now());
|
|
3117
3377
|
localNextDelay = updateMinOutboxDelay(localNextDelay, wait);
|
|
@@ -3123,6 +3383,7 @@ class BncrBridgeRuntime {
|
|
|
3123
3383
|
availableConnIds,
|
|
3124
3384
|
decision,
|
|
3125
3385
|
localNextDelay,
|
|
3386
|
+
ackTimeoutMs,
|
|
3126
3387
|
});
|
|
3127
3388
|
await this.sleepMs(PUSH_DRAIN_INTERVAL_MS);
|
|
3128
3389
|
break;
|
|
@@ -3213,17 +3474,36 @@ class BncrBridgeRuntime {
|
|
|
3213
3474
|
|
|
3214
3475
|
private async waitForMessageAck(messageId: string, waitMs: number): Promise<'acked' | 'timeout'> {
|
|
3215
3476
|
const key = asString(messageId).trim();
|
|
3216
|
-
const timeoutMs =
|
|
3477
|
+
const timeoutMs = clampFiniteNumber(
|
|
3478
|
+
waitMs,
|
|
3479
|
+
0,
|
|
3480
|
+
0,
|
|
3481
|
+
RECOMMENDED_ACK_TIMEOUT_MAX_MS,
|
|
3482
|
+
);
|
|
3217
3483
|
if (!key || !timeoutMs) return 'timeout';
|
|
3218
3484
|
|
|
3219
|
-
|
|
3220
|
-
|
|
3485
|
+
const existing = this.messageAckWaiters.get(key);
|
|
3486
|
+
if (existing) {
|
|
3487
|
+
this.logWarn(
|
|
3488
|
+
'outbox',
|
|
3489
|
+
`message-ack-waiter-reuse ${JSON.stringify({ bridge: this.bridgeId, messageId: key })}`,
|
|
3490
|
+
{ debugOnly: true },
|
|
3491
|
+
);
|
|
3492
|
+
return await existing.promise;
|
|
3493
|
+
}
|
|
3494
|
+
|
|
3495
|
+
let timer: NodeJS.Timeout;
|
|
3496
|
+
let resolveWaiter!: (result: 'acked' | 'timeout') => void;
|
|
3497
|
+
const promise = new Promise<'acked' | 'timeout'>((resolve) => {
|
|
3498
|
+
resolveWaiter = resolve;
|
|
3499
|
+
timer = setTimeout(() => {
|
|
3221
3500
|
this.messageAckWaiters.delete(key);
|
|
3222
3501
|
resolve('timeout');
|
|
3223
3502
|
}, timeoutMs);
|
|
3224
|
-
|
|
3225
|
-
this.messageAckWaiters.set(key, { resolve, timer });
|
|
3226
3503
|
});
|
|
3504
|
+
|
|
3505
|
+
this.messageAckWaiters.set(key, { promise, resolve: resolveWaiter, timer: timer! });
|
|
3506
|
+
return await promise;
|
|
3227
3507
|
}
|
|
3228
3508
|
|
|
3229
3509
|
private connectionKey(accountId: string, clientId?: string): string {
|
|
@@ -3253,6 +3533,9 @@ class BncrBridgeRuntime {
|
|
|
3253
3533
|
{ debugOnly: true },
|
|
3254
3534
|
);
|
|
3255
3535
|
this.connections.delete(key);
|
|
3536
|
+
if (this.activeConnectionByAccount.get(c.accountId) === key) {
|
|
3537
|
+
this.activeConnectionByAccount.delete(c.accountId);
|
|
3538
|
+
}
|
|
3256
3539
|
}
|
|
3257
3540
|
}
|
|
3258
3541
|
|
|
@@ -3267,11 +3550,23 @@ class BncrBridgeRuntime {
|
|
|
3267
3550
|
|
|
3268
3551
|
private cleanupFileTransfers() {
|
|
3269
3552
|
const t = now();
|
|
3553
|
+
const keepMsForTransfer = (st: { status: string; startedAt: number; terminalAt?: number }) => {
|
|
3554
|
+
const startedAt = finiteNumberOr(st.startedAt, t);
|
|
3555
|
+
if (st.status === 'completed' || st.status === 'aborted') {
|
|
3556
|
+
return {
|
|
3557
|
+
since: finiteNumberOr(st.terminalAt, startedAt),
|
|
3558
|
+
keepMs: FILE_TRANSFER_TERMINAL_KEEP_MS,
|
|
3559
|
+
};
|
|
3560
|
+
}
|
|
3561
|
+
return { since: startedAt, keepMs: FILE_TRANSFER_KEEP_MS };
|
|
3562
|
+
};
|
|
3270
3563
|
for (const [id, st] of this.fileSendTransfers.entries()) {
|
|
3271
|
-
|
|
3564
|
+
const keep = keepMsForTransfer(st);
|
|
3565
|
+
if (t - keep.since > keep.keepMs) this.fileSendTransfers.delete(id);
|
|
3272
3566
|
}
|
|
3273
3567
|
for (const [id, st] of this.fileRecvTransfers.entries()) {
|
|
3274
|
-
|
|
3568
|
+
const keep = keepMsForTransfer(st);
|
|
3569
|
+
if (t - keep.since > keep.keepMs) this.fileRecvTransfers.delete(id);
|
|
3275
3570
|
}
|
|
3276
3571
|
for (const [key, ack] of this.earlyFileAcks.entries()) {
|
|
3277
3572
|
if (t - ack.at > FILE_TRANSFER_ACK_TTL_MS) this.earlyFileAcks.delete(key);
|
|
@@ -3638,7 +3933,7 @@ class BncrBridgeRuntime {
|
|
|
3638
3933
|
const canonicalAgentId =
|
|
3639
3934
|
this.canonicalAgentId ||
|
|
3640
3935
|
this.ensureCanonicalAgentId({
|
|
3641
|
-
cfg: this.api
|
|
3936
|
+
cfg: getOpenClawRuntimeConfigOrDefault(this.api, {}),
|
|
3642
3937
|
accountId: acc,
|
|
3643
3938
|
channelId: CHANNEL_ID,
|
|
3644
3939
|
peer: { kind: 'direct', id: route.groupId === '0' ? route.userId : route.groupId },
|
|
@@ -3671,10 +3966,19 @@ class BncrBridgeRuntime {
|
|
|
3671
3966
|
}
|
|
3672
3967
|
|
|
3673
3968
|
private fileAckKey(transferId: string, stage: string, chunkIndex?: number): string {
|
|
3674
|
-
const
|
|
3969
|
+
const n = Number(chunkIndex);
|
|
3970
|
+
const idx = Number.isInteger(n) && n >= 0 ? String(n) : '-';
|
|
3675
3971
|
return `${transferId}|${stage}|${idx}`;
|
|
3676
3972
|
}
|
|
3677
3973
|
|
|
3974
|
+
private fileAckOwnerInfo(transferId: string) {
|
|
3975
|
+
const st = this.fileSendTransfers.get(transferId);
|
|
3976
|
+
return {
|
|
3977
|
+
...(st?.ownerConnId ? { ownerConnId: st.ownerConnId } : {}),
|
|
3978
|
+
...(st?.ownerClientId ? { ownerClientId: st.ownerClientId } : {}),
|
|
3979
|
+
};
|
|
3980
|
+
}
|
|
3981
|
+
|
|
3678
3982
|
private waitForFileAck(params: {
|
|
3679
3983
|
transferId: string;
|
|
3680
3984
|
stage: string;
|
|
@@ -3684,10 +3988,8 @@ class BncrBridgeRuntime {
|
|
|
3684
3988
|
const transferId = asString(params.transferId).trim();
|
|
3685
3989
|
const stage = asString(params.stage).trim();
|
|
3686
3990
|
const key = this.fileAckKey(transferId, stage, params.chunkIndex);
|
|
3687
|
-
const timeoutMs =
|
|
3688
|
-
|
|
3689
|
-
Math.min(Number(params.timeoutMs || FILE_ACK_TIMEOUT_MS), 120_000),
|
|
3690
|
-
);
|
|
3991
|
+
const timeoutMs = clampFiniteNumber(params.timeoutMs, FILE_ACK_TIMEOUT_MS, 1_000, 120_000);
|
|
3992
|
+
const ownerInfo = this.fileAckOwnerInfo(transferId);
|
|
3691
3993
|
|
|
3692
3994
|
const cached = this.earlyFileAcks.get(key);
|
|
3693
3995
|
if (cached) {
|
|
@@ -3698,9 +4000,13 @@ class BncrBridgeRuntime {
|
|
|
3698
4000
|
bridge: this.bridgeId,
|
|
3699
4001
|
transferId,
|
|
3700
4002
|
stage,
|
|
4003
|
+
ackStage: stage,
|
|
4004
|
+
ackOutcome: cached.ok ? 'acked' : 'failed',
|
|
4005
|
+
waiterReused: false,
|
|
3701
4006
|
chunkIndex:
|
|
3702
4007
|
Number.isFinite(Number(params.chunkIndex)) ? Number(params.chunkIndex) : undefined,
|
|
3703
4008
|
key,
|
|
4009
|
+
...ownerInfo,
|
|
3704
4010
|
ok: cached.ok,
|
|
3705
4011
|
payload: cached.payload,
|
|
3706
4012
|
}),
|
|
@@ -3714,22 +4020,52 @@ class BncrBridgeRuntime {
|
|
|
3714
4020
|
);
|
|
3715
4021
|
}
|
|
3716
4022
|
|
|
4023
|
+
const existing = this.fileAckWaiters.get(key);
|
|
4024
|
+
if (existing) {
|
|
4025
|
+
this.logWarn(
|
|
4026
|
+
'file-ack-waiter-reuse',
|
|
4027
|
+
JSON.stringify({
|
|
4028
|
+
bridge: this.bridgeId,
|
|
4029
|
+
transferId,
|
|
4030
|
+
stage,
|
|
4031
|
+
ackStage: stage,
|
|
4032
|
+
ackOutcome: 'waiter-reused',
|
|
4033
|
+
waiterReused: true,
|
|
4034
|
+
chunkIndex:
|
|
4035
|
+
Number.isFinite(Number(params.chunkIndex)) ? Number(params.chunkIndex) : undefined,
|
|
4036
|
+
key,
|
|
4037
|
+
...ownerInfo,
|
|
4038
|
+
}),
|
|
4039
|
+
{ debugOnly: true },
|
|
4040
|
+
);
|
|
4041
|
+
return existing.promise;
|
|
4042
|
+
}
|
|
4043
|
+
|
|
3717
4044
|
this.logInfo(
|
|
3718
4045
|
'file-ack-wait',
|
|
3719
4046
|
JSON.stringify({
|
|
3720
4047
|
bridge: this.bridgeId,
|
|
3721
4048
|
transferId,
|
|
3722
4049
|
stage,
|
|
4050
|
+
ackStage: stage,
|
|
4051
|
+
ackOutcome: 'waiting',
|
|
4052
|
+
waiterReused: false,
|
|
3723
4053
|
chunkIndex:
|
|
3724
4054
|
Number.isFinite(Number(params.chunkIndex)) ? Number(params.chunkIndex) : undefined,
|
|
3725
4055
|
key,
|
|
4056
|
+
...ownerInfo,
|
|
3726
4057
|
timeoutMs,
|
|
3727
4058
|
}),
|
|
3728
4059
|
{ debugOnly: true },
|
|
3729
4060
|
);
|
|
3730
4061
|
|
|
3731
|
-
|
|
3732
|
-
|
|
4062
|
+
let timer: NodeJS.Timeout;
|
|
4063
|
+
let resolveWaiter!: (payload: Record<string, unknown>) => void;
|
|
4064
|
+
let rejectWaiter!: (err: Error) => void;
|
|
4065
|
+
const promise = new Promise<Record<string, unknown>>((resolve, reject) => {
|
|
4066
|
+
resolveWaiter = resolve;
|
|
4067
|
+
rejectWaiter = reject;
|
|
4068
|
+
timer = setTimeout(() => {
|
|
3733
4069
|
this.fileAckWaiters.delete(key);
|
|
3734
4070
|
this.logWarn(
|
|
3735
4071
|
OUTBOUND_TERMINAL_REASON.FILE_ACK_TIMEOUT,
|
|
@@ -3737,17 +4073,27 @@ class BncrBridgeRuntime {
|
|
|
3737
4073
|
bridge: this.bridgeId,
|
|
3738
4074
|
transferId,
|
|
3739
4075
|
stage,
|
|
4076
|
+
ackStage: stage,
|
|
4077
|
+
ackOutcome: 'timeout',
|
|
4078
|
+
waiterReused: false,
|
|
3740
4079
|
chunkIndex:
|
|
3741
4080
|
Number.isFinite(Number(params.chunkIndex)) ? Number(params.chunkIndex) : undefined,
|
|
3742
4081
|
key,
|
|
4082
|
+
...ownerInfo,
|
|
3743
4083
|
timeoutMs,
|
|
3744
4084
|
}),
|
|
3745
4085
|
{ debugOnly: true },
|
|
3746
4086
|
);
|
|
3747
4087
|
reject(new Error(`file ack timeout: ${key}`));
|
|
3748
4088
|
}, timeoutMs);
|
|
3749
|
-
this.fileAckWaiters.set(key, { resolve, reject, timer });
|
|
3750
4089
|
});
|
|
4090
|
+
this.fileAckWaiters.set(key, {
|
|
4091
|
+
promise,
|
|
4092
|
+
resolve: resolveWaiter,
|
|
4093
|
+
reject: rejectWaiter,
|
|
4094
|
+
timer: timer!,
|
|
4095
|
+
});
|
|
4096
|
+
return promise;
|
|
3751
4097
|
}
|
|
3752
4098
|
|
|
3753
4099
|
private resolveFileAck(params: {
|
|
@@ -3760,9 +4106,10 @@ class BncrBridgeRuntime {
|
|
|
3760
4106
|
const transferId = asString(params.transferId).trim();
|
|
3761
4107
|
const stage = asString(params.stage).trim();
|
|
3762
4108
|
const key = this.fileAckKey(transferId, stage, params.chunkIndex);
|
|
4109
|
+
const ownerInfo = this.fileAckOwnerInfo(transferId);
|
|
3763
4110
|
const waiter = this.fileAckWaiters.get(key);
|
|
3764
4111
|
if (!waiter) {
|
|
3765
|
-
this.
|
|
4112
|
+
this.rememberEarlyFileAck(key, {
|
|
3766
4113
|
payload: params.payload,
|
|
3767
4114
|
ok: params.ok,
|
|
3768
4115
|
at: now(),
|
|
@@ -3773,9 +4120,13 @@ class BncrBridgeRuntime {
|
|
|
3773
4120
|
bridge: this.bridgeId,
|
|
3774
4121
|
transferId,
|
|
3775
4122
|
stage,
|
|
4123
|
+
ackStage: stage,
|
|
4124
|
+
ackOutcome: params.ok ? 'early-acked' : 'early-failed',
|
|
4125
|
+
waiterReused: false,
|
|
3776
4126
|
chunkIndex:
|
|
3777
4127
|
Number.isFinite(Number(params.chunkIndex)) ? Number(params.chunkIndex) : undefined,
|
|
3778
4128
|
key,
|
|
4129
|
+
...ownerInfo,
|
|
3779
4130
|
ok: params.ok,
|
|
3780
4131
|
payload: params.payload,
|
|
3781
4132
|
cached: true,
|
|
@@ -3792,9 +4143,13 @@ class BncrBridgeRuntime {
|
|
|
3792
4143
|
bridge: this.bridgeId,
|
|
3793
4144
|
transferId,
|
|
3794
4145
|
stage,
|
|
4146
|
+
ackStage: stage,
|
|
4147
|
+
ackOutcome: params.ok ? 'acked' : 'failed',
|
|
4148
|
+
waiterReused: false,
|
|
3795
4149
|
chunkIndex:
|
|
3796
4150
|
Number.isFinite(Number(params.chunkIndex)) ? Number(params.chunkIndex) : undefined,
|
|
3797
4151
|
key,
|
|
4152
|
+
...ownerInfo,
|
|
3798
4153
|
ok: params.ok,
|
|
3799
4154
|
payload: params.payload,
|
|
3800
4155
|
}),
|
|
@@ -3841,38 +4196,6 @@ class BncrBridgeRuntime {
|
|
|
3841
4196
|
return mt || 'file';
|
|
3842
4197
|
}
|
|
3843
4198
|
|
|
3844
|
-
private resolveInboundFilesDir(): string {
|
|
3845
|
-
const dir = path.join(process.cwd(), '.openclaw', 'media', 'inbound', 'bncr');
|
|
3846
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
3847
|
-
return dir;
|
|
3848
|
-
}
|
|
3849
|
-
|
|
3850
|
-
private async materializeRecvTransfer(
|
|
3851
|
-
st: FileRecvTransferState,
|
|
3852
|
-
): Promise<{ path: string; fileSha256: string }> {
|
|
3853
|
-
const dir = this.resolveInboundFilesDir();
|
|
3854
|
-
const safeName = asString(st.fileName).trim() || `${st.transferId}.bin`;
|
|
3855
|
-
const finalPath = path.join(dir, safeName);
|
|
3856
|
-
|
|
3857
|
-
const ordered: Buffer[] = [];
|
|
3858
|
-
for (let i = 0; i < st.totalChunks; i++) {
|
|
3859
|
-
const chunk = st.bufferByChunk.get(i);
|
|
3860
|
-
if (!chunk) throw new Error(`missing chunk ${i}`);
|
|
3861
|
-
ordered.push(chunk);
|
|
3862
|
-
}
|
|
3863
|
-
const merged = Buffer.concat(ordered);
|
|
3864
|
-
if (Number(st.fileSize || 0) > 0 && merged.length !== Number(st.fileSize || 0)) {
|
|
3865
|
-
throw new Error(`size mismatch expected=${st.fileSize} got=${merged.length}`);
|
|
3866
|
-
}
|
|
3867
|
-
|
|
3868
|
-
const sha = createHash('sha256').update(merged).digest('hex');
|
|
3869
|
-
if (st.fileSha256 && sha !== st.fileSha256) {
|
|
3870
|
-
throw new Error(`sha256 mismatch expected=${st.fileSha256} got=${sha}`);
|
|
3871
|
-
}
|
|
3872
|
-
|
|
3873
|
-
fs.writeFileSync(finalPath, merged);
|
|
3874
|
-
return { path: finalPath, fileSha256: sha };
|
|
3875
|
-
}
|
|
3876
4199
|
|
|
3877
4200
|
private buildRuntimeQueueSnapshot(accountId: string) {
|
|
3878
4201
|
const pending = Array.from(this.outbox.values()).filter((v) => v.accountId === accountId).length;
|
|
@@ -3898,6 +4221,213 @@ class BncrBridgeRuntime {
|
|
|
3898
4221
|
};
|
|
3899
4222
|
}
|
|
3900
4223
|
|
|
4224
|
+
private computeRecommendedAckTimeoutReason(args: {
|
|
4225
|
+
lateAckOkCount: number;
|
|
4226
|
+
recentAckTimeoutCount: number;
|
|
4227
|
+
lastLateAckPushLatencyMs: number | null;
|
|
4228
|
+
lastLateAckOkAt?: number | null;
|
|
4229
|
+
adaptiveAckRecoveryOkCount?: number;
|
|
4230
|
+
recommendedAckTimeoutMs?: number;
|
|
4231
|
+
nowMs?: number;
|
|
4232
|
+
}) {
|
|
4233
|
+
if (args.recentAckTimeoutCount <= 0) return 'no-timeout-evidence';
|
|
4234
|
+
if (args.lateAckOkCount <= 0) return 'no-late-ack-evidence';
|
|
4235
|
+
if (typeof args.lastLateAckPushLatencyMs !== 'number') return 'missing-latency';
|
|
4236
|
+
const lastLateAckOkAt = typeof args.lastLateAckOkAt === 'number' ? args.lastLateAckOkAt : null;
|
|
4237
|
+
const nowMs = typeof args.nowMs === 'number' ? args.nowMs : now();
|
|
4238
|
+
if (
|
|
4239
|
+
typeof lastLateAckOkAt === 'number' &&
|
|
4240
|
+
lastLateAckOkAt > 0 &&
|
|
4241
|
+
nowMs - lastLateAckOkAt > ADAPTIVE_ACK_TIMEOUT_OBSERVATION_TTL_MS
|
|
4242
|
+
) {
|
|
4243
|
+
return 'late-ack-expired';
|
|
4244
|
+
}
|
|
4245
|
+
if (
|
|
4246
|
+
typeof args.adaptiveAckRecoveryOkCount === 'number' &&
|
|
4247
|
+
args.adaptiveAckRecoveryOkCount >= ADAPTIVE_ACK_TIMEOUT_RECOVERY_OK_THRESHOLD
|
|
4248
|
+
) {
|
|
4249
|
+
return 'recovered';
|
|
4250
|
+
}
|
|
4251
|
+
if (args.recommendedAckTimeoutMs === RECOMMENDED_ACK_TIMEOUT_MAX_MS) return 'capped-max';
|
|
4252
|
+
return 'late-ack-observed';
|
|
4253
|
+
}
|
|
4254
|
+
|
|
4255
|
+
private computeRecommendedAckTimeoutMs(args: {
|
|
4256
|
+
lateAckOkCount: number;
|
|
4257
|
+
recentAckTimeoutCount: number;
|
|
4258
|
+
lastLateAckPushLatencyMs: number | null;
|
|
4259
|
+
lastLateAckOkAt?: number | null;
|
|
4260
|
+
adaptiveAckRecoveryOkCount?: number;
|
|
4261
|
+
nowMs?: number;
|
|
4262
|
+
}) {
|
|
4263
|
+
const lastLateAckOkAt = typeof args.lastLateAckOkAt === 'number' ? args.lastLateAckOkAt : null;
|
|
4264
|
+
const nowMs = typeof args.nowMs === 'number' ? args.nowMs : now();
|
|
4265
|
+
const lateAckExpired =
|
|
4266
|
+
typeof lastLateAckOkAt === 'number' &&
|
|
4267
|
+
lastLateAckOkAt > 0 &&
|
|
4268
|
+
nowMs - lastLateAckOkAt > ADAPTIVE_ACK_TIMEOUT_OBSERVATION_TTL_MS;
|
|
4269
|
+
const recovered =
|
|
4270
|
+
typeof args.adaptiveAckRecoveryOkCount === 'number' &&
|
|
4271
|
+
args.adaptiveAckRecoveryOkCount >= ADAPTIVE_ACK_TIMEOUT_RECOVERY_OK_THRESHOLD;
|
|
4272
|
+
if (
|
|
4273
|
+
args.lateAckOkCount <= 0 ||
|
|
4274
|
+
args.recentAckTimeoutCount <= 0 ||
|
|
4275
|
+
typeof args.lastLateAckPushLatencyMs !== 'number' ||
|
|
4276
|
+
lateAckExpired ||
|
|
4277
|
+
recovered
|
|
4278
|
+
) {
|
|
4279
|
+
return PUSH_ACK_TIMEOUT_MS;
|
|
4280
|
+
}
|
|
4281
|
+
const recommended = Math.ceil(args.lastLateAckPushLatencyMs * 1.25);
|
|
4282
|
+
return Math.min(
|
|
4283
|
+
RECOMMENDED_ACK_TIMEOUT_MAX_MS,
|
|
4284
|
+
Math.max(RECOMMENDED_ACK_TIMEOUT_MIN_MS, recommended),
|
|
4285
|
+
);
|
|
4286
|
+
}
|
|
4287
|
+
|
|
4288
|
+
private maybeLogAdaptiveAckTimeout(args: {
|
|
4289
|
+
accountId: string;
|
|
4290
|
+
timeoutMs: number;
|
|
4291
|
+
reason: string;
|
|
4292
|
+
lastLateAckPushLatencyMs: number | null;
|
|
4293
|
+
nowMs?: number;
|
|
4294
|
+
}) {
|
|
4295
|
+
if (args.timeoutMs <= PUSH_ACK_TIMEOUT_MS) return;
|
|
4296
|
+
const t = typeof args.nowMs === 'number' ? args.nowMs : now();
|
|
4297
|
+
const previous = this.adaptiveAckTimeoutLogStateByAccount.get(args.accountId);
|
|
4298
|
+
if (
|
|
4299
|
+
previous &&
|
|
4300
|
+
previous.timeoutMs === args.timeoutMs &&
|
|
4301
|
+
previous.reason === args.reason &&
|
|
4302
|
+
t - previous.at < ADAPTIVE_ACK_TIMEOUT_LOG_THROTTLE_MS
|
|
4303
|
+
) {
|
|
4304
|
+
return;
|
|
4305
|
+
}
|
|
4306
|
+
this.adaptiveAckTimeoutLogStateByAccount.set(args.accountId, {
|
|
4307
|
+
at: t,
|
|
4308
|
+
timeoutMs: args.timeoutMs,
|
|
4309
|
+
reason: args.reason,
|
|
4310
|
+
});
|
|
4311
|
+
const parts = [
|
|
4312
|
+
args.accountId,
|
|
4313
|
+
`current=${args.timeoutMs}`,
|
|
4314
|
+
`default=${PUSH_ACK_TIMEOUT_MS}`,
|
|
4315
|
+
`reason=${args.reason}`,
|
|
4316
|
+
];
|
|
4317
|
+
if (typeof args.lastLateAckPushLatencyMs === 'number') {
|
|
4318
|
+
parts.push(`latePushMs=${args.lastLateAckPushLatencyMs}`);
|
|
4319
|
+
}
|
|
4320
|
+
this.logInfo('outbox ack timeout-adaptive', parts.join('|'));
|
|
4321
|
+
}
|
|
4322
|
+
|
|
4323
|
+
private resolveMessageAckTimeoutMs(accountId?: string) {
|
|
4324
|
+
if (!ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED) return PUSH_ACK_TIMEOUT_MS;
|
|
4325
|
+
const acc = normalizeAccountId(accountId || BNCR_DEFAULT_ACCOUNT_ID);
|
|
4326
|
+
const lateAckOkCount = this.getCounter(this.lateAckOkCountByAccount, acc);
|
|
4327
|
+
const recentAckTimeoutCount = this.getCounter(this.ackTimeoutCountByAccount, acc);
|
|
4328
|
+
const lastLateAckPushLatencyMs = this.lastLateAckPushLatencyMsByAccount.get(acc) || null;
|
|
4329
|
+
const lastLateAckOkAt = this.lastLateAckOkByAccount.get(acc) || null;
|
|
4330
|
+
const adaptiveAckRecoveryOkCount = this.getCounter(this.adaptiveAckRecoveryOkCountByAccount, acc);
|
|
4331
|
+
const nowMs = now();
|
|
4332
|
+
const timeoutMs = this.computeRecommendedAckTimeoutMs({
|
|
4333
|
+
lateAckOkCount,
|
|
4334
|
+
recentAckTimeoutCount,
|
|
4335
|
+
lastLateAckPushLatencyMs,
|
|
4336
|
+
lastLateAckOkAt,
|
|
4337
|
+
adaptiveAckRecoveryOkCount,
|
|
4338
|
+
nowMs,
|
|
4339
|
+
});
|
|
4340
|
+
const reason = this.computeRecommendedAckTimeoutReason({
|
|
4341
|
+
lateAckOkCount,
|
|
4342
|
+
recentAckTimeoutCount,
|
|
4343
|
+
lastLateAckPushLatencyMs,
|
|
4344
|
+
lastLateAckOkAt,
|
|
4345
|
+
adaptiveAckRecoveryOkCount,
|
|
4346
|
+
recommendedAckTimeoutMs: timeoutMs,
|
|
4347
|
+
nowMs,
|
|
4348
|
+
});
|
|
4349
|
+
this.maybeLogAdaptiveAckTimeout({
|
|
4350
|
+
accountId: acc,
|
|
4351
|
+
timeoutMs,
|
|
4352
|
+
reason,
|
|
4353
|
+
lastLateAckPushLatencyMs,
|
|
4354
|
+
nowMs,
|
|
4355
|
+
});
|
|
4356
|
+
return timeoutMs;
|
|
4357
|
+
}
|
|
4358
|
+
|
|
4359
|
+
private buildRuntimeAckObservability(accountId: string) {
|
|
4360
|
+
const acc = normalizeAccountId(accountId);
|
|
4361
|
+
const recentAckTimeoutCount = this.getCounter(this.ackTimeoutCountByAccount, acc);
|
|
4362
|
+
const lateAckOkCount = this.getCounter(this.lateAckOkCountByAccount, acc);
|
|
4363
|
+
const lastLateAckPushLatencyMs = this.lastLateAckPushLatencyMsByAccount.get(acc) || null;
|
|
4364
|
+
const lastLateAckOkAt = this.lastLateAckOkByAccount.get(acc) || null;
|
|
4365
|
+
const nowMs = now();
|
|
4366
|
+
const lastLateAckAgeMs =
|
|
4367
|
+
typeof lastLateAckOkAt === 'number' && lastLateAckOkAt > 0 ? Math.max(0, nowMs - lastLateAckOkAt) : null;
|
|
4368
|
+
const lateAckObservationTtlMs = ADAPTIVE_ACK_TIMEOUT_OBSERVATION_TTL_MS;
|
|
4369
|
+
const lateAckObservationExpired =
|
|
4370
|
+
typeof lastLateAckAgeMs === 'number' && lastLateAckAgeMs > lateAckObservationTtlMs;
|
|
4371
|
+
const adaptiveAckRecoveryOkCount = this.getCounter(this.adaptiveAckRecoveryOkCountByAccount, acc);
|
|
4372
|
+
const adaptiveAckRecovered = adaptiveAckRecoveryOkCount >= ADAPTIVE_ACK_TIMEOUT_RECOVERY_OK_THRESHOLD;
|
|
4373
|
+
const recommendedAckTimeoutMs = this.computeRecommendedAckTimeoutMs({
|
|
4374
|
+
lateAckOkCount,
|
|
4375
|
+
recentAckTimeoutCount,
|
|
4376
|
+
lastLateAckPushLatencyMs,
|
|
4377
|
+
lastLateAckOkAt,
|
|
4378
|
+
adaptiveAckRecoveryOkCount,
|
|
4379
|
+
nowMs,
|
|
4380
|
+
});
|
|
4381
|
+
const currentAckTimeoutMs = this.resolveMessageAckTimeoutMs(acc);
|
|
4382
|
+
return {
|
|
4383
|
+
lastAckOkAt: this.lastAckOkByAccount.get(acc) || null,
|
|
4384
|
+
lastAckTimeoutAt: this.lastAckTimeoutByAccount.get(acc) || null,
|
|
4385
|
+
recentAckTimeoutCount,
|
|
4386
|
+
lateAckOkCount,
|
|
4387
|
+
lastLateAckOkAt,
|
|
4388
|
+
lastLateAckAgeMs,
|
|
4389
|
+
lateAckObservationTtlMs,
|
|
4390
|
+
lateAckObservationExpired,
|
|
4391
|
+
adaptiveAckRecoveryOkCount,
|
|
4392
|
+
adaptiveAckRecoveryOkThreshold: ADAPTIVE_ACK_TIMEOUT_RECOVERY_OK_THRESHOLD,
|
|
4393
|
+
adaptiveAckRecovered,
|
|
4394
|
+
lastAckQueueLatencyMs: this.lastAckQueueLatencyMsByAccount.get(acc) || null,
|
|
4395
|
+
lastAckPushLatencyMs: this.lastAckPushLatencyMsByAccount.get(acc) || null,
|
|
4396
|
+
lastLateAckQueueLatencyMs: this.lastLateAckQueueLatencyMsByAccount.get(acc) || null,
|
|
4397
|
+
lastLateAckPushLatencyMs,
|
|
4398
|
+
adaptiveAckTimeoutEnabled: ADAPTIVE_ACK_TIMEOUT_DEFAULT_ENABLED,
|
|
4399
|
+
defaultAckTimeoutMs: PUSH_ACK_TIMEOUT_MS,
|
|
4400
|
+
currentAckTimeoutMs,
|
|
4401
|
+
recommendedAckTimeoutMs,
|
|
4402
|
+
recommendedAckTimeoutReason: this.computeRecommendedAckTimeoutReason({
|
|
4403
|
+
lateAckOkCount,
|
|
4404
|
+
recentAckTimeoutCount,
|
|
4405
|
+
lastLateAckPushLatencyMs,
|
|
4406
|
+
lastLateAckOkAt,
|
|
4407
|
+
adaptiveAckRecoveryOkCount,
|
|
4408
|
+
recommendedAckTimeoutMs,
|
|
4409
|
+
nowMs,
|
|
4410
|
+
}),
|
|
4411
|
+
};
|
|
4412
|
+
}
|
|
4413
|
+
|
|
4414
|
+
private buildRuntimeAckStrategy(ackObservability: Record<string, any>) {
|
|
4415
|
+
const currentMs = finiteNumberOr(ackObservability.currentAckTimeoutMs, PUSH_ACK_TIMEOUT_MS);
|
|
4416
|
+
const defaultMs = finiteNumberOr(ackObservability.defaultAckTimeoutMs, PUSH_ACK_TIMEOUT_MS);
|
|
4417
|
+
const reason = asString(ackObservability.recommendedAckTimeoutReason || 'unknown') || 'unknown';
|
|
4418
|
+
return {
|
|
4419
|
+
mode: ackObservability.adaptiveAckTimeoutEnabled === true ? 'adaptive' : 'fixed',
|
|
4420
|
+
currentMs,
|
|
4421
|
+
defaultMs,
|
|
4422
|
+
maxMs: RECOMMENDED_ACK_TIMEOUT_MAX_MS,
|
|
4423
|
+
reason,
|
|
4424
|
+
active: currentMs > defaultMs,
|
|
4425
|
+
lastLateAckAgeMs: ackObservability.lastLateAckAgeMs ?? null,
|
|
4426
|
+
lateAckObservationTtlMs: ackObservability.lateAckObservationTtlMs ?? null,
|
|
4427
|
+
recovered: ackObservability.adaptiveAckRecovered === true,
|
|
4428
|
+
};
|
|
4429
|
+
}
|
|
4430
|
+
|
|
3901
4431
|
private buildRuntimeActivitySnapshot(accountId: string) {
|
|
3902
4432
|
return {
|
|
3903
4433
|
activeConnections: this.activeConnectionCount(accountId),
|
|
@@ -3927,7 +4457,29 @@ class BncrBridgeRuntime {
|
|
|
3927
4457
|
}
|
|
3928
4458
|
|
|
3929
4459
|
getAccountRuntimeSnapshot(accountId: string) {
|
|
3930
|
-
|
|
4460
|
+
const snapshot = buildAccountRuntimeSnapshot(this.buildRuntimeStatusInput(accountId, { running: true }));
|
|
4461
|
+
const ackObservability = this.buildRuntimeAckObservability(accountId);
|
|
4462
|
+
const ackStrategy = this.buildRuntimeAckStrategy(ackObservability);
|
|
4463
|
+
return {
|
|
4464
|
+
...snapshot,
|
|
4465
|
+
ackObservability,
|
|
4466
|
+
ackStrategy,
|
|
4467
|
+
diagnostics: {
|
|
4468
|
+
...(snapshot.diagnostics || {}),
|
|
4469
|
+
ackObservability,
|
|
4470
|
+
ackStrategy,
|
|
4471
|
+
},
|
|
4472
|
+
meta: {
|
|
4473
|
+
...(snapshot.meta || {}),
|
|
4474
|
+
ackObservability,
|
|
4475
|
+
ackStrategy,
|
|
4476
|
+
diagnostics: {
|
|
4477
|
+
...(snapshot.meta?.diagnostics || {}),
|
|
4478
|
+
ackObservability,
|
|
4479
|
+
ackStrategy,
|
|
4480
|
+
},
|
|
4481
|
+
},
|
|
4482
|
+
};
|
|
3931
4483
|
}
|
|
3932
4484
|
|
|
3933
4485
|
private buildStatusHeadline(accountId: string): string {
|
|
@@ -3996,7 +4548,7 @@ class BncrBridgeRuntime {
|
|
|
3996
4548
|
this.deadLetter = appendDeadLetter({
|
|
3997
4549
|
deadLetter: this.deadLetter,
|
|
3998
4550
|
entry: dead,
|
|
3999
|
-
maxEntries:
|
|
4551
|
+
maxEntries: MAX_DEAD_LETTER_ENTRIES,
|
|
4000
4552
|
});
|
|
4001
4553
|
this.outbox.delete(entry.messageId);
|
|
4002
4554
|
this.resolveMessageAck(entry.messageId, 'timeout');
|
|
@@ -4029,7 +4581,7 @@ class BncrBridgeRuntime {
|
|
|
4029
4581
|
mediaUrl: string,
|
|
4030
4582
|
mediaLocalRoots?: readonly string[],
|
|
4031
4583
|
): Promise<{ mediaBase64: string; mimeType?: string; fileName?: string }> {
|
|
4032
|
-
const loaded = await this.api
|
|
4584
|
+
const loaded = await loadOpenClawWebMedia(this.api, mediaUrl, {
|
|
4033
4585
|
localRoots: mediaLocalRoots,
|
|
4034
4586
|
maxBytes: 20 * 1024 * 1024,
|
|
4035
4587
|
});
|
|
@@ -4044,12 +4596,12 @@ class BncrBridgeRuntime {
|
|
|
4044
4596
|
mediaUrl: string;
|
|
4045
4597
|
mediaLocalRoots?: readonly string[];
|
|
4046
4598
|
}): Promise<{
|
|
4047
|
-
loaded:
|
|
4599
|
+
loaded: OpenClawLoadedMedia;
|
|
4048
4600
|
size: number;
|
|
4049
4601
|
mimeType?: string;
|
|
4050
4602
|
fileName: string;
|
|
4051
4603
|
}> {
|
|
4052
|
-
const loaded = await this.api
|
|
4604
|
+
const loaded = await loadOpenClawWebMedia(this.api, params.mediaUrl, {
|
|
4053
4605
|
localRoots: params.mediaLocalRoots,
|
|
4054
4606
|
maxBytes: 50 * 1024 * 1024,
|
|
4055
4607
|
});
|
|
@@ -4336,52 +4888,37 @@ class BncrBridgeRuntime {
|
|
|
4336
4888
|
}
|
|
4337
4889
|
|
|
4338
4890
|
private async sleepMs(ms: number): Promise<void> {
|
|
4339
|
-
await new Promise<void>((resolve) =>
|
|
4891
|
+
await new Promise<void>((resolve) =>
|
|
4892
|
+
setTimeout(resolve, clampFiniteNumber(ms, 0, 0, INTERNAL_SLEEP_MAX_MS)),
|
|
4893
|
+
);
|
|
4340
4894
|
}
|
|
4341
4895
|
|
|
4342
|
-
private waitChunkAck(params: {
|
|
4896
|
+
private async waitChunkAck(params: {
|
|
4343
4897
|
transferId: string;
|
|
4344
4898
|
chunkIndex: number;
|
|
4345
4899
|
timeoutMs?: number;
|
|
4346
4900
|
}): Promise<void> {
|
|
4347
4901
|
// Refactor boundary note (file-transfer / ACK coupling):
|
|
4348
4902
|
// Chunk-level ACK waiting is part of the file-transfer sub-protocol, but it depends directly on
|
|
4349
|
-
// mutable transfer runtime state in fileSendTransfers.
|
|
4350
|
-
//
|
|
4903
|
+
// mutable transfer runtime state in fileSendTransfers. Keep state prechecks here, while ACK wakeup
|
|
4904
|
+
// uses the shared event-style fileAckWaiters path instead of polling transfer state.
|
|
4351
4905
|
const { transferId, chunkIndex } = params;
|
|
4352
|
-
const
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
}
|
|
4365
|
-
if (st.failedChunks.has(chunkIndex)) {
|
|
4366
|
-
reject(new Error(st.failedChunks.get(chunkIndex) || `chunk ${chunkIndex} failed`));
|
|
4367
|
-
return;
|
|
4368
|
-
}
|
|
4369
|
-
if (st.ackedChunks.has(chunkIndex)) {
|
|
4370
|
-
resolve();
|
|
4371
|
-
return;
|
|
4372
|
-
}
|
|
4373
|
-
if (now() - started >= timeoutMs) {
|
|
4374
|
-
reject(new Error(`chunk ack timeout index=${chunkIndex}`));
|
|
4375
|
-
return;
|
|
4376
|
-
}
|
|
4377
|
-
await this.sleepMs(120);
|
|
4378
|
-
void tick();
|
|
4379
|
-
};
|
|
4380
|
-
void tick();
|
|
4906
|
+
const st = this.fileSendTransfers.get(transferId);
|
|
4907
|
+
if (!st) throw new Error('transfer state missing');
|
|
4908
|
+
if (st.failedChunks.has(chunkIndex)) {
|
|
4909
|
+
throw new Error(st.failedChunks.get(chunkIndex) || `chunk ${chunkIndex} failed`);
|
|
4910
|
+
}
|
|
4911
|
+
if (st.ackedChunks.has(chunkIndex)) return;
|
|
4912
|
+
|
|
4913
|
+
await this.waitForFileAck({
|
|
4914
|
+
transferId,
|
|
4915
|
+
stage: 'chunk',
|
|
4916
|
+
chunkIndex,
|
|
4917
|
+
timeoutMs: clampFiniteNumber(params.timeoutMs, FILE_TRANSFER_ACK_TTL_MS, 1_000, 60_000),
|
|
4381
4918
|
});
|
|
4382
4919
|
}
|
|
4383
4920
|
|
|
4384
|
-
private waitCompleteAck(params: {
|
|
4921
|
+
private async waitCompleteAck(params: {
|
|
4385
4922
|
transferId: string;
|
|
4386
4923
|
timeoutMs?: number;
|
|
4387
4924
|
}): Promise<{ path: string }> {
|
|
@@ -4390,33 +4927,20 @@ class BncrBridgeRuntime {
|
|
|
4390
4927
|
// transfer status transitions performed elsewhere in channel.ts. Keep completion wait behavior and
|
|
4391
4928
|
// transfer-state mutation boundaries aligned if/when file-transfer pieces are moved out.
|
|
4392
4929
|
const { transferId } = params;
|
|
4393
|
-
const
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
return
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
}
|
|
4403
|
-
if (st.status === 'aborted') {
|
|
4404
|
-
reject(new Error(st.error || 'transfer aborted'));
|
|
4405
|
-
return;
|
|
4406
|
-
}
|
|
4407
|
-
if (st.status === 'completed' && st.completedPath) {
|
|
4408
|
-
resolve({ path: st.completedPath });
|
|
4409
|
-
return;
|
|
4410
|
-
}
|
|
4411
|
-
if (now() - started >= timeoutMs) {
|
|
4412
|
-
reject(new Error('complete ack timeout'));
|
|
4413
|
-
return;
|
|
4414
|
-
}
|
|
4415
|
-
await this.sleepMs(150);
|
|
4416
|
-
void tick();
|
|
4417
|
-
};
|
|
4418
|
-
void tick();
|
|
4930
|
+
const st = this.fileSendTransfers.get(transferId);
|
|
4931
|
+
if (!st) throw new Error('transfer state missing');
|
|
4932
|
+
if (st.status === 'aborted') throw new Error(st.error || 'transfer aborted');
|
|
4933
|
+
if (st.status === 'completed' && st.completedPath) return { path: st.completedPath };
|
|
4934
|
+
|
|
4935
|
+
const payload = await this.waitForFileAck({
|
|
4936
|
+
transferId,
|
|
4937
|
+
stage: 'complete',
|
|
4938
|
+
timeoutMs: clampFiniteNumber(params.timeoutMs, 60_000, 2_000, 120_000),
|
|
4419
4939
|
});
|
|
4940
|
+
const updated = this.fileSendTransfers.get(transferId);
|
|
4941
|
+
const path = asString(payload?.path || updated?.completedPath || '').trim();
|
|
4942
|
+
if (!path) throw new Error('complete ack missing path');
|
|
4943
|
+
return { path };
|
|
4420
4944
|
}
|
|
4421
4945
|
|
|
4422
4946
|
private async transferMediaToBncrClient(params: {
|
|
@@ -4596,6 +5120,7 @@ class BncrBridgeRuntime {
|
|
|
4596
5120
|
|
|
4597
5121
|
if (!ok) {
|
|
4598
5122
|
st.status = 'aborted';
|
|
5123
|
+
st.terminalAt = now();
|
|
4599
5124
|
st.error = String((lastErr as any)?.message || lastErr || `chunk-${idx}-failed`);
|
|
4600
5125
|
this.fileSendTransfers.set(transferId, st);
|
|
4601
5126
|
ctx.broadcastToConnIds(
|
|
@@ -4896,7 +5421,7 @@ class BncrBridgeRuntime {
|
|
|
4896
5421
|
|
|
4897
5422
|
handleDiagnostics = async ({ params, respond }: GatewayRequestHandlerOptions) => {
|
|
4898
5423
|
const accountId = normalizeAccountId(asString(params?.accountId || ''));
|
|
4899
|
-
const cfg = this.api
|
|
5424
|
+
const cfg = getOpenClawRuntimeConfig(this.api);
|
|
4900
5425
|
const runtime = this.getAccountRuntimeSnapshot(accountId);
|
|
4901
5426
|
const diagnostics = this.buildExtendedDiagnostics(accountId);
|
|
4902
5427
|
|
|
@@ -4949,15 +5474,34 @@ class BncrBridgeRuntime {
|
|
|
4949
5474
|
const sessionKey = asString(params?.sessionKey || '').trim();
|
|
4950
5475
|
const fileName = asString(params?.fileName || '').trim() || 'file.bin';
|
|
4951
5476
|
const mimeType = asString(params?.mimeType || '').trim() || 'application/octet-stream';
|
|
4952
|
-
const fileSize =
|
|
4953
|
-
const chunkSize =
|
|
4954
|
-
const totalChunks =
|
|
5477
|
+
const fileSize = finiteNonNegativeNumberOrNull(params?.fileSize);
|
|
5478
|
+
const chunkSize = finiteNonNegativeNumberOrNull(params?.chunkSize ?? 256 * 1024);
|
|
5479
|
+
const totalChunks = finiteNonNegativeNumberOrNull(params?.totalChunks);
|
|
4955
5480
|
const fileSha256 = asString(params?.fileSha256 || '').trim();
|
|
4956
5481
|
|
|
4957
5482
|
if (!transferId || !sessionKey || !fileSize || !chunkSize || !totalChunks) {
|
|
4958
5483
|
respond(false, { error: 'transferId/sessionKey/fileSize/chunkSize/totalChunks required' });
|
|
4959
5484
|
return;
|
|
4960
5485
|
}
|
|
5486
|
+
if (fileSize > INBOUND_FILE_TRANSFER_MAX_BYTES) {
|
|
5487
|
+
respond(false, {
|
|
5488
|
+
error: `fileSize too large size=${fileSize} max=${INBOUND_FILE_TRANSFER_MAX_BYTES}`,
|
|
5489
|
+
});
|
|
5490
|
+
return;
|
|
5491
|
+
}
|
|
5492
|
+
if (totalChunks > INBOUND_FILE_TRANSFER_MAX_CHUNKS) {
|
|
5493
|
+
respond(false, {
|
|
5494
|
+
error: `totalChunks too large total=${totalChunks} max=${INBOUND_FILE_TRANSFER_MAX_CHUNKS}`,
|
|
5495
|
+
});
|
|
5496
|
+
return;
|
|
5497
|
+
}
|
|
5498
|
+
const expectedTotalChunks = Math.ceil(fileSize / chunkSize);
|
|
5499
|
+
if (totalChunks !== expectedTotalChunks) {
|
|
5500
|
+
respond(false, {
|
|
5501
|
+
error: `totalChunks mismatch total=${totalChunks} expected=${expectedTotalChunks}`,
|
|
5502
|
+
});
|
|
5503
|
+
return;
|
|
5504
|
+
}
|
|
4961
5505
|
|
|
4962
5506
|
const normalized = normalizeStoredSessionKey(sessionKey);
|
|
4963
5507
|
if (!normalized) {
|
|
@@ -5015,13 +5559,13 @@ class BncrBridgeRuntime {
|
|
|
5015
5559
|
const clientId = asString((params as any)?.clientId || '').trim() || undefined;
|
|
5016
5560
|
|
|
5017
5561
|
const transferId = asString(params?.transferId || '').trim();
|
|
5018
|
-
const chunkIndex =
|
|
5019
|
-
const offset =
|
|
5020
|
-
const size =
|
|
5562
|
+
const chunkIndex = finiteNonNegativeNumberOrNull(params?.chunkIndex);
|
|
5563
|
+
const offset = finiteNonNegativeNumberOrNull(params?.offset ?? 0);
|
|
5564
|
+
const size = finiteNonNegativeNumberOrNull(params?.size ?? 0);
|
|
5021
5565
|
const chunkSha256 = asString(params?.chunkSha256 || '').trim();
|
|
5022
5566
|
const base64 = asString(params?.base64 || '');
|
|
5023
5567
|
|
|
5024
|
-
if (!transferId || chunkIndex
|
|
5568
|
+
if (!transferId || chunkIndex == null || !base64) {
|
|
5025
5569
|
respond(false, { error: 'transferId/chunkIndex/base64 required' });
|
|
5026
5570
|
return;
|
|
5027
5571
|
}
|
|
@@ -5031,6 +5575,10 @@ class BncrBridgeRuntime {
|
|
|
5031
5575
|
respond(false, { error: 'transfer not found' });
|
|
5032
5576
|
return;
|
|
5033
5577
|
}
|
|
5578
|
+
if (chunkIndex >= st.totalChunks) {
|
|
5579
|
+
respond(false, { error: `chunkIndex out of range index=${chunkIndex} total=${st.totalChunks}` });
|
|
5580
|
+
return;
|
|
5581
|
+
}
|
|
5034
5582
|
|
|
5035
5583
|
const staleObserved = this.observeLease('file.chunk', params ?? {});
|
|
5036
5584
|
if (staleObserved.stale) {
|
|
@@ -5061,7 +5609,7 @@ class BncrBridgeRuntime {
|
|
|
5061
5609
|
|
|
5062
5610
|
try {
|
|
5063
5611
|
const buf = Buffer.from(base64, 'base64');
|
|
5064
|
-
if (size > 0 && buf.length !== size) {
|
|
5612
|
+
if (size != null && size > 0 && buf.length !== size) {
|
|
5065
5613
|
throw new Error(`chunk size mismatch expected=${size} got=${buf.length}`);
|
|
5066
5614
|
}
|
|
5067
5615
|
if (chunkSha256) {
|
|
@@ -5168,7 +5716,8 @@ class BncrBridgeRuntime {
|
|
|
5168
5716
|
throw new Error('file sha256 mismatch');
|
|
5169
5717
|
}
|
|
5170
5718
|
|
|
5171
|
-
const saved = await
|
|
5719
|
+
const saved = await saveOpenClawChannelMediaBuffer(
|
|
5720
|
+
this.api,
|
|
5172
5721
|
merged,
|
|
5173
5722
|
st.mimeType,
|
|
5174
5723
|
'inbound',
|
|
@@ -5177,6 +5726,7 @@ class BncrBridgeRuntime {
|
|
|
5177
5726
|
);
|
|
5178
5727
|
st.completedPath = saved.path;
|
|
5179
5728
|
st.status = 'completed';
|
|
5729
|
+
st.terminalAt = now();
|
|
5180
5730
|
this.fileRecvTransfers.set(transferId, st);
|
|
5181
5731
|
|
|
5182
5732
|
respond(
|
|
@@ -5205,6 +5755,7 @@ class BncrBridgeRuntime {
|
|
|
5205
5755
|
);
|
|
5206
5756
|
} catch (error) {
|
|
5207
5757
|
st.status = 'aborted';
|
|
5758
|
+
st.terminalAt = now();
|
|
5208
5759
|
st.error = String((error as any)?.message || error || 'complete failed');
|
|
5209
5760
|
this.fileRecvTransfers.set(transferId, st);
|
|
5210
5761
|
respond(false, { error: st.error });
|
|
@@ -5256,6 +5807,7 @@ class BncrBridgeRuntime {
|
|
|
5256
5807
|
}
|
|
5257
5808
|
|
|
5258
5809
|
st.status = 'aborted';
|
|
5810
|
+
st.terminalAt = now();
|
|
5259
5811
|
st.error = asString(params?.reason || 'aborted');
|
|
5260
5812
|
this.fileRecvTransfers.set(transferId, st);
|
|
5261
5813
|
|
|
@@ -5285,7 +5837,7 @@ class BncrBridgeRuntime {
|
|
|
5285
5837
|
const transferId = asString(params?.transferId || '').trim();
|
|
5286
5838
|
const stage = asString(params?.stage || '').trim();
|
|
5287
5839
|
const ok = params?.ok !== false;
|
|
5288
|
-
const chunkIndex =
|
|
5840
|
+
const chunkIndex = finiteNonNegativeNumberOrNull(params?.chunkIndex);
|
|
5289
5841
|
|
|
5290
5842
|
this.logInfo(
|
|
5291
5843
|
'file-ack-inbound',
|
|
@@ -5296,8 +5848,10 @@ class BncrBridgeRuntime {
|
|
|
5296
5848
|
clientId: clientId || null,
|
|
5297
5849
|
transferId,
|
|
5298
5850
|
stage,
|
|
5851
|
+
ackStage: stage,
|
|
5852
|
+
ackOutcome: ok ? 'acked' : 'failed',
|
|
5299
5853
|
ok,
|
|
5300
|
-
chunkIndex: chunkIndex
|
|
5854
|
+
chunkIndex: chunkIndex != null ? chunkIndex : undefined,
|
|
5301
5855
|
errorCode: asString(params?.errorCode || ''),
|
|
5302
5856
|
errorMessage: asString(params?.errorMessage || ''),
|
|
5303
5857
|
path: asString(params?.path || '').trim(),
|
|
@@ -5355,15 +5909,19 @@ class BncrBridgeRuntime {
|
|
|
5355
5909
|
const code = asString(params?.errorCode || 'ACK_FAILED');
|
|
5356
5910
|
const msg = asString(params?.errorMessage || 'ack failed');
|
|
5357
5911
|
st.error = `${code}:${msg}`;
|
|
5358
|
-
if (stage === 'chunk' && chunkIndex
|
|
5359
|
-
if (stage === 'complete')
|
|
5912
|
+
if (stage === 'chunk' && chunkIndex != null) st.failedChunks.set(chunkIndex, st.error);
|
|
5913
|
+
if (stage === 'complete') {
|
|
5914
|
+
st.status = 'aborted';
|
|
5915
|
+
st.terminalAt = now();
|
|
5916
|
+
}
|
|
5360
5917
|
} else {
|
|
5361
|
-
if (stage === 'chunk' && chunkIndex
|
|
5918
|
+
if (stage === 'chunk' && chunkIndex != null) {
|
|
5362
5919
|
st.ackedChunks.add(chunkIndex);
|
|
5363
5920
|
st.status = 'transferring';
|
|
5364
5921
|
}
|
|
5365
5922
|
if (stage === 'complete') {
|
|
5366
5923
|
st.status = 'completed';
|
|
5924
|
+
st.terminalAt = now();
|
|
5367
5925
|
st.completedPath = asString(params?.path || '').trim() || st.completedPath;
|
|
5368
5926
|
}
|
|
5369
5927
|
}
|
|
@@ -5374,7 +5932,7 @@ class BncrBridgeRuntime {
|
|
|
5374
5932
|
this.resolveFileAck({
|
|
5375
5933
|
transferId,
|
|
5376
5934
|
stage,
|
|
5377
|
-
chunkIndex: chunkIndex
|
|
5935
|
+
chunkIndex: chunkIndex != null ? chunkIndex : undefined,
|
|
5378
5936
|
payload: {
|
|
5379
5937
|
ok,
|
|
5380
5938
|
transferId,
|
|
@@ -5494,14 +6052,14 @@ class BncrBridgeRuntime {
|
|
|
5494
6052
|
this.lastInboundAtGlobal = now();
|
|
5495
6053
|
this.incrementCounter(this.inboundEventsByAccount, accountId);
|
|
5496
6054
|
|
|
5497
|
-
const cfg = this.api
|
|
6055
|
+
const cfg = getOpenClawRuntimeConfig(this.api);
|
|
5498
6056
|
const canonicalAgentId = this.ensureCanonicalAgentId({
|
|
5499
6057
|
cfg,
|
|
5500
6058
|
accountId,
|
|
5501
6059
|
peer,
|
|
5502
6060
|
channelId: CHANNEL_ID,
|
|
5503
6061
|
});
|
|
5504
|
-
const acceptance = this.prepareInboundAcceptance({ parsed, canonicalAgentId });
|
|
6062
|
+
const acceptance = await this.prepareInboundAcceptance({ parsed, canonicalAgentId });
|
|
5505
6063
|
if (!acceptance.ok) {
|
|
5506
6064
|
respond(acceptance.status, acceptance.payload);
|
|
5507
6065
|
return;
|
|
@@ -5633,20 +6191,19 @@ class BncrBridgeRuntime {
|
|
|
5633
6191
|
|
|
5634
6192
|
tick();
|
|
5635
6193
|
const timer = setInterval(tick, 5_000);
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
await new Promise<void>((resolve) => {
|
|
6194
|
+
let worker!: ChannelAccountWorkerHandle;
|
|
6195
|
+
const done = new Promise<void>((resolve) => {
|
|
5639
6196
|
let settled = false;
|
|
5640
6197
|
const finish = (reason: string) => {
|
|
5641
6198
|
if (settled) return;
|
|
5642
6199
|
settled = true;
|
|
5643
|
-
const
|
|
5644
|
-
if (
|
|
5645
|
-
|
|
5646
|
-
this.channelAccountTimers.delete(accountId);
|
|
5647
|
-
} else {
|
|
5648
|
-
clearInterval(timer);
|
|
6200
|
+
const activeWorker = this.channelAccountWorkers.get(accountId);
|
|
6201
|
+
if (activeWorker === worker) {
|
|
6202
|
+
this.channelAccountWorkers.delete(accountId);
|
|
5649
6203
|
}
|
|
6204
|
+
clearInterval(timer);
|
|
6205
|
+
worker.cleanupAbortListener?.();
|
|
6206
|
+
worker.cleanupAbortListener = undefined;
|
|
5650
6207
|
this.logInfo(
|
|
5651
6208
|
'health',
|
|
5652
6209
|
`status-worker finished ${JSON.stringify({ bridge: this.bridgeId, accountId, reason })}`,
|
|
@@ -5656,15 +6213,23 @@ class BncrBridgeRuntime {
|
|
|
5656
6213
|
resolve();
|
|
5657
6214
|
};
|
|
5658
6215
|
|
|
6216
|
+
worker = { timer, finish };
|
|
6217
|
+
this.channelAccountWorkers.set(accountId, worker);
|
|
6218
|
+
|
|
5659
6219
|
const onAbort = () => finish('abort');
|
|
6220
|
+
const abortSignal = ctx.abortSignal;
|
|
5660
6221
|
|
|
5661
|
-
if (
|
|
6222
|
+
if (abortSignal?.aborted) {
|
|
5662
6223
|
onAbort();
|
|
5663
6224
|
return;
|
|
5664
6225
|
}
|
|
5665
6226
|
|
|
5666
|
-
|
|
6227
|
+
abortSignal?.addEventListener?.('abort', onAbort, { once: true });
|
|
6228
|
+
if (abortSignal?.removeEventListener) {
|
|
6229
|
+
worker.cleanupAbortListener = () => abortSignal.removeEventListener('abort', onAbort);
|
|
6230
|
+
}
|
|
5667
6231
|
});
|
|
6232
|
+
await done;
|
|
5668
6233
|
};
|
|
5669
6234
|
|
|
5670
6235
|
channelStopAccount = async (ctx: any) => {
|
|
@@ -5749,6 +6314,7 @@ class BncrBridgeRuntime {
|
|
|
5749
6314
|
accountId,
|
|
5750
6315
|
to,
|
|
5751
6316
|
text: asString(ctx.text || ''),
|
|
6317
|
+
kind: ctx?.kind,
|
|
5752
6318
|
replyToId,
|
|
5753
6319
|
mediaLocalRoots: ctx.mediaLocalRoots,
|
|
5754
6320
|
resolveVerifiedTarget: (to, accountId) => this.resolveVerifiedTarget(to, accountId),
|
|
@@ -5790,6 +6356,7 @@ class BncrBridgeRuntime {
|
|
|
5790
6356
|
mediaUrls: Array.isArray(ctx?.mediaUrls) ? ctx.mediaUrls : undefined,
|
|
5791
6357
|
asVoice,
|
|
5792
6358
|
audioAsVoice,
|
|
6359
|
+
kind: ctx?.kind,
|
|
5793
6360
|
replyToId,
|
|
5794
6361
|
mediaLocalRoots: ctx.mediaLocalRoots,
|
|
5795
6362
|
resolveVerifiedTarget: (to, accountId) => this.resolveVerifiedTarget(to, accountId),
|
|
@@ -5799,6 +6366,65 @@ class BncrBridgeRuntime {
|
|
|
5799
6366
|
createMessageId: () => randomUUID(),
|
|
5800
6367
|
});
|
|
5801
6368
|
};
|
|
6369
|
+
|
|
6370
|
+
private async enqueueChannelMessageHandoff(ctx: any, payload: ReplyPayloadInput) {
|
|
6371
|
+
const accountId = normalizeAccountId(ctx.accountId);
|
|
6372
|
+
const to = asString(ctx.to || '').trim();
|
|
6373
|
+
const verified = this.resolveVerifiedTarget(to, accountId);
|
|
6374
|
+
this.rememberSessionRoute(verified.sessionKey, accountId, verified.route);
|
|
6375
|
+
const before = new Set(this.outbox.keys());
|
|
6376
|
+
await this.enqueueFromReply({
|
|
6377
|
+
accountId,
|
|
6378
|
+
sessionKey: verified.sessionKey,
|
|
6379
|
+
route: verified.route,
|
|
6380
|
+
payload,
|
|
6381
|
+
mediaLocalRoots: ctx.mediaLocalRoots,
|
|
6382
|
+
});
|
|
6383
|
+
const entries = Array.from(this.outbox.values()).filter((entry) => !before.has(entry.messageId));
|
|
6384
|
+
if (!entries.length) {
|
|
6385
|
+
throw new Error('bncr channel.message handoff did not enqueue an outbox entry');
|
|
6386
|
+
}
|
|
6387
|
+
return entries[entries.length - 1];
|
|
6388
|
+
}
|
|
6389
|
+
|
|
6390
|
+
channelMessageSendText = async (ctx: any) => {
|
|
6391
|
+
const entry = await this.enqueueChannelMessageHandoff(ctx, {
|
|
6392
|
+
text: asString(ctx.text || ''),
|
|
6393
|
+
kind: ctx?.kind,
|
|
6394
|
+
replyToId: this.resolveChannelSendReplyToId(ctx),
|
|
6395
|
+
});
|
|
6396
|
+
return buildBncrDurableQueuedResult({ entry });
|
|
6397
|
+
};
|
|
6398
|
+
|
|
6399
|
+
channelMessageSendMedia = async (ctx: any) => {
|
|
6400
|
+
const entry = await this.enqueueChannelMessageHandoff(ctx, {
|
|
6401
|
+
text: asString(ctx.text || ''),
|
|
6402
|
+
mediaUrl: asString(ctx.mediaUrl || ''),
|
|
6403
|
+
mediaUrls: Array.isArray(ctx?.mediaUrls) ? ctx.mediaUrls : undefined,
|
|
6404
|
+
asVoice: ctx?.asVoice === true,
|
|
6405
|
+
audioAsVoice: ctx?.audioAsVoice === true,
|
|
6406
|
+
kind: ctx?.kind,
|
|
6407
|
+
replyToId: this.resolveChannelSendReplyToId(ctx),
|
|
6408
|
+
});
|
|
6409
|
+
return buildBncrDurableQueuedResult({ entry });
|
|
6410
|
+
};
|
|
6411
|
+
|
|
6412
|
+
channelMessageSendPayload = async (ctx: any) => {
|
|
6413
|
+
const payload = ctx?.payload || {};
|
|
6414
|
+
if (!payload || typeof payload !== 'object') {
|
|
6415
|
+
throw new Error('bncr channel.message payload must be an object');
|
|
6416
|
+
}
|
|
6417
|
+
const entry = await this.enqueueChannelMessageHandoff(ctx, {
|
|
6418
|
+
text: asString(payload.text || payload.message || payload.caption || ''),
|
|
6419
|
+
mediaUrl: asString(payload.mediaUrl || ''),
|
|
6420
|
+
mediaUrls: Array.isArray(payload.mediaUrls) ? payload.mediaUrls : undefined,
|
|
6421
|
+
asVoice: payload.asVoice === true,
|
|
6422
|
+
audioAsVoice: payload.audioAsVoice === true,
|
|
6423
|
+
kind: payload.kind,
|
|
6424
|
+
replyToId: asString(payload.replyToId || ctx?.replyToId || ctx?.replyToMessageId || '').trim() || undefined,
|
|
6425
|
+
});
|
|
6426
|
+
return buildBncrDurableQueuedResult({ entry });
|
|
6427
|
+
};
|
|
5802
6428
|
}
|
|
5803
6429
|
|
|
5804
6430
|
export function createBncrBridge(api: OpenClawPluginApi) {
|
|
@@ -5834,7 +6460,7 @@ export function createBncrChannelPlugin(getBridge: () => BncrBridgeRuntime) {
|
|
|
5834
6460
|
};
|
|
5835
6461
|
},
|
|
5836
6462
|
supportsAction: ({ action }) => action === 'send',
|
|
5837
|
-
extractToolSend: ({ args })
|
|
6463
|
+
extractToolSend: ({ args }) => extractOpenClawToolSend(args, 'sendMessage'),
|
|
5838
6464
|
handleAction: async ({ action, params, accountId, mediaLocalRoots }) => {
|
|
5839
6465
|
if (action !== 'send')
|
|
5840
6466
|
throw new Error(`Action ${action} is not supported for provider ${CHANNEL_ID}.`);
|
|
@@ -5872,7 +6498,7 @@ export function createBncrChannelPlugin(getBridge: () => BncrBridgeRuntime) {
|
|
|
5872
6498
|
createMessageId: () => randomUUID(),
|
|
5873
6499
|
});
|
|
5874
6500
|
|
|
5875
|
-
return
|
|
6501
|
+
return openClawJsonResult({ ok: true, ...result });
|
|
5876
6502
|
},
|
|
5877
6503
|
};
|
|
5878
6504
|
|
|
@@ -5887,6 +6513,17 @@ export function createBncrChannelPlugin(getBridge: () => BncrBridgeRuntime) {
|
|
|
5887
6513
|
aliases: ['bncr'],
|
|
5888
6514
|
},
|
|
5889
6515
|
actions: messageActions,
|
|
6516
|
+
message: {
|
|
6517
|
+
receive: {
|
|
6518
|
+
defaultAckPolicy: 'manual' as const,
|
|
6519
|
+
supportedAckPolicies: ['manual'] as const,
|
|
6520
|
+
},
|
|
6521
|
+
send: {
|
|
6522
|
+
text: async (ctx: any) => getBridge().channelMessageSendText(ctx),
|
|
6523
|
+
media: async (ctx: any) => getBridge().channelMessageSendMedia(ctx),
|
|
6524
|
+
payload: async (ctx: any) => getBridge().channelMessageSendPayload(ctx),
|
|
6525
|
+
},
|
|
6526
|
+
},
|
|
5890
6527
|
capabilities: {
|
|
5891
6528
|
chatTypes: ['direct'] as ChatType[],
|
|
5892
6529
|
media: true,
|
|
@@ -5975,7 +6612,7 @@ export function createBncrChannelPlugin(getBridge: () => BncrBridgeRuntime) {
|
|
|
5975
6612
|
listAccountIds,
|
|
5976
6613
|
resolveAccount,
|
|
5977
6614
|
setAccountEnabled: ({ cfg, accountId, enabled }: any) =>
|
|
5978
|
-
|
|
6615
|
+
setOpenClawAccountEnabledInConfigSection({
|
|
5979
6616
|
cfg,
|
|
5980
6617
|
sectionKey: CHANNEL_ID,
|
|
5981
6618
|
accountId,
|
|
@@ -5999,7 +6636,7 @@ export function createBncrChannelPlugin(getBridge: () => BncrBridgeRuntime) {
|
|
|
5999
6636
|
},
|
|
6000
6637
|
setup: {
|
|
6001
6638
|
applyAccountName: ({ cfg, accountId, name }: any) =>
|
|
6002
|
-
|
|
6639
|
+
applyOpenClawAccountNameToChannelSection({
|
|
6003
6640
|
cfg,
|
|
6004
6641
|
channelKey: CHANNEL_ID,
|
|
6005
6642
|
accountId,
|
|
@@ -6051,7 +6688,7 @@ export function createBncrChannelPlugin(getBridge: () => BncrBridgeRuntime) {
|
|
|
6051
6688
|
}),
|
|
6052
6689
|
},
|
|
6053
6690
|
status: {
|
|
6054
|
-
defaultRuntime:
|
|
6691
|
+
defaultRuntime: createOpenClawDefaultChannelRuntimeState(BNCR_DEFAULT_ACCOUNT_ID, {
|
|
6055
6692
|
mode: 'ws-offline',
|
|
6056
6693
|
}),
|
|
6057
6694
|
buildChannelSummary: async ({ defaultAccountId }: any) => {
|