@xmoxmo/bncr 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +7 -3
- package/index.ts +6 -0
- package/openclaw.plugin.json +21 -0
- package/package.json +1 -1
- package/scripts/check-pack.mjs +97 -17
- package/scripts/check-register-drift.mjs +91 -65
- package/scripts/selfcheck.mjs +79 -3
- package/src/channel.ts +477 -635
- package/src/core/connection-capability.ts +2 -2
- package/src/core/connection-reachability.ts +106 -0
- package/src/core/dead-letter-diagnostics.ts +91 -0
- package/src/core/diagnostic-counters.ts +61 -0
- package/src/core/diagnostics.ts +9 -5
- package/src/core/downlink-health.ts +12 -7
- package/src/core/extended-diagnostics.ts +2 -0
- package/src/core/logging.ts +98 -0
- package/src/core/outbox-entry-builders.ts +13 -2
- package/src/core/persisted-outbox-entry.ts +53 -0
- package/src/core/probe.ts +33 -13
- package/src/core/register-trace.ts +48 -0
- package/src/core/status-meta.ts +77 -0
- package/src/core/status.ts +50 -57
- package/src/messaging/inbound/commands.ts +25 -86
- package/src/messaging/inbound/dispatch.ts +9 -36
- package/src/messaging/inbound/last-route.ts +46 -0
- package/src/messaging/inbound/native-command.ts +49 -0
- package/src/messaging/inbound/native-reply-delivery.ts +43 -0
- package/src/messaging/outbound/diagnostics.ts +221 -2
- package/src/messaging/outbound/reply-enqueue.ts +56 -1
- package/src/messaging/outbound/reply-target-policy.ts +4 -1
- package/src/messaging/outbound/send-params.ts +56 -0
- package/src/openclaw/runtime-surface.ts +29 -0
- package/src/plugin/gateway-methods.ts +2 -0
- package/src/plugin/status.ts +10 -4
- package/src/runtime/outbound-ack-timeout.ts +73 -0
- package/src/runtime/register-trace-runtime.ts +102 -0
- package/src/runtime/status-snapshots.ts +7 -3
- package/src/runtime/status-worker.ts +70 -11
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import {
|
|
2
|
+
appendBoundedRegisterTrace,
|
|
3
|
+
buildRegisterDriftSnapshot,
|
|
4
|
+
buildRegisterTraceEntry,
|
|
5
|
+
buildRegisterTraceSummary,
|
|
6
|
+
type RegisterDriftSnapshot,
|
|
7
|
+
type RegisterTraceEntry,
|
|
8
|
+
type RegisterTraceSummary,
|
|
9
|
+
} from '../core/register-trace.ts';
|
|
10
|
+
|
|
11
|
+
export type RegisterTraceRuntimeState = {
|
|
12
|
+
registerCount: number;
|
|
13
|
+
apiGeneration: number;
|
|
14
|
+
firstRegisterAt: number | null;
|
|
15
|
+
lastRegisterAt: number | null;
|
|
16
|
+
lastApiRebindAt: number | null;
|
|
17
|
+
pluginSource: string | null;
|
|
18
|
+
pluginVersion: string | null;
|
|
19
|
+
lastApiInstanceId: string | null;
|
|
20
|
+
lastRegistryFingerprint: string | null;
|
|
21
|
+
lastDriftSnapshot: RegisterDriftSnapshot | null;
|
|
22
|
+
registerTraceRecent: RegisterTraceEntry[];
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
export type RegisterTraceRuntimeMeta = {
|
|
26
|
+
source?: string;
|
|
27
|
+
pluginVersion?: string;
|
|
28
|
+
apiRebound?: boolean;
|
|
29
|
+
apiInstanceId?: string;
|
|
30
|
+
registryFingerprint?: string;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export function buildRegisterTraceRuntimeSummary(args: {
|
|
34
|
+
state: Pick<RegisterTraceRuntimeState, 'registerTraceRecent' | 'firstRegisterAt'>;
|
|
35
|
+
warmupWindowMs: number;
|
|
36
|
+
}): RegisterTraceSummary {
|
|
37
|
+
return buildRegisterTraceSummary({
|
|
38
|
+
traceRecent: args.state.registerTraceRecent,
|
|
39
|
+
firstRegisterAt: args.state.firstRegisterAt,
|
|
40
|
+
warmupWindowMs: args.warmupWindowMs,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function noteRegisterTraceRuntime(args: {
|
|
45
|
+
state: RegisterTraceRuntimeState;
|
|
46
|
+
meta: RegisterTraceRuntimeMeta;
|
|
47
|
+
ts: number;
|
|
48
|
+
stack: string;
|
|
49
|
+
bridgeId: string;
|
|
50
|
+
gatewayPid: number;
|
|
51
|
+
warmupWindowMs: number;
|
|
52
|
+
maxTraceEntries?: number;
|
|
53
|
+
}): { trace: RegisterTraceEntry; summary: RegisterTraceSummary; capturedDriftSnapshot: boolean } {
|
|
54
|
+
const { state, meta } = args;
|
|
55
|
+
state.registerCount += 1;
|
|
56
|
+
if (state.firstRegisterAt == null) state.firstRegisterAt = args.ts;
|
|
57
|
+
state.lastRegisterAt = args.ts;
|
|
58
|
+
if (meta.apiRebound) {
|
|
59
|
+
state.apiGeneration += 1;
|
|
60
|
+
state.lastApiRebindAt = args.ts;
|
|
61
|
+
} else if (state.registerCount === 1 && state.apiGeneration === 0) {
|
|
62
|
+
state.apiGeneration = 1;
|
|
63
|
+
}
|
|
64
|
+
if (meta.source) state.pluginSource = meta.source;
|
|
65
|
+
if (meta.pluginVersion) state.pluginVersion = meta.pluginVersion;
|
|
66
|
+
if (meta.apiInstanceId) state.lastApiInstanceId = meta.apiInstanceId;
|
|
67
|
+
if (meta.registryFingerprint) state.lastRegistryFingerprint = meta.registryFingerprint;
|
|
68
|
+
|
|
69
|
+
const trace = buildRegisterTraceEntry({
|
|
70
|
+
ts: args.ts,
|
|
71
|
+
bridgeId: args.bridgeId,
|
|
72
|
+
gatewayPid: args.gatewayPid,
|
|
73
|
+
registerCount: state.registerCount,
|
|
74
|
+
apiGeneration: state.apiGeneration,
|
|
75
|
+
apiRebound: meta.apiRebound === true,
|
|
76
|
+
apiInstanceId: state.lastApiInstanceId,
|
|
77
|
+
registryFingerprint: state.lastRegistryFingerprint,
|
|
78
|
+
source: state.pluginSource,
|
|
79
|
+
pluginVersion: state.pluginVersion,
|
|
80
|
+
stack: args.stack,
|
|
81
|
+
});
|
|
82
|
+
appendBoundedRegisterTrace(state.registerTraceRecent, trace, args.maxTraceEntries ?? 12);
|
|
83
|
+
|
|
84
|
+
const summary = buildRegisterTraceRuntimeSummary({
|
|
85
|
+
state,
|
|
86
|
+
warmupWindowMs: args.warmupWindowMs,
|
|
87
|
+
});
|
|
88
|
+
const capturedDriftSnapshot = summary.postWarmupRegisterCount > 0;
|
|
89
|
+
if (capturedDriftSnapshot) {
|
|
90
|
+
state.lastDriftSnapshot = buildRegisterDriftSnapshot({
|
|
91
|
+
capturedAt: args.ts,
|
|
92
|
+
registerCount: state.registerCount,
|
|
93
|
+
apiGeneration: state.apiGeneration,
|
|
94
|
+
summary,
|
|
95
|
+
apiInstanceId: state.lastApiInstanceId,
|
|
96
|
+
registryFingerprint: state.lastRegistryFingerprint,
|
|
97
|
+
traceRecent: state.registerTraceRecent,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return { trace, summary, capturedDriftSnapshot };
|
|
102
|
+
}
|
|
@@ -46,6 +46,10 @@ export function buildRuntimeEventCounters(args: {
|
|
|
46
46
|
};
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
function nullableMapNumber(map: Map<string, number>, key: string): number | null {
|
|
50
|
+
return map.get(key) ?? null;
|
|
51
|
+
}
|
|
52
|
+
|
|
49
53
|
export function buildRuntimeActivitySnapshot(args: {
|
|
50
54
|
accountId: string;
|
|
51
55
|
activeConnectionCount: (accountId: string) => number;
|
|
@@ -58,9 +62,9 @@ export function buildRuntimeActivitySnapshot(args: {
|
|
|
58
62
|
return {
|
|
59
63
|
activeConnections: args.activeConnectionCount(accountId),
|
|
60
64
|
lastSession: args.lastSessionByAccount.get(accountId) || null,
|
|
61
|
-
lastActivityAt: args.lastActivityByAccount
|
|
62
|
-
lastInboundAt: args.lastInboundByAccount
|
|
63
|
-
lastOutboundAt: args.lastOutboundByAccount
|
|
65
|
+
lastActivityAt: nullableMapNumber(args.lastActivityByAccount, accountId),
|
|
66
|
+
lastInboundAt: nullableMapNumber(args.lastInboundByAccount, accountId),
|
|
67
|
+
lastOutboundAt: nullableMapNumber(args.lastOutboundByAccount, accountId),
|
|
64
68
|
};
|
|
65
69
|
}
|
|
66
70
|
|
|
@@ -15,8 +15,58 @@ export type ChannelAccountWorkerHandle = {
|
|
|
15
15
|
timer: NodeJS.Timeout;
|
|
16
16
|
finish: (reason: string) => void;
|
|
17
17
|
cleanupAbortListener?: () => void;
|
|
18
|
+
healthLogState?: HealthStatusLogState;
|
|
18
19
|
};
|
|
19
20
|
|
|
21
|
+
export const HEALTH_STATUS_STABLE_WINDOW_MS = 10_000;
|
|
22
|
+
|
|
23
|
+
export type HealthStatusLogState = {
|
|
24
|
+
emittedSig: string | null;
|
|
25
|
+
pendingSig: string | null;
|
|
26
|
+
pendingSince: number;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export function createHealthStatusLogState(): HealthStatusLogState {
|
|
30
|
+
return { emittedSig: null, pendingSig: null, pendingSince: 0 };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function finiteNumberOr(value: unknown, fallback: number): number {
|
|
34
|
+
const n = Number(value);
|
|
35
|
+
return Number.isFinite(n) ? n : fallback;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function nonNegativeFiniteNumberOr(value: unknown, fallback: number): number {
|
|
39
|
+
return Math.max(0, finiteNumberOr(value, fallback));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function updateHealthStatusLogState(args: {
|
|
43
|
+
state: HealthStatusLogState;
|
|
44
|
+
sig: string;
|
|
45
|
+
nowMs: number;
|
|
46
|
+
stableWindowMs?: number;
|
|
47
|
+
}): 'pending' | 'stable' | 'unchanged' {
|
|
48
|
+
const stableWindowMs = nonNegativeFiniteNumberOr(
|
|
49
|
+
args.stableWindowMs,
|
|
50
|
+
HEALTH_STATUS_STABLE_WINDOW_MS,
|
|
51
|
+
);
|
|
52
|
+
const nowMs = finiteNumberOr(args.nowMs, 0);
|
|
53
|
+
if (args.state.emittedSig === args.sig) {
|
|
54
|
+
args.state.pendingSig = null;
|
|
55
|
+
args.state.pendingSince = 0;
|
|
56
|
+
return 'unchanged';
|
|
57
|
+
}
|
|
58
|
+
if (args.state.pendingSig !== args.sig) {
|
|
59
|
+
args.state.pendingSig = args.sig;
|
|
60
|
+
args.state.pendingSince = nowMs;
|
|
61
|
+
if (stableWindowMs > 0) return 'pending';
|
|
62
|
+
}
|
|
63
|
+
if (nowMs - args.state.pendingSince < stableWindowMs) return 'pending';
|
|
64
|
+
args.state.emittedSig = args.sig;
|
|
65
|
+
args.state.pendingSig = null;
|
|
66
|
+
args.state.pendingSince = 0;
|
|
67
|
+
return 'stable';
|
|
68
|
+
}
|
|
69
|
+
|
|
20
70
|
type StatusWorkerHooks = {
|
|
21
71
|
isOnline: (accountId: string) => boolean;
|
|
22
72
|
hasRecentInboundReachability: (accountId: string) => boolean;
|
|
@@ -30,6 +80,8 @@ type StatusWorkerHooks = {
|
|
|
30
80
|
message: string,
|
|
31
81
|
options: { key: string; sig: string; debugOnly?: boolean; windowMs?: number },
|
|
32
82
|
) => void;
|
|
83
|
+
now?: () => number;
|
|
84
|
+
healthStableWindowMs?: number;
|
|
33
85
|
};
|
|
34
86
|
|
|
35
87
|
type StatusWorkerRuntime = {
|
|
@@ -66,6 +118,7 @@ export async function startBncrStatusWorker(
|
|
|
66
118
|
) {
|
|
67
119
|
const accountId = normalizeAccountId(ctx.accountId);
|
|
68
120
|
clearBncrStatusWorker(runtime, accountId, 'start-replace');
|
|
121
|
+
let worker!: ChannelAccountWorkerHandle;
|
|
69
122
|
|
|
70
123
|
const tick = () => {
|
|
71
124
|
const previous = ctx.getStatus?.() || {};
|
|
@@ -84,14 +137,20 @@ export async function startBncrStatusWorker(
|
|
|
84
137
|
activeConnections,
|
|
85
138
|
});
|
|
86
139
|
const conns = activeConnections.length;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
);
|
|
140
|
+
const healthLogState = worker.healthLogState || createHealthStatusLogState();
|
|
141
|
+
worker.healthLogState = healthLogState;
|
|
142
|
+
const healthLogDecision = updateHealthStatusLogState({
|
|
143
|
+
state: healthLogState,
|
|
144
|
+
sig: healthSig,
|
|
145
|
+
nowMs: runtime.hooks.now?.() ?? Date.now(),
|
|
146
|
+
stableWindowMs: runtime.hooks.healthStableWindowMs,
|
|
147
|
+
});
|
|
148
|
+
if (healthLogDecision === 'stable') {
|
|
149
|
+
runtime.hooks.logInfo(
|
|
150
|
+
'health',
|
|
151
|
+
`status-tick ${accountId}|stable|${connected ? 'linked' : 'configured'}|onlineByConn=${onlineByConn}|recentInboundReachable=${recentInboundReachable}|conns=${conns}`,
|
|
152
|
+
);
|
|
153
|
+
}
|
|
95
154
|
runtime.hooks.logInfoDedup('health', `status-tick ${healthSig}`, {
|
|
96
155
|
key: `health-status-tick-debug:${accountId}`,
|
|
97
156
|
sig: healthSig,
|
|
@@ -111,9 +170,7 @@ export async function startBncrStatusWorker(
|
|
|
111
170
|
});
|
|
112
171
|
};
|
|
113
172
|
|
|
114
|
-
tick();
|
|
115
173
|
const timer = setInterval(tick, 5_000);
|
|
116
|
-
let worker!: ChannelAccountWorkerHandle;
|
|
117
174
|
const done = new Promise<void>((resolve) => {
|
|
118
175
|
let settled = false;
|
|
119
176
|
const finish = (reason: string) => {
|
|
@@ -135,9 +192,11 @@ export async function startBncrStatusWorker(
|
|
|
135
192
|
resolve();
|
|
136
193
|
};
|
|
137
194
|
|
|
138
|
-
worker = { timer, finish };
|
|
195
|
+
worker = { timer, finish, healthLogState: createHealthStatusLogState() };
|
|
139
196
|
runtime.workers.set(accountId, worker);
|
|
140
197
|
|
|
198
|
+
tick();
|
|
199
|
+
|
|
141
200
|
const onAbort = () => finish('abort');
|
|
142
201
|
const abortSignal = ctx.abortSignal;
|
|
143
202
|
|