@xmoxmo/bncr 0.3.6 → 0.3.7

package/src/plugin/runtime-diagnostics-helpers.ts

@@ -0,0 +1,302 @@
+import {
+  buildDeadLetterDiagnostics as buildDeadLetterDiagnosticsFromRuntime,
+  filterDeadLetterEntries as filterDeadLetterEntriesFromRuntime,
+} from '../core/dead-letter-diagnostics.ts';
+import type {
+  RegisterDriftSnapshot,
+  RegisterTraceEntry,
+  RegisterTraceSummary,
+} from '../core/register-trace.ts';
+import type { buildIntegratedDiagnostics as buildIntegratedDiagnosticsFromRuntime } from '../core/status.ts';
+import type {
+  BncrAckObservability,
+  BncrConnection,
+  BncrOutboxQueueDiagnostics,
+  OutboxEntry,
+} from '../core/types.ts';
+import { buildBncrRuntimeAckObservability } from '../runtime/outbound-ack-timeout.ts';
+import {
+  buildBncrActiveConnectionDebugList,
+  buildBncrDeadLetterSummaryMessage,
+} from './runtime-diagnostics-payload-builders.ts';
+
+type RuntimeStatusInput = Parameters<typeof buildIntegratedDiagnosticsFromRuntime>[0];
+type IntegratedDiagnostics = ReturnType<typeof buildIntegratedDiagnosticsFromRuntime>;
+type BncrConnectionDebugView = BncrConnection & {
+  outboundReadyUntil?: number;
+  preferredForOutboundUntil?: number;
+  lastAckOkAt?: number;
+  lastPushTimeoutAt?: number;
+  pushFailureScore?: number;
+};
+
+export type ExtendedDiagnosticsAssemblerRuntime = {
+  normalizeAccountId: (accountId: string) => string;
+  buildIntegratedDiagnostics: (
+    accountId: string,
+    runtimeStatusInput?: RuntimeStatusInput,
+  ) => IntegratedDiagnostics;
+  buildOutboxDiagnostics: (accountId: string) => BncrOutboxQueueDiagnostics;
+  buildRuntimeAckObservability: (accountId: string) => BncrAckObservability;
+  getCounter: (map: Map<string, number>, accountId: string) => number;
+  prePushGuardSkipCountByAccount: Map<string, number>;
+  lastPrePushGuardSkipAtByAccount: Map<string, number>;
+  lastPrePushGuardSkipReasonByAccount: Map<string, string>;
+  hasGatewayContext: () => boolean;
+  buildRuntimeSurfaceDiagnostics: () => ReturnType<
+    typeof import('../openclaw/runtime-surface.ts')['buildOpenClawChannelRuntimeSurfaceDiagnostics']
+  >;
+  getRegisterRuntime: () => {
+    bridgeId: string;
+    gatewayPid: number;
+    pluginVersion: string | null;
+    pluginSource: string | null;
+    lastApiInstanceId: string | null;
+    lastRegistryFingerprint: string | null;
+    registerCount: number;
+    firstRegisterAt: number | null;
+    lastRegisterAt: number | null;
+    lastApiRebindAt: number | null;
+    apiGeneration: number;
+    registerTraceRecent: RegisterTraceEntry[];
+    lastDriftSnapshot: RegisterDriftSnapshot | null;
+  };
+  buildRegisterTraceSummary: () => RegisterTraceSummary;
+  activeConnectionCount: (accountId: string) => number;
+  getConnectionRuntime: () => {
+    lastGatewayContextAt: number | null;
+    primaryLeaseId: string | null;
+    connectionEpoch: number;
+    acceptedConnections: number;
+    lastConnectAt: number | null;
+    lastDisconnectAt: number | null;
+    lastActivityAtGlobal: number | null;
+    lastInboundAtGlobal: number | null;
+    lastAckAtGlobal: number | null;
+    recentConnections: Map<
+      string,
+      { epoch: number; connectedAt: number; lastActivityAt: number | null; isPrimary: boolean }
+    >;
+  };
+  getOutboundRuntime: () => {
+    outboundEnqueueCountByAccount: Map<string, number>;
+    lastOutboundEnqueueAtByAccount: Map<string, number>;
+  };
+  buildDeadLetterDiagnostics: (
+    accountId: string,
+  ) => ReturnType<typeof buildBncrDeadLetterDiagnosticsSnapshot>;
+  bridgeVersion: number;
+  staleCounters: {
+    staleConnect: number;
+    staleInbound: number;
+    staleActivity: number;
+    staleAck: number;
+    staleFileInit: number;
+    staleFileChunk: number;
+    staleFileComplete: number;
+    staleFileAbort: number;
+    lastStaleAt: number | null;
+  };
+  now: () => number;
+};
+
+export type ExtendedDiagnosticsAssemblerOptions = {
+  runtimeStatusInput?: RuntimeStatusInput;
+  integratedDiagnostics?: IntegratedDiagnostics;
+};
+
+export function buildBncrDeadLetterDiagnosticsSnapshot(args: {
+  accountId: string;
+  entries: OutboxEntry[];
+  allAccountsTotal: number;
+  sinceStart: number;
+  cappedAt: number;
+}) {
+  return buildDeadLetterDiagnosticsFromRuntime({
+    entries: args.entries,
+    allAccountsTotal: args.allAccountsTotal,
+    sinceStart: args.sinceStart,
+    cappedAt: args.cappedAt,
+  });
+}
+
+export function createBncrOutboxDiagnosticsHelpers(runtime: {
+  normalizeAccountId: (accountId: string) => string;
+  outboxValues: () => Iterable<OutboxEntry>;
+  pendingAllAccounts: () => number;
+  resolvePushConnIds: (accountId: string) => Set<string>;
+  buildOutboxQueueDiagnostics: typeof import('../messaging/outbound/diagnostics.ts').buildOutboxQueueDiagnostics;
+  buildBncrOutboxQueueDiagnosticsInput: typeof import('./runtime-diagnostics-payload-builders.ts').buildBncrOutboxQueueDiagnosticsInput;
+}) {
+  const buildOutboxDiagnostics = (accountId: string) => {
+    const acc = runtime.normalizeAccountId(accountId);
+    return runtime.buildOutboxQueueDiagnostics(
+      runtime.buildBncrOutboxQueueDiagnosticsInput({
+        accountId: acc,
+        outboxEntries: runtime.outboxValues(),
+        pendingAllAccounts: runtime.pendingAllAccounts(),
+        pushConnIds: runtime.resolvePushConnIds(acc),
+      }),
+    );
+  };
+
+  return { buildOutboxDiagnostics };
+}
+
+export function createBncrRuntimeAckObservabilityBuilder(runtime: {
+  normalizeAccountId: (accountId: string) => string;
+  getCounter: (map: Map<string, number>, accountId: string) => number;
+  ackTimeoutCountByAccount: Map<string, number>;
+  lateAckOkCountByAccount: Map<string, number>;
+  lastLateAckPushLatencyMsByAccount: Map<string, number>;
+  lastLateAckOkByAccount: Map<string, number>;
+  adaptiveAckRecoveryOkCountByAccount: Map<string, number>;
+  lastAckOkByAccount: Map<string, number>;
+  lastAckTimeoutByAccount: Map<string, number>;
+  lastAckQueueLatencyMsByAccount: Map<string, number>;
+  lastAckPushLatencyMsByAccount: Map<string, number>;
+  lastLateAckQueueLatencyMsByAccount: Map<string, number>;
+  adaptiveAckTimeoutEnabled: boolean;
+  defaultAckTimeoutMs: number;
+  resolveMessageAckTimeoutMs: (accountId: string) => number;
+  minAckTimeoutMs: number;
+  maxAckTimeoutMs: number;
+  lateAckObservationTtlMs: number;
+  recoveryOkThreshold: number;
+  now: () => number;
+}) {
+  return (accountId: string): BncrAckObservability => {
+    const acc = runtime.normalizeAccountId(accountId);
+    const recentAckTimeoutCount = runtime.getCounter(runtime.ackTimeoutCountByAccount, acc);
+    const lateAckOkCount = runtime.getCounter(runtime.lateAckOkCountByAccount, acc);
+    const lastLateAckPushLatencyMs = runtime.lastLateAckPushLatencyMsByAccount.get(acc) || null;
+    const lastLateAckOkAt = runtime.lastLateAckOkByAccount.get(acc) || null;
+    const nowMs = runtime.now();
+    const adaptiveAckRecoveryOkCount = runtime.getCounter(
+      runtime.adaptiveAckRecoveryOkCountByAccount,
+      acc,
+    );
+    return buildBncrRuntimeAckObservability({
+      lastAckOkAt: runtime.lastAckOkByAccount.get(acc) || null,
+      lastAckTimeoutAt: runtime.lastAckTimeoutByAccount.get(acc) || null,
+      recentAckTimeoutCount,
+      lateAckOkCount,
+      lastLateAckOkAt,
+      adaptiveAckRecoveryOkCount,
+      lastAckQueueLatencyMs: runtime.lastAckQueueLatencyMsByAccount.get(acc) || null,
+      lastAckPushLatencyMs: runtime.lastAckPushLatencyMsByAccount.get(acc) || null,
+      lastLateAckQueueLatencyMs: runtime.lastLateAckQueueLatencyMsByAccount.get(acc) || null,
+      lastLateAckPushLatencyMs,
+      adaptiveAckTimeoutEnabled: runtime.adaptiveAckTimeoutEnabled,
+      defaultAckTimeoutMs: runtime.defaultAckTimeoutMs,
+      currentAckTimeoutMs: runtime.resolveMessageAckTimeoutMs(acc),
+      minAckTimeoutMs: runtime.minAckTimeoutMs,
+      maxAckTimeoutMs: runtime.maxAckTimeoutMs,
+      lateAckObservationTtlMs: runtime.lateAckObservationTtlMs,
+      recoveryOkThreshold: runtime.recoveryOkThreshold,
+      nowMs,
+    });
+  };
+}
+
+export function createBncrDeadLetterDiagnosticsHelpers(runtime: {
+  normalizeAccountId: (accountId: string) => string;
+  getDeadLetterEntries: () => OutboxEntry[];
+  maxDeadLetterEntries: number;
+  getCounter: (map: Map<string, number>, accountId: string) => number;
+  deadLetterSinceStartByAccount: Map<string, number>;
+  getAccountDeadLetterEntries: (accountId: string) => OutboxEntry[];
+  logInfo: (scope: string, message: string, options?: { debugOnly?: boolean }) => void;
+  logInfoDedup: (
+    scope: string,
+    message: string,
+    options: { key: string; sig: string; windowMs?: number },
+  ) => void;
+}) {
+  const buildDeadLetterDiagnostics = (accountId: string) => {
+    const acc = runtime.normalizeAccountId(accountId);
+    return buildBncrDeadLetterDiagnosticsSnapshot({
+      accountId: acc,
+      entries: runtime.getAccountDeadLetterEntries(acc),
+      allAccountsTotal: runtime.getDeadLetterEntries().length,
+      sinceStart: runtime.getCounter(runtime.deadLetterSinceStartByAccount, acc),
+      cappedAt: runtime.maxDeadLetterEntries,
+    });
+  };
+
+  const logDeadLetterSummary = (
+    accountId: string,
+    options?: { force?: boolean; source?: string },
+  ) => {
+    const acc = runtime.normalizeAccountId(accountId);
+    const summary = buildDeadLetterDiagnostics(acc);
+    const message = buildBncrDeadLetterSummaryMessage({
+      accountId: acc,
+      summary,
+      source: options?.source,
+    });
+    if (options?.force) {
+      runtime.logInfo('deadLetter summary', message);
+      return;
+    }
+    runtime.logInfoDedup('deadLetter summary', message, {
+      key: `dead-letter-summary:${acc}:update`,
+      sig: 'dead-letter-summary',
+      windowMs: 5 * 60 * 1000,
+    });
+  };
+
+  const filterDeadLetterEntries = (params: {
+    accountId: string;
+    reason?: string | null;
+    olderThan?: number | null;
+  }) => {
+    return filterDeadLetterEntriesFromRuntime({
+      accountId: params.accountId,
+      entries: runtime.getDeadLetterEntries(),
+      reason: params.reason,
+      olderThan: params.olderThan,
+    });
+  };
+
+  return {
+    buildDeadLetterDiagnostics,
+    filterDeadLetterEntries,
+    logDeadLetterSummary,
+  };
+}
+
+export function createBncrDiagnosticsSelectionHelpers(runtime: {
+  normalizeAccountId: (accountId: string) => string;
+  outboxValues: () => Iterable<OutboxEntry>;
+  getDeadLetterEntries: () => OutboxEntry[];
+  connectionsValues: () => Iterable<BncrConnection>;
+}) {
+  const getAccountPendingOutboxEntries = (accountId: string) => {
+    const acc = runtime.normalizeAccountId(accountId);
+    return Array.from(runtime.outboxValues()).filter((entry) => entry.accountId === acc);
+  };
+
+  const getAccountDeadLetterEntries = (accountId: string) => {
+    const acc = runtime.normalizeAccountId(accountId);
+    return runtime.getDeadLetterEntries().filter((entry) => entry.accountId === acc);
+  };
+
+  const buildActiveConnectionDebugList = (
+    accountId: string,
+    options?: { includeOutboundState?: boolean },
+  ) => {
+    const acc = runtime.normalizeAccountId(accountId);
+    return buildBncrActiveConnectionDebugList({
+      accountId: acc,
+      connections: runtime.connectionsValues() as Iterable<BncrConnectionDebugView>,
+      options,
+    });
+  };
+
+  return {
+    getAccountPendingOutboxEntries,
+    getAccountDeadLetterEntries,
+    buildActiveConnectionDebugList,
+  };
+}

package/src/plugin/runtime-diagnostics-payload-builders.ts

@@ -0,0 +1,171 @@
+import { formatDeadLetterTopReasons } from '../core/dead-letter-diagnostics.ts';
+import type {
+  RegisterDriftSnapshot,
+  RegisterTraceEntry,
+  RegisterTraceSummary,
+} from '../core/register-trace.ts';
+import type {
+  BncrAckObservability,
+  BncrDeadLetterDiagnosticsSummary,
+  BncrOutboxQueueDiagnostics,
+  OutboxEntry,
+} from '../core/types.ts';
+
+export function buildBncrActiveConnectionDebugList(args: {
+  accountId: string;
+  connections: Iterable<{
+    accountId: string;
+    connId: string;
+    clientId?: string;
+    connectedAt: number;
+    lastSeenAt: number;
+    outboundReadyUntil?: number;
+    preferredForOutboundUntil?: number;
+    inboundOnly?: boolean;
+  }>;
+  options?: { includeOutboundState?: boolean };
+}) {
+  return Array.from(args.connections)
+    .filter((conn) => conn.accountId === args.accountId)
+    .map((conn) => ({
+      accountId: conn.accountId,
+      connId: conn.connId,
+      clientId: conn.clientId,
+      connectedAt: conn.connectedAt,
+      lastSeenAt: conn.lastSeenAt,
+      ...(args.options?.includeOutboundState
+        ? {
+            outboundReadyUntil: conn.outboundReadyUntil || null,
+            preferredForOutboundUntil: conn.preferredForOutboundUntil || null,
+            inboundOnly: conn.inboundOnly === true,
+          }
+        : {}),
+    }));
+}
+
+export function buildBncrExtendedRegisterDiagnostics(args: {
+  bridgeId: string;
+  gatewayPid: number;
+  pluginVersion: string | null;
+  source: string | null;
+  apiInstanceId: string | null;
+  registryFingerprint: string | null;
+  registerCount: number;
+  firstRegisterAt: number | null;
+  lastRegisterAt: number | null;
+  lastApiRebindAt: number | null;
+  apiGeneration: number;
+  traceRecent: RegisterTraceEntry[];
+  traceSummary: RegisterTraceSummary;
+  lastDriftSnapshot: RegisterDriftSnapshot | null;
+}) {
+  return {
+    bridgeId: args.bridgeId,
+    gatewayPid: args.gatewayPid,
+    pluginVersion: args.pluginVersion,
+    source: args.source,
+    apiInstanceId: args.apiInstanceId,
+    registryFingerprint: args.registryFingerprint,
+    registerCount: args.registerCount,
+    firstRegisterAt: args.firstRegisterAt,
+    lastRegisterAt: args.lastRegisterAt,
+    lastApiRebindAt: args.lastApiRebindAt,
+    apiGeneration: args.apiGeneration,
+    traceRecent: args.traceRecent,
+    traceSummary: args.traceSummary,
+    lastDriftSnapshot: args.lastDriftSnapshot ?? null,
+  };
+}
+
+export function buildBncrExtendedConnectionDiagnostics(args: {
+  active: number;
+  hasGatewayContext: boolean;
+  lastGatewayContextAt: number | null;
+  primaryLeaseId: string | null;
+  primaryEpoch: number | null;
+  acceptedConnections: number;
+  lastConnectAt: number | null;
+  lastDisconnectAt: number | null;
+  lastActivityAt: number | null;
+  lastInboundAt: number | null;
+  lastAckAt: number | null;
+  recentConnections: Map<
+    string,
+    { epoch: number; connectedAt: number; lastActivityAt: number | null; isPrimary: boolean }
+  >;
+}) {
+  return {
+    active: args.active,
+    hasGatewayContext: args.hasGatewayContext,
+    lastGatewayContextAt: args.lastGatewayContextAt,
+    primaryLeaseId: args.primaryLeaseId,
+    primaryEpoch: args.primaryEpoch,
+    acceptedConnections: args.acceptedConnections,
+    lastConnectAt: args.lastConnectAt,
+    lastDisconnectAt: args.lastDisconnectAt,
+    lastActivityAt: args.lastActivityAt,
+    lastInboundAt: args.lastInboundAt,
+    lastAckAt: args.lastAckAt,
+    recent: Array.from(args.recentConnections.entries()).map(([leaseId, entry]) => ({
+      leaseId,
+      epoch: entry.epoch,
+      connectedAt: entry.connectedAt,
+      lastActivityAt: entry.lastActivityAt,
+      isPrimary: entry.isPrimary,
+    })),
+  };
+}
+
+export function buildBncrExtendedOutboundDiagnosticsInput(args: {
+  outbox: BncrOutboxQueueDiagnostics;
+  enqueueCount: number;
+  lastEnqueueAt: number | null;
+  prePushGuardSkipCount: number;
+  lastPrePushGuardSkipAt: number | null;
+  lastPrePushGuardSkipReason: string | null;
+  hasGatewayContext: boolean;
+  lastGatewayContextAt: number | null;
+  ackObservability: BncrAckObservability;
+  nowMs: number;
+}) {
+  return {
+    outbox: args.outbox,
+    enqueueCount: args.enqueueCount,
+    lastEnqueueAt: args.lastEnqueueAt,
+    prePushGuardSkipCount: args.prePushGuardSkipCount,
+    lastPrePushGuardSkipAt: args.lastPrePushGuardSkipAt,
+    lastPrePushGuardSkipReason: args.lastPrePushGuardSkipReason,
+    hasGatewayContext: args.hasGatewayContext,
+    lastGatewayContextAt: args.lastGatewayContextAt,
+    ackObservability: args.ackObservability,
+    nowMs: args.nowMs,
+  };
+}
+
+export function buildBncrDeadLetterSummaryMessage(args: {
+  accountId: string;
+  summary: BncrDeadLetterDiagnosticsSummary;
+  source?: string;
+}) {
+  return [
+    `${args.accountId}|total=${args.summary.total}`,
+    `all=${args.summary.allAccountsTotal}`,
+    `sinceStart=${args.summary.sinceStart}`,
+    `top=${formatDeadLetterTopReasons(args.summary.topReasons)}`,
+    `source=${args.source || 'update'}`,
+  ].join('|');
+}
+
+export function buildBncrOutboxQueueDiagnosticsInput(args: {
+  accountId: string;
+  outboxEntries: Iterable<OutboxEntry>;
+  pendingAllAccounts: number;
+  pushConnIds: Set<string>;
+}) {
+  return {
+    accountId: args.accountId,
+    outboxEntries: args.outboxEntries,
+    pendingAllAccounts: args.pendingAllAccounts,
+    pushConnIds: args.pushConnIds,
+  };
+}

package/src/plugin/runtime-diagnostics-snapshot.ts

@@ -0,0 +1,31 @@
+import type { OutboxEntry } from '../core/types.ts';
+import { buildOutboxQueueDiagnostics } from '../messaging/outbound/diagnostics.ts';
+import { createBncrOutboxDiagnosticsHelpers as createBncrOutboxDiagnosticsHelpersBase } from './runtime-diagnostics-helpers.ts';
+import { buildBncrOutboxQueueDiagnosticsInput } from './runtime-diagnostics-payload-builders.ts';
+
+export {
+  buildBncrExtendedDiagnosticsSnapshot,
+  createBncrExtendedDiagnosticsAssembler,
+} from './runtime-diagnostics-assembler.ts';
+
+export {
+  buildBncrDeadLetterDiagnosticsSnapshot,
+  createBncrDeadLetterDiagnosticsHelpers,
+  createBncrDiagnosticsSelectionHelpers,
+  createBncrRuntimeAckObservabilityBuilder,
+  type ExtendedDiagnosticsAssemblerOptions,
+  type ExtendedDiagnosticsAssemblerRuntime,
+} from './runtime-diagnostics-helpers.ts';
+
+export function createBncrOutboxDiagnosticsHelpers(runtime: {
+  normalizeAccountId: (accountId: string) => string;
+  outboxValues: () => Iterable<OutboxEntry>;
+  pendingAllAccounts: () => number;
+  resolvePushConnIds: (accountId: string) => Set<string>;
+}) {
+  return createBncrOutboxDiagnosticsHelpersBase({
+    ...runtime,
+    buildOutboxQueueDiagnostics,
+    buildBncrOutboxQueueDiagnosticsInput,
+  });
+}

package/src/plugin/setup.ts

@@ -1,8 +1,34 @@
 import { CHANNEL_ID } from '../core/accounts.ts';
 import { applyOpenClawAccountNameToChannelSection } from '../openclaw/sdk-helpers.ts';
 
+type BncrSetupAccountConfig = {
+  enabled?: boolean;
+  name?: string;
+} & Record<string, unknown>;
+
+type BncrSetupChannelConfig = {
+  accounts?: Record<string, BncrSetupAccountConfig>;
+} & Record<string, unknown>;
+
+type BncrSetupRootConfig = {
+  channels?: Record<string, unknown> & {
+    [CHANNEL_ID]?: BncrSetupChannelConfig;
+  };
+};
+
+type BncrApplyAccountNameArgs = {
+  cfg: BncrSetupRootConfig;
+  accountId: string;
+  name?: string;
+};
+
+type BncrApplyAccountConfigArgs = {
+  cfg: BncrSetupRootConfig | null | undefined;
+  accountId: string;
+};
+
 export const BNCR_SETUP_SURFACE = {
-  applyAccountName: ({ cfg, accountId, name }: any) =>
+  applyAccountName: ({ cfg, accountId, name }: BncrApplyAccountNameArgs) =>
     applyOpenClawAccountNameToChannelSection({
       cfg,
       channelKey: CHANNEL_ID,
@@ -10,13 +36,14 @@ export const BNCR_SETUP_SURFACE = {
       name,
       alwaysUseAccounts: true,
     }),
-  applyAccountConfig: ({ cfg, accountId }: any) => {
-    const next = { ...(cfg || {}) } as any;
+  applyAccountConfig: ({ cfg, accountId }: BncrApplyAccountConfigArgs) => {
+    const next: BncrSetupRootConfig = { ...(cfg || {}) };
     next.channels = next.channels || {};
     next.channels[CHANNEL_ID] = next.channels[CHANNEL_ID] || {};
-    next.channels[CHANNEL_ID].accounts = next.channels[CHANNEL_ID].accounts || {};
-    next.channels[CHANNEL_ID].accounts[accountId] = {
-      ...(next.channels[CHANNEL_ID].accounts[accountId] || {}),
+    const channelCfg = next.channels[CHANNEL_ID] as BncrSetupChannelConfig;
+    channelCfg.accounts = channelCfg.accounts || {};
+    channelCfg.accounts[accountId] = {
+      ...(channelCfg.accounts[accountId] || {}),
       enabled: true,
     };
     return next;