@xmoxmo/bncr 0.3.6 → 0.3.7

package/src/plugin/outbox-drain-ack.ts

@@ -0,0 +1,145 @@
+import type { BncrRoute, OutboxEntry } from '../core/types.ts';
+import {
+  buildOutboxAckDebugInfo,
+  buildOutboxScheduleDebugInfo,
+} from '../messaging/outbound/diagnostics.ts';
+import { computeOutboxRetryWait } from '../messaging/outbound/queue-selectors.ts';
+import {
+  OUTBOUND_SCHEDULE_SOURCE,
+  type OutboundScheduleSource,
+} from '../messaging/outbound/reasons.ts';
+import type { RetryRerouteDecision } from '../messaging/outbound/retry-policy.ts';
+import { applyBncrRetryRerouteDecisionToEntry } from '../runtime/outbox-transitions.ts';
+
+export type BncrOutboxDrainAckRuntime = {
+  bridgeId: string;
+  pushEvent: string;
+  now: () => number;
+  defaultAckTimeoutMs: number;
+  adaptiveAckTimeoutEnabled: boolean;
+  outboxSize: () => number;
+  formatDisplayScope: (route: BncrRoute) => string;
+  isFileTransferEntry: (entry: OutboxEntry) => boolean;
+  setOutboxEntry: (messageId: string, entry: OutboxEntry) => void;
+  scheduleSave: () => void;
+  moveToDeadLetter: (entry: OutboxEntry, reason: string) => void;
+  recordAckTimeoutTelemetry: (accountId: string) => void;
+  logInfo: (scope: string, message: string, options?: { debugOnly?: boolean }) => void;
+  logOutboxAckReroute: (args: {
+    accountId: string;
+    entry: OutboxEntry;
+    requireAck: boolean;
+    currentConnId: string;
+    availableConnIds: string[];
+    decision: Extract<RetryRerouteDecision, { kind: 'retry' }>;
+    localNextDelay: number | null;
+    ackTimeoutMs?: number | null;
+  }) => void;
+};
+
+export function createBncrOutboxDrainAck(runtime: BncrOutboxDrainAckRuntime) {
+  const logAckWaitStart = (args: {
+    entry: OutboxEntry;
+    requireAck: boolean;
+    ackTimeoutMs?: number | null;
+    onlineNow: boolean;
+    recentInboundReachable: boolean;
+  }) => {
+    runtime.logInfo(
+      'outbox',
+      `ack wait-start ${JSON.stringify(
+        buildOutboxAckDebugInfo({
+          messageId: args.entry.messageId,
+          accountId: args.entry.accountId,
+          sessionKey: args.entry.sessionKey,
+          to: runtime.formatDisplayScope(args.entry.route),
+          kind: runtime.isFileTransferEntry(args.entry) ? 'file-transfer' : undefined,
+          requireAck: args.requireAck,
+          ackResult: 'timeout',
+          ackStage: 'message',
+          ackOutcome: 'waiting',
+          ackTimeoutMs: args.ackTimeoutMs || runtime.defaultAckTimeoutMs,
+          adaptiveAckTimeoutEnabled: runtime.adaptiveAckTimeoutEnabled,
+          onlineNow: args.onlineNow,
+          recentInboundReachable: args.recentInboundReachable,
+          connIds: args.entry.lastPushConnId ? [args.entry.lastPushConnId] : [],
+          ownerConnId: args.entry.lastPushConnId,
+          ownerClientId: args.entry.lastPushClientId,
+          event: runtime.pushEvent,
+        }),
+      )}`,
+      { debugOnly: true },
+    );
+  };
+
+  const handleAckTimeoutReroute = (args: {
+    accountId: string;
+    entry: OutboxEntry;
+    requireAck: boolean;
+    currentConnId: string;
+    availableConnIds: string[];
+    decision:
+      | { kind: 'dead-letter'; terminalReason: string; [key: string]: unknown }
+      | Extract<RetryRerouteDecision, { kind: 'retry' }>;
+    localNextDelay: number | null;
+    ackTimeoutMs?: number | null;
+    updateMinOutboxDelay: (current: number | null, next: number | null) => number | null;
+  }) => {
+    if (args.decision.kind === 'dead-letter') {
+      runtime.logInfo(
+        'outbox ack fatal',
+        `mid=${args.entry.messageId}|q=${runtime.outboxSize()}|err=${args.decision.terminalReason}`,
+      );
+      runtime.moveToDeadLetter(args.entry, args.decision.terminalReason);
+      return { kind: 'dead-letter' as const, localNextDelay: args.localNextDelay };
+    }
+
+    const nextEntry = applyBncrRetryRerouteDecisionToEntry(args.entry, args.decision);
+    runtime.setOutboxEntry(args.entry.messageId, nextEntry);
+    runtime.scheduleSave();
+    if (args.requireAck) runtime.recordAckTimeoutTelemetry(args.accountId);
+    const wait = computeOutboxRetryWait(args.decision.nextAttemptAt, runtime.now());
+    const localNextDelay = args.updateMinOutboxDelay(args.localNextDelay, wait);
+    runtime.logOutboxAckReroute({
+      accountId: args.accountId,
+      entry: nextEntry,
+      requireAck: args.requireAck,
+      currentConnId: args.currentConnId,
+      availableConnIds: args.availableConnIds,
+      decision: args.decision,
+      localNextDelay,
+      ackTimeoutMs: args.ackTimeoutMs,
+    });
+    return { kind: 'retry' as const, nextEntry, wait, localNextDelay };
+  };
+
+  const logSchedule = (args: {
+    accountId: string;
+    messageId: string;
+    wait: number;
+    localNextDelay: number | null;
+    source: OutboundScheduleSource;
+  }) => {
+    runtime.logInfo(
+      'outbox',
+      `schedule ${JSON.stringify(
+        buildOutboxScheduleDebugInfo({
+          bridgeId: runtime.bridgeId,
+          accountId: args.accountId,
+          messageId: args.messageId,
+          source: args.source,
+          wait: args.wait,
+          localNextDelay: args.localNextDelay,
+        }),
+      )}`,
+      { debugOnly: true },
+    );
+  };
+
+  return {
+    logAckWaitStart,
+    handleAckTimeoutReroute,
+    logSchedule,
+    scheduleSource: OUTBOUND_SCHEDULE_SOURCE,
+  };
+}

package/src/plugin/outbox-drain-failure.ts

@@ -0,0 +1,84 @@
+import type { OutboxEntry } from '../core/types.ts';
+import { computeOutboxRetryWait } from '../messaging/outbound/queue-selectors.ts';
+import { OUTBOUND_SCHEDULE_SOURCE } from '../messaging/outbound/reasons.ts';
+import { computePushFailureDecision } from '../messaging/outbound/retry-policy.ts';
+import { applyBncrPushFailureDecisionToEntry } from '../runtime/outbox-transitions.ts';
+
+type UpdateMinOutboxDelay = (current: number | null, candidate: number | null) => number | null;
+
+type BncrOutboxDrainScheduleRuntime = {
+  scheduleAccountWait: (args: {
+    accountId: string;
+    messageId?: string;
+    source: string;
+    wait: number;
+    localNextDelay: number | null;
+    updateMinOutboxDelay: UpdateMinOutboxDelay;
+  }) => number | null;
+};
+
+type BncrOutboxDrainFailureRuntime = {
+  backoffMs: (retryCount: number) => number;
+  outbox: Map<string, OutboxEntry>;
+  isPrePushGuardDeferral: (entry: OutboxEntry) => boolean;
+  moveToDeadLetter: (entry: OutboxEntry, reason: string) => void;
+  scheduleSave: () => void;
+  outboxDrainSchedule: BncrOutboxDrainScheduleRuntime;
+  maxRetry: number;
+  prePushGuardRetryDelayMs: number;
+};
+
+export function createBncrOutboxDrainFailure(runtime: BncrOutboxDrainFailureRuntime) {
+  return function handleFailedDrainEntry(args: {
+    accountId: string;
+    entry: OutboxEntry;
+    localNextDelay: number | null;
+    attemptedAt: number;
+    updateMinOutboxDelay: UpdateMinOutboxDelay;
+  }): { action: 'continue' | 'break'; localNextDelay: number | null } {
+    const { accountId, entry, attemptedAt, updateMinOutboxDelay } = args;
+    let { localNextDelay } = args;
+
+    if (runtime.isPrePushGuardDeferral(entry)) {
+      const wait = runtime.prePushGuardRetryDelayMs;
+      localNextDelay = runtime.outboxDrainSchedule.scheduleAccountWait({
+        accountId,
+        messageId: entry.messageId,
+        source: OUTBOUND_SCHEDULE_SOURCE.PRE_PUSH_GUARD_WAIT,
+        wait,
+        localNextDelay,
+        updateMinOutboxDelay,
+      });
+      return { action: 'break', localNextDelay };
+    }
+
+    const decision = computePushFailureDecision(
+      {
+        nowMs: attemptedAt,
+        maxRetry: runtime.maxRetry,
+        currentRetryCount: entry.retryCount,
+        lastError: entry.lastError,
+      },
+      { backoffMs: runtime.backoffMs },
+    );
+    if (decision.kind === 'dead-letter') {
+      runtime.moveToDeadLetter(entry, decision.terminalReason);
+      return { action: 'continue', localNextDelay };
+    }
+
+    const nextEntry = applyBncrPushFailureDecisionToEntry(entry, decision);
+    runtime.outbox.set(entry.messageId, nextEntry);
+    runtime.scheduleSave();
+
+    const wait = computeOutboxRetryWait(decision.nextAttemptAt, attemptedAt);
+    localNextDelay = runtime.outboxDrainSchedule.scheduleAccountWait({
+      accountId,
+      messageId: entry.messageId,
+      source: OUTBOUND_SCHEDULE_SOURCE.PUSH_FAIL_WAIT,
+      wait,
+      localNextDelay,
+      updateMinOutboxDelay,
+    });
+    return { action: 'break', localNextDelay };
+  };
+}