@kronos-ts/axon-server 0.5.0 → 0.7.0

package/src/platform-service.ts

@@ -105,7 +105,16 @@ export type PlatformConnection = {
 }
 
 export type PlatformServiceOptions = {
-  /** Heartbeat interval in ms. Default: 10000 */
+  /**
+   * Heartbeat interval in ms. Default: 2500.
+   *
+   * Axon Server's `client-heartbeat-timeout` defaults to 5000 ms and is
+   * refreshed ONLY by heartbeats the client sends (`HeartbeatMonitor`), checked
+   * every second. A 10 s cadence — Axon Framework's own default — survives
+   * there only because the framework also answers every heartbeat the SERVER
+   * sends; this client does both, and beats at half the server's window so a
+   * paused runner still fits.
+   */
   heartbeatIntervalMs?: number
   /** Heartbeat timeout in ms. If no response within this window, reconnect. Default: 7500 */
   heartbeatTimeoutMs?: number
@@ -133,7 +142,7 @@ export function platformConnection(
   connection: AxonServerConnection,
   options?: PlatformServiceOptions,
 ): PlatformConnection {
-  const heartbeatIntervalMs = options?.heartbeatIntervalMs ?? 10000
+  const heartbeatIntervalMs = options?.heartbeatIntervalMs ?? 2500
   const heartbeatTimeoutMs = options?.heartbeatTimeoutMs ?? 7500
   const processorsNotificationRateMs = options?.processorsNotificationRateMs ?? 500
   const processorsNotificationInitialDelayMs = options?.processorsNotificationInitialDelayMs ?? 5000
@@ -154,12 +163,22 @@ export function platformConnection(
    */
   const pendingInstructions: PlatformInstruction[] = []
   let isConnected = false
+  let monitoringArmed = false
+  let recoveryTimer: ReturnType<typeof setTimeout> | undefined
   let heartbeatTimer: ReturnType<typeof setInterval> | null = null
   let heartbeatTimeoutTimer: ReturnType<typeof setTimeout> | null = null
+  let processorStatusInitialTimer: ReturnType<typeof setTimeout> | undefined
   let processorStatusTimer: ReturnType<typeof setInterval> | null = null
   /** Guards against arming the status-report timer twice — see `startProcessorStatusReporting`. */
   let processorStatusArmed = false
   let lastHeartbeatResponse = Date.now()
+  // Whether this server has EVER sent a heartbeat. Axon Server only beats at
+  // clients whose registered framework version it recognises (4.2.1+); a client
+  // it does not beat at cannot be judged by the silence — that judgement fell
+  // every window and tore down a healthy stream, every 10 s, in production.
+  // Dead-channel detection for a server that never beats is gRPC keepalive's
+  // job (see `connection.ts`), not this timer's.
+  let serverBeats = false
   let outbound: ReturnType<typeof outboundStream<PlatformInboundInstruction>> | null = null
   /**
    * Latches once Axon Server sends its first inbound message after
@@ -176,13 +195,37 @@ export function platformConnection(
     grpcMetadata.set("AxonIQ-Access-Token", connection.config.token)
   }
 
-  async function processInboundInstructions(inbound: AsyncIterable<any>) {
+  function scheduleRecovery() {
+    if (!monitoringArmed || recoveryTimer || connection.state === "closed" || connection.state === "reconnecting") return
+    recoveryTimer = setTimeout(() => {
+      recoveryTimer = undefined
+      if (!monitoringArmed || connection.state === "closed") return
+      void connection.reconnect().catch((error) => console.error("Platform recovery failed", error))
+    }, connection.config.reconnectIntervalMs ?? 2000)
+  }
+
+  connection.onDisconnect?.(() => {
+    isConnected = false
+    outbound?.close()
+  })
+
+  connection.onReconnect(() => {
+    if (!monitoringArmed) return
+    clearTimeout(recoveryTimer)
+    recoveryTimer = undefined
+    isConnected = false
+    openPlatformStream()
+  })
+
+  async function processInboundInstructions(inbound: AsyncIterable<any>, stream: typeof outbound) {
     try {
       for await (const message of inbound) {
+        if (stream !== outbound || !monitoringArmed) return
         // First inbound message after start() = the platform has accepted our
         // registration and is talking back. Latch the ack flag (mirror of
         // kronosdb Plan 09-03 / D-102 — replaces the legacy 1s sleep).
         acked = true
+        if (message.requestReconnect) scheduleRecovery()
         // Parse instruction type
         const instruction = parseInstruction(message)
         if (instruction) {
@@ -200,16 +243,27 @@ export function platformConnection(
           }
         }
 
-        // Handle heartbeat response — track last response time for timeout detection
+        // A heartbeat from the server is BOTH the liveness signal this side
+        // tracks and a request: Axon Server measures client inactivity by the
+        // heartbeats it receives, so every one it sends is answered at once
+        // (what Axon Framework's connector does). Without the echo, the only
+        // activity the server ever sees is the periodic beat below.
         if (message.heartbeat) {
+          serverBeats = true
           lastHeartbeatResponse = Date.now()
+          outbound?.send({ heartbeat: { clientId: connection.config.clientId }, instructionId: "" })
         }
       }
     } catch (err) {
-      if (isConnected) {
+      if (stream === outbound && monitoringArmed && isConnected) {
         console.error("Platform stream error:", err)
         isConnected = false
       }
+    } finally {
+      if (stream === outbound && monitoringArmed) {
+        isConnected = false
+        scheduleRecovery()
+      }
     }
   }
 
@@ -266,10 +320,13 @@ export function platformConnection(
 
     heartbeatTimer = setInterval(() => {
       if (!isConnected || !outbound) return
+      // Not before the server has acknowledged the registration: a heartbeat
+      // from a stream it has not filed yet is an error on its side, not a beat.
+      if (!acked) return
 
-      // Check if last heartbeat response was too long ago
+      // Judge silence only from a server that has beaten before.
       const timeSinceLastResponse = Date.now() - lastHeartbeatResponse
-      if (timeSinceLastResponse > heartbeatTimeoutMs) {
+      if (serverBeats && timeSinceLastResponse > heartbeatTimeoutMs) {
         console.warn(
           `Platform heartbeat timeout: no response in ${timeSinceLastResponse}ms ` +
           `(threshold: ${heartbeatTimeoutMs}ms). Marking connection as lost.`,
@@ -302,9 +359,9 @@ export function platformConnection(
     if (processorStatusTimer) clearInterval(processorStatusTimer)
 
     // Initial delay before first report
-    setTimeout(() => {
-      if (!isConnected) return
-      reportProcessorStatus()
+    processorStatusInitialTimer = setTimeout(() => {
+      if (!monitoringArmed) return
+      if (isConnected) void reportProcessorStatus()
 
       // Then report at the configured rate
       processorStatusTimer = setInterval(() => {
@@ -352,6 +409,7 @@ export function platformConnection(
 
     // Re-arm the ack latch so a stop/start cycle correctly re-waits.
     acked = false
+    serverBeats = false
     outbound = outboundStream<PlatformInboundInstruction>()
 
     // Register with Axon Server
@@ -375,7 +433,7 @@ export function platformConnection(
     // timeout, which is what the command/query buses hang their stream
     // re-establishment off. It belongs to every service, administered or not.
     startHeartbeat()
-    processInboundInstructions(inbound)
+    void processInboundInstructions(inbound, outbound)
 
     // Axon Server's PlatformService does NOT proactively emit an inbound
     // frame in response to `register` — the stream is held open silently
@@ -399,15 +457,21 @@ export function platformConnection(
 
   return {
     async armConnectionMonitoring() {
+      monitoringArmed = true
       openPlatformStream()
     },
 
     async start() {
+      monitoringArmed = true
       openPlatformStream()
       startProcessorStatusReporting()
     },
 
     stop() {
+      monitoringArmed = false
+      clearTimeout(processorStatusInitialTimer)
+      clearTimeout(recoveryTimer)
+      recoveryTimer = undefined
       isConnected = false
       // A stopped stream's un-routed backlog is stale — do not replay it if a
       // handler registers later.

package/src/resilience.ts

@@ -39,9 +39,11 @@ export type ResilienceConfig = {
   log?: (msg: string) => void
   /** Per-attempt classification: returns false to short-circuit (terminal error). */
   isRetryable?: (err: unknown) => boolean
+  /** How a retry waits. Defaults to `setTimeout`; a test injects one to observe delays without touching globals. */
+  sleep?: (ms: number) => Promise<void>
 }
 
-const DEFAULTS: Omit<ResilienceConfig, "log" | "isRetryable"> = {
+const DEFAULTS: Omit<ResilienceConfig, "log" | "isRetryable" | "sleep"> = {
   initialDelayMs: 100,
   maxDelayMs: 30_000,
   maxAttempts: 10,
@@ -105,12 +107,14 @@ export async function withRetry<T>(
             (err as Error)?.message ?? String(err)
           }`,
       )
-      await new Promise((r) => setTimeout(r, delay))
+      await (cfg.sleep ?? defaultSleep)(delay)
     }
   }
   throw lastErr
 }
 
+const defaultSleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms))
+
 /**
  * Run a health-check probe with warn-then-continue semantics (D-100).
  *

package/src/shutdown-latch.ts

@@ -15,6 +15,7 @@ export type ShutdownLatch = {
    * Register an in-flight activity. Throws if shutdown is in progress.
    * Call `end()` on the returned handle when the activity completes.
    */
+  onShutdown(callback: () => void): () => void
   registerActivity(): ActivityHandle
 
   /**
@@ -44,8 +45,10 @@ export class ShutdownInProgressError extends Error {
 }
 
 export function shutdownLatch(): ShutdownLatch {
+  const callbacks = new Set<() => void>()
   let activeCount = 0
   let shuttingDown = false
+  let drainPromise: Promise<void> | undefined
   let drainResolve: (() => void) | null = null
 
   function checkDrained() {
@@ -56,6 +59,11 @@ export function shutdownLatch(): ShutdownLatch {
   }
 
   return {
+    onShutdown(callback) {
+      if (shuttingDown) callback()
+      else callbacks.add(callback)
+      return () => { callbacks.delete(callback) }
+    },
     registerActivity(): ActivityHandle {
       if (shuttingDown) {
         throw new ShutdownInProgressError()
@@ -76,14 +84,19 @@ export function shutdownLatch(): ShutdownLatch {
 
     initiateShutdown(): Promise<void> {
       shuttingDown = true
+      for (const callback of callbacks) {
+        try { callback() } catch (error) { console.error("Messaging shutdown callback failed", error) }
+      }
+      callbacks.clear()
 
       if (activeCount === 0) {
         return Promise.resolve()
       }
 
-      return new Promise((resolve) => {
+      drainPromise ??= new Promise((resolve) => {
         drainResolve = resolve
       })
+      return drainPromise
     },
 
     get shuttingDown() {

package/src/stream-recovery.ts

@@ -0,0 +1,72 @@
+import type { ResilienceConfig } from "./resilience.js"
+
+/** Retry failures of the async stream, not just synchronous stream creation. */
+export function streamRecovery(
+  reopen: () => void,
+  canReconnect: () => boolean,
+  config: Partial<ResilienceConfig> = {},
+) {
+  const log = (message: string) => {
+    try {
+      ;(config.log ?? console.warn)(message)
+    } catch {
+      /* Diagnostics cannot prevent recovery. */
+    }
+  }
+  let timer: ReturnType<typeof setTimeout> | undefined
+  let stopped = false
+  let attempts = 0
+  let openedAt = Date.now()
+
+  function failed(error: unknown) {
+    if (stopped || timer || !canReconnect()) return
+    if (config.isRetryable?.(error) === false || attempts >= (config.maxAttempts ?? 30)) {
+      log(`Provider stream recovery exhausted: ${String(error)}`)
+      return
+    }
+    // Keep a positive floor even when jitter is zero. An immediately-ended
+    // async generator must never create a microtask reconnect loop.
+    const cap = Math.min(
+      (config.initialDelayMs ?? 100) * (config.multiplier ?? 2) ** attempts++,
+      config.maxDelayMs ?? 30000,
+    )
+    const delay = Math.max(1, cap * (0.5 + Math.random() * 0.5))
+    log(`Provider stream failed; reconnecting in ${Math.round(delay)}ms: ${String(error)}`)
+    timer = setTimeout(() => {
+      timer = undefined
+      if (stopped || !canReconnect()) return
+      openedAt = Date.now()
+      try {
+        reopen()
+      } catch (error) {
+        failed(error)
+      }
+    }, delay)
+    timer.unref?.()
+  }
+
+  return {
+    failed,
+    received() {
+      // Subscription acknowledgements alone do not prove a stable connection.
+      if (Date.now() - openedAt >= 30000) attempts = 0
+    },
+    restart() {
+      if (stopped || !canReconnect()) return
+      clearTimeout(timer)
+      timer = undefined
+      attempts = 0
+      openedAt = Date.now()
+      try {
+        reopen()
+      } catch (error) {
+        failed(error)
+      }
+    },
+    stop() {
+      stopped = true
+      clearTimeout(timer)
+      timer = undefined
+    },
+  }
+}