@opengeni/events 0.4.30 → 0.4.32-canary.36199476632001

package/dist/index.d.ts

@@ -9,6 +9,14 @@ export type EventBusOptions = {
     logger?: EventLogger;
     /** Test/host transport seam; production defaults to the nats.js connector. */
     connect?: typeof connect;
+    /**
+     * Metrics seam: called once for every subscription that ends without being
+     * unsubscribed (see {@link natsSubscriptionTerminationCounter}). A throwing
+     * sink is ignored.
+     */
+    onSubscriptionTerminated?: (termination: NatsSubscriptionTermination) => void;
+    /** Test seam for the responder resubscribe backoff; production uses {@link natsResubscribeDelayMs}. */
+    resubscribeDelayMs?: (attempt: number) => number;
 };
 export { SESSION_EVENT_COALESCED_TEXT_TARGET_BYTES, coalesceSessionEventDeltas, coalesceSessionEventDeltasWithCoverage, type CoalescedSessionEventPage, } from "./coalesce.js";
 /** Comfortably below NATS Core's common 1 MiB max_payload default. */
@@ -21,6 +29,54 @@ export declare const SESSION_EVENT_HTTP_PAGE_MAX_BYTES: number;
 export declare const WORKSPACE_CONTROL_NATS_MESSAGE_MAX_BYTES: number;
 /** Count+byte envelope for one workspace-control REST replay page. */
 export declare const WORKSPACE_CONTROL_HTTP_PAGE_MAX_BYTES: number;
+/**
+ * Closed label set for {@link NatsSubscriptionTermination}. Each value names one
+ * subscription role this package owns; no subject, tenant, or payload
+ * dimension is ever admitted, so a metric sink can use them as labels.
+ */
+export declare const NATS_SUBSCRIPTION_KINDS: readonly ["session_events", "workspace_control", "request_responder", "agent_events", "auth_callout"];
+export type NatsSubscriptionKind = (typeof NATS_SUBSCRIPTION_KINDS)[number];
+/**
+ * What happens after an unexpected end: a per-consumer subscription tells its
+ * consumer (`consumer_reconnect`, e.g. session SSE fails retryably and the
+ * client replays from Postgres), a process-lifetime responder resubscribes with
+ * bounded backoff (`resubscribe`), and a consumer that registered no hook only
+ * relies on its own durable re-read (`none`).
+ */
+export declare const NATS_SUBSCRIPTION_RECOVERIES: readonly ["consumer_reconnect", "resubscribe", "none"];
+export type NatsSubscriptionRecovery = (typeof NATS_SUBSCRIPTION_RECOVERIES)[number];
+export type NatsSubscriptionTermination = {
+    kind: NatsSubscriptionKind;
+    recovery: NatsSubscriptionRecovery;
+};
+/** Prometheus counter name for {@link natsSubscriptionTerminationCounter}. */
+export declare const NATS_SUBSCRIPTION_TERMINATIONS_METRIC = "opengeni_nats_subscription_terminations_total";
+/**
+ * Adapt a counter sink (the API/worker `Observability`) into the
+ * `onSubscriptionTerminated` hook. Labels are the closed kind/recovery sets.
+ */
+export declare function natsSubscriptionTerminationCounter(sink: {
+    incrementCounter(input: {
+        name: string;
+        help?: string;
+        labels?: Record<string, string>;
+    }): void;
+}): (termination: NatsSubscriptionTermination) => void;
+/** Per-subscription consumer options for {@link EventBus.subscribe}. */
+export type BusSubscriptionOptions = {
+    /**
+     * Called at most once when the live transport subscription ends while the
+     * consumer still holds it (a broker-side error such as a permissions
+     * violation, or an iterator end the consumer did not request). Live events
+     * stop after this call, so the consumer must reconcile from Postgres: session
+     * and workspace-control SSE fail the stream retryably and the client
+     * reconnects and replays. Never called after the returned unsubscribe runs or
+     * while the bus itself is closing.
+     */
+    onTerminated?: (error: unknown) => void;
+};
+/** Bounded resubscribe backoff for process-lifetime responders: 1 s doubling to 30 s, plus up to 1 s jitter. */
+export declare function natsResubscribeDelayMs(attempt: number, random?: () => number): number;
 export { decodeAuthRequest, mintAuthResponse, mintUserJwt, parseAgentConnectionName, workspaceAgentPermissions, type DecodedAuthRequest, type MintAuthResponseInput, type MintUserJwtInput, type NatsPermission, type NatsPermissions, } from "./nats-jwt.js";
 export { connect, nkeys, type NatsConnection } from "nats";
 /**
@@ -107,11 +163,18 @@ export type EventBus = {
      * publish promise when this capability is unavailable.
      */
     publishConfirmed?: (workspaceId: string, sessionId: string, events: SessionEvent[]) => Promise<void>;
-    subscribe: (workspaceId: string, sessionId: string, onEvents: (events: SessionEvent[]) => void | Promise<void>) => Promise<() => void>;
+    /**
+     * Live session-event fanout. `options.onTerminated` reports a transport
+     * subscription that ended while still held; implementations whose
+     * subscriptions can end on their own should call it so the consumer
+     * reconciles from Postgres. Buses that never end a held subscription may
+     * ignore it.
+     */
+    subscribe: (workspaceId: string, sessionId: string, onEvents: (events: SessionEvent[]) => void | Promise<void>, options?: BusSubscriptionOptions) => Promise<() => void>;
     /** Best-effort live invalidation; the event is already durable in Postgres. */
     publishWorkspaceControl: (workspaceId: string, event: WorkspaceControlEvent) => Promise<void>;
     /** One workspace subscription fans a control change to every open descendant view. */
-    subscribeWorkspaceControl: (workspaceId: string, onEvent: (event: WorkspaceControlEvent) => void | Promise<void>) => Promise<() => void>;
+    subscribeWorkspaceControl: (workspaceId: string, onEvent: (event: WorkspaceControlEvent) => void | Promise<void>, options?: BusSubscriptionOptions) => Promise<() => void>;
     /**
      * Issue a binary request/reply on a subject over the bus's NATS connection
      * (the selfhosted control plane's exact claimed process subject). A new usage of what was
@@ -208,6 +271,8 @@ export declare function createResponderConnection(natsUrl: string, auth: NatsCon
     name?: string;
     logger?: EventLogger;
     connect?: typeof connect;
+    onSubscriptionTerminated?: (termination: NatsSubscriptionTermination) => void;
+    resubscribeDelayMs?: (attempt: number) => number;
 }): Promise<ResponderConnection>;
 /**
  * Optional timing seam for {@link appendAndPublishEvents}: `onAppend` fires after

package/dist/index.js

@@ -316,6 +316,7 @@ var silentLogger = {
 var RECONNECT_OPTIONS = {
   reconnect: true,
   maxReconnectAttempts: -1,
+  ignoreAuthErrorAbort: true,
   reconnectTimeWait: 2e3,
   reconnectJitter: 1e3,
   reconnectJitterTLS: 1e3,
@@ -380,6 +381,165 @@ function logConnectionStatus(nc, label, logger = silentLogger, onStatus) {
 function isWarnNatsStatus(type) {
   return type === "disconnect" || type === "error" || type === "staleConnection";
 }
+function subscriptionFailureAttributes(error) {
+  if (!error || typeof error !== "object") return { errorType: typeof error };
+  try {
+    const candidate = error;
+    return {
+      errorName: typeof candidate.name === "string" ? candidate.name.slice(0, 64) : "unknown",
+      ...typeof candidate.code === "string" ? { errorCode: candidate.code.slice(0, 64) } : {}
+    };
+  } catch {
+    return { errorName: "unknown" };
+  }
+}
+var NATS_SUBSCRIPTION_KINDS = [
+  "session_events",
+  "workspace_control",
+  "request_responder",
+  "agent_events",
+  "auth_callout"
+];
+var NATS_SUBSCRIPTION_RECOVERIES = ["consumer_reconnect", "resubscribe", "none"];
+var NATS_SUBSCRIPTION_TERMINATIONS_METRIC = "opengeni_nats_subscription_terminations_total";
+function natsSubscriptionTerminationCounter(sink) {
+  return ({ kind, recovery }) => {
+    sink.incrementCounter({
+      name: NATS_SUBSCRIPTION_TERMINATIONS_METRIC,
+      help: "NATS subscriptions that ended without being unsubscribed, by subscription kind and recovery path.",
+      labels: { kind, recovery }
+    });
+  };
+}
+function natsResubscribeDelayMs(attempt, random = Math.random) {
+  const exponent = Number.isFinite(attempt) ? Math.min(Math.max(0, Math.trunc(attempt) - 1), 5) : 0;
+  const sample = random();
+  const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
+  return Math.min(3e4, 1e3 * 2 ** exponent) + Math.floor(unit * 1e3);
+}
+function pumpSubscription(subscription, context, onMessage, onEnded) {
+  const warn = context.logger?.warn ?? silentLogger.warn;
+  void (async () => {
+    let end = {};
+    try {
+      for await (const message of subscription) {
+        try {
+          await onMessage(message);
+        } catch (error) {
+          try {
+            warn("NATS subscription message dropped", {
+              label: context.label,
+              subject: context.subject,
+              ...subscriptionFailureAttributes(error)
+            });
+          } catch {
+          }
+        }
+      }
+    } catch (error) {
+      end = { error };
+    }
+    try {
+      onEnded(end);
+    } catch {
+    }
+  })();
+}
+function reportSubscriptionTermination(context, recovery, end) {
+  try {
+    (context.logger?.warn ?? silentLogger.warn)(
+      "error" in end && end.error !== void 0 ? "NATS subscription ended with an error" : "NATS subscription ended unexpectedly",
+      {
+        label: context.label,
+        subject: context.subject,
+        recovery,
+        ..."error" in end && end.error !== void 0 ? subscriptionFailureAttributes(end.error) : {}
+      }
+    );
+  } catch {
+  }
+  try {
+    context.onSubscriptionTerminated?.({ kind: context.kind, recovery });
+  } catch {
+  }
+}
+function connectionShuttingDown(nc) {
+  return nc.isClosed() || nc.isDraining();
+}
+function subscribeWithTermination(nc, context, onMessage, options) {
+  const sub = nc.subscribe(context.subject);
+  let released = false;
+  const onTerminated = options?.onTerminated;
+  pumpSubscription(sub, context, onMessage, (end) => {
+    if (released || connectionShuttingDown(nc)) return;
+    reportSubscriptionTermination(context, onTerminated ? "consumer_reconnect" : "none", end);
+    try {
+      onTerminated?.("error" in end ? end.error : void 0);
+    } catch {
+    }
+  });
+  return () => {
+    released = true;
+    sub.unsubscribe();
+  };
+}
+function subscribeWithResubscribe(nc, context, onMessage, resubscribeDelayMs = natsResubscribeDelayMs) {
+  let stopped = false;
+  let current = null;
+  let timer = null;
+  let consecutiveFailures = 0;
+  const pump = (sub) => {
+    current = sub;
+    pumpSubscription(
+      sub,
+      context,
+      async (message) => {
+        consecutiveFailures = 0;
+        await onMessage(message);
+      },
+      (end) => {
+        if (current === sub) current = null;
+        if (stopped || connectionShuttingDown(nc)) return;
+        reportSubscriptionTermination(context, "resubscribe", end);
+        schedule();
+      }
+    );
+  };
+  const schedule = () => {
+    if (stopped || connectionShuttingDown(nc)) return;
+    consecutiveFailures += 1;
+    let delayMs = 3e4;
+    try {
+      delayMs = resubscribeDelayMs(consecutiveFailures);
+    } catch {
+    }
+    timer = setTimeout(() => {
+      timer = null;
+      if (stopped || connectionShuttingDown(nc)) return;
+      let sub;
+      try {
+        sub = nc.subscribe(context.subject);
+      } catch (error) {
+        reportSubscriptionTermination(context, "resubscribe", { error });
+        schedule();
+        return;
+      }
+      pump(sub);
+    }, delayMs);
+    timer.unref?.();
+  };
+  pump(nc.subscribe(context.subject));
+  return () => {
+    stopped = true;
+    if (timer) {
+      clearTimeout(timer);
+      timer = null;
+    }
+    const sub = current;
+    current = null;
+    sub?.unsubscribe();
+  };
+}
 var SESSION_EVENT_DURABLE_FANOUT_CAPABILITY_VERSION = 1;
 function requireSessionEventDurableFanoutCapability(bus) {
   const capability = bus?.sessionEventDurableFanout;
@@ -484,7 +644,7 @@ async function createNatsEventBus(natsUrl, auth, options = {}) {
       publishSessionEvents(workspaceId, sessionId, events);
       await flushConfirmedWithTimeout(nc, PUBLISH_FLUSH_TIMEOUT_MS);
     },
-    subscribe: async (workspaceId, sessionId, onEvents) => subscribeSession(nc, workspaceId, sessionId, onEvents),
+    subscribe: async (workspaceId, sessionId, onEvents, subscriptionOptions) => subscribeSession(nc, workspaceId, sessionId, onEvents, options, subscriptionOptions),
     publishWorkspaceControl: async (workspaceId, event) => {
       try {
         const encoded = workspaceControlEventNatsPayload(event);
@@ -502,22 +662,27 @@ async function createNatsEventBus(natsUrl, auth, options = {}) {
       }
       await flushWithTimeout(nc, PUBLISH_FLUSH_TIMEOUT_MS);
     },
-    subscribeWorkspaceControl: async (workspaceId, onEvent) => {
-      const sub = nc.subscribe(workspaceControlSubject(workspaceId));
-      void (async () => {
-        for await (const msg of sub) {
-          await onEvent(
-            boundWorkspaceControlEvent(codec.decode(msg.data), {
-              surface: "nats_legacy_guard"
-            })
-          );
-        }
-      })();
-      return () => sub.unsubscribe();
-    },
+    subscribeWorkspaceControl: async (workspaceId, onEvent, subscriptionOptions) => subscribeWithTermination(
+      nc,
+      {
+        label: "workspace-control",
+        kind: "workspace_control",
+        subject: workspaceControlSubject(workspaceId),
+        logger: options.logger,
+        onSubscriptionTerminated: options.onSubscriptionTerminated
+      },
+      async (msg) => {
+        await onEvent(
+          boundWorkspaceControlEvent(codec.decode(msg.data), {
+            surface: "nats_legacy_guard"
+          })
+        );
+      },
+      subscriptionOptions
+    ),
     request: async (subject, payload, opts) => requestReply(nc, subject, payload, opts.timeoutMs),
-    subscribeRequests: (subject, handler) => subscribeRequests(nc, subject, handler),
-    subscribeAgentEvents: (subject, handler) => subscribeAgentEvents(nc, subject, handler),
+    subscribeRequests: (subject, handler) => subscribeRequests(nc, subject, handler, options),
+    subscribeAgentEvents: (subject, handler) => subscribeAgentEvents(nc, subject, handler, options),
     getRequestConnection: () => requestConnection,
     getOpStreamConnection: () => opStreamConnection,
     isConnected: () => connected && !nc.isClosed() && !nc.isDraining(),
@@ -544,22 +709,30 @@ async function createResponderConnection(natsUrl, auth, subject, handler, option
     options.name ? `auth-callout:${options.name}` : "auth-callout",
     options.logger
   );
-  const sub = nc.subscribe(subject);
-  void (async () => {
-    for await (const msg of sub) {
+  const unsubscribe = subscribeWithResubscribe(
+    nc,
+    {
+      label: options.name ? `auth-callout:${options.name}` : "auth-callout",
+      kind: "auth_callout",
+      subject,
+      logger: options.logger,
+      onSubscriptionTerminated: options.onSubscriptionTerminated
+    },
+    async (msg) => {
       if (!msg.reply) {
-        continue;
+        return;
       }
       try {
         const reply = await handler(msg.data, msg.subject);
         msg.respond(reply);
       } catch {
       }
-    }
-  })();
+    },
+    options.resubscribeDelayMs
+  );
   return {
     close: async () => {
-      sub.unsubscribe();
+      unsubscribe();
       await nc.drain();
     }
   };
@@ -647,56 +820,71 @@ async function appendAndPublishTurnEventsFenced(db, bus, workspaceId, sessionId,
   observeSince(observe?.onPublish, publishStartedAt, result.events.length);
   return result;
 }
-function subscribeSession(nc, workspaceId, sessionId, onEvents) {
-  const sub = nc.subscribe(sessionSubject(workspaceId, sessionId));
-  void (async () => {
-    for await (const msg of sub) {
+function subscribeSession(nc, workspaceId, sessionId, onEvents, busOptions, subscriptionOptions) {
+  return subscribeWithTermination(
+    nc,
+    {
+      label: "session-events",
+      kind: "session_events",
+      subject: sessionSubject(workspaceId, sessionId),
+      logger: busOptions.logger,
+      onSubscriptionTerminated: busOptions.onSubscriptionTerminated
+    },
+    async (msg) => {
       const decoded = codec.decode(msg.data);
       const events = ("events" in decoded ? decoded.events : [decoded]).map(
         (event) => boundSessionEventForSurface(event, "nats_legacy_guard")
       );
       await onEvents(events);
-    }
-  })();
-  return () => {
-    sub.unsubscribe();
-  };
+    },
+    subscriptionOptions
+  );
 }
 async function requestReply(nc, subject, payload, timeout) {
   const msg = await nc.request(subject, payload, { timeout });
   return { data: msg.data };
 }
-function subscribeRequests(nc, subject, handler) {
-  const sub = nc.subscribe(subject);
-  void (async () => {
-    for await (const msg of sub) {
+function subscribeRequests(nc, subject, handler, busOptions) {
+  return subscribeWithResubscribe(
+    nc,
+    {
+      label: "request-responder",
+      kind: "request_responder",
+      subject,
+      logger: busOptions.logger,
+      onSubscriptionTerminated: busOptions.onSubscriptionTerminated
+    },
+    async (msg) => {
       if (!msg.reply) {
-        continue;
+        return;
       }
       try {
         const reply = await handler(msg.data, msg.subject);
         msg.respond(reply);
       } catch {
       }
-    }
-  })();
-  return () => {
-    sub.unsubscribe();
-  };
+    },
+    busOptions.resubscribeDelayMs
+  );
 }
-function subscribeAgentEvents(nc, subject, handler) {
-  const sub = nc.subscribe(subject);
-  void (async () => {
-    for await (const msg of sub) {
+function subscribeAgentEvents(nc, subject, handler, busOptions) {
+  return subscribeWithResubscribe(
+    nc,
+    {
+      label: "agent-events",
+      kind: "agent_events",
+      subject,
+      logger: busOptions.logger,
+      onSubscriptionTerminated: busOptions.onSubscriptionTerminated
+    },
+    async (msg) => {
       try {
         await handler(msg.data, msg.subject);
       } catch {
       }
-    }
-  })();
-  return () => {
-    sub.unsubscribe();
-  };
+    },
+    busOptions.resubscribeDelayMs
+  );
 }
 function formatSse(event, idSequence = event.sequence) {
   const trustedId = Number.isSafeInteger(idSequence) && idSequence >= event.sequence ? idSequence : event.sequence;
@@ -853,6 +1041,9 @@ function workspaceControlSubject(workspaceId) {
   return `workspaces.${workspaceId}.control`;
 }
 export {
+  NATS_SUBSCRIPTION_KINDS,
+  NATS_SUBSCRIPTION_RECOVERIES,
+  NATS_SUBSCRIPTION_TERMINATIONS_METRIC,
   SESSION_EVENT_COALESCED_TEXT_TARGET_BYTES,
   SESSION_EVENT_DURABLE_FANOUT_CAPABILITY_VERSION,
   SESSION_EVENT_HTTP_PAGE_MAX_BYTES,
@@ -875,6 +1066,8 @@ export {
   formatWorkspaceControlEventSse,
   mintAuthResponse,
   mintUserJwt,
+  natsResubscribeDelayMs,
+  natsSubscriptionTerminationCounter,
   nkeys2 as nkeys,
   observeSince,
   parseAgentConnectionName,