@open-mercato/events 0.6.8-develop.6917.1.af45bc96e2 → 0.6.8-develop.6924.1.a8d208fcdc

package/src/bus.ts

@@ -19,6 +19,7 @@ import type {
   SubscriberDescriptor,
   EventPayload,
   EmitOptions,
+  QueuedDispatchResult,
 } from './types'
 
 /** Queue name for persistent events */
@@ -38,10 +39,16 @@ type RegisteredSubscriber = {
  * When enabled, a persistent emit delivers each subscriber on exactly one path:
  * persistent-marked subscribers are skipped inline (the events worker dispatches
  * them via pattern match, so wildcard persistent subscribers are reached), while
- * ephemeral subscribers keep running inline. Defaults ON; the server bootstrap
- * reconciles the env against worker availability (and may rewrite it to `false`
- * for a worker-less process). Both the bus and the events worker read the same
- * (possibly reconciled) env var, so they always agree within a process.
+ * ephemeral subscribers keep running inline. Defaults ON.
+ *
+ * The bus never second-guesses this against "is a worker running": the queue is
+ * durable, so a persistent emit with no worker yet is delayed, not lost, and a
+ * worker started later drains the backlog. Delivering inline instead would move
+ * the work back onto the caller's request path — exactly what a split app/worker
+ * deployment sets `AUTO_SPAWN_WORKERS=false` to avoid.
+ *
+ * The events worker asks this same bus for its subscribers via `dispatchQueued`,
+ * so both halves of single-delivery read one decision.
  */
 function isSingleDeliveryEnabled(): boolean {
   return isSingleDeliveryRequested()
@@ -78,6 +85,14 @@ type EventJobData = {
   event: string
   payload: EventPayload
   options?: EmitOptions
+  /**
+   * Set by the producer when it already ran the persistent subscribers inline
+   * (single-delivery reconciled off). The worker skips such jobs, so the app and
+   * the worker staying exactly-once no longer depends on both processes agreeing
+   * about `OM_EVENTS_SINGLE_DELIVERY`. Absent on jobs queued before this field
+   * existed, which keeps their behavior unchanged.
+   */
+  persistentDeliveredInline?: boolean
 }
 
 // Process-wide cache of the async (BullMQ) persistent-events producer queue.
@@ -201,13 +216,20 @@ export function createEventBus(opts: CreateBusOptions): EventBus {
   /**
    * Delivers an event to all registered in-memory handlers.
    * Supports wildcard pattern matching for event patterns.
+   *
+   * Handler errors are logged and delivery continues - one bad subscriber must
+   * not stop the others. The count of failed PERSISTENT handlers is returned so
+   * the caller can decide whether the inline run may be recorded as complete:
+   * a persistent subscriber that threw inline has no retry of its own, so the
+   * queued job must stay unstamped and let the worker run it again.
    */
   async function deliver(
     event: string,
     payload: EventPayload,
     options?: EmitOptions,
     skipPersistent = false,
-  ): Promise<void> {
+  ): Promise<{ persistentFailures: number }> {
+    let persistentFailures = 0
     // Check all registered patterns (including wildcards)
     for (const [pattern, handlers] of listeners) {
       if (!matchEventPattern(event, pattern)) continue
@@ -235,6 +257,9 @@ export function createEventBus(opts: CreateBusOptions): EventBus {
             }),
           )
         } catch (error) {
+          if (persistentSubscribers.has(subscriber)) {
+            persistentFailures += 1
+          }
           logger.error('Handler error', { event, pattern, err: error })
           if (options?.rethrowHandlerErrors) {
             throw error
@@ -242,6 +267,97 @@ export function createEventBus(opts: CreateBusOptions): EventBus {
         }
       }
     }
+    return { persistentFailures }
+  }
+
+  /**
+   * Selects the subscribers the events worker owns for a queued event: every
+   * pattern matching `event`, keeping persistent subscribers only, so wildcard
+   * (`event: '*'`) persistent subscribers are reached exactly once here.
+   *
+   * Deliberately does NOT read `OM_EVENTS_SINGLE_DELIVERY`. Whether inline
+   * delivery already happened is carried by the job's `persistentDeliveredInline`
+   * stamp, so the producer owns that decision entirely and the worker cannot
+   * disagree with it across processes. Reading the flag here would reintroduce
+   * exactly that skew: a worker whose env has the flag off would fall back to
+   * exact-match, re-running ephemeral subscribers the producer already ran inline
+   * and never reaching wildcard persistent subscribers - the failure this whole
+   * change exists to remove.
+   *
+   * Every job that reaches this point wants persistent-pattern selection: a job
+   * is only dispatched when its stamp is false, which means either the producer
+   * had single-delivery on, or the emit was enqueue-only (`deliverInline: false`,
+   * documented as "every subscriber to the event is persistent").
+   */
+  function selectQueuedSubscribers(event: string): RegisteredSubscriber[] {
+    const matched: RegisteredSubscriber[] = []
+    for (const [pattern, handlers] of listeners) {
+      if (!matchEventPattern(event, pattern)) {
+        continue
+      }
+      for (const subscriber of handlers) {
+        if (subscriber.persistent) {
+          matched.push(subscriber)
+        }
+      }
+    }
+    return matched
+  }
+
+  /**
+   * Dispatches a queued event on behalf of the events worker.
+   *
+   * Handler failures are returned instead of logged-and-swallowed the way
+   * `deliver()` does them: the worker needs them to propagate so the queue retries
+   * the job and eventually dead-letters it.
+   *
+   * Subscribers run against the caller's `resolve` when one is given, falling back
+   * to the bus's own. The events worker passes its per-job `ctx.resolve`, which is
+   * the container `createPerJobWorkerHandler` built for that job - the handle the
+   * queue contract already hands it. Defaulting to `opts.resolve` instead would
+   * bind subscribers to whichever container built the bus; that is the same object
+   * on the default configuration, but not under `OM_BOOTSTRAP_CACHE`, where the
+   * cached bus is replayed into later containers while its captured resolver stays
+   * bound to the first.
+   */
+  async function dispatchQueued(
+    event: string,
+    payload: EventPayload,
+    options?: EmitOptions,
+    resolve?: <T = unknown>(name: string) => T,
+  ): Promise<QueuedDispatchResult[]> {
+    const subscribers = selectQueuedSubscribers(event)
+    if (subscribers.length === 0) {
+      return []
+    }
+
+    const resolver = resolve ?? opts.resolve
+
+    const settled = await Promise.allSettled(
+      subscribers.map((subscriber) =>
+        withModuleResourceUsage(
+          {
+            moduleId: subscriber.moduleId ?? inferModuleIdFromResourceId(subscriber.id),
+            surface: 'subscriber',
+            operation: `${event} -> ${subscriber.id ?? subscriber.event}`,
+            resourceId: subscriber.id ?? subscriber.event,
+          },
+          () => subscriber.handler(payload, {
+            resolve: resolver,
+            eventName: event,
+            tenantId: options?.tenantId ?? null,
+            organizationId: options?.organizationId ?? null,
+          }),
+        ),
+      ),
+    )
+
+    return settled.map((result, index) => {
+      const subscriberId = subscribers[index].id ?? subscribers[index].event
+      return result.status === 'rejected'
+        ? { subscriberId, ok: false, error: result.reason }
+        : { subscriberId, ok: true }
+    })
   }
 
   /**
@@ -309,9 +425,13 @@ export function createEventBus(opts: CreateBusOptions): EventBus {
     // happens). Without this, a persistent emit dual-dispatches by default and
     // runs the subscriber inline in the caller's request too.
     const enqueueOnly = Boolean(options?.persistent) && options?.deliverInline === false
+    // Read the mode once so the inline skip and the stamp below cannot disagree.
+    const singleDelivery = isSingleDeliveryEnabled()
+    let inlinePersistentFailed = false
     if (!enqueueOnly) {
-      const skipPersistentInline = Boolean(options?.persistent) && isSingleDeliveryEnabled()
-      await deliver(event, payload, options, skipPersistentInline)
+      const skipPersistentInline = Boolean(options?.persistent) && singleDelivery
+      const delivered = await deliver(event, payload, options, skipPersistentInline)
+      inlinePersistentFailed = delivered.persistentFailures > 0
     }
 
     if (isBroadcastEvent(event) && hasTenantScope(payload)) {
@@ -325,7 +445,19 @@ export function createEventBus(opts: CreateBusOptions): EventBus {
     // If persistent, also enqueue for async processing
     if (options?.persistent) {
       const q = getQueue()
-      await q.enqueue({ event, payload, options })
+      await q.enqueue({
+        event,
+        payload,
+        options,
+        // Only record the inline run as complete when every persistent
+        // subscriber actually succeeded. Inline delivery logs-and-continues, so a
+        // handler that threw has no retry of its own; leaving the job unstamped
+        // hands it to the worker, where a failure gets queue retry and
+        // dead-lettering. Without this, flipping the flag off (which the
+        // `mercato server` guard can do automatically) silently downgrades
+        // persistent delivery to at-most-once.
+        persistentDeliveredInline: !enqueueOnly && !singleDelivery && !inlinePersistentFailed,
+      })
     }
   }
 
@@ -345,6 +477,7 @@ export function createEventBus(opts: CreateBusOptions): EventBus {
     emitEvent, // Alias for backward compatibility
     on,
     registerModuleSubscribers,
+    dispatchQueued,
     clearQueue,
   }
 }