@sixthsummit/ship-api 0.0.0-stage → 4.1.1

package/lib/modules/outbox/EventEmitter.ts

@@ -0,0 +1,106 @@
+import type { ShipDb } from "@sixthsummit/ship-api/app/db/client";
+import { type NewOutboxEvent, type OutboxEvent, outboxEvents } from "./schema.js";
+
+/**
+ * Producer-side helper for the transactional outbox pattern.
+ *
+ * Domain operations call `emitter.emit(...)` *inside the same
+ * transaction* as the business write. Postgres atomicity guarantees
+ * either both commit or neither — the outbox row is never orphaned
+ * from the domain change, and the domain change is never published
+ * without an outbox row to drain.
+ *
+ * Usage:
+ *
+ * ```ts
+ * await db.transaction(async (tx) => {
+ *   const account = await accountService.create({...}, { tx });
+ *   await emitter.emit({
+ *     eventType: "account.created",
+ *     aggregateType: "account",
+ *     aggregateId: account.id,
+ *     payload: { id: account.id, email: account.email },
+ *   }, { tx });
+ * });
+ * ```
+ *
+ * The drain worker (`OutboxDrainer`) picks up the row asynchronously
+ * and dispatches it to the registered `EventHandler`s. Producers
+ * never see the dispatch chain — they just write the event.
+ *
+ * Why a class instead of a free function: the emitter is constructed
+ * once per app with the framework's `db` client; consumers inject it
+ * into services / route handlers via the `deps` parameter.
+ */
+export class OutboxEventEmitter<
+  TSchema extends Record<string, unknown> = { outboxEvents: typeof outboxEvents },
+> {
+  constructor(private readonly db: ShipDb<TSchema>) {}
+
+  /**
+   * Append an event to the outbox. Pass a transaction client (`tx`)
+   * via `opts.tx` to atomically pair the event with a domain write —
+   * **strongly recommended**. Without `tx`, the event is appended in
+   * its own implicit transaction and could decouple from the domain
+   * write (defeats the purpose of the outbox pattern).
+   */
+  async emit(input: EmitEventInput, opts: EmitEventOptions = {}): Promise<OutboxEvent> {
+    const dbClient = (opts.tx ?? this.db) as unknown as ShipDb<TSchema>;
+    const values: NewOutboxEvent = {
+      eventType: input.eventType,
+      eventVersion: input.eventVersion ?? 1,
+      occurredAt: input.occurredAt ?? new Date(),
+      aggregateType: input.aggregateType ?? null,
+      aggregateId: input.aggregateId ?? null,
+      ownerUserId: input.ownerUserId ?? null,
+      tenantId: input.tenantId ?? null,
+      dedupeKey: input.dedupeKey ?? null,
+      correlationId: input.correlationId ?? null,
+      payload: input.payload,
+      maxRetries: input.maxRetries ?? 10,
+    };
+    const inserter = dbClient as unknown as {
+      insert: (table: typeof outboxEvents) => {
+        values: (v: unknown) => { returning: () => Promise<OutboxEvent[]> };
+      };
+    };
+    const [row] = await inserter.insert(outboxEvents).values(values).returning();
+    return row;
+  }
+}
+
+export interface EmitEventInput {
+  /** Domain event type — matches a registered handler's `type`. */
+  eventType: string;
+  /** Schema version for the payload. Bump when payload shape changes. */
+  eventVersion?: number;
+  /** When the domain operation occurred. Defaults to `now()`. */
+  occurredAt?: Date;
+  /** Aggregate type ("account", "post", ...). */
+  aggregateType?: string | null;
+  /** Aggregate id within `aggregateType`. */
+  aggregateId?: string | null;
+  /** Owner user id (typically `accounts.id`). */
+  ownerUserId?: string | null;
+  /** Multi-tenant scope. */
+  tenantId?: string | null;
+  /** Stable dedup key for upstream-deduplication. */
+  dedupeKey?: string | null;
+  /** Trace correlation id. */
+  correlationId?: string | null;
+  /** Event payload. Consumer-shaped JSON. */
+  payload: Record<string, unknown>;
+  /** Override the default 10-attempt retry budget. */
+  maxRetries?: number;
+}
+
+export interface EmitEventOptions {
+  /**
+   * Transaction client to participate in. Pass the same `tx` you're
+   * using for the domain write so the outbox row commits atomically
+   * with the business data. Without `tx`, the event is appended in
+   * its own transaction — usable for fire-and-forget signals where
+   * the outbox semantics don't matter.
+   */
+  tx?: unknown;
+}

package/lib/modules/outbox/OutboxDrainer.ts

@@ -0,0 +1,163 @@
+import type { EventHandlerRegistry } from "./handlers/EventHandler.js";
+import { hasExhaustedRetries, type OutboxEvent } from "./schema.js";
+import type { OutboxEventService } from "./services/OutboxEventService.js";
+
+/**
+ * Consumer-side drain orchestrator. One pass:
+ *
+ *   1. `findReadyToPublish(batchSize)` — pull pending rows
+ *   2. `claimForProcessing(ids)` — atomic pending→processing
+ *      transition (concurrent-safe — competing workers see different
+ *      claimed subsets)
+ *   3. For each claimed row:
+ *      - Resolve the handler via the registry
+ *      - Invoke `handler.handle(event, ctx)`
+ *      - On success → `markPublished(id)`
+ *      - On throw → check the retry budget:
+ *          - If exhausted → `markFailed(id, error)` (terminal)
+ *          - Otherwise → `markForRetry(id, error, attempts)` (revert to pending,
+ *            schedule via exponential backoff)
+ *      - If no handler matches → mark for retry with a "no handler"
+ *        message; ops backfills the handler and the next drain
+ *        retries automatically.
+ *   4. Returns a summary — published / failed / retried counts.
+ *
+ * `runOnce()` does one pass. `runUntilEmpty()` keeps passing until
+ * the database returns 0 ready rows (polling-friendly worker entry
+ * point — wraps with `setInterval` outside this class).
+ *
+ * Honors `signal` end-to-end: handler invocations receive the same
+ * AbortSignal, the loop exits between batches, and the loop also
+ * exits *between* per-event handler calls so a SIGTERM mid-batch
+ * doesn't have to wait for the entire batch to finish.
+ */
+
+export interface OutboxDrainerOptions {
+  /**
+   * Rows to claim per drain pass. Larger = fewer round trips but
+   * longer per-pass latency. Default 50.
+   */
+  batchSize?: number;
+  /** AbortSignal to short-circuit between handler invocations. */
+  signal?: AbortSignal;
+  /**
+   * Per-handler context the registry forwards to each handler.
+   * Typical contents: `{ requestId, traceId }`.
+   */
+  metadata?: Record<string, unknown>;
+}
+
+export interface OutboxDrainerResult {
+  /** Total events processed in this pass. */
+  processed: number;
+  /** Successfully published — terminal success. */
+  published: number;
+  /** Permanently failed — exhausted retry budget. */
+  failed: number;
+  /** Scheduled for retry — non-terminal failure. */
+  retried: number;
+  /** Skipped because no handler is registered for the event type. */
+  skippedNoHandler: number;
+}
+
+const ZERO_RESULT: OutboxDrainerResult = {
+  processed: 0,
+  published: 0,
+  failed: 0,
+  retried: 0,
+  skippedNoHandler: 0,
+};
+
+export class OutboxDrainer {
+  constructor(
+    private readonly service: OutboxEventService,
+    private readonly handlers: EventHandlerRegistry,
+  ) {}
+
+  async runOnce(opts: OutboxDrainerOptions = {}): Promise<OutboxDrainerResult> {
+    const batchSize = opts.batchSize ?? 50;
+    if (opts.signal?.aborted) return { ...ZERO_RESULT };
+
+    const candidates = await this.service.findReadyToPublish(batchSize);
+    if (candidates.length === 0) return { ...ZERO_RESULT };
+
+    const claimedIds = await this.service.claimForProcessing(candidates.map((c) => c.id));
+    const claimed = candidates.filter((c) => claimedIds.includes(c.id));
+
+    const result: OutboxDrainerResult = { ...ZERO_RESULT };
+    result.processed = claimed.length;
+
+    for (const event of claimed) {
+      if (opts.signal?.aborted) break;
+      await this.processOne(event, opts, result);
+    }
+
+    return result;
+  }
+
+  /**
+   * Loop `runOnce` until a pass returns `processed: 0` (queue
+   * drained) or the signal aborts. Useful for cron-driven flushes
+   * and for catch-up runs after an outage.
+   */
+  async runUntilEmpty(opts: OutboxDrainerOptions = {}): Promise<OutboxDrainerResult> {
+    const total: OutboxDrainerResult = { ...ZERO_RESULT };
+    while (!opts.signal?.aborted) {
+      const pass = await this.runOnce(opts);
+      total.processed += pass.processed;
+      total.published += pass.published;
+      total.failed += pass.failed;
+      total.retried += pass.retried;
+      total.skippedNoHandler += pass.skippedNoHandler;
+      if (pass.processed === 0) break;
+    }
+    return total;
+  }
+
+  private async processOne(
+    event: OutboxEvent,
+    opts: OutboxDrainerOptions,
+    result: OutboxDrainerResult,
+  ): Promise<void> {
+    const handler = this.handlers.findHandler({
+      eventType: event.eventType,
+      eventVersion: event.eventVersion,
+    });
+
+    if (!handler) {
+      // No handler registered. Don't fail-permanent — ops may add
+      // the handler later and the next drain should retry. Mark
+      // for retry with a clear marker, but don't increment the
+      // attempt charge against the row (claimForProcessing already
+      // bumped it).
+      const message = `no handler registered for ${event.eventType} (v${event.eventVersion})`;
+      if (hasExhaustedRetries(event)) {
+        await this.service.markFailed(event.id, message);
+        result.failed += 1;
+      } else {
+        await this.service.markForRetry(event.id, message, event.publishAttempts);
+        result.retried += 1;
+      }
+      result.skippedNoHandler += 1;
+      return;
+    }
+
+    try {
+      await handler.handle(event as never, {
+        signal: opts.signal,
+        metadata: opts.metadata,
+      });
+      await this.service.markPublished(event.id);
+      result.published += 1;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      if (hasExhaustedRetries(event)) {
+        await this.service.markFailed(event.id, message);
+        result.failed += 1;
+      } else {
+        await this.service.markForRetry(event.id, message, event.publishAttempts);
+        result.retried += 1;
+      }
+    }
+  }
+}

package/lib/modules/outbox/handlers/EventHandler.ts

@@ -0,0 +1,115 @@
+import type { OutboxEvent } from "../schema.js";
+
+/**
+ * Consumer-side event handler contract.
+ *
+ * One handler maps to one event type. The drain worker dispatches each
+ * pending row to the matching handler; the handler does the actual
+ * publish (queue enqueue, webhook POST, log emit, …) and returns
+ * normally on success or throws on failure. Handler throws are
+ * captured by the drainer and translated to retry / mark-failed
+ * decisions based on the row's retry budget.
+ *
+ * Handlers MUST be idempotent. The outbox guarantees at-least-once
+ * delivery — a handler may be re-invoked for the same event after a
+ * crash, after stale-processing recovery, or after an exhausted
+ * retry budget gets re-enabled by an admin. Use `dedupeKey` /
+ * `eventId` for downstream deduplication where the consumer system
+ * supports it.
+ *
+ * Handlers SHOULD honor `ctx.signal` for graceful shutdown — when
+ * the worker is asked to stop mid-publish, the signal aborts and
+ * the handler should clean up rather than push past.
+ */
+export interface EventHandler<TPayload = Record<string, unknown>> {
+  /** Stable event type matching `OutboxEvent.eventType`. */
+  readonly eventType: string;
+  /**
+   * Schema version this handler supports. When omitted, the handler
+   * accepts any version (caller-side responsibility to keep handlers
+   * forward-compatible). When set, the drainer skips events with a
+   * different version (or routes to a versioned alternative if
+   * registered).
+   */
+  readonly eventVersion?: number;
+  /** Run the handler. Throw to signal failure (drainer decides retry vs final-fail). */
+  handle(event: OutboxEvent & { payload: TPayload }, ctx: HandlerContext): Promise<void>;
+}
+
+export interface HandlerContext {
+  /** Aborts when the drain worker is stopping. Handlers honor this. */
+  signal?: AbortSignal;
+  /** Per-attempt scoped logger / metadata. Caller decides shape. */
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Registry of `EventHandler`s keyed by `eventType` (+ optional
+ * `eventVersion` discriminator). The drainer does
+ * `registry.findHandler(event)` to dispatch each row.
+ *
+ * Replaces v3's `EventHandlerRegistry` static-class pattern with an
+ * instance-based registry. Same justification as `OAuthAdapterRegistry`:
+ * lets consumers compose handler sets per app / per tenant without
+ * touching framework globals.
+ */
+export class EventHandlerRegistry {
+  // Two-level map: eventType → (versionKey → handler). versionKey is
+  // either the literal version number or the sentinel "*" for
+  // version-agnostic handlers.
+  private readonly handlers = new Map<string, Map<string, EventHandler>>();
+
+  register(handler: EventHandler): this {
+    const versionKey = handler.eventVersion === undefined ? "*" : String(handler.eventVersion);
+    let versions = this.handlers.get(handler.eventType);
+    if (!versions) {
+      versions = new Map();
+      this.handlers.set(handler.eventType, versions);
+    }
+    if (versions.has(versionKey)) {
+      throw new Error(
+        `EventHandlerRegistry: duplicate handler for ${handler.eventType} v${versionKey}`,
+      );
+    }
+    versions.set(versionKey, handler);
+    return this;
+  }
+
+  /**
+   * Find the handler for an event. Resolution order:
+   *   1. Exact `(eventType, eventVersion)` match
+   *   2. Version-agnostic `(eventType, *)` fallback
+   * Returns `undefined` when nothing is registered for the event type.
+   */
+  findHandler(
+    event: Pick<{ eventType: string; eventVersion: number }, "eventType" | "eventVersion">,
+  ): EventHandler | undefined {
+    const versions = this.handlers.get(event.eventType);
+    if (!versions) return undefined;
+    const exact = versions.get(String(event.eventVersion));
+    if (exact) return exact;
+    return versions.get("*");
+  }
+
+  /** Adapter-equivalent of `has` — true when *any* handler matches. */
+  hasHandler(eventType: string): boolean {
+    return this.handlers.has(eventType);
+  }
+
+  /** All event types with at least one registered handler. */
+  eventTypes(): string[] {
+    return [...this.handlers.keys()];
+  }
+
+  size(): number {
+    let total = 0;
+    for (const versions of this.handlers.values()) total += versions.size;
+    return total;
+  }
+}
+
+export function eventHandlerRegistry(...handlers: EventHandler[]): EventHandlerRegistry {
+  const registry = new EventHandlerRegistry();
+  for (const h of handlers) registry.register(h);
+  return registry;
+}

package/lib/modules/outbox/index.ts

@@ -0,0 +1,43 @@
+/**
+ * Outbox module — public barrel.
+ *
+ * Postgres-backed transactional event outbox: emit events inside the
+ * same DB transaction as the domain write, then drain to handlers
+ * (which typically enqueue BullMQ jobs) outside the transaction.
+ * Atomicity with the domain write requires Postgres; Redis-backed
+ * BullMQ can't participate.
+ *
+ * Surface: outbox events table + service + status / lifecycle
+ * transition methods + retry/recover helpers + EventEmitter +
+ * EventHandlerRegistry + OutboxDrainer + HTTP routes.
+ */
+
+export { type EmitEventInput, type EmitEventOptions, OutboxEventEmitter } from "./EventEmitter.js";
+export {
+  type EventHandler,
+  EventHandlerRegistry,
+  eventHandlerRegistry,
+  type HandlerContext,
+} from "./handlers/EventHandler.js";
+export {
+  OutboxDrainer,
+  type OutboxDrainerOptions,
+  type OutboxDrainerResult,
+} from "./OutboxDrainer.js";
+export {
+  OUTBOX_PERMISSION_DEFINITIONS,
+  OUTBOX_PERMISSIONS,
+  type OutboxPermission,
+} from "./permissions.js";
+export { buildOutboxApp, defineOutboxRoutes, type OutboxRoutesDeps } from "./routes.js";
+export {
+  computeNextAttempt,
+  hasExhaustedRetries,
+  isReadyForPublish,
+  type NewOutboxEvent,
+  type OutboxEvent,
+  type OutboxEventStatus,
+  outboxEventStatus,
+  outboxEvents,
+} from "./schema.js";
+export { OutboxEventService } from "./services/OutboxEventService.js";

package/lib/modules/outbox/module.json

@@ -0,0 +1,15 @@
+{
+  "name": "outbox",
+  "description": "Transactional outbox on the v4 substrate. The v3 `operations` module bundled outbox events + a job queue; v4 splits them — the job runtime moves entirely into BullMQ (`defineJob` / `JobsManager`) and this module owns the Postgres-backed outbox (event row commits in the same tx as the domain write).",
+  "version": "2.0.0-alpha.1",
+  "dependencies": ["accounts"],
+  "npmDependencies": {},
+  "requiredEnvVars": [],
+  "optionalEnvVars": [],
+  "postInstall": [
+    "Add the schema to your composed schema: `createApp({ schema: { ...accountsSchema, outboxEvents } })`",
+    "Wire the routes: `defineOutboxRoutes(app, { service, drainer, provider })`",
+    "Generate the migration: `npm run db:generate`",
+    "If you're migrating from v3 operations: the `Job` entity + JobController are NOT ported. Move job processing to `defineJob` from @sixthsummit/ship-api (BullMQ-backed)."
+  ]
+}

package/lib/modules/outbox/permissions.ts

@@ -0,0 +1,36 @@
+/**
+ * Permission slugs for the outbox module.
+ *
+ * v3 had no permissions for the JobController routes (operational
+ * tooling assumed admin-only). v4 makes the permission boundary
+ * explicit so the same module can ship to consumers who route admin
+ * traffic through different RBAC layers.
+ */
+
+import type { PermissionDefinition } from "@sixthsummit/ship-api";
+
+export const OUTBOX_PERMISSIONS = {
+  /** View outbox events (for one's own scope or all, depending on RBAC). */
+  outboxRead: "outbox:read",
+  /** Trigger drain manually (admin operation). */
+  outboxDrain: "outbox:drain",
+  /** Recover stale processing events (admin operation). */
+  outboxRecover: "outbox:recover",
+} as const;
+
+export type OutboxPermission = (typeof OUTBOX_PERMISSIONS)[keyof typeof OUTBOX_PERMISSIONS];
+
+export const OUTBOX_PERMISSION_DEFINITIONS = [
+  {
+    slug: OUTBOX_PERMISSIONS.outboxRead,
+    label: "View Outbox Events",
+  },
+  {
+    slug: OUTBOX_PERMISSIONS.outboxDrain,
+    label: "Drain Outbox (Admin)",
+  },
+  {
+    slug: OUTBOX_PERMISSIONS.outboxRecover,
+    label: "Recover Stale Outbox Events (Admin)",
+  },
+] as const satisfies readonly PermissionDefinition[];