@fullstackhouse/open-mercato-durable-work 0.1.0

package/src/transport/memory.ts

@@ -0,0 +1,191 @@
+// An in-process transport, for unit tests and fault injection.
+//
+// Not a toy: it implements the same delivery semantics as the real adapters — attempts,
+// backoff, delayed availability, hand-back that spends no attempt, a real abort signal on
+// close — and it runs the same conformance suite. Its purpose is to make failures *reachable*
+// (duplicate a delivery, stall one, drop one) that a real broker only produces by accident.
+//
+// Explicitly not for production: nothing survives the process.
+
+import { deliveryId } from '../core/ids'
+import type { Delivery, RetrySettings } from '../core/types'
+import type {
+  BindOptions,
+  BoundWorker,
+  DeliveryHandler,
+  DeliveryState,
+  EnqueueOptions,
+  TransportAdapter,
+} from './types'
+
+type Entry = {
+  id: string
+  queue: string
+  delivery: Delivery
+  availableAt: number
+  attempt: number
+  retry: RetrySettings
+  state: DeliveryState
+}
+
+function backoffFor(retry: RetrySettings, attempt: number): number {
+  const { type, delayMs, maxDelayMs } = retry.backoff
+  const raw = type === 'fixed' ? delayMs : delayMs * 2 ** Math.max(0, attempt - 1)
+  return Math.min(raw, maxDelayMs)
+}
+
+export type MemoryFaults = {
+  /** Deliver each accepted delivery twice. The lease must refuse the second. */
+  duplicateDeliveries?: boolean
+  /** Swallow enqueues matching this predicate, simulating a broker that lost the message.
+   *  The reconciler is what must notice. */
+  dropEnqueue?: (delivery: Delivery) => boolean
+  /** Hold a delivery indefinitely instead of running it. */
+  stall?: (delivery: Delivery) => boolean
+}
+
+export class MemoryTransport implements TransportAdapter {
+  readonly name = 'memory' as const
+  readonly supportsTransactionalEnqueue = false
+
+  private readonly entries = new Map<string, Entry>()
+  private readonly handlers = new Map<string, { handler: DeliveryHandler; opts: BindOptions }>()
+  private readonly ticks = new Map<string, { queue: string; everyMs: number; timer: NodeJS.Timeout }>()
+  private readonly inFlight = new Set<Promise<void>>()
+  private readonly abort = new AbortController()
+  private pump: NodeJS.Timeout | null = null
+  private closed = false
+
+  constructor(private readonly faults: MemoryFaults = {}) {}
+
+  async enqueue(queue: string, delivery: Delivery, opts: EnqueueOptions): Promise<{ transportJobId: string }> {
+    const id = deliveryId(delivery)
+    if (this.faults.dropEnqueue?.(delivery)) return { transportJobId: id }
+    // Same id, same delivery: the broker deduplicates, which is what a caller-supplied job id
+    // buys. The database refuses a duplicate anyway; this only saves the wasted claim.
+    this.entries.set(id, {
+      id,
+      queue,
+      delivery,
+      availableAt: Date.now() + (opts.delayMs ?? 0),
+      attempt: 0,
+      retry: opts.retry,
+      state: (opts.delayMs ?? 0) > 0 ? 'delayed' : 'waiting',
+    })
+    this.start()
+    return { transportJobId: id }
+  }
+
+  async remove(_queue: string, transportJobId: string): Promise<void> {
+    this.entries.delete(transportJobId)
+  }
+
+  async getState(_queue: string, transportJobId: string): Promise<DeliveryState> {
+    return this.entries.get(transportJobId)?.state ?? 'unknown'
+  }
+
+  async upsertTick(opts: { id: string; queue: string; everyMs: number }): Promise<void> {
+    this.ticks.get(opts.id)?.timer.unref?.()
+    clearInterval(this.ticks.get(opts.id)?.timer)
+    const timer = setInterval(() => {
+      void this.enqueue(opts.queue, { jobId: opts.id, seq: 0, redrives: 0 }, { retry: TICK_RETRY })
+    }, opts.everyMs)
+    timer.unref?.()
+    this.ticks.set(opts.id, { queue: opts.queue, everyMs: opts.everyMs, timer })
+  }
+
+  async bind(queue: string, handler: DeliveryHandler, opts: BindOptions): Promise<BoundWorker> {
+    this.handlers.set(queue, { handler, opts })
+    this.start()
+    return {
+      queue,
+      close: async () => {
+        this.handlers.delete(queue)
+      },
+    }
+  }
+
+  async close(opts: { timeoutMs?: number } = {}): Promise<void> {
+    this.closed = true
+    this.abort.abort()
+    for (const tick of this.ticks.values()) clearInterval(tick.timer)
+    this.ticks.clear()
+    if (this.pump) clearInterval(this.pump)
+    this.pump = null
+    // Bounded: a handler that ignores its abort signal must not be able to hang shutdown
+    // forever. Dropping the wait is the honest outcome — the lease still protects the row.
+    const deadline = new Promise<void>((resolve) => setTimeout(resolve, opts.timeoutMs ?? 30_000).unref?.())
+    await Promise.race([Promise.allSettled([...this.inFlight]).then(() => undefined), deadline])
+  }
+
+  /** Test hook: how many deliveries the broker still holds. */
+  size(): number {
+    return this.entries.size
+  }
+
+  private start(): void {
+    if (this.pump || this.closed) return
+    this.pump = setInterval(() => void this.drain(), 5)
+    this.pump.unref?.()
+  }
+
+  private async drain(): Promise<void> {
+    if (this.closed) return
+    const now = Date.now()
+    for (const entry of [...this.entries.values()]) {
+      if (entry.state === 'active' || entry.availableAt > now) continue
+      const bound = this.handlers.get(entry.queue)
+      if (!bound) continue
+      if (this.faults.stall?.(entry.delivery)) continue
+      const running = [...this.inFlight].length
+      if (running >= bound.opts.concurrency) return
+      entry.state = 'active'
+      entry.attempt += 1
+      const promise = this.run(entry, bound.handler, bound.opts).finally(() => this.inFlight.delete(promise))
+      this.inFlight.add(promise)
+      if (this.faults.duplicateDeliveries) {
+        const twin = this.run({ ...entry }, bound.handler, bound.opts).finally(() => this.inFlight.delete(twin))
+        this.inFlight.add(twin)
+      }
+    }
+  }
+
+  private async run(entry: Entry, handler: DeliveryHandler, opts: BindOptions): Promise<void> {
+    let handedBack = false
+    try {
+      await handler(entry.delivery, {
+        transportJobId: entry.id,
+        attempt: entry.attempt,
+        maxAttempts: entry.retry.attempts,
+        signal: this.abort.signal,
+        handBack: async (next, handBackOpts) => {
+          handedBack = true
+          this.entries.delete(entry.id)
+          await this.enqueue(entry.queue, next, { retry: entry.retry, delayMs: handBackOpts?.delayMs })
+        },
+      })
+      if (!handedBack) {
+        entry.state = 'completed'
+        this.entries.delete(entry.id)
+      }
+    } catch (error) {
+      if ((error as { name?: string })?.name === 'NoFurtherAttempts') {
+        entry.state = 'failed'
+        this.entries.delete(entry.id)
+        return
+      }
+      if (entry.attempt >= entry.retry.attempts) {
+        entry.state = 'failed'
+        this.entries.delete(entry.id)
+        return
+      }
+      entry.state = 'delayed'
+      entry.availableAt = Date.now() + backoffFor(entry.retry, entry.attempt)
+    } finally {
+      void opts
+    }
+  }
+}
+
+/** The tick is a heartbeat, not work: one attempt, no backoff to reason about. */
+const TICK_RETRY: RetrySettings = { attempts: 1, backoff: { type: 'fixed', delayMs: 0, maxDelayMs: 0 } }

package/src/transport/pgboss.ts

@@ -0,0 +1,250 @@
+// The pg-boss adapter: production without Redis, and the only adapter that can enqueue a
+// delivery inside the caller's transaction.
+//
+// That one capability is why it exists. `send(..., { db })` composes its statements on a
+// client the caller supplies, so a domain row, its job row and its delivery all commit or all
+// roll back. With any other transport there is a window between commit and enqueue where a
+// crash leaves a job nobody will ever deliver — the reconciler closes it, but closing it after
+// fifteen minutes is not the same as never opening it.
+//
+// pg-boss is a peer dependency and is imported lazily.
+
+import { deliveryId } from '../core/ids'
+import type { Delivery, SqlExecutor } from '../core/types'
+import type {
+  BindOptions,
+  BoundWorker,
+  DeliveryHandler,
+  DeliveryState,
+  EnqueueOptions,
+  TransportAdapter,
+} from './types'
+
+type PgBossModule = typeof import('pg-boss')
+// pg-boss 12 exports the class by name, not as a default.
+type PgBossInstance = InstanceType<PgBossModule['PgBoss']>
+type PgBossJob = import('pg-boss').Job<Delivery> & { signal?: AbortSignal }
+
+export type PgBossTransportOptions = {
+  connectionString: string
+  /** Keeps pg-boss's own tables out of `public`, so they are obviously not the app's. */
+  schema?: string
+  /** Reuse an already-started instance instead of owning its lifecycle. */
+  instance?: PgBossInstance
+}
+
+let cached: PgBossModule | null = null
+async function pgboss(): Promise<PgBossModule> {
+  if (cached) return cached
+  try {
+    cached = await import('pg-boss')
+    return cached
+  } catch (error) {
+    throw new Error(
+      'The pgboss transport requires the optional peer dependency `pg-boss`. Install it, or use DURABLE_WORK_TRANSPORT=bullmq.',
+      { cause: error },
+    )
+  }
+}
+
+/**
+ * Adapts a `SqlExecutor` to the shape pg-boss expects from a caller-supplied client.
+ *
+ * pg-boss only ever calls `executeSql`, so the whole surface is one method. Passing our own
+ * executor through — rather than requiring a raw `pg` client — is what lets the caller's
+ * transaction be a MikroORM one, a node-postgres one, or the harness's, without any of them
+ * knowing about the others.
+ */
+function asDb(tx: SqlExecutor) {
+  return {
+    async executeSql(text: string, values: unknown[]) {
+      const result = await tx.query(text, values)
+      return { rows: result.rows as Record<string, unknown>[], rowCount: result.rowCount }
+    },
+  }
+}
+
+/** A tick, in the same shape as any other delivery. */
+const tickDelivery = (id: string): Delivery => ({ jobId: id, seq: 0, redrives: 0 })
+
+export class PgBossTransport implements TransportAdapter {
+  readonly name = 'pgboss' as const
+  readonly supportsTransactionalEnqueue = true
+
+  private boss: PgBossInstance | null = null
+  private starting: Promise<PgBossInstance> | null = null
+  private readonly ownsInstance: boolean
+  private readonly ensuredQueues = new Set<string>()
+  private readonly workerIds: Array<{ queue: string; id: string }> = []
+  private readonly ticks: NodeJS.Timeout[] = []
+  private readonly shutdown = new AbortController()
+
+  constructor(private readonly options: PgBossTransportOptions) {
+    this.boss = options.instance ?? null
+    this.ownsInstance = !options.instance
+  }
+
+  private async ready(): Promise<PgBossInstance> {
+    if (this.boss) return this.boss
+    if (!this.starting) {
+      this.starting = (async () => {
+        const { PgBoss } = await pgboss()
+        const instance = new PgBoss({ connectionString: this.options.connectionString, schema: this.options.schema ?? 'durable_work_boss' })
+        await instance.start()
+        this.boss = instance
+        return instance
+      })()
+    }
+    return this.starting
+  }
+
+  /** pg-boss 10+ requires a queue to exist before anything is sent to it. `expireInSeconds`
+   *  belongs to the queue, not to the worker: it is how long a delivery may stay active before
+   *  pg-boss reclaims it, so it must exceed a whole slice or work that is still running gets
+   *  handed to a second worker — which the lease then refuses, wasting the slice. */
+  private async ensureQueue(name: string, expireInSeconds?: number): Promise<PgBossInstance> {
+    const boss = await this.ready()
+    if (this.ensuredQueues.has(name)) return boss
+    await boss.createQueue(name, expireInSeconds ? { expireInSeconds } : undefined)
+    this.ensuredQueues.add(name)
+    return boss
+  }
+
+  async enqueue(queue: string, delivery: Delivery, opts: EnqueueOptions): Promise<{ transportJobId: string }> {
+    const boss = await this.ensureQueue(queue)
+    const key = deliveryId(delivery)
+    const sent = await boss.send(queue, delivery as unknown as object, {
+      // pg-boss job ids are uuids, so the delivery identity travels as the singleton key —
+      // which is also what makes a re-enqueue of the same delivery a no-op.
+      singletonKey: key,
+      startAfter: opts.delayMs && opts.delayMs > 0 ? Math.ceil(opts.delayMs / 1000) : undefined,
+      retryLimit: opts.retry.attempts,
+      retryDelay: Math.max(1, Math.round(opts.retry.backoff.delayMs / 1000)),
+      // pg-boss rejects a max delay unless backoff is on, so the cap travels only with it.
+      ...(opts.retry.backoff.type === 'exponential'
+        ? { retryBackoff: true, retryDelayMax: Math.max(1, Math.round(opts.retry.backoff.maxDelayMs / 1000)) }
+        : { retryBackoff: false }),
+      ...(opts.tx ? { db: asDb(opts.tx) } : {}),
+    })
+    // `send` returns null when the singleton key collapsed this into an existing job. That is
+    // the intended outcome, not a failure: the delivery is already scheduled.
+    return { transportJobId: sent ?? key }
+  }
+
+  async remove(queue: string, transportJobId: string): Promise<void> {
+    const boss = await this.ready()
+    await boss.deleteJob(queue, transportJobId).catch(() => undefined)
+  }
+
+  async getState(queue: string, transportJobId: string): Promise<DeliveryState> {
+    const boss = await this.ready()
+    const job = await boss.getJobById(queue, transportJobId).catch(() => null)
+    if (!job) return 'unknown'
+    switch (job.state) {
+      case 'created':
+      case 'retry':
+        return 'waiting'
+      case 'active':
+        return 'active'
+      case 'completed':
+        return 'completed'
+      case 'cancelled':
+      case 'failed':
+        return 'failed'
+      default:
+        return 'unknown'
+    }
+  }
+
+  async upsertTick(opts: { id: string; queue: string; everyMs: number }): Promise<void> {
+    await this.ensureQueue(opts.queue)
+    // pg-boss's own scheduler is cron-based, so its finest granularity is a minute — too
+    // coarse for a repair loop. A per-process timer with a singleton key gives the cadence we
+    // need and still collapses the fleet's ticks into one job per window. The trade-off is
+    // stated rather than hidden: with zero workers up there is no tick, exactly as with a
+    // broker-owned schedule that nobody polls.
+    const everySeconds = Math.max(1, Math.round(opts.everyMs / 1000))
+    const fire = async () => {
+      if (this.shutdown.signal.aborted) return
+      const boss = await this.ready()
+      // Delivery-shaped, like every other payload on every adapter: a tick is a delivery
+      // whose handler happens to ignore it, not a second kind of message.
+      await boss
+        .send(opts.queue, tickDelivery(opts.id), { singletonKey: opts.id, singletonSeconds: everySeconds })
+        .catch(() => undefined)
+    }
+    void fire()
+    const timer = setInterval(() => void fire(), opts.everyMs)
+    timer.unref?.()
+    this.ticks.push(timer)
+  }
+
+  async bind(queue: string, handler: DeliveryHandler, opts: BindOptions): Promise<BoundWorker> {
+    const boss = await this.ensureQueue(queue, Math.ceil(opts.activeTimeoutMs / 1000))
+
+    const workerId = await boss.work<Delivery>(
+      queue,
+      { batchSize: opts.concurrency },
+      async (jobs: PgBossJob[]) => {
+        for (const job of jobs) {
+          // Whatever arrived is handed on unexamined. An adapter that inspects payloads
+          // decides what counts as a real delivery, and this one used to skip anything
+          // without a `jobId` — which silently swallowed every reconciler tick.
+          const delivery = job.data
+
+          const combined = new AbortController()
+          const relay = () => combined.abort()
+          job.signal?.addEventListener('abort', relay, { once: true })
+          this.shutdown.signal.addEventListener('abort', relay, { once: true })
+
+          try {
+            await handler(delivery, {
+              transportJobId: job.id,
+              // pg-boss does not expose the attempt on the job, so the adapter reports the
+              // first attempt and lets its own retry policy carry the rest. The consequence is
+              // narrow: the delay written to `next_run_at` is the base rather than a backed-off
+              // one, and pg-boss's own `retryBackoff` still spaces the real deliveries.
+              attempt: 1,
+              maxAttempts: 1,
+              signal: combined.signal,
+              handBack: async (next, handBackOpts) => {
+                // No native hand-back: send the next delivery and let this one complete. The
+                // row is already at `seq + 1`, so the new key cannot collide with this job.
+                await this.enqueue(queue, next, {
+                  delayMs: handBackOpts?.delayMs,
+                  retry: { attempts: 1, backoff: { type: 'fixed', delayMs: 0, maxDelayMs: 0 } },
+                })
+              },
+            })
+          } catch (error) {
+            if ((error as { name?: string })?.name === 'NoFurtherAttempts') continue // settled; no retry wanted
+            throw error
+          } finally {
+            job.signal?.removeEventListener('abort', relay)
+            this.shutdown.signal.removeEventListener('abort', relay)
+          }
+        }
+      },
+    )
+
+    this.workerIds.push({ queue, id: workerId })
+    return {
+      queue,
+      close: async () => {
+        const instance = await this.ready()
+        await instance.offWork(queue, { id: workerId }).catch(() => undefined)
+      },
+    }
+  }
+
+  async close(opts: { timeoutMs?: number } = {}): Promise<void> {
+    this.shutdown.abort()
+    for (const timer of this.ticks) clearInterval(timer)
+    this.ticks.length = 0
+    if (!this.boss) return
+    if (!this.ownsInstance) return
+    await this.boss.stop({ graceful: true, close: true, timeout: opts.timeoutMs ?? 30_000 }).catch(() => undefined)
+    this.boss = null
+    this.starting = null
+  }
+}

package/src/transport/types.ts

@@ -0,0 +1,81 @@
+// What the mechanism needs from a broker, and nothing more.
+//
+// The interface is small on purpose. Lease, epoch fencing, slices and budgets all live above
+// it, in the job row — because none of the candidate brokers fences (pg-boss completes a job
+// on `state = 'active'` alone; neither it nor Graphile Worker has an epoch column), so that
+// guarantee had to be ours regardless. Once it is ours, what remains for a transport is
+// genuinely just delivery, and a transport becomes a replaceable detail rather than the
+// design's centre. See docs/adr/0001-transport-adapters.md.
+
+import type { Delivery, RetrySettings, SqlExecutor } from '../core/types'
+
+export type TransportName = 'memory' | 'bullmq' | 'pgboss'
+
+/** The state of a delivery, as far as the broker knows. Used to tell "the broker still holds
+ *  this" from "nothing is scheduled" — the reconciler's two very different situations. */
+export type DeliveryState = 'waiting' | 'delayed' | 'active' | 'completed' | 'failed' | 'unknown'
+
+export type EnqueueOptions = {
+  /** How long the broker should hold the delivery before making it available. */
+  delayMs?: number
+  retry: RetrySettings
+  /**
+   * Enqueue inside the caller's transaction, so the job row and its delivery commit together.
+   *
+   * Only pg-boss can honour this — it accepts a caller-supplied client on `send`. Adapters
+   * that cannot must ignore it, and the service must therefore enqueue *after* commit for
+   * them. This is the one capability difference between the adapters that callers can see.
+   */
+  tx?: SqlExecutor
+}
+
+export type HandlerContext = {
+  transportJobId: string
+  /** 1-based. Together with the kind's retry settings this decides the next delay. */
+  attempt: number
+  maxAttempts: number
+  /** Aborts when the process is shutting down, so a slice can stop at a batch boundary
+   *  instead of being killed between two writes. */
+  signal: AbortSignal
+  /**
+   * Hands the rest of the work back as a fresh delivery, without spending a retry attempt.
+   *
+   * Native where the broker supports it (BullMQ moves the job to delayed and keeps its id);
+   * emulated elsewhere by enqueuing the next delivery and completing the current one. Either
+   * way the contract is the same: after `handBack` resolves, the handler must return.
+   */
+  handBack(next: Delivery, opts?: { delayMs?: number }): Promise<void>
+}
+
+export type DeliveryHandler = (delivery: Delivery, ctx: HandlerContext) => Promise<void>
+
+export type BindOptions = {
+  concurrency: number
+  /** How long a delivery may be in flight before the broker considers the worker dead. Must
+   *  exceed the slice budget, or the broker will redeliver work that is still running — which
+   *  the lease then refuses, wasting a whole slice. */
+  activeTimeoutMs: number
+}
+
+export interface BoundWorker {
+  queue: string
+  close(opts?: { timeoutMs?: number }): Promise<void>
+}
+
+export interface TransportAdapter {
+  readonly name: TransportName
+  /** True when `enqueue` honours `tx`. Callers branch on this rather than on the name. */
+  readonly supportsTransactionalEnqueue: boolean
+
+  enqueue(queue: string, delivery: Delivery, opts: EnqueueOptions): Promise<{ transportJobId: string }>
+  remove(queue: string, transportJobId: string): Promise<void>
+  getState(queue: string, transportJobId: string): Promise<DeliveryState>
+
+  /** A repeating delivery, used for the reconciler tick. Idempotent by `id`. */
+  upsertTick(opts: { id: string; queue: string; everyMs: number }): Promise<void>
+
+  bind(queue: string, handler: DeliveryHandler, opts: BindOptions): Promise<BoundWorker>
+
+  /** Stops accepting work and waits, up to `timeoutMs`, for in-flight deliveries to end. */
+  close(opts?: { timeoutMs?: number }): Promise<void>
+}