@fullstackhouse/open-mercato-durable-work 0.1.0

package/src/core/reconciler.ts

@@ -0,0 +1,242 @@
+// The server-side repair loop: what makes "no job stays running forever" true.
+//
+// Three queries, in a fixed order, each selecting a bounded batch with `for update skip
+// locked` in its own short transaction. The order is not cosmetic — see the note on Q3.
+//
+// Everything here is decided from the row, never from anything a process remembers. That is
+// what lets any process run the reconciler, lets two run at once, and lets the whole fleet
+// restart without losing track of a single job.
+
+import { errorMessageOf } from './errors'
+import type { KindRegistry, ResolvedKind } from './registry'
+import { runAfterTransition, runTerminalTransition } from './terminal'
+import {
+  cancelPending,
+  markMirrored,
+  park,
+  redrivePending,
+  selectCancelling,
+  selectOrphans,
+  selectStalePending,
+  takeOrphan,
+} from './store'
+import type { DurableJob, Lease, Scope, SqlTransactor } from './types'
+
+export type ReconcileReport = {
+  scanned: number
+  cancelled: number
+  redriven: number
+  parked: number
+  errors: number
+}
+
+export type ReconcilerDeps = {
+  sql: SqlTransactor
+  registry: KindRegistry
+  /** Enqueues a delivery for a job that has just been re-driven. */
+  enqueue: (job: DurableJob) => Promise<void>
+  /** How long past its expiry a lease is tolerated before the job counts as orphaned. Short,
+   *  because lease expiry is a database-clock fact about a driver. */
+  graceMs?: number
+  batchSize?: number
+  /** Repair only this tenant's jobs. Unset means every tenant, which is what a single worker
+   *  should do; set it to shard the loop across a fleet. */
+  tenantId?: string
+  log?: (event: string, fields: Record<string, unknown>) => void
+}
+
+const DEFAULT_GRACE_MS = 20_000
+const DEFAULT_BATCH = 100
+/** Backoff between successive re-drives of the same job, so a job that keeps orphaning does
+ *  not spin. Doubles per re-drive since the last committed unit, capped. */
+const REDRIVE_BASE_MS = 15_000
+const REDRIVE_CAP_MS = 600_000
+
+const scopeOf = (job: DurableJob): Scope => ({ tenantId: job.tenantId, organizationId: job.organizationId })
+const leaseOf = (job: DurableJob): Lease => ({ jobId: job.id, owner: job.leaseOwner ?? 'reconciler', epoch: job.leaseEpoch, ttlMs: 0 })
+
+/**
+ * One pass. Safe to run concurrently with itself: every query takes its rows with
+ * `skip locked`, so two reconcilers partition the work rather than fighting over it.
+ */
+export async function reconcileOnce(deps: ReconcilerDeps): Promise<ReconcileReport> {
+  const report: ReconcileReport = { scanned: 0, cancelled: 0, redriven: 0, parked: 0, errors: 0 }
+  const limit = deps.batchSize ?? DEFAULT_BATCH
+  const graceMs = deps.graceMs ?? DEFAULT_GRACE_MS
+
+  // Q3 — cancellations first.
+  //
+  // A running job whose driver died after an operator asked to cancel it matches both "dead
+  // cancel" and "orphan". Without a precedence, the orphan query would park it as orphaned
+  // with the cancellation never honoured, or re-drive it — restarting work somebody
+  // explicitly asked to stop. So cancellations are settled before anything else looks.
+  for (const job of await select(deps, (tx) => selectCancelling(tx, limit, deps.tenantId))) {
+    report.scanned += 1
+    try {
+      if (await endCancelled(deps, job)) report.cancelled += 1
+    } catch (error) {
+      report.errors += 1
+      deps.log?.('durable_work.reconcile_cancel_failed', { jobId: job.id, error: errorMessageOf(error) })
+    }
+  }
+
+  // Q1 — orphans: a job whose driver stopped heartbeating.
+  for (const job of await select(deps, (tx) => selectOrphans(tx, { graceMs, pendingTtlMs: widestPendingTtl(deps), limit, tenantId: deps.tenantId }))) {
+    report.scanned += 1
+    try {
+      const outcome = await repairOrphan(deps, job, graceMs)
+      if (outcome === 'redriven') report.redriven += 1
+      if (outcome === 'parked') report.parked += 1
+    } catch (error) {
+      report.errors += 1
+      deps.log?.('durable_work.reconcile_orphan_failed', { jobId: job.id, error: errorMessageOf(error) })
+    }
+  }
+
+  // Q2 — pending jobs whose delivery never arrived: a lost hand-back, or an enqueue that
+  // never reached the broker because the process died between commit and enqueue.
+  for (const job of await select(deps, (tx) => selectStalePending(tx, { pendingTtlMs: widestPendingTtl(deps), limit, tenantId: deps.tenantId }))) {
+    report.scanned += 1
+    try {
+      const outcome = await repairPending(deps, job)
+      if (outcome === 'redriven') report.redriven += 1
+      if (outcome === 'parked') report.parked += 1
+    } catch (error) {
+      report.errors += 1
+      deps.log?.('durable_work.reconcile_pending_failed', { jobId: job.id, error: errorMessageOf(error) })
+    }
+  }
+
+  return report
+}
+
+/** Selection commits — and so releases its locks — before any per-row work runs. Holding a
+ *  row lock across a domain mirror would block a second reconciler for the length of that
+ *  mirror, and turn a slow domain into a stalled repair loop. */
+async function select(deps: ReconcilerDeps, query: (tx: Parameters<Parameters<SqlTransactor['transaction']>[0]>[0]) => Promise<DurableJob[]>): Promise<DurableJob[]> {
+  return deps.sql.transaction(query)
+}
+
+/** Selection is a coarse filter, so it uses the widest tolerance any registered kind declares
+ *  and lets the per-row statements re-check with that row's own kind. Selecting on the
+ *  narrowest instead would silently exclude jobs of a more tolerant kind from being repaired
+ *  at all; over-selecting only costs a re-check. */
+function widestPendingTtl(deps: ReconcilerDeps): number {
+  const kinds = deps.registry.list()
+  return kinds.length ? Math.max(...kinds.map((k) => k.lease.pendingTtlMs)) : 900_000
+}
+
+async function endCancelled(deps: ReconcilerDeps, job: DurableJob): Promise<boolean> {
+  const kind = deps.registry.get(job.kind)
+
+  if (job.status === 'pending') {
+    // No lease to fence on; `pending` is the fence.
+    const ended = await deps.sql.transaction(async (tx) => {
+      const row = await cancelPending(tx, job.id)
+      if (!row) return null
+      if (kind?.onCancel) await kind.onCancel(row, scopeOf(row), tx)
+      if (kind?.onTransition) {
+        const { matched } = await kind.onTransition(row, scopeOf(row), tx)
+        if (matched < 1) throw new Error(`Domain mirror matched no rows for job ${row.id}`)
+      }
+      await markMirrored(tx, row.id)
+      return row
+    })
+    if (!ended) return false
+    if (kind) await runAfterTransition(kind, ended, scopeOf(ended))
+    deps.log?.('durable_work.job_cancelled', { jobId: job.id, by: 'reconciler' })
+    return true
+  }
+
+  // A `running` job whose lease has expired: nobody is driving it, so the reconciler settles
+  // the cancellation on its behalf. A live lease is left alone — its own slice will observe
+  // the request at the next heartbeat, which is both faster and safer.
+  if (job.leaseExpiresAt && job.leaseExpiresAt.getTime() > Date.now()) return false
+
+  if (!kind) {
+    const parked = await park(deps.sql, job.id, 'no_handler', 'No handler registered for this kind')
+    return parked != null
+  }
+  const result = await runTerminalTransition(deps.sql, kind, leaseOf(job), scopeOf(job), { type: 'cancel' })
+  if (!result) return false
+  await runAfterTransition(kind, result.job, scopeOf(result.job))
+  deps.log?.('durable_work.job_cancelled', { jobId: job.id, by: 'reconciler' })
+  return true
+}
+
+async function repairOrphan(deps: ReconcilerDeps, job: DurableJob, graceMs: number): Promise<'redriven' | 'parked' | 'skipped'> {
+  const kind = deps.registry.get(job.kind)
+
+  // In order; the first match wins.
+  if (!kind) return (await parkJob(deps, job, undefined, 'no_handler', 'No handler registered for this kind')) ? 'parked' : 'skipped'
+  if (job.errorCode === 'unrecoverable' || job.errorCode === 'retry_exhausted') {
+    // The slice already reached a conclusion but could not commit it. Park with that verdict
+    // preserved — the orphan policy is never consulted for a job that has already decided.
+    return (await parkJob(deps, job, kind, job.errorCode, job.errorMessage)) ? 'parked' : 'skipped'
+  }
+  if (kind.orphanPolicy !== 'redrive') {
+    return (await parkJob(deps, job, kind, 'orphaned', job.errorMessage ?? 'Worker stopped without releasing the lease')) ? 'parked' : 'skipped'
+  }
+  if (job.redrivesSinceCommit >= kind.budget.poisonRedrivesWithoutCommit) {
+    // Re-driven this many times without committing anything: the job is not making progress
+    // and re-running it again is guessing. A human decides from here.
+    return (await parkJob(deps, job, kind, 'poison', 'Re-driven repeatedly without committing progress')) ? 'parked' : 'skipped'
+  }
+
+  const backoffMs = Math.min(REDRIVE_BASE_MS * 2 ** job.redrivesSinceCommit, REDRIVE_CAP_MS)
+  const taken = await takeOrphan(deps.sql, job.id, { graceMs, pendingTtlMs: kind.lease.pendingTtlMs, backoffMs })
+  if (!taken) return 'skipped' // the row moved under us; another pass will see it
+  await deps.enqueue(taken)
+  deps.log?.('durable_work.job_orphaned', { jobId: job.id, redrives: taken.redrives, backoffMs })
+  return 'redriven'
+}
+
+async function repairPending(deps: ReconcilerDeps, job: DurableJob): Promise<'redriven' | 'parked' | 'skipped'> {
+  const kind = deps.registry.get(job.kind)
+  if (!kind) return (await parkJob(deps, job, undefined, 'no_handler', 'No handler registered for this kind')) ? 'parked' : 'skipped'
+
+  // A lost hand-back is cheap and is not evidence that the work is bad, so this budget is
+  // deliberately wider than the poison budget the orphan path uses.
+  if (job.redrivesSinceCommit >= kind.budget.maxRedrives) {
+    return (await parkJob(deps, job, kind, 'never_started', 'Delivery never arrived after repeated re-drives')) ? 'parked' : 'skipped'
+  }
+
+  const redriven = await redrivePending(deps.sql, job.id, { pendingTtlMs: kind.lease.pendingTtlMs })
+  if (!redriven) return 'skipped'
+  await deps.enqueue(redriven)
+  deps.log?.('durable_work.job_redriven', { jobId: job.id, redrives: redriven.redrives, reason: 'never_started' })
+  return 'redriven'
+}
+
+/** Parks through the terminal protocol when the kind has a mirror to run, and through the
+ *  plain statement when it does not — an unregistered kind has no domain row to agree with,
+ *  so "no mirror" is a satisfied mirror rather than a pending one. */
+async function parkJob(
+  deps: ReconcilerDeps,
+  job: DurableJob,
+  kind: ResolvedKind | undefined,
+  reason: string,
+  message: string | null,
+): Promise<boolean> {
+  if (!kind?.onTransition) {
+    const parked = await park(deps.sql, job.id, reason as never, message)
+    if (parked) {
+      await deps.sql.query(`update durable_work_jobs set domain_mirrored_at = now() where id = $1`, [job.id])
+      deps.log?.('durable_work.job_parked', { jobId: job.id, reason })
+    }
+    return parked != null
+  }
+
+  const parked = await deps.sql.transaction(async (tx) => {
+    const row = await park(tx, job.id, reason as never, message)
+    if (!row) return null
+    const { matched } = await kind.onTransition!(row, scopeOf(row), tx)
+    if (matched < 1) throw new Error(`Domain mirror matched no rows for job ${row.id}`)
+    await markMirrored(tx, row.id)
+    return row
+  })
+  if (!parked) return false
+  await runAfterTransition(kind, parked, scopeOf(parked))
+  deps.log?.('durable_work.job_parked', { jobId: job.id, reason })
+  return true
+}

package/src/core/registry.ts

@@ -0,0 +1,199 @@
+// What a caller declares when they hand work to the mechanism, and the process-wide map of
+// those declarations.
+//
+// The registry is process-wide rather than per-container because a job's handler has to be
+// resolvable in every process that can act on the job: the worker runs its slices, the web
+// process re-drives and cancels it, the reconciler parks it. A registration attached to a
+// request-scoped container would be invisible to the next request.
+
+import { UnknownKindError } from './errors'
+import type {
+  BudgetSettings,
+  DurableJob,
+  ErrorClass,
+  Lease,
+  LeaseSettings,
+  RetrySettings,
+  Scope,
+  SliceOutcome,
+  SqlExecutor,
+} from './types'
+
+/** What a slice is given. Everything it needs to make progress and to stop safely. */
+export interface SliceContext<TInput = unknown, TCheckpoint = unknown> {
+  job: DurableJob
+  scope: Scope
+  lease: Lease
+  input: TInput
+  checkpoint: TCheckpoint | null
+
+  /** Aborts on shutdown, on cancellation, and when the lease is lost. A slice that checks it
+   *  at batch boundaries is the difference between a clean stop and a killed process. */
+  signal: AbortSignal
+
+  /** Milliseconds this slice may run before it should hand back. */
+  budgetMs: number
+
+  /** Stable per (job, slice). Forward it to any external side effect so a redelivered slice
+   *  is recognised as the same request rather than a second one. */
+  idempotencyKey: string
+
+  /** Extends the lease and reports progress. `committed: true` records that a unit of work
+   *  is durably written, which resets the failure and orphan budgets. Throws `LeaseLostError`
+   *  when the lease is gone. */
+  heartbeat(patch?: { processedCount?: number; totalCount?: number | null; committed?: boolean }): Promise<void>
+
+  /** Records resume state and counts as a committed unit, under the fence. */
+  checkpoint_(state: TCheckpoint, patch?: { processedCount?: number; totalCount?: number | null }): Promise<void>
+
+  /** Runs `fn` in a transaction that also re-asserts this lease. If the lease is gone the
+   *  transaction rolls back and `LeaseLostError` is thrown — so a worker that lost its lease
+   *  mid-write cannot land a write that outlives its right to make one. */
+  fencedWrite<T>(fn: (tx: SqlExecutor) => Promise<T>): Promise<T>
+
+  /** True once the slice budget is spent or a stop was requested. Check it at boundaries. */
+  shouldYield(): boolean
+}
+
+export interface KindDefinition<TInput = unknown, TCheckpoint = unknown> {
+  kind: string
+  /** Which queue carries this kind. Kinds sharing a queue share a worker's concurrency. */
+  queue: string
+  /** ACL features an operator needs to re-drive this kind, beyond `durable_work.operate`. */
+  requiredFeatures?: string[]
+  concurrency?: number
+
+  lease?: Partial<LeaseSettings>
+  budget?: Partial<BudgetSettings>
+  retry?: { attempts?: number; backoff?: Partial<RetrySettings['backoff']> }
+
+  /** What the reconciler does with an orphan. Defaults to `park`: a job nobody declared
+   *  idempotent is not re-run automatically just because its worker died. */
+  orphanPolicy?: 'redrive' | 'park'
+
+  /** One slice under a held lease. Return when the budget is spent or the signal aborts. */
+  step(ctx: SliceContext<TInput, TCheckpoint>): Promise<SliceOutcome>
+
+  /**
+   * Mirrors a terminal transition onto the domain row, inside the terminal transaction.
+   *
+   * Must be idempotent and must return how many domain rows its update matched: `matched: 0`
+   * is treated exactly like a throw, because "mirrored" means "the domain row agrees", not
+   * "the callback ran". No events, no enqueues — those belong in `onAfterTransition`, which
+   * runs after the commit.
+   */
+  onTransition?(job: DurableJob, scope: Scope, tx: SqlExecutor): Promise<{ matched: number }>
+
+  /**
+   * Re-opens the domain row when an operator re-drives. The mirror image of `onTransition`,
+   * in the same transaction, with the same contract.
+   *
+   * Required whenever `onTransition` is declared — a mirror with no way back would leave an
+   * operator able to re-drive the job while the domain row stays terminal.
+   */
+  onRedrive?(job: DurableJob, scope: Scope, tx: SqlExecutor): Promise<{ matched: number }>
+
+  /** Release external resources before a cancellation commits. Same transaction, same
+   *  idempotency rule as `onTransition`. */
+  onCancel?(job: DurableJob, scope: Scope, tx: SqlExecutor): Promise<void>
+
+  /** After-commit hooks. Best-effort and at-most-once: a throw is logged, never retried, and
+   *  never affects the committed row. Domain events and log writes belong here. */
+  onAfterTransition?(job: DurableJob, scope: Scope): Promise<void>
+  onAfterRedrive?(job: DurableJob, scope: Scope): Promise<void>
+
+  /** Override the default classification for errors this kind understands. */
+  classify?(error: unknown): ErrorClass | null
+}
+
+export const DEFAULT_LEASE: LeaseSettings = { ttlMs: 60_000, sliceBudgetMs: 300_000, pendingTtlMs: 900_000 }
+export const DEFAULT_BUDGET: BudgetSettings = { maxRedrives: 10, maxConsecutiveFailures: 5, poisonRedrivesWithoutCommit: 3 }
+export const DEFAULT_RETRY: RetrySettings = { attempts: 5, backoff: { type: 'exponential', delayMs: 5_000, maxDelayMs: 300_000 } }
+
+export type ResolvedKind<TInput = unknown, TCheckpoint = unknown> = KindDefinition<TInput, TCheckpoint> & {
+  lease: LeaseSettings
+  budget: BudgetSettings
+  retry: RetrySettings
+  orphanPolicy: 'redrive' | 'park'
+  concurrency: number
+}
+
+export function resolveKind<TInput, TCheckpoint>(definition: KindDefinition<TInput, TCheckpoint>): ResolvedKind<TInput, TCheckpoint> {
+  if (definition.onTransition && !definition.onRedrive) {
+    throw new Error(
+      `Kind ${JSON.stringify(definition.kind)} declares onTransition without onRedrive: a domain mirror with no way back would leave an operator able to re-drive the job while the domain row stays terminal.`,
+    )
+  }
+  return {
+    ...definition,
+    lease: { ...DEFAULT_LEASE, ...definition.lease },
+    budget: { ...DEFAULT_BUDGET, ...definition.budget },
+    retry: {
+      attempts: definition.retry?.attempts ?? DEFAULT_RETRY.attempts,
+      backoff: { ...DEFAULT_RETRY.backoff, ...definition.retry?.backoff },
+    },
+    orphanPolicy: definition.orphanPolicy ?? 'park',
+    concurrency: definition.concurrency ?? 1,
+  }
+}
+
+/** The delay before the transport's next attempt, or null when none is coming. */
+export function nextAttemptDelayMs(kind: ResolvedKind, attempt: number): number | null {
+  if (attempt >= kind.retry.attempts) return null
+  const { type, delayMs, maxDelayMs } = kind.retry.backoff
+  const raw = type === 'fixed' ? delayMs : delayMs * 2 ** Math.max(0, attempt - 1)
+  return Math.min(raw, maxDelayMs)
+}
+
+export class KindRegistry {
+  private readonly kinds = new Map<string, ResolvedKind<never, never>>()
+
+  register<TInput, TCheckpoint>(definition: KindDefinition<TInput, TCheckpoint>): void {
+    const resolved = resolveKind(definition)
+    const existing = this.kinds.get(definition.kind)
+    // Re-registering the identical definition is a no-op so a module loaded twice (two entry
+    // points, a test re-import) is not a crash; a *different* definition under the same id is
+    // a genuine conflict and must not be resolved silently by last-write-wins.
+    if (existing && existing.step !== definition.step) {
+      throw new Error(`Duplicate durable job kind ${JSON.stringify(definition.kind)}: two different handlers registered under one id.`)
+    }
+    this.kinds.set(definition.kind, resolved as unknown as ResolvedKind<never, never>)
+  }
+
+  get(kind: string): ResolvedKind | undefined {
+    return this.kinds.get(kind) as ResolvedKind | undefined
+  }
+
+  require(kind: string): ResolvedKind {
+    const found = this.get(kind)
+    if (!found) throw new UnknownKindError(kind)
+    return found
+  }
+
+  has(kind: string): boolean {
+    return this.kinds.has(kind)
+  }
+
+  list(): ResolvedKind[] {
+    return [...this.kinds.values()] as ResolvedKind[]
+  }
+
+  queues(): string[] {
+    return [...new Set(this.list().map((k) => k.queue))]
+  }
+
+  clear(): void {
+    this.kinds.clear()
+  }
+}
+
+/**
+ * The process-wide registry. Modules register into this at import time.
+ *
+ * Module-scoped, so there is exactly one per copy of this package in the process — which is why
+ * anything registering kinds must depend on this package as a PEER, never as a dependency. A
+ * nested second copy would give the adopter its own registry: kinds would register into one,
+ * the worker would read the other, and nothing would run. No error, no warning, just jobs that
+ * sit pending forever while the reconciler eventually parks them `no_handler`.
+ */
+export const registry = new KindRegistry()