@fullstackhouse/open-mercato-durable-work 0.1.0

package/src/core/terminal.ts

@@ -0,0 +1,107 @@
+// The one code path by which a job reaches `completed`, `failed` or `cancelled`.
+//
+// Everything terminal goes through here so there is a single place where "the job row and the
+// domain row move together, or neither moves" is true. The job row is what every surface reads
+// — the UI, the cancel route, the reconciler — so a domain row that went terminal while the
+// job row still said `running` would be repaired by the reconciler as an orphan and re-driven,
+// which is worse than the inconsistency it came from.
+
+import { bumpMirrorAttempts, cancelCas, completeCas, failTerminalCas, markMirrored } from './store'
+import type { ResolvedKind } from './registry'
+import type { DurableJob, ErrorClass, Lease, Scope, SqlExecutor, SqlTransactor } from './types'
+
+/** The domain mirror ran but matched no row: the domain record is gone, or another path
+ *  already moved it. Treated exactly like a throw — "mirrored" means the domain row agrees,
+ *  not that the callback was called. */
+export class DomainMirrorMismatchError extends Error {
+  constructor(readonly jobId: string) {
+    super(`Domain mirror matched no rows for job ${jobId}`)
+    this.name = 'DomainMirrorMismatchError'
+  }
+}
+
+export type Transition =
+  | { type: 'complete'; patch?: { processedCount?: number; totalCount?: number | null } }
+  | { type: 'fail'; code: string; class: ErrorClass; message: string | null }
+  | { type: 'cancel' }
+
+export type TerminalResult = { job: DurableJob; mirrored: boolean }
+
+/**
+ * Runs the terminal CAS and the kind's domain mirror in one transaction.
+ *
+ * Three distinguishable outcomes, and keeping them distinguishable is the point:
+ *   - `null`      the CAS matched no rows. The lease was lost or taken; this delivery has no
+ *                 say any more and should end quietly.
+ *   - a result    committed.
+ *   - a throw     the CAS matched but the mirror failed, so everything rolled back. The caller
+ *                 decides whether that costs a retry (it does on the ordinary completion path,
+ *                 and deliberately does not on the cancellation path).
+ *
+ * An earlier draft returned `null` for both the refused fence and the rolled-back mirror. The
+ * caller then could not tell "someone else owns this" from "try again", which are opposite
+ * instructions.
+ */
+export async function runTerminalTransition(
+  sql: SqlTransactor,
+  kind: ResolvedKind,
+  lease: Lease,
+  scope: Scope,
+  transition: Transition,
+): Promise<TerminalResult | null> {
+  let casMatched = false
+  try {
+    const result = await sql.transaction(async (tx) => {
+      const job = await applyCas(tx, lease, transition)
+      if (!job) return null
+      casMatched = true
+
+      if (transition.type === 'cancel' && kind.onCancel) await kind.onCancel(job, scope, tx)
+
+      if (kind.onTransition) {
+        const { matched } = await kind.onTransition(job, scope, tx)
+        if (matched < 1) throw new DomainMirrorMismatchError(job.id)
+      }
+
+      // Recorded inside the same transaction as the mirror it describes: a job that says its
+      // domain row agrees, when the write that made it agree rolled back, is the exact lie
+      // this protocol exists to prevent.
+      await markMirrored(tx, job.id)
+      return { job, mirrored: true }
+    })
+    return result
+  } catch (error) {
+    // Outside the rolled-back transaction on purpose — a counter written inside it would roll
+    // back with it, and the job would retry its mirror forever with nothing to show for it.
+    if (casMatched) await bumpMirrorAttempts(sql, lease.jobId).catch(() => undefined)
+    throw error
+  }
+}
+
+async function applyCas(tx: SqlExecutor, lease: Lease, transition: Transition): Promise<DurableJob | null> {
+  switch (transition.type) {
+    case 'complete':
+      return completeCas(tx, lease, transition.patch)
+    case 'fail':
+      return failTerminalCas(tx, lease, { code: transition.code, class: transition.class, message: transition.message })
+    case 'cancel':
+      return cancelCas(tx, lease)
+  }
+}
+
+/** Runs the kind's after-commit hook. Best-effort and at-most-once by design: it has already
+ *  been decided that the job is terminal, and a hook that throws must not undo that or be
+ *  retried into a duplicate side effect. */
+export async function runAfterTransition(
+  kind: ResolvedKind,
+  job: DurableJob,
+  scope: Scope,
+  onError?: (error: unknown) => void,
+): Promise<void> {
+  if (!kind.onAfterTransition) return
+  try {
+    await kind.onAfterTransition(job, scope)
+  } catch (error) {
+    onError?.(error)
+  }
+}

package/src/core/types.ts

@@ -0,0 +1,120 @@
+// The public vocabulary of the mechanism. Kept free of Open Mercato and of any transport:
+// `core/` talks to Postgres through `SqlExecutor` and to a broker through `TransportAdapter`,
+// so the same code runs in the OM module, in the CLI worker and in the failure harness.
+
+export type DurableJobStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'
+
+/** Why a slice returned. `budget` hands the rest of the work back without spending a retry. */
+export type SliceOutcome = 'drained' | 'budget' | 'cancelled'
+
+/** What the reconciler does with a job whose driver died. Default `park`: a job that nobody
+ *  declared idempotent is not re-run automatically. */
+export type OrphanPolicy = 'redrive' | 'park'
+
+/** How an error was classified. `transient` is the default for anything unrecognised — the
+ *  transport retries it. `terminal` fails the job now but leaves it re-drivable. `unrecoverable`
+ *  fails it now and requires `{ force: true }` to re-drive. */
+export type ErrorClass = 'transient' | 'terminal' | 'unrecoverable'
+
+/** Written by the mechanism into `error_code`. Park reasons come from the reconciler, verdicts
+ *  from a slice; an operator re-drive clears either. */
+export type ParkReason = 'orphaned' | 'poison' | 'never_started' | 'no_handler'
+export type SliceVerdict = 'unrecoverable' | 'retry_exhausted'
+
+/** Every statement is tenant- and organization-scoped. A null organization is a real value
+ *  (it means tenant-wide), which is why the lock-key index coalesces it to the zero uuid
+ *  rather than relying on NULL-distinctness. */
+export type Scope = { tenantId: string; organizationId: string | null }
+
+/** What a worker holds while it runs a slice. There is deliberately no worker-clock expiry
+ *  here: expiry is a fact about the database's clock, and only the database may judge it. */
+export type Lease = { jobId: string; owner: string; epoch: number; ttlMs: number }
+
+/** The identity of one delivery. A job may be delivered many times; only the delivery whose
+ *  (seq, redrives) still matches the row may claim it. */
+export type Delivery = { jobId: string; seq: number; redrives: number }
+
+/** One row of `durable_work_jobs`, in camelCase. The row IS the authority for liveness:
+ *  status, lease, budgets and cancellation intent are read from nowhere else. */
+export type DurableJob = {
+  id: string
+  tenantId: string
+  organizationId: string | null
+  kind: string
+  status: DurableJobStatus
+  createdBy: string | null
+  createdAt: Date
+  updatedAt: Date
+
+  input: unknown
+  checkpoint: unknown
+  meta: Record<string, unknown> | null
+
+  idempotencyKey: string | null
+  lockKey: string | null
+  subjectType: string | null
+  subjectId: string | null
+  progressJobId: string | null
+
+  leaseOwner: string | null
+  leaseEpoch: number
+  leaseExpiresAt: Date | null
+  heartbeatAt: Date | null
+
+  queueName: string | null
+  queueJobId: string | null
+  continuationSeq: number
+  redrives: number
+  nextRunAt: Date | null
+  pendingSince: Date | null
+
+  redrivesSinceCommit: number
+  consecutiveFailures: number
+  interruptions: number
+  mirrorAttempts: number
+  lastCommittedAt: Date | null
+
+  startedAt: Date | null
+  finishedAt: Date | null
+  parkedAt: Date | null
+  cancelRequestedAt: Date | null
+  cancelledBy: string | null
+  errorClass: ErrorClass | null
+  errorCode: string | null
+  errorMessage: string | null
+  domainMirroredAt: Date | null
+
+  processedCount: number
+  totalCount: number | null
+}
+
+/** The minimum a caller must supply to create a job. `kind` selects the handler; `lockKey`
+ *  is the single-runner key; `idempotencyKey` makes `start` safe to call twice. */
+export type StartJobInput = {
+  kind: string
+  input?: unknown
+  idempotencyKey?: string | null
+  lockKey?: string | null
+  subject?: { type: string; id: string } | null
+  progressJobId?: string | null
+  createdBy?: string | null
+  meta?: Record<string, unknown> | null
+  totalCount?: number | null
+}
+
+/** A minimal SQL surface. Deliberately not MikroORM: the statements are hand-written because
+ *  every one of them is a compare-and-set whose predicate is the actual specification, and an
+ *  ORM would put a layer between that predicate and review. */
+export interface SqlExecutor {
+  query<R = Record<string, unknown>>(text: string, params?: readonly unknown[]): Promise<{ rows: R[]; rowCount: number }>
+}
+
+export interface SqlTransactor extends SqlExecutor {
+  /** Runs `fn` inside one transaction. A throw rolls back and the throw propagates. */
+  transaction<T>(fn: (tx: SqlExecutor) => Promise<T>): Promise<T>
+}
+
+/** Effective per-kind settings after defaults are applied. */
+export type LeaseSettings = { ttlMs: number; sliceBudgetMs: number; pendingTtlMs: number }
+export type BudgetSettings = { maxRedrives: number; maxConsecutiveFailures: number; poisonRedrivesWithoutCommit: number }
+export type RetrySettings = { attempts: number; backoff: { type: 'exponential' | 'fixed'; delayMs: number; maxDelayMs: number } }

package/src/core/worker.ts

@@ -0,0 +1,169 @@
+// The worker process: binds each kind's queue, runs the reconciler tick, and drains on
+// shutdown instead of being killed mid-batch.
+
+import { deliveryId, makeOwnerId, parseDeliveryId, queueNameFor } from './ids'
+import { errorMessageOf, NoFurtherAttempts } from './errors'
+import { reconcileOnce, type ReconcileReport } from './reconciler'
+import { registry as globalRegistry, type KindRegistry, type ResolvedKind } from './registry'
+import { runSlice } from './run-slice'
+import { getJob, recordEnqueue } from './store'
+import type { DurableJob, Delivery, Scope, SqlTransactor } from './types'
+import type { BoundWorker, TransportAdapter } from '../transport/types'
+
+export const RECONCILE_TICK_ID = 'durable-work-reconcile'
+export const RECONCILE_QUEUE = queueNameFor('reconcile')
+
+export type WorkerOptions = {
+  sql: SqlTransactor
+  transport: TransportAdapter
+  registry?: KindRegistry
+  /** Restrict this process to a subset of kinds. Everything registered runs by default. */
+  kinds?: string[]
+  concurrency?: number
+  /** How often the reconciler runs. */
+  tickMs?: number
+  reconcilerGraceMs?: number
+  drainTimeoutMs?: number
+  owner?: string
+  log?: (event: string, fields: Record<string, unknown>) => void
+}
+
+export type DurableWorker = {
+  owner: string
+  /** Runs one reconciler pass immediately. Exposed for the CLI and for tests that would
+   *  otherwise have to wait out a tick. */
+  reconcile(): Promise<ReconcileReport>
+  stop(): Promise<void>
+}
+
+/** Enqueues a delivery for a job, and records the id so a cancellation can remove it. */
+export async function enqueueJob(
+  sql: SqlTransactor,
+  transport: TransportAdapter,
+  kind: ResolvedKind,
+  job: DurableJob,
+  opts: { tx?: Parameters<typeof recordEnqueue>[0] } = {},
+): Promise<void> {
+  const delivery: Delivery = { jobId: job.id, seq: job.continuationSeq, redrives: job.redrives }
+  // The delay comes from the row's own `next_run_at`, computed on the database clock. Reading
+  // it back as a duration here — rather than passing a timestamp to the broker — keeps the
+  // two clocks from having to agree.
+  const delayMs = job.nextRunAt ? Math.max(0, job.nextRunAt.getTime() - Date.now()) : 0
+  const { transportJobId } = await transport.enqueue(job.queueName ?? kind.queue, delivery, {
+    delayMs,
+    retry: kind.retry,
+    tx: opts.tx,
+  })
+  await recordEnqueue(opts.tx ?? sql, job.id, transportJobId, job.queueName ?? kind.queue)
+}
+
+export async function startWorker(options: WorkerOptions): Promise<DurableWorker> {
+  const registry = options.registry ?? globalRegistry
+  const owner = options.owner ?? makeOwnerId()
+  const log = options.log ?? (() => undefined)
+  const { sql, transport } = options
+
+  const kinds = registry.list().filter((k) => !options.kinds || options.kinds.includes(k.kind))
+  const byQueue = new Map<string, ResolvedKind[]>()
+  for (const kind of kinds) byQueue.set(kind.queue, [...(byQueue.get(kind.queue) ?? []), kind])
+
+  const bound: BoundWorker[] = []
+
+  for (const [queue, queueKinds] of byQueue) {
+    const concurrency = options.concurrency ?? Math.max(...queueKinds.map((k) => k.concurrency))
+    // The broker must tolerate a delivery being in flight for longer than a whole slice, or it
+    // redelivers work that is still running — which the lease then refuses, wasting the slice.
+    const activeTimeoutMs = Math.max(...queueKinds.map((k) => k.lease.sliceBudgetMs)) * 2
+
+    bound.push(
+      await transport.bind(
+        queue,
+        async (delivery, ctx) => {
+          const job = await loadJob(sql, delivery)
+          if (!job) {
+            log('durable_work.delivery_orphaned', { jobId: delivery.jobId, queue })
+            return
+          }
+          const kind = registry.get(job.kind)
+          if (!kind) {
+            // Nothing in this process can run it. Leave it alone rather than failing it: a
+            // rolling deploy legitimately has processes that do not yet know a new kind, and
+            // the reconciler parks it if nobody ever claims it.
+            log('durable_work.no_handler', { jobId: job.id, kind: job.kind })
+            return
+          }
+          const scope: Scope = { tenantId: job.tenantId, organizationId: job.organizationId }
+          const result = await runSlice({ sql, kind, owner, log }, delivery, scope, ctx)
+          if (result.outcome === 'yielded' && !ctx.signal.aborted) {
+            // The transport's hand-back may have been refused (a lock lost at exactly the
+            // wrong moment). The row is already `pending` at the next seq, so re-enqueuing
+            // under the new identity cannot collide with anything the broker still holds.
+            const current = await getJob(sql, job.id, scope)
+            if (current && current.status === 'pending') await enqueueJob(sql, transport, kind, current).catch((error) => {
+              log('durable_work.reenqueue_failed', { jobId: job.id, error: errorMessageOf(error) })
+            })
+          }
+        },
+        { concurrency, activeTimeoutMs },
+      ),
+    )
+  }
+
+  const reconcile = () =>
+    reconcileOnce({
+      sql,
+      registry,
+      graceMs: options.reconcilerGraceMs,
+      log,
+      enqueue: async (job) => {
+        const kind = registry.get(job.kind)
+        if (kind) await enqueueJob(sql, transport, kind, job)
+      },
+    })
+
+  // The tick is a repeating delivery owned by the broker rather than a job that re-enqueues
+  // itself: a self-re-enqueue is lost the moment one tick fails, and nothing would notice.
+  let tickWorker: BoundWorker | null = null
+  if (options.tickMs !== 0) {
+    tickWorker = await transport.bind(
+      RECONCILE_QUEUE,
+      async () => {
+        const report = await reconcile()
+        if (report.scanned) log('durable_work.reconciled', report as unknown as Record<string, unknown>)
+      },
+      { concurrency: 1, activeTimeoutMs: 120_000 },
+    )
+    await transport.upsertTick({ id: RECONCILE_TICK_ID, queue: RECONCILE_QUEUE, everyMs: options.tickMs ?? 15_000 })
+    bound.push(tickWorker)
+  }
+
+  return {
+    owner,
+    reconcile,
+    async stop() {
+      // Close the transport first: it stops accepting new deliveries and aborts the signal
+      // every in-flight slice is watching, so they hand back at their next boundary rather
+      // than being cut off between two writes.
+      await transport.close({ timeoutMs: options.drainTimeoutMs ?? 30_000 })
+      await Promise.allSettled(bound.map((worker) => worker.close({ timeoutMs: options.drainTimeoutMs ?? 30_000 })))
+    },
+  }
+}
+
+async function loadJob(sql: SqlTransactor, delivery: Delivery): Promise<DurableJob | null> {
+  // The delivery carries only ids, so the row is read unscoped here and every statement after
+  // it re-scopes from the row's own tenant. A delivery cannot name a scope it should not see:
+  // it can only name a job id that already exists.
+  const result = await sql.query<Record<string, unknown>>(
+    `select tenant_id, organization_id from durable_work_jobs where id = $1`,
+    [delivery.jobId],
+  )
+  if (!result.rows.length) return null
+  const row = result.rows[0]!
+  return getJob(sql, delivery.jobId, {
+    tenantId: String(row.tenant_id),
+    organizationId: row.organization_id == null ? null : String(row.organization_id),
+  })
+}
+
+export { deliveryId, parseDeliveryId, NoFurtherAttempts }

package/src/index.ts

@@ -0,0 +1,100 @@
+// Public API of @fullstackhouse/open-mercato-durable-work.
+//
+// The OM module itself lives at ./modules/durable_work and is loaded by the host through
+// `{ id: 'durable_work', from: '@fullstackhouse/open-mercato-durable-work' }`. Everything
+// exported here is usable without Open Mercato: `core/` speaks to Postgres through
+// `SqlExecutor` and to a broker through `TransportAdapter`, which is what lets the failure
+// harness run the real mechanism with no app around it.
+
+export { metadata } from './modules/durable_work/index'
+export { features } from './modules/durable_work/acl'
+
+export type {
+  Delivery,
+  DurableJob,
+  DurableJobStatus,
+  ErrorClass,
+  Lease,
+  LeaseSettings,
+  BudgetSettings,
+  RetrySettings,
+  OrphanPolicy,
+  ParkReason,
+  Scope,
+  SliceOutcome,
+  SliceVerdict,
+  SqlExecutor,
+  SqlTransactor,
+  StartJobInput,
+} from './core/types'
+
+export {
+  TransientError,
+  TerminalError,
+  UnrecoverableError,
+  LeaseLostError,
+  LockKeyHeldError,
+  NoFurtherAttempts,
+  UnknownKindError,
+  classifyError,
+} from './core/errors'
+
+export {
+  DEFAULT_BUDGET,
+  DEFAULT_LEASE,
+  DEFAULT_RETRY,
+  KindRegistry,
+  nextAttemptDelayMs,
+  registry,
+  resolveKind,
+} from './core/registry'
+export type { KindDefinition, ResolvedKind, SliceContext } from './core/registry'
+
+export {
+  CREATE_INDEXES,
+  CREATE_TABLE,
+  DROP_INDEXES,
+  DROP_TABLE,
+  NO_ORG,
+  SCHEMA_STATEMENTS,
+  TABLE,
+} from './core/schema'
+
+export * as store from './core/store'
+export { runSlice } from './core/run-slice'
+export type { RunSliceDeps, RunSliceResult } from './core/run-slice'
+export { DomainMirrorMismatchError, runAfterTransition, runTerminalTransition } from './core/terminal'
+export type { TerminalResult, Transition } from './core/terminal'
+export { DurableWorkService } from './core/service'
+export type { DurableWorkServiceDeps, RedriveRefusal, StartResult } from './core/service'
+export { reconcileOnce } from './core/reconciler'
+export type { ReconcileReport, ReconcilerDeps } from './core/reconciler'
+export { RECONCILE_QUEUE, RECONCILE_TICK_ID, enqueueJob, startWorker } from './core/worker'
+export type { DurableWorker, WorkerOptions } from './core/worker'
+export { PORTABLE_QUEUE_NAME, deliveryId, makeOwnerId, parseDeliveryId, queueNameFor, sliceIdempotencyKey } from './core/ids'
+
+export type {
+  BindOptions,
+  BoundWorker,
+  DeliveryHandler,
+  DeliveryState,
+  EnqueueOptions,
+  HandlerContext,
+  TransportAdapter,
+  TransportName,
+} from './transport/types'
+export { createTransport, readConfig } from './om/config'
+export type { DurableWorkConfig } from './om/config'
+export { mikroExecutor, mikroTx } from './om/sql-executor-mikro'
+export { createProgressMirror } from './om/progress-mirror'
+export type { ProgressMirror, ProgressServiceLike } from './om/progress-mirror'
+export { DurableWorkJob } from './modules/durable_work/data/entities'
+
+export { MemoryTransport } from './transport/memory'
+export type { MemoryFaults } from './transport/memory'
+export { BullMQTransport } from './transport/bullmq'
+export type { BullMQTransportOptions } from './transport/bullmq'
+export { PgBossTransport } from './transport/pgboss'
+export type { PgBossTransportOptions } from './transport/pgboss'
+export { transportConformance } from './transport/conformance'
+export type { ConformanceHooks } from './transport/conformance'

package/src/modules/durable_work/__integration__/TC-DW-001.spec.ts

@@ -0,0 +1,51 @@
+// The operator surface, against a running app.
+//
+// What these check is the contract an operator actually relies on: that the list is scoped and
+// gated, that a job they cannot re-drive is not offered as re-drivable, and that a refusal says
+// which refusal it is — because "wait for the other run" and "this is already done" call for
+// opposite actions.
+
+import { expect, test } from '@playwright/test'
+import { apiRequest, getAuthToken } from '@open-mercato/core/helpers/integration/api'
+
+test.describe('durable_work operator API', () => {
+  test('requires authentication', async ({ request }) => {
+    const response = await request.get('/api/durable_work/jobs')
+    expect(response.status()).toBe(401)
+  })
+
+  test('lists jobs for an authorised operator', async ({ request }) => {
+    const token = await getAuthToken(request, 'admin')
+    const response = await apiRequest(request, 'GET', '/api/durable_work/jobs', { token })
+
+    expect(response.status()).toBe(200)
+    const body = (await response.json()) as { items: unknown[]; total: number }
+    expect(Array.isArray(body.items)).toBe(true)
+    expect(typeof body.total).toBe('number')
+  })
+
+  test('answers 404 for a job that does not exist, rather than leaking that it might', async ({ request }) => {
+    const token = await getAuthToken(request, 'admin')
+    const response = await apiRequest(request, 'GET', '/api/durable_work/jobs/00000000-0000-0000-0000-000000000000', { token })
+    expect(response.status()).toBe(404)
+  })
+
+  test('refuses to re-drive a job that does not exist', async ({ request }) => {
+    const token = await getAuthToken(request, 'admin')
+    const response = await apiRequest(request, 'POST', '/api/durable_work/jobs/00000000-0000-0000-0000-000000000000/redrive', {
+      token,
+      data: {},
+    })
+    // 409 with a code, not a bare failure: the caller needs to know *which* refusal it is.
+    expect(response.status()).toBe(409)
+    expect((await response.json()) as { error: string }).toMatchObject({ error: 'not_redrivable' })
+  })
+
+  test('rejects a page size a caller could use to pull the whole table', async ({ request }) => {
+    const token = await getAuthToken(request, 'admin')
+    const response = await apiRequest(request, 'GET', '/api/durable_work/jobs?pageSize=100000', { token })
+    expect(response.status()).toBe(200)
+    const body = (await response.json()) as { items: unknown[] }
+    expect(body.items.length).toBeLessThanOrEqual(200)
+  })
+})

package/src/modules/durable_work/__tests__/metadata.test.ts

@@ -0,0 +1,13 @@
+import { metadata } from '../index'
+import { features } from '../acl'
+
+describe('durable_work module metadata', () => {
+  it('declares the module id the host registers', () => {
+    expect(metadata.name).toBe('durable_work')
+  })
+
+  it('declares view and operate features scoped to the module', () => {
+    expect(features.map((f) => f.id)).toEqual(['durable_work.view', 'durable_work.operate'])
+    expect(features.every((f) => f.module === 'durable_work')).toBe(true)
+  })
+})

package/src/modules/durable_work/__tests__/schema-agreement.test.ts

@@ -0,0 +1,52 @@
+// `core/schema.ts` is the definition of the table; the MikroORM entity is a second description
+// of the same thing, for the host's benefit. Two descriptions drift, and the way this
+// particular drift would surface is ugly: the host's migration creates one shape while the
+// statements — every one of them a compare-and-set naming columns explicitly — expect another,
+// so the failure lands at runtime on a predicate rather than at migration time on a schema.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { CREATE_TABLE } from '../../../core/schema'
+
+const HERE = path.dirname(fileURLToPath(import.meta.url))
+
+/** Column names as they appear in the DDL. */
+function ddlColumns(): string[] {
+  const body = CREATE_TABLE.slice(CREATE_TABLE.indexOf('(') + 1, CREATE_TABLE.lastIndexOf(')'))
+  return body
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .map((line) => line.split(/\s+/)[0]!)
+    .filter((name) => /^[a-z_]+$/.test(name))
+    .sort()
+}
+
+/** Column names the entity declares, read from the source rather than from decorator metadata:
+ *  loading metadata would need a configured ORM, and this only has to compare two lists. */
+function entityColumns(): string[] {
+  const source = fs.readFileSync(path.join(HERE, '..', 'data', 'entities.ts'), 'utf8')
+  const names = [...source.matchAll(/@(?:Property|PrimaryKey)\(\{[^}]*name:\s*'([a-z_]+)'/g)].map((m) => m[1]!)
+  // The primary key declares no `name`, so it is added explicitly.
+  return [...new Set([...names, 'id'])].sort()
+}
+
+describe('durable_work_jobs schema', () => {
+  it('declares the same columns in the DDL and in the entity', () => {
+    const ddl = ddlColumns()
+    const entity = entityColumns()
+    expect(ddl.length).toBeGreaterThan(30)
+    expect(entity).toEqual(ddl)
+  })
+
+  it('keeps the columns the fence depends on', () => {
+    // Named individually because losing any one of them silently removes a guarantee rather
+    // than breaking a build: no epoch is no fence, no lock key is no single-runner, no
+    // idempotency key turns a retried start into a second job.
+    for (const column of ['lease_epoch', 'lease_owner', 'lease_expires_at', 'lock_key', 'idempotency_key', 'continuation_seq', 'redrives']) {
+      expect(ddlColumns()).toContain(column)
+    }
+  })
+})

package/src/modules/durable_work/acl.ts

@@ -0,0 +1,6 @@
+export const features = [
+  { id: 'durable_work.view', title: 'View durable jobs', module: 'durable_work' },
+  { id: 'durable_work.operate', title: 'Re-drive and cancel durable jobs', module: 'durable_work' },
+]
+
+export default features

package/src/modules/durable_work/api/jobs/[id]/redrive.ts

@@ -0,0 +1,27 @@
+import { NextResponse } from 'next/server'
+
+import { routeContext, toDto } from '../../../lib/route-helpers'
+
+export const metadata = {
+  POST: { requireAuth: true, requireFeatures: ['durable_work.operate'] },
+}
+
+/**
+ * Runs a stopped job again.
+ *
+ * Each refusal is a 409 with a code rather than a generic failure, because the three have
+ * different answers: wait for or cancel the job holding the lock key; nothing to re-drive;
+ * or say explicitly that re-running an unrecoverable failure is right.
+ */
+export async function POST(req: Request, { params }: { params: { id: string } }) {
+  const ctx = await routeContext(req)
+  if (ctx instanceof NextResponse) return ctx
+
+  const body = (await req.json().catch(() => ({}))) as { force?: boolean }
+  const result = await ctx.service.redrive(params.id, ctx.scope, { force: body.force === true })
+
+  if ('refused' in result) {
+    return NextResponse.json({ error: result.refused, heldBy: result.heldBy }, { status: 409 })
+  }
+  return NextResponse.json(toDto(result))
+}