@fullstackhouse/open-mercato-durable-work 0.1.0

package/src/core/run-slice.ts

@@ -0,0 +1,343 @@
+// The worker body: what happens between a delivery arriving and the job row being right again.
+//
+// The shape is dictated by one rule — after every fenced statement, "matched zero rows" means
+// somebody else owns this job now. When that happens the delivery ends *quietly*: no
+// hand-back (it would redeliver an identity the row refuses), no rethrow (that costs a
+// transport attempt and redelivers the same refused identity), no failure counted (the row is
+// not ours to count against). The new driver owns the row and this delivery is history.
+
+import { LeaseLostError, NoFurtherAttempts, classifyError, errorCodeOf, errorMessageOf } from './errors'
+import { nextAttemptDelayMs, type ResolvedKind, type SliceContext } from './registry'
+import { sliceIdempotencyKey } from './ids'
+import {
+  assertLease,
+  claim,
+  failSlice,
+  heartbeat,
+  releaseLease,
+  writeCheckpoint,
+  yieldSlice,
+} from './store'
+import { runAfterTransition, runTerminalTransition } from './terminal'
+import type { Delivery, DurableJob, Lease, Scope, SliceOutcome, SqlExecutor, SqlTransactor } from './types'
+import type { HandlerContext } from '../transport/types'
+
+export type RunSliceDeps = {
+  sql: SqlTransactor
+  kind: ResolvedKind
+  owner: string
+  /** Structured observability. Every branch below reports; silence is the one outcome that
+   *  would make a stuck job impossible to explain after the fact. */
+  log?: (event: string, fields: Record<string, unknown>) => void
+  now?: () => number
+}
+
+export type RunSliceResult =
+  | { outcome: 'refused'; reason: 'identity' | 'leased' }
+  | { outcome: 'lease_lost' }
+  | { outcome: 'completed' }
+  | { outcome: 'failed'; verdict: 'unrecoverable' | 'retry_exhausted' }
+  | { outcome: 'cancelled' }
+  | { outcome: 'yielded'; seq: number; redrives: number }
+  | { outcome: 'retry' }
+
+/**
+ * Runs one slice of one job.
+ *
+ * @param delivery what the transport delivered — the identity that has to still match the row
+ * @param ctx      the transport's handle on this delivery (attempt number, signal, hand-back)
+ */
+export async function runSlice(
+  deps: RunSliceDeps,
+  delivery: Delivery,
+  scope: Scope,
+  ctx: HandlerContext,
+): Promise<RunSliceResult> {
+  const { sql, kind, owner } = deps
+  const log = deps.log ?? (() => undefined)
+  const now = deps.now ?? (() => Date.now())
+
+  const claimed = await claim(sql, delivery.jobId, scope, delivery, owner, kind.lease.ttlMs)
+  if (!claimed) {
+    // Either the identity moved on (a straggling redelivery) or the lease is still alive
+    // (a duplicate delivery racing its twin). Both end the same way; the distinction is
+    // recorded because it is the difference between a broker quirk and a real overlap.
+    log('durable_work.delivery_refused', { jobId: delivery.jobId, seq: delivery.seq, redrives: delivery.redrives })
+    return { outcome: 'refused', reason: 'identity' }
+  }
+
+  const lease: Lease = { jobId: claimed.id, owner, epoch: claimed.leaseEpoch, ttlMs: kind.lease.ttlMs }
+  const abort = new AbortController()
+  const deadline = now() + kind.lease.sliceBudgetMs
+  let cancelObserved = false
+  let leaseLost = false
+
+  const onExternalAbort = () => abort.abort()
+  ctx.signal.addEventListener('abort', onExternalAbort, { once: true })
+
+  // The heartbeat is what keeps the lease alive and what notices a cancellation. A third of
+  // the TTL leaves room for two missed beats before anyone else may take the job.
+  const beat = async (patch?: Parameters<typeof heartbeat>[2]) => {
+    const result = await heartbeat(sql, lease, patch)
+    if (!result) {
+      leaseLost = true
+      abort.abort()
+      throw new LeaseLostError(lease)
+    }
+    if (result.cancelRequested && !cancelObserved) {
+      cancelObserved = true
+      abort.abort()
+    }
+  }
+  const timer = setInterval(() => {
+    void beat().catch(() => undefined)
+  }, Math.max(1_000, Math.floor(kind.lease.ttlMs / 3)))
+  timer.unref?.()
+
+  const finish = () => {
+    clearInterval(timer)
+    ctx.signal.removeEventListener('abort', onExternalAbort)
+  }
+
+  try {
+    // A verdict already on the claimed row means a previous delivery decided this job is over
+    // but could not commit that decision — the domain mirror failed. Retry the decision, never
+    // the work: the page that produced the verdict is not run again.
+    const verdict = claimed.errorCode
+    if (verdict === 'unrecoverable' || verdict === 'retry_exhausted') {
+      return await retryTerminalFail(deps, claimed, lease, scope, ctx, verdict, log)
+    }
+
+    // The step's errors and the terminal paths' errors need opposite handling, so they are
+    // caught separately. Collapsing them into one catch let an error raised *by* the cancel
+    // path fall back into that same path and mirror the cancellation twice.
+    let outcome: SliceOutcome
+    try {
+      outcome = await kind.step(makeContext(deps, claimed, lease, scope, abort, deadline, beat, now))
+    } catch (error) {
+      if (leaseLost || error instanceof LeaseLostError) {
+        log('durable_work.lease_lost', { jobId: lease.jobId, seq: delivery.seq, redrives: delivery.redrives, epoch: lease.epoch })
+        return { outcome: 'lease_lost' }
+      }
+      if (cancelObserved) return await cancel(deps, lease, scope, ctx, log)
+      // An abort from the transport (shutdown) is not a failure: hand the rest back so the
+      // next process resumes from the committed cursor rather than replaying the slice.
+      if (ctx.signal.aborted) return await handBack(deps, lease, ctx, { interrupted: true }, log)
+      return await fail(deps, lease, scope, ctx, error, log)
+    }
+
+    switch (cancelObserved ? 'cancelled' : outcome) {
+      case 'drained':
+        try {
+          return await complete(deps, lease, scope, ctx, log)
+        } catch (error) {
+          // A mirror rollback on the completion path IS treated as a slice failure: the work
+          // is done but the system does not yet agree it is, and retrying the slice is how
+          // that gets resolved — `step` runs, finds nothing left, drains again, and the
+          // terminal transaction is retried. The cancellation path deliberately differs.
+          return await fail(deps, lease, scope, ctx, error, log)
+        }
+      case 'cancelled':
+        return await cancel(deps, lease, scope, ctx, log)
+      case 'budget':
+        return await handBack(deps, lease, ctx, { interrupted: ctx.signal.aborted }, log)
+    }
+  } finally {
+    finish()
+  }
+}
+
+function makeContext(
+  deps: RunSliceDeps,
+  job: DurableJob,
+  lease: Lease,
+  scope: Scope,
+  abort: AbortController,
+  deadline: number,
+  beat: (patch?: Parameters<typeof heartbeat>[2]) => Promise<void>,
+  now: () => number,
+): SliceContext {
+  const { sql, kind } = deps
+  return {
+    job,
+    scope,
+    lease,
+    input: job.input,
+    checkpoint: job.checkpoint ?? null,
+    signal: abort.signal,
+    budgetMs: kind.lease.sliceBudgetMs,
+    idempotencyKey: sliceIdempotencyKey(job.id, job.continuationSeq),
+    heartbeat: beat,
+    checkpoint_: async (state, patch) => {
+      const ok = await writeCheckpoint(sql, lease, state, patch)
+      if (!ok) {
+        abort.abort()
+        throw new LeaseLostError(lease)
+      }
+    },
+    fencedWrite: async <T>(fn: (tx: SqlExecutor) => Promise<T>): Promise<T> => {
+      return sql.transaction(async (tx) => {
+        // Asserted *inside* the transaction, before the caller's writes, so the lease check
+        // and those writes commit or roll back together. Checking it outside would prove only
+        // that the lease was held a moment before.
+        if (!(await assertLease(tx, lease))) {
+          abort.abort()
+          throw new LeaseLostError(lease)
+        }
+        return fn(tx)
+      })
+    },
+    shouldYield: () => abort.signal.aborted || now() >= deadline,
+  }
+}
+
+async function complete(
+  deps: RunSliceDeps,
+  lease: Lease,
+  scope: Scope,
+  ctx: HandlerContext,
+  log: NonNullable<RunSliceDeps['log']>,
+): Promise<RunSliceResult> {
+  const result = await runTerminalTransition(deps.sql, deps.kind, lease, scope, { type: 'complete' })
+  if (!result) {
+    log('durable_work.lease_lost', { jobId: lease.jobId, at: 'complete' })
+    return { outcome: 'lease_lost' }
+  }
+  log('durable_work.job_completed', { jobId: lease.jobId })
+  await runAfterTransition(deps.kind, result.job, scope, (e) => log('durable_work.after_transition_failed', { jobId: lease.jobId, error: errorMessageOf(e) }))
+  void ctx
+  return { outcome: 'completed' }
+}
+
+async function cancel(
+  deps: RunSliceDeps,
+  lease: Lease,
+  scope: Scope,
+  ctx: HandlerContext,
+  log: NonNullable<RunSliceDeps['log']>,
+): Promise<RunSliceResult> {
+  try {
+    const result = await runTerminalTransition(deps.sql, deps.kind, lease, scope, { type: 'cancel' })
+    if (!result) return { outcome: 'lease_lost' }
+    log('durable_work.job_cancelled', { jobId: lease.jobId })
+    await runAfterTransition(deps.kind, result.job, scope, (e) => log('durable_work.after_transition_failed', { jobId: lease.jobId, error: errorMessageOf(e) }))
+    return { outcome: 'cancelled' }
+  } catch (error) {
+    // A mirror failure during a cancellation is not a slice failure. Counting it would burn
+    // the retry budget and could end the job as `failed` — but a cancellation must always end
+    // as `cancelled`, or the operator who asked for it is told something untrue.
+    await releaseLease(deps.sql, lease, { nextAttemptDelayMs: nextAttemptDelayMs(deps.kind, ctx.attempt) })
+    log('durable_work.cancel_mirror_failed', { jobId: lease.jobId, error: errorMessageOf(error) })
+    throw error
+  }
+}
+
+async function handBack(
+  deps: RunSliceDeps,
+  lease: Lease,
+  ctx: HandlerContext,
+  opts: { interrupted: boolean },
+  log: NonNullable<RunSliceDeps['log']>,
+): Promise<RunSliceResult> {
+  const next = await yieldSlice(deps.sql, lease, opts)
+  if (!next) {
+    log('durable_work.lease_lost', { jobId: lease.jobId, at: 'yield' })
+    return { outcome: 'lease_lost' }
+  }
+  // The row is already `pending` at seq + 1, so this id cannot collide with anything the
+  // broker still holds. If the hand-back itself fails, the reconciler is the backstop — which
+  // is why it is safe to log and move on rather than unwind.
+  try {
+    await ctx.handBack({ jobId: lease.jobId, seq: next.seq, redrives: next.redrives })
+  } catch (error) {
+    log('durable_work.hand_back_failed', { jobId: lease.jobId, seq: next.seq, error: errorMessageOf(error) })
+  }
+  log('durable_work.job_yielded', { jobId: lease.jobId, seq: next.seq, interrupted: opts.interrupted })
+  return { outcome: 'yielded', ...next }
+}
+
+async function fail(
+  deps: RunSliceDeps,
+  lease: Lease,
+  scope: Scope,
+  ctx: HandlerContext,
+  error: unknown,
+  log: NonNullable<RunSliceDeps['log']>,
+): Promise<RunSliceResult> {
+  const { sql, kind } = deps
+  const errorClass = kind.classify?.(error) ?? classifyError(error)
+  const delayMs = errorClass === 'transient' ? nextAttemptDelayMs(kind, ctx.attempt) : null
+
+  const outcome = await failSlice(
+    sql,
+    lease,
+    { message: errorMessageOf(error), code: errorCodeOf(error), class: errorClass },
+    { nextAttemptDelayMs: delayMs, maxConsecutiveFailures: kind.budget.maxConsecutiveFailures },
+  )
+  if (!outcome) {
+    log('durable_work.lease_lost', { jobId: lease.jobId, at: 'fail' })
+    return { outcome: 'lease_lost' }
+  }
+
+  // A `terminal` error has no verdict of its own but must not be retried either; the statement
+  // above only mints `retry_exhausted` and `unrecoverable`, so terminal is decided here.
+  const verdict = outcome.verdict ?? (errorClass === 'terminal' ? 'retry_exhausted' : null)
+  if (!verdict) {
+    log('durable_work.slice_failed', { jobId: lease.jobId, consecutiveFailures: outcome.consecutiveFailures, error: errorMessageOf(error) })
+    throw error // the transport retries this delivery; the released lease lets the retry claim
+  }
+
+  try {
+    const result = await runTerminalTransition(sql, kind, lease, scope, {
+      type: 'fail',
+      code: verdict,
+      class: errorClass,
+      message: errorMessageOf(error),
+    })
+    if (!result) return { outcome: 'lease_lost' }
+    log('durable_work.job_failed', { jobId: lease.jobId, verdict, error: errorMessageOf(error) })
+    await runAfterTransition(kind, result.job, scope, (e) => log('durable_work.after_transition_failed', { jobId: lease.jobId, error: errorMessageOf(e) }))
+    // Ends the delivery without a further attempt: the row is already `failed`, so a retry
+    // would claim a row that refuses it and waste the attempt.
+    throw new NoFurtherAttempts(verdict)
+  } catch (terminalError) {
+    if (terminalError instanceof NoFurtherAttempts) throw terminalError
+    // The mirror failed. The row keeps the verdict and stays `running` with its lease
+    // released, so the transport's next attempt re-runs the *decision* and never the work;
+    // when the attempts run out the reconciler parks it with the verdict preserved.
+    log('durable_work.terminal_mirror_failed', { jobId: lease.jobId, verdict, error: errorMessageOf(terminalError) })
+    throw error
+  }
+}
+
+/** A delivery that claimed a row already carrying a verdict: retry the decision, not the work. */
+async function retryTerminalFail(
+  deps: RunSliceDeps,
+  job: DurableJob,
+  lease: Lease,
+  scope: Scope,
+  ctx: HandlerContext,
+  verdict: 'unrecoverable' | 'retry_exhausted',
+  log: NonNullable<RunSliceDeps['log']>,
+): Promise<RunSliceResult> {
+  try {
+    const result = await runTerminalTransition(deps.sql, deps.kind, lease, scope, {
+      type: 'fail',
+      code: verdict,
+      class: job.errorClass ?? 'terminal',
+      message: job.errorMessage,
+    })
+    if (!result) return { outcome: 'lease_lost' }
+    log('durable_work.job_failed', { jobId: lease.jobId, verdict, retriedTerminal: true })
+    await runAfterTransition(deps.kind, result.job, scope, (e) => log('durable_work.after_transition_failed', { jobId: lease.jobId, error: errorMessageOf(e) }))
+    throw new NoFurtherAttempts(verdict)
+  } catch (error) {
+    if (error instanceof NoFurtherAttempts) throw error
+    // This path re-acquired the lease (the claim did), and `failSlice` never ran — so nothing
+    // else will release it. Without this release the row would sit under a live lease, every
+    // remaining transport retry would be refused by `claim`, and the retry chain would end.
+    await releaseLease(deps.sql, lease, { nextAttemptDelayMs: nextAttemptDelayMs(deps.kind, ctx.attempt) })
+    log('durable_work.terminal_mirror_failed', { jobId: lease.jobId, verdict, retriedTerminal: true, error: errorMessageOf(error) })
+    throw error
+  }
+}

package/src/core/schema.ts

@@ -0,0 +1,114 @@
+// The table. One statement list, used by the OM module's migration and by the harness, so
+// what CI exercises and what a host migrates are the same DDL rather than two texts that
+// drift. Re-homed from core's `progress_jobs` onto a package-owned table (ADR 0003); the
+// column set and index set are the archived spec's §2, unchanged apart from the home.
+
+export const TABLE = 'durable_work_jobs'
+
+/** The zero uuid stands in for "no organization" in the single-runner index.
+ *  Postgres treats NULLs as distinct in a unique index, so a null organization would let two
+ *  tenant-wide jobs hold the same lock key. Coalescing is what makes the guarantee real, and
+ *  it keeps PG14 support (PG15's NULLS NOT DISTINCT was the alternative). */
+export const NO_ORG = '00000000-0000-0000-0000-000000000000'
+
+export const CREATE_TABLE = `
+create table if not exists ${TABLE} (
+  id                      uuid primary key,
+  tenant_id               uuid not null,
+  organization_id         uuid null,
+  kind                    text not null,
+  status                  text not null,
+  created_by              uuid null,
+  created_at              timestamptz not null default now(),
+  updated_at              timestamptz not null default now(),
+
+  input                   jsonb null,
+  checkpoint              jsonb null,
+  meta                    jsonb null,
+
+  idempotency_key         text null,
+  lock_key                text null,
+  subject_type            text null,
+  subject_id              text null,
+  progress_job_id         uuid null,
+
+  lease_owner             text null,
+  lease_epoch             bigint not null default 0,
+  lease_expires_at        timestamptz null,
+  heartbeat_at            timestamptz null,
+
+  queue_name              text null,
+  queue_job_id            text null,
+  continuation_seq        int not null default 0,
+  redrives                int not null default 0,
+  next_run_at             timestamptz null,
+  pending_since           timestamptz null,
+
+  redrives_since_commit   int not null default 0,
+  consecutive_failures    int not null default 0,
+  interruptions           int not null default 0,
+  mirror_attempts         int not null default 0,
+  last_committed_at       timestamptz null,
+
+  started_at              timestamptz null,
+  finished_at             timestamptz null,
+  parked_at               timestamptz null,
+  cancel_requested_at     timestamptz null,
+  cancelled_by            uuid null,
+  error_class             text null,
+  error_code              text null,
+  error_message           text null,
+  domain_mirrored_at      timestamptz null,
+
+  processed_count         int not null default 0,
+  total_count             int null
+)`
+
+/** fillfactor 80 leaves room for HOT updates. Heartbeats rewrite the row every few seconds for
+ *  the length of a multi-day run, and a HOT update avoids touching any index — which is only
+ *  true while no indexed column is written by the heartbeat statement. That is a real
+ *  constraint on the statements below, not a hint: `heartbeat_at` is deliberately unindexed. */
+export const SET_FILLFACTOR = `alter table ${TABLE} set (fillfactor = 80)`
+
+export const CREATE_INDEXES: readonly string[] = [
+  // Single-runner. At most one live job per (lock_key, tenant, org).
+  `create unique index if not exists durable_work_jobs_one_live_per_lock_key
+     on ${TABLE} (lock_key, tenant_id, coalesce(organization_id, '${NO_ORG}'::uuid))
+   where lock_key is not null and status in ('pending','running')`,
+
+  // Idempotency. Re-issuing the same key returns the existing job instead of starting a second.
+  `create unique index if not exists durable_work_jobs_idempotency_uq
+     on ${TABLE} (tenant_id, idempotency_key)
+   where idempotency_key is not null`,
+
+  // Reconciler scans. Each predicate deliberately excludes every column a heartbeat writes,
+  // so a heartbeat never has to update an index (see fillfactor above).
+  `create index if not exists durable_work_jobs_running_idx on ${TABLE} (tenant_id) where status = 'running'`,
+  `create index if not exists durable_work_jobs_pending_idx on ${TABLE} (pending_since) where status = 'pending'`,
+  `create index if not exists durable_work_jobs_cancelling_idx on ${TABLE} (cancel_requested_at)
+   where cancel_requested_at is not null and status in ('pending','running')`,
+  `create index if not exists durable_work_jobs_subject_idx on ${TABLE} (subject_type, subject_id) where subject_type is not null`,
+  `create index if not exists durable_work_jobs_retention_idx on ${TABLE} (finished_at)
+   where status in ('completed','failed','cancelled')`,
+]
+
+export const DROP_INDEXES: readonly string[] = [
+  'drop index if exists durable_work_jobs_one_live_per_lock_key',
+  'drop index if exists durable_work_jobs_idempotency_uq',
+  'drop index if exists durable_work_jobs_running_idx',
+  'drop index if exists durable_work_jobs_pending_idx',
+  'drop index if exists durable_work_jobs_cancelling_idx',
+  'drop index if exists durable_work_jobs_subject_idx',
+  'drop index if exists durable_work_jobs_retention_idx',
+]
+
+export const DROP_TABLE = `drop table if exists ${TABLE}`
+
+/** Every DDL statement in order. Idempotent: safe to run against a database that already has
+ *  the table, which is what makes it usable from both the migration and a test's setup. */
+export const SCHEMA_STATEMENTS: readonly string[] = [CREATE_TABLE, SET_FILLFACTOR, ...CREATE_INDEXES]
+
+/** The index name Postgres reports on a single-runner violation. The store maps that specific
+ *  violation to `LockKeyHeldError`; every other unique violation is a real bug and propagates. */
+export const LOCK_KEY_INDEX = 'durable_work_jobs_one_live_per_lock_key'
+export const IDEMPOTENCY_INDEX = 'durable_work_jobs_idempotency_uq'

package/src/core/service.ts

@@ -0,0 +1,222 @@
+// The API everything else talks to: starting work, operating on it, and reading it.
+//
+// Deliberately thin. The guarantees live in the statements and in `runSlice`; this is where
+// they are composed into the handful of operations a caller actually performs.
+
+import { randomUUID } from 'node:crypto'
+
+import { LockKeyHeldError } from './errors'
+import { queueNameFor } from './ids'
+import { reconcileOnce, type ReconcileReport } from './reconciler'
+import { registry as globalRegistry, type KindRegistry } from './registry'
+import { runAfterTransition } from './terminal'
+import {
+  cancelPending,
+  findLiveByLockKey,
+  getJob,
+  listJobs,
+  markMirrored,
+  operatorRedrive,
+  requestCancel,
+  type ListFilter,
+} from './store'
+import * as store from './store'
+import { enqueueJob } from './worker'
+import type { DurableJob, Scope, SqlExecutor, SqlTransactor, StartJobInput } from './types'
+import type { TransportAdapter } from '../transport/types'
+
+export type StartResult = {
+  job: DurableJob
+  /** False when an existing job was returned for a repeated idempotency key. */
+  created: boolean
+  /**
+   * Publishes the delivery.
+   *
+   * Separate from `start` on purpose. With a transport that cannot enqueue inside the caller's
+   * transaction, enqueuing before the commit would publish a delivery for a job that may never
+   * exist. So the caller commits first and then calls this — and if the process dies in
+   * between, the reconciler picks the job up.
+   */
+  enqueue: () => Promise<void>
+}
+
+export type RedriveRefusal = { refused: 'lock_key_held' | 'not_redrivable' | 'unrecoverable_requires_force'; heldBy?: string }
+
+export type DurableWorkServiceDeps = {
+  sql: SqlTransactor
+  transport: TransportAdapter
+  registry?: KindRegistry
+  graceMs?: number
+  log?: (event: string, fields: Record<string, unknown>) => void
+}
+
+export class DurableWorkService {
+  private readonly registry: KindRegistry
+
+  constructor(private readonly deps: DurableWorkServiceDeps) {
+    this.registry = deps.registry ?? globalRegistry
+  }
+
+  /**
+   * Creates a job.
+   *
+   * `tx` is the caller's transaction, and passing it is the whole point for anyone whose
+   * domain row and job row must agree: they commit together or not at all.
+   */
+  async start(input: StartJobInput, scope: Scope, opts: { tx?: SqlExecutor } = {}): Promise<StartResult> {
+    const kind = this.registry.require(input.kind)
+    const sql = opts.tx ?? this.deps.sql
+    const queue = kind.queue || queueNameFor('default')
+
+    const { job, created } = await store.insertJob(sql, randomUUID(), scope, input, queue)
+
+    return {
+      job,
+      created,
+      enqueue: async () => {
+        if (!created) return // the existing job already has, or will get, a delivery
+        await enqueueJob(this.deps.sql, this.deps.transport, kind, job)
+      },
+    }
+  }
+
+  /** Starts a job and publishes it, transactionally where the transport allows it. */
+  async startAndEnqueue(input: StartJobInput, scope: Scope): Promise<StartResult> {
+    if (this.deps.transport.supportsTransactionalEnqueue) {
+      const kind = this.registry.require(input.kind)
+      return this.deps.sql.transaction(async (tx) => {
+        const started = await this.start(input, scope, { tx })
+        if (started.created) await enqueueJob(this.deps.sql, this.deps.transport, kind, started.job, { tx })
+        return { ...started, enqueue: async () => undefined }
+      })
+    }
+    const started = await this.start(input, scope)
+    await started.enqueue()
+    return started
+  }
+
+  get(id: string, scope: Scope): Promise<DurableJob | null> {
+    return getJob(this.deps.sql, id, scope)
+  }
+
+  list(scope: Scope, filter: ListFilter = {}): Promise<{ items: DurableJob[]; total: number }> {
+    return listJobs(this.deps.sql, scope, filter)
+  }
+
+  /**
+   * Asks a job to stop.
+   *
+   * Never writes a terminal status directly on a running job: the driver has to be given the
+   * chance to stop cleanly at a boundary, and it observes this at its next heartbeat. A job
+   * that nobody is driving is ended here and now, because there is nobody to observe anything.
+   */
+  async cancel(id: string, scope: Scope, by: string | null = null): Promise<DurableJob | null> {
+    const requested = await requestCancel(this.deps.sql, id, scope, by)
+    if (!requested) return null
+
+    if (requested.status === 'pending') {
+      const kind = this.registry.get(requested.kind)
+      const ended = await this.deps.sql
+        .transaction(async (tx) => {
+          const row = await cancelPending(tx, id)
+          if (!row) return null
+          if (kind?.onCancel) await kind.onCancel(row, scope, tx)
+          if (kind?.onTransition) {
+            const { matched } = await kind.onTransition(row, scope, tx)
+            if (matched < 1) throw new Error(`Domain mirror matched no rows for job ${row.id}`)
+          }
+          await markMirrored(tx, row.id)
+          return row
+        })
+        // A failed mirror must not lose the operator's request: the row keeps
+        // `cancel_requested_at`, so the reconciler settles it on a later tick.
+        .catch(() => null)
+
+      if (ended) {
+        if (requested.queueJobId && requested.queueName) {
+          await this.deps.transport.remove(requested.queueName, requested.queueJobId).catch(() => undefined)
+        }
+        if (kind) await runAfterTransition(kind, ended, scope)
+        return ended
+      }
+    }
+    return requested
+  }
+
+  /**
+   * The operator's way back from a failed job.
+   *
+   * Refuses rather than guesses in three situations, because each has a different answer:
+   * another live job holds the lock key (wait or cancel that one), the job is not in a state
+   * a re-drive applies to (completed and cancelled jobs are done), and an unrecoverable
+   * failure (someone must say explicitly that running it again is right).
+   */
+  async redrive(
+    id: string,
+    scope: Scope,
+    opts: { force?: boolean } = {},
+  ): Promise<DurableJob | RedriveRefusal> {
+    const existing = await getJob(this.deps.sql, id, scope)
+    if (!existing) return { refused: 'not_redrivable' }
+    if (!opts.force && existing.errorCode === 'unrecoverable') return { refused: 'unrecoverable_requires_force' }
+
+    if (existing.lockKey) {
+      const holder = await findLiveByLockKey(this.deps.sql, scope, existing.lockKey)
+      if (holder && holder.id !== id) return { refused: 'lock_key_held', heldBy: holder.id }
+    }
+
+    const kind = this.registry.get(existing.kind)
+    try {
+      const redriven = await this.deps.sql.transaction(async (tx) => {
+        const row = await operatorRedrive(tx, id, scope, {
+          graceMs: this.deps.graceMs ?? 20_000,
+          pendingTtlMs: kind?.lease.pendingTtlMs ?? 900_000,
+          force: opts.force ?? false,
+        })
+        if (!row) return null
+        // The domain row is re-opened in the same transaction that re-opens the job row. A
+        // mirror with no way back would leave an operator able to restart the job while the
+        // domain record stayed terminal.
+        if (kind?.onRedrive) {
+          const { matched } = await kind.onRedrive(row, scope, tx)
+          if (matched < 1) throw new Error(`Domain re-open matched no rows for job ${row.id}`)
+        }
+        return row
+      })
+      if (!redriven) return { refused: 'not_redrivable' }
+
+      if (kind) {
+        await enqueueJob(this.deps.sql, this.deps.transport, kind, redriven)
+        if (kind.onAfterRedrive) await kind.onAfterRedrive(redriven, scope).catch(() => undefined)
+      }
+      this.deps.log?.('durable_work.job_redriven', { jobId: id, by: 'operator', redrives: redriven.redrives })
+      return redriven
+    } catch (error) {
+      // The partial unique index is the last word on the single-runner guarantee: a job that
+      // started between the check above and this statement raises here, and the answer is the
+      // same refusal rather than a second live runner.
+      if ((error as { code?: string })?.code === '23505') {
+        const holder = existing.lockKey ? await findLiveByLockKey(this.deps.sql, scope, existing.lockKey) : null
+        return { refused: 'lock_key_held', heldBy: holder?.id }
+      }
+      throw error
+    }
+  }
+
+  reconcile(opts: { batchSize?: number; tenantId?: string } = {}): Promise<ReconcileReport> {
+    return reconcileOnce({
+      sql: this.deps.sql,
+      registry: this.registry,
+      graceMs: this.deps.graceMs,
+      log: this.deps.log,
+      batchSize: opts.batchSize,
+      tenantId: opts.tenantId,
+      enqueue: async (job) => {
+        const kind = this.registry.get(job.kind)
+        if (kind) await enqueueJob(this.deps.sql, this.deps.transport, kind, job)
+      },
+    })
+  }
+}
+
+export { LockKeyHeldError }