@ultimat3/jobs 1.2.0 → 2.0.0

package/src/execute.ts

@@ -0,0 +1,289 @@
+// One claimed job run to completion, suspension or failure and settled with the driver — the
+// single execution path the worker loop and `x jobs run` share. It owns the run's deadline, and
+// a deadline here means CANCEL: the nack that follows makes the job claimable again, so a body
+// still running past it would be a second copy of one job, racing the attempt that replaced it.
+
+import type { Actor, Clock, Ctx, ServiceBag } from '@ultimat3/core';
+import {
+  anonymousActor,
+  isUltimateError,
+  logger,
+  reportError,
+  runWithContext,
+  useContext,
+  withChildContext,
+} from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { ClaimedJob, JobDriver } from './driver';
+import { JobAbortedError, JobTimeoutError } from './errors';
+import { eventBus } from './events';
+import type { AnyJobHandle } from './job';
+import { nextRetry } from './retry';
+import type { EventLookup, StepRecord } from './steps';
+import { createStepRunner, isStepSuspension } from './steps';
+import { jobRunActor } from './tenant';
+
+export type JobOutcome = 'completed' | 'suspended' | 'retried' | 'dead-lettered';
+
+/** Stands in for a caller with nothing to cancel, so the composition below has one shape. */
+const NEVER_ABORTED = new AbortController().signal;
+
+/**
+ * `Ctx.signal` is non-optional in the type and `createContext` always sets it — but a context can
+ * still arrive across a cast (`@ultimat3/http`'s `asCtx`, a test's `{} as Ctx`) without one, and
+ * a job that crashed on a missing field would be a far worse answer than a job with no caller to
+ * follow. Read it, do not assume it.
+ */
+function callerSignal(ctx: Ctx): AbortSignal {
+  return ctx.signal instanceof AbortSignal ? ctx.signal : NEVER_ABORTED;
+}
+
+/**
+ * The same defensive read, for the same reason: `Ctx.actor` is non-optional in the type and
+ * `createContext` always sets it, but `WorkerOptions.context()` is the app's own function and a
+ * cast context (`{} as Ctx`) reaches here without one. Anonymous is the honest stand-in — it
+ * carries no org, so the job's declared tenant is the only thing that can put one on the run.
+ */
+function callerActor(ctx: Ctx): Actor {
+  const actor: Actor | undefined = ctx.actor;
+  return actor === undefined ? anonymousActor() : actor;
+}
+
+const NO_SERVICES: ServiceBag = Object.freeze({});
+
+/**
+ * The same defensive read a third time, and this one is load-bearing rather than merely kind:
+ * `withChildContext` ITERATES the parent's bag to decide what carries forward, so a cast context
+ * with no `services` would fail the run with a `TypeError` before the body ever started.
+ */
+function callerServices(ctx: Ctx): ServiceBag {
+  const services: ServiceBag | undefined = ctx.services;
+  return services === undefined ? NO_SERVICES : services;
+}
+
+export interface JobExecution {
+  readonly outcome: JobOutcome;
+  readonly jobId: string;
+  readonly job: string;
+  readonly attempt: number;
+  readonly durationMs: number;
+  readonly resumeAt?: number;
+  readonly error?: string;
+  readonly steps: readonly StepRecord[];
+  readonly replayed: readonly string[];
+}
+
+export interface ExecuteJobOptions {
+  readonly driver: JobDriver;
+  readonly claimed: ClaimedJob;
+  readonly handle: AnyJobHandle;
+  readonly ctx: Ctx;
+  readonly clock?: Clock;
+  readonly events?: EventLookup;
+}
+
+/**
+ * Run one claimed job to completion, suspension or failure, and settle it with the driver.
+ * Shared by the worker loop and `x jobs run` so both take exactly the same code path.
+ */
+export async function executeJob(options: ExecuteJobOptions): Promise<JobExecution> {
+  const { driver, claimed, handle } = options;
+  const startedAt = nowMs(options.clock);
+  // This attempt's cancellation. `ctx.signal` is the framework's ONE cancellation seam — the same
+  // one `throwIfAborted` reads in an action — so a job body learns its deadline passed exactly
+  // where every other body does, with no jobs-only parameter to know about. Composed with the
+  // caller's signal rather than replacing it: a ctx that was already going away still is.
+  const cancel = new AbortController();
+  const signal = AbortSignal.any([callerSignal(options.ctx), cancel.signal]);
+  const ctx: Ctx = Object.freeze({
+    ...options.ctx,
+    signal,
+    services: callerServices(options.ctx),
+  });
+  const runner = createStepRunner({
+    runId: claimed.runId,
+    jobName: handle.name,
+    store: driver.steps,
+    signal,
+    ...(options.clock === undefined ? {} : { clock: options.clock }),
+    events: options.events ?? eventBus(),
+  });
+
+  const settle = async (outcome: JobExecution): Promise<JobExecution> => {
+    const steps = await driver.steps.list(claimed.runId);
+    return { ...outcome, steps, replayed: runner.replayedNames() };
+  };
+
+  try {
+    const input = handle.parse(claimed.input);
+    // The job's DECLARED tenant, on the actor the body runs as. `tenant: 'none'` strips the org
+    // rather than inheriting the worker's, so a tenant-scoped read inside such a job fails closed.
+    const runActor = jobRunActor(callerActor(ctx), handle.tenantFor(input));
+    // Installed as the AMBIENT context and not only handed over as a parameter. This is the whole
+    // of the fix: `@ultimat3/entity`'s tenant guard derives from `tryUseContext()`, so a ctx passed
+    // as an argument was read by nobody — `actorTenant` answered `undefined`, `scopedPlan` derived
+    // no predicate, `verifyScope` returned early, and a row naming another org was written by a
+    // job while the identical write over HTTP was refused as `X_TENANCY_ACTOR_MISMATCH`.
+    //
+    // `withChildContext` and NOT a spread of `ctx`, for the reason it exists: a registered service
+    // CLOSES OVER the context it was built for (`defineService`), so the worker's `ctx.posts` would
+    // still answer the worker's org while every ambient repository call answered the job's — one
+    // run acting as two tenants, which is the same hole one layer up. It rebuilds every managed
+    // factory against `runActor` and carries only the services no factory owns.
+    const work = runWithContext(ctx, () =>
+      withChildContext({ actor: runActor }, () =>
+        handle.run({
+          input,
+          step: runner.step,
+          // The child itself, never a rebuilt sibling: the ctx a body is HANDED and the ctx the
+          // entity guard READS have to be one object, which is what `tenancy-cross-surface` pins.
+          ctx: useContext(),
+          attempt: claimed.attempt,
+          jobId: claimed.id,
+          runId: claimed.runId,
+        }),
+      ),
+    );
+
+    await (handle.timeoutMs === undefined
+      ? work
+      : raceTimeout(work, handle.timeoutMs, handle.name, cancel));
+  } catch (error) {
+    if (isStepSuspension(error)) {
+      const delayMs = Math.max(0, error.resumeAt - nowMs(options.clock));
+      // countsAsAttempt: false — parking a run is not a failure.
+      await driver.nack(claimed.id, { delayMs, countsAsAttempt: false });
+      return settle({
+        outcome: 'suspended',
+        jobId: claimed.id,
+        job: handle.name,
+        attempt: claimed.attempt,
+        durationMs: nowMs(options.clock) - startedAt,
+        resumeAt: error.resumeAt,
+        steps: [],
+        replayed: [],
+      });
+    }
+
+    const message = error instanceof Error ? error.message : String(error);
+    const decision = nextRetry(handle.retry, claimed.attempt);
+    await driver.nack(claimed.id, {
+      delayMs: decision.delayMs,
+      error: message,
+      countsAsAttempt: true,
+      deadLetter: !decision.retry && decision.deadLetter,
+    });
+    logger.warn('jobs.attempt.failed', {
+      job: handle.name,
+      jobId: claimed.id,
+      attempt: claimed.attempt,
+      retry: decision.retry,
+      error: message,
+    });
+    // This package's ONE error-reporting call site, and it is here rather than in the loop because
+    // this is the only frame that still holds the thrown value — the loop sees a message string.
+    // A retry is a failure the framework recovered from, so it is a `warning`; a dead letter is
+    // one nobody recovered from. `x jobs run` takes this path too, which is the point: one
+    // execution path means one place a failed job can become visible.
+    reportError(error, {
+      source: 'job',
+      severity: decision.retry ? 'warning' : 'error',
+      scope: {
+        operation: handle.name,
+        extra: {
+          jobId: claimed.id,
+          runId: claimed.runId,
+          attempt: claimed.attempt,
+          retry: decision.retry,
+        },
+      },
+    });
+    return settle({
+      outcome: decision.retry ? 'retried' : 'dead-lettered',
+      jobId: claimed.id,
+      job: handle.name,
+      attempt: claimed.attempt,
+      durationMs: nowMs(options.clock) - startedAt,
+      error: message,
+      steps: [],
+      replayed: [],
+    });
+  } finally {
+    // The attempt is over however it ended, so nothing from it may still be writing: a step left
+    // in flight by a handler that returned without awaiting it settles into a run the next
+    // attempt already owns. The runner fences its writes on this signal. A second `abort()` keeps
+    // the first reason, so a timed-out run still reports the timeout, not this.
+    cancel.abort(new JobAbortedError({ job: handle.name }));
+  }
+
+  // Only reachable when the BODY succeeded, and settlement is deliberately outside the catch
+  // above: an `ack` that rejects — a pool timeout, a reset on that one statement — is not an
+  // attempt failure. Nacking it would re-queue work that already ran to completion and report
+  // the run as `retried`, so `jobs_total{outcome}` would count a failure that never happened.
+  // Let it reach the worker instead, which logs `jobs.worker.settle-failed`; the lease then
+  // lapses and the queue re-delivers, which is the honest outcome for "we could not say it ended".
+  await driver.ack(claimed.id);
+  return settle({
+    outcome: 'completed',
+    jobId: claimed.id,
+    job: handle.name,
+    attempt: claimed.attempt,
+    durationMs: nowMs(options.clock) - startedAt,
+    steps: [],
+    replayed: [],
+  });
+}
+
+/**
+ * The run's deadline. It CANCELS before it rejects, and the order is the whole point: the caller
+ * nacks on this rejection and the queue hands the job straight to another worker, so the body has
+ * to have been told to stop before that becomes possible.
+ *
+ * Nothing in JS can kill a body that ignores the signal, so what is left is to say so: a run that
+ * settles after its deadline logs `jobs.timeout.abandoned`, which is how an app finds the handler
+ * that never reads `ctx.signal`. A body that stopped BECAUSE it was cancelled is the intended end
+ * and stays quiet.
+ */
+function raceTimeout(
+  work: Promise<unknown>,
+  timeoutMs: number,
+  job: string,
+  cancel: AbortController,
+): Promise<unknown> {
+  return new Promise((resolve, reject) => {
+    let expired = false;
+    const timer = setTimeout(() => {
+      expired = true;
+      const failure = new JobTimeoutError({ job, timeoutMs });
+      cancel.abort(failure);
+      reject(failure);
+    }, timeoutMs);
+    const abandoned = (ended: 'resolved' | 'rejected', error?: unknown): void => {
+      logger.warn('jobs.timeout.abandoned', {
+        job,
+        timeoutMs,
+        ended,
+        ...(error === undefined
+          ? {}
+          : { error: error instanceof Error ? error.message : String(error) }),
+      });
+    };
+    work.then(
+      (value) => {
+        clearTimeout(timer);
+        if (expired) abandoned('resolved');
+        else resolve(value);
+      },
+      (error) => {
+        clearTimeout(timer);
+        if (!expired) reject(error);
+        else if (!isCancellation(error, cancel.signal.reason)) abandoned('rejected', error);
+      },
+    );
+  });
+}
+
+/** The body stopped because we cancelled it: our own reason back, or a fenced step write. */
+function isCancellation(error: unknown, reason: unknown): boolean {
+  return error === reason || (isUltimateError(error) && error.code === 'X_ABORTED');
+}

package/src/heartbeat.ts

@@ -0,0 +1,146 @@
+// One claimed job's lease: the heartbeat that renews it, and the fact no driver reports — that
+// renewals stopped landing long enough for the queue to hand this job to another worker. A lease
+// that lapses in silence is one job running twice with nothing in the log saying so, which is the
+// hardest bug in a queue to see from the outside and the easiest to name from in here.
+
+import type { Clock } from '@ultimat3/core';
+import { logger, recordLeaseLost } from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { ClaimedJob, JobDriver } from './driver';
+import { LeaseLostError } from './errors';
+
+export interface LeaseHeartbeatOptions {
+  /** Only `heartbeat` is used — a lease renews itself and settles nothing. */
+  readonly driver: Pick<JobDriver, 'heartbeat'>;
+  readonly claimed: ClaimedJob;
+  readonly visibilityTimeoutMs: number;
+  readonly intervalMs: number;
+  readonly workerId: string;
+  readonly clock?: Clock;
+}
+
+export interface LeaseHeartbeat {
+  /** One renewal. The interval drives it; tests drive it instead of the timer. */
+  renew(): Promise<void>;
+  /** True once the lease lapsed: the queue may already have delivered this job again. */
+  lost(): boolean;
+  /**
+   * Aborts when the lease is gone. The worker folds it into the `Ctx` it hands `executeJob`, so a
+   * job cancelled from outside — or one whose lease lapsed and whose row another worker
+   * re-claimed — stops at the next renewal instead of running to the end beside its replacement.
+   * `steps.ts` refuses every write past it, which is what unwinds a body that ignores the signal.
+   */
+  readonly signal: AbortSignal;
+  /** Stop renewing. Idempotent. */
+  stop(): void;
+}
+
+/**
+ * Start renewing the lease `claim()` bought, and say so out loud when renewal stops working.
+ *
+ * The lease is measured on THIS process's clock, not on `claimed.visibleAt`: that timestamp comes
+ * from the driver's clock, and comparing the two makes every lease decision a function of clock
+ * skew. Measured from the moment renewal started, the window can only ever be read as shorter than
+ * it really is — reporting late is safe, reporting early would cry loss over a live lease.
+ */
+export function startLeaseHeartbeat(options: LeaseHeartbeatOptions): LeaseHeartbeat {
+  const { claimed, visibilityTimeoutMs, workerId } = options;
+  const now = (): number => nowMs(options.clock);
+  let renewedAt = now();
+  let renewing = false;
+  let lost = false;
+  let timer: ReturnType<typeof setInterval> | undefined;
+  const gone = new AbortController();
+
+  const stop = (): void => {
+    if (timer !== undefined) clearInterval(timer);
+    timer = undefined;
+  };
+
+  const lapsed = (): boolean => now() - renewedAt >= visibilityTimeoutMs;
+
+  /**
+   * Reported once, then renewal stops. Once the window is gone this job is claimable by anyone,
+   * so a further renewal would extend a lease this process no longer holds — and a counter that
+   * ticked once per interval would count intervals, not jobs.
+   */
+  const reportLost = (error?: unknown, reason?: 'expired' | 'not-ours'): void => {
+    if (lost) return;
+    lost = true;
+    stop();
+    logger.error('jobs.lease.lost', {
+      workerId,
+      job: claimed.name,
+      jobId: claimed.id,
+      queue: claimed.queue,
+      attempt: claimed.attempt,
+      visibilityTimeoutMs,
+      reason: reason ?? 'expired',
+      ...(error === undefined
+        ? {}
+        : { error: error instanceof Error ? error.message : String(error) }),
+    });
+    recordLeaseLost(claimed.queue);
+    // Cancel LAST, so the loss is logged and counted before the body it unwinds starts throwing.
+    gone.abort(new LeaseLostError({ job: claimed.name, jobId: claimed.id }));
+  };
+
+  const renew = async (): Promise<void> => {
+    if (lost) return;
+    // Expiry is decided BEFORE the driver is asked, because the failure that loses a lease most
+    // quietly is the one that never answers: a heartbeat hung on a dead connection neither
+    // resolves nor rejects, so a check that ran only on rejection would never run at all.
+    if (lapsed()) {
+      reportLost();
+      return;
+    }
+    // One renewal in flight at a time. A driver slower than the interval would otherwise stack a
+    // request per tick onto the connection that is already the thing failing.
+    if (renewing) return;
+    renewing = true;
+    try {
+      const held = await options.driver.heartbeat(claimed.id, { visibilityTimeoutMs, workerId });
+      // The driver answered, and it said the row is not ours. That is a DIFFERENT fact from an
+      // expired window and the only one an operator can cause on purpose: `x jobs cancel` writes
+      // a terminal state, and the renewal that misses it is what tells this attempt to stop. It
+      // is not a failure of the queue, so it is reported once and the attempt is unwound.
+      // `=== false`, not `!held`: `heartbeat` only recently began answering, and a driver from
+      // before that — a hand-rolled one, a test double — resolves `undefined`. Reading that as a
+      // lost lease would cancel every job on every renewal. Only an explicit "no" is a no.
+      if (held === false) {
+        reportLost(undefined, 'not-ours');
+        return;
+      }
+      // Asked again AFTER the call, because a renewal that SUCCEEDS late is still late: an
+      // event-loop stall or a driver that answered at the end of a connect timeout can land this
+      // past the window it was renewing, and `renewedAt = now()` there would restart the clock on
+      // a lease the queue has already re-delivered — the loss hidden by the very call meant to
+      // prevent it.
+      if (lapsed()) {
+        reportLost();
+        return;
+      }
+      renewedAt = now();
+    } catch (error) {
+      // One failed renewal is not a lost lease: there is a whole visibility window left to land
+      // the next one, and the default interval gives three tries inside it. Say it at warn and
+      // let the window decide.
+      logger.warn('jobs.heartbeat.failed', {
+        workerId,
+        job: claimed.name,
+        jobId: claimed.id,
+        attempt: claimed.attempt,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      if (lapsed()) reportLost(error);
+    } finally {
+      renewing = false;
+    }
+  };
+
+  timer = setInterval(() => {
+    void renew();
+  }, options.intervalMs);
+
+  return { renew, lost: () => lost, signal: gone.signal, stop };
+}

package/src/index.ts

@@ -10,6 +10,54 @@ import './register';
 /** Re-exported so a `job`/`task` file needs one import, not two. Same object as schema's. */
 export type { Infer } from '@ultimat3/schema';
 export { t } from '@ultimat3/schema';
+export type {
+  BackfillBatch,
+  BackfillDefinition,
+  BackfillInput,
+  BackfillReport,
+} from './backfill';
+export { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
+export type { BackfillGate, BackfillGateInput } from './backfill-gate';
+export { checkBackfillEnvironment, gateBackfill } from './backfill-gate';
+export type { BackfillProgress } from './backfill-inspect';
+export { backfillForRun, inspectBackfills, toBackfillProgress } from './backfill-inspect';
+export type {
+  BackfillFilter,
+  BackfillLedger,
+  BackfillRun,
+  BackfillStatus,
+  BackfillVerdict,
+} from './backfill-ledger';
+export {
+  BACKFILL_STATUSES,
+  backfillChecksum,
+  createMemoryBackfillLedger,
+  decideBackfill,
+  isBackfillStatus,
+} from './backfill-ledger';
+export type {
+  BackfillPendingReport,
+  BackfillState,
+  BackfillStateRow,
+} from './backfill-pending';
+export {
+  BACKFILL_STATES,
+  isPendingBackfillState,
+  PENDING_BACKFILL_STATES,
+  pendingBackfills,
+} from './backfill-pending';
+export type { Pacer, PacerOptions } from './backfill-rate';
+export { createPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
+export type { BackfillCount, BackfillDeclaration, BackfillOrigin } from './backfill-registry';
+// `stampBackfill` is deliberately absent, for the reason `registerJob` is: a second way to make a
+// handle claim it is a backfill would let a plain `job()` inherit the pending diff and the gate.
+export {
+  backfillOrigin,
+  declarationOf,
+  getBackfill,
+  isBackfill,
+  registeredBackfills,
+} from './backfill-registry';
 export type { JobDescriptor } from './describe';
 export type {
   ClaimedJob,
@@ -17,6 +65,7 @@ export type {
   ConflictPolicy,
   EnqueueRequest,
   EnqueueResult,
+  HeartbeatOptions,
   JobDriver,
   JobFilter,
   JobIntrospection,
@@ -41,11 +90,27 @@ export { createPgDriver, createPgLeader } from './driver-pg';
 export {
   SQL_ACK,
   SQL_ADVISORY_UNLOCK,
+  SQL_BACKFILL_FINISH,
+  SQL_BACKFILL_LIST,
+  SQL_BACKFILL_PROGRESS,
+  SQL_BACKFILL_START,
+  SQL_CANCEL,
   SQL_CLAIM,
   SQL_ENQUEUE,
   SQL_HEARTBEAT,
   SQL_JOBS_TABLE,
+  SQL_LEADER_ACQUIRE,
+  SQL_LEADER_RELEASE,
+  SQL_LEASE_ACQUIRE,
+  SQL_LEASE_RELEASE,
+  SQL_LEASE_RENEW,
   SQL_NACK,
+  SQL_OUTBOX_CLAIM,
+  SQL_OUTBOX_MARK_PUBLISHED,
+  SQL_OUTBOX_STAGE,
+  SQL_OUTBOX_TABLE,
+  SQL_SCHEDULER_STATE_GET,
+  SQL_SCHEDULER_STATE_MARK,
   SQL_STATS,
   SQL_STEP_GET,
   SQL_STEP_PUT,
@@ -55,20 +120,38 @@ export type { RedisDriverOptions } from './driver-redis';
 export { createRedisDriver } from './driver-redis';
 export type { JobErrorCode } from './errors';
 export {
+  BackfillAppliedError,
+  BackfillEnvironmentError,
+  BackfillMigrationPendingError,
+  BackfillPendingError,
+  BackfillRunningError,
+  BackfillStalledError,
+  BackfillUnknownError,
+  CancelUnsupportedError,
+  ConcurrencyUnenforceableError,
   DriverUnavailableError,
   IdempotencyRequiredError,
   JOB_ERROR_CODES,
   JOB_ERROR_TITLES,
+  JobAbortedError,
   JobDuplicateError,
   JobMaxAttemptsError,
   JobNameTakenError,
+  JobNotCancellableError,
+  JobSlotLostError,
   JobsNotImplementedError,
+  JobTenantRequiredError,
   JobTimeoutError,
+  LeaseLostError,
   OutboxNoTxError,
   StepDuplicateError,
 } from './errors';
 export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
 export { createMemoryEventBus, eventBus, publishEvent, setEventBus } from './events';
+export type { PgEventBusOptions } from './events-pg';
+export { createPgEventBus } from './events-pg';
+export type { ExecuteJobOptions, JobExecution, JobOutcome } from './execute';
+export { executeJob } from './execute';
 export type {
   DeadLetterEntry,
   JobsManifest,
@@ -77,6 +160,7 @@ export type {
   StepTrace,
 } from './inspect';
 export {
+  cancelJob,
   inspectDeadLetters,
   inspectJob,
   inspectJobList,
@@ -86,6 +170,8 @@ export {
 } from './inspect';
 export type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } from './job';
 export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
+export type { HeldLease, LeaseStore, MemoryLeaseStoreOptions } from './leases';
+export { createMemoryLeaseStore, jobLeaseKey } from './leases';
 export type {
   Lease,
   LimitConfig,
@@ -96,9 +182,16 @@ export type {
   RateLimit,
 } from './limits';
 export { createLimiter, NO_TENANT, tenantKeyFrom } from './limits';
+export {
+  queueDeadJobs,
+  queueOldestReady,
+  recordQueueDeadJobs,
+  recordQueueOldestReady,
+} from './metrics';
 export type {
   EnqueueOptions,
   JobsFacade,
+  MemoryOutboxStore,
   OutboxDeps,
   OutboxRecord,
   OutboxRelay,
@@ -112,39 +205,29 @@ export {
   enqueueInTx,
   jobsFacade,
   resetJobsFacade,
-  SQL_OUTBOX_CLAIM,
-  SQL_OUTBOX_MARK_PUBLISHED,
-  SQL_OUTBOX_STAGE,
-  SQL_OUTBOX_TABLE,
   setJobsFacade,
 } from './outbox';
+export type { PgOutboxOptions } from './outbox-pg';
+export { createPgOutboxStore } from './outbox-pg';
 
 export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
 export { backoffDelayMs, DEFAULT_RETRY, nextRetry, retrySchedule } from './retry';
 export type {
-  CatchUpPolicy,
   CronResolver,
   DispatchedOccurrence,
   LeaderElection,
   Scheduler,
   SchedulerOptions,
   SchedulerState,
-  TaskDefinition,
-  TaskDescriptor,
-  TaskEnqueueEntry,
-  TaskHandle,
-  TaskJobResult,
 } from './scheduler';
+export { createMemorySchedulerState, createScheduler, soleLeader } from './scheduler';
+export type { PgLeaseLeaderOptions } from './scheduler-pg';
 export {
-  createMemorySchedulerState,
-  createScheduler,
-  getTask,
-  isTaskHandle,
-  registeredTasks,
-  resetTasks,
-  soleLeader,
-  task,
-} from './scheduler';
+  createPgLeaseLeader,
+  currentLeader,
+  DEFAULT_LEADER_TTL_MS,
+  pgSchedulerState,
+} from './scheduler-pg';
 export type {
   EventLookup,
   StepApi,
@@ -159,14 +242,25 @@ export {
   createMemoryStepStore,
   createStepRunner,
   isStepSuspension,
+  MAX_TRACE_NAMES,
   StepSuspension,
 } from './steps';
 export type {
-  ExecuteJobOptions,
-  JobExecution,
-  JobOutcome,
-  Worker,
-  WorkerOptions,
-  WorkerStats,
-} from './worker';
-export { createWorker, executeJob } from './worker';
+  CatchUpPolicy,
+  TaskDefinition,
+  TaskDescriptor,
+  TaskEnqueueEntry,
+  TaskHandle,
+  TaskJobResult,
+} from './task';
+export { getTask, isTaskHandle, registeredTasks, resetTasks, task } from './task';
+/**
+ * The tenant a job's body runs under. The TYPE only: `NO_JOB_TENANT`, `jobRunActor` and
+ * `jobTenantFor` stay unexported. The first would be a second spelling of `'none'` (axiom 1 — the
+ * literal is what the type says and what a declaration reads as), and the other two are
+ * `executeJob`'s and `job()`'s: a second caller deriving a run's org would be a second answer to
+ * "whose tenant is this", which is the thing this declaration exists to make singular.
+ */
+export type { JobTenant } from './tenant';
+export type { Worker, WorkerOptions, WorkerStats } from './worker';
+export { createWorker } from './worker';