@ultimat3/jobs 1.2.0 → 3.0.0

package/src/index.ts

@@ -10,6 +10,54 @@ import './register';
 /** Re-exported so a `job`/`task` file needs one import, not two. Same object as schema's. */
 export type { Infer } from '@ultimat3/schema';
 export { t } from '@ultimat3/schema';
+export type {
+  BackfillBatch,
+  BackfillDefinition,
+  BackfillInput,
+  BackfillReport,
+} from './backfill';
+export { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
+export type { BackfillGate, BackfillGateInput } from './backfill-gate';
+export { checkBackfillEnvironment, gateBackfill } from './backfill-gate';
+export type { BackfillProgress } from './backfill-inspect';
+export { backfillForRun, inspectBackfills, toBackfillProgress } from './backfill-inspect';
+export type {
+  BackfillFilter,
+  BackfillLedger,
+  BackfillRun,
+  BackfillStatus,
+  BackfillVerdict,
+} from './backfill-ledger';
+export {
+  BACKFILL_STATUSES,
+  backfillChecksum,
+  createMemoryBackfillLedger,
+  decideBackfill,
+  isBackfillStatus,
+} from './backfill-ledger';
+export type {
+  BackfillPendingReport,
+  BackfillState,
+  BackfillStateRow,
+} from './backfill-pending';
+export {
+  BACKFILL_STATES,
+  isPendingBackfillState,
+  PENDING_BACKFILL_STATES,
+  pendingBackfills,
+} from './backfill-pending';
+export type { Pacer, PacerOptions } from './backfill-rate';
+export { createPacer, DEFAULT_BACKFILL_RATE } from './backfill-rate';
+export type { BackfillCount, BackfillDeclaration, BackfillOrigin } from './backfill-registry';
+// `stampBackfill` is deliberately absent, for the reason `registerJob` is: a second way to make a
+// handle claim it is a backfill would let a plain `job()` inherit the pending diff and the gate.
+export {
+  backfillOrigin,
+  declarationOf,
+  getBackfill,
+  isBackfill,
+  registeredBackfills,
+} from './backfill-registry';
 export type { JobDescriptor } from './describe';
 export type {
   ClaimedJob,
@@ -17,6 +65,7 @@ export type {
   ConflictPolicy,
   EnqueueRequest,
   EnqueueResult,
+  HeartbeatOptions,
   JobDriver,
   JobFilter,
   JobIntrospection,
@@ -41,11 +90,28 @@ export { createPgDriver, createPgLeader } from './driver-pg';
 export {
   SQL_ACK,
   SQL_ADVISORY_UNLOCK,
+  SQL_BACKFILL_FINISH,
+  SQL_BACKFILL_LIST,
+  SQL_BACKFILL_PROGRESS,
+  SQL_BACKFILL_START,
+  SQL_CANCEL,
   SQL_CLAIM,
   SQL_ENQUEUE,
   SQL_HEARTBEAT,
   SQL_JOBS_TABLE,
+  SQL_LEADER_ACQUIRE,
+  SQL_LEADER_RELEASE,
+  SQL_LEASE_ACQUIRE,
+  SQL_LEASE_RELEASE,
+  SQL_LEASE_RENEW,
   SQL_NACK,
+  SQL_OUTBOX_CLAIM,
+  SQL_OUTBOX_MARK_PUBLISHED,
+  SQL_OUTBOX_RELEASE,
+  SQL_OUTBOX_STAGE,
+  SQL_OUTBOX_TABLE,
+  SQL_SCHEDULER_STATE_GET,
+  SQL_SCHEDULER_STATE_MARK,
   SQL_STATS,
   SQL_STEP_GET,
   SQL_STEP_PUT,
@@ -55,20 +121,38 @@ export type { RedisDriverOptions } from './driver-redis';
 export { createRedisDriver } from './driver-redis';
 export type { JobErrorCode } from './errors';
 export {
+  BackfillAppliedError,
+  BackfillEnvironmentError,
+  BackfillMigrationPendingError,
+  BackfillPendingError,
+  BackfillRunningError,
+  BackfillStalledError,
+  BackfillUnknownError,
+  CancelUnsupportedError,
+  ConcurrencyUnenforceableError,
   DriverUnavailableError,
   IdempotencyRequiredError,
   JOB_ERROR_CODES,
   JOB_ERROR_TITLES,
+  JobAbortedError,
   JobDuplicateError,
   JobMaxAttemptsError,
   JobNameTakenError,
+  JobNotCancellableError,
+  JobSlotLostError,
   JobsNotImplementedError,
+  JobTenantRequiredError,
   JobTimeoutError,
+  LeaseLostError,
   OutboxNoTxError,
   StepDuplicateError,
 } from './errors';
 export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
 export { createMemoryEventBus, eventBus, publishEvent, setEventBus } from './events';
+export type { PgEventBusOptions } from './events-pg';
+export { createPgEventBus } from './events-pg';
+export type { ExecuteJobOptions, JobExecution, JobOutcome } from './execute';
+export { executeJob } from './execute';
 export type {
   DeadLetterEntry,
   JobsManifest,
@@ -77,6 +161,7 @@ export type {
   StepTrace,
 } from './inspect';
 export {
+  cancelJob,
   inspectDeadLetters,
   inspectJob,
   inspectJobList,
@@ -86,6 +171,8 @@ export {
 } from './inspect';
 export type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } from './job';
 export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
+export type { HeldLease, LeaseStore, MemoryLeaseStoreOptions } from './leases';
+export { createMemoryLeaseStore, jobLeaseKey } from './leases';
 export type {
   Lease,
   LimitConfig,
@@ -96,9 +183,17 @@ export type {
   RateLimit,
 } from './limits';
 export { createLimiter, NO_TENANT, tenantKeyFrom } from './limits';
+export {
+  queueDeadJobs,
+  queueOldestReady,
+  recordQueueDeadJobs,
+  recordQueueOldestReady,
+} from './metrics';
 export type {
   EnqueueOptions,
   JobsFacade,
+  MemoryOutboxOptions,
+  MemoryOutboxStore,
   OutboxDeps,
   OutboxRecord,
   OutboxRelay,
@@ -112,39 +207,34 @@ export {
   enqueueInTx,
   jobsFacade,
   resetJobsFacade,
-  SQL_OUTBOX_CLAIM,
-  SQL_OUTBOX_MARK_PUBLISHED,
-  SQL_OUTBOX_STAGE,
-  SQL_OUTBOX_TABLE,
   setJobsFacade,
 } from './outbox';
+// One definition of the lease, consumed by both stores — a memory default and a pg default that
+// could drift are two answers to "how long is a claim mine for", and the shorter one duplicates.
+export { DEFAULT_OUTBOX_CLAIM_LEASE_MS } from './outbox-lease';
+export type { PgOutboxOptions } from './outbox-pg';
+export { createPgOutboxStore } from './outbox-pg';
 
 export type { BackoffStrategy, Random, RetryDecision, RetryPolicy } from './retry';
 export { backoffDelayMs, DEFAULT_RETRY, nextRetry, retrySchedule } from './retry';
+export type { JobRetryDecision, JobStopReason } from './retry-classification';
+export { classifyThrown, nextRetryForError } from './retry-classification';
 export type {
-  CatchUpPolicy,
   CronResolver,
   DispatchedOccurrence,
   LeaderElection,
   Scheduler,
   SchedulerOptions,
   SchedulerState,
-  TaskDefinition,
-  TaskDescriptor,
-  TaskEnqueueEntry,
-  TaskHandle,
-  TaskJobResult,
 } from './scheduler';
+export { createMemorySchedulerState, createScheduler, soleLeader } from './scheduler';
+export type { PgLeaseLeaderOptions } from './scheduler-pg';
 export {
-  createMemorySchedulerState,
-  createScheduler,
-  getTask,
-  isTaskHandle,
-  registeredTasks,
-  resetTasks,
-  soleLeader,
-  task,
-} from './scheduler';
+  createPgLeaseLeader,
+  currentLeader,
+  DEFAULT_LEADER_TTL_MS,
+  pgSchedulerState,
+} from './scheduler-pg';
 export type {
   EventLookup,
   StepApi,
@@ -159,14 +249,25 @@ export {
   createMemoryStepStore,
   createStepRunner,
   isStepSuspension,
+  MAX_TRACE_NAMES,
   StepSuspension,
 } from './steps';
 export type {
-  ExecuteJobOptions,
-  JobExecution,
-  JobOutcome,
-  Worker,
-  WorkerOptions,
-  WorkerStats,
-} from './worker';
-export { createWorker, executeJob } from './worker';
+  CatchUpPolicy,
+  TaskDefinition,
+  TaskDescriptor,
+  TaskEnqueueEntry,
+  TaskHandle,
+  TaskJobResult,
+} from './task';
+export { getTask, isTaskHandle, registeredTasks, resetTasks, task } from './task';
+/**
+ * The tenant a job's body runs under. The TYPE only: `NO_JOB_TENANT`, `jobRunActor` and
+ * `jobTenantFor` stay unexported. The first would be a second spelling of `'none'` (axiom 1 — the
+ * literal is what the type says and what a declaration reads as), and the other two are
+ * `executeJob`'s and `job()`'s: a second caller deriving a run's org would be a second answer to
+ * "whose tenant is this", which is the thing this declaration exists to make singular.
+ */
+export type { JobTenant } from './tenant';
+export type { Worker, WorkerOptions, WorkerStats } from './worker';
+export { createWorker } from './worker';

package/src/inspect.ts

@@ -2,13 +2,15 @@
 // object so `x jobs ... --json` and the MCP tool share one shape — an agent debugging a stuck
 // queue reads exactly what the dashboard renders.
 
+import type { BackfillProgress } from './backfill-inspect';
+import { backfillForRun } from './backfill-inspect';
 import type { JobDriver, JobFilter, JobRecord, QueueStats } from './driver';
-import { JobsNotImplementedError } from './errors';
+import { CancelUnsupportedError, JobNotCancellableError, JobsNotImplementedError } from './errors';
 import { registeredJobs } from './job';
 import { retrySchedule } from './retry';
 import type { Scheduler } from './scheduler';
-import { registeredTasks } from './scheduler';
 import type { StepRecord } from './steps';
+import { registeredTasks } from './task';
 
 export interface QueueDepthReport {
   readonly driver: string;
@@ -67,9 +69,19 @@ export interface JobTrace {
   readonly runAt: string;
   readonly lastError: string | null;
   readonly tenantId: string | null;
+  /** W3C `traceparent` of the request that queued it — paste it into the trace viewer. */
+  readonly traceparent: string | null;
+  /** Actor id of whoever asked. Audit only: the body ran with system authority. */
+  readonly enqueuedBy: string | null;
   readonly steps: readonly StepTrace[];
   /** Remaining retry delays in ms, jitter excluded. */
   readonly retryDelaysMs: readonly number[];
+  /**
+   * The `x_backfills` row this run wrote, when the job is a `backfill()`. `null` for every other
+   * job and for a driver with no ledger — a step trace says which batch is next, and this says how
+   * many rows the pass has actually put behind it.
+   */
+  readonly backfill: BackfillProgress | null;
 }
 
 const iso = (ms: number | undefined): string | null =>
@@ -103,6 +115,7 @@ export async function inspectJob(driver: JobDriver, jobId: string): Promise<JobT
   if (record === undefined) return undefined;
   const steps = await driver.steps.list(record.runId);
   const handle = registeredJobs().find((candidate) => candidate.name === record.name);
+  const backfill = await backfillForRun(driver, record.runId);
   return {
     id: record.id,
     name: record.name,
@@ -115,8 +128,11 @@ export async function inspectJob(driver: JobDriver, jobId: string): Promise<JobT
     runAt: new Date(record.runAt).toISOString(),
     lastError: record.lastError ?? null,
     tenantId: record.tenantId ?? null,
+    traceparent: record.traceparent ?? null,
+    enqueuedBy: record.enqueuedBy ?? null,
     steps: steps.map(toStepTrace),
     retryDelaysMs: handle === undefined ? [] : [...retrySchedule(handle.retry)],
+    backfill: backfill ?? null,
   };
 }
 
@@ -169,6 +185,31 @@ export async function retryFromStep(
   return inspectJob(driver, jobId);
 }
 
+/**
+ * Stop a job from outside — the surface `x jobs cancel <id>` binds to. The hard half was already
+ * built: `execute.ts` cancels the attempt and `steps.ts` fences every write behind that signal.
+ * What was missing was anything that could TRIGGER it, so a runaway backfill against production
+ * left two options: scale the worker to zero (stopping every job) or `UPDATE x_jobs` by hand,
+ * which the worker's next ack overwrote because `SQL_ACK` had no state guard.
+ *
+ * Refuses loudly rather than answering "nothing happened": an operator cancelling a 40M-row sweep
+ * has to know whether they stopped it or missed it.
+ */
+export async function cancelJob(
+  driver: JobDriver,
+  jobId: string,
+  reason?: string,
+): Promise<JobTrace | undefined> {
+  const introspect = requireIntrospection(driver);
+  if (introspect.cancel === undefined) throw new CancelUnsupportedError({ driver: driver.name });
+  const record = await introspect.cancel(jobId, reason);
+  if (record === undefined) {
+    const current = await introspect.job(jobId);
+    throw new JobNotCancellableError({ jobId, state: current?.state ?? 'missing' });
+  }
+  return inspectJob(driver, jobId);
+}
+
 export interface JobsManifest {
   readonly jobs: readonly {
     readonly name: string;

package/src/job.ts

@@ -24,10 +24,18 @@ import { jobsFacade } from './outbox';
 import type { RetryPolicy } from './retry';
 import { DEFAULT_RETRY } from './retry';
 import type { StepApi } from './steps';
+import type { JobTenant } from './tenant';
+import { assertJobTenant, jobTenantFor } from './tenant';
 
 export interface JobRunArgs<I> {
   readonly input: I;
   readonly step: StepApi;
+  /**
+   * `ctx.signal` aborts when this attempt's `timeout` passes — the same seam an action reads, so
+   * `throwIfAborted(ctx)` and `fetch(url, { signal: ctx.signal })` work here unchanged. Past it
+   * the run belongs to whoever claims it next and `step.run` refuses to write, so a loop that
+   * never checks it is a body running beside its own retry.
+   */
   readonly ctx: Ctx;
   /** 1-based. Assume at-least-once: never branch on `attempt === 1` for correctness. */
   readonly attempt: number;
@@ -44,11 +52,50 @@ export interface JobDefinition<I> {
   readonly input: StandardSchemaV1<unknown, I>;
   /** REQUIRED. See the file header — this is the whole point. */
   readonly idempotencyKey: (input: I) => string;
+  /**
+   * REQUIRED, and the org this job's body runs under. `tenant: (input) => input.orgId` derives it
+   * from the payload; `tenant: 'none'` says this job belongs to no tenant, and then every
+   * tenant-scoped read inside it fails closed with `X_TENANCY_ACTOR_ORG_REQUIRED`.
+   *
+   * There is no default, because both candidates are wrong. Until this field existed the worker ran
+   * a body with no ambient context at all, so `@ultimat3/entity`'s tenant guard — which derives
+   * from `tryUseContext()` and not from the ctx it is handed — read no actor, added no predicate
+   * and accepted a caller-named `orgId` unchecked: the same write refused over HTTP as
+   * `X_TENANCY_ACTOR_MISMATCH` was ACCEPTED through the job surface. A boot-supplied service actor
+   * would close that with ONE identity for every job, which is a cross-tenant read waiting for the
+   * first job that takes an org id in its input. So the job declares it, per job, from its own
+   * payload — the value an author already had to pass anyway.
+   */
+  readonly tenant: JobTenant<I>;
   readonly retry: RetryPolicy;
   readonly queue?: string;
-  /** Max in-flight runs of THIS job across the fleet. Omit for the queue-wide cap. */
+  /**
+   * Max in-flight runs of THIS job across the fleet. Omit for the queue-wide cap.
+   *
+   * Enforced by `JobDriver.leases` — a row every replica can see — and NOT by `limits.ts`, which
+   * counts one process's heap. A driver with no lease store cannot hold this cap, so
+   * `createWorker().start()` refuses to boot rather than let it pass silently
+   * (`X_JOB_CONCURRENCY_UNENFORCEABLE`): this field was declared, documented and in the manifest
+   * while nothing read it, which is exactly what axiom 3 exists to prevent.
+   */
   readonly concurrency?: number;
   readonly timeout?: DurationInput;
+  /**
+   * Ceiling for ONE `step.run`, where `timeout` is the ceiling for the whole attempt. Folded into
+   * the signal the step body is handed, so a body reads one signal and sees whichever deadline
+   * lands first — and it ABORTS before it rejects, because the attempt that replaces this one is
+   * claimable the moment the nack lands.
+   *
+   * Declared here and nowhere else: the runner has implemented this ceiling since 1.0 and no
+   * declaration could ask for it, which is a documented guarantee that does nothing.
+   */
+  readonly stepTimeout?: DurationInput;
+  /**
+   * How long a `step.waitForEvent` parks between polls. Default 30s. Lower it for a wait a user
+   * is watching; the step suspends for exactly this long each time, so it is also the resolution
+   * of the resume, never a busy loop.
+   */
+  readonly eventPoll?: DurationInput;
   run(args: JobRunArgs<I>): Promise<unknown>;
 }
 
@@ -58,6 +105,18 @@ export interface JobDefinition<I> {
  */
 export interface JobActor {
   readonly orgId?: string | undefined;
+  /**
+   * The actor's id, recorded as `enqueuedBy` — ATTRIBUTION, never authority.
+   *
+   * The framework picks one answer to "whose permissions does a job run with" (axiom 1) and it is
+   * this: a job body runs with SYSTEM authority and this is an audit column. Impersonating the
+   * enqueuer at claim time is the defensible alternative and is rejected for one reason — a job
+   * that sleeps three days, or dead-letters and is retried next quarter, would then act as
+   * somebody whose role, org membership or employment has changed since. `02-primitives.md`
+   * already frames a job as server-authoritative work. A job that must act FOR a user takes that
+   * user's id in its input and re-authorises it in the body, where the check is visible in review.
+   */
+  readonly id?: string | undefined;
 }
 
 /**
@@ -71,9 +130,20 @@ export interface JobHandle<I = unknown> {
   readonly retry: RetryPolicy;
   readonly concurrency: number | undefined;
   readonly timeoutMs: number | undefined;
+  /** The declared per-step ceiling in ms; `executeJob` hands it to the step runner. */
+  readonly stepTimeoutMs: number | undefined;
+  /** The declared event-poll interval in ms; `undefined` leaves the runner's 30s default. */
+  readonly eventPollMs: number | undefined;
   readonly input: StandardSchemaV1<unknown, I>;
   parse(raw: unknown): I;
   idempotencyKeyFor(input: I): string;
+  /**
+   * The org THIS payload's run acts under — `undefined` for `tenant: 'none'`. A method and never a
+   * `readonly tenant: JobTenant<I>` field: a function-typed property is contravariant in its
+   * parameter, so `JobHandle<OrgInput>` would stop being assignable to `AnyJobHandle` and the
+   * registry, the worker and a task's enqueue list could no longer hold heterogeneous handles.
+   */
+  tenantFor(input: I): string | undefined;
   run(args: JobRunArgs<I>): Promise<unknown>;
   /**
    * Put this job on the queue. Joins the caller's transaction when the app installed the
@@ -112,15 +182,48 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
   anonymous += 1;
   const name = definition.name ?? `anonymous-job-${anonymous}`;
 
-  // Runtime backstop for generated code and JS callers; TS already forbids omitting it.
+  // Runtime backstops for generated code and JS callers; TS already forbids omitting either.
   if (typeof definition.idempotencyKey !== 'function') {
     throw new IdempotencyRequiredError({ job: name });
   }
+  assertJobTenant(name, definition.tenant);
   assert(
     definition.retry.attempts >= 1,
     `job "${name}" needs retry.attempts >= 1, got ${String(definition.retry.attempts)}`,
     `set retry: { attempts: 1 } or higher on job("${name}") — 0 attempts means the job is never executed at all, not that it never retries`,
   );
+  // `concurrency: 0` is not "no cap" — it is a fleet slot table that grants nothing.
+  // `createFleetSlots.acquire` reads `limit === undefined` as uncapped, so a declared `0` reaches
+  // `leases.acquire(key, 0, …)`, answers `false` forever with no log line, and the job is
+  // permanently unrunnable. Refused where it is written, the way `createPacer` refuses `rate: 0`.
+  assert(
+    definition.concurrency === undefined ||
+      (Number.isInteger(definition.concurrency) && definition.concurrency >= 1),
+    `job "${name}" declares concurrency ${String(definition.concurrency)}, which no worker can ever fill`,
+    `set a whole concurrency of 1 or more on job("${name}"), or omit the field for no cap at all`,
+  );
+
+  const stepTimeoutMs =
+    definition.stepTimeout === undefined ? undefined : toMs(definition.stepTimeout);
+  const eventPollMs = definition.eventPoll === undefined ? undefined : toMs(definition.eventPoll);
+  // `withStepTimeout` reads `<= 0` as "no ceiling at all" and a poll of zero is a suspension that
+  // resumes immediately, forever. Both are an author who asked for a limit and got the opposite,
+  // so they are refused where they are written — the same answer `concurrency: 0` gets.
+  //
+  // FINITE, not merely positive: `> 0` admits `Infinity`, which is the same defect spelled the
+  // other way. `eventPoll: Infinity` parks a waiting step and schedules the poll that would wake
+  // it for never; `stepTimeout: Infinity` is a ceiling no step can reach. `NaN` fails `> 0` on its
+  // own, and is covered here so the predicate says what it means rather than passing by accident.
+  assert(
+    stepTimeoutMs === undefined || (Number.isFinite(stepTimeoutMs) && stepTimeoutMs > 0),
+    `job "${name}" declares stepTimeout ${String(definition.stepTimeout)}, which is no ceiling at all`,
+    `set a finite positive stepTimeout on job("${name}") — "30s" or 30_000 — or omit the field for no per-step ceiling`,
+  );
+  assert(
+    eventPollMs === undefined || (Number.isFinite(eventPollMs) && eventPollMs > 0),
+    `job "${name}" declares eventPoll ${String(definition.eventPoll)}, which parks a waiting step for no time at all`,
+    `set a finite positive eventPoll on job("${name}") — "5s" or 5_000 — or omit the field for the 30s default`,
+  );
 
   const handle: JobHandle<I> = {
     kind: 'job',
@@ -129,6 +232,8 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
     retry: { ...DEFAULT_RETRY, ...definition.retry },
     concurrency: definition.concurrency,
     timeoutMs: definition.timeout === undefined ? undefined : toMs(definition.timeout),
+    stepTimeoutMs,
+    eventPollMs,
     input: definition.input,
     parse(raw: unknown): I {
       return parse(definition.input, raw) as I;
@@ -142,6 +247,9 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
       );
       return key;
     },
+    tenantFor(input: I): string | undefined {
+      return jobTenantFor(name, definition.tenant, input);
+    },
     run(args: JobRunArgs<I>): Promise<unknown> {
       return definition.run(args);
     },
@@ -152,9 +260,15 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
       const tenantId = options.tenantId ?? tenantFor(actor);
       // `NO_TENANT` is the limiter's own bucket for an absent tenant, so leaving the column
       // empty is the same limit and one less fake org id on the row.
+      //
+      // `enqueuedBy` is the actor's id and NOTHING ELSE crosses: the body runs with system
+      // authority, so this is an audit column, not a principal. See `JobActor.id` for why the
+      // framework chose attribution over impersonation.
+      const enqueuedBy = options.enqueuedBy ?? actor?.id;
       return handle.enqueue(input, {
         ...options,
         ...(tenantId === NO_TENANT ? {} : { tenantId }),
+        ...(enqueuedBy === undefined ? {} : { enqueuedBy }),
       });
     },
     // Reads `handle`, never the captured `name`: `nameJobs()` rebinds the property in place.
@@ -241,8 +355,18 @@ export function getJob(name: string): AnyJobHandle | undefined {
   return registry.get(name);
 }
 
+/**
+ * Code-unit compare, never `localeCompare`. This list is projected into `x.manifest.json`, which
+ * both tracked apps COMMIT and `x verify`'s drift step diffs byte for byte — and `localeCompare`
+ * with no locale argument answers from the runtime's ICU default and collation version, so the
+ * same source could sort two ways on two machines. `@ultimat3/http`'s `describeRoutes` states the
+ * same rule; the comparator is restated rather than imported because `http` is not below this
+ * package on the tier table.
+ */
+const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
+
 export function registeredJobs(): readonly AnyJobHandle[] {
-  return [...registry.values()].sort((a, b) => a.name.localeCompare(b.name));
+  return [...registry.values()].sort((a, b) => byName(a.name, b.name));
 }
 
 export function resetJobs(): void {

package/src/leases.ts

@@ -0,0 +1,90 @@
+// Fleet-wide slot counting: the seam that makes `job.concurrency` true. `limits.ts` counts slots
+// in ONE process's heap, so `perTenant: 2` on twenty pods is forty concurrent runs — the number a
+// downstream API rate-limits you for. A lease is a row somewhere every replica can see, held for
+// a TTL and renewed by the same heartbeat that renews the visibility lease, so a killed worker
+// gives its slot back by expiry rather than by cleanup nobody runs.
+
+import type { Clock } from '@ultimat3/core';
+import { systemClock } from '@ultimat3/core';
+import { nowMs } from './clock';
+
+/** A granted slot. `slot` plus `holder` is what renew and release are addressed by. */
+export interface HeldLease {
+  readonly key: string;
+  readonly slot: number;
+  readonly holder: string;
+}
+
+export interface LeaseStore {
+  /**
+   * Take a slot under `limit` for `key`, or answer `undefined`. Never over-grants; under
+   * contention it may refuse a slot that is genuinely free, which costs one poll interval.
+   */
+  acquire(
+    key: string,
+    limit: number,
+    ttlMs: number,
+    holder: string,
+  ): Promise<HeldLease | undefined>;
+  /** Push the expiry out. `false` means the slot is no longer this holder's. */
+  renew(lease: HeldLease, ttlMs: number): Promise<boolean>;
+  release(lease: HeldLease): Promise<void>;
+  /** Live slots for `key`. Diagnostics only — never the acquire decision, which must be atomic. */
+  held(key: string): Promise<number>;
+}
+
+export interface MemoryLeaseStoreOptions {
+  readonly clock?: Clock;
+}
+
+/**
+ * The `x dev` / test implementation. One heap, so it is not a fleet gate — it exists so the
+ * memory driver enforces `concurrency` with the same code path the pg driver does, and so the
+ * "a second worker is refused" test is a real test rather than a pg-only one.
+ */
+export function createMemoryLeaseStore(options: MemoryLeaseStoreOptions = {}): LeaseStore {
+  const clock = options.clock ?? systemClock;
+  const slots = new Map<string, Map<number, { holder: string; expiresAt: number }>>();
+
+  const live = (key: string): Map<number, { holder: string; expiresAt: number }> => {
+    const at = nowMs(clock);
+    const held = slots.get(key) ?? new Map();
+    for (const [slot, entry] of held) if (entry.expiresAt <= at) held.delete(slot);
+    slots.set(key, held);
+    return held;
+  };
+
+  return {
+    acquire(key, limit, ttlMs, holder) {
+      const held = live(key);
+      for (let slot = 0; slot < limit; slot += 1) {
+        if (held.has(slot)) continue;
+        held.set(slot, { holder, expiresAt: nowMs(clock) + ttlMs });
+        return Promise.resolve({ key, slot, holder });
+      }
+      return Promise.resolve(undefined);
+    },
+    renew(lease, ttlMs) {
+      const entry = live(lease.key).get(lease.slot);
+      if (entry === undefined || entry.holder !== lease.holder) return Promise.resolve(false);
+      entry.expiresAt = nowMs(clock) + ttlMs;
+      return Promise.resolve(true);
+    },
+    release(lease) {
+      const held = slots.get(lease.key);
+      const entry = held?.get(lease.slot);
+      // Only the holder gives it back: releasing a slot the TTL already handed to someone else
+      // would let two runs share it, which is the whole failure this store exists to prevent.
+      if (entry !== undefined && entry.holder === lease.holder) held?.delete(lease.slot);
+      return Promise.resolve();
+    },
+    held(key) {
+      return Promise.resolve(live(key).size);
+    },
+  };
+}
+
+/** The lease key for a job's own fleet-wide cap. One shape, so pg and memory agree on it. */
+export function jobLeaseKey(jobName: string): string {
+  return `job:${jobName}`;
+}

package/src/metrics.ts

@@ -0,0 +1,35 @@
+// The two queue gauges an alert can actually be written against. `queue_depth` and `jobs_total`
+// live in `@ultimat3/core`'s `runtime-metrics.ts` because the deploy chart scales on them; these
+// two are not scaling signals, they are the ones every queue team pages on, and neither existed:
+//
+//   "page if the oldest job in payments is older than 5 minutes" — no series carried
+//     `oldestReadyMs`, which `QueueStats` computes, `inspectQueues` renders and nothing published.
+//     `queue_depth` cannot tell 10 jobs stuck for an hour from 10 enqueued a second ago.
+//   "page if the dead-letter queue is not empty" — `jobs_total{outcome="dead"}` is a COUNTER, so
+//     a DLQ that filled overnight and stopped growing has a flat rate and alerts on nothing.
+//
+// Declared here rather than in core because they are this package's facts and core is another
+// package's file; if they move next to `queueDepth` later, these two functions are the only
+// callers to redirect.
+
+import type { Gauge } from '@ultimat3/core';
+import { gauge } from '@ultimat3/core';
+
+/** Seconds and not milliseconds: every Prometheus duration is seconds, and the alert is `> 300`. */
+export const queueOldestReady: Gauge = gauge('queue_oldest_ready_seconds', {
+  unit: 's',
+  description: 'Age of the oldest claimable job, by queue — 0 when the queue is empty',
+});
+
+export const queueDeadJobs: Gauge = gauge('queue_dead_jobs', {
+  unit: '{job}',
+  description: 'Jobs sitting in the dead-letter state, by queue',
+});
+
+export function recordQueueOldestReady(queue: string, oldestReadyMs: number): void {
+  queueOldestReady.record(oldestReadyMs / 1000, { queue });
+}
+
+export function recordQueueDeadJobs(queue: string, dead: number): void {
+  queueDeadJobs.record(dead, { queue });
+}