@ultimat3/jobs 2.0.0 → 4.0.0

package/src/driver-memory.ts

@@ -38,7 +38,18 @@ export interface MemoryDriverOptions {
 
 const LIVE_STATES = new Set(['ready', 'delayed', 'running', 'suspended']);
 
-export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver {
+/**
+ * The in-memory driver's own type: `JobDriver` with `close` REQUIRED.
+ *
+ * `JobDriver.close` is optional because a driver may hold nothing to release. This one always
+ * does — it clears the job map — and every wrapper in the test suite delegates through
+ * `base.close()`. Declaring it here is what makes that delegation a CHECKED call: against a plain
+ * `JobDriver` the only way to write it is `base.close?.()`, which a driver that quietly stopped
+ * shipping a `close` would satisfy in silence.
+ */
+export type MemoryJobDriver = JobDriver & { close(): Promise<void> };
+
+export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJobDriver {
   const clock = options.clock ?? systemClock;
   const steps = options.steps ?? createMemoryStepStore();
   const backfills = options.backfills ?? createMemoryBackfillLedger(clock);
@@ -46,12 +57,20 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver
     options.leases ?? createMemoryLeaseStore(options.clock === undefined ? {} : { clock });
   const jobs = new Map<string, JobRecord>();
 
-  // Keyed by NAME and key, exactly as `x_jobs_name_idempotency_live_idx` is. A global key
-  // namespace let two unrelated jobs that derived the same natural key dedupe against each
-  // other: the second enqueue returned the first's id and its work never ran.
-  const liveByKey = (name: string, key: string): JobRecord | undefined => {
+  // Keyed by NAME, TENANT and key, exactly as `x_jobs_name_tenant_idempotency_live_idx` is. A
+  // global key namespace let two unrelated jobs that derived the same natural key dedupe against
+  // each other: the second enqueue returned the first's id and its work never ran. A tenant-blind
+  // one did the same ACROSS tenants, where the id handed back belongs to somebody else and is
+  // valid on every id-addressed surface. `?? ''` mirrors the index's `coalesce`, so all tenantless
+  // rows share one namespace rather than each becoming its own.
+  const liveByKey = (name: string, key: string, tenantId?: string): JobRecord | undefined => {
     for (const record of jobs.values()) {
-      if (record.name === name && record.idempotencyKey === key && LIVE_STATES.has(record.state)) {
+      if (
+        record.name === name &&
+        record.idempotencyKey === key &&
+        (record.tenantId ?? '') === (tenantId ?? '') &&
+        LIVE_STATES.has(record.state)
+      ) {
         return record;
       }
     }
@@ -124,7 +143,7 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver
     introspect,
 
     enqueue(request: EnqueueRequest): Promise<EnqueueResult> {
-      const existing = liveByKey(request.name, request.idempotencyKey);
+      const existing = liveByKey(request.name, request.idempotencyKey, request.tenantId);
       if (existing !== undefined) {
         if (request.onConflict === 'error') {
           throw new JobDuplicateError({
@@ -206,7 +225,14 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): JobDriver
       const at = nowMs(clock);
       const counts = nackOptions.countsAsAttempt !== false;
       const patch: Partial<JobRecord> = {
-        state: nackOptions.deadLetter === true ? 'dead' : counts ? 'ready' : 'suspended',
+        // `park`, never `counts`: parking is what leaves the ready bucket, and burning an attempt
+        // is a separate fact. A shed sets neither and stays `ready`, which is what it is.
+        state:
+          nackOptions.deadLetter === true
+            ? 'dead'
+            : nackOptions.park === true
+              ? 'suspended'
+              : 'ready',
         runAt: at + nackOptions.delayMs,
         // A suspension must not burn an attempt, or a 3-day sleep dead-letters the run. Floored
         // where `SQL_NACK` floors it (`greatest(attempt - 1, 0)`): the fence above is what keeps

package/src/driver-pg-ddl.ts

@@ -39,19 +39,33 @@ alter table x_jobs add column if not exists traceparent text;
 
 alter table x_jobs add column if not exists enqueued_by text;
 
--- Partial unique index: one LIVE job per (name, idempotency key). Completed rows stay for
+-- Partial unique index: one LIVE job per (name, tenant, idempotency key). Completed rows stay for
 -- history, so re-running the same work tomorrow is allowed and re-delivering it today is not.
 --
 -- The NAME is in the key, and its absence was silent data loss: two jobs that happened to derive
 -- the same natural key from the same input ("user:42") shared one namespace, so the second
 -- enqueue deduped against the FIRST jobs row and returned its id. The work never ran, no error
--- was raised, and the queue showed one healthy job. The old index is dropped rather than left
--- beside the new one — it is strictly narrower, so keeping it would keep enforcing exactly the
--- collision this fixes.
+-- was raised, and the queue showed one healthy job.
+--
+-- The TENANT is in the key for that same argument with one word substituted, and it is the worse
+-- half. Every natural key an app writes is unique only WITHIN a tenant, invoice:1001 and
+-- order:5540 being the shapes the docs suggest, so tenant B enqueuing while tenant A held that
+-- key deduped into tenant A row: tenant B work never ran, and tenant B caller received tenant A
+-- job id, which is valid on every id-addressed surface. Cancel takes an id with no tenant
+-- predicate.
+--
+-- coalesce rather than the bare column, because a null tenant_id compares unequal to every other
+-- null under a unique index and a tenantless queue would lose its dedupe entirely. All tenantless
+-- rows share one namespace instead, which is exactly what they had before tenancy existed.
+--
+-- Each superseded index is dropped rather than left beside the new one. Each is strictly narrower,
+-- so keeping it would keep enforcing exactly the collision this fixes.
 drop index if exists x_jobs_idempotency_live_idx;
 
-create unique index if not exists x_jobs_name_idempotency_live_idx
-  on x_jobs (name, idempotency_key)
+drop index if exists x_jobs_name_idempotency_live_idx;
+
+create unique index if not exists x_jobs_name_tenant_idempotency_live_idx
+  on x_jobs (name, (coalesce(tenant_id, '')), idempotency_key)
   where state in ('ready', 'delayed', 'running', 'suspended');
 
 create index if not exists x_jobs_claim_idx
@@ -109,6 +123,15 @@ create table if not exists x_outbox (
 create index if not exists x_outbox_unpublished_idx
   on x_outbox (staged_at) where published_at is null;
 
+-- The relays claim lease. A claim stamped in the same statement that locks the row is what stops
+-- two relays publishing one row twice: for update skip locked holds its locks only until that
+-- statement ends, which under autocommit is before claim returns. claimed_at is also what gives
+-- back the rows of a relay that died mid-batch, since a claim nothing can expire strands them.
+-- Added by alter because x_outbox shipped without them.
+alter table x_outbox add column if not exists claimed_at timestamptz;
+
+alter table x_outbox add column if not exists claimed_by text;
+
 -- The scheduler watermark. Without a durable one a redeployed scheduler has no idea what the
 -- pod it replaced already fired, so runRound takes the arming branch and every occurrence
 -- between the two processes is dropped with nothing logged.
@@ -177,4 +200,6 @@ create table if not exists x_outbox (
 );
 create index if not exists x_outbox_unpublished_idx
   on x_outbox (staged_at) where published_at is null;
+alter table x_outbox add column if not exists claimed_at timestamptz;
+alter table x_outbox add column if not exists claimed_by text;
 `.trim();

package/src/driver-pg-rows.ts

@@ -3,9 +3,10 @@
 // decoding a row is not control flow — every number arrives as `number | string` (a bigint is a
 // string in every client) and every absent column as `null`, and that translation is its own job.
 
-import type { BackfillRun, BackfillStatus } from './backfill-ledger';
-import type { JobRecord } from './driver';
-import type { StepRecord } from './steps';
+import { BACKFILL_STATUSES, type BackfillRun, isBackfillStatus } from './backfill-ledger';
+import { isJobState, JOB_STATES, type JobRecord, type JobState } from './driver';
+import { JobRowStatusUnknownError } from './errors';
+import { isStepStatus, STEP_STATUSES, type StepRecord, type StepStatus } from './steps';
 
 export interface StepRow {
   readonly run_id: string;
@@ -54,6 +55,21 @@ export interface JobRow {
   readonly enqueued_by?: string | null;
 }
 
+/**
+ * The one narrowing for all three status columns. `as` is not a check, and these rows cross a
+ * process boundary — a queue row was written by whatever build was deployed when the job was
+ * enqueued, which on a rolling deploy is not this one. `isBackfillStatus`'s own doc already stated
+ * the rule ("Never a cast — the list decides") and this file was the caller ignoring it.
+ */
+const statusIn = <T extends string>(
+  known: readonly T[],
+  is: (value: string) => value is T,
+  input: { table: string; column: string; value: string },
+): T => {
+  if (is(input.value)) return input.value;
+  throw new JobRowStatusUnknownError({ ...input, known });
+};
+
 export const num = (value: number | string | null | undefined): number =>
   value === null || value === undefined ? 0 : Number(value);
 
@@ -71,7 +87,11 @@ export function toJobRecord(row: JobRow): JobRecord {
     runId: row.run_id,
     attempt: row.attempt,
     maxAttempts: row.max_attempts,
-    state: row.state as JobRecord['state'],
+    state: statusIn<JobState>(JOB_STATES, isJobState, {
+      table: 'ultimate_jobs',
+      column: 'state',
+      value: row.state,
+    }),
     runAt: num(row.run_at),
     createdAt: num(row.created_at),
     updatedAt: num(row.updated_at),
@@ -94,7 +114,11 @@ export function toStepRecord(row: StepRow): StepRecord {
   return {
     runId: row.run_id,
     name: row.name,
-    status: row.status as StepRecord['status'],
+    status: statusIn<StepStatus>(STEP_STATUSES, isStepStatus, {
+      table: 'ultimate_job_steps',
+      column: 'status',
+      value: row.status,
+    }),
     output: row.output,
     startedAt: num(row.started_at),
     attempts: row.attempts,
@@ -112,7 +136,11 @@ export function toBackfillRun(row: BackfillRow): BackfillRun {
     runId: row.run_id,
     name: row.name,
     checksum: row.checksum,
-    status: row.status as BackfillStatus,
+    status: statusIn(BACKFILL_STATUSES, isBackfillStatus, {
+      table: 'ultimate_backfills',
+      column: 'status',
+      value: row.status,
+    }),
     appVersion: row.app_version,
     // `rows_processed` is a bigint, which every Postgres client hands back as a string.
     rows: num(row.rows_processed),

package/src/driver-pg-sql.ts

@@ -17,17 +17,26 @@ values
   ($1, $2, $3, $4::jsonb, $5, $6, $7,
    case when to_timestamp($8 / 1000.0) > now() then 'delayed' else 'ready' end,
    to_timestamp($8 / 1000.0), $9, $10, $11)
-on conflict (name, idempotency_key)
+on conflict (name, (coalesce(tenant_id, '')), idempotency_key)
   where state in ('ready', 'delayed', 'running', 'suspended')
   do nothing
 returning id, run_id
 `.trim();
 
-/** Scoped by NAME as well as key — the index is, so a lookup that was not would find a stranger. */
+/**
+ * Scoped by NAME and TENANT as well as key, because the index is — a lookup narrower than the
+ * index it reads answers with whichever stranger holds the key, and `{ deduped: true, id: <another
+ * tenant's> }` is a cross-tenant handle, not just a missed run.
+ *
+ * `coalesce` on both sides, matching the index expression exactly: a bare `tenant_id = $3` never
+ * matches a null row, so every tenantless enqueue would fall through to the "rejected but no live
+ * row holds its idempotency key" refusal instead of finding its own live job.
+ */
 export const SQL_FIND_LIVE_BY_KEY = `
 select id, run_id from x_jobs
  where name = $1
    and idempotency_key = $2
+   and coalesce(tenant_id, '') = coalesce($3::text, '')
    and state in ('ready', 'delayed', 'running', 'suspended')
  limit 1
 `.trim();
@@ -271,20 +280,81 @@ values ($1, $2, $3, $4::jsonb, $5, $6, to_timestamp($7 / 1000.0), to_timestamp($
         $9, $10, $11)
 `.trim();
 
+/**
+ * The claim, and it has to be ONE statement. `for update skip locked` in a bare select holds its
+ * row locks only until that statement ends — under autocommit, before `claim()` even resolves — so
+ * two relays polling 200ms apart read the same unpublished rows and both hand them to `enqueue`.
+ * `SQL_ENQUEUE` collapses the repeat only while the first job is still LIVE: its conflict target
+ * is a partial index over the live states, so a second publish landing after that job reached a
+ * terminal state inserts a second row and the handler runs again. (The mechanism is Postgres
+ * semantics; how often the two orderings line up in a deployment was never measured.)
+ *
+ * So the lock and the claim commit together, the CTE shape `SQL_CLAIM` already uses, and
+ * `claimed_at` is a LEASE: `$2` is the window after which a row a dead relay was holding is
+ * claimable again, because a claim nothing can expire strands its rows forever.
+ *
+ * The outer `select ... order by staged_at, id` is not cosmetic. `update ... returning` has no
+ * defined row order and the relay publishes in the order it is handed rows, so an app staging
+ * `createInvoice` then `chargeCard` in one transaction depends on this line.
+ *
+ * `, id` is what makes that key TOTAL, and the CTE needs it as much as the projection does: every
+ * row staged in one transaction shares a `staged_at`, so `staged_at` alone leaves the tie to the
+ * planner — which rows a `limit` takes, and in which order they publish, then differ between two
+ * relays and between two runs of one relay. No column was added for it: `id` is a UUIDv7 minted by
+ * `uuid()`, monotonic and already the primary key, so the tiebreak IS stage order.
+ */
 export const SQL_OUTBOX_CLAIM = `
+with claimable as (
+  select id
+    from x_outbox
+   where published_at is null
+     and (claimed_at is null
+          or claimed_at <= now() - ($2::bigint * interval '1 millisecond'))
+   order by staged_at, id
+   limit $1
+     for update skip locked
+), claimed as (
+  update x_outbox o
+     set claimed_at = now(), claimed_by = $3
+    from claimable c
+   where o.id = c.id
+  returning o.id, o.job, o.queue, o.input, o.idempotency_key, o.max_attempts, o.tenant_id,
+            o.traceparent, o.enqueued_by, o.claimed_by, o.run_at, o.staged_at
+)
 select id, job, queue, input, idempotency_key, max_attempts, tenant_id,
-       traceparent, enqueued_by,
+       traceparent, enqueued_by, claimed_by,
        (extract(epoch from run_at) * 1000)::bigint    as run_at,
        (extract(epoch from staged_at) * 1000)::bigint as staged_at
-  from x_outbox
- where published_at is null
- order by staged_at
- limit $1
-   for update skip locked
+  from claimed
+ order by staged_at, id
 `.trim();
 
+/**
+ * Hand a claim back early. The relay stops its batch on the first publish that fails, and without
+ * this the rows behind it would wait out the whole lease before any relay could retry them — a
+ * pool blip during a failover becoming tens of seconds of unpublished, committed work.
+ *
+ * Fenced on `published_at is null` so it can never unclaim a row some other pass already
+ * published, AND on `claimed_by` so it can never unclaim one a NEWER claimant now holds: a relay
+ * whose lease lapsed while it stalled wakes into a world where its batch is another relay's, and
+ * an unfenced release frees rows that relay is mid-publish on — a third relay claims them and
+ * publishes them again, which is the duplicate the lease exists to prevent.
+ */
+export const SQL_OUTBOX_RELEASE = `
+update x_outbox
+   set claimed_at = null, claimed_by = null
+ where id = any($1::uuid[]) and published_at is null and claimed_by = $2
+`.trim();
+
+/**
+ * Same fence, and here it is the more expensive one to miss: marking a row published is LOSING it,
+ * so a lapsed claimant stamping a row the current one has not published yet drops that job with
+ * nothing to notice. `published_at is null` makes the stamp first-writer-wins rather than a
+ * rewrite of an audit timestamp.
+ */
 export const SQL_OUTBOX_MARK_PUBLISHED = `
-update x_outbox set published_at = to_timestamp($2 / 1000.0) where id = $1
+update x_outbox set published_at = to_timestamp($2 / 1000.0)
+ where id = $1 and published_at is null and claimed_by = $3
 `.trim();
 
 export const SQL_OUTBOX_PENDING_COUNT = `

package/src/driver-pg.ts

@@ -271,19 +271,22 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
         return { id: inserted.id, runId: inserted.run_id, deduped: false };
       }
 
-      // `do nothing` fired: a live job OF THIS NAME already owns this idempotency key. The name
-      // is in the lookup because it is in the index — without it this returned whichever other
-      // job happened to derive the same natural key, and the caller's work silently never ran.
+      // `do nothing` fired: a live job OF THIS NAME, IN THIS TENANT, already owns this idempotency
+      // key. Both are in the lookup because both are in the index — without the name this returned
+      // whichever other job derived the same natural key; without the tenant it returned another
+      // TENANT's row, so the caller's work silently never ran AND the caller was handed an id it
+      // has no right to, on a surface (`cancel`) that takes an id with no tenant predicate.
       const existing = await exec().query<{ id: string; run_id: string }>(SQL_FIND_LIVE_BY_KEY, [
         request.name,
         request.idempotencyKey,
+        request.tenantId ?? null,
       ]);
       const found = existing[0];
       if (found === undefined) {
         throw new DriverUnavailableError({
           driver: 'pg',
           cause: `enqueue of "${request.name}" was rejected but no live row holds its idempotency key`,
-          fix: 'x db migrate   # reapplies SQL_JOBS_TABLE, whose x_jobs_name_idempotency_live_idx is what this lookup reads',
+          fix: 'x db migrate   # reapplies SQL_JOBS_TABLE, whose x_jobs_name_tenant_idempotency_live_idx is what this lookup reads',
         });
       }
       if (request.onConflict === 'error') {
@@ -321,7 +324,14 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
 
     async nack(jobId: string, nackOptions: NackOptions): Promise<void> {
       const counts = nackOptions.countsAsAttempt !== false;
-      const state = nackOptions.deadLetter === true ? 'dead' : counts ? 'ready' : 'suspended';
+      // The same three-way the memory driver takes, and it reads `park` rather than `counts`: the
+      // attempt counter and the ready bucket are two facts, and a shed only ever meant the first.
+      const state =
+        nackOptions.deadLetter === true
+          ? 'dead'
+          : nackOptions.park === true
+            ? 'suspended'
+            : 'ready';
       await exec().query(SQL_NACK, [
         jobId,
         state,

package/src/driver.ts

@@ -17,15 +17,22 @@ import type { StepStore } from './steps';
  * a cancellation is work an operator stopped on purpose and `x jobs retry` must not resurrect by
  * accident. It appears in no claim predicate, so the queue never hands a cancelled row out again.
  */
-export type JobState =
-  | 'ready'
-  | 'delayed'
-  | 'running'
-  | 'suspended'
-  | 'done'
-  | 'failed'
-  | 'dead'
-  | 'cancelled';
+export const JOB_STATES = [
+  'ready',
+  'delayed',
+  'running',
+  'suspended',
+  'done',
+  'failed',
+  'dead',
+  'cancelled',
+] as const;
+
+export type JobState = (typeof JOB_STATES)[number];
+
+/** Narrows a state read back off a queue row. Never a cast — the list decides. */
+export const isJobState = (value: string): value is JobState =>
+  (JOB_STATES as readonly string[]).includes(value);
 
 export interface JobRecord {
   readonly id: string;
@@ -114,10 +121,23 @@ export interface NackOptions {
   readonly delayMs: number;
   readonly error?: string;
   /**
-   * False for a suspension (`step.sleep`): parking a run is not a failure and must not burn
-   * a retry attempt, or a 3-day sleep would dead-letter the job.
+   * The ATTEMPT COUNTER, and nothing else. False for a suspension and for a shed alike: neither is
+   * a failure, and a 3-day sleep that burned an attempt would dead-letter the job.
    */
   readonly countsAsAttempt?: boolean;
+  /**
+   * True for a SUSPENSION — `step.sleep`, or a name this deploy does not know — which leaves the
+   * ready bucket and is counted `suspended`. Absent for a limiter or `job.concurrency` shed, which
+   * is a job still WAITING to run.
+   *
+   * The two were one flag until 2026-08: `countsAsAttempt: false` decided the state as well as the
+   * counter, so a shed was filed beside a 3-day sleep and `stats()` excluded it from `ready` and
+   * from `oldestReadyMs` — the two numbers the worker publishes as `queue_depth` and
+   * `queue_oldest_ready_seconds`. Under sustained overload the shed fraction approaches 100%, so
+   * the HPA signal and the "oldest job older than 5 minutes" page both went quiet exactly when the
+   * queue was saturated.
+   */
+  readonly park?: boolean;
   readonly deadLetter?: boolean;
 }
 
@@ -128,7 +148,11 @@ export interface QueueStats {
   readonly running: number;
   readonly suspended: number;
   readonly dead: number;
-  /** Age in ms of the oldest claimable job — the number that decides autoscaling. */
+  /**
+   * Age in ms of the oldest job that is READY and due — the number that decides autoscaling.
+   * Not "claimable": a `suspended` row is claimable once its `runAt` passes and is deliberately
+   * excluded here, which is exactly why a limiter shed may not be filed as a suspension.
+   */
   readonly oldestReadyMs: number;
 }