@ultimat3/jobs 1.2.0 → 2.0.0

package/src/inspect.ts

@@ -2,13 +2,15 @@
 // object so `x jobs ... --json` and the MCP tool share one shape — an agent debugging a stuck
 // queue reads exactly what the dashboard renders.
 
+import type { BackfillProgress } from './backfill-inspect';
+import { backfillForRun } from './backfill-inspect';
 import type { JobDriver, JobFilter, JobRecord, QueueStats } from './driver';
-import { JobsNotImplementedError } from './errors';
+import { CancelUnsupportedError, JobNotCancellableError, JobsNotImplementedError } from './errors';
 import { registeredJobs } from './job';
 import { retrySchedule } from './retry';
 import type { Scheduler } from './scheduler';
-import { registeredTasks } from './scheduler';
 import type { StepRecord } from './steps';
+import { registeredTasks } from './task';
 
 export interface QueueDepthReport {
   readonly driver: string;
@@ -67,9 +69,19 @@ export interface JobTrace {
   readonly runAt: string;
   readonly lastError: string | null;
   readonly tenantId: string | null;
+  /** W3C `traceparent` of the request that queued it — paste it into the trace viewer. */
+  readonly traceparent: string | null;
+  /** Actor id of whoever asked. Audit only: the body ran with system authority. */
+  readonly enqueuedBy: string | null;
   readonly steps: readonly StepTrace[];
   /** Remaining retry delays in ms, jitter excluded. */
   readonly retryDelaysMs: readonly number[];
+  /**
+   * The `x_backfills` row this run wrote, when the job is a `backfill()`. `null` for every other
+   * job and for a driver with no ledger — a step trace says which batch is next, and this says how
+   * many rows the pass has actually put behind it.
+   */
+  readonly backfill: BackfillProgress | null;
 }
 
 const iso = (ms: number | undefined): string | null =>
@@ -103,6 +115,7 @@ export async function inspectJob(driver: JobDriver, jobId: string): Promise<JobT
   if (record === undefined) return undefined;
   const steps = await driver.steps.list(record.runId);
   const handle = registeredJobs().find((candidate) => candidate.name === record.name);
+  const backfill = await backfillForRun(driver, record.runId);
   return {
     id: record.id,
     name: record.name,
@@ -115,8 +128,11 @@ export async function inspectJob(driver: JobDriver, jobId: string): Promise<JobT
     runAt: new Date(record.runAt).toISOString(),
     lastError: record.lastError ?? null,
     tenantId: record.tenantId ?? null,
+    traceparent: record.traceparent ?? null,
+    enqueuedBy: record.enqueuedBy ?? null,
     steps: steps.map(toStepTrace),
     retryDelaysMs: handle === undefined ? [] : [...retrySchedule(handle.retry)],
+    backfill: backfill ?? null,
   };
 }
 
@@ -169,6 +185,31 @@ export async function retryFromStep(
   return inspectJob(driver, jobId);
 }
 
+/**
+ * Stop a job from outside — the surface `x jobs cancel <id>` binds to. The hard half was already
+ * built: `execute.ts` cancels the attempt and `steps.ts` fences every write behind that signal.
+ * What was missing was anything that could TRIGGER it, so a runaway backfill against production
+ * left two options: scale the worker to zero (stopping every job) or `UPDATE x_jobs` by hand,
+ * which the worker's next ack overwrote because `SQL_ACK` had no state guard.
+ *
+ * Refuses loudly rather than answering "nothing happened": an operator cancelling a 40M-row sweep
+ * has to know whether they stopped it or missed it.
+ */
+export async function cancelJob(
+  driver: JobDriver,
+  jobId: string,
+  reason?: string,
+): Promise<JobTrace | undefined> {
+  const introspect = requireIntrospection(driver);
+  if (introspect.cancel === undefined) throw new CancelUnsupportedError({ driver: driver.name });
+  const record = await introspect.cancel(jobId, reason);
+  if (record === undefined) {
+    const current = await introspect.job(jobId);
+    throw new JobNotCancellableError({ jobId, state: current?.state ?? 'missing' });
+  }
+  return inspectJob(driver, jobId);
+}
+
 export interface JobsManifest {
   readonly jobs: readonly {
     readonly name: string;

package/src/job.ts

@@ -24,10 +24,18 @@ import { jobsFacade } from './outbox';
 import type { RetryPolicy } from './retry';
 import { DEFAULT_RETRY } from './retry';
 import type { StepApi } from './steps';
+import type { JobTenant } from './tenant';
+import { assertJobTenant, jobTenantFor } from './tenant';
 
 export interface JobRunArgs<I> {
   readonly input: I;
   readonly step: StepApi;
+  /**
+   * `ctx.signal` aborts when this attempt's `timeout` passes — the same seam an action reads, so
+   * `throwIfAborted(ctx)` and `fetch(url, { signal: ctx.signal })` work here unchanged. Past it
+   * the run belongs to whoever claims it next and `step.run` refuses to write, so a loop that
+   * never checks it is a body running beside its own retry.
+   */
   readonly ctx: Ctx;
   /** 1-based. Assume at-least-once: never branch on `attempt === 1` for correctness. */
   readonly attempt: number;
@@ -44,9 +52,32 @@ export interface JobDefinition<I> {
   readonly input: StandardSchemaV1<unknown, I>;
   /** REQUIRED. See the file header — this is the whole point. */
   readonly idempotencyKey: (input: I) => string;
+  /**
+   * REQUIRED, and the org this job's body runs under. `tenant: (input) => input.orgId` derives it
+   * from the payload; `tenant: 'none'` says this job belongs to no tenant, and then every
+   * tenant-scoped read inside it fails closed with `X_TENANCY_ACTOR_ORG_REQUIRED`.
+   *
+   * There is no default, because both candidates are wrong. Until this field existed the worker ran
+   * a body with no ambient context at all, so `@ultimat3/entity`'s tenant guard — which derives
+   * from `tryUseContext()` and not from the ctx it is handed — read no actor, added no predicate
+   * and accepted a caller-named `orgId` unchecked: the same write refused over HTTP as
+   * `X_TENANCY_ACTOR_MISMATCH` was ACCEPTED through the job surface. A boot-supplied service actor
+   * would close that with ONE identity for every job, which is a cross-tenant read waiting for the
+   * first job that takes an org id in its input. So the job declares it, per job, from its own
+   * payload — the value an author already had to pass anyway.
+   */
+  readonly tenant: JobTenant<I>;
   readonly retry: RetryPolicy;
   readonly queue?: string;
-  /** Max in-flight runs of THIS job across the fleet. Omit for the queue-wide cap. */
+  /**
+   * Max in-flight runs of THIS job across the fleet. Omit for the queue-wide cap.
+   *
+   * Enforced by `JobDriver.leases` — a row every replica can see — and NOT by `limits.ts`, which
+   * counts one process's heap. A driver with no lease store cannot hold this cap, so
+   * `createWorker().start()` refuses to boot rather than let it pass silently
+   * (`X_JOB_CONCURRENCY_UNENFORCEABLE`): this field was declared, documented and in the manifest
+   * while nothing read it, which is exactly what axiom 3 exists to prevent.
+   */
   readonly concurrency?: number;
   readonly timeout?: DurationInput;
   run(args: JobRunArgs<I>): Promise<unknown>;
@@ -58,6 +89,18 @@ export interface JobDefinition<I> {
  */
 export interface JobActor {
   readonly orgId?: string | undefined;
+  /**
+   * The actor's id, recorded as `enqueuedBy` — ATTRIBUTION, never authority.
+   *
+   * The framework picks one answer to "whose permissions does a job run with" (axiom 1) and it is
+   * this: a job body runs with SYSTEM authority and this is an audit column. Impersonating the
+   * enqueuer at claim time is the defensible alternative and is rejected for one reason — a job
+   * that sleeps three days, or dead-letters and is retried next quarter, would then act as
+   * somebody whose role, org membership or employment has changed since. `02-primitives.md`
+   * already frames a job as server-authoritative work. A job that must act FOR a user takes that
+   * user's id in its input and re-authorises it in the body, where the check is visible in review.
+   */
+  readonly id?: string | undefined;
 }
 
 /**
@@ -74,6 +117,13 @@ export interface JobHandle<I = unknown> {
   readonly input: StandardSchemaV1<unknown, I>;
   parse(raw: unknown): I;
   idempotencyKeyFor(input: I): string;
+  /**
+   * The org THIS payload's run acts under — `undefined` for `tenant: 'none'`. A method and never a
+   * `readonly tenant: JobTenant<I>` field: a function-typed property is contravariant in its
+   * parameter, so `JobHandle<OrgInput>` would stop being assignable to `AnyJobHandle` and the
+   * registry, the worker and a task's enqueue list could no longer hold heterogeneous handles.
+   */
+  tenantFor(input: I): string | undefined;
   run(args: JobRunArgs<I>): Promise<unknown>;
   /**
    * Put this job on the queue. Joins the caller's transaction when the app installed the
@@ -112,15 +162,26 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
   anonymous += 1;
   const name = definition.name ?? `anonymous-job-${anonymous}`;
 
-  // Runtime backstop for generated code and JS callers; TS already forbids omitting it.
+  // Runtime backstops for generated code and JS callers; TS already forbids omitting either.
   if (typeof definition.idempotencyKey !== 'function') {
     throw new IdempotencyRequiredError({ job: name });
   }
+  assertJobTenant(name, definition.tenant);
   assert(
     definition.retry.attempts >= 1,
     `job "${name}" needs retry.attempts >= 1, got ${String(definition.retry.attempts)}`,
     `set retry: { attempts: 1 } or higher on job("${name}") — 0 attempts means the job is never executed at all, not that it never retries`,
   );
+  // `concurrency: 0` is not "no cap" — it is a fleet slot table that grants nothing.
+  // `createFleetSlots.acquire` reads `limit === undefined` as uncapped, so a declared `0` reaches
+  // `leases.acquire(key, 0, …)`, answers `false` forever with no log line, and the job is
+  // permanently unrunnable. Refused where it is written, the way `createPacer` refuses `rate: 0`.
+  assert(
+    definition.concurrency === undefined ||
+      (Number.isInteger(definition.concurrency) && definition.concurrency >= 1),
+    `job "${name}" declares concurrency ${String(definition.concurrency)}, which no worker can ever fill`,
+    `set a whole concurrency of 1 or more on job("${name}"), or omit the field for no cap at all`,
+  );
 
   const handle: JobHandle<I> = {
     kind: 'job',
@@ -142,6 +203,9 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
       );
       return key;
     },
+    tenantFor(input: I): string | undefined {
+      return jobTenantFor(name, definition.tenant, input);
+    },
     run(args: JobRunArgs<I>): Promise<unknown> {
       return definition.run(args);
     },
@@ -152,9 +216,15 @@ export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
       const tenantId = options.tenantId ?? tenantFor(actor);
       // `NO_TENANT` is the limiter's own bucket for an absent tenant, so leaving the column
       // empty is the same limit and one less fake org id on the row.
+      //
+      // `enqueuedBy` is the actor's id and NOTHING ELSE crosses: the body runs with system
+      // authority, so this is an audit column, not a principal. See `JobActor.id` for why the
+      // framework chose attribution over impersonation.
+      const enqueuedBy = options.enqueuedBy ?? actor?.id;
       return handle.enqueue(input, {
         ...options,
         ...(tenantId === NO_TENANT ? {} : { tenantId }),
+        ...(enqueuedBy === undefined ? {} : { enqueuedBy }),
       });
     },
     // Reads `handle`, never the captured `name`: `nameJobs()` rebinds the property in place.

package/src/leases.ts

@@ -0,0 +1,90 @@
+// Fleet-wide slot counting: the seam that makes `job.concurrency` true. `limits.ts` counts slots
+// in ONE process's heap, so `perTenant: 2` on twenty pods is forty concurrent runs — the number a
+// downstream API rate-limits you for. A lease is a row somewhere every replica can see, held for
+// a TTL and renewed by the same heartbeat that renews the visibility lease, so a killed worker
+// gives its slot back by expiry rather than by cleanup nobody runs.
+
+import type { Clock } from '@ultimat3/core';
+import { systemClock } from '@ultimat3/core';
+import { nowMs } from './clock';
+
+/** A granted slot. `slot` plus `holder` is what renew and release are addressed by. */
+export interface HeldLease {
+  readonly key: string;
+  readonly slot: number;
+  readonly holder: string;
+}
+
+export interface LeaseStore {
+  /**
+   * Take a slot under `limit` for `key`, or answer `undefined`. Never over-grants; under
+   * contention it may refuse a slot that is genuinely free, which costs one poll interval.
+   */
+  acquire(
+    key: string,
+    limit: number,
+    ttlMs: number,
+    holder: string,
+  ): Promise<HeldLease | undefined>;
+  /** Push the expiry out. `false` means the slot is no longer this holder's. */
+  renew(lease: HeldLease, ttlMs: number): Promise<boolean>;
+  release(lease: HeldLease): Promise<void>;
+  /** Live slots for `key`. Diagnostics only — never the acquire decision, which must be atomic. */
+  held(key: string): Promise<number>;
+}
+
+export interface MemoryLeaseStoreOptions {
+  readonly clock?: Clock;
+}
+
+/**
+ * The `x dev` / test implementation. One heap, so it is not a fleet gate — it exists so the
+ * memory driver enforces `concurrency` with the same code path the pg driver does, and so the
+ * "a second worker is refused" test is a real test rather than a pg-only one.
+ */
+export function createMemoryLeaseStore(options: MemoryLeaseStoreOptions = {}): LeaseStore {
+  const clock = options.clock ?? systemClock;
+  const slots = new Map<string, Map<number, { holder: string; expiresAt: number }>>();
+
+  const live = (key: string): Map<number, { holder: string; expiresAt: number }> => {
+    const at = nowMs(clock);
+    const held = slots.get(key) ?? new Map();
+    for (const [slot, entry] of held) if (entry.expiresAt <= at) held.delete(slot);
+    slots.set(key, held);
+    return held;
+  };
+
+  return {
+    acquire(key, limit, ttlMs, holder) {
+      const held = live(key);
+      for (let slot = 0; slot < limit; slot += 1) {
+        if (held.has(slot)) continue;
+        held.set(slot, { holder, expiresAt: nowMs(clock) + ttlMs });
+        return Promise.resolve({ key, slot, holder });
+      }
+      return Promise.resolve(undefined);
+    },
+    renew(lease, ttlMs) {
+      const entry = live(lease.key).get(lease.slot);
+      if (entry === undefined || entry.holder !== lease.holder) return Promise.resolve(false);
+      entry.expiresAt = nowMs(clock) + ttlMs;
+      return Promise.resolve(true);
+    },
+    release(lease) {
+      const held = slots.get(lease.key);
+      const entry = held?.get(lease.slot);
+      // Only the holder gives it back: releasing a slot the TTL already handed to someone else
+      // would let two runs share it, which is the whole failure this store exists to prevent.
+      if (entry !== undefined && entry.holder === lease.holder) held?.delete(lease.slot);
+      return Promise.resolve();
+    },
+    held(key) {
+      return Promise.resolve(live(key).size);
+    },
+  };
+}
+
+/** The lease key for a job's own fleet-wide cap. One shape, so pg and memory agree on it. */
+export function jobLeaseKey(jobName: string): string {
+  return `job:${jobName}`;
+}

package/src/metrics.ts

@@ -0,0 +1,35 @@
+// The two queue gauges an alert can actually be written against. `queue_depth` and `jobs_total`
+// live in `@ultimat3/core`'s `runtime-metrics.ts` because the deploy chart scales on them; these
+// two are not scaling signals, they are the ones every queue team pages on, and neither existed:
+//
+//   "page if the oldest job in payments is older than 5 minutes" — no series carried
+//     `oldestReadyMs`, which `QueueStats` computes, `inspectQueues` renders and nothing published.
+//     `queue_depth` cannot tell 10 jobs stuck for an hour from 10 enqueued a second ago.
+//   "page if the dead-letter queue is not empty" — `jobs_total{outcome="dead"}` is a COUNTER, so
+//     a DLQ that filled overnight and stopped growing has a flat rate and alerts on nothing.
+//
+// Declared here rather than in core because they are this package's facts and core is another
+// package's file; if they move next to `queueDepth` later, these two functions are the only
+// callers to redirect.
+
+import type { Gauge } from '@ultimat3/core';
+import { gauge } from '@ultimat3/core';
+
+/** Seconds and not milliseconds: every Prometheus duration is seconds, and the alert is `> 300`. */
+export const queueOldestReady: Gauge = gauge('queue_oldest_ready_seconds', {
+  unit: 's',
+  description: 'Age of the oldest claimable job, by queue — 0 when the queue is empty',
+});
+
+export const queueDeadJobs: Gauge = gauge('queue_dead_jobs', {
+  unit: '{job}',
+  description: 'Jobs sitting in the dead-letter state, by queue',
+});
+
+export function recordQueueOldestReady(queue: string, oldestReadyMs: number): void {
+  queueOldestReady.record(oldestReadyMs / 1000, { queue });
+}
+
+export function recordQueueDeadJobs(queue: string, dead: number): void {
+  queueDeadJobs.record(dead, { queue });
+}

package/src/outbox-pg.ts

@@ -0,0 +1,137 @@
+// The `x_outbox` implementation of `OutboxStore` — the half of the transactional outbox that
+// makes it transactional. `stage()` runs on the CALLER'S OWN connection, so the queue row is in
+// the same Postgres transaction as the business rows and commits or vanishes with them. The
+// memory store hangs rows off the `Tx` object and is correct only inside one process; this is
+// what a deployment installs.
+//
+// `txExecutor` is injected rather than resolved here for the reason `createPgDriver` takes a
+// `PgExecutor`: this package holds no `@ultimat3/db` dependency, and "which connection is this
+// `Tx` on" is a question only boot can answer. Boot has `currentTx()` — a `DbTx` IS a client on
+// the transaction's connection — so the wiring is one line there and no tier crossing here.
+
+import type { Clock } from '@ultimat3/core';
+import type { Tx } from '@ultimat3/entity';
+import { nowMs } from './clock';
+import type { PgExecutor } from './driver-pg';
+import {
+  SQL_OUTBOX_CLAIM,
+  SQL_OUTBOX_MARK_PUBLISHED,
+  SQL_OUTBOX_PENDING_COUNT,
+  SQL_OUTBOX_STAGE,
+} from './driver-pg-sql';
+import type { OutboxRecord, OutboxStore } from './outbox';
+
+interface OutboxRow {
+  readonly id: string;
+  readonly job: string;
+  readonly queue: string;
+  readonly input: unknown;
+  readonly idempotency_key: string;
+  readonly max_attempts: number;
+  readonly run_at: number | string;
+  readonly staged_at: number | string;
+  readonly tenant_id: string | null;
+  readonly traceparent: string | null;
+  readonly enqueued_by: string | null;
+}
+
+export interface PgOutboxOptions {
+  /**
+   * The pooled executor the RELAY uses: `claim`, `markPublished` and `pendingCount` all run after
+   * the caller's transaction is gone, so they must not be bound to it.
+   */
+  readonly executor: PgExecutor;
+  /**
+   * The executor bound to `tx`'s own connection. Boot supplies it; without it `stage()` would
+   * write on a second connection and the outbox would guarantee nothing at all.
+   */
+  readonly txExecutor: (tx: Tx) => PgExecutor;
+  readonly clock?: Clock;
+}
+
+function toRecord(row: OutboxRow): OutboxRecord {
+  return {
+    id: row.id,
+    job: row.job,
+    queue: row.queue,
+    input: row.input,
+    idempotencyKey: row.idempotency_key,
+    maxAttempts: Number(row.max_attempts),
+    runAt: Number(row.run_at),
+    stagedAt: Number(row.staged_at),
+    ...(row.tenant_id === null ? {} : { tenantId: row.tenant_id }),
+    ...(row.traceparent === null ? {} : { traceparent: row.traceparent }),
+    ...(row.enqueued_by === null ? {} : { enqueuedBy: row.enqueued_by }),
+  };
+}
+
+export function createPgOutboxStore(options: PgOutboxOptions): OutboxStore {
+  // What each open transaction has staged, for `commit()`'s return value only. Never the source
+  // of truth — that is the row, and the row's fate is the transaction's. A WeakMap so a `Tx` that
+  // is neither committed nor rolled back (a process killed mid-request) leaves nothing behind.
+  const staged = new WeakMap<object, OutboxRecord[]>();
+  const key = (tx: Tx): object => tx as unknown as object;
+
+  return {
+    async stage(tx, record) {
+      await options
+        .txExecutor(tx)
+        .query(SQL_OUTBOX_STAGE, [
+          record.id,
+          record.job,
+          record.queue,
+          JSON.stringify(record.input ?? null),
+          record.idempotencyKey,
+          record.maxAttempts,
+          record.runAt,
+          record.stagedAt,
+          record.tenantId ?? null,
+          record.traceparent ?? null,
+          record.enqueuedBy ?? null,
+        ]);
+      const bucket = staged.get(key(tx)) ?? [];
+      bucket.push(record);
+      staged.set(key(tx), bucket);
+    },
+
+    /**
+     * Nothing to do but report. The rows became visible when Postgres committed them — there is
+     * no second write here, and that absence IS the guarantee: a commit hook that had to run
+     * would be one more thing between the business rows and the job row.
+     */
+    commit(tx) {
+      const bucket = staged.get(key(tx)) ?? [];
+      staged.delete(key(tx));
+      return Promise.resolve(bucket);
+    },
+
+    /** Also nothing: the ROLLBACK already took the rows. Only the bookkeeping is ours to drop. */
+    rollback(tx) {
+      staged.delete(key(tx));
+      return Promise.resolve();
+    },
+
+    /**
+     * `for update skip locked` is in the statement, and under autocommit its row locks last only
+     * for that statement — so two relays can hand the same row to `enqueue`. That is the
+     * at-least-once the whole design already assumes and the idempotency key already collapses;
+     * what the clause buys is that two relays running side by side do not serialise on each other.
+     */
+    async claim(limit) {
+      const rows = await options.executor.query<OutboxRow>(SQL_OUTBOX_CLAIM, [limit]);
+      return rows.map(toRecord);
+    },
+
+    async markPublished(id, at) {
+      await options.executor.query(SQL_OUTBOX_MARK_PUBLISHED, [id, at || nowMs(options.clock)]);
+    },
+
+    async pendingCount() {
+      const rows = await options.executor.query<{ pending: number | string }>(
+        SQL_OUTBOX_PENDING_COUNT,
+        [],
+      );
+      return Number(rows[0]?.pending ?? 0);
+    },
+  };
+}