@ultimat3/jobs 1.2.0 → 2.0.0

package/src/errors.ts

@@ -10,15 +10,27 @@ export const JOB_OWNED_ERROR_CODES = [
   'X_JOB_MAX_ATTEMPTS',
   'X_DRIVER_UNAVAILABLE',
   'X_IDEMPOTENCY_REQUIRED',
+  'X_JOB_TENANT_REQUIRED',
+  'X_JOB_CONCURRENCY_UNENFORCEABLE',
+  'X_JOB_LEASE_LOST',
+  'X_JOB_SLOT_LOST',
+  'X_JOB_NOT_CANCELLABLE',
   'X_OUTBOX_NO_TX',
+  'X_BACKFILL_PENDING',
+  'X_BACKFILL_APPLIED',
+  'X_BACKFILL_ENVIRONMENT',
+  'X_BACKFILL_MIGRATION_PENDING',
+  'X_BACKFILL_RUNNING',
+  'X_BACKFILL_STALLED',
+  'X_BACKFILL_UNKNOWN',
 ] as const;
 
 /**
- * `X_NOT_IMPLEMENTED` is `@ultimat3/core`'s. `JobsNotImplementedError` below throws it; jobs keeps
- * no title for it, because the copy this file used to hold was a second title that nothing would
- * have failed on once core's changed.
+ * `X_NOT_IMPLEMENTED` and `X_ABORTED` are `@ultimat3/core`'s. `JobsNotImplementedError` and
+ * `JobAbortedError` below throw them; jobs keeps no title for either, because the copy this file
+ * used to hold was a second title that nothing would have failed on once core's changed.
  */
-export const JOB_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED'] as const;
+export const JOB_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED', 'X_ABORTED'] as const;
 
 /** Every code jobs can throw: the ones it owns plus the one it borrows. */
 export const JOB_ERROR_CODES = [...JOB_OWNED_ERROR_CODES, ...JOB_BORROWED_ERROR_CODES] as const;
@@ -33,7 +45,19 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
   X_JOB_MAX_ATTEMPTS: 'the job exhausted its retries',
   X_DRIVER_UNAVAILABLE: 'the queue driver is unreachable',
   X_IDEMPOTENCY_REQUIRED: 'the job has no idempotencyKey',
+  X_JOB_TENANT_REQUIRED: 'the job declares no tenant',
+  X_JOB_CONCURRENCY_UNENFORCEABLE: 'job.concurrency is declared and cannot be enforced',
+  X_JOB_LEASE_LOST: 'the queue took this job back mid-run',
+  X_JOB_SLOT_LOST: 'the fleet concurrency slot was taken by another worker',
+  X_JOB_NOT_CANCELLABLE: 'the job cannot be cancelled',
   X_OUTBOX_NO_TX: 'enqueue outside a transaction',
+  X_BACKFILL_PENDING: 'a declared backfill has never completed',
+  X_BACKFILL_APPLIED: 'the ledger already holds a completed pass',
+  X_BACKFILL_ENVIRONMENT: 'the backfill is not declared for this environment',
+  X_BACKFILL_MIGRATION_PENDING: 'the migration this backfill requires is not applied',
+  X_BACKFILL_RUNNING: 'a pass under this name is already live',
+  X_BACKFILL_STALLED: 'the sweep ended with rows its own count still matches',
+  X_BACKFILL_UNKNOWN: 'no declaration carries this backfill name',
 };
 
 // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -103,6 +127,30 @@ export class JobTimeoutError extends UltimateError {
   }
 }
 
+/**
+ * This attempt was cancelled — its deadline passed, or the caller went away — and something in it
+ * tried to keep going. The run belongs to whoever claims it next, so a step write from here would
+ * land on their history.
+ *
+ * Core's `X_ABORTED` rather than a code of jobs' own: the framework already means exactly one
+ * thing by "the signal fired, stop work", and a second name for it would make an agent ask which
+ * one it is looking at. `X_JOB_TIMEOUT` stays the code the ATTEMPT fails with; this is the code
+ * the work inside it stops with.
+ */
+export class JobAbortedError extends UltimateError {
+  constructor(input: { job: string; step?: string }) {
+    super({
+      code: 'X_ABORTED',
+      cause:
+        input.step === undefined
+          ? `job "${input.job}" was cancelled — this attempt no longer owns the run`
+          : `job "${input.job}" was cancelled before step "${input.step}" could be recorded`,
+      fix: 'add throwIfAborted(ctx) before expensive work, or pass fetch(url, { signal: ctx.signal }) — the queue re-runs the job, so stop at the deadline instead of running past it',
+      docs: docsFor('X_ABORTED'),
+    });
+  }
+}
+
 /** Retries exhausted. The job is in the dead-letter queue, not lost. */
 export class JobMaxAttemptsError extends UltimateError {
   constructor(input: { job: string; jobId: string; attempts: number; lastError: string }) {
@@ -141,6 +189,116 @@ export class IdempotencyRequiredError extends UltimateError {
   }
 }
 
+/**
+ * A job declared no `tenant`. The type already requires it; this is the runtime backstop for
+ * generated code and JS callers, so the guarantee holds at both ends — the shape
+ * `IdempotencyRequiredError` above already has.
+ *
+ * It is refused rather than defaulted because both defaults are wrong. `'none'` silently reopens
+ * the hole this field closes: the body would run with no org, and before this field existed that
+ * meant `@ultimat3/entity`'s guard read no actor at all and accepted a caller-named tenant
+ * unchecked. Inheriting the worker's org would make one identity serve every job, which is the
+ * cross-tenant read the declaration exists to prevent.
+ */
+export class JobTenantRequiredError extends UltimateError {
+  constructor(input: { job: string }) {
+    super({
+      code: 'X_JOB_TENANT_REQUIRED',
+      cause: `job "${input.job}" declares no tenant — a job body runs with no request behind it, so every tenant-scoped read inside it would be unscoped`,
+      // `'none'` reads differently either side of `backfill()` and the fix has to say so: for a
+      // plain job it means "touches no tenant-scoped table", because the org is STRIPPED and any
+      // scoped read fails closed — but a backfill declaring it is how a sweep says it spans every
+      // tenant, and the pass opens the cross-tenant scope for exactly that declaration. Half the
+      // callers of this code arrive through `backfill()`, which forwards its `tenant` to `job()`.
+      fix: `add tenant: (input) => input.orgId to job("${input.job}") — or tenant: 'none', which declares NO org: right for a job that touches no tenant-scoped table, and the spelling a backfill() uses to sweep every tenant`,
+      docs: docsFor('X_JOB_TENANT_REQUIRED'),
+    });
+  }
+}
+
+/**
+ * The queue took this job back while it was still running: `x jobs cancel` wrote a terminal state,
+ * or this worker's lease lapsed and another one re-claimed the row. Its own code and not
+ * `X_ABORTED`, because the response is different — `X_ABORTED` is "your deadline passed, make the
+ * work smaller", this is "somebody else owns this run now, stop writing to it".
+ */
+export class LeaseLostError extends UltimateError {
+  constructor(input: { job: string; jobId: string }) {
+    super({
+      code: 'X_JOB_LEASE_LOST',
+      cause: `job "${input.job}" (${input.jobId}) is no longer claimed by this worker — it was cancelled, or its visibility lease lapsed and the queue re-delivered it`,
+      fix: `x jobs show ${input.jobId} --json`,
+      docs: docsFor('X_JOB_LEASE_LOST'),
+    });
+  }
+}
+
+/**
+ * The fleet slot this run holds under `job.concurrency` is somebody else's now: renewal answered
+ * "not yours", which is the one thing `LeaseStore.renew` can say that a retry cannot fix. Its own
+ * code and not `X_JOB_LEASE_LOST`, because they are different rows on different clocks — the
+ * queue may still consider this worker the owner of the JOB while another worker is already
+ * running one under the same cap, which is precisely the guarantee `concurrency` sells.
+ */
+export class JobSlotLostError extends UltimateError {
+  constructor(input: { job: string; jobId: string; slot: number }) {
+    super({
+      code: 'X_JOB_SLOT_LOST',
+      cause: `job "${input.job}" (${input.jobId}) no longer holds fleet concurrency slot ${input.slot} — its lease expired and another worker took it`,
+      fix: `x jobs show ${input.jobId} --json`,
+      docs: docsFor('X_JOB_SLOT_LOST'),
+    });
+  }
+}
+
+/**
+ * `x jobs cancel` reached a job that already finished. Not a failure of the command — the work is
+ * done — but never a silent success either: an operator cancelling a runaway pass has to know
+ * whether they stopped it or missed it.
+ */
+export class JobNotCancellableError extends UltimateError {
+  constructor(input: { jobId: string; state: string }) {
+    super({
+      code: 'X_JOB_NOT_CANCELLABLE',
+      cause:
+        input.state === 'missing'
+          ? `no job ${input.jobId} exists in this queue`
+          : `job ${input.jobId} is "${input.state}" and only a job that has not finished can be cancelled`,
+      fix: `x jobs ls --state running --json`,
+      docs: docsFor('X_JOB_NOT_CANCELLABLE'),
+    });
+  }
+}
+
+/** The driver has no `introspect.cancel`. The redis/nats stubs, and any hand-rolled driver. */
+export class CancelUnsupportedError extends UltimateError {
+  constructor(input: { driver: string }) {
+    super({
+      code: 'X_JOB_NOT_CANCELLABLE',
+      cause: `the "${input.driver}" jobs driver cannot cancel a single job`,
+      fix: "set jobs: { driver: 'postgres' } in app.config.ts, then: x jobs cancel <id> --json",
+      docs: docsFor('X_JOB_NOT_CANCELLABLE'),
+    });
+  }
+}
+
+/**
+ * `job.concurrency` is declared and this driver has no `leases`, so the cap is per PROCESS and the
+ * fleet runs `concurrency x replicas`. Thrown at worker start rather than logged, because a
+ * documented guarantee that silently does nothing is exactly what axiom 3 exists to refuse — the
+ * worker refuses to start instead of running with the wrong number.
+ */
+export class ConcurrencyUnenforceableError extends UltimateError {
+  constructor(input: { driver: string; jobs: readonly string[] }) {
+    super({
+      code: 'X_JOB_CONCURRENCY_UNENFORCEABLE',
+      cause: `${input.jobs.join(', ')} declare concurrency and the "${input.driver}" jobs driver has no lease store, so the cap would hold per process and the fleet would run concurrency x replicas`,
+      fix: `remove concurrency from job("${input.jobs[0] ?? 'the job'}"), or set jobs: { driver: 'postgres' } in app.config.ts`,
+      docs: docsFor('X_JOB_CONCURRENCY_UNENFORCEABLE'),
+    });
+  }
+}
+
 /** An outbox enqueue happened with no ambient transaction to join. */
 export class OutboxNoTxError extends UltimateError {
   constructor(input: { job: string }) {
@@ -153,6 +311,134 @@ export class OutboxNoTxError extends UltimateError {
   }
 }
 
+/**
+ * The seven backfill codes below all answer one question — "why is this sweep not running?" — and
+ * each is here because it sends the reader somewhere different: run it, force it, change
+ * environment, migrate first, wait, fix the predicates, or fix the name. A code that shared a fix
+ * line with another one would be code inflation, which is why `X_BACKFILL_WRITE_UNCONFIRMED` was
+ * considered and rejected: a dry run that wrote nothing did exactly what it was asked to.
+ *
+ * Every `fix` below is ONE line something can execute — a shell command, or the edit to make.
+ * Never a command with prose appended: a `fix:` is copied and run verbatim, so a trailing clause
+ * turns a working command into a syntax error at the one moment the reader is following it
+ * literally. Explanations belong in `cause`, which is read and never run.
+ */
+
+/**
+ * Declared and never completed. The alarm the framework did not have: an author could
+ * `x g backfill`, merge and deploy, and nothing anywhere said the pass had not run.
+ */
+export class BackfillPendingError extends UltimateError {
+  constructor(input: { backfill: string; environment: string }) {
+    super({
+      code: 'X_BACKFILL_PENDING',
+      cause: `backfill "${input.backfill}" is declared and x_backfills holds no completed pass for it in ${input.environment}`,
+      fix: `x db backfill ${input.backfill} --write --json`,
+      docs: docsFor('X_BACKFILL_PENDING'),
+    });
+  }
+}
+
+/** Already swept. A rerun is legitimate, so this names the flag rather than refusing outright. */
+export class BackfillAppliedError extends UltimateError {
+  constructor(input: { backfill: string; runId: string; completedAt: string }) {
+    super({
+      code: 'X_BACKFILL_APPLIED',
+      cause: `backfill "${input.backfill}" completed as run ${input.runId} at ${input.completedAt}; a forced rerun writes a NEW ledger row and never edits that one`,
+      fix: `x db backfill ${input.backfill} --write --force --json`,
+      docs: docsFor('X_BACKFILL_APPLIED'),
+    });
+  }
+}
+
+/**
+ * The declaration names the environments it belongs to and this is not one. Declared DATA, never a
+ * hardcoded "cleanups are production": a staging rehearsal is correct practice, so which
+ * environments a sweep belongs to is the app's convention and this is only the mechanism carrying
+ * it (axiom 8).
+ */
+export class BackfillEnvironmentError extends UltimateError {
+  constructor(input: { backfill: string; environment: string; declared: readonly string[] }) {
+    // The first declared environment, because the fix has to be ONE runnable line and the list is
+    // ordered by the author. The empty case cannot arise from `checkBackfillEnvironment`, which
+    // treats an empty list as "every environment" — but this constructor is public, so it answers
+    // with the command that lists what IS declared rather than an `ULTIMATE_ENV=undefined`.
+    const target = input.declared[0];
+    super({
+      code: 'X_BACKFILL_ENVIRONMENT',
+      cause: `backfill "${input.backfill}" declares environments: ${input.declared.join(', ')} and this process resolved ${input.environment} — add "${input.environment}" to that list if this deploy should sweep too`,
+      fix:
+        target === undefined
+          ? 'x db backfill --pending --json'
+          : `ULTIMATE_ENV=${target} x db backfill ${input.backfill} --write --json`,
+      docs: docsFor('X_BACKFILL_ENVIRONMENT'),
+    });
+  }
+}
+
+/**
+ * `requires` names a migration the ledger has not applied. Checked where `x_migrations` is
+ * readable — this package holds no `@ultimat3/db` dependency and growing one to read a ledger
+ * would put the migration engine on the tier-3 queue's import graph.
+ */
+export class BackfillMigrationPendingError extends UltimateError {
+  constructor(input: { backfill: string; migration: string }) {
+    super({
+      code: 'X_BACKFILL_MIGRATION_PENDING',
+      cause: `backfill "${input.backfill}" requires migration ${input.migration}, which x_migrations does not record as applied`,
+      fix: 'x db migrate --json',
+      docs: docsFor('X_BACKFILL_MIGRATION_PENDING'),
+    });
+  }
+}
+
+/**
+ * The enqueue deduped: one live pass per name, so this run is the one already going. Distinct from
+ * `X_BACKFILL_APPLIED`, which is a pass that finished — the response there is `--force`, and the
+ * response here is to look at the run that is holding the key.
+ */
+export class BackfillRunningError extends UltimateError {
+  constructor(input: { backfill: string; jobId: string }) {
+    super({
+      code: 'X_BACKFILL_RUNNING',
+      cause: `backfill "${input.backfill}" already has a live pass queued as ${input.jobId}, and one name holds one live pass; its step trace names the batch it is on, and a pass that is not advancing is a worker that lost its lease`,
+      fix: `x jobs show ${input.jobId} --json`,
+      docs: docsFor('X_BACKFILL_RUNNING'),
+    });
+  }
+}
+
+/**
+ * The source ran out of rows and the declaration's own `count()` still matches some. Two
+ * predicates that disagree is an authoring bug in any business — the sweep reported success over
+ * rows it never visited — so the pass fails rather than writing a completed row nobody can trust.
+ */
+export class BackfillStalledError extends UltimateError {
+  constructor(input: { backfill: string; remaining: number; swept: number }) {
+    super({
+      code: 'X_BACKFILL_STALLED',
+      cause: `backfill "${input.backfill}" swept ${input.swept} rows, exhausted its source, and count() still matches ${input.remaining} — a WHERE the sweep narrows and the count does not is what leaves rows behind`,
+      fix: `make count() select on exactly what source() selects on in backfill("${input.backfill}")`,
+      docs: docsFor('X_BACKFILL_STALLED'),
+    });
+  }
+}
+
+/** A name no declaration in this app carries — a typo, or a backfill whose module was deleted. */
+export class BackfillUnknownError extends UltimateError {
+  constructor(input: { backfill: string; known: readonly string[] }) {
+    super({
+      code: 'X_BACKFILL_UNKNOWN',
+      cause:
+        input.known.length === 0
+          ? `no backfill named "${input.backfill}" is declared, and this app declares none at all`
+          : `no backfill named "${input.backfill}" is declared (declared: ${input.known.join(', ')})`,
+      fix: 'x db backfill --pending --json',
+      docs: docsFor('X_BACKFILL_UNKNOWN'),
+    });
+  }
+}
+
 export class JobsNotImplementedError extends UltimateError {
   constructor(input: { feature: string; fix: string }) {
     super({

package/src/events-pg.ts

@@ -0,0 +1,121 @@
+// `x_job_events`: the event bus `step.waitForEvent` needs in a real deployment. The memory bus is
+// one process's heap, and in a deployment the publisher and the resumer are never the same pod —
+// a Stripe webhook lands on web-3 and the worker that resumes the run is worker-7, so `find`
+// answers `undefined` until the 24h timeout and the run dead-letters with nothing logged first.
+//
+// Stored and not broadcast, exactly as the memory bus is: a step that suspends at 12:00 and
+// resumes at 12:00:30 must still see an event published at 12:00:10.
+
+import type { Clock } from '@ultimat3/core';
+import { logger, systemClock, uuid } from '@ultimat3/core';
+import type { DurationInput } from './clock';
+import { nowMs, toMs } from './clock';
+import type { PgExecutor } from './driver-pg';
+import {
+  SQL_EVENT_FIND,
+  SQL_EVENT_LIST,
+  SQL_EVENT_PUBLISH,
+  SQL_EVENT_PURGE,
+} from './driver-pg-sql';
+import type { EventBus, JobEvent } from './events';
+
+interface EventRow {
+  readonly id: string;
+  readonly name: string;
+  readonly payload: unknown;
+  readonly correlation_key: string | null;
+  readonly published_at: number | string;
+  readonly expires_at: number | string;
+}
+
+export interface PgEventBusOptions {
+  readonly executor: PgExecutor;
+  readonly clock?: Clock;
+  /** How long an event stays matchable. Default 7d — longer than any sane wait. */
+  readonly defaultTtl?: DurationInput;
+  /** Rows returned by `list()`. Diagnostics only; `find()` is what a step uses. */
+  readonly listLimit?: number;
+}
+
+/**
+ * `purgeExpired()` is SYNCHRONOUS in `EventBus` because the memory bus can be — it walks a Map.
+ * Over SQL the delete is a round trip, so this fires it and answers 0: the count is a diagnostic
+ * the memory bus offers and this one cannot, and blocking a step's resume on a housekeeping
+ * DELETE would be a far worse trade than an unanswered number. The index on `(name, published_at)`
+ * plus `expires_at > now()` in `find` means an unpurged row costs a filter, never a wrong answer.
+ */
+export function createPgEventBus(options: PgEventBusOptions): EventBus {
+  const clock = options.clock ?? systemClock;
+  const defaultTtl = options.defaultTtl ?? 604_800_000;
+  const listLimit = options.listLimit ?? 1_000;
+  const exec = options.executor;
+
+  const purgeExpired = (): number => {
+    void exec.query(SQL_EVENT_PURGE, []).catch((error: unknown) => {
+      // Housekeeping never costs a publish: an unpurged row is filtered out of every read.
+      logger.warn('jobs.event.purge-failed', {
+        error: error instanceof Error ? error.message : String(error),
+      });
+    });
+    return 0;
+  };
+
+  return {
+    async publish(name, payload, publishOptions = {}) {
+      const at = nowMs(clock);
+      const event: JobEvent = {
+        id: uuid(),
+        name,
+        payload,
+        publishedAt: at,
+        expiresAt: at + toMs(publishOptions.ttl ?? defaultTtl),
+        ...(publishOptions.correlationKey === undefined
+          ? {}
+          : { correlationKey: publishOptions.correlationKey }),
+      };
+      await exec.query(SQL_EVENT_PUBLISH, [
+        event.id,
+        event.name,
+        JSON.stringify(event.payload ?? null),
+        event.correlationKey ?? null,
+        event.publishedAt,
+        event.expiresAt,
+      ]);
+      logger.debug('jobs.event.published', {
+        event: name,
+        correlationKey: publishOptions.correlationKey ?? null,
+      });
+      return event;
+    },
+
+    /** Earliest match at or after `afterMs`, so a resumed step consumes events in order. */
+    async find(name, correlationKey, afterMs) {
+      const rows = await exec.query<{ payload: unknown; published_at: number | string }>(
+        SQL_EVENT_FIND,
+        [name, correlationKey ?? null, afterMs],
+      );
+      const row = rows[0];
+      return row === undefined
+        ? undefined
+        : { payload: row.payload, publishedAt: Number(row.published_at) };
+    },
+
+    async list(name) {
+      const rows = await exec.query<EventRow>(SQL_EVENT_LIST, [name ?? null, listLimit]);
+      return rows.map((row) => ({
+        id: row.id,
+        name: row.name,
+        payload: row.payload,
+        publishedAt: Number(row.published_at),
+        expiresAt: Number(row.expires_at),
+        ...(row.correlation_key === null ? {} : { correlationKey: row.correlation_key }),
+      }));
+    },
+
+    purgeExpired,
+    // The memory bus's `size()` is its Map's; over SQL a `count(*)` is a round trip and every
+    // caller of this is a test asserting on a bound the memory bus has. `-1` is the honest
+    // "not a number this bus keeps" — never `0`, which reads as an empty bus.
+    size: () => -1,
+  };
+}

package/src/events.ts

@@ -115,7 +115,13 @@ export function createMemoryEventBus(options: MemoryEventBusOptions = {}): Event
 
 let ambientBus: EventBus = createMemoryEventBus();
 
-/** Swapped at boot for the NATS/Redis-streams bus in a multi-node deployment. */
+/**
+ * **Install `createPgEventBus({ executor })` here in any deployment with more than one process.**
+ * The default above is one process's heap: the pod that publishes and the pod that resumes are
+ * never the same one, so a webhook landing on web-3 strands a run on worker-7 until its 24h
+ * timeout dead-letters it, with nothing logged before then. This line used to promise a
+ * "NATS/Redis-streams bus swapped at boot"; no such bus existed and boot installed the memory one.
+ */
 export function setEventBus(bus: EventBus): void {
   ambientBus = bus;
 }