@ultimat3/jobs 20.2.1 → 22.0.0

package/README.md

@@ -679,7 +679,7 @@ between them — swapping is `setJobDriver(other)`, and there is **no `jobs.driv
 
 | Driver | Status | Backing | Use |
 |---|---|---|---|
-| `pg` | **default** | `SELECT ... FOR UPDATE SKIP LOCKED`, a partial unique index on `(name, idempotency_key)`, lease-based leader, `x_job_leases` | zero-infra start, most apps |
+| `pg` | **default** | `SELECT ... FOR UPDATE SKIP LOCKED`, a partial unique index on `(name, coalesce(tenant_id, ''), idempotency_key)` over live rows (`x_jobs_name_tenant_idempotency_live_idx`), lease-based leader, `x_job_leases` | zero-infra start, most apps |
 | `memory` | complete | in-process maps | `x dev`, tests |
 | `redis` | interface-complete, `X_NOT_IMPLEMENTED` | Streams + consumer groups | planned |
 | `nats` | interface-complete, `X_NOT_IMPLEMENTED` | JetStream work queue | planned |
@@ -764,10 +764,10 @@ non-empty string is not a timezone: `tz: 'Bogota'` would resolve every occurrenc
 run five hours off, silently, forever. `0 3 * * *` in a DST zone runs twice or zero times on
 the switch day. Catch-up after downtime is explicit: `skip` (default) fires the latest missed
 occurrence and drops the older ones, `run-once` fires the earliest missed one, `run-all` fires
-every one of them. `maxCatchUp` (default 10) bounds the WALK for every mode, not just `run-all`:
-one tick walks at most that many occurrences forward from the last fire, and the policy then
-picks from what that walk found — so after a long outage `skip` fires the latest occurrence
-*within the cap*, not the true latest missed.
+every one of them. `maxCatchUp` (default 10) bounds the WALK for `run-all` and `run-once`: one
+tick walks at most that many occurrences forward from the last fire. `skip` is not bounded by it —
+after a long outage it fires the TRUE latest missed occurrence, found by bisection over the gap
+(about 25 cron evaluations for three hours, 35 for a year), and moves the watermark there.
 
 `run-once` fires **once**, not once per tick. Dropping the rest means the watermark passes them
 too, so a scheduler back up after a day down enqueues one catch-up and then waits for the next
@@ -778,7 +778,10 @@ hourly task fire 24 catch-ups a second apart.
 
 `{ attempts, backoff: 'exponential' | 'linear' | 'fixed', delay, maxDelay, jitter }`.
 Equal jitter is on by default so a burst of failures does not retry in lockstep.
-Exhausted jobs are dead-lettered, never dropped: `x jobs retry <id>`.
+Exhausted jobs are dead-lettered, never dropped: `x jobs retry <id>`. Only a finished job — dead,
+cancelled, failed or done — can be requeued: a live one is `X_JOB_NOT_REQUEUEABLE` (requeueing a
+running job used to run it twice), and one whose idempotency key a live job now holds is
+`X_JOB_DUPLICATE`. `--from-step <name>` drops that step and every step that started after it.
 
 ```
 retrySchedule({ attempts: 5, backoff: 'exponential', delay: 1000 })
@@ -902,6 +905,53 @@ FOR a user takes that user's id in its input and re-authorises it in the body.
 | `X_JOB_CONCURRENCY_UNENFORCEABLE` | a registered job declares `concurrency` and the driver has no lease store |
 | `X_NOT_IMPLEMENTED` | redis / nats driver |
 
+### Error classes
+
+Every error class `src/index.ts` exports, for `instanceof` inside one process. Across a wire or
+a job boundary the class is gone and the `code` is what survives — match on that.
+
+| Class | Code | Declared in |
+|---|---|---|
+| `ActionJobUnbridgedError` | `X_ACTION_JOB_UNBRIDGED` | `src/errors.ts` |
+| `BackfillAppliedError` | `X_BACKFILL_APPLIED` | `src/backfill-errors.ts` |
+| `BackfillEnvironmentError` | `X_BACKFILL_ENVIRONMENT` | `src/backfill-errors.ts` |
+| `BackfillMigrationPendingError` | `X_BACKFILL_MIGRATION_PENDING` | `src/backfill-errors.ts` |
+| `BackfillPendingError` | `X_BACKFILL_PENDING` | `src/backfill-errors.ts` |
+| `BackfillRunningError` | `X_BACKFILL_RUNNING` | `src/backfill-errors.ts` |
+| `BackfillStalledError` | `X_BACKFILL_STALLED` | `src/backfill-errors.ts` |
+| `BackfillUnknownError` | `X_BACKFILL_UNKNOWN` | `src/backfill-errors.ts` |
+| `CancelUnsupportedError` | `X_JOB_NOT_CANCELLABLE` | `src/errors.ts` |
+| `ClaimQueuesEmptyError` | `X_JOB_CLAIM_QUEUES_EMPTY` | `src/errors.ts` |
+| `ConcurrencyUnenforceableError` | `X_JOB_CONCURRENCY_UNENFORCEABLE` | `src/errors.ts` |
+| `DriverUnavailableError` | `X_DRIVER_UNAVAILABLE` | `src/errors.ts` |
+| `ExportPartTooLargeError` | `X_EXPORT_PART_TOO_LARGE` | `src/export-errors.ts` |
+| `ExportRowInvalidError` | `X_EXPORT_ROW_INVALID` | `src/export-errors.ts` |
+| `IdempotencyRequiredError` | `X_IDEMPOTENCY_REQUIRED` | `src/errors.ts` |
+| `JobAbortedError` | `X_ABORTED` | `src/errors.ts` |
+| `JobDeclarationInvalidError` | `X_JOB_DECLARATION_INVALID` | `src/errors-declaration.ts` |
+| `JobDrainedError` | `X_DRAINING` | `src/errors.ts` |
+| `JobDuplicateError` | `X_JOB_DUPLICATE` | `src/errors.ts` |
+| `JobMaxAttemptsError` | `X_JOB_MAX_ATTEMPTS` | `src/errors.ts` |
+| `JobNameTakenError` | `X_JOB_DUPLICATE` | `src/errors.ts` |
+| `JobNotCancellableError` | `X_JOB_NOT_CANCELLABLE` | `src/errors.ts` |
+| `JobNotRequeueableError` | `X_JOB_NOT_REQUEUEABLE` | `src/errors-requeue.ts` |
+| `JobRowStatusUnknownError` | `X_JOB_ROW_STATUS_UNKNOWN` | `src/errors.ts` |
+| `JobSlotLostError` | `X_JOB_SLOT_LOST` | `src/errors.ts` |
+| `JobsNotImplementedError` | `X_NOT_IMPLEMENTED` | `src/errors.ts` |
+| `JobTenantRequiredError` | `X_JOB_TENANT_REQUIRED` | `src/errors.ts` |
+| `JobTimeoutError` | `X_JOB_TIMEOUT` | `src/errors.ts` |
+| `LeaseLostError` | `X_JOB_LEASE_LOST` | `src/errors.ts` |
+| `OutboxNoTxError` | `X_OUTBOX_NO_TX` | `src/errors.ts` |
+| `StepDuplicateError` | `X_STEP_DUPLICATE` | `src/errors.ts` |
+| `WebhookDeliveryFailedError` | `X_WEBHOOK_DELIVERY_FAILED` | `src/webhook-errors.ts` |
+| `WebhookDeliveryRejectedError` | `X_WEBHOOK_DELIVERY_REJECTED` | `src/webhook-errors.ts` |
+| `WebhookDeliveryThrottledError` | `X_WEBHOOK_DELIVERY_THROTTLED` | `src/webhook-errors.ts` |
+| `WebhookEndpointDisabledError` | `X_WEBHOOK_ENDPOINT_DISABLED` | `src/webhook-errors.ts` |
+| `WebhookEndpointInvalidError` | `X_WEBHOOK_ENDPOINT_INVALID` | `src/webhook-errors.ts` |
+| `WebhookEndpointUnknownError` | `X_WEBHOOK_ENDPOINT_UNKNOWN` | `src/webhook-errors.ts` |
+| `WebhookEventInvalidError` | `X_WEBHOOK_EVENT_INVALID` | `src/webhook-errors.ts` |
+| `WebhookEventUnknownError` | `X_WEBHOOK_EVENT_UNKNOWN` | `src/webhook-errors.ts` |
+
 ## Boundary
 
 Tier 3. Imports `@ultimat3/core`, `@ultimat3/schema`, `@ultimat3/entity`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/jobs",
-  "version": "20.2.1",
+  "version": "22.0.0",
   "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
   "license": "MIT",
   "type": "module",
@@ -32,10 +32,10 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "20.2.1",
-    "@ultimat3/db": "20.2.1",
-    "@ultimat3/entity": "20.2.1",
-    "@ultimat3/schema": "20.2.1",
-    "@ultimat3/time": "20.2.1"
+    "@ultimat3/core": "22.0.0",
+    "@ultimat3/db": "22.0.0",
+    "@ultimat3/entity": "22.0.0",
+    "@ultimat3/schema": "22.0.0",
+    "@ultimat3/time": "22.0.0"
   }
 }

package/src/driver-memory.ts

@@ -20,8 +20,9 @@ import type {
   NackOptions,
   QueueStats,
 } from './driver';
-import { assertClaimBounds, assertClaimQueues, DEFAULT_QUEUE } from './driver';
+import { assertClaimBounds, assertClaimQueues, DEFAULT_QUEUE, REQUEUEABLE_STATES } from './driver';
 import { JobDuplicateError } from './errors';
+import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
 import type { LeaseStore } from './leases';
 import { createMemoryLeaseStore } from './leases';
 import type { StepStore } from './steps';
@@ -133,11 +134,30 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
         'requeue a job id returned by enqueue() or stats() — the memory driver holds no state across processes, so an id from another run will not resolve',
       );
       const record = existing;
+      // The two refusals `SQL_JOB_REQUEUE` and `SQL_JOB_LIVE_HOLDER` give, before anything moves.
+      if (!REQUEUEABLE_STATES.has(record.state)) {
+        throw new JobNotRequeueableError({ jobId, state: record.state });
+      }
+      const holder = liveByKey(record.name, record.idempotencyKey, record.tenantId);
+      if (holder !== undefined && holder.id !== jobId) {
+        throw requeueKeyTaken({ ...record, holderId: holder.id });
+      }
       if (requeueOptions?.fromStep !== undefined) {
-        // Drop the target step so it re-executes; earlier steps stay memoized.
-        await steps.del(record.runId, requeueOptions.fromStep);
+        // The target step and every step that started AFTER it — `SQL_STEPS_FROM`, whose header
+        // says why a tie is kept. Earlier steps stay memoized; a later one replaying the old run's
+        // result is what this prevents.
+        const all = await steps.list(record.runId);
+        const target = all.find((step) => step.name === requeueOptions.fromStep);
+        if (target !== undefined) {
+          for (const step of all) {
+            if (step === target || step.startedAt > target.startedAt) {
+              await steps.del(record.runId, step.name);
+            }
+          }
+        }
       }
-      update(jobId, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
+      // `settle` releases the claim — `SQL_JOB_REQUEUE` writes `claimed_by = null` too.
+      settle(jobId, { state: 'ready', attempt: 0, runAt: nowMs(clock) });
       const next = jobs.get(jobId);
       return next ?? record;
     },
@@ -306,10 +326,15 @@ export function createMemoryDriver(options: MemoryDriverOptions = {}): MemoryJob
           oldestReadyMs: 0,
         };
         const next = { ...current };
-        if (record.state === 'ready' && record.runAt <= at) {
+        // Due is due, whatever state the row was WRITTEN in: a job enqueued `delayed` stays
+        // `delayed` once its `runAt` passes (the claim scan reads `runAt`, nothing rewrites the
+        // state), and counted by state it never reached `ready` or `oldestReadyMs` — the backlog
+        // the autoscaler reads. `SQL_STATS` makes the same split.
+        const waiting = record.state === 'ready' || record.state === 'delayed';
+        if (waiting && record.runAt <= at) {
           next.ready += 1;
           next.oldestReadyMs = Math.max(next.oldestReadyMs, at - record.runAt);
-        } else if (record.state === 'ready' || record.state === 'delayed') next.delayed += 1;
+        } else if (waiting) next.delayed += 1;
         else if (record.state === 'running') next.running += 1;
         else if (record.state === 'suspended') next.suspended += 1;
         else if (record.state === 'dead') next.dead += 1;

package/src/driver-pg-jobs-sql.ts

@@ -36,10 +36,41 @@ select ${JOB_ROW_COLUMNS}
   from x_jobs where state = 'dead' order by updated_at desc limit $1
 `.trim();
 
-/** `run_at = now()` makes the requeued job due immediately; the attempt counter starts over. */
+/**
+ * `run_at = now()` makes the requeued job due immediately; the attempt counter starts over. Fenced
+ * to FINISHED states and releasing the claim: requeueing a running row left `claimed_by` set with
+ * state `ready`, and a second worker claimed it — the job ran twice. `REQUEUEABLE_STATES`
+ * (`driver.ts`) is the same set; a row this matches nothing for was refused before it was sent,
+ * or raced to live in between.
+ */
 export const SQL_JOB_REQUEUE = `
 update x_jobs
-   set state = 'ready', attempt = 0, run_at = now(), updated_at = now()
- where id = $1
+   set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
+       claimed_by = null, visible_at = null
+ where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
 returning ${JOB_ROW_COLUMNS}
 `.trim();
+
+/**
+ * The LIVE row holding a requeued row's key, if any — `x_jobs_name_tenant_idempotency_live_idx`'s
+ * predicate, read before the update so the answer is `X_JOB_DUPLICATE` rather than a raw 23505.
+ */
+export const SQL_JOB_LIVE_HOLDER = `
+select id from x_jobs
+ where name = $1 and coalesce(tenant_id, '') = coalesce($2, '') and idempotency_key = $3
+   and id <> $4 and state in ('ready', 'delayed', 'running', 'suspended')
+ limit 1
+`.trim();
+
+/**
+ * "From that step onward": the step `fromStep` names and every step that started AFTER it.
+ * Deleting the named step alone left later steps replaying results computed from the old run.
+ * Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
+ * later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
+ */
+export const SQL_STEPS_FROM = `
+delete from x_job_steps
+ where run_id = $1
+   and (name = $2
+        or started_at > (select started_at from x_job_steps where run_id = $1 and name = $2))
+`.trim();

package/src/driver-pg-sql.ts

@@ -17,7 +17,9 @@ export {
   SQL_JOB_DEAD_LETTERS,
   SQL_JOB_GET,
   SQL_JOB_LIST,
+  SQL_JOB_LIVE_HOLDER,
   SQL_JOB_REQUEUE,
+  SQL_STEPS_FROM,
 } from './driver-pg-jobs-sql';
 
 import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
@@ -150,14 +152,13 @@ returning id
  */
 export const SQL_STATS = `
 select queue,
-       count(*) filter (where state = 'ready' and run_at <= now())              as ready,
-       count(*) filter (where state = 'delayed'
-                           or (state = 'ready' and run_at > now()))             as delayed,
+       count(*) filter (where state in ('ready', 'delayed') and run_at <= now()) as ready,
+       count(*) filter (where state in ('ready', 'delayed') and run_at > now())  as delayed,
        count(*) filter (where state = 'running')                                as running,
        count(*) filter (where state = 'suspended')                              as suspended,
        count(*) filter (where state = 'dead')                                   as dead,
        coalesce(max(extract(epoch from now() - run_at)) filter
-         (where state = 'ready' and run_at <= now()), 0) * 1000                 as oldest_ready_ms
+         (where state in ('ready', 'delayed') and run_at <= now()), 0) * 1000   as oldest_ready_ms
   from x_jobs
  group by queue
  order by queue

package/src/driver-pg.ts

@@ -20,7 +20,7 @@ import type {
   NackOptions,
   QueueStats,
 } from './driver';
-import { assertClaimBounds, assertClaimQueues, DEFAULT_QUEUE } from './driver';
+import { assertClaimBounds, assertClaimQueues, DEFAULT_QUEUE, REQUEUEABLE_STATES } from './driver';
 import type { BackfillRow, JobRow, StepRow } from './driver-pg-rows';
 import { num, toBackfillRun, toJobRecord, toStepRecord } from './driver-pg-rows';
 import {
@@ -38,6 +38,7 @@ import {
   SQL_JOB_DEAD_LETTERS,
   SQL_JOB_GET,
   SQL_JOB_LIST,
+  SQL_JOB_LIVE_HOLDER,
   SQL_JOB_REQUEUE,
   SQL_LEASE_ACQUIRE,
   SQL_LEASE_RELEASE,
@@ -47,9 +48,11 @@ import {
   SQL_STEP_GET,
   SQL_STEP_LIST,
   SQL_STEP_PUT,
+  SQL_STEPS_FROM,
   SQL_TRY_ADVISORY_LOCK,
 } from './driver-pg-sql';
 import { DriverUnavailableError, JobDuplicateError } from './errors';
+import { JobNotRequeueableError, requeueKeyTaken } from './errors-requeue';
 import type { HeldLease, LeaseStore } from './leases';
 import type { StepStore } from './steps';
 
@@ -213,24 +216,33 @@ export function createPgDriver(options: PgDriverOptions = {}): JobDriver {
       return rows.map(toJobRecord);
     },
     async requeue(jobId, requeueOptions) {
-      if (requeueOptions?.fromStep !== undefined) {
-        const current = await this.job(jobId);
-        if (current !== undefined) {
-          await exec().query(`delete from x_job_steps where run_id = $1 and name = $2`, [
-            current.runId,
-            requeueOptions.fromStep,
-          ]);
-        }
-      }
-      const rows = await exec().query<JobRow>(SQL_JOB_REQUEUE, [jobId]);
-      const row = rows[0];
-      if (row === undefined) {
+      const current = await this.job(jobId);
+      if (current === undefined) {
         throw new DriverUnavailableError({
           driver: 'pg',
           cause: `job ${jobId} does not exist`,
           fix: 'x jobs ls --state dead --json',
         });
       }
+      // Both refusals BEFORE anything is deleted or updated — a refused requeue changes nothing.
+      if (!REQUEUEABLE_STATES.has(current.state)) {
+        throw new JobNotRequeueableError({ jobId, state: current.state });
+      }
+      const holders = await exec().query<{ id: string }>(SQL_JOB_LIVE_HOLDER, [
+        current.name,
+        current.tenantId ?? null,
+        current.idempotencyKey,
+        jobId,
+      ]);
+      const holder = holders[0];
+      if (holder !== undefined) throw requeueKeyTaken({ ...current, holderId: holder.id });
+      if (requeueOptions?.fromStep !== undefined) {
+        await exec().query(SQL_STEPS_FROM, [current.runId, requeueOptions.fromStep]);
+      }
+      const rows = await exec().query<JobRow>(SQL_JOB_REQUEUE, [jobId]);
+      const row = rows[0];
+      // Read as finished a moment ago and live now: another requeue won the race.
+      if (row === undefined) throw new JobNotRequeueableError({ jobId, state: 'live' });
       return toJobRecord(row);
     },
     async cancel(jobId, reason) {

package/src/driver.ts

@@ -32,6 +32,17 @@ export const JOB_STATES = [
 
 export type JobState = (typeof JOB_STATES)[number];
 
+/**
+ * The states `requeue` — `x jobs retry` — accepts. Everything else is live: requeueing a running
+ * row ran it twice, and a queued one has nothing to retry. `SQL_JOB_REQUEUE` fences on the same set.
+ */
+export const REQUEUEABLE_STATES: ReadonlySet<JobState> = new Set<JobState>([
+  'dead',
+  'cancelled',
+  'done',
+  'failed',
+]);
+
 /** Narrows a state read back off a queue row. Never a cast — the list decides. */
 export const isJobState = (value: string): value is JobState =>
   (JOB_STATES as readonly string[]).includes(value);
@@ -175,7 +186,11 @@ export interface JobIntrospection {
   job(jobId: string): Promise<JobRecord | undefined>;
   list(filter?: JobFilter): Promise<readonly JobRecord[]>;
   deadLetters(limit?: number): Promise<readonly JobRecord[]>;
-  /** Re-queue a dead/failed job. `fromStep` drops step records from that step onward. */
+  /**
+   * Re-queue a finished job (`REQUEUEABLE_STATES`); a live one is `X_JOB_NOT_REQUEUEABLE`, and a
+   * key a live job holds is `X_JOB_DUPLICATE`. `fromStep` drops that step and every step that
+   * started after it.
+   */
   requeue(jobId: string, options?: { readonly fromStep?: string }): Promise<JobRecord>;
   /**
    * Stop a job from outside. The only answer to a runaway pass that was otherwise "scale the

package/src/errors-declaration.ts

@@ -0,0 +1,34 @@
+// Single responsibility: the refusal a `job()` declaration earns when it lacks a field every job
+// declares. Its own module because `errors.ts` sits at the size ceiling; the CODE is registered
+// there, beside every other jobs code, and only the constructor lives here.
+
+import { UltimateError } from '@ultimat3/core';
+
+/** The edit each required `job()` field takes, for `JobDeclarationInvalidError`'s fix line. */
+const REQUIRED_FIELD_FIX = Object.freeze<Record<string, string>>({
+  input: 'input: t.object({ … })',
+  idempotencyKey: `idempotencyKey: (input) => \`<name>:\${input.id}\``,
+  tenant: "tenant: (input) => input.orgId — or tenant: 'none' for a job no org owns",
+  retry: "retry: { attempts: 5, backoff: 'exponential' }",
+  run: 'run: async ({ input, step }) => { … }',
+});
+
+/**
+ * `job()` called without one or more fields the type requires — generated code and JS callers.
+ * EVERY missing field in one refusal: a missing `retry` used to crash the app loader with a bare
+ * `TypeError … definition.retry.attempts`, reported as `X_CLI_UNEXPECTED`, and fixing fields one
+ * restart at a time is the slow way to learn a declaration.
+ */
+export class JobDeclarationInvalidError extends UltimateError {
+  constructor(input: { job: string; missing: readonly string[] }) {
+    const edits = input.missing.map((field) =>
+      Object.hasOwn(REQUIRED_FIELD_FIX, field) ? REQUIRED_FIELD_FIX[field] : field,
+    );
+    super({
+      code: 'X_JOB_DECLARATION_INVALID',
+      cause: `job "${input.job}" is missing ${input.missing.join(', ')}, which every job declares`,
+      fix: `set ${edits.join('; ')} on job('${input.job}')`,
+      meta: { job: input.job, missing: [...input.missing] },
+    });
+  }
+}

package/src/errors-requeue.ts

@@ -0,0 +1,34 @@
+// Single responsibility: the two refusals `requeue` — `x jobs retry` — gives before it moves a
+// row. Its own module because `errors.ts` sits at the size ceiling; the codes are registered there.
+
+import { renderFixShellArg, UltimateError } from '@ultimat3/core';
+import { JobDuplicateError } from './errors';
+
+/**
+ * `x jobs retry` on a job that is still live. Requeueing a RUNNING row left `claimed_by` set with
+ * state `ready`, so a second worker claimed it and it ran twice; a queued one has nothing to retry.
+ */
+export class JobNotRequeueableError extends UltimateError {
+  constructor(input: { jobId: string; state: string }) {
+    super({
+      code: 'X_JOB_NOT_REQUEUEABLE',
+      cause: `job ${input.jobId} is ${input.state}, and only a dead, cancelled, failed or done job can be requeued — a live one would run twice`,
+      fix: `x jobs cancel ${renderFixShellArg(input.jobId, '<job id>')} first, then x jobs retry ${renderFixShellArg(input.jobId, '<job id>')}`,
+      meta: { jobId: input.jobId, state: input.state },
+    });
+  }
+}
+
+/** The error a requeue gives when a LIVE job already holds the requeued row's key. */
+export const requeueKeyTaken = (record: {
+  readonly id: string;
+  readonly name: string;
+  readonly idempotencyKey: string;
+  readonly holderId: string;
+}): JobDuplicateError =>
+  new JobDuplicateError({
+    job: record.name,
+    idempotencyKey: record.idempotencyKey,
+    existingId: record.holderId,
+    fix: `x jobs show ${renderFixShellArg(record.holderId, '<live job id>')} — that is the live run of this key; cancel it first (x jobs cancel ${renderFixShellArg(record.holderId, '<live job id>')}) only if this one must run instead`,
+  });

package/src/errors.ts

@@ -36,6 +36,8 @@ export const JOB_OWNED_ERROR_CODES = [
   'X_WEBHOOK_DELIVERY_REJECTED',
   'X_EXPORT_ROW_INVALID',
   'X_EXPORT_PART_TOO_LARGE',
+  'X_JOB_DECLARATION_INVALID',
+  'X_JOB_NOT_REQUEUEABLE',
 ] as const;
 
 /**
@@ -86,6 +88,8 @@ export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
   X_WEBHOOK_DELIVERY_REJECTED: 'the endpoint refused the delivery in a way a retry cannot change',
   X_EXPORT_ROW_INVALID: 'row() answered something the declared columns do not carry',
   X_EXPORT_PART_TOO_LARGE: 'one page encoded to more bytes than a part may hold',
+  X_JOB_DECLARATION_INVALID: 'the job declaration is missing a required field',
+  X_JOB_NOT_REQUEUEABLE: 'the job is still live and cannot be requeued',
 };
 
 // One unconditional call, so a second package claiming one of jobs' codes throws
@@ -148,11 +152,12 @@ registerErrorRetry({
 
 /** An enqueue collided with a live job holding the same idempotency key under `onConflict: 'error'`. */
 export class JobDuplicateError extends UltimateError {
-  constructor(input: { job: string; idempotencyKey: string; existingId: string }) {
+  /** `fix` for a caller the enqueue-time instruction does not fit — a requeue, where it is moot. */
+  constructor(input: { job: string; idempotencyKey: string; existingId: string; fix?: string }) {
     super({
       code: 'X_JOB_DUPLICATE',
       cause: `job "${input.job}" already queued as ${input.existingId} with idempotencyKey "${input.idempotencyKey}"`,
-      fix: 'pass onConflict: "dedupe" to enqueue, or make idempotencyKey narrower',
+      fix: input.fix ?? 'pass onConflict: "dedupe" to enqueue, or make idempotencyKey narrower',
     });
   }
 }
@@ -320,11 +325,20 @@ export class DriverUnavailableError extends UltimateError {
  * generated code and JS callers, so the guarantee holds at both ends.
  */
 export class IdempotencyRequiredError extends UltimateError {
-  constructor(input: { job: string }) {
+  /**
+   * `positional` is a name `job()` minted from a counter (`anonymous-job-3`). Spliced into the key
+   * template it would be written into persisted idempotency keys — and it MOVES when an import is
+   * added above the declaration, so every key written before the move stops matching. The fix
+   * names the edit instead of the counter.
+   */
+  constructor(input: { job: string; positional?: boolean }) {
     super({
       code: 'X_IDEMPOTENCY_REQUIRED',
       cause: `job "${input.job}" has no idempotencyKey — at-least-once delivery would run it twice`,
-      fix: `add idempotencyKey: (input) => \`${input.job}:\${input.id}\` to the job definition`,
+      fix:
+        input.positional === true
+          ? `give the job a stable name first — job({ name: 'syncInvoices', … }), or register it in apps/web/api/index.ts so its export name is stamped — then add idempotencyKey: (input) => \`syncInvoices:\${input.id}\``
+          : `add idempotencyKey: (input) => \`${input.job}:\${input.id}\` to the job definition`,
     });
   }
 }

package/src/index.ts

@@ -173,6 +173,8 @@ export {
   OutboxNoTxError,
   StepDuplicateError,
 } from './errors';
+export { JobDeclarationInvalidError } from './errors-declaration';
+export { JobNotRequeueableError } from './errors-requeue';
 export type { EventBus, JobEvent, MemoryEventBusOptions, PublishOptions } from './events';
 export { createMemoryEventBus, eventBus, publishEvent, setEventBus } from './events';
 export type { PgEventBusOptions } from './events-pg';

package/src/job.ts

@@ -18,6 +18,7 @@ import { describeJob } from './describe';
 import type { EnqueueResult } from './driver';
 import { DEFAULT_QUEUE } from './driver';
 import { IdempotencyRequiredError, JobNameTakenError } from './errors';
+import { JobDeclarationInvalidError } from './errors-declaration';
 import { NO_TENANT, tenantKeyFrom } from './limits';
 import type { EnqueueOptions } from './outbox';
 import { jobsFacade } from './outbox';
@@ -178,13 +179,31 @@ interface JobOrigin {
 
 const origin = new WeakMap<object, JobOrigin>();
 
+/** The required fields a definition lacks, in declaration order. Read off the value, not the type. */
+function missingJobFields(definition: object): string[] {
+  const loose = definition as unknown as Readonly<Record<string, unknown>>;
+  const missing: string[] = [];
+  if (typeof loose['input'] !== 'object' || loose['input'] === null) missing.push('input');
+  if (typeof loose['idempotencyKey'] !== 'function') missing.push('idempotencyKey');
+  if (loose['tenant'] === undefined) missing.push('tenant');
+  if (typeof loose['retry'] !== 'object' || loose['retry'] === null) missing.push('retry');
+  if (typeof loose['run'] !== 'function') missing.push('run');
+  return missing;
+}
+
 export function job<I>(definition: JobDefinition<I>): JobHandle<I> {
   anonymous += 1;
   const name = definition.name ?? `anonymous-job-${anonymous}`;
 
-  // Runtime backstops for generated code and JS callers; TS already forbids omitting either.
-  if (typeof definition.idempotencyKey !== 'function') {
-    throw new IdempotencyRequiredError({ job: name });
+  // Runtime backstops for generated code and JS callers; TS already forbids omitting any of them.
+  // Collected, never one at a time — see `JobDeclarationInvalidError`. The two fields with a code
+  // of their own keep it when they are the only thing missing: shipped codes keep their meaning.
+  const missing = missingJobFields(definition);
+  if (missing.length === 1 && missing[0] === 'idempotencyKey') {
+    throw new IdempotencyRequiredError({ job: name, positional: definition.name === undefined });
+  }
+  if (missing.length > 0 && !(missing.length === 1 && missing[0] === 'tenant')) {
+    throw new JobDeclarationInvalidError({ job: name, missing });
   }
   assertJobTenant(name, definition.tenant);
   assert(