@ultimat3/jobs 29.4.0 → 30.0.0

package/src/error-codes.ts

@@ -0,0 +1,233 @@
+// The X_* codes owned by @ultimat3/jobs, and nothing else: the two lists, their titles, the one
+// registration call and the retry classification. `bun run new-error-code --package jobs` writes
+// here. The classes that throw these codes live in `./errors`, which imports this file for its
+// registration, so a module that can construct one of them has the code registered.
+import { registerErrorCodes, registerErrorRetry } from '@ultimat3/core';
+
+/** Codes this package declares and owns. */
+export const JOB_OWNED_ERROR_CODES = [
+  'X_JOB_DUPLICATE',
+  'X_STEP_DUPLICATE',
+  'X_JOB_TIMEOUT',
+  'X_JOB_MAX_ATTEMPTS',
+  'X_DRIVER_UNAVAILABLE',
+  'X_IDEMPOTENCY_REQUIRED',
+  'X_JOB_TENANT_REQUIRED',
+  'X_JOB_CONCURRENCY_UNENFORCEABLE',
+  'X_JOB_LEASE_LOST',
+  'X_JOB_SLOT_LOST',
+  'X_JOB_NOT_CANCELLABLE',
+  'X_OUTBOX_NO_TX',
+  'X_BACKFILL_PENDING',
+  'X_BACKFILL_APPLIED',
+  'X_BACKFILL_ENVIRONMENT',
+  'X_BACKFILL_MIGRATION_PENDING',
+  'X_BACKFILL_RUNNING',
+  'X_BACKFILL_STALLED',
+  'X_BACKFILL_UNKNOWN',
+  'X_JOB_ROW_STATUS_UNKNOWN',
+  'X_ACTION_JOB_UNBRIDGED',
+  'X_JOB_CLAIM_QUEUES_EMPTY',
+  'X_WEBHOOK_ENDPOINT_UNKNOWN',
+  'X_WEBHOOK_ENDPOINT_INVALID',
+  'X_WEBHOOK_ENDPOINT_DISABLED',
+  'X_WEBHOOK_EVENT_UNKNOWN',
+  'X_WEBHOOK_EVENT_INVALID',
+  'X_WEBHOOK_DELIVERY_FAILED',
+  'X_WEBHOOK_DELIVERY_THROTTLED',
+  'X_WEBHOOK_DELIVERY_REJECTED',
+  'X_EXPORT_ROW_INVALID',
+  'X_EXPORT_PART_TOO_LARGE',
+  'X_JOB_DECLARATION_INVALID',
+  'X_JOB_NOT_REQUEUEABLE',
+  'X_JOB_KEY_BUSY',
+  'X_JOB_NOT_REMOVABLE',
+  'X_JOB_NOT_PROMOTABLE',
+  'X_JOB_PAGE_INVALID',
+  'X_JOB_ON_SETTLED_FAILED',
+  'X_JOB_NOT_FOUND',
+  'X_JOB_TENANT_MISMATCH',
+  'X_STEP_OUTPUT_UNSERIALIZABLE',
+  'X_JOB_EVENT_INVALID',
+  'X_IMPORT_HEADER_MISMATCH',
+  'X_IMPORT_RECORD_TOO_LARGE',
+  'X_IMPORT_TOO_MANY_ERRORS',
+  'X_JOB_ENQUEUE_BATCH_TOO_LARGE',
+  'X_WEBHOOK_RECEIVER_INVALID',
+  'X_WEBHOOK_RECEIVER_UNAVAILABLE',
+  'X_WEBHOOK_EVENT_ID_INVALID',
+  'X_JOB_COMPENSATION_FAILED',
+  'X_STEP_OUTCOME_UNKNOWN',
+  'X_CONSUMER_INVALID',
+  'X_CONSUMER_SELF_ENQUEUE',
+  'X_BACKFILL_SOURCE_INVALID',
+  'X_STEP_NOT_RESOLVABLE',
+  'X_JOB_ROLLUP_INVALID',
+  'X_JOB_REFRESH_VIEW_INVALID',
+  'X_SUBJECT_TABLE_UNDECLARED',
+  'X_SUBJECT_TREATMENT_INVALID',
+  'X_SUBJECT_CYCLE',
+] as const;
+
+/**
+ * `X_NOT_IMPLEMENTED`, `X_ABORTED` and `X_DRAINING` are `@ultimat3/core`'s. The first is thrown as
+ * core's own `NotImplementedError` (`inspect.ts`), the other two by
+ * `JobAbortedError` and `JobDrainedError` below; jobs keeps no title for any of the three, because the copy this file used to hold was a second title that nothing would have failed
+ * on once core's changed. Listed here all the same, so `JobErrorCode` can name every code a job
+ * can see — `X_DRAINING` was thrown for a day before it was, and the type said it could not be.
+ */
+export const JOB_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED', 'X_ABORTED', 'X_DRAINING'] as const;
+
+/** Every code jobs can throw: the ones it owns plus the ones it borrows. */
+export const JOB_ERROR_CODES = [...JOB_OWNED_ERROR_CODES, ...JOB_BORROWED_ERROR_CODES] as const;
+
+export type JobOwnedErrorCode = (typeof JOB_OWNED_ERROR_CODES)[number];
+export type JobErrorCode = (typeof JOB_ERROR_CODES)[number];
+
+export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
+  X_JOB_DUPLICATE: 'a live key already has a job',
+  X_STEP_DUPLICATE: 'two step.run calls share a name',
+  X_JOB_TIMEOUT: 'a job exceeded its wall-clock limit',
+  X_JOB_MAX_ATTEMPTS: 'the job exhausted its retries',
+  X_DRIVER_UNAVAILABLE: 'the queue driver is unreachable',
+  X_IDEMPOTENCY_REQUIRED: 'the job has no idempotencyKey',
+  X_JOB_TENANT_REQUIRED: 'the job declares no tenant',
+  X_JOB_CONCURRENCY_UNENFORCEABLE: 'job.concurrency is declared and cannot be enforced',
+  X_JOB_LEASE_LOST: 'the queue took this job back mid-run',
+  X_JOB_SLOT_LOST: 'the fleet concurrency slot was taken by another worker',
+  X_JOB_NOT_CANCELLABLE: 'the job cannot be cancelled',
+  X_OUTBOX_NO_TX: 'enqueue outside a transaction',
+  X_BACKFILL_PENDING: 'a declared backfill has never completed',
+  X_BACKFILL_APPLIED: 'the ledger already holds a completed pass',
+  X_BACKFILL_ENVIRONMENT: 'the backfill is not declared for this environment',
+  X_BACKFILL_MIGRATION_PENDING: 'the migration this backfill requires is not applied',
+  X_BACKFILL_RUNNING: 'a pass under this name is already live',
+  X_BACKFILL_STALLED: 'the sweep ended with rows its own count still matches',
+  X_BACKFILL_UNKNOWN: 'no declaration carries this backfill name',
+  X_JOB_ROW_STATUS_UNKNOWN: 'a queue row carries a status this build does not know',
+  X_ACTION_JOB_UNBRIDGED: 'an action projection was registered as a job',
+  X_JOB_CLAIM_QUEUES_EMPTY: 'a claim named no queue, and the drivers do not agree what that means',
+  X_WEBHOOK_ENDPOINT_UNKNOWN: 'no endpoint carries this id',
+  X_WEBHOOK_ENDPOINT_INVALID: 'the endpoint cannot be delivered to as declared',
+  X_WEBHOOK_ENDPOINT_DISABLED: 'the endpoint takes no deliveries',
+  X_WEBHOOK_EVENT_UNKNOWN: 'no event carries this id',
+  X_WEBHOOK_EVENT_INVALID: 'the event cannot be signed as declared',
+  X_WEBHOOK_DELIVERY_FAILED: 'the endpoint did not accept the delivery',
+  X_WEBHOOK_DELIVERY_THROTTLED: 'the endpoint asked for the delivery later, and said when',
+  X_WEBHOOK_DELIVERY_REJECTED: 'the endpoint refused the delivery in a way a retry cannot change',
+  X_EXPORT_ROW_INVALID: 'row() answered something the declared columns do not carry',
+  X_EXPORT_PART_TOO_LARGE: 'one page encoded to more bytes than a part may hold',
+  X_JOB_DECLARATION_INVALID: 'the job declaration is missing or mis-declares a field',
+  X_JOB_NOT_REQUEUEABLE: 'the job is still live and cannot be requeued',
+  X_JOB_KEY_BUSY: 'the concurrency key already holds its limit of runs',
+  X_JOB_NOT_REMOVABLE: 'the job is running and cannot be removed',
+  X_JOB_NOT_PROMOTABLE: 'the job is not waiting on its run time',
+  X_JOB_PAGE_INVALID: 'a job list page was asked for outside its bounds',
+  X_JOB_ON_SETTLED_FAILED: 'the onSettled hook failed after the run was settled',
+  X_JOB_NOT_FOUND: 'the queue holds no job with this id',
+  X_JOB_TENANT_MISMATCH: "a job's payload names an org other than the tenant it runs under",
+  X_STEP_OUTPUT_UNSERIALIZABLE: 'a step returned a value JSON cannot hold',
+  X_JOB_EVENT_INVALID: 'a waited event failed its schema',
+  X_IMPORT_HEADER_MISMATCH: 'the csv header does not name the declared columns',
+  X_IMPORT_RECORD_TOO_LARGE: 'one record runs past the bytes a record may hold',
+  X_IMPORT_TOO_MANY_ERRORS: 'more rows were rejected than the import allows',
+  X_JOB_ENQUEUE_BATCH_TOO_LARGE: 'enqueueMany was handed more inputs than one call may stage',
+  X_WEBHOOK_RECEIVER_INVALID: 'a webhookReceiver() declaration cannot be served',
+  X_WEBHOOK_RECEIVER_UNAVAILABLE:
+    'an inbound webhook could not be queued, so it was not acknowledged',
+  X_WEBHOOK_EVENT_ID_INVALID:
+    'the event id a webhook receiver verified is not one it can key a job by',
+  X_JOB_COMPENSATION_FAILED: 'a compensator could not undo its step',
+  X_STEP_OUTCOME_UNKNOWN: 'a once step started and never completed',
+  X_CONSUMER_INVALID: 'a consumer() declaration cannot be run as declared',
+  X_CONSUMER_SELF_ENQUEUE: 'a consumer enqueued itself from inside its own run',
+  X_BACKFILL_SOURCE_INVALID:
+    'a backfill() declares both or neither of source and pages, or input with no scope',
+  X_STEP_NOT_RESOLVABLE: 'the step is not an unknown once outcome an operator can resolve',
+  X_JOB_ROLLUP_INVALID: 'a rollup declaration cannot be run',
+  X_JOB_REFRESH_VIEW_INVALID: 'a refreshView declaration names no materialized view it can refresh',
+  X_SUBJECT_TABLE_UNDECLARED:
+    'a table reaches the declared subject and defineSubject gives it no treatment',
+  X_SUBJECT_TREATMENT_INVALID: 'a defineSubject treatment cannot be carried out',
+  X_SUBJECT_CYCLE: 'the subject erase has no deletion order',
+};
+
+// One unconditional call, so a second package claiming one of jobs' codes throws
+// X_ERROR_CODE_DUPLICATE instead of losing silently to whichever module imported first.
+registerErrorCodes(
+  Object.fromEntries(Object.entries(JOB_ERROR_TITLES).map(([code, title]) => [code, { title }])),
+);
+
+/**
+ * The codes of this package's that can be thrown INSIDE a job body, classified — `executeJob`
+ * reads this, so a `terminal` one dead-letters on the attempt it happened instead of spending the
+ * whole policy on an answer that cannot change. Same rule every package uses: retryable means the
+ * same code, run again, has a real chance of a different answer.
+ *
+ * Two are deliberately absent. `X_JOB_LEASE_LOST` and `X_JOB_SLOT_LOST` mean the row is somebody
+ * else's now, so this attempt's verdict is not this attempt's to give: dead-lettering would settle
+ * a job another worker is running. They keep the attempt-count path, which ends in the queue
+ * re-delivering — the honest outcome for "we stopped owning it".
+ */
+registerErrorRetry({
+  X_JOB_TIMEOUT: 'retryable',
+  X_DRIVER_UNAVAILABLE: 'retryable',
+  // A second `step.run` under one name is a defect in the handler, replayed identically forever.
+  X_STEP_DUPLICATE: 'terminal',
+  // A sweep whose source ran dry while its own count still matches rows: the next attempt resumes
+  // at the cursor that just ran dry and diverges again.
+  X_BACKFILL_STALLED: 'terminal',
+  X_BACKFILL_ENVIRONMENT: 'terminal',
+  X_BACKFILL_APPLIED: 'terminal',
+  // The one retryable webhook code, and the reason the split exists at all: a 5xx, a 429 or a
+  // connection that never opened is the same request landing later, which is what the job's retry
+  // policy is for.
+  X_WEBHOOK_DELIVERY_FAILED: 'retryable',
+  // `retry-after` and not `retryable`: the receiver NAMED a delay, so `statedDelayMs` reads
+  // `meta.retryAfterSeconds` and the nack waits that long (clamped by the policy's `maxDelay`)
+  // instead of guessing a curve against an answer it was already given.
+  X_WEBHOOK_DELIVERY_THROTTLED: 'retry-after',
+  // The rest are `terminal` and every one is LISTED rather than left to the default, because
+  // `classifyThrown` reads an unregistered code carrying `terminal` as UNCLASSIFIED — which spends
+  // a delivery's whole retry policy re-proving that a 404 endpoint is still a 404.
+  X_WEBHOOK_DELIVERY_REJECTED: 'terminal',
+  X_WEBHOOK_ENDPOINT_UNKNOWN: 'terminal',
+  X_WEBHOOK_ENDPOINT_INVALID: 'terminal',
+  X_WEBHOOK_ENDPOINT_DISABLED: 'terminal',
+  X_WEBHOOK_EVENT_UNKNOWN: 'terminal',
+  X_WEBHOOK_EVENT_INVALID: 'terminal',
+  // Both export codes are refusals of the DECLARATION and not of the data: the same `row()` over
+  // the same page answers the same way on every attempt, so an unclassified reading would spend
+  // the whole retry policy proving it.
+  X_EXPORT_ROW_INVALID: 'terminal',
+  X_EXPORT_PART_TOO_LARGE: 'terminal',
+  // An import reads the same bytes on every attempt: a header, an unclosed quote and a count of
+  // refused rows answer identically, so the repair is the file and a new enqueue, never a retry.
+  X_IMPORT_HEADER_MISMATCH: 'terminal',
+  X_IMPORT_RECORD_TOO_LARGE: 'terminal',
+  X_IMPORT_TOO_MANY_ERRORS: 'terminal',
+  // The worker settles a busy key itself; listed so a body that RETHROWS one it was handed stops.
+  X_JOB_KEY_BUSY: 'terminal',
+  // The payload names the wrong org, or none its tenant can run under — a fact about the ROW, so
+  // every attempt and every `x jobs retry` is refused identically: the repair is a new enqueue.
+  X_JOB_TENANT_MISMATCH: 'terminal',
+  // The same value is refused on attempt five; the step's side effect has already happened.
+  X_STEP_OUTPUT_UNSERIALIZABLE: 'terminal',
+  // The stored event is the same on every attempt: the repair is the publisher or the schema.
+  X_JOB_EVENT_INVALID: 'terminal',
+  // The same list is the same length on every attempt: the repair is the caller's slicing.
+  X_JOB_ENQUEUE_BATCH_TOO_LARGE: 'terminal',
+  // A step name the framework reserves, or `compensate` on a job without `compensates: true`: the
+  // body is the same on every attempt, so the repair is the declaration.
+  X_JOB_DECLARATION_INVALID: 'terminal',
+  // Only a person or the vendor can say whether a `once` step ran (`x jobs resolve-step`).
+  X_STEP_OUTCOME_UNKNOWN: 'terminal',
+  // The unwind's own attempts are spent, or it met an unknown `once` step: the repair is a person.
+  X_JOB_COMPENSATION_FAILED: 'terminal',
+  // The body that enqueued itself does so on every attempt: the repair is deleting the call.
+  X_CONSUMER_SELF_ENQUEUE: 'terminal',
+  // A subject's plan is read off the declarations: the same entities plan the same way every time.
+  X_SUBJECT_TABLE_UNDECLARED: 'terminal',
+  X_SUBJECT_TREATMENT_INVALID: 'terminal',
+  X_SUBJECT_CYCLE: 'terminal',
+});

package/src/errors-concurrency.ts

@@ -1,7 +1,7 @@
 // Single responsibility: the refusals `job.concurrency` earns — a cap mis-declared, a key that
 // cannot name a lease, a key already holding its limit, and a driver that cannot hold the cap.
-// Its own module because `errors.ts` sits at the size ceiling; the CODES are registered there,
-// beside every other jobs code.
+// Its own module because `errors.ts` sits at the size ceiling; the CODES are registered in
+// `error-codes.ts`, beside every other jobs code.
 
 import { isFixShellSafe, renderCauseValue, renderFixShellArg, UltimateError } from '@ultimat3/core';
 

package/src/errors-declaration.ts

@@ -1,6 +1,7 @@
 // Single responsibility: the refusal a `job()` declaration earns when it lacks a field every job
-// declares. Its own module because `errors.ts` sits at the size ceiling; the CODE is registered
-// there, beside every other jobs code, and only the constructor lives here.
+// declares. Its own module because `errors.ts` sits at the size ceiling; the CODE is registered in
+// `error-codes.ts`, beside every other jobs code; the constructor and the read that finds the
+// missing fields live here.
 
 import { UltimateError } from '@ultimat3/core';
 
@@ -32,3 +33,15 @@ export class JobDeclarationInvalidError extends UltimateError {
     });
   }
 }
+
+/** The required fields a definition lacks, in declaration order. Read off the value, not the type. */
+export function missingJobFields(definition: object): string[] {
+  const loose = definition as unknown as Readonly<Record<string, unknown>>;
+  const missing: string[] = [];
+  if (typeof loose['input'] !== 'object' || loose['input'] === null) missing.push('input');
+  if (typeof loose['idempotencyKey'] !== 'function') missing.push('idempotencyKey');
+  if (loose['tenant'] === undefined) missing.push('tenant');
+  if (typeof loose['retry'] !== 'object' || loose['retry'] === null) missing.push('retry');
+  if (typeof loose['run'] !== 'function') missing.push('run');
+  return missing;
+}

package/src/errors-operator.ts

@@ -1,7 +1,7 @@
 // Single responsibility: the refusals the operator surface gives — a job that cannot be removed
 // or promoted, a page it cannot answer — and the one failure it only ever reports, a settled
 // run's `onSettled` hook. Its own
-// module because `errors.ts` sits at the size ceiling; the codes are registered there.
+// module because `errors.ts` sits at the size ceiling; the codes are registered in `error-codes.ts`.
 
 import { isFixShellSafe, renderFixShellArg, renderThrowable, UltimateError } from '@ultimat3/core';
 

package/src/errors-requeue.ts

@@ -1,6 +1,6 @@
 // Single responsibility: the three refusals `requeue` — `x jobs retry` — gives before it moves a
 // row, raised identically by `driver-memory.ts` and `driver-pg.ts`: one answer for both drivers is
-// the point, and neither driver imports the other. The codes are registered in `errors.ts`.
+// the point, and neither driver imports the other. The codes are registered in `error-codes.ts`.
 
 import { renderCauseValue, renderFixShellArg, UltimateError } from '@ultimat3/core';
 import { JobDuplicateError } from './errors';

package/src/errors-saga.ts

@@ -0,0 +1,87 @@
+// The refusals of durable compensation and at-most-once steps: a step name the framework reserves,
+// a `compensate` its job did not declare, a `once` step whose outcome nobody knows, an unwind that
+// could not finish, and a resolve with nothing to resolve. The codes are in `error-codes.ts`.
+
+import { renderCauseValue, renderFixShellArg, UltimateError } from '@ultimat3/core';
+
+/** The step the framework writes before the first compensator: the durable "this run unwinds". */
+export const UNWINDING_STEP = 'unwinding';
+/** Each compensator runs as an ordinary step under this prefix, so a replay skips what ran. */
+export const COMPENSATE_PREFIX = 'compensate:';
+
+/** A body named a step the framework writes itself. Refused for every job, `compensates` or not. */
+export class StepNameReservedError extends UltimateError {
+  constructor(input: { job: string; step: string }) {
+    super({
+      code: 'X_JOB_DECLARATION_INVALID',
+      cause: `job "${input.job}" names a step "${input.step}", and "${UNWINDING_STEP}" and "${COMPENSATE_PREFIX}*" are the names compensation writes`,
+      fix: `step.run('<another name>', …)   # in job('${input.job}'), instead of "${input.step}": "${UNWINDING_STEP}" and "${COMPENSATE_PREFIX}*" are compensation's`,
+      meta: { job: input.job, step: input.step },
+    });
+  }
+}
+
+/** `step.run(…, { compensate })` in a job that never declared it compensates. */
+export class StepCompensateUndeclaredError extends UltimateError {
+  constructor(input: { job: string; step: string }) {
+    super({
+      code: 'X_JOB_DECLARATION_INVALID',
+      cause: `job "${input.job}" step "${input.step}" passes compensate, and the job does not declare compensates: true, so nothing would ever run it`,
+      fix: `job({ name: '${input.job}', compensates: true, … })   # or drop compensate from step.run('${input.step}', …)`,
+      meta: { job: input.job, step: input.step },
+    });
+  }
+}
+
+/** A `once` step has a `started` (or `failed`) record and no `completed` one: did it happen? */
+export class StepOutcomeUnknownError extends UltimateError {
+  constructor(input: { job: string; step: string }) {
+    super({
+      code: 'X_STEP_OUTCOME_UNKNOWN',
+      cause: `job "${input.job}" step "${input.step}" is declared once, it started on an earlier attempt and never completed, so whether it ran is unknown and running it again could do it twice`,
+      fix: `x jobs resolve-step <id> ${renderFixShellArg(input.step, '<step>')} --ran --output '<json>'   # or --not-ran to run it again; then x jobs retry <id>`,
+      meta: { job: input.job, step: input.step },
+    });
+  }
+}
+
+/**
+ * The unwind stopped: a compensator spent the unwind's attempts, or it reached a `once` step whose
+ * outcome is unknown — undoing something that may not have happened is a person's call.
+ */
+export class JobCompensationFailedError extends UltimateError {
+  constructor(input: {
+    job: string;
+    step: string;
+    outcome: 'failed' | 'unknown';
+    reason: string;
+  }) {
+    const unknown = input.outcome === 'unknown';
+    super({
+      code: 'X_JOB_COMPENSATION_FAILED',
+      cause: unknown
+        ? `job "${input.job}" is unwinding and step "${input.step}" is once with an unknown outcome (X_STEP_OUTCOME_UNKNOWN), so its compensator cannot know whether there is anything to undo`
+        : `job "${input.job}" is unwinding and the compensator of step "${input.step}" failed: ${input.reason}`,
+      fix: unknown
+        ? `x jobs resolve-step <id> ${renderFixShellArg(input.step, '<step>')} --ran --output '<json>'   # or --not-ran; then x jobs retry <id>, and completed compensators replay`
+        : `x jobs retry <id>   # once the compensate of step.run('${input.step}', …) in job('${input.job}') is repaired: completed compensators replay`,
+      meta: { job: input.job, step: input.step, outcome: input.outcome },
+    });
+  }
+}
+
+/** Whether `name` is one the framework writes: refused when a body asks for it. */
+export const isReservedStepName = (name: string): boolean =>
+  name === UNWINDING_STEP || name.startsWith(COMPENSATE_PREFIX);
+
+/** `x jobs resolve-step` on a running job, or on a step that has no unknown outcome to answer. */
+export class StepNotResolvableError extends UltimateError {
+  constructor(input: { jobId: string; step: string; state: string; status: string }) {
+    super({
+      code: 'X_STEP_NOT_RESOLVABLE',
+      cause: `job ${renderCauseValue(input.jobId)} is ${input.state} and its step ${renderCauseValue(input.step)} is ${input.status}: only a step that started (or failed) and never completed, on a job no worker holds, has an outcome to resolve`,
+      fix: `x jobs show ${renderFixShellArg(input.jobId, '<job id>')} --json   # read the steps: resolve one whose status is started or failed, once the job has settled`,
+      meta: { jobId: input.jobId, step: input.step, state: input.state, status: input.status },
+    });
+  }
+}

package/src/errors-steps.ts

@@ -0,0 +1,35 @@
+// The two refusals a step answer can raise: a value JSON cannot hold, and a waited event its schema
+// refuses. Both are terminal, and their codes stay declared in `error-codes.ts`.
+
+import { UltimateError } from '@ultimat3/core';
+
+/** `step.run`'s body answered what a store cannot hold faithfully, so no attempt could replay it. */
+export class StepOutputUnserializableError extends UltimateError {
+  constructor(input: { job: string; step: string; reason: string }) {
+    super({
+      code: 'X_STEP_OUTPUT_UNSERIALIZABLE',
+      cause: `job "${input.job}" step "${input.step}" answered a value JSON cannot hold (${input.reason}), so no attempt could replay it`,
+      fix: `step.run('${input.step}', () => String(id))   # return plain JSON: String(id) for a bigint, the ids rather than the object graph, null rather than NaN or an invalid Date, text with no NUL and no lone surrogate`,
+      meta: { job: input.job, step: input.step },
+    });
+  }
+}
+
+/**
+ * The event `step.waitForEvent` found fails the wait's own schema, on a hit or on a replay.
+ *
+ * Terminal rather than skipped: skipping hides the publisher's bug until the timeout. `issues` come
+ * from `formatIssues`, and the payload itself is never echoed, because an event can carry anything.
+ * The wait records the refused event as its floor, so after the retry the fix names, it answers to
+ * the next one published rather than finding this one again.
+ */
+export class JobEventInvalidError extends UltimateError {
+  constructor(input: { job: string; step: string; event: string; issues: readonly string[] }) {
+    super({
+      code: 'X_JOB_EVENT_INVALID',
+      cause: `job "${input.job}" step "${input.step}" found event "${input.event}", and it fails the wait's schema: ${input.issues.join('; ')}`,
+      fix: `x jobs retry <id>   # after the publisher of "${input.event}" sends what the schema of step.waitForEvent('${input.step}', …) declares, or that schema is widened`,
+      meta: { job: input.job, step: input.step, event: input.event },
+    });
+  }
+}

package/src/errors-tenant.ts

@@ -1,6 +1,6 @@
 // The one refusal for a job whose PAYLOAD and declared tenant disagree — a webhook delivery's
 // `orgId` that is not the org the run is under, a tenant that derives no org from the payload, an
-// agent actor resolved in another org. Its code stays declared in `errors.ts`, classified terminal.
+// agent actor resolved in another org. Its code stays declared in `error-codes.ts`, classified terminal.
 
 import { UltimateError } from '@ultimat3/core';
 

package/src/errors.ts

@@ -1,166 +1,8 @@
-// The X_* codes owned by @ultimat3/jobs. Every one names the command or code change that
-// fixes it — a job failure an agent cannot act on is a job failure that gets retried forever.
-import { registerErrorCodes, registerErrorRetry, UltimateError } from '@ultimat3/core';
-
-/** Codes this package declares and owns. */
-export const JOB_OWNED_ERROR_CODES = [
-  'X_JOB_DUPLICATE',
-  'X_STEP_DUPLICATE',
-  'X_JOB_TIMEOUT',
-  'X_JOB_MAX_ATTEMPTS',
-  'X_DRIVER_UNAVAILABLE',
-  'X_IDEMPOTENCY_REQUIRED',
-  'X_JOB_TENANT_REQUIRED',
-  'X_JOB_CONCURRENCY_UNENFORCEABLE',
-  'X_JOB_LEASE_LOST',
-  'X_JOB_SLOT_LOST',
-  'X_JOB_NOT_CANCELLABLE',
-  'X_OUTBOX_NO_TX',
-  'X_BACKFILL_PENDING',
-  'X_BACKFILL_APPLIED',
-  'X_BACKFILL_ENVIRONMENT',
-  'X_BACKFILL_MIGRATION_PENDING',
-  'X_BACKFILL_RUNNING',
-  'X_BACKFILL_STALLED',
-  'X_BACKFILL_UNKNOWN',
-  'X_JOB_ROW_STATUS_UNKNOWN',
-  'X_ACTION_JOB_UNBRIDGED',
-  'X_JOB_CLAIM_QUEUES_EMPTY',
-  'X_WEBHOOK_ENDPOINT_UNKNOWN',
-  'X_WEBHOOK_ENDPOINT_INVALID',
-  'X_WEBHOOK_ENDPOINT_DISABLED',
-  'X_WEBHOOK_EVENT_UNKNOWN',
-  'X_WEBHOOK_EVENT_INVALID',
-  'X_WEBHOOK_DELIVERY_FAILED',
-  'X_WEBHOOK_DELIVERY_THROTTLED',
-  'X_WEBHOOK_DELIVERY_REJECTED',
-  'X_EXPORT_ROW_INVALID',
-  'X_EXPORT_PART_TOO_LARGE',
-  'X_JOB_DECLARATION_INVALID',
-  'X_JOB_NOT_REQUEUEABLE',
-  'X_JOB_KEY_BUSY',
-  'X_JOB_NOT_REMOVABLE',
-  'X_JOB_NOT_PROMOTABLE',
-  'X_JOB_PAGE_INVALID',
-  'X_JOB_ON_SETTLED_FAILED',
-  'X_JOB_NOT_FOUND',
-  'X_JOB_TENANT_MISMATCH',
-] as const;
-
-/**
- * `X_NOT_IMPLEMENTED`, `X_ABORTED` and `X_DRAINING` are `@ultimat3/core`'s. The first is thrown as
- * core's own `NotImplementedError` (`inspect.ts`), the other two by
- * `JobAbortedError` and `JobDrainedError` below; jobs keeps no title for any of the three, because the copy this file used to hold was a second title that nothing would have failed
- * on once core's changed. Listed here all the same, so `JobErrorCode` can name every code a job
- * can see — `X_DRAINING` was thrown for a day before it was, and the type said it could not be.
- */
-export const JOB_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED', 'X_ABORTED', 'X_DRAINING'] as const;
-
-/** Every code jobs can throw: the ones it owns plus the ones it borrows. */
-export const JOB_ERROR_CODES = [...JOB_OWNED_ERROR_CODES, ...JOB_BORROWED_ERROR_CODES] as const;
-
-export type JobOwnedErrorCode = (typeof JOB_OWNED_ERROR_CODES)[number];
-export type JobErrorCode = (typeof JOB_ERROR_CODES)[number];
-
-export const JOB_ERROR_TITLES: Readonly<Record<JobOwnedErrorCode, string>> = {
-  X_JOB_DUPLICATE: 'a live key already has a job',
-  X_STEP_DUPLICATE: 'two step.run calls share a name',
-  X_JOB_TIMEOUT: 'a job exceeded its wall-clock limit',
-  X_JOB_MAX_ATTEMPTS: 'the job exhausted its retries',
-  X_DRIVER_UNAVAILABLE: 'the queue driver is unreachable',
-  X_IDEMPOTENCY_REQUIRED: 'the job has no idempotencyKey',
-  X_JOB_TENANT_REQUIRED: 'the job declares no tenant',
-  X_JOB_CONCURRENCY_UNENFORCEABLE: 'job.concurrency is declared and cannot be enforced',
-  X_JOB_LEASE_LOST: 'the queue took this job back mid-run',
-  X_JOB_SLOT_LOST: 'the fleet concurrency slot was taken by another worker',
-  X_JOB_NOT_CANCELLABLE: 'the job cannot be cancelled',
-  X_OUTBOX_NO_TX: 'enqueue outside a transaction',
-  X_BACKFILL_PENDING: 'a declared backfill has never completed',
-  X_BACKFILL_APPLIED: 'the ledger already holds a completed pass',
-  X_BACKFILL_ENVIRONMENT: 'the backfill is not declared for this environment',
-  X_BACKFILL_MIGRATION_PENDING: 'the migration this backfill requires is not applied',
-  X_BACKFILL_RUNNING: 'a pass under this name is already live',
-  X_BACKFILL_STALLED: 'the sweep ended with rows its own count still matches',
-  X_BACKFILL_UNKNOWN: 'no declaration carries this backfill name',
-  X_JOB_ROW_STATUS_UNKNOWN: 'a queue row carries a status this build does not know',
-  X_ACTION_JOB_UNBRIDGED: 'an action projection was registered as a job',
-  X_JOB_CLAIM_QUEUES_EMPTY: 'a claim named no queue, and the drivers do not agree what that means',
-  X_WEBHOOK_ENDPOINT_UNKNOWN: 'no endpoint carries this id',
-  X_WEBHOOK_ENDPOINT_INVALID: 'the endpoint cannot be delivered to as declared',
-  X_WEBHOOK_ENDPOINT_DISABLED: 'the endpoint takes no deliveries',
-  X_WEBHOOK_EVENT_UNKNOWN: 'no event carries this id',
-  X_WEBHOOK_EVENT_INVALID: 'the event cannot be signed as declared',
-  X_WEBHOOK_DELIVERY_FAILED: 'the endpoint did not accept the delivery',
-  X_WEBHOOK_DELIVERY_THROTTLED: 'the endpoint asked for the delivery later, and said when',
-  X_WEBHOOK_DELIVERY_REJECTED: 'the endpoint refused the delivery in a way a retry cannot change',
-  X_EXPORT_ROW_INVALID: 'row() answered something the declared columns do not carry',
-  X_EXPORT_PART_TOO_LARGE: 'one page encoded to more bytes than a part may hold',
-  X_JOB_DECLARATION_INVALID: 'the job declaration is missing or mis-declares a field',
-  X_JOB_NOT_REQUEUEABLE: 'the job is still live and cannot be requeued',
-  X_JOB_KEY_BUSY: 'the concurrency key already holds its limit of runs',
-  X_JOB_NOT_REMOVABLE: 'the job is running and cannot be removed',
-  X_JOB_NOT_PROMOTABLE: 'the job is not waiting on its run time',
-  X_JOB_PAGE_INVALID: 'a job list page was asked for outside its bounds',
-  X_JOB_ON_SETTLED_FAILED: 'the onSettled hook failed after the run was settled',
-  X_JOB_NOT_FOUND: 'the queue holds no job with this id',
-  X_JOB_TENANT_MISMATCH: "a job's payload names an org other than the tenant it runs under",
-};
-
-// One unconditional call, so a second package claiming one of jobs' codes throws
-// X_ERROR_CODE_DUPLICATE instead of losing silently to whichever module imported first.
-registerErrorCodes(
-  Object.fromEntries(Object.entries(JOB_ERROR_TITLES).map(([code, title]) => [code, { title }])),
-);
-
-/**
- * The codes of this package's that can be thrown INSIDE a job body, classified — `executeJob`
- * reads this, so a `terminal` one dead-letters on the attempt it happened instead of spending the
- * whole policy on an answer that cannot change. Same rule every package uses: retryable means the
- * same code, run again, has a real chance of a different answer.
- *
- * Two are deliberately absent. `X_JOB_LEASE_LOST` and `X_JOB_SLOT_LOST` mean the row is somebody
- * else's now, so this attempt's verdict is not this attempt's to give: dead-lettering would settle
- * a job another worker is running. They keep the attempt-count path, which ends in the queue
- * re-delivering — the honest outcome for "we stopped owning it".
- */
-registerErrorRetry({
-  X_JOB_TIMEOUT: 'retryable',
-  X_DRIVER_UNAVAILABLE: 'retryable',
-  // A second `step.run` under one name is a defect in the handler, replayed identically forever.
-  X_STEP_DUPLICATE: 'terminal',
-  // A sweep whose source ran dry while its own count still matches rows: the next attempt resumes
-  // at the cursor that just ran dry and diverges again.
-  X_BACKFILL_STALLED: 'terminal',
-  X_BACKFILL_ENVIRONMENT: 'terminal',
-  X_BACKFILL_APPLIED: 'terminal',
-  // The one retryable webhook code, and the reason the split exists at all: a 5xx, a 429 or a
-  // connection that never opened is the same request landing later, which is what the job's retry
-  // policy is for.
-  X_WEBHOOK_DELIVERY_FAILED: 'retryable',
-  // `retry-after` and not `retryable`: the receiver NAMED a delay, so `statedDelayMs` reads
-  // `meta.retryAfterSeconds` and the nack waits that long (clamped by the policy's `maxDelay`)
-  // instead of guessing a curve against an answer it was already given.
-  X_WEBHOOK_DELIVERY_THROTTLED: 'retry-after',
-  // The rest are `terminal` and every one is LISTED rather than left to the default, because
-  // `classifyThrown` reads an unregistered code carrying `terminal` as UNCLASSIFIED — which spends
-  // a delivery's whole retry policy re-proving that a 404 endpoint is still a 404.
-  X_WEBHOOK_DELIVERY_REJECTED: 'terminal',
-  X_WEBHOOK_ENDPOINT_UNKNOWN: 'terminal',
-  X_WEBHOOK_ENDPOINT_INVALID: 'terminal',
-  X_WEBHOOK_ENDPOINT_DISABLED: 'terminal',
-  X_WEBHOOK_EVENT_UNKNOWN: 'terminal',
-  X_WEBHOOK_EVENT_INVALID: 'terminal',
-  // Both export codes are refusals of the DECLARATION and not of the data: the same `row()` over
-  // the same page answers the same way on every attempt, so an unclassified reading would spend
-  // the whole retry policy proving it.
-  X_EXPORT_ROW_INVALID: 'terminal',
-  X_EXPORT_PART_TOO_LARGE: 'terminal',
-  // The worker settles a busy key itself; listed so a body that RETHROWS one it was handed stops.
-  X_JOB_KEY_BUSY: 'terminal',
-  // The payload names the wrong org, or none its tenant can run under — a fact about the ROW, so
-  // every attempt and every `x jobs retry` is refused identically: the repair is a new enqueue.
-  X_JOB_TENANT_MISMATCH: 'terminal',
-});
+// The error classes @ultimat3/jobs throws. Every one names the command or code change that fixes
+// it — a job failure an agent cannot act on is a job failure that gets retried forever. The codes,
+// their titles and their registration are `./error-codes`, imported here for that side effect.
+import { UltimateError } from '@ultimat3/core';
+import './error-codes';
 
 // No `docs:` on any class below, here or in `backfill-errors.ts`. `UltimateError` fills it from
 // `describeErrorCode(code).docs`, which is `@ultimat3/core`'s `ERROR_DOCS_URL` — one page for every
@@ -262,14 +104,16 @@ export class StepDuplicateError extends UltimateError {
 }
 
 export class JobTimeoutError extends UltimateError {
-  constructor(input: { job: string; timeoutMs: number; step?: string }) {
+  /** `fix` for a ceiling the job's `timeout` does not set: the `onSettled` hook's. */
+  constructor(input: { job: string; timeoutMs: number; step?: string; fix?: string }) {
     super({
       code: 'X_JOB_TIMEOUT',
       cause:
         input.step === undefined
           ? `job "${input.job}" exceeded its ${input.timeoutMs}ms timeout`
           : `job "${input.job}" step "${input.step}" exceeded its ${input.timeoutMs}ms timeout`,
-      fix: `raise timeout on the job definition, or split the work into step.run() calls`,
+      fix:
+        input.fix ?? `raise timeout on the job definition, or split the work into step.run() calls`,
     });
   }
 }
@@ -450,6 +294,22 @@ export class CancelUnsupportedError extends UltimateError {
   }
 }
 
+/**
+ * `enqueueMany` handed more inputs than one call takes. Refused before any key is derived or any
+ * row is written, so nothing of the batch lands: a list that size is a table, and a table is a
+ * `backfill()` that pages it, not one call holding every row in memory.
+ */
+export class JobEnqueueBatchTooLargeError extends UltimateError {
+  constructor(input: { job: string; count: number; max: number }) {
+    super({
+      code: 'X_JOB_ENQUEUE_BATCH_TOO_LARGE',
+      cause: `${input.job}.enqueueMany was handed ${input.count} inputs, past the ${input.max} one call may stage`,
+      fix: `await handle.enqueueMany(inputs.slice(i, i + ${input.max})) for i = 0, ${input.max}, … with handle = ${input.job} — or page the source with backfill() from @ultimat3/jobs`,
+      meta: { job: input.job, count: input.count, max: input.max },
+    });
+  }
+}
+
 /** An outbox enqueue happened with no ambient transaction to join. */
 export class OutboxNoTxError extends UltimateError {
   constructor(input: { job: string }) {