@ultimat3/jobs 29.4.0 → 30.0.0

package/src/job-enqueue-many.ts

@@ -0,0 +1,108 @@
+// `handle.enqueueMany(inputs)`: many jobs of one kind in one statement per chunk, on the same two
+// paths `enqueue` takes — staged in the caller's transaction, or straight to the driver. A loop of
+// `enqueue` is the N+1 write the detector refuses (`X_N_PLUS_ONE_WRITE`): 1,000 rows were 1,000
+// statements and a second on Postgres, against one statement and 18 ms (measured 2026-11-02).
+
+import { uuidV7 } from '@ultimat3/core';
+import { refuseAbandoned } from '@ultimat3/db';
+import type { Tx } from '@ultimat3/entity';
+import { nowMs } from './clock';
+import type { EnqueueResult } from './driver';
+import { ambientTraceparent, directRequest, stagedRecord } from './enqueue-record';
+import { signalEnqueued, signalStaged } from './enqueue-signal';
+import { JobEnqueueBatchTooLargeError, OutboxNoTxError } from './errors';
+import type { JobHandle } from './job';
+import type { EnqueueOptions, OutboxDeps, OutboxRecord, OutboxStore } from './outbox';
+import { refuseEnqueueTenant } from './tenant';
+
+/**
+ * The most inputs one call takes. A list longer than this is a table, and a table is paged by a
+ * `backfill()` — never held whole in memory and staged inside one request's transaction.
+ */
+export const MAX_ENQUEUE_BATCH = 10_000;
+
+/** Rows per statement: bounds one bind's size, and so the time one statement holds its locks. */
+export const ENQUEUE_CHUNK = 1_000;
+
+/**
+ * One set of options for the whole batch, and no `runId`: one id names one run. A batch whose
+ * inputs need options of their own is a loop of `enqueue`.
+ */
+export type EnqueueManyOptions = Omit<EnqueueOptions, 'runId'>;
+
+function chunked<T>(rows: readonly T[]): readonly (readonly T[])[] {
+  const out: T[][] = [];
+  for (let at = 0; at < rows.length; at += ENQUEUE_CHUNK) {
+    out.push(rows.slice(at, at + ENQUEUE_CHUNK));
+  }
+  return out;
+}
+
+/** Stage `records` in `tx` in one statement when the store can, else one by one, in order. */
+export async function stageAll(
+  store: OutboxStore,
+  tx: Tx,
+  records: readonly OutboxRecord[],
+): Promise<void> {
+  if (store.stageMany !== undefined) return store.stageMany(tx, records);
+  for (const record of records) await store.stage(tx, record);
+}
+
+/**
+ * The facade's `enqueueMany`. Every refusal is decided before the first write: the abandoned
+ * scope, the size, and each input's idempotency and concurrency keys — an empty key in input 900
+ * refuses the call, not the rows after 899. Answers in input order; two inputs sharing a key are
+ * one job, the second answered `deduped: true` on the direct path.
+ *
+ * ATOMIC PER CHUNK on the direct path: a failure past a chunk boundary leaves the earlier chunks
+ * enqueued. Inside `withTransaction` the batch is staged, and commits or vanishes whole.
+ */
+export async function enqueueMany<I>(
+  deps: OutboxDeps,
+  tx: Tx | undefined,
+  handle: JobHandle<I>,
+  inputs: readonly I[],
+  options: EnqueueManyOptions = {},
+): Promise<readonly EnqueueResult[]> {
+  refuseAbandoned();
+  if (inputs.length > MAX_ENQUEUE_BATCH) {
+    throw new JobEnqueueBatchTooLargeError({
+      job: handle.name,
+      count: inputs.length,
+      max: MAX_ENQUEUE_BATCH,
+    });
+  }
+  if (inputs.length === 0) return [];
+  const at = nowMs(deps.clock);
+  const trace = options.traceparent ?? ambientTraceparent();
+  // Allocated here on both paths, so the answer needs no read per row.
+  const rows = inputs.map((input) => {
+    handle.concurrencyKeyFor(input);
+    refuseEnqueueTenant(handle.name, handle.tenantFor(input), options.tenantId);
+    const idempotencyKey = handle.idempotencyKeyFor(input);
+    return { input, facts: { id: uuidV7(), runId: uuidV7(), idempotencyKey, at, trace } };
+  });
+
+  if (tx === undefined) {
+    if (deps.mode === 'required' && options.outbox !== false) {
+      throw new OutboxNoTxError({ job: handle.name });
+    }
+    const requests = rows.map((row) => directRequest(handle, row.input, options, row.facts));
+    const results: EnqueueResult[] = [];
+    for (const chunk of chunked(requests)) {
+      // Before EVERY chunk: a direct write is one no transaction guard sees, and a scope abandoned
+      // while chunk 3 was in flight must not start chunks 4 to 10.
+      refuseAbandoned();
+      results.push(...(await deps.driver.enqueueMany(chunk)));
+    }
+    signalEnqueued(options.queue ?? handle.queue);
+    return results;
+  }
+
+  const records = rows.map((row) => stagedRecord(handle, row.input, options, row.facts));
+  for (const chunk of chunked(records)) await stageAll(deps.store, tx, chunk);
+  signalStaged();
+  // As `enqueue` answers a staged row: the ids are the jobs' to be, and only the publish knows
+  // whether a LIVE job already held the key.
+  return records.map((record) => ({ id: record.id, runId: record.runId, deduped: false }));
+}

package/src/job-types.ts

@@ -0,0 +1,229 @@
+// The `job` primitive's declared shapes: what a definition holds, what a run receives, and the
+// handle `job()` returns. Types only, so the factory in `job.ts` stays one responsibility.
+
+import type { Ctx } from '@ultimat3/core';
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import type { DurationInput } from './clock';
+import type { JobConcurrency, WhenBusy } from './concurrency';
+import type { JobDescriptor } from './describe';
+import type { EnqueueResult } from './driver';
+import type { EnqueueManyOptions } from './job-enqueue-many';
+import type { EnqueueOptions } from './outbox';
+import type { ProgressFn } from './progress';
+import type { RetryPolicy } from './retry';
+import type { JobSettled } from './settled';
+import type { StepApi } from './steps';
+import type { JobTenant } from './tenant';
+
+export interface JobRunArgs<I> {
+  readonly input: I;
+  readonly step: StepApi;
+  /**
+   * `ctx.signal` aborts when this attempt's `timeout` passes — the same seam an action reads, so
+   * `throwIfAborted(ctx)` and `fetch(url, { signal: ctx.signal })` work here unchanged. Past it
+   * the run belongs to whoever claims it next and `step.run` refuses to write, so a loop that
+   * never checks it is a body running beside its own retry.
+   */
+  readonly ctx: Ctx;
+  /** 1-based. Assume at-least-once: never branch on `attempt === 1` for correctness. */
+  readonly attempt: number;
+  /**
+   * True when a failure of THIS attempt is not retried for want of attempts — the same boolean
+   * the runner dead-letters on (`isFinalAttempt` in `retry.ts`), so a body never re-derives it
+   * from `retry.attempts`. The place to release what a retry would have reused. REQUIRED: a test
+   * that drives a body by hand states it, `isFinalAttempt(handle.retry, attempt)` being the
+   * runner's own answer.
+   *
+   * Not "this body runs once more at most": a `terminal` error stops an earlier attempt, and an
+   * attempt handed back UNCOUNTED — a `step.sleep`, a drain — is presented again under the same
+   * number, so the last attempt of a job that sleeps sees `true` on every resume.
+   */
+  readonly finalAttempt: boolean;
+  /**
+   * `progress(done, total, note?)` — how far this run has got, shown by `x jobs show` and the
+   * dashboard. Call it as often as the loop turns: it is written to the row at most once a
+   * second, and the last call before the run settles is always written.
+   */
+  readonly progress: ProgressFn;
+  readonly jobId: string;
+  readonly runId: string;
+}
+
+export interface JobDefinition<I, R = unknown> {
+  /**
+   * Omit it: `defineApi({ jobs })` assigns the export name. Set it only to pin a queue key the
+   * export name must not decide — a framework job like `mail.send`, or a name rows already carry.
+   */
+  readonly name?: string;
+  readonly input: StandardSchemaV1<unknown, I>;
+  /** REQUIRED. See the file header — this is the whole point. */
+  readonly idempotencyKey: (input: I) => string;
+  /**
+   * REQUIRED, and the org this job's body runs under. `tenant: (input) => input.orgId` derives it
+   * from the payload; `tenant: 'none'` says this job belongs to no tenant, and then every
+   * tenant-scoped read inside it fails closed with `X_TENANCY_ACTOR_ORG_REQUIRED`.
+   *
+   * There is no default, because both candidates are wrong. Until this field existed the worker ran
+   * a body with no ambient context at all, so `@ultimat3/entity`'s tenant guard — which derives
+   * from `tryUseContext()` and not from the ctx it is handed — read no actor, added no predicate
+   * and accepted a caller-named `orgId` unchecked: the same write refused over HTTP as
+   * `X_TENANCY_ACTOR_MISMATCH` was ACCEPTED through the job surface. A boot-supplied service actor
+   * would close that with ONE identity for every job, which is a cross-tenant read waiting for the
+   * first job that takes an org id in its input. So the job declares it, per job, from its own
+   * payload — the value an author already had to pass anyway.
+   */
+  readonly tenant: JobTenant<I>;
+  readonly retry: RetryPolicy;
+  readonly queue?: string;
+  /**
+   * Max in-flight runs of THIS job across the fleet. Omit for the queue-wide cap.
+   *
+   * A number is one cap for the job. `{ key, limit, whenBusy }` is that cap PER KEY — "one run
+   * per account" — and says what a claim over it does: `'wait'` (default) leaves the run queued,
+   * `'fail'` settles it `failed` with `X_JOB_KEY_BUSY` without running its body (`concurrency.ts`).
+   *
+   * Enforced by `JobDriver.leases` — a row every replica can see — and NOT by `limits.ts`, which
+   * counts one process's heap. A driver with no lease store cannot hold this cap, so
+   * `jobWorker().start()` refuses to boot rather than let it pass silently
+   * (`X_JOB_CONCURRENCY_UNENFORCEABLE`): this field was declared, documented and in the manifest
+   * while nothing read it, which is exactly what axiom 3 exists to prevent.
+   */
+  readonly concurrency?: JobConcurrency<I>;
+  readonly timeout?: DurationInput;
+  /**
+   * Ceiling for ONE `step.run`, where `timeout` is the ceiling for the whole attempt. Folded into
+   * the signal the step body is handed, so a body reads one signal and sees whichever deadline
+   * lands first — and it ABORTS before it rejects, because the attempt that replaces this one is
+   * claimable the moment the nack lands.
+   *
+   * Declared here and nowhere else: the runner has implemented this ceiling since 1.0 and no
+   * declaration could ask for it, which is a documented guarantee that does nothing.
+   */
+  readonly stepTimeout?: DurationInput;
+  /**
+   * Ceiling for `onSettled`, every try included. Default `ON_SETTLED_TIMEOUT_MS`. It aborts the
+   * hook's `ctx.signal` with `X_JOB_TIMEOUT`, then the worker moves on, so a hook on a dead host
+   * cannot hold a slot for good.
+   */
+  readonly onSettledTimeout?: DurationInput;
+  /**
+   * How long a `step.waitForEvent` parks between polls. Default 30s — 15 min when the event bus
+   * wakes its waiters (`postgresEventBus` over the queue's database, as boot installs it), where
+   * the poll is only the backstop. Lower it for a wait a user is watching on a bus that does not
+   * wake; the step suspends for exactly this long each time, never a busy loop.
+   */
+  readonly eventPoll?: DurationInput;
+  /**
+   * This job undoes its steps when it fails FOR GOOD: `step.run(name, fn, { compensate })` is
+   * legal, and the row is enqueued with `UNWIND_MARGIN` attempts past `retry.attempts`, so a
+   * crash on the final attempt still unwinds. Default false: no path of the job changes.
+   */
+  readonly compensates?: boolean;
+  run(args: JobRunArgs<I>): Promise<R>;
+  /**
+   * How the run ENDED, once: `completed` with what `run` returned, `dead-lettered`, `dropped`
+   * (`retry.deadLetter: false`) or `refused` (`whenBusy: 'fail'` over a busy key — the body never
+   * ran). Called after the row is settled, by the worker whose settle landed, under the job's
+   * tenant. Not called for a retry, a suspension, a drain, or a row an operator cancelled.
+   *
+   * AT MOST ONCE across a crash: a worker killed between the settle and this call never makes it.
+   * In one process it gets `ON_SETTLED_ATTEMPTS` tries; if all throw, the failure is logged and
+   * reported (`X_JOB_ON_SETTLED_FAILED`) and the row is untouched. What must be recorded for
+   * certain belongs in the body, inside a `step.run`.
+   */
+  onSettled?(settled: JobSettled<I, R>): Promise<void>;
+}
+
+/**
+ * Whoever the enqueue is for. Structural, exactly like `tenantKeyFrom` in `limits.ts`: the
+ * queue needs the actor's org and nothing else, so this package never imports the auth types.
+ */
+export interface JobActor {
+  readonly orgId?: string | undefined;
+  /**
+   * The actor's id, recorded as `enqueuedBy` — ATTRIBUTION, never authority.
+   *
+   * The framework picks one answer to "whose permissions does a job run with" (axiom 1) and it is
+   * this: a job body runs with SYSTEM authority and this is an audit column. Impersonating the
+   * enqueuer at claim time is the defensible alternative and is rejected for one reason — a job
+   * that sleeps three days, or dead-letters and is retried next quarter, would then act as
+   * somebody whose role, org membership or employment has changed since. `02-primitives.md`
+   * already frames a job as server-authoritative work. A job that must act FOR a user takes that
+   * user's id in its input and re-authorises it in the body, where the check is visible in review.
+   */
+  readonly id?: string | undefined;
+}
+
+/**
+ * Methods (not function-typed properties) throughout, so `JobHandle<Specific>` is assignable
+ * to `AnyJobHandle` and heterogeneous handles can share a registry and a task's enqueue list.
+ */
+export interface JobHandle<I = unknown> {
+  readonly kind: 'job';
+  readonly name: string;
+  readonly queue: string;
+  readonly retry: RetryPolicy;
+  /** The declared cap, resolved: the plain number, or a keyed declaration's `limit`. */
+  readonly concurrency: number | undefined;
+  /**
+   * What a claim over the cap does — set exactly when the cap is KEYED, `undefined` for a plain
+   * number and for no cap. A resolved field and never the declaration itself: `key` is a function
+   * of `I`, and a function-typed property would break `AnyJobHandle` — see `tenantFor`.
+   */
+  readonly whenBusy: WhenBusy | undefined;
+  readonly timeoutMs: number | undefined;
+  /** The declared per-step ceiling in ms; `executeJob` hands it to the step runner. */
+  readonly stepTimeoutMs: number | undefined;
+  readonly onSettledTimeoutMs: number;
+  /** The declared event-poll interval in ms; `undefined` leaves the runner's default. */
+  readonly eventPollMs: number | undefined;
+  /** `compensates: true` was declared. `executeJob` unwinds a run that failed for good. */
+  readonly compensates: boolean;
+  readonly input: StandardSchemaV1<unknown, I>;
+  parse(raw: unknown): I;
+  idempotencyKeyFor(input: I): string;
+  /**
+   * The org THIS payload's run acts under — `undefined` for `tenant: 'none'`. A method and never a
+   * `readonly tenant: JobTenant<I>` field: a function-typed property is contravariant in its
+   * parameter, so `JobHandle<OrgInput>` would stop being assignable to `AnyJobHandle` and the
+   * registry, the worker and a task's enqueue list could no longer hold heterogeneous handles.
+   */
+  tenantFor(input: I): string | undefined;
+  /**
+   * The concurrency key THIS payload's run counts under — `undefined` when the cap is not keyed.
+   * Refuses an empty key (`X_JOB_DECLARATION_INVALID`): the facade asks it at enqueue so the
+   * refusal reaches the caller, and the worker asks it again at claim.
+   */
+  concurrencyKeyFor(input: I): string | undefined;
+  run(args: JobRunArgs<I>): Promise<unknown>;
+  /** Whether the definition declared `onSettled` — published by `describe()`. */
+  readonly declaresOnSettled: boolean;
+  /**
+   * The declared hook; a no-op when there is none. A method, for `tenantFor`'s variance reason,
+   * and `unknown`-typed on both sides for the same one: the handle carries no result generic, so
+   * the definition's own `onSettled` is where `result` and `input` are typed.
+   */
+  onSettled(settled: JobSettled<unknown>): Promise<void>;
+  /**
+   * Put this job on the queue. Joins the caller's transaction when the app installed the
+   * outbox — same call site in a request handler, a job, a script or a test.
+   */
+  enqueue(input: I, options?: EnqueueOptions): Promise<EnqueueResult>;
+  /**
+   * Many inputs of this job, one round trip per 1,000, answered in input order — never a loop of
+   * `enqueue` (`X_N_PLUS_ONE_WRITE`). At most `MAX_ENQUEUE_BATCH`; see `job-enqueue-many.ts`.
+   */
+  enqueueMany(
+    inputs: readonly I[],
+    options?: EnqueueManyOptions,
+  ): Promise<readonly EnqueueResult[]>;
+  /**
+   * Enqueue on behalf of `actor`: fills `tenantId` from the actor's org so per-tenant limits
+   * apply. It queues rather than running inline because a job's execution surface IS the
+   * queue — an inline run would be a second execution path alongside `executeJob`.
+   */
+  as(actor: JobActor | null, input: I, options?: EnqueueOptions): Promise<EnqueueResult>;
+  describe(): JobDescriptor;
+}
+
+export type AnyJobHandle = JobHandle<unknown>;

package/src/job.ts

@@ -7,219 +7,28 @@
 // orgs) surfaces in production under load, never in a test. Requiring it by construction
 // deletes that class of bug: there is no way to define a job that cannot be deduped.
 
-import type { Ctx } from '@ultimat3/core';
 import { assert, finiteCount } from '@ultimat3/core';
-import type { StandardSchemaV1 } from '@ultimat3/schema';
 import { parse } from '@ultimat3/schema';
-import type { DurationInput } from './clock';
 import { finiteDurationMs } from './clock';
-import type { JobConcurrency, WhenBusy } from './concurrency';
 import { resolveConcurrency } from './concurrency';
 import type { JobDescriptor } from './describe';
 import { describeJob } from './describe';
 import type { EnqueueResult } from './driver';
 import { DEFAULT_QUEUE } from './driver';
 import { IdempotencyRequiredError, JobNameTakenError } from './errors';
-import { JobDeclarationInvalidError } from './errors-declaration';
+import { JobDeclarationInvalidError, missingJobFields } from './errors-declaration';
+import type { EnqueueManyOptions } from './job-enqueue-many';
 import { NO_TENANT, tenantKeyFrom } from './limits';
 import type { EnqueueOptions } from './outbox';
 import { jobsFacade } from './outbox';
-import type { ProgressFn } from './progress';
-import type { RetryPolicy } from './retry';
 import { DEFAULT_RETRY } from './retry';
 import type { JobSettled } from './settled';
-import type { StepApi } from './steps';
-import type { JobTenant } from './tenant';
+import { ON_SETTLED_TIMEOUT_MS } from './settled';
 import { assertJobTenant, jobTenantFor } from './tenant';
 
-export interface JobRunArgs<I> {
-  readonly input: I;
-  readonly step: StepApi;
-  /**
-   * `ctx.signal` aborts when this attempt's `timeout` passes — the same seam an action reads, so
-   * `throwIfAborted(ctx)` and `fetch(url, { signal: ctx.signal })` work here unchanged. Past it
-   * the run belongs to whoever claims it next and `step.run` refuses to write, so a loop that
-   * never checks it is a body running beside its own retry.
-   */
-  readonly ctx: Ctx;
-  /** 1-based. Assume at-least-once: never branch on `attempt === 1` for correctness. */
-  readonly attempt: number;
-  /**
-   * True when a failure of THIS attempt is not retried for want of attempts — the same boolean
-   * the runner dead-letters on (`isFinalAttempt` in `retry.ts`), so a body never re-derives it
-   * from `retry.attempts`. The place to release what a retry would have reused. REQUIRED: a test
-   * that drives a body by hand states it, `isFinalAttempt(handle.retry, attempt)` being the
-   * runner's own answer.
-   *
-   * Not "this body runs once more at most": a `terminal` error stops an earlier attempt, and an
-   * attempt handed back UNCOUNTED — a `step.sleep`, a drain — is presented again under the same
-   * number, so the last attempt of a job that sleeps sees `true` on every resume.
-   */
-  readonly finalAttempt: boolean;
-  /**
-   * `progress(done, total, note?)` — how far this run has got, shown by `x jobs show` and the
-   * dashboard. Call it as often as the loop turns: it is written to the row at most once a
-   * second, and the last call before the run settles is always written.
-   */
-  readonly progress: ProgressFn;
-  readonly jobId: string;
-  readonly runId: string;
-}
-
-export interface JobDefinition<I, R = unknown> {
-  /**
-   * Omit it: `defineApi({ jobs })` assigns the export name. Set it only to pin a queue key the
-   * export name must not decide — a framework job like `mail.send`, or a name rows already carry.
-   */
-  readonly name?: string;
-  readonly input: StandardSchemaV1<unknown, I>;
-  /** REQUIRED. See the file header — this is the whole point. */
-  readonly idempotencyKey: (input: I) => string;
-  /**
-   * REQUIRED, and the org this job's body runs under. `tenant: (input) => input.orgId` derives it
-   * from the payload; `tenant: 'none'` says this job belongs to no tenant, and then every
-   * tenant-scoped read inside it fails closed with `X_TENANCY_ACTOR_ORG_REQUIRED`.
-   *
-   * There is no default, because both candidates are wrong. Until this field existed the worker ran
-   * a body with no ambient context at all, so `@ultimat3/entity`'s tenant guard — which derives
-   * from `tryUseContext()` and not from the ctx it is handed — read no actor, added no predicate
-   * and accepted a caller-named `orgId` unchecked: the same write refused over HTTP as
-   * `X_TENANCY_ACTOR_MISMATCH` was ACCEPTED through the job surface. A boot-supplied service actor
-   * would close that with ONE identity for every job, which is a cross-tenant read waiting for the
-   * first job that takes an org id in its input. So the job declares it, per job, from its own
-   * payload — the value an author already had to pass anyway.
-   */
-  readonly tenant: JobTenant<I>;
-  readonly retry: RetryPolicy;
-  readonly queue?: string;
-  /**
-   * Max in-flight runs of THIS job across the fleet. Omit for the queue-wide cap.
-   *
-   * A number is one cap for the job. `{ key, limit, whenBusy }` is that cap PER KEY — "one run
-   * per account" — and says what a claim over it does: `'wait'` (default) leaves the run queued,
-   * `'fail'` settles it `failed` with `X_JOB_KEY_BUSY` without running its body (`concurrency.ts`).
-   *
-   * Enforced by `JobDriver.leases` — a row every replica can see — and NOT by `limits.ts`, which
-   * counts one process's heap. A driver with no lease store cannot hold this cap, so
-   * `jobWorker().start()` refuses to boot rather than let it pass silently
-   * (`X_JOB_CONCURRENCY_UNENFORCEABLE`): this field was declared, documented and in the manifest
-   * while nothing read it, which is exactly what axiom 3 exists to prevent.
-   */
-  readonly concurrency?: JobConcurrency<I>;
-  readonly timeout?: DurationInput;
-  /**
-   * Ceiling for ONE `step.run`, where `timeout` is the ceiling for the whole attempt. Folded into
-   * the signal the step body is handed, so a body reads one signal and sees whichever deadline
-   * lands first — and it ABORTS before it rejects, because the attempt that replaces this one is
-   * claimable the moment the nack lands.
-   *
-   * Declared here and nowhere else: the runner has implemented this ceiling since 1.0 and no
-   * declaration could ask for it, which is a documented guarantee that does nothing.
-   */
-  readonly stepTimeout?: DurationInput;
-  /**
-   * How long a `step.waitForEvent` parks between polls. Default 30s. Lower it for a wait a user
-   * is watching; the step suspends for exactly this long each time, so it is also the resolution
-   * of the resume, never a busy loop.
-   */
-  readonly eventPoll?: DurationInput;
-  run(args: JobRunArgs<I>): Promise<R>;
-  /**
-   * How the run ENDED, once: `completed` with what `run` returned, `dead-lettered`, `dropped`
-   * (`retry.deadLetter: false`) or `refused` (`whenBusy: 'fail'` over a busy key — the body never
-   * ran). Called after the row is settled, by the worker whose settle landed, under the job's
-   * tenant. Not called for a retry, a suspension, a drain, or a row an operator cancelled.
-   *
-   * AT MOST ONCE across a crash: a worker killed between the settle and this call never makes it.
-   * In one process it gets `ON_SETTLED_ATTEMPTS` tries; if all throw, the failure is logged and
-   * reported (`X_JOB_ON_SETTLED_FAILED`) and the row is untouched. What must be recorded for
-   * certain belongs in the body, inside a `step.run`.
-   */
-  onSettled?(settled: JobSettled<I, R>): Promise<void>;
-}
-
-/**
- * Whoever the enqueue is for. Structural, exactly like `tenantKeyFrom` in `limits.ts`: the
- * queue needs the actor's org and nothing else, so this package never imports the auth types.
- */
-export interface JobActor {
-  readonly orgId?: string | undefined;
-  /**
-   * The actor's id, recorded as `enqueuedBy` — ATTRIBUTION, never authority.
-   *
-   * The framework picks one answer to "whose permissions does a job run with" (axiom 1) and it is
-   * this: a job body runs with SYSTEM authority and this is an audit column. Impersonating the
-   * enqueuer at claim time is the defensible alternative and is rejected for one reason — a job
-   * that sleeps three days, or dead-letters and is retried next quarter, would then act as
-   * somebody whose role, org membership or employment has changed since. `02-primitives.md`
-   * already frames a job as server-authoritative work. A job that must act FOR a user takes that
-   * user's id in its input and re-authorises it in the body, where the check is visible in review.
-   */
-  readonly id?: string | undefined;
-}
-
-/**
- * Methods (not function-typed properties) throughout, so `JobHandle<Specific>` is assignable
- * to `AnyJobHandle` and heterogeneous handles can share a registry and a task's enqueue list.
- */
-export interface JobHandle<I = unknown> {
-  readonly kind: 'job';
-  readonly name: string;
-  readonly queue: string;
-  readonly retry: RetryPolicy;
-  /** The declared cap, resolved: the plain number, or a keyed declaration's `limit`. */
-  readonly concurrency: number | undefined;
-  /**
-   * What a claim over the cap does — set exactly when the cap is KEYED, `undefined` for a plain
-   * number and for no cap. A resolved field and never the declaration itself: `key` is a function
-   * of `I`, and a function-typed property would break `AnyJobHandle` — see `tenantFor`.
-   */
-  readonly whenBusy: WhenBusy | undefined;
-  readonly timeoutMs: number | undefined;
-  /** The declared per-step ceiling in ms; `executeJob` hands it to the step runner. */
-  readonly stepTimeoutMs: number | undefined;
-  /** The declared event-poll interval in ms; `undefined` leaves the runner's 30s default. */
-  readonly eventPollMs: number | undefined;
-  readonly input: StandardSchemaV1<unknown, I>;
-  parse(raw: unknown): I;
-  idempotencyKeyFor(input: I): string;
-  /**
-   * The org THIS payload's run acts under — `undefined` for `tenant: 'none'`. A method and never a
-   * `readonly tenant: JobTenant<I>` field: a function-typed property is contravariant in its
-   * parameter, so `JobHandle<OrgInput>` would stop being assignable to `AnyJobHandle` and the
-   * registry, the worker and a task's enqueue list could no longer hold heterogeneous handles.
-   */
-  tenantFor(input: I): string | undefined;
-  /**
-   * The concurrency key THIS payload's run counts under — `undefined` when the cap is not keyed.
-   * Refuses an empty key (`X_JOB_DECLARATION_INVALID`): the facade asks it at enqueue so the
-   * refusal reaches the caller, and the worker asks it again at claim.
-   */
-  concurrencyKeyFor(input: I): string | undefined;
-  run(args: JobRunArgs<I>): Promise<unknown>;
-  /** Whether the definition declared `onSettled` — published by `describe()`. */
-  readonly declaresOnSettled: boolean;
-  /**
-   * The declared hook; a no-op when there is none. A method, for `tenantFor`'s variance reason,
-   * and `unknown`-typed on both sides for the same one: the handle carries no result generic, so
-   * the definition's own `onSettled` is where `result` and `input` are typed.
-   */
-  onSettled(settled: JobSettled<unknown>): Promise<void>;
-  /**
-   * Put this job on the queue. Joins the caller's transaction when the app installed the
-   * outbox — same call site in a request handler, a job, a script or a test.
-   */
-  enqueue(input: I, options?: EnqueueOptions): Promise<EnqueueResult>;
-  /**
-   * Enqueue on behalf of `actor`: fills `tenantId` from the actor's org so per-tenant limits
-   * apply. It queues rather than running inline because a job's execution surface IS the
-   * queue — an inline run would be a second execution path alongside `executeJob`.
-   */
-  as(actor: JobActor | null, input: I, options?: EnqueueOptions): Promise<EnqueueResult>;
-  describe(): JobDescriptor;
-}
+export type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } from './job-types';
 
-export type AnyJobHandle = JobHandle<unknown>;
+import type { AnyJobHandle, JobActor, JobDefinition, JobHandle, JobRunArgs } from './job-types';
 
 const registry = new Map<string, AnyJobHandle>();
 /** Process-monotonic — `resetJobs()` deliberately leaves it alone. */
@@ -239,18 +48,6 @@ interface JobOrigin {
 
 const origin = new WeakMap<object, JobOrigin>();
 
-/** The required fields a definition lacks, in declaration order. Read off the value, not the type. */
-function missingJobFields(definition: object): string[] {
-  const loose = definition as unknown as Readonly<Record<string, unknown>>;
-  const missing: string[] = [];
-  if (typeof loose['input'] !== 'object' || loose['input'] === null) missing.push('input');
-  if (typeof loose['idempotencyKey'] !== 'function') missing.push('idempotencyKey');
-  if (loose['tenant'] === undefined) missing.push('tenant');
-  if (typeof loose['retry'] !== 'object' || loose['retry'] === null) missing.push('retry');
-  if (typeof loose['run'] !== 'function') missing.push('run');
-  return missing;
-}
-
 export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<I> {
   anonymous += 1;
   const name = definition.name ?? `anonymous-job-${anonymous}`;
@@ -278,6 +75,15 @@ export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<
     definition.stepTimeout === undefined
       ? undefined
       : finiteDurationMs(definition.stepTimeout, `job "${name}"`, 'stepTimeout');
+  const onSettledTimeoutMs =
+    definition.onSettledTimeout === undefined
+      ? ON_SETTLED_TIMEOUT_MS
+      : finiteDurationMs(definition.onSettledTimeout, `job "${name}"`, 'onSettledTimeout');
+  assert(
+    onSettledTimeoutMs > 0,
+    `job "${name}" declares onSettledTimeout ${String(definition.onSettledTimeout)}, which is no ceiling at all`,
+    `set a finite positive onSettledTimeout on job("${name}") — "30s" or 30_000 — or omit the field for the default`,
+  );
   const eventPollMs =
     definition.eventPoll === undefined
       ? undefined
@@ -313,7 +119,9 @@ export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<
         ? undefined
         : finiteDurationMs(definition.timeout, `job "${name}"`, 'timeout'),
     stepTimeoutMs,
+    onSettledTimeoutMs,
     eventPollMs,
+    compensates: definition.compensates === true,
     input: definition.input,
     parse(raw: unknown): I {
       return parse(definition.input, raw) as I;
@@ -346,6 +154,9 @@ export function job<I, R = unknown>(definition: JobDefinition<I, R>): JobHandle<
     enqueue(input: I, options?: EnqueueOptions): Promise<EnqueueResult> {
       return jobsFacade().enqueue(handle, input, options);
     },
+    enqueueMany(inputs: readonly I[], options?: EnqueueManyOptions) {
+      return jobsFacade().enqueueMany(handle, inputs, options);
+    },
     as(actor: JobActor | null, input: I, options: EnqueueOptions = {}): Promise<EnqueueResult> {
       const tenantId = options.tenantId ?? tenantFor(actor);
       // `NO_TENANT` is the limiter's own bucket for an absent tenant, so leaving the column

package/src/metrics.ts

@@ -49,6 +49,16 @@ export const queueDeadJobs: Gauge = gauge('queue_dead_jobs', {
   description: 'Jobs sitting in the dead-letter state, by queue',
 });
 
+/** `SCALING_METRICS['queue-demand']`: what the chart's worker HPA divides by a pod's capacity. */
+export const queueDemand: Gauge = gauge('queue_demand', {
+  unit: '{job}',
+  description: 'Jobs a serving worker must hold, by queue: running on one, or due and claimable',
+});
+
+export function recordQueueDemand(queue: string, demand: number): void {
+  queueDemand.record(demand, { queue });
+}
+
 export function recordQueueOldestReady(queue: string, oldestReadyMs: number): void {
   queueOldestReady.record(oldestReadyMs / 1000, { queue });
 }