@ultimat3/jobs 30.9.0 → 31.0.0

package/src/lock.ts

@@ -0,0 +1,176 @@
+// `withLock(key, fn, { ttl })`: one holder fleet-wide for `fn`, held in the job driver's
+// `LeaseStore` (one slot, `lock:<key>`) — the same rows `job.concurrency` counts, so a lock needs
+// no table, no Redis and no second mechanism. Not a primitive: a helper a job or action body calls.
+// The holder renews at a third of `ttl`; a lost lease aborts `fn`'s signal with `X_LOCK_LOST`.
+
+import type { SleepClock } from '@ultimat3/core';
+import { ambientClock, assert, logger, renderThrowable, uuidV7 } from '@ultimat3/core';
+import type { DurationInput } from './clock';
+import { finiteDurationMs, nowMs } from './clock';
+import { jobDriver } from './driver';
+import type { HeldLease, LeaseStore } from './leases';
+import { LockLostError, LockNotAcquiredError, LockStoreMissingError } from './lock-errors';
+import type { IntervalScheduler, RenewalTimer } from './renewal-timer';
+import { startRenewalTimer } from './renewal-timer';
+
+/** Between two acquire attempts while another holder has the key. Never above `wait`'s remainder. */
+export const DEFAULT_LOCK_POLL_MS = 100;
+
+export interface WithLockOptions {
+  /** REQUIRED. How long the key stays held if this process dies; renewed while `fn` runs. */
+  readonly ttl: DurationInput;
+  /** How long to keep trying before `X_LOCK_NOT_ACQUIRED`. Default: one `ttl`. `0`: one attempt. */
+  readonly wait?: DurationInput;
+  /** Between attempts. Default `DEFAULT_LOCK_POLL_MS`. */
+  readonly poll?: DurationInput;
+  /** Default: the installed job driver's `leases` (`postgresJobDriver` over `x_job_leases`). */
+  readonly store?: LeaseStore;
+  /** Default `ambientClock()`: the wait, the expiry bookkeeping and the token's timestamp. */
+  readonly clock?: SleepClock;
+  /** Gives up the WAIT (rejects with its reason); once held, it aborts `fn`'s signal too. */
+  readonly signal?: AbortSignal;
+  /** The renewal interval's seam — `renewal-timer.ts`'s, so a test renews when it says so. */
+  readonly schedule?: IntervalScheduler;
+}
+
+/** What `fn` holds: the key, the fencing token and the signal a lost lease aborts. */
+export interface HeldLock {
+  readonly key: string;
+  /**
+   * The FENCING token: the store's number for this grant (`HeldLease.fence`), greater than every
+   * token any earlier holder of any key got — the database's counter, never a clock. Write it
+   * beside whatever `fn` changes and refuse a write carrying a smaller one: that is what stops a
+   * holder that paused past its `ttl` (GC, a frozen VM) from writing after the next one took over.
+   */
+  readonly token: number;
+  readonly signal: AbortSignal;
+}
+
+/** The lease key: its own prefix, so no lock ever shares a slot table with a job's cap. */
+export const lockLeaseKey = (key: string): string => `lock:${encodeURIComponent(key)}`;
+
+const storeFor = (key: string, options: WithLockOptions): LeaseStore => {
+  if (options.store !== undefined) return options.store;
+  const driver = jobDriver();
+  if (driver?.leases === undefined) throw new LockStoreMissingError({ key, driver: driver?.name });
+  return driver.leases;
+};
+
+async function acquire(
+  store: LeaseStore,
+  key: string,
+  token: string,
+  bounds: { ttlMs: number; waitMs: number; pollMs: number },
+  clock: SleepClock,
+  signal: AbortSignal | undefined,
+  sent: (atMs: number) => void,
+): Promise<HeldLease> {
+  const leaseKey = lockLeaseKey(key);
+  const deadline = nowMs(clock) + bounds.waitMs;
+  for (;;) {
+    signal?.throwIfAborted();
+    const sentAt = nowMs(clock);
+    const lease = await store.acquire(leaseKey, 1, bounds.ttlMs, token);
+    if (lease !== undefined) {
+      sent(sentAt);
+      return lease;
+    }
+    const left = deadline - nowMs(clock);
+    if (left <= 0) {
+      const holders = await store.holders(leaseKey);
+      throw new LockNotAcquiredError({ key, waitMs: bounds.waitMs, holders });
+    }
+    await clock.sleep(Math.min(bounds.pollMs, left), signal);
+  }
+}
+
+/**
+ * Run `fn` while this process alone holds `key` across the fleet, then release it. Refuses with
+ * `X_LOCK_NOT_ACQUIRED` when `wait` passes first. If the lease is lost mid-run, `fn`'s signal
+ * aborts and the call rejects with `X_LOCK_LOST` even when `fn` resolved: its work was not exclusive.
+ */
+export async function withLock<T>(
+  key: string,
+  fn: (lock: HeldLock) => Promise<T> | T,
+  options: WithLockOptions,
+): Promise<T> {
+  const ttlMs = finiteDurationMs(options.ttl, 'withLock', 'ttl');
+  assert(
+    Number.isSafeInteger(ttlMs) && ttlMs >= 3,
+    `withLock(${JSON.stringify(key)}) declares ttl: ${String(options.ttl)} — renewed at a third of it, so at least 3ms`,
+    `withLock(${JSON.stringify(key)}, fn, { ttl: '30s' })   # in the calling file`,
+  );
+  const waitMs = Math.max(0, finiteDurationMs(options.wait ?? ttlMs, 'withLock', 'wait'));
+  const pollMs = Math.max(
+    1,
+    finiteDurationMs(options.poll ?? DEFAULT_LOCK_POLL_MS, 'withLock', 'poll'),
+  );
+  const clock = options.clock ?? ambientClock();
+  const store = storeFor(key, options);
+  const holder = uuidV7(clock);
+  // Stamped BEFORE the acquire that won: the store dates the expiry at its statement.
+  let renewedAt = nowMs(clock);
+  const lease = await acquire(
+    store,
+    key,
+    holder,
+    { ttlMs, waitMs, pollMs },
+    clock,
+    options.signal,
+    (sentAt) => {
+      renewedAt = sentAt;
+    },
+  );
+
+  const controller = new AbortController();
+  const onOuterAbort = (): void => controller.abort(options.signal?.reason);
+  let lost: LockLostError | undefined;
+  let timer: RenewalTimer | undefined;
+  // Everything after the acquire is inside the `try`: a throw anywhere still hands the lease back.
+  try {
+    const token = lease.fence;
+    if (token === undefined)
+      throw new LockStoreMissingError({ key, driver: undefined, fence: true });
+    options.signal?.addEventListener('abort', onOuterAbort, { once: true });
+    // An abort that landed while the acquire that SUCCEEDED was in flight fired no listener.
+    if (options.signal?.aborted === true) controller.abort(options.signal.reason);
+    const lapsed = (): boolean => nowMs(clock) - renewedAt >= ttlMs;
+    const lose = (): void => {
+      lost ??= new LockLostError({ key, token, ttlMs });
+      if (!controller.signal.aborted) controller.abort(lost);
+    };
+    let renewing = false;
+    // `worker-fleet-slots.ts`'s renewal: a renew hung on a dead socket never settles, so the
+    // lapse is decided BEFORE the call, one renewal is in flight at a time, and a reply that
+    // lands after the expiry it would have extended is still a loss.
+    const renew = async (): Promise<void> => {
+      if (lost !== undefined || timer?.stopped() === true) return;
+      if (lapsed()) return lose();
+      if (renewing) return;
+      renewing = true;
+      const sentAt = nowMs(clock);
+      try {
+        const held = await store.renew(lease, ttlMs);
+        if (timer?.stopped() === true) return;
+        if (held === false || lapsed()) return lose();
+        renewedAt = sentAt;
+      } catch (error: unknown) {
+        logger.warn('jobs.lock.renew_failed', { key, error: renderThrowable(error) });
+        if (timer?.stopped() !== true && lapsed()) lose();
+      } finally {
+        renewing = false;
+      }
+    };
+    timer = startRenewalTimer(Math.max(1, Math.floor(ttlMs / 3)), renew, options.schedule);
+    const result = await fn({ key, token, signal: controller.signal });
+    if (lost !== undefined) throw lost;
+    return result;
+  } finally {
+    timer?.stop();
+    options.signal?.removeEventListener('abort', onOuterAbort);
+    // Released only by this holder (the store checks `holder`); a failure leaves it to the TTL.
+    await store.release(lease).catch((error: unknown) => {
+      logger.warn('jobs.lock.release_failed', { key, error: renderThrowable(error) });
+    });
+  }
+}

package/src/outbox-relay.ts

@@ -188,6 +188,10 @@ export function outboxRelay(options: RelayOptions): OutboxRelay {
     // The loop's next wait, decided by the pass itself so `tick()` and the timer agree: the floor
     // while rows are moving, doubling to the ceiling while there are none.
     delayMs = backoff.next(batch.length > 0);
+    // A FULL batch, all of it published, says there is more behind it: the next pass now, as the
+    // worker's `again`. At the floor a 500-row batch every 200 ms capped the relay at 2,500 rows a
+    // second, a 100k backlog at 40 s. A batch that stopped at a failing row keeps the floor.
+    if (batch.length === batchSize && published === batchSize) delayMs = 0;
     return published;
   };
 

package/src/outbox.ts

@@ -356,7 +356,7 @@ let ambient: JobsFacade | undefined;
 let fallback: JobsFacade | undefined;
 
 /**
- * Installed once at boot, next to `setJobDriver`, with the app's outbox store and its
+ * Installed once at boot, next to the job driver, with the app's outbox store and its
  * transaction accessor. `handle.enqueue()` then joins the caller's transaction wherever it is
  * called from — which is the only reason the outbox protects anything.
  */
@@ -373,7 +373,7 @@ export function jobsFacade(): JobsFacade {
   if (ambient !== undefined) return ambient;
   fallback ??= outboxJobsFacade(
     {
-      // A getter, not a snapshot: `setJobDriver()` after the first enqueue is honoured, and a
+      // A getter, not a snapshot: a driver installed after the first enqueue is honoured, and a
       // missing driver is an error at the call rather than at import time.
       get driver(): JobDriver {
         const installed = jobDriver();
@@ -381,7 +381,7 @@ export function jobsFacade(): JobsFacade {
           throw new DriverUnavailableError({
             driver: 'none',
             cause: 'no queue driver is installed in this process',
-            fix: 'call setJobDriver(memoryJobDriver()) before enqueuing, or setJobDriver(postgresJobDriver()) for a real queue',
+            fix: 'memoryJobDriver()   # jobs: in apps/<app>/runtime.ts — or enqueue from a process the boot started (x dev, or the serve entry), which builds postgresJobDriver() over DATABASE_URL',
           });
         }
         return installed;

package/src/partitions.ts

@@ -0,0 +1,119 @@
+// `partitions()`: the maintenance every `entity({ partitionBy })` needs — partitions created ahead,
+// expired ones dropped — as a FACTORY over `job()` plus a `.schedule` `task()` (default daily), the
+// shape `consumer()`'s `.supervisor` has. Never a ninth primitive. The work is `@ultimat3/db`'s
+// `maintainPartitions` (bounded lock waits, `detach … concurrently`), judged at THIS job's clock.
+
+import type { Clock } from '@ultimat3/core';
+import { assert, liveClock } from '@ultimat3/core';
+import { db, maintainPartitions } from '@ultimat3/db';
+import type { EntityDescription } from '@ultimat3/entity';
+import { registeredEntities } from '@ultimat3/entity';
+import { t } from '@ultimat3/schema';
+import type { DurationInput } from './clock';
+import { nowMs } from './clock';
+import type { JobHandle } from './job';
+import { job } from './job';
+import type { RetryPolicy } from './retry';
+import { DEFAULT_RETRY } from './retry';
+import type { TaskHandle } from './task';
+import { task } from './task';
+
+/** 03:17 UTC daily: off the hour every other cron in a fleet picks, and well before month-end. */
+export const DEFAULT_PARTITIONS_CRON = '17 3 * * *';
+
+export const DEFAULT_PARTITIONS_NAME = 'partitions';
+
+export interface PartitionsDefinition {
+  /** The job's durable name; the task is `<name>.schedule`. Default `DEFAULT_PARTITIONS_NAME`. */
+  readonly name?: string;
+  /** Default `DEFAULT_PARTITIONS_CRON`. Daily is enough for any interval but `day` with `ahead: 0`. */
+  readonly cron?: string;
+  /** Default `'UTC'`: partition bounds are UTC, so the schedule is too. */
+  readonly tz?: string;
+  /** The instant partitions are judged at — the framework clock, so a test can move it. */
+  readonly clock?: Clock;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  readonly timeout?: DurationInput;
+}
+
+/** One partitioned table's maintenance: the partitions this run created and dropped. */
+export interface PartitionsTableReport {
+  readonly table: string;
+  readonly created: readonly string[];
+  readonly dropped: readonly string[];
+}
+
+export interface PartitionsReport {
+  readonly tables: readonly PartitionsTableReport[];
+}
+
+/**
+ * The job's input. The schedule enqueues `{}`. Before a `backfill()` copies rows older than the
+ * table's first partition, enqueue it once with `from`: the oldest row's instant (ISO). That run
+ * creates every partition back to it and sweeps nothing, so the copy routes instead of `23514`.
+ */
+export interface PartitionsInput {
+  readonly from?: string | undefined;
+}
+
+/** The maintenance job, with the `task()` that enqueues it — as `consumer()` carries `.supervisor`. */
+export interface PartitionsJob extends JobHandle<PartitionsInput> {
+  readonly schedule: TaskHandle;
+}
+
+/** Every registered entity declaring `partitionBy`, described, in registry name order. */
+export const partitionedEntities = (): readonly EntityDescription[] =>
+  registeredEntities()
+    .map((entry) => entry.describe())
+    .filter((described) => described.partition != null);
+
+/** `from` as an instant, or refused where it was enqueued: a bad bound would create nothing. */
+function fromOf(name: string, from: string | undefined): Date | undefined {
+  if (from === undefined) return undefined;
+  const at = new Date(from);
+  assert(
+    !Number.isNaN(at.getTime()),
+    `${name}: from "${from}" is no ISO instant`,
+    `enqueue ${name} with { from: '2025-01-01T00:00:00.000Z' } — the oldest row's instant`,
+  );
+  return at;
+}
+
+export function partitions(definition: PartitionsDefinition = {}): PartitionsJob {
+  const name = definition.name ?? DEFAULT_PARTITIONS_NAME;
+  const clock = definition.clock ?? liveClock;
+  const handle = job<PartitionsInput>({
+    name,
+    input: t.object({ from: t.string.optional() }),
+    // Partitions are tables, not any tenant's rows.
+    tenant: 'none',
+    // One live pass: a second tick while one runs is the same pass, and the SQL is idempotent anyway.
+    // A `from` run is a different pass: it must not fold into the schedule's.
+    idempotencyKey: (input) => (input.from === undefined ? name : `${name}:from:${input.from}`),
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
+    async run({ input, step }): Promise<PartitionsReport> {
+      const from = fromOf(name, input.from);
+      const tables: PartitionsTableReport[] = [];
+      // Read per run, never at declaration: an entity module may load after this one.
+      for (const described of partitionedEntities()) {
+        // One step per table: a retry resumes at the table that failed, never re-dropping.
+        const report = await step.run(`partitions:${described.table}`, async () => {
+          const run = await maintainPartitions(db(), described, new Date(nowMs(clock)), { from });
+          return { table: described.table, created: run.created, dropped: run.dropped };
+        });
+        tables.push(report);
+      }
+      return { tables };
+    },
+  });
+  const schedule = task({
+    name: `${name}.schedule`,
+    cron: definition.cron ?? DEFAULT_PARTITIONS_CRON,
+    tz: definition.tz ?? 'UTC',
+    enqueue: () => [[handle, {}]],
+  });
+  return Object.assign(handle, { schedule });
+}

package/src/reindex-sweep.ts

@@ -0,0 +1,94 @@
+// `reindex()`'s second phase: the documents an index holds whose row the source no longer reads —
+// deleted while the index was down, deleted by a store that names no rows, or moved to another
+// tenant. One `list` page at a time, each looked up by its key, the absent ones removed.
+
+import type { Ctx } from '@ultimat3/core';
+import type { EntityCore, ReadBuilder, SearchIndex, SearchListEntry } from '@ultimat3/entity';
+// The barrel's leaves, never `@ultimat3/entity/adapter`: that drags the driver kit into ROLE=migrate.
+import { kindOf, MAX_PAGE_SIZE, rowKey } from '@ultimat3/entity';
+
+/** A row's place in the index: its tenant and its key, as one comparable string. */
+const placeOf = (tenant: string | null, id: string): string => JSON.stringify([tenant, id]);
+
+/** The key values an index id spells (`rowKey`): one, or a composite key's JSON array of them. */
+const keyValues = (entity: EntityCore<unknown>, id: string): readonly string[] => {
+  if (entity.$primaryKey.length === 1) return [id];
+  try {
+    const parsed: unknown = JSON.parse(id);
+    return Array.isArray(parsed) && parsed.every((each) => typeof each === 'string') ? parsed : [];
+  } catch {
+    return [];
+  }
+};
+
+/**
+ * A key spelling as the column compares it: `rowKey` writes an `integer` by value and an instant
+ * in ISO, both as text — handed back as the number and the `Date` the column holds.
+ */
+const typed = (entity: EntityCore<unknown>, property: string, spelled: string): unknown => {
+  const kind = kindOf(entity, property);
+  if (kind === 'integer') return Number(spelled);
+  if (kind === 'timestamptz') return new Date(spelled);
+  return spelled;
+};
+
+/**
+ * The places among `entries` the source still reads. One read per page: each key column `in` the
+ * values the page names — exact for a single key, a superset for a composite one, matched here.
+ */
+const present = async <Row>(
+  entity: EntityCore<Row>,
+  source: ReadBuilder<Row>,
+  entries: readonly SearchListEntry[],
+): Promise<ReadonlySet<string>> => {
+  const core = entity as EntityCore<unknown>;
+  let chain = source;
+  for (const [position, property] of core.$primaryKey.entries()) {
+    const values = new Set(entries.flatMap((entry) => keyValues(core, entry.id)[position] ?? []));
+    if (values.size === 0) return new Set();
+    chain = chain.andWhere(
+      property as keyof Row & string,
+      'in',
+      [...values].map((value) => typed(core, property, value)),
+    );
+  }
+  const tenantColumn = core.$tenantColumn;
+  const found = new Set<string>();
+  for await (const batch of chain.inBatches(MAX_PAGE_SIZE)) {
+    for (const row of batch) {
+      const tenant =
+        tenantColumn === null ? null : (row as Readonly<Record<string, unknown>>)[tenantColumn];
+      found.add(placeOf(typeof tenant === 'string' ? tenant : null, rowKey(core, row)));
+    }
+  }
+  return found;
+};
+
+/**
+ * Remove every entry of one `list` page whose row the source no longer reads. The tombstone sits
+ * ONE millisecond above the version listed, never at the sweep's own instant: a write that put the
+ * id back after the list is newer than that, and keeps its document.
+ */
+export const sweepListed = async <Row>(args: {
+  readonly entity: EntityCore<Row>;
+  readonly source: (args: { readonly ctx: Ctx }) => ReadBuilder<Row>;
+  readonly ctx: Ctx;
+  readonly index: SearchIndex;
+  readonly entries: readonly SearchListEntry[];
+}): Promise<number> => {
+  const { entity, entries, index } = args;
+  if (entries.length === 0) return 0;
+  const still = await present(entity, args.source({ ctx: args.ctx }), entries);
+  const gone = new Map<string, { tenant: string | null; version: number; ids: string[] }>();
+  for (const entry of entries) {
+    if (still.has(placeOf(entry.tenant, entry.id))) continue;
+    const at = JSON.stringify([entry.tenant, entry.version]);
+    const group = gone.get(at) ?? { tenant: entry.tenant, version: entry.version, ids: [] };
+    group.ids.push(entry.id);
+    gone.set(at, group);
+  }
+  for (const { tenant, version, ids } of gone.values()) {
+    await index.remove({ entity: entity.$name, tenant }, ids, version + 1);
+  }
+  return [...gone.values()].reduce((sum, group) => sum + group.ids.length, 0);
+};

package/src/reindex.ts

@@ -0,0 +1,147 @@
+// `reindex()`: rebuild a `SearchIndex` from the table, as a `backfill()` — so a FACTORY over a
+// factory over `job()`, never a ninth primitive. Run after swapping the built-in Postgres index for
+// Meilisearch or Typesense, or after the index lost data or missed deletes: every row through
+// entity's ONE document builder (`searchDocumentOf`), then a sweep of what the table lost.
+
+import type { Ctx, Environment } from '@ultimat3/core';
+import { assert, finiteCount } from '@ultimat3/core';
+import type { EntityCore, ReadBuilder, SearchIndex, SearchListEntry } from '@ultimat3/entity';
+import {
+  MAX_PAGE_SIZE,
+  searchDocumentOf,
+  searchIndex,
+  searchVersionColumn,
+} from '@ultimat3/entity';
+import type { BackfillInput, BackfillPage } from './backfill';
+import { backfill, DEFAULT_BACKFILL_BATCH } from './backfill';
+import type { DurationInput } from './clock';
+import type { JobHandle } from './job';
+import { sweepListed } from './reindex-sweep';
+import type { RetryPolicy } from './retry';
+import type { JobTenant } from './tenant';
+import { NO_JOB_TENANT } from './tenant';
+
+export interface ReindexDefinition<Row> {
+  /** REQUIRED: the backfill's durable key — its ledger row and queue key carry it. */
+  readonly name: string;
+  /** The entity whose `.searchable()` columns are indexed. */
+  readonly entity: EntityCore<Row>;
+  /**
+   * The rows to index — the WHOLE table: `() => db.posts`. Read per page. The sweep removes every
+   * document whose row this does not read, so a narrowed source empties the index of the rest.
+   */
+  source(args: { readonly ctx: Ctx }): ReadBuilder<Row>;
+  /** As on `backfill()`: `'none'` for a sweep of every tenant, each document keeping its own. */
+  readonly tenant: JobTenant<BackfillInput>;
+  /** Default: the installed index (`runtime.ts`'s `searchIndex`, else Postgres' built-in). */
+  readonly index?: () => SearchIndex;
+  readonly batch?: number;
+  readonly rate?: number;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  readonly timeout?: DurationInput;
+  readonly environments?: readonly Environment[];
+}
+
+/** True when at least one column of `entity` is declared `.searchable()`. */
+const isSearchable = (entity: EntityCore<unknown>): boolean =>
+  Object.values(entity.$columns).some((column) => column.$meta.searchable !== undefined);
+
+/** One item of the pass: a source row to index, or an index entry the sweep checks. */
+type ReindexItem<Row> = { readonly row: Row } | { readonly entry: SearchListEntry };
+
+const ROWS = 'rows:';
+const SWEEP = 'sweep:';
+
+/** The phase a checkpointed cursor names, and the position inside it (`null`: from the top). */
+const phaseOf = (
+  cursor: string | null,
+): { readonly sweep: boolean; readonly at: string | null } => {
+  if (cursor?.startsWith(SWEEP) === true) {
+    const at = cursor.slice(SWEEP.length);
+    return { sweep: true, at: at === '' ? null : at };
+  }
+  const at = cursor?.slice(ROWS.length) ?? '';
+  return { sweep: false, at: at === '' ? null : at };
+};
+
+/** One batch of the source at `at`, and where the next starts — the sweep, once it is exhausted. */
+async function rowsPage<Row>(
+  source: ReindexDefinition<Row>['source'],
+  ctx: Ctx,
+  at: string | null,
+  size: number,
+): Promise<BackfillPage<ReindexItem<Row>>> {
+  const batches = source({ ctx }).after(at).inBatches(size);
+  try {
+    const first = await batches[Symbol.asyncIterator]().next();
+    const rows = first.done === true ? [] : first.value;
+    const next = first.done === true ? null : batches.cursor;
+    return { rows: rows.map((row) => ({ row })), next: next === null ? SWEEP : `${ROWS}${next}` };
+  } finally {
+    await batches.close();
+  }
+}
+
+/**
+ * Two phases, one `backfill()`, one durable step per page. First every source row's document is
+ * written — at least once per batch: each carries its row's `onUpdateNow()` version and the index
+ * keeps only newer, so a replayed batch is a no-op and a batch that read a row before a write (or
+ * before its delete's tombstone) cannot put the old text back. Then the SWEEP: every document the
+ * index `list`s whose row `source` no longer reads is removed (`reindex-sweep.ts`) — so `source` is
+ * the whole table, and after a pass the index holds what it reads and nothing else.
+ */
+export function reindex<Row>(definition: ReindexDefinition<Row>): JobHandle<BackfillInput> {
+  const { entity } = definition;
+  assert(
+    isSearchable(entity as EntityCore<unknown>),
+    `reindex "${definition.name}" names entity ${entity.$name}, which has no .searchable() column — there is nothing to index`,
+    `title: text().searchable()   # in entity('${entity.$name}'): mark the columns a search reads, or drop reindex "${definition.name}"`,
+  );
+  // At declaration, not at the first batch: a document without a version is one an index cannot
+  // order against a write, so the entity must declare its `onUpdateNow()` column first.
+  searchVersionColumn(entity as EntityCore<unknown>);
+  const index = definition.index ?? searchIndex;
+  // Screened here, before `backfill()` sees it: `size` also bounds the sweep's `list` page.
+  const size = finiteCount(
+    `reindex "${definition.name}"`,
+    'batch',
+    definition.batch ?? DEFAULT_BACKFILL_BATCH,
+    1,
+  );
+  const everyTenant = definition.tenant === NO_JOB_TENANT;
+  return backfill<ReindexItem<Row>>({
+    name: definition.name,
+    tenant: definition.tenant,
+    pages: async ({ ctx, cursor }) => {
+      const phase = phaseOf(cursor);
+      if (!phase.sweep) return rowsPage(definition.source, ctx, phase.at, size);
+      // A declared tenant sweeps its own documents; `'none'` every tenant's, under the pass's
+      // cross-tenant scope (`backfill-scope.ts`).
+      const page = await index().list({
+        entity: entity.$name,
+        ...(everyTenant ? {} : { tenant: ctx.actor?.orgId ?? null }),
+        after: phase.at,
+        limit: Math.min(size, MAX_PAGE_SIZE),
+      });
+      return {
+        rows: page.entries.map((entry) => ({ entry })),
+        next: page.next === null ? null : `${SWEEP}${page.next}`,
+      };
+    },
+    handle: async ({ rows, ctx }) => {
+      const documents = rows.flatMap((item) =>
+        'row' in item ? [searchDocumentOf(entity, item.row)] : [],
+      );
+      if (documents.length > 0) await index().index(documents);
+      const entries = rows.flatMap((item) => ('entry' in item ? [item.entry] : []));
+      await sweepListed({ entity, source: definition.source, ctx, index: index(), entries });
+    },
+    ...(definition.batch === undefined ? {} : { batch: definition.batch }),
+    ...(definition.rate === undefined ? {} : { rate: definition.rate }),
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    ...(definition.retry === undefined ? {} : { retry: definition.retry }),
+    ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
+    ...(definition.environments === undefined ? {} : { environments: definition.environments }),
+  });
+}

package/src/retry-classification.ts

@@ -72,9 +72,10 @@ export function nextRetryForError(
   // on top is core's `jitterStatedDelay` — the one rule for a named delay — because an HTTP-date
   // hands every delivery to one receiver the SAME instant, and waking them together is the herd.
   const cap = finiteDurationMs(policy.maxDelay ?? DEFAULT_RETRY.maxDelay, 'retry', 'maxDelay');
-  // `jitter: false` is the policy's own decision and is honoured: the bare floor.
+  // `jitter: 'none'` is the policy's own decision and is honoured: the bare floor. `'equal'` and
+  // `'full'` both take the one spread — a stated delay is already a floor, so none is dropped.
   const floor = Math.min(stated, cap);
-  const jittered = (policy.jitter ?? DEFAULT_RETRY.jitter) === true;
+  const jittered = (policy.jitter ?? DEFAULT_RETRY.jitter) !== 'none';
   const delayMs = jittered ? jitterStatedDelay(floor, cap, random) : floor;
   return { ...decision, delayMs, stoppedBy: undefined, classification };
 }

package/src/retry-jitter.ts

@@ -0,0 +1,32 @@
+// Single responsibility: refuse a `retry.jitter` that is not core's `JitterMode`, at `job()`. A v30
+// boolean from JS or JSON config otherwise splits in two: `backoffDelay` reads `true` as no jitter,
+// the stated-delay path (`retry-classification.ts`) as jitter — one declaration, two meanings.
+
+import type { JitterMode } from '@ultimat3/core';
+import { renderCauseValue, UltimateError } from '@ultimat3/core';
+
+const MODES: readonly JitterMode[] = ['none', 'equal', 'full'];
+
+/**
+ * The shipped `X_JOB_DECLARATION_INVALID`, not a code of its own: the same statement — this `job()`
+ * call cannot be seated as written — and the same repair, an edit to the declaration.
+ */
+export class JobRetryJitterInvalidError extends UltimateError {
+  constructor(input: { job: string; declared: unknown }) {
+    // What a v30 boolean meant, so the fix names the exact replacement: true → 'equal', false → 'none'.
+    const mapped: JitterMode = input.declared === false ? 'none' : 'equal';
+    super({
+      code: 'X_JOB_DECLARATION_INVALID',
+      cause: `job "${input.job}" declares retry.jitter ${renderCauseValue(input.declared)}, which is not one of 'none' | 'equal' | 'full' — a v30 boolean would mean no jitter to the backoff and jitter to a stated delay`,
+      fix: `jitter: '${mapped}',   # in job('${input.job}')'s retry — v30's boolean maps true → 'equal', false → 'none'; omitted is 'equal'`,
+      meta: { job: input.job, field: 'retry.jitter' },
+    });
+  }
+}
+
+/** `undefined` is the default; anything else must be a `JitterMode`, read as `unknown` on purpose. */
+export function assertRetryJitter(job: string, jitter: unknown): void {
+  if (jitter === undefined) return;
+  if (MODES.some((mode) => mode === jitter)) return;
+  throw new JobRetryJitterInvalidError({ job, declared: jitter });
+}