@ultimat3/jobs 27.10.3 → 27.11.0

package/README.md

@@ -416,6 +416,19 @@ The stored event bus is a target too: `eventsPurgeTarget(bus)` answers
 answers the rows it removed — `x_job_events` only grew while nothing called it. Add it to
 `targets`: `targets: () => [eventsPurgeTarget(eventBus())]`.
 
+Finished jobs are one too: `jobsPurgeTarget({ executor, doneRetentionMs, deadRetentionMs })`
+deletes `done` rows past the first window and `dead`, `failed` and `cancelled` rows past the
+second, each with its `x_job_steps` once no row of the run still stands. A pass runs at most
+`maxBatches` (50) batches of `batchSize` (1,000) rows, `skip locked`; a batch is one statement, plus a
+follow-up when a row it could not see deleted still shared a run (two sweeps that split a run settle
+its steps after both commit). After the batches, one more statement removes up to `batchSize` step
+records whose run has no row left, what a sweep that died before its follow-up stranded. A live row is never deleted, nor one an
+unpublished `x_outbox` row or a `running` `x_backfills` row still names. The windows come from
+`app.config.ts` (`jobs.doneRetentionMs`, 14 days, and `jobs.deadRetentionMs`, 90 days).
+`outboxPurgeTarget({ executor, publishedRetentionMs })` is the same for `x_outbox`: published
+rows only, measured against the job's clock (the relay stamps `published_at` from its own), at
+`jobs.outboxRetentionMs` (7 days). An unpublished row is a job not yet handed over, never deleted.
+
 `@ultimat3/cli`'s boot declares both halves over the three tables it owns, so an app gets the sweep
 without writing any of the above. It needs a `worker` to run it and a `scheduler` to fire it: a
 deployment with neither has no background work at all, and this is one more thing it does not do.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/jobs",
-  "version": "27.10.3",
+  "version": "27.11.0",
   "description": "Durable background work: steps, transactional outbox, cron tasks, one driver interface",
   "license": "MIT",
   "type": "module",
@@ -33,10 +33,10 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "27.10.3",
-    "@ultimat3/db": "27.10.3",
-    "@ultimat3/entity": "27.10.3",
-    "@ultimat3/schema": "27.10.3",
-    "@ultimat3/time": "27.10.3"
+    "@ultimat3/core": "27.11.0",
+    "@ultimat3/db": "27.11.0",
+    "@ultimat3/entity": "27.11.0",
+    "@ultimat3/schema": "27.11.0",
+    "@ultimat3/time": "27.11.0"
   }
 }

package/src/driver-pg-ddl.ts

@@ -123,6 +123,10 @@ create table if not exists x_outbox (
 create index if not exists x_outbox_unpublished_idx
   on x_outbox (staged_at) where published_at is null;
 
+-- What the retention sweep reads, the other half of the table: published rows, oldest first.
+create index if not exists x_outbox_published_idx
+  on x_outbox (published_at) where published_at is not null;
+
 -- The relays claim lease. A claim stamped in the same statement that locks the row is what stops
 -- two relays publishing one row twice: for update skip locked holds its locks only until that
 -- statement ends, which under autocommit is before claim returns. claimed_at is also what gives
@@ -207,6 +211,18 @@ alter table x_jobs add column if not exists claims int not null default 0;
 -- range and never a sort of the whole table.
 create index if not exists x_jobs_created_idx on x_jobs (created_at desc, id desc);
 
+-- What the retention sweep reads. Partial over the four terminal states, so a live row never pays
+-- for it and the oldest finished rows are an index range rather than a scan of the whole queue.
+create index if not exists x_jobs_finished_idx
+  on x_jobs (updated_at)
+  where state in ('done', 'failed', 'dead', 'cancelled');
+
+-- The rows of one run, by run id: the retention sweep asks, per deleted row, whether another row of
+-- the run still stands before it deletes the step records of that run, and its follow-up and its
+-- orphan pass ask again. Without it each question is a scan of the whole queue. Not partial: run_id
+-- is not null, so a "where run_id is not null" predicate would index the same rows for nothing.
+create index if not exists x_jobs_run_idx on x_jobs (run_id);
+
 -- A paused queue or task, one row each. A queue pause is a fact about the QUEUE and not a column
 -- on every one of its rows: the claim reads this table with not exists, and a pause that had to
 -- rewrite a million rows would not take effect within one poll interval.

package/src/index.ts

@@ -235,6 +235,8 @@ export type {
   JobRunArgs,
 } from './job';
 export { describeJobs, getJob, isJobHandle, job, registeredJobs, resetJobs } from './job';
+export type { JobsRetentionOptions } from './jobs-retention';
+export { jobsPurgeTarget } from './jobs-retention';
 export type { HeldLease, LeaseStore, MemoryLeaseStore, MemoryLeaseStoreOptions } from './leases';
 export { jobLeaseKey, memoryLeaseStore } from './leases';
 export type {
@@ -277,6 +279,8 @@ export type { PostgresOutboxStoreOptions } from './outbox-pg';
 export { postgresOutboxStore } from './outbox-pg';
 export type { OutboxRelay, RelayOptions } from './outbox-relay';
 export { outboxRelay } from './outbox-relay';
+export type { OutboxRetentionOptions } from './outbox-retention';
+export { outboxPurgeTarget } from './outbox-retention';
 export type { ProgressFn } from './progress';
 export type {
   PurgeDefinition,
@@ -286,6 +290,7 @@ export type {
   PurgeTarget,
 } from './purge';
 export { DEFAULT_PURGE_CRON, purge } from './purge';
+export type { PurgeBatching } from './purge-batches';
 export type { PgListener, QueueWake, QueueWakeOptions } from './queue-wake';
 export { startQueueWake } from './queue-wake';
 export type { IntervalScheduler } from './renewal-timer';

package/src/jobs-retention.ts

@@ -0,0 +1,159 @@
+// Retention for FINISHED jobs: the `x_jobs` rows whose work is over, and the `x_job_steps` records
+// under their runs, as one `PurgeTarget` the boot's hourly `purge()` sweeps. Until this existed
+// nothing deleted a finished row, so the table grew by every job ever run — and a `mail.send` row's
+// input is a recipient address and a rendered body, personal data kept for the life of the database.
+
+import type { PgExecutor } from '@ultimat3/core';
+import { finiteCount } from '@ultimat3/core';
+import type { PurgeTarget } from './purge';
+import {
+  type PurgeBatch,
+  type PurgeBatching,
+  purgeBatching,
+  purgeInBatches,
+} from './purge-batches';
+
+/**
+ * One batch, counted in the statement (`PgExecutor` answers rows, never a command tag).
+ *
+ * Aged by `updated_at`, which every settle stamps with the server's `now()` — so the cutoff is the
+ * server's clock too, and the job's `nowMs` is not read: measuring a server-written instant against
+ * a caller's clock is the two-clock mismatch `PurgeTarget.purgeExpired` warns about.
+ *
+ * Only the four TERMINAL states, never a live one (`LIVE_STATES`). Two vetoes on top, each a row
+ * that still names the job as unfinished work: an `x_outbox` row not yet published under this id
+ * (deleting the job would let the relay insert it again, a second run) and an `x_backfills` ledger
+ * row still `running` under this run. `skip locked` so a sweep never waits on a worker or an
+ * operator holding a row.
+ *
+ * A run's step records go in the same statement when no row outside this batch shares the run.
+ * When one does (`held`), they stay and the run is answered: that row may be another sweep's,
+ * deleted but not yet committed — invisible as gone to this snapshot, just as this batch's rows are
+ * to that one — so each statement alone would keep the steps forever. `SQL_JOBS_RETENTION_STEPS`
+ * settles a held run after this commits.
+ *
+ * `held` is a LATERAL probe with a `limit`, not an `exists`: the planner flattens an `exists` into
+ * a semi-join and, at a 1,000-row batch, hashes the whole of `x_jobs` for it; the lateral form is one
+ * `x_jobs_run_idx` lookup per deleted run (EXPLAIN, 63,000 rows, 2026-10-17).
+ */
+export const SQL_JOBS_RETENTION_BATCH = `
+with doomed as (
+  select j.id, j.run_id from x_jobs j
+   where ((j.state = 'done'
+           and j.updated_at < statement_timestamp() - ($1::bigint * interval '1 millisecond'))
+       or (j.state in ('failed', 'dead', 'cancelled')
+           and j.updated_at < statement_timestamp() - ($2::bigint * interval '1 millisecond')))
+     and not exists (select 1 from x_outbox o where o.id = j.id and o.published_at is null)
+     and not exists (select 1 from x_backfills b where b.run_id = j.run_id and b.status = 'running')
+   order by j.updated_at
+   limit $3
+     for update of j skip locked
+), gone as (
+  delete from x_jobs where id in (select id from doomed) returning run_id
+), held as (
+  select g.run_id from (select distinct run_id from gone) g
+   cross join lateral (
+     select 1 from x_jobs l
+      where l.run_id = g.run_id and l.id not in (select id from doomed)
+      limit 1
+   ) standing
+), steps as (
+  delete from x_job_steps s
+   where s.run_id in (select run_id from gone)
+     and s.run_id not in (select run_id from held)
+)
+select (select count(*) from gone)::int as removed,
+       (select coalesce(json_agg(run_id), '[]'::json) from held)::text as held
+`.trim();
+
+/**
+ * The held runs' step records, in a statement of its own — so its snapshot is taken AFTER the batch
+ * that answered them committed. Of two sweeps that each deleted one row of a shared run, the one
+ * whose follow-up starts last sees both deletes, and removes the steps. A run with a row still
+ * standing (a retry, a younger terminal row) keeps them, for the sweep that takes that row.
+ * The ids travel as JSON text, never a bound array (issue #384).
+ */
+export const SQL_JOBS_RETENTION_STEPS = `
+delete from x_job_steps s
+ where s.run_id in (select value::uuid from json_array_elements_text($1::json))
+   and not exists (select 1 from x_jobs j where j.run_id = s.run_id)
+`.trim();
+
+/**
+ * Step records whose run has NO `x_jobs` row left — what the follow-up above would have removed had
+ * its process lived to send it. A sweep that commits a batch and dies before the follow-up strands
+ * a held run's steps, and no later batch names that run again: this is the pass that does. Once
+ * per pass, after the batches, at most `$2` records.
+ *
+ * Only records started before the shorter window (`$1`): a job is deleted no sooner than that after
+ * its last settle, and every step of it started before that settle, so a true orphan always
+ * qualifies — and a record younger than the window belongs to a run whose row cannot have been
+ * swept yet. Each candidate is one `x_jobs_run_idx` probe, a LATERAL with a `limit` for the reason
+ * `held` is one: as `not exists` the planner hashes the whole of `x_jobs` for a few thousand
+ * candidates (EXPLAIN, 2026-10-17), a cost that grows with the queue rather than with the old step
+ * records. The delete is the `x_job_steps` primary key. `skip locked` for the reason the batch gives.
+ */
+export const SQL_JOBS_RETENTION_ORPHANS = `
+with orphaned as (
+  select s.run_id, s.name from x_job_steps s
+    left join lateral (
+      select 1 as standing from x_jobs j where j.run_id = s.run_id limit 1
+    ) l on true
+   where s.started_at < statement_timestamp() - ($1::bigint * interval '1 millisecond')
+     and l.standing is null
+   limit $2
+     for update of s skip locked
+), gone as (
+  delete from x_job_steps s using orphaned o
+   where s.run_id = o.run_id and s.name = o.name
+  returning 1
+)
+select count(*)::int as removed from gone
+`.trim();
+
+/**
+ * The two windows are REQUIRED here: their defaults are `app.config.ts`'s (`jobs.doneRetentionMs`,
+ * `jobs.deadRetentionMs`, in `@ultimat3/core`), and a second default in this file would be a second
+ * answer to "how long is a finished job kept".
+ */
+export interface JobsRetentionOptions extends PurgeBatching {
+  readonly executor: PgExecutor;
+  /** Age past which a `done` row goes. */
+  readonly doneRetentionMs: number;
+  /** Age past which a `dead`, `failed` or `cancelled` row goes. */
+  readonly deadRetentionMs: number;
+}
+
+const SUBJECT = 'jobsPurgeTarget';
+
+/** `x_jobs` as a purge target, batched (`purge-batches.ts`). */
+export function jobsPurgeTarget(options: JobsRetentionOptions): PurgeTarget {
+  const done = finiteCount(SUBJECT, 'doneRetentionMs', options.doneRetentionMs, 1);
+  const dead = finiteCount(SUBJECT, 'deadRetentionMs', options.deadRetentionMs, 1);
+  const bounds = purgeBatching(SUBJECT, options);
+  return {
+    name: 'x_jobs',
+    // The count is `x_jobs` rows, the target's name; orphaned step records are its upkeep.
+    purgeExpired: async (): Promise<number> => {
+      const removed = await purgeInBatches(jobsBatch(options.executor, done, dead), bounds);
+      await options.executor.query(SQL_JOBS_RETENTION_ORPHANS, [
+        Math.min(done, dead),
+        bounds.batch,
+      ]);
+      return removed;
+    },
+  };
+}
+
+/** One batch of `SQL_JOBS_RETENTION_BATCH`, then the held runs' steps if it answered any. */
+function jobsBatch(executor: PgExecutor, done: number, dead: number): PurgeBatch {
+  return async (size) => {
+    const rows = await executor.query<{ removed: number | string; held: string | null }>(
+      SQL_JOBS_RETENTION_BATCH,
+      [done, dead, size],
+    );
+    const held = rows[0]?.held ?? '[]';
+    if (held !== '[]') await executor.query(SQL_JOBS_RETENTION_STEPS, [held]);
+    return Number(rows[0]?.removed ?? 0);
+  };
+}

package/src/outbox-retention.ts

@@ -0,0 +1,57 @@
+// Retention for PUBLISHED `x_outbox` rows, as one `PurgeTarget` the boot's hourly `purge()` sweeps.
+// A published row's work is over — the job it staged is in `x_jobs` — and nothing deleted it, so
+// the table kept every enqueue a transaction ever staged, `mail.send`'s address and body included.
+
+import type { PgExecutor } from '@ultimat3/core';
+import { finiteCount } from '@ultimat3/core';
+import type { PurgeTarget } from './purge';
+import { countedBatch, type PurgeBatching, purgeBatching, purgeInBatches } from './purge-batches';
+
+/**
+ * One batch, counted. Only `published_at is not null`: an unpublished row is a job the relay has
+ * not handed over yet, and deleting it is losing that job. The `x_jobs` sweep vetoes a job whose
+ * outbox row is still unpublished; this one never touches such a row, so the two cannot disagree.
+ *
+ * The cutoff is the JOB's clock (`$1`): `SQL_OUTBOX_MARK_PUBLISHED` stamps `published_at` from the
+ * relay's clock, so measuring it against the server's `now()` is the two-clock mismatch
+ * `PurgeTarget.purgeExpired` warns about.
+ */
+export const SQL_OUTBOX_RETENTION_BATCH = `
+with doomed as (
+  select id from x_outbox
+   where published_at is not null
+     and published_at < to_timestamp(($1::bigint - $2::bigint) / 1000.0)
+   order by published_at
+   limit $3
+     for update skip locked
+), gone as (
+  delete from x_outbox where id in (select id from doomed) returning 1
+)
+select count(*)::int as removed from gone
+`.trim();
+
+/**
+ * The window is REQUIRED: its default is `app.config.ts`'s (`jobs.outboxRetentionMs`, in
+ * `@ultimat3/core`), for the reason `JobsRetentionOptions` gives.
+ */
+export interface OutboxRetentionOptions extends PurgeBatching {
+  readonly executor: PgExecutor;
+  /** Age of `published_at` past which a published row goes. */
+  readonly publishedRetentionMs: number;
+}
+
+const SUBJECT = 'outboxPurgeTarget';
+
+/** `x_outbox` as a purge target, batched (`purge-batches.ts`). */
+export function outboxPurgeTarget(options: OutboxRetentionOptions): PurgeTarget {
+  const window = finiteCount(SUBJECT, 'publishedRetentionMs', options.publishedRetentionMs, 1);
+  const bounds = purgeBatching(SUBJECT, options);
+  return {
+    name: 'x_outbox',
+    purgeExpired: (nowMs: number): Promise<number> =>
+      purgeInBatches(
+        countedBatch(options.executor, SQL_OUTBOX_RETENTION_BATCH, [Math.trunc(nowMs), window]),
+        bounds,
+      ),
+  };
+}

package/src/purge-batches.ts

@@ -0,0 +1,68 @@
+// The batch loop every row-count retention target shares: one counted DELETE per `batchSize` rows,
+// repeated until a batch comes back short or `maxBatches` ran. A backlog of months is drained over
+// several passes rather than in one statement that locks every row it will delete.
+
+import type { PgExecutor } from '@ultimat3/core';
+import { finiteCount } from '@ultimat3/core';
+
+/** Rows per DELETE. Small enough that one statement holds its row locks for milliseconds. */
+export const DEFAULT_RETENTION_BATCH = 1_000;
+/**
+ * Batches per pass — a cap on BATCHES, not statements: a target may send more than one statement
+ * per batch (`jobsPurgeTarget`'s follow-up). An hourly pass clears 50,000 rows, and a backlog
+ * drains over hours.
+ */
+export const DEFAULT_RETENTION_MAX_BATCHES = 50;
+
+export interface PurgeBatching {
+  readonly batchSize?: number;
+  readonly maxBatches?: number;
+}
+
+/** The two bounds, screened once at construction under the factory's name. */
+export function purgeBatching(
+  subject: string,
+  options: PurgeBatching,
+): { readonly batch: number; readonly rounds: number } {
+  return {
+    batch: finiteCount(subject, 'batchSize', options.batchSize ?? DEFAULT_RETENTION_BATCH, 1),
+    rounds: finiteCount(
+      subject,
+      'maxBatches',
+      options.maxBatches ?? DEFAULT_RETENTION_MAX_BATCHES,
+      1,
+    ),
+  };
+}
+
+/** One batch: up to `size` rows removed, answering how many were. */
+export type PurgeBatch = (size: number) => Promise<number>;
+
+/**
+ * The batch most targets are: one statement answering one row `{ removed }`, taking `params` with
+ * the batch size appended last.
+ */
+export function countedBatch(
+  executor: PgExecutor,
+  sql: string,
+  params: readonly unknown[],
+): PurgeBatch {
+  return async (size) => {
+    const rows = await executor.query<{ removed: number | string }>(sql, [...params, size]);
+    return Number(rows[0]?.removed ?? 0);
+  };
+}
+
+/** Run `batch` until it removes fewer than `bounds.batch` rows or `rounds` ran; answer the total. */
+export async function purgeInBatches(
+  batch: PurgeBatch,
+  bounds: { readonly batch: number; readonly rounds: number },
+): Promise<number> {
+  let removed = 0;
+  for (let round = 0; round < bounds.rounds; round += 1) {
+    const count = await batch(bounds.batch);
+    removed += count;
+    if (count < bounds.batch) break;
+  }
+  return removed;
+}