@ultimat3/jobs 23.0.0 → 25.0.0

package/src/driver-pg-jobs-sql.ts

@@ -104,14 +104,35 @@ select ${JOB_ROW_COLUMNS}
  * (`driver.ts`) is the same set; a row this matches nothing for was refused before it was sent,
  * or raced to live in between. `woke` announces the row in the statement that made it claimable
  * (`driver-pg-wake-sql.ts`): `x jobs retry` is a job starting now, not at an idle worker's poll.
+ *
+ * $2 is `fromStep`, or null — and the steps are dropped HERE, off the row this statement moved.
+ * They were a statement of their own, sent first: a requeue that then matched nothing (the row
+ * raced to live) had already deleted a running job's steps, and a crash between the two left a
+ * dead letter with its steps gone and no retry queued.
+ *
+ * "From that step onward": the step `fromStep` names and every step that started AFTER it.
+ * Deleting the named step alone left later steps replaying results computed from the old run.
+ * Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
+ * later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
  */
 export const SQL_JOB_REQUEUE = `
-update x_jobs
-   set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
-       claimed_by = null, visible_at = null
- where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
-returning ${JOB_ROW_COLUMNS},
-          pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
+with requeued as (
+  update x_jobs
+     set state = 'ready', attempt = 0, run_at = now(), updated_at = now(),
+         claimed_by = null, visible_at = null
+   where id = $1 and state in ('dead', 'cancelled', 'done', 'failed')
+  returning ${JOB_ROW_COLUMNS},
+            pg_notify('${JOBS_WAKE_CHANNEL}', queue)::text as woke
+), dropped as (
+  delete from x_job_steps s
+   using requeued r
+   where $2::text is not null
+     and s.run_id = r.run_id
+     and (s.name = $2::text
+          or s.started_at > (select t.started_at from x_job_steps t
+                              where t.run_id = r.run_id and t.name = $2::text))
+)
+select ${JOB_ROW_NAMES}, woke from requeued
 `.trim();
 
 /**
@@ -124,16 +145,3 @@ select id from x_jobs
    and id <> $4 and state in ('ready', 'delayed', 'running', 'suspended')
  limit 1
 `.trim();
-
-/**
- * "From that step onward": the step `fromStep` names and every step that started AFTER it.
- * Deleting the named step alone left later steps replaying results computed from the old run.
- * Strictly after, plus the step itself: a step sharing the target's millisecond is not provably
- * later, and re-running an EARLIER step — the charge before the receipt — is the worse mistake.
- */
-export const SQL_STEPS_FROM = `
-delete from x_job_steps
- where run_id = $1
-   and (name = $2
-        or started_at > (select started_at from x_job_steps where run_id = $1 and name = $2))
-`.trim();

package/src/driver-pg-operator-sql.ts

@@ -15,15 +15,18 @@ const bulkMatch = (alias = ''): string =>
   `${alias}state = $1 and ($2::text is null or ${alias}queue = $2) and ($3::text is null or ${alias}name = $3) and ($5::text is null or ${alias}tenant_id = $5)`;
 
 /**
- * Re-queue up to $4 finished rows. `distinct on` the idempotency namespace, because two dead rows
- * of one key would both go `ready` and the second would break the live index — and `not exists`
- * a live holder, the single requeue's `X_JOB_DUPLICATE`, answered here as "not this row".
- * `matching` is counted in the same statement, before the update, so the caller's `remaining` is
- * `matching - affected` with no second read.
+ * Re-queue up to $4 finished rows. `eligible` is every row of the filter no LIVE job holds the key
+ * of — the single requeue's `X_JOB_DUPLICATE`, answered here as "not this row" — and `distinct on`
+ * the idempotency namespace picks one of them per key, because two dead rows of one key would both
+ * go `ready` and the second would break the live index.
+ *
+ * `remaining` is what a second call could still move, counted in the same statement off the same
+ * snapshot: the eligible rows whose key this call did not just take. A held row was counted too,
+ * so a caller told to "call again until it is zero" never stopped.
  */
 export const SQL_JOB_REQUEUE_MANY = `
-with candidates as (
-  select distinct on (j.name, coalesce(j.tenant_id, ''), j.idempotency_key) j.id
+with eligible as (
+  select j.id, j.name, coalesce(j.tenant_id, '') as tenant, j.idempotency_key, j.created_at
     from x_jobs j
    where ${bulkMatch('j.')}
      and not exists (
@@ -33,7 +36,10 @@ with candidates as (
           and l.idempotency_key = j.idempotency_key
           and l.state in ${LIVE}
      )
-   order by j.name, coalesce(j.tenant_id, ''), j.idempotency_key, j.created_at
+), candidates as (
+  select distinct on (name, tenant, idempotency_key) id, name, tenant, idempotency_key
+    from eligible
+   order by name, tenant, idempotency_key, created_at
    limit $4
 ), moved as (
   update x_jobs
@@ -43,7 +49,13 @@ with candidates as (
   returning id
 )
 select (select count(*) from moved)::int as affected,
-       (select count(*) from x_jobs where ${bulkMatch()})::int as matching
+       (select count(*) from eligible e
+         where not exists (
+           select 1 from candidates c
+             join moved m on m.id = c.id
+            where c.name = e.name and c.tenant = e.tenant
+              and c.idempotency_key = e.idempotency_key
+         ))::int as remaining
 `.trim();
 
 /** Delete up to $4 rows and their step records. `running` is refused before this is sent. */
@@ -267,17 +279,25 @@ select task_name,
  * `woken` announces each queue that received a job, once, when the statement commits — read in
  * the final select because a plain-select CTE nothing reads is never run.
  *
- * $1 task, $2 occurrence (ms), $3 the jobs as a JSON array.
+ * The fence is the OCCURRENCE and the watermark may land past it: `run-once` runs the earliest
+ * missed occurrence and drops the rest, and dropping is the watermark moving beyond them. That was
+ * a second statement behind this one, and a crash between the two fired a second "one catch-up".
+ * `greatest` ignores a null, so with no `$4` the watermark lands on the occurrence — and a `$4`
+ * behind the occurrence can never leave the watermark short of what just fired.
+ *
+ * $1 task, $2 occurrence (ms), $3 the jobs as a JSON array, $4 where the watermark lands (ms) or
+ * null for the occurrence itself.
  */
 export const SQL_SCHEDULER_FIRE = `
 with moved as (
   insert into x_scheduler_state
     (task_name, last_fired_at, updated_at, fired_occurrence_at, fired_at)
-  values ($1, to_timestamp($2::bigint / 1000.0), now(), to_timestamp($2::bigint / 1000.0), now())
+  values ($1, to_timestamp(greatest($2::bigint, $4::bigint) / 1000.0), now(),
+          to_timestamp($2::bigint / 1000.0), now())
   on conflict (task_name) do update
      set last_fired_at = excluded.last_fired_at, updated_at = now(),
          fired_occurrence_at = excluded.fired_occurrence_at, fired_at = excluded.fired_at
-   where x_scheduler_state.last_fired_at < excluded.last_fired_at
+   where x_scheduler_state.last_fired_at < excluded.fired_occurrence_at
   returning task_name
 ), queued as (
   insert into x_jobs

package/src/driver-pg-operator.ts

@@ -3,9 +3,9 @@
 // that file is the claim/settle driver and sits near its size ceiling; this is the read-and-repair
 // half a dashboard and `x jobs` call.
 
+import type { PgExecutor } from '@ultimat3/core';
 import { assert, finiteOption } from '@ultimat3/core';
 import { REQUEUEABLE_STATES } from './driver';
-import type { PgExecutor } from './driver-pg';
 import { SQL_JOB_LIST, SQL_JOB_LIST_BEFORE } from './driver-pg-jobs-sql';
 import {
   SQL_COUNTER_DROP,
@@ -126,7 +126,14 @@ export function pgOperator(
       );
       const limit = pageLimit('the pg driver list', filter.limit);
       const raw = filter.before ?? filter.after;
-      const cursor = raw === undefined ? undefined : parseJobCursor(raw, 'the pg driver list');
+      const cursor =
+        raw === undefined
+          ? undefined
+          : parseJobCursor(
+              raw,
+              'the pg driver list',
+              filter.before === undefined ? 'after' : 'before',
+            );
       const rows = await exec().query<JobRow>(
         filter.before === undefined ? SQL_JOB_LIST : SQL_JOB_LIST_BEFORE,
         [
@@ -162,7 +169,16 @@ export function pgOperator(
         `requeueMany was asked for ${filter.state} jobs, and only a finished job can be requeued`,
         "pass state: 'dead', 'failed', 'cancelled' or 'done' to requeueMany",
       );
-      const result = await bulk(SQL_JOB_REQUEUE_MANY, filter);
+      // Its own read, not `bulk`: the statement answers `remaining` itself — the rows a second
+      // call could move — where `matching - affected` counted the held rows it never will.
+      const rows = await exec().query<{ affected: Count; remaining: Count }>(SQL_JOB_REQUEUE_MANY, [
+        filter.state,
+        filter.queue ?? null,
+        filter.name ?? null,
+        MAX_BULK_ROWS,
+        filter.tenantId ?? null,
+      ]);
+      const result = { affected: num(rows[0]?.affected), remaining: num(rows[0]?.remaining) };
       if (result.affected > 0) await wake(filter.queue);
       return result;
     },

package/src/driver-pg-rows.ts

@@ -94,7 +94,7 @@ export function toJobRecord(row: JobRow): JobRecord {
     attempt: row.attempt,
     maxAttempts: row.max_attempts,
     state: statusIn<JobState>(JOB_STATES, isJobState, {
-      table: 'ultimate_jobs',
+      table: 'x_jobs',
       column: 'state',
       value: row.state,
     }),
@@ -146,7 +146,7 @@ export function toStepRecord(row: StepRow): StepRecord {
     runId: row.run_id,
     name: row.name,
     status: statusIn<StepStatus>(STEP_STATUSES, isStepStatus, {
-      table: 'ultimate_job_steps',
+      table: 'x_job_steps',
       column: 'status',
       value: row.status,
     }),
@@ -168,7 +168,7 @@ export function toBackfillRun(row: BackfillRow): BackfillRun {
     name: row.name,
     checksum: row.checksum,
     status: statusIn(BACKFILL_STATUSES, isBackfillStatus, {
-      table: 'ultimate_backfills',
+      table: 'x_backfills',
       column: 'status',
       value: row.status,
     }),

package/src/driver-pg-settle-sql.ts

@@ -17,8 +17,7 @@ const BUCKET = `${COUNTER_BUCKET_MS}, date_trunc('minute', now())`;
  * that claimed it again is the SAME one: its id did not change, only the claim's ordinal did.
  * `returning` is how the caller learns whether its settle landed.
  *
- * $1 id, $2 worker, $3 the attempt's duration in ms, $4 the claim, $5 whether it is counted — a
- * row `x jobs drain` moved is `done` here and is not a completed run.
+ * $1 id, $2 worker, $3 the attempt's duration in ms, $4 the claim.
  */
 export const SQL_ACK = `
 with settled as (
@@ -28,7 +27,7 @@ with settled as (
   returning name
 ), counted as (
   insert into x_job_counters (job, bucket_ms, bucket_start, done, duration_ms)
-  select name, ${BUCKET}, 1, $3::bigint from settled where $5::boolean
+  select name, ${BUCKET}, 1, $3::bigint from settled
   on conflict (job, bucket_ms, bucket_start) do update
      set done        = x_job_counters.done + 1,
          duration_ms = x_job_counters.duration_ms + excluded.duration_ms

package/src/driver-pg-sql.ts

@@ -19,7 +19,6 @@ export {
   SQL_JOB_LIST,
   SQL_JOB_LIVE_HOLDER,
   SQL_JOB_REQUEUE,
-  SQL_STEPS_FROM,
 } from './driver-pg-jobs-sql';
 
 export {
@@ -32,8 +31,23 @@ export {
 } from './driver-pg-outbox-sql';
 export { SQL_ACK, SQL_NACK } from './driver-pg-settle-sql';
 
+import { LEASE_LAPSED_FINAL_ATTEMPT } from './driver';
 import { JOB_ROW_COLUMNS } from './driver-pg-jobs-sql';
 import { notifyJobReady } from './driver-pg-wake-sql';
+import { COUNTER_BUCKET_MS } from './introspection';
+
+/** What both arms of the claim return: a `JobRow`, read off the updated row `j`. */
+const CLAIM_ROW_COLUMNS = `j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
+          j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
+          j.traceparent, j.enqueued_by,
+          (extract(epoch from j.run_at)     * 1000)::bigint as run_at,
+          (extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
+          (extract(epoch from j.created_at) * 1000)::bigint as created_at,
+          (extract(epoch from j.updated_at) * 1000)::bigint as updated_at`;
+
+const CLAIM_ROW_NAMES = `id, name, queue, input, idempotency_key, run_id, attempt, max_attempts,
+       state, tenant_id, last_error, claimed_by, claims, traceparent, enqueued_by, run_at,
+       visible_at, created_at, updated_at`;
 
 /**
  * The insert, and its wake: `woke` is the notification's own column, evaluated once per inserted
@@ -85,10 +99,19 @@ select id, run_id from x_jobs
  * lock and throughput collapses to one worker. `visible_at` in the predicate reclaims leases
  * abandoned by a crashed worker. `cancelled` is absent from every branch by construction — a
  * cancelled row is terminal, so it is never handed to a worker again.
+ *
+ * TWO arms over one pick. A `running` row is picked only with a lapsed lease, and one that lapsed
+ * on its FINAL attempt (`attempt >= max_attempts` — `isFinalAttempt`, `retry.ts`) is `buried`:
+ * settled `dead` here, counted, and answered with its new state so the driver reports it instead
+ * of handing it out. Re-claimed, it was an attempt past `max_attempts` every lease, forever — a
+ * job that kills its worker took one per visibility timeout and never reached the dead letters.
+ * `claimed_by` is cleared with the lease, so the body that stalled rather than died cannot settle
+ * the row it lost. A buried row spends a slot of `$2`: the pass after it claims what it displaced.
+ * `$5` names the jobs declaring `retry.deadLetter: false`: their rows are buried `failed`.
  */
 export const SQL_CLAIM = `
-with claimed as (
-  select id
+with picked as (
+  select id, (state = 'running' and attempt >= max_attempts) as exhausted
     from x_jobs
    where queue = any($1::text[])
      and run_at <= now()
@@ -102,35 +125,56 @@ with claimed as (
    order by run_at
    limit $2
      for update skip locked
+), buried as (
+  update x_jobs j
+     set state            = case when j.name = any($5::text[]) then 'failed' else 'dead' end,
+         visible_at       = null,
+         claimed_by       = null,
+         last_error       = '${LEASE_LAPSED_FINAL_ATTEMPT}',
+         last_error_stack = null,
+         updated_at       = now()
+    from picked p
+   where j.id = p.id and p.exhausted
+  returning ${CLAIM_ROW_COLUMNS}
+), counted as (
+  insert into x_job_counters (job, bucket_ms, bucket_start, dead, failed, duration_ms)
+  select name, ${COUNTER_BUCKET_MS}, date_trunc('minute', now()),
+         (count(*) filter (where state = 'dead'))::int,
+         (count(*) filter (where state = 'failed'))::int, 0
+    from buried
+   group by name
+  on conflict (job, bucket_ms, bucket_start) do update
+     set dead   = x_job_counters.dead + excluded.dead,
+         failed = x_job_counters.failed + excluded.failed
+), claimed as (
+  update x_jobs j
+     set state      = 'running',
+         attempt    = j.attempt + 1,
+         claims     = j.claims + 1,
+         claimed_by = $3,
+         visible_at = now() + ($4::bigint * interval '1 millisecond'),
+         updated_at = now()
+    from picked p
+   where j.id = p.id and not p.exhausted
+  returning ${CLAIM_ROW_COLUMNS}
 )
-update x_jobs j
-   set state      = 'running',
-       attempt    = j.attempt + 1,
-       claims     = j.claims + 1,
-       claimed_by = $3,
-       visible_at = now() + ($4::bigint * interval '1 millisecond'),
-       updated_at = now()
-  from claimed c
- where j.id = c.id
-returning j.id, j.name, j.queue, j.input, j.idempotency_key, j.run_id, j.attempt,
-          j.max_attempts, j.state, j.tenant_id, j.last_error, j.claimed_by, j.claims,
-          j.traceparent, j.enqueued_by,
-          (extract(epoch from j.run_at)     * 1000)::bigint as run_at,
-          (extract(epoch from j.visible_at) * 1000)::bigint as visible_at,
-          (extract(epoch from j.created_at) * 1000)::bigint as created_at,
-          (extract(epoch from j.updated_at) * 1000)::bigint as updated_at
+select ${CLAIM_ROW_NAMES} from claimed
+union all
+select ${CLAIM_ROW_NAMES} from buried
 `.trim();
 
 /**
- * Stop a job from outside. `state <> 'done'` and not `state = 'ready'`: the runaway backfill this
- * exists for is `running`, and a job that already finished has nothing to stop. The worker holding
- * it learns on its next heartbeat, which no longer matches `state = 'running'`.
+ * Stop a job from outside — a LIVE one. The runaway backfill this exists for is `running`; the
+ * worker holding it learns on its next heartbeat, which no longer matches `state = 'running'`.
+ * Fenced on the four live states and not on `state <> 'done'`, which let a cancel REWRITE a dead
+ * letter: `dead / "card declined"` became `cancelled / "oops wrong id"` and left the dead-letter
+ * queue. A finished row is how the job ended, and stays that.
  */
 export const SQL_CANCEL = `
 update x_jobs
    set state = 'cancelled', visible_at = null, claimed_by = null,
        last_error = coalesce($2, last_error), updated_at = now()
- where id = $1 and state <> 'done'
+ where id = $1 and state in ('ready', 'delayed', 'running', 'suspended')
 returning ${JOB_ROW_COLUMNS}
 `.trim();
 
@@ -167,12 +211,12 @@ select queue,
          (where state in ('ready', 'delayed') and run_at <= now()), 0) * 1000   as oldest_ready_ms
   from x_jobs
  group by queue
- order by queue
+ order by queue collate "C"
 `.trim();
 
 /**
  * Session-scoped, so a crashed node's lock releases itself — and released just as surely when a
- * POOLED connection goes back to the pool, which is why `createPgLeader` is not what a scheduler
+ * POOLED connection goes back to the pool, which is why `postgresLeader` is not what a scheduler
  * running on a shared executor should use. See `SQL_LEADER_ACQUIRE` below.
  */
 export const SQL_TRY_ADVISORY_LOCK = 'select pg_try_advisory_lock($1) as locked';
@@ -324,7 +368,17 @@ select id, name, payload, correlation_key,
  limit $2
 `.trim();
 
-export const SQL_EVENT_PURGE = `delete from x_job_events where expires_at <= statement_timestamp()`;
+/**
+ * The sweep, COUNTED in the statement: `PgExecutor` answers rows and never a command tag, and
+ * `returning` alone would ship every deleted id back to count them. The statement's own time, as
+ * every other event statement reads — an unpurged row costs `find` a filter, never a wrong answer.
+ */
+export const SQL_EVENT_PURGE_COUNTED = `
+with gone as (
+  delete from x_job_events where expires_at <= statement_timestamp() returning 1
+)
+select count(*)::int as removed from gone
+`.trim();
 
 export const SQL_STEP_GET = `
 select run_id, name, status, output, attempts, error,
@@ -351,18 +405,33 @@ select run_id, name, status, output, attempts, error,
   from x_job_steps where run_id = $1 order by started_at
 `.trim();
 
+/**
+ * FENCED on the claim when one is given, exactly as `SQL_ACK` is: `running`, the same worker, the
+ * same claim ordinal. The runner's own fence is an in-process signal, and it cannot stop a body in
+ * ANOTHER process — a worker stalled past its lease wrote its step over the re-claimer's. A null
+ * `$12` is an unfenced write: a transfer between drivers, with no row on this one to hold yet.
+ * Answers one row when the write landed, none when the fence refused it.
+ *
+ * $12 the job id or null, $13 the worker, $14 the claim.
+ */
 export const SQL_STEP_PUT = `
 insert into x_job_steps
   (run_id, name, status, output, started_at, completed_at, wake_at, event,
    correlation_key, attempts, error)
-values ($1, $2, $3, $4::jsonb, to_timestamp($5 / 1000.0),
-        case when $6::bigint is null then null else to_timestamp($6 / 1000.0) end,
-        case when $7::bigint is null then null else to_timestamp($7 / 1000.0) end,
-        $8, $9, $10, $11)
+select $1::uuid, $2, $3, $4::jsonb, to_timestamp($5::bigint / 1000.0),
+       case when $6::bigint is null then null else to_timestamp($6::bigint / 1000.0) end,
+       case when $7::bigint is null then null else to_timestamp($7::bigint / 1000.0) end,
+       $8, $9, $10::int, $11
+ where $12::uuid is null
+    or exists (
+      select 1 from x_jobs
+       where id = $12::uuid and state = 'running' and claimed_by = $13 and claims = $14::int
+    )
 on conflict (run_id, name) do update
    set status = excluded.status, output = excluded.output,
        completed_at = excluded.completed_at, wake_at = excluded.wake_at,
        attempts = excluded.attempts, error = excluded.error
+returning 1 as written
 `.trim();
 
 /**