@ultimat3/jobs 22.14.0 → 23.0.0

package/src/driver-pg-operator.ts

@@ -0,0 +1,300 @@
+// The pg driver's operator surface: every `JobIntrospection` member the queue grew for an
+// operator, each ONE statement from `driver-pg-operator-sql.ts`. Apart from `driver-pg.ts` because
+// that file is the claim/settle driver and sits near its size ceiling; this is the read-and-repair
+// half a dashboard and `x jobs` call.
+
+import { assert, finiteOption } from '@ultimat3/core';
+import { REQUEUEABLE_STATES } from './driver';
+import type { PgExecutor } from './driver-pg';
+import { SQL_JOB_LIST, SQL_JOB_LIST_BEFORE } from './driver-pg-jobs-sql';
+import {
+  SQL_COUNTER_DROP,
+  SQL_COUNTER_FOLD,
+  SQL_COUNTER_TOTALS,
+  SQL_COUNTERS,
+  SQL_JOB_PROGRESS,
+  SQL_JOB_PROMOTE,
+  SQL_JOB_PROMOTE_MANY,
+  SQL_JOB_REMOVE,
+  SQL_JOB_REMOVE_MANY,
+  SQL_JOB_REQUEUE_MANY,
+  SQL_PAUSE,
+  SQL_PAUSED,
+  SQL_RESUME,
+  SQL_TASK_FIRE_RECORD,
+  SQL_TASK_FIRES,
+  SQL_WORKER_ANNOUNCE,
+  SQL_WORKER_FORGET,
+  SQL_WORKERS,
+} from './driver-pg-operator-sql';
+import type { JobRow } from './driver-pg-rows';
+import { num, toJobRecord } from './driver-pg-rows';
+import { JOBS_WAKE_CHANNEL, SQL_WAKE } from './driver-pg-wake-sql';
+import { JobNotRemovableError } from './errors-operator';
+import type {
+  BulkFilter,
+  BulkResult,
+  CounterBucketMs,
+  JobIntrospection,
+  PausedName,
+} from './introspection';
+import {
+  COUNTER_TIERS,
+  MAX_BULK_ROWS,
+  MAX_TASK_FIRES,
+  MAX_WORKER_IN_FLIGHT,
+  PROMOTABLE_STATES,
+  pageLimit,
+  parseJobCursor,
+} from './introspection';
+
+export type PgOperatorMembers = Omit<
+  JobIntrospection,
+  'job' | 'deadLetters' | 'requeue' | 'cancel'
+>;
+
+type Count = number | string;
+
+interface CounterRow {
+  readonly job: string;
+  readonly bucket_ms?: Count;
+  readonly bucket_start?: Count;
+  readonly done: Count;
+  readonly retried: Count;
+  readonly failed: Count;
+  readonly dead: Count;
+  readonly duration_ms: Count;
+}
+
+const counts = (row: CounterRow) => ({
+  job: row.job,
+  done: num(row.done),
+  retried: num(row.retried),
+  failed: num(row.failed),
+  dead: num(row.dead),
+  durationMs: num(row.duration_ms),
+});
+
+/** A `jsonb` list of strings, from a client with a type map or without one. */
+function stringList(raw: unknown): readonly string[] {
+  let value: unknown = raw;
+  if (typeof raw === 'string') {
+    try {
+      value = JSON.parse(raw) as unknown;
+    } catch {
+      return [];
+    }
+  }
+  return Array.isArray(value) ? value.filter((entry) => typeof entry === 'string') : [];
+}
+
+export function pgOperator(
+  exec: () => PgExecutor,
+  job: JobIntrospection['job'],
+): PgOperatorMembers {
+  const bulk = async (sql: string, filter: BulkFilter): Promise<BulkResult> => {
+    const rows = await exec().query<{ affected: Count; matching: Count }>(sql, [
+      filter.state,
+      filter.queue ?? null,
+      filter.name ?? null,
+      MAX_BULK_ROWS,
+      filter.tenantId ?? null,
+    ]);
+    const affected = num(rows[0]?.affected);
+    return { affected, remaining: Math.max(0, num(rows[0]?.matching) - affected) };
+  };
+
+  const paused = async (kind: 'queue' | 'task'): Promise<readonly PausedName[]> => {
+    const rows = await exec().query<{ name: string; paused_at: Count }>(SQL_PAUSED, [kind]);
+    return rows.map((row) => ({ name: row.name, pausedAt: num(row.paused_at) }));
+  };
+
+  /**
+   * An operator made rows claimable — on `queue`, or on any when none is named. Announced, so
+   * `x jobs retry` is a job starting now rather than at the end of an idle worker's wait.
+   */
+  const wake = async (queue?: string): Promise<void> => {
+    await exec().query(SQL_WAKE, [JOBS_WAKE_CHANNEL, queue ?? '']);
+  };
+
+  return {
+    async list(filter = {}) {
+      assert(
+        filter.after === undefined || filter.before === undefined,
+        'list was asked for a page both after and before a cursor',
+        'pass one of after: jobCursor(lastRow) or before: jobCursor(firstRow)',
+      );
+      const limit = pageLimit('the pg driver list', filter.limit);
+      const raw = filter.before ?? filter.after;
+      const cursor = raw === undefined ? undefined : parseJobCursor(raw, 'the pg driver list');
+      const rows = await exec().query<JobRow>(
+        filter.before === undefined ? SQL_JOB_LIST : SQL_JOB_LIST_BEFORE,
+        [
+          filter.queue ?? null,
+          filter.name ?? null,
+          filter.state ?? null,
+          limit,
+          filter.idPrefix ?? null,
+          filter.createdFrom ?? null,
+          filter.createdTo ?? null,
+          cursor?.createdAt ?? null,
+          cursor?.id ?? null,
+          filter.tenantId ?? null,
+        ],
+      );
+      return rows.map(toJobRecord);
+    },
+
+    async remove(jobId) {
+      const current = await job(jobId);
+      if (current === undefined) return undefined;
+      if (current.state === 'running') throw new JobNotRemovableError({ jobId });
+      const gone = await exec().query<{ run_id: string }>(SQL_JOB_REMOVE, [jobId]);
+      // Read as queued a moment ago and claimed since: the claim won, and the row is a worker's.
+      if (gone.length === 0) throw new JobNotRemovableError({ jobId });
+      return current;
+    },
+
+    // `async`, so a refused state REJECTS — the memory driver's answer, never a synchronous throw.
+    async requeueMany(filter) {
+      assert(
+        REQUEUEABLE_STATES.has(filter.state),
+        `requeueMany was asked for ${filter.state} jobs, and only a finished job can be requeued`,
+        "pass state: 'dead', 'failed', 'cancelled' or 'done' to requeueMany",
+      );
+      const result = await bulk(SQL_JOB_REQUEUE_MANY, filter);
+      if (result.affected > 0) await wake(filter.queue);
+      return result;
+    },
+
+    // `async`, so a refused state REJECTS, as `requeueMany` does.
+    async promoteMany(filter) {
+      assert(
+        PROMOTABLE_STATES.has(filter.state),
+        `promoteMany was asked for ${filter.state} jobs, and only a job waiting on its run time can be promoted`,
+        "pass state: 'delayed' or 'ready' to promoteMany",
+      );
+      const rows = await exec().query<{ affected: Count; matching: Count }>(SQL_JOB_PROMOTE_MANY, [
+        filter.state,
+        filter.queue ?? null,
+        filter.name ?? null,
+        MAX_BULK_ROWS,
+        filter.tenantId ?? null,
+      ]);
+      return { affected: num(rows[0]?.affected), remaining: num(rows[0]?.matching) };
+    },
+
+    async removeMany(filter) {
+      if (filter.state === 'running') throw new JobNotRemovableError({});
+      return bulk(SQL_JOB_REMOVE_MANY, filter);
+    },
+
+    async promote(jobId) {
+      const rows = await exec().query<JobRow>(SQL_JOB_PROMOTE, [jobId]);
+      const row = rows[0];
+      return row === undefined ? undefined : toJobRecord(row);
+    },
+
+    async pauseQueue(queue) {
+      await exec().query(SQL_PAUSE, ['queue', queue]);
+    },
+    async resumeQueue(queue) {
+      await exec().query(SQL_RESUME, ['queue', queue]);
+      await wake(queue);
+    },
+    pausedQueues: () => paused('queue'),
+    async pauseTask(task) {
+      await exec().query(SQL_PAUSE, ['task', task]);
+    },
+    async resumeTask(task) {
+      await exec().query(SQL_RESUME, ['task', task]);
+    },
+    pausedTasks: () => paused('task'),
+
+    async recordTaskFire({ task, occurrenceMs }) {
+      await exec().query(SQL_TASK_FIRE_RECORD, [task, occurrenceMs]);
+    },
+    async taskFires() {
+      const rows = await exec().query<{ task_name: string; occurrence: Count; fired_at: Count }>(
+        SQL_TASK_FIRES,
+        [MAX_TASK_FIRES],
+      );
+      return rows.map((row) => ({
+        task: row.task_name,
+        occurrenceMs: num(row.occurrence),
+        firedAt: num(row.fired_at),
+      }));
+    },
+
+    async announceWorker(worker, ttlMs) {
+      await exec().query(SQL_WORKER_ANNOUNCE, [
+        worker.id,
+        worker.host,
+        JSON.stringify(worker.queues),
+        worker.concurrency,
+        JSON.stringify(worker.inFlight.slice(0, MAX_WORKER_IN_FLIGHT)),
+        worker.startedAt,
+        finiteOption('the pg worker registry', 'ttlMs', ttlMs),
+      ]);
+    },
+    async forgetWorker(workerId) {
+      await exec().query(SQL_WORKER_FORGET, [workerId]);
+    },
+    async workers() {
+      const rows = await exec().query<{
+        id: string;
+        host: string;
+        queues: unknown;
+        concurrency: Count;
+        in_flight: unknown;
+        started_at: Count;
+        heartbeat_at: Count;
+      }>(SQL_WORKERS, []);
+      return rows.map((row) => ({
+        id: row.id,
+        host: row.host,
+        startedAt: num(row.started_at),
+        queues: stringList(row.queues),
+        concurrency: num(row.concurrency),
+        inFlight: stringList(row.in_flight),
+        heartbeatAt: num(row.heartbeat_at),
+      }));
+    },
+
+    async recordProgress(jobId, by, progress) {
+      await exec().query(SQL_JOB_PROGRESS, [
+        jobId,
+        by.workerId,
+        JSON.stringify(progress),
+        by.claim,
+      ]);
+    },
+
+    async counters(query) {
+      const rows = await exec().query<CounterRow>(SQL_COUNTERS, [query.job, query.sinceMs]);
+      return rows.map((row) => ({
+        ...counts(row),
+        bucketStart: num(row.bucket_start),
+        bucketMs: num(row.bucket_ms) as CounterBucketMs,
+      }));
+    },
+    async counterTotals(sinceMs) {
+      const rows = await exec().query<CounterRow>(SQL_COUNTER_TOTALS, [sinceMs]);
+      return rows.map(counts);
+    },
+    async rollupCounters() {
+      let moved = 0;
+      for (const [index, tier] of COUNTER_TIERS.entries()) {
+        const next = COUNTER_TIERS[index + 1];
+        const rows = await exec().query<{ moved: Count }>(
+          next === undefined ? SQL_COUNTER_DROP : SQL_COUNTER_FOLD,
+          next === undefined
+            ? [tier.bucketMs, tier.keepMs]
+            : [tier.bucketMs, tier.keepMs, next.bucketMs],
+        );
+        moved += num(rows[0]?.moved);
+      }
+      return moved;
+    },
+  };
+}

package/src/driver-pg-outbox-sql.ts

@@ -0,0 +1,110 @@
+// Every statement the `x_outbox` store runs. Split off `driver-pg-sql.ts` at its size ceiling and
+// re-exported from it, the way the settle and whole-row statements are.
+
+import { NOTIFY_STAGED } from './driver-pg-wake-sql';
+
+/**
+ * The stage, and its wake. One statement on the CALLER's connection, so the notification is in
+ * the business transaction with the row: delivered when it commits, never when it rolls back.
+ */
+export const SQL_OUTBOX_STAGE = `
+with staged as (
+  insert into x_outbox
+    (id, job, queue, input, idempotency_key, max_attempts, run_at, staged_at, tenant_id,
+     traceparent, enqueued_by, run_id)
+  values ($1, $2, $3, $4::jsonb, $5, $6, to_timestamp($7 / 1000.0), to_timestamp($8 / 1000.0),
+          $9, $10, $11, $12)
+  returning id
+)
+select id,${NOTIFY_STAGED}::text as woke
+  from staged
+`.trim();
+
+/**
+ * The claim, and it has to be ONE statement. `for update skip locked` in a bare select holds its
+ * row locks only until that statement ends — under autocommit, before `claim()` even resolves — so
+ * two relays polling 200ms apart read the same unpublished rows and both hand them to `enqueue`.
+ * `SQL_ENQUEUE` collapses the repeat only while the first job is still LIVE: its conflict target
+ * is a partial index over the live states, so a second publish landing after that job reached a
+ * terminal state inserts a second row and the handler runs again. (The mechanism is Postgres
+ * semantics; how often the two orderings line up in a deployment was never measured.)
+ *
+ * So the lock and the claim commit together, the CTE shape `SQL_CLAIM` already uses, and
+ * `claimed_at` is a LEASE: `$2` is the window after which a row a dead relay was holding is
+ * claimable again, because a claim nothing can expire strands its rows forever.
+ *
+ * The outer `select ... order by staged_at, id` is not cosmetic. `update ... returning` has no
+ * defined row order and the relay publishes in the order it is handed rows, so an app staging
+ * `createInvoice` then `chargeCard` in one transaction depends on this line.
+ *
+ * `, id` is what makes that key TOTAL, and the CTE needs it as much as the projection does: every
+ * row staged in one transaction shares a `staged_at`, so `staged_at` alone leaves the tie to the
+ * planner — which rows a `limit` takes, and in which order they publish, then differ between two
+ * relays and between two runs of one relay. No column was added for it: `id` is a UUIDv7 minted by
+ * `uuid()`, monotonic and already the primary key, so the tiebreak IS stage order.
+ */
+export const SQL_OUTBOX_CLAIM = `
+with claimable as (
+  select id
+    from x_outbox
+   where published_at is null
+     and (claimed_at is null
+          or claimed_at <= now() - ($2::bigint * interval '1 millisecond'))
+   order by staged_at, id
+   limit $1
+     for update skip locked
+), claimed as (
+  update x_outbox o
+     set claimed_at = now(), claimed_by = $3
+    from claimable c
+   where o.id = c.id
+  returning o.id, o.run_id, o.job, o.queue, o.input, o.idempotency_key, o.max_attempts,
+            o.tenant_id, o.traceparent, o.enqueued_by, o.claimed_by, o.run_at, o.staged_at
+)
+select id, run_id, job, queue, input, idempotency_key, max_attempts, tenant_id,
+       traceparent, enqueued_by, claimed_by,
+       (extract(epoch from run_at) * 1000)::bigint    as run_at,
+       (extract(epoch from staged_at) * 1000)::bigint as staged_at
+  from claimed
+ order by staged_at, id
+`.trim();
+
+/**
+ * Hand a claim back early. The relay stops its batch on the first publish that fails, and without
+ * this the rows behind it would wait out the whole lease before any relay could retry them — a
+ * pool blip during a failover becoming tens of seconds of unpublished, committed work.
+ *
+ * Fenced on `published_at is null` so it can never unclaim a row some other pass already
+ * published, AND on `claimed_by` so it can never unclaim one a NEWER claimant now holds: a relay
+ * whose lease lapsed while it stalled wakes into a world where its batch is another relay's, and
+ * an unfenced release frees rows that relay is mid-publish on — a third relay claims them and
+ * publishes them again, which is the duplicate the lease exists to prevent.
+ */
+export const SQL_OUTBOX_RELEASE = `
+update x_outbox
+   set claimed_at = null, claimed_by = null
+ where id = any($1::uuid[]) and published_at is null and claimed_by = $2
+`.trim();
+
+/**
+ * Same fence, and here it is the more expensive one to miss: marking a row published is LOSING it,
+ * so a lapsed claimant stamping a row the current one has not published yet drops that job with
+ * nothing to notice. `published_at is null` makes the stamp first-writer-wins rather than a
+ * rewrite of an audit timestamp.
+ */
+export const SQL_OUTBOX_MARK_PUBLISHED = `
+update x_outbox set published_at = to_timestamp($2 / 1000.0)
+ where id = $1 and published_at is null and claimed_by = $3
+`.trim();
+
+/**
+ * The job a staged row's publish made, by the id the two share. Read when `SQL_ENQUEUE` inserted
+ * nothing for a request that named its id: the answer is that publish, repeated.
+ */
+export const SQL_OUTBOX_PUBLISHED_JOB = `
+select id, run_id from x_jobs where id = $1
+`.trim();
+
+export const SQL_OUTBOX_PENDING_COUNT = `
+select count(*)::bigint as pending from x_outbox where published_at is null
+`.trim();

package/src/driver-pg-rows.ts

@@ -6,6 +6,7 @@
 import { BACKFILL_STATUSES, type BackfillRun, isBackfillStatus } from './backfill-ledger';
 import { isJobState, JOB_STATES, type JobRecord, type JobState } from './driver';
 import { JobRowStatusUnknownError } from './errors';
+import type { JobProgress } from './introspection';
 import { isStepStatus, STEP_STATUSES, type StepRecord, type StepStatus } from './steps';
 
 export interface StepRow {
@@ -47,12 +48,17 @@ export interface JobRow {
   readonly tenant_id: string | null;
   readonly last_error: string | null;
   readonly claimed_by: string | null;
+  /** Absent from a row an older statement projected. */
+  readonly claims?: number | string | null;
   readonly run_at: number | string;
   readonly visible_at: number | string | null;
   readonly created_at: number | string;
   readonly updated_at: number | string;
   readonly traceparent?: string | null;
   readonly enqueued_by?: string | null;
+  /** `jsonb`: an object from a client with a type map, its text from one without. */
+  readonly progress?: unknown;
+  readonly last_error_stack?: string | null;
 }
 
 /**
@@ -98,6 +104,7 @@ export function toJobRecord(row: JobRow): JobRecord {
     ...(row.tenant_id === null ? {} : { tenantId: row.tenant_id }),
     ...(row.last_error === null ? {} : { lastError: row.last_error }),
     ...(row.claimed_by === null ? {} : { claimedBy: row.claimed_by }),
+    ...(num(row.claims ?? 0) > 0 ? { claim: num(row.claims ?? 0) } : {}),
     ...(visibleAt === undefined ? {} : { visibleAt }),
     ...(row.traceparent === null || row.traceparent === undefined
       ? {}
@@ -105,9 +112,33 @@ export function toJobRecord(row: JobRow): JobRecord {
     ...(row.enqueued_by === null || row.enqueued_by === undefined
       ? {}
       : { enqueuedBy: row.enqueued_by }),
+    ...(row.last_error_stack === null || row.last_error_stack === undefined
+      ? {}
+      : { lastErrorStack: row.last_error_stack }),
+    ...progressOf(row.progress),
   };
 }
 
+/**
+ * The `progress` column, or nothing. Total: the value was written by `recordProgress` in whatever
+ * build was deployed then, and a row that carries something else has no progress, not a throw.
+ */
+function progressOf(raw: unknown): { readonly progress?: JobProgress } {
+  const value: unknown = typeof raw === 'string' ? parseJson(raw) : raw;
+  if (typeof value !== 'object' || value === null) return {};
+  const { done, total, note, at } = value as Readonly<Record<string, unknown>>;
+  if (typeof done !== 'number' || typeof total !== 'number' || typeof at !== 'number') return {};
+  return { progress: { done, total, at, ...(typeof note === 'string' ? { note } : {}) } };
+}
+
+function parseJson(text: string): unknown {
+  try {
+    return JSON.parse(text) as unknown;
+  } catch {
+    return undefined;
+  }
+}
+
 export function toStepRecord(row: StepRow): StepRecord {
   const completedAt = optionalNum(row.completed_at);
   const wakeAt = optionalNum(row.wake_at);

package/src/driver-pg-settle-sql.ts

@@ -0,0 +1,71 @@
+// The two statements that SETTLE a claimed row, each also adding the settle to its job's
+// one-minute counter bucket. One statement, not two: a count that can disagree with the rows is
+// worse than no count, and a second round trip per job is a cost every job would pay. Split off
+// `driver-pg-sql.ts` at its size ceiling and re-exported from it.
+
+import { COUNTER_BUCKET_MS } from './introspection';
+
+/** `x_job_counters`'s conflict target and the bucket every settle writes — spelled once. */
+const BUCKET = `${COUNTER_BUCKET_MS}, date_trunc('minute', now())`;
+
+/**
+ * `and state = 'running'`, `and claimed_by = $2` and `and claims = $4` are all FENCES, not
+ * filters. Without the first, an ack from a worker that was cancelled overwrites the
+ * cancellation. Without the second, an ack from a worker whose lease lapsed marks `done` a run
+ * ANOTHER worker has since claimed and is still executing — the row is `running` again, so the
+ * state fence alone lets it through. Without the third, the same thing happens when the worker
+ * that claimed it again is the SAME one: its id did not change, only the claim's ordinal did.
+ * `returning` is how the caller learns whether its settle landed.
+ *
+ * $1 id, $2 worker, $3 the attempt's duration in ms, $4 the claim, $5 whether it is counted — a
+ * row `x jobs drain` moved is `done` here and is not a completed run.
+ */
+export const SQL_ACK = `
+with settled as (
+  update x_jobs
+     set state = 'done', visible_at = null, claimed_by = null, updated_at = now()
+   where id = $1 and state = 'running' and claimed_by = $2 and claims = $4::int
+  returning name
+), counted as (
+  insert into x_job_counters (job, bucket_ms, bucket_start, done, duration_ms)
+  select name, ${BUCKET}, 1, $3::bigint from settled where $5::boolean
+  on conflict (job, bucket_ms, bucket_start) do update
+     set done        = x_job_counters.done + 1,
+         duration_ms = x_job_counters.duration_ms + excluded.duration_ms
+)
+select name from settled
+`.trim();
+
+/**
+ * The same three fences. The counter row is written only when one of the three counts is set: a
+ * shed, a suspension and a drained attempt are handed back UNCOUNTED and are not history.
+ *
+ * $1 id, $2 state, $3 counts-as-attempt, $4 delay ms, $5 error, $6 worker, $7 stack,
+ * $8 retried, $9 failed, $10 dead (each 0 or 1), $11 the attempt's duration in ms, $12 the claim.
+ */
+export const SQL_NACK = `
+with settled as (
+  update x_jobs
+     set state      = $2,
+         attempt    = case when $3::boolean then attempt else greatest(attempt - 1, 0) end,
+         run_at     = now() + ($4::bigint * interval '1 millisecond'),
+         visible_at = null,
+         claimed_by = null,
+         last_error = coalesce($5::text, last_error),
+         last_error_stack = case when $5::text is null then last_error_stack else $7::text end,
+         updated_at = now()
+   where id = $1 and state = 'running' and claimed_by = $6 and claims = $12::int
+  returning name
+), counted as (
+  insert into x_job_counters (job, bucket_ms, bucket_start, retried, failed, dead, duration_ms)
+  select name, ${BUCKET}, $8::int, $9::int, $10::int, $11::bigint
+    from settled
+   where $8::int + $9::int + $10::int > 0
+  on conflict (job, bucket_ms, bucket_start) do update
+     set retried     = x_job_counters.retried + excluded.retried,
+         failed      = x_job_counters.failed + excluded.failed,
+         dead        = x_job_counters.dead + excluded.dead,
+         duration_ms = x_job_counters.duration_ms + excluded.duration_ms
+)
+select name from settled
+`.trim();