@ultimat3/jobs 1.2.0 → 2.0.0

package/src/worker-fleet-slots.ts

@@ -0,0 +1,124 @@
+// The fleet slot an in-flight job holds: `job.concurrency` as a row per HELD SLOT in the driver's
+// lease store, taken at claim time, renewed while the job runs and handed back when it settles.
+// Apart from `worker.ts` because the claim loop's question is "may I start this one?" — which job
+// holds which slot, and who gives it back, is bookkeeping of its own.
+
+import { logger } from '@ultimat3/core';
+import type { ClaimedJob } from './driver';
+import { getJob } from './job';
+import type { HeldLease, LeaseStore } from './leases';
+import { jobLeaseKey } from './leases';
+
+/**
+ * A renewal that REJECTED is not a lost slot: there is a TTL behind it and the interval gets
+ * several tries inside it, exactly as `heartbeat.ts` treats a failed `driver.heartbeat`. The
+ * heartbeat cannot cover for this one either way — it renews `x_jobs.visible_at`, a different row
+ * on a different clock, and knows nothing about `x_job_leases`.
+ */
+const noop = (): void => undefined;
+
+export interface FleetSlotOptions {
+  /** The driver's lease store, or `undefined` for a driver that ships none. */
+  readonly leases: LeaseStore | undefined;
+  readonly workerId: string;
+  /**
+   * How long a fleet slot survives without renewal. The worker passes its visibility timeout, so
+   * a worker that is SIGKILLed gives its slot back on exactly the schedule the queue gives its job
+   * back — a longer TTL would leave `concurrency: 1` unfillable while the job it guarded is
+   * already re-delivered.
+   */
+  readonly ttlMs: number;
+  readonly renewIntervalMs: number;
+}
+
+export interface FleetSlots {
+  /**
+   * A fleet slot for this job's declared `concurrency`, or `false` when the fleet is full.
+   * `true` means "no cap declared" — a job with no `concurrency` never touches the lease table.
+   *
+   * A driver with no lease store cannot reach here: `createWorker().start()` refuses to boot when
+   * a registered job declares `concurrency` and the driver has none, because a cap that silently
+   * holds per process is the documented-guarantee-that-does-nothing axiom 3 exists to make
+   * impossible.
+   */
+  acquire(claimed: ClaimedJob): Promise<boolean>;
+  /**
+   * Keeps this job's slot alive until the returned stop is called. A no-op when it holds none.
+   *
+   * `onLost` fires once, when a renewal comes back `false` — this worker no longer holds the slot,
+   * either because another holder took it (this run and that one are both live under a cap of one)
+   * or because it lapsed and is now free for anyone's next `acquire`. Both drivers answer `false`
+   * to both cases; the pg statement fenced only on the holder until 2026-08, so a lapsed slot
+   * revived itself there and cancelled the run under `x dev`. The caller cancels the run on it;
+   * renewal stops here either way, because extending a slot this worker no longer holds would push
+   * out somebody else's expiry.
+   */
+  startRenewal(jobId: string, onLost?: (slot: HeldLease) => void): () => void;
+  release(jobId: string): Promise<void>;
+}
+
+export function createFleetSlots(options: FleetSlotOptions): FleetSlots {
+  /** The fleet slot each in-flight job holds, so the renewal finds it and the drain frees it. */
+  const held = new Map<string, HeldLease>();
+
+  return {
+    async acquire(claimed) {
+      const limit = getJob(claimed.name)?.concurrency;
+      if (limit === undefined || options.leases === undefined) return true;
+      const slot = await options.leases.acquire(
+        jobLeaseKey(claimed.name),
+        limit,
+        options.ttlMs,
+        `${options.workerId}:${claimed.id}`,
+      );
+      if (slot === undefined) return false;
+      held.set(claimed.id, slot);
+      return true;
+    },
+
+    startRenewal(jobId, onLost) {
+      const slot = held.get(jobId);
+      if (slot === undefined) return noop;
+      const stop = (): void => {
+        clearInterval(timer);
+      };
+      // Renewed on the lease heartbeat's own interval and released in the same `finally`: one
+      // clock for "this worker still owns the job" and "this worker still owns the slot" is one
+      // fewer way for them to disagree.
+      const timer = setInterval(() => {
+        void options.leases
+          ?.renew(slot, options.ttlMs)
+          .then((renewed) => {
+            // `=== false`, never `!renewed`, for the reason `heartbeat.ts` reads `held` that way:
+            // a store written before this return value existed resolves `undefined`, and treating
+            // that as a loss would cancel every job on every renewal. Only an explicit no is one.
+            if (renewed !== false) return;
+            stop();
+            logger.error('jobs.worker.slot-lost', {
+              workerId: options.workerId,
+              jobId,
+              leaseKey: slot.key,
+              slot: slot.slot,
+            });
+            onLost?.(slot);
+          })
+          .catch(noop);
+      }, options.renewIntervalMs);
+      return stop;
+    },
+
+    async release(jobId) {
+      const slot = held.get(jobId);
+      if (slot === undefined) return;
+      held.delete(jobId);
+      // Never lets a settle fail over bookkeeping: an unreleased slot expires on its own TTL.
+      await options.leases?.release(slot).catch((error: unknown) => {
+        logger.warn('jobs.worker.lease-release-failed', {
+          workerId: options.workerId,
+          jobId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      });
+    },
+  };
+}

package/src/worker-run.ts

@@ -0,0 +1,132 @@
+// One claimed job, wired: its lease heartbeat, the fleet slot it holds, the signal the run is
+// cancelled by, and the span it is executed under — all started together and handed back in one
+// `finally`. Apart from `worker.ts` because that file's job is the claim loop and the drain; this
+// one is everything that has to be true for the duration of a single run.
+
+import type { Clock, Ctx } from '@ultimat3/core';
+import { parseTraceparent, withSpan } from '@ultimat3/core';
+import type { ClaimedJob, JobDriver } from './driver';
+import { JobSlotLostError } from './errors';
+import type { JobExecution } from './execute';
+import { executeJob } from './execute';
+import { startLeaseHeartbeat } from './heartbeat';
+import { getJob } from './job';
+import { createRunSignal } from './run-signal';
+import type { EventLookup } from './steps';
+import type { FleetSlots } from './worker-fleet-slots';
+
+export interface RunClaimedOptions {
+  readonly driver: JobDriver;
+  readonly claimed: ClaimedJob;
+  /** Supplies the ambient Ctx for this run; the app wires ALS + tenant here. */
+  readonly context: () => Ctx;
+  readonly fleetSlots: FleetSlots;
+  readonly workerId: string;
+  readonly visibilityTimeoutMs: number;
+  readonly heartbeatIntervalMs: number;
+  readonly clock?: Clock;
+  readonly events?: EventLookup;
+}
+
+/** A name this deploy does not know, parked rather than failed — almost always a deploy skew. */
+function unknownJob(claimed: ClaimedJob): JobExecution {
+  return {
+    outcome: 'suspended',
+    jobId: claimed.id,
+    job: claimed.name,
+    attempt: claimed.attempt,
+    durationMs: 0,
+    error: `no job registered as "${claimed.name}"`,
+    steps: [],
+    replayed: [],
+  };
+}
+
+/** Run one claimed job under its lease, its slot and its span, and settle it with the driver. */
+export async function runClaimedJob(options: RunClaimedOptions): Promise<JobExecution> {
+  const { claimed, driver, fleetSlots, workerId, visibilityTimeoutMs } = options;
+  const handle = getJob(claimed.name);
+  if (handle === undefined) {
+    // Park it, do not burn attempts: the job may well be registered by the pod next to this one.
+    await driver.nack(claimed.id, {
+      delayMs: 30_000,
+      error: `no job registered as "${claimed.name}"`,
+      countsAsAttempt: false,
+    });
+    return unknownJob(claimed);
+  }
+
+  // Read BEFORE the timers start. `context()` is the app's own function and it can throw; started
+  // first, a heartbeat interval and a slot renewal were left running for a job that never ran,
+  // with nothing left holding a reference to stop them.
+  const base = options.context();
+
+  // The lease, kept alive and NOT kept quiet: a renewal that stops landing means the queue hands
+  // this job to another worker while this one is still running it, and `.catch(() => undefined)`
+  // made that — the one failure a queue cannot recover from on its own — indistinguishable from a
+  // healthy run.
+  const heartbeat = startLeaseHeartbeat({
+    driver,
+    claimed,
+    visibilityTimeoutMs,
+    intervalMs: options.heartbeatIntervalMs,
+    workerId,
+    ...(options.clock === undefined ? {} : { clock: options.clock }),
+  });
+
+  // The caller's context plus this lease's cancellation, so a job cancelled from outside — or one
+  // this worker lost the lease on — stops at the next renewal. `steps.ts` refuses every write past
+  // the signal, which is what unwinds a body that never reads it. Composed through a controller
+  // this worker owns rather than `AbortSignal.any`, for two reasons: it is handed BACK when the run
+  // settles (an app whose `context()` carries a process-lifetime signal was accumulating one
+  // composite per job), and the worker can abort it itself — which is the only way a fleet slot
+  // taken by somebody else reaches the body running under it.
+  const runSignal = createRunSignal([base.signal, heartbeat.signal]);
+  const ctx: Ctx = { ...base, signal: runSignal.signal };
+
+  // The fleet slot this job already holds, kept alive for as long as the lease is and stopped in
+  // the same `finally`: one clock for "this worker still owns the job" and "this worker still owns
+  // the slot" is one fewer way for them to disagree. A renewal answering "not yours" means another
+  // worker is already running this job under a cap of one, so it CANCELS — discarding that boolean
+  // made `job.concurrency` a number the framework prints and does not hold.
+  const stopSlotRenewal = fleetSlots.startRenewal(claimed.id, (slot) => {
+    runSignal.abort(
+      new JobSlotLostError({ job: claimed.name, jobId: claimed.id, slot: slot.slot }),
+    );
+  });
+
+  // The job's span is a CHILD of the request that queued it when the row carries a trace. That
+  // link is what `04-jobs.md` promised and no column existed to hold: without it a checkout's
+  // `chargeCard` opens a fresh root two seconds later with nothing pointing back.
+  const parent = parseTraceparent(claimed.traceparent);
+
+  try {
+    return await withSpan(
+      `job.${handle.name}`,
+      () =>
+        executeJob({
+          driver,
+          claimed,
+          handle,
+          ctx,
+          ...(options.clock === undefined ? {} : { clock: options.clock }),
+          ...(options.events === undefined ? {} : { events: options.events }),
+        }),
+      {
+        ...(parent === undefined ? {} : { parent }),
+        attributes: {
+          'job.name': handle.name,
+          'job.id': claimed.id,
+          'job.attempt': claimed.attempt,
+          ...(claimed.enqueuedBy === undefined ? {} : { 'job.enqueued_by': claimed.enqueuedBy }),
+        },
+      },
+    );
+  } finally {
+    stopSlotRenewal();
+    heartbeat.stop();
+    // Nothing of the caller's is held past the run: `dispose` is what makes the composition above
+    // reversible, and it is the whole reason this is not `AbortSignal.any`.
+    runSignal.dispose();
+  }
+}