@12-apps/jobs 2.0.0 → 4.0.0

package/src/drivers/bullmq.ts

@@ -7,10 +7,10 @@
  * never the ledger. Payloads carry ids (see `JobDefinition`'s payload rule),
  * every handler re-reads its row, and every job is paired with a durable table
  * a sweep can re-derive the work from. Losing Redis therefore costs a delayed
- * run — not a lost charge, and not a double one.
+ * run — not a lost write, and not a double one.
  *
  * That split is what makes the enqueue-outside-the-transaction problem
- * survivable. A Prisma commit followed by an enqueue is not atomic: crash in
+ * survivable. A database commit followed by an enqueue is not atomic: crash in
  * between and the job never lands. The paired sweep is the answer — the row is
  * committed, so the next tick finds it.
  *
@@ -25,32 +25,29 @@
 
 import { Queue, UnrecoverableError, Worker, type JobsOptions } from "bullmq";
 
+import { createEventEmitter, type EmitJobEvent } from "../core/events";
+import { DEFAULT_QUEUE } from "../core/queues";
+import { resolveRegisteredJob } from "../core/registry";
 import type {
   AnyJobDefinition,
   EnqueueOptions,
   EnqueueResult,
   JobContext,
   JobDriver,
+  JobEvents,
   JobLogger,
+  JobRetention,
 } from "../core/types";
 
+import {
+  DEFAULT_CONCURRENCY,
+  DEFAULT_JOB_RETENTION,
+  isTerminalFailure,
+  resolveConcurrency,
+  retentionOptions,
+} from "./bullmq-policy";
 import { parseRedisUrl, type RedisConnectionOptions } from "./redis-url";
 
-/** The queue a definition lands on when it does not name one. */
-const DEFAULT_QUEUE = "default";
-/** Worker concurrency when no definition in the queue asks for more. */
-const DEFAULT_CONCURRENCY = 5;
-
-/**
- * Retention. Bounded on purpose: an unbounded completed-set is the classic way
- * a small Redis fills up and starts refusing writes. A day of successes is
- * enough to answer "did it run?"; a week of failures is enough to debug one.
- */
-const RETENTION: Pick<JobsOptions, "removeOnComplete" | "removeOnFail"> = {
-  removeOnComplete: { age: 24 * 3600, count: 1_000 },
-  removeOnFail: { age: 7 * 24 * 3600, count: 5_000 },
-};
-
 /** The one ioredis command this driver reads outside BullMQ's own surface. */
 interface RedisConfigReader {
   config(command: "GET", parameter: string): Promise<unknown>;
@@ -65,6 +62,12 @@ export interface BullMqJobDriverOptions {
    * worker consuming production's jobs.
    */
   prefix?: string;
+  /** Defaults to {@link DEFAULT_JOB_RETENTION}. */
+  retention?: JobRetention;
+  /** Per-queue concurrency when no definition on the queue states one. */
+  defaultConcurrency?: number;
+  /** Where completions, dead-letters and removed schedules are reported. */
+  events?: JobEvents;
 }
 
 /** Everything the driver's helpers need, threaded instead of closed over. */
@@ -72,18 +75,23 @@ interface DriverState {
   connection: RedisConnectionOptions;
   logger: JobLogger;
   prefix?: string;
+  retention: Pick<JobsOptions, "removeOnComplete" | "removeOnFail">;
+  defaultConcurrency: number;
+  /** Reports to the host's observer; see `core/events`. Never throws. */
+  emit: EmitJobEvent;
   queues: Map<string, Queue>;
   workers: Worker[];
 }
 
 /** Turn a definition's retry policy into BullMQ's per-job options. */
 function jobOptionsFor(
+  state: DriverState,
   definition: AnyJobDefinition,
   enqueueOptions: EnqueueOptions,
 ): JobsOptions {
   const options: JobsOptions = {
     attempts: Math.max(1, definition.attempts ?? 1),
-    ...RETENTION,
+    ...state.retention,
   };
   if (definition.backoff) {
     options.backoff = {
@@ -108,7 +116,7 @@ function queueFor(state: DriverState, name: string): Queue {
   const queue = new Queue(name, {
     connection: state.connection,
     ...(state.prefix ? { prefix: state.prefix } : {}),
-    defaultJobOptions: RETENTION,
+    defaultJobOptions: state.retention,
   });
   // Without a listener an emitted 'error' is an unhandled exception that takes
   // the process down — a Redis blip must not kill the web server.
@@ -165,7 +173,7 @@ async function reconcileSchedules(
     await queue.upsertJobScheduler(
       name,
       { pattern: schedule.pattern, tz: schedule.timezone ?? "UTC" },
-      { name, data: {}, opts: jobOptionsFor(job, {}) },
+      { name, data: {}, opts: jobOptionsFor(state, job, {}) },
     );
     state.logger.info(
       `schedule installed: ${name} (${schedule.pattern} ${schedule.timezone ?? "UTC"})`,
@@ -178,41 +186,37 @@ async function reconcileSchedules(
     state.logger.warn(
       `removed stale schedule "${scheduler.key}" (no longer defined in code).`,
     );
+    // The destructive half of the reconcile, and the one worth auditing: a
+    // deploy just cancelled a recurring job, permanently, from the queue's
+    // point of view.
+    state.emit((events) =>
+      events.onScheduleRemoved?.({ name: scheduler.key, queue: queue.name }),
+    );
   }
 }
 
-/**
- * A queue's worker concurrency.
- *
- * The default applies only when NO definition on the queue asked for a
- * specific number. A stated `concurrency: 1` means single-flight and must be
- * honoured — taking `max(default, …)` would quietly raise it back to the
- * default and undo the very property the caller asked for.
- */
-function resolveConcurrency(group: readonly AnyJobDefinition[]): number {
-  const stated = group
-    .map((definition) => definition.concurrency)
-    .filter((value): value is number => typeof value === "number" && value > 0);
-  return stated.length > 0 ? Math.max(...stated) : DEFAULT_CONCURRENCY;
-}
-
 /** Consume one queue, dispatching by job name to the definitions it carries. */
 function startWorker(
   state: DriverState,
   queueName: string,
   group: readonly AnyJobDefinition[],
 ): Worker {
-  const byName = new Map(group.map((definition) => [definition.name, definition]));
-  const concurrency = resolveConcurrency(group);
+  const onThisQueue = new Set(group.map((definition) => definition.name));
+  const concurrency = resolveConcurrency(group, state.defaultConcurrency);
 
   const worker = new Worker(
     queueName,
     async (job) => {
-      const definition = byName.get(job.name);
+      // THE GATE, execution side — `resolveRegisteredJob` is the same function
+      // the enqueue path calls, so the two cannot come to disagree about what
+      // a runnable job is. The queue membership check on top of it keeps a job
+      // from being run by another queue's worker.
+      const registered = resolveRegisteredJob(job.name);
+      const definition = registered && onThisQueue.has(job.name) ? registered : undefined;
       if (!definition) {
-        // A job left in Redis by a previous deploy whose handler is gone.
-        // Retrying cannot help, so fail it terminally instead of burning every
-        // attempt on it.
+        // A job left in the backend by a previous deploy whose handler is
+        // gone. Retrying cannot help, so fail it terminally instead of burning
+        // every attempt on it.
         throw new UnrecoverableError(`No handler registered for job "${job.name}".`);
       }
       const context: JobContext = {
@@ -222,6 +226,15 @@ function startWorker(
         logger: state.logger,
       };
       await definition.handle(job.data as never, context);
+      state.emit((events) =>
+        events.onJobCompleted?.({
+          name: job.name,
+          queue: queueName,
+          runId: context.runId,
+          attempt: context.attempt,
+          maxAttempts: context.maxAttempts,
+        }),
+      );
     },
     {
       connection: state.connection,
@@ -231,11 +244,25 @@ function startWorker(
   );
 
   worker.on("failed", (job, error) => {
-    const attempts = job ? `${job.attemptsMade}/${job.opts.attempts ?? 1}` : "?";
+    const maxAttempts = job?.opts.attempts ?? 1;
+    const attempts = job ? `${job.attemptsMade}/${maxAttempts}` : "?";
     state.logger.error(
       `job "${job?.name ?? queueName}" failed (attempt ${attempts}):`,
       error,
     );
+    if (!job) return;
+    const terminal = isTerminalFailure(job.attemptsMade, maxAttempts, error);
+    state.emit((events) =>
+      events.onJobFailed?.({
+        name: job.name,
+        queue: queueName,
+        runId: job.id ?? `${job.name}:unknown`,
+        attempt: job.attemptsMade,
+        maxAttempts,
+        error,
+        terminal,
+      }),
+    );
   });
   worker.on("error", (error) =>
     state.logger.error(`worker "${queueName}" error:`, error),
@@ -262,6 +289,12 @@ export function createBullMqJobDriver(options: BullMqJobDriverOptions): JobDrive
     connection: parseRedisUrl(options.redisUrl),
     logger: options.logger,
     prefix: options.prefix,
+    retention: retentionOptions(options.retention ?? DEFAULT_JOB_RETENTION),
+    defaultConcurrency:
+      typeof options.defaultConcurrency === "number" && options.defaultConcurrency > 0
+        ? options.defaultConcurrency
+        : DEFAULT_CONCURRENCY,
+    emit: createEventEmitter(options.events, options.logger),
     queues: new Map(),
     workers: [],
   };
@@ -270,8 +303,22 @@ export function createBullMqJobDriver(options: BullMqJobDriverOptions): JobDrive
     kind: "bullmq",
 
     async enqueue(definition, payload, enqueueOptions): Promise<EnqueueResult> {
+      // The same gate as the worker's dispatch, at the other end of the wire:
+      // a name this deployment cannot run must not be written to Redis, where
+      // it would sit until a consumer dead-lettered it. Reached directly only
+      // by a host that built the driver itself — `enqueueJob` refuses first.
+      if (!resolveRegisteredJob(definition)) {
+        state.logger.error(
+          `refused to enqueue "${definition.name}": it is not the job registered under that name.`,
+        );
+        return { enqueued: false, reason: "unregistered" };
+      }
       const queue = queueFor(state, definition.queue ?? DEFAULT_QUEUE);
-      await queue.add(definition.name, payload, jobOptionsFor(definition, enqueueOptions));
+      await queue.add(
+        definition.name,
+        payload,
+        jobOptionsFor(state, definition, enqueueOptions),
+      );
       return { enqueued: true };
     },
 
@@ -296,7 +343,15 @@ export function createBullMqJobDriver(options: BullMqJobDriverOptions): JobDrive
 }
 
 /**
- * Internals exposed for tests only — the concurrency rule is silent when
- * wrong, so it is pinned directly rather than inferred from a live Worker.
+ * The policy rules, re-exposed for tests through the driver's own subpath.
+ *
+ * Both are SILENT when wrong — a queue that quietly runs at the default
+ * instead of single-flight, and a dead-letter that quietly reports itself as
+ * one more retry — so they are pinned directly rather than inferred from a
+ * live Worker, which would need a Redis to exist.
  */
-export const __testables = { resolveConcurrency, DEFAULT_CONCURRENCY };
+export const __testables = {
+  resolveConcurrency,
+  isTerminalFailure,
+  DEFAULT_CONCURRENCY,
+};

package/src/drivers/inline.ts

@@ -1,20 +1,26 @@
 /**
  * The INLINE driver: run the handler in the calling process, immediately.
  *
- * For unit and integration tests (which boot PGlite and must not need a Redis
- * container) and for local development without `docker compose up`. It is a
- * real driver, not a stub — `attempts` are honoured so retry behaviour stays
- * testable — but three things it deliberately does NOT do:
+ * For unit and integration tests (which must not need a queue backend running)
+ * and for local development with no container. It is a real driver, not a stub
+ * — `attempts` are honoured so retry behaviour stays testable — but three
+ * things it deliberately does NOT do:
  *
  *   - **No delay.** `delayMs` is ignored and logged. A test asserting a
- *     three-day dunning gap should assert on the enqueue, not the clock.
+ *     three-day gap should assert on the enqueue, not on the clock.
  *   - **No schedules.** A cron job is registered but never fires; drive it
  *     directly in a test by calling the handler.
  *   - **No backoff waits.** Retries happen back-to-back so suites stay fast.
  *
  * Which is why it must never reach production: work would run inside the
- * request that scheduled it, and a crash between the two would lose it. The
- * host's driver factory is responsible for refusing it there.
+ * request that scheduled it, and a crash between the two would lose it.
+ * `createApiJobs` refuses it there, by name and by instance.
+ *
+ * It also does no registry lookup, so it has no write/run gate of its own to
+ * keep in step: it runs the definition OBJECT it was handed, not a name it
+ * resolves. `enqueueJob` applies the registry gate before the driver is
+ * reached; a test calling `driver.enqueue(definition, …)` directly is
+ * deliberately below that line.
  */
 
 import type {
@@ -23,9 +29,13 @@ import type {
   EnqueueResult,
   JobContext,
   JobDriver,
+  JobEvents,
   JobLogger,
 } from "../core/types";
 
+import { createEventEmitter, type EmitJobEvent } from "../core/events";
+import { DEFAULT_QUEUE } from "../core/queues";
+
 export interface InlineJobDriverOptions {
   logger?: JobLogger;
   /**
@@ -34,6 +44,12 @@ export interface InlineJobDriverOptions {
    * sees the side effect.
    */
   await?: boolean;
+  /**
+   * Where completions and dead-letters are reported. The same port the BullMQ
+   * driver takes, so a host's observer is exercised by its test suite instead
+   * of only in production.
+   */
+  events?: JobEvents;
 }
 
 /** A record of what ran, for assertions. */
@@ -51,45 +67,64 @@ export interface InlineJobDriver extends JobDriver {
   clearRuns(): void;
 }
 
-export function createInlineJobDriver(
-  options: InlineJobDriverOptions = {},
-): InlineJobDriver {
-  const logger = options.logger ?? console;
-  const awaitHandlers = options.await ?? true;
-  const runs: InlineJobRun[] = [];
+/** What one inline execution needs from the driver that owns it. */
+interface InlineRunner {
+  logger: JobLogger;
+  runs: InlineJobRun[];
+  emit: EmitJobEvent;
+}
 
-  async function run(
-    definition: AnyJobDefinition,
-    payload: unknown,
-  ): Promise<void> {
-    const maxAttempts = Math.max(1, definition.attempts ?? 1);
-    const record: InlineJobRun = { name: definition.name, payload, attempts: 0 };
-    runs.push(record);
+/**
+ * Run one job to its conclusion, retrying up to `attempts` with no waits.
+ *
+ * Hoisted out of the factory so the factory stays a wiring function: the
+ * attempt loop is the part with the rules in it, and it reads better with its
+ * inputs named than closed over.
+ */
+async function runInline(
+  runner: InlineRunner,
+  definition: AnyJobDefinition,
+  payload: unknown,
+): Promise<void> {
+  const { logger, runs, emit } = runner;
+  const maxAttempts = Math.max(1, definition.attempts ?? 1);
+  const queue = definition.queue ?? DEFAULT_QUEUE;
+  const record: InlineJobRun = { name: definition.name, payload, attempts: 0 };
+  runs.push(record);
 
-    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
-      record.attempts = attempt;
-      const context: JobContext = {
-        runId: `inline:${definition.name}:${runs.length}:${attempt}`,
-        attempt,
-        maxAttempts,
-        logger,
-      };
-      try {
-        await definition.handle(payload as never, context);
-        record.error = undefined;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    record.attempts = attempt;
+    const runId = `inline:${definition.name}:${runs.length}:${attempt}`;
+    const context: JobContext = { runId, attempt, maxAttempts, logger };
+    const event = { name: definition.name, queue, runId, attempt, maxAttempts };
+    try {
+      await definition.handle(payload as never, context);
+      record.error = undefined;
+      emit((events) => events.onJobCompleted?.(event));
+      return;
+    } catch (error) {
+      record.error = error;
+      const terminal = attempt === maxAttempts;
+      emit((events) => events.onJobFailed?.({ ...event, error, terminal }));
+      if (terminal) {
+        logger.error(`"${definition.name}" failed after ${attempt} attempt(s):`, error);
         return;
-      } catch (error) {
-        record.error = error;
-        if (attempt === maxAttempts) {
-          logger.error(
-            `"${definition.name}" failed after ${attempt} attempt(s):`,
-            error,
-          );
-          return;
-        }
       }
     }
   }
+}
+
+export function createInlineJobDriver(
+  options: InlineJobDriverOptions = {},
+): InlineJobDriver {
+  const logger = options.logger ?? console;
+  const awaitHandlers = options.await ?? true;
+  const runs: InlineJobRun[] = [];
+  const runner: InlineRunner = {
+    logger,
+    runs,
+    emit: createEventEmitter(options.events, logger),
+  };
 
   return {
     kind: "inline",
@@ -107,7 +142,7 @@ export function createInlineJobDriver(
           `inline driver ignored a ${enqueueOptions.delayMs}ms delay on "${definition.name}".`,
         );
       }
-      const execution = run(definition, payload);
+      const execution = runInline(runner, definition, payload);
       if (awaitHandlers) await execution;
       else void execution;
       return { enqueued: true };

package/src/hono/index.ts

@@ -21,9 +21,10 @@ import type { JobsRoute } from "../server/create-api-jobs";
  * must call `start()` on the same instance whose routes report health, and a
  * router that built its own would answer for a runtime nobody started.
  *
- * Auth stays the host's: mount this under whatever guard the deployment's
- * internal probes live behind (future-pay's `/api/internal/*` is answered
- * only machine-to-machine; the reverse proxy 404s it from the internet).
+ * Auth stays the host's: mount it under whatever guard the deployment's
+ * internal probes live behind — a prefix answered only machine-to-machine,
+ * with the reverse proxy refusing it from the internet, is the usual shape.
+ * This package holds zero authorization logic.
  */
 export function jobsRouter(api: { routes: JobsRoute[] }): Hono {
   const app = new Hono();

package/src/index.ts

@@ -1,17 +1,17 @@
 /**
- * `@12-apps/jobs` — typed background jobs with retries, backoff and cron, behind
- * a swappable driver.
+ * `@12-apps/jobs` — typed background jobs with retries, backoff and cron,
+ * behind a swappable driver.
  *
  *     // where the domain lives
- *     export const dispatchNotification = defineJob<{ notificationId: string }>({
- *       name: "notifications.dispatch",
+ *     export const renderReport = defineJob<{ reportId: string }>({
+ *       name: "reports.render",
  *       attempts: 5,
  *       backoff: { type: "exponential", delayMs: 5_000 },
- *       handle: async ({ notificationId }) => dispatchDeliveries(notificationId),
+ *       handle: async ({ reportId }) => renderAndStore(reportId),
  *     });
  *
  *     // at an emit site
- *     await dispatchNotification.enqueue({ notificationId }, { dedupeKey: notificationId });
+ *     await renderReport.enqueue({ reportId }, { dedupeKey: reportId });
  *
  *     // at process start
  *     configureJobs({ driver: createBullMqJobDriver({ redisUrl, logger }), logger });
@@ -30,14 +30,29 @@
  * `createApiJobs` in `@12-apps/jobs/server` (mount it with
  * `@12-apps/jobs/hono` or your own adapter). What this root adds to it:
  *
- *   - `SWEEP_QUEUE` — the single-flight queue the scheduled sweeps share.
+ *   - `DEFAULT_QUEUE` / `SWEEP_QUEUE` — this package's own queue vocabulary,
+ *     `SWEEP_QUEUE` being the single-flight queue the scheduled sweeps share.
  *   - `createSweepLease` — the named, time-bounded claim that keeps a sweep
  *     to ONE pass per tick across a multi-worker deployment. Its `SweepLease`
  *     table ships in `prisma/jobs.prisma` with its migration; the host syncs
  *     both (see ADOPTING.md).
+ *
+ * Every guard this package has lives on THIS path as well as on the factory:
+ * `defineJob` refuses a definition that cannot run, `startJobWorkers` refuses
+ * an empty registry, and `enqueueJob` refuses a job the registry does not
+ * hold. A host that wires the runtime by hand is not a host with fewer checks.
  */
 
-export { defineJob, findJob, listJobs, clearJobs, DuplicateJobError } from "./core/registry";
+export {
+  defineJob,
+  findJob,
+  listJobs,
+  clearJobs,
+  resolveRegisteredJob,
+  DuplicateJobError,
+  InvalidJobDefinitionError,
+  NoJobsRegisteredError,
+} from "./core/registry";
 export type { RegisteredJob } from "./core/registry";
 
 export {
@@ -56,21 +71,32 @@ export type {
   EnqueueResult,
   EnqueueSkipReason,
   JobBackoff,
+  JobCompletedEvent,
   JobContext,
   JobDefinition,
   JobDriver,
+  JobEvents,
+  JobFailedEvent,
   JobHandler,
   JobLogger,
+  JobRetention,
+  JobRetentionWindow,
+  JobRunEvent,
   JobSchedule,
+  ScheduleRemovedEvent,
 } from "./core/types";
 
+// Retention validation lives in `core` so this barrel can carry it without
+// pulling `bullmq` (and ioredis) into a bundle that only ever enqueues.
+export { assertValidRetention, InvalidJobRetentionError } from "./core/retention";
+
 export { createInlineJobDriver } from "./drivers/inline";
 export type { InlineJobDriver, InlineJobRun } from "./drivers/inline";
 
 export { parseRedisUrl, InvalidRedisUrlError } from "./drivers/redis-url";
 export type { RedisConnectionOptions } from "./drivers/redis-url";
 
-export { SWEEP_QUEUE } from "./core/queues";
+export { DEFAULT_QUEUE, SWEEP_QUEUE } from "./core/queues";
 
 export { createSweepLease } from "./lease/sweep-lease";
 export type {