@zerotal/queue 1.3.0 → 1.5.0

package/CHANGELOG.md

@@ -8,6 +8,52 @@ follows the Zerotal monorepo's unified versioning.
 
 ## [Unreleased]
 
+## [1.5.0] — 2026-08-15
+
+### Added
+
+- **Debounced jobs — `debounce` on a `Job` collapses repeated dispatches into one run.**
+  A document saved eight times in a minute rebuilt its search index eight times, seven
+  of them wasted, and the eighth was the only one whose result anybody saw.
+
+  It is a **trailing** debounce and the name is accurate: each dispatch pushes the run
+  further out, and the job runs once, after the dispatches stop. The surviving job
+  carries the **newest** payload, because the premise is that the earlier ones are stale.
+
+  The default key is the class name plus the serialised payload, so `ReindexDocument(1)`
+  and `ReindexDocument(2)` never collapse into each other and the common case needs no
+  configuration. Override `debounceKey()` when two payloads mean the same work — a job
+  carrying a timestamp is unique on every dispatch and would otherwise collapse with
+  nothing.
+
+  **The key lives in the queue's backing store, so it is stable across processes.**
+  Collapsing has to be atomic or two processes dispatching at the same instant both find
+  nothing pending and both enqueue, which is the failure the feature exists to prevent:
+  `sqlite` does it as one `INSERT … ON CONFLICT` against a partial unique index covering
+  only unreserved rows, `redis` as one `EVAL`. A driver that cannot promise atomicity
+  throws `E_QUEUE_DEBOUNCE_UNSUPPORTED` rather than degrading to a per-process debounce —
+  which would appear to work in development and do nothing in production.
+
+  A job a worker has already claimed is never collapsed into: it is running, so the next
+  dispatch is genuinely new work. `sync` ignores `debounce`, because every job runs inline
+  and there is no window for a second dispatch to arrive in.
+
+### Added
+
+- **The worker runs under `bun zt dev`, in its own tab.** An app with a queue needed
+  a second terminal and the discipline to restart the right one after a change; now
+  `QueueProvider` registers `queue:work` as a dev process and the runner supervises
+  it beside the server, restartable on its own with `r`.
+
+  It declines to appear when it would have nothing to do — under the `sync` driver,
+  where every dispatched job runs inline on the request, and when `queue.workers` is
+  set, where the server's own thread pool is already draining the queue. A tab that
+  polls an empty queue forever is worse than no tab, because it looks like it is
+  working.
+
+  Drop it with `app.dev.disable: ["queue"]`, or replace it by registering `queue`
+  yourself in `app.dev.processes`.
+
 ## [1.0.3] — 2026-08-07
 
 ### Changed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zerotal/queue",
-  "version": "1.3.0",
+  "version": "1.5.0",
   "license": "MIT",
   "maturity": "stable",
   "private": false,
@@ -30,8 +30,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@zerotal/core": "1.3.0",
-    "@zerotal/orm": "1.3.0"
+    "@zerotal/core": "1.5.0",
+    "@zerotal/orm": "1.5.0"
   },
   "devDependencies": {
     "typescript": "^5.8.0"

package/src/Job.ts

@@ -16,6 +16,59 @@ export abstract class Job {
   /** Set by Bus.chain() — remaining jobs to dispatch after this one succeeds. */
   _chain: SerializedJob[] | undefined = undefined;
 
+  /**
+   * Collapse repeated dispatches into a single run, `debounce` **seconds** after
+   * the last one. Unset (the default) dispatches immediately, as before.
+   *
+   * This is a **trailing** debounce, and the name is accurate: each dispatch
+   * pushes the run further out, and the job runs once, after the dispatches
+   * stop. It is the shape the problem has — a document saved eight times in a
+   * minute should rebuild its search index once, and the only rebuild anyone
+   * sees is the last one. A *leading* behaviour, where the first dispatch runs
+   * and the rest are dropped, is a different thing and is not this.
+   *
+   * **The last payload wins.** When eight dispatches collapse, the surviving
+   * job carries the eighth one's data, because the whole premise is that the
+   * earlier ones are stale.
+   *
+   * @example
+   * ```ts
+   * export class ReindexDocument extends Job {
+   *   override readonly debounce = 30;
+   *   constructor(private documentId: number) { super(); }
+   *   override payload() { return { documentId: this.documentId }; }
+   *   async handle() { await search.reindex(this.documentId); }
+   * }
+   * ```
+   */
+  readonly debounce?: number;
+
+  /**
+   * What counts as "the same job" for {@link debounce}.
+   *
+   * Defaults to the class name plus the serialised payload, so
+   * `ReindexDocument(1)` and `ReindexDocument(2)` are different work and do not
+   * collapse into each other — which is what makes the common case need no
+   * configuration at all.
+   *
+   * Override when two payloads mean the same work. A job carrying a timestamp,
+   * or a request id, is unique on every dispatch and would otherwise never
+   * collapse with anything:
+   *
+   * ```ts
+   * override debounceKey(): string {
+   *   return `reindex:${this.documentId}`;   // ignore the requestedAt field
+   * }
+   * ```
+   *
+   * The key is stored in the queue's own backing store, so it is stable across
+   * processes. A debounce that only held inside one worker would appear to work
+   * in development and do nothing in production, where there is more than one.
+   */
+  debounceKey(): string {
+    return `${this.className}:${JSON.stringify(this.payload())}`;
+  }
+
   /** The job's work — implement this in every subclass */
   abstract handle(): Promise<void>;
 

package/src/QueueManager.ts

@@ -1,5 +1,9 @@
 import { randomUUIDv7 } from "bun";
-import { QueueShuttingDownError, QueueBatchingUnsupportedError } from "./errors.ts";
+import {
+  QueueShuttingDownError,
+  QueueBatchingUnsupportedError,
+  QueueDebounceUnsupportedError,
+} from "./errors.ts";
 import type { QueueDriver, JobRecord } from "./drivers/QueueDriver.ts";
 import type { Job } from "./Job.ts";
 import type { PendingBatch } from "./PendingBatch.ts";
@@ -115,16 +119,29 @@ export class QueueManager {
       payload["__chain"] = job._chain;
     }
 
-    await this._driver.push({
+    // A debounced job runs `debounce` seconds after the *last* dispatch, so each
+    // one sets the run-at forward from now; the driver collapses it into whatever
+    // is already pending under the same key.
+    const debounceSeconds = job.debounce;
+    const record = {
       queue: job.queue,
       className: job.className,
       payload: JSON.stringify(payload),
       attempts: 0,
       maxAttempts: job.maxAttempts,
       retryDelay: job.retryDelay,
-      availableAt: Math.floor(Date.now() / 1000),
+      availableAt: Math.floor(Date.now() / 1000) + (debounceSeconds ?? 0),
       batchId: job.batchId,
-    });
+    };
+
+    if (debounceSeconds !== undefined && debounceSeconds > 0 && !this._isSyncDriver()) {
+      if (!this._driver.pushDebounced) {
+        throw new QueueDebounceUnsupportedError(job.className, this._driverName());
+      }
+      await this._driver.pushDebounced(record, job.debounceKey());
+    } else {
+      await this._driver.push(record);
+    }
 
     _fireJobEvent(job.className, job.queue, "dispatched", 0);
 
@@ -424,4 +441,9 @@ export class QueueManager {
   private _isSyncDriver(): boolean {
     return this._driver instanceof SyncDriver;
   }
+
+  /** The driver's class name, for an error that has to name what to change. */
+  private _driverName(): string {
+    return this._driver.constructor.name;
+  }
 }

package/src/drivers/QueueDriver.ts

@@ -9,6 +9,8 @@ export interface JobRecord {
   availableAt: number;
   createdAt: string;
   batchId?: string | undefined;
+  /** Debounce identity, when the job declared one. See `Job.debounce`. */
+  debounceKey?: string | undefined;
 }
 
 /**
@@ -32,6 +34,21 @@ export interface FailedRecord {
 export interface QueueDriver {
   // ── Active queue ──────────────────────────────────────────────────────────
   push(record: Omit<JobRecord, "id" | "createdAt">): Promise<void>;
+  /**
+   * Enqueue, or push an already-pending job with the same `key` out to this
+   * record's `availableAt`, replacing its payload.
+   *
+   * Optional, because it is the one operation a backing store has to be able to
+   * do *atomically*: two processes dispatching at once must produce one pending
+   * job, not two. A driver that cannot promise that does not implement this, and
+   * `Job.debounce` refuses rather than silently degrading to a per-process
+   * debounce — which would appear to work in development and do nothing in
+   * production, where there is more than one worker.
+   *
+   * Only jobs not yet reserved are collapsed into: once a worker has claimed one,
+   * it is running, and the next dispatch is genuinely new work.
+   */
+  pushDebounced?(record: Omit<JobRecord, "id" | "createdAt">, key: string): Promise<void>;
   pop(queue?: string): Promise<JobRecord | null>;
   fail(record: JobRecord, error: string): Promise<void>;
   retry(record: JobRecord): Promise<void>;

package/src/drivers/RedisDriver.ts

@@ -84,6 +84,10 @@ export class RedisDriver implements QueueDriver {
   private _failKey(queue: string): string {
     return `${this._prefix}${queue}:failed`;
   }
+  /** Hash: debounce key → the delayed-set member currently holding it. */
+  private _debounceKey(queue: string): string {
+    return `${this._prefix}${queue}:debounce`;
+  }
   /** Set of queue names seen by push() — lets flush() find every key family. */
   private _queuesKey(): string {
     return `${this._prefix}_queues`;
@@ -117,6 +121,47 @@ export class RedisDriver implements QueueDriver {
     }
   }
 
+  /**
+   * Collapse a dispatch into whatever is already pending under the same key.
+   *
+   * One EVAL, because the read ("is something pending?") and the write have to be
+   * one step: two processes dispatching at the same instant would otherwise both
+   * see nothing pending and both enqueue, which is the failure the whole feature
+   * exists to prevent.
+   *
+   * The hash maps key → the delayed-set member holding it. `ZREM` returning 0
+   * means that member is gone — already promoted to ready, or claimed — so the
+   * dispatch becomes a fresh job rather than collapsing into something running.
+   */
+  private static readonly _DEBOUNCE_LUA =
+    // KEYS: 1 delayed zset, 2 debounce hash. ARGV: 1 key, 2 score, 3 payload.
+    "local prev = redis.call('HGET', KEYS[2], ARGV[1]) " +
+    "if prev then redis.call('ZREM', KEYS[1], prev) end " +
+    "redis.call('ZADD', KEYS[1], ARGV[2], ARGV[3]) " +
+    "redis.call('HSET', KEYS[2], ARGV[1], ARGV[3]) " +
+    "return 1";
+
+  async pushDebounced(record: Omit<JobRecord, "id" | "createdAt">, key: string): Promise<void> {
+    const redis = await this._redis();
+    const id = Number(await redis.incr(this._idKey()));
+    const payload = JSON.stringify({
+      ...record,
+      id,
+      createdAt: new Date().toISOString(),
+      debounceKey: key,
+    });
+    await redis.sadd(this._queuesKey(), record.queue);
+    await redis.send("EVAL", [
+      RedisDriver._DEBOUNCE_LUA,
+      "2",
+      this._delayedKey(record.queue),
+      this._debounceKey(record.queue),
+      key,
+      String(record.availableAt),
+      payload,
+    ]);
+  }
+
   /**
    * Move every member of sorted set `from` with score ≤ `maxScore` onto the
    * ready list. `ZREM` returns 0 when another worker won the race for a
@@ -129,7 +174,19 @@ export class RedisDriver implements QueueDriver {
     if (!due || due.length === 0) return;
     for (const member of due) {
       const removed = Number(await redis.send("ZREM", [from, member]));
-      if (removed === 1) await redis.lpush(this._key(queue), member);
+      if (removed !== 1) continue;
+      await redis.lpush(this._key(queue), member);
+      // The job is runnable now, so it no longer holds its debounce key — the
+      // next dispatch is new work rather than a reschedule of something a worker
+      // is about to pick up.
+      try {
+        const parsed = JSON.parse(member) as { debounceKey?: string };
+        if (parsed.debounceKey) {
+          await redis.send("HDEL", [this._debounceKey(queue), parsed.debounceKey]);
+        }
+      } catch {
+        // An unparseable member is already on the ready list; nothing to release.
+      }
     }
   }
 
@@ -238,6 +295,7 @@ export class RedisDriver implements QueueDriver {
       await redis.del(this._delayedKey(queue));
       await redis.del(this._reservedKey(queue));
       await redis.del(this._failKey(queue));
+      await redis.del(this._debounceKey(queue));
     }
     await redis.del(this._queuesKey());
     await redis.del(this._idKey());

package/src/drivers/SqliteDriver.ts

@@ -14,6 +14,7 @@ type JobRow = {
   reserved_at: number | null;
   created_at: string;
   batch_id: string | null;
+  debounce_key: string | null;
 };
 
 type FailedRow = {
@@ -105,10 +106,26 @@ export class SqliteDriver implements QueueDriver {
     } catch {
       // Column already exists — safe to ignore.
     }
+    // Add debounce_key column to existing databases that predate this migration.
+    try {
+      await this._sql`ALTER TABLE zerotal_jobs ADD COLUMN debounce_key TEXT`;
+    } catch {
+      // Column already exists — safe to ignore.
+    }
     await this._sql`
       CREATE INDEX IF NOT EXISTS zerotal_jobs_pop_idx
       ON zerotal_jobs (queue, available_at, reserved_at)
     `;
+    // The debounce upsert's conflict target, and the thing that makes it atomic:
+    // at most one *unreserved* pending job may hold a given key. Partial, so a
+    // job already claimed by a worker no longer occupies the key — the next
+    // dispatch is new work, not a collapse into something already running — and
+    // so the overwhelming majority of jobs, which set no key, are not indexed.
+    await this._sql`
+      CREATE UNIQUE INDEX IF NOT EXISTS zerotal_jobs_debounce_idx
+      ON zerotal_jobs (debounce_key)
+      WHERE debounce_key IS NOT NULL AND reserved_at IS NULL
+    `;
   }
 
   private _mapRow(row: JobRow): JobRecord {
@@ -121,6 +138,7 @@ export class SqliteDriver implements QueueDriver {
       maxAttempts: row.max_attempts,
       retryDelay: row.retry_delay,
       availableAt: row.available_at,
+      debounceKey: row.debounce_key ?? undefined,
       createdAt: row.created_at,
       batchId: row.batch_id ?? undefined,
     };
@@ -154,6 +172,35 @@ export class SqliteDriver implements QueueDriver {
     `;
   }
 
+  /**
+   * One statement, so two processes dispatching at once cannot both insert.
+   *
+   * `ON CONFLICT` targets the partial unique index on `debounce_key`, which only
+   * covers unreserved rows — so a dispatch collapses into a pending job and never
+   * into one a worker has already claimed. The update takes the *new* payload and
+   * run-at, because the point of collapsing is that the earlier dispatch is stale.
+   */
+  async pushDebounced(record: Omit<JobRecord, "id" | "createdAt">, key: string): Promise<void> {
+    await this._ready;
+    const now = new Date().toISOString();
+    const batchId = record.batchId ?? null;
+    await this._sql`
+      INSERT INTO zerotal_jobs
+        (queue, class_name, payload, attempts, max_attempts, retry_delay, available_at, created_at, batch_id, debounce_key)
+      VALUES
+        (${record.queue}, ${record.className}, ${record.payload},
+         ${record.attempts}, ${record.maxAttempts}, ${record.retryDelay},
+         ${record.availableAt}, ${now}, ${batchId}, ${key})
+      ON CONFLICT (debounce_key) WHERE debounce_key IS NOT NULL AND reserved_at IS NULL
+      DO UPDATE SET
+        available_at = excluded.available_at,
+        payload      = excluded.payload,
+        queue        = excluded.queue,
+        max_attempts = excluded.max_attempts,
+        retry_delay  = excluded.retry_delay
+    `;
+  }
+
   async pop(queue = "default"): Promise<JobRecord | null> {
     await this._ready;
     const now = Math.floor(Date.now() / 1000);

package/src/errors.ts

@@ -40,3 +40,21 @@ export class QueueBatchingUnsupportedError extends QueueError {
     );
   }
 }
+
+/** Thrown when a job declares `debounce` on a driver that cannot collapse atomically. */
+export class QueueDebounceUnsupportedError extends QueueError {
+  constructor(className: string, driver: string) {
+    super(
+      `[Zerotal] ${className} declares debounce, but the ${driver} queue driver cannot ` +
+        `collapse dispatches atomically.
+` +
+        `Collapsing in this process instead would appear to work in development and do ` +
+        `nothing in production, where more than one worker dispatches — so this refuses ` +
+        `rather than degrading quietly.
+` +
+        `Fix: use the sqlite or redis driver, or remove \`debounce\` from ${className}.`,
+      "E_QUEUE_DEBOUNCE_UNSUPPORTED",
+      500,
+    );
+  }
+}

package/src/provider/QueueProvider.ts

@@ -1,5 +1,5 @@
 import { ServiceProvider } from "@zerotal/core";
-import type { AppEnvironment } from "@zerotal/core";
+import type { AppEnvironment, DevProcessDefinition } from "@zerotal/core";
 import type { ConfigManager } from "@zerotal/core/config";
 import { _getConnection } from "@zerotal/orm";
 import { QueueManager } from "../QueueManager.ts";
@@ -29,6 +29,33 @@ export class QueueProvider extends ServiceProvider {
   private _workerPool: WorkerPool | undefined = undefined;
   private _disposeObservability: (() => void) | undefined = undefined;
 
+  /**
+   * The worker that `bun zt dev` runs beside the server.
+   *
+   * Two conditions, and both are about not putting a tab on screen that has
+   * nothing to do. The `sync` driver runs every dispatched job inline on the
+   * request, so a worker would poll an empty queue forever; and an in-process
+   * `workers` pool means the server thread is already draining the queue, so a
+   * second consumer is duplicated work rather than more of it.
+   */
+  override devProcesses(): DevProcessDefinition[] {
+    return [
+      {
+        name: "queue",
+        command: "queue:work",
+        enabled: () => {
+          const config = this.app.container.makeSync("config") as ConfigManager;
+          const driver = config.get<string>("queue.driver", "sqlite");
+          const workers = config.get<number>("queue.workers", 0);
+          return driver !== "sync" && (workers ?? 0) === 0;
+        },
+        // A worker that exits cleanly has been told to stop; one that crashes
+        // has not, and the developer wants it back.
+        restart: "on-failure",
+      },
+    ];
+  }
+
   override onRegister(): void {
     // Refuse a production boot on structural queue misconfiguration (unknown
     // driver, workers without a bootstrap module). Runs in the boot-time config pass.