@maestro-js/queue 1.0.0-alpha.0

package/README.md

@@ -0,0 +1,279 @@
+# @maestro-js/queue
+
+Database-backed job queue that stores messages in MySQL via a pluggable driver and processes them asynchronously. Follows the Provider pattern from `@maestro-js/service-registry` with a producer/consumer split supporting scheduled execution, concurrency, and distributed trace propagation.
+
+## Quick Setup
+
+```ts
+import { Queue } from '@maestro-js/queue'
+import { Db } from '@maestro-js/db'
+import { Log } from '@maestro-js/log'
+
+// Register the queue service with the db driver
+Queue.Provider.register(
+  'default',
+  Queue.Provider.create({
+    driver: Queue.drivers.db({
+      db: Db.provider('default'),
+      databaseTable: 'jobs'
+    }),
+    logger: Log
+  })
+)
+
+// Create a named queue with a work function
+const orderQueue = Queue.create({
+  name: 'fulfill-orders',
+  workFn: async (body) => {
+    await fulfillOrder(body.orderId)
+    return { fulfilled: true }
+  },
+  concurrency: 3
+})
+
+// Enqueue a message
+await orderQueue.addMessage({ orderId: 'ord_123' }, { scheduledDate: null })
+
+// Start processing
+orderQueue.startProcessing()
+```
+
+## Database Schema
+
+The db driver expects a MySQL table (name configurable via `databaseTable`). The column-to-field mapping used internally:
+
+| Column          | Type          | Description                                              |
+| --------------- | ------------- | -------------------------------------------------------- |
+| `id`            | int (PK, AI)  | Message ID                                               |
+| `batch_id`      | int / NULL    | Lock batch for concurrent dequeue                        |
+| `job_type`      | varchar       | Queue name string                                        |
+| `body`          | text (JSON)   | Serialized message body                                  |
+| `trace`         | text (JSON)   | Serialized `DehydratedTrace` for distributed tracing     |
+| `status`        | enum          | `waiting`, `processing`, `success`, `error`, `stuck`     |
+| `scheduledDate` | datetime      | Earliest time the message can be processed               |
+| `enqueuedDate`  | datetime      | When the message was inserted                            |
+| `started_ts`    | datetime      | When processing began                                    |
+| `result`        | text          | JSON result on success, error stack on failure           |
+| `recovered`     | tinyint       | 1 if the message was recovered from stuck state          |
+| `running_time`  | int / NULL    | Seconds elapsed during processing                        |
+| `unique_key`    | int / NULL    | MD5-derived dedup key (nullable)                         |
+| `priority`      | bigint / NULL | Lower value = higher priority (defaults to `Date.now()`) |
+
+The driver requires a `FORCE INDEX (locking_update)` index on `(batch_id, status, job_type, scheduledDate)` for efficient locking queries.
+
+## API Reference
+
+### Provider Setup
+
+```ts
+Queue.Provider.create(config: {
+  driver: Queue.Driver
+  logger: Log.Logger<[Queue.LogMessage]>
+  tracing?: Queue.Tracing  // optional distributed tracing adapter
+}): Queue.QueueService
+```
+
+```ts
+Queue.Provider.register(name: Queue.Provider.Key, service: Queue.QueueService): void
+```
+
+### Drivers
+
+```ts
+Queue.drivers.db(config: {
+  db: { update, select, insert }  // Db provider interface
+  databaseTable: string
+  extraFields?: string[]           // additional columns to insert
+  fastestPollingRateMs?: number    // default 100
+  slowestPollingRateMs?: number    // default 10_000
+  pollingBackoffRate?: number      // default 1.1
+}): Queue.Driver
+```
+
+### Queue Instance Methods
+
+Create a queue:
+
+```ts
+const queue = Queue.create<MessageBody, Result>({
+  name: string,
+  workFn: (body: MessageBody) => Promise<Result>,
+  logger?: Log.LogFunctions<[Queue.LogMessage]>,  // additional per-queue logger
+  concurrency?: number  // default 1
+})
+```
+
+Dispatch a message:
+
+```ts
+await queue.addMessage(body, {
+  scheduledDate: Iso.Instant | null,   // null = process immediately
+  tracingContext?: DehydratedTrace | null
+})
+```
+
+Update a waiting message:
+
+```ts
+await queue.updateMessage({
+  messageId: number,
+  scheduledDate: Iso.Instant,
+  body: MessageBody
+})
+```
+
+List messages (paginated):
+
+```ts
+const messages = await queue.listMessages({ pageSize: 50, pageNumber: 0 })
+```
+
+Processing control:
+
+```ts
+queue.startProcessing()     // begin polling and processing
+queue.stopProcessing()      // stop polling
+await queue.processUntilEmpty({ pollingMs?: number })  // process then resolve when empty
+```
+
+### Service-Level Methods
+
+```ts
+Queue.list() // names of all registered queues
+Queue.get<Body, Result>(name) // retrieve a queue by name
+Queue.startProcessing() // start all queues
+Queue.stopProcessing() // stop all queues
+```
+
+### Message Statuses
+
+| Status       | Meaning                                                        |
+| ------------ | -------------------------------------------------------------- |
+| `waiting`    | Queued and eligible for processing when scheduled time arrives |
+| `processing` | Locked by a worker and currently executing                     |
+| `success`    | Work function completed without error                          |
+| `error`      | Work function threw or timed out                               |
+| `stuck`      | Processing exceeded timeout; recovered or marked stuck         |
+
+### Log Events
+
+The queue emits structured log messages with these `event` values:
+
+- `messageEnqueued` -- message added to the queue
+- `messageDequeued` -- message picked up by a worker
+- `messageProcessed` -- work function completed successfully
+- `messageFailed` -- work function threw an error
+- `queueStarted` -- queue processing started
+- `queueStopped` -- queue processing stopped
+
+## Common Patterns
+
+### Schedule a Future Job
+
+```ts
+import { instantFns } from 'iso-fns2'
+
+// Schedule 1 hour from now
+const oneHourLater = instantFns.add(instantFns.now(), { hours: 1 })
+await queue.addMessage({ task: 'reminder' }, { scheduledDate: oneHourLater })
+```
+
+### Deduplication with Unique Keys
+
+The db driver supports unique keys via the `addMessage` options on the driver level. When a duplicate `uniqueKey` is inserted, the row is not duplicated -- instead the priority is updated if the new priority is higher (lower number).
+
+### Process Until Empty (Worker Scripts)
+
+```ts
+// Useful for one-off batch processing or tests
+await queue.processUntilEmpty({ pollingMs: 1000 })
+// Resolves when no waiting/processing messages remain
+```
+
+### Concurrency Control
+
+```ts
+// High-concurrency queue for lightweight jobs
+const emailQueue = Queue.create({
+  name: 'send-emails',
+  workFn: async (body) => sendEmail(body),
+  concurrency: 10
+})
+```
+
+### Stuck Job Recovery
+
+The db driver automatically recovers stuck jobs every 60 seconds. Jobs in `processing` status that exceed the timeout (default 45 minutes) are reset to `waiting` and reprocessed. Set `shouldRecoverStuckJobs: false` in the driver `start` options to mark them as `stuck` instead of recovering.
+
+### Distributed Tracing
+
+Pass a `tracing` adapter when creating the service to propagate trace context across producer/consumer boundaries:
+
+```ts
+Queue.Provider.register(
+  'default',
+  Queue.Provider.create({
+    driver: Queue.drivers.db({ db: Db.provider('default'), databaseTable: 'jobs' }),
+    logger: Log,
+    tracing: {
+      hydrate: (trace, callback) => {
+        /* restore trace context */ return callback()
+      },
+      dehydrate: () => {
+        /* return current trace context */ return null
+      },
+      startActiveSpan: (name, fn) => fn(),
+      getActiveSpan: () => undefined
+    }
+  })
+)
+```
+
+The trace data is serialized as `DehydratedTrace` (traceId, spanId, traceFlags, traceState) and stored in the `trace` column.
+
+## Testing
+
+This package has no tests yet (`pnpm --filter @maestro-js/queue test` prints "No tests yet"). To test queue behavior in consuming packages, mock the driver interface:
+
+```ts
+const mockDriver: Queue.Driver = {
+  start: () => {},
+  stop: () => {},
+  listMessages: async () => [],
+  addMessage: async () => ({ id: 1 }),
+  updateMessage: async () => {},
+  isQueueEmpty: async () => true
+}
+
+Queue.Provider.register(
+  'default',
+  Queue.Provider.create({
+    driver: mockDriver,
+    logger: Log
+  })
+)
+```
+
+## Cross-Package Integration
+
+### Mail (async email sending)
+
+The `@maestro-js/mail` package uses Queue to dispatch email-sending jobs asynchronously. Mail enqueues messages; a queue consumer processes them via the configured mail driver.
+
+### Recurring Jobs (cron scheduling)
+
+The `@maestro-js/recurring-jobs` package uses Queue to execute scheduled/cron-based jobs. It dispatches queue messages on a schedule and relies on the queue's processing infrastructure.
+
+### Db (database driver)
+
+The db driver requires a `Db` provider that exposes `select`, `update`, and `insert` methods. Pass `Db.provider('default')` as the `db` config option.
+
+### Tracing
+
+Integrate with `@maestro-js/tracing` by implementing the `Queue.Tracing` interface to propagate OpenTelemetry-compatible trace context across the producer/consumer boundary.
+
+## Source Files
+
+- `packages/queue/src/index.ts` -- Provider pattern, facade, queue creation and lifecycle
+- `packages/queue/src/queue-types.ts` -- TypeScript interfaces for QueueMessage, QueueDriver, QueueTracing, log events
+- `packages/queue/src/db-queue-driver.ts` -- MySQL-backed driver with polling, batched locking, stuck job recovery

package/dist/index.d.ts

@@ -0,0 +1,300 @@
+import { Iso } from 'iso-fns2';
+import { Log } from '@maestro-js/log';
+
+interface DehydratedTrace {
+    traceId: string;
+    spanId: string;
+    traceFlags: number;
+    traceState: string | undefined;
+}
+interface QueueTracing {
+    hydrate: <T>(trace: DehydratedTrace | null, callback: () => T) => T;
+    dehydrate: () => DehydratedTrace | null;
+    startActiveSpan: <R>(name: string | {
+        name: string;
+        attributes?: Record<string, unknown>;
+    }, fn: () => R) => R;
+    getActiveSpan: () => {
+        setAttribute(key: string, value: unknown): void;
+        setStatus(status: {
+            code: number;
+            message: string;
+        }): void;
+    } | undefined;
+}
+interface QueueMessage<MessageBody extends Record<string, any> = Record<string, any>, Result = any> {
+    id: number;
+    batchId: number | null;
+    queue: string;
+    scheduledDate: Iso.Instant;
+    enqueuedDate: Iso.Instant;
+    startedDate: Iso.Instant | null;
+    body: MessageBody;
+    trace: DehydratedTrace | null;
+    status: 'waiting' | 'processing' | 'success' | 'error' | 'stuck';
+    result: Result;
+    recovered: number;
+    runningTimeSeconds: number | null;
+    uniqueKey: number | null;
+    priority: number | null;
+}
+type QueueWorkFunction<MessageBody extends Record<string, any> = Record<string, any>> = (args: {
+    body: MessageBody;
+    id: number;
+    trace: DehydratedTrace | null;
+    scheduledDate: Iso.Instant;
+}) => any;
+interface QueueDriver {
+    start<MessageBody extends Record<string, any> = Record<string, any>>(options: {
+        queueName: string;
+        workFn: QueueWorkFunction<MessageBody>;
+        concurrency?: number;
+    }): void;
+    stop(options: {
+        queueName: string;
+    }): void;
+    listMessages<MessageBody extends Record<string, any> = Record<string, any>, Result = any>(options: {
+        pageSize: number;
+        pageNumber: number;
+        queueName: string;
+    }): Promise<QueueMessage<MessageBody, Result>[]>;
+    addMessage<MessageBody extends Record<string, any> = Record<string, any>>(options: {
+        body: MessageBody;
+        scheduledDate: Iso.Instant | null;
+        queueName: string;
+        traceData: DehydratedTrace | null;
+    }): Promise<{
+        id: number;
+    }>;
+    updateMessage<MessageBody extends Record<string, any> = Record<string, any>>(options: {
+        queueName: string;
+        messageId: number;
+        scheduledDate: Iso.Instant;
+        body: MessageBody;
+    }): Promise<void>;
+    isQueueEmpty(options: {
+        queueName: string;
+    }): Promise<boolean>;
+}
+type QueueLogMessage = {
+    event: 'messageEnqueued';
+    queue: string;
+    driver: QueueDriver;
+    scheduledDate: Iso.Instant | null;
+    body: Record<string, any>;
+    messageId: number;
+} | {
+    event: 'messageDequeued';
+    queue: string;
+    driver: QueueDriver;
+    body: Record<string, any>;
+    messageId: number;
+} | {
+    event: 'messageProcessed';
+    queue: string;
+    driver: QueueDriver;
+    body: Record<string, any>;
+    result: unknown;
+    messageId: number;
+} | {
+    event: 'messageFailed';
+    queue: string;
+    driver: QueueDriver;
+    body: Record<string, any>;
+    error: unknown;
+    messageId: number;
+} | {
+    event: 'queueStarted';
+    queue: string;
+    driver: QueueDriver;
+} | {
+    event: 'queueStopped';
+    queue: string;
+    driver: QueueDriver;
+};
+
+interface DbQueueDriverConfig {
+    db: {
+        update(query: string, params?: any[]): Promise<{
+            changedRows: number;
+        }>;
+        select(query: string, params?: any[]): Promise<Array<Record<string, any>>>;
+        insert(query: string, params?: any[]): Promise<{
+            insertId: number;
+        }>;
+    };
+    databaseTable: string;
+    extraFields?: string[];
+    fastestPollingRateMs?: number;
+    slowestPollingRateMs?: number;
+    pollingBackoffRate?: number;
+}
+declare function DbQueueDriver({ db, databaseTable, extraFields, fastestPollingRateMs, slowestPollingRateMs, pollingBackoffRate }: DbQueueDriverConfig): QueueDriver;
+
+declare function createService(config: Queue.Provider.QueueServiceConfig): {
+    list: () => string[];
+    get: <MessageBody extends Record<string, any>, Result = any>(name: string) => Queue.Queue<MessageBody, Result> | undefined;
+    create: <MessageBody extends Record<string, any>, Result = any>(options: Queue.QueueOptions<MessageBody, Result>) => Queue.Queue<MessageBody, Result>;
+    startProcessing: () => void;
+    stopProcessing: () => void;
+};
+declare function provider(key: Queue.Provider.Key): {
+    list: () => string[];
+    get: <MessageBody extends Record<string, any>, Result = any>(name: string) => Queue.Queue<MessageBody, Result> | undefined;
+    create: <MessageBody extends Record<string, any>, Result = any>(options: Queue.QueueOptions<MessageBody, Result>) => Queue.Queue<MessageBody, Result>;
+    startProcessing: () => void;
+    stopProcessing: () => void;
+};
+/**
+ * Job queue for deferring work beyond the current request.
+ *
+ * Messages are stored by a pluggable driver and processed asynchronously by a
+ * worker — either in the same process or a dedicated consumer. Each message
+ * tracks its full lifecycle (waiting → processing → success/error/stuck) and
+ * supports scheduled future execution, concurrent processing, and distributed
+ * trace propagation across the producer/consumer boundary.
+ *
+ * @example
+ * ```ts
+ * // Create and register a queue service
+ * Queue.Provider.register('default', Queue.Provider.create({
+ *   driver: Queue.drivers.db({ db: Db.provider('default'), databaseTable: 'jobs' }),
+ *   logger: Log.provider('default')
+ * }))
+ *
+ * // Create a queue with a work function
+ * const orderQueue = Queue.create({
+ *   name: 'fulfill-orders',
+ *   workFn: async (body) => {
+ *     await fulfillOrder(body.orderId)
+ *     return { fulfilled: true }
+ *   }
+ * })
+ *
+ * // Enqueue a message and start processing
+ * await orderQueue.addMessage({ orderId: 'ord_123' }, { scheduledDate: null })
+ * orderQueue.startProcessing()
+ * ```
+ */
+declare namespace Queue {
+    /** Per-queue configuration passed to {@link QueueService.create} */
+    interface QueueOptions<T extends Record<string, any> = Record<string, any>, Result = any> {
+        name: string;
+        workFn(body: T): Promise<Result>;
+        logger?: Log.LogFunctions<[QueueLogMessage]>;
+        concurrency?: number;
+    }
+    /** Pluggable storage backend that handles message persistence, polling, and locking */
+    type Driver = QueueDriver;
+    /** A single message stored in the queue with its full lifecycle metadata */
+    type QueueMessage<MessageBody extends Record<string, any> = Record<string, any>, Result = any> = QueueMessage<MessageBody, Result>;
+    /** Union of all queue lifecycle log event shapes */
+    type LogMessage = QueueLogMessage;
+    /** Distributed tracing adapter for propagating trace context across producer/consumer boundaries */
+    type Tracing = QueueTracing;
+    /**
+     * A named queue instance with methods for adding messages and controlling processing.
+     *
+     * Each queue has its own work function, concurrency setting, and independent
+     * start/stop lifecycle. Messages are typed via the `MessageBody` and `Result`
+     * generics — the body is what the producer enqueues, the result is what the
+     * work function returns on success.
+     *
+     * @example
+     * ```ts
+     * const queue = Queue.create({
+     *   name: 'send-emails',
+     *   workFn: async (body) => sendEmail(body.to, body.subject),
+     *   concurrency: 3
+     * })
+     * await queue.addMessage({ to: 'alice@example.com', subject: 'Welcome' }, { scheduledDate: null })
+     * queue.startProcessing()
+     * ```
+     */
+    type Queue<MessageBody extends Record<string, any> = Record<string, any>, Result = any> = {
+        /** Starts polling for and processing messages on this queue */
+        startProcessing(): void;
+        /** Starts processing and resolves when no waiting or in-progress messages remain */
+        processUntilEmpty(options?: {
+            pollingMs?: number;
+        }): Promise<void>;
+        /** Stops polling for new messages on this queue */
+        stopProcessing(): void;
+        name: string;
+        /** Returns a paginated list of messages in this queue */
+        listMessages(options: {
+            pageSize: number;
+            pageNumber: number;
+        }): Promise<QueueMessage<MessageBody, Result>[]>;
+        /** Adds a message to this queue for immediate or scheduled processing */
+        addMessage(body: MessageBody, options: {
+            scheduledDate: Iso.Instant | null;
+            tracingContext?: DehydratedTrace | null;
+        }): Promise<void>;
+        /** Updates the body and scheduled date of a waiting message */
+        updateMessage(options: {
+            messageId: number;
+            scheduledDate: Iso.Instant;
+            body: MessageBody;
+        }): Promise<void>;
+    };
+    namespace Provider {
+        interface QueueServiceConfig {
+            driver: Driver;
+            logger: Log.Logger<[QueueLogMessage]>;
+            tracing?: QueueTracing;
+        }
+        type Key = keyof Keys;
+        interface Keys {
+            default: unknown;
+        }
+    }
+    /**
+     * Container that manages multiple named queues sharing the same driver and logger.
+     *
+     * Create individual queues with {@link create}, then start or stop processing
+     * across all queues at once or individually. Each queue has its own work function,
+     * concurrency, and message lifecycle.
+     *
+     * @example
+     * ```ts
+     * const orders = Queue.create({ name: 'orders', workFn: processOrder })
+     * const emails = Queue.create({ name: 'emails', workFn: sendEmail, concurrency: 5 })
+     * Queue.startProcessing() // starts both queues
+     * ```
+     */
+    interface QueueService {
+        /** Returns the names of all queues registered with this service */
+        list(): string[];
+        /** Returns the queue with the given name, or `undefined` if not found */
+        get<MessageBody extends Record<string, any>, Result = any>(name: string): Queue.Queue<MessageBody, Result> | undefined;
+        /** Creates a new named queue with the given work function and options */
+        create<MessageBody extends Record<string, any>, Result = any>(options: Queue.QueueOptions<MessageBody, Result>): Queue<MessageBody, Result>;
+        /** Stops processing messages on all queues */
+        stopProcessing(): void;
+        /** Starts processing messages on all queues */
+        startProcessing(): void;
+    }
+}
+/**
+ * Job queue for deferring work beyond the current request.
+ * Call `Queue.Provider.create()` with a driver, register it, then use `Queue.create()` to define queues.
+ */
+declare const Queue: {
+    drivers: {
+        db: typeof DbQueueDriver;
+    };
+    Provider: {
+        create: typeof createService;
+        register: (name: Queue.Provider.Key, item: Queue.QueueService) => void;
+        list: () => "default"[];
+    };
+    provider: typeof provider;
+    list: () => string[];
+    get: <MessageBody extends Record<string, any>, Result = any>(name: string) => Queue.Queue<MessageBody, Result> | undefined;
+    create: <MessageBody extends Record<string, any>, Result = any>(options: Queue.QueueOptions<MessageBody, Result>) => Queue.Queue<MessageBody, Result>;
+    startProcessing: () => void;
+    stopProcessing: () => void;
+};
+
+export { Queue };

package/dist/index.js

@@ -0,0 +1,545 @@
+// src/index.ts
+import { instantFns as instantFns2 } from "iso-fns2";
+import { Log } from "@maestro-js/log";
+import { ServiceRegistry } from "@maestro-js/service-registry";
+
+// src/db-queue-driver.ts
+import crypto from "crypto";
+import { instantFns } from "iso-fns2";
+import assert from "assert";
+function DbQueueDriver({
+  db,
+  databaseTable,
+  extraFields,
+  fastestPollingRateMs = 100,
+  slowestPollingRateMs = 1e4,
+  pollingBackoffRate = 1.1
+}) {
+  function logError(msg) {
+    console.error(`Oxen Queue: ${msg}`);
+  }
+  const processingMap = /* @__PURE__ */ new Map();
+  function getIsProcessing(queueName) {
+    return !!processingMap.get(queueName);
+  }
+  function setIsProcessing(queueName, processing) {
+    processingMap.set(queueName, processing);
+  }
+  const jobRecoveryIntervalMap = /* @__PURE__ */ new Map();
+  function getJobRecoveryInterval(queueName) {
+    return jobRecoveryIntervalMap.get(queueName) ?? null;
+  }
+  function setJobRecoveryInterval(queueName, jobRecoveryInterval) {
+    jobRecoveryIntervalMap.set(queueName, jobRecoveryInterval);
+  }
+  async function start({
+    workFn,
+    concurrency = 3,
+    timeout = 60 * 45,
+    shouldRecoverStuckJobs = true,
+    queueName
+  }) {
+    if (getIsProcessing(queueName)) {
+      throw new Error(`This queue is already processing`);
+    }
+    if (!workFn) {
+      throw new Error(
+        `Missing work_fn argument, nothing to do! Remember that the process() function takes an object as its single argument.`
+      );
+    }
+    if (typeof concurrency !== "number") {
+      throw new Error(`The concurrency argument must be a number`);
+    }
+    let jobTimeoutSeconds = Math.floor(timeout);
+    let inProcess = 0;
+    let pollingRate = fastestPollingRateMs;
+    let currentlyFetching = false;
+    const workingJobBatch = [];
+    setIsProcessing(queueName, true);
+    async function fillJobBatch() {
+      const batchId = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+      const lockedBatch = await db.update(
+        `
+            UPDATE ${databaseTable} AS main
+            INNER JOIN (
+                SELECT id FROM ${databaseTable} FORCE INDEX (locking_update)
+                WHERE batchId IS NULL
+                AND STATUS = "waiting"
+                AND jobType = ?
+                AND scheduledDate <= NOW()
+                ORDER BY priority ASC LIMIT ${concurrency}
+            ) sub
+            ON sub.id = main.id
+            SET batchId = ?, STATUS = "processing", startedDate = NOW()`,
+        [queueName, batchId]
+      );
+      if (lockedBatch.changedRows === 0) {
+        return false;
+      }
+      const nextJobs = await db.select(
+        `SELECT id, scheduledDate, body, trace FROM ${databaseTable} WHERE batchId = ? ORDER BY priority ASC LIMIT ${concurrency}`,
+        [batchId]
+      );
+      if (nextJobs.length === 0) {
+        return false;
+      }
+      nextJobs.map((db2) => ({
+        id: db2.id,
+        scheduledDate: new Date(db2.scheduledDate).toISOString(),
+        body: db2.body ? JSON.parse(db2.body) : db2.body,
+        trace: db2.trace ? JSON.parse(db2.trace) : db2.trace
+      })).forEach((job) => {
+        workingJobBatch.push(job);
+      });
+      return true;
+    }
+    async function getNextJob() {
+      if (!currentlyFetching && workingJobBatch.length < concurrency) {
+        currentlyFetching = true;
+        await fillJobBatch().catch((error) => {
+          logError("There was an error while trying to get the next set of jobs:");
+          logError(error);
+        });
+        currentlyFetching = false;
+      }
+      return workingJobBatch.shift();
+    }
+    async function doWork() {
+      const job = await getNextJob();
+      if (!job) {
+        if (pollingRate < slowestPollingRateMs) {
+          pollingRate *= pollingBackoffRate;
+        }
+        if (pollingRate >= slowestPollingRateMs) {
+          pollingRate = slowestPollingRateMs;
+        }
+        return Promise.resolve("no_job");
+      }
+      pollingRate = fastestPollingRateMs;
+      const promises = [
+        workFn({ body: job.body, id: job.id, trace: job.trace, scheduledDate: job.scheduledDate }),
+        new Promise((_, reject) => {
+          setTimeout(
+            () => reject(new Error(`timeout for job_id ${job.id} (over ${jobTimeoutSeconds} seconds)`)),
+            typeof jobTimeoutSeconds === "number" ? jobTimeoutSeconds * 1e3 : 0
+          );
+        })
+      ];
+      return Promise.race(promises).then(async (jobResult) => {
+        return db.update(
+          `
+                  update ${databaseTable} 
+                  set 
+                      result = ?, 
+                      uniqueKey = NULL,
+                      status="success",
+                      runningTimeSeconds = TIMESTAMPDIFF(SECOND,startedDate,NOW())
+                  where id =?
+                  LIMIT 1`,
+          [JSON.stringify(jobResult), job.id]
+        );
+      }).catch(async (error) => {
+        return db.update(
+          `
+                  update ${databaseTable} 
+                  set 
+                      result = ?, 
+                      uniqueKey = NULL,
+                      status="error",
+                      runningTimeSeconds = TIMESTAMPDIFF(SECOND,startedDate,NOW())
+                  where id = ?
+                  LIMIT 1`,
+          [error.stack, job.id]
+        );
+      });
+    }
+    async function loop() {
+      if (inProcess < concurrency) {
+        inProcess++;
+        doWork().then(() => {
+          inProcess--;
+        }).catch((error) => {
+          inProcess--;
+          logError("Unhandled Oxen Queue error:");
+          logError(error);
+        });
+      } else if (pollingRate < slowestPollingRateMs) {
+        pollingRate *= pollingBackoffRate;
+        if (pollingRate >= slowestPollingRateMs) {
+          pollingRate = slowestPollingRateMs;
+        }
+      }
+      if (!getIsProcessing(queueName)) {
+        return;
+      }
+      setTimeout(function() {
+        if (!getIsProcessing(queueName)) {
+          return;
+        }
+        loop();
+      }, pollingRate ?? 0);
+    }
+    async function recoverStuckJobs() {
+      return db.update(
+        `
+            UPDATE ${databaseTable} 
+            SET 
+                STATUS="waiting",
+                batchId = NULL,
+                startedDate = NULL,
+                recovered = 1
+            WHERE
+                STATUS="processing" AND
+                startedDate < (NOW() - INTERVAL ${jobTimeoutSeconds} SECOND) AND
+                jobType = ?
+            `,
+        [queueName]
+      );
+    }
+    async function markStuckJobs() {
+      return db.update(
+        `
+            UPDATE ${databaseTable} 
+            SET 
+                STATUS="stuck",
+                uniqueKey = NULL,
+                recovered = 1
+            WHERE
+                STATUS="processing" AND
+                startedDate < (NOW() - INTERVAL ${jobTimeoutSeconds} SECOND) AND
+                jobType = ?
+            `,
+        [queueName]
+      );
+    }
+    loop();
+    setJobRecoveryInterval(
+      queueName,
+      setInterval(function() {
+        if (shouldRecoverStuckJobs) {
+          recoverStuckJobs().catch(function(error) {
+            logError("Unable to recover stuck jobs:");
+            logError(error);
+          });
+        } else {
+          markStuckJobs().catch(function(error) {
+            logError("Unable to mark stuck jobs:");
+            logError(error);
+          });
+        }
+      }, 1e3 * 60)
+      //every minute
+    );
+  }
+  function stop(options) {
+    setIsProcessing(options.queueName, false);
+    const jobRecoveryInterval = getJobRecoveryInterval(options.queueName);
+    if (jobRecoveryInterval) clearInterval(jobRecoveryInterval);
+  }
+  async function listMessages({
+    pageNumber,
+    pageSize,
+    queueName
+  }) {
+    const skip = pageNumber * pageSize;
+    const results = await db.select(`SELECT * FROM ${databaseTable} WHERE jobType = ? ORDER BY id DESC LIMIT ? OFFSET ?`, [
+      queueName,
+      pageSize,
+      skip
+    ]);
+    return results.map((r) => dbToJob(r));
+  }
+  async function addMessage(job) {
+    const uniqueKey = job.uniqueKey === void 0 ? null : parseInt(crypto.createHash("md5").update(`${job.uniqueKey}|${job.queueName}`).digest("hex").slice(0, 8), 16);
+    const inserts = [
+      JSON.stringify(job.body),
+      job.traceData ? JSON.stringify(job.traceData) : null,
+      job.queueName,
+      uniqueKey,
+      typeof job.priority === "number" ? job.priority : Date.now()
+    ];
+    extraFields?.forEach((field) => {
+      inserts.push(job[field]);
+    });
+    const fields = ["body", "trace", "jobType", "uniqueKey", "priority", ...extraFields ?? []];
+    const { insertId } = await db.insert(
+      `
+                insert into ${databaseTable} (
+                    ${fields.join(",")}, scheduledDate
+                ) 
+                VALUES (?, GREATEST(?, NOW())) 
+                ON DUPLICATE KEY UPDATE 
+                priority = IF(priority > VALUES(priority), VALUES(priority), priority)
+            `,
+      [inserts, instantFns.formatISO9075(job.scheduledDate ?? instantFns.now())]
+    );
+    if (!insertId) {
+      const results = await db.select(`SELECT id FROM ${databaseTable} WHERE uniqueKey = ?`, [uniqueKey]);
+      const row = results[0];
+      assert.ok(row, "Could not find inserted row");
+      return { id: row["id"] };
+    } else {
+      return { id: insertId };
+    }
+  }
+  async function updateMessage({
+    queueName,
+    messageId,
+    scheduledDate,
+    body
+  }) {
+    const result = instantFns.formatISO9075(scheduledDate);
+    await db.update(
+      `UPDATE ${databaseTable} SET scheduledDate = GREATEST(?, NOW()), body = ? WHERE jobType = ? AND id = ? AND status = "waiting"`,
+      [result, JSON.stringify(body), queueName, messageId]
+    );
+  }
+  async function isQueueEmpty({ queueName }) {
+    const results = await db.select(
+      `
+        SELECT id FROM ${databaseTable} FORCE INDEX (locking_update)
+        WHERE ((batchId IS NULL AND STATUS = "waiting") OR STATUS = "processing")
+        AND jobType = ?
+        AND scheduledDate <= NOW()
+        ORDER BY priority ASC LIMIT 1`,
+      [queueName]
+    );
+    return !results.length;
+  }
+  return { start, stop, listMessages, addMessage, updateMessage, isQueueEmpty };
+}
+function dbToJob(db) {
+  return {
+    id: db.id,
+    batchId: db.batchId,
+    queue: db.jobType,
+    scheduledDate: new Date(db.scheduledDate).toISOString(),
+    enqueuedDate: new Date(db.enqueuedDate).toISOString(),
+    startedDate: db.startedDate ? new Date(db.startedDate).toISOString() : null,
+    body: db.body ? JSON.parse(db.body) : db.body,
+    trace: db.trace ? JSON.parse(db.trace) : db.trace,
+    status: db.status,
+    result: db.result,
+    recovered: db.recovered,
+    runningTimeSeconds: db.runningTimeSeconds,
+    uniqueKey: db.uniqueKey,
+    priority: db.priority
+  };
+}
+
+// src/index.ts
+var noopTracing = {
+  hydrate: (_trace, callback) => callback(),
+  dehydrate: () => null,
+  startActiveSpan: (_name, fn) => fn(),
+  getActiveSpan: () => void 0
+};
+function createService(config) {
+  const queues = [];
+  const tracing = config.tracing ?? noopTracing;
+  function list() {
+    return queues.map((q) => q.name);
+  }
+  function get(name) {
+    return queues.find((q) => q.name === name);
+  }
+  function create(options) {
+    const queueName = options.name;
+    const concurrency = options.concurrency ?? 1;
+    const logger = Log.create({
+      transports: [config.logger, ...options.logger ? [options.logger] : []]
+    });
+    async function workFn({
+      body,
+      id,
+      trace,
+      scheduledDate
+    }) {
+      try {
+        logger.info({
+          event: "messageDequeued",
+          queue: queueName,
+          driver: config.driver,
+          body,
+          messageId: id
+        });
+        const result = await tracing.hydrate(trace, async () => {
+          return tracing.startActiveSpan(
+            { name: "queue_consumer_transaction", attributes: { "messaging.destination.name": options.name } },
+            () => {
+              const parent = tracing.getActiveSpan();
+              return tracing.startActiveSpan(
+                {
+                  name: "queue_consumer",
+                  attributes: {
+                    op: "queue.process",
+                    "sentry.op": "queue.process",
+                    "messaging.message.id": id,
+                    "messaging.destination.name": options.name,
+                    "messaging.message.receive.latency": instantFns2.getEpochMilliseconds(instantFns2.now()) - instantFns2.getEpochMilliseconds(scheduledDate)
+                  }
+                },
+                async () => {
+                  const res = await options.workFn(body);
+                  parent?.setStatus({ code: 1, message: "ok" });
+                  return res;
+                }
+              );
+            }
+          );
+        });
+        logger.info({
+          event: "messageProcessed",
+          queue: queueName,
+          driver: config.driver,
+          body,
+          result,
+          messageId: id
+        });
+        return result;
+      } catch (e) {
+        logger.error({
+          event: "messageFailed",
+          queue: queueName,
+          driver: config.driver,
+          body,
+          error: e,
+          messageId: id
+        });
+        throw e;
+      }
+    }
+    async function listMessages({ pageSize, pageNumber }) {
+      return config.driver.listMessages({ pageSize, pageNumber, queueName });
+    }
+    async function addMessage(body, options2) {
+      const { id } = await tracing.hydrate(
+        options2.tracingContext === void 0 ? tracing.dehydrate() : options2.tracingContext,
+        async () => {
+          return tracing.startActiveSpan(
+            {
+              name: "queue_producer",
+              attributes: {
+                op: "queue.publish",
+                "sentry.op": "queue.publish",
+                "messaging.destination.name": queueName
+              }
+            },
+            async () => {
+              const span = tracing.getActiveSpan();
+              const { id: id2 } = await config.driver.addMessage({
+                body,
+                scheduledDate: options2.scheduledDate,
+                queueName,
+                traceData: tracing.dehydrate()
+              });
+              span?.setAttribute("messaging.message.id", id2);
+              return { id: id2 };
+            }
+          );
+        }
+      );
+      logger.info({
+        event: "messageEnqueued",
+        queue: queueName,
+        body,
+        scheduledDate: options2.scheduledDate ?? null,
+        driver: config.driver,
+        messageId: id
+      });
+    }
+    async function updateMessage({
+      messageId,
+      scheduledDate,
+      body
+    }) {
+      return config.driver.updateMessage({ messageId, body, scheduledDate, queueName });
+    }
+    function processUntilEmpty(options2) {
+      const interval = setInterval(() => {
+        config.driver.isQueueEmpty({ queueName }).then((isEmpty2) => {
+          if (isEmpty2) {
+            declareEmpty(null);
+          }
+        });
+      }, options2?.pollingMs ?? 3e3);
+      let declareEmpty = (value) => {
+      };
+      const isEmpty = new Promise((resolve) => {
+        declareEmpty = resolve;
+      }).then(() => clearInterval(interval)).then(() => {
+        return stopProcessing2();
+      });
+      startProcessing2();
+      return isEmpty;
+    }
+    function startProcessing2() {
+      config.driver.start({ queueName, workFn, concurrency });
+      logger.info({
+        event: "queueStarted",
+        queue: queueName,
+        driver: config.driver
+      });
+    }
+    function stopProcessing2() {
+      config.driver.stop({ queueName });
+      logger.info({
+        event: "queueStopped",
+        queue: queueName,
+        driver: config.driver
+      });
+    }
+    const queue = {
+      listMessages,
+      addMessage,
+      updateMessage,
+      startProcessing: startProcessing2,
+      processUntilEmpty,
+      stopProcessing: stopProcessing2,
+      name: options.name
+    };
+    queues.push(queue);
+    return queue;
+  }
+  function startProcessing() {
+    queues.forEach((q) => q.startProcessing());
+  }
+  function stopProcessing() {
+    queues.forEach((q) => q.stopProcessing());
+  }
+  return { list, get, create, startProcessing, stopProcessing };
+}
+var registry = ServiceRegistry.createRegistry(
+  ServiceRegistry.proxyFunctionsForObject
+);
+var Provider = {
+  create: createService,
+  register: registry.register,
+  list: registry.list
+};
+function provider(key) {
+  const service = registry.resolve(key);
+  return {
+    list: service.list,
+    get: service.get,
+    create: service.create,
+    startProcessing: service.startProcessing,
+    stopProcessing: service.stopProcessing
+  };
+}
+var facade = provider("default");
+var Queue = {
+  drivers: {
+    db: DbQueueDriver
+  },
+  Provider,
+  provider,
+  list: facade.list,
+  get: facade.get,
+  create: facade.create,
+  startProcessing: facade.startProcessing,
+  stopProcessing: facade.stopProcessing
+};
+export {
+  Queue
+};

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@maestro-js/queue",
+  "description": "Database-backed job queue for deferring work beyond the current request with scheduled execution, concurrency control, and distributed tracing. Use when working with @maestro-js/queue, including dispatching jobs, processing queues, configuring the db driver, writing queue consumers, handling retries/stuck jobs, integrating with mail or recurring-jobs, and understanding the queue database schema.",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "iso-fns2": "npm:iso-fns@2.0.0-alpha.26",
+    "@maestro-js/log": "1.0.0-alpha.0",
+    "@maestro-js/service-registry": "1.0.0-alpha.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.11"
+  },
+  "version": "1.0.0-alpha.0",
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "dist"
+  ],
+  "license": "UNLICENSED",
+  "engines": {
+    "node": ">=22.18.0"
+  },
+  "repository": "https://github.com/Marcato-Partners/maestro-js",
+  "scripts": {
+    "build": "tsup --config ../../tsup.config.ts",
+    "typecheck": "tsc --noEmit",
+    "test": "echo 'No tests yet'",
+    "format": "prettier --write src/",
+    "lint": "prettier --check src/"
+  }
+}