@maestro-js/queue 1.0.0-alpha.19 → 1.0.0-alpha.2

package/dist/index.js

@@ -1,491 +1,544 @@
 // src/index.ts
-import { randomUUID } from "crypto";
-import { instantFns as instantFns3 } from "iso-fns2";
+import { instantFns as instantFns2 } from "iso-fns2";
+import { Log } from "@maestro-js/log";
 import { ServiceRegistry } from "@maestro-js/service-registry";
 
 // src/db-queue-driver.ts
-import { randomInt } from "crypto";
+import crypto from "crypto";
 import { instantFns } from "iso-fns2";
+import assert from "assert";
 function DbQueueDriver({
   db,
   databaseTable,
-  queue: queueFilter,
-  batchingMaxWaitSeconds
+  extraFields,
+  fastestPollingRateMs = 100,
+  slowestPollingRateMs = 1e4,
+  pollingBackoffRate = 1.1
 }) {
-  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(databaseTable)) {
-    throw new Error(`Invalid databaseTable name: ${databaseTable}`);
+  function logError(msg) {
+    console.error(`Oxen Queue: ${msg}`);
   }
-  const queueFilterClause = queueFilter ? Array.isArray(queueFilter) ? `AND queue IN (${queueFilter.map(() => "?").join(", ")})` : "AND queue = ?" : "";
-  const queueFilterParams = queueFilter ? Array.isArray(queueFilter) ? queueFilter : [queueFilter] : [];
-  async function size() {
-    const rows = await db.select(
-      `SELECT COUNT(*) AS count FROM ${databaseTable} WHERE status = 'waiting' AND scheduledDate <= NOW() ${queueFilterClause}`,
-      queueFilterParams
-    );
-    return Number(rows[0]?.["count"] ?? 0);
+  const processingMap = /* @__PURE__ */ new Map();
+  function getIsProcessing(queueName) {
+    return !!processingMap.get(queueName);
   }
-  async function push(message) {
-    const { insertId } = await db.insert(
-      `INSERT INTO ${databaseTable} (queue, body, scheduledDate) VALUES (?, ?, GREATEST(?, NOW()))`,
-      [message.queue, JSON.stringify(message.body), instantFns.formatISO9075(message.scheduledDate)]
-    );
-    return String(insertId);
+  function setIsProcessing(queueName, processing) {
+    processingMap.set(queueName, processing);
   }
-  async function bulk(messages) {
-    if (messages.length === 0) return;
-    const placeholders = messages.map(() => "(?, ?, GREATEST(?, NOW()))").join(", ");
-    const params = [];
-    for (const message of messages) {
-      params.push(message.queue, JSON.stringify(message.body), instantFns.formatISO9075(message.scheduledDate));
-    }
-    await db.insert(`INSERT INTO ${databaseTable} (queue, body, scheduledDate) VALUES ${placeholders}`, params);
+  const jobRecoveryIntervalMap = /* @__PURE__ */ new Map();
+  function getJobRecoveryInterval(queueName) {
+    return jobRecoveryIntervalMap.get(queueName) ?? null;
   }
-  async function batchPop(count) {
-    const waiting = await db.select(
-      `SELECT 1 FROM ${databaseTable} WHERE lockId IS NULL AND status = 'waiting' AND scheduledDate <= NOW() ${queueFilterClause} LIMIT 1`,
-      queueFilterParams
-    );
-    if (waiting.length === 0) {
-      return [];
+  function setJobRecoveryInterval(queueName, jobRecoveryInterval) {
+    jobRecoveryIntervalMap.set(queueName, jobRecoveryInterval);
+  }
+  async function start({
+    workFn,
+    concurrency = 3,
+    timeout = 60 * 45,
+    shouldRecoverStuckJobs = true,
+    queueName
+  }) {
+    if (getIsProcessing(queueName)) {
+      throw new Error(`This queue is already processing`);
     }
-    const lockId = randomInt(0, 2 ** 48 - 1);
-    const locked = await db.update(
-      `UPDATE ${databaseTable}
-       INNER JOIN (
-         SELECT id, attempt FROM ${databaseTable} FORCE INDEX (locking_pop)
-         WHERE lockId IS NULL
-         AND status = 'waiting'
-         AND scheduledDate <= NOW()
-         ${queueFilterClause}
-         ORDER BY scheduledDate ASC
-         LIMIT ?
-       ) AS sub ON sub.id = ${databaseTable}.id AND sub.attempt = ${databaseTable}.attempt
-       SET lockId = ?, status = 'processing', startedDate = NOW()`,
-      [...queueFilterParams, count, lockId]
-    );
-    if (locked.changedRows === 0) {
-      return [];
+    if (!workFn) {
+      throw new Error(
+        `Missing work_fn argument, nothing to do! Remember that the process() function takes an object as its single argument.`
+      );
     }
-    const rows = await db.select(`SELECT id, queue, body, attempt FROM ${databaseTable} WHERE lockId = ?`, [lockId]);
-    return rows.map(dbRowToMessage);
-  }
-  const pop = makePop({ batchPop, maxWaitSeconds: batchingMaxWaitSeconds });
-  async function fail({ id, attempts, error }) {
-    await db.update(
-      `UPDATE ${databaseTable} SET status = 'failedPermanently', completedDate = NOW(), result = ? WHERE id = ? AND attempt = ? LIMIT 1`,
-      [error.stack, id, attempts]
-    );
-  }
-  async function success(options) {
-    await db.update(
-      `UPDATE ${databaseTable} SET status = 'succeeded', completedDate = NOW(), result = ? WHERE id = ? AND attempt = ? LIMIT 1`,
-      [JSON.stringify(options.result), options.id, options.attempts]
-    );
-  }
-  async function release(options) {
-    await db.update(`UPDATE ${databaseTable} SET status = 'failed', result = ? WHERE id = ? AND attempt = ? LIMIT 1`, [
-      options.error.stack,
-      options.id,
-      options.attempts
-    ]);
-    await db.insert(
-      `INSERT INTO ${databaseTable} (id, attempt, queue, body, scheduledDate)
-       SELECT id, ?, queue, body, DATE_ADD(NOW(), INTERVAL ? SECOND)
-       FROM ${databaseTable}
-       WHERE id = ? AND attempt = ? LIMIT 1`,
-      [options.attempts + 1, options.delaySeconds, options.id, options.attempts]
-    );
-  }
-  return { size, push, bulk, pop, fail, success, release };
-}
-function dbRowToMessage(row) {
-  return {
-    id: String(row["id"]),
-    queue: row["queue"],
-    body: typeof row["body"] === "string" ? JSON.parse(row["body"]) : row["body"],
-    attempts: row["attempt"]
-  };
-}
-function makePop({
-  batchPop,
-  maxWaitSeconds = 0
-}) {
-  let waiting = [];
-  function processBatch() {
-    const batch = waiting;
-    waiting = [];
-    batchPop(batch.length).then((messages) => {
-      batch.forEach((b, index) => {
-        if (index < messages.length) {
-          b[0](messages[index]);
-        } else {
-          b[0](null);
-        }
+    if (typeof concurrency !== "number") {
+      throw new Error(`The concurrency argument must be a number`);
+    }
+    let jobTimeoutSeconds = Math.floor(timeout);
+    let inProcess = 0;
+    let pollingRate = fastestPollingRateMs;
+    let currentlyFetching = false;
+    const workingJobBatch = [];
+    setIsProcessing(queueName, true);
+    async function fillJobBatch() {
+      const batchId = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+      const lockedBatch = await db.update(
+        `
+            UPDATE ${databaseTable} AS main
+            INNER JOIN (
+                SELECT id FROM ${databaseTable} FORCE INDEX (locking_update)
+                WHERE batchId IS NULL
+                AND STATUS = "waiting"
+                AND jobType = ?
+                AND scheduledDate <= NOW()
+                ORDER BY priority ASC LIMIT ${concurrency}
+            ) sub
+            ON sub.id = main.id
+            SET batchId = ?, STATUS = "processing", startedDate = NOW()`,
+        [queueName, batchId]
+      );
+      if (lockedBatch.changedRows === 0) {
+        return false;
+      }
+      const nextJobs = await db.select(
+        `SELECT id, scheduledDate, body, trace FROM ${databaseTable} WHERE batchId = ? ORDER BY priority ASC LIMIT ${concurrency}`,
+        [batchId]
+      );
+      if (nextJobs.length === 0) {
+        return false;
+      }
+      nextJobs.map((db2) => ({
+        id: db2.id,
+        scheduledDate: new Date(db2.scheduledDate).toISOString(),
+        body: db2.body ? JSON.parse(db2.body) : db2.body,
+        trace: db2.trace ? JSON.parse(db2.trace) : db2.trace
+      })).forEach((job) => {
+        workingJobBatch.push(job);
       });
-    }).catch((e) => batch.forEach((b) => b[1](e)));
-  }
-  function pop() {
-    if (maxWaitSeconds <= 0) {
-      return batchPop(1).then((m) => m.length ? m[0] : null);
-    } else {
-      const promise = new Promise((resolve, reject) => waiting.push([resolve, reject]));
-      if (waiting.length === 1) {
-        setTimeout(processBatch, maxWaitSeconds * 1e3);
+      return true;
+    }
+    async function getNextJob() {
+      if (!currentlyFetching && workingJobBatch.length < concurrency) {
+        currentlyFetching = true;
+        await fillJobBatch().catch((error) => {
+          logError("There was an error while trying to get the next set of jobs:");
+          logError(error);
+        });
+        currentlyFetching = false;
       }
-      return promise;
+      return workingJobBatch.shift();
     }
-  }
-  return pop;
-}
-
-// src/local-queue-driver.ts
-import { instantFns as instantFns2 } from "iso-fns2";
-function localQueueDriver({ queue: queueFilter } = {}) {
-  const filterSet = queueFilter ? new Set(Array.isArray(queueFilter) ? queueFilter : [queueFilter]) : null;
-  const messages = [];
-  let nextId = 1;
-  function matchesFilter(queue) {
-    return !filterSet || filterSet.has(queue);
-  }
-  function eligible(msg) {
-    return msg.status === "waiting" && msg.scheduledDate <= instantFns2.now() && matchesFilter(msg.queue);
-  }
-  function find(id, attempts) {
-    return messages.find((m) => m.id === Number(id) && m.attempt === attempts);
-  }
-  async function size() {
-    return messages.filter(eligible).length;
-  }
-  async function push(message) {
-    const id = nextId++;
-    const now = instantFns2.now();
-    messages.push({
-      id,
-      attempt: 0,
-      queue: message.queue,
-      status: "waiting",
-      scheduledDate: message.scheduledDate > now ? message.scheduledDate : now,
-      body: structuredClone(message.body)
-    });
-    return String(id);
-  }
-  async function bulk(input) {
-    if (input.length === 0) return;
-    const now = instantFns2.now();
-    for (const message of input) {
-      const id = nextId++;
-      messages.push({
-        id,
-        attempt: 0,
-        queue: message.queue,
-        status: "waiting",
-        scheduledDate: message.scheduledDate > now ? message.scheduledDate : now,
-        body: structuredClone(message.body)
+    async function doWork() {
+      const job = await getNextJob();
+      if (!job) {
+        if (pollingRate < slowestPollingRateMs) {
+          pollingRate *= pollingBackoffRate;
+        }
+        if (pollingRate >= slowestPollingRateMs) {
+          pollingRate = slowestPollingRateMs;
+        }
+        return Promise.resolve("no_job");
+      }
+      pollingRate = fastestPollingRateMs;
+      const promises = [
+        workFn({ body: job.body, id: job.id, trace: job.trace, scheduledDate: job.scheduledDate }),
+        new Promise((_, reject) => {
+          setTimeout(
+            () => reject(new Error(`timeout for job_id ${job.id} (over ${jobTimeoutSeconds} seconds)`)),
+            typeof jobTimeoutSeconds === "number" ? jobTimeoutSeconds * 1e3 : 0
+          );
+        })
+      ];
+      return Promise.race(promises).then(async (jobResult) => {
+        return db.update(
+          `
+                  update ${databaseTable} 
+                  set 
+                      result = ?, 
+                      uniqueKey = NULL,
+                      status="success",
+                      runningTimeSeconds = TIMESTAMPDIFF(SECOND,startedDate,NOW())
+                  where id =?
+                  LIMIT 1`,
+          [JSON.stringify(jobResult), job.id]
+        );
+      }).catch(async (error) => {
+        return db.update(
+          `
+                  update ${databaseTable} 
+                  set 
+                      result = ?, 
+                      uniqueKey = NULL,
+                      status="error",
+                      runningTimeSeconds = TIMESTAMPDIFF(SECOND,startedDate,NOW())
+                  where id = ?
+                  LIMIT 1`,
+          [error.stack, job.id]
+        );
       });
     }
+    async function loop() {
+      if (inProcess < concurrency) {
+        inProcess++;
+        doWork().then(() => {
+          inProcess--;
+        }).catch((error) => {
+          inProcess--;
+          logError("Unhandled Oxen Queue error:");
+          logError(error);
+        });
+      } else if (pollingRate < slowestPollingRateMs) {
+        pollingRate *= pollingBackoffRate;
+        if (pollingRate >= slowestPollingRateMs) {
+          pollingRate = slowestPollingRateMs;
+        }
+      }
+      if (!getIsProcessing(queueName)) {
+        return;
+      }
+      setTimeout(function() {
+        if (!getIsProcessing(queueName)) {
+          return;
+        }
+        loop();
+      }, pollingRate ?? 0);
+    }
+    async function recoverStuckJobs() {
+      return db.update(
+        `
+            UPDATE ${databaseTable} 
+            SET 
+                STATUS="waiting",
+                batchId = NULL,
+                startedDate = NULL,
+                recovered = 1
+            WHERE
+                STATUS="processing" AND
+                startedDate < (NOW() - INTERVAL ${jobTimeoutSeconds} SECOND) AND
+                jobType = ?
+            `,
+        [queueName]
+      );
+    }
+    async function markStuckJobs() {
+      return db.update(
+        `
+            UPDATE ${databaseTable} 
+            SET 
+                STATUS="stuck",
+                uniqueKey = NULL,
+                recovered = 1
+            WHERE
+                STATUS="processing" AND
+                startedDate < (NOW() - INTERVAL ${jobTimeoutSeconds} SECOND) AND
+                jobType = ?
+            `,
+        [queueName]
+      );
+    }
+    loop();
+    setJobRecoveryInterval(
+      queueName,
+      setInterval(function() {
+        if (shouldRecoverStuckJobs) {
+          recoverStuckJobs().catch(function(error) {
+            logError("Unable to recover stuck jobs:");
+            logError(error);
+          });
+        } else {
+          markStuckJobs().catch(function(error) {
+            logError("Unable to mark stuck jobs:");
+            logError(error);
+          });
+        }
+      }, 1e3 * 60)
+      //every minute
+    );
   }
-  async function pop() {
-    const ready = messages.filter(eligible).sort((a, b) => a.scheduledDate < b.scheduledDate ? -1 : 1);
-    const msg = ready[0];
-    if (!msg) return null;
-    msg.status = "processing";
-    return { id: String(msg.id), queue: msg.queue, body: structuredClone(msg.body), attempts: msg.attempt };
+  function stop(options) {
+    setIsProcessing(options.queueName, false);
+    const jobRecoveryInterval = getJobRecoveryInterval(options.queueName);
+    if (jobRecoveryInterval) clearInterval(jobRecoveryInterval);
   }
-  async function fail({ id, attempts, error }) {
-    const msg = find(id, attempts);
-    if (msg) {
-      msg.status = "failedPermanently";
-      msg.result = error.stack;
-    }
+  async function listMessages({
+    pageNumber,
+    pageSize,
+    queueName
+  }) {
+    const skip = pageNumber * pageSize;
+    const results = await db.select(`SELECT * FROM ${databaseTable} WHERE jobType = ? ORDER BY id DESC LIMIT ? OFFSET ?`, [
+      queueName,
+      pageSize,
+      skip
+    ]);
+    return results.map((r) => dbToJob(r));
   }
-  async function success({ id, attempts, result }) {
-    const msg = find(id, attempts);
-    if (msg) {
-      msg.status = "succeeded";
-      msg.result = JSON.stringify(result);
+  async function addMessage(job) {
+    const uniqueKey = job.uniqueKey === void 0 ? null : parseInt(crypto.createHash("md5").update(`${job.uniqueKey}|${job.queueName}`).digest("hex").slice(0, 8), 16);
+    const inserts = [
+      JSON.stringify(job.body),
+      job.traceData ? JSON.stringify(job.traceData) : null,
+      job.queueName,
+      uniqueKey,
+      typeof job.priority === "number" ? job.priority : Date.now()
+    ];
+    extraFields?.forEach((field) => {
+      inserts.push(job[field]);
+    });
+    const fields = ["body", "trace", "jobType", "uniqueKey", "priority", ...extraFields ?? []];
+    const { insertId } = await db.insert(
+      `
+                insert into ${databaseTable} (
+                    ${fields.join(",")}, scheduledDate
+                ) 
+                VALUES (?, GREATEST(?, NOW())) 
+                ON DUPLICATE KEY UPDATE 
+                priority = IF(priority > VALUES(priority), VALUES(priority), priority)
+            `,
+      [inserts, instantFns.formatISO9075(job.scheduledDate ?? instantFns.now())]
+    );
+    if (!insertId) {
+      const results = await db.select(`SELECT id FROM ${databaseTable} WHERE uniqueKey = ?`, [uniqueKey]);
+      const row = results[0];
+      assert.ok(row, "Could not find inserted row");
+      return { id: row["id"] };
+    } else {
+      return { id: insertId };
     }
   }
-  async function release({
-    id,
-    attempts,
-    error,
-    delaySeconds
+  async function updateMessage({
+    queueName,
+    messageId,
+    scheduledDate,
+    body
   }) {
-    const msg = find(id, attempts);
-    if (msg) {
-      msg.status = "failed";
-      msg.result = error.stack;
-      messages.push({
-        id: msg.id,
-        attempt: attempts + 1,
-        queue: msg.queue,
-        status: "waiting",
-        scheduledDate: instantFns2.add(instantFns2.now(), { seconds: delaySeconds }),
-        body: structuredClone(msg.body)
-      });
-    }
+    const result = instantFns.formatISO9075(scheduledDate);
+    await db.update(
+      `UPDATE ${databaseTable} SET scheduledDate = GREATEST(?, NOW()), body = ? WHERE jobType = ? AND id = ? AND status = "waiting"`,
+      [result, JSON.stringify(body), queueName, messageId]
+    );
+  }
+  async function isQueueEmpty({ queueName }) {
+    const results = await db.select(
+      `
+        SELECT id FROM ${databaseTable} FORCE INDEX (locking_update)
+        WHERE ((batchId IS NULL AND STATUS = "waiting") OR STATUS = "processing")
+        AND jobType = ?
+        AND scheduledDate <= NOW()
+        ORDER BY priority ASC LIMIT 1`,
+      [queueName]
+    );
+    return !results.length;
   }
-  return { size, push, bulk, pop, fail, success, release };
+  return { start, stop, listMessages, addMessage, updateMessage, isQueueEmpty };
 }
-
-// src/index.ts
-function combineLoggers(...loggers) {
-  const active = loggers.filter((l) => !!l);
+function dbToJob(db) {
   return {
-    info(msg) {
-      for (const l of active) l.info(msg);
-    },
-    warn(msg) {
-      for (const l of active) l.warn(msg);
-    },
-    error(msg) {
-      for (const l of active) l.error(msg);
-    }
+    id: db.id,
+    batchId: db.batchId,
+    queue: db.jobType,
+    scheduledDate: new Date(db.scheduledDate).toISOString(),
+    enqueuedDate: new Date(db.enqueuedDate).toISOString(),
+    startedDate: db.startedDate ? new Date(db.startedDate).toISOString() : null,
+    body: db.body ? JSON.parse(db.body) : db.body,
+    trace: db.trace ? JSON.parse(db.trace) : db.trace,
+    status: db.status,
+    result: db.result,
+    recovered: db.recovered,
+    runningTimeSeconds: db.runningTimeSeconds,
+    uniqueKey: db.uniqueKey,
+    priority: db.priority
   };
 }
-function createService({ driver, logger: serviceLogger }) {
-  const allQueues = /* @__PURE__ */ new Map();
-  function getDefaultDelaySeconds({
-    queueConfig,
-    queueMessage
-  }) {
-    if (typeof queueConfig.backoffSeconds === "number") {
-      return queueConfig.backoffSeconds;
-    } else if (!Array.isArray(queueConfig.backoffSeconds) || !queueConfig.backoffSeconds.length) {
-      return 60;
-    } else {
-      return queueConfig.backoffSeconds.length > queueMessage.attempts ? queueConfig.backoffSeconds[queueMessage.attempts] : queueConfig.backoffSeconds.at(-1);
-    }
+
+// src/index.ts
+var noopTracing = {
+  hydrate: (_trace, callback) => callback(),
+  dehydrate: () => null,
+  startActiveSpan: (_name, fn) => fn(),
+  getActiveSpan: () => void 0
+};
+function createService(config) {
+  const queues = [];
+  const tracing = config.tracing ?? noopTracing;
+  function list() {
+    return queues.map((q) => q.name);
   }
-  async function execute(options) {
-    const actions = {
-      fail(error) {
-        throw new FailQueueMessageError(error);
-      },
-      release(error, delaySeconds) {
-        throw new ReleaseQueueMessageError(error, delaySeconds);
-      }
-    };
-    const middleware = options.queueConfig.middleware ?? [];
-    const executeFn = middleware.reduceRight(
-      (acc, cur) => {
-        return (body) => cur(body, { ...actions, next: () => acc(body) });
-      },
-      (body) => options.queueConfig.handle(body, actions)
-    );
-    return executeFn(options.queueMessage.body);
+  function get(name) {
+    return queues.find((q) => q.name === name);
   }
-  async function processMessage(queueMessage) {
-    const queueConfig = allQueues.get(queueMessage.queue);
-    if (!queueConfig) {
-      throw new Error(`Queue not found with name "${queueMessage.queue}"`);
-    }
-    const logger = combineLoggers(serviceLogger, queueConfig.logger);
-    logger.info({ event: "messageDequeued", queue: queueMessage.queue, body: queueMessage.body, messageId: queueMessage.id });
-    try {
-      const result = await execute({ queueConfig, queueMessage });
-      await driver.success({ id: queueMessage.id, attempts: queueMessage.attempts, result });
-      logger.info({
-        event: "messageProcessed",
-        queue: queueMessage.queue,
-        body: queueMessage.body,
-        messageId: queueMessage.id,
-        result
-      });
-      return result;
-    } catch (e) {
-      if (e instanceof FailQueueMessageError) {
-        await driver.fail({ id: queueMessage.id, attempts: queueMessage.attempts, error: e.cause });
+  function create(options) {
+    const queueName = options.name;
+    const concurrency = options.concurrency ?? 1;
+    const logger = Log.create({
+      transports: [config.logger, ...options.logger ? [options.logger] : []]
+    });
+    async function workFn({
+      body,
+      id,
+      trace,
+      scheduledDate
+    }) {
+      try {
+        logger.info({
+          event: "messageDequeued",
+          queue: queueName,
+          driver: config.driver,
+          body,
+          messageId: id
+        });
+        const result = await tracing.hydrate(trace, async () => {
+          return tracing.startActiveSpan(
+            { name: "queue_consumer_transaction", attributes: { "messaging.destination.name": options.name } },
+            () => {
+              const parent = tracing.getActiveSpan();
+              return tracing.startActiveSpan(
+                {
+                  name: "queue_consumer",
+                  attributes: {
+                    op: "queue.process",
+                    "sentry.op": "queue.process",
+                    "messaging.message.id": id,
+                    "messaging.destination.name": options.name,
+                    "messaging.message.receive.latency": instantFns2.getEpochMilliseconds(instantFns2.now()) - instantFns2.getEpochMilliseconds(scheduledDate)
+                  }
+                },
+                async () => {
+                  const res = await options.workFn(body);
+                  parent?.setStatus({ code: 1, message: "ok" });
+                  return res;
+                }
+              );
+            }
+          );
+        });
+        logger.info({
+          event: "messageProcessed",
+          queue: queueName,
+          driver: config.driver,
+          body,
+          result,
+          messageId: id
+        });
+        return result;
+      } catch (e) {
         logger.error({
           event: "messageFailed",
-          queue: queueMessage.queue,
-          body: queueMessage.body,
-          messageId: queueMessage.id,
-          error: e.cause
+          queue: queueName,
+          driver: config.driver,
+          body,
+          error: e,
+          messageId: id
         });
         throw e;
-      } else {
-        const isLastAttempt = queueMessage.attempts + 1 >= (queueConfig.tries ?? 1);
-        const error = e instanceof ReleaseQueueMessageError ? e.cause : e instanceof Error ? e : new Error(String(e));
-        if (isLastAttempt) {
-          await driver.fail({ id: queueMessage.id, attempts: queueMessage.attempts, error });
-          logger.error({
-            event: "messageFailed",
-            queue: queueMessage.queue,
-            body: queueMessage.body,
-            messageId: queueMessage.id,
-            error
-          });
-        } else {
-          await driver.release({
-            id: queueMessage.id,
-            attempts: queueMessage.attempts,
-            error,
-            delaySeconds: e instanceof ReleaseQueueMessageError ? e.delaySeconds ?? getDefaultDelaySeconds({ queueConfig, queueMessage }) : getDefaultDelaySeconds({ queueConfig, queueMessage })
-          });
-          logger.warn({
-            event: "messageReleased",
-            queue: queueMessage.queue,
-            body: queueMessage.body,
-            messageId: queueMessage.id,
-            error,
-            attempt: queueMessage.attempts + 1
-          });
-        }
-        throw e;
       }
     }
-  }
-  let workPromises = [];
-  function work({ maxJobs, maxTimeSeconds, stopWhenEmpty, sleepSeconds, concurrency = 1 }) {
-    const workAbortController = new AbortController();
-    let messagesProcessed = 0;
-    let messagesPopping = 0;
-    let timeoutToClear = null;
-    if (maxTimeSeconds) {
-      timeoutToClear = setTimeout(() => workAbortController.abort("max time exceeded"), maxTimeSeconds * 1e3);
+    async function listMessages({ pageSize, pageNumber }) {
+      return config.driver.listMessages({ pageSize, pageNumber, queueName });
     }
-    async function pollForWork() {
-      let didThrowDuringPop = false;
-      while (!workAbortController.signal.aborted) {
-        try {
-          if (didThrowDuringPop) {
-            await new Promise((r) => setTimeout(r, 1e3));
-            didThrowDuringPop = false;
-          }
-          if (typeof maxJobs === "number" && messagesProcessed >= maxJobs) {
-            workAbortController.abort("max jobs reached");
-          } else {
-            if (typeof maxJobs === "number" && messagesProcessed + messagesPopping >= maxJobs) {
-              await new Promise((r) => setTimeout(r, 1e3 * (sleepSeconds || 1)));
-            } else {
-              messagesPopping++;
-              const message = await driver.pop().catch((e) => {
-                didThrowDuringPop = true;
-                messagesPopping--;
-                throw e;
-              });
-              messagesPopping--;
-              if (!message && stopWhenEmpty) {
-                workAbortController.abort("queue empty");
-              } else if (!message) {
-                await new Promise((r) => setTimeout(r, 1e3 * (sleepSeconds || 1)));
-              } else {
-                messagesProcessed++;
-                await processMessage(message);
+    async function addMessage(body, options2) {
+      const { id } = await tracing.hydrate(
+        options2.tracingContext === void 0 ? tracing.dehydrate() : options2.tracingContext,
+        async () => {
+          return tracing.startActiveSpan(
+            {
+              name: "queue_producer",
+              attributes: {
+                op: "queue.publish",
+                "sentry.op": "queue.publish",
+                "messaging.destination.name": queueName
               }
+            },
+            async () => {
+              const span = tracing.getActiveSpan();
+              const { id: id2 } = await config.driver.addMessage({
+                body,
+                scheduledDate: options2.scheduledDate,
+                queueName,
+                traceData: tracing.dehydrate()
+              });
+              span?.setAttribute("messaging.message.id", id2);
+              return { id: id2 };
             }
-          }
-          await new Promise((r) => setTimeout(r, 0));
-        } catch (e) {
+          );
         }
-      }
-    }
-    const promise = Promise.all(Array.from({ length: concurrency }).map(() => pollForWork())).then(() => {
-    });
-    const result = Object.assign(promise, {
-      abort: (reason) => {
-        workAbortController.abort(reason);
-        if (timeoutToClear) clearTimeout(timeoutToClear);
-      }
-    });
-    workPromises.push(result);
-    result.finally(() => {
-      workPromises = workPromises.filter((w) => w !== result);
-      if (timeoutToClear) {
-        clearTimeout(timeoutToClear);
-      }
-    });
-    return result;
-  }
-  async function stopAllWork() {
-    const current = [...workPromises];
-    current.forEach((w) => w.abort());
-    await Promise.all(current);
-  }
-  function create(queueConfig) {
-    if (allQueues.has(queueConfig.name)) {
-      throw new Error(`Cannot have multiple queues with name "${queueConfig.name}"`);
+      );
+      logger.info({
+        event: "messageEnqueued",
+        queue: queueName,
+        body,
+        scheduledDate: options2.scheduledDate ?? null,
+        driver: config.driver,
+        messageId: id
+      });
     }
-    allQueues.set(queueConfig.name, queueConfig);
-    const logger = combineLoggers(serviceLogger, queueConfig.logger);
-    async function dispatch(body) {
-      await driver.push({ queue: queueConfig.name, body: body ?? {}, scheduledDate: instantFns3.now() });
-      logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
+    async function updateMessage({
+      messageId,
+      scheduledDate,
+      body
+    }) {
+      return config.driver.updateMessage({ messageId, body, scheduledDate, queueName });
     }
-    async function dispatchWithDelay(...params) {
-      const body = typeof params[0] === "string" ? {} : params[0];
-      const scheduledDate = typeof params[0] === "string" ? params[0] : params[1];
-      await driver.push({ queue: queueConfig.name, body: params, scheduledDate });
-      logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
+    function processUntilEmpty(options2) {
+      const interval = setInterval(() => {
+        config.driver.isQueueEmpty({ queueName }).then((isEmpty2) => {
+          if (isEmpty2) {
+            declareEmpty(null);
+          }
+        });
+      }, options2?.pollingMs ?? 3e3);
+      let declareEmpty = (value) => {
+      };
+      const isEmpty = new Promise((resolve) => {
+        declareEmpty = resolve;
+      }).then(() => clearInterval(interval)).then(() => {
+        return stopProcessing2();
+      });
+      startProcessing2();
+      return isEmpty;
     }
-    async function dispatchSync(body) {
-      logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
-      const queueMessage = { id: randomUUID(), body, queue: queueConfig.name, attempts: 0 };
-      const result = await execute({ queueConfig, queueMessage });
-      logger.info({ event: "messageProcessed", queue: queueConfig.name, body, messageId: queueMessage.id, result });
-      return result;
+    function startProcessing2() {
+      config.driver.start({ queueName, workFn, concurrency });
+      logger.info({
+        event: "queueStarted",
+        queue: queueName,
+        driver: config.driver
+      });
     }
-    async function bulk(messages) {
-      await driver.bulk(
-        messages.map(({ body, scheduledDate }) => ({
-          queue: queueConfig.name,
-          body,
-          scheduledDate: scheduledDate ?? instantFns3.now()
-        }))
-      );
-      for (const { body } of messages) {
-        logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
-      }
+    function stopProcessing2() {
+      config.driver.stop({ queueName });
+      logger.info({
+        event: "queueStopped",
+        queue: queueName,
+        driver: config.driver
+      });
     }
-    return { dispatch, dispatchWithDelay, dispatchSync, bulk };
+    const queue = {
+      listMessages,
+      addMessage,
+      updateMessage,
+      startProcessing: startProcessing2,
+      processUntilEmpty,
+      stopProcessing: stopProcessing2,
+      name: options.name
+    };
+    queues.push(queue);
+    return queue;
   }
-  return { work, stopAllWork, create, processMessage };
+  function startProcessing() {
+    queues.forEach((q) => q.startProcessing());
+  }
+  function stopProcessing() {
+    queues.forEach((q) => q.stopProcessing());
+  }
+  return { list, get, create, startProcessing, stopProcessing };
 }
 var registry = ServiceRegistry.createRegistry(
   ServiceRegistry.proxyFunctionsForObject
 );
 var Provider = {
   create: createService,
-  register: registry.register
+  register: registry.register,
+  list: registry.list
 };
 function provider(key) {
   const service = registry.resolve(key);
   return {
-    work: service.work,
-    stopAllWork: service.stopAllWork,
-    processMessage: service.processMessage,
-    create: service.create
+    list: service.list,
+    get: service.get,
+    create: service.create,
+    startProcessing: service.startProcessing,
+    stopProcessing: service.stopProcessing
   };
 }
 var facade = provider("default");
-var FailQueueMessageError = class extends Error {
-  cause;
-  constructor(cause) {
-    super(void 0, { cause });
-    this.cause = cause;
-  }
-};
-var ReleaseQueueMessageError = class extends Error {
-  cause;
-  delaySeconds = void 0;
-  constructor(cause, delaySeconds) {
-    super(void 0, { cause });
-    this.delaySeconds = delaySeconds;
-    this.cause = cause;
-  }
-};
 var Queue = {
+  drivers: {
+    db: DbQueueDriver
+  },
   Provider,
   provider,
-  work: facade.work,
-  stopAllWork: facade.stopAllWork,
-  processMessage: facade.processMessage,
+  list: facade.list,
+  get: facade.get,
   create: facade.create,
-  drivers: {
-    db: DbQueueDriver,
-    local: localQueueDriver
-  }
+  startProcessing: facade.startProcessing,
+  stopProcessing: facade.stopProcessing
 };
 export {
   Queue