@maestro-js/queue 1.0.0-alpha.1 → 1.0.0-alpha.11

package/dist/index.js

@@ -1,544 +1,491 @@
 // src/index.ts
-import { instantFns as instantFns2 } from "iso-fns2";
-import { Log } from "@maestro-js/log";
+import { randomUUID } from "crypto";
+import { instantFns as instantFns3 } from "iso-fns2";
 import { ServiceRegistry } from "@maestro-js/service-registry";
 
 // src/db-queue-driver.ts
-import crypto from "crypto";
+import { randomInt } from "crypto";
 import { instantFns } from "iso-fns2";
-import assert from "assert";
 function DbQueueDriver({
   db,
   databaseTable,
-  extraFields,
-  fastestPollingRateMs = 100,
-  slowestPollingRateMs = 1e4,
-  pollingBackoffRate = 1.1
+  queue: queueFilter,
+  batchingMaxWaitSeconds
 }) {
-  function logError(msg) {
-    console.error(`Oxen Queue: ${msg}`);
-  }
-  const processingMap = /* @__PURE__ */ new Map();
-  function getIsProcessing(queueName) {
-    return !!processingMap.get(queueName);
-  }
-  function setIsProcessing(queueName, processing) {
-    processingMap.set(queueName, processing);
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(databaseTable)) {
+    throw new Error(`Invalid databaseTable name: ${databaseTable}`);
   }
-  const jobRecoveryIntervalMap = /* @__PURE__ */ new Map();
-  function getJobRecoveryInterval(queueName) {
-    return jobRecoveryIntervalMap.get(queueName) ?? null;
+  const queueFilterClause = queueFilter ? Array.isArray(queueFilter) ? `AND queue IN (${queueFilter.map(() => "?").join(", ")})` : "AND queue = ?" : "";
+  const queueFilterParams = queueFilter ? Array.isArray(queueFilter) ? queueFilter : [queueFilter] : [];
+  async function size() {
+    const rows = await db.select(
+      `SELECT COUNT(*) AS count FROM ${databaseTable} WHERE status = 'waiting' AND scheduledDate <= NOW() ${queueFilterClause}`,
+      queueFilterParams
+    );
+    return Number(rows[0]?.["count"] ?? 0);
   }
-  function setJobRecoveryInterval(queueName, jobRecoveryInterval) {
-    jobRecoveryIntervalMap.set(queueName, jobRecoveryInterval);
+  async function push(message) {
+    const { insertId } = await db.insert(
+      `INSERT INTO ${databaseTable} (queue, body, scheduledDate) VALUES (?, ?, GREATEST(?, NOW()))`,
+      [message.queue, JSON.stringify(message.body), instantFns.formatISO9075(message.scheduledDate)]
+    );
+    return String(insertId);
   }
-  async function start({
-    workFn,
-    concurrency = 3,
-    timeout = 60 * 45,
-    shouldRecoverStuckJobs = true,
-    queueName
-  }) {
-    if (getIsProcessing(queueName)) {
-      throw new Error(`This queue is already processing`);
-    }
-    if (!workFn) {
-      throw new Error(
-        `Missing work_fn argument, nothing to do! Remember that the process() function takes an object as its single argument.`
-      );
-    }
-    if (typeof concurrency !== "number") {
-      throw new Error(`The concurrency argument must be a number`);
+  async function bulk(messages) {
+    if (messages.length === 0) return;
+    const placeholders = messages.map(() => "(?, ?, GREATEST(?, NOW()))").join(", ");
+    const params = [];
+    for (const message of messages) {
+      params.push(message.queue, JSON.stringify(message.body), instantFns.formatISO9075(message.scheduledDate));
     }
-    let jobTimeoutSeconds = Math.floor(timeout);
-    let inProcess = 0;
-    let pollingRate = fastestPollingRateMs;
-    let currentlyFetching = false;
-    const workingJobBatch = [];
-    setIsProcessing(queueName, true);
-    async function fillJobBatch() {
-      const batchId = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
-      const lockedBatch = await db.update(
-        `
-            UPDATE ${databaseTable} AS main
-            INNER JOIN (
-                SELECT id FROM ${databaseTable} FORCE INDEX (locking_update)
-                WHERE batchId IS NULL
-                AND STATUS = "waiting"
-                AND jobType = ?
-                AND scheduledDate <= NOW()
-                ORDER BY priority ASC LIMIT ${concurrency}
-            ) sub
-            ON sub.id = main.id
-            SET batchId = ?, STATUS = "processing", startedDate = NOW()`,
-        [queueName, batchId]
-      );
-      if (lockedBatch.changedRows === 0) {
-        return false;
-      }
-      const nextJobs = await db.select(
-        `SELECT id, scheduledDate, body, trace FROM ${databaseTable} WHERE batchId = ? ORDER BY priority ASC LIMIT ${concurrency}`,
-        [batchId]
-      );
-      if (nextJobs.length === 0) {
-        return false;
-      }
-      nextJobs.map((db2) => ({
-        id: db2.id,
-        scheduledDate: new Date(db2.scheduledDate).toISOString(),
-        body: db2.body ? JSON.parse(db2.body) : db2.body,
-        trace: db2.trace ? JSON.parse(db2.trace) : db2.trace
-      })).forEach((job) => {
-        workingJobBatch.push(job);
-      });
-      return true;
+    await db.insert(`INSERT INTO ${databaseTable} (queue, body, scheduledDate) VALUES ${placeholders}`, params);
+  }
+  async function batchPop(count) {
+    const waiting = await db.select(
+      `SELECT 1 FROM ${databaseTable} WHERE lockId IS NULL AND status = 'waiting' AND scheduledDate <= NOW() ${queueFilterClause} LIMIT 1`,
+      queueFilterParams
+    );
+    if (waiting.length === 0) {
+      return [];
     }
-    async function getNextJob() {
-      if (!currentlyFetching && workingJobBatch.length < concurrency) {
-        currentlyFetching = true;
-        await fillJobBatch().catch((error) => {
-          logError("There was an error while trying to get the next set of jobs:");
-          logError(error);
-        });
-        currentlyFetching = false;
-      }
-      return workingJobBatch.shift();
+    const lockId = randomInt(0, 2 ** 48 - 1);
+    const locked = await db.update(
+      `UPDATE ${databaseTable}
+       INNER JOIN (
+         SELECT id, attempt FROM ${databaseTable} FORCE INDEX (locking_pop)
+         WHERE lockId IS NULL
+         AND status = 'waiting'
+         AND scheduledDate <= NOW()
+         ${queueFilterClause}
+         ORDER BY scheduledDate ASC
+         LIMIT ?
+       ) AS sub ON sub.id = ${databaseTable}.id AND sub.attempt = ${databaseTable}.attempt
+       SET lockId = ?, status = 'processing', startedDate = NOW()`,
+      [...queueFilterParams, count, lockId]
+    );
+    if (locked.changedRows === 0) {
+      return [];
     }
-    async function doWork() {
-      const job = await getNextJob();
-      if (!job) {
-        if (pollingRate < slowestPollingRateMs) {
-          pollingRate *= pollingBackoffRate;
-        }
-        if (pollingRate >= slowestPollingRateMs) {
-          pollingRate = slowestPollingRateMs;
+    const rows = await db.select(`SELECT id, queue, body, attempt FROM ${databaseTable} WHERE lockId = ?`, [lockId]);
+    return rows.map(dbRowToMessage);
+  }
+  const pop = makePop({ batchPop, maxWaitSeconds: batchingMaxWaitSeconds });
+  async function fail({ id, attempts, error }) {
+    await db.update(
+      `UPDATE ${databaseTable} SET status = 'failedPermanently', completedDate = NOW(), result = ? WHERE id = ? AND attempt = ? LIMIT 1`,
+      [error.stack, id, attempts]
+    );
+  }
+  async function success(options) {
+    await db.update(
+      `UPDATE ${databaseTable} SET status = 'succeeded', completedDate = NOW(), result = ? WHERE id = ? AND attempt = ? LIMIT 1`,
+      [JSON.stringify(options.result), options.id, options.attempts]
+    );
+  }
+  async function release(options) {
+    await db.update(`UPDATE ${databaseTable} SET status = 'failed', result = ? WHERE id = ? AND attempt = ? LIMIT 1`, [
+      options.error.stack,
+      options.id,
+      options.attempts
+    ]);
+    await db.insert(
+      `INSERT INTO ${databaseTable} (id, attempt, queue, body, scheduledDate)
+       SELECT id, ?, queue, body, DATE_ADD(NOW(), INTERVAL ? SECOND)
+       FROM ${databaseTable}
+       WHERE id = ? AND attempt = ? LIMIT 1`,
+      [options.attempts + 1, options.delaySeconds, options.id, options.attempts]
+    );
+  }
+  return { size, push, bulk, pop, fail, success, release };
+}
+function dbRowToMessage(row) {
+  return {
+    id: String(row["id"]),
+    queue: row["queue"],
+    body: typeof row["body"] === "string" ? JSON.parse(row["body"]) : row["body"],
+    attempts: row["attempt"]
+  };
+}
+function makePop({
+  batchPop,
+  maxWaitSeconds = 0
+}) {
+  let waiting = [];
+  function processBatch() {
+    const batch = waiting;
+    waiting = [];
+    batchPop(batch.length).then((messages) => {
+      batch.forEach((b, index) => {
+        if (index < messages.length) {
+          b[0](messages[index]);
+        } else {
+          b[0](null);
         }
-        return Promise.resolve("no_job");
-      }
-      pollingRate = fastestPollingRateMs;
-      const promises = [
-        workFn({ body: job.body, id: job.id, trace: job.trace, scheduledDate: job.scheduledDate }),
-        new Promise((_, reject) => {
-          setTimeout(
-            () => reject(new Error(`timeout for job_id ${job.id} (over ${jobTimeoutSeconds} seconds)`)),
-            typeof jobTimeoutSeconds === "number" ? jobTimeoutSeconds * 1e3 : 0
-          );
-        })
-      ];
-      return Promise.race(promises).then(async (jobResult) => {
-        return db.update(
-          `
-                  update ${databaseTable} 
-                  set 
-                      result = ?, 
-                      uniqueKey = NULL,
-                      status="success",
-                      runningTimeSeconds = TIMESTAMPDIFF(SECOND,startedDate,NOW())
-                  where id =?
-                  LIMIT 1`,
-          [JSON.stringify(jobResult), job.id]
-        );
-      }).catch(async (error) => {
-        return db.update(
-          `
-                  update ${databaseTable} 
-                  set 
-                      result = ?, 
-                      uniqueKey = NULL,
-                      status="error",
-                      runningTimeSeconds = TIMESTAMPDIFF(SECOND,startedDate,NOW())
-                  where id = ?
-                  LIMIT 1`,
-          [error.stack, job.id]
-        );
       });
-    }
-    async function loop() {
-      if (inProcess < concurrency) {
-        inProcess++;
-        doWork().then(() => {
-          inProcess--;
-        }).catch((error) => {
-          inProcess--;
-          logError("Unhandled Oxen Queue error:");
-          logError(error);
-        });
-      } else if (pollingRate < slowestPollingRateMs) {
-        pollingRate *= pollingBackoffRate;
-        if (pollingRate >= slowestPollingRateMs) {
-          pollingRate = slowestPollingRateMs;
-        }
-      }
-      if (!getIsProcessing(queueName)) {
-        return;
+    }).catch((e) => batch.forEach((b) => b[1](e)));
+  }
+  function pop() {
+    if (maxWaitSeconds <= 0) {
+      return batchPop(1).then((m) => m.length ? m[0] : null);
+    } else {
+      const promise = new Promise((resolve, reject) => waiting.push([resolve, reject]));
+      if (waiting.length === 1) {
+        setTimeout(processBatch, maxWaitSeconds * 1e3);
       }
-      setTimeout(function() {
-        if (!getIsProcessing(queueName)) {
-          return;
-        }
-        loop();
-      }, pollingRate ?? 0);
-    }
-    async function recoverStuckJobs() {
-      return db.update(
-        `
-            UPDATE ${databaseTable} 
-            SET 
-                STATUS="waiting",
-                batchId = NULL,
-                startedDate = NULL,
-                recovered = 1
-            WHERE
-                STATUS="processing" AND
-                startedDate < (NOW() - INTERVAL ${jobTimeoutSeconds} SECOND) AND
-                jobType = ?
-            `,
-        [queueName]
-      );
-    }
-    async function markStuckJobs() {
-      return db.update(
-        `
-            UPDATE ${databaseTable} 
-            SET 
-                STATUS="stuck",
-                uniqueKey = NULL,
-                recovered = 1
-            WHERE
-                STATUS="processing" AND
-                startedDate < (NOW() - INTERVAL ${jobTimeoutSeconds} SECOND) AND
-                jobType = ?
-            `,
-        [queueName]
-      );
+      return promise;
     }
-    loop();
-    setJobRecoveryInterval(
-      queueName,
-      setInterval(function() {
-        if (shouldRecoverStuckJobs) {
-          recoverStuckJobs().catch(function(error) {
-            logError("Unable to recover stuck jobs:");
-            logError(error);
-          });
-        } else {
-          markStuckJobs().catch(function(error) {
-            logError("Unable to mark stuck jobs:");
-            logError(error);
-          });
-        }
-      }, 1e3 * 60)
-      //every minute
-    );
   }
-  function stop(options) {
-    setIsProcessing(options.queueName, false);
-    const jobRecoveryInterval = getJobRecoveryInterval(options.queueName);
-    if (jobRecoveryInterval) clearInterval(jobRecoveryInterval);
+  return pop;
+}
+
+// src/local-queue-driver.ts
+import { instantFns as instantFns2 } from "iso-fns2";
+function localQueueDriver({ queue: queueFilter } = {}) {
+  const filterSet = queueFilter ? new Set(Array.isArray(queueFilter) ? queueFilter : [queueFilter]) : null;
+  const messages = [];
+  let nextId = 1;
+  function matchesFilter(queue) {
+    return !filterSet || filterSet.has(queue);
+  }
+  function eligible(msg) {
+    return msg.status === "waiting" && msg.scheduledDate <= instantFns2.now() && matchesFilter(msg.queue);
   }
-  async function listMessages({
-    pageNumber,
-    pageSize,
-    queueName
-  }) {
-    const skip = pageNumber * pageSize;
-    const results = await db.select(`SELECT * FROM ${databaseTable} WHERE jobType = ? ORDER BY id DESC LIMIT ? OFFSET ?`, [
-      queueName,
-      pageSize,
-      skip
-    ]);
-    return results.map((r) => dbToJob(r));
+  function find(id, attempts) {
+    return messages.find((m) => m.id === Number(id) && m.attempt === attempts);
   }
-  async function addMessage(job) {
-    const uniqueKey = job.uniqueKey === void 0 ? null : parseInt(crypto.createHash("md5").update(`${job.uniqueKey}|${job.queueName}`).digest("hex").slice(0, 8), 16);
-    const inserts = [
-      JSON.stringify(job.body),
-      job.traceData ? JSON.stringify(job.traceData) : null,
-      job.queueName,
-      uniqueKey,
-      typeof job.priority === "number" ? job.priority : Date.now()
-    ];
-    extraFields?.forEach((field) => {
-      inserts.push(job[field]);
+  async function size() {
+    return messages.filter(eligible).length;
+  }
+  async function push(message) {
+    const id = nextId++;
+    const now = instantFns2.now();
+    messages.push({
+      id,
+      attempt: 0,
+      queue: message.queue,
+      status: "waiting",
+      scheduledDate: message.scheduledDate > now ? message.scheduledDate : now,
+      body: structuredClone(message.body)
     });
-    const fields = ["body", "trace", "jobType", "uniqueKey", "priority", ...extraFields ?? []];
-    const { insertId } = await db.insert(
-      `
-                insert into ${databaseTable} (
-                    ${fields.join(",")}, scheduledDate
-                ) 
-                VALUES (?, GREATEST(?, NOW())) 
-                ON DUPLICATE KEY UPDATE 
-                priority = IF(priority > VALUES(priority), VALUES(priority), priority)
-            `,
-      [inserts, instantFns.formatISO9075(job.scheduledDate ?? instantFns.now())]
-    );
-    if (!insertId) {
-      const results = await db.select(`SELECT id FROM ${databaseTable} WHERE uniqueKey = ?`, [uniqueKey]);
-      const row = results[0];
-      assert.ok(row, "Could not find inserted row");
-      return { id: row["id"] };
-    } else {
-      return { id: insertId };
+    return String(id);
+  }
+  async function bulk(input) {
+    if (input.length === 0) return;
+    const now = instantFns2.now();
+    for (const message of input) {
+      const id = nextId++;
+      messages.push({
+        id,
+        attempt: 0,
+        queue: message.queue,
+        status: "waiting",
+        scheduledDate: message.scheduledDate > now ? message.scheduledDate : now,
+        body: structuredClone(message.body)
+      });
     }
   }
-  async function updateMessage({
-    queueName,
-    messageId,
-    scheduledDate,
-    body
-  }) {
-    const result = instantFns.formatISO9075(scheduledDate);
-    await db.update(
-      `UPDATE ${databaseTable} SET scheduledDate = GREATEST(?, NOW()), body = ? WHERE jobType = ? AND id = ? AND status = "waiting"`,
-      [result, JSON.stringify(body), queueName, messageId]
-    );
+  async function pop() {
+    const ready = messages.filter(eligible).sort((a, b) => a.scheduledDate < b.scheduledDate ? -1 : 1);
+    const msg = ready[0];
+    if (!msg) return null;
+    msg.status = "processing";
+    return { id: String(msg.id), queue: msg.queue, body: structuredClone(msg.body), attempts: msg.attempt };
   }
-  async function isQueueEmpty({ queueName }) {
-    const results = await db.select(
-      `
-        SELECT id FROM ${databaseTable} FORCE INDEX (locking_update)
-        WHERE ((batchId IS NULL AND STATUS = "waiting") OR STATUS = "processing")
-        AND jobType = ?
-        AND scheduledDate <= NOW()
-        ORDER BY priority ASC LIMIT 1`,
-      [queueName]
-    );
-    return !results.length;
+  async function fail({ id, attempts, error }) {
+    const msg = find(id, attempts);
+    if (msg) {
+      msg.status = "failedPermanently";
+      msg.result = error.stack;
+    }
   }
-  return { start, stop, listMessages, addMessage, updateMessage, isQueueEmpty };
+  async function success({ id, attempts, result }) {
+    const msg = find(id, attempts);
+    if (msg) {
+      msg.status = "succeeded";
+      msg.result = JSON.stringify(result);
+    }
+  }
+  async function release({
+    id,
+    attempts,
+    error,
+    delaySeconds
+  }) {
+    const msg = find(id, attempts);
+    if (msg) {
+      msg.status = "failed";
+      msg.result = error.stack;
+      messages.push({
+        id: msg.id,
+        attempt: attempts + 1,
+        queue: msg.queue,
+        status: "waiting",
+        scheduledDate: instantFns2.add(instantFns2.now(), { seconds: delaySeconds }),
+        body: structuredClone(msg.body)
+      });
+    }
+  }
+  return { size, push, bulk, pop, fail, success, release };
 }
-function dbToJob(db) {
+
+// src/index.ts
+function combineLoggers(...loggers) {
+  const active = loggers.filter((l) => !!l);
   return {
-    id: db.id,
-    batchId: db.batchId,
-    queue: db.jobType,
-    scheduledDate: new Date(db.scheduledDate).toISOString(),
-    enqueuedDate: new Date(db.enqueuedDate).toISOString(),
-    startedDate: db.startedDate ? new Date(db.startedDate).toISOString() : null,
-    body: db.body ? JSON.parse(db.body) : db.body,
-    trace: db.trace ? JSON.parse(db.trace) : db.trace,
-    status: db.status,
-    result: db.result,
-    recovered: db.recovered,
-    runningTimeSeconds: db.runningTimeSeconds,
-    uniqueKey: db.uniqueKey,
-    priority: db.priority
+    info(msg) {
+      for (const l of active) l.info(msg);
+    },
+    warn(msg) {
+      for (const l of active) l.warn(msg);
+    },
+    error(msg) {
+      for (const l of active) l.error(msg);
+    }
   };
 }
-
-// src/index.ts
-var noopTracing = {
-  hydrate: (_trace, callback) => callback(),
-  dehydrate: () => null,
-  startActiveSpan: (_name, fn) => fn(),
-  getActiveSpan: () => void 0
-};
-function createService(config) {
-  const queues = [];
-  const tracing = config.tracing ?? noopTracing;
-  function list() {
-    return queues.map((q) => q.name);
+function createService({ driver, logger: serviceLogger }) {
+  const allQueues = /* @__PURE__ */ new Map();
+  function getDefaultDelaySeconds({
+    queueConfig,
+    queueMessage
+  }) {
+    if (typeof queueConfig.backoffSeconds === "number") {
+      return queueConfig.backoffSeconds;
+    } else if (!Array.isArray(queueConfig.backoffSeconds) || !queueConfig.backoffSeconds.length) {
+      return 60;
+    } else {
+      return queueConfig.backoffSeconds.length > queueMessage.attempts ? queueConfig.backoffSeconds[queueMessage.attempts] : queueConfig.backoffSeconds.at(-1);
+    }
   }
-  function get(name) {
-    return queues.find((q) => q.name === name);
+  async function execute(options) {
+    const actions = {
+      fail(error) {
+        throw new FailQueueMessageError(error);
+      },
+      release(error, delaySeconds) {
+        throw new ReleaseQueueMessageError(error, delaySeconds);
+      }
+    };
+    const middleware = options.queueConfig.middleware ?? [];
+    const executeFn = middleware.reduceRight(
+      (acc, cur) => {
+        return (body) => cur(body, { ...actions, next: () => acc(body) });
+      },
+      (body) => options.queueConfig.handle(body, actions)
+    );
+    return executeFn(options.queueMessage.body);
   }
-  function create(options) {
-    const queueName = options.name;
-    const concurrency = options.concurrency ?? 1;
-    const logger = Log.create({
-      transports: [config.logger, ...options.logger ? [options.logger] : []]
-    });
-    async function workFn({
-      body,
-      id,
-      trace,
-      scheduledDate
-    }) {
-      try {
-        logger.info({
-          event: "messageDequeued",
-          queue: queueName,
-          driver: config.driver,
-          body,
-          messageId: id
-        });
-        const result = await tracing.hydrate(trace, async () => {
-          return tracing.startActiveSpan(
-            { name: "queue_consumer_transaction", attributes: { "messaging.destination.name": options.name } },
-            () => {
-              const parent = tracing.getActiveSpan();
-              return tracing.startActiveSpan(
-                {
-                  name: "queue_consumer",
-                  attributes: {
-                    op: "queue.process",
-                    "sentry.op": "queue.process",
-                    "messaging.message.id": id,
-                    "messaging.destination.name": options.name,
-                    "messaging.message.receive.latency": instantFns2.getEpochMilliseconds(instantFns2.now()) - instantFns2.getEpochMilliseconds(scheduledDate)
-                  }
-                },
-                async () => {
-                  const res = await options.workFn(body);
-                  parent?.setStatus({ code: 1, message: "ok" });
-                  return res;
-                }
-              );
-            }
-          );
-        });
-        logger.info({
-          event: "messageProcessed",
-          queue: queueName,
-          driver: config.driver,
-          body,
-          result,
-          messageId: id
-        });
-        return result;
-      } catch (e) {
+  async function processMessage(queueMessage) {
+    const queueConfig = allQueues.get(queueMessage.queue);
+    if (!queueConfig) {
+      throw new Error(`Queue not found with name "${queueMessage.queue}"`);
+    }
+    const logger = combineLoggers(serviceLogger, queueConfig.logger);
+    logger.info({ event: "messageDequeued", queue: queueMessage.queue, body: queueMessage.body, messageId: queueMessage.id });
+    try {
+      const result = await execute({ queueConfig, queueMessage });
+      await driver.success({ id: queueMessage.id, attempts: queueMessage.attempts, result });
+      logger.info({
+        event: "messageProcessed",
+        queue: queueMessage.queue,
+        body: queueMessage.body,
+        messageId: queueMessage.id,
+        result
+      });
+      return result;
+    } catch (e) {
+      if (e instanceof FailQueueMessageError) {
+        await driver.fail({ id: queueMessage.id, attempts: queueMessage.attempts, error: e.cause });
         logger.error({
           event: "messageFailed",
-          queue: queueName,
-          driver: config.driver,
-          body,
-          error: e,
-          messageId: id
+          queue: queueMessage.queue,
+          body: queueMessage.body,
+          messageId: queueMessage.id,
+          error: e.cause
         });
         throw e;
+      } else {
+        const isLastAttempt = queueMessage.attempts + 1 >= (queueConfig.tries ?? 1);
+        const error = e instanceof ReleaseQueueMessageError ? e.cause : e instanceof Error ? e : new Error(String(e));
+        if (isLastAttempt) {
+          await driver.fail({ id: queueMessage.id, attempts: queueMessage.attempts, error });
+          logger.error({
+            event: "messageFailed",
+            queue: queueMessage.queue,
+            body: queueMessage.body,
+            messageId: queueMessage.id,
+            error
+          });
+        } else {
+          await driver.release({
+            id: queueMessage.id,
+            attempts: queueMessage.attempts,
+            error,
+            delaySeconds: e instanceof ReleaseQueueMessageError ? e.delaySeconds ?? getDefaultDelaySeconds({ queueConfig, queueMessage }) : getDefaultDelaySeconds({ queueConfig, queueMessage })
+          });
+          logger.warn({
+            event: "messageReleased",
+            queue: queueMessage.queue,
+            body: queueMessage.body,
+            messageId: queueMessage.id,
+            error,
+            attempt: queueMessage.attempts + 1
+          });
+        }
+        throw e;
       }
     }
-    async function listMessages({ pageSize, pageNumber }) {
-      return config.driver.listMessages({ pageSize, pageNumber, queueName });
+  }
+  let workPromises = [];
+  function work({ maxJobs, maxTimeSeconds, stopWhenEmpty, sleepSeconds, concurrency = 1 }) {
+    const workAbortController = new AbortController();
+    let messagesProcessed = 0;
+    let messagesPopping = 0;
+    let timeoutToClear = null;
+    if (maxTimeSeconds) {
+      timeoutToClear = setTimeout(() => workAbortController.abort("max time exceeded"), maxTimeSeconds * 1e3);
     }
-    async function addMessage(body, options2) {
-      const { id } = await tracing.hydrate(
-        options2.tracingContext === void 0 ? tracing.dehydrate() : options2.tracingContext,
-        async () => {
-          return tracing.startActiveSpan(
-            {
-              name: "queue_producer",
-              attributes: {
-                op: "queue.publish",
-                "sentry.op": "queue.publish",
-                "messaging.destination.name": queueName
-              }
-            },
-            async () => {
-              const span = tracing.getActiveSpan();
-              const { id: id2 } = await config.driver.addMessage({
-                body,
-                scheduledDate: options2.scheduledDate,
-                queueName,
-                traceData: tracing.dehydrate()
+    async function pollForWork() {
+      let didThrowDuringPop = false;
+      while (!workAbortController.signal.aborted) {
+        try {
+          if (didThrowDuringPop) {
+            await new Promise((r) => setTimeout(r, 1e3));
+            didThrowDuringPop = false;
+          }
+          if (typeof maxJobs === "number" && messagesProcessed >= maxJobs) {
+            workAbortController.abort("max jobs reached");
+          } else {
+            if (typeof maxJobs === "number" && messagesProcessed + messagesPopping >= maxJobs) {
+              await new Promise((r) => setTimeout(r, 1e3 * (sleepSeconds || 1)));
+            } else {
+              messagesPopping++;
+              const message = await driver.pop().catch((e) => {
+                didThrowDuringPop = true;
+                messagesPopping--;
+                throw e;
               });
-              span?.setAttribute("messaging.message.id", id2);
-              return { id: id2 };
+              messagesPopping--;
+              if (!message && stopWhenEmpty) {
+                workAbortController.abort("queue empty");
+              } else if (!message) {
+                await new Promise((r) => setTimeout(r, 1e3 * (sleepSeconds || 1)));
+              } else {
+                messagesProcessed++;
+                await processMessage(message);
+              }
             }
-          );
+          }
+          await new Promise((r) => setTimeout(r, 0));
+        } catch (e) {
         }
-      );
-      logger.info({
-        event: "messageEnqueued",
-        queue: queueName,
-        body,
-        scheduledDate: options2.scheduledDate ?? null,
-        driver: config.driver,
-        messageId: id
-      });
+      }
     }
-    async function updateMessage({
-      messageId,
-      scheduledDate,
-      body
-    }) {
-      return config.driver.updateMessage({ messageId, body, scheduledDate, queueName });
+    const promise = Promise.all(Array.from({ length: concurrency }).map(() => pollForWork())).then(() => {
+    });
+    const result = Object.assign(promise, {
+      abort: (reason) => {
+        workAbortController.abort(reason);
+        if (timeoutToClear) clearTimeout(timeoutToClear);
+      }
+    });
+    workPromises.push(result);
+    result.finally(() => {
+      workPromises = workPromises.filter((w) => w !== result);
+      if (timeoutToClear) {
+        clearTimeout(timeoutToClear);
+      }
+    });
+    return result;
+  }
+  async function stopAllWork() {
+    const current = [...workPromises];
+    current.forEach((w) => w.abort());
+    await Promise.all(current);
+  }
+  function create(queueConfig) {
+    if (allQueues.has(queueConfig.name)) {
+      throw new Error(`Cannot have multiple queues with name "${queueConfig.name}"`);
     }
-    function processUntilEmpty(options2) {
-      const interval = setInterval(() => {
-        config.driver.isQueueEmpty({ queueName }).then((isEmpty2) => {
-          if (isEmpty2) {
-            declareEmpty(null);
-          }
-        });
-      }, options2?.pollingMs ?? 3e3);
-      let declareEmpty = (value) => {
-      };
-      const isEmpty = new Promise((resolve) => {
-        declareEmpty = resolve;
-      }).then(() => clearInterval(interval)).then(() => {
-        return stopProcessing2();
-      });
-      startProcessing2();
-      return isEmpty;
+    allQueues.set(queueConfig.name, queueConfig);
+    const logger = combineLoggers(serviceLogger, queueConfig.logger);
+    async function dispatch(body) {
+      await driver.push({ queue: queueConfig.name, body: body ?? {}, scheduledDate: instantFns3.now() });
+      logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
     }
-    function startProcessing2() {
-      config.driver.start({ queueName, workFn, concurrency });
-      logger.info({
-        event: "queueStarted",
-        queue: queueName,
-        driver: config.driver
-      });
+    async function dispatchWithDelay(...params) {
+      const body = typeof params[0] === "string" ? {} : params[0];
+      const scheduledDate = typeof params[0] === "string" ? params[0] : params[1];
+      await driver.push({ queue: queueConfig.name, body: params, scheduledDate });
+      logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
     }
-    function stopProcessing2() {
-      config.driver.stop({ queueName });
-      logger.info({
-        event: "queueStopped",
-        queue: queueName,
-        driver: config.driver
-      });
+    async function dispatchSync(body) {
+      logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
+      const queueMessage = { id: randomUUID(), body, queue: queueConfig.name, attempts: 0 };
+      const result = await execute({ queueConfig, queueMessage });
+      logger.info({ event: "messageProcessed", queue: queueConfig.name, body, messageId: queueMessage.id, result });
+      return result;
     }
-    const queue = {
-      listMessages,
-      addMessage,
-      updateMessage,
-      startProcessing: startProcessing2,
-      processUntilEmpty,
-      stopProcessing: stopProcessing2,
-      name: options.name
-    };
-    queues.push(queue);
-    return queue;
-  }
-  function startProcessing() {
-    queues.forEach((q) => q.startProcessing());
-  }
-  function stopProcessing() {
-    queues.forEach((q) => q.stopProcessing());
+    async function bulk(messages) {
+      await driver.bulk(
+        messages.map(({ body, scheduledDate }) => ({
+          queue: queueConfig.name,
+          body,
+          scheduledDate: scheduledDate ?? instantFns3.now()
+        }))
+      );
+      for (const { body } of messages) {
+        logger.info({ event: "messageEnqueued", queue: queueConfig.name, body });
+      }
+    }
+    return { dispatch, dispatchWithDelay, dispatchSync, bulk };
   }
-  return { list, get, create, startProcessing, stopProcessing };
+  return { work, stopAllWork, create, processMessage };
 }
 var registry = ServiceRegistry.createRegistry(
   ServiceRegistry.proxyFunctionsForObject
 );
 var Provider = {
   create: createService,
-  register: registry.register,
-  list: registry.list
+  register: registry.register
 };
 function provider(key) {
   const service = registry.resolve(key);
   return {
-    list: service.list,
-    get: service.get,
-    create: service.create,
-    startProcessing: service.startProcessing,
-    stopProcessing: service.stopProcessing
+    work: service.work,
+    stopAllWork: service.stopAllWork,
+    processMessage: service.processMessage,
+    create: service.create
   };
 }
 var facade = provider("default");
+var FailQueueMessageError = class extends Error {
+  cause;
+  constructor(cause) {
+    super(void 0, { cause });
+    this.cause = cause;
+  }
+};
+var ReleaseQueueMessageError = class extends Error {
+  cause;
+  delaySeconds = void 0;
+  constructor(cause, delaySeconds) {
+    super(void 0, { cause });
+    this.delaySeconds = delaySeconds;
+    this.cause = cause;
+  }
+};
 var Queue = {
-  drivers: {
-    db: DbQueueDriver
-  },
   Provider,
   provider,
-  list: facade.list,
-  get: facade.get,
+  work: facade.work,
+  stopAllWork: facade.stopAllWork,
+  processMessage: facade.processMessage,
   create: facade.create,
-  startProcessing: facade.startProcessing,
-  stopProcessing: facade.stopProcessing
+  drivers: {
+    db: DbQueueDriver,
+    local: localQueueDriver
+  }
 };
 export {
   Queue