@getstrata/core 1.1.8 → 2.0.0

package/dist/entries/queue/publicQueue.js

@@ -124,7 +124,7 @@ var queueConfig = {
 };
 
 // ../../src/core/queue/jobRunner.ts
-async function runQueueJob(envelope, failedJobs) {
+async function runQueueJob(envelope, failedJobs, options = {}) {
   const job = jobRegistry.create(envelope.name);
   if (!job) {
     throw new Error(`Unknown job "${envelope.name}".`);
@@ -141,7 +141,7 @@ async function runQueueJob(envelope, failedJobs) {
       await runQueueJob({
         ...envelope,
         attempts: nextAttempt
-      }, failedJobs);
+      }, failedJobs, options);
       return;
     }
     await failedJobs.recordFailure({
@@ -149,6 +149,7 @@ async function runQueueJob(envelope, failedJobs) {
       payload: envelope.payload,
       exception: error instanceof Error ? error.stack ?? error.message : String(error)
     });
+    options.onFailureRecorded?.();
     throw error;
   }
 }
@@ -256,6 +257,30 @@ var QUEUE_LIST_KEY = queueListKey();
 var QUEUE_HIGH_KEY = queueHighKey();
 var QUEUE_LOW_KEY = queueLowKey();
 var QUEUE_KEYS = [QUEUE_HIGH_KEY, QUEUE_LIST_KEY, QUEUE_LOW_KEY];
+var DEFAULT_VISIBILITY_MS = 60000;
+function queueProcessingKey(queueKey) {
+  return `${queueKey}:processing`;
+}
+function queueProcessingLeaseKey(queueKey) {
+  return `${queueKey}:processing:leases`;
+}
+function readVisibilityMs() {
+  const parsed = Number(process.env.QUEUE_VISIBILITY_MS ?? String(DEFAULT_VISIBILITY_MS));
+  if (!Number.isSafeInteger(parsed) || parsed < 30) {
+    throw new Error("QUEUE_VISIBILITY_MS must be an integer of at least 30 milliseconds.");
+  }
+  return parsed;
+}
+function reservationExpired(lease, now, visibilityMs) {
+  if (lease === null || lease.length === 0) {
+    return true;
+  }
+  const reservedAt = Number(lease);
+  if (!Number.isFinite(reservedAt)) {
+    return true;
+  }
+  return now - reservedAt >= visibilityMs;
+}
 function queueKeyForPriority(priority = "default") {
   switch (priority) {
     case "high":
@@ -297,6 +322,129 @@ function parseQueueJobEnvelope(rawPayload) {
     attempts: typeof envelope.attempts === "number" ? envelope.attempts : 0
   };
 }
+var SCRIPT_HEADER = `
+local function checkType(key, expected)
+  local actual = redis.call('TYPE', key).ok
+  if actual ~= 'none' and actual ~= expected then
+    return redis.error_reply('Unexpected queue key type: ' .. key)
+  end
+end
+local function nowMs()
+  local time = redis.call('TIME')
+  return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
+end
+`;
+var RESERVE_JOB = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
+if err then return err end
+local payload = redis.call('LINDEX', KEYS[1], -1)
+if not payload then return nil end
+local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
+local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
+redis.call('LPUSH', KEYS[2], record)
+redis.call('HSET', KEYS[3], ARGV[1], lease)
+redis.call('RPOP', KEYS[1])
+return record
+`;
+var RENEW_OR_ACK = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
+if err then return err end
+local raw = redis.call('HGET', KEYS[2], ARGV[1])
+if not raw then return 0 end
+local ok, lease = pcall(cjson.decode, raw)
+local now = nowMs()
+if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
+  return 0
+end
+if ARGV[4] == 'renew' then
+  redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
+  return 1
+end
+if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
+local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
+redis.call('HDEL', KEYS[2], ARGV[1])
+return removed
+`;
+var RECLAIM_JOBS = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
+if err then return err end
+local now = nowMs()
+local reclaimed = 0
+for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
+  local payload, field = record, record
+  local modern = false
+  local ok, reservation = pcall(cjson.decode, record)
+  if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
+    payload, field, modern = reservation.payload, reservation.id, true
+  end
+  local raw = redis.call('HGET', KEYS[3], field)
+  local expired = true
+  if raw then
+    if modern then
+      local leaseOk, lease = pcall(cjson.decode, raw)
+      expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
+    else
+      local reservedAt = tonumber(raw)
+      expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
+    end
+  end
+  if expired then
+    redis.call('LPUSH', KEYS[1], payload)
+    redis.call('LREM', KEYS[2], 1, record)
+    redis.call('HDEL', KEYS[3], field)
+    reclaimed = reclaimed + 1
+  end
+end
+return reclaimed
+`;
+function queueInvalidKey(queueKey) {
+  return `${queueKey}:invalid`;
+}
+async function reserveQueueJob(client, queueKey, owner) {
+  const raw = await client.send("EVAL", [
+    RESERVE_JOB,
+    "3",
+    queueKey,
+    queueProcessingKey(queueKey),
+    queueProcessingLeaseKey(queueKey),
+    crypto.randomUUID(),
+    owner,
+    String(readVisibilityMs())
+  ]);
+  if (typeof raw !== "string")
+    return null;
+  const reservation = JSON.parse(raw);
+  return { ...reservation, record: raw };
+}
+async function updateQueueReservation(client, queueKey, reservation, action) {
+  return Number(await client.send("EVAL", [
+    RENEW_OR_ACK,
+    "3",
+    queueProcessingKey(queueKey),
+    queueProcessingLeaseKey(queueKey),
+    queueInvalidKey(queueKey),
+    reservation.id,
+    reservation.owner,
+    reservation.record,
+    action,
+    String(readVisibilityMs()),
+    reservation.payload
+  ])) === 1;
+}
+async function reclaimExpiredQueueReservations(client, queueKeys = QUEUE_KEYS) {
+  let reclaimed = 0;
+  for (const key of queueKeys) {
+    reclaimed += Number(await client.send("EVAL", [
+      RECLAIM_JOBS,
+      "3",
+      key,
+      queueProcessingKey(key),
+      queueProcessingLeaseKey(key),
+      String(readVisibilityMs())
+    ]));
+  }
+  return reclaimed;
+}
 
 class RedisQueue {
   client;
@@ -321,12 +469,15 @@ class RedisQueue {
 class QueueWorker {
   failedJobs;
   timeoutSeconds;
+  queueKeys;
+  owner = crypto.randomUUID();
   running = false;
   stopping = false;
   client;
-  constructor(redisUrl, failedJobs, timeoutSeconds = 5) {
+  constructor(redisUrl, failedJobs, timeoutSeconds = 5, queueKeys = QUEUE_KEYS) {
     this.failedJobs = failedJobs;
     this.timeoutSeconds = timeoutSeconds;
+    this.queueKeys = queueKeys;
     this.client = new RedisClient(redisUrl);
   }
   requestStop() {
@@ -335,38 +486,81 @@ class QueueWorker {
   isRunning() {
     return this.running;
   }
-  async processNext() {
-    let result = null;
-    for (const queueKey of QUEUE_KEYS) {
-      result = await this.client.brpop(queueKey, 1);
-      if (result) {
-        break;
+  async reserve() {
+    const deadline = performance.now() + this.timeoutSeconds * 1000;
+    do {
+      for (const key of this.queueKeys) {
+        if (this.stopping)
+          return null;
+        const reservation = await reserveQueueJob(this.client, key, this.owner);
+        if (reservation)
+          return { queueKey: key, reservation };
       }
-    }
-    if (!result) {
-      result = await this.client.brpop(QUEUE_LIST_KEY, this.timeoutSeconds);
-    }
-    if (!result) {
+      if (this.stopping || performance.now() >= deadline)
+        return null;
+      await Bun.sleep(Math.min(100, Math.max(1, deadline - performance.now())));
+    } while (!this.stopping);
+    return null;
+  }
+  async processNext() {
+    if (this.stopping)
       return false;
-    }
-    const [, rawPayload] = result;
-    const envelope = parseQueueJobEnvelope(rawPayload);
+    await reclaimExpiredQueueReservations(this.client, this.queueKeys);
+    const reserved = await this.reserve();
+    if (!reserved)
+      return false;
+    const { queueKey, reservation } = reserved;
+    const envelope = parseQueueJobEnvelope(reservation.payload);
     if (!envelope) {
+      await updateQueueReservation(this.client, queueKey, reservation, "quarantine");
       return true;
     }
+    let acknowledge = false;
+    let ownershipLost = false;
+    let renewal;
+    const heartbeat = setInterval(() => {
+      if (renewal || ownershipLost)
+        return;
+      renewal = (async () => {
+        try {
+          if (!await updateQueueReservation(this.client, queueKey, reservation, "renew")) {
+            ownershipLost = true;
+          }
+        } catch (error) {
+          ownershipLost = true;
+          console.error("[QueueWorker] Lease renewal failed:", error);
+        }
+      })().finally(() => {
+        renewal = undefined;
+      });
+    }, Math.max(1, Math.floor(readVisibilityMs() / 3)));
     try {
-      await runQueueJob(envelope, this.failedJobs);
+      await runQueueJob(envelope, this.failedJobs, {
+        onFailureRecorded: () => {
+          acknowledge = true;
+        }
+      });
+      acknowledge = true;
     } catch (error) {
       console.error("[QueueWorker] Job failed:", error);
+    } finally {
+      clearInterval(heartbeat);
+      await renewal;
+    }
+    if (acknowledge && !ownershipLost) {
+      await updateQueueReservation(this.client, queueKey, reservation, "ack");
     }
     return true;
   }
   async run() {
     this.running = true;
-    while (!this.stopping) {
-      await this.processNext();
+    try {
+      while (!this.stopping) {
+        await this.processNext();
+      }
+    } finally {
+      this.running = false;
     }
-    this.running = false;
   }
   close() {
     this.client.close();

package/dist/entries/queue/queueMetrics.js

@@ -127,7 +127,7 @@ var queueConfig = {
 };
 
 // ../../src/core/queue/jobRunner.ts
-async function runQueueJob(envelope, failedJobs) {
+async function runQueueJob(envelope, failedJobs, options = {}) {
   const job = jobRegistry.create(envelope.name);
   if (!job) {
     throw new Error(`Unknown job "${envelope.name}".`);
@@ -144,7 +144,7 @@ async function runQueueJob(envelope, failedJobs) {
       await runQueueJob({
         ...envelope,
         attempts: nextAttempt
-      }, failedJobs);
+      }, failedJobs, options);
       return;
     }
     await failedJobs.recordFailure({
@@ -152,6 +152,7 @@ async function runQueueJob(envelope, failedJobs) {
       payload: envelope.payload,
       exception: error instanceof Error ? error.stack ?? error.message : String(error)
     });
+    options.onFailureRecorded?.();
     throw error;
   }
 }
@@ -259,6 +260,30 @@ var QUEUE_LIST_KEY = queueListKey();
 var QUEUE_HIGH_KEY = queueHighKey();
 var QUEUE_LOW_KEY = queueLowKey();
 var QUEUE_KEYS = [QUEUE_HIGH_KEY, QUEUE_LIST_KEY, QUEUE_LOW_KEY];
+var DEFAULT_VISIBILITY_MS = 60000;
+function queueProcessingKey(queueKey) {
+  return `${queueKey}:processing`;
+}
+function queueProcessingLeaseKey(queueKey) {
+  return `${queueKey}:processing:leases`;
+}
+function readVisibilityMs() {
+  const parsed = Number(process.env.QUEUE_VISIBILITY_MS ?? String(DEFAULT_VISIBILITY_MS));
+  if (!Number.isSafeInteger(parsed) || parsed < 30) {
+    throw new Error("QUEUE_VISIBILITY_MS must be an integer of at least 30 milliseconds.");
+  }
+  return parsed;
+}
+function reservationExpired(lease, now, visibilityMs) {
+  if (lease === null || lease.length === 0) {
+    return true;
+  }
+  const reservedAt = Number(lease);
+  if (!Number.isFinite(reservedAt)) {
+    return true;
+  }
+  return now - reservedAt >= visibilityMs;
+}
 function queueKeyForPriority(priority = "default") {
   switch (priority) {
     case "high":
@@ -300,6 +325,129 @@ function parseQueueJobEnvelope(rawPayload) {
     attempts: typeof envelope.attempts === "number" ? envelope.attempts : 0
   };
 }
+var SCRIPT_HEADER = `
+local function checkType(key, expected)
+  local actual = redis.call('TYPE', key).ok
+  if actual ~= 'none' and actual ~= expected then
+    return redis.error_reply('Unexpected queue key type: ' .. key)
+  end
+end
+local function nowMs()
+  local time = redis.call('TIME')
+  return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
+end
+`;
+var RESERVE_JOB = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
+if err then return err end
+local payload = redis.call('LINDEX', KEYS[1], -1)
+if not payload then return nil end
+local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
+local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
+redis.call('LPUSH', KEYS[2], record)
+redis.call('HSET', KEYS[3], ARGV[1], lease)
+redis.call('RPOP', KEYS[1])
+return record
+`;
+var RENEW_OR_ACK = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
+if err then return err end
+local raw = redis.call('HGET', KEYS[2], ARGV[1])
+if not raw then return 0 end
+local ok, lease = pcall(cjson.decode, raw)
+local now = nowMs()
+if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
+  return 0
+end
+if ARGV[4] == 'renew' then
+  redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
+  return 1
+end
+if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
+local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
+redis.call('HDEL', KEYS[2], ARGV[1])
+return removed
+`;
+var RECLAIM_JOBS = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
+if err then return err end
+local now = nowMs()
+local reclaimed = 0
+for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
+  local payload, field = record, record
+  local modern = false
+  local ok, reservation = pcall(cjson.decode, record)
+  if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
+    payload, field, modern = reservation.payload, reservation.id, true
+  end
+  local raw = redis.call('HGET', KEYS[3], field)
+  local expired = true
+  if raw then
+    if modern then
+      local leaseOk, lease = pcall(cjson.decode, raw)
+      expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
+    else
+      local reservedAt = tonumber(raw)
+      expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
+    end
+  end
+  if expired then
+    redis.call('LPUSH', KEYS[1], payload)
+    redis.call('LREM', KEYS[2], 1, record)
+    redis.call('HDEL', KEYS[3], field)
+    reclaimed = reclaimed + 1
+  end
+end
+return reclaimed
+`;
+function queueInvalidKey(queueKey) {
+  return `${queueKey}:invalid`;
+}
+async function reserveQueueJob(client, queueKey, owner) {
+  const raw = await client.send("EVAL", [
+    RESERVE_JOB,
+    "3",
+    queueKey,
+    queueProcessingKey(queueKey),
+    queueProcessingLeaseKey(queueKey),
+    crypto.randomUUID(),
+    owner,
+    String(readVisibilityMs())
+  ]);
+  if (typeof raw !== "string")
+    return null;
+  const reservation = JSON.parse(raw);
+  return { ...reservation, record: raw };
+}
+async function updateQueueReservation(client, queueKey, reservation, action) {
+  return Number(await client.send("EVAL", [
+    RENEW_OR_ACK,
+    "3",
+    queueProcessingKey(queueKey),
+    queueProcessingLeaseKey(queueKey),
+    queueInvalidKey(queueKey),
+    reservation.id,
+    reservation.owner,
+    reservation.record,
+    action,
+    String(readVisibilityMs()),
+    reservation.payload
+  ])) === 1;
+}
+async function reclaimExpiredQueueReservations(client, queueKeys = QUEUE_KEYS) {
+  let reclaimed = 0;
+  for (const key of queueKeys) {
+    reclaimed += Number(await client.send("EVAL", [
+      RECLAIM_JOBS,
+      "3",
+      key,
+      queueProcessingKey(key),
+      queueProcessingLeaseKey(key),
+      String(readVisibilityMs())
+    ]));
+  }
+  return reclaimed;
+}
 
 class RedisQueue {
   client;
@@ -324,12 +472,15 @@ class RedisQueue {
 class QueueWorker {
   failedJobs;
   timeoutSeconds;
+  queueKeys;
+  owner = crypto.randomUUID();
   running = false;
   stopping = false;
   client;
-  constructor(redisUrl, failedJobs, timeoutSeconds = 5) {
+  constructor(redisUrl, failedJobs, timeoutSeconds = 5, queueKeys = QUEUE_KEYS) {
     this.failedJobs = failedJobs;
     this.timeoutSeconds = timeoutSeconds;
+    this.queueKeys = queueKeys;
     this.client = new RedisClient(redisUrl);
   }
   requestStop() {
@@ -338,38 +489,81 @@ class QueueWorker {
   isRunning() {
     return this.running;
   }
-  async processNext() {
-    let result = null;
-    for (const queueKey of QUEUE_KEYS) {
-      result = await this.client.brpop(queueKey, 1);
-      if (result) {
-        break;
+  async reserve() {
+    const deadline = performance.now() + this.timeoutSeconds * 1000;
+    do {
+      for (const key of this.queueKeys) {
+        if (this.stopping)
+          return null;
+        const reservation = await reserveQueueJob(this.client, key, this.owner);
+        if (reservation)
+          return { queueKey: key, reservation };
       }
-    }
-    if (!result) {
-      result = await this.client.brpop(QUEUE_LIST_KEY, this.timeoutSeconds);
-    }
-    if (!result) {
+      if (this.stopping || performance.now() >= deadline)
+        return null;
+      await Bun.sleep(Math.min(100, Math.max(1, deadline - performance.now())));
+    } while (!this.stopping);
+    return null;
+  }
+  async processNext() {
+    if (this.stopping)
       return false;
-    }
-    const [, rawPayload] = result;
-    const envelope = parseQueueJobEnvelope(rawPayload);
+    await reclaimExpiredQueueReservations(this.client, this.queueKeys);
+    const reserved = await this.reserve();
+    if (!reserved)
+      return false;
+    const { queueKey, reservation } = reserved;
+    const envelope = parseQueueJobEnvelope(reservation.payload);
     if (!envelope) {
+      await updateQueueReservation(this.client, queueKey, reservation, "quarantine");
       return true;
     }
+    let acknowledge = false;
+    let ownershipLost = false;
+    let renewal;
+    const heartbeat = setInterval(() => {
+      if (renewal || ownershipLost)
+        return;
+      renewal = (async () => {
+        try {
+          if (!await updateQueueReservation(this.client, queueKey, reservation, "renew")) {
+            ownershipLost = true;
+          }
+        } catch (error) {
+          ownershipLost = true;
+          console.error("[QueueWorker] Lease renewal failed:", error);
+        }
+      })().finally(() => {
+        renewal = undefined;
+      });
+    }, Math.max(1, Math.floor(readVisibilityMs() / 3)));
     try {
-      await runQueueJob(envelope, this.failedJobs);
+      await runQueueJob(envelope, this.failedJobs, {
+        onFailureRecorded: () => {
+          acknowledge = true;
+        }
+      });
+      acknowledge = true;
     } catch (error) {
       console.error("[QueueWorker] Job failed:", error);
+    } finally {
+      clearInterval(heartbeat);
+      await renewal;
+    }
+    if (acknowledge && !ownershipLost) {
+      await updateQueueReservation(this.client, queueKey, reservation, "ack");
     }
     return true;
   }
   async run() {
     this.running = true;
-    while (!this.stopping) {
-      await this.processNext();
+    try {
+      while (!this.stopping) {
+        await this.processNext();
+      }
+    } finally {
+      this.running = false;
     }
-    this.running = false;
   }
   close() {
     this.client.close();