@boringnode/queue 0.5.1 → 0.6.0

package/build/src/drivers/redis_adapter.js

@@ -3,6 +3,23 @@ import {
   calculateScore,
   resolveRetention
 } from "../../chunk-QEFYHCL7.js";
+import {
+  ACQUIRE_JOB_SCRIPT,
+  CLAIM_SCHEDULE_SCRIPT,
+  FINALIZE_JOB_SCRIPT,
+  GET_JOB_SCRIPT,
+  PUSH_DEDUP_JOB_SCRIPT,
+  PUSH_DELAYED_JOB_SCRIPT,
+  PUSH_JOB_SCRIPT,
+  RECOVER_STALLED_JOBS_SCRIPT,
+  REMOVE_JOB_SCRIPT,
+  RENEW_JOBS_SCRIPT,
+  RETRY_JOB_SCRIPT
+} from "../../chunk-S37X3CBO.js";
+import {
+  encodeRedisJobPayloadOverlay,
+  hydrateRedisJob
+} from "../../chunk-6IO4P6RB.js";
 import "../../chunk-PZ5AY32C.js";
 
 // src/drivers/redis_adapter.ts
@@ -11,361 +28,6 @@ import { Redis } from "ioredis";
 var redisKey = "jobs";
 var schedulesKey = "schedules";
 var schedulesIndexKey = "schedules::index";
-var PUSH_JOB_SCRIPT = `
-  local data_key = KEYS[1]
-  local pending_key = KEYS[2]
-  local job_id = ARGV[1]
-  local job_data = ARGV[2]
-  local score = tonumber(ARGV[3])
-
-  redis.call('HSET', data_key, job_id, job_data)
-  redis.call('ZADD', pending_key, score, job_id)
-
-  return 1
-`;
-var PUSH_DELAYED_JOB_SCRIPT = `
-  local data_key = KEYS[1]
-  local delayed_key = KEYS[2]
-  local job_id = ARGV[1]
-  local job_data = ARGV[2]
-  local execute_at = tonumber(ARGV[3])
-
-  redis.call('HSET', data_key, job_id, job_data)
-  redis.call('ZADD', delayed_key, execute_at, job_id)
-
-  return 1
-`;
-var ACQUIRE_JOB_SCRIPT = `
-  local data_key = KEYS[1]
-  local pending_key = KEYS[2]
-  local active_key = KEYS[3]
-  local delayed_key = KEYS[4]
-  local worker_id = ARGV[1]
-  local now = tonumber(ARGV[2])
-
-  -- Process delayed jobs: move ready jobs to pending
-  local ready_job_ids = redis.call('ZRANGEBYSCORE', delayed_key, 0, now)
-  if #ready_job_ids > 0 then
-    for i = 1, #ready_job_ids do
-      local job_id = ready_job_ids[i]
-      local job_data = redis.call('HGET', data_key, job_id)
-      if job_data then
-        local job = cjson.decode(job_data)
-        local priority = job.priority or 5
-        local score = priority * 10000000000000 + now
-        redis.call('ZADD', pending_key, score, job_id)
-        redis.call('ZREM', delayed_key, job_id)
-      end
-    end
-  end
-
-  -- Pop highest priority job (lowest score)
-  local result = redis.call('ZPOPMIN', pending_key)
-  if not result or #result == 0 then
-    return nil
-  end
-
-  local job_id = result[1]
-  local job_data = redis.call('HGET', data_key, job_id)
-  if not job_data then
-    return nil
-  end
-
-  -- Store in active hash (without data, it's in data_key)
-  local active_data = cjson.encode({
-    workerId = worker_id,
-    acquiredAt = now
-  })
-  redis.call('HSET', active_key, job_id, active_data)
-
-  -- Return job with acquiredAt
-  local job = cjson.decode(job_data)
-  job.acquiredAt = now
-  return cjson.encode(job)
-`;
-var REMOVE_JOB_SCRIPT = `
-  local data_key = KEYS[1]
-  local active_key = KEYS[2]
-  local job_id = ARGV[1]
-
-  if redis.call('HEXISTS', active_key, job_id) == 0 then
-    return 0
-  end
-
-  redis.call('HDEL', active_key, job_id)
-  redis.call('HDEL', data_key, job_id)
-
-  return 1
-`;
-var FINALIZE_JOB_SCRIPT = `
-  local data_key = KEYS[1]
-  local active_key = KEYS[2]
-  local history_key = KEYS[3]
-  local index_key = KEYS[4]
-  local job_id = ARGV[1]
-  local now = tonumber(ARGV[2])
-  local max_age = tonumber(ARGV[3])
-  local max_count = tonumber(ARGV[4])
-  local error_message = ARGV[5]
-
-  -- Verify job is active
-  if redis.call('HEXISTS', active_key, job_id) == 0 then
-    return 0
-  end
-
-  -- Remove from active
-  redis.call('HDEL', active_key, job_id)
-
-  -- Store finalization info (data stays in data_key)
-  local record = {
-    finishedAt = now
-  }
-  if error_message and error_message ~= '' then
-    record.error = error_message
-  end
-  redis.call('HSET', history_key, job_id, cjson.encode(record))
-  redis.call('ZADD', index_key, now, job_id)
-
-  -- Prune by age
-  if max_age and max_age > 0 then
-    local cutoff = now - max_age
-    local expired = redis.call('ZRANGEBYSCORE', index_key, 0, cutoff)
-    if #expired > 0 then
-      redis.call('ZREM', index_key, unpack(expired))
-      redis.call('HDEL', history_key, unpack(expired))
-      redis.call('HDEL', data_key, unpack(expired))
-    end
-  end
-
-  -- Prune by count
-  if max_count and max_count > 0 then
-    local size = tonumber(redis.call('ZCARD', index_key))
-    if size > max_count then
-      local excess = size - max_count
-      local stale = redis.call('ZRANGE', index_key, 0, excess - 1)
-      if #stale > 0 then
-        redis.call('ZREM', index_key, unpack(stale))
-        redis.call('HDEL', history_key, unpack(stale))
-        redis.call('HDEL', data_key, unpack(stale))
-      end
-    end
-  end
-
-  return 1
-`;
-var RETRY_JOB_SCRIPT = `
-  local data_key = KEYS[1]
-  local active_key = KEYS[2]
-  local pending_key = KEYS[3]
-  local delayed_key = KEYS[4]
-  local job_id = ARGV[1]
-  local retry_at = tonumber(ARGV[2])
-  local now = tonumber(ARGV[3])
-
-  -- Verify job is active
-  if redis.call('HEXISTS', active_key, job_id) == 0 then
-    return 0
-  end
-
-  -- Get job data
-  local job_data = redis.call('HGET', data_key, job_id)
-  if not job_data then
-    return 0
-  end
-
-  -- Remove from active
-  redis.call('HDEL', active_key, job_id)
-
-  -- Increment attempts and update data
-  local job = cjson.decode(job_data)
-  job.attempts = (job.attempts or 0) + 1
-  redis.call('HSET', data_key, job_id, cjson.encode(job))
-
-  -- Add back to pending or delayed
-  if retry_at and retry_at > now then
-    redis.call('ZADD', delayed_key, retry_at, job_id)
-  else
-    -- Score = priority * 1e13 + timestamp
-    -- Lower score = higher priority, FIFO within same priority
-    local priority = job.priority or 5
-    local score = priority * 10000000000000 + now
-    redis.call('ZADD', pending_key, score, job_id)
-  end
-
-  return 1
-`;
-var RECOVER_STALLED_JOBS_SCRIPT = `
-  local data_key = KEYS[1]
-  local active_key = KEYS[2]
-  local pending_key = KEYS[3]
-  local now = tonumber(ARGV[1])
-  local stalled_threshold = tonumber(ARGV[2])
-  local max_stalled_count = tonumber(ARGV[3])
-
-  local recovered = 0
-  local stalled_cutoff = now - stalled_threshold
-
-  -- Get all active jobs
-  local active_jobs = redis.call('HGETALL', active_key)
-
-  -- HGETALL returns [field1, value1, field2, value2, ...]
-  for i = 1, #active_jobs, 2 do
-    local job_id = active_jobs[i]
-    local active_data = active_jobs[i + 1]
-    local active = cjson.decode(active_data)
-
-    -- Check if job is stalled
-    if active.acquiredAt < stalled_cutoff then
-      local job_data = redis.call('HGET', data_key, job_id)
-      if job_data then
-        local job = cjson.decode(job_data)
-        local current_stalled_count = job.stalledCount or 0
-
-        -- Remove from active hash
-        redis.call('HDEL', active_key, job_id)
-
-        -- Check if job has exceeded max stalled count
-        if current_stalled_count >= max_stalled_count then
-          -- Job failed permanently, remove data too
-          redis.call('HDEL', data_key, job_id)
-        else
-          -- Recover: increment stalledCount and put back in pending
-          job.stalledCount = current_stalled_count + 1
-          redis.call('HSET', data_key, job_id, cjson.encode(job))
-          -- Score = priority * 1e13 + timestamp
-          local priority = job.priority or 5
-          local score = priority * 10000000000000 + now
-          redis.call('ZADD', pending_key, score, job_id)
-          recovered = recovered + 1
-        end
-      end
-    end
-  end
-
-  return recovered
-`;
-var GET_JOB_SCRIPT = `
-  local data_key = KEYS[1]
-  local pending_key = KEYS[2]
-  local delayed_key = KEYS[3]
-  local active_key = KEYS[4]
-  local completed_key = KEYS[5]
-  local failed_key = KEYS[6]
-  local job_id = ARGV[1]
-
-  local job_data = redis.call('HGET', data_key, job_id)
-  if not job_data then
-    return nil
-  end
-
-  local status = nil
-  local finished_at = nil
-  local error_msg = nil
-
-  -- Check status in order
-  if redis.call('HEXISTS', active_key, job_id) == 1 then
-    status = 'active'
-  elseif redis.call('ZSCORE', pending_key, job_id) then
-    status = 'pending'
-  elseif redis.call('ZSCORE', delayed_key, job_id) then
-    status = 'delayed'
-  else
-    local completed_data = redis.call('HGET', completed_key, job_id)
-    if completed_data then
-      status = 'completed'
-      local record = cjson.decode(completed_data)
-      finished_at = record.finishedAt
-    else
-      local failed_data = redis.call('HGET', failed_key, job_id)
-      if failed_data then
-        status = 'failed'
-        local record = cjson.decode(failed_data)
-        finished_at = record.finishedAt
-        error_msg = record.error
-      end
-    end
-  end
-
-  if not status then
-    return nil
-  end
-
-  return cjson.encode({
-    status = status,
-    data = cjson.decode(job_data),
-    finishedAt = finished_at,
-    error = error_msg
-  })
-`;
-var CLAIM_SCHEDULE_SCRIPT = `
-  local schedule_key = KEYS[1]
-  local now = tonumber(ARGV[1])
-
-  -- Get schedule data
-  local data = redis.call('HGETALL', schedule_key)
-  if #data == 0 then
-    return nil
-  end
-
-  -- Convert HGETALL result to table
-  local schedule = {}
-  for j = 1, #data, 2 do
-    schedule[data[j]] = data[j + 1]
-  end
-
-  -- Check if schedule is due
-  if schedule.status ~= 'active' then
-    return nil
-  end
-
-  local next_run_at = tonumber(schedule.next_run_at)
-  if not next_run_at or next_run_at > now then
-    return nil
-  end
-
-  local run_count = tonumber(schedule.run_count or '0')
-  local run_limit = schedule.run_limit and tonumber(schedule.run_limit) or nil
-  local to_date = schedule.to_date and tonumber(schedule.to_date) or nil
-
-  -- Check limits
-  if run_limit and run_count >= run_limit then
-    return nil
-  end
-
-  if to_date and now > to_date then
-    return nil
-  end
-
-  -- This schedule is claimable - atomically update it
-  local new_run_count = run_count + 1
-
-  -- Calculate new next_run_at (simple interval-based for now)
-  -- Complex cron calculation happens in the caller
-  local new_next_run_at = ''
-  local every_ms = schedule.every_ms and tonumber(schedule.every_ms) or nil
-  if every_ms then
-    new_next_run_at = tostring(now + every_ms)
-  end
-
-  -- Check if we've hit the limit after this run
-  if run_limit and new_run_count >= run_limit then
-    new_next_run_at = ''
-  end
-
-  -- Check if past end date
-  if to_date and new_next_run_at ~= '' and tonumber(new_next_run_at) > to_date then
-    new_next_run_at = ''
-  end
-
-  -- Update the schedule atomically
-  redis.call('HSET', schedule_key,
-    'next_run_at', new_next_run_at,
-    'last_run_at', tostring(now),
-    'run_count', tostring(new_run_count))
-
-  -- Return the schedule data (before update) as JSON
-  return cjson.encode(schedule)
-`;
 function redis(config) {
   return () => {
     if (config instanceof Redis) {
@@ -396,12 +58,19 @@ var RedisAdapter = class {
       pending: `${redisKey}::${queue}::pending`,
       delayed: `${redisKey}::${queue}::delayed`,
       active: `${redisKey}::${queue}::active`,
+      overlay: `${redisKey}::${queue}::metadata`,
       completed: `${redisKey}::${queue}::completed`,
       completedIndex: `${redisKey}::${queue}::completed::index`,
       failed: `${redisKey}::${queue}::failed`,
       failedIndex: `${redisKey}::${queue}::failed::index`
     };
   }
+  #getDedupKey(queue, dedupId) {
+    return `${this.#getDedupPrefix(queue)}${dedupId}`;
+  }
+  #getDedupPrefix(queue) {
+    return `${redisKey}::${queue}::dedup::`;
+  }
   setWorkerId(workerId) {
     this.#workerId = workerId;
   }
@@ -418,59 +87,83 @@ var RedisAdapter = class {
     const now = Date.now();
     const result = await this.#connection.eval(
       ACQUIRE_JOB_SCRIPT,
-      4,
+      5,
       keys.data,
       keys.pending,
       keys.active,
       keys.delayed,
+      keys.overlay,
       this.#workerId,
       now.toString()
     );
     if (!result) {
       return null;
     }
-    return JSON.parse(result);
+    const { data, overlay, acquiredAt } = JSON.parse(result);
+    return { ...hydrateRedisJob(data, overlay), acquiredAt };
   }
   async completeJob(jobId, queue, removeOnComplete) {
     const keys = this.#getKeys(queue);
+    const dedupPrefix = this.#getDedupPrefix(queue);
     const { keep, maxAge, maxCount } = resolveRetention(removeOnComplete);
     if (!keep) {
-      await this.#connection.eval(REMOVE_JOB_SCRIPT, 2, keys.data, keys.active, jobId);
+      await this.#connection.eval(
+        REMOVE_JOB_SCRIPT,
+        3,
+        keys.data,
+        keys.active,
+        keys.overlay,
+        jobId,
+        dedupPrefix
+      );
       return;
     }
     await this.#connection.eval(
       FINALIZE_JOB_SCRIPT,
-      4,
+      5,
       keys.data,
       keys.active,
       keys.completed,
       keys.completedIndex,
+      keys.overlay,
       jobId,
       Date.now().toString(),
       maxAge.toString(),
       maxCount.toString(),
-      ""
+      "",
+      dedupPrefix
     );
   }
   async failJob(jobId, queue, error, removeOnFail) {
     const keys = this.#getKeys(queue);
+    const dedupPrefix = this.#getDedupPrefix(queue);
     const { keep, maxAge, maxCount } = resolveRetention(removeOnFail);
     if (!keep) {
-      await this.#connection.eval(REMOVE_JOB_SCRIPT, 2, keys.data, keys.active, jobId);
+      await this.#connection.eval(
+        REMOVE_JOB_SCRIPT,
+        3,
+        keys.data,
+        keys.active,
+        keys.overlay,
+        jobId,
+        dedupPrefix
+      );
       return;
     }
     await this.#connection.eval(
       FINALIZE_JOB_SCRIPT,
-      4,
+      5,
       keys.data,
       keys.active,
       keys.failed,
       keys.failedIndex,
+      keys.overlay,
       jobId,
       Date.now().toString(),
       maxAge.toString(),
       maxCount.toString(),
-      error?.message || ""
+      error?.message || "",
+      dedupPrefix
     );
   }
   async retryJob(jobId, queue, retryAt) {
@@ -478,11 +171,12 @@ var RedisAdapter = class {
     const now = Date.now();
     await this.#connection.eval(
       RETRY_JOB_SCRIPT,
-      4,
+      5,
       keys.data,
       keys.active,
       keys.pending,
       keys.delayed,
+      keys.overlay,
       jobId,
       retryAt ? retryAt.getTime().toString() : "0",
       now.toString()
@@ -492,19 +186,21 @@ var RedisAdapter = class {
     const keys = this.#getKeys(queue);
     const result = await this.#connection.eval(
       GET_JOB_SCRIPT,
-      6,
+      7,
       keys.data,
       keys.pending,
       keys.delayed,
       keys.active,
       keys.completed,
       keys.failed,
+      keys.overlay,
       jobId
     );
     if (!result) {
       return null;
     }
-    return JSON.parse(result);
+    const record = JSON.parse(result);
+    return { ...record, data: hydrateRedisJob(record.data, record.overlay) };
   }
   push(jobData) {
     return this.pushOn("default", jobData);
@@ -515,11 +211,34 @@ var RedisAdapter = class {
   async pushLaterOn(queue, jobData, delay) {
     const keys = this.#getKeys(queue);
     const executeAt = Date.now() + delay;
+    if (jobData.dedup) {
+      const dedupKey = this.#getDedupKey(queue, jobData.dedup.id);
+      const [payloadData, payloadIsUndefined] = encodeRedisJobPayloadOverlay(jobData.payload);
+      const result = await this.#connection.eval(
+        PUSH_DEDUP_JOB_SCRIPT,
+        5,
+        keys.data,
+        keys.delayed,
+        dedupKey,
+        keys.pending,
+        keys.overlay,
+        jobData.id,
+        JSON.stringify(jobData),
+        executeAt.toString(),
+        (jobData.dedup.ttl ?? 0).toString(),
+        jobData.dedup.extend ? "1" : "0",
+        jobData.dedup.replace ? "1" : "0",
+        payloadData,
+        payloadIsUndefined
+      );
+      return { outcome: result[0], jobId: result[1] };
+    }
     await this.#connection.eval(
       PUSH_DELAYED_JOB_SCRIPT,
-      2,
+      3,
       keys.data,
       keys.delayed,
+      keys.overlay,
       jobData.id,
       JSON.stringify(jobData),
       executeAt.toString()
@@ -530,11 +249,34 @@ var RedisAdapter = class {
     const priority = jobData.priority ?? DEFAULT_PRIORITY;
     const timestamp = Date.now();
     const score = calculateScore(priority, timestamp);
+    if (jobData.dedup) {
+      const dedupKey = this.#getDedupKey(queue, jobData.dedup.id);
+      const [payloadData, payloadIsUndefined] = encodeRedisJobPayloadOverlay(jobData.payload);
+      const result = await this.#connection.eval(
+        PUSH_DEDUP_JOB_SCRIPT,
+        5,
+        keys.data,
+        keys.pending,
+        dedupKey,
+        keys.delayed,
+        keys.overlay,
+        jobData.id,
+        JSON.stringify(jobData),
+        score.toString(),
+        (jobData.dedup.ttl ?? 0).toString(),
+        jobData.dedup.extend ? "1" : "0",
+        jobData.dedup.replace ? "1" : "0",
+        payloadData,
+        payloadIsUndefined
+      );
+      return { outcome: result[0], jobId: result[1] };
+    }
     await this.#connection.eval(
       PUSH_JOB_SCRIPT,
-      2,
+      3,
       keys.data,
       keys.pending,
+      keys.overlay,
       jobData.id,
       JSON.stringify(jobData),
       score.toString()
@@ -545,12 +287,16 @@ var RedisAdapter = class {
   }
   async pushManyOn(queue, jobs) {
     if (jobs.length === 0) return;
+    if (jobs.some((j) => j.dedup)) {
+      throw new Error("dedup is not supported in batch dispatch; use single dispatch");
+    }
     const keys = this.#getKeys(queue);
     const now = Date.now();
     const multi = this.#connection.multi();
     for (const job of jobs) {
       const priority = job.priority ?? DEFAULT_PRIORITY;
       const score = calculateScore(priority, now);
+      multi.hdel(keys.overlay, job.id);
       multi.hset(keys.data, job.id, JSON.stringify(job));
       multi.zadd(keys.pending, score, job.id);
     }
@@ -568,16 +314,34 @@ var RedisAdapter = class {
     const now = Date.now();
     const recovered = await this.#connection.eval(
       RECOVER_STALLED_JOBS_SCRIPT,
-      3,
+      4,
       keys.data,
       keys.active,
       keys.pending,
+      keys.overlay,
       now.toString(),
       stalledThreshold.toString(),
-      maxStalledCount.toString()
+      maxStalledCount.toString(),
+      this.#getDedupPrefix(queue)
     );
     return recovered;
   }
+  async renewJobs(queue, jobIds) {
+    if (jobIds.length === 0) {
+      return 0;
+    }
+    const keys = this.#getKeys(queue);
+    const now = Date.now();
+    const renewed = await this.#connection.eval(
+      RENEW_JOBS_SCRIPT,
+      1,
+      keys.active,
+      now.toString(),
+      this.#workerId,
+      ...jobIds
+    );
+    return renewed;
+  }
   async upsertSchedule(config) {
     const id = config.id ?? randomUUID();
     const now = Date.now();
@@ -665,40 +429,40 @@ var RedisAdapter = class {
   }
   async claimDueSchedule() {
     const now = Date.now();
-    const ids = await this.#connection.smembers(schedulesIndexKey);
-    for (const id of ids) {
-      const scheduleKey = `${schedulesKey}::${id}`;
-      const result = await this.#connection.eval(
-        CLAIM_SCHEDULE_SCRIPT,
-        1,
-        scheduleKey,
-        now.toString()
-      );
-      if (!result) {
-        continue;
-      }
-      const data = JSON.parse(result);
-      if (data.cron_expression) {
-        const { CronExpressionParser } = await import("cron-parser");
-        const cron = CronExpressionParser.parse(data.cron_expression, {
-          currentDate: new Date(now),
-          tz: data.timezone || "UTC"
-        });
-        const nextRun = cron.next().toDate().getTime();
-        const runCount = Number.parseInt(data.run_count || "0", 10) + 1;
-        const runLimit = data.run_limit ? Number.parseInt(data.run_limit, 10) : null;
-        const toDate = data.to_date ? Number.parseInt(data.to_date, 10) : null;
-        let newNextRunAt = nextRun;
-        if (runLimit !== null && runCount >= runLimit) {
-          newNextRunAt = "";
-        } else if (toDate && nextRun > toDate) {
-          newNextRunAt = "";
-        }
-        await this.#connection.hset(scheduleKey, "next_run_at", newNextRunAt.toString());
+    const result = await this.#connection.eval(
+      CLAIM_SCHEDULE_SCRIPT,
+      2,
+      schedulesIndexKey,
+      `${schedulesKey}::`,
+      now.toString()
+    );
+    if (!result) {
+      return null;
+    }
+    const data = JSON.parse(result);
+    if (data.cron_expression) {
+      const { CronExpressionParser } = await import("cron-parser");
+      const cron = CronExpressionParser.parse(data.cron_expression, {
+        currentDate: new Date(now),
+        tz: data.timezone || "UTC"
+      });
+      const nextRun = cron.next().toDate().getTime();
+      const runCount = Number.parseInt(data.run_count || "0", 10) + 1;
+      const runLimit = data.run_limit ? Number.parseInt(data.run_limit, 10) : null;
+      const toDate = data.to_date ? Number.parseInt(data.to_date, 10) : null;
+      let newNextRunAt = nextRun;
+      if (runLimit !== null && runCount >= runLimit) {
+        newNextRunAt = "";
+      } else if (toDate && nextRun > toDate) {
+        newNextRunAt = "";
       }
-      return this.#hashToScheduleData(data);
+      await this.#connection.hset(
+        `${schedulesKey}::${data.id}`,
+        "next_run_at",
+        newNextRunAt.toString()
+      );
     }
-    return null;
+    return this.#hashToScheduleData(data);
   }
   #hashToScheduleData(data) {
     return {