@boringnode/queue 0.2.0 → 0.3.0

package/build/src/drivers/redis_adapter.js

@@ -1,6 +1,7 @@
 import {
-  calculateScore
-} from "../../chunk-NPQKBCCY.js";
+  calculateScore,
+  resolveRetention
+} from "../../chunk-PBGPIFI5.js";
 import {
   DEFAULT_PRIORITY
 } from "../../chunk-SMOKFZ46.js";
@@ -11,26 +12,51 @@ import { Redis } from "ioredis";
 var redisKey = "jobs";
 var schedulesKey = "schedules";
 var schedulesIndexKey = "schedules::index";
+var PUSH_JOB_SCRIPT = `
+  local data_key = KEYS[1]
+  local pending_key = KEYS[2]
+  local job_id = ARGV[1]
+  local job_data = ARGV[2]
+  local score = tonumber(ARGV[3])
+
+  redis.call('HSET', data_key, job_id, job_data)
+  redis.call('ZADD', pending_key, score, job_id)
+
+  return 1
+`;
+var PUSH_DELAYED_JOB_SCRIPT = `
+  local data_key = KEYS[1]
+  local delayed_key = KEYS[2]
+  local job_id = ARGV[1]
+  local job_data = ARGV[2]
+  local execute_at = tonumber(ARGV[3])
+
+  redis.call('HSET', data_key, job_id, job_data)
+  redis.call('ZADD', delayed_key, execute_at, job_id)
+
+  return 1
+`;
 var ACQUIRE_JOB_SCRIPT = `
-  local pending_key = KEYS[1]
-  local active_key = KEYS[2]
-  local delayed_key = KEYS[3]
+  local data_key = KEYS[1]
+  local pending_key = KEYS[2]
+  local active_key = KEYS[3]
+  local delayed_key = KEYS[4]
   local worker_id = ARGV[1]
-  local now = ARGV[2]
-
-  -- First, process delayed jobs
-  local ready_jobs = redis.call('ZRANGEBYSCORE', delayed_key, 0, now)
-  if #ready_jobs > 0 then
-    for i = 1, #ready_jobs do
-      local job_data = ready_jobs[i]
-      local job = cjson.decode(job_data)
-      -- Score = priority * 1e13 + timestamp
-      -- Lower score = higher priority, FIFO within same priority
-      local priority = job.priority or 5
-      local timestamp = tonumber(now)
-      local score = priority * 10000000000000 + timestamp
-      redis.call('ZADD', pending_key, score, job_data)
-      redis.call('ZREM', delayed_key, job_data)
+  local now = tonumber(ARGV[2])
+
+  -- Process delayed jobs: move ready jobs to pending
+  local ready_job_ids = redis.call('ZRANGEBYSCORE', delayed_key, 0, now)
+  if #ready_job_ids > 0 then
+    for i = 1, #ready_job_ids do
+      local job_id = ready_job_ids[i]
+      local job_data = redis.call('HGET', data_key, job_id)
+      if job_data then
+        local job = cjson.decode(job_data)
+        local priority = job.priority or 5
+        local score = priority * 10000000000000 + now
+        redis.call('ZADD', pending_key, score, job_id)
+        redis.call('ZREM', delayed_key, job_id)
+      end
     end
   end
 
@@ -40,76 +66,139 @@ var ACQUIRE_JOB_SCRIPT = `
     return nil
   end
 
-  local job_data = result[1]
-  local job = cjson.decode(job_data)
+  local job_id = result[1]
+  local job_data = redis.call('HGET', data_key, job_id)
+  if not job_data then
+    return nil
+  end
 
-  -- Store in active hash: jobId -> {workerId, acquiredAt, data}
+  -- Store in active hash (without data, it's in data_key)
   local active_data = cjson.encode({
     workerId = worker_id,
-    acquiredAt = tonumber(now),
-    data = job
+    acquiredAt = now
   })
-  redis.call('HSET', active_key, job.id, active_data)
+  redis.call('HSET', active_key, job_id, active_data)
 
   -- Return job with acquiredAt
-  job.acquiredAt = tonumber(now)
+  local job = cjson.decode(job_data)
+  job.acquiredAt = now
   return cjson.encode(job)
 `;
-var COMPLETE_JOB_SCRIPT = `
-  local active_key = KEYS[1]
+var REMOVE_JOB_SCRIPT = `
+  local data_key = KEYS[1]
+  local active_key = KEYS[2]
   local job_id = ARGV[1]
 
+  if redis.call('HEXISTS', active_key, job_id) == 0 then
+    return 0
+  end
+
   redis.call('HDEL', active_key, job_id)
+  redis.call('HDEL', data_key, job_id)
+
   return 1
 `;
-var FAIL_JOB_SCRIPT = `
-  local active_key = KEYS[1]
+var FINALIZE_JOB_SCRIPT = `
+  local data_key = KEYS[1]
+  local active_key = KEYS[2]
+  local history_key = KEYS[3]
+  local index_key = KEYS[4]
   local job_id = ARGV[1]
+  local now = tonumber(ARGV[2])
+  local max_age = tonumber(ARGV[3])
+  local max_count = tonumber(ARGV[4])
+  local error_message = ARGV[5]
+
+  -- Verify job is active
+  if redis.call('HEXISTS', active_key, job_id) == 0 then
+    return 0
+  end
 
+  -- Remove from active
   redis.call('HDEL', active_key, job_id)
+
+  -- Store finalization info (data stays in data_key)
+  local record = {
+    finishedAt = now
+  }
+  if error_message and error_message ~= '' then
+    record.error = error_message
+  end
+  redis.call('HSET', history_key, job_id, cjson.encode(record))
+  redis.call('ZADD', index_key, now, job_id)
+
+  -- Prune by age
+  if max_age and max_age > 0 then
+    local cutoff = now - max_age
+    local expired = redis.call('ZRANGEBYSCORE', index_key, 0, cutoff)
+    if #expired > 0 then
+      redis.call('ZREM', index_key, unpack(expired))
+      redis.call('HDEL', history_key, unpack(expired))
+      redis.call('HDEL', data_key, unpack(expired))
+    end
+  end
+
+  -- Prune by count
+  if max_count and max_count > 0 then
+    local size = tonumber(redis.call('ZCARD', index_key))
+    if size > max_count then
+      local excess = size - max_count
+      local stale = redis.call('ZRANGE', index_key, 0, excess - 1)
+      if #stale > 0 then
+        redis.call('ZREM', index_key, unpack(stale))
+        redis.call('HDEL', history_key, unpack(stale))
+        redis.call('HDEL', data_key, unpack(stale))
+      end
+    end
+  end
+
   return 1
 `;
 var RETRY_JOB_SCRIPT = `
-  local active_key = KEYS[1]
-  local pending_key = KEYS[2]
-  local delayed_key = KEYS[3]
+  local data_key = KEYS[1]
+  local active_key = KEYS[2]
+  local pending_key = KEYS[3]
+  local delayed_key = KEYS[4]
   local job_id = ARGV[1]
   local retry_at = tonumber(ARGV[2])
   local now = tonumber(ARGV[3])
 
-  -- Get job from active hash
-  local active_data = redis.call('HGET', active_key, job_id)
-  if not active_data then
+  -- Verify job is active
+  if redis.call('HEXISTS', active_key, job_id) == 0 then
     return 0
   end
 
-  local active = cjson.decode(active_data)
-  local job = active.data
+  -- Get job data
+  local job_data = redis.call('HGET', data_key, job_id)
+  if not job_data then
+    return 0
+  end
 
   -- Remove from active
   redis.call('HDEL', active_key, job_id)
 
-  -- Increment attempts
+  -- Increment attempts and update data
+  local job = cjson.decode(job_data)
   job.attempts = (job.attempts or 0) + 1
-
-  local job_data = cjson.encode(job)
+  redis.call('HSET', data_key, job_id, cjson.encode(job))
 
   -- Add back to pending or delayed
   if retry_at and retry_at > now then
-    redis.call('ZADD', delayed_key, retry_at, job_data)
+    redis.call('ZADD', delayed_key, retry_at, job_id)
   else
     -- Score = priority * 1e13 + timestamp
     -- Lower score = higher priority, FIFO within same priority
     local priority = job.priority or 5
     local score = priority * 10000000000000 + now
-    redis.call('ZADD', pending_key, score, job_data)
+    redis.call('ZADD', pending_key, score, job_id)
   end
 
   return 1
 `;
 var RECOVER_STALLED_JOBS_SCRIPT = `
-  local active_key = KEYS[1]
-  local pending_key = KEYS[2]
+  local data_key = KEYS[1]
+  local active_key = KEYS[2]
+  local pending_key = KEYS[3]
   local now = tonumber(ARGV[1])
   local stalled_threshold = tonumber(ARGV[2])
   local max_stalled_count = tonumber(ARGV[3])
@@ -128,31 +217,87 @@ var RECOVER_STALLED_JOBS_SCRIPT = `
 
     -- Check if job is stalled
     if active.acquiredAt < stalled_cutoff then
-      local job = active.data
-      local current_stalled_count = job.stalledCount or 0
-
-      -- Remove from active hash
-      redis.call('HDEL', active_key, job_id)
-
-      -- Check if job has exceeded max stalled count
-      if current_stalled_count >= max_stalled_count then
-        -- Job failed permanently, just remove (already done above)
-      else
-        -- Recover: increment stalledCount and put back in pending
-        job.stalledCount = current_stalled_count + 1
-        local job_data = cjson.encode(job)
-        -- Score = priority * 1e13 + timestamp
-        -- Lower score = higher priority, FIFO within same priority
-        local priority = job.priority or 5
-        local score = priority * 10000000000000 + now
-        redis.call('ZADD', pending_key, score, job_data)
-        recovered = recovered + 1
+      local job_data = redis.call('HGET', data_key, job_id)
+      if job_data then
+        local job = cjson.decode(job_data)
+        local current_stalled_count = job.stalledCount or 0
+
+        -- Remove from active hash
+        redis.call('HDEL', active_key, job_id)
+
+        -- Check if job has exceeded max stalled count
+        if current_stalled_count >= max_stalled_count then
+          -- Job failed permanently, remove data too
+          redis.call('HDEL', data_key, job_id)
+        else
+          -- Recover: increment stalledCount and put back in pending
+          job.stalledCount = current_stalled_count + 1
+          redis.call('HSET', data_key, job_id, cjson.encode(job))
+          -- Score = priority * 1e13 + timestamp
+          local priority = job.priority or 5
+          local score = priority * 10000000000000 + now
+          redis.call('ZADD', pending_key, score, job_id)
+          recovered = recovered + 1
+        end
       end
     end
   end
 
   return recovered
 `;
+var GET_JOB_SCRIPT = `
+  local data_key = KEYS[1]
+  local pending_key = KEYS[2]
+  local delayed_key = KEYS[3]
+  local active_key = KEYS[4]
+  local completed_key = KEYS[5]
+  local failed_key = KEYS[6]
+  local job_id = ARGV[1]
+
+  local job_data = redis.call('HGET', data_key, job_id)
+  if not job_data then
+    return nil
+  end
+
+  local status = nil
+  local finished_at = nil
+  local error_msg = nil
+
+  -- Check status in order
+  if redis.call('HEXISTS', active_key, job_id) == 1 then
+    status = 'active'
+  elseif redis.call('ZSCORE', pending_key, job_id) then
+    status = 'pending'
+  elseif redis.call('ZSCORE', delayed_key, job_id) then
+    status = 'delayed'
+  else
+    local completed_data = redis.call('HGET', completed_key, job_id)
+    if completed_data then
+      status = 'completed'
+      local record = cjson.decode(completed_data)
+      finished_at = record.finishedAt
+    else
+      local failed_data = redis.call('HGET', failed_key, job_id)
+      if failed_data then
+        status = 'failed'
+        local record = cjson.decode(failed_data)
+        finished_at = record.finishedAt
+        error_msg = record.error
+      end
+    end
+  end
+
+  if not status then
+    return nil
+  end
+
+  return cjson.encode({
+    status = status,
+    data = cjson.decode(job_data),
+    finishedAt = finished_at,
+    error = error_msg
+  })
+`;
 var CLAIM_SCHEDULE_SCRIPT = `
   local schedule_key = KEYS[1]
   local now = tonumber(ARGV[1])
@@ -246,6 +391,18 @@ var RedisAdapter = class {
     this.#connection = connection;
     this.#ownsConnection = ownsConnection;
   }
+  #getKeys(queue) {
+    return {
+      data: `${redisKey}::${queue}::data`,
+      pending: `${redisKey}::${queue}::pending`,
+      delayed: `${redisKey}::${queue}::delayed`,
+      active: `${redisKey}::${queue}::active`,
+      completed: `${redisKey}::${queue}::completed`,
+      completedIndex: `${redisKey}::${queue}::completed::index`,
+      failed: `${redisKey}::${queue}::failed`,
+      failedIndex: `${redisKey}::${queue}::failed::index`
+    };
+  }
   setWorkerId(workerId) {
     this.#workerId = workerId;
   }
@@ -258,16 +415,15 @@ var RedisAdapter = class {
     return this.popFrom("default");
   }
   async popFrom(queue) {
+    const keys = this.#getKeys(queue);
     const now = Date.now();
-    const pendingKey = `${redisKey}::${queue}`;
-    const activeKey = `${redisKey}::${queue}::active`;
-    const delayedKey = `${redisKey}::delayed::${queue}`;
     const result = await this.#connection.eval(
       ACQUIRE_JOB_SCRIPT,
-      3,
-      pendingKey,
-      activeKey,
-      delayedKey,
+      4,
+      keys.data,
+      keys.pending,
+      keys.active,
+      keys.delayed,
       this.#workerId,
       now.toString()
     );
@@ -276,30 +432,81 @@ var RedisAdapter = class {
     }
     return JSON.parse(result);
   }
-  async completeJob(jobId, queue) {
-    const activeKey = `${redisKey}::${queue}::active`;
-    await this.#connection.eval(COMPLETE_JOB_SCRIPT, 1, activeKey, jobId);
+  async completeJob(jobId, queue, removeOnComplete) {
+    const keys = this.#getKeys(queue);
+    const { keep, maxAge, maxCount } = resolveRetention(removeOnComplete);
+    if (!keep) {
+      await this.#connection.eval(REMOVE_JOB_SCRIPT, 2, keys.data, keys.active, jobId);
+      return;
+    }
+    await this.#connection.eval(
+      FINALIZE_JOB_SCRIPT,
+      4,
+      keys.data,
+      keys.active,
+      keys.completed,
+      keys.completedIndex,
+      jobId,
+      Date.now().toString(),
+      maxAge.toString(),
+      maxCount.toString(),
+      ""
+    );
   }
-  async failJob(jobId, queue, _error) {
-    const activeKey = `${redisKey}::${queue}::active`;
-    await this.#connection.eval(FAIL_JOB_SCRIPT, 1, activeKey, jobId);
+  async failJob(jobId, queue, error, removeOnFail) {
+    const keys = this.#getKeys(queue);
+    const { keep, maxAge, maxCount } = resolveRetention(removeOnFail);
+    if (!keep) {
+      await this.#connection.eval(REMOVE_JOB_SCRIPT, 2, keys.data, keys.active, jobId);
+      return;
+    }
+    await this.#connection.eval(
+      FINALIZE_JOB_SCRIPT,
+      4,
+      keys.data,
+      keys.active,
+      keys.failed,
+      keys.failedIndex,
+      jobId,
+      Date.now().toString(),
+      maxAge.toString(),
+      maxCount.toString(),
+      error?.message || ""
+    );
   }
   async retryJob(jobId, queue, retryAt) {
+    const keys = this.#getKeys(queue);
     const now = Date.now();
-    const activeKey = `${redisKey}::${queue}::active`;
-    const pendingKey = `${redisKey}::${queue}`;
-    const delayedKey = `${redisKey}::delayed::${queue}`;
     await this.#connection.eval(
       RETRY_JOB_SCRIPT,
-      3,
-      activeKey,
-      pendingKey,
-      delayedKey,
+      4,
+      keys.data,
+      keys.active,
+      keys.pending,
+      keys.delayed,
       jobId,
       retryAt ? retryAt.getTime().toString() : "0",
       now.toString()
     );
   }
+  async getJob(jobId, queue) {
+    const keys = this.#getKeys(queue);
+    const result = await this.#connection.eval(
+      GET_JOB_SCRIPT,
+      6,
+      keys.data,
+      keys.pending,
+      keys.delayed,
+      keys.active,
+      keys.completed,
+      keys.failed,
+      jobId
+    );
+    if (!result) {
+      return null;
+    }
+    return JSON.parse(result);
+  }
   push(jobData) {
     return this.pushOn("default", jobData);
   }
@@ -307,31 +514,65 @@ var RedisAdapter = class {
     return this.pushLaterOn("default", jobData, delay);
   }
   async pushLaterOn(queue, jobData, delay) {
+    const keys = this.#getKeys(queue);
     const executeAt = Date.now() + delay;
-    const delayedKey = `${redisKey}::delayed::${queue}`;
-    await this.#connection.zadd(delayedKey, executeAt, JSON.stringify(jobData));
+    await this.#connection.eval(
+      PUSH_DELAYED_JOB_SCRIPT,
+      2,
+      keys.data,
+      keys.delayed,
+      jobData.id,
+      JSON.stringify(jobData),
+      executeAt.toString()
+    );
   }
   async pushOn(queue, jobData) {
+    const keys = this.#getKeys(queue);
     const priority = jobData.priority ?? DEFAULT_PRIORITY;
     const timestamp = Date.now();
     const score = calculateScore(priority, timestamp);
-    await this.#connection.zadd(`${redisKey}::${queue}`, score, JSON.stringify(jobData));
+    await this.#connection.eval(
+      PUSH_JOB_SCRIPT,
+      2,
+      keys.data,
+      keys.pending,
+      jobData.id,
+      JSON.stringify(jobData),
+      score.toString()
+    );
+  }
+  pushMany(jobs) {
+    return this.pushManyOn("default", jobs);
+  }
+  async pushManyOn(queue, jobs) {
+    if (jobs.length === 0) return;
+    const keys = this.#getKeys(queue);
+    const now = Date.now();
+    const multi = this.#connection.multi();
+    for (const job of jobs) {
+      const priority = job.priority ?? DEFAULT_PRIORITY;
+      const score = calculateScore(priority, now);
+      multi.hset(keys.data, job.id, JSON.stringify(job));
+      multi.zadd(keys.pending, score, job.id);
+    }
+    await multi.exec();
   }
   size() {
     return this.sizeOf("default");
   }
   sizeOf(queue) {
-    return this.#connection.zcard(`${redisKey}::${queue}`);
+    const keys = this.#getKeys(queue);
+    return this.#connection.zcard(keys.pending);
   }
   async recoverStalledJobs(queue, stalledThreshold, maxStalledCount) {
+    const keys = this.#getKeys(queue);
     const now = Date.now();
-    const activeKey = `${redisKey}::${queue}::active`;
-    const pendingKey = `${redisKey}::${queue}`;
     const recovered = await this.#connection.eval(
       RECOVER_STALLED_JOBS_SCRIPT,
-      2,
-      activeKey,
-      pendingKey,
+      3,
+      keys.data,
+      keys.active,
+      keys.pending,
       now.toString(),
       stalledThreshold.toString(),
       maxStalledCount.toString()