@nicnocquee/dataqueue 1.34.0 → 1.35.0-beta.20260224110011

package/src/backends/redis-scripts.ts

@@ -31,7 +31,8 @@ const SCORE_RANGE = '1000000000000000'; // 1e15
  * ADD JOB
  * KEYS: [prefix]
  * ARGV: [jobType, payloadJson, maxAttempts, priority, runAtMs, timeoutMs,
- *        forceKillOnTimeout, tagsJson, idempotencyKey, nowMs]
+ *        forceKillOnTimeout, tagsJson, idempotencyKey, nowMs,
+ *        retryDelay, retryBackoff, retryDelayMax, groupId, groupTier]
  * Returns: job ID (number)
  */
 export const ADD_JOB_SCRIPT = `
@@ -46,6 +47,11 @@ local forceKillOnTimeout = ARGV[7]
 local tagsJson = ARGV[8] -- "null" or JSON array string
 local idempotencyKey = ARGV[9] -- "null" string if not set
 local nowMs = tonumber(ARGV[10])
+local retryDelay = ARGV[11]       -- "null" or seconds string
+local retryBackoff = ARGV[12]     -- "null" or "true"/"false"
+local retryDelayMax = ARGV[13]    -- "null" or seconds string
+local groupId = ARGV[14]          -- "null" or group ID
+local groupTier = ARGV[15]        -- "null" or group tier
 
 -- Idempotency check
 if idempotencyKey ~= "null" then
@@ -89,7 +95,12 @@ redis.call('HMSET', jobKey,
   'idempotencyKey', idempotencyKey,
   'waitUntil', 'null',
   'waitTokenId', 'null',
-  'stepData', 'null'
+  'stepData', 'null',
+  'retryDelay', retryDelay,
+  'retryBackoff', retryBackoff,
+  'retryDelayMax', retryDelayMax,
+  'groupId', groupId,
+  'groupTier', groupTier
 )
 
 -- Status index
@@ -131,11 +142,138 @@ end
 return id
 `;
 
+/**
+ * ADD JOBS (batch)
+ * KEYS: [prefix]
+ * ARGV: [jobsJson, nowMs]
+ *   jobsJson is a JSON array of objects, each with:
+ *     jobType, payload (already JSON string), maxAttempts, priority,
+ *     runAtMs, timeoutMs, forceKillOnTimeout, tags (JSON or "null"),
+ *     idempotencyKey
+ * Returns: array of job IDs (one per input job, in order)
+ */
+export const ADD_JOBS_SCRIPT = `
+local prefix = KEYS[1]
+local jobsJson = ARGV[1]
+local nowMs = tonumber(ARGV[2])
+
+local jobs = cjson.decode(jobsJson)
+local results = {}
+
+for i, job in ipairs(jobs) do
+  local jobType = job.jobType
+  local payloadJson = job.payload
+  local maxAttempts = tonumber(job.maxAttempts)
+  local priority = tonumber(job.priority)
+  local runAtMs = tostring(job.runAtMs)
+  local timeoutMs = tostring(job.timeoutMs)
+  local forceKillOnTimeout = tostring(job.forceKillOnTimeout)
+  local tagsJson = tostring(job.tags)
+  local idempotencyKey = tostring(job.idempotencyKey)
+  local retryDelay = tostring(job.retryDelay)
+  local retryBackoff = tostring(job.retryBackoff)
+  local retryDelayMax = tostring(job.retryDelayMax)
+  local groupId = tostring(job.groupId)
+  local groupTier = tostring(job.groupTier)
+
+  -- Idempotency check
+  local skip = false
+  if idempotencyKey ~= "null" then
+    local existing = redis.call('GET', prefix .. 'idempotency:' .. idempotencyKey)
+    if existing then
+      results[i] = tonumber(existing)
+      skip = true
+    end
+  end
+
+  if not skip then
+    -- Generate ID
+    local id = redis.call('INCR', prefix .. 'id_seq')
+    local jobKey = prefix .. 'job:' .. id
+    local runAt = runAtMs ~= "0" and tonumber(runAtMs) or nowMs
+
+    -- Store the job hash
+    redis.call('HMSET', jobKey,
+      'id', id,
+      'jobType', jobType,
+      'payload', payloadJson,
+      'status', 'pending',
+      'maxAttempts', maxAttempts,
+      'attempts', 0,
+      'priority', priority,
+      'runAt', runAt,
+      'timeoutMs', timeoutMs,
+      'forceKillOnTimeout', forceKillOnTimeout,
+      'createdAt', nowMs,
+      'updatedAt', nowMs,
+      'lockedAt', 'null',
+      'lockedBy', 'null',
+      'nextAttemptAt', 'null',
+      'pendingReason', 'null',
+      'errorHistory', '[]',
+      'failureReason', 'null',
+      'completedAt', 'null',
+      'startedAt', 'null',
+      'lastRetriedAt', 'null',
+      'lastFailedAt', 'null',
+      'lastCancelledAt', 'null',
+      'tags', tagsJson,
+      'idempotencyKey', idempotencyKey,
+      'waitUntil', 'null',
+      'waitTokenId', 'null',
+      'stepData', 'null',
+      'retryDelay', retryDelay,
+      'retryBackoff', retryBackoff,
+      'retryDelayMax', retryDelayMax,
+      'groupId', groupId,
+      'groupTier', groupTier
+    )
+
+    -- Status index
+    redis.call('SADD', prefix .. 'status:pending', id)
+
+    -- Type index
+    redis.call('SADD', prefix .. 'type:' .. jobType, id)
+
+    -- Tag indexes
+    if tagsJson ~= "null" then
+      local tags = cjson.decode(tagsJson)
+      for _, tag in ipairs(tags) do
+        redis.call('SADD', prefix .. 'tag:' .. tag, id)
+      end
+      for _, tag in ipairs(tags) do
+        redis.call('SADD', prefix .. 'job:' .. id .. ':tags', tag)
+      end
+    end
+
+    -- Idempotency mapping
+    if idempotencyKey ~= "null" then
+      redis.call('SET', prefix .. 'idempotency:' .. idempotencyKey, id)
+    end
+
+    -- All-jobs sorted set
+    redis.call('ZADD', prefix .. 'all', nowMs, id)
+
+    -- Queue or delayed
+    if runAt <= nowMs then
+      local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - nowMs)
+      redis.call('ZADD', prefix .. 'queue', score, id)
+    else
+      redis.call('ZADD', prefix .. 'delayed', runAt, id)
+    end
+
+    results[i] = id
+  end
+end
+
+return results
+`;
+
 /**
  * GET NEXT BATCH
  * Atomically: move ready delayed/retry jobs into queue, then pop N jobs.
  * KEYS: [prefix]
- * ARGV: [workerId, batchSize, nowMs, jobTypeFilter]
+ * ARGV: [workerId, batchSize, nowMs, jobTypeFilter, groupConcurrency]
  * jobTypeFilter: "null" or a JSON array like ["email","sms"] or a string like "email"
  * Returns: array of job field arrays (flat: [field1, val1, field2, val2, ...] per job)
  */
@@ -145,6 +283,12 @@ local workerId = ARGV[1]
 local batchSize = tonumber(ARGV[2])
 local nowMs = tonumber(ARGV[3])
 local jobTypeFilter = ARGV[4] -- "null" or JSON array or single string
+local groupConcurrencyRaw = ARGV[5] -- "null" or positive integer
+local groupConcurrency = nil
+if groupConcurrencyRaw ~= "null" then
+  groupConcurrency = tonumber(groupConcurrencyRaw)
+end
+local groupActiveKey = prefix .. 'group:active'
 
 -- 1. Move ready delayed jobs into queue
 local delayed = redis.call('ZRANGEBYSCORE', prefix .. 'delayed', '-inf', nowMs, 'LIMIT', 0, 200)
@@ -245,36 +389,53 @@ for i = 1, #candidates, 2 do
         -- Not ready yet: move to delayed
         redis.call('ZADD', prefix .. 'delayed', runAt, jobId)
       else
-        -- Claim this job
-        local attempts = tonumber(redis.call('HGET', jk, 'attempts'))
-        local startedAt = redis.call('HGET', jk, 'startedAt')
-        local lastRetriedAt = redis.call('HGET', jk, 'lastRetriedAt')
-        if startedAt == 'null' then startedAt = nowMs end
-        if attempts > 0 then lastRetriedAt = nowMs end
-
-        redis.call('HMSET', jk,
-          'status', 'processing',
-          'lockedAt', nowMs,
-          'lockedBy', workerId,
-          'attempts', attempts + 1,
-          'updatedAt', nowMs,
-          'pendingReason', 'null',
-          'startedAt', startedAt,
-          'lastRetriedAt', lastRetriedAt
-        )
-
-        -- Update status sets
-        redis.call('SREM', prefix .. 'status:pending', jobId)
-        redis.call('SADD', prefix .. 'status:processing', jobId)
+        local groupId = redis.call('HGET', jk, 'groupId')
+        local hasGroup = groupId and groupId ~= 'null'
+        local canClaim = true
+        if hasGroup and groupConcurrency then
+          local activeCount = tonumber(redis.call('HGET', groupActiveKey, groupId) or '0')
+          if activeCount >= groupConcurrency then
+            table.insert(putBack, score)
+            table.insert(putBack, jobId)
+            canClaim = false
+          end
+        end
 
-        -- Return job data as flat array
-        local data = redis.call('HGETALL', jk)
-        for _, v in ipairs(data) do
-          table.insert(results, v)
+        if canClaim then
+          -- Claim this job
+          local attempts = tonumber(redis.call('HGET', jk, 'attempts'))
+          local startedAt = redis.call('HGET', jk, 'startedAt')
+          local lastRetriedAt = redis.call('HGET', jk, 'lastRetriedAt')
+          if startedAt == 'null' then startedAt = nowMs end
+          if attempts > 0 then lastRetriedAt = nowMs end
+
+          redis.call('HMSET', jk,
+            'status', 'processing',
+            'lockedAt', nowMs,
+            'lockedBy', workerId,
+            'attempts', attempts + 1,
+            'updatedAt', nowMs,
+            'pendingReason', 'null',
+            'startedAt', startedAt,
+            'lastRetriedAt', lastRetriedAt
+          )
+
+          -- Update status sets
+          redis.call('SREM', prefix .. 'status:pending', jobId)
+          redis.call('SADD', prefix .. 'status:processing', jobId)
+          if hasGroup and groupConcurrency then
+            redis.call('HINCRBY', groupActiveKey, groupId, 1)
+          end
+
+          -- Return job data as flat array
+          local data = redis.call('HGETALL', jk)
+          for _, v in ipairs(data) do
+            table.insert(results, v)
+          end
+          -- Separator
+          table.insert(results, '__JOB_SEP__')
+          jobsClaimed = jobsClaimed + 1
         end
-        -- Separator
-        table.insert(results, '__JOB_SEP__')
-        jobsClaimed = jobsClaimed + 1
       end
     end
   end
@@ -291,24 +452,40 @@ return results
 /**
  * COMPLETE JOB
  * KEYS: [prefix]
- * ARGV: [jobId, nowMs]
+ * ARGV: [jobId, nowMs, outputJson]
  */
 export const COMPLETE_JOB_SCRIPT = `
 local prefix = KEYS[1]
 local jobId = ARGV[1]
 local nowMs = ARGV[2]
+local outputJson = ARGV[3]
 local jk = prefix .. 'job:' .. jobId
+local groupId = redis.call('HGET', jk, 'groupId')
 
-redis.call('HMSET', jk,
+local fields = {
   'status', 'completed',
   'updatedAt', nowMs,
   'completedAt', nowMs,
   'stepData', 'null',
   'waitUntil', 'null',
   'waitTokenId', 'null'
-)
+}
+
+if outputJson ~= '__NONE__' then
+  fields[#fields + 1] = 'output'
+  fields[#fields + 1] = outputJson
+end
+
+redis.call('HMSET', jk, unpack(fields))
 redis.call('SREM', prefix .. 'status:processing', jobId)
 redis.call('SADD', prefix .. 'status:completed', jobId)
+if groupId and groupId ~= 'null' then
+  local activeKey = prefix .. 'group:active'
+  local remaining = redis.call('HINCRBY', activeKey, groupId, -1)
+  if tonumber(remaining) <= 0 then
+    redis.call('HDEL', activeKey, groupId)
+  end
+end
 
 return 1
 `;
@@ -326,15 +503,43 @@ local errorJson = ARGV[2]
 local failureReason = ARGV[3]
 local nowMs = tonumber(ARGV[4])
 local jk = prefix .. 'job:' .. jobId
+local groupId = redis.call('HGET', jk, 'groupId')
 
 local attempts = tonumber(redis.call('HGET', jk, 'attempts'))
 local maxAttempts = tonumber(redis.call('HGET', jk, 'maxAttempts'))
 
--- Compute next_attempt_at: 2^attempts minutes from now
+-- Read per-job retry config (may be "null")
+local rdRaw = redis.call('HGET', jk, 'retryDelay')
+local rbRaw = redis.call('HGET', jk, 'retryBackoff')
+local rmRaw = redis.call('HGET', jk, 'retryDelayMax')
+
 local nextAttemptAt = 'null'
 if attempts < maxAttempts then
-  local delayMs = math.pow(2, attempts) * 60000
-  nextAttemptAt = nowMs + delayMs
+  local allNull = (rdRaw == 'null' or rdRaw == false)
+               and (rbRaw == 'null' or rbRaw == false)
+               and (rmRaw == 'null' or rmRaw == false)
+  if allNull then
+    -- Legacy formula: 2^attempts minutes
+    local delayMs = math.pow(2, attempts) * 60000
+    nextAttemptAt = nowMs + delayMs
+  else
+    local retryDelaySec = 60
+    if rdRaw and rdRaw ~= 'null' then retryDelaySec = tonumber(rdRaw) end
+    local useBackoff = true
+    if rbRaw and rbRaw ~= 'null' then useBackoff = (rbRaw == 'true') end
+    local maxDelaySec = nil
+    if rmRaw and rmRaw ~= 'null' then maxDelaySec = tonumber(rmRaw) end
+
+    local delaySec
+    if useBackoff then
+      delaySec = retryDelaySec * math.pow(2, attempts)
+      if maxDelaySec then delaySec = math.min(delaySec, maxDelaySec) end
+      delaySec = delaySec * (0.5 + 0.5 * math.random())
+    else
+      delaySec = retryDelaySec
+    end
+    nextAttemptAt = nowMs + math.floor(delaySec * 1000)
+  end
 end
 
 -- Append to error_history
@@ -356,6 +561,13 @@ redis.call('HMSET', jk,
 )
 redis.call('SREM', prefix .. 'status:processing', jobId)
 redis.call('SADD', prefix .. 'status:failed', jobId)
+if groupId and groupId ~= 'null' then
+  local activeKey = prefix .. 'group:active'
+  local remaining = redis.call('HINCRBY', activeKey, groupId, -1)
+  if tonumber(remaining) <= 0 then
+    redis.call('HDEL', activeKey, groupId)
+  end
+end
 
 -- Schedule retry if applicable
 if nextAttemptAt ~= 'null' then
@@ -378,6 +590,7 @@ local jk = prefix .. 'job:' .. jobId
 
 local oldStatus = redis.call('HGET', jk, 'status')
 if oldStatus ~= 'failed' and oldStatus ~= 'processing' then return 0 end
+local groupId = redis.call('HGET', jk, 'groupId')
 
 redis.call('HMSET', jk,
   'status', 'pending',
@@ -391,6 +604,13 @@ redis.call('HMSET', jk,
 -- Remove from old status, add to pending
 redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
 redis.call('SADD', prefix .. 'status:pending', jobId)
+if oldStatus == 'processing' and groupId and groupId ~= 'null' then
+  local activeKey = prefix .. 'group:active'
+  local remaining = redis.call('HINCRBY', activeKey, groupId, -1)
+  if tonumber(remaining) <= 0 then
+    redis.call('HDEL', activeKey, groupId)
+  end
+end
 
 -- Remove from retry sorted set if present
 redis.call('ZREM', prefix .. 'retry', jobId)
@@ -496,6 +716,14 @@ for _, jobId in ipairs(processing) do
         )
         redis.call('SREM', prefix .. 'status:processing', jobId)
         redis.call('SADD', prefix .. 'status:pending', jobId)
+        local groupId = redis.call('HGET', jk, 'groupId')
+        if groupId and groupId ~= 'null' then
+          local activeKey = prefix .. 'group:active'
+          local remaining = redis.call('HINCRBY', activeKey, groupId, -1)
+          if tonumber(remaining) <= 0 then
+            redis.call('HDEL', activeKey, groupId)
+          end
+        end
 
         -- Re-add to queue
         local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
@@ -584,6 +812,7 @@ local jk = prefix .. 'job:' .. jobId
 
 local status = redis.call('HGET', jk, 'status')
 if status ~= 'processing' then return 0 end
+local groupId = redis.call('HGET', jk, 'groupId')
 
 redis.call('HMSET', jk,
   'status', 'waiting',
@@ -596,6 +825,13 @@ redis.call('HMSET', jk,
 )
 redis.call('SREM', prefix .. 'status:processing', jobId)
 redis.call('SADD', prefix .. 'status:waiting', jobId)
+if groupId and groupId ~= 'null' then
+  local activeKey = prefix .. 'group:active'
+  local remaining = redis.call('HINCRBY', activeKey, groupId, -1)
+  if tonumber(remaining) <= 0 then
+    redis.call('HDEL', activeKey, groupId)
+  end
+end
 
 -- Add to waiting sorted set if time-based wait
 if waitUntilMs ~= 'null' then