groupmq-plus 1.1.0

package/dist/lua/check-stalled.lua

@@ -0,0 +1,86 @@
+-- Check for stalled jobs and move them back to waiting or fail them
+-- KEYS: namespace, currentTime, gracePeriod, maxStalledCount
+-- Returns: array of [jobId, groupId, action] for each stalled job found
+--   action: "recovered" or "failed"
+
+local ns = KEYS[1]
+local now = tonumber(ARGV[1])
+local gracePeriod = tonumber(ARGV[2]) or 0
+local maxStalledCount = tonumber(ARGV[3]) or 1
+
+-- Circuit breaker for high concurrency: limit stalled job recovery
+local circuitBreakerKey = ns .. ":stalled:circuit"
+local lastCheck = redis.call("GET", circuitBreakerKey)
+if lastCheck then
+  local lastCheckTime = tonumber(lastCheck)
+  local circuitBreakerInterval = 2000
+  if lastCheckTime and (now - lastCheckTime) < circuitBreakerInterval then
+    return {}
+  end
+end
+redis.call("SET", circuitBreakerKey, now, "PX", 3000)
+
+local processingKey = ns .. ":processing"
+local groupsKey = ns .. ":groups"
+
+-- Candidates: jobs whose deadlines are past
+local candidates = redis.call("ZRANGEBYSCORE", processingKey, 0, now - gracePeriod, "LIMIT", 0, 100)
+if not candidates or #candidates == 0 then
+  return {}
+end
+
+local results = {}
+
+for _, jobId in ipairs(candidates) do
+  local jobKey = ns .. ":job:" .. jobId
+  local h = redis.call("HMGET", jobKey, "groupId","stalledCount","maxAttempts","attempts","status","finishedOn","score")
+  local groupId = h[1]
+  if groupId then
+    local stalledCount = tonumber(h[2]) or 0
+    local maxAttempts = tonumber(h[3]) or 3
+    local status = h[5]
+    local finishedOn = tonumber(h[6] or "0")
+    -- CRITICAL: Don't recover jobs that are completing (prevents race with completion)
+    -- "completing" is a temporary state set by complete-with-metadata.lua to prevent races
+    if status == "processing" then
+      stalledCount = stalledCount + 1
+      redis.call("HSET", jobKey, "stalledCount", stalledCount)
+      -- BullMQ-style: Remove from per-group active list
+      local groupActiveKey = ns .. ":g:" .. groupId .. ":active"
+      redis.call("LREM", groupActiveKey, 1, jobId)
+      
+      if stalledCount >= maxStalledCount and maxStalledCount > 0 then
+        redis.call("ZREM", processingKey, jobId)
+        local groupKey = ns .. ":g:" .. groupId
+        redis.call("ZREM", groupKey, jobId)
+        redis.call("DEL", ns .. ":processing:" .. jobId)
+        redis.call("HSET", jobKey, "status","failed","finishedOn", now,
+                   "failedReason", "Job stalled " .. stalledCount .. " times (max: " .. maxStalledCount .. ")")
+        redis.call("ZADD", ns .. ":failed", now, jobId)
+        table.insert(results, jobId); table.insert(results, groupId); table.insert(results, "failed")
+      else
+        local stillInProcessing = redis.call("ZSCORE", processingKey, jobId)
+        if stillInProcessing then
+          redis.call("ZREM", processingKey, jobId)
+          redis.call("DEL", ns .. ":processing:" .. jobId)
+          local score = tonumber(h[7])
+          if score then
+            local groupKey2 = ns .. ":g:" .. groupId
+            redis.call("ZADD", groupKey2, score, jobId)
+            local head = redis.call("ZRANGE", groupKey2, 0, 0, "WITHSCORES")
+            if head and #head >= 2 then
+              local headScore = tonumber(head[2])
+              redis.call("ZADD", ns .. ":ready", headScore, groupId)
+            end
+            redis.call("SADD", groupsKey, groupId)
+          end
+          redis.call("HSET", jobKey, "status", "waiting")
+          table.insert(results, jobId); table.insert(results, groupId); table.insert(results, "recovered")
+        end
+      end
+    end
+  end
+end
+
+return results
+

package/dist/lua/clean-status.lua

@@ -0,0 +1,64 @@
+-- argv: ns, status, graceAtMs, limit
+local ns = KEYS[1]
+local status = ARGV[1]
+local graceAt = tonumber(ARGV[2]) or 0
+local limit = tonumber(ARGV[3]) or 1000
+
+local setKey = nil
+if status == 'completed' then
+  setKey = ns .. ':completed'
+elseif status == 'failed' then
+  setKey = ns .. ':failed'
+elseif status == 'delayed' then
+  setKey = ns .. ':delayed'
+else
+  -- unsupported status for clean
+  return 0
+end
+
+-- Fetch up to 'limit' job ids with score <= graceAt
+local ids = redis.call('ZRANGEBYSCORE', setKey, '-inf', graceAt, 'LIMIT', 0, limit)
+
+local removed = 0
+for i = 1, #ids do
+  local id = ids[i]
+  local jobKey = ns .. ':job:' .. id
+
+  -- Remove from the primary set first to avoid reprocessing
+  redis.call('ZREM', setKey, id)
+
+  -- Remove from group and update ready queue for ALL statuses
+  -- This prevents poisoned groups when completed/failed jobs are cleaned
+  local groupId = redis.call('HGET', jobKey, 'groupId')
+  if groupId then
+    local gZ = ns .. ':g:' .. groupId
+    local readyKey = ns .. ':ready'
+    redis.call('ZREM', gZ, id)
+    local jobCount = redis.call('ZCARD', gZ)
+    if jobCount == 0 then
+      redis.call('ZREM', readyKey, groupId)
+      -- Clean up empty group
+      redis.call('DEL', gZ)
+      redis.call('SREM', ns .. ':groups', groupId)
+    elseif status == 'delayed' then
+      -- Only update ready queue score for delayed jobs
+      -- (completed/failed jobs shouldn't affect ready queue)
+      local head = redis.call('ZRANGE', gZ, 0, 0, 'WITHSCORES')
+      if head and #head >= 2 then
+        local headScore = tonumber(head[2])
+        redis.call('ZADD', readyKey, headScore, groupId)
+      end
+    end
+  end
+
+  -- Delete job hash, idempotence key and flow results
+  redis.call('DEL', jobKey)
+  redis.call('DEL', ns .. ':unique:' .. id)
+  redis.call('DEL', ns .. ':flow:results:' .. id)
+
+  removed = removed + 1
+end
+
+return removed
+
+

package/dist/lua/cleanup-poisoned-group.lua

@@ -0,0 +1,46 @@
+-- argv: ns, groupId, now
+local ns = KEYS[1]
+local groupId = ARGV[1]
+local now = tonumber(ARGV[2])
+
+local readyKey = ns .. ":ready"
+local gZ = ns .. ":g:" .. groupId
+local lockKey = ns .. ":lock:" .. groupId
+
+-- Check if group has any jobs at all
+local jobCount = redis.call("ZCARD", gZ)
+if jobCount == 0 then
+  redis.call("ZREM", readyKey, groupId)
+  return "empty"
+end
+
+-- Check if group is currently locked by another worker
+local lockValue = redis.call("GET", lockKey)
+if lockValue then
+  local lockTtl = redis.call("PTTL", lockKey)
+  if lockTtl > 0 then
+    return "locked"
+  end
+end
+
+-- Check if all jobs in the group have exceeded max attempts
+local jobs = redis.call("ZRANGE", gZ, 0, -1)
+local reservableJobs = 0
+for i = 1, #jobs do
+  local jobId = jobs[i]
+  local jobKey = ns .. ":job:" .. jobId
+  local attempts = tonumber(redis.call("HGET", jobKey, "attempts"))
+  local maxAttempts = tonumber(redis.call("HGET", jobKey, "maxAttempts"))
+  if attempts and maxAttempts and attempts < maxAttempts then
+    reservableJobs = reservableJobs + 1
+  end
+end
+
+if reservableJobs == 0 then
+  redis.call("ZREM", readyKey, groupId)
+  return "poisoned"
+end
+
+return "ok"
+
+

package/dist/lua/cleanup.lua

@@ -0,0 +1,46 @@
+-- argv: ns, nowEpochMs
+local ns = KEYS[1]
+local now = tonumber(ARGV[1])
+
+local readyKey = ns .. ":ready"
+local processingKey = ns .. ":processing"
+local cleaned = 0
+
+local expiredJobs = redis.call("ZRANGEBYSCORE", processingKey, 0, now)
+for _, jobId in ipairs(expiredJobs) do
+  -- CRITICAL: Verify job is STILL in processing to avoid race conditions
+  -- If job was completed between our snapshot and now, don't re-add it
+  local stillInProcessing = redis.call("ZSCORE", processingKey, jobId)
+  
+  if stillInProcessing then
+    local procKey = ns .. ":processing:" .. jobId
+    local procData = redis.call("HMGET", procKey, "groupId", "deadlineAt")
+    local gid = procData[1]
+    local deadlineAt = tonumber(procData[2])
+    if gid and deadlineAt and now > deadlineAt then
+      local jobKey = ns .. ":job:" .. jobId
+      local jobScore = redis.call("HGET", jobKey, "score")
+      if jobScore then
+        local gZ = ns .. ":g:" .. gid
+        redis.call("ZADD", gZ, tonumber(jobScore), jobId)
+        local head = redis.call("ZRANGE", gZ, 0, 0, "WITHSCORES")
+        if head and #head >= 2 then
+          local headScore = tonumber(head[2])
+          redis.call("ZADD", readyKey, headScore, gid)
+        end
+        redis.call("DEL", ns .. ":lock:" .. gid)
+        redis.call("DEL", procKey)
+        redis.call("ZREM", processingKey, jobId)
+        
+        -- No counter operations - use ZCARD for counts
+        
+        cleaned = cleaned + 1
+      end
+    end
+  end
+  -- If not still in processing, it was completed - don't re-add it!
+end
+
+return cleaned
+
+

package/dist/lua/complete-and-reserve-next-with-metadata.lua

@@ -0,0 +1,221 @@
+-- Complete a job with metadata and atomically reserve the next job from the same group
+-- argv: ns, completedJobId, groupId, status, timestamp, resultOrError, keepCompleted, keepFailed,
+--       processedOn, finishedOn, attempts, maxAttempts, now, vt
+local ns = KEYS[1]
+local completedJobId = ARGV[1]
+local gid = ARGV[2]
+local status = ARGV[3]
+local timestamp = tonumber(ARGV[4])
+local resultOrError = ARGV[5]
+local keepCompleted = tonumber(ARGV[6])
+local keepFailed = tonumber(ARGV[7])
+local processedOn = ARGV[8]
+local finishedOn = ARGV[9]
+local attempts = ARGV[10]
+local maxAttempts = ARGV[11]
+local now = tonumber(ARGV[12])
+local vt = tonumber(ARGV[13])
+
+local jobKey = ns .. ":job:" .. completedJobId
+
+-- [PHASE 3 MODIFICATION START: Get parentId before potentially deleting the job]
+local parentId = redis.call("HGET", jobKey, "parentId")
+-- [PHASE 3 MODIFICATION END]
+
+-- Part 1: Atomically verify and mark completion (prevent duplicate processing)
+local processingKey = ns .. ":processing"
+
+-- CRITICAL: Check both status AND processing set membership atomically
+-- This prevents race with stalled job recovery
+local jobStatus = redis.call("HGET", jobKey, "status")
+local stillInProcessing = redis.call("ZSCORE", processingKey, completedJobId)
+
+-- If job is not in "processing" state OR not in processing set, this is late/duplicate
+if jobStatus ~= "processing" or not stillInProcessing then
+  return nil
+end
+
+-- Atomically mark as completed and remove from processing
+-- This prevents stalled checker from racing with us
+redis.call("HSET", jobKey, "status", "completing") -- Temporary status to block stalled checker
+redis.call("DEL", ns .. ":processing:" .. completedJobId)
+redis.call("ZREM", processingKey, completedJobId)
+
+-- Part 3: Record job metadata (completed or failed)
+
+if status == "completed" then
+  local completedKey = ns .. ":completed"
+  
+  -- CRITICAL: Always set final status first, even if job will be deleted
+  -- This ensures any concurrent reads see "completed", not "completing"
+  redis.call("HSET", jobKey, "status", "completed")
+
+  -- [PHASE 3 MODIFICATION START: Update parent flow if exists]
+  if parentId then
+    local parentKey = ns .. ":job:" .. parentId
+    -- 1. Store child result in flow:results hash (CRITICAL: was missing!)
+    local flowResultsKey = ns .. ":flow:results:" .. parentId
+    redis.call("HSET", flowResultsKey, completedJobId, resultOrError)
+    
+    -- 2. Decrement remaining counter
+    local remaining = redis.call("HINCRBY", parentKey, "flowRemaining", -1)
+    
+    -- 3. If all children done, move parent to waiting
+    if remaining <= 0 then
+      local parentStatus = redis.call("HGET", parentKey, "status")
+      if parentStatus == "waiting-children" then
+        redis.call("HSET", parentKey, "status", "waiting")
+        local parentGroupId = redis.call("HGET", parentKey, "groupId")
+        local parentScore = tonumber(redis.call("HGET", parentKey, "score"))
+        if not parentScore then
+          parentScore = tonumber(now)
+        end
+        
+        local pGZ = ns .. ":g:" .. parentGroupId
+        redis.call("ZADD", pGZ, parentScore, parentId)
+        redis.call("SADD", ns .. ":groups", parentGroupId)
+        
+        -- Check if should add to ready queue (if head)
+        local pHead = redis.call("ZRANGE", pGZ, 0, 0, "WITHSCORES")
+        if pHead and #pHead >= 2 then
+           local pHeadScore = tonumber(pHead[2])
+           redis.call("ZADD", ns .. ":ready", pHeadScore, parentGroupId)
+        end
+      end
+    end
+  end
+  -- [PHASE 3 MODIFICATION END]
+  
+  if keepCompleted > 0 then
+    -- Store full job metadata and add to completed set
+    redis.call("HSET", jobKey, 
+      "processedOn", processedOn,
+      "finishedOn", finishedOn,
+      "attempts", attempts,
+      "maxAttempts", maxAttempts,
+      "returnvalue", resultOrError
+    )
+    redis.call("ZADD", completedKey, timestamp, completedJobId)
+    
+    -- Trim old entries atomically
+    local zcount = redis.call("ZCARD", completedKey)
+    local toRemove = zcount - keepCompleted
+    if toRemove > 0 then
+      local oldIds = redis.call("ZRANGE", completedKey, 0, toRemove - 1)
+      if #oldIds > 0 then
+        redis.call("ZREMRANGEBYRANK", completedKey, 0, toRemove - 1)
+        for i = 1, #oldIds do
+          local oldId = oldIds[i]
+          redis.call("DEL", ns .. ":job:" .. oldId)
+          redis.call("DEL", ns .. ":unique:" .. oldId)
+          redis.call("DEL", ns .. ":flow:results:" .. oldId)
+        end
+      end
+    end
+  else
+    -- keepCompleted == 0: Delete immediately (status already set above)
+    redis.call("DEL", jobKey)
+    redis.call("DEL", ns .. ":unique:" .. completedJobId)
+    redis.call("DEL", ns .. ":flow:results:" .. completedJobId)
+  end
+  
+elseif status == "failed" then
+  local failedKey = ns .. ":failed"
+  local errorInfo = cjson.decode(resultOrError)
+  
+  -- CRITICAL: Always set final status first, even if job will be deleted
+  redis.call("HSET", jobKey, "status", "failed")
+  
+  if keepFailed > 0 then
+    redis.call("HSET", jobKey,
+      "failedReason", errorInfo.message or "Error",
+      "failedName", errorInfo.name or "Error",
+      "stacktrace", errorInfo.stack or "",
+      "processedOn", processedOn,
+      "finishedOn", finishedOn,
+      "attempts", attempts,
+      "maxAttempts", maxAttempts
+    )
+    redis.call("ZADD", failedKey, timestamp, completedJobId)
+  else
+    -- Delete job (status already set above)
+    redis.call("DEL", jobKey)
+    redis.call("DEL", ns .. ":unique:" .. completedJobId)
+    redis.call("DEL", ns .. ":flow:results:" .. completedJobId)
+  end
+end
+
+-- Part 3: Handle group active list and get next job (BullMQ-style)
+local groupActiveKey = ns .. ":g:" .. gid .. ":active"
+local activeJobId = redis.call("LINDEX", groupActiveKey, 0)
+
+-- Always clean up this job from active list, even if not at head
+-- This prevents stale active lists from race conditions
+if activeJobId == completedJobId then
+  -- Normal case: this job is at the head of active list
+  redis.call("LPOP", groupActiveKey)
+else
+  -- Race condition: job is not at head (maybe already removed, or wrong job)
+  -- Clean it up anyway to prevent stale entries
+  redis.call("LREM", groupActiveKey, 1, completedJobId)
+  
+  -- If active list had a different job or was empty, don't try to reserve next
+  -- Return nil to indicate no chaining
+  return nil
+end
+
+local gZ = ns .. ":g:" .. gid
+local zpop = redis.call("ZPOPMIN", gZ, 1)
+if not zpop or #zpop == 0 then
+  -- Clean up empty group
+  local jobCount = redis.call("ZCARD", gZ)
+  if jobCount == 0 then
+    redis.call("DEL", gZ)
+    redis.call("SREM", ns .. ":groups", gid)
+    redis.call("ZREM", ns .. ":ready", gid)
+  end
+  -- No next job
+  return nil
+end
+
+local nextJobId = zpop[1]
+local nextJobKey = ns .. ":job:" .. nextJobId
+local job = redis.call("HMGET", nextJobKey, "id","groupId","data","attempts","maxAttempts","seq","timestamp","orderMs","score")
+local id, groupId, payload, attempts, maxAttempts, seq, enq, orderMs, score = job[1], job[2], job[3], job[4], job[5], job[6], job[7], job[8], job[9]
+
+-- Validate job data exists (handle corrupted/missing job hash)
+if not id or id == false then
+  -- Job hash is missing/corrupted, clean up and return completion only
+  -- Re-add next job to ready queue if exists
+  local nextHead = redis.call("ZRANGE", gZ, 0, 0, "WITHSCORES")
+  if nextHead and #nextHead >= 2 then
+    local nextScore = tonumber(nextHead[2])
+    local readyKey = ns .. ":ready"
+    redis.call("ZADD", readyKey, nextScore, groupId)
+  end
+  
+  -- Return nil to indicate no next job was reserved
+  return nil
+end
+
+-- Push next job to active list (chaining)
+redis.call("LPUSH", groupActiveKey, id)
+
+local procKey = ns .. ":processing:" .. id
+local deadline = now + vt
+redis.call("HSET", procKey, "groupId", groupId, "deadlineAt", tostring(deadline))
+
+local processingKey = ns .. ":processing"
+redis.call("ZADD", processingKey, deadline, id)
+
+-- Mark next job as processing for accurate stalled detection
+redis.call("HSET", nextJobKey, "status", "processing")
+
+local nextHead = redis.call("ZRANGE", gZ, 0, 0, "WITHSCORES")
+if nextHead and #nextHead >= 2 then
+  local nextScore = tonumber(nextHead[2])
+  local readyKey = ns .. ":ready"
+  redis.call("ZADD", readyKey, nextScore, groupId)
+end
+
+return id .. "|||" .. groupId .. "|||" .. payload .. "|||" .. attempts .. "|||" .. maxAttempts .. "|||" .. seq .. "|||" .. enq .. "|||" .. orderMs .. "|||" .. score .. "|||" .. deadline

package/dist/lua/complete-with-metadata.lua

@@ -0,0 +1,190 @@
+-- Complete a job: unlock group AND record metadata atomically in one call
+-- argv: ns, jobId, groupId, status, timestamp, resultOrError, keepCompleted, keepFailed,
+--       processedOn, finishedOn, attempts, maxAttempts
+local ns = KEYS[1]
+local jobId = ARGV[1]
+local gid = ARGV[2]
+local status = ARGV[3]
+local timestamp = tonumber(ARGV[4])
+local resultOrError = ARGV[5]
+local keepCompleted = tonumber(ARGV[6])
+local keepFailed = tonumber(ARGV[7])
+local processedOn = ARGV[8]
+local finishedOn = ARGV[9]
+local attempts = ARGV[10]
+local maxAttempts = ARGV[11]
+
+local jobKey = ns .. ":job:" .. jobId
+
+-- [PHASE 3 MODIFICATION START: Get parentId before potentially deleting the job]
+local parentId = redis.call("HGET", jobKey, "parentId")
+-- [PHASE 3 MODIFICATION END]
+
+-- Part 1: Atomically verify and mark completion (prevent duplicate processing)
+local processingKey = ns .. ":processing"
+
+-- CRITICAL: Check both status AND processing set membership atomically
+-- This prevents race with stalled job recovery
+local jobStatus = redis.call("HGET", jobKey, "status")
+local stillInProcessing = redis.call("ZSCORE", processingKey, jobId)
+
+-- If job is not in "processing" state OR not in processing set, this is late/duplicate
+if jobStatus ~= "processing" or not stillInProcessing then
+  -- Job was already handled (recovered, failed, or completed by another worker)
+  -- Return 0 to indicate this completion was ignored
+  return 0
+end
+
+-- Atomically mark as completed and remove from processing
+-- This prevents stalled checker from racing with us
+redis.call("HSET", jobKey, "status", "completing") -- Temporary status to block stalled checker
+redis.call("DEL", ns .. ":processing:" .. jobId)
+redis.call("ZREM", processingKey, jobId)
+
+-- Always remove this job from active list to prevent stale entries
+local groupActiveKey = ns .. ":g:" .. gid .. ":active"
+local activeJobId = redis.call("LINDEX", groupActiveKey, 0)
+local wasActive = (activeJobId == jobId)
+
+if wasActive then
+  -- Normal case: remove from head of active list
+  redis.call("LPOP", groupActiveKey)
+else
+  -- Race condition: not at head, but still remove to prevent stale entries
+  redis.call("LREM", groupActiveKey, 1, jobId)
+end
+
+-- Check if there are more jobs in this group
+local gZ = ns .. ":g:" .. gid
+local jobCount = redis.call("ZCARD", gZ)
+if jobCount == 0 then
+  -- Remove empty group
+  redis.call("DEL", gZ)
+  redis.call("DEL", groupActiveKey)
+  redis.call("SREM", ns .. ":groups", gid)
+  redis.call("ZREM", ns .. ":ready", gid)
+  redis.call("DEL", ns .. ":buffer:" .. gid)
+  redis.call("ZREM", ns .. ":buffering", gid)
+else
+  -- Group has more jobs, re-add to ready if not buffering
+  local groupBufferKey = ns .. ":buffer:" .. gid
+  local isBuffering = redis.call("EXISTS", groupBufferKey)
+  
+  if isBuffering == 0 then
+    local nextHead = redis.call("ZRANGE", gZ, 0, 0, "WITHSCORES")
+    if nextHead and #nextHead >= 2 then
+      local nextScore = tonumber(nextHead[2])
+      local readyKey = ns .. ":ready"
+      redis.call("ZADD", readyKey, nextScore, gid)
+    end
+  end
+end
+
+-- [PHASE 3 MODIFICATION START: Update Flow Parent]
+if parentId then
+  local parentKey = ns .. ":job:" .. parentId
+  -- 1. Store child result in a separate hash to define parent's "childrenValues"
+  -- Key: flow:results:{parentId}, Field: {childId}
+  local flowResultsKey = ns .. ":flow:results:" .. parentId
+  redis.call("HSET", flowResultsKey, jobId, resultOrError)
+  
+  -- 2. Decrement remaining counter
+  local remaining = redis.call("HINCRBY", parentKey, "flowRemaining", -1)
+  
+  -- 3. If all children done, move parent to waiting
+  if remaining <= 0 then
+    local parentStatus = redis.call("HGET", parentKey, "status")
+    if parentStatus == "waiting-children" then
+      redis.call("HSET", parentKey, "status", "waiting")
+      
+      -- Add parent to its group and ready queue
+      local parentGroupId = redis.call("HGET", parentKey, "groupId")
+      local parentScore = tonumber(redis.call("HGET", parentKey, "score"))
+      if not parentScore then
+        parentScore = tonumber(redis.call("TIME")[1]) * 1000
+      end
+      
+      local pGZ = ns .. ":g:" .. parentGroupId
+      redis.call("ZADD", pGZ, parentScore, parentId)
+      redis.call("SADD", ns .. ":groups", parentGroupId)
+      
+      -- Check if should add to ready queue (if head)
+      local pHead = redis.call("ZRANGE", pGZ, 0, 0, "WITHSCORES")
+      if pHead and #pHead >= 2 then
+         local pHeadScore = tonumber(pHead[2])
+         redis.call("ZADD", ns .. ":ready", pHeadScore, parentGroupId)
+      end
+    end
+  end
+end
+-- [PHASE 3 MODIFICATION END]
+
+-- Part 2: Record job metadata (completed or failed)
+if status == "completed" then
+  local completedKey = ns .. ":completed"
+  
+  -- CRITICAL: Always set final status first, even if job will be deleted
+  -- This ensures any concurrent reads see "completed", not "completing"
+  redis.call("HSET", jobKey, "status", "completed")
+  
+  if keepCompleted > 0 then
+    -- Store full job metadata and add to completed set
+    redis.call("HSET", jobKey, 
+      "processedOn", processedOn,
+      "finishedOn", finishedOn,
+      "attempts", attempts,
+      "maxAttempts", maxAttempts,
+      "returnvalue", resultOrError
+    )
+    redis.call("ZADD", completedKey, timestamp, jobId)
+    
+    -- Trim old entries atomically
+    local zcount = redis.call("ZCARD", completedKey)
+    local toRemove = zcount - keepCompleted
+    if toRemove > 0 then
+      local oldIds = redis.call("ZRANGE", completedKey, 0, toRemove - 1)
+      if #oldIds > 0 then
+        redis.call("ZREMRANGEBYRANK", completedKey, 0, toRemove - 1)
+        for i = 1, #oldIds do
+          local oldId = oldIds[i]
+          redis.call("DEL", ns .. ":job:" .. oldId)
+          redis.call("DEL", ns .. ":unique:" .. oldId)
+          redis.call("DEL", ns .. ":flow:results:" .. oldId)
+        end
+      end
+    end
+  else
+    -- keepCompleted == 0: Delete immediately (status already set above)
+    redis.call("DEL", jobKey)
+    redis.call("DEL", ns .. ":unique:" .. jobId)
+    redis.call("DEL", ns .. ":flow:results:" .. jobId)
+  end
+  
+elseif status == "failed" then
+  local failedKey = ns .. ":failed"
+  local errorInfo = cjson.decode(resultOrError)
+  
+  -- CRITICAL: Always set final status first, even if job will be deleted
+  redis.call("HSET", jobKey, "status", "failed")
+  
+  if keepFailed > 0 then
+    redis.call("HSET", jobKey,
+      "failedReason", errorInfo.message or "Error",
+      "failedName", errorInfo.name or "Error",
+      "stacktrace", errorInfo.stack or "",
+      "processedOn", processedOn,
+      "finishedOn", finishedOn,
+      "attempts", attempts,
+      "maxAttempts", maxAttempts
+    )
+    redis.call("ZADD", failedKey, timestamp, jobId)
+  else
+    -- Delete job (status already set above)
+    redis.call("DEL", jobKey)
+    redis.call("DEL", ns .. ":unique:" .. jobId)
+    redis.call("DEL", ns .. ":flow:results:" .. jobId)
+  end
+end
+
+return 1
+