sidekiq-unique-jobs 4.0.18 → 7.1.28

data/lib/sidekiq_unique_jobs/lua/queue.lua

@@ -0,0 +1,87 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id    = ARGV[1]      -- The job_id that was previously primed
+local pttl      = tonumber(ARGV[2])
+local lock_type = ARGV[3]
+local limit     = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+---------  END injected arguments ---------
+
+
+--------  BEGIN Variables --------
+local queued_count = redis.call("LLEN", queued)
+local locked_count = redis.call("HLEN", locked)
+local within_limit = limit > locked_count
+local limit_exceeded = not within_limit
+--------   END Variables  --------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN queue.lua --------
+log_debug("BEGIN queue with key:", digest, "for job:", job_id)
+
+if redis.call("HEXISTS", locked, job_id) == 1 then
+  log_debug("HEXISTS", locked, job_id, "== 1")
+  log("Duplicate")
+  return job_id
+end
+
+local prev_jid = redis.call("GET", digest)
+log_debug("job_id:", job_id, "prev_jid:", prev_jid)
+if not prev_jid or prev_jid == false then
+  log_debug("SET", digest, job_id)
+  redis.call("SET", digest, job_id)
+elseif prev_jid == job_id then
+  log_debug(digest, "already queued with job_id:", job_id)
+  log("Duplicate")
+  return job_id
+else
+  -- TODO: Consider constraining the total count of both locked and queued?
+  if within_limit and queued_count < limit then
+    log_debug("Within limit:", digest, "(",  locked_count, "of", limit, ")", "queued (", queued_count, "of", limit, ")")
+    log_debug("SET", digest, job_id, "(was", prev_jid, ")")
+    redis.call("SET", digest, job_id)
+  else
+    log_debug("Limit exceeded:", digest, "(",  locked_count, "of", limit, ")")
+    log("Limit exceeded", prev_jid)
+    return prev_jid
+  end
+end
+
+log_debug("LPUSH", queued, job_id)
+redis.call("LPUSH", queued, job_id)
+
+-- The Sidekiq client should only set pttl for until_expired
+-- The Sidekiq server should set pttl for all other jobs
+if pttl and pttl > 0 then
+  log_debug("PEXPIRE", digest, pttl)
+  redis.call("PEXPIRE", digest, pttl)
+  log_debug("PEXPIRE", queued, pttl)
+  redis.call("PEXPIRE", queued, pttl)
+end
+
+log("Queued")
+log_debug("END queue with key:", digest, "for job:", job_id)
+return job_id
+--------  END queue.lua --------

data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua

@@ -0,0 +1,122 @@
+redis.replicate_commands()
+
+-------- BEGIN keys ---------
+local digests_set          = KEYS[1]
+local expiring_digests_set = KEYS[2]
+local schedule_set         = KEYS[3]
+local retry_set            = KEYS[4]
+--------  END keys  ---------
+
+-------- BEGIN argv ---------
+local reaper_count = tonumber(ARGV[1])
+local threshold    = tonumber(ARGV[2])
+--------  END argv  ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[3])
+local debug_lua    = ARGV[4] == "true"
+local max_history  = tonumber(ARGV[5])
+local script_name  = ARGV[6] .. ".lua"
+local redisversion = ARGV[7]
+---------  END injected arguments ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_find_digest_in_queues.lua" %>
+<%= include_partial "shared/_find_digest_in_sorted_set.lua" %>
+<%= include_partial "shared/_find_digest_in_process_set.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN delete_orphaned.lua --------
+log_debug("BEGIN")
+local found     = false
+local per       = 50
+local total     = redis.call("ZCARD", digests_set)
+local index     = 0
+local del_count = 0
+local redis_ver = toversion(redisversion)
+local del_cmd   = "DEL"
+
+if tonumber(redis_ver["major"]) >= 4 then del_cmd = "UNLINK"; end
+
+repeat
+  log_debug("Interating through:", digests_set, "for orphaned locks")
+  local digests  = redis.call("ZREVRANGE", digests_set, index, index + per -1)
+
+  for _, digest in pairs(digests) do
+    log_debug("Searching for digest:", digest, "in", schedule_set)
+    found = find_digest_in_sorted_set(schedule_set, digest)
+
+    if found ~= true then
+      log_debug("Searching for digest:", digest, "in", retry_set)
+      found = find_digest_in_sorted_set(retry_set, digest)
+    end
+
+    if found ~= true then
+      log_debug("Searching for digest:", digest, "in all queues")
+      local queue = find_digest_in_queues(digest)
+
+      if queue then
+        log_debug("found digest:", digest, "in queue:", queue)
+        found = true
+      end
+    end
+
+    -- TODO: Add check for jobs checked out by process
+    if found ~= true then
+      log_debug("Searching for digest:", digest, "in process sets")
+      found = find_digest_in_process_set(digest, threshold)
+    end
+
+    if found ~= true then
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", digests_set, digest)
+      del_count = del_count + 1
+    end
+  end
+
+  index = index + per
+until index >= total or del_count >= reaper_count
+
+if del_count < reaper_count then
+  index = 0
+  total = redis.call("ZCOUNT", expiring_digests_set, 0, current_time)
+  repeat
+    local digests = redis.call("ZRANGEBYSCORE", expiring_digests_set, 0, current_time, "LIMIT", index, index + per -1)
+
+    for _, digest in pairs(digests) do
+      local queued     = digest .. ":QUEUED"
+      local primed     = digest .. ":PRIMED"
+      local locked     = digest .. ":LOCKED"
+      local info       = digest .. ":INFO"
+      local run_digest = digest .. ":RUN"
+      local run_queued = digest .. ":RUN:QUEUED"
+      local run_primed = digest .. ":RUN:PRIMED"
+      local run_locked = digest .. ":RUN:LOCKED"
+      local run_info   = digest .. ":RUN:INFO"
+
+      redis.call(del_cmd, digest, queued, primed, locked, info, run_digest, run_queued, run_primed, run_locked, run_info)
+
+      redis.call("ZREM", expiring_digests_set, digest)
+      del_count = del_count + 1
+    end
+
+    index = index + per
+  until index >= total or del_count >= reaper_count
+end
+
+log_debug("END")
+return del_count

data/lib/sidekiq_unique_jobs/lua/shared/_common.lua

@@ -0,0 +1,40 @@
+local function toversion(version)
+  local _, _, maj, min, pat = string.find(version, "(%d+)%.(%d+)%.(%d+)")
+
+  return {
+    ["version"] = version,
+    ["major"]   = tonumber(maj),
+    ["minor"]   = tonumber(min),
+    ["patch"]   = tonumber(pat)
+  }
+end
+
+local function toboolean(val)
+  val = tostring(val)
+  return val == "1" or val == "true"
+end
+
+local function log_debug( ... )
+  if debug_lua ~= true then return end
+
+  local result = ""
+  for _,v in ipairs(arg) do
+    result = result .. " " .. tostring(v)
+  end
+  redis.log(redis.LOG_DEBUG, script_name .. " -" ..  result)
+end
+
+local function log(message, prev_jid)
+  if not max_history or max_history == 0 then return end
+  local entry = cjson.encode({digest = digest, job_id = job_id, script = script_name, message = message, time = current_time, prev_jid = prev_jid })
+
+  log_debug("ZADD", changelog, current_time, entry);
+  redis.call("ZADD", changelog, current_time, entry);
+  local total_entries = redis.call("ZCARD", changelog)
+  local removed_entries = redis.call("ZREMRANGEBYRANK", changelog, 0, -1 * max_history)
+  if removed_entries > 0 then
+    log_debug("Removing", removed_entries , "entries from changelog (total entries", total_entries, "exceeds max_history:", max_history ..")");
+  end
+  log_debug("PUBLISH", changelog, entry);
+  redis.call("PUBLISH", changelog, entry);
+end

data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua

@@ -0,0 +1,8 @@
+local function current_time()
+  local time = redis.call("time")
+  local s = time[1]
+  local ms = time[2]
+  local number = tonumber((s .. "." .. ms))
+
+  return number
+end

data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua

@@ -0,0 +1,22 @@
+local function delete_from_queue(queue, digest)
+  local per    = 50
+  local total  = redis.call("LLEN", queue)
+  local index  = 0
+  local result = nil
+
+  while (index < total) do
+    local items = redis.call("LRANGE", queue, index, index + per -1)
+    if #items == 0 then
+      break
+    end
+    for _, item in pairs(items) do
+      if string.find(item, digest) then
+        redis.call("LREM", queue, 1, item)
+        result = item
+        break
+      end
+    end
+    index = index + per
+  end
+  return result
+end

data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua

@@ -0,0 +1,18 @@
+local function delete_from_sorted_set(name, digest)
+  local per   = 50
+  local total = redis.call("zcard", name)
+  local index = 0
+  local result
+  while (index < total) do
+    local items = redis.call("ZRANGE", name, index, index + per -1)
+    for _, item in pairs(items) do
+      if string.find(item, digest) then
+        redis.call("ZREM", name, item)
+        result = item
+        break
+      end
+    end
+    index = index + per
+  end
+  return result
+end

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua

@@ -0,0 +1,53 @@
+local function find_digest_in_process_set(digest, threshold)
+  local process_cursor = 0
+  local job_cursor     = 0
+  local pattern        = "*" .. digest .. "*"
+  local found          = false
+
+  log_debug("Searching in process list",
+            "for digest:", digest,
+            "cursor:", process_cursor)
+
+  repeat
+    local process_paginator   = redis.call("SSCAN", "processes", process_cursor, "MATCH", "*")
+    local next_process_cursor = process_paginator[1]
+    local processes           = process_paginator[2]
+    log_debug("Found number of processes:", #processes, "next cursor:", next_process_cursor)
+
+    for _, process in ipairs(processes) do
+      local workers_key = process .. ":workers"
+      log_debug("searching in process set:", process,
+                "for digest:", digest,
+                "cursor:", process_cursor)
+
+      local jobs = redis.call("HGETALL", workers_key)
+
+      if #jobs == 0 then
+        log_debug("No entries in:", workers_key)
+      else
+        for i = 1, #jobs, 2 do
+          local jobstr = jobs[i +1]
+          if string.find(string.gsub(jobstr, ':RUN', ''), string.gsub(digest, ':RUN', '')) then
+            log_debug("Found digest", digest, "in:", workers_key)
+            found = true
+            break
+          end
+
+          local job = cjson.decode(jobstr)
+          if job.payload.created_at > threshold then
+            found = true
+            break
+          end
+        end
+      end
+
+      if found == true then
+        break
+      end
+    end
+
+    process_cursor = next_process_cursor
+  until found == true or process_cursor == "0"
+
+  return found
+end

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua

@@ -0,0 +1,43 @@
+local function find_digest_in_queues(digest)
+  local cursor = "0"
+  local count  = 50
+  local result = nil
+  local found  = false
+
+  repeat
+    log_debug("searching all queues for a matching digest:", digest)
+    local pagination  = redis.call("SCAN", cursor, "MATCH", "queue:*", "COUNT", count)
+    local next_cursor = pagination[1]
+    local queues      = pagination[2]
+
+    for _, queue in ipairs(queues) do
+      local per    = 50
+      local total  = redis.call("LLEN", queue)
+      local index  = 0
+
+      log_debug("searching in:", queue,
+                "for digest:", digest,
+                "from:", index,
+                "to:", total,
+                "(per: " .. per .. ")",
+                "cursor:", cursor)
+
+      while (index < total) do
+        local items = redis.call("LRANGE", queue, index, index + per -1)
+        for _, item in pairs(items) do
+          if string.find(item, digest) then
+            log_debug("Found digest:", digest, "in queue:", queue)
+            result = cjson.decode(item).queue
+            found = true
+            break
+          end
+        end
+        index = index + per
+      end
+    end
+
+    cursor = next_cursor
+  until found == true or cursor == "0"
+
+  return result
+end

data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua

@@ -0,0 +1,24 @@
+local function find_digest_in_sorted_set(name, digest)
+  local cursor  = 0
+  local count   = 5
+  local pattern = "*" .. digest .. "*"
+  local found   = false
+
+  log_debug("searching in:", name,
+            "for digest:", digest,
+            "cursor:", cursor)
+  repeat
+    local pagination  = redis.call("ZSCAN", name, cursor, "MATCH", pattern, "COUNT", count)
+    local next_cursor = pagination[1]
+    local items       = pagination[2]
+
+    if #items > 0 then
+      log_debug("Found digest", digest, "in zset:", name)
+      found = true
+    end
+
+    cursor = next_cursor
+  until found == true or cursor == "0"
+
+  return found
+end

data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua

@@ -0,0 +1,13 @@
+local hgetall = function (key)
+  local bulk = redis.call("HGETALL", key)
+  local result = {}
+  local nextkey
+  for i, v in ipairs(bulk) do
+    if i % 2 == 1 then
+      nextkey = v
+    else
+      result[nextkey] = v
+    end
+  end
+  return result
+end

data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua

@@ -0,0 +1,3 @@
+local function upgrade_v6_to_v7()
+
+end

data/lib/sidekiq_unique_jobs/lua/unlock.lua

@@ -0,0 +1,107 @@
+-------- BEGIN keys ---------
+local digest    = KEYS[1]
+local queued    = KEYS[2]
+local primed    = KEYS[3]
+local locked    = KEYS[4]
+local info      = KEYS[5]
+local changelog = KEYS[6]
+local digests   = KEYS[7]
+-------- END keys ---------
+
+
+-------- BEGIN lock arguments ---------
+local job_id    = ARGV[1]
+local pttl      = tonumber(ARGV[2])
+local lock_type = ARGV[3]
+local limit     = tonumber(ARGV[4])
+-------- END lock arguments -----------
+
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+
+--------  BEGIN Variables --------
+local queued_count = redis.call("LLEN", queued)
+local primed_count = redis.call("LLEN", primed)
+local locked_count = redis.call("HLEN", locked)
+---------  END Variables ---------
+
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+---------  Begin unlock.lua ---------
+log_debug("BEGIN unlock digest:", digest, "(job_id: " .. job_id ..")")
+
+log_debug("HEXISTS", locked, job_id)
+if redis.call("HEXISTS", locked, job_id) == 0 then
+  -- TODO: Improve orphaned lock detection
+  if queued_count == 0 and primed_count == 0 and locked_count == 0 then
+    log_debug("Orphaned lock")
+  else
+    local result = ""
+    for i,v in ipairs(redis.call("HKEYS", locked)) do
+      result = result .. v .. ","
+    end
+    result = locked .. " (" .. result .. ")"
+    log("Yielding to: " .. result)
+    log_debug("Yielding to", result, locked, "by job", job_id)
+    return nil
+  end
+end
+
+-- Just in case something went wrong
+log_debug("LREM", queued, -1, job_id)
+redis.call("LREM", queued, -1, job_id)
+
+log_debug("LREM", primed, -1, job_id)
+redis.call("LREM", primed, -1, job_id)
+
+local redis_version = toversion(redisversion)
+local del_cmd       = "DEL"
+
+if tonumber(redis_version["major"]) >= 4 then del_cmd =  "UNLINK"; end
+
+if lock_type ~= "until_expired" then
+  log_debug(del_cmd, digest, info)
+  redis.call(del_cmd, digest, info)
+
+  log_debug("HDEL", locked, job_id)
+  redis.call("HDEL", locked, job_id)
+end
+
+local locked_count = redis.call("HLEN", locked)
+
+if locked_count and locked_count < 1 then
+  log_debug(del_cmd, locked)
+  redis.call(del_cmd, locked)
+end
+
+if redis.call("LLEN", primed) == 0 then
+  log_debug(del_cmd, primed)
+  redis.call(del_cmd, primed)
+end
+
+if limit and limit <= 1 and locked_count and locked_count <= 1 then
+  log_debug("ZREM", digests, digest)
+  redis.call("ZREM", digests, digest)
+end
+
+log_debug("LPUSH", queued, "1")
+redis.call("LPUSH", queued, "1")
+
+log_debug("PEXPIRE", queued, 5000)
+redis.call("PEXPIRE", queued, 5000)
+
+log("Unlocked")
+log_debug("END unlock digest:", digest, "(job_id: " .. job_id ..")")
+return job_id
+---------  END unlock.lua ---------

data/lib/sidekiq_unique_jobs/lua/update_version.lua

@@ -0,0 +1,40 @@
+-------- BEGIN keys ---------
+local live_key = KEYS[1]
+local dead_key = KEYS[2]
+-------- END keys ---------
+
+-------- BEGIN lock arguments ---------
+local version  = ARGV[1]
+-------- END lock arguments -----------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[2])
+local debug_lua    = ARGV[3] == "true"
+local max_history  = tonumber(ARGV[4])
+local script_name  = tostring(ARGV[5]) .. ".lua"
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+----------  END local functions ----------
+
+
+--------  BEGIN set_version.lua --------
+log_debug("BEGIN setting version:", version)
+
+local updated     = false
+local old_version = redis.call("GETSET", live_key, version)
+
+if not old_version then
+  log_debug("No previous version found")
+  updated = true
+elseif old_version ~= version then
+  log_debug("Old version:", old_version, "differs from:", version)
+  redis.call("SET", dead_key, old_version)
+  updated = true
+end
+
+return updated
+--------  END delete.lua --------
+
+

data/lib/sidekiq_unique_jobs/lua/upgrade.lua

@@ -0,0 +1,68 @@
+redis.replicate_commands()
+-------- BEGIN keys ---------
+local live_version = KEYS[1]
+local dead_version = KEYS[2]
+-------- END keys ---------
+
+--------  BEGIN injected arguments --------
+local current_time = tonumber(ARGV[5])
+local debug_lua    = ARGV[6] == "true"
+local max_history  = tonumber(ARGV[7])
+local script_name  = tostring(ARGV[8]) .. ".lua"
+local redisversion = ARGV[9]
+---------  END injected arguments ---------
+
+--------  BEGIN local functions --------
+<%= include_partial "shared/_common.lua" %>
+<%= include_partial "shared/_upgrades.lua" %>
+----------  END local functions ----------
+
+
+local new_version   = redis.call("GET", live_version)
+local old_version   = redis.call("GET", dead_version)
+local redis_version = toversion(redisversion)
+local upgraded      = 0
+local del_cmd       = "DEL"
+
+if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
+--------  BEGIN delete.lua --------
+
+log_debug("BEGIN upgrading from: ", old_version, "to:", new_version)
+
+-- 1. Loop through all uniquejobs:jajshdahdas:GRABBED
+local cursor = "0"
+local per   = 50
+repeat
+  local pagination   = redis.call("SCAN", cursor, "MATCH", "*:GRABBED", "COUNT", per)
+  local next_cursor  = pagination[1]
+  local grabbed_keys = pagination[2]
+
+  for _, grabbed in ipairs(grabbed_keys) do
+    local locked_key = grabbed.gsub(":GRABBED", ":LOCKED")
+    local locks      = redis.call("HGETALL", grabbed)
+
+    if #locks == 0 then
+      log_debug("No entries in:", grabbed)
+    else
+      log_debug("HMSET", locked_key, unpack(locks))
+      redis.call("HMSET", locked_key, unpack(locks))
+    end
+
+    log_debug("DEL", grabbed)
+    redis.call("DEL", grabbed)
+
+    upgraded = upgraded + 1
+  end
+
+  cursor = next_cursor
+  if cursor == "0" then
+    log_debug("Looped through all grabbed keys, stopping iteration")
+  end
+until cursor == "0"
+
+
+
+log_debug("END upgrading from: ", old_version, "to:", new_version)
+
+return
+--------  END delete.lua --------