sidekiq-unique-jobs 7.0.0.beta28 → 7.0.3

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +98 -18
  3. data/README.md +169 -37
  4. data/lib/sidekiq-unique-jobs.rb +0 -2
  5. data/lib/sidekiq_unique_jobs.rb +4 -1
  6. data/lib/sidekiq_unique_jobs/batch_delete.rb +1 -1
  7. data/lib/sidekiq_unique_jobs/changelog.rb +11 -4
  8. data/lib/sidekiq_unique_jobs/constants.rb +1 -0
  9. data/lib/sidekiq_unique_jobs/digests.rb +1 -1
  10. data/lib/sidekiq_unique_jobs/json.rb +7 -1
  11. data/lib/sidekiq_unique_jobs/lock.rb +31 -1
  12. data/lib/sidekiq_unique_jobs/lock_config.rb +2 -0
  13. data/lib/sidekiq_unique_jobs/lua/lock.lua +10 -11
  14. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +8 -7
  15. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +9 -2
  16. data/lib/sidekiq_unique_jobs/middleware.rb +0 -57
  17. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +9 -8
  18. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +1 -1
  19. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +1 -1
  20. data/lib/sidekiq_unique_jobs/orphans/manager.rb +13 -3
  21. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +10 -0
  22. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +14 -5
  23. data/lib/sidekiq_unique_jobs/redis/entity.rb +9 -3
  24. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +27 -0
  25. data/lib/sidekiq_unique_jobs/server.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/timer_task.rb +78 -0
  27. data/lib/sidekiq_unique_jobs/version.rb +1 -1
  28. data/lib/sidekiq_unique_jobs/web.rb +26 -9
  29. data/lib/sidekiq_unique_jobs/web/helpers.rb +24 -3
  30. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  31. data/lib/sidekiq_unique_jobs/web/views/locks.erb +1 -1
  32. metadata +13 -8
  33. data/lib/sidekiq_unique_jobs/profiler.rb +0 -55
@@ -1,5 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq_unique_jobs"
4
-
5
- SidekiqUniqueJobs::Middleware.configure
@@ -3,9 +3,10 @@
3
3
  require "brpoplpush/redis_script"
4
4
  require "concurrent/future"
5
5
  require "concurrent/promises"
6
- require "concurrent/timer_task"
7
6
  require "concurrent/map"
8
7
  require "concurrent/mutable_struct"
8
+ require "concurrent/timer_task"
9
+ require "concurrent/executor/ruby_single_thread_executor"
9
10
  require "digest"
10
11
  require "digest/sha1"
11
12
  require "erb"
@@ -14,6 +15,7 @@ require "json"
14
15
  require "pathname"
15
16
  require "sidekiq"
16
17
 
18
+ require "sidekiq_unique_jobs/timer_task"
17
19
  require "sidekiq_unique_jobs/version"
18
20
  require "sidekiq_unique_jobs/version_check"
19
21
  require "sidekiq_unique_jobs/constants"
@@ -71,3 +73,4 @@ require "sidekiq_unique_jobs/config"
71
73
  require "sidekiq_unique_jobs/sidekiq_unique_jobs"
72
74
  require "sidekiq_unique_jobs/update_version"
73
75
  require "sidekiq_unique_jobs/upgrade_locks"
76
+ require "sidekiq_unique_jobs/server"
@@ -44,7 +44,7 @@ module SidekiqUniqueJobs
44
44
  #
45
45
  # @return [void]
46
46
  #
47
- def self.call(digests, conn)
47
+ def self.call(digests, conn = nil)
48
48
  new(digests, conn).call
49
49
  end
50
50
 
@@ -7,6 +7,13 @@ module SidekiqUniqueJobs
7
7
  # @author Mikael Henriksson <mikael@mhenrixon.com>
8
8
  #
9
9
  class Changelog < Redis::SortedSet
10
+ #
11
+ # @return [Integer] the number of matches to return by default
12
+ DEFAULT_COUNT = 1_000
13
+ #
14
+ # @return [String] the default pattern to use for matching
15
+ SCAN_PATTERN = "*"
16
+
10
17
  def initialize
11
18
  super(CHANGELOGS)
12
19
  end
@@ -34,10 +41,10 @@ module SidekiqUniqueJobs
34
41
  #
35
42
  # @return [Array<Hash>] an array of entries
36
43
  #
37
- def entries(pattern: "*", count: nil)
44
+ def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
38
45
  options = {}
39
- options[:match] = pattern if pattern
40
- options[:count] = count if count
46
+ options[:match] = pattern
47
+ options[:count] = count
41
48
 
42
49
  redis do |conn|
43
50
  conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
@@ -53,7 +60,7 @@ module SidekiqUniqueJobs
53
60
  #
54
61
  # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
55
62
  #
56
- def page(cursor, pattern: "*", page_size: 100)
63
+ def page(cursor: 0, pattern: "*", page_size: 100)
57
64
  redis do |conn|
58
65
  total_size, result = conn.multi do
59
66
  conn.zcard(key)
@@ -11,6 +11,7 @@ module SidekiqUniqueJobs
11
11
  AT ||= "at"
12
12
  CHANGELOGS ||= "uniquejobs:changelog"
13
13
  CLASS ||= "class"
14
+ CREATED_AT ||= "created_at"
14
15
  DEAD_VERSION ||= "uniquejobs:dead"
15
16
  DIGESTS ||= "uniquejobs:digests"
16
17
  ERRORS ||= "errors"
@@ -78,7 +78,7 @@ module SidekiqUniqueJobs
78
78
  def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
79
79
  options = {}
80
80
  options[:match] = pattern
81
- options[:count] = count if count
81
+ options[:count] = count
82
82
 
83
83
  result = redis { |conn| conn.zscan_each(key, **options).to_a }
84
84
 
@@ -15,11 +15,17 @@ module SidekiqUniqueJobs
15
15
  # @return [Object]
16
16
  #
17
17
  def load_json(string)
18
- return unless string && !string.empty?
18
+ return if string.nil? || string.empty?
19
19
 
20
20
  ::JSON.parse(string)
21
21
  end
22
22
 
23
+ def safe_load_json(string)
24
+ return string if string.is_a?(Hash)
25
+
26
+ load_json(string)
27
+ end
28
+
23
29
  #
24
30
  # Dumps an object into a JSON string
25
31
  #
@@ -53,7 +53,7 @@ module SidekiqUniqueJobs
53
53
  #
54
54
  # Locks a job_id
55
55
  #
56
- # @note intended only for testing purposez
56
+ # @note intended only for testing purposes
57
57
  #
58
58
  # @param [String] job_id a sidekiq JID
59
59
  # @param [Hash] lock_info information about the lock
@@ -73,6 +73,36 @@ module SidekiqUniqueJobs
73
73
  end
74
74
  end
75
75
 
76
+ #
77
+ # Create the :QUEUED key
78
+ #
79
+ # @note intended only for testing purposes
80
+ #
81
+ # @param [String] job_id a sidekiq JID
82
+ #
83
+ # @return [void]
84
+ #
85
+ def queue(job_id)
86
+ redis do |conn|
87
+ conn.lpush(key.queued, job_id)
88
+ end
89
+ end
90
+
91
+ #
92
+ # Create the :PRIMED key
93
+ #
94
+ # @note intended only for testing purposes
95
+ #
96
+ # @param [String] job_id a sidekiq JID
97
+ #
98
+ # @return [void]
99
+ #
100
+ def prime(job_id)
101
+ redis do |conn|
102
+ conn.lpush(key.primed, job_id)
103
+ end
104
+ end
105
+
76
106
  #
77
107
  # Unlock a specific job_id
78
108
  #
@@ -98,6 +98,8 @@ module SidekiqUniqueJobs
98
98
  # @return [String]
99
99
  #
100
100
  def errors_as_string
101
+ return if valid?
102
+
101
103
  @errors_as_string ||= begin
102
104
  error_msg = +"\t"
103
105
  error_msg << errors.map { |key, val| "#{key}: :#{val}" }.join("\n\t")
@@ -69,25 +69,24 @@ redis.call("LREM", queued, -1, job_id)
69
69
  log_debug("LREM", primed, 1, job_id)
70
70
  redis.call("LREM", primed, 1, job_id)
71
71
 
72
- -- The Sidekiq client should only set pttl for until_expired
73
- -- The Sidekiq server should set pttl for all other jobs
72
+ -- The Sidekiq client sets pttl
74
73
  if pttl and pttl > 0 then
75
74
  log_debug("PEXPIRE", digest, pttl)
76
75
  redis.call("PEXPIRE", digest, pttl)
77
76
 
78
- log_debug("PEXPIRE", queued, pttl)
79
- redis.call("PEXPIRE", queued, pttl)
80
-
81
- log_debug("PEXPIRE", primed, pttl)
82
- redis.call("PEXPIRE", primed, pttl)
83
-
84
77
  log_debug("PEXPIRE", locked, pttl)
85
78
  redis.call("PEXPIRE", locked, pttl)
86
-
87
- log_debug("PEXPIRE", info, pttl)
88
- redis.call("PEXPIRE", info, pttl)
89
79
  end
90
80
 
81
+ log_debug("PEXPIRE", queued, 1000)
82
+ redis.call("PEXPIRE", queued, 1000)
83
+
84
+ log_debug("PEXPIRE", primed, 1000)
85
+ redis.call("PEXPIRE", primed, 1000)
86
+
87
+ log_debug("PEXPIRE", info, 1000)
88
+ redis.call("PEXPIRE", info, 1000)
89
+
91
90
  log("Locked")
92
91
  log_debug("END lock digest:", digest, "job_id:", job_id)
93
92
  return job_id
@@ -7,15 +7,16 @@ local retry_set = KEYS[3]
7
7
  -------- END keys ---------
8
8
 
9
9
  -------- BEGIN argv ---------
10
- local reaper_count = tonumber(ARGV[1])
10
+ local reaper_count = tonumber(ARGV[1])
11
+ local threshold = tonumber(ARGV[2])
11
12
  -------- END argv ---------
12
13
 
13
14
  -------- BEGIN injected arguments --------
14
- local current_time = tonumber(ARGV[2])
15
- local debug_lua = ARGV[3] == "true"
16
- local max_history = tonumber(ARGV[4])
17
- local script_name = ARGV[5] .. ".lua"
18
- local redisversion = ARGV[6]
15
+ local current_time = tonumber(ARGV[3])
16
+ local debug_lua = ARGV[4] == "true"
17
+ local max_history = tonumber(ARGV[5])
18
+ local script_name = ARGV[6] .. ".lua"
19
+ local redisversion = ARGV[7]
19
20
  --------- END injected arguments ---------
20
21
 
21
22
 
@@ -65,7 +66,7 @@ repeat
65
66
  -- TODO: Add check for jobs checked out by process
66
67
  if found ~= true then
67
68
  log_debug("Searching for digest:", digest, "in process sets")
68
- found = find_digest_in_process_set(digest)
69
+ found = find_digest_in_process_set(digest, threshold)
69
70
  end
70
71
 
71
72
  if found ~= true then
@@ -1,4 +1,4 @@
1
- local function find_digest_in_process_set(digest)
1
+ local function find_digest_in_process_set(digest, threshold)
2
2
  local process_cursor = 0
3
3
  local job_cursor = 0
4
4
  local pattern = "*" .. digest .. "*"
@@ -26,11 +26,18 @@ local function find_digest_in_process_set(digest)
26
26
  log_debug("No entries in:", workers_key)
27
27
  else
28
28
  for i = 1, #jobs, 2 do
29
- if string.find(jobs[i +1], digest) then
29
+ local jobstr = jobs[i +1]
30
+ if string.find(jobstr, digest) then
30
31
  log_debug("Found digest", digest, "in:", workers_key)
31
32
  found = true
32
33
  break
33
34
  end
35
+
36
+ local job = cjson.decode(jobstr)
37
+ if job.payload.created_at > threshold then
38
+ found = true
39
+ break
40
+ end
34
41
  end
35
42
  end
36
43
 
@@ -11,63 +11,6 @@ module SidekiqUniqueJobs
11
11
  include SidekiqUniqueJobs::OptionsWithFallback
12
12
  include SidekiqUniqueJobs::JSON
13
13
 
14
- #
15
- # Configure both server and client
16
- #
17
- def self.configure
18
- configure_server
19
- configure_client
20
- end
21
-
22
- #
23
- # Configures the Sidekiq server
24
- #
25
- def self.configure_server # rubocop:disable Metrics/MethodLength
26
- Sidekiq.configure_server do |config|
27
- config.client_middleware do |chain|
28
- if defined?(Apartment::Sidekiq::Middleware::Client)
29
- chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
30
- else
31
- chain.add SidekiqUniqueJobs::Middleware::Client
32
- end
33
- end
34
-
35
- config.server_middleware do |chain|
36
- if defined?(Apartment::Sidekiq::Middleware::Server)
37
- chain.insert_after Apartment::Sidekiq::Middleware::Server, SidekiqUniqueJobs::Middleware::Server
38
- else
39
- chain.add SidekiqUniqueJobs::Middleware::Server
40
- end
41
- end
42
-
43
- config.on(:startup) do
44
- SidekiqUniqueJobs::UpdateVersion.call
45
- SidekiqUniqueJobs::UpgradeLocks.call
46
-
47
- SidekiqUniqueJobs::Orphans::Manager.start
48
- end
49
-
50
- config.on(:shutdown) do
51
- SidekiqUniqueJobs::Orphans::Manager.stop
52
- end
53
- end
54
- end
55
-
56
- #
57
- # Configures the Sidekiq client
58
- #
59
- def self.configure_client
60
- Sidekiq.configure_client do |config|
61
- config.client_middleware do |chain|
62
- if defined?(Apartment::Sidekiq::Middleware::Client)
63
- chain.insert_after Apartment::Sidekiq::Middleware::Client, SidekiqUniqueJobs::Middleware::Client
64
- else
65
- chain.add SidekiqUniqueJobs::Middleware::Client
66
- end
67
- end
68
- end
69
- end
70
-
71
14
  # The sidekiq job hash
72
15
  # @return [Hash] the Sidekiq job hash
73
16
  attr_reader :item
@@ -11,9 +11,9 @@ module SidekiqUniqueJobs
11
11
  # @return [String] rthe sidekiq queue this job belongs to
12
12
  attr_reader :queue
13
13
  #
14
- # @!attribute [r] unique_digest
14
+ # @!attribute [r] lock_digest
15
15
  # @return [String] the unique digest to use for locking
16
- attr_reader :unique_digest
16
+ attr_reader :lock_digest
17
17
 
18
18
  #
19
19
  # Initialize a new Replace strategy
@@ -22,8 +22,8 @@ module SidekiqUniqueJobs
22
22
  #
23
23
  def initialize(item, redis_pool = nil)
24
24
  super(item, redis_pool)
25
- @queue = item[QUEUE]
26
- @unique_digest = item[LOCK_DIGEST]
25
+ @queue = item[QUEUE]
26
+ @lock_digest = item[LOCK_DIGEST]
27
27
  end
28
28
 
29
29
  #
@@ -37,10 +37,11 @@ module SidekiqUniqueJobs
37
37
  def call(&block)
38
38
  return unless (deleted_job = delete_job_by_digest)
39
39
 
40
- log_info("Deleting job: #{deleted_job}")
40
+ log_info("Deleted job: #{deleted_job}")
41
41
  if (del_count = delete_lock)
42
- log_info("Deleted `#{del_count}` keys for #{unique_digest}")
42
+ log_info("Deleted `#{del_count}` keys for #{lock_digest}")
43
43
  end
44
+
44
45
  block&.call
45
46
  end
46
47
 
@@ -54,7 +55,7 @@ module SidekiqUniqueJobs
54
55
  def delete_job_by_digest
55
56
  call_script(:delete_job_by_digest,
56
57
  keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
57
- argv: [unique_digest])
58
+ argv: [lock_digest])
58
59
  end
59
60
 
60
61
  #
@@ -64,7 +65,7 @@ module SidekiqUniqueJobs
64
65
  # @return [Integer] the number of keys deleted
65
66
  #
66
67
  def delete_lock
67
- digests.delete_by_digest(unique_digest)
68
+ digests.delete_by_digest(lock_digest)
68
69
  end
69
70
 
70
71
  #
@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
21
21
  def call
22
22
  if sidekiq_worker_class?
23
23
  log_info("Rescheduling #{item[LOCK_DIGEST]}")
24
- worker_class&.perform_in(5, *item[ARGS])
24
+ worker_class.perform_in(5, *item[ARGS])
25
25
  else
26
26
  log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
27
27
  end
@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
21
21
  :reap_orphans,
22
22
  conn,
23
23
  keys: [DIGESTS, SCHEDULE, RETRY, PROCESSES],
24
- argv: [reaper_count],
24
+ argv: [reaper_count, (Time.now - reaper_timeout).to_f],
25
25
  )
26
26
  end
27
27
  end
@@ -20,12 +20,14 @@ module SidekiqUniqueJobs
20
20
  # Starts a separate thread that periodically reaps orphans
21
21
  #
22
22
  #
23
- # @return [Concurrent::TimerTask] the task that was started
23
+ # @return [SidekiqUniqueJobs::TimerTask] the task that was started
24
24
  #
25
- def start # rubocop:disable
25
+ def start(test_task = nil) # rubocop:disable
26
26
  return if disabled?
27
27
  return if registered?
28
28
 
29
+ self.task = test_task || default_task
30
+
29
31
  with_logging_context do
30
32
  register_reaper_process
31
33
  log_info("Starting Reaper")
@@ -59,7 +61,11 @@ module SidekiqUniqueJobs
59
61
  # @return [<type>] <description>
60
62
  #
61
63
  def task
62
- @task ||= Concurrent::TimerTask.new(timer_task_options) do
64
+ @task ||= default_task
65
+ end
66
+
67
+ def default_task
68
+ SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
63
69
  with_logging_context do
64
70
  redis do |conn|
65
71
  refresh_reaper_mutex
@@ -69,6 +75,10 @@ module SidekiqUniqueJobs
69
75
  end
70
76
  end
71
77
 
78
+ def task=(task)
79
+ @task = task
80
+ end
81
+
72
82
  #
73
83
  # Arguments passed on to the timer task
74
84
  #