sidekiq-unique-jobs 8.0.10 → 8.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +54 -6
  3. data/README.md +62 -49
  4. data/lib/sidekiq_unique_jobs/cli.rb +2 -2
  5. data/lib/sidekiq_unique_jobs/config.rb +65 -33
  6. data/lib/sidekiq_unique_jobs/digests.rb +1 -1
  7. data/lib/sidekiq_unique_jobs/exceptions.rb +2 -2
  8. data/lib/sidekiq_unique_jobs/job.rb +1 -1
  9. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +8 -4
  10. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +7 -4
  11. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +1 -1
  12. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +1 -1
  13. data/lib/sidekiq_unique_jobs/lock.rb +1 -1
  14. data/lib/sidekiq_unique_jobs/lock_args.rb +3 -3
  15. data/lib/sidekiq_unique_jobs/lock_digest.rb +6 -1
  16. data/lib/sidekiq_unique_jobs/lock_ttl.rb +34 -8
  17. data/lib/sidekiq_unique_jobs/locksmith.rb +25 -7
  18. data/lib/sidekiq_unique_jobs/logging.rb +2 -2
  19. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +8 -3
  20. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +11 -0
  21. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +5 -1
  22. data/lib/sidekiq_unique_jobs/lua/unlock.lua +20 -12
  23. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +10 -1
  24. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +3 -3
  25. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +1 -1
  26. data/lib/sidekiq_unique_jobs/on_conflict.rb +2 -2
  27. data/lib/sidekiq_unique_jobs/orphans/manager.rb +3 -3
  28. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +36 -9
  29. data/lib/sidekiq_unique_jobs/reflections.rb +3 -3
  30. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +3 -1
  31. data/lib/sidekiq_unique_jobs/script/client.rb +11 -3
  32. data/lib/sidekiq_unique_jobs/script/lua_error.rb +2 -0
  33. data/lib/sidekiq_unique_jobs/script/scripts.rb +42 -46
  34. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +2 -2
  35. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +4 -4
  36. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +1 -1
  37. data/lib/sidekiq_unique_jobs/testing.rb +2 -2
  38. data/lib/sidekiq_unique_jobs/version.rb +1 -1
  39. data/lib/sidekiq_unique_jobs/web/helpers.rb +29 -1
  40. data/lib/sidekiq_unique_jobs/web.rb +38 -30
  41. metadata +5 -8
@@ -27,20 +27,26 @@ module SidekiqUniqueJobs
27
27
  @scripts = Scripts.fetch(config.scripts_path)
28
28
  end
29
29
 
30
+ #
31
+ # Maximum number of retries for script execution errors
32
+ #
33
+ MAX_RETRIES = 3
34
+
30
35
  #
31
36
  # Execute a lua script with the provided script_name
32
37
  #
33
38
  # @note this method is recursive if we need to load a lua script
34
- # that wasn't previously loaded.
39
+ # that wasn't previously loaded. Limited to MAX_RETRIES to prevent stack overflow.
35
40
  #
36
41
  # @param [Symbol] script_name the name of the script to execute
37
42
  # @param [Redis] conn the redis connection to use for execution
38
43
  # @param [Array<String>] keys script keys
39
44
  # @param [Array<Object>] argv script arguments
45
+ # @param [Integer] retries number of retries remaining (internal use)
40
46
  #
41
47
  # @return value from script
42
48
  #
43
- def execute(script_name, conn, keys: [], argv: [])
49
+ def execute(script_name, conn, keys: [], argv: [], retries: MAX_RETRIES)
44
50
  result, elapsed = timed do
45
51
  scripts.execute(script_name, conn, keys: keys, argv: argv)
46
52
  end
@@ -48,8 +54,10 @@ module SidekiqUniqueJobs
48
54
  logger.debug("Executed #{script_name}.lua in #{elapsed}ms")
49
55
  result
50
56
  rescue ::RedisClient::CommandError => ex
57
+ raise if retries <= 0
58
+
51
59
  handle_error(script_name, conn, ex) do
52
- execute(script_name, conn, keys: keys, argv: argv)
60
+ execute(script_name, conn, keys: keys, argv: argv, retries: retries - 1)
53
61
  end
54
62
  end
55
63
 
@@ -88,9 +88,11 @@ module SidekiqUniqueJobs
88
88
  end
89
89
 
90
90
  # :nocov:
91
+ # rubocop:disable Naming/PredicateMethod
91
92
  def line_from_gem(line)
92
93
  line.split(":").first.include?(LIB_PATH)
93
94
  end
95
+ # rubocop:enable Naming/PredicateMethod
94
96
  end
95
97
  end
96
98
  end
@@ -11,42 +11,16 @@ module SidekiqUniqueJobs
11
11
  SCRIPT_PATHS = Concurrent::Map.new
12
12
 
13
13
  #
14
- # Fetch a scripts configuration for path
14
+ # Fetch or create a scripts configuration for path
15
15
  #
16
- # @param [Pathname] root_path the path to scripts
17
- #
18
- # @return [Scripts] a collection of scripts
19
- #
20
- def self.fetch(root_path)
21
- if (scripts = SCRIPT_PATHS.get(root_path))
22
- return scripts
23
- end
24
-
25
- create(root_path)
26
- end
27
-
28
- #
29
- # Create a new scripts collection based on path
16
+ # Uses Concurrent::Map#fetch_or_store for thread-safe lazy initialization
30
17
  #
31
18
  # @param [Pathname] root_path the path to scripts
32
19
  #
33
20
  # @return [Scripts] a collection of scripts
34
21
  #
35
- def self.create(root_path)
36
- scripts = new(root_path)
37
- store(scripts)
38
- end
39
-
40
- #
41
- # Store the scripts collection in memory
42
- #
43
- # @param [Scripts] scripts the path to scripts
44
- #
45
- # @return [Scripts] the scripts instance that was stored
46
- #
47
- def self.store(scripts)
48
- SCRIPT_PATHS.put(scripts.root_path, scripts)
49
- scripts
22
+ def self.fetch(root_path)
23
+ SCRIPT_PATHS.fetch_or_store(root_path) { new(root_path) }
50
24
  end
51
25
 
52
26
  #
@@ -66,35 +40,57 @@ module SidekiqUniqueJobs
66
40
  @root_path = path
67
41
  end
68
42
 
43
+ #
44
+ # Fetch or load a script by name
45
+ #
46
+ # Uses Concurrent::Map#fetch_or_store for thread-safe lazy loading
47
+ #
48
+ # @param [Symbol, String] name the script name
49
+ # @param [Redis] conn the redis connection
50
+ #
51
+ # @return [Script] the loaded script
52
+ #
69
53
  def fetch(name, conn)
70
- if (script = scripts.get(name.to_sym))
71
- return script
72
- end
73
-
74
- load(name, conn)
54
+ scripts.fetch_or_store(name.to_sym) { load(name, conn) }
75
55
  end
76
56
 
57
+ #
58
+ # Load a script from disk, store in Redis, and cache in memory
59
+ #
60
+ # @param [Symbol, String] name the script name
61
+ # @param [Redis] conn the redis connection
62
+ #
63
+ # @return [Script] the loaded script
64
+ #
77
65
  def load(name, conn)
78
66
  script = Script.load(name, root_path, conn)
79
67
  scripts.put(name.to_sym, script)
80
-
81
68
  script
82
69
  end
83
70
 
71
+ #
72
+ # Delete a script from the collection
73
+ #
74
+ # @param [Script, Symbol, String] script the script or script name to delete
75
+ #
76
+ # @return [Script, nil] the deleted script
77
+ #
84
78
  def delete(script)
85
- if script.is_a?(Script)
86
- scripts.delete(script.name)
87
- else
88
- scripts.delete(script.to_sym)
89
- end
79
+ key = script.is_a?(Script) ? script.name : script.to_sym
80
+ scripts.delete(key)
90
81
  end
91
82
 
83
+ #
84
+ # Kill a running Redis script
85
+ #
86
+ # @param [Redis] conn the redis connection
87
+ #
88
+ # @return [String] Redis response
89
+ #
92
90
  def kill(conn)
93
- if conn.respond_to?(:namespace)
94
- conn.redis.script(:kill)
95
- else
96
- conn.script(:kill)
97
- end
91
+ # Handle both namespaced and non-namespaced Redis connections
92
+ redis = conn.respond_to?(:namespace) ? conn.redis : conn
93
+ redis.script(:kill)
98
94
  end
99
95
 
100
96
  #
@@ -60,7 +60,7 @@ module Sidekiq
60
60
  #
61
61
  def delete(score, job_id)
62
62
  entry = find_job(job_id)
63
- SidekiqUniqueJobs::Unlockable.delete!(entry.item) if super(score, job_id)
63
+ SidekiqUniqueJobs::Unlockable.delete!(entry.item) if super
64
64
  entry
65
65
  end
66
66
  end
@@ -132,7 +132,7 @@ module Sidekiq
132
132
  # @param [String] value a sidekiq job hash
133
133
  #
134
134
  def delete_by_value(name, value)
135
- SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(value)) if super(name, value)
135
+ SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(value)) if super
136
136
  end
137
137
  end
138
138
 
@@ -4,7 +4,7 @@
4
4
  # Contains configuration and utility methods that belongs top level
5
5
  #
6
6
  # @author Mikael Henriksson <mikael@mhenrixon.com>
7
- module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
7
+ module SidekiqUniqueJobs
8
8
  include SidekiqUniqueJobs::Connection
9
9
  extend SidekiqUniqueJobs::JSON
10
10
 
@@ -17,7 +17,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
17
17
  # @return [SidekiqUniqueJobs::Config] the gem configuration
18
18
  #
19
19
  def config
20
- @config ||= reset! # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
20
+ @config ||= reset! # rubocop:disable ThreadSafety/ClassInstanceVariable
21
21
  end
22
22
 
23
23
  #
@@ -108,7 +108,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
108
108
  # @return [SidekiqUniqueJobs::Config] a default gem configuration
109
109
  #
110
110
  def reset!
111
- @config = SidekiqUniqueJobs::Config.default # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
111
+ @config = SidekiqUniqueJobs::Config.default # rubocop:disable ThreadSafety/ClassInstanceVariable
112
112
  end
113
113
 
114
114
  #
@@ -288,7 +288,7 @@ module SidekiqUniqueJobs # rubocop:disable Metrics/ModuleLength
288
288
  # @return [Reflections]
289
289
  #
290
290
  def reflections
291
- @reflections ||= Reflections.new # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
291
+ @reflections ||= Reflections.new # rubocop:disable ThreadSafety/ClassInstanceVariable
292
292
  end
293
293
 
294
294
  #
@@ -39,7 +39,7 @@ module SidekiqUniqueJobs
39
39
 
40
40
  # The hook to call after a successful unlock
41
41
  # @return [Proc]
42
- def after_unlock_hook # rubocop:disable Metrics/MethodLength
42
+ def after_unlock_hook
43
43
  lambda do
44
44
  if @original_job_class.respond_to?(:after_unlock)
45
45
  # instance method in sidekiq v6
@@ -21,7 +21,7 @@ module Sidekiq
21
21
  #
22
22
  # @param [Hash<Symbol, Object>] tmp_config the temporary config to use
23
23
  #
24
- def self.use_options(tmp_config = {}) # rubocop:disable Metrics/MethodLength
24
+ def self.use_options(tmp_config = {})
25
25
  if respond_to?(:default_job_options)
26
26
  default_job_options.clear
27
27
  self.default_job_options = tmp_config
@@ -87,7 +87,7 @@ module Sidekiq
87
87
  def sidekiq_options(options = {})
88
88
  SidekiqUniqueJobs.validate_worker!(options) if SidekiqUniqueJobs.config.raise_on_config_error
89
89
 
90
- super(options)
90
+ super
91
91
  end
92
92
 
93
93
  #
@@ -3,5 +3,5 @@
3
3
  module SidekiqUniqueJobs
4
4
  #
5
5
  # @return [String] the current SidekiqUniqueJobs version
6
- VERSION = "8.0.10"
6
+ VERSION = "8.0.12"
7
7
  end
@@ -116,7 +116,7 @@ module SidekiqUniqueJobs
116
116
  #
117
117
  # @return a redirect to the new subpath
118
118
  #
119
- def redirect_to(subpath)
119
+ def safe_redirect_to(subpath)
120
120
  if respond_to?(:to)
121
121
  # Sinatra-based web UI
122
122
  redirect to(subpath)
@@ -170,6 +170,34 @@ module SidekiqUniqueJobs
170
170
  Time.parse(time.to_s)
171
171
  end
172
172
  end
173
+
174
+ # Copied from sidekiq for compatibility with older versions
175
+
176
+ # stuff after ? or form input
177
+ # uses String keys, no Symbols!
178
+ def safe_url_params(key)
179
+ return url_params(key) if Sidekiq::MAJOR >= 8
180
+
181
+ if key.is_a?(Symbol)
182
+ warn do
183
+ "URL parameter `#{key}` should be accessed via String, not Symbol (at #{caller(3..3).first})"
184
+ end
185
+ end
186
+ request.params[key.to_s]
187
+ end
188
+
189
+ # variables embedded in path, `/metrics/:name`
190
+ # uses Symbol keys, no Strings!
191
+ def safe_route_params(key)
192
+ return route_params(key) if Sidekiq::MAJOR >= 8
193
+
194
+ if key.is_a?(String)
195
+ warn do
196
+ "Route parameter `#{key}` should be accessed via Symbol, not String (at #{caller(3..3).first})"
197
+ end
198
+ end
199
+ env["rack.route_params"][key.to_sym]
200
+ end
173
201
  end
174
202
  end
175
203
  end
@@ -7,17 +7,15 @@ module SidekiqUniqueJobs
7
7
  # Useful for deleting keys that for whatever reason wasn't deleted
8
8
  # @author Mikael Henriksson <mikael@mhenrixon.com>
9
9
  module Web
10
- def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
11
- app.helpers do
12
- include Web::Helpers
13
- end
10
+ def self.registered(app)
11
+ app.helpers Web::Helpers
14
12
 
15
13
  app.get "/changelogs" do
16
- @filter = h(params[:filter] || "*")
14
+ @filter = h(safe_url_params("filter") || "*")
17
15
  @filter = "*" if @filter == ""
18
- @count = h(params[:count] || 100).to_i
19
- @current_cursor = h(params[:cursor]).to_i
20
- @prev_cursor = h(params[:prev_cursor]).to_i
16
+ @count = h(safe_url_params("count") || 100).to_i
17
+ @current_cursor = h(safe_url_params("cursor")).to_i
18
+ @prev_cursor = h(safe_url_params("prev_cursor")).to_i
21
19
  @total_size, @next_cursor, @changelogs = changelog.page(
22
20
  cursor: @current_cursor,
23
21
  pattern: @filter,
@@ -29,15 +27,15 @@ module SidekiqUniqueJobs
29
27
 
30
28
  app.get "/changelogs/delete_all" do
31
29
  changelog.clear
32
- redirect_to :changelogs
30
+ safe_redirect_to :changelogs
33
31
  end
34
32
 
35
33
  app.get "/locks" do
36
- @filter = h(params[:filter]) || "*"
34
+ @filter = h(safe_url_params("filter") || "*")
37
35
  @filter = "*" if @filter == ""
38
- @count = h(params[:count] || 100).to_i
39
- @current_cursor = h(params[:cursor]).to_i
40
- @prev_cursor = h(params[:prev_cursor]).to_i
36
+ @count = h(safe_url_params("count") || 100).to_i
37
+ @current_cursor = h(safe_url_params("cursor")).to_i
38
+ @prev_cursor = h(safe_url_params("prev_cursor")).to_i
41
39
 
42
40
  @total_size, @next_cursor, @locks = digests.page(
43
41
  cursor: @current_cursor,
@@ -49,11 +47,11 @@ module SidekiqUniqueJobs
49
47
  end
50
48
 
51
49
  app.get "/expiring_locks" do
52
- @filter = h(params[:filter]) || "*"
50
+ @filter = h(safe_url_params("filter") || "*")
53
51
  @filter = "*" if @filter == ""
54
- @count = h(params[:count] || 100).to_i
55
- @current_cursor = h(params[:cursor]).to_i
56
- @prev_cursor = h(params[:prev_cursor]).to_i
52
+ @count = h(safe_url_params("count") || 100).to_i
53
+ @current_cursor = h(safe_url_params("cursor")).to_i
54
+ @prev_cursor = h(safe_url_params("prev_cursor")).to_i
57
55
 
58
56
  @total_size, @next_cursor, @locks = expiring_digests.page(
59
57
  cursor: @current_cursor,
@@ -67,29 +65,29 @@ module SidekiqUniqueJobs
67
65
  app.get "/locks/delete_all" do
68
66
  digests.delete_by_pattern("*", count: digests.count)
69
67
  expiring_digests.delete_by_pattern("*", count: digests.count)
70
- redirect_to :locks
68
+ safe_redirect_to :locks
71
69
  end
72
70
 
73
71
  app.get "/locks/:digest" do
74
- @digest = h(params[:digest])
72
+ @digest = h(safe_route_params(:digest))
75
73
  @lock = SidekiqUniqueJobs::Lock.new(@digest)
76
74
 
77
75
  erb(unique_template(:lock))
78
76
  end
79
77
 
80
78
  app.get "/locks/:digest/delete" do
81
- digests.delete_by_digest(h(params[:digest]))
82
- expiring_digests.delete_by_digest(h(params[:digest]))
83
- redirect_to :locks
79
+ digests.delete_by_digest(h(safe_route_params(:digest)))
80
+ expiring_digests.delete_by_digest(h(safe_route_params(:digest)))
81
+ safe_redirect_to :locks
84
82
  end
85
83
 
86
84
  app.get "/locks/:digest/jobs/:job_id/delete" do
87
- @digest = h(params[:digest])
88
- @job_id = h(params[:job_id])
85
+ @digest = h(safe_route_params(:digest))
86
+ @job_id = h(safe_route_params(:job_id))
89
87
  @lock = SidekiqUniqueJobs::Lock.new(@digest)
90
88
  @lock.unlock(@job_id)
91
89
 
92
- redirect_to "locks/#{@lock.key}"
90
+ safe_redirect_to "locks/#{@lock.key}"
93
91
  end
94
92
  end
95
93
  end
@@ -99,11 +97,21 @@ begin
99
97
  require "delegate" unless defined?(DelegateClass)
100
98
  require "sidekiq/web" unless defined?(Sidekiq::Web)
101
99
 
102
- Sidekiq::Web.register(SidekiqUniqueJobs::Web)
103
- Sidekiq::Web.tabs["Locks"] = "locks"
104
- Sidekiq::Web.tabs["Expiring Locks"] = "expiring_locks"
105
- Sidekiq::Web.tabs["Changelogs"] = "changelogs"
106
- Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
100
+ if Sidekiq::MAJOR >= 8
101
+ Sidekiq::Web.configure do |config|
102
+ config.register_extension(
103
+ SidekiqUniqueJobs::Web,
104
+ name: "unique_jobs",
105
+ tab: ["Locks", "Expiring Locks", "Changelogs"],
106
+ index: %w[locks/ expiring_locks/ changelogs/],
107
+ )
108
+ end
109
+ else
110
+ Sidekiq::Web.register(SidekiqUniqueJobs::Web)
111
+ Sidekiq::Web.tabs["Locks"] = "locks"
112
+ Sidekiq::Web.tabs["Expiring Locks"] = "expiring_locks"
113
+ Sidekiq::Web.tabs["Changelogs"] = "changelogs"
114
+ end
107
115
  rescue NameError, LoadError => ex
108
116
  SidekiqUniqueJobs.logger.error(ex)
109
117
  end
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-unique-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 8.0.10
4
+ version: 8.0.12
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mikael Henriksson
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2024-02-22 00:00:00.000000000 Z
10
+ date: 1980-01-02 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: concurrent-ruby
@@ -39,7 +38,7 @@ dependencies:
39
38
  version: 7.0.0
40
39
  - - "<"
41
40
  - !ruby/object:Gem::Version
42
- version: 8.0.0
41
+ version: 9.0.0
43
42
  type: :runtime
44
43
  prerelease: false
45
44
  version_requirements: !ruby/object:Gem::Requirement
@@ -49,7 +48,7 @@ dependencies:
49
48
  version: 7.0.0
50
49
  - - "<"
51
50
  - !ruby/object:Gem::Version
52
- version: 8.0.0
51
+ version: 9.0.0
53
52
  - !ruby/object:Gem::Dependency
54
53
  name: thor
55
54
  requirement: !ruby/object:Gem::Requirement
@@ -208,7 +207,6 @@ licenses:
208
207
  - MIT
209
208
  metadata:
210
209
  rubygems_mfa_required: 'true'
211
- post_install_message:
212
210
  rdoc_options: []
213
211
  require_paths:
214
212
  - lib
@@ -223,8 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
223
221
  - !ruby/object:Gem::Version
224
222
  version: '0'
225
223
  requirements: []
226
- rubygems_version: 3.5.6
227
- signing_key:
224
+ rubygems_version: 3.7.2
228
225
  specification_version: 4
229
226
  summary: Sidekiq middleware that prevents duplicates jobs
230
227
  test_files: []