sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +155 -20
  3. data/README.md +349 -112
  4. data/lib/sidekiq-unique-jobs.rb +2 -0
  5. data/lib/sidekiq_unique_jobs.rb +43 -6
  6. data/lib/sidekiq_unique_jobs/batch_delete.rb +121 -0
  7. data/lib/sidekiq_unique_jobs/changelog.rb +71 -0
  8. data/lib/sidekiq_unique_jobs/cli.rb +20 -29
  9. data/lib/sidekiq_unique_jobs/config.rb +193 -0
  10. data/lib/sidekiq_unique_jobs/connection.rb +5 -4
  11. data/lib/sidekiq_unique_jobs/constants.rb +36 -24
  12. data/lib/sidekiq_unique_jobs/core_ext.rb +38 -0
  13. data/lib/sidekiq_unique_jobs/digests.rb +78 -93
  14. data/lib/sidekiq_unique_jobs/exceptions.rb +152 -8
  15. data/lib/sidekiq_unique_jobs/job.rb +3 -3
  16. data/lib/sidekiq_unique_jobs/json.rb +34 -0
  17. data/lib/sidekiq_unique_jobs/key.rb +93 -0
  18. data/lib/sidekiq_unique_jobs/lock.rb +295 -0
  19. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +49 -43
  20. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  21. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  22. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +8 -17
  23. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +5 -5
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +1 -23
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +65 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +12 -8
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +1 -1
  28. data/lib/sidekiq_unique_jobs/lock_config.rb +95 -0
  29. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  30. data/lib/sidekiq_unique_jobs/locksmith.rb +255 -99
  31. data/lib/sidekiq_unique_jobs/logging.rb +148 -22
  32. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  33. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  34. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +46 -0
  35. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  36. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  37. data/lib/sidekiq_unique_jobs/lua/find_digest_in_sorted_set.lua +24 -0
  38. data/lib/sidekiq_unique_jobs/lua/lock.lua +91 -0
  39. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  40. data/lib/sidekiq_unique_jobs/lua/queue.lua +83 -0
  41. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +86 -0
  42. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  43. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  44. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +19 -0
  45. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  46. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +46 -0
  47. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/find_digest_in_sorted_set.lua +24 -0
  51. data/lib/sidekiq_unique_jobs/lua/unlock.lua +99 -0
  52. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  53. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  54. data/lib/sidekiq_unique_jobs/middleware.rb +62 -31
  55. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  56. data/lib/sidekiq_unique_jobs/middleware/server.rb +27 -0
  57. data/lib/sidekiq_unique_jobs/normalizer.rb +3 -3
  58. data/lib/sidekiq_unique_jobs/on_conflict.rb +22 -9
  59. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +8 -4
  60. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +59 -13
  61. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +42 -13
  62. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +4 -4
  63. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +24 -5
  64. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +47 -23
  65. data/lib/sidekiq_unique_jobs/orphans/manager.rb +100 -0
  66. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  67. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +201 -0
  68. data/lib/sidekiq_unique_jobs/profiler.rb +51 -0
  69. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  70. data/lib/sidekiq_unique_jobs/redis/entity.rb +94 -0
  71. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  72. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  73. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  74. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +59 -0
  75. data/lib/sidekiq_unique_jobs/redis/string.rb +49 -0
  76. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +19 -0
  77. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +43 -0
  78. data/lib/sidekiq_unique_jobs/{scripts.rb → script.rb} +43 -29
  79. data/lib/sidekiq_unique_jobs/script/caller.rb +125 -0
  80. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  81. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +92 -65
  82. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +166 -28
  83. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +10 -11
  84. data/lib/sidekiq_unique_jobs/testing.rb +47 -15
  85. data/lib/sidekiq_unique_jobs/time_calculator.rb +103 -0
  86. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  87. data/lib/sidekiq_unique_jobs/unique_args.rb +19 -21
  88. data/lib/sidekiq_unique_jobs/unlockable.rb +11 -2
  89. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  90. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +151 -0
  91. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  92. data/lib/sidekiq_unique_jobs/version_check.rb +1 -1
  93. data/lib/sidekiq_unique_jobs/web.rb +25 -19
  94. data/lib/sidekiq_unique_jobs/web/helpers.rb +98 -6
  95. data/lib/sidekiq_unique_jobs/web/views/lock.erb +108 -0
  96. data/lib/sidekiq_unique_jobs/web/views/locks.erb +52 -0
  97. data/lib/tasks/changelog.rake +4 -3
  98. metadata +70 -35
  99. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  100. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  101. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  102. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  103. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  104. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  105. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  106. data/redis/acquire_lock.lua +0 -21
  107. data/redis/convert_legacy_lock.lua +0 -13
  108. data/redis/delete.lua +0 -14
  109. data/redis/delete_by_digest.lua +0 -23
  110. data/redis/delete_job_by_digest.lua +0 -60
  111. data/redis/lock.lua +0 -62
  112. data/redis/release_stale_locks.lua +0 -90
  113. data/redis/unlock.lua +0 -35
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Interface to dealing with .lua files
5
+ #
6
+ # @author Mikael Henriksson <mikael@zoolutions.se>
7
+ module Script
8
+ #
9
+ # Class Template provides LUA script partial template rendering
10
+ #
11
+ # @author Mikael Henriksson <mikael@zoolutions.se>
12
+ #
13
+ class Template
14
+ def initialize(script_path)
15
+ @script_path = script_path
16
+ end
17
+
18
+ #
19
+ # Renders a Lua script and includes any partials in that file
20
+ # all `<%= include_partial '' %>` replaced with the actual contents of the partial
21
+ #
22
+ # @param [Pathname] pathname the path to the
23
+ #
24
+ # @return [String] the rendered Luascript
25
+ #
26
+ def render(pathname)
27
+ @partial_templates ||= {}
28
+ ERB.new(File.read(pathname)).result(binding)
29
+ end
30
+
31
+ # helper method to include a lua partial within another lua script
32
+ #
33
+ def include_partial(relative_path)
34
+ return if @partial_templates.key?(relative_path)
35
+
36
+ @partial_templates[relative_path] = nil
37
+ render(Pathname.new("#{@script_path}/#{relative_path}"))
38
+ end
39
+ end
40
+ end
41
+ end
@@ -2,113 +2,140 @@
2
2
 
3
3
  require "sidekiq/api"
4
4
 
5
+ #
6
+ # @api private
7
+ #
5
8
  module Sidekiq
9
+ # See Sidekiq::Api
6
10
  class SortedEntry
11
+ #
12
+ # Provides extensions for unlocking jobs that are removed and deleted
13
+ #
14
+ # @author Mikael Henriksson <mikael@zoolutions.se>
15
+ #
7
16
  module UniqueExtension
8
- def self.included(base)
9
- base.class_eval do
10
- alias_method :delete_orig, :delete
11
- alias_method :delete, :delete_ext
12
- alias_method :remove_job_orig, :remove_job
13
- alias_method :remove_job, :remove_job_ext
14
- end
15
- end
16
-
17
- def delete_ext
18
- SidekiqUniqueJobs::Unlockable.unlock(item) if delete_orig
17
+ #
18
+ # Wraps the original method to ensure locks for the job are deleted
19
+ #
20
+ #
21
+ # @return [Hash] the deleted sidekiq job hash
22
+ #
23
+ def delete
24
+ SidekiqUniqueJobs::Unlockable.delete!(item) if super
25
+ item
19
26
  end
20
27
 
21
28
  private
22
29
 
23
- def remove_job_ext
24
- remove_job_orig do |message|
25
- SidekiqUniqueJobs::Unlockable.unlock(Sidekiq.load_json(message))
30
+ #
31
+ # Wraps the original method to ensure locks for the job are deleted
32
+ #
33
+ #
34
+ # @yieldparam [Hash] message the sidekiq job hash
35
+ def remove_job
36
+ super do |message|
37
+ SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(message))
26
38
  yield message
27
39
  end
28
40
  end
29
41
  end
30
42
 
31
- include UniqueExtension
43
+ prepend UniqueExtension
32
44
  end
33
45
 
46
+ # See Sidekiq::Api
34
47
  class ScheduledSet
48
+ #
49
+ # Provides extensions for unlocking jobs that are removed and deleted
50
+ #
51
+ # @author Mikael Henriksson <mikael@zoolutions.se>
52
+ #
35
53
  module UniqueExtension
36
- def self.included(base)
37
- base.class_eval do
38
- alias_method :delete_orig, :delete
39
- alias_method :delete, :delete_ext
40
- end
41
- end
42
-
43
- def delete_ext
44
- SidekiqUniqueJobs::Unlockable.unlock(item) if delete_orig
54
+ #
55
+ # Wraps the original method to ensure locks for the job are deleted
56
+ #
57
+ #
58
+ # @param [Integer, Float] score the score in the scheduled set
59
+ # @param [String] job_id the Sidekiq JID
60
+ #
61
+ def delete(score, job_id)
62
+ entry = find_job(job_id)
63
+ SidekiqUniqueJobs::Unlockable.delete!(entry.item) if super(score, job_id)
64
+ entry
45
65
  end
46
66
  end
47
- include UniqueExtension
67
+
68
+ prepend UniqueExtension
48
69
  end
49
70
 
71
+ # See Sidekiq::Api
50
72
  class Job
73
+ #
74
+ # Provides extensions for unlocking jobs that are removed and deleted
75
+ #
76
+ # @author Mikael Henriksson <mikael@zoolutions.se>
77
+ #
51
78
  module UniqueExtension
52
- def self.included(base)
53
- base.class_eval do
54
- alias_method :delete_orig, :delete
55
- alias_method :delete, :delete_ext
56
- end
57
- end
58
-
59
- def delete_ext
60
- delete_orig
61
- SidekiqUniqueJobs::Unlockable.unlock(item)
79
+ #
80
+ # Wraps the original method to ensure locks for the job are deleted
81
+ #
82
+ def delete
83
+ SidekiqUniqueJobs::Unlockable.delete!(item)
84
+ super
62
85
  end
63
86
  end
64
87
 
65
- include UniqueExtension
88
+ prepend UniqueExtension
66
89
  end
67
90
 
91
+ # See Sidekiq::Api
68
92
  class Queue
93
+ #
94
+ # Provides extensions for unlocking jobs that are removed and deleted
95
+ #
96
+ # @author Mikael Henriksson <mikael@zoolutions.se>
97
+ #
69
98
  module UniqueExtension
70
- def self.included(base)
71
- base.class_eval do
72
- alias_method :clear_orig, :clear
73
- alias_method :clear, :clear_ext
74
- end
75
- end
76
-
77
- def clear_ext
99
+ #
100
+ # Wraps the original method to ensure locks for the job are deleted
101
+ #
102
+ def clear
78
103
  each(&:delete)
79
- clear_orig
104
+ super
80
105
  end
81
106
  end
82
107
 
83
- include UniqueExtension
108
+ prepend UniqueExtension
84
109
  end
85
110
 
111
+ # See Sidekiq::Api
86
112
  class JobSet
113
+ #
114
+ # Provides extensions for unlocking jobs that are removed and deleted
115
+ #
116
+ # @author Mikael Henriksson <mikael@zoolutions.se>
117
+ #
87
118
  module UniqueExtension
88
- def self.included(base)
89
- base.class_eval do
90
- if base.method_defined?(:clear)
91
- alias_method :clear_orig, :clear
92
- alias_method :clear, :clear_ext
93
- end
94
-
95
- if base.method_defined?(:delete_by_value)
96
- alias_method :delete_by_value_orig, :delete_by_value
97
- alias_method :delete_by_value, :delete_by_value_ext
98
- end
99
- end
100
- end
101
-
102
- def clear_ext
119
+ #
120
+ # Wraps the original method to ensure locks for the job are deleted
121
+ #
122
+ def clear
103
123
  each(&:delete)
104
- clear_orig
124
+ super
105
125
  end
106
126
 
107
- def delete_by_value_ext(name, value)
108
- SidekiqUniqueJobs::Unlockable.unlock(Sidekiq.load_json(value)) if delete_by_value_orig(name, value)
127
+ #
128
+ # Wraps the original method to ensure locks for the job are deleted
129
+ #
130
+ #
131
+ # @param [String] name the name of the key
132
+ # @param [String] value a sidekiq job hash
133
+ #
134
+ def delete_by_value(name, value)
135
+ SidekiqUniqueJobs::Unlockable.delete!(Sidekiq.load_json(value)) if super(name, value)
109
136
  end
110
137
  end
111
138
 
112
- include UniqueExtension
139
+ prepend UniqueExtension
113
140
  end
114
141
  end
@@ -6,53 +6,79 @@
6
6
  # @author Mikael Henriksson <mikael@zoolutions.se>
7
7
  module SidekiqUniqueJobs
8
8
  include SidekiqUniqueJobs::Connection
9
+ extend SidekiqUniqueJobs::JSON
9
10
 
10
11
  module_function
11
12
 
12
- Config = Concurrent::MutableStruct.new(
13
- :default_lock_timeout,
14
- :enabled,
15
- :unique_prefix,
16
- :logger,
17
- )
18
-
13
+ #
19
14
  # The current configuration (See: {.configure} on how to configure)
15
+ #
16
+ #
17
+ # @return [SidekiqUniqueJobs::Config] the gem configuration
18
+ #
20
19
  def config
21
20
  # Arguments here need to match the definition of the new class (see above)
22
- @config ||= Config.new(
23
- 0,
24
- true,
25
- "uniquejobs",
26
- Sidekiq.logger,
27
- )
21
+ @config ||= SidekiqUniqueJobs::Config.default
22
+ end
23
+
24
+ #
25
+ # The current strategies
26
+ #
27
+ #
28
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Strategy>] the configured locks
29
+ #
30
+ def strategies
31
+ config.strategies
32
+ end
33
+
34
+ #
35
+ # The current locks
36
+ #
37
+ #
38
+ # @return [Hash<Symbol, SidekiqUniqueJobs::BaseLock>] the configured locks
39
+ #
40
+ def locks
41
+ config.locks
28
42
  end
29
43
 
44
+ #
30
45
  # The current logger
46
+ #
47
+ #
31
48
  # @return [Logger] the configured logger
49
+ #
32
50
  def logger
33
51
  config.logger
34
52
  end
35
53
 
36
- # :reek:ManualDispatch
37
- def with_context(context)
38
- if logger.respond_to?(:with_context)
39
- logger.with_context(context) { yield }
40
- elsif defined?(Sidekiq::Context)
41
- Sidekiq::Context.with(context) { yield }
42
- elsif defined?(Sidekiq::Logging)
43
- Sidekiq::Logging.with_context(context) { yield }
44
- else
45
- yield
46
- end
54
+ #
55
+ # The current gem version
56
+ #
57
+ #
58
+ # @return [String] the current gem version
59
+ #
60
+ def version
61
+ VERSION
47
62
  end
48
63
 
64
+ #
49
65
  # Set a new logger
50
- # @param [Logger] other a new logger
66
+ #
67
+ # @param [Logger] other another logger
68
+ #
69
+ # @return [Logger] the new logger
70
+ #
51
71
  def logger=(other)
52
72
  config.logger = other
53
73
  end
54
74
 
55
- # Change global configuration while yielding
75
+ #
76
+ # Temporarily use another configuration and reset to the old config after yielding
77
+ #
78
+ # @param [Hash] tmp_config the temporary configuration to use
79
+ #
80
+ # @return [void]
81
+ #
56
82
  # @yield control to the caller
57
83
  def use_config(tmp_config)
58
84
  raise ::ArgumentError, "#{name}.#{__method__} needs a block" unless block_given?
@@ -63,6 +89,66 @@ module SidekiqUniqueJobs
63
89
  configure(old_config)
64
90
  end
65
91
 
92
+ #
93
+ # Enable SidekiqUniuqeJobs either temporarily in a block or for good
94
+ #
95
+ #
96
+ # @return [true] when not given a block
97
+ # @return [true, false] the previous value of enable when given a block
98
+ #
99
+ # @yieldreturn [void] temporarily enable sidekiq unique jobs while executing a block of code
100
+ def enable!(&block)
101
+ toggle(true, &block)
102
+ end
103
+
104
+ #
105
+ # Disable SidekiqUniuqeJobs either temporarily in a block or for good
106
+ #
107
+ #
108
+ # @return [false] when not given a block
109
+ # @return [true, false] the previous value of enable when given a block
110
+ #
111
+ # @yieldreturn [void] temporarily disable sidekiq unique jobs while executing a block of code
112
+ def disable!(&block)
113
+ toggle(false, &block)
114
+ end
115
+
116
+ #
117
+ # Checks if the gem has been disabled
118
+ #
119
+ # @return [true] when config.enabled is true
120
+ # @return [false] when config.enabled is false
121
+ #
122
+ def enabled?
123
+ config.enabled
124
+ end
125
+
126
+ #
127
+ # Checks if the gem has been disabled
128
+ #
129
+ # @return [true] when config.enabled is false
130
+ # @return [false] when config.enabled is true
131
+ #
132
+ def disabled?
133
+ !enabled?
134
+ end
135
+
136
+ #
137
+ # Toggles enabled on or off
138
+ #
139
+ # @api private
140
+ # :nodoc:
141
+ def toggle(enabled)
142
+ if block_given?
143
+ enabled_was = config.enabled
144
+ config.enabled = enabled
145
+ yield
146
+ config.enabled = enabled_was
147
+ else
148
+ config.enabled = enabled
149
+ end
150
+ end
151
+
66
152
  # Configure the gem
67
153
  #
68
154
  # This is usually called once at startup of an application
@@ -82,7 +168,59 @@ module SidekiqUniqueJobs
82
168
  end
83
169
  end
84
170
 
85
- def redis_version
86
- @redis_version ||= redis { |conn| conn.info("server")["redis_version"] }
171
+ #
172
+ # Returns the current redis version
173
+ #
174
+ #
175
+ # @return [String] a string like `5.0.2`
176
+ #
177
+ def fetch_redis_version
178
+ redis { |conn| conn.info("server")["redis_version"] }
179
+ end
180
+
181
+ #
182
+ # Current time as float
183
+ #
184
+ #
185
+ # @return [Float]
186
+ #
187
+ def now_f
188
+ now.to_f
189
+ end
190
+
191
+ #
192
+ # Current time
193
+ #
194
+ #
195
+ # @return [Time]
196
+ #
197
+ def now
198
+ Time.now
199
+ end
200
+
201
+ #
202
+ # Checks that the worker is valid with the given options
203
+ #
204
+ # @param [Hash] options the `sidekiq_options` to validate
205
+ #
206
+ # @return [Boolean]
207
+ #
208
+ def validate_worker(options)
209
+ raise NotUniqueWorker, options: options unless (lock_type = options[LOCK])
210
+
211
+ lock_class = locks[lock_type]
212
+ lock_class.validate_options(options)
213
+ end
214
+
215
+ #
216
+ # Checks that the worker is valid with the given options
217
+ #
218
+ # @param [Hash] options the `sidekiq_options` to validate
219
+ #
220
+ # @raise [InvalidWorker] when {#validate_worker} returns false or nil
221
+ #
222
+ def validate_worker!(options)
223
+ lock_config = validate_worker(options)
224
+ raise InvalidWorker, lock_config unless lock_config.errors.empty?
87
225
  end
88
226
  end