sidekiq-unique-jobs 6.0.25 → 7.1.33

Sign up to get free protection for your applications and to get access to all the features.
Files changed (130) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1157 -41
  3. data/README.md +825 -291
  4. data/lib/sidekiq_unique_jobs/batch_delete.rb +124 -0
  5. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  6. data/lib/sidekiq_unique_jobs/cli.rb +57 -29
  7. data/lib/sidekiq_unique_jobs/config.rb +319 -0
  8. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  9. data/lib/sidekiq_unique_jobs/constants.rb +46 -25
  10. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  11. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  12. data/lib/sidekiq_unique_jobs/digests.rb +70 -102
  13. data/lib/sidekiq_unique_jobs/exceptions.rb +88 -12
  14. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  15. data/lib/sidekiq_unique_jobs/job.rb +46 -12
  16. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  17. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  18. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +111 -82
  19. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  20. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  21. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +40 -15
  22. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +30 -7
  23. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +26 -2
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +27 -15
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +26 -12
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -3
  28. data/lib/sidekiq_unique_jobs/lock.rb +342 -0
  29. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  30. data/lib/sidekiq_unique_jobs/lock_config.rb +126 -0
  31. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  32. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  33. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  34. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  35. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  36. data/lib/sidekiq_unique_jobs/locksmith.rb +305 -101
  37. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  38. data/lib/sidekiq_unique_jobs/logging.rb +202 -33
  39. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  40. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  41. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  42. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  43. data/lib/sidekiq_unique_jobs/lua/lock.lua +99 -0
  44. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  45. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  46. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  47. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +122 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  57. data/lib/sidekiq_unique_jobs/lua/unlock.lua +107 -0
  58. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  59. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  60. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  61. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  62. data/lib/sidekiq_unique_jobs/middleware.rb +29 -43
  63. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  64. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  65. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  66. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  67. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +63 -17
  68. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  69. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +16 -5
  70. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +25 -6
  71. data/lib/sidekiq_unique_jobs/on_conflict.rb +23 -10
  72. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +39 -36
  73. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  74. data/lib/sidekiq_unique_jobs/orphans/manager.rb +241 -0
  75. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  76. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  77. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  78. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  79. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +298 -0
  80. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  81. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  82. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  83. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  84. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  85. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  86. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  87. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  88. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  89. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  90. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/script/caller.rb +127 -0
  92. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  93. data/lib/sidekiq_unique_jobs/server.rb +61 -0
  94. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +114 -65
  95. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +252 -36
  96. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +47 -32
  97. data/lib/sidekiq_unique_jobs/testing.rb +102 -29
  98. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  99. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  100. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  101. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  102. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  103. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  104. data/lib/sidekiq_unique_jobs/version_check.rb +23 -4
  105. data/lib/sidekiq_unique_jobs/web/helpers.rb +138 -13
  106. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +4 -4
  107. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  108. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  109. data/lib/sidekiq_unique_jobs/web/views/locks.erb +54 -0
  110. data/lib/sidekiq_unique_jobs/web.rb +82 -32
  111. data/lib/sidekiq_unique_jobs.rb +54 -7
  112. data/lib/tasks/changelog.rake +16 -16
  113. metadata +134 -177
  114. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  115. data/lib/sidekiq_unique_jobs/scripts.rb +0 -118
  116. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  117. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  118. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  119. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -150
  120. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  121. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  122. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  123. data/redis/acquire_lock.lua +0 -21
  124. data/redis/convert_legacy_lock.lua +0 -13
  125. data/redis/delete.lua +0 -14
  126. data/redis/delete_by_digest.lua +0 -23
  127. data/redis/delete_job_by_digest.lua +0 -60
  128. data/redis/lock.lua +0 -62
  129. data/redis/release_stale_locks.lua +0 -90
  130. data/redis/unlock.lua +0 -35
@@ -0,0 +1,170 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Orphans
5
+ # Restarts orphan manager if it is considered dead
6
+ module ReaperResurrector
7
+ module_function
8
+
9
+ include SidekiqUniqueJobs::Connection
10
+ include SidekiqUniqueJobs::Logging
11
+
12
+ DRIFT_FACTOR = 0.1
13
+ REAPERS = [:ruby, :lua].freeze
14
+
15
+ #
16
+ # Starts reaper resurrector that watches orphans reaper
17
+ #
18
+ # @return [SidekiqUniqueJobs::TimerTask] the task that was started
19
+ #
20
+ def start
21
+ return if resurrector_disabled?
22
+ return if reaper_disabled?
23
+
24
+ with_logging_context do
25
+ run_task
26
+ end
27
+ end
28
+
29
+ #
30
+ # Runs reaper resurrector task
31
+ #
32
+ # @return [SidekiqUniqueJobs::TimerTask]
33
+ def run_task
34
+ log_info("Starting Reaper Resurrector")
35
+ task.execute
36
+ task
37
+ end
38
+
39
+ #
40
+ # The task that runs the resurrector
41
+ #
42
+ # @return [SidekiqUniqueJobs::TimerTask]
43
+ def task
44
+ SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
45
+ with_logging_context do
46
+ restart_if_dead
47
+ end
48
+ end
49
+ end
50
+
51
+ #
52
+ # Starts new instance of orphan reaper if reaper is considered dead (reaper mutex has not been refreshed lately)
53
+ #
54
+ def restart_if_dead
55
+ return if reaper_registered?
56
+
57
+ log_info("Reaper is considered dead. Starting new reaper instance")
58
+ orphans_manager.start
59
+ end
60
+
61
+ #
62
+ # Returns orphan manager
63
+ #
64
+ # @return [SidekiqUniqueJobs::Orphans::Manager]
65
+ def orphans_manager
66
+ SidekiqUniqueJobs::Orphans::Manager
67
+ end
68
+
69
+ #
70
+ # Checks if resurrector is disabled
71
+ #
72
+ # @see resurrector_enabled?
73
+ #
74
+ # @return [true, false]
75
+ def resurrector_disabled?
76
+ !resurrector_enabled?
77
+ end
78
+
79
+ #
80
+ # Checks if resurrector is enabled
81
+ #
82
+ # @return [true, false]
83
+ def resurrector_enabled?
84
+ SidekiqUniqueJobs.config.reaper_resurrector_enabled
85
+ end
86
+
87
+ #
88
+ # Checks if reaping is disabled
89
+ #
90
+ # @see reaper_enabled?
91
+ #
92
+ # @return [true, false]
93
+ #
94
+ def reaper_disabled?
95
+ !reaper_enabled?
96
+ end
97
+
98
+ #
99
+ # Checks if reaping is enabled
100
+ #
101
+ # @return [true, false]
102
+ #
103
+ def reaper_enabled?
104
+ REAPERS.include?(reaper)
105
+ end
106
+
107
+ #
108
+ # Checks if reaper is registered
109
+ #
110
+ # @return [true, false]
111
+ def reaper_registered?
112
+ redis do |conn|
113
+ conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
114
+ end
115
+ end
116
+
117
+ #
118
+ # @see SidekiqUniqueJobs::Config#reaper
119
+ #
120
+ def reaper
121
+ SidekiqUniqueJobs.config.reaper
122
+ end
123
+
124
+ #
125
+ # Arguments passed on to the timer task
126
+ #
127
+ #
128
+ # @return [Hash]
129
+ #
130
+ def timer_task_options
131
+ { run_now: false,
132
+ execution_interval: reaper_resurrector_interval }
133
+ end
134
+
135
+ #
136
+ # A context to use for all log entries
137
+ #
138
+ #
139
+ # @return [Hash] when logger responds to `:with_context`
140
+ # @return [String] when logger does not responds to `:with_context`
141
+ #
142
+ def logging_context
143
+ if logger_context_hash?
144
+ { "uniquejobs" => "reaper-resurrector" }
145
+ else
146
+ "uniquejobs=reaper-resurrector"
147
+ end
148
+ end
149
+
150
+ #
151
+ # @see SidekiqUniqueJobs::Config#reaper_resurrector_interval
152
+ #
153
+ def reaper_resurrector_interval
154
+ SidekiqUniqueJobs.config.reaper_resurrector_interval
155
+ end
156
+
157
+ def reaper_interval
158
+ SidekiqUniqueJobs.config.reaper_interval
159
+ end
160
+
161
+ def drift_reaper_interval
162
+ reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
163
+ end
164
+
165
+ def current_timestamp
166
+ Time.now.to_i
167
+ end
168
+ end
169
+ end
170
+ end
@@ -0,0 +1,298 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Orphans
5
+ #
6
+ # Class DeleteOrphans provides deletion of orphaned digests
7
+ #
8
+ # @note this is a much slower version of the lua script but does not crash redis
9
+ #
10
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
11
+ #
12
+ # rubocop:disable Metrics/ClassLength
13
+ class RubyReaper < Reaper
14
+ include SidekiqUniqueJobs::Timing
15
+
16
+ #
17
+ # @return [String] the suffix for :RUN locks
18
+ RUN_SUFFIX = ":RUN"
19
+ #
20
+ # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
21
+ MAX_QUEUE_LENGTH = 1000
22
+ #
23
+ # @!attribute [r] digests
24
+ # @return [SidekiqUniqueJobs::Digests] digest collection
25
+ attr_reader :digests
26
+ #
27
+ # @!attribute [r] scheduled
28
+ # @return [Redis::SortedSet] the Sidekiq ScheduleSet
29
+ attr_reader :scheduled
30
+ #
31
+ # @!attribute [r] retried
32
+ # @return [Redis::SortedSet] the Sidekiq RetrySet
33
+ attr_reader :retried
34
+
35
+ #
36
+ # @!attribute [r] start_time
37
+ # @return [Integer] The timestamp this execution started represented as Time (used for locks)
38
+ attr_reader :start_time
39
+
40
+ #
41
+ # @!attribute [r] start_time
42
+ # @return [Integer] The clock stamp this execution started represented as integer
43
+ # (used for redis compatibility as it is more accurate than time)
44
+ attr_reader :start_source
45
+
46
+ #
47
+ # @!attribute [r] timeout_ms
48
+ # @return [Integer] The allowed ms before timeout
49
+ attr_reader :timeout_ms
50
+
51
+ #
52
+ # Initialize a new instance of DeleteOrphans
53
+ #
54
+ # @param [Redis] conn a connection to redis
55
+ #
56
+ def initialize(conn)
57
+ super(conn)
58
+ @digests = SidekiqUniqueJobs::Digests.new
59
+ @scheduled = Redis::SortedSet.new(SCHEDULE)
60
+ @retried = Redis::SortedSet.new(RETRY)
61
+ @start_time = Time.now
62
+ @start_source = time_source.call
63
+ @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
64
+ end
65
+
66
+ #
67
+ # Delete orphaned digests
68
+ #
69
+ #
70
+ # @return [Integer] the number of reaped locks
71
+ #
72
+ def call
73
+ return if queues_very_full?
74
+
75
+ BatchDelete.call(expired_digests, conn)
76
+ BatchDelete.call(orphans, conn)
77
+ end
78
+
79
+ def expired_digests
80
+ max_score = (start_time - reaper_timeout).to_f
81
+
82
+ if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && VersionCheck.satisfied?(::Redis::VERSION, ">= 4.6.0")
83
+ conn.zrange(EXPIRING_DIGESTS, 0, max_score, byscore: true)
84
+ else
85
+ conn.zrangebyscore(EXPIRING_DIGESTS, 0, max_score)
86
+ end
87
+ end
88
+
89
+ #
90
+ # Find orphaned digests
91
+ #
92
+ #
93
+ # @return [Array<String>] an array of orphaned digests
94
+ #
95
+ def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
96
+ page = 0
97
+ per = reaper_count * 2
98
+ orphans = []
99
+ results = conn.zrange(digests.key, page * per, (page + 1) * per)
100
+
101
+ while results.size.positive?
102
+ results.each do |digest|
103
+ break if timeout?
104
+ next if belongs_to_job?(digest)
105
+
106
+ orphans << digest
107
+ break if orphans.size >= reaper_count
108
+ end
109
+
110
+ break if timeout?
111
+ break if orphans.size >= reaper_count
112
+
113
+ page += 1
114
+ results = conn.zrange(digests.key, page * per, (page + 1) * per)
115
+ end
116
+
117
+ orphans
118
+ end
119
+
120
+ def timeout?
121
+ elapsed_ms >= timeout_ms
122
+ end
123
+
124
+ def elapsed_ms
125
+ time_source.call - start_source
126
+ end
127
+
128
+ #
129
+ # Checks if the digest has a matching job.
130
+ # 1. It checks the scheduled set
131
+ # 2. It checks the retry set
132
+ # 3. It goes through all queues
133
+ #
134
+ #
135
+ # @param [String] digest the digest to search for
136
+ #
137
+ # @return [true] when either of the checks return true
138
+ # @return [false] when no job was found for this digest
139
+ #
140
+ def belongs_to_job?(digest)
141
+ scheduled?(digest) || retried?(digest) || enqueued?(digest) || active?(digest)
142
+ end
143
+
144
+ #
145
+ # Checks if the digest exists in the Sidekiq::ScheduledSet
146
+ #
147
+ # @param [String] digest the current digest
148
+ #
149
+ # @return [true] when digest exists in scheduled set
150
+ #
151
+ def scheduled?(digest)
152
+ in_sorted_set?(SCHEDULE, digest)
153
+ end
154
+
155
+ #
156
+ # Checks if the digest exists in the Sidekiq::RetrySet
157
+ #
158
+ # @param [String] digest the current digest
159
+ #
160
+ # @return [true] when digest exists in retry set
161
+ #
162
+ def retried?(digest)
163
+ in_sorted_set?(RETRY, digest)
164
+ end
165
+
166
+ #
167
+ # Checks if the digest exists in a Sidekiq::Queue
168
+ #
169
+ # @param [String] digest the current digest
170
+ #
171
+ # @return [true] when digest exists in any queue
172
+ #
173
+ def enqueued?(digest)
174
+ Sidekiq.redis do |conn|
175
+ queues(conn) do |queue|
176
+ entries(conn, queue) do |entry|
177
+ return true if entry.include?(digest)
178
+ end
179
+ end
180
+
181
+ false
182
+ end
183
+ end
184
+
185
+ def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
186
+ Sidekiq.redis do |conn|
187
+ procs = conn.sscan_each("processes").to_a
188
+ return false if procs.empty?
189
+
190
+ procs.sort.each do |key|
191
+ valid, workers = conn.pipelined do |pipeline|
192
+ # TODO: Remove the if statement in the future
193
+ if pipeline.respond_to?(:exists?)
194
+ pipeline.exists?(key)
195
+ else
196
+ pipeline.exists(key)
197
+ end
198
+ pipeline.hgetall("#{key}:work")
199
+ end
200
+
201
+ next unless valid
202
+ next unless workers.any?
203
+
204
+ workers.each_pair do |_tid, job|
205
+ next unless (item = safe_load_json(job))
206
+
207
+ payload = safe_load_json(item[PAYLOAD])
208
+
209
+ return true if match?(digest, payload[LOCK_DIGEST])
210
+ return true if considered_active?(payload[CREATED_AT])
211
+ end
212
+ end
213
+
214
+ false
215
+ end
216
+ end
217
+
218
+ def match?(key_one, key_two)
219
+ return false if key_one.nil? || key_two.nil?
220
+
221
+ key_one.delete_suffix(RUN_SUFFIX) == key_two.delete_suffix(RUN_SUFFIX)
222
+ end
223
+
224
+ def considered_active?(time_f)
225
+ (Time.now - reaper_timeout).to_f < time_f
226
+ end
227
+
228
+ #
229
+ # Loops through all the redis queues and yields them one by one
230
+ #
231
+ # @param [Redis] conn the connection to use for fetching queues
232
+ #
233
+ # @return [void]
234
+ #
235
+ # @yield queues one at a time
236
+ #
237
+ def queues(conn, &block)
238
+ conn.sscan_each("queues", &block)
239
+ end
240
+
241
+ def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
242
+ queue_key = "queue:#{queue}"
243
+ initial_size = conn.llen(queue_key)
244
+ deleted_size = 0
245
+ page = 0
246
+ page_size = 50
247
+
248
+ loop do
249
+ range_start = (page * page_size) - deleted_size
250
+
251
+ range_end = range_start + page_size - 1
252
+ entries = conn.lrange(queue_key, range_start, range_end)
253
+ page += 1
254
+
255
+ break if entries.empty?
256
+
257
+ entries.each(&block)
258
+
259
+ deleted_size = initial_size - conn.llen(queue_key)
260
+
261
+ # The queue is growing, not shrinking, just keep looping
262
+ deleted_size = 0 if deleted_size.negative?
263
+ end
264
+ end
265
+
266
+ # If sidekiq queues are very full, it becomes highly inefficient for the reaper
267
+ # because it must check every queued job to verify a digest is safe to delete
268
+ # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
269
+ # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
270
+ def queues_very_full?
271
+ total_queue_size = 0
272
+ Sidekiq.redis do |conn|
273
+ queues(conn) do |queue|
274
+ total_queue_size += conn.llen("queue:#{queue}")
275
+
276
+ return true if total_queue_size > MAX_QUEUE_LENGTH
277
+ end
278
+ end
279
+ false
280
+ end
281
+
282
+ #
283
+ # Checks a sorted set for the existance of this digest
284
+ #
285
+ #
286
+ # @param [String] key the key for the sorted set
287
+ # @param [String] digest the digest to scan for
288
+ #
289
+ # @return [true] when found
290
+ # @return [false] when missing
291
+ #
292
+ def in_sorted_set?(key, digest)
293
+ conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
294
+ end
295
+ end
296
+ # rubocop:enable Metrics/ClassLength
297
+ end
298
+ end
@@ -0,0 +1,112 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Entity functions as a base class for redis types
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Entity
11
+ # includes "SidekiqUniqueJobs::Logging"
12
+ # @!parse include SidekiqUniqueJobs::Logging
13
+ include SidekiqUniqueJobs::Logging
14
+
15
+ # includes "SidekiqUniqueJobs::Script::Caller"
16
+ # @!parse include SidekiqUniqueJobs::Script::Caller
17
+ include SidekiqUniqueJobs::Script::Caller
18
+
19
+ # includes "SidekiqUniqueJobs::JSON"
20
+ # @!parse include SidekiqUniqueJobs::JSON
21
+ include SidekiqUniqueJobs::JSON
22
+
23
+ # includes "SidekiqUniqueJobs::Timing"
24
+ # @!parse include SidekiqUniqueJobs::Timing
25
+ include SidekiqUniqueJobs::Timing
26
+
27
+ #
28
+ # @!attribute [r] key
29
+ # @return [String] the redis key for this entity
30
+ attr_reader :key
31
+
32
+ #
33
+ # Initialize a new Entity
34
+ #
35
+ # @param [String] key the redis key for this entity
36
+ #
37
+ def initialize(key)
38
+ @key = key
39
+ end
40
+
41
+ #
42
+ # Checks if the key for this entity exists in redis
43
+ #
44
+ #
45
+ # @return [true] when exists
46
+ # @return [false] when not exists
47
+ #
48
+ def exist?
49
+ redis do |conn|
50
+ # TODO: Remove the if statement in the future
51
+ value =
52
+ if conn.respond_to?(:exists?)
53
+ conn.exists?(key)
54
+ else
55
+ conn.exists(key)
56
+ end
57
+
58
+ return value if boolean?(value)
59
+
60
+ value.to_i.positive?
61
+ end
62
+ end
63
+
64
+ #
65
+ # The number of microseconds until the key expires
66
+ #
67
+ #
68
+ # @return [Integer] expiration in milliseconds
69
+ #
70
+ def pttl
71
+ redis { |conn| conn.pttl(key) }
72
+ end
73
+
74
+ #
75
+ # The number of seconds until the key expires
76
+ #
77
+ #
78
+ # @return [Integer] expiration in seconds
79
+ #
80
+ def ttl
81
+ redis { |conn| conn.ttl(key) }
82
+ end
83
+
84
+ #
85
+ # Check if the entity has expiration
86
+ #
87
+ #
88
+ # @return [true] when entity is set to exire
89
+ # @return [false] when entity isn't expiring
90
+ #
91
+ def expires?
92
+ pttl.positive? || ttl.positive?
93
+ end
94
+
95
+ #
96
+ # Returns the number of entries in this entity
97
+ #
98
+ #
99
+ # @return [Integer] 0
100
+ #
101
+ def count
102
+ 0
103
+ end
104
+
105
+ private
106
+
107
+ def boolean?(value)
108
+ [TrueClass, FalseClass].any? { |klazz| value.is_a?(klazz) }
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Hash provides convenient access to redis hashes
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Hash < Entity
11
+ #
12
+ # Return entries for this hash
13
+ #
14
+ # @param [true,false] with_values false return hash
15
+ #
16
+ # @return [Array<Object>] when given with_values: false
17
+ # @return [Hash<String, String>] when given with_values: true
18
+ #
19
+ def entries(with_values: false)
20
+ if with_values
21
+ redis { |conn| conn.hgetall(key) }
22
+ else
23
+ redis { |conn| conn.hkeys(key) }
24
+ end
25
+ end
26
+
27
+ #
28
+ # Removes the key from redis
29
+ #
30
+ def del(*fields)
31
+ redis { |conn| conn.hdel(key, *fields) }
32
+ end
33
+
34
+ #
35
+ # Get a members value
36
+ #
37
+ # @param [String] member the member who's value to get
38
+ #
39
+ # @return [Object] whatever is stored on this hash member
40
+ #
41
+ def [](member)
42
+ redis { |conn| conn.hget(key, member) }
43
+ end
44
+
45
+ #
46
+ # Returns the count for this hash
47
+ #
48
+ #
49
+ # @return [Integer] the length of this hash
50
+ #
51
+ def count
52
+ redis { |conn| conn.hlen(key) }
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class List provides convenient access to redis hashes
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class List < Entity
11
+ #
12
+ # Entries in this list
13
+ #
14
+ #
15
+ # @return [Array<Object>] the elements in this list
16
+ #
17
+ def entries
18
+ redis { |conn| conn.lrange(key, 0, -1) }
19
+ end
20
+
21
+ #
22
+ # The number of entries in this list
23
+ #
24
+ #
25
+ # @return [Integer] the total number of entries
26
+ #
27
+ def count
28
+ redis { |conn| conn.llen(key) }
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Set provides convenient access to redis sets
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Set < Entity
11
+ #
12
+ # Return entries for this set
13
+ #
14
+ #
15
+ # @return [Array<String>]
16
+ #
17
+ def entries
18
+ redis { |conn| conn.smembers(key) }
19
+ end
20
+
21
+ #
22
+ # Returns the count for this sorted set
23
+ #
24
+ #
25
+ # @return [Integer] the number of entries
26
+ #
27
+ def count
28
+ redis { |conn| conn.scard(key) }
29
+ end
30
+ end
31
+ end
32
+ end