sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,313 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Orphans
5
+ #
6
+ # Class DeleteOrphans provides deletion of orphaned digests
7
+ #
8
+ # @note this is a much slower version of the lua script but does not crash redis
9
+ #
10
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
11
+ #
12
+ # rubocop:disable Metrics/ClassLength
13
+ class RubyReaper < Reaper
14
+ include SidekiqUniqueJobs::Timing
15
+
16
+ #
17
+ # @return [Integer] a best guess of Sidekiq::Launcher::BEAT_PAUSE
18
+ SIDEKIQ_BEAT_PAUSE = 10
19
+ #
20
+ # @return [String] the suffix for :RUN locks
21
+ RUN_SUFFIX = ":RUN"
22
+ #
23
+ # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
24
+ MAX_QUEUE_LENGTH = 1000
25
+ #
26
+ # @!attribute [r] digests
27
+ # @return [SidekiqUniqueJobs::Digests] digest collection
28
+ attr_reader :digests
29
+ #
30
+ # @!attribute [r] scheduled
31
+ # @return [Redis::SortedSet] the Sidekiq ScheduleSet
32
+ attr_reader :scheduled
33
+ #
34
+ # @!attribute [r] retried
35
+ # @return [Redis::SortedSet] the Sidekiq RetrySet
36
+ attr_reader :retried
37
+
38
+ #
39
+ # @!attribute [r] start_time
40
+ # @return [Integer] The timestamp this execution started represented as Time (used for locks)
41
+ attr_reader :start_time
42
+
43
+ #
44
+ # @!attribute [r] start_time
45
+ # @return [Integer] The clock stamp this execution started represented as integer
46
+ # (used for redis compatibility as it is more accurate than time)
47
+ attr_reader :start_source
48
+
49
+ #
50
+ # @!attribute [r] timeout_ms
51
+ # @return [Integer] The allowed ms before timeout
52
+ attr_reader :timeout_ms
53
+
54
+ #
55
+ # Initialize a new instance of DeleteOrphans
56
+ #
57
+ # @param [Redis] conn a connection to redis
58
+ #
59
+ def initialize(conn)
60
+ super(conn)
61
+ @digests = SidekiqUniqueJobs::Digests.new
62
+ @scheduled = Redis::SortedSet.new(SCHEDULE)
63
+ @retried = Redis::SortedSet.new(RETRY)
64
+ @start_time = Time.now
65
+ @start_source = time_source.call
66
+ @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
67
+ end
68
+
69
+ #
70
+ # Delete orphaned digests
71
+ #
72
+ #
73
+ # @return [Integer] the number of reaped locks
74
+ #
75
+ def call
76
+ return if queues_very_full?
77
+
78
+ BatchDelete.call(expired_digests, conn)
79
+ BatchDelete.call(orphans, conn)
80
+
81
+ # orphans.each_slice(500) do |chunk|
82
+ # conn.pipelined do |pipeline|
83
+ # chunk.each do |digest|
84
+ # next if belongs_to_job?(digest)
85
+
86
+ # pipeline.zadd(ORPHANED_DIGESTS, now_f, digest)
87
+ # end
88
+ # end
89
+ # end
90
+ end
91
+
92
+ def expired_digests
93
+ conn.zrange(EXPIRING_DIGESTS, 0, max_score, "byscore")
94
+ end
95
+
96
+ def orphaned_digests
97
+ conn.zrange(ORPHANED_DIGESTS, 0, max_score, "byscore")
98
+ end
99
+
100
+ def max_score
101
+ (start_time - reaper_timeout - SIDEKIQ_BEAT_PAUSE).to_f
102
+ end
103
+
104
+ #
105
+ # Find orphaned digests
106
+ #
107
+ #
108
+ # @return [Array<String>] an array of orphaned digests
109
+ #
110
+ def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
111
+ orphans = []
112
+ page = 0
113
+ per = reaper_count * 2
114
+ results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
115
+
116
+ while results.size.positive?
117
+ results.each do |digest|
118
+ break if timeout?
119
+ next if belongs_to_job?(digest)
120
+
121
+ orphans << digest
122
+ break if orphans.size >= reaper_count
123
+ end
124
+
125
+ break if timeout?
126
+ break if orphans.size >= reaper_count
127
+
128
+ page += 1
129
+ results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
130
+ end
131
+
132
+ orphans
133
+ end
134
+
135
+ def timeout?
136
+ elapsed_ms >= timeout_ms
137
+ end
138
+
139
+ def elapsed_ms
140
+ time_source.call - start_source
141
+ end
142
+
143
+ #
144
+ # Checks if the digest has a matching job.
145
+ # 1. It checks the scheduled set
146
+ # 2. It checks the retry set
147
+ # 3. It goes through all queues
148
+ #
149
+ #
150
+ # @param [String] digest the digest to search for
151
+ #
152
+ # @return [true] when either of the checks return true
153
+ # @return [false] when no job was found for this digest
154
+ #
155
+ def belongs_to_job?(digest)
156
+ scheduled?(digest) || retried?(digest) || enqueued?(digest) || active?(digest)
157
+ end
158
+
159
+ #
160
+ # Checks if the digest exists in the Sidekiq::ScheduledSet
161
+ #
162
+ # @param [String] digest the current digest
163
+ #
164
+ # @return [true] when digest exists in scheduled set
165
+ #
166
+ def scheduled?(digest)
167
+ in_sorted_set?(SCHEDULE, digest)
168
+ end
169
+
170
+ #
171
+ # Checks if the digest exists in the Sidekiq::RetrySet
172
+ #
173
+ # @param [String] digest the current digest
174
+ #
175
+ # @return [true] when digest exists in retry set
176
+ #
177
+ def retried?(digest)
178
+ in_sorted_set?(RETRY, digest)
179
+ end
180
+
181
+ #
182
+ # Checks if the digest exists in a Sidekiq::Queue
183
+ #
184
+ # @param [String] digest the current digest
185
+ #
186
+ # @return [true] when digest exists in any queue
187
+ #
188
+ def enqueued?(digest)
189
+ Sidekiq.redis do |conn|
190
+ queues(conn) do |queue|
191
+ entries(conn, queue) do |entry|
192
+ return true if entry.include?(digest)
193
+ end
194
+ end
195
+
196
+ false
197
+ end
198
+ end
199
+
200
+ def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
201
+ Sidekiq.redis do |conn|
202
+ procs = conn.sscan("processes").to_a
203
+ return false if procs.empty?
204
+
205
+ procs.sort.each do |key|
206
+ valid, workers = conn.pipelined do |pipeline|
207
+ # TODO: Remove the if statement in the future
208
+ if pipeline.respond_to?(:exists?)
209
+ pipeline.exists?(key)
210
+ else
211
+ pipeline.exists(key)
212
+ end
213
+ pipeline.hgetall("#{key}:work")
214
+ end
215
+
216
+ next unless valid
217
+ next unless workers.any?
218
+
219
+ workers.each_pair do |_tid, job|
220
+ next unless (item = safe_load_json(job))
221
+
222
+ payload = safe_load_json(item[PAYLOAD])
223
+
224
+ return true if match?(digest, payload[LOCK_DIGEST])
225
+ return true if considered_active?(payload[CREATED_AT])
226
+ end
227
+ end
228
+
229
+ false
230
+ end
231
+ end
232
+
233
+ def match?(key_one, key_two)
234
+ return false if key_one.nil? || key_two.nil?
235
+
236
+ key_one.delete_suffix(RUN_SUFFIX) == key_two.delete_suffix(RUN_SUFFIX)
237
+ end
238
+
239
+ def considered_active?(time_f)
240
+ max_score < time_f
241
+ end
242
+
243
+ #
244
+ # Loops through all the redis queues and yields them one by one
245
+ #
246
+ # @param [Redis] conn the connection to use for fetching queues
247
+ #
248
+ # @return [void]
249
+ #
250
+ # @yield queues one at a time
251
+ #
252
+ def queues(conn, &block)
253
+ conn.sscan("queues").each(&block)
254
+ end
255
+
256
+ def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
257
+ queue_key = "queue:#{queue}"
258
+ initial_size = conn.llen(queue_key)
259
+ deleted_size = 0
260
+ page = 0
261
+ page_size = 50
262
+
263
+ loop do
264
+ range_start = (page * page_size) - deleted_size
265
+
266
+ range_end = range_start + page_size - 1
267
+ entries = conn.lrange(queue_key, range_start, range_end)
268
+ page += 1
269
+
270
+ break if entries.empty?
271
+
272
+ entries.each(&block)
273
+
274
+ deleted_size = initial_size - conn.llen(queue_key)
275
+
276
+ # The queue is growing, not shrinking, just keep looping
277
+ deleted_size = 0 if deleted_size.negative?
278
+ end
279
+ end
280
+
281
+ # If sidekiq queues are very full, it becomes highly inefficient for the reaper
282
+ # because it must check every queued job to verify a digest is safe to delete
283
+ # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
284
+ # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
285
+ def queues_very_full?
286
+ total_queue_size = 0
287
+ Sidekiq.redis do |conn|
288
+ queues(conn) do |queue|
289
+ total_queue_size += conn.llen("queue:#{queue}")
290
+
291
+ return true if total_queue_size > MAX_QUEUE_LENGTH
292
+ end
293
+ end
294
+ false
295
+ end
296
+
297
+ #
298
+ # Checks a sorted set for the existance of this digest
299
+ #
300
+ #
301
+ # @param [String] key the key for the sorted set
302
+ # @param [String] digest the digest to scan for
303
+ #
304
+ # @return [true] when found
305
+ # @return [false] when missing
306
+ #
307
+ def in_sorted_set?(key, digest)
308
+ conn.zscan(key, match: "*#{digest}*", count: 1).to_a.any?
309
+ end
310
+ end
311
+ # rubocop:enable Metrics/ClassLength
312
+ end
313
+ end
@@ -0,0 +1,112 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Entity functions as a base class for redis types
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Entity
11
+ # includes "SidekiqUniqueJobs::Logging"
12
+ # @!parse include SidekiqUniqueJobs::Logging
13
+ include SidekiqUniqueJobs::Logging
14
+
15
+ # includes "SidekiqUniqueJobs::Script::Caller"
16
+ # @!parse include SidekiqUniqueJobs::Script::Caller
17
+ include SidekiqUniqueJobs::Script::Caller
18
+
19
+ # includes "SidekiqUniqueJobs::JSON"
20
+ # @!parse include SidekiqUniqueJobs::JSON
21
+ include SidekiqUniqueJobs::JSON
22
+
23
+ # includes "SidekiqUniqueJobs::Timing"
24
+ # @!parse include SidekiqUniqueJobs::Timing
25
+ include SidekiqUniqueJobs::Timing
26
+
27
+ #
28
+ # @!attribute [r] key
29
+ # @return [String] the redis key for this entity
30
+ attr_reader :key
31
+
32
+ #
33
+ # Initialize a new Entity
34
+ #
35
+ # @param [String] key the redis key for this entity
36
+ #
37
+ def initialize(key)
38
+ @key = key
39
+ end
40
+
41
+ #
42
+ # Checks if the key for this entity exists in redis
43
+ #
44
+ #
45
+ # @return [true] when exists
46
+ # @return [false] when not exists
47
+ #
48
+ def exist?
49
+ redis do |conn|
50
+ # TODO: Remove the if statement in the future
51
+ value =
52
+ if conn.respond_to?(:exists?)
53
+ conn.exists?(key)
54
+ else
55
+ conn.exists(key)
56
+ end
57
+
58
+ return value if boolean?(value)
59
+
60
+ value.to_i.positive?
61
+ end
62
+ end
63
+
64
+ #
65
+ # The number of microseconds until the key expires
66
+ #
67
+ #
68
+ # @return [Integer] expiration in milliseconds
69
+ #
70
+ def pttl
71
+ redis { |conn| conn.pttl(key) }
72
+ end
73
+
74
+ #
75
+ # The number of seconds until the key expires
76
+ #
77
+ #
78
+ # @return [Integer] expiration in seconds
79
+ #
80
+ def ttl
81
+ redis { |conn| conn.ttl(key) }
82
+ end
83
+
84
+ #
85
+ # Check if the entity has expiration
86
+ #
87
+ #
88
+ # @return [true] when entity is set to exire
89
+ # @return [false] when entity isn't expiring
90
+ #
91
+ def expires?
92
+ pttl.positive? || ttl.positive?
93
+ end
94
+
95
+ #
96
+ # Returns the number of entries in this entity
97
+ #
98
+ #
99
+ # @return [Integer] 0
100
+ #
101
+ def count
102
+ 0
103
+ end
104
+
105
+ private
106
+
107
+ def boolean?(value)
108
+ [TrueClass, FalseClass].any? { |klazz| value.is_a?(klazz) }
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Hash provides convenient access to redis hashes
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Hash < Entity
11
+ #
12
+ # Return entries for this hash
13
+ #
14
+ # @param [true,false] with_values false return hash
15
+ #
16
+ # @return [Array<Object>] when given with_values: false
17
+ # @return [Hash<String, String>] when given with_values: true
18
+ #
19
+ def entries(with_values: false)
20
+ if with_values
21
+ redis { |conn| conn.hgetall(key) }
22
+ else
23
+ redis { |conn| conn.hkeys(key) }
24
+ end
25
+ end
26
+
27
+ #
28
+ # Removes the key from redis
29
+ #
30
+ def del(*fields)
31
+ redis { |conn| conn.hdel(key, *fields) }
32
+ end
33
+
34
+ #
35
+ # Get a members value
36
+ #
37
+ # @param [String] member the member who's value to get
38
+ #
39
+ # @return [Object] whatever is stored on this hash member
40
+ #
41
+ def [](member)
42
+ redis { |conn| conn.hget(key, member) }
43
+ end
44
+
45
+ #
46
+ # Returns the count for this hash
47
+ #
48
+ #
49
+ # @return [Integer] the length of this hash
50
+ #
51
+ def count
52
+ redis { |conn| conn.hlen(key) }
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class List provides convenient access to redis hashes
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class List < Entity
11
+ #
12
+ # Entries in this list
13
+ #
14
+ #
15
+ # @return [Array<Object>] the elements in this list
16
+ #
17
+ def entries
18
+ redis { |conn| conn.lrange(key, 0, -1) }
19
+ end
20
+
21
+ #
22
+ # The number of entries in this list
23
+ #
24
+ #
25
+ # @return [Integer] the total number of entries
26
+ #
27
+ def count
28
+ redis { |conn| conn.llen(key) }
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Set provides convenient access to redis sets
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Set < Entity
11
+ #
12
+ # Return entries for this set
13
+ #
14
+ #
15
+ # @return [Array<String>]
16
+ #
17
+ def entries
18
+ redis { |conn| conn.smembers(key) }
19
+ end
20
+
21
+ #
22
+ # Returns the count for this sorted set
23
+ #
24
+ #
25
+ # @return [Integer] the number of entries
26
+ #
27
+ def count
28
+ redis { |conn| conn.scard(key) }
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class SortedSet provides convenient access to redis sorted sets
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class SortedSet < Entity
11
+ #
12
+ # @return [Integer] the number of matches to return by default
13
+ DEFAULT_COUNT = 1_000
14
+ #
15
+ # @return [String] the default pattern to use for matching
16
+ SCAN_PATTERN = "*"
17
+
18
+ #
19
+ # Return entries for this sorted set
20
+ #
21
+ # @param [true,false] with_scores true return
22
+ #
23
+ # @return [Array<Object>] when given with_scores: false
24
+ # @return [Hash] when given with_scores: true
25
+ #
26
+ def entries(with_scores: true)
27
+ return redis { |conn| conn.zrange(key, 0, -1) } unless with_scores
28
+
29
+ redis { |conn| conn.zrange(key, 0, -1, "withscores") }.each_with_object({}) do |pair, hash|
30
+ hash[pair[0]] = pair[1]
31
+ end
32
+ end
33
+
34
+ #
35
+ # Adds a value to the sorted set
36
+ #
37
+ # @param [Array<Float, String>, String] values the values to add
38
+ #
39
+ # @return [Boolean, Integer] <description>
40
+ #
41
+ def add(values)
42
+ redis do |conn|
43
+ if values.is_a?(Array)
44
+ conn.zadd(key, *values)
45
+ else
46
+ conn.zadd(key, now_f, values)
47
+ end
48
+ end
49
+ end
50
+
51
+ def byscore(min, max, offset: nil, count: nil)
52
+ redis do |conn|
53
+ return conn.zrange(key, min, max, "byscore") unless offset && count
54
+
55
+ conn.zrange(key, min, max, "byscore", "limit", offset, count)
56
+ end
57
+ end
58
+
59
+ #
60
+ # Return the zrak of the member
61
+ #
62
+ # @param [String] member the member to pull rank on
63
+ #
64
+ # @return [Integer]
65
+ #
66
+ def rank(member)
67
+ redis { |conn| conn.zrank(key, member) }
68
+ end
69
+
70
+ #
71
+ # Return score for a member
72
+ #
73
+ # @param [String] member the member for which score
74
+ #
75
+ # @return [Integer, Float]
76
+ #
77
+ def score(member)
78
+ redis { |conn| conn.zscore(key, member) }
79
+ end
80
+
81
+ #
82
+ # Clears the sorted set from all entries
83
+ #
84
+ #
85
+ # @return [Integer] number of entries removed
86
+ #
87
+ def clear
88
+ redis { |conn| conn.zremrangebyrank(key, 0, count) }
89
+ end
90
+
91
+ #
92
+ # Returns the count for this sorted set
93
+ #
94
+ #
95
+ # @return [Integer] the number of entries
96
+ #
97
+ def count
98
+ redis { |conn| conn.zcard(key) }
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class String provides convenient access to redis strings
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class String < Entity
11
+ #
12
+ # Returns the value of the key
13
+ #
14
+ #
15
+ # @return [String]
16
+ #
17
+ def value
18
+ redis { |conn| conn.get(key) }
19
+ end
20
+
21
+ #
22
+ # Sets the value of the key to given object
23
+ #
24
+ # @param [String] obj the object to update the key with
25
+ #
26
+ # @return [true, false]
27
+ #
28
+ def set(obj, pipeline = nil)
29
+ return pipeline.set(key, obj) if pipeline
30
+
31
+ redis { |conn| conn.set(key, obj) }
32
+ end
33
+
34
+ #
35
+ # Removes the key from redis
36
+ #
37
+ def del(*)
38
+ redis { |conn| conn.del(key) }
39
+ end
40
+
41
+ #
42
+ # Used only for compatibility with other keys
43
+ #
44
+ # @return [1] when key exists
45
+ # @return [0] when key does not exists
46
+ def count
47
+ exist? ? 1 : 0
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Augments and enhances redis with object oriented methods
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ module Redis
10
+ end
11
+ end