sidekiq-unique-jobs 6.0.7 → 7.1.33

Sign up to get free protection for your applications and to get access to all the features.
Files changed (194) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1880 -156
  3. data/README.md +818 -238
  4. data/bin/uniquejobs +2 -2
  5. data/lib/sidekiq-unique-jobs.rb +1 -1
  6. data/lib/sidekiq_unique_jobs/batch_delete.rb +124 -0
  7. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  8. data/lib/sidekiq_unique_jobs/cli.rb +68 -25
  9. data/lib/sidekiq_unique_jobs/config.rb +319 -0
  10. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  11. data/lib/sidekiq_unique_jobs/constants.rb +50 -20
  12. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  13. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  14. data/lib/sidekiq_unique_jobs/digests.rb +70 -87
  15. data/lib/sidekiq_unique_jobs/exceptions.rb +88 -12
  16. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  17. data/lib/sidekiq_unique_jobs/job.rb +47 -13
  18. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  19. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  20. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +111 -82
  21. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  22. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  23. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +45 -5
  24. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +30 -4
  25. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +26 -2
  26. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +27 -15
  27. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  28. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +26 -7
  29. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -11
  30. data/lib/sidekiq_unique_jobs/lock.rb +342 -0
  31. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  32. data/lib/sidekiq_unique_jobs/lock_config.rb +126 -0
  33. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  34. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  35. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  36. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  37. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  38. data/lib/sidekiq_unique_jobs/locksmith.rb +324 -95
  39. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  40. data/lib/sidekiq_unique_jobs/logging.rb +208 -30
  41. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  44. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  45. data/lib/sidekiq_unique_jobs/lua/lock.lua +99 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  47. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  48. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  49. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +122 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  59. data/lib/sidekiq_unique_jobs/lua/unlock.lua +107 -0
  60. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  61. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  62. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  63. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  64. data/lib/sidekiq_unique_jobs/middleware.rb +33 -30
  65. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  66. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  67. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  68. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +2 -2
  69. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +63 -17
  70. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  71. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +16 -5
  72. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +26 -7
  73. data/lib/sidekiq_unique_jobs/on_conflict.rb +34 -16
  74. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +39 -36
  75. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  76. data/lib/sidekiq_unique_jobs/orphans/manager.rb +241 -0
  77. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  78. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  79. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  81. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +298 -0
  82. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  83. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  84. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  85. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  87. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  88. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  89. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  90. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  91. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  93. data/lib/sidekiq_unique_jobs/script/caller.rb +127 -0
  94. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  95. data/lib/sidekiq_unique_jobs/server.rb +61 -0
  96. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +115 -66
  97. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  98. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +51 -29
  99. data/lib/sidekiq_unique_jobs/testing.rb +104 -31
  100. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  101. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  102. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  103. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  104. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  105. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  106. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  107. data/lib/sidekiq_unique_jobs/web/helpers.rb +140 -15
  108. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +4 -4
  109. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  110. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  111. data/lib/sidekiq_unique_jobs/web/views/locks.erb +54 -0
  112. data/lib/sidekiq_unique_jobs/web.rb +86 -29
  113. data/lib/sidekiq_unique_jobs.rb +78 -105
  114. data/lib/tasks/changelog.rake +23 -0
  115. metadata +154 -218
  116. data/.codeclimate.yml +0 -35
  117. data/.csslintrc +0 -2
  118. data/.dockerignore +0 -4
  119. data/.editorconfig +0 -14
  120. data/.eslintignore +0 -1
  121. data/.eslintrc +0 -213
  122. data/.fasterer.yml +0 -23
  123. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -31
  124. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -17
  125. data/.gitignore +0 -28
  126. data/.reek.yml +0 -87
  127. data/.rspec +0 -2
  128. data/.rubocop.yml +0 -127
  129. data/.simplecov +0 -20
  130. data/.travis.yml +0 -47
  131. data/.yardopts +0 -7
  132. data/Appraisals +0 -29
  133. data/CODE_OF_CONDUCT.md +0 -74
  134. data/Gemfile +0 -24
  135. data/Guardfile +0 -55
  136. data/Rakefile +0 -12
  137. data/_config.yml +0 -1
  138. data/assets/unique_digests_1.png +0 -0
  139. data/assets/unique_digests_2.png +0 -0
  140. data/bin/bench +0 -20
  141. data/examples/another_unique_job.rb +0 -15
  142. data/examples/custom_queue_job.rb +0 -12
  143. data/examples/custom_queue_job_with_filter_method.rb +0 -13
  144. data/examples/custom_queue_job_with_filter_proc.rb +0 -16
  145. data/examples/expiring_job.rb +0 -12
  146. data/examples/inline_worker.rb +0 -12
  147. data/examples/just_a_worker.rb +0 -13
  148. data/examples/long_running_job.rb +0 -14
  149. data/examples/main_job.rb +0 -14
  150. data/examples/my_job.rb +0 -12
  151. data/examples/my_unique_job.rb +0 -15
  152. data/examples/my_unique_job_with_filter_method.rb +0 -21
  153. data/examples/my_unique_job_with_filter_proc.rb +0 -19
  154. data/examples/notify_worker.rb +0 -14
  155. data/examples/plain_class.rb +0 -13
  156. data/examples/simple_worker.rb +0 -15
  157. data/examples/spawn_simple_worker.rb +0 -12
  158. data/examples/test_class.rb +0 -9
  159. data/examples/unique_across_workers_job.rb +0 -20
  160. data/examples/unique_job_on_conflict_raise.rb +0 -14
  161. data/examples/unique_job_on_conflict_reject.rb +0 -14
  162. data/examples/unique_job_on_conflict_reschedule.rb +0 -14
  163. data/examples/unique_job_with_conditional_parameter.rb +0 -18
  164. data/examples/unique_job_with_filter_method.rb +0 -21
  165. data/examples/unique_job_with_nil_unique_args.rb +0 -20
  166. data/examples/unique_job_with_no_unique_args_method.rb +0 -16
  167. data/examples/unique_job_withthout_unique_args_parameter.rb +0 -18
  168. data/examples/unique_on_all_queues_job.rb +0 -16
  169. data/examples/until_and_while_executing_job.rb +0 -17
  170. data/examples/until_executed_2_job.rb +0 -24
  171. data/examples/until_executed_job.rb +0 -25
  172. data/examples/until_executing_job.rb +0 -11
  173. data/examples/until_expired_job.rb +0 -12
  174. data/examples/until_global_expired_job.rb +0 -12
  175. data/examples/while_executing_job.rb +0 -15
  176. data/examples/while_executing_reject_job.rb +0 -14
  177. data/examples/without_argument_job.rb +0 -13
  178. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  179. data/lib/sidekiq_unique_jobs/scripts.rb +0 -89
  180. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -40
  181. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  182. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  183. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -149
  184. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  185. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  186. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -42
  187. data/redis/acquire_lock.lua +0 -21
  188. data/redis/delete.lua +0 -14
  189. data/redis/delete_by_digest.lua +0 -24
  190. data/redis/delete_job_by_digest.lua +0 -60
  191. data/redis/lock.lua +0 -58
  192. data/redis/release_stale_locks.lua +0 -90
  193. data/redis/unlock.lua +0 -32
  194. data/sidekiq-unique-jobs.gemspec +0 -42
data/bin/uniquejobs CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
  # frozen_string_literal: true
3
3
 
4
- require 'bundler/setup'
5
- require 'sidekiq_unique_jobs'
4
+ require "bundler/setup"
5
+ require "sidekiq_unique_jobs"
6
6
 
7
7
  SidekiqUniqueJobs::Cli.start(ARGV)
@@ -1,3 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq_unique_jobs'
3
+ require "sidekiq_unique_jobs"
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class BatchDelete provides batch deletion of digests
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class BatchDelete
10
+ #
11
+ # @return [Integer] the default batch size
12
+ BATCH_SIZE = 100
13
+
14
+ #
15
+ # @return [Array<String>] Supported key suffixes
16
+ SUFFIXES = %w[
17
+ QUEUED
18
+ PRIMED
19
+ LOCKED
20
+ INFO
21
+ ].freeze
22
+
23
+ # includes "SidekiqUniqueJobs::Connection"
24
+ # @!parse include SidekiqUniqueJobs::Connection
25
+ include SidekiqUniqueJobs::Connection
26
+ # includes "SidekiqUniqueJobs::Logging"
27
+ # @!parse include SidekiqUniqueJobs::Logging
28
+ include SidekiqUniqueJobs::Logging
29
+
30
+ #
31
+ # @!attribute [r] digests
32
+ # @return [Array<String>] a collection of digests to be deleted
33
+ attr_reader :digests
34
+ #
35
+ # @!attribute [r] conn
36
+ # @return [Redis, RedisConnection, ConnectionPool] a redis connection
37
+ attr_reader :conn
38
+
39
+ #
40
+ # Executes a batch deletion of the provided digests
41
+ #
42
+ # @param [Array<String>] digests the digests to delete
43
+ # @param [Redis] conn the connection to use for deletion
44
+ #
45
+ # @return [void]
46
+ #
47
+ def self.call(digests, conn = nil)
48
+ new(digests, conn).call
49
+ end
50
+
51
+ #
52
+ # Initialize a new batch delete instance
53
+ #
54
+ # @param [Array<String>] digests the digests to delete
55
+ # @param [Redis] conn the connection to use for deletion
56
+ #
57
+ def initialize(digests, conn)
58
+ @count = 0
59
+ @digests = digests
60
+ @conn = conn
61
+ @digests ||= []
62
+ @digests.compact!
63
+ redis_version # Avoid pipelined calling redis_version and getting a future.
64
+ end
65
+
66
+ #
67
+ # Executes a batch deletion of the provided digests
68
+ # @note Just wraps batch_delete to be able to provide no connection
69
+ #
70
+ #
71
+ def call
72
+ return log_info("Nothing to delete; exiting.") if digests.none?
73
+
74
+ log_info("Deleting batch with #{digests.size} digests")
75
+ return batch_delete(conn) if conn
76
+
77
+ redis { |rcon| batch_delete(rcon) }
78
+ end
79
+
80
+ private
81
+
82
+ #
83
+ # Does the actual batch deletion
84
+ #
85
+ #
86
+ # @return [Integer] the number of deleted digests
87
+ #
88
+ def batch_delete(conn)
89
+ digests.each_slice(BATCH_SIZE) do |chunk|
90
+ conn.pipelined do |pipeline|
91
+ chunk.each do |digest|
92
+ del_digest(pipeline, digest)
93
+ pipeline.zrem(SidekiqUniqueJobs::DIGESTS, digest)
94
+ pipeline.zrem(SidekiqUniqueJobs::EXPIRING_DIGESTS, digest)
95
+ @count += 1
96
+ end
97
+ end
98
+ end
99
+
100
+ @count
101
+ end
102
+
103
+ def del_digest(pipeline, digest)
104
+ removable_keys = keys_for_digest(digest)
105
+
106
+ if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
107
+ pipeline.unlink(*removable_keys)
108
+ else
109
+ pipeline.del(*removable_keys)
110
+ end
111
+ end
112
+
113
+ def keys_for_digest(digest)
114
+ [digest, "#{digest}:RUN"].each_with_object([]) do |key, digest_keys|
115
+ digest_keys.push(key)
116
+ digest_keys.concat(SUFFIXES.map { |suffix| "#{key}:#{suffix}" })
117
+ end
118
+ end
119
+
120
+ def redis_version
121
+ @redis_version ||= SidekiqUniqueJobs.config.redis_version
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class Changelogs provides access to the changelog entries
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class Changelog < Redis::SortedSet
10
+ #
11
+ # @return [Integer] the number of matches to return by default
12
+ DEFAULT_COUNT = 1_000
13
+ #
14
+ # @return [String] the default pattern to use for matching
15
+ SCAN_PATTERN = "*"
16
+
17
+ def initialize
18
+ super(CHANGELOGS)
19
+ end
20
+
21
+ #
22
+ # Adds a new changelog entry
23
+ #
24
+ # @param [String] message a descriptive message about the entry
25
+ # @param [String] digest a unique digest
26
+ # @param [String] job_id a Sidekiq JID
27
+ # @param [String] script the name of the script adding the entry
28
+ #
29
+ # @return [void]
30
+ #
31
+ def add(message:, digest:, job_id:, script:)
32
+ message = dump_json(message: message, digest: digest, job_id: job_id, script: script)
33
+ redis { |conn| conn.zadd(key, now_f, message) }
34
+ end
35
+
36
+ #
37
+ # The change log entries
38
+ #
39
+ # @param [String] pattern the pattern to match
40
+ # @param [Integer] count the number of matches to return
41
+ #
42
+ # @return [Array<Hash>] an array of entries
43
+ #
44
+ def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
45
+ options = {}
46
+ options[:match] = pattern
47
+ options[:count] = count
48
+
49
+ redis do |conn|
50
+ conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
51
+ end
52
+ end
53
+
54
+ #
55
+ # Paginate the changelog entries
56
+ #
57
+ # @param [Integer] cursor the cursor for this iteration
58
+ # @param [String] pattern "*" the pattern to match
59
+ # @param [Integer] page_size 100 the number of matches to return
60
+ #
61
+ # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
62
+ #
63
+ def page(cursor: 0, pattern: "*", page_size: 100)
64
+ redis do |conn|
65
+ total_size, result = conn.multi do |pipeline|
66
+ pipeline.zcard(key)
67
+ pipeline.zscan(key, cursor, match: pattern, count: page_size)
68
+ end
69
+
70
+ [
71
+ total_size.to_i,
72
+ result[0].to_i, # next_cursor
73
+ result[1].map { |entry| load_json(entry[0]) }, # entries
74
+ ]
75
+ end
76
+ end
77
+ end
78
+ end
@@ -1,52 +1,95 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'thor'
3
+ require "thor"
4
4
 
5
5
  module SidekiqUniqueJobs
6
+ #
7
+ # Command line interface for unique jobs
8
+ #
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
10
+ #
6
11
  class Cli < Thor
7
- def self.banner(command, _namespace = nil, _subcommand = false)
8
- "jobs #{@package_name} #{command.usage}"
12
+ # :nodoc:
13
+ # rubocop:disable Style/OptionalBooleanParameter
14
+ def self.banner(command, _namespace = nil, _subcommand = false) # rubocop:disable Style/OptionalBooleanParameter
15
+ "jobs #{@package_name} #{command.usage}" # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
9
16
  end
17
+ # rubocop:enable Style/OptionalBooleanParameter
10
18
 
11
- desc 'keys PATTERN', 'list all unique keys and their expiry time'
12
- option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
13
- def keys(pattern = '*')
14
- keys = Util.keys(pattern, options[:count])
15
- say "Found #{keys.size} keys matching '#{pattern}':"
16
- print_in_columns(keys.sort) if keys.any?
19
+ desc "list PATTERN", "list all unique digests and their expiry time"
20
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
21
+ # :nodoc:
22
+ def list(pattern = "*")
23
+ max_count = options[:count]
24
+ say "Searching for regular digests"
25
+ list_entries(digests.entries(pattern: pattern, count: max_count), pattern)
26
+ say "Searching for expiring digests"
27
+ list_entries(expiring_digests.entries(pattern: pattern, count: max_count), pattern)
17
28
  end
18
29
 
19
- desc 'del PATTERN', 'deletes unique keys from redis by pattern'
20
- option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
21
- option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
30
+ desc "del PATTERN", "deletes unique digests from redis by pattern"
31
+ option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
32
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
33
+ # :nodoc:
22
34
  def del(pattern)
23
35
  max_count = options[:count]
24
36
  if options[:dry_run]
25
- keys = Util.keys(pattern, max_count)
26
- say "Would delete #{keys.size} keys matching '#{pattern}'"
37
+ count_entries_for_del(max_count, pattern)
27
38
  else
28
- deleted_count = Util.del(pattern, max_count)
29
- say "Deleted #{deleted_count} keys matching '#{pattern}'"
39
+ del_entries(max_count, pattern)
30
40
  end
31
41
  end
32
42
 
33
- desc 'console', 'drop into a console with easy access to helper methods'
43
+ desc "console", "drop into a console with easy access to helper methods"
44
+ # :nodoc:
34
45
  def console
35
- say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
36
- say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
37
- say "Use `del '*', 1000, false to delete the first 1000 keys matching '*'"
38
- Object.include SidekiqUniqueJobs::Util
46
+ say "Use `list '*', 1000 to display the first 1000 unique digests matching '*'"
47
+ say "Use `del '*', 1000, true (default) to see how many digests would be deleted for the pattern '*'"
48
+ say "Use `del '*', 1000, false to delete the first 1000 digests matching '*'"
49
+
50
+ # Object.include SidekiqUniqueJobs::Api
39
51
  console_class.start
40
52
  end
41
53
 
42
- no_commands do
54
+ no_commands do # rubocop:disable Metrics/BlockLength
55
+ # :nodoc:
56
+ def digests
57
+ @digests ||= SidekiqUniqueJobs::Digests.new
58
+ end
59
+
60
+ # :nodoc:
61
+ def expiring_digests
62
+ @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
63
+ end
64
+
65
+ # :nodoc:
43
66
  def console_class
44
- require 'pry'
67
+ require "pry"
45
68
  Pry
46
- rescue LoadError
47
- require 'irb'
69
+ rescue NameError, LoadError
70
+ require "irb"
48
71
  IRB
49
72
  end
73
+
74
+ # :nodoc:
75
+ def list_entries(entries, pattern)
76
+ say "Found #{entries.size} digests matching '#{pattern}':"
77
+ print_in_columns(entries.sort) if entries.any?
78
+ end
79
+
80
+ # :nodoc:
81
+ def count_entries_for_del(max_count, pattern)
82
+ count = digests.entries(pattern: pattern, count: max_count).size +
83
+ expiring_digests.entries(pattern: pattern, count: max_count).size
84
+ say "Would delete #{count} digests matching '#{pattern}'"
85
+ end
86
+
87
+ # :nodoc:
88
+ def del_entries(max_count, pattern)
89
+ deleted_count = digests.delete_by_pattern(pattern, count: max_count).to_i +
90
+ expiring_digests.delete_by_pattern(pattern, count: max_count).to_i
91
+ say "Deleted #{deleted_count} digests matching '#{pattern}'"
92
+ end
50
93
  end
51
94
  end
52
95
  end
@@ -0,0 +1,319 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # ThreadSafe config exists to be able to document the config class without errors
5
+ ThreadSafeConfig = Concurrent::MutableStruct.new("ThreadSafeConfig",
6
+ :lock_timeout,
7
+ :lock_ttl,
8
+ :enabled,
9
+ :lock_prefix,
10
+ :logger,
11
+ :logger_enabled,
12
+ :locks,
13
+ :strategies,
14
+ :debug_lua,
15
+ :max_history,
16
+ :reaper,
17
+ :reaper_count,
18
+ :reaper_interval,
19
+ :reaper_timeout,
20
+ :reaper_resurrector_interval,
21
+ :reaper_resurrector_enabled,
22
+ :lock_info,
23
+ :raise_on_config_error,
24
+ :current_redis_version)
25
+
26
+ #
27
+ # Shared class for dealing with gem configuration
28
+ #
29
+ # @author Mauro Berlanda <mauro.berlanda@gmail.com>
30
+ # rubocop:disable Metrics/ClassLength
31
+ class Config < ThreadSafeConfig
32
+ #
33
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available queued locks
34
+ LOCKS_WHILE_ENQUEUED = {
35
+ until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
36
+ while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
37
+ }.freeze
38
+
39
+ #
40
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available fulltime locks
41
+ LOCKS_FROM_PUSH_TO_PROCESSED = {
42
+ until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
43
+ until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
44
+ until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
45
+ until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
46
+ until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
47
+ until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
48
+ }.freeze
49
+
50
+ #
51
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available locks without unlock
52
+ LOCKS_WITHOUT_UNLOCK = {
53
+ until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
54
+ }.freeze
55
+
56
+ #
57
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available runtime/client locks
58
+ LOCKS_WHEN_BUSY = {
59
+ around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
60
+ while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
61
+ while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
62
+ while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
63
+ while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
64
+ }.freeze
65
+
66
+ #
67
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available default locks
68
+ LOCKS =
69
+ LOCKS_WHEN_BUSY.dup
70
+ .merge(LOCKS_WHILE_ENQUEUED.dup)
71
+ .merge(LOCKS_WITHOUT_UNLOCK.dup)
72
+ .merge(LOCKS_FROM_PUSH_TO_PROCESSED.dup)
73
+ .freeze
74
+
75
+ #
76
+ # @return [Hash<Symbol, SidekiqUniqueJobs::OnConflict::Strategy] all available default strategies
77
+ STRATEGIES = {
78
+ log: SidekiqUniqueJobs::OnConflict::Log,
79
+ raise: SidekiqUniqueJobs::OnConflict::Raise,
80
+ reject: SidekiqUniqueJobs::OnConflict::Reject,
81
+ replace: SidekiqUniqueJobs::OnConflict::Replace,
82
+ reschedule: SidekiqUniqueJobs::OnConflict::Reschedule,
83
+ }.freeze
84
+
85
+ #
86
+ # @return ['uniquejobs'] by default we use this prefix
87
+ PREFIX = "uniquejobs"
88
+ #
89
+ # @return [0] by default don't wait for locks
90
+ LOCK_TIMEOUT = 0
91
+ #
92
+ # @return [nil]
93
+ LOCK_TTL = nil
94
+ #
95
+ # @return [true,false] by default false (don't disable logger)
96
+ LOGGER_ENABLED = true
97
+ #
98
+ # @return [true] by default the gem is enabled
99
+ ENABLED = true
100
+ #
101
+ # @return [false] by default we don't debug the lua scripts because it is slow
102
+ DEBUG_LUA = false
103
+ #
104
+ # @return [1_000] use a changelog history of 1_000 entries by default
105
+ MAX_HISTORY = 1_000
106
+ #
107
+ # @return [:ruby] prefer the ruby reaper by default since the lua reaper still has problems
108
+ REAPER = :ruby
109
+ #
110
+ # @return [1_000] reap 1_000 orphaned locks at a time by default
111
+ REAPER_COUNT = 1_000
112
+ #
113
+ # @return [600] reap locks every 10 minutes
114
+ REAPER_INTERVAL = 600
115
+ #
116
+ # @return [10] stop reaper after 10 seconds
117
+ REAPER_TIMEOUT = 10
118
+ #
119
+ # @return [3600] check if reaper is dead each 3600 seconds
120
+ REAPER_RESURRECTOR_INTERVAL = 3600
121
+
122
+ #
123
+ # @return [false] enable reaper resurrector
124
+ REAPER_RESURRECTOR_ENABLED = false
125
+
126
+ #
127
+ # @return [false] while useful it also adds overhead so disable lock_info by default
128
+ USE_LOCK_INFO = false
129
+ #
130
+ # @return [false] by default we don't raise validation errors for workers
131
+ RAISE_ON_CONFIG_ERROR = false
132
+ #
133
+ # @return [0.0.0] default redis version is only to avoid NoMethodError on nil
134
+ REDIS_VERSION = "0.0.0"
135
+
136
+ #
137
+ # Returns a default configuration
138
+ #
139
+ # @example
140
+ # SidekiqUniqueJobs::Config.default => <concurrent/mutable_struct/thread_safe_config SidekiqUniqueJobs::Config {
141
+ # default_lock_timeout: 0,
142
+ # default_lock_ttl: nil,
143
+ # enabled: true,
144
+ # lock_prefix: "uniquejobs",
145
+ # logger: #<Sidekiq::Logger:0x00007f81e096b0e0 @level=1 ...>,
146
+ # locks: {
147
+ # around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
148
+ # while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
149
+ # while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
150
+ # while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
151
+ # while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
152
+ # until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
153
+ # while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
154
+ # until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
155
+ # until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
156
+ # until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
157
+ # until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
158
+ # until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
159
+ # until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
160
+ # until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted
161
+ # },
162
+ # strategies: {
163
+ # log: SidekiqUniqueJobs::OnConflict::Log,
164
+ # raise: SidekiqUniqueJobs::OnConflict::Raise,
165
+ # reject: SidekiqUniqueJobs::OnConflict::Reject,
166
+ # replace: SidekiqUniqueJobs::OnConflict::Replace,
167
+ # reschedule: SidekiqUniqueJobs::OnConflict::Reschedule
168
+ # },
169
+ # debug_lua: false,
170
+ # max_history: 1000,
171
+ # reaper:: ruby,
172
+ # reaper_count: 1000,
173
+ # lock_info: false,
174
+ # raise_on_config_error: false,
175
+ # }>
176
+ #
177
+ #
178
+ # @return [SidekiqUniqueJobs::Config] a default configuration
179
+ #
180
+ def self.default # rubocop:disable Metrics/MethodLength
181
+ new(
182
+ LOCK_TIMEOUT,
183
+ LOCK_TTL,
184
+ ENABLED,
185
+ PREFIX,
186
+ Sidekiq.logger,
187
+ LOGGER_ENABLED,
188
+ LOCKS,
189
+ STRATEGIES,
190
+ DEBUG_LUA,
191
+ MAX_HISTORY,
192
+ REAPER,
193
+ REAPER_COUNT,
194
+ REAPER_INTERVAL,
195
+ REAPER_TIMEOUT,
196
+ REAPER_RESURRECTOR_INTERVAL,
197
+ REAPER_RESURRECTOR_ENABLED,
198
+ USE_LOCK_INFO,
199
+ RAISE_ON_CONFIG_ERROR,
200
+ REDIS_VERSION,
201
+ )
202
+ end
203
+
204
+ #
205
+ # Set the default_lock_ttl
206
+ # @deprecated
207
+ #
208
+ # @param [Integer] obj value to set (seconds)
209
+ #
210
+ # @return [<type>] <description>
211
+ #
212
+ def default_lock_ttl=(obj)
213
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
214
+ " Please use `#{class_name}#lock_ttl=` instead."
215
+ self.lock_ttl = obj
216
+ end
217
+
218
+ #
219
+ # Set new value for default_lock_timeout
220
+ # @deprecated
221
+ #
222
+ # @param [Integer] obj value to set (seconds)
223
+ #
224
+ # @return [Integer]
225
+ #
226
+ def default_lock_timeout=(obj)
227
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
228
+ " Please use `#{class_name}#lock_timeout=` instead."
229
+ self.lock_timeout = obj
230
+ end
231
+
232
+ #
233
+ # Default lock TTL (Time To Live)
234
+ # @deprecated
235
+ #
236
+ # @return [nil, Integer] configured value or nil
237
+ #
238
+ def default_lock_ttl
239
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
240
+ " Please use `#{class_name}#lock_ttl` instead."
241
+ lock_ttl
242
+ end
243
+
244
+ #
245
+ # Default Lock Timeout
246
+ # @deprecated
247
+ #
248
+ #
249
+ # @return [nil, Integer] configured value or nil
250
+ #
251
+ def default_lock_timeout
252
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated." \
253
+ " Please use `#{class_name}#lock_timeout` instead."
254
+ lock_timeout
255
+ end
256
+
257
+ #
258
+ # Memoized variable to get the class name
259
+ #
260
+ #
261
+ # @return [String] name of the class
262
+ #
263
+ def class_name
264
+ @class_name ||= self.class.name
265
+ end
266
+
267
+ #
268
+ # Adds a lock type to the configuration. It will raise if the lock exists already
269
+ #
270
+ # @example Add a custom lock
271
+ # add_lock(:my_lock, CustomLocks::MyLock)
272
+ #
273
+ # @raise DuplicateLock when the name already exists
274
+ #
275
+ # @param [String, Symbol] name the name of the lock
276
+ # @param [Class] klass the class describing the lock
277
+ #
278
+ # @return [void]
279
+ #
280
+ def add_lock(name, klass)
281
+ lock_sym = name.to_sym
282
+ raise DuplicateLock, ":#{name} already defined, please use another name" if locks.key?(lock_sym)
283
+
284
+ new_locks = locks.dup.merge(lock_sym => klass).freeze
285
+ self.locks = new_locks
286
+ end
287
+
288
+ #
289
+ # Adds an on_conflict strategy to the configuration.
290
+ #
291
+ # @example Add a custom strategy
292
+ # add_lock(:my_strategy, CustomStrategies::MyStrategy)
293
+ #
294
+ # @raise [DuplicateStrategy] when the name already exists
295
+ #
296
+ # @param [String] name the name of the custom strategy
297
+ # @param [Class] klass the class describing the strategy
298
+ #
299
+ def add_strategy(name, klass)
300
+ strategy_sym = name.to_sym
301
+ raise DuplicateStrategy, ":#{name} already defined, please use another name" if strategies.key?(strategy_sym)
302
+
303
+ new_strategies = strategies.dup.merge(strategy_sym => klass).freeze
304
+ self.strategies = new_strategies
305
+ end
306
+
307
+ #
308
+ # The current version of redis
309
+ #
310
+ #
311
+ # @return [String] a version string eg. `5.0.1`
312
+ #
313
+ def redis_version
314
+ self.current_redis_version = SidekiqUniqueJobs.fetch_redis_version if current_redis_version == REDIS_VERSION
315
+ current_redis_version
316
+ end
317
+ end
318
+ # rubocop:enable Metrics/ClassLength
319
+ end
@@ -3,7 +3,7 @@
3
3
  module SidekiqUniqueJobs
4
4
  # Shared module for dealing with redis connections
5
5
  #
6
- # @author Mikael Henriksson <mikael@zoolutions.se>
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
7
  module Connection
8
8
  def self.included(base)
9
9
  base.send(:extend, self)
@@ -11,11 +11,12 @@ module SidekiqUniqueJobs
11
11
 
12
12
  # Creates a connection to redis
13
13
  # @return [Sidekiq::RedisConnection, ConnectionPool] a connection to redis
14
- def redis(redis_pool = nil)
15
- if redis_pool
16
- redis_pool.with { |conn| yield conn }
14
+ def redis(r_pool = nil, &block)
15
+ r_pool ||= defined?(redis_pool) ? redis_pool : r_pool
16
+ if r_pool
17
+ r_pool.with(&block)
17
18
  else
18
- Sidekiq.redis { |conn| yield conn }
19
+ Sidekiq.redis(&block)
19
20
  end
20
21
  end
21
22
  end