sidekiq-unique-jobs 6.0.24 → 7.0.4

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +707 -25
  3. data/README.md +516 -105
  4. data/lib/sidekiq_unique_jobs.rb +48 -7
  5. data/lib/sidekiq_unique_jobs/batch_delete.rb +123 -0
  6. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  7. data/lib/sidekiq_unique_jobs/cli.rb +34 -31
  8. data/lib/sidekiq_unique_jobs/config.rb +263 -0
  9. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  10. data/lib/sidekiq_unique_jobs/constants.rb +46 -24
  11. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  12. data/lib/sidekiq_unique_jobs/digests.rb +71 -100
  13. data/lib/sidekiq_unique_jobs/exceptions.rb +78 -12
  14. data/lib/sidekiq_unique_jobs/job.rb +41 -12
  15. data/lib/sidekiq_unique_jobs/json.rb +40 -0
  16. data/lib/sidekiq_unique_jobs/key.rb +93 -0
  17. data/lib/sidekiq_unique_jobs/lock.rb +325 -0
  18. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +66 -50
  19. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  20. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  21. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +7 -10
  22. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +6 -6
  23. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +1 -1
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +4 -21
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +13 -9
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -3
  28. data/lib/sidekiq_unique_jobs/lock_args.rb +123 -0
  29. data/lib/sidekiq_unique_jobs/lock_config.rb +122 -0
  30. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  31. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  32. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  33. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  34. data/lib/sidekiq_unique_jobs/locksmith.rb +261 -101
  35. data/lib/sidekiq_unique_jobs/logging.rb +149 -23
  36. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  37. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  38. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  39. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  40. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  41. data/lib/sidekiq_unique_jobs/lua/lock.lua +93 -0
  42. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  43. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  44. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +94 -0
  45. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  46. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  47. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  54. data/lib/sidekiq_unique_jobs/lua/unlock.lua +95 -0
  55. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  56. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  57. data/lib/sidekiq_unique_jobs/middleware.rb +29 -31
  58. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  59. data/lib/sidekiq_unique_jobs/middleware/server.rb +27 -0
  60. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  61. data/lib/sidekiq_unique_jobs/on_conflict.rb +23 -10
  62. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  63. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  64. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  65. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +61 -15
  66. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  67. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +12 -5
  68. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +25 -6
  69. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +41 -27
  70. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  71. data/lib/sidekiq_unique_jobs/orphans/manager.rb +212 -0
  72. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  73. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  74. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  75. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +201 -0
  76. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  77. data/lib/sidekiq_unique_jobs/redis/entity.rb +106 -0
  78. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  79. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  80. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  81. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  82. data/lib/sidekiq_unique_jobs/redis/string.rb +49 -0
  83. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  84. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  85. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  86. data/lib/sidekiq_unique_jobs/script/caller.rb +125 -0
  87. data/lib/sidekiq_unique_jobs/server.rb +48 -0
  88. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +92 -65
  89. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +185 -34
  90. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +11 -5
  91. data/lib/sidekiq_unique_jobs/testing.rb +62 -21
  92. data/lib/sidekiq_unique_jobs/timer_task.rb +78 -0
  93. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  94. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  95. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  96. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  97. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  98. data/lib/sidekiq_unique_jobs/version_check.rb +23 -4
  99. data/lib/sidekiq_unique_jobs/web.rb +50 -27
  100. data/lib/sidekiq_unique_jobs/web/helpers.rb +125 -10
  101. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  102. data/lib/sidekiq_unique_jobs/web/views/lock.erb +108 -0
  103. data/lib/sidekiq_unique_jobs/web/views/locks.erb +52 -0
  104. data/lib/tasks/changelog.rake +5 -5
  105. metadata +117 -177
  106. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  107. data/lib/sidekiq_unique_jobs/scripts.rb +0 -118
  108. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  109. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  110. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  111. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -150
  112. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  113. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  114. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  115. data/redis/acquire_lock.lua +0 -21
  116. data/redis/convert_legacy_lock.lua +0 -13
  117. data/redis/delete.lua +0 -14
  118. data/redis/delete_by_digest.lua +0 -23
  119. data/redis/delete_job_by_digest.lua +0 -60
  120. data/redis/lock.lua +0 -62
  121. data/redis/release_stale_locks.lua +0 -90
  122. data/redis/unlock.lua +0 -35
@@ -1,25 +1,59 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "forwardable"
3
+ require "brpoplpush/redis_script"
4
+ require "concurrent/future"
5
+ require "concurrent/promises"
6
+ require "concurrent/map"
4
7
  require "concurrent/mutable_struct"
8
+ require "concurrent/timer_task"
9
+ require "concurrent/executor/ruby_single_thread_executor"
10
+ require "digest"
11
+ require "digest/sha1"
12
+ require "erb"
13
+ require "forwardable"
14
+ require "json"
15
+ require "pathname"
16
+ require "sidekiq"
5
17
 
18
+ require "sidekiq_unique_jobs/timer_task"
6
19
  require "sidekiq_unique_jobs/version"
7
20
  require "sidekiq_unique_jobs/version_check"
8
21
  require "sidekiq_unique_jobs/constants"
22
+ require "sidekiq_unique_jobs/json"
9
23
  require "sidekiq_unique_jobs/logging"
24
+ require "sidekiq_unique_jobs/logging/middleware_context"
25
+ require "sidekiq_unique_jobs/timing"
10
26
  require "sidekiq_unique_jobs/sidekiq_worker_methods"
11
27
  require "sidekiq_unique_jobs/connection"
12
28
  require "sidekiq_unique_jobs/exceptions"
29
+ require "sidekiq_unique_jobs/script"
30
+ require "sidekiq_unique_jobs/script/caller"
31
+ require "sidekiq_unique_jobs/normalizer"
13
32
  require "sidekiq_unique_jobs/job"
14
- require "sidekiq_unique_jobs/util"
15
- require "sidekiq_unique_jobs/digests"
33
+ require "sidekiq_unique_jobs/redis"
34
+ require "sidekiq_unique_jobs/redis/entity"
35
+ require "sidekiq_unique_jobs/redis/hash"
36
+ require "sidekiq_unique_jobs/redis/list"
37
+ require "sidekiq_unique_jobs/redis/set"
38
+ require "sidekiq_unique_jobs/redis/sorted_set"
39
+ require "sidekiq_unique_jobs/redis/string"
40
+ require "sidekiq_unique_jobs/batch_delete"
41
+ require "sidekiq_unique_jobs/orphans/reaper"
42
+ require "sidekiq_unique_jobs/orphans/observer"
43
+ require "sidekiq_unique_jobs/orphans/manager"
16
44
  require "sidekiq_unique_jobs/cli"
17
45
  require "sidekiq_unique_jobs/core_ext"
18
- require "sidekiq_unique_jobs/timeout"
19
- require "sidekiq_unique_jobs/scripts"
20
- require "sidekiq_unique_jobs/unique_args"
46
+ require "sidekiq_unique_jobs/lock_timeout"
47
+ require "sidekiq_unique_jobs/lock_ttl"
48
+ require "sidekiq_unique_jobs/lock_args"
49
+ require "sidekiq_unique_jobs/lock_digest"
21
50
  require "sidekiq_unique_jobs/unlockable"
51
+ require "sidekiq_unique_jobs/key"
22
52
  require "sidekiq_unique_jobs/locksmith"
53
+ require "sidekiq_unique_jobs/options_with_fallback"
54
+ require "sidekiq_unique_jobs/lock"
55
+ require "sidekiq_unique_jobs/lock_config"
56
+ require "sidekiq_unique_jobs/lock_info"
23
57
  require "sidekiq_unique_jobs/lock/base_lock"
24
58
  require "sidekiq_unique_jobs/lock/until_executed"
25
59
  require "sidekiq_unique_jobs/lock/until_executing"
@@ -27,9 +61,16 @@ require "sidekiq_unique_jobs/lock/until_expired"
27
61
  require "sidekiq_unique_jobs/lock/while_executing"
28
62
  require "sidekiq_unique_jobs/lock/while_executing_reject"
29
63
  require "sidekiq_unique_jobs/lock/until_and_while_executing"
30
- require "sidekiq_unique_jobs/options_with_fallback"
31
64
  require "sidekiq_unique_jobs/middleware"
65
+ require "sidekiq_unique_jobs/middleware/client"
66
+ require "sidekiq_unique_jobs/middleware/server"
32
67
  require "sidekiq_unique_jobs/sidekiq_unique_ext"
33
68
  require "sidekiq_unique_jobs/on_conflict"
69
+ require "sidekiq_unique_jobs/changelog"
70
+ require "sidekiq_unique_jobs/digests"
34
71
 
72
+ require "sidekiq_unique_jobs/config"
35
73
  require "sidekiq_unique_jobs/sidekiq_unique_jobs"
74
+ require "sidekiq_unique_jobs/update_version"
75
+ require "sidekiq_unique_jobs/upgrade_locks"
76
+ require "sidekiq_unique_jobs/server"
@@ -0,0 +1,123 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class BatchDelete provides batch deletion of digests
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class BatchDelete
10
+ #
11
+ # @return [Integer] the default batch size
12
+ BATCH_SIZE = 100
13
+
14
+ #
15
+ # @return [Array<String>] Supported key suffixes
16
+ SUFFIXES = %w[
17
+ QUEUED
18
+ PRIMED
19
+ LOCKED
20
+ INFO
21
+ ].freeze
22
+
23
+ # includes "SidekiqUniqueJobs::Connection"
24
+ # @!parse include SidekiqUniqueJobs::Connection
25
+ include SidekiqUniqueJobs::Connection
26
+ # includes "SidekiqUniqueJobs::Logging"
27
+ # @!parse include SidekiqUniqueJobs::Logging
28
+ include SidekiqUniqueJobs::Logging
29
+
30
+ #
31
+ # @!attribute [r] digests
32
+ # @return [Array<String>] a collection of digests to be deleted
33
+ attr_reader :digests
34
+ #
35
+ # @!attribute [r] conn
36
+ # @return [Redis, RedisConnection, ConnectionPool] a redis connection
37
+ attr_reader :conn
38
+
39
+ #
40
+ # Executes a batch deletion of the provided digests
41
+ #
42
+ # @param [Array<String>] digests the digests to delete
43
+ # @param [Redis] conn the connection to use for deletion
44
+ #
45
+ # @return [void]
46
+ #
47
+ def self.call(digests, conn = nil)
48
+ new(digests, conn).call
49
+ end
50
+
51
+ #
52
+ # Initialize a new batch delete instance
53
+ #
54
+ # @param [Array<String>] digests the digests to delete
55
+ # @param [Redis] conn the connection to use for deletion
56
+ #
57
+ def initialize(digests, conn)
58
+ @count = 0
59
+ @digests = digests
60
+ @conn = conn
61
+ @digests ||= []
62
+ @digests.compact!
63
+ redis_version # Avoid pipelined calling redis_version and getting a future.
64
+ end
65
+
66
+ #
67
+ # Executes a batch deletion of the provided digests
68
+ # @note Just wraps batch_delete to be able to provide no connection
69
+ #
70
+ #
71
+ def call
72
+ return log_info("Nothing to delete; exiting.") if digests.none?
73
+
74
+ log_info("Deleting batch with #{digests.size} digests")
75
+ return batch_delete(conn) if conn
76
+
77
+ redis { |rcon| batch_delete(rcon) }
78
+ end
79
+
80
+ private
81
+
82
+ #
83
+ # Does the actual batch deletion
84
+ #
85
+ #
86
+ # @return [Integer] the number of deleted digests
87
+ #
88
+ def batch_delete(conn)
89
+ digests.each_slice(BATCH_SIZE) do |chunk|
90
+ conn.pipelined do
91
+ chunk.each do |digest|
92
+ del_digest(conn, digest)
93
+ conn.zrem(SidekiqUniqueJobs::DIGESTS, digest)
94
+ @count += 1
95
+ end
96
+ end
97
+ end
98
+
99
+ @count
100
+ end
101
+
102
+ def del_digest(conn, digest)
103
+ removable_keys = keys_for_digest(digest)
104
+
105
+ if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
106
+ conn.unlink(*removable_keys)
107
+ else
108
+ conn.del(*removable_keys)
109
+ end
110
+ end
111
+
112
+ def keys_for_digest(digest)
113
+ [digest, "#{digest}:RUN"].each_with_object([]) do |key, digest_keys|
114
+ digest_keys.concat([key])
115
+ digest_keys.concat(SUFFIXES.map { |suffix| "#{key}:#{suffix}" })
116
+ end
117
+ end
118
+
119
+ def redis_version
120
+ @redis_version ||= SidekiqUniqueJobs.config.redis_version
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class Changelogs provides access to the changelog entries
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class Changelog < Redis::SortedSet
10
+ #
11
+ # @return [Integer] the number of matches to return by default
12
+ DEFAULT_COUNT = 1_000
13
+ #
14
+ # @return [String] the default pattern to use for matching
15
+ SCAN_PATTERN = "*"
16
+
17
+ def initialize
18
+ super(CHANGELOGS)
19
+ end
20
+
21
+ #
22
+ # Adds a new changelog entry
23
+ #
24
+ # @param [String] message a descriptive message about the entry
25
+ # @param [String] digest a unique digest
26
+ # @param [String] job_id a Sidekiq JID
27
+ # @param [String] script the name of the script adding the entry
28
+ #
29
+ # @return [void]
30
+ #
31
+ def add(message:, digest:, job_id:, script:)
32
+ message = dump_json(message: message, digest: digest, job_id: job_id, script: script)
33
+ redis { |conn| conn.zadd(key, now_f, message) }
34
+ end
35
+
36
+ #
37
+ # The change log entries
38
+ #
39
+ # @param [String] pattern the pattern to match
40
+ # @param [Integer] count the number of matches to return
41
+ #
42
+ # @return [Array<Hash>] an array of entries
43
+ #
44
+ def entries(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
45
+ options = {}
46
+ options[:match] = pattern
47
+ options[:count] = count
48
+
49
+ redis do |conn|
50
+ conn.zscan_each(key, **options).to_a.map { |entry| load_json(entry[0]) }
51
+ end
52
+ end
53
+
54
+ #
55
+ # Paginate the changelog entries
56
+ #
57
+ # @param [Integer] cursor the cursor for this iteration
58
+ # @param [String] pattern "*" the pattern to match
59
+ # @param [Integer] page_size 100 the number of matches to return
60
+ #
61
+ # @return [Array<Integer, Integer, Array<Hash>] the total size, next cursor and changelog entries
62
+ #
63
+ def page(cursor: 0, pattern: "*", page_size: 100)
64
+ redis do |conn|
65
+ total_size, result = conn.multi do
66
+ conn.zcard(key)
67
+ conn.zscan(key, cursor, match: pattern, count: page_size)
68
+ end
69
+
70
+ [
71
+ total_size,
72
+ result[0], # next_cursor
73
+ result[1].map { |entry| load_json(entry[0]) }, # entries
74
+ ]
75
+ end
76
+ end
77
+ end
78
+ end
@@ -6,61 +6,64 @@ module SidekiqUniqueJobs
6
6
  #
7
7
  # Command line interface for unique jobs
8
8
  #
9
- # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
10
10
  #
11
11
  class Cli < Thor
12
- def self.banner(command, _namespace = nil, _subcommand = false)
13
- "jobs #{@package_name} #{command.usage}"
12
+ # :nodoc:
13
+ # rubocop:disable Style/OptionalBooleanParameter
14
+ def self.banner(command, _namespace = nil, _subcommand = false) # rubocop:disable Style/OptionalBooleanParameter
15
+ "jobs #{@package_name} #{command.usage}" # rubocop:disable ThreadSafety/InstanceVariableInClassMethod
14
16
  end
17
+ # rubocop:enable Style/OptionalBooleanParameter
15
18
 
16
- desc "keys PATTERN", "list all unique keys and their expiry time"
17
- option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
18
- def keys(pattern = "*")
19
- keys = Util.keys(pattern, options[:count])
20
- say "Found #{keys.size} keys matching '#{pattern}':"
21
- print_in_columns(keys.sort) if keys.any?
19
+ desc "list PATTERN", "list all unique digests and their expiry time"
20
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
21
+ # :nodoc:
22
+ def list(pattern = "*")
23
+ entries = digests.entries(pattern: pattern, count: options[:count])
24
+ say "Found #{entries.size} digests matching '#{pattern}':"
25
+ print_in_columns(entries.sort) if entries.any?
22
26
  end
23
27
 
24
- desc "del PATTERN", "deletes unique keys from redis by pattern"
28
+ desc "del PATTERN", "deletes unique digests from redis by pattern"
25
29
  option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
26
- option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
30
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of digests to return"
31
+ # :nodoc:
27
32
  def del(pattern)
28
33
  max_count = options[:count]
29
34
  if options[:dry_run]
30
- keys = Util.keys(pattern, max_count)
31
- say "Would delete #{keys.size} keys matching '#{pattern}'"
35
+ result = digests.entries(pattern: pattern, count: max_count)
36
+ say "Would delete #{result.size} digests matching '#{pattern}'"
32
37
  else
33
- deleted_count = Util.del(pattern, max_count)
34
- say "Deleted #{deleted_count} keys matching '#{pattern}'"
38
+ deleted_count = digests.delete_by_pattern(pattern, count: max_count)
39
+ say "Deleted #{deleted_count} digests matching '#{pattern}'"
35
40
  end
36
41
  end
37
42
 
38
43
  desc "console", "drop into a console with easy access to helper methods"
44
+ # :nodoc:
39
45
  def console
40
- say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
41
- say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
42
- say "Use `del '*', 1000, false to delete the first 1000 keys matching '*'"
43
- Object.include SidekiqUniqueJobs::Util
46
+ say "Use `list '*', 1000 to display the first 1000 unique digests matching '*'"
47
+ say "Use `del '*', 1000, true (default) to see how many digests would be deleted for the pattern '*'"
48
+ say "Use `del '*', 1000, false to delete the first 1000 digests matching '*'"
49
+
50
+ # Object.include SidekiqUniqueJobs::Api
44
51
  console_class.start
45
52
  end
46
53
 
47
54
  no_commands do
48
- def console_class
49
- return irb if RUBY_PLATFORM == "JAVA"
50
-
51
- pry
52
- end
53
-
54
- def irb
55
- require "irb"
56
- IRB
55
+ # :nodoc:
56
+ def digests
57
+ @digests ||= SidekiqUniqueJobs::Digests.new
57
58
  end
58
59
 
59
- def pry
60
+ # :nodoc:
61
+ def console_class
60
62
  require "pry"
61
63
  Pry
62
- rescue LoadError, NameError
63
- irb
64
+ rescue NameError, LoadError
65
+ require "irb"
66
+ IRB
64
67
  end
65
68
  end
66
69
  end
@@ -0,0 +1,263 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # ThreadSafe config exists to be able to document the config class without errors
5
+ ThreadSafeConfig = Concurrent::MutableStruct.new("ThreadSafeConfig",
6
+ :lock_timeout,
7
+ :lock_ttl,
8
+ :enabled,
9
+ :lock_prefix,
10
+ :logger,
11
+ :locks,
12
+ :strategies,
13
+ :debug_lua,
14
+ :max_history,
15
+ :reaper,
16
+ :reaper_count,
17
+ :reaper_interval,
18
+ :reaper_timeout,
19
+ :lock_info,
20
+ :raise_on_config_error,
21
+ :current_redis_version)
22
+
23
+ #
24
+ # Shared class for dealing with gem configuration
25
+ #
26
+ # @author Mauro Berlanda <mauro.berlanda@gmail.com>
27
+ # rubocop:disable Metrics/ClassLength
28
+ class Config < ThreadSafeConfig
29
+ #
30
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available queued locks
31
+ LOCKS_WHILE_ENQUEUED = {
32
+ until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
33
+ while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
34
+ }.freeze
35
+
36
+ #
37
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available fulltime locks
38
+ LOCKS_FROM_PUSH_TO_PROCESSED = {
39
+ until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
40
+ until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
41
+ until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
42
+ until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
43
+ until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
44
+ until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
45
+ }.freeze
46
+
47
+ #
48
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available locks without unlock
49
+ LOCKS_WITHOUT_UNLOCK = {
50
+ until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
51
+ }.freeze
52
+
53
+ #
54
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available runtime/client locks
55
+ LOCKS_WHEN_BUSY = {
56
+ around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
57
+ while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
58
+ while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
59
+ while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
60
+ while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
61
+ }.freeze
62
+
63
+ #
64
+ # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available default locks
65
+ LOCKS =
66
+ LOCKS_WHEN_BUSY.dup
67
+ .merge(LOCKS_WHILE_ENQUEUED.dup)
68
+ .merge(LOCKS_WITHOUT_UNLOCK.dup)
69
+ .merge(LOCKS_FROM_PUSH_TO_PROCESSED.dup)
70
+ .freeze
71
+
72
+ #
73
+ # @return [Hash<Symbol, SidekiqUniqueJobs::OnConflict::Strategy] all available default strategies
74
+ STRATEGIES = {
75
+ log: SidekiqUniqueJobs::OnConflict::Log,
76
+ raise: SidekiqUniqueJobs::OnConflict::Raise,
77
+ reject: SidekiqUniqueJobs::OnConflict::Reject,
78
+ replace: SidekiqUniqueJobs::OnConflict::Replace,
79
+ reschedule: SidekiqUniqueJobs::OnConflict::Reschedule,
80
+ }.freeze
81
+
82
+ #
83
+ # @return ['uniquejobs'] by default we use this prefix
84
+ PREFIX = "uniquejobs"
85
+ #
86
+ # @return [0] by default don't wait for locks
87
+ LOCK_TIMEOUT = 0
88
+ #
89
+ # @return [nil]
90
+ LOCK_TTL = nil
91
+ #
92
+ # @return [true] by default the gem is enabled
93
+ ENABLED = true
94
+ #
95
+ # @return [false] by default we don't debug the lua scripts because it is slow
96
+ DEBUG_LUA = false
97
+ #
98
+ # @return [1_000] use a changelog history of 1_000 entries by default
99
+ MAX_HISTORY = 1_000
100
+ #
101
+ # @return [:ruby] prefer the ruby reaper by default since the lua reaper still has problems
102
+ REAPER = :ruby
103
+ #
104
+ # @return [1_000] reap 1_000 orphaned locks at a time by default
105
+ REAPER_COUNT = 1_000
106
+ #
107
+ # @return [600] reap locks every 10 minutes
108
+ REAPER_INTERVAL = 600
109
+ #
110
+ # @return [10] stop reaper after 10 seconds
111
+ REAPER_TIMEOUT = 10
112
+ #
113
+ # @return [false] while useful it also adds overhead so disable lock_info by default
114
+ USE_LOCK_INFO = false
115
+ #
116
+ # @return [false] by default we don't raise validation errors for workers
117
+ RAISE_ON_CONFIG_ERROR = false
118
+ #
119
+ # @return [0.0.0] default redis version is only to avoid NoMethodError on nil
120
+ REDIS_VERSION = "0.0.0"
121
+
122
+ #
123
+ # Returns a default configuration
124
+ #
125
+ # @example
126
+ # SidekiqUniqueJobs::Config.default => <concurrent/mutable_struct/thread_safe_config SidekiqUniqueJobs::Config {
127
+ # default_lock_timeout: 0,
128
+ # default_lock_ttl: nil,
129
+ # enabled: true,
130
+ # lock_prefix: "uniquejobs",
131
+ # logger: #<Sidekiq::Logger:0x00007f81e096b0e0 @level=1 ...>,
132
+ # locks: {
133
+ # around_perform: SidekiqUniqueJobs::Lock::WhileExecuting,
134
+ # while_busy: SidekiqUniqueJobs::Lock::WhileExecuting,
135
+ # while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
136
+ # while_working: SidekiqUniqueJobs::Lock::WhileExecuting,
137
+ # while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
138
+ # until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
139
+ # while_enqueued: SidekiqUniqueJobs::Lock::UntilExecuting,
140
+ # until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
141
+ # until_completed: SidekiqUniqueJobs::Lock::UntilExecuted,
142
+ # until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
143
+ # until_performed: SidekiqUniqueJobs::Lock::UntilExecuted,
144
+ # until_processed: SidekiqUniqueJobs::Lock::UntilExecuted,
145
+ # until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
146
+ # until_successfully_completed: SidekiqUniqueJobs::Lock::UntilExecuted
147
+ # },
148
+ # strategies: {
149
+ # log: SidekiqUniqueJobs::OnConflict::Log,
150
+ # raise: SidekiqUniqueJobs::OnConflict::Raise,
151
+ # reject: SidekiqUniqueJobs::OnConflict::Reject,
152
+ # replace: SidekiqUniqueJobs::OnConflict::Replace,
153
+ # reschedule: SidekiqUniqueJobs::OnConflict::Reschedule
154
+ # },
155
+ # debug_lua: false,
156
+ # max_history: 1000,
157
+ # reaper:: ruby,
158
+ # reaper_count: 1000,
159
+ # lock_info: false,
160
+ # raise_on_config_error: false,
161
+ # }>
162
+ #
163
+ #
164
+ # @return [SidekiqUniqueJobs::Config] a default configuration
165
+ #
166
+ def self.default # rubocop:disable Metrics/MethodLength
167
+ new(
168
+ LOCK_TIMEOUT,
169
+ LOCK_TTL,
170
+ ENABLED,
171
+ PREFIX,
172
+ Sidekiq.logger,
173
+ LOCKS,
174
+ STRATEGIES,
175
+ DEBUG_LUA,
176
+ MAX_HISTORY,
177
+ REAPER,
178
+ REAPER_COUNT,
179
+ REAPER_INTERVAL,
180
+ REAPER_TIMEOUT,
181
+ USE_LOCK_INFO,
182
+ RAISE_ON_CONFIG_ERROR,
183
+ REDIS_VERSION,
184
+ )
185
+ end
186
+
187
+ def default_lock_ttl=(obj)
188
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl=` instead."
189
+ self.lock_ttl = obj
190
+ end
191
+
192
+ def default_lock_timeout=(obj)
193
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout=` instead."
194
+ self.lock_timeout = obj
195
+ end
196
+
197
+ def default_lock_ttl
198
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_ttl` instead."
199
+ lock_ttl
200
+ end
201
+
202
+ def default_lock_timeout
203
+ warn "[DEPRECATION] `#{class_name}##{__method__}` is deprecated. Please use `#{class_name}#lock_timeout` instead."
204
+ lock_timeout
205
+ end
206
+
207
+ def class_name
208
+ @class_name ||= self.class.name
209
+ end
210
+
211
+ #
212
+ # Adds a lock type to the configuration. It will raise if the lock exists already
213
+ #
214
+ # @example Add a custom lock
215
+ # add_lock(:my_lock, CustomLocks::MyLock)
216
+ #
217
+ # @raise DuplicateLock when the name already exists
218
+ #
219
+ # @param [String, Symbol] name the name of the lock
220
+ # @param [Class] klass the class describing the lock
221
+ #
222
+ # @return [void]
223
+ #
224
+ def add_lock(name, klass)
225
+ lock_sym = name.to_sym
226
+ raise DuplicateLock, ":#{name} already defined, please use another name" if locks.key?(lock_sym)
227
+
228
+ new_locks = locks.dup.merge(lock_sym => klass).freeze
229
+ self.locks = new_locks
230
+ end
231
+
232
+ #
233
+ # Adds an on_conflict strategy to the configuration.
234
+ #
235
+ # @example Add a custom strategy
236
+ # add_lock(:my_strategy, CustomStrategies::MyStrategy)
237
+ #
238
+ # @raise [DuplicateStrategy] when the name already exists
239
+ #
240
+ # @param [String] name the name of the custom strategy
241
+ # @param [Class] klass the class describing the strategy
242
+ #
243
+ def add_strategy(name, klass)
244
+ strategy_sym = name.to_sym
245
+ raise DuplicateStrategy, ":#{name} already defined, please use another name" if strategies.key?(strategy_sym)
246
+
247
+ new_strategies = strategies.dup.merge(strategy_sym => klass).freeze
248
+ self.strategies = new_strategies
249
+ end
250
+
251
+ #
252
+ # The current version of redis
253
+ #
254
+ #
255
+ # @return [String] a version string eg. `5.0.1`
256
+ #
257
+ def redis_version
258
+ self.current_redis_version = SidekiqUniqueJobs.fetch_redis_version if current_redis_version == REDIS_VERSION
259
+ current_redis_version
260
+ end
261
+ end
262
+ # rubocop:enable Metrics/ClassLength
263
+ end