sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Middleware
5
+ # The unique sidekiq middleware for the client push
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Client
9
+ include Sidekiq::ClientMiddleware
10
+
11
+ # prepend "SidekiqUniqueJobs::Middleware"
12
+ # @!parse prepends SidekiqUniqueJobs::Middleware
13
+ prepend SidekiqUniqueJobs::Middleware
14
+ # includes "SidekiqUniqueJobs::Reflectable"
15
+ # @!parse include SidekiqUniqueJobs::Reflectable
16
+ include SidekiqUniqueJobs::Reflectable
17
+
18
+ # Calls this client middleware
19
+ # Used from Sidekiq.process_single
20
+ #
21
+ # @see SidekiqUniqueJobs::Middleware#call
22
+ #
23
+ # @see https://github.com/mperham/sidekiq/wiki/Job-Format
24
+ # @see https://github.com/mperham/sidekiq/wiki/Middleware
25
+ #
26
+ # @yield when uniqueness is disable
27
+ # @yield when the lock is successful
28
+ def call(*, &block)
29
+ lock(&block)
30
+ end
31
+
32
+ private
33
+
34
+ def lock
35
+ lock_instance.lock do
36
+ reflect(:locked, item)
37
+ return yield
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Middleware
5
+ # The unique sidekiq middleware for the server processor
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Server
9
+ include Sidekiq::ServerMiddleware
10
+
11
+ # prepend "SidekiqUniqueJobs::Middleware"
12
+ # @!parse prepends SidekiqUniqueJobs::Middleware
13
+ prepend SidekiqUniqueJobs::Middleware
14
+
15
+ #
16
+ #
17
+ # Runs the server middleware (used from Sidekiq::Processor#process)
18
+ #
19
+ # @see SidekiqUniqueJobs::Middleware#call
20
+ #
21
+ # @see https://github.com/mperham/sidekiq/wiki/Job-Format
22
+ # @see https://github.com/mperham/sidekiq/wiki/Middleware
23
+ #
24
+ # @yield when uniqueness is disabled
25
+ # @yield when owning the lock
26
+ def call(*, &block)
27
+ lock_instance.execute(&block)
28
+ end
29
+ end
30
+ end
31
+ end
@@ -1,19 +1,45 @@
1
- require 'sidekiq'
1
+ # frozen_string_literal: true
2
2
 
3
- Sidekiq.configure_server do |config|
4
- config.server_middleware do |chain|
5
- require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
6
- chain.add SidekiqUniqueJobs::Middleware::Server::UniqueJobs
7
- end
8
- config.client_middleware do |chain|
9
- require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
10
- chain.add SidekiqUniqueJobs::Middleware::Client::UniqueJobs
11
- end
12
- end
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Provides the sidekiq middleware that makes the gem work
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ module Middleware
10
+ include SidekiqUniqueJobs::Logging::Middleware
11
+ include SidekiqUniqueJobs::OptionsWithFallback
12
+ include SidekiqUniqueJobs::JSON
13
+
14
+ # The sidekiq job hash
15
+ # @return [Hash] the Sidekiq job hash
16
+ attr_reader :item
17
+
18
+ #
19
+ # This method runs before (prepended) the actual middleware implementation.
20
+ # This is done to reduce duplication
21
+ #
22
+ # @param [Sidekiq::Job] worker_class
23
+ # @param [Hash] item a sidekiq job hash
24
+ # @param [String] queue name of the queue
25
+ # @param [ConnectionPool] redis_pool only used for compatility reasons
26
+ #
27
+ # @return [yield<super>] call the rest of the middleware stack
28
+ #
29
+ # @yieldparam [void] if uniquejobs is disable
30
+ # @yieldreturn [void] delegate back to other sidekiq middleware
31
+ def call(worker_class, item, queue, redis_pool = nil)
32
+ @item = item
33
+ @queue = queue
34
+ @redis_pool = redis_pool
35
+ self.job_class = worker_class
36
+ return yield if unique_disabled?
37
+
38
+ SidekiqUniqueJobs::Job.prepare(item)
13
39
 
14
- Sidekiq.configure_client do |config|
15
- config.client_middleware do |chain|
16
- require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
17
- chain.add SidekiqUniqueJobs::Middleware::Client::UniqueJobs
40
+ with_logging_context do
41
+ super
42
+ end
43
+ end
18
44
  end
19
45
  end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Normalizes hashes by dumping them to json and loading them from json
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ module Normalizer
8
+ extend SidekiqUniqueJobs::JSON
9
+
10
+ # Changes hash to a json compatible hash
11
+ # @param [Hash] args
12
+ # @return [Hash] a json compatible hash
13
+ def self.jsonify(args)
14
+ load_json(dump_json(args))
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module OnConflict
5
+ # Strategy to log information about conflict
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Log < OnConflict::Strategy
9
+ include SidekiqUniqueJobs::Logging
10
+
11
+ #
12
+ # Logs an informational message about that the job was not unique
13
+ #
14
+ #
15
+ # @return [void]
16
+ #
17
+ def call
18
+ log_info(<<~MESSAGE.chomp)
19
+ Skipping job with id (#{item[JID]}) because lock_digest: (#{item[LOCK_DIGEST]}) already exists
20
+ MESSAGE
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module OnConflict
5
+ # Default conflict strategy class that does nothing
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class NullStrategy < OnConflict::Strategy
9
+ # Do nothing on conflict
10
+ # @return [nil]
11
+ def call
12
+ # NOOP
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module OnConflict
5
+ # Strategy to raise an error on conflict
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Raise < OnConflict::Strategy
9
+ # Raise an error on conflict.
10
+ # This will cause Sidekiq to retry the job
11
+ # @raise [SidekiqUniqueJobs::Conflict]
12
+ def call
13
+ raise SidekiqUniqueJobs::Conflict, item
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module OnConflict
5
+ # Strategy to send jobs to dead queue
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Reject < OnConflict::Strategy
9
+ include SidekiqUniqueJobs::Timing
10
+
11
+ # Send jobs to dead queue
12
+ def call
13
+ log_info { "Adding dead #{item[CLASS]} job #{item[JID]}" }
14
+
15
+ if kill_with_options?
16
+ kill_job_with_options
17
+ else
18
+ kill_job_without_options
19
+ end
20
+ end
21
+
22
+ #
23
+ # Sidekiq version compatibility check
24
+ # @api private
25
+ #
26
+ #
27
+ # @return [true] when Sidekiq::Deadset#kill takes more than 1 argument
28
+ # @return [false] when Sidekiq::Deadset#kill does not take multiple arguments
29
+ #
30
+ def kill_with_options?
31
+ Sidekiq::DeadSet.instance_method(:kill).arity > 1
32
+ end
33
+
34
+ #
35
+ # Executes the kill instructions without arguments
36
+ # @api private
37
+ #
38
+ # @return [void]
39
+ #
40
+ def kill_job_without_options
41
+ deadset.kill(payload)
42
+ end
43
+
44
+ #
45
+ # Executes the kill instructions with arguments
46
+ # @api private
47
+ #
48
+ # @return [void]
49
+ #
50
+ def kill_job_with_options
51
+ deadset.kill(payload, notify_failure: false)
52
+ end
53
+
54
+ #
55
+ # An instance of Sidekiq::Deadset
56
+ # @api private
57
+ #
58
+ # @return [Sidekiq::Deadset]>
59
+ #
60
+ def deadset
61
+ @deadset ||= Sidekiq::DeadSet.new
62
+ end
63
+
64
+ #
65
+ # The Sidekiq job hash as JSON
66
+ #
67
+ #
68
+ # @return [String] a JSON formatted string
69
+ #
70
+ def payload
71
+ @payload ||= dump_json(item)
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module OnConflict
5
+ # Strategy to replace the job on conflict
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Replace < OnConflict::Strategy
9
+ #
10
+ # @!attribute [r] queue
11
+ # @return [String] rthe sidekiq queue this job belongs to
12
+ attr_reader :queue
13
+ #
14
+ # @!attribute [r] lock_digest
15
+ # @return [String] the unique digest to use for locking
16
+ attr_reader :lock_digest
17
+
18
+ #
19
+ # Initialize a new Replace strategy
20
+ #
21
+ # @param [Hash] item sidekiq job hash
22
+ #
23
+ def initialize(item, redis_pool = nil)
24
+ super(item, redis_pool)
25
+ @queue = item[QUEUE]
26
+ @lock_digest = item[LOCK_DIGEST]
27
+ end
28
+
29
+ #
30
+ # Replace the old job in the queue
31
+ #
32
+ #
33
+ # @return [void] <description>
34
+ #
35
+ # @yield to retry the lock after deleting the old one
36
+ #
37
+ def call(&block)
38
+ return unless (deleted_job = delete_job_by_digest)
39
+
40
+ log_info("Deleted job: #{deleted_job}")
41
+ if (del_count = delete_lock)
42
+ log_info("Deleted `#{del_count}` keys for #{lock_digest}")
43
+ end
44
+
45
+ block&.call
46
+ end
47
+
48
+ #
49
+ # Delete the job from either schedule, retry or the queue
50
+ #
51
+ #
52
+ # @return [String] the deleted job hash
53
+ # @return [nil] when deleting nothing
54
+ #
55
+ def delete_job_by_digest
56
+ call_script(:delete_job_by_digest,
57
+ keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
58
+ argv: [lock_digest])
59
+ end
60
+
61
+ #
62
+ # Delete the keys belonging to the job
63
+ #
64
+ #
65
+ # @return [Integer] the number of keys deleted
66
+ #
67
+ def delete_lock
68
+ digests.delete_by_digest(lock_digest, runtime: false)
69
+ end
70
+
71
+ #
72
+ # Access to the {Digests}
73
+ #
74
+ #
75
+ # @return [Digests] and instance with digests
76
+ #
77
+ def digests
78
+ @digests ||= SidekiqUniqueJobs::Digests.new
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module OnConflict
5
+ # Strategy to reschedule job on conflict
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Reschedule < OnConflict::Strategy
9
+ include SidekiqUniqueJobs::SidekiqWorkerMethods
10
+ include SidekiqUniqueJobs::Logging
11
+ include SidekiqUniqueJobs::JSON
12
+ include SidekiqUniqueJobs::Reflectable
13
+
14
+ # @param [Hash] item sidekiq job hash
15
+ def initialize(item, redis_pool = nil)
16
+ super(item, redis_pool)
17
+ self.job_class = item[CLASS]
18
+ end
19
+
20
+ # Create a new job from the current one.
21
+ # This will mess up sidekiq stats because a new job is created
22
+ def call
23
+ if sidekiq_job_class?
24
+ if job_class.set(queue: item["queue"].to_sym).perform_in(schedule_in, *item[ARGS])
25
+ reflect(:rescheduled, item)
26
+ else
27
+ reflect(:reschedule_failed, item)
28
+ end
29
+ else
30
+ reflect(:unknown_sidekiq_worker, item)
31
+ end
32
+ end
33
+
34
+ def schedule_in
35
+ job_class.get_sidekiq_options["schedule_in"] || 5
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module OnConflict
5
+ # Abstract conflict strategy class
6
+ #
7
+ # @abstract
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ class Strategy
10
+ include SidekiqUniqueJobs::JSON
11
+ include SidekiqUniqueJobs::Logging
12
+ include SidekiqUniqueJobs::Script::Caller
13
+ include SidekiqUniqueJobs::Timing
14
+
15
+ # @!attribute [r] item
16
+ # @return [Hash] sidekiq job hash
17
+ attr_reader :item
18
+ # @!attribute [r] redis_pool
19
+ # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
20
+ attr_reader :redis_pool
21
+
22
+ #
23
+ # Initialize a new Strategy
24
+ #
25
+ # @param [Hash] item sidekiq job hash
26
+ # @param [ConnectionPool] redis_pool the connection pool instance
27
+ #
28
+ def initialize(item, redis_pool = nil)
29
+ @item = item
30
+ @redis_pool = redis_pool
31
+ end
32
+
33
+ # Use strategy on conflict
34
+ # @raise [NotImplementedError] needs to be implemented in child class
35
+ def call
36
+ raise NotImplementedError, "needs to be implemented in child class"
37
+ end
38
+
39
+ #
40
+ # Check if the strategy is kind of {Replace}
41
+ #
42
+ #
43
+ # @return [true] when the strategy is a {Replace}
44
+ # @return [false] when the strategy is not a {Replace}
45
+ #
46
+ def replace?
47
+ is_a?(Replace)
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "on_conflict/strategy"
4
+ require_relative "on_conflict/null_strategy"
5
+ require_relative "on_conflict/log"
6
+ require_relative "on_conflict/raise"
7
+ require_relative "on_conflict/reject"
8
+ require_relative "on_conflict/replace"
9
+ require_relative "on_conflict/reschedule"
10
+
11
+ module SidekiqUniqueJobs
12
+ #
13
+ # Provides lock conflict resolutions
14
+ #
15
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
16
+ #
17
+ module OnConflict
18
+ # A convenience method for using the configured strategies
19
+ def self.strategies
20
+ SidekiqUniqueJobs.strategies
21
+ end
22
+
23
+ #
24
+ # Find a strategy to use for conflicting locks
25
+ #
26
+ # @param [Symbol] strategy the key for the strategy
27
+ #
28
+ # @return [OnConflict::Strategy] when found
29
+ # @return [OnConflict::NullStrategy] when no other could be found
30
+ #
31
+ def self.find_strategy(strategy)
32
+ return OnConflict::NullStrategy unless strategy
33
+
34
+ strategies.fetch(strategy.to_sym) do
35
+ SidekiqUniqueJobs.logger.warn(
36
+ "No matching implementation for strategy: #{strategy}, returning OnConflict::NullStrategy." \
37
+ " Available strategies are (#{strategies.inspect})",
38
+ )
39
+
40
+ OnConflict::NullStrategy
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Module containing methods shared between client and server middleware
5
+ #
6
+ # Requires the following methods to be defined in the including class
7
+ # 1. item (required)
8
+ # 2. options (can be nil)
9
+ # 3. job_class (required, can be anything)
10
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
11
+ module OptionsWithFallback
12
+ def self.included(base)
13
+ base.send(:include, SidekiqUniqueJobs::SidekiqWorkerMethods)
14
+ end
15
+
16
+ # A convenience method for using the configured locks
17
+ def locks
18
+ SidekiqUniqueJobs.locks
19
+ end
20
+
21
+ # Check if unique has been enabled
22
+ # @return [true, false] indicate if the gem has been enabled
23
+ def unique_enabled?
24
+ SidekiqUniqueJobs.enabled? && lock_type
25
+ end
26
+
27
+ # Check if unique has been disabled
28
+ def unique_disabled?
29
+ !unique_enabled?
30
+ end
31
+
32
+ #
33
+ # A new lock for this Sidekiq Job
34
+ #
35
+ #
36
+ # @return [Lock::BaseLock] an instance of a lock implementation
37
+ #
38
+ def lock_instance
39
+ @lock_instance ||= lock_class.new(item, after_unlock_hook, @redis_pool)
40
+ end
41
+
42
+ #
43
+ # Returns the corresponding class for the lock_type
44
+ #
45
+ #
46
+ # @return [Class]
47
+ #
48
+ def lock_class
49
+ @lock_class ||= locks.fetch(lock_type.to_sym) do
50
+ raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
51
+ end
52
+ end
53
+
54
+ #
55
+ # The type of lock for this worker
56
+ #
57
+ #
58
+ # @return [Symbol, NilClass]
59
+ #
60
+ def lock_type
61
+ @lock_type ||= options[LOCK] || item[LOCK]
62
+ end
63
+
64
+ #
65
+ # The default options with any matching keys overridden from worker options
66
+ #
67
+ #
68
+ # @return [Hash<String, Object>]
69
+ #
70
+ def options
71
+ @options ||= begin
72
+ opts = default_job_options.dup
73
+ opts.merge!(job_options) if sidekiq_job_class?
74
+ (opts || {}).stringify_keys
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Orphans
5
+ #
6
+ # Class DeleteOrphans provides deletion of orphaned digests
7
+ #
8
+ # @note this is a much slower version of the lua script but does not crash redis
9
+ #
10
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
11
+ #
12
+ class LuaReaper < Reaper
13
+ #
14
+ # Delete orphaned digests
15
+ #
16
+ #
17
+ # @return [Integer] the number of reaped locks
18
+ #
19
+ def call
20
+ call_script(
21
+ :reap_orphans,
22
+ conn,
23
+ keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
24
+ argv: [reaper_count, (Time.now - reaper_timeout).to_f],
25
+ )
26
+ end
27
+ end
28
+ end
29
+ end