sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Class Info provides information about a lock
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ class LockInfo < Redis::String
10
+ #
11
+ # Returns the value for this key as a hash
12
+ #
13
+ #
14
+ # @return [Hash]
15
+ #
16
+ def value
17
+ @value ||= load_json(super)
18
+ end
19
+
20
+ #
21
+ # Check if this redis string is blank
22
+ #
23
+ #
24
+ # @return [Boolean]
25
+ #
26
+ def none?
27
+ value.nil? || value.empty?
28
+ end
29
+
30
+ #
31
+ # Check if this redis string has a value
32
+ #
33
+ #
34
+ # @return [Boolean]
35
+ #
36
+ def present?
37
+ !none?
38
+ end
39
+
40
+ #
41
+ # Quick access to the hash members for the value
42
+ #
43
+ # @param [String, Symbol] key the key who's value to retrieve
44
+ #
45
+ # @return [Object]
46
+ #
47
+ def [](key)
48
+ value[key.to_s] if value.is_a?(Hash)
49
+ end
50
+
51
+ #
52
+ # Writes the lock info to redis
53
+ #
54
+ # @param [Hash] obj the information to store at key
55
+ #
56
+ # @return [Hash]
57
+ #
58
+ def set(obj, pipeline = nil)
59
+ return unless SidekiqUniqueJobs.config.lock_info
60
+ raise InvalidArgument, "argument `obj` (#{obj}) needs to be a hash" unless obj.is_a?(Hash)
61
+
62
+ json = dump_json(obj)
63
+ @value = load_json(json)
64
+ super(json, pipeline)
65
+ value
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Calculates timeout and expiration
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ class LockTimeout
8
+ # includes "SidekiqUniqueJobs::SidekiqWorkerMethods"
9
+ # @!parse include SidekiqUniqueJobs::SidekiqWorkerMethods
10
+ include SidekiqUniqueJobs::SidekiqWorkerMethods
11
+
12
+ #
13
+ # Calculates the timeout for a Sidekiq job
14
+ #
15
+ # @param [Hash] item sidekiq job hash
16
+ #
17
+ # @return [Integer] timeout in seconds
18
+ #
19
+ def self.calculate(item)
20
+ new(item).calculate
21
+ end
22
+
23
+ # @!attribute [r] item
24
+ # @return [Hash] the Sidekiq job hash
25
+ attr_reader :item
26
+
27
+ # @param [Hash] item the Sidekiq job hash
28
+ # @option item [Integer, nil] :lock_ttl the configured lock expiration
29
+ # @option item [Integer, nil] :lock_timeout the configured lock timeout
30
+ # @option item [String] :class the class of the sidekiq worker
31
+ # @option item [Float] :at the unix time the job is scheduled at
32
+ def initialize(item)
33
+ @item = item
34
+ self.job_class = item[CLASS]
35
+ end
36
+
37
+ #
38
+ # Finds a lock timeout in either of
39
+ # default worker options, {default_lock_timeout} or provided worker_options
40
+ #
41
+ #
42
+ # @return [Integer, nil]
43
+ #
44
+ def calculate
45
+ timeout = default_job_options[LOCK_TIMEOUT]
46
+ timeout = default_lock_timeout if default_lock_timeout
47
+ timeout = job_options[LOCK_TIMEOUT] if job_options.key?(LOCK_TIMEOUT)
48
+ timeout
49
+ end
50
+
51
+ #
52
+ # The configured default_lock_timeout
53
+ # @see SidekiqUniqueJobs::Config#lock_timeout
54
+ #
55
+ #
56
+ # @return [Integer, nil]
57
+ #
58
+ def default_lock_timeout
59
+ SidekiqUniqueJobs.config.lock_timeout
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Calculates timeout and expiration
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ class LockTTL
8
+ # includes "SidekiqUniqueJobs::SidekiqWorkerMethods"
9
+ # @!parse include SidekiqUniqueJobs::SidekiqWorkerMethods
10
+ include SidekiqUniqueJobs::SidekiqWorkerMethods
11
+
12
+ #
13
+ # Computes lock ttl from job arguments, sidekiq_options.
14
+ # Falls back to {SidekiqUniqueJobs::Config#lock_ttl}
15
+ #
16
+ # @note this method takes into consideration the time
17
+ # until a job is scheduled
18
+ #
19
+ #
20
+ # @return [Integer] the number of seconds to live
21
+ #
22
+ def self.calculate(item)
23
+ new(item).calculate
24
+ end
25
+
26
+ # @!attribute [r] item
27
+ # @return [Hash] the Sidekiq job hash
28
+ attr_reader :item
29
+
30
+ # @param [Hash] item the Sidekiq job hash
31
+ # @option item [Integer, nil] :lock_ttl the configured lock expiration
32
+ # @option item [Integer, nil] :lock_timeout the configured lock timeout
33
+ # @option item [String] :class the class of the sidekiq worker
34
+ # @option item [Float] :at the unix time the job is scheduled at
35
+ def initialize(item)
36
+ @item = item
37
+ self.job_class = item[CLASS]
38
+ end
39
+
40
+ #
41
+ # Calculates the time until the job is scheduled starting from now
42
+ #
43
+ #
44
+ # @return [Integer] the number of seconds until job is scheduled
45
+ #
46
+ def time_until_scheduled
47
+ return 0 unless scheduled_at
48
+
49
+ scheduled_at.to_i - Time.now.utc.to_i
50
+ end
51
+
52
+ # The time a job is scheduled
53
+ # @return [Float] the exact unix time the job is scheduled at
54
+ def scheduled_at
55
+ @scheduled_at ||= item[AT]
56
+ end
57
+
58
+ #
59
+ # Computes lock ttl from job arguments, sidekiq_options.
60
+ # Falls back to {SidekiqUniqueJobs::Config#lock_ttl}
61
+ #
62
+ # @note this method takes into consideration the time
63
+ # until a job is scheduled
64
+ #
65
+ #
66
+ # @return [Integer] the number of seconds to live
67
+ #
68
+ def calculate
69
+ ttl = item[LOCK_TTL]
70
+ ttl ||= job_options[LOCK_TTL]
71
+ ttl ||= item[LOCK_EXPIRATION] # TODO: Deprecate at some point
72
+ ttl ||= job_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
73
+ ttl ||= SidekiqUniqueJobs.config.lock_ttl
74
+ ttl && (ttl.to_i + time_until_scheduled)
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Calculates the lock type
5
+ #
6
+ class LockType
7
+ # includes "SidekiqUniqueJobs::SidekiqWorkerMethods"
8
+ # @!parse include SidekiqUniqueJobs::SidekiqWorkerMethods
9
+ include SidekiqUniqueJobs::SidekiqWorkerMethods
10
+
11
+ #
12
+ # Computes lock type from job arguments, sidekiq_options.
13
+ #
14
+ # @return [Symbol] the lock type
15
+ # @return [NilClass] if no lock type is found.
16
+ #
17
+ def self.call(item)
18
+ new(item).call
19
+ end
20
+
21
+ # @!attribute [r] item
22
+ # @return [Hash] the Sidekiq job hash
23
+ attr_reader :item
24
+
25
+ # @param [Hash] item the Sidekiq job hash
26
+ # @option item [Symbol, nil] :lock the type of lock to use.
27
+ # @option item [String] :class the class of the sidekiq worker
28
+ def initialize(item)
29
+ @item = item
30
+ self.job_class = item[CLASS]
31
+ end
32
+
33
+ def call
34
+ item[LOCK] || job_options[LOCK] || default_job_options[LOCK]
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,390 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Lock manager class that handles all the various locks
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ class Locksmith # rubocop:disable Metrics/ClassLength
8
+ # includes "SidekiqUniqueJobs::Connection"
9
+ # @!parse include SidekiqUniqueJobs::Connection
10
+ include SidekiqUniqueJobs::Connection
11
+
12
+ # includes "SidekiqUniqueJobs::Logging"
13
+ # @!parse include SidekiqUniqueJobs::Logging
14
+ include SidekiqUniqueJobs::Logging
15
+
16
+ # includes "SidekiqUniqueJobs::Reflectable"
17
+ # @!parse include SidekiqUniqueJobs::Reflectable
18
+ include SidekiqUniqueJobs::Reflectable
19
+
20
+ # includes "SidekiqUniqueJobs::Timing"
21
+ # @!parse include SidekiqUniqueJobs::Timing
22
+ include SidekiqUniqueJobs::Timing
23
+
24
+ # includes "SidekiqUniqueJobs::Script::Caller"
25
+ # @!parse include SidekiqUniqueJobs::Script::Caller
26
+ include SidekiqUniqueJobs::Script::Caller
27
+
28
+ # includes "SidekiqUniqueJobs::JSON"
29
+ # @!parse include SidekiqUniqueJobs::JSON
30
+ include SidekiqUniqueJobs::JSON
31
+
32
+ #
33
+ # @return [Float] used to take into consideration the inaccuracy of redis timestamps
34
+ CLOCK_DRIFT_FACTOR = 0.01
35
+ NETWORK_FACTOR = 0.04
36
+
37
+ #
38
+ # @!attribute [r] key
39
+ # @return [Key] the key used for locking
40
+ attr_reader :key
41
+ #
42
+ # @!attribute [r] job_id
43
+ # @return [String] a sidekiq JID
44
+ attr_reader :job_id
45
+ #
46
+ # @!attribute [r] config
47
+ # @return [LockConfig] the configuration for this lock
48
+ attr_reader :config
49
+ #
50
+ # @!attribute [r] item
51
+ # @return [Hash] a sidekiq job hash
52
+ attr_reader :item
53
+
54
+ #
55
+ # Initialize a new Locksmith instance
56
+ #
57
+ # @param [Hash] item a Sidekiq job hash
58
+ # @option item [Integer] :lock_ttl the configured expiration
59
+ # @option item [String] :jid the sidekiq job id
60
+ # @option item [String] :unique_digest the unique digest (See: {LockDigest#lock_digest})
61
+ # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
62
+ #
63
+ def initialize(item, redis_pool = nil)
64
+ @item = item
65
+ @key = Key.new(item[LOCK_DIGEST] || item[UNIQUE_DIGEST]) # fallback until can be removed
66
+ @job_id = item[JID]
67
+ @config = LockConfig.new(item)
68
+ @redis_pool = redis_pool
69
+ end
70
+
71
+ #
72
+ # Deletes the lock unless it has a pttl set
73
+ #
74
+ #
75
+ def delete
76
+ return if config.pttl.positive?
77
+
78
+ delete!
79
+ end
80
+
81
+ #
82
+ # Deletes the lock regardless of if it has a pttl set
83
+ #
84
+ def delete!
85
+ call_script(:delete, key.to_a, argv).to_i.positive?
86
+ end
87
+
88
+ #
89
+ # Create a lock for the Sidekiq job
90
+ #
91
+ # @return [String] the Sidekiq job_id that was locked/queued
92
+ #
93
+ def lock(wait: nil)
94
+ method_name = wait ? :primed_async : :primed_sync
95
+ redis(redis_pool) do |conn|
96
+ lock!(conn, method(method_name), wait) do
97
+ return job_id
98
+ end
99
+ end
100
+ end
101
+
102
+ def execute(&block)
103
+ raise SidekiqUniqueJobs::InvalidArgument, "#execute needs a block" unless block
104
+
105
+ redis(redis_pool) do |conn|
106
+ lock!(conn, method(:primed_async), &block)
107
+ end
108
+ end
109
+
110
+ #
111
+ # Removes the lock keys from Redis if locked by the provided jid/token
112
+ #
113
+ # @return [false] unless locked?
114
+ # @return [String] Sidekiq job_id (jid) if successful
115
+ #
116
+ def unlock(conn = nil)
117
+ return false unless locked?(conn)
118
+
119
+ unlock!(conn)
120
+ end
121
+
122
+ #
123
+ # Removes the lock keys from Redis
124
+ #
125
+ # @return [false] unless locked?
126
+ # @return [String] Sidekiq job_id (jid) if successful
127
+ #
128
+ def unlock!(conn = nil)
129
+ call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
130
+ if unlocked_jid == job_id
131
+ reflect(:debug, :unlocked, item, unlocked_jid)
132
+ reflect(:unlocked, item)
133
+ end
134
+
135
+ unlocked_jid
136
+ end
137
+ end
138
+
139
+ # Checks if this instance is considered locked
140
+ #
141
+ # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
142
+ #
143
+ # @return [true, false] true when the :LOCKED hash contains the job_id
144
+ #
145
+ def locked?(conn = nil)
146
+ return taken?(conn) if conn
147
+
148
+ redis { |rcon| taken?(rcon) }
149
+ end
150
+
151
+ #
152
+ # Nicely formatted string with information about self
153
+ #
154
+ #
155
+ # @return [String]
156
+ #
157
+ def to_s
158
+ "Locksmith##{object_id}(digest=#{key} job_id=#{job_id} locked=#{locked?})"
159
+ end
160
+
161
+ #
162
+ # @see to_s
163
+ #
164
+ def inspect
165
+ to_s
166
+ end
167
+
168
+ #
169
+ # Compare this locksmith with another
170
+ #
171
+ # @param [Locksmith] other the locksmith to compare with
172
+ #
173
+ # @return [true, false]
174
+ #
175
+ def ==(other)
176
+ key == other.key && job_id == other.job_id
177
+ end
178
+
179
+ private
180
+
181
+ attr_reader :redis_pool
182
+
183
+ #
184
+ # Used to reduce some duplication from the two methods
185
+ #
186
+ # @see lock
187
+ # @see execute
188
+ #
189
+ # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
190
+ # @param [Method] primed_method reference to the method to use for getting a primed token
191
+ # @param [nil, Integer, Float] wait time to wait before timeout
192
+ #
193
+ # @yieldparam [string] job_id the sidekiq JID
194
+ # @yieldreturn [void] whatever the calling block returns
195
+ def lock!(conn, primed_method, wait = nil)
196
+ return yield if locked?(conn)
197
+
198
+ enqueue(conn) do |queued_jid|
199
+ reflect(:debug, :queued, item, queued_jid)
200
+
201
+ primed_method.call(conn, wait) do |primed_jid|
202
+ reflect(:debug, :primed, item, primed_jid)
203
+ locked_jid = call_script(:lock, key.to_a, argv, conn)
204
+
205
+ if locked_jid
206
+ reflect(:debug, :locked, item, locked_jid)
207
+ return yield
208
+ end
209
+ end
210
+ end
211
+ end
212
+
213
+ #
214
+ # Prepares all the various lock data
215
+ #
216
+ # @param [Redis] conn a redis connection
217
+ #
218
+ # @return [nil] when redis was already prepared for this lock
219
+ # @return [yield<String>] when successfully enqueued
220
+ #
221
+ def enqueue(conn)
222
+ queued_jid, elapsed = timed do
223
+ call_script(:queue, key.to_a, argv, conn)
224
+ end
225
+
226
+ return unless queued_jid
227
+ return unless [job_id, "1"].include?(queued_jid)
228
+
229
+ validity = config.pttl - elapsed - drift(config.pttl)
230
+ return unless validity >= 0 || config.pttl.zero?
231
+
232
+ write_lock_info(conn)
233
+ yield job_id
234
+ end
235
+
236
+ #
237
+ # Pops an enqueued token
238
+ # @note Used for runtime locks to avoid problems with blocking commands
239
+ # in current thread
240
+ #
241
+ # @param [Redis] conn a redis connection
242
+ #
243
+ # @return [nil] when lock was not possible
244
+ # @return [Object] whatever the block returns when lock was acquired
245
+ #
246
+ def primed_async(conn, wait = nil, &block) # rubocop:disable Metrics/MethodLength
247
+ timeout = (wait || config.timeout).to_i
248
+ timeout = 1 if timeout.zero?
249
+
250
+ brpoplpush_timeout = timeout
251
+ concurrent_timeout = add_drift(timeout)
252
+
253
+ reflect(:debug, :timeouts, item,
254
+ timeouts: {
255
+ brpoplpush_timeout: brpoplpush_timeout,
256
+ concurrent_timeout: concurrent_timeout,
257
+ })
258
+
259
+ # NOTE: When debugging, change .value to .value!
260
+ primed_jid = Concurrent::Promises
261
+ .future(conn) { |red_con| pop_queued(red_con, timeout) }
262
+ .value
263
+
264
+ handle_primed(primed_jid, &block)
265
+ end
266
+
267
+ #
268
+ # Pops an enqueued token
269
+ # @note Used for non-runtime locks
270
+ #
271
+ # @param [Redis] conn a redis connection
272
+ #
273
+ # @return [nil] when lock was not possible
274
+ # @return [Object] whatever the block returns when lock was acquired
275
+ #
276
+ def primed_sync(conn, wait = nil, &block)
277
+ primed_jid = pop_queued(conn, wait)
278
+ handle_primed(primed_jid, &block)
279
+ end
280
+
281
+ def handle_primed(primed_jid)
282
+ return yield job_id if [job_id, "1"].include?(primed_jid)
283
+
284
+ reflect(:timeout, item) unless config.wait_for_lock?
285
+ end
286
+
287
+ #
288
+ # Does the actual popping of the enqueued token
289
+ #
290
+ # @param [Redis] conn a redis connection
291
+ #
292
+ # @return [String] a previously enqueued token (now taken off the queue)
293
+ #
294
+ def pop_queued(conn, wait = 1)
295
+ wait ||= config.timeout if config.wait_for_lock?
296
+
297
+ if wait.nil?
298
+ rpoplpush(conn)
299
+ else
300
+ brpoplpush(conn, wait)
301
+ end
302
+ end
303
+
304
+ #
305
+ # @api private
306
+ #
307
+ def brpoplpush(conn, wait)
308
+ # passing timeout 0 to brpoplpush causes it to block indefinitely
309
+ raise InvalidArgument, "wait must be an integer" unless wait.is_a?(Integer)
310
+
311
+ conn.blmove(key.queued, key.primed, "RIGHT", "LEFT", wait)
312
+ end
313
+
314
+ #
315
+ # @api private
316
+ #
317
+ def rpoplpush(conn)
318
+ conn.lmove(key.queued, key.primed, "RIGHT", "LEFT")
319
+ end
320
+
321
+ #
322
+ # Writes lock information to redis.
323
+ # The lock information contains information about worker, queue, limit etc.
324
+ #
325
+ #
326
+ # @return [void]
327
+ #
328
+ def write_lock_info(conn)
329
+ return unless config.lock_info?
330
+
331
+ conn.set(key.info, lock_info)
332
+ end
333
+
334
+ #
335
+ # Used to combat redis imprecision with ttl/pttl
336
+ #
337
+ # @param [Integer] val the value to compute drift for
338
+ #
339
+ # @return [Integer] a computed drift value
340
+ #
341
+ def drift(val)
342
+ # Add 2 milliseconds to the drift to account for Redis expires
343
+ # precision, which is 1 millisecond, plus 1 millisecond min drift
344
+ # for small TTLs.
345
+ (val + 2).to_f * CLOCK_DRIFT_FACTOR
346
+ end
347
+
348
+ def add_drift(val)
349
+ val = val.to_f
350
+ val + drift(val)
351
+ end
352
+
353
+ #
354
+ # Checks if the lock has been taken
355
+ #
356
+ # @param [Redis] conn a redis connection
357
+ #
358
+ # @return [true, false]
359
+ #
360
+ def taken?(conn)
361
+ conn.hexists(key.locked, job_id) != 0
362
+ end
363
+
364
+ def argv
365
+ [job_id, config.pttl, config.type, config.limit, lock_score]
366
+ end
367
+
368
+ def lock_score
369
+ item[AT].to_s
370
+ end
371
+
372
+ def lock_info
373
+ @lock_info ||= dump_json(
374
+ WORKER => item[CLASS],
375
+ QUEUE => item[QUEUE],
376
+ LIMIT => item[LOCK_LIMIT],
377
+ TIMEOUT => item[LOCK_TIMEOUT],
378
+ TTL => item[LOCK_TTL],
379
+ TYPE => config.type,
380
+ LOCK_ARGS => item[LOCK_ARGS],
381
+ TIME => now_f,
382
+ AT => item[AT],
383
+ )
384
+ end
385
+
386
+ def redis_version
387
+ @redis_version ||= SidekiqUniqueJobs.config.redis_version
388
+ end
389
+ end
390
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Provides the sidekiq middleware that makes the gem work
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ module Logging
10
+ #
11
+ # Context aware logging for Sidekiq Middlewares
12
+ #
13
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
14
+ #
15
+ module Middleware
16
+ include Logging
17
+
18
+ def self.included(base)
19
+ base.class_eval do
20
+ extend Logging::Middleware
21
+ end
22
+ end
23
+
24
+ #
25
+ # Provides a logging context for Sidekiq Middlewares
26
+ #
27
+ #
28
+ # @return [Hash] when logger responds to `:with_context`
29
+ # @return [String] when logger does not responds to `:with_context`
30
+ #
31
+ def logging_context
32
+ middleware = is_a?(SidekiqUniqueJobs::Middleware::Client) ? :client : :server
33
+ digest = item[LOCK_DIGEST]
34
+ lock_type = item[LOCK]
35
+
36
+ if logger_context_hash?
37
+ { "uniquejobs" => middleware, lock_type => digest }
38
+ else
39
+ "uniquejobs-#{middleware} #{"DIG-#{digest}" if digest}"
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end