sidekiq-unique-jobs 6.0.24 → 7.0.4

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +707 -25
  3. data/README.md +516 -105
  4. data/lib/sidekiq_unique_jobs.rb +48 -7
  5. data/lib/sidekiq_unique_jobs/batch_delete.rb +123 -0
  6. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  7. data/lib/sidekiq_unique_jobs/cli.rb +34 -31
  8. data/lib/sidekiq_unique_jobs/config.rb +263 -0
  9. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  10. data/lib/sidekiq_unique_jobs/constants.rb +46 -24
  11. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  12. data/lib/sidekiq_unique_jobs/digests.rb +71 -100
  13. data/lib/sidekiq_unique_jobs/exceptions.rb +78 -12
  14. data/lib/sidekiq_unique_jobs/job.rb +41 -12
  15. data/lib/sidekiq_unique_jobs/json.rb +40 -0
  16. data/lib/sidekiq_unique_jobs/key.rb +93 -0
  17. data/lib/sidekiq_unique_jobs/lock.rb +325 -0
  18. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +66 -50
  19. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  20. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  21. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +7 -10
  22. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +6 -6
  23. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +1 -1
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +4 -21
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +13 -9
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -3
  28. data/lib/sidekiq_unique_jobs/lock_args.rb +123 -0
  29. data/lib/sidekiq_unique_jobs/lock_config.rb +122 -0
  30. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  31. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  32. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  33. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  34. data/lib/sidekiq_unique_jobs/locksmith.rb +261 -101
  35. data/lib/sidekiq_unique_jobs/logging.rb +149 -23
  36. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  37. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  38. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  39. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  40. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  41. data/lib/sidekiq_unique_jobs/lua/lock.lua +93 -0
  42. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  43. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  44. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +94 -0
  45. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  46. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  47. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  54. data/lib/sidekiq_unique_jobs/lua/unlock.lua +95 -0
  55. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  56. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  57. data/lib/sidekiq_unique_jobs/middleware.rb +29 -31
  58. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  59. data/lib/sidekiq_unique_jobs/middleware/server.rb +27 -0
  60. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  61. data/lib/sidekiq_unique_jobs/on_conflict.rb +23 -10
  62. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  63. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  64. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  65. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +61 -15
  66. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  67. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +12 -5
  68. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +25 -6
  69. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +41 -27
  70. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  71. data/lib/sidekiq_unique_jobs/orphans/manager.rb +212 -0
  72. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  73. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  74. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  75. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +201 -0
  76. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  77. data/lib/sidekiq_unique_jobs/redis/entity.rb +106 -0
  78. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  79. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  80. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  81. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  82. data/lib/sidekiq_unique_jobs/redis/string.rb +49 -0
  83. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  84. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  85. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  86. data/lib/sidekiq_unique_jobs/script/caller.rb +125 -0
  87. data/lib/sidekiq_unique_jobs/server.rb +48 -0
  88. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +92 -65
  89. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +185 -34
  90. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +11 -5
  91. data/lib/sidekiq_unique_jobs/testing.rb +62 -21
  92. data/lib/sidekiq_unique_jobs/timer_task.rb +78 -0
  93. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  94. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  95. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  96. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  97. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  98. data/lib/sidekiq_unique_jobs/version_check.rb +23 -4
  99. data/lib/sidekiq_unique_jobs/web.rb +50 -27
  100. data/lib/sidekiq_unique_jobs/web/helpers.rb +125 -10
  101. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  102. data/lib/sidekiq_unique_jobs/web/views/lock.erb +108 -0
  103. data/lib/sidekiq_unique_jobs/web/views/locks.erb +52 -0
  104. data/lib/tasks/changelog.rake +5 -5
  105. metadata +117 -177
  106. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  107. data/lib/sidekiq_unique_jobs/scripts.rb +0 -118
  108. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  109. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  110. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  111. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -150
  112. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  113. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  114. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  115. data/redis/acquire_lock.lua +0 -21
  116. data/redis/convert_legacy_lock.lua +0 -13
  117. data/redis/delete.lua +0 -14
  118. data/redis/delete_by_digest.lua +0 -23
  119. data/redis/delete_job_by_digest.lua +0 -60
  120. data/redis/lock.lua +0 -62
  121. data/redis/release_stale_locks.lua +0 -90
  122. data/redis/unlock.lua +0 -35
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # Augments and enhances redis with object oriented methods
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ module Redis
10
+ end
11
+ end
@@ -0,0 +1,106 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Entity functions as a base class for redis types
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Entity
11
+ # includes "SidekiqUniqueJobs::Logging"
12
+ # @!parse include SidekiqUniqueJobs::Logging
13
+ include SidekiqUniqueJobs::Logging
14
+
15
+ # includes "SidekiqUniqueJobs::Script::Caller"
16
+ # @!parse include SidekiqUniqueJobs::Script::Caller
17
+ include SidekiqUniqueJobs::Script::Caller
18
+
19
+ # includes "SidekiqUniqueJobs::JSON"
20
+ # @!parse include SidekiqUniqueJobs::JSON
21
+ include SidekiqUniqueJobs::JSON
22
+
23
+ # includes "SidekiqUniqueJobs::Timing"
24
+ # @!parse include SidekiqUniqueJobs::Timing
25
+ include SidekiqUniqueJobs::Timing
26
+
27
+ #
28
+ # @!attribute [r] key
29
+ # @return [String] the redis key for this entity
30
+ attr_reader :key
31
+
32
+ #
33
+ # Initialize a new Entity
34
+ #
35
+ # @param [String] key the redis key for this entity
36
+ #
37
+ def initialize(key)
38
+ @key = key
39
+ end
40
+
41
+ #
42
+ # Checks if the key for this entity exists in redis
43
+ #
44
+ #
45
+ # @return [true] when exists
46
+ # @return [false] when not exists
47
+ #
48
+ def exist?
49
+ redis do |conn|
50
+ value = conn.exists(key)
51
+
52
+ return value if boolean?(value)
53
+
54
+ value.to_i.positive?
55
+ end
56
+ end
57
+
58
+ #
59
+ # The number of microseconds until the key expires
60
+ #
61
+ #
62
+ # @return [Integer] expiration in milliseconds
63
+ #
64
+ def pttl
65
+ redis { |conn| conn.pttl(key) }
66
+ end
67
+
68
+ #
69
+ # The number of seconds until the key expires
70
+ #
71
+ #
72
+ # @return [Integer] expiration in seconds
73
+ #
74
+ def ttl
75
+ redis { |conn| conn.ttl(key) }
76
+ end
77
+
78
+ #
79
+ # Check if the entity has expiration
80
+ #
81
+ #
82
+ # @return [true] when entity is set to exire
83
+ # @return [false] when entity isn't expiring
84
+ #
85
+ def expires?
86
+ pttl.positive? || ttl.positive?
87
+ end
88
+
89
+ #
90
+ # Returns the number of entries in this entity
91
+ #
92
+ #
93
+ # @return [Integer] 0
94
+ #
95
+ def count
96
+ 0
97
+ end
98
+
99
+ private
100
+
101
+ def boolean?(value)
102
+ [TrueClass, FalseClass].any? { |klazz| value.is_a?(klazz) }
103
+ end
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Hash provides convenient access to redis hashes
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Hash < Entity
11
+ #
12
+ # Return entries for this hash
13
+ #
14
+ # @param [true,false] with_values false return hash
15
+ #
16
+ # @return [Array<Object>] when given with_values: false
17
+ # @return [Hash<String, String>] when given with_values: true
18
+ #
19
+ def entries(with_values: false)
20
+ if with_values
21
+ redis { |conn| conn.hgetall(key) }
22
+ else
23
+ redis { |conn| conn.hkeys(key) }
24
+ end
25
+ end
26
+
27
+ #
28
+ # Removes the key from redis
29
+ #
30
+ def del(*fields)
31
+ redis { |conn| conn.hdel(key, *fields) }
32
+ end
33
+
34
+ #
35
+ # Get a members value
36
+ #
37
+ # @param [String] member the member who's value to get
38
+ #
39
+ # @return [Object] whatever is stored on this hash member
40
+ #
41
+ def [](member)
42
+ redis { |conn| conn.hget(key, member) }
43
+ end
44
+
45
+ #
46
+ # Returns the count for this hash
47
+ #
48
+ #
49
+ # @return [Integer] the length of this hash
50
+ #
51
+ def count
52
+ redis { |conn| conn.hlen(key) }
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class List provides convenient access to redis hashes
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class List < Entity
11
+ #
12
+ # Entries in this list
13
+ #
14
+ #
15
+ # @return [Array<Object>] the elements in this list
16
+ #
17
+ def entries
18
+ redis { |conn| conn.lrange(key, 0, -1) }
19
+ end
20
+
21
+ #
22
+ # The number of entries in this list
23
+ #
24
+ #
25
+ # @return [Integer] the total number of entries
26
+ #
27
+ def count
28
+ redis { |conn| conn.llen(key) }
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class Set provides convenient access to redis sets
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class Set < Entity
11
+ #
12
+ # Return entries for this set
13
+ #
14
+ #
15
+ # @return [Array<String>]
16
+ #
17
+ def entries
18
+ redis { |conn| conn.smembers(key) }
19
+ end
20
+
21
+ #
22
+ # Returns the count for this sorted set
23
+ #
24
+ #
25
+ # @return [Integer] the number of entries
26
+ #
27
+ def count
28
+ redis { |conn| conn.scard(key) }
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class SortedSet provides convenient access to redis sorted sets
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class SortedSet < Entity
11
+ #
12
+ # Return entries for this sorted set
13
+ #
14
+ # @param [true,false] with_scores true return
15
+ #
16
+ # @return [Array<Object>] when given with_scores: false
17
+ # @return [Hash] when given with_scores: true
18
+ #
19
+ def entries(with_scores: true)
20
+ entrys = redis { |conn| conn.zrange(key, 0, -1, with_scores: with_scores) }
21
+ return entrys unless with_scores
22
+
23
+ entrys.each_with_object({}) { |pair, hash| hash[pair[0]] = pair[1] }
24
+ end
25
+
26
+ #
27
+ # Adds a value to the sorted set
28
+ #
29
+ # @param [Array<Float, String>, String] the values to add
30
+ #
31
+ # @return [Boolean, Integer] <description>
32
+ #
33
+ def add(values)
34
+ redis do |conn|
35
+ if values.is_a?(Array)
36
+ conn.zadd(key, values)
37
+ else
38
+ conn.zadd(key, now_f, values)
39
+ end
40
+ end
41
+ end
42
+
43
+ #
44
+ # Return the zrak of the member
45
+ #
46
+ # @param [String] member the member to pull rank on
47
+ #
48
+ # @return [Integer]
49
+ #
50
+ def rank(member)
51
+ redis { |conn| conn.zrank(key, member) }
52
+ end
53
+
54
+ #
55
+ # Return score for a member
56
+ #
57
+ # @param [String] member the member for which score
58
+ #
59
+ # @return [Integer, Float]
60
+ #
61
+ def score(member)
62
+ redis { |conn| conn.zscore(key, member) }
63
+ end
64
+
65
+ #
66
+ # Clears the sorted set from all entries
67
+ #
68
+ #
69
+ # @return [Integer] number of entries removed
70
+ #
71
+ def clear
72
+ redis { |conn| conn.zremrangebyrank(key, 0, count) }
73
+ end
74
+
75
+ #
76
+ # Returns the count for this sorted set
77
+ #
78
+ #
79
+ # @return [Integer] the number of entries
80
+ #
81
+ def count
82
+ redis { |conn| conn.zcard(key) }
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Redis
5
+ #
6
+ # Class String provides convenient access to redis strings
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ class String < Entity
11
+ #
12
+ # Returns the value of the key
13
+ #
14
+ #
15
+ # @return [String]
16
+ #
17
+ def value
18
+ redis { |conn| conn.get(key) }
19
+ end
20
+
21
+ #
22
+ # Sets the value of the key to given object
23
+ #
24
+ # @param [String] obj the object to update the key with
25
+ #
26
+ # @return [true, false]
27
+ #
28
+ def set(obj)
29
+ redis { |conn| conn.set(key, obj) }
30
+ end
31
+
32
+ #
33
+ # Removes the key from redis
34
+ #
35
+ def del(*)
36
+ redis { |conn| conn.del(key) }
37
+ end
38
+
39
+ #
40
+ # Used only for compatibility with other keys
41
+ #
42
+ # @return [1] when key exists
43
+ # @return [0] when key does not exists
44
+ def count
45
+ exist? ? 1 : 0
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ #
5
+ # We all know what the hell RSpec is no?
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ #
9
+ module RSpec
10
+ #
11
+ # Module Matchers provides RSpec matcher for your workers
12
+ #
13
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
14
+ #
15
+ module Matchers
16
+ end
17
+ end
18
+ end
19
+
20
+ if defined?(RSpec)
21
+ require_relative "matchers/have_valid_sidekiq_options"
22
+
23
+ RSpec.configure do |config|
24
+ config.include SidekiqUniqueJobs::RSpec::Matchers
25
+ end
26
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module RSpec
5
+ #
6
+ # Module Matchers provides RSpec matcher for your workers
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ #
10
+ module Matchers
11
+ #
12
+ # Class HaveValidSidekiqOptions validates the unique/lock configuration for a worker.
13
+ #
14
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
15
+ #
16
+ class HaveValidSidekiqOptions
17
+ attr_reader :worker, :lock_config, :sidekiq_options
18
+
19
+ def matches?(worker)
20
+ @worker = worker
21
+ @sidekiq_options = worker.get_sidekiq_options
22
+ @lock_config = SidekiqUniqueJobs.validate_worker(sidekiq_options)
23
+ lock_config.valid?
24
+ end
25
+
26
+ # :nodoc:
27
+ def failure_message
28
+ <<~FAILURE_MESSAGE
29
+ Expected #{worker} to have valid sidekiq options but found the following problems:
30
+ #{lock_config.errors_as_string}
31
+ FAILURE_MESSAGE
32
+ end
33
+
34
+ # :nodoc:
35
+ def description
36
+ "have valid sidekiq options"
37
+ end
38
+ end
39
+
40
+ #
41
+ # RSpec matcher method for validating that a sidekiq worker has valid unique/lock configuration
42
+ #
43
+ #
44
+ # @return [HaveValidSidekiqOptions] an RSpec matcher
45
+ #
46
+ def have_valid_sidekiq_options(*args) # rubocop:disable Naming/PredicateName
47
+ HaveValidSidekiqOptions.new(*args)
48
+ end
49
+ end
50
+ end
51
+ end