sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -1,70 +0,0 @@
1
- require 'spec_helper'
2
- require 'sidekiq/api'
3
- require 'sidekiq/worker'
4
- require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
5
- require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
6
- require 'sidekiq_unique_jobs/sidekiq_unique_ext'
7
-
8
- class JustAWorker
9
- include Sidekiq::Worker
10
-
11
- sidekiq_options unique: true, queue: 'testqueue'
12
-
13
- def perform
14
- end
15
- end
16
-
17
- describe Sidekiq::Job::UniqueExtension do
18
- before do
19
- Sidekiq.redis = REDIS
20
- Sidekiq.redis(&:flushdb)
21
- end
22
-
23
- it 'deletes uniqueness lock on delete' do
24
- params = { foo: 'bar' }
25
- payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('JustAWorker', 'testqueue', [params])
26
- jid = JustAWorker.perform_async(foo: 'bar')
27
- queue = Sidekiq::Queue.new('testqueue')
28
- job = queue.find_job(jid)
29
- job.delete
30
- Sidekiq.redis do |c|
31
- expect(c.exists(payload_hash)).to be_falsy
32
- end
33
- end
34
- end
35
-
36
- describe Sidekiq::Queue::UniqueExtension do
37
- before do
38
- Sidekiq.redis = REDIS
39
- Sidekiq.redis(&:flushdb)
40
- end
41
-
42
- it 'deletes uniqueness locks on clear' do
43
- params = { foo: 'bar' }
44
- payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('JustAWorker', 'testqueue', [params])
45
- JustAWorker.perform_async(foo: 'bar')
46
- queue = Sidekiq::Queue.new('testqueue')
47
- queue.clear
48
- Sidekiq.redis do |c|
49
- expect(c.exists(payload_hash)).to be_falsy
50
- end
51
- end
52
- end
53
-
54
- describe Sidekiq::JobSet::UniqueExtension, sidekiq_ver: 3 do
55
- before do
56
- Sidekiq.redis = REDIS
57
- Sidekiq.redis(&:flushdb)
58
- end
59
-
60
- it 'deletes uniqueness locks on clear' do
61
- params = { foo: 'bar' }
62
- payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('JustAWorker', 'testqueue', [params])
63
- JustAWorker.perform_in(60 * 60 * 3, foo: 'bar')
64
- set = Sidekiq::JobSet.new('schedule')
65
- set.clear
66
- Sidekiq.redis do |c|
67
- expect(c.exists(payload_hash)).to be_falsy
68
- end
69
- end
70
- end
@@ -1,64 +0,0 @@
1
- require 'spec_helper'
2
- require 'sidekiq/api'
3
- require 'sidekiq/worker'
4
- require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
5
-
6
- describe 'Unlock order' do
7
- QUEUE = 'unlock_ordering'
8
-
9
- class BeforeYieldOrderingWorker
10
- include Sidekiq::Worker
11
-
12
- sidekiq_options unique: true, unique_unlock_order: :before_yield, queue: QUEUE
13
-
14
- def perform
15
- end
16
- end
17
-
18
- class AfterYieldOrderingWorker
19
- include Sidekiq::Worker
20
-
21
- sidekiq_options unique: true, unique_unlock_order: :after_yield, queue: QUEUE
22
-
23
- def perform
24
- end
25
- end
26
-
27
- describe 'with real redis' do
28
- before do
29
- Sidekiq.redis = REDIS
30
- Sidekiq.redis(&:flushdb)
31
- @middleware = SidekiqUniqueJobs::Middleware::Server::UniqueJobs.new
32
- end
33
-
34
- describe ':before_yield' do
35
- it 'removes the lock before yielding to the worker' do
36
- jid = BeforeYieldOrderingWorker.perform_async
37
- item = Sidekiq::Queue.new(QUEUE).find_job(jid).item
38
-
39
- result = @middleware.call(BeforeYieldOrderingWorker.new, item, QUEUE) do
40
- Sidekiq.redis do |c|
41
- c.get(SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args']))
42
- end
43
- end
44
-
45
- expect(result).to eq nil
46
- end
47
- end
48
-
49
- describe ':after_yield' do
50
- it 'removes the lock after yielding to the worker' do
51
- jid = AfterYieldOrderingWorker.perform_async
52
- item = Sidekiq::Queue.new(QUEUE).find_job(jid).item
53
-
54
- result = @middleware.call(AfterYieldOrderingWorker.new, item, QUEUE) do
55
- Sidekiq.redis do |c|
56
- c.get(SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args']))
57
- end
58
- end
59
-
60
- expect(result).to eq '1'
61
- end
62
- end
63
- end
64
- end
data/spec/spec_helper.rb DELETED
@@ -1,37 +0,0 @@
1
- begin
2
- require 'pry'
3
- rescue LoadError
4
- puts 'Pry unavailable'
5
- end
6
-
7
- require 'rspec'
8
-
9
- require 'celluloid/test'
10
- require 'sidekiq'
11
- require 'sidekiq/util'
12
- require 'sidekiq-unique-jobs'
13
- Sidekiq.logger.level = Logger::ERROR
14
-
15
- require 'rspec-sidekiq'
16
-
17
- Sidekiq::Testing.disable!
18
-
19
- require 'sidekiq/redis_connection'
20
- redis_url = ENV['REDIS_URL'] || 'redis://localhost/15'
21
- REDIS = Sidekiq::RedisConnection.create(url: redis_url, namespace: 'testy')
22
-
23
- Dir[File.join(File.dirname(__FILE__), 'support', '**', '*.rb')].each { |f| require f }
24
- RSpec.configure do |_config|
25
- # config.treat_symbols_as_metadata_keys_with_true_values = true
26
- end
27
-
28
- RSpec::Sidekiq.configure do |config|
29
- # Clears all job queues before each example
30
- config.clear_all_enqueued_jobs = true # default => true
31
-
32
- # Whether to use terminal colours when outputting messages
33
- config.enable_terminal_colours = true # default => true
34
-
35
- # Warn when jobs are not enqueued to Redis but to a job array
36
- config.warn_when_jobs_not_processed_by_sidekiq = true # default => true
37
- end
@@ -1,13 +0,0 @@
1
- class MyWorker
2
- include Sidekiq::Worker
3
- sidekiq_options queue: :working, retry: 1, backtrace: 10
4
- sidekiq_options unique: false
5
-
6
- sidekiq_retries_exhausted do |msg|
7
- Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
8
- end
9
-
10
- def perform(param)
11
- puts param
12
- end
13
- end
@@ -1,17 +0,0 @@
1
- RSpec.configure do |config|
2
- config.before(:each) do |example|
3
- if (sidekiq = example.metadata[:sidekiq])
4
- sidekiq = :fake if sidekiq == true
5
- Sidekiq::Testing.send("#{sidekiq}!")
6
- end
7
-
8
- sidekiq_ver = example.metadata[:sidekiq_ver]
9
- if sidekiq_ver && Sidekiq::VERSION[0].to_s != sidekiq_ver.to_s
10
- skip("Not relevant for version #{sidekiq_ver}")
11
- end
12
- end
13
-
14
- config.after(:each) do |example|
15
- Sidekiq::Testing.disable! unless example.metadata[:sidekiq].nil?
16
- end
17
- end
@@ -1,13 +0,0 @@
1
- class UniqueWorker
2
- include Sidekiq::Worker
3
- sidekiq_options queue: :working, retry: 1, backtrace: 10
4
- sidekiq_options unique: true
5
-
6
- sidekiq_retries_exhausted do |msg|
7
- Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
8
- end
9
-
10
- def perform(param)
11
- puts param
12
- end
13
- end