sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -1,76 +0,0 @@
1
- require 'digest'
2
- require 'sidekiq_unique_jobs/connectors'
3
-
4
- module SidekiqUniqueJobs
5
- module Middleware
6
- module Client
7
- module Strategies
8
- class Unique
9
- def self.elegible?
10
- true
11
- end
12
-
13
- def self.review(worker_class, item, queue, redis_pool = nil)
14
- new(worker_class, item, queue, redis_pool).review { yield }
15
- end
16
-
17
- def initialize(worker_class, item, queue, redis_pool = nil)
18
- @worker_class = SidekiqUniqueJobs.worker_class_constantize(worker_class)
19
- @item = item
20
- @queue = queue
21
- @redis_pool = redis_pool
22
- end
23
-
24
- def review
25
- item['unique_hash'] = payload_hash
26
- return unless unique_for_connection?
27
- yield
28
- end
29
-
30
- private
31
-
32
- attr_reader :item, :worker_class, :redis_pool, :queue
33
-
34
- # rubocop:disable MethodLength
35
- def unique_for_connection?
36
- unique = false
37
- connection do |conn|
38
- conn.watch(payload_hash)
39
-
40
- if conn.get(payload_hash).to_i == 1 ||
41
- (conn.get(payload_hash).to_i == 2 && item['at'])
42
- # if the job is already queued, or is already scheduled and
43
- # we're trying to schedule again, abort
44
- conn.unwatch
45
- else
46
- # if the job was previously scheduled and is now being queued,
47
- # or we've never seen it before
48
- expires_at = unique_job_expiration || SidekiqUniqueJobs.config.default_expiration
49
- expires_at = ((Time.at(item['at']) - Time.now.utc) + expires_at).to_i if item['at']
50
-
51
- unique = conn.multi do
52
- # set value of 2 for scheduled jobs, 1 for queued jobs.
53
- conn.setex(payload_hash, expires_at, item['at'] ? 2 : 1)
54
- end
55
- end
56
- end
57
- unique
58
- end
59
- # rubocop:enable MethodLength
60
-
61
- def connection(&block)
62
- SidekiqUniqueJobs::Connectors.connection(redis_pool, &block)
63
- end
64
-
65
- def payload_hash
66
- SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args'])
67
- end
68
-
69
- def unique_job_expiration
70
- worker_class.get_sidekiq_options['unique_job_expiration']
71
- end
72
- end
73
- end
74
- end
75
- end
76
- end
@@ -1,39 +0,0 @@
1
- require 'sidekiq_unique_jobs/middleware/client/strategies/unique'
2
- require 'sidekiq_unique_jobs/middleware/client/strategies/testing_inline'
3
-
4
- module SidekiqUniqueJobs
5
- module Middleware
6
- module Client
7
- class UniqueJobs
8
- STRATEGIES = [
9
- Strategies::TestingInline,
10
- Strategies::Unique
11
- ]
12
-
13
- attr_reader :item, :worker_class, :redis_pool
14
-
15
- def call(worker_class, item, queue, redis_pool = nil)
16
- @worker_class = SidekiqUniqueJobs.worker_class_constantize(worker_class)
17
- @item = item
18
- @redis_pool = redis_pool
19
-
20
- if unique_enabled?
21
- strategy.review(worker_class, item, queue, redis_pool) { yield }
22
- else
23
- yield
24
- end
25
- end
26
-
27
- private
28
-
29
- def unique_enabled?
30
- worker_class.get_sidekiq_options['unique'] || item['unique']
31
- end
32
-
33
- def strategy
34
- STRATEGIES.detect(&:elegible?)
35
- end
36
- end
37
- end
38
- end
39
- end
@@ -1,69 +0,0 @@
1
- require 'digest'
2
- require 'sidekiq_unique_jobs/connectors'
3
-
4
- module SidekiqUniqueJobs
5
- module Middleware
6
- module Server
7
- class UniqueJobs
8
- attr_reader :unlock_order, :redis_pool
9
-
10
- def call(worker, item, _queue, redis_pool = nil)
11
- @redis_pool = redis_pool
12
-
13
- decide_unlock_order(worker.class)
14
- lock_key = payload_hash(item)
15
- unlocked = before_yield? ? unlock(lock_key).inspect : 0
16
-
17
- yield
18
- ensure
19
- if after_yield? || !defined? unlocked || unlocked != 1
20
- unlock(lock_key)
21
- end
22
- end
23
-
24
- def decide_unlock_order(klass)
25
- @unlock_order = if unlock_order_configured?(klass)
26
- klass.get_sidekiq_options['unique_unlock_order']
27
- else
28
- default_unlock_order
29
- end
30
- end
31
-
32
- def unlock_order_configured?(klass)
33
- klass.respond_to?(:get_sidekiq_options) &&
34
- !klass.get_sidekiq_options['unique_unlock_order'].nil?
35
- end
36
-
37
- def default_unlock_order
38
- SidekiqUniqueJobs.config.default_unlock_order
39
- end
40
-
41
- def before_yield?
42
- unlock_order == :before_yield
43
- end
44
-
45
- def after_yield?
46
- unlock_order == :after_yield
47
- end
48
-
49
- protected
50
-
51
- def payload_hash(item)
52
- SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args'])
53
- end
54
-
55
- def unlock(payload_hash)
56
- connection { |c| c.del(payload_hash) }
57
- end
58
-
59
- def logger
60
- Sidekiq.logger
61
- end
62
-
63
- def connection(&block)
64
- SidekiqUniqueJobs::Connectors.connection(redis_pool, &block)
65
- end
66
- end
67
- end
68
- end
69
- end
@@ -1,42 +0,0 @@
1
- module SidekiqUniqueJobs
2
- class PayloadHelper
3
- def self.config
4
- SidekiqUniqueJobs.config
5
- end
6
-
7
- def self.get_payload(klass, queue, *args)
8
- unique_on_all_queues = false
9
- if config.unique_args_enabled
10
- worker_class = klass.constantize
11
- args = yield_unique_args(worker_class, *args)
12
- unique_on_all_queues =
13
- worker_class.get_sidekiq_options['unique_on_all_queues']
14
- end
15
- md5_arguments = { class: klass, args: args }
16
- md5_arguments[:queue] = queue unless unique_on_all_queues
17
- "#{config.unique_prefix}:" \
18
- "#{Digest::MD5.hexdigest(Sidekiq.dump_json(md5_arguments))}"
19
- end
20
-
21
- def self.yield_unique_args(worker_class, args)
22
- unique_args = worker_class.get_sidekiq_options['unique_args']
23
- filtered_args(worker_class, unique_args, args)
24
- rescue NameError
25
- # fallback to not filtering args when class can't be instantiated
26
- args
27
- end
28
-
29
- def self.filtered_args(worker_class, unique_args, args)
30
- case unique_args
31
- when Proc
32
- unique_args.call(args)
33
- when Symbol
34
- if worker_class.respond_to?(unique_args)
35
- worker_class.send(unique_args, *args)
36
- end
37
- else
38
- args
39
- end
40
- end
41
- end
42
- end
@@ -1,27 +0,0 @@
1
- # -*- encoding: utf-8 -*-
2
- require File.expand_path('../lib/sidekiq_unique_jobs/version', __FILE__)
3
-
4
- Gem::Specification.new do |gem|
5
- gem.authors = ['Mikael Henriksson']
6
- gem.email = ['mikael@zoolutions.se']
7
- gem.description = gem.summary = 'The unique jobs that were removed from sidekiq'
8
- gem.homepage = 'https://github.com/mhenrixon/sidekiq-unique-jobs'
9
- gem.license = 'LGPL-3.0'
10
-
11
- # gem.executables = ['']
12
- gem.files = `git ls-files`.split("\n")
13
- gem.test_files = `git ls-files -- test/*`.split("\n")
14
- gem.name = 'sidekiq-unique-jobs'
15
- gem.require_paths = ['lib']
16
- gem.post_install_message = 'If you are relying on `mock_redis` you will now have to add' \
17
- 'gem "mock_redis" to your desired bundler group.'
18
- gem.version = SidekiqUniqueJobs::VERSION
19
- gem.add_dependency 'sidekiq', '>= 2.6'
20
- gem.add_development_dependency 'mock_redis'
21
- gem.add_development_dependency 'rspec', '~> 3.1.0'
22
- gem.add_development_dependency 'rake'
23
- gem.add_development_dependency 'rspec-sidekiq'
24
- gem.add_development_dependency 'activesupport', '>= 3'
25
- gem.add_development_dependency 'rubocop'
26
- gem.add_development_dependency 'simplecov'
27
- end
@@ -1,173 +0,0 @@
1
- require 'spec_helper'
2
- require 'celluloid'
3
- require 'sidekiq/worker'
4
- require 'sidekiq-unique-jobs'
5
- require 'sidekiq/scheduled'
6
- require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
7
-
8
- describe 'Client' do
9
- describe 'with real redis' do
10
- before do
11
- Sidekiq.redis = REDIS
12
- Sidekiq.redis(&:flushdb)
13
- QueueWorker.sidekiq_options unique: nil, unique_job_expiration: nil
14
- end
15
-
16
- class QueueWorker
17
- include Sidekiq::Worker
18
- sidekiq_options queue: 'customqueue'
19
- def perform(_x)
20
- end
21
- end
22
-
23
- class PlainClass
24
- def run(_x)
25
- end
26
- end
27
-
28
- it 'does not push duplicate messages when configured for unique only' do
29
- QueueWorker.sidekiq_options unique: true
30
- 10.times { Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2]) }
31
- result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
32
- expect(result).to eq 1
33
- end
34
-
35
- it 'does push duplicate messages to different queues' do
36
- QueueWorker.sidekiq_options unique: true
37
- Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2])
38
- Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue2', 'args' => [1, 2])
39
- q1_length = Sidekiq.redis { |c| c.llen('queue:customqueue') }
40
- q2_length = Sidekiq.redis { |c| c.llen('queue:customqueue2') }
41
- expect(q1_length).to eq 1
42
- expect(q2_length).to eq 1
43
- end
44
-
45
- it 'does not queue duplicates when when calling delay' do
46
- 10.times { PlainClass.delay(unique: true, queue: 'customqueue').run(1) }
47
- result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
48
- expect(result).to eq 1
49
- end
50
-
51
- it 'does not schedule duplicates when calling perform_in' do
52
- QueueWorker.sidekiq_options unique: true
53
- 10.times { QueueWorker.perform_in(60, [1, 2]) }
54
- result = Sidekiq.redis { |c| c.zcount('schedule', -1, Time.now.to_f + 2 * 60) }
55
- expect(result).to eq 1
56
- end
57
-
58
- it 'enqueues previously scheduled job' do
59
- QueueWorker.sidekiq_options unique: true
60
- QueueWorker.perform_in(60 * 60, 1, 2)
61
-
62
- # time passes and the job is pulled off the schedule:
63
- Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2])
64
-
65
- result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
66
- expect(result).to eq 1
67
- end
68
-
69
- it 'sets an expiration when provided by sidekiq options' do
70
- one_hour_expiration = 60 * 60
71
- QueueWorker.sidekiq_options unique: true, unique_job_expiration: one_hour_expiration
72
- Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2])
73
-
74
- payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('QueueWorker', 'customqueue', [1, 2])
75
- actual_expires_at = Sidekiq.redis { |c| c.ttl(payload_hash) }
76
-
77
- Sidekiq.redis { |c| c.llen('queue:customqueue') }
78
- expect(actual_expires_at).to be_within(2).of(one_hour_expiration)
79
- end
80
-
81
- it 'does push duplicate messages when not configured for unique only' do
82
- QueueWorker.sidekiq_options unique: false
83
- 10.times { Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2]) }
84
- expect(Sidekiq.redis { |c| c.llen('queue:customqueue') }).to eq 10
85
-
86
- result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
87
- expect(result).to eq 10
88
- end
89
-
90
- describe 'when unique_args is defined' do
91
- before { SidekiqUniqueJobs.config.unique_args_enabled = true }
92
- after { SidekiqUniqueJobs.config.unique_args_enabled = false }
93
-
94
- class QueueWorkerWithFilterMethod < QueueWorker
95
- sidekiq_options unique: true, unique_args: :args_filter
96
-
97
- def self.args_filter(*args)
98
- args.first
99
- end
100
- end
101
-
102
- class QueueWorkerWithFilterProc < QueueWorker
103
- # slightly contrived example of munging args to the
104
- # worker and removing a random bit.
105
- sidekiq_options unique: true, unique_args: (lambda do |args|
106
- a = args.last.dup
107
- a.delete(:random)
108
- [args.first, a]
109
- end)
110
- end
111
-
112
- it 'does not push duplicate messages based on args filter method' do
113
- expect(QueueWorkerWithFilterMethod).to respond_to(:args_filter)
114
- expect(QueueWorkerWithFilterMethod.get_sidekiq_options['unique_args']).to eq :args_filter
115
-
116
- (0..10).each do |i|
117
- Sidekiq::Client.push(
118
- 'class' => QueueWorkerWithFilterMethod,
119
- 'queue' => 'customqueue',
120
- 'args' => [1, i]
121
- )
122
- end
123
- result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
124
- expect(result).to eq 1
125
- end
126
-
127
- it 'does not push duplicate messages based on args filter proc' do
128
- expect(QueueWorkerWithFilterProc.get_sidekiq_options['unique_args']).to be_a(Proc)
129
-
130
- 10.times do
131
- Sidekiq::Client.push(
132
- 'class' => QueueWorkerWithFilterProc,
133
- 'queue' => 'customqueue',
134
- 'args' => [1, { random: rand, name: 'foobar' }]
135
- )
136
- end
137
- result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
138
- expect(result).to eq 1
139
- end
140
-
141
- describe 'when unique_on_all_queues is set' do
142
- before { QueueWorker.sidekiq_options unique: true, unique_on_all_queues: true }
143
- before { QueueWorker.sidekiq_options unique: true }
144
- it 'does not push duplicate messages on different queues' do
145
- Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2])
146
- Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue2', 'args' => [1, 2])
147
- q1_length = Sidekiq.redis { |c| c.llen('queue:customqueue') }
148
- q2_length = Sidekiq.redis { |c| c.llen('queue:customqueue2') }
149
- expect(q1_length).to eq 1
150
- expect(q2_length).to eq 0
151
- end
152
- end
153
- end
154
-
155
- # TODO: If anyone know of a better way to check that the expiration for scheduled
156
- # jobs are set around the same time as the scheduled job itself feel free to improve.
157
- it 'expires the payload_hash when a scheduled job is scheduled at' do
158
- require 'active_support/all'
159
- QueueWorker.sidekiq_options unique: true
160
-
161
- at = 15.minutes.from_now
162
- expected_expires_at = (Time.at(at) - Time.now.utc) + SidekiqUniqueJobs.config.default_expiration
163
-
164
- QueueWorker.perform_in(at, 'mike')
165
- payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('QueueWorker', 'customqueue', ['mike'])
166
-
167
- # deconstruct this into a time format we can use to get a decent delta for
168
- actual_expires_at = Sidekiq.redis { |c| c.ttl(payload_hash) }
169
-
170
- expect(actual_expires_at).to be_within(2).of(expected_expires_at)
171
- end
172
- end
173
- end
@@ -1,81 +0,0 @@
1
- require 'spec_helper'
2
-
3
- module SidekiqUniqueJobs
4
- module Middleware
5
- module Server
6
- describe UniqueJobs do
7
- describe '#unlock_order_configured?' do
8
- context "when class isn't a Sidekiq::Worker" do
9
- it 'returns false' do
10
- expect(subject.unlock_order_configured?(Class))
11
- .to eq(false)
12
- end
13
- end
14
-
15
- context 'when get_sidekiq_options[:unique_unlock_order] is nil' do
16
- it 'returns false' do
17
- expect(subject.unlock_order_configured?(MyWorker))
18
- .to eq(false)
19
- end
20
- end
21
-
22
- it 'returns true when unique_unlock_order has been set' do
23
- UniqueWorker.sidekiq_options unique_unlock_order: :before_yield
24
- expect(subject.unlock_order_configured?(UniqueWorker))
25
- .to eq(true)
26
- end
27
- end
28
-
29
- describe '#decide_unlock_order' do
30
- context 'when worker has specified unique_unlock_order' do
31
- it 'changes unlock_order to the configured value' do
32
- UniqueWorker.sidekiq_options unique_unlock_order: :before_yield
33
- expect do
34
- subject.decide_unlock_order(UniqueWorker)
35
- end.to change { subject.unlock_order }.to :before_yield
36
- end
37
- end
38
-
39
- context "when worker hasn't specified unique_unlock_order" do
40
- it 'falls back to configured default_unlock_order' do
41
- SidekiqUniqueJobs.config.default_unlock_order = :before_yield
42
- expect do
43
- subject.decide_unlock_order(UniqueWorker)
44
- end.to change { subject.unlock_order }.to :before_yield
45
- end
46
- end
47
- end
48
-
49
- describe '#before_yield?' do
50
- it 'returns unlock_order == :before_yield' do
51
- allow(subject).to receive(:unlock_order).and_return(:after_yield)
52
- expect(subject.before_yield?).to eq(false)
53
-
54
- allow(subject).to receive(:unlock_order).and_return(:before_yield)
55
- expect(subject.before_yield?).to eq(true)
56
- end
57
- end
58
-
59
- describe '#after_yield?' do
60
- it 'returns unlock_order == :before_yield' do
61
- allow(subject).to receive(:unlock_order).and_return(:before_yield)
62
- expect(subject.after_yield?).to eq(false)
63
-
64
- allow(subject).to receive(:unlock_order).and_return(:after_yield)
65
- expect(subject.after_yield?).to eq(true)
66
- end
67
- end
68
-
69
- describe '#default_unlock_order' do
70
- it 'returns the default value from config' do
71
- SidekiqUniqueJobs.config.default_unlock_order = :before_yield
72
- expect(subject.default_unlock_order).to eq(:before_yield)
73
-
74
- SidekiqUniqueJobs.config.default_unlock_order = :after_yield
75
- expect(subject.default_unlock_order).to eq(:after_yield)
76
- end
77
- end
78
- end
79
- end
80
- end
81
- end
@@ -1,123 +0,0 @@
1
- require 'spec_helper'
2
- require 'sidekiq/worker'
3
- require 'sidekiq-unique-jobs'
4
- require 'sidekiq/scheduled'
5
- require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
6
- require 'active_support/testing/time_helpers'
7
- require 'rspec-sidekiq'
8
-
9
- describe 'When Sidekiq::Testing is enabled' do
10
- describe 'when set to :fake!', sidekiq: :fake do
11
- context 'with unique worker' do
12
- it 'does not push duplicate messages' do
13
- param = 'work'
14
- expect(UniqueWorker.jobs.size).to eq(0)
15
- UniqueWorker.perform_async(param)
16
- expect(UniqueWorker.jobs.size).to eq(1)
17
- expect(UniqueWorker).to have_enqueued_job(param)
18
- UniqueWorker.perform_async(param)
19
- expect(UniqueWorker.jobs.size).to eq(1)
20
- end
21
-
22
- it 'adds the unique_hash to the message' do
23
- param = 'hash'
24
- hash = SidekiqUniqueJobs::PayloadHelper.get_payload(UniqueWorker, :working, [param])
25
- UniqueWorker.perform_async(param)
26
- expect(UniqueWorker.jobs.size).to eq(1)
27
- expect(UniqueWorker.jobs.first['unique_hash']).to eq(hash)
28
- end
29
- end
30
-
31
- context 'with non-unique worker' do
32
- it 'pushes duplicates messages' do
33
- param = 'work'
34
- expect(MyWorker.jobs.size).to eq(0)
35
- MyWorker.perform_async(param)
36
- expect(MyWorker.jobs.size).to eq(1)
37
- expect(MyWorker).to have_enqueued_job(param)
38
- MyWorker.perform_async(param)
39
- expect(MyWorker.jobs.size).to eq(2)
40
- end
41
- end
42
- end
43
-
44
- describe 'when set to :inline!', sidekiq: :inline do
45
- class InlineWorker
46
- include Sidekiq::Worker
47
- sidekiq_options unique: true
48
-
49
- def perform(x)
50
- TestClass.run(x)
51
- end
52
- end
53
-
54
- class InlineUnlockOrderWorker
55
- include Sidekiq::Worker
56
- sidekiq_options unique: true, unique_unlock_order: :never
57
-
58
- def perform(x)
59
- TestClass.run(x)
60
- end
61
- end
62
-
63
- class InlineUnlockOrderWorker
64
- include Sidekiq::Worker
65
- sidekiq_options unique: true, unique_unlock_order: :never
66
-
67
- def perform(x)
68
- TestClass.run(x)
69
- end
70
- end
71
-
72
- class InlineExpirationWorker
73
- include Sidekiq::Worker
74
- sidekiq_options unique: true, unique_unlock_order: :never,
75
- unique_job_expiration: 10 * 60
76
- def perform(x)
77
- TestClass.run(x)
78
- end
79
- end
80
-
81
- class TestClass
82
- def self.run(_x)
83
- end
84
- end
85
-
86
- it 'once the job is completed allows to run another one' do
87
- expect(TestClass).to receive(:run).exactly(2).times
88
-
89
- InlineWorker.perform_async('test')
90
- InlineWorker.perform_async('test')
91
- end
92
-
93
- it 'if the unique is kept forever it does not allows to run the job again' do
94
- expect(TestClass).to receive(:run).once
95
-
96
- InlineUnlockOrderWorker.perform_async('test')
97
- InlineUnlockOrderWorker.perform_async('test')
98
- end
99
-
100
- describe 'when a job is set to run once in 10 minutes' do
101
- include ActiveSupport::Testing::TimeHelpers
102
- it 'only allows 1 call per 10 minutes' do
103
- allow(TestClass).to receive(:run).with(1).and_return(true)
104
- allow(TestClass).to receive(:run).with(2).and_return(true)
105
-
106
- InlineExpirationWorker.perform_async(1)
107
- expect(TestClass).to have_received(:run).with(1).once
108
- 100.times do
109
- InlineExpirationWorker.perform_async(1)
110
- end
111
- expect(TestClass).to have_received(:run).with(1).once
112
- InlineExpirationWorker.perform_async(2)
113
- expect(TestClass).to have_received(:run).with(1).once
114
- expect(TestClass).to have_received(:run).with(2).once
115
- travel_to(Time.now + (11 * 60)) do
116
- InlineExpirationWorker.perform_async(1)
117
- end
118
-
119
- expect(TestClass).to have_received(:run).with(1).twice
120
- end
121
- end
122
- end
123
- end