sidekiq-unique-jobs 3.0.11 → 8.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +2163 -25
- data/LICENSE.txt +21 -0
- data/README.md +984 -47
- data/bin/uniquejobs +7 -0
- data/lib/sidekiq-unique-jobs.rb +2 -36
- data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
- data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
- data/lib/sidekiq_unique_jobs/cli.rb +95 -0
- data/lib/sidekiq_unique_jobs/config.rb +306 -33
- data/lib/sidekiq_unique_jobs/connection.rb +20 -0
- data/lib/sidekiq_unique_jobs/constants.rb +55 -0
- data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
- data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
- data/lib/sidekiq_unique_jobs/digests.rb +134 -0
- data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
- data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
- data/lib/sidekiq_unique_jobs/job.rb +63 -0
- data/lib/sidekiq_unique_jobs/json.rb +47 -0
- data/lib/sidekiq_unique_jobs/key.rb +98 -0
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
- data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
- data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
- data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
- data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
- data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
- data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
- data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
- data/lib/sidekiq_unique_jobs/lock.rb +348 -0
- data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
- data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
- data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
- data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
- data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
- data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
- data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
- data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
- data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
- data/lib/sidekiq_unique_jobs/logging.rb +236 -0
- data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
- data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
- data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
- data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
- data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
- data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
- data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
- data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
- data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
- data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
- data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
- data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
- data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
- data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
- data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
- data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
- data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
- data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
- data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
- data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
- data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
- data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
- data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
- data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
- data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
- data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
- data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
- data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
- data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
- data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
- data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
- data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
- data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
- data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
- data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
- data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
- data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
- data/lib/sidekiq_unique_jobs/redis.rb +11 -0
- data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
- data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
- data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
- data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
- data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
- data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
- data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
- data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
- data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
- data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
- data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
- data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
- data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
- data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
- data/lib/sidekiq_unique_jobs/script.rb +46 -0
- data/lib/sidekiq_unique_jobs/server.rb +62 -0
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
- data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
- data/lib/sidekiq_unique_jobs/testing.rb +132 -9
- data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
- data/lib/sidekiq_unique_jobs/timing.rb +58 -0
- data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
- data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
- data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
- data/lib/sidekiq_unique_jobs/version.rb +5 -1
- data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
- data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
- data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
- data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
- data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
- data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
- data/lib/sidekiq_unique_jobs/web.rb +109 -0
- data/lib/sidekiq_unique_jobs.rb +83 -0
- data/lib/tasks/changelog.rake +23 -0
- metadata +157 -126
- data/.gitignore +0 -10
- data/.rspec +0 -3
- data/.rubocop.yml +0 -36
- data/.travis.yml +0 -25
- data/Appraisals +0 -20
- data/Gemfile +0 -5
- data/LICENSE +0 -22
- data/Rakefile +0 -11
- data/gemfiles/sidekiq_2.15.gemfile +0 -9
- data/gemfiles/sidekiq_2.16.gemfile +0 -9
- data/gemfiles/sidekiq_2.17.gemfile +0 -9
- data/gemfiles/sidekiq_3.0.gemfile +0 -9
- data/gemfiles/sidekiq_develop.gemfile +0 -9
- data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
- data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
- data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
- data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
- data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
- data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
- data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
- data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
- data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
- data/sidekiq-unique-jobs.gemspec +0 -27
- data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
- data/spec/lib/client_spec.rb +0 -173
- data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
- data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
- data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
- data/spec/lib/unlock_order_spec.rb +0 -64
- data/spec/spec_helper.rb +0 -37
- data/spec/support/my_worker.rb +0 -13
- data/spec/support/sidekiq_meta.rb +0 -17
- data/spec/support/unique_worker.rb +0 -13
@@ -1,70 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'sidekiq/api'
|
3
|
-
require 'sidekiq/worker'
|
4
|
-
require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
|
5
|
-
require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
|
6
|
-
require 'sidekiq_unique_jobs/sidekiq_unique_ext'
|
7
|
-
|
8
|
-
class JustAWorker
|
9
|
-
include Sidekiq::Worker
|
10
|
-
|
11
|
-
sidekiq_options unique: true, queue: 'testqueue'
|
12
|
-
|
13
|
-
def perform
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
describe Sidekiq::Job::UniqueExtension do
|
18
|
-
before do
|
19
|
-
Sidekiq.redis = REDIS
|
20
|
-
Sidekiq.redis(&:flushdb)
|
21
|
-
end
|
22
|
-
|
23
|
-
it 'deletes uniqueness lock on delete' do
|
24
|
-
params = { foo: 'bar' }
|
25
|
-
payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('JustAWorker', 'testqueue', [params])
|
26
|
-
jid = JustAWorker.perform_async(foo: 'bar')
|
27
|
-
queue = Sidekiq::Queue.new('testqueue')
|
28
|
-
job = queue.find_job(jid)
|
29
|
-
job.delete
|
30
|
-
Sidekiq.redis do |c|
|
31
|
-
expect(c.exists(payload_hash)).to be_falsy
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
describe Sidekiq::Queue::UniqueExtension do
|
37
|
-
before do
|
38
|
-
Sidekiq.redis = REDIS
|
39
|
-
Sidekiq.redis(&:flushdb)
|
40
|
-
end
|
41
|
-
|
42
|
-
it 'deletes uniqueness locks on clear' do
|
43
|
-
params = { foo: 'bar' }
|
44
|
-
payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('JustAWorker', 'testqueue', [params])
|
45
|
-
JustAWorker.perform_async(foo: 'bar')
|
46
|
-
queue = Sidekiq::Queue.new('testqueue')
|
47
|
-
queue.clear
|
48
|
-
Sidekiq.redis do |c|
|
49
|
-
expect(c.exists(payload_hash)).to be_falsy
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
describe Sidekiq::JobSet::UniqueExtension, sidekiq_ver: 3 do
|
55
|
-
before do
|
56
|
-
Sidekiq.redis = REDIS
|
57
|
-
Sidekiq.redis(&:flushdb)
|
58
|
-
end
|
59
|
-
|
60
|
-
it 'deletes uniqueness locks on clear' do
|
61
|
-
params = { foo: 'bar' }
|
62
|
-
payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('JustAWorker', 'testqueue', [params])
|
63
|
-
JustAWorker.perform_in(60 * 60 * 3, foo: 'bar')
|
64
|
-
set = Sidekiq::JobSet.new('schedule')
|
65
|
-
set.clear
|
66
|
-
Sidekiq.redis do |c|
|
67
|
-
expect(c.exists(payload_hash)).to be_falsy
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
@@ -1,64 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'sidekiq/api'
|
3
|
-
require 'sidekiq/worker'
|
4
|
-
require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
|
5
|
-
|
6
|
-
describe 'Unlock order' do
|
7
|
-
QUEUE = 'unlock_ordering'
|
8
|
-
|
9
|
-
class BeforeYieldOrderingWorker
|
10
|
-
include Sidekiq::Worker
|
11
|
-
|
12
|
-
sidekiq_options unique: true, unique_unlock_order: :before_yield, queue: QUEUE
|
13
|
-
|
14
|
-
def perform
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
class AfterYieldOrderingWorker
|
19
|
-
include Sidekiq::Worker
|
20
|
-
|
21
|
-
sidekiq_options unique: true, unique_unlock_order: :after_yield, queue: QUEUE
|
22
|
-
|
23
|
-
def perform
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
describe 'with real redis' do
|
28
|
-
before do
|
29
|
-
Sidekiq.redis = REDIS
|
30
|
-
Sidekiq.redis(&:flushdb)
|
31
|
-
@middleware = SidekiqUniqueJobs::Middleware::Server::UniqueJobs.new
|
32
|
-
end
|
33
|
-
|
34
|
-
describe ':before_yield' do
|
35
|
-
it 'removes the lock before yielding to the worker' do
|
36
|
-
jid = BeforeYieldOrderingWorker.perform_async
|
37
|
-
item = Sidekiq::Queue.new(QUEUE).find_job(jid).item
|
38
|
-
|
39
|
-
result = @middleware.call(BeforeYieldOrderingWorker.new, item, QUEUE) do
|
40
|
-
Sidekiq.redis do |c|
|
41
|
-
c.get(SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args']))
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
expect(result).to eq nil
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
describe ':after_yield' do
|
50
|
-
it 'removes the lock after yielding to the worker' do
|
51
|
-
jid = AfterYieldOrderingWorker.perform_async
|
52
|
-
item = Sidekiq::Queue.new(QUEUE).find_job(jid).item
|
53
|
-
|
54
|
-
result = @middleware.call(AfterYieldOrderingWorker.new, item, QUEUE) do
|
55
|
-
Sidekiq.redis do |c|
|
56
|
-
c.get(SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args']))
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
expect(result).to eq '1'
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
data/spec/spec_helper.rb
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
begin
|
2
|
-
require 'pry'
|
3
|
-
rescue LoadError
|
4
|
-
puts 'Pry unavailable'
|
5
|
-
end
|
6
|
-
|
7
|
-
require 'rspec'
|
8
|
-
|
9
|
-
require 'celluloid/test'
|
10
|
-
require 'sidekiq'
|
11
|
-
require 'sidekiq/util'
|
12
|
-
require 'sidekiq-unique-jobs'
|
13
|
-
Sidekiq.logger.level = Logger::ERROR
|
14
|
-
|
15
|
-
require 'rspec-sidekiq'
|
16
|
-
|
17
|
-
Sidekiq::Testing.disable!
|
18
|
-
|
19
|
-
require 'sidekiq/redis_connection'
|
20
|
-
redis_url = ENV['REDIS_URL'] || 'redis://localhost/15'
|
21
|
-
REDIS = Sidekiq::RedisConnection.create(url: redis_url, namespace: 'testy')
|
22
|
-
|
23
|
-
Dir[File.join(File.dirname(__FILE__), 'support', '**', '*.rb')].each { |f| require f }
|
24
|
-
RSpec.configure do |_config|
|
25
|
-
# config.treat_symbols_as_metadata_keys_with_true_values = true
|
26
|
-
end
|
27
|
-
|
28
|
-
RSpec::Sidekiq.configure do |config|
|
29
|
-
# Clears all job queues before each example
|
30
|
-
config.clear_all_enqueued_jobs = true # default => true
|
31
|
-
|
32
|
-
# Whether to use terminal colours when outputting messages
|
33
|
-
config.enable_terminal_colours = true # default => true
|
34
|
-
|
35
|
-
# Warn when jobs are not enqueued to Redis but to a job array
|
36
|
-
config.warn_when_jobs_not_processed_by_sidekiq = true # default => true
|
37
|
-
end
|
data/spec/support/my_worker.rb
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
class MyWorker
|
2
|
-
include Sidekiq::Worker
|
3
|
-
sidekiq_options queue: :working, retry: 1, backtrace: 10
|
4
|
-
sidekiq_options unique: false
|
5
|
-
|
6
|
-
sidekiq_retries_exhausted do |msg|
|
7
|
-
Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
|
8
|
-
end
|
9
|
-
|
10
|
-
def perform(param)
|
11
|
-
puts param
|
12
|
-
end
|
13
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
RSpec.configure do |config|
|
2
|
-
config.before(:each) do |example|
|
3
|
-
if (sidekiq = example.metadata[:sidekiq])
|
4
|
-
sidekiq = :fake if sidekiq == true
|
5
|
-
Sidekiq::Testing.send("#{sidekiq}!")
|
6
|
-
end
|
7
|
-
|
8
|
-
sidekiq_ver = example.metadata[:sidekiq_ver]
|
9
|
-
if sidekiq_ver && Sidekiq::VERSION[0].to_s != sidekiq_ver.to_s
|
10
|
-
skip("Not relevant for version #{sidekiq_ver}")
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
config.after(:each) do |example|
|
15
|
-
Sidekiq::Testing.disable! unless example.metadata[:sidekiq].nil?
|
16
|
-
end
|
17
|
-
end
|
@@ -1,13 +0,0 @@
|
|
1
|
-
class UniqueWorker
|
2
|
-
include Sidekiq::Worker
|
3
|
-
sidekiq_options queue: :working, retry: 1, backtrace: 10
|
4
|
-
sidekiq_options unique: true
|
5
|
-
|
6
|
-
sidekiq_retries_exhausted do |msg|
|
7
|
-
Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
|
8
|
-
end
|
9
|
-
|
10
|
-
def perform(param)
|
11
|
-
puts param
|
12
|
-
end
|
13
|
-
end
|