sidekiq-unique-jobs 3.0.14 → 4.0.0

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,3 +1,3 @@
 module SidekiqUniqueJobs
-  VERSION = '3.0.14'
+  VERSION = '4.0.0'
 end

data/redis/aquire_lock.lua

@@ -0,0 +1,9 @@
+local unique_key     = KEYS[1]
+local job_id         = ARGV[1]
+local expires        = ARGV[2]
+
+if redis.pcall('set', unique_key, job_id, 'nx', 'ex', expires) then
+  return 1
+else
+  return 0
+end

data/redis/release_lock.lua

@@ -0,0 +1,14 @@
+local unique_key = KEYS[1]
+local job_id     = ARGV[1]
+local stored_jid = redis.pcall('get', unique_key)
+
+if stored_jid then
+  if stored_jid == job_id then
+    return redis.pcall('del', unique_key)
+  else
+    return 0
+  end
+else
+  return -1
+end
+

data/redis/synchronize.lua

@@ -0,0 +1,15 @@
+local unique_key     = KEYS[1]
+local time           = ARGV[1]
+
+if redis.pcall('set', unique_key, time + 60, 'nx', 'ex', 60) then
+  return 1
+end
+
+local stored_time = redis.pcall('get', unique_key)
+if stored_time and stored_time < time then
+  if redis.call('set', unique_key, time + 60, 'nx', 'ex') then
+    return 1
+  end
+end
+
+return 0

data/sidekiq-unique-jobs.gemspec

@@ -13,11 +13,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = `git ls-files -- test/*`.split("\n")
   gem.name          = 'sidekiq-unique-jobs'
   gem.require_paths = ['lib']
-  gem.post_install_message = 'If you are relying on `mock_redis` you will now have to add' \
-                             'gem "mock_redis" to your desired bundler group.'
-  gem.version       = SidekiqUniqueJobs::VERSION
+  gem.post_install_message = 'WARNING: VERSION 4.0.0 HAS BREAKING CHANGES! Please see Readme for info'
+  gem.version = SidekiqUniqueJobs::VERSION
   gem.add_dependency 'sidekiq', '>= 2.6'
-  gem.add_development_dependency 'mock_redis'
   gem.add_development_dependency 'rspec', '~> 3.1.0'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'rspec-sidekiq'

data/spec/lib/sidekiq_unique_jobs/client/middleware_spec.rb

@@ -0,0 +1,195 @@
+require 'spec_helper'
+require 'celluloid/current'
+require 'sidekiq/worker'
+require 'sidekiq-unique-jobs'
+require 'sidekiq/scheduled'
+
+RSpec.describe SidekiqUniqueJobs::Client::Middleware do
+  def digest_for(item)
+    SidekiqUniqueJobs::UniqueArgs.digest(item)
+  end
+
+  describe 'with real redis' do
+    before do
+      Sidekiq.redis = REDIS
+      Sidekiq.redis(&:flushdb)
+      QueueWorker.sidekiq_options unique: nil, unique_expiration: nil
+    end
+
+    describe 'when a job is already scheduled' do
+      context '#new_unique_for' do
+        it 'rejects new scheduled jobs with the same argument' do
+          MyUniqueWorker.perform_in(3600, 1)
+          expect(MyUniqueWorker.perform_in(3600, 1)).to eq(nil)
+        end
+
+        it 'will run a job in real time with the same arguments' do
+          WhileExecutingWorker.perform_in(3600, 1)
+          expect(WhileExecutingWorker.perform_async(1)).not_to eq(nil)
+        end
+
+        it 'schedules new jobs when arguments differ' do
+          [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20].each do |x|
+            MainJob.perform_in(x.seconds.from_now, x)
+          end
+
+          Sidekiq.redis do |c|
+            count = c.zcount('schedule', -1, Time.now.to_f + 2 * 60)
+            expect(count).to eq(20)
+          end
+        end
+      end
+    end
+
+    it 'does not push duplicate messages when configured for unique only' do
+      item = { 'class' => MyUniqueWorker, 'queue' => 'customqueue', 'args' => [1, 2] }
+      10.times { Sidekiq::Client.push(item) }
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq(1)
+      end
+    end
+
+    it 'does push duplicate messages to different queues' do
+      Sidekiq::Client.push('class' => MyUniqueWorker, 'queue' => 'customqueue', 'args' => [1, 2])
+      Sidekiq::Client.push('class' => MyUniqueWorker, 'queue' => 'customqueue2', 'args' => [1, 2])
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq 1
+        expect(c.llen('queue:customqueue2')).to eq 1
+      end
+    end
+
+    it 'does not queue duplicates when when calling delay' do
+      10.times { PlainClass.delay(unique_lock: :until_executed, unique: true, queue: 'customqueue').run(1) }
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq(1)
+      end
+    end
+
+    it 'does not schedule duplicates when calling perform_in' do
+      10.times { MyUniqueWorker.perform_in(60, [1, 2]) }
+      Sidekiq.redis do |c|
+        expect(c.zcount('schedule', -1, Time.now.to_f + 2 * 60))
+          .to eq(1)
+      end
+    end
+
+    it 'enqueues previously scheduled job' do
+      jid = WhileExecutingWorker.perform_in(60 * 60, 1, 2)
+      item = { 'class' => WhileExecutingWorker, 'queue' => 'customqueue', 'args' => [1, 2], 'jid' => jid }
+
+      # time passes and the job is pulled off the schedule:
+      Sidekiq::Client.push(item)
+
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq 1
+      end
+    end
+
+    it 'sets an expiration when provided by sidekiq options' do
+      item = { 'class' => ExpiringWorker, 'queue' => 'customqueue', 'args' => [1, 2] }
+      Sidekiq::Client.push(item)
+
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq(1)
+        expect(c.ttl(digest_for(item)))
+          .to eq(ExpiringWorker.get_sidekiq_options['unique_expiration'])
+      end
+    end
+
+    it 'does push duplicate messages when not configured for unique only' do
+      10.times { Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2]) }
+
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq(10)
+      end
+    end
+
+    describe 'when unique_args is defined' do
+      before(:all) { SidekiqUniqueJobs.config.unique_args_enabled = true }
+      after(:all)  { SidekiqUniqueJobs.config.unique_args_enabled = false }
+
+      it 'does not push duplicate messages based on args filter method' do
+        expect(QueueWorkerWithFilterMethod).to respond_to(:args_filter)
+        expect(QueueWorkerWithFilterMethod.get_sidekiq_options['unique_args']).to eq :args_filter
+
+        (0..10).each do |i|
+          Sidekiq::Client.push(
+            'class' => QueueWorkerWithFilterMethod,
+            'queue' => 'customqueue',
+            'args' => [1, i]
+          )
+        end
+
+        Sidekiq.redis do |c|
+          expect(c.llen('queue:customqueue')).to eq(1)
+        end
+      end
+
+      it 'does not push duplicate messages based on args filter proc' do
+        expect(QueueWorkerWithFilterProc.get_sidekiq_options['unique_args']).to be_a(Proc)
+
+        100.times do
+          Sidekiq::Client.push(
+            'class' => QueueWorkerWithFilterProc,
+            'queue' => 'customqueue',
+            'args' => [1, { random: rand, name: 'foobar' }]
+          )
+        end
+
+        Sidekiq.redis do |c|
+          expect(c.llen('queue:customqueue')).to eq(1)
+        end
+      end
+
+      describe 'when unique_on_all_queues is set' do
+        it 'does not push duplicate messages on different queues' do
+          item = { 'class' => UniqueOnAllQueuesWorker, 'args' => [1, 2] }
+          Sidekiq::Client.push(item.merge('queue' => 'customqueue'))
+          Sidekiq::Client.push(item.merge('queue' => 'customqueue2'))
+          Sidekiq.redis do |c|
+            expect(c.llen('queue:customqueue')).to eq(1)
+            expect(c.llen('queue:customqueue2')).to eq(0)
+          end
+        end
+      end
+    end
+
+    # TODO: If anyone know of a better way to check that the expiration for scheduled
+    # jobs are set around the same time as the scheduled job itself feel free to improve.
+    it 'expires the digest when a scheduled job is scheduled at' do
+      expected_expires_at =
+        (Time.at(15.minutes.from_now) - Time.now.utc) + SidekiqUniqueJobs.config.default_expiration
+      jid = MyUniqueWorker.perform_in(expected_expires_at, 'mike')
+      item = { 'class' => MyUniqueWorker,
+               'queue' => 'customqueue',
+               'args' => ['mike'],
+               'at' => expected_expires_at }
+      digest = digest_for(item.merge('jid' => jid))
+      Sidekiq.redis do |c|
+        expect(c.ttl(digest)).to eq(9_899)
+      end
+    end
+
+    it 'logs duplicate payload when config turned on' do
+      expect(Sidekiq.logger).to receive(:warn).with(/^payload is not unique/)
+      UniqueWorker.sidekiq_options log_duplicate_payload: true
+      2.times { Sidekiq::Client.push('class' => UniqueWorker, 'queue' => 'customqueue', 'args' => [1, 2]) }
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq 1
+      end
+      UniqueWorker.sidekiq_options log_duplicate_payload: true
+    end
+
+    it 'does not log duplicate payload when config turned off' do
+      expect(Sidekiq.logger).to_not receive(:warn).with(/^payload is not unique/)
+
+      UniqueWorker.sidekiq_options log_duplicate_payload: false
+
+      2.times { Sidekiq::Client.push('class' => UniqueWorker, 'queue' => 'customqueue', 'args' => [1, 2]) }
+      Sidekiq.redis do |c|
+        expect(c.llen('queue:customqueue')).to eq 1
+      end
+      UniqueWorker.sidekiq_options log_duplicate_payload: true
+    end
+  end
+end

data/spec/lib/sidekiq_unique_jobs/core_ext_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+RSpec.describe Hash do
+  subject { { test: :me, not: :me } }
+
+  describe '#slice' do
+    specify { expect(subject.slice(:test)).to eq(test: :me) }
+  end
+
+  describe '#slice!' do
+    specify { expect { subject.slice!(:test) }.to change { subject }.to(test: :me) }
+  end
+end
+
+RSpec.describe String do
+  describe '#classify' do
+    subject { 'under_scored_string' }
+    its(:classify) { is_expected.to eq('UnderScoredString') }
+  end
+
+  describe '#camelize' do
+    subject { 'under_scored_string' }
+    its(:camelize) { is_expected.to eq('UnderScoredString') }
+  end
+end

data/spec/lib/sidekiq_unique_jobs/lock/time_calculator_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+RSpec.describe SidekiqUniqueJobs::Lock::TimeCalculator do
+  include ActiveSupport::Testing::TimeHelpers
+  shared_context 'undefined worker class' do
+    subject { described_class.new('class' => 'test') }
+  end
+
+  shared_context 'item not scheduled' do
+    subject { described_class.new('class' => 'MyUniqueWorker') }
+  end
+
+  describe 'public api' do
+    subject { described_class.new(nil) }
+    it { is_expected.to respond_to(:time_until_scheduled) }
+    it { is_expected.to respond_to(:unique_expiration) }
+    it { is_expected.to respond_to(:worker_class_unique_expiration) }
+    it { is_expected.to respond_to(:worker_class) }
+    it { is_expected.to respond_to(:seconds) }
+  end
+
+  describe '.for_item' do
+    it 'initializes a new calculator' do
+      expect(described_class).to receive(:new).with('WAT')
+      described_class.for_item('WAT')
+    end
+  end
+
+  describe '#seconds' do
+    subject { described_class.new(nil) }
+
+    before do
+      allow(subject).to receive(:time_until_scheduled).and_return(10)
+      allow(subject).to receive(:unique_expiration).and_return(9)
+    end
+    its(:seconds) { is_expected.to eq(19) }
+  end
+
+  describe '#time_until_scheduled' do
+    it_behaves_like 'item not scheduled' do
+      its(:time_until_scheduled) { is_expected.to eq(0) }
+    end
+
+    subject { described_class.new('class' => 'MyUniqueWorker', 'at' => schedule_time) }
+    let(:schedule_time) { 1.day.from_now.to_i }
+    let(:now_in_utc) { Time.now.utc.to_i }
+
+    its(:time_until_scheduled) do
+      travel_to(Time.at(now_in_utc)) do
+        is_expected.to eq(schedule_time - now_in_utc)
+      end
+    end
+  end
+
+  describe '#unique_expiration' do
+    it_behaves_like 'undefined worker class' do
+      its(:unique_expiration) { is_expected.to eq(SidekiqUniqueJobs.config.default_expiration) }
+    end
+
+    subject { described_class.new('class' => 'MyUniqueWorker') }
+    its(:unique_expiration) { is_expected.to eq(7_200) }
+  end
+
+  describe '#worker_class_unique_expiration' do
+    it_behaves_like 'undefined worker class' do
+      its(:worker_class_unique_expiration) { is_expected.to eq(nil) }
+    end
+
+    subject { described_class.new('class' => 'MyUniqueWorker') }
+    its(:worker_class_unique_expiration) { is_expected.to eq(7_200) }
+  end
+
+  describe '#worker_class' do
+    it_behaves_like 'undefined worker class' do
+      its(:worker_class) { is_expected.to eq('test') }
+    end
+
+    subject { described_class.new('class' => 'MyWorker') }
+    its(:worker_class) { is_expected.to eq(MyWorker) }
+  end
+end

data/spec/lib/sidekiq_unique_jobs/lock/while_executing_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+RSpec.describe SidekiqUniqueJobs::Lock::WhileExecuting do
+  it 'allows only one mutex object to have the lock at a time' do
+    mutexes = (1..10).map do
+      described_class.new('test_mutex_key')
+    end
+
+    x = 0
+    mutexes.map do |m|
+      Thread.new do
+        m.synchronize do
+          y = x
+          sleep 0.001
+          x = y + 1
+        end
+      end
+    end.map(&:join)
+
+    expect(x).to eq(10)
+  end
+
+  it 'handles auto cleanup correctly' do
+    m = described_class.new('test_mutex_key')
+
+    SidekiqUniqueJobs.connection do |conn|
+      conn.set 'test_mutex_key', Time.now.to_i - 1, nx: true
+    end
+
+    start = Time.now.to_i
+    m.synchronize do
+      'nop'
+    end
+
+    # no longer than a second
+    expect(Time.now.to_i).to be <= start + 1
+  end
+
+  it 'maintains mutex semantics' do
+    m = described_class.new('test_mutex_key')
+
+    expect do
+      m.synchronize do
+        m.synchronize {}
+      end
+    end.to raise_error(ThreadError)
+  end
+end

data/spec/lib/sidekiq_unique_jobs/normalizer_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+RSpec.describe SidekiqUniqueJobs::Normalizer do
+  def jsonify(args)
+    described_class.jsonify(args)
+  end
+
+  describe '.jsonify' do
+    specify do
+      original = [1, :test, [test: :test]]
+      expected = [1, 'test', ['test' => 'test']]
+      expect(jsonify(original)).to eq(expected)
+    end
+
+    specify do
+      original = [1, :test, [test: [test: :test]]]
+      expected = [1, 'test', ['test' => ['test' => 'test']]]
+      expect(jsonify(original)).to eq(expected)
+    end
+  end
+end

data/spec/lib/sidekiq_unique_jobs/scripts_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+RSpec.describe SidekiqUniqueJobs::Scripts do
+  include ActiveSupport::Testing::TimeHelpers
+  MD5_DIGEST ||= 'unique'.freeze
+  UNIQUE_KEY ||= 'uniquejobs:unique'.freeze
+  JID ||= 'fuckit'.freeze
+  ANOTHER_JID ||= 'anotherjid'.freeze
+
+  context 'class methods' do
+    subject { SidekiqUniqueJobs::Scripts }
+
+    it { is_expected.to respond_to(:call).with(3).arguments }
+    it { is_expected.to respond_to(:logger) }
+    it { is_expected.to respond_to(:script_shas) }
+    it { is_expected.to respond_to(:connection).with(1).arguments }
+    it { is_expected.to respond_to(:script_source).with(1).arguments }
+    it { is_expected.to respond_to(:script_path).with(1).arguments }
+
+    describe '.script_shas' do
+      its(:script_shas) { is_expected.to be_a(Hash) }
+    end
+
+    describe '.logger' do
+      its(:logger) { is_expected.to eq(Sidekiq.logger) }
+    end
+
+    def lock_for(seconds = 1, jid = JID, key = UNIQUE_KEY)
+      subject.call(:aquire_lock, nil,         keys: [key], argv: [jid, seconds])
+    end
+
+    def unlock(key = UNIQUE_KEY, jid = JID)
+      subject.call(:release_lock, nil, keys: [key], argv: [jid])
+    end
+
+    describe '.aquire_lock' do
+      context 'when job is unique' do
+        specify { expect(lock_for).to eq(1) }
+        specify do
+          expect(lock_for(0.5.seconds)).to eq(1)
+          expect(Redis)
+            .to have_key(UNIQUE_KEY)
+            .for_seconds(1)
+            .with_value('fuckit')
+          sleep 0.5
+          expect(lock_for).to eq(1)
+        end
+
+        context 'when job is locked' do
+          before  { expect(lock_for).to eq(1) }
+          specify { expect(lock_for).to eq(0) }
+        end
+      end
+    end
+
+    describe '.release_lock' do
+      context 'when job is locked by another jid' do
+        before  { expect(lock_for(10.seconds, 'anotherjid')).to eq(1) }
+        specify { expect(unlock).to eq(0) }
+        after { unlock(UNIQUE_KEY, ANOTHER_JID) }
+      end
+
+      context 'when job is not locked at all' do
+        specify { expect(unlock).to eq(-1) }
+      end
+
+      context 'when job is locked by the same jid' do
+        specify do
+          expect(lock_for(10.seconds)).to eq(1)
+          expect(unlock).to eq(1)
+        end
+      end
+    end
+  end
+end