sidekiq-unique-jobs 3.0.11 → 8.0.10

data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb

@@ -1,76 +0,0 @@
-require 'digest'
-require 'sidekiq_unique_jobs/connectors'
-
-module SidekiqUniqueJobs
-  module Middleware
-    module Client
-      module Strategies
-        class Unique
-          def self.elegible?
-            true
-          end
-
-          def self.review(worker_class, item, queue, redis_pool = nil)
-            new(worker_class, item, queue, redis_pool).review { yield }
-          end
-
-          def initialize(worker_class, item, queue, redis_pool = nil)
-            @worker_class = SidekiqUniqueJobs.worker_class_constantize(worker_class)
-            @item = item
-            @queue = queue
-            @redis_pool = redis_pool
-          end
-
-          def review
-            item['unique_hash'] = payload_hash
-            return unless unique_for_connection?
-            yield
-          end
-
-          private
-
-          attr_reader :item, :worker_class, :redis_pool, :queue
-
-          # rubocop:disable MethodLength
-          def unique_for_connection?
-            unique = false
-            connection do |conn|
-              conn.watch(payload_hash)
-
-              if conn.get(payload_hash).to_i == 1 ||
-                 (conn.get(payload_hash).to_i == 2 && item['at'])
-                # if the job is already queued, or is already scheduled and
-                # we're trying to schedule again, abort
-                conn.unwatch
-              else
-                # if the job was previously scheduled and is now being queued,
-                # or we've never seen it before
-                expires_at = unique_job_expiration || SidekiqUniqueJobs.config.default_expiration
-                expires_at = ((Time.at(item['at']) - Time.now.utc) + expires_at).to_i if item['at']
-
-                unique = conn.multi do
-                  # set value of 2 for scheduled jobs, 1 for queued jobs.
-                  conn.setex(payload_hash, expires_at, item['at'] ? 2 : 1)
-                end
-              end
-            end
-            unique
-          end
-          # rubocop:enable MethodLength
-
-          def connection(&block)
-            SidekiqUniqueJobs::Connectors.connection(redis_pool, &block)
-          end
-
-          def payload_hash
-            SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args'])
-          end
-
-          def unique_job_expiration
-            worker_class.get_sidekiq_options['unique_job_expiration']
-          end
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb

@@ -1,39 +0,0 @@
-require 'sidekiq_unique_jobs/middleware/client/strategies/unique'
-require 'sidekiq_unique_jobs/middleware/client/strategies/testing_inline'
-
-module SidekiqUniqueJobs
-  module Middleware
-    module Client
-      class UniqueJobs
-        STRATEGIES = [
-          Strategies::TestingInline,
-          Strategies::Unique
-        ]
-
-        attr_reader :item, :worker_class, :redis_pool
-
-        def call(worker_class, item, queue, redis_pool = nil)
-          @worker_class = SidekiqUniqueJobs.worker_class_constantize(worker_class)
-          @item = item
-          @redis_pool = redis_pool
-
-          if unique_enabled?
-            strategy.review(worker_class, item, queue, redis_pool) { yield }
-          else
-            yield
-          end
-        end
-
-        private
-
-        def unique_enabled?
-          worker_class.get_sidekiq_options['unique'] || item['unique']
-        end
-
-        def strategy
-          STRATEGIES.detect(&:elegible?)
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb

@@ -1,69 +0,0 @@
-require 'digest'
-require 'sidekiq_unique_jobs/connectors'
-
-module SidekiqUniqueJobs
-  module Middleware
-    module Server
-      class UniqueJobs
-        attr_reader :unlock_order, :redis_pool
-
-        def call(worker, item, _queue, redis_pool = nil)
-          @redis_pool = redis_pool
-
-          decide_unlock_order(worker.class)
-          lock_key = payload_hash(item)
-          unlocked = before_yield? ? unlock(lock_key).inspect : 0
-
-          yield
-        ensure
-          if after_yield? || !defined? unlocked || unlocked != 1
-            unlock(lock_key)
-          end
-        end
-
-        def decide_unlock_order(klass)
-          @unlock_order = if unlock_order_configured?(klass)
-                            klass.get_sidekiq_options['unique_unlock_order']
-                          else
-                            default_unlock_order
-                          end
-        end
-
-        def unlock_order_configured?(klass)
-          klass.respond_to?(:get_sidekiq_options) &&
-            !klass.get_sidekiq_options['unique_unlock_order'].nil?
-        end
-
-        def default_unlock_order
-          SidekiqUniqueJobs.config.default_unlock_order
-        end
-
-        def before_yield?
-          unlock_order == :before_yield
-        end
-
-        def after_yield?
-          unlock_order == :after_yield
-        end
-
-        protected
-
-        def payload_hash(item)
-          SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args'])
-        end
-
-        def unlock(payload_hash)
-          connection { |c| c.del(payload_hash) }
-        end
-
-        def logger
-          Sidekiq.logger
-        end
-
-        def connection(&block)
-          SidekiqUniqueJobs::Connectors.connection(redis_pool, &block)
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq_unique_jobs/payload_helper.rb

@@ -1,42 +0,0 @@
-module SidekiqUniqueJobs
-  class PayloadHelper
-    def self.config
-      SidekiqUniqueJobs.config
-    end
-
-    def self.get_payload(klass, queue, *args)
-      unique_on_all_queues = false
-      if config.unique_args_enabled
-        worker_class = klass.constantize
-        args = yield_unique_args(worker_class, *args)
-        unique_on_all_queues =
-          worker_class.get_sidekiq_options['unique_on_all_queues']
-      end
-      md5_arguments = { class: klass, args: args }
-      md5_arguments[:queue] = queue unless unique_on_all_queues
-      "#{config.unique_prefix}:" \
-        "#{Digest::MD5.hexdigest(Sidekiq.dump_json(md5_arguments))}"
-    end
-
-    def self.yield_unique_args(worker_class, args)
-      unique_args = worker_class.get_sidekiq_options['unique_args']
-      filtered_args(worker_class, unique_args, args)
-    rescue NameError
-      # fallback to not filtering args when class can't be instantiated
-      args
-    end
-
-    def self.filtered_args(worker_class, unique_args, args)
-      case unique_args
-      when Proc
-        unique_args.call(args)
-      when Symbol
-        if worker_class.respond_to?(unique_args)
-          worker_class.send(unique_args, *args)
-        end
-      else
-        args
-      end
-    end
-  end
-end

data/sidekiq-unique-jobs.gemspec

@@ -1,27 +0,0 @@
-# -*- encoding: utf-8 -*-
-require File.expand_path('../lib/sidekiq_unique_jobs/version', __FILE__)
-
-Gem::Specification.new do |gem|
-  gem.authors       = ['Mikael Henriksson']
-  gem.email         = ['mikael@zoolutions.se']
-  gem.description   = gem.summary = 'The unique jobs that were removed from sidekiq'
-  gem.homepage      = 'https://github.com/mhenrixon/sidekiq-unique-jobs'
-  gem.license       = 'LGPL-3.0'
-
-  # gem.executables   = ['']
-  gem.files         = `git ls-files`.split("\n")
-  gem.test_files    = `git ls-files -- test/*`.split("\n")
-  gem.name          = 'sidekiq-unique-jobs'
-  gem.require_paths = ['lib']
-  gem.post_install_message = 'If you are relying on `mock_redis` you will now have to add' \
-                             'gem "mock_redis" to your desired bundler group.'
-  gem.version       = SidekiqUniqueJobs::VERSION
-  gem.add_dependency 'sidekiq', '>= 2.6'
-  gem.add_development_dependency 'mock_redis'
-  gem.add_development_dependency 'rspec', '~> 3.1.0'
-  gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'rspec-sidekiq'
-  gem.add_development_dependency 'activesupport', '>= 3'
-  gem.add_development_dependency 'rubocop'
-  gem.add_development_dependency 'simplecov'
-end

data/spec/lib/client_spec.rb

@@ -1,173 +0,0 @@
-require 'spec_helper'
-require 'celluloid'
-require 'sidekiq/worker'
-require 'sidekiq-unique-jobs'
-require 'sidekiq/scheduled'
-require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
-
-describe 'Client' do
-  describe 'with real redis' do
-    before do
-      Sidekiq.redis = REDIS
-      Sidekiq.redis(&:flushdb)
-      QueueWorker.sidekiq_options unique: nil, unique_job_expiration: nil
-    end
-
-    class QueueWorker
-      include Sidekiq::Worker
-      sidekiq_options queue: 'customqueue'
-      def perform(_x)
-      end
-    end
-
-    class PlainClass
-      def run(_x)
-      end
-    end
-
-    it 'does not push duplicate messages when configured for unique only' do
-      QueueWorker.sidekiq_options unique: true
-      10.times { Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue',  'args' => [1, 2]) }
-      result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-      expect(result).to eq 1
-    end
-
-    it 'does push duplicate messages to different queues' do
-      QueueWorker.sidekiq_options unique: true
-      Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue',  'args' => [1, 2])
-      Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue2',  'args' => [1, 2])
-      q1_length = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-      q2_length = Sidekiq.redis { |c| c.llen('queue:customqueue2') }
-      expect(q1_length).to eq 1
-      expect(q2_length).to eq 1
-    end
-
-    it 'does not queue duplicates when when calling delay' do
-      10.times { PlainClass.delay(unique: true, queue: 'customqueue').run(1) }
-      result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-      expect(result).to eq 1
-    end
-
-    it 'does not schedule duplicates when calling perform_in' do
-      QueueWorker.sidekiq_options unique: true
-      10.times { QueueWorker.perform_in(60, [1, 2]) }
-      result = Sidekiq.redis { |c| c.zcount('schedule', -1, Time.now.to_f + 2 * 60) }
-      expect(result).to eq 1
-    end
-
-    it 'enqueues previously scheduled job' do
-      QueueWorker.sidekiq_options unique: true
-      QueueWorker.perform_in(60 * 60, 1, 2)
-
-      # time passes and the job is pulled off the schedule:
-      Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue', 'args' => [1, 2])
-
-      result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-      expect(result).to eq 1
-    end
-
-    it 'sets an expiration when provided by sidekiq options' do
-      one_hour_expiration = 60 * 60
-      QueueWorker.sidekiq_options unique: true, unique_job_expiration: one_hour_expiration
-      Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue',  'args' => [1, 2])
-
-      payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('QueueWorker', 'customqueue', [1, 2])
-      actual_expires_at = Sidekiq.redis { |c| c.ttl(payload_hash) }
-
-      Sidekiq.redis { |c| c.llen('queue:customqueue') }
-      expect(actual_expires_at).to be_within(2).of(one_hour_expiration)
-    end
-
-    it 'does push duplicate messages when not configured for unique only' do
-      QueueWorker.sidekiq_options unique: false
-      10.times { Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue',  'args' => [1, 2]) }
-      expect(Sidekiq.redis { |c| c.llen('queue:customqueue') }).to eq 10
-
-      result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-      expect(result).to eq 10
-    end
-
-    describe 'when unique_args is defined' do
-      before { SidekiqUniqueJobs.config.unique_args_enabled = true }
-      after  { SidekiqUniqueJobs.config.unique_args_enabled = false }
-
-      class QueueWorkerWithFilterMethod < QueueWorker
-        sidekiq_options unique: true, unique_args: :args_filter
-
-        def self.args_filter(*args)
-          args.first
-        end
-      end
-
-      class QueueWorkerWithFilterProc < QueueWorker
-        # slightly contrived example of munging args to the
-        # worker and removing a random bit.
-        sidekiq_options unique: true, unique_args: (lambda do |args|
-          a = args.last.dup
-          a.delete(:random)
-          [args.first, a]
-        end)
-      end
-
-      it 'does not push duplicate messages based on args filter method' do
-        expect(QueueWorkerWithFilterMethod).to respond_to(:args_filter)
-        expect(QueueWorkerWithFilterMethod.get_sidekiq_options['unique_args']).to eq :args_filter
-
-        (0..10).each do |i|
-          Sidekiq::Client.push(
-            'class' => QueueWorkerWithFilterMethod,
-            'queue' => 'customqueue',
-            'args' => [1, i]
-          )
-        end
-        result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-        expect(result).to eq 1
-      end
-
-      it 'does not push duplicate messages based on args filter proc' do
-        expect(QueueWorkerWithFilterProc.get_sidekiq_options['unique_args']).to be_a(Proc)
-
-        10.times do
-          Sidekiq::Client.push(
-            'class' => QueueWorkerWithFilterProc,
-            'queue' => 'customqueue',
-            'args' => [1, { random: rand, name: 'foobar' }]
-          )
-        end
-        result = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-        expect(result).to eq 1
-      end
-
-      describe 'when unique_on_all_queues is set' do
-        before { QueueWorker.sidekiq_options unique: true, unique_on_all_queues: true }
-        before { QueueWorker.sidekiq_options unique: true }
-        it 'does not push duplicate messages on different queues' do
-          Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue',  'args' => [1, 2])
-          Sidekiq::Client.push('class' => QueueWorker, 'queue' => 'customqueue2',  'args' => [1, 2])
-          q1_length = Sidekiq.redis { |c| c.llen('queue:customqueue') }
-          q2_length = Sidekiq.redis { |c| c.llen('queue:customqueue2') }
-          expect(q1_length).to eq 1
-          expect(q2_length).to eq 0
-        end
-      end
-    end
-
-    # TODO: If anyone know of a better way to check that the expiration for scheduled
-    # jobs are set around the same time as the scheduled job itself feel free to improve.
-    it 'expires the payload_hash when a scheduled job is scheduled at' do
-      require 'active_support/all'
-      QueueWorker.sidekiq_options unique: true
-
-      at = 15.minutes.from_now
-      expected_expires_at = (Time.at(at) - Time.now.utc) + SidekiqUniqueJobs.config.default_expiration
-
-      QueueWorker.perform_in(at, 'mike')
-      payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload('QueueWorker', 'customqueue', ['mike'])
-
-      # deconstruct this into a time format we can use to get a decent delta for
-      actual_expires_at = Sidekiq.redis { |c| c.ttl(payload_hash) }
-
-      expect(actual_expires_at).to be_within(2).of(expected_expires_at)
-    end
-  end
-end

data/spec/lib/middleware/server/unique_jobs_spec.rb

@@ -1,81 +0,0 @@
-require 'spec_helper'
-
-module SidekiqUniqueJobs
-  module Middleware
-    module Server
-      describe UniqueJobs do
-        describe '#unlock_order_configured?' do
-          context "when class isn't a Sidekiq::Worker" do
-            it 'returns false' do
-              expect(subject.unlock_order_configured?(Class))
-                .to eq(false)
-            end
-          end
-
-          context 'when get_sidekiq_options[:unique_unlock_order] is nil' do
-            it 'returns false' do
-              expect(subject.unlock_order_configured?(MyWorker))
-                .to eq(false)
-            end
-          end
-
-          it 'returns true when unique_unlock_order has been set' do
-            UniqueWorker.sidekiq_options unique_unlock_order: :before_yield
-            expect(subject.unlock_order_configured?(UniqueWorker))
-              .to eq(true)
-          end
-        end
-
-        describe '#decide_unlock_order' do
-          context 'when worker has specified unique_unlock_order' do
-            it 'changes unlock_order to the configured value' do
-              UniqueWorker.sidekiq_options unique_unlock_order: :before_yield
-              expect do
-                subject.decide_unlock_order(UniqueWorker)
-              end.to change { subject.unlock_order }.to :before_yield
-            end
-          end
-
-          context "when worker hasn't specified unique_unlock_order" do
-            it 'falls back to configured default_unlock_order' do
-              SidekiqUniqueJobs.config.default_unlock_order = :before_yield
-              expect do
-                subject.decide_unlock_order(UniqueWorker)
-              end.to change { subject.unlock_order }.to :before_yield
-            end
-          end
-        end
-
-        describe '#before_yield?' do
-          it 'returns unlock_order == :before_yield' do
-            allow(subject).to receive(:unlock_order).and_return(:after_yield)
-            expect(subject.before_yield?).to eq(false)
-
-            allow(subject).to receive(:unlock_order).and_return(:before_yield)
-            expect(subject.before_yield?).to eq(true)
-          end
-        end
-
-        describe '#after_yield?' do
-          it 'returns unlock_order == :before_yield' do
-            allow(subject).to receive(:unlock_order).and_return(:before_yield)
-            expect(subject.after_yield?).to eq(false)
-
-            allow(subject).to receive(:unlock_order).and_return(:after_yield)
-            expect(subject.after_yield?).to eq(true)
-          end
-        end
-
-        describe '#default_unlock_order' do
-          it 'returns the default value from config' do
-            SidekiqUniqueJobs.config.default_unlock_order = :before_yield
-            expect(subject.default_unlock_order).to eq(:before_yield)
-
-            SidekiqUniqueJobs.config.default_unlock_order = :after_yield
-            expect(subject.default_unlock_order).to eq(:after_yield)
-          end
-        end
-      end
-    end
-  end
-end

data/spec/lib/sidekiq_testing_enabled_spec.rb

@@ -1,123 +0,0 @@
-require 'spec_helper'
-require 'sidekiq/worker'
-require 'sidekiq-unique-jobs'
-require 'sidekiq/scheduled'
-require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
-require 'active_support/testing/time_helpers'
-require 'rspec-sidekiq'
-
-describe 'When Sidekiq::Testing is enabled' do
-  describe 'when set to :fake!', sidekiq: :fake do
-    context 'with unique worker' do
-      it 'does not push duplicate messages' do
-        param = 'work'
-        expect(UniqueWorker.jobs.size).to eq(0)
-        UniqueWorker.perform_async(param)
-        expect(UniqueWorker.jobs.size).to eq(1)
-        expect(UniqueWorker).to have_enqueued_job(param)
-        UniqueWorker.perform_async(param)
-        expect(UniqueWorker.jobs.size).to eq(1)
-      end
-
-      it 'adds the unique_hash to the message' do
-        param = 'hash'
-        hash = SidekiqUniqueJobs::PayloadHelper.get_payload(UniqueWorker, :working, [param])
-        UniqueWorker.perform_async(param)
-        expect(UniqueWorker.jobs.size).to eq(1)
-        expect(UniqueWorker.jobs.first['unique_hash']).to eq(hash)
-      end
-    end
-
-    context 'with non-unique worker' do
-      it 'pushes duplicates messages' do
-        param = 'work'
-        expect(MyWorker.jobs.size).to eq(0)
-        MyWorker.perform_async(param)
-        expect(MyWorker.jobs.size).to eq(1)
-        expect(MyWorker).to have_enqueued_job(param)
-        MyWorker.perform_async(param)
-        expect(MyWorker.jobs.size).to eq(2)
-      end
-    end
-  end
-
-  describe 'when set to :inline!', sidekiq: :inline do
-    class InlineWorker
-      include Sidekiq::Worker
-      sidekiq_options unique: true
-
-      def perform(x)
-        TestClass.run(x)
-      end
-    end
-
-    class InlineUnlockOrderWorker
-      include Sidekiq::Worker
-      sidekiq_options unique: true, unique_unlock_order: :never
-
-      def perform(x)
-        TestClass.run(x)
-      end
-    end
-
-    class InlineUnlockOrderWorker
-      include Sidekiq::Worker
-      sidekiq_options unique: true, unique_unlock_order: :never
-
-      def perform(x)
-        TestClass.run(x)
-      end
-    end
-
-    class InlineExpirationWorker
-      include Sidekiq::Worker
-      sidekiq_options unique: true, unique_unlock_order: :never,
-                      unique_job_expiration: 10 * 60
-      def perform(x)
-        TestClass.run(x)
-      end
-    end
-
-    class TestClass
-      def self.run(_x)
-      end
-    end
-
-    it 'once the job is completed allows to run another one' do
-      expect(TestClass).to receive(:run).exactly(2).times
-
-      InlineWorker.perform_async('test')
-      InlineWorker.perform_async('test')
-    end
-
-    it 'if the unique is kept forever it does not allows to run the job again' do
-      expect(TestClass).to receive(:run).once
-
-      InlineUnlockOrderWorker.perform_async('test')
-      InlineUnlockOrderWorker.perform_async('test')
-    end
-
-    describe 'when a job is set to run once in 10 minutes' do
-      include ActiveSupport::Testing::TimeHelpers
-      it 'only allows 1 call per 10 minutes' do
-        allow(TestClass).to receive(:run).with(1).and_return(true)
-        allow(TestClass).to receive(:run).with(2).and_return(true)
-
-        InlineExpirationWorker.perform_async(1)
-        expect(TestClass).to have_received(:run).with(1).once
-        100.times do
-          InlineExpirationWorker.perform_async(1)
-        end
-        expect(TestClass).to have_received(:run).with(1).once
-        InlineExpirationWorker.perform_async(2)
-        expect(TestClass).to have_received(:run).with(1).once
-        expect(TestClass).to have_received(:run).with(2).once
-        travel_to(Time.now + (11 * 60)) do
-          InlineExpirationWorker.perform_async(1)
-        end
-
-        expect(TestClass).to have_received(:run).with(1).twice
-      end
-    end
-  end
-end