sidekiq-unique-jobs 2.3.2 → 2.6.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 668f0e7f7a11db5e1ebf8e1d85943911b18c044e
+  data.tar.gz: 337c9f52503823719953a55a38fcae0f8f17a6d5
+SHA512:
+  metadata.gz: c0e0c720b227b5eb4b574aaf6064bf27f97e6e79faa253287df3d012bad2248ed098e341379cf9a34f20ba5f744e4f9bdd1dd3dcac79afd995644ce8be185311
+  data.tar.gz: 5a14117bed073f1e222eb44ee86591ce82dcb4660723bca1a138cfc9f45d9e208b47fafc5d454f29ec23d5756933487a83e6471dd6589590b21bb4241d56135f

data/.gitignore

@@ -4,3 +4,5 @@
 .DS_Store
 
 Gemfile.lock
+
+.ruby-version

data/.travis.yml

@@ -2,14 +2,17 @@ language: ruby
 rvm:
   - 1.9.3
   - jruby-19mode
+  - rbx-19mode
+  - 2.0.0
 branches:
   only:
     - master
-# notifications:
-#   irc: "irc.freenode.org#sidekiq"
-#   email:
-#     recipients:
-#       - sidekiq@librelist.org
+notifications:
+  email:
+    recipients:
+      - mikael@zoolutions.se
 matrix:
   allow_failures:
-    - rvm: jruby-19mode
+    - rvm: jruby-19mode
+    - rvm: rbx-19mode
+    - rvm: 2.0.0

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## v2.6.5
+- via @sax - possibility to set which arguments should be counted as unique - https://github.com/form26/sidekiq-unique-jobs/pull/12
+- via @eduardosasso - possibility to set which arguments should be counted as unique - https://github.com/form26/sidekiq-unique-jobs/pull/11
+- via @KensoDev - configuration of default expiration - https://github.com/form26/sidekiq-unique-jobs/pull/9
+
 ## v2.1.0
 
 Extracted the unique jobs portion from sidekiq main repo since @mperham dropped support for it.

data/Gemfile

@@ -1,14 +1,2 @@
 source 'http://rubygems.org'
-gemspec
-
-group :test do
-  gem 'simplecov', :require => false
-end
-
-group :development do
-  gem 'activesupport'
-  gem 'pry'
-  gem 'pry-doc'
-  gem 'pry-stack_explorer'
-  gem 'pry-debugger'
-end
+gemspec

data/README.md

@@ -1,4 +1,4 @@
-# SidekiqUniqueJobs [![Build Status](https://secure.travis-ci.org/form26/sidekiq-unique-jobs.png)](http://travis-ci.org/form26/sidekiq-unique-jobs)
+# SidekiqUniqueJobs [![Build Status](https://travis-ci.org/form26/sidekiq-unique-jobs.png?branch=master)](https://travis-ci.org/form26/sidekiq-unique-jobs)
 
 The missing unique jobs for sidekiq
 
@@ -32,6 +32,45 @@ sidekiq_options unique: true, unique_job_expiration: 120 * 60 # 2 hours
 
 Requiring the gem in your gemfile should be sufficient to enable unique jobs.
 
+### Finer Control over Uniqueness
+
+Sometimes it is desired to have a finer control over which arguments are used in determining uniqueness of the job, and others may be _transient_. For this use-case, you need to
+set `SidekiqUniqueJobs::Config.unique_args_enabled` to true in an initializer, and then defined either `unique_args` method, or a ruby proc.
+
+```ruby
+SidekiqUniqueJobs::Config.unique_args_enabled = true
+```
+
+The method or the proc can return a modified version of args without the transient arguments included, as shown below:
+
+```ruby
+class UniqueJobWithFilterMethod
+  include Sidekiq::Worker
+  sidekiq_options unique: true,
+                  unique_args: :unique_args
+
+  def self.unique_args(name, id, options)
+    [ name, options[:type] ]
+  end
+
+  ...
+
+end
+
+class UniqueJobWithFilterProc
+  include Sidekiq::Worker
+  sidekiq_options unique: true,
+                  unique_args: ->(args) { [ args.first ] }
+
+  ...
+
+end
+```
+
+Note that objects passed into workers are converted to JSON *after* running through client middleware. In server
+middleware, the JSON is passed directly to the worker `#perform` method. So, you may run into issues where the
+arguments are different when enqueuing than they are when performing. Your `unique_args` method may need to
+account for this.
 
 ## Contributing
 
@@ -40,3 +79,9 @@ Requiring the gem in your gemfile should be sufficient to enable unique jobs.
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+## Contributors
+
+- @sax
+- @eduardosasso
+- @KensoDev

data/lib/sidekiq-unique-jobs.rb

@@ -1,2 +1,11 @@
+require 'yaml' if RUBY_VERSION.include?('2.0.0')
 require 'sidekiq-unique-jobs/middleware'
-require "sidekiq-unique-jobs/version"
+require "sidekiq-unique-jobs/version"
+require "sidekiq-unique-jobs/config"
+require "sidekiq-unique-jobs/payload_helper"
+
+module SidekiqUniqueJobs
+  def self.config
+    SidekiqUniqueJobs::Config
+  end
+end

data/lib/sidekiq-unique-jobs/config.rb

@@ -0,0 +1,27 @@
+module SidekiqUniqueJobs
+  class Config
+    def self.unique_prefix=(prefix)
+      @unique_prefix = prefix
+    end
+
+    def self.unique_prefix
+      @unique_prefix || "sidekiq_unique"
+    end
+
+    def self.unique_args_enabled=(enabled)
+      @unique_args_enabled = enabled
+    end
+
+    def self.unique_args_enabled?
+      @unique_args_enabled || false
+    end
+
+    def self.default_expiration=(expiration)
+      @expiration = expiration
+    end
+
+    def self.default_expiration
+      @expiration || 30 * 60
+    end
+  end
+end

data/lib/sidekiq-unique-jobs/middleware.rb

@@ -5,6 +5,11 @@ Sidekiq.configure_server do |config|
     require 'sidekiq-unique-jobs/middleware/server/unique_jobs'
     chain.add SidekiqUniqueJobs::Middleware::Server::UniqueJobs
   end
+  config.client_middleware do |chain|
+    require 'sidekiq-unique-jobs/middleware/client/unique_jobs'
+    chain.add SidekiqUniqueJobs::Middleware::Client::UniqueJobs
+  end
+
 end
 
 Sidekiq.configure_client do |config|

data/lib/sidekiq-unique-jobs/middleware/client/unique_jobs.rb

@@ -4,17 +4,14 @@ module SidekiqUniqueJobs
   module Middleware
     module Client
       class UniqueJobs
-        HASH_KEY_EXPIRATION = 30 * 60
-
         def call(worker_class, item, queue)
 
-          enabled = worker_class.get_sidekiq_options['unique']
+          enabled = worker_class.get_sidekiq_options['unique'] || item['unique']
           unique_job_expiration = worker_class.get_sidekiq_options['unique_job_expiration']
 
           if enabled
 
-            md5_arguments = {:class => item['class'], :queue => item['queue'], :args => item['args']}
-            payload_hash = Digest::MD5.hexdigest(Sidekiq.dump_json(md5_arguments))
+            payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args'])
 
             unique = false
 
@@ -25,7 +22,7 @@ module SidekiqUniqueJobs
               if conn.get(payload_hash)
                 conn.unwatch
               else
-                expires_at = unique_job_expiration || HASH_KEY_EXPIRATION
+                expires_at = unique_job_expiration || SidekiqUniqueJobs::Config.default_expiration
                 expires_at = ((Time.at(item['at']) - Time.now.utc) + expires_at).to_i if item['at']
 
                 unique = conn.multi do

data/lib/sidekiq-unique-jobs/middleware/server/unique_jobs.rb

@@ -5,14 +5,12 @@ module SidekiqUniqueJobs
     module Server
       class UniqueJobs
         def call(*args)
-          logger.info("About to process a job with args #{args.inspect}")
           yield
-          logger.info("Done processing a job with args #{args.inspect}")
         ensure
           item = args[1]
-          md5_arguments = {:class => item['class'], :queue => item['queue'], :args => item['args']}
-          hash = Digest::MD5.hexdigest(Sidekiq.dump_json(md5_arguments))
-          Sidekiq.redis {|conn| conn.del(hash) }
+          payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload(item['class'], item['queue'], item['args'])
+
+          Sidekiq.redis {|conn| conn.del(payload_hash) }
         end
 
         protected

data/lib/sidekiq-unique-jobs/payload_helper.rb

@@ -0,0 +1,25 @@
+module SidekiqUniqueJobs
+  class PayloadHelper
+    def self.get_payload(klass, queue, *args)
+      args = yield_unique_args(klass, *args) if SidekiqUniqueJobs::Config.unique_args_enabled?
+      md5_arguments = {:class => klass, :queue => queue, :args => args}
+      "#{SidekiqUniqueJobs::Config.unique_prefix}:#{Digest::MD5.hexdigest(Sidekiq.dump_json(md5_arguments))}"
+    end
+
+    def self.yield_unique_args(klass, args)
+      worker_class = klass.constantize
+      unique_args = worker_class.get_sidekiq_options['unique_args']
+      filtered_args = if unique_args
+                        case unique_args
+                          when Proc
+                            unique_args.call(args)
+                          when Symbol
+                            worker_class.send(unique_args, *args) if worker_class.respond_to?(unique_args)
+                        end
+                      end
+      filtered_args || args
+    rescue NameError # if we can't instantiate the class, we just fallback to not filtering args
+      args
+    end
+  end
+end

data/lib/sidekiq-unique-jobs/version.rb

@@ -1,3 +1,3 @@
 module SidekiqUniqueJobs
-  VERSION = "2.3.2"
+  VERSION = "2.6.5"
 end

data/sidekiq-unique-jobs.gemspec

@@ -14,11 +14,11 @@ Gem::Specification.new do |gem|
   gem.name          = "sidekiq-unique-jobs"
   gem.require_paths = ["lib"]
   gem.version       = SidekiqUniqueJobs::VERSION
-  gem.add_dependency                  'sidekiq', '>= 2.2.0'
+  gem.add_dependency                  'sidekiq', '~> 2.6'
   gem.add_development_dependency      'minitest', '~> 3'
   gem.add_development_dependency      'sinatra'
   gem.add_development_dependency      'slim'
   gem.add_development_dependency      'rake'
-  gem.add_development_dependency      'actionmailer', '~> 3'
-  gem.add_development_dependency      'activerecord', '~> 3'
+  gem.add_development_dependency      'activesupport', '~> 3'
+  gem.add_development_dependency      'simplecov'
 end

data/test/helper.rb

@@ -3,7 +3,6 @@ if ENV.has_key?("SIMPLECOV")
   require 'simplecov'
   SimpleCov.start
 end
-require 'pry'
 require 'minitest/unit'
 require 'minitest/pride'
 require 'minitest/autorun'

data/test/lib/sidekiq/another_test_client.rb

@@ -0,0 +1,28 @@
+require 'helper'
+require 'sidekiq/worker'
+require "sidekiq-unique-jobs"
+require 'sidekiq/scheduled'
+require 'sidekiq-unique-jobs/middleware/server/unique_jobs'
+
+class AnotherTestClient < MiniTest::Unit::TestCase
+  describe 'with real redis' do
+    before do
+      Sidekiq.redis = REDIS
+      Sidekiq.redis {|c| c.flushdb }
+      QueueWorker.sidekiq_options :unique => nil, :unique_job_expiration => nil
+    end
+
+    class QueueWorker
+      include Sidekiq::Worker
+      sidekiq_options :queue => 'customqueue'
+      def perform(x)
+      end
+    end
+
+    it 'does not push duplicate messages when configured for unique only' do
+      QueueWorker.sidekiq_options :unique => true
+      10.times { Sidekiq::Client.push('class' => TestClient::QueueWorker, 'queue' => 'customqueue',  'args' => [1, 2]) }
+      assert_equal 1, Sidekiq.redis {|c| c.llen("queue:customqueue") }
+    end
+  end
+end

data/test/lib/sidekiq/test_client.rb

@@ -19,24 +19,9 @@ class TestClient < MiniTest::Unit::TestCase
       end
     end
 
-    # This spec sometimes fails (unless it's the only spec that runs)
-    # Not sure why, we tried a wide variety of ways to make sure that
-    # there aren't side effects between tests and it still happens
-    it 'is able to enqueue after the server middleware completes' do
-      QueueWorker.sidekiq_options :unique => true
-      request_item = {'class' => TestClient::QueueWorker, 'queue' => 'customqueue', 'args' => ["some arg"]}
-
-      Sidekiq::Client.push(request_item.dup)
-      assert_equal 1, Sidekiq.redis {|c| c.llen("queue:customqueue") }
-
-      # Simulate sidekiq processing the job
-      Sidekiq.redis {|c| c.lpop("queue:customqueue")}
-      assert_equal 0, Sidekiq.redis {|c| c.llen("queue:customqueue") }
-
-      SidekiqUniqueJobs::Middleware::Server::UniqueJobs.new.call("dummy arg", request_item.dup) {}
-
-      Sidekiq::Client.push(request_item.dup) 
-      assert_equal 1, Sidekiq.redis {|c| c.llen("queue:customqueue") }
+    class PlainClass
+      def run(x)
+      end
     end
 
     it 'does not push duplicate messages when configured for unique only' do
@@ -45,13 +30,17 @@ class TestClient < MiniTest::Unit::TestCase
       assert_equal 1, Sidekiq.redis {|c| c.llen("queue:customqueue") }
     end
 
+    it 'does not queue duplicates when when calling delay' do
+      10.times { PlainClass.delay(unique: true, queue: 'customqueue').run(1) }
+      assert_equal 1, Sidekiq.redis {|c| c.llen("queue:customqueue") }
+    end
+
     it 'sets an expiration when provided by sidekiq options' do
       one_hour_expiration = 60 * 60
       QueueWorker.sidekiq_options :unique => true, :unique_job_expiration => one_hour_expiration
       Sidekiq::Client.push('class' => TestClient::QueueWorker, 'queue' => 'customqueue',  'args' => [1, 2])
 
-      md5_arguments = {:class => "TestClient::QueueWorker", :queue => "customqueue", :args => [1, 2]}
-      payload_hash = Digest::MD5.hexdigest(Sidekiq.dump_json(md5_arguments))
+      payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload("TestClient::QueueWorker", "customqueue", [1, 2])
       actual_expires_at = Sidekiq.redis {|c| c.ttl(payload_hash) }
 
       assert_in_delta one_hour_expiration, actual_expires_at, 2
@@ -63,6 +52,43 @@ class TestClient < MiniTest::Unit::TestCase
       assert_equal 10, Sidekiq.redis {|c| c.llen("queue:customqueue") }
     end
 
+    describe 'when unique_args is defined' do
+      before { SidekiqUniqueJobs::Config.unique_args_enabled = true }
+      after  { SidekiqUniqueJobs::Config.unique_args_enabled = false }
+
+      class QueueWorkerWithFilterMethod < QueueWorker
+        sidekiq_options :unique => true, :unique_args => :args_filter
+
+        def self.args_filter(*args)
+          args.first
+        end
+      end
+
+      class QueueWorkerWithFilterProc < QueueWorker
+        # slightly contrived example of munging args to the worker and removing a random bit.
+        sidekiq_options :unique => true, :unique_args => lambda { |args| a = args.last.dup; a.delete(:random); [ args.first, a ] }
+      end
+
+      it 'does not push duplicate messages based on args filter method' do
+        assert TestClient::QueueWorkerWithFilterMethod.respond_to?(:args_filter)
+        assert_equal :args_filter, TestClient::QueueWorkerWithFilterMethod.get_sidekiq_options['unique_args']
+
+        for i in (0..10).to_a
+          Sidekiq::Client.push('class' => TestClient::QueueWorkerWithFilterMethod, 'queue' => 'customqueue', 'args' => [1, i])
+        end
+        assert_equal 1, Sidekiq.redis {|c| c.llen("queue:customqueue") }
+      end
+
+      it 'does not push duplicate messages based on args filter proc' do
+        assert_kind_of Proc, TestClient::QueueWorkerWithFilterProc.get_sidekiq_options['unique_args']
+
+        10.times do
+          Sidekiq::Client.push('class' => TestClient::QueueWorkerWithFilterProc, 'queue' => 'customqueue', 'args' => [ 1, {:random => rand(), :name => "foobar"} ])
+        end
+        assert_equal 1, Sidekiq.redis {|c| c.llen("queue:customqueue") }
+      end
+    end
+
     # TODO: If anyone know of a better way to check that the expiration for scheduled
     # jobs are set around the same time as the scheduled job itself feel free to improve.
     it 'expires the payload_hash when a scheduled job is scheduled at' do
@@ -70,11 +96,10 @@ class TestClient < MiniTest::Unit::TestCase
       QueueWorker.sidekiq_options :unique => true
 
       at = 15.minutes.from_now
-      expected_expires_at = (Time.at(at) - Time.now.utc) + SidekiqUniqueJobs::Middleware::Client::UniqueJobs::HASH_KEY_EXPIRATION
+      expected_expires_at = (Time.at(at) - Time.now.utc) + SidekiqUniqueJobs::Config.default_expiration
 
       QueueWorker.perform_in(at, 'mike')
-      md5_arguments = {:class => "TestClient::QueueWorker", :queue => "customqueue", :args => ['mike']}
-      payload_hash = Digest::MD5.hexdigest(Sidekiq.dump_json(md5_arguments))
+      payload_hash = SidekiqUniqueJobs::PayloadHelper.get_payload("TestClient::QueueWorker", "customqueue", ['mike'])
 
       # deconstruct this into a time format we can use to get a decent delta for
       actual_expires_at = Sidekiq.redis {|c| c.ttl(payload_hash) }

metadata

@@ -1,36 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 2.3.2
-  prerelease: 
+  version: 2.6.5
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-27 00:00:00.000000000 Z
+date: 2013-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,55 +41,48 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: slim
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: actionmailer
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -102,27 +90,24 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
 description: The unique jobs that were removed from sidekiq
 email:
 - mikael@zoolutions.se
@@ -138,39 +123,42 @@ files:
 - README.md
 - Rakefile
 - lib/sidekiq-unique-jobs.rb
+- lib/sidekiq-unique-jobs/config.rb
 - lib/sidekiq-unique-jobs/middleware.rb
 - lib/sidekiq-unique-jobs/middleware/client/unique_jobs.rb
 - lib/sidekiq-unique-jobs/middleware/server/unique_jobs.rb
+- lib/sidekiq-unique-jobs/payload_helper.rb
 - lib/sidekiq-unique-jobs/version.rb
 - sidekiq-unique-jobs.gemspec
 - test/helper.rb
+- test/lib/sidekiq/another_test_client.rb
 - test/lib/sidekiq/test_client.rb
 homepage: http://mperham.github.com/sidekiq
 licenses:
 - LGPL-3.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: The unique jobs that were removed from sidekiq
 test_files:
 - test/helper.rb
+- test/lib/sidekiq/another_test_client.rb
 - test/lib/sidekiq/test_client.rb
 has_rdoc: