sidekiq-middleware 0.0.6 → 0.1.0

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+dump.rdb

data/CHANGES.md

@@ -1,3 +1,9 @@
+0.1.0
+-----------
+
+- Added ability to set custom lock key ([dimko](https://github.com/dimko))
+- Removed forever option due to race condition issues. Added ability to manually operate unique locks instead
+
 0.0.6
 -----------
 
@@ -11,12 +17,12 @@
 0.0.4
 -----------
 
-- Fixed Hash#slice (bnorton)
+- Fixed Hash#slice ([bnorton](https://github.com/bnorton))
 
 0.0.3
 -----------
 
-- Refactored and simplified the UniqueJobs middleware server and client as well as only enforcing the uniqueness of the payload across the keys for class, queue, args, and at (bnorton)
+- Refactored and simplified the UniqueJobs middleware server and client as well as only enforcing the uniqueness of the payload across the keys for class, queue, args, and at ([bnorton](https://github.com/bnorton))
 
 0.0.2
 -----------

data/README.md

@@ -37,11 +37,8 @@ class UniqueWorker
     # or :all (enables uniqueness for both async and scheduled jobs)
     unique: :all,
 
-    # Set this to true in case your job schedules itself
-    forever: true,
-
     # Unique expiration (optional, default is 30 minutes)
-    # For scheduled jobs calculates automatically if not provided
+    # For scheduled jobs calculates automatically
     expiration: 24 * 60 * 60
   })
 
@@ -51,6 +48,47 @@ class UniqueWorker
 end
 ```
 
+Custom lock key and manual expiration:
+
+```ruby
+class UniqueWorker
+  include Sidekiq::Worker
+
+  sidekiq_options({
+    unique: :all,
+    expiration: 24 * 60 * 60,
+    
+    # Set this to true when you need to handle locks manually.
+    # You'll be able to handle unique expiration inside your worker.
+    # Please see example below.
+    manual: true
+  })
+  
+  # Implement your own lock string
+  def self.lock(id)
+    "locks:unique:#{id}"
+  end
+  
+  # Implement method to handle lock removing manually
+  def self.unlock!(id)
+  	lock = self.lock(id)
+    Sidekiq.redis { |conn| conn.del(lock) }
+  end
+
+  def perform(id)
+    # Your code goes here
+    # You are able to re-schedule job from perform method,
+    # Just remove lock manually before performing job again.
+    sleep 5
+    
+    # Re-schedule!
+    self.class.unlock!(id)
+    self.class.perform_async(id)
+  end
+end
+```
+
+
 ## Contributing
 
 1. Fork it

data/lib/sidekiq-middleware/client/unique_jobs.rb

@@ -13,11 +13,12 @@ module Sidekiq
 
             # Enabled unique scheduled
             if enabled == :all && payload.has_key?('at')
-              expiration = (payload['at'].to_i - Time.now.to_i)
+              scheduled_expiration = (payload['at'].to_i - Time.now.to_i)
+              expiration = scheduled_expiration if scheduled_expiration > expiration
               payload.delete('at')
             end
 
-            payload_hash = "locks:unique:#{Digest::MD5.hexdigest(Sidekiq.dump_json(payload))}"
+            payload_hash = Sidekiq::Middleware::UniqueKey.generate(worker_class, payload)
 
             Sidekiq.redis do |conn|
               conn.watch(payload_hash)

data/lib/sidekiq-middleware/middleware.rb

@@ -11,4 +11,4 @@ Sidekiq.configure_client do |config|
   config.client_middleware do |chain|
     chain.add Sidekiq::Middleware::Client::UniqueJobs
   end
-end
+end

data/lib/sidekiq-middleware/server/unique_jobs.rb

@@ -4,31 +4,31 @@ module Sidekiq
       class UniqueJobs
 
         def call(worker_instance, item, queue)
-          forever = worker_instance.class.get_sidekiq_options['forever']
-
-          # Delete lock first if forever is set
-          # Used for jobs which may scheduling self in future
-          clear(worker_instance, item, queue) if forever
+          manual = worker_instance.class.get_sidekiq_options['manual']
 
           begin
             yield
           ensure
-            clear(worker_instance, item, queue) unless forever
+            clear(worker_instance, item, queue) unless manual
           end
         end
 
-        def clear(worker_instance, item, queue)
+        def unique_lock_key(worker_instance, item, queue)
           # Only enforce uniqueness across class, queue, args, and at.
           # Useful when middleware uses the payload to store metadata.
           enabled, payload = worker_instance.class.get_sidekiq_options['unique'],
             item.clone.slice(*%w(class queue args at))
 
-          # Enabled unique scheduled 
+          # Enabled unique scheduled
           if enabled == :all && payload.has_key?('at')
             payload.delete('at')
           end
 
-          Sidekiq.redis { |conn| conn.del "locks:unique:#{Digest::MD5.hexdigest(Sidekiq.dump_json(payload))}" }
+          Sidekiq::Middleware::UniqueKey.generate(worker_instance.class, payload)
+        end
+
+        def clear(worker_instance, item, queue)
+          Sidekiq.redis { |conn| conn.del unique_lock_key(worker_instance, item, queue) }
         end
       end
     end

data/lib/sidekiq-middleware/unique_key.rb

@@ -0,0 +1,14 @@
+module Sidekiq
+  module Middleware
+    module UniqueKey
+      def self.generate(worker_class, payload)
+        if worker_class.respond_to?(:lock)
+          worker_class.lock(*payload['args'])
+        else
+          json = Sidekiq.dump_json(payload)
+          "locks:unique:#{Digest::MD5.hexdigest(json)}"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-middleware/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Middleware
-    VERSION = "0.0.6"
+    VERSION = "0.1.0"
   end
 end

data/lib/sidekiq-middleware.rb

@@ -1,6 +1,7 @@
 require "digest"
 require "sidekiq-middleware/version"
 require "sidekiq-middleware/core_ext"
+require "sidekiq-middleware/unique_key"
 require "sidekiq-middleware/server/unique_jobs"
 require "sidekiq-middleware/client/unique_jobs"
-require "sidekiq-middleware/middleware"
+require "sidekiq-middleware/middleware"

data/test/helper.rb

@@ -12,6 +12,8 @@ require 'celluloid'
 Celluloid.logger = nil
 
 require 'sidekiq'
+require 'sidekiq/cli'
+require 'sidekiq/processor'
 require 'sidekiq/util'
 require 'sidekiq-middleware'
 Sidekiq.logger.level = Logger::ERROR

data/test/test_unique_jobs.rb

@@ -10,11 +10,25 @@ class TestUniqueJobs < MiniTest::Unit::TestCase
     before do
       @boss = MiniTest::Mock.new
       @processor = ::Sidekiq::Processor.new(@boss)
-      
+
       Sidekiq.redis = REDIS
       Sidekiq.redis {|c| c.flushdb }
     end
 
+    UnitOfWork = Struct.new(:queue, :message) do
+      def acknowledge
+        # nothing to do
+      end
+
+      def queue_name
+        queue
+      end
+
+      def requeue
+        # nothing to do
+      end
+    end
+
     class UniqueWorker
       include Sidekiq::Worker
       sidekiq_options queue: :unique_queue, unique: true
@@ -48,7 +62,7 @@ class TestUniqueJobs < MiniTest::Unit::TestCase
 
     class UniqueScheduledWorker
       include Sidekiq::Worker
-      sidekiq_options queue: :unique_scheduled_queue, unique: :all, forever: true
+      sidekiq_options queue: :unique_scheduled_queue, unique: :all, manual: true
 
       def perform(x)
         UniqueScheduledWorker.perform_in(60, x)
@@ -60,26 +74,65 @@ class TestUniqueJobs < MiniTest::Unit::TestCase
       assert_equal 1, Sidekiq.redis { |c| c.zcard('schedule') }
     end
 
-    it 'allows the job to reschedule itself with enabled forever option' do
+    class CustomUniqueWorker
+      include Sidekiq::Worker
+      sidekiq_options queue: :custom_unique_queue, unique: :all, manual: true
+
+      def self.lock(id, unlock)
+        "custom:unique:lock:#{id}"
+      end
+
+      def self.unlock!(id, unlock)
+        lock = self.lock(id, unlock)
+        Sidekiq.redis { |conn| conn.del(lock) }
+      end
+
+      def perform(id, unlock)
+        self.class.unlock!(id, unlock) if unlock
+        CustomUniqueWorker.perform_in(60, id, unlock)
+      end
+    end
+
+    it 'does not duplicate messages with enabled unique option and custom unique lock key' do
+      5.times { CustomUniqueWorker.perform_async('args', false) }
+      assert_equal 1, Sidekiq.redis { |c| c.llen('queue:custom_unique_queue') }
+      assert_equal 1, Sidekiq.redis { |c| c.get('custom:unique:lock:args').to_i }
+    end
+
+    it 'does not allow the job to be duplicated when processing job with manual option' do
       5.times {
-        msg = Sidekiq.dump_json('class' => UniqueScheduledWorker.to_s, 'args' => ['something'])
+        msg = Sidekiq.dump_json('class' => CustomUniqueWorker.to_s, 'args' => ['something', false])
         actor = MiniTest::Mock.new
         actor.expect(:processor_done, nil, [@processor])
         @boss.expect(:async, actor, [])
-        @processor.process(msg, 'default')
+        work = UnitOfWork.new('default', msg)
+        @processor.process(work)
       }
       assert_equal 1, Sidekiq.redis { |c| c.zcard('schedule') }
     end
 
     it 'discards non critical information about the message' do
       5.times {|i|
-        msg = Sidekiq.dump_json('class' => UniqueScheduledWorker.to_s, 'args' => ['something'], 'sent_at' => (Time.now + i*60).to_f)
+        msg = Sidekiq.dump_json('class' => CustomUniqueWorker.to_s, 'args' => ['something', false], 'sent_at' => (Time.now + i*60).to_f)
         actor = MiniTest::Mock.new
         actor.expect(:processor_done, nil, [@processor])
         @boss.expect(:async, actor, [])
-        @processor.process(msg, 'default')
+        work = UnitOfWork.new('default', msg)
+        @processor.process(work)
       }
       assert_equal 1, Sidekiq.redis { |c| c.zcard('schedule') }
     end
+
+    it 'allows a job to be rescheduled when processing using unlock' do
+      5.times {
+        msg = Sidekiq.dump_json('class' => CustomUniqueWorker.to_s, 'args' => ['something', true])
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        @boss.expect(:async, actor, [])
+        work = UnitOfWork.new('default', msg)
+        @processor.process(work)
+      }
+      assert_equal 5, Sidekiq.redis { |c| c.zcard('schedule') }
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-middleware
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-07 00:00:00.000000000 Z
+date: 2013-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -78,6 +78,7 @@ files:
 - lib/sidekiq-middleware/core_ext.rb
 - lib/sidekiq-middleware/middleware.rb
 - lib/sidekiq-middleware/server/unique_jobs.rb
+- lib/sidekiq-middleware/unique_key.rb
 - lib/sidekiq-middleware/version.rb
 - sidekiq-middleware.gemspec
 - test/helper.rb
@@ -95,12 +96,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1078771584105605648
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1078771584105605648
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24