resque-workers-lock 1.9 → 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 46137f8bebbbe0ced3b3c8bc255ecdc72fb718e3
+  data.tar.gz: d2185948f051bbc073ae22a1f41782f985dd3ad5
+SHA512:
+  metadata.gz: 3f0aa3cc230357b85e8ba84e4fb8230dceb3bee78d39c7eb764dd93c35267b587f2f39f38efe83561fff19e027c988ed8c6ab7071d9cbd2ff4ac06b414884251
+  data.tar.gz: 4f4920fcfc473b25589cfa07e75d7cd5c473fa49baad5827f0f94a88f93e7d031e07e1053d5e698ef7e9aa46eb36cf7fe0e81d83ed851d464d24ccfd9bacc076

data/README.md

@@ -5,13 +5,14 @@ This is a [resque](https://github.com/defunkt/resque) plugin inspired by [resque
 gem 'resque-workers-lock'
 ```
 
-**Important notice** - As of this gem version 1.7, Resque Workers Lock no longer includes a enqueue lock but focusses solely on a workers lock. If you're also looking for enqueue lock functionality, just add [resque-lock](https://github.com/defunkt/resque-lock) or another plugin in the mix.
+**Important notice** - As of this gem version 1.7, Resque Workers Lock no longer includes an enqueue lock but focusses solely on a workers lock. If you're also looking for enqueue lock functionality, just add [resque-lock](https://github.com/defunkt/resque-lock) or another plugin in the mix.
 
 ## What does it do?
-If resque jobs have the same lock applied this means that those jobs cannot be processed simultaneously by two or more workers. When this situation occurs the second job gets pushed back to the queue.
+If resque jobs have the same lock(s) applied this means that those jobs cannot be processed simultaneously by two or more workers. When this situation occurs the second job gets pushed back to the queue.
 
 ## What is the default lock?
 By default the lock is the instance name + arguments (just like the classic resque-lock). Override this lock to lock on specific arguments.
+You can specify only one lock or an array of locks. A job with an array of locks will only be processed when all the (individual) locks are not being processed by workers.
 
 ## How does it differ from resque-lock?
 Resque-lock will not let you enqueue jobs when you locked them. Resque-workers-lock locks on a workers-level and will requeue the locked jobs. If a worker takes on a job that is already being processed by another worker it will put the job back up in the queue!
@@ -66,3 +67,4 @@ Do a delayed resque (re)queue. However, this will have approximately the same re
 ## Authors/Contributors
 [nicholaides](https://github.com/nicholaides)
 [jgarber](https://github.com/jgarber)
+[mohawke](https://github.com/mohawke)

data/lib/resque/plugins/workers/lock.rb

@@ -19,13 +19,21 @@ module Resque
           3600
         end
 
-        # Override in your job to control the workers lock key.
+        # Override in your job to control the workers lock key(s).
         def lock_workers(*args)
           "#{name}-#{args.to_s}"
         end
 
         def get_lock_workers(*args)
-          "workerslock:"+lock_workers(*args).to_s
+          lock_result = lock_workers(*args)
+
+          if lock_result.kind_of?(Array)
+            lock_result.map do |lock|
+              "workerslock:#{lock}"
+            end
+          else
+            ["workerslock:#{lock_result}"]
+          end
         end
 
         # Override in your job to change the perform requeue delay
@@ -37,8 +45,12 @@ module Resque
         # If it raises Resque::Job::DontPerform, the job is aborted.
         def before_perform_workers_lock(*args)
           if lock_workers(*args)
-            if Resque.redis.setnx(get_lock_workers(*args), true)
-              Resque.redis.expire(get_lock_workers(*args), worker_lock_timeout(*args))
+            lock_result = get_lock_workers(*args)
+
+            if Resque.redis.msetnx lock_result.zip([true]*lock_result.size).flatten
+              lock_result.each do |lock|
+                Resque.redis.expire(lock, worker_lock_timeout(*args))
+              end
             else
               sleep(requeue_perform_delay)
               Resque.enqueue(self, *args)
@@ -48,7 +60,11 @@ module Resque
         end
 
         def clear_workers_lock(*args)
-          Resque.redis.del(get_lock_workers(*args))
+          lock_result = get_lock_workers(*args)
+
+          lock_result.each do |lock|
+            Resque.redis.del(lock)
+          end
         end
 
         def around_perform_workers_lock(*args)

data/test/lock_mkey_test.rb

@@ -0,0 +1,137 @@
+require 'test/unit'
+require File.expand_path('../../lib/resque/plugins/workers/lock', __FILE__)
+require 'tempfile'
+require 'timeout'
+
+require_relative 'unique_job_mkey'
+
+class LockMKeyTest < Test::Unit::TestCase
+
+
+  def setup
+    Resque.redis.keys('UniqueJobMkey*').each do |key|
+      Resque.redis.del key
+    end
+  end
+
+  def test_lint
+    assert_nothing_raised do
+      Resque::Plugin.lint(Resque::Plugins::Workers::Lock)
+    end
+  end
+
+  def test_workers_dont_work_simultaneously
+    assert_locking_works_with jobs: 2, workers: 2
+  end
+
+  def test_worker_locks_timeout
+    output_file = Tempfile.new 'output_file'
+
+    Resque.enqueue UniqueJobMKey, job: 'interrupted-job', output_file: output_file.path, sleep: 1000
+
+    worker_pid = start_worker
+    wait_until(10){ lock_has_been_acquired }
+    kill_worker(worker_pid)
+
+    Resque.enqueue UniqueJobMKey, job: 'completing-job', output_file: output_file.path, sleep: 0
+    process_jobs workers: 1, timeout: UniqueJobMKey.worker_lock_timeout + 2
+
+    lines = File.readlines(output_file).map(&:chomp)
+    assert_equal ['starting interrupted-job', 'starting completing-job', 'finished completing-job'], lines
+  end
+
+  private
+
+  def lock_has_been_acquired
+    Resque.redis.exists(UniqueJobMKey.get_lock_workers.first)
+  end
+
+  def kill_worker(worker_pid)
+    Process.kill("TERM", worker_pid)
+    Process.waitpid(worker_pid)
+  end
+
+  def start_worker
+    fork.tap do |pid|
+      if !pid
+        worker = Resque::Worker.new('*')
+        worker.term_child = true
+        worker.reconnect
+        worker.work(0.5)
+        exit!
+      end
+    end
+  end
+
+  def assert_worker_lock_exists(job_class, *args)
+    assert Resque.redis.exists(job_class.get_lock_workers(*args).first, "lock does not exist")
+  end
+
+  def assert_locking_works_with options
+    jobs = (1..options[:jobs]).map{|job| "Job #{job}" }
+    output_file = Tempfile.new 'output_file'
+
+    jobs.each do |job|
+      Resque.enqueue UniqueJobMKey, job: job, output_file: output_file.path
+    end
+
+    process_jobs workers: options[:workers], timeout: 10
+
+    lines = File.readlines(output_file).map(&:chomp)
+    lines.each_slice(2) do |a,b|
+      assert_equal a.split.last,b.split.last, "#{a} was interrupted by #{b}"
+    end
+  end
+
+  def process_jobs options
+    with_workers options[:workers] do
+      wait_until(options[:timeout]) do
+         no_busy_workers && no_queued_jobs
+      end
+    end
+  end
+
+  def with_workers n
+    pids = []
+    n.times do
+      if pid = fork
+        pids << pid
+      else
+        pids = [] # Don't kill from child's ensure
+        worker = Resque::Worker.new('*')
+        worker.term_child = true
+        worker.reconnect
+        worker.work(0.5)
+        exit!
+      end
+    end
+
+    yield
+
+  ensure
+    pids.each do |pid|
+      Process.kill("QUIT", pid)
+    end
+
+    pids.each do |pid|
+      Process.waitpid(pid)
+    end
+  end
+
+  def no_busy_workers
+    Resque::Worker.working.size == 0
+  end
+
+  def no_queued_jobs
+    Resque.redis.llen("queue:mlock_test") == 0
+  end
+
+  def wait_until(timeout)
+    Timeout::timeout(timeout) do
+      loop do
+        return if yield
+        sleep 1
+      end
+    end
+  end
+end

data/test/unique_job_mkey.rb

@@ -0,0 +1,26 @@
+require File.expand_path('../../lib/resque/plugins/workers/lock', __FILE__)
+
+class UniqueJobMKey
+  extend Resque::Plugins::Workers::Lock
+  @queue = :mlock_test
+
+  def self.worker_lock_timeout(*)
+    5
+  end
+
+  def self.lock_workers(*args)
+    [self.name + '1', "#{self.name}#{args.to_s}"]
+  end
+
+  def self.append_output filename, string
+    File.open(filename, 'a') do |output_file|
+      output_file.puts string
+    end
+  end
+
+  def self.perform params
+    append_output params['output_file'], "starting #{params['job']}"
+    sleep(params['sleep'] || 1)
+    append_output params['output_file'], "finished #{params['job']}"
+  end
+end

metadata

@@ -1,90 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: resque-workers-lock
 version: !ruby/object:Gem::Version
-  version: '1.9'
-  prerelease: 
+  version: 2.0.0
 platform: ruby
 authors:
 - Bart Olsthoorn
 - Mike Nicholaides
 - Jason Garber
+- Tijs Planckaert
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-27 00:00:00.000000000 Z
+date: 2014-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ! 'A Resque plugin. Two or more jobs with the same lock cannot be processed
-  simultaneously by multiple workers.
-
+description: |
+  A Resque plugin. Two or more jobs with the same lock cannot be processed simultaneously by multiple workers.
   When this situation occurs the second job gets pushed back to the queue.
-
-'
 email: bartolsthoorn@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
 - README.md
 - Rakefile
-- LICENSE
 - lib/resque/plugins/workers/lock.rb
+- test/lock_mkey_test.rb
 - test/lock_test.rb
 - test/unique_job.rb
+- test/unique_job_mkey.rb
 homepage: http://github.com/bartolsthoorn/resque-workers-lock
-licenses: []
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Resque plugin, prevent specific jobs to be processed simultaneously by multiple
   workers.
 test_files: []