resque-serial-queues 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34c9b997a83c81205f422f22b616de56ba0516b4
-  data.tar.gz: ed19138a00d01994dce31852244ad817b7e7e0b4
+  metadata.gz: cd531373558ed3d2a84f9e421a3233c9ab905d1f
+  data.tar.gz: 9de3eb0f08eb657bd08c24aea3631e63da08373c
 SHA512:
-  metadata.gz: 6b2a21f5b1fd9ac020d3a538fd6b56ff76c98c76cf695d6c83ab8a607e662b6c3285424655fdc8dcab78a0a8d969192db9a93f1e0d59120eeb64a1088360d371
-  data.tar.gz: 045083e0b65352dc9aa272792130f00a38a8d2fb1da124c72df2f4bd84acbff244b265bf0c67eaeb1399e8b29586ba7433708c1ddb7d27394cd49bcf70f0d959
+  metadata.gz: 8fff5b737059347db4d065cfcdc542199a7bde5cd41b3debecadaa1b115de2ce2f1d7080dc3558a87730fde750ff94012b39d5d41e5d9d51740988045e0955be
+  data.tar.gz: cfc9029142c28796b5b500a533eaaf06eb39ee659517724daddd4edbee16d47cdfd024b2b36c502f5d7e5911b762bd0ee04057b6702bcf2781c9cac1d63eb693

data/README.md

@@ -1,6 +1,13 @@
 # Resque::Plugins::SerialQueues
 
-Declaring resque queues as serial.
+This gem is tested on the latest version from Resque 1.x.
+
+Declaring resque queues as serial. Jobs from a serial queue won't be processed more than one at a time event if you have multiple workers / servers. There are similar solutions but none of them worked for me. To lock jobs from running in parallel I used `Redis#setnx`. This implementation has a few sensitive things:
+- `Resque::Job.reserve` is overriden (here the queue locking is done)
+- Added `Object#after_perform_unlock_queue_if_serial` (here the queue is unlocked)
+- Added `Object#on_failure_unlock_queue_if_serial` (here the queue is unlocked)
+
+I think serial queues are not a scalable. You shouldn't have to have serial queues. This should be a temporary drop-in solution without changing the deployment process. If you really need serial job processing you should start a worker for each serial queue that will listen for jobs only on that queue or migrate to a background processing / messaging tool that provides serial processing out of the box.
 
 ## Installation
 
@@ -16,15 +23,20 @@ Or install it yourself as:
 
     $ gem install resque-serial-queues
 
-## Usage
+## Usage (Rails)
 
-```ruby
+Create a `.rb` file in config/initializers
 
+```ruby
 Resque::Plugins::SerialQueues.configure do |config|
   config.serial_queues = [:serial_jobs]
   config.lock_timeout  = 120  #default 3600 seconds
 end
+```
+
+and use the declared serial queues in your jobs:
 
+```ruby
 class MyJob
 
   @queue = :serial_jobs
@@ -33,7 +45,6 @@ class MyJob
   end
 
 end
-
 ```
 
 ## Contributing

data/lib/resque-serial-queues/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module SerialQueues
-      VERSION = "0.0.1"
+      VERSION = "0.0.2"
     end
   end
 end

data/lib/resque/plugins/serial_queues/resque_extension.rb

@@ -3,8 +3,9 @@ require 'resque/job'
 
 class Resque::Job
   def self.reserve(queue)
-    return if is_serial_queue?(queue) and not lock_queue(queue)
+    return if is_serial_queue?(queue) and is_queue_locked?(queue)
     return unless payload = Resque.pop(queue)
+    lock_queue(queue)
     new(queue, payload)
   end
 
@@ -13,13 +14,21 @@ class Resque::Job
       Resque::Plugins::SerialQueues.is_queue_serial?(queue)
     end
 
+    def self.is_queue_locked?(queue)
+      Resque::Plugins::SerialQueues.is_queue_locked?(queue)
+    end
+
     def self.lock_queue(queue)
       Resque::Plugins::SerialQueues.lock_queue(queue)
     end
 end
 
 class Object
-  def self.after_perform_unlock_serial_queue(*)
+  def self.after_perform_unlock_queue_if_serial(*)
+    Resque::Plugins::SerialQueues.unlock_queue(@queue) if Resque::Plugins::SerialQueues.is_queue_serial?(@queue)
+  end
+
+  def self.on_failure_unlock_queue_if_serial(*)
     Resque::Plugins::SerialQueues.unlock_queue(@queue) if Resque::Plugins::SerialQueues.is_queue_serial?(@queue)
   end
 end

data/test/cases/lint_test.rb

@@ -0,0 +1,14 @@
+require 'minitest_helper'
+require 'resque/plugin'
+class LintTest < Minitest::Test
+
+  def test_resque_plugin_lint
+    begin
+      Resque::Plugin.lint(Resque::Plugins::SerialQueues)
+    rescue Exception => e
+      assert false, "Resque::Plugin.lint raised #{e.inspect}"
+    end
+    assert true
+  end
+
+end

data/test/cases/locking_test.rb

@@ -14,7 +14,7 @@ class LockingTest < ResqueSerialQueuesTest
   end
 
   def test_locking_a_queue
-    assert Resque::Plugins::SerialQueues.lock_queue(:serial_jobs)
+    assert Resque::Plugins::SerialQueues.lock_queue(:serial_jobs), "Should be able to lock a queue but couldn't"
     assert_queue_locked :serial_jobs
   end
 
@@ -51,9 +51,9 @@ class LockingTest < ResqueSerialQueuesTest
   end
 
   def test_running_many_concurrent_jobs
-    jobs = 1000
+    jobs = 10
     max_sleep = 0.05
-    workers = 20
+    workers = 2
     self.class.start_redis_workers workers
     jobs.times do
       Resque.enqueue BenchmarkJob, max_sleep
@@ -61,7 +61,7 @@ class LockingTest < ResqueSerialQueuesTest
     assert_queue_locked :serial_jobs
     while Resque.info[:pending]>0
       sleep 0.1
-      assert Resque.info[:working]<=1
+      assert Resque.info[:working]<=1, "Should not have more than one working worker"
     end
     assert_queue_unlocked :serial_jobs
     results = []

data/test/minitest_helper.rb

@@ -28,19 +28,19 @@ class ResqueSerialQueuesTest < Minitest::Test
   end
 
   def assert_queue_locked(queue)
-    Resque::Plugins::SerialQueues.is_queue_locked?(queue)
+    assert Resque::Plugins::SerialQueues.is_queue_locked?(queue), "Queue #{queue} should be locked but it's unlocked"
   end
 
   def assert_queue_unlocked(queue)
-    !Resque::Plugins::SerialQueues.is_queue_locked?(queue)
+    assert !Resque::Plugins::SerialQueues.is_queue_locked?(queue), "Queue #{queue} should be unlocked but it's locked"
   end
 
   def assert_serial_queue(queue)
-    Resque::Plugins::SerialQueues.is_queue_serial?(queue)
+    assert Resque::Plugins::SerialQueues.is_queue_serial?(queue), "Queue #{queue} should be serial but it's concurrent"
   end
 
   def assert_concurrent_queue(queue)
-    !assert_serial_queue(queue)
+    assert !Resque::Plugins::SerialQueues.is_queue_serial?(queue), "Queue #{queue} should be concurrent but it's serial"
   end
 
   def log(message)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-serial-queues
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Cristian Bica
@@ -114,6 +114,7 @@ files:
 - lib/resque/plugins/serial_queues/config.rb
 - lib/resque/plugins/serial_queues/resque_extension.rb
 - resque-serial-queues.gemspec
+- test/cases/lint_test.rb
 - test/cases/locking_test.rb
 - test/jobs/benchmark_job.rb
 - test/jobs/delayed_failing_job.rb
@@ -144,6 +145,7 @@ signing_key:
 specification_version: 4
 summary: Declare resque queues serial
 test_files:
+- test/cases/lint_test.rb
 - test/cases/locking_test.rb
 - test/jobs/benchmark_job.rb
 - test/jobs/delayed_failing_job.rb