resque-serial-queues 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 34c9b997a83c81205f422f22b616de56ba0516b4
+  data.tar.gz: ed19138a00d01994dce31852244ad817b7e7e0b4
+SHA512:
+  metadata.gz: 6b2a21f5b1fd9ac020d3a538fd6b56ff76c98c76cf695d6c83ab8a607e662b6c3285424655fdc8dcab78a0a8d969192db9a93f1e0d59120eeb64a1088360d371
+  data.tar.gz: 045083e0b65352dc9aa272792130f00a38a8d2fb1da124c72df2f4bd84acbff244b265bf0c67eaeb1399e8b29586ba7433708c1ddb7d27394cd49bcf70f0d959

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.0.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in resque-serial-queues.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Cristian Bica
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# Resque::Plugins::SerialQueues
+
+Declaring resque queues as serial.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'resque-serial-queues'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install resque-serial-queues
+
+## Usage
+
+```ruby
+
+Resque::Plugins::SerialQueues.configure do |config|
+  config.serial_queues = [:serial_jobs]
+  config.lock_timeout  = 120  #default 3600 seconds
+end
+
+class MyJob
+
+  @queue = :serial_jobs
+
+  def perform(*)
+  end
+
+end
+
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/cristianbica/resque-serial-queues/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/cases/**/*.rb']
+  t.verbose = true
+end
+
+task :default => :test
+

data/lib/resque-serial-queues.rb

@@ -0,0 +1,5 @@
+require 'resque-serial-queues/version'
+require 'resque/plugins/serial_queues'
+require 'resque/plugins/serial_queues/config'
+require 'resque/plugins/serial_queues/resque_extension'
+

data/lib/resque-serial-queues/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Plugins
+    module SerialQueues
+      VERSION = "0.0.1"
+    end
+  end
+end

data/lib/resque/plugins/serial_queues.rb

@@ -0,0 +1,38 @@
+module Resque
+  module Plugins
+    module SerialQueues extend self
+      def config
+        @config ||= Config.new
+      end
+
+      def configure
+        yield(config) if block_given?
+      end
+
+      def redis
+        @redis ||= Redis::Namespace.new(:serial_queues, redis: Resque.redis)
+      end
+
+      def self.is_queue_serial?(queue)
+        config.serial_queues.include?(queue.to_s)
+      end
+
+      def self.lock_queue(queue)
+        if redis.setnx("queue-lock:#{queue}", 1)
+          redis.expire("queue-lock:#{queue}", config.lock_timeout)
+          true
+        else
+          false
+        end
+      end
+
+      def self.is_queue_locked?(queue)
+        redis.exists("queue-lock:#{queue}")
+      end
+
+      def self.unlock_queue(queue)
+        redis.del("queue-lock:#{queue}")
+      end
+    end
+  end
+end

data/lib/resque/plugins/serial_queues/config.rb

@@ -0,0 +1,20 @@
+module Resque
+  module Plugins
+    module SerialQueues
+      class Config
+        attr_accessor :serial_queues
+        attr_accessor :lock_timeout
+
+        def initialize
+          self.serial_queues = []
+          self.lock_timeout  = 60*60
+        end
+
+        def serial_queues=(queues)
+          raise "queues should be an Array" unless queues.is_a?(Array)
+          @serial_queues = queues.map(&:to_s)
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/serial_queues/resque_extension.rb

@@ -0,0 +1,25 @@
+require 'resque'
+require 'resque/job'
+
+class Resque::Job
+  def self.reserve(queue)
+    return if is_serial_queue?(queue) and not lock_queue(queue)
+    return unless payload = Resque.pop(queue)
+    new(queue, payload)
+  end
+
+  protected
+    def self.is_serial_queue?(queue)
+      Resque::Plugins::SerialQueues.is_queue_serial?(queue)
+    end
+
+    def self.lock_queue(queue)
+      Resque::Plugins::SerialQueues.lock_queue(queue)
+    end
+end
+
+class Object
+  def self.after_perform_unlock_serial_queue(*)
+    Resque::Plugins::SerialQueues.unlock_queue(@queue) if Resque::Plugins::SerialQueues.is_queue_serial?(@queue)
+  end
+end

data/resque-serial-queues.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque-serial-queues/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "resque-serial-queues"
+  spec.version       = Resque::Plugins::SerialQueues::VERSION
+  spec.authors       = ["Cristian Bica"]
+  spec.email         = ["cristian.bica@gmail.com"]
+  spec.summary       = %q{Declare resque queues serial}
+  spec.description   = %q{Declare resque queues serial and jobs in that queue will be run in serial mode}
+  spec.homepage      = "http://github.com/cristianbica/resque-serial-queues"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'resque', '~>1.0'
+  spec.add_dependency 'redis-namespace'
+  spec.add_dependency 'activesupport'
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+end

data/test/cases/locking_test.rb

@@ -0,0 +1,75 @@
+require 'minitest_helper'
+require 'jobs/sleeping_job'
+require 'jobs/delayed_failing_job'
+require 'jobs/benchmark_job'
+
+class LockingTest < ResqueSerialQueuesTest
+
+  def test_checking_if_a_queue_is_serial
+    assert_serial_queue :serial_jobs
+  end
+
+  def test_checking_if_a_queue_is_not_serial
+    assert_concurrent_queue :parallel_jobs
+  end
+
+  def test_locking_a_queue
+    assert Resque::Plugins::SerialQueues.lock_queue(:serial_jobs)
+    assert_queue_locked :serial_jobs
+  end
+
+  def test_queue_lock_timeout
+    timeout_was = Resque::Plugins::SerialQueues.config.lock_timeout
+    Resque::Plugins::SerialQueues.config.lock_timeout = 1
+    Resque::Plugins::SerialQueues.lock_queue(:serial_jobs)
+    assert_queue_locked :serial_jobs
+    sleep 1.1
+    assert_queue_unlocked :serial_jobs
+  ensure
+    Resque::Plugins::SerialQueues.config.lock_timeout = timeout_was
+  end
+
+  def test_a_queue_is_locked_while_running
+    self.class.start_redis_workers 1
+    Resque.enqueue SleepingJob, 1
+    sleep 0.5
+    assert_queue_locked :serial_jobs
+  end
+
+  def test_a_queue_is_not_locked_after_running
+    self.class.start_redis_workers 1
+    Resque.enqueue SleepingJob, 0.1
+    sleep 0.4
+    assert_queue_unlocked :serial_jobs
+  end
+
+  def test_a_queue_is_not_locked_after_a_job_fails
+    self.class.start_redis_workers 1
+    Resque.enqueue DelayedFailingJob, 0.1
+    sleep 0.4
+    assert_queue_unlocked :serial_jobs
+  end
+
+  def test_running_many_concurrent_jobs
+    jobs = 1000
+    max_sleep = 0.05
+    workers = 20
+    self.class.start_redis_workers workers
+    jobs.times do
+      Resque.enqueue BenchmarkJob, max_sleep
+    end
+    assert_queue_locked :serial_jobs
+    while Resque.info[:pending]>0
+      sleep 0.1
+      assert Resque.info[:working]<=1
+    end
+    assert_queue_unlocked :serial_jobs
+    results = []
+    while row = Resque::Plugins::SerialQueues.redis.lpop("benchmark-results")
+      results << row.split("-").map(&:to_f)
+    end
+    results.flatten!
+    assert_equal results, results.sort
+  end
+
+end

data/test/jobs/benchmark_job.rb

@@ -0,0 +1,11 @@
+class BenchmarkJob
+  @queue = :serial_jobs
+
+  def self.perform(max_seconds)
+    t0 = Time.now.to_f
+    sleep rand*1000%max_seconds
+    t1 = Time.now.to_f
+    Resque::Plugins::SerialQueues.redis.rpush "benchmark-results", [t0, t1].join("-")
+  end
+
+end

data/test/jobs/delayed_failing_job.rb

@@ -0,0 +1,8 @@
+class DelayedFailingJob
+  @queue = :serial_jobs
+
+  def self.perform(seconds)
+    sleep seconds
+    raise
+  end
+end

data/test/jobs/sleeping_job.rb

@@ -0,0 +1,7 @@
+class SleepingJob
+  @queue = :serial_jobs
+
+  def self.perform(seconds)
+    sleep seconds
+  end
+end

data/test/minitest_helper.rb

@@ -0,0 +1,125 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'resque-serial-queues'
+require "minitest"
+require 'minitest/test'
+Minitest.autorun
+
+Resque::Plugins::SerialQueues.configure do |config|
+  config.serial_queues = [:serial_jobs]
+  config.lock_timeout  = 60
+end
+
+Resque.redis = "127.0.0.1:6379:10/resque-serial-queues:resque"
+
+$redis_workers_pids = []
+
+class ResqueSerialQueuesTest < Minitest::Test
+
+  #alias_method :run_without_resque_serial_queue_hooks, :run
+  def run(*args)
+    log "Running test #{self.name}"
+    self.class.clear_redis_keys
+    r = super
+    self.class.stop_redis_workers
+    self.class.clear_redis_keys
+    log "Finished running test #{self.name}"
+    r
+  end
+
+  def assert_queue_locked(queue)
+    Resque::Plugins::SerialQueues.is_queue_locked?(queue)
+  end
+
+  def assert_queue_unlocked(queue)
+    !Resque::Plugins::SerialQueues.is_queue_locked?(queue)
+  end
+
+  def assert_serial_queue(queue)
+    Resque::Plugins::SerialQueues.is_queue_serial?(queue)
+  end
+
+  def assert_concurrent_queue(queue)
+    !assert_serial_queue(queue)
+  end
+
+  def log(message)
+    self.class.log(message)
+  end
+
+  def self.log(message)
+    puts "[PID: #{Process.pid}] [#{Time.now.to_f}] #{message}"
+  end
+
+  protected
+    def self.start_redis_workers(number_of_workers = 1)
+      number_of_workers.times do
+        start_worker
+      end
+    end
+
+    def self.start_worker
+      if pid = Kernel.fork
+        log "Forked to pid #{pid} to start worker"
+        started = wait_for_worker_to_start(pid)
+        log "Waiting for worker to start"
+        if started
+          log "Worker started"
+          $redis_workers_pids << pid
+        else
+          raise "Failed to start worker"
+        end
+      else
+        Resque.redis.client.reconnect
+        worker = Resque::Worker.new("*")
+        worker.term_timeout = 4.0
+        worker.term_child = true
+        Resque.logger.level = Logger::DEBUG
+        log "Worker started working"
+        worker.work 0.1
+        Kernel.exit!
+      end
+    end
+
+    def self.wait_for_worker_to_start(pid)
+      worker_started = false
+      25.times do
+        worker_started = worker_exists?(pid)
+        break if worker_started
+        sleep 0.2
+      end
+      worker_started
+    end
+
+    def self.wait_for_worker_to_finish(pid)
+      worker_finished = false
+      25.times do
+        worker_finished = !worker_exists?(pid)
+        break if worker_finished
+        sleep 0.2
+      end
+      worker_finished
+    end
+
+    def self.worker_exists?(pid)
+      Resque::Worker.all.map(&:to_s).grep(/\:#{pid}\:/).any?
+    end
+
+    def self.stop_redis_workers
+      while pid = $redis_workers_pids.pop do
+        log "Stopping worker with pid #{pid} (QUIT)"
+        Process.kill "QUIT", pid
+        log "Stopping worker with pid #{pid} (TERM)" and Process.kill "TERM", pid unless wait_for_worker_to_finish(pid)
+        log "Stopped worker with pid #{pid}"
+        Process.kill 9, pid rescue nil
+      end
+    end
+
+    def self.clear_redis_keys
+      log "Clear redis keys"
+      Resque.redis.keys("*").each do |key|
+        Resque.redis.del(key)
+      end
+    end
+
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: resque-serial-queues
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Cristian Bica
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Declare resque queues serial and jobs in that queue will be run in serial
+  mode
+email:
+- cristian.bica@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/resque-serial-queues.rb
+- lib/resque-serial-queues/version.rb
+- lib/resque/plugins/serial_queues.rb
+- lib/resque/plugins/serial_queues/config.rb
+- lib/resque/plugins/serial_queues/resque_extension.rb
+- resque-serial-queues.gemspec
+- test/cases/locking_test.rb
+- test/jobs/benchmark_job.rb
+- test/jobs/delayed_failing_job.rb
+- test/jobs/sleeping_job.rb
+- test/minitest_helper.rb
+homepage: http://github.com/cristianbica/resque-serial-queues
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Declare resque queues serial
+test_files:
+- test/cases/locking_test.rb
+- test/jobs/benchmark_job.rb
+- test/jobs/delayed_failing_job.rb
+- test/jobs/sleeping_job.rb
+- test/minitest_helper.rb
+has_rdoc: