delayed_job_active_record_threaded 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 981142e7ce0bc0a93febb2fd65d0b9d6154a19f0
-  data.tar.gz: 8d8fb58d77e0cb70776512c82ad293565816def5
+  metadata.gz: 56c9fe953525b72cc2c50364109a34b1ef26e445
+  data.tar.gz: a87b84d2c0372b7591fd1be25a9d934f97edcc6f
 SHA512:
-  metadata.gz: 0c6a4748e4f00dfce80ae4c83bf17ceb9a59feb6a8315987adafcab9dbc8e8573fe79b34ba97bd3d3b936105ed7ad630ecf9cf18ebbfc2c4d2161484ec5b38d9
-  data.tar.gz: 6634d85d67102b7097c73213dce1e42607f8dd7b04592d67c2a843e22765a77aa3af9ded22e82800a790388ea6d7d835ccd75100dbb86c7e19648619245f01cc
+  metadata.gz: 1348de2ee8adf7da6ef92aa7655f3db0d6765f343513b8dec9f05d5b019e58cac2f485cbc1b131de288428f99123a9e5700cf2e881cafc50fdd002b8ac6efc71
+  data.tar.gz: a203944b180322e06e456692e87090c14c3d5b59eaadd1299672006fd37eecc4e36fe160db27dcb224789eeb8317702a1f846bac73a38bcd08fea358683bd738

data/.gitignore

@@ -17,3 +17,4 @@ test/version_tmp
 tmp
 .idea/workspace.xml
 .idea/
+.DS_Store

data/.travis.yml

@@ -8,9 +8,13 @@ only:
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1.0
+  - 2.1.2
 env:
   matrix:
     - "RAILS_VERSION=\"~> 3.0.0\""
     - "RAILS_VERSION=\"~> 3.1.0\""
     - "RAILS_VERSION=\"~> 3.2.0\""
-    - "RAILS_VERSION=\"~> 4.0.0\""
+    - "RAILS_VERSION=\"~> 4.0.0\""
+    - "RAILS_VERSION=\"~> 4.1.0\""
+    - "RAILS_VERSION=\"~> 4.1.1\""

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## v0.0.5
+* enabled for rails 4.1+
+
+## v0.0.4
+
+* added tests
+* fixed issues with default queue
+* fixed little bugs on HomeManager exit
+* enhanced documentation
+
 ## v0.0.3
 
 * fixed tests and travis related stuff

data/README.md

@@ -1,10 +1,12 @@
 # DelayedJobActiveRecordThreaded [![Build Status](https://travis-ci.org/zxiest/delayed_job_active_record_threaded.png)](http://travis-ci.org/zxiest/delayed_job_active_record_threaded)
 
+<br/>
 DelayedJob allows you to execute long-running jobs at a later time. For more information on how to create and delay a job, follow https://github.com/collectiveidea/delayed_job
 
-This gem processes your delayed jobs with a single threaded process (instead of multiple processes). 
+This gem processes your delayed jobs with a single multi-threaded process (instead of multiple processes).
 This helps avoid database deadlocks and saves computing resources.
 
+<hr/>
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -19,9 +21,8 @@ Or install it yourself as:
 
     $ gem install delayed_job_active_record_threaded
 
-
 ## Setup
-Make sure to pick the correct setup method prior to proceeding:
+<b>Make sure to pick the correct setup method prior to proceeding</b>
 
 ### Fresh setup
 Follow this setup method if you do have not been using delayed_job_active_record in your project and do not have a "delayed_jobs" table created:
@@ -30,36 +31,77 @@ Follow this setup method if you do have not been using delayed_job_active_record
     $ rake db:migrate
 
 ### Upgrading from delayed_job_active_record
-
 If you were previously using delayed_job_active_record, follow the steps below:
 
     $ rails generate delayed_job:upgrade
     $ rake db:migrate
     
-This will add columns to your delayed_jobs table.
+The lines above will add additional columns to your delayed_jobs table.
 
 ## Usage
+### Delaying work
+Any object that respond_to? perform can be delayed.
+
+    class EbookProcessor
+        def initialize(ebook_id)
+            @ebook_id = ebook_id
+        end
+
+        def perform
+            ebook = Ebook.find_by_id(@ebook_id)
+            # run long-running process
+        end
+    end
+
+In order to create a job and push it to the queue, do the following:
+
+    Delayed::Job.enqueue EbookProcessor.new(Ebook.find_by_name('The Power of Habit').id), :queue => 'ebooks_queue'
+
+When :queue is not provided, the job will be pushed to the default queue (queue with name 'default' or nil). 
+Read below if you wish to process multiple queues simultaneously.
+
+Another way of delaying work is to call the method delay on an object:
+
+    EbookProcessor.new(Ebook.find_by_name('The Power of Habit').id).delay
+
+### Delaying mailers
+
+Mailers work differently and can be delayed this way:
 
-For information on how to create delayed_jobs and enqueue them, follow https://github.com/collectiveidea/delayed_job
+    UserMailer.delay(:queue => 'mailers').welcome_user(13)
 
-In order to start the threaded process this gem provides, use the following:
+<hr/>
+### WARNING
+As you may have noticed in call to welcome_user above, we did not pass a user object but rather the user's id. This is a very important notion when using delayed_job.
+Never pass an object to delayed job unless you are certain it can be deserialized.
+Especially avoid passing ActiveRecord::Base objects. If you do, you will be wondering why your delayed_jobs are failing silently.
 
-### Defaults
+For instance, the following code is <b>BAD</b>:
+
+    # WARNING: NEVER DO THIS
+    @user = User.find(13)
+    UserMailer.delay(:queue => 'mailers').welcome_user(@user)
+
+</span></p><br/>
+For more information about delayed_jobs, check out https://github.com/collectiveidea/delayed_job
+
+<hr/>
+## Running Queue Processors
+
+### Default
+The commands below are equivalent and run a process that goes through the jobs in the delayed_jobs table:
 
     $ bundle exec rake delayed_job:start
-or
     $ bundle exec rake dj:start
 
-### With Options
+### Multiple Queues and Extra Options
+The default task processes all the jobs in all queues. In order to allow queue separation and prioritization, you would want to categorize your jobs and push them to different queues.
 
-The default task processes items belonging to all the queues. This is, most of times, not desirable as tasks with higher priority in a given queue take precedence over tasks in another queue.
+Also, you can have multiple queues processed simultaneously, each with its own options. In the example below, we assume you have an "ebooks" and an "albums" queues.
 
-You can have multiple queues processed simultaneously, each with its own options.
-In the example below, we assume you have an "ebooks" and an "albums" queues.
+The commands below will process the queues ebooks and albums simultaneously:
 
-The command below will process the queues ebooks and albums simultaneously
     $ bundle exec rake "dj:start[ebooks&albums]"
-
     $ bundle exec rake "dj:start[ebooks[workers_number]=16&ebooks[worker_timeout]=60&albums[workers_number]=32&albums[worker_timeout]=120]"
 
 The expression "ebooks[workers_number]=16&ebooks[worker_timeout]=60" is a parsed similarly to a URL's query string and would be converted into the following hash:
@@ -68,7 +110,20 @@ The expression "ebooks[workers_number]=16&ebooks[worker_timeout]=60" is a parsed
 
 The rake task above will start a process in the background and process two queues, "ebooks" with 16 working threads and times out after 60 seconds and "albums" with 32 working threads and times out after 120 seconds.
 
-Default options:
+Note that, if you specify the name of one queue, you must specify the other queues as well if you wish them to be processed.
+
+### Starting/Stopping on Production
+In order to start and stop delayed_job on production (or any other environenment), pass along the RAILS_ENV variable.
+
+    $ RAILS_ENV=production bundle exec rake dj:start
+    $ RAILS_ENV=production bundle exec rake dj:stop
+    
+The pid file will be in tmp/pids/delayed_job_production.pid and the log file will be in log/delayed_job_production.log
+
+Note that the task dj:start spawns a background process that calls dj:run and does not die when you disconnect from the terminal.
+
+<b>Default options:</b>
+    
     workers_number: 16
     worker_timeout: 60
 

data/delayed_job_active_record_threaded.gemspec

@@ -3,30 +3,29 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
-  spec.name          = "delayed_job_active_record_threaded"
-  spec.authors       = ["Abdo Achkar"]
-  spec.email         = ["abdo.achkar@gmail.com"]
+  spec.name          = 'delayed_job_active_record_threaded'
+  spec.authors       = ['Abdo Achkar']
+  spec.email         = ['abdo.achkar@gmail.com']
   spec.description   = %q{Allows going through delayed job queues using threads instead of processes}
   spec.summary       = %q{Process the delayed job queue using threads.}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.homepage      = ''
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files`.split($/)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "minitest", "~> 4.7.3"
-  spec.add_development_dependency "minitest-rails", "~> 0.9.1"
-  spec.add_development_dependency "rails", ['>= 3.0', '< 4.1']
-  spec.add_development_dependency "protected_attributes"
-  spec.add_development_dependency "turn"
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'minitest', '~> 4.7.3'
+  spec.add_development_dependency 'minitest-rails', '~> 0.9.1'
+  spec.add_development_dependency 'rails', ['>= 3.0']
+  spec.add_development_dependency 'protected_attributes'
+  spec.add_development_dependency 'rake'
   spec.add_development_dependency 'sqlite3-ruby', '>= 1.3.1'
-  spec.add_development_dependency "mysql2"
-  spec.add_dependency   'activerecord', ['>= 3.0', '< 4.1']
-  spec.add_dependency   'delayed_job',  ['>= 3.0', '< 4.1']
+  spec.add_development_dependency 'mysql2'
+  spec.add_dependency   'activerecord', ['>= 3.0']
+  spec.add_dependency   'delayed_job',  ['>= 3.0']
   spec.add_dependency   'celluloid', ['>=0.14.1']
-  spec.version = "0.0.3"
+  spec.version = '0.0.5'
 end

data/lib/delayed/home_manager.rb

@@ -31,6 +31,8 @@ module Delayed
       @workers_number ||= DEFAULT_WORKERS_NUMBER
       @max_attempts ||= DEFAULT_MAX_ATTEMPTS
       @worker_options ||= {}
+
+      # if @queue is nil, we'll pull from all queues
     end
 
     def start
@@ -49,11 +51,11 @@ module Delayed
       @timer = every(@sleep_time) {
         begin
           if (!@alive)
-            @timer.stop
+            @timer.cancel
+          else
+            jobs = pull_next(@queue, @workers_pool.idle_size)
+            jobs.each { |j| @workers_pool.async.work(j) }
           end
-
-          jobs = pull_next(@queue, @workers_pool.idle_size)
-          jobs.each { |j| @workers_pool.async.work(j) }
         rescue
           # logging error watch
           begin
@@ -77,10 +79,15 @@ module Delayed
     def pull_next(queue=nil, n=15)
       ids = []
       Delayed::Job.transaction do
-        query = Delayed::Job.where("(run_at is null or run_at < ?) and locked_at is null", DateTime.now).order("priority asc, run_at asc, id asc")
-        if (queue)
+        query = Delayed::Job.where('(run_at is null or run_at < ?) and locked_at is null', DateTime.now).order('priority asc, run_at asc, id asc')
+        if (queue && queue != 'default')
           query = query.where(:queue => queue)
+        # if the queue is 'default' or nil, this would be the "default queue"
+        elsif queue == 'default'
+          query = query.where(:queue => ['default', nil])
         end
+        # if no queue name is provided, the "queue" column would be ignored
+        # therefore returning the list of all jobs in all queues
 
         query = query.limit(n)
         ids = query.pluck(:id)

data/lib/delayed/home_worker.rb

@@ -37,6 +37,8 @@ module Delayed
             job.attempts += 1
             job.run_at = (job.attempts ** 4 + 5).seconds.from_now
             job.save
+          else
+            job.delete
           end
         end
 

data/lib/tasks/dj.rake

@@ -52,7 +52,7 @@ namespace :delayed_job do
       workers_timeout ||= DEFAULT_WORKERS_TIMEOUT
 
       # set to nil of queue_name is default
-      queue_name = nil if (queue_name == "default")
+      queue_name = nil if (queue_name == 'default')
 
       puts "queue: #{queue_name}, workers_number: #{workers_number}, workers_timeout: #{workers_timeout}"
 
@@ -66,14 +66,20 @@ namespace :delayed_job do
   end
 
   def dj_pid
-    return 'tmp/pids/delayed_job.pid'
+    return "tmp/pids/delayed_job_#{Rails.env}.pid" if Rails.env
+    'tmp/pids/delayed_job.pid'
+  end
+
+  def delayed_job_output
+    return "log/delayed_job_#{Rails.env}.log" if Rails.env
+    'log/delayed_job.log'
   end
 
   task :start, [ :args_expr ] => :environment do |t, args|
     puts "#{args[:args_expr]}"
 
     cmd = %(if [ -f #{dj_pid} ] && [ -n `cat #{dj_pid}` ] && ps -p `cat #{dj_pid}` > /dev/null; then kill `cat #{dj_pid}`; fi
-            (bundle exec rake "delayed_job:run[#{args[:args_expr]}]" >> log/delayed_job.log 2>&1) & (echo $! > #{dj_pid})
+            (RAILS_ENV=#{Rails.env} bundle exec rake "delayed_job:run[#{args[:args_expr]}]" >> #{delayed_job_output} 2>&1) & (echo $! > #{dj_pid})
            )
 
     puts "executing: #{cmd}"

data/test/delayed_job_active_record_threaded_test.rb

@@ -1,8 +1,8 @@
-require "test_helper"
-require "delayed_job_active_record_threaded"
-require "rails"
-require "rails/test_help"
-require "minitest/rails"
+require 'test_helper'
+require 'delayed_job_active_record_threaded'
+require 'rails'
+require 'rails/test_help'
+require 'minitest/rails'
 
 class TestJob < Struct.new(:id)
   def perform
@@ -12,24 +12,48 @@ class TestJob < Struct.new(:id)
   end
 end
 
+class CrashingJob < TestJob
+  def perform
+    raise 'failing job'
+  end
+end
+
 class DelayedJobActiveRecordThreadedTest < ActiveSupport::TestCase #MiniTest::Unit::TestCase
   self.use_transactional_fixtures = false
 
-  QUEUE_NAME = "StoriesQueue"
+  QUEUE_NAME = 'StoriesQueue'
+  WORKING_STORIES = 100
+  CRASHING_STORIES = 10
+  STORIES_WITH_NO_QUEUE_NAME = 10
+
+  def prepare_stories_with_no_queue_name
+    STORIES_WITH_NO_QUEUE_NAME.times { |i|
+      s = Story.create!(:text => "Story #{i}")
+      Delayed::Job.enqueue(CrashingJob.new(s.id), :priority => i, :delayable_id => s.id, :delayable_type => s.class.name, :run_at => 10.seconds.ago)
+    }
+  end
+
+  def prepare_failing_stories
+    CRASHING_STORIES.times { |i|
+      s = Story.create!(:text => "Story #{i}")
+      Delayed::Job.enqueue(CrashingJob.new(s.id), :queue => QUEUE_NAME, :priority => i, :delayable_id => s.id, :delayable_type => s.class.name, :run_at => 10.seconds.ago)
+    }
+  end
 
   before do
     #Delayed::Job.new(:run_at => 10.seconds.ago).save
-    100.times { |i|
+    WORKING_STORIES.times { |i|
       s = Story.create!(:text => "Story #{i}")
       Delayed::Job.enqueue(TestJob.new(s.id), :queue => QUEUE_NAME, :priority => i, :delayable_id => s.id, :delayable_type => s.class.name, :run_at => 10.seconds.ago)
     }
   end
 
   after do
+    Story.delete_all
   end
 
   # this is VERY dependent on the database pool size
-  test "should process jobs without crashing" do
+  test 'should process jobs without crashing' do
     Celluloid.boot # fixes Celluloid issue "Thread pool is not running"
     mgr = Delayed::HomeManager.new({ :sleep_time => 1, :workers_number => 25, :queue => QUEUE_NAME })
 
@@ -43,6 +67,65 @@ class DelayedJobActiveRecordThreadedTest < ActiveSupport::TestCase #MiniTest::Un
 
     puts "Number of threads: #{Thread.list.count}"
 
-    assert Delayed::Job.where("queue = ?", QUEUE_NAME).count == 0, "Expected to have an empty queue after 10 seconds. Queue size was #{Delayed::Job.count} instead"
+    assert Delayed::Job.where('queue = ?', QUEUE_NAME).count == 0, "Expected to have an empty queue after 10 seconds. Queue size was #{Delayed::Job.count} instead"
+  end
+
+  test 'pool should maintain the number of workers and not crash' do
+    Celluloid.boot # fixes Celluloid issue "Thread pool is not running"
+    mgr = Delayed::HomeManager.new({ :sleep_time => 0.5, :workers_number => 50, :queue => QUEUE_NAME })
+    mgr.start
+    sleep(3)
+
+    assert mgr.workers_pool.size == 50, "Pool is expected to maintain its size, got size of #{mgr.workers_pool.size} instead"
+
+    assert mgr.alive, 'Expected Manager to be alive after completing queue'
+
+    mgr.kill
+  end
+
+  test 'should succeed and maintain pool size even if workers have crashing job' do
+    prepare_failing_stories
+
+    assert (Story.count == (CRASHING_STORIES+WORKING_STORIES)), "test expects to start with #{(CRASHING_STORIES+WORKING_STORIES)} stories, got #{Story.count} instead"
+
+    Celluloid.boot # fixes Celluloid issue "Thread pool is not running"
+    mgr = Delayed::HomeManager.new({ :sleep_time => 0.5, :workers_number => 50, :queue => QUEUE_NAME })
+
+    mgr.start
+    sleep(3)
+
+    assert mgr.workers_pool.size == 50, "Pool is expected to maintain its size, got size of #{mgr.workers_pool.size} instead"
+
+    assert mgr.alive, 'Expected Manager to be alive after completing queue'
+
+    mgr.kill
+  end
+
+  test 'should work if things are queued without a queue name' do
+    prepare_stories_with_no_queue_name
+
+    Celluloid.boot # fixes Celluloid issue "Thread pool is not running"
+    mgr = Delayed::HomeManager.new({ :sleep_time => 0.5, :workers_number => 50 })
+
+    mgr.start
+    sleep(3)
+
+    assert mgr.workers_pool.size == 50, "Pool is expected to maintain its size, got size of #{mgr.workers_pool.size} instead"
+
+    assert mgr.alive, 'Expected Manager to be alive after completing queue'
+
+    mgr.kill
+  end
+
+  test 'should not touch a queue that we are not processing' do
+    prepare_stories_with_no_queue_name
+
+    Celluloid.boot # fixes Celluloid issue "Thread pool is not running"
+    mgr = Delayed::HomeManager.new({ :sleep_time => 0.5, :workers_number => 50, :queue => QUEUE_NAME })
+
+    mgr.start
+    sleep(3)
+
+    assert Delayed::Job.where('queue is null').count == STORIES_WITH_NO_QUEUE_NAME, "Expected jobs in queue NIL to remain untouched. Instead, got only #{Delayed::Job.count} remaining"
   end
 end

data/test/test_helper.rb

@@ -1,6 +1,5 @@
 require 'minitest/unit'
 require 'minitest/autorun'
-require 'turn'
 
 
 require 'yaml'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: delayed_job_active_record_threaded
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Abdo Achkar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-18 00:00:00.000000000 Z
+date: 2014-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -59,9 +59,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '4.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -69,9 +66,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: protected_attributes
   requirement: !ruby/object:Gem::Requirement
@@ -86,20 +80,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: turn
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -149,9 +129,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -159,9 +136,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: delayed_job
   requirement: !ruby/object:Gem::Requirement
@@ -169,9 +143,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -179,9 +150,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
@@ -246,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Process the delayed job queue using threads.