delayed_job_celluloid 0.1.0 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YmNjODUyMWU4YjE2ZWE2ZDBjNjk5OTIxM2U0ZTNmNWU2YTcyODczOA==
-  data.tar.gz: !binary |-
-    ZGZjMTFhZmE4ZjhhNTQ5ZjU5YjFiYjM0ZWM0MjA3YTE3N2UzZWU4MQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    MjJkYjY2OWFlYmVjZjQ1Zjc0MGI3ODYxYWM5M2UxYmUwNzFkMWQzYWQ0OTlm
-    MjFiYzhhZDU0ZDExNWNjNjY4NGNhMmQ5NGIyNWJiZjA3NmI2YzAwYmQzYzBk
-    YjI0MDZkOGY1YmQ1YzIwMzQ1OGQ1ODhmMGMxODIyMDI0OGUxMGM=
-  data.tar.gz: !binary |-
-    NjAzMDMyNjNlNWQ5Njg1NjA2NzhkMjUxNzRlOTNhMGMwNmI4MzBkYzFlZWRi
-    ZDVhNjg5NmE5ZWM2ZDlkYWEwNGMzZjQxNTQ5NjE5OTRlZGEyOGRmYWJkMWNh
-    MTUxYzRkYjBlMTQ1ZjM1MWRiMmU5Y2JhN2IzMDI3NWMwZjUwZTM=
+SHA1:
+  metadata.gz: 13f46b03b4a215168b557850cf06ff0722e4dc6b
+  data.tar.gz: d5e2e5876c613d67497f6ea7dda79fd79c6bae99
+SHA512:
+  metadata.gz: 1fd4632b88ea294c93b012de18da796d977ba8618c42131930449794c6c01f39c6cb05de110a98fa7587fad15be637bc4ce9c1761ee4190bb5d3e1821a2a8e11
+  data.tar.gz: 1dac98c0b4c006d422d03052811196514bb05718971f00c0775d6fb12a8261c060cc492079eef75cb34891a12d5cd2084b91c100241a6d33d041f3689ced79a7

data/README.md

@@ -6,7 +6,7 @@ Based on awesome gems like Sidekiq and Suckerpunch, DelayedJobCelluloid allows d
 
 Add delayed_job_celluloid to your gem file
 
-	gem 'delayed_job_celluloid
+	gem 'delayed_job_celluloid'
 
 Run bundle install
 
@@ -36,7 +36,20 @@ One important thing to bear in mind with this is that you should have a database
   	  	host: localhost
   	  	pool: 5
 
-Currently the gem does not support daemonization of the main process because I haven't needed it as I am using it in conjuction with Unicorn on Heroku.  If you are running your app on Heroku, this gem will allow you to run multiple workers in a single Unicorn process.  An example unicorn config is as follows:
+The gem also does not work particularly well with Sqlite since Sqlite does not handle concurrent reads/writes.  You will see locking errors, although delayed job handles these gracefully by retrying the job later.
+
+## Daemonization
+
+    script/delayed_job_celluloid -d start
+    script/delayed_job_celluloid -d stop
+
+You can also start the daemon with a monitor process that will restart the daemon if it crashes
+
+	script/delayed_job_celluloid -d -m start
+	
+##Heroku
+
+If you are running your app on Heroku, this gem will allow you to run multiple workers in a single Unicorn process.  An example unicorn config is as follows:
 
 	# config/unicorn.rb
 	worker_processes Integer(ENV["WEB_CONCURRENCY"] || 2)
@@ -67,8 +80,6 @@ Currently the gem does not support daemonization of the main process because I h
 	    ActiveRecord::Base.establish_connection
 	end
 
-If you are interested in adding daemonization to the gem itself, feel free to fork it and submit a pull request.
-
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -1,11 +1,30 @@
 require "bundler/gem_tasks"
 require 'rake/testtask'
 
-Rake::TestTask.new(:test) do |test|
+Rake::TestTask.new(:test_launcher) do |test|
   test.libs << 'test'
   #SO MUCH NOISE
   #test.warning = true
-  test.pattern = 'spec/**/*_spec.rb'
+  test.pattern = 'spec/launcher_spec.rb'
 end
 
-task :default => :test
+Rake::TestTask.new(:test_manager) do |test|
+  test.libs << 'test'
+  #SO MUCH NOISE
+  #test.warning = true
+  test.pattern = 'spec/manager_spec.rb'
+end
+
+Rake::TestTask.new(:test_worker) do |test|
+  test.libs << 'test'
+  #SO MUCH NOISE
+  #test.warning = true
+  test.pattern = 'spec/worker_spec.rb'
+end
+
+Rake::TestTask.new(:test_command) do |test|
+  test.libs << 'test'
+  #SO MUCH NOISE
+  #test.warning = true
+  test.pattern = 'spec/command_spec.rb'
+end

data/delayed_job_celluloid.gemspec

@@ -13,16 +13,18 @@ Gem::Specification.new do |s|
   s.homepage      = ""
   s.license       = "MIT"
 
-  s.files         = `git ls-files`.split($/)
+  s.files         = `git ls-files | grep -Ev '^(testapp)'`.split("\n")
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ["lib"]
 
-  s.add_dependency 'delayed_job'
-  s.add_dependency 'celluloid'
+  s.add_dependency 'delayed_job', ">= 4.0.0"
+  s.add_dependency 'celluloid', ">= 0.16.0"
 
   s.add_development_dependency "bundler", "~> 1.3"
   s.add_development_dependency "rake"
   s.add_development_dependency "sqlite3"
-  s.add_development_dependency "delayed_job_active_record"
+  s.add_development_dependency "delayed_job_active_record", ">= 4.0.0"
+  s.add_development_dependency "rails", "~> 4.1.0"
+  s.add_development_dependency "minitest", "~> 5.5.0"
 end

data/lib/delayed_job_celluloid/command.rb

@@ -1,7 +1,7 @@
 $stdout.sync = true
 
 require 'optparse'
-require 'celluloid/autostart'
+require 'celluloid'
 
 module DelayedJobCelluloid
   
@@ -11,32 +11,114 @@ module DelayedJobCelluloid
     
     attr_accessor :worker_count
 
+    def logger
+      DelayedJobCelluloid.logger
+    end
+
     def initialize(args)
       parse_options(args)
     end
-  
+
+    def daemonize 
+      begin
+        require 'daemons'
+        dir = @options[:pid_dir]
+        Dir.mkdir(dir) unless File.exist?(dir)
+        
+        before_fork
+        Daemons.run_proc('delayed_job_celluloid', :dir => @options[:pid_dir], :dir_mode => :normal, :monitor => @monitor, :ARGV => @args) do |*_args|
+          Celluloid.register_shutdown
+          Celluloid.start
+          launch_celluloid(false)
+        end
+        
+      rescue LoadError
+        raise "You need to add gem 'daemons' to your Gemfile if you wish to daemonize delayed_job_celluloid."
+      end
+    end
+
+    def before_fork
+      @files_to_reopen = []
+        ObjectSpace.each_object(File) do |file|
+        @files_to_reopen << file unless file.closed?
+      end
+    end
+    
+    def after_fork
+      @files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, "a+"
+          file.sync = true
+        rescue ::Exception
+        end
+      end
+    end
+
+    # Run Celluloid in the foreground
     def run
-      self_read, self_write = IO.pipe
+
+      # Run in the background if daemonizing
+      (daemonize; return) if @options[:daemonize]
+
+      # Otherwise, run in the foreground
+      launch_celluloid(true)
+    end
+
+    def launch_celluloid(in_foreground = true)
       
-      %w(INT TERM).each do |sig|
-        trap sig do
-          self_write.puts(sig)
+      if in_foreground
+        Celluloid.start
+        self_read, self_write = IO.pipe
+        %w(INT TERM).each do |sig|
+          trap sig do
+            self_write.puts(sig)
+          end
         end
       end
-      
+
       require 'delayed_job_celluloid/launcher'
       @launcher = Launcher.new(@options, @worker_count)
-      
+
+      unless in_foreground
+        after_fork
+        log_file = @options[:log_file] ||= 'delayed_job_celluloid.log'
+        DelayedJobCelluloid.logger = Logger.new(File.join(Rails.root, 'log', log_file))
+        DelayedJobCelluloid.logger.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime}: #{msg}\n"
+        end
+        Delayed::Worker.logger ||= Logger.new(File.join(Rails.root, 'log', 'delayed_job.log'))
+        DelayedJobCelluloid.logger.info 'delayed_job_celluloid daemon started'
+        
+        # Daemonized - wait to receive a signal
+        %w(INT TERM).each do |sig|
+          trap sig do
+            handle_signal(sig)
+          end
+        end
+      end
+
       begin
         @launcher.run
-        
-        while readable_io = IO.select([self_read])
-          signal = readable_io.first[0].gets.strip
-          handle_signal(signal)
+
+        if in_foreground
+          while readable_io = IO.select([self_read])
+            signal = readable_io.first[0].gets.strip
+            handle_signal(signal)
+          end
+        else
+          # Sleep for a bit
+          while true
+            sleep 60
+          end
         end
       rescue Interrupt
+        logger.info 'Shutting down delayed_job_celluloid'
         @launcher.stop
         exit(0)
+
+      rescue => e
+        DelayedJobCelluloid.logger.error "Exception: #{e.message}"
+        DelayedJobCelluloid.logger.info Kernel.caller
       end
     end
     
@@ -45,16 +127,17 @@ module DelayedJobCelluloid
       when 'INT','TERM'
         raise Interrupt
       end
-        
     end
     
     def parse_options(args)
       @options = {
         :quiet => true,
-        :timeout => 8
+        :timeout => 8,
+        :pid_dir => "#{Rails.root}/tmp/pids"
       }
 
       @worker_count = 2
+      @monitor = false
 
       opts = OptionParser.new do |opts|
         opts.banner = "Usage: #{File.basename($0)} [options] start|stop|restart|run"
@@ -90,12 +173,27 @@ module DelayedJobCelluloid
         opts.on('--queue=queue', "Specify which queue DJ must look up for jobs") do |queue|
           @options[:queues] = queue.split(',')
         end
+        opts.on('--pool=queue1[,queue2][:worker_count]', 'Specify queues and number of workers for a worker pool') do |pool|
+          parse_worker_pool(pool)
+        end
         opts.on('--exit-on-complete', "Exit when no more jobs are available to run. This will exit if all jobs are scheduled to run in the future.") do
           @options[:exit_on_complete] = true
         end
         opts.on('-t', '--timeout NUM', "Shutdown timeout") do |prefix|
           @options[:timeout] = Integer(arg)
         end
+        opts.on('-d', '--daemonize', "Daemonize process") do
+          @options[:daemonize] = true
+        end
+        opts.on('-L', '--log FILE', "Name of log file") do |log_file|
+          @options[:log_file] = log_file
+        end
+        opts.on('-m', '--monitor', 'Start monitor process.') do
+          @monitor = true
+        end
+        opts.on('--pid-dir=DIR', 'Specifies an alternate directory in which to store the process ids.') do |dir|
+          @options[:pid_dir] = dir
+        end
       end
       @args = opts.parse!(args)
     end

data/lib/delayed_job_celluloid/launcher.rb

@@ -9,6 +9,7 @@ module DelayedJobCelluloid
     end
 
     def run
+      DelayedJobCelluloid.logger.info 'Launching delayed_job_celluloid'
       manager.async.start
     end
 

data/lib/delayed_job_celluloid/manager.rb

@@ -29,6 +29,7 @@ module DelayedJobCelluloid
     def start
       @ready.each_with_index do |worker, index|
         worker.name = "delayed_job.#{index}"
+        DelayedJobCelluloid.logger.info "Starting worker #{worker.name}"
         worker.async.start 
       end
     end
@@ -90,14 +91,6 @@ module DelayedJobCelluloid
         # They must die but their messages shall live on.
         info "Still waiting for #{@busy.size} busy workers"
 
-        # Re-enqueue terminated jobs
-        # NOTE: You may notice that we may push a job back to redis before
-        # the worker thread is terminated. This is ok because Sidekiq's
-        # contract says that jobs are run AT LEAST once. Process termination
-        # is delayed until we're certain the jobs are back in Redis because
-        # it is worse to lose a job than to run it twice.
-        #Sidekiq::Fetcher.strategy.bulk_requeue(@in_progress.values)
-
         debug "Terminating #{@busy.size} busy worker threads"
         @busy.each do |worker|
           if worker.alive? && t = @threads.delete(worker.object_id)

data/lib/delayed_job_celluloid/version.rb

@@ -1,3 +1,3 @@
 module DelayedJobCelluloid
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/delayed_job_celluloid/worker.rb

@@ -1,5 +1,4 @@
 require 'delayed_job'
-require 'delayed/performable_mailer'
 
 module DelayedJobCelluloid
   class Worker < Delayed::Worker

data/spec/{command_test.rb → command_spec.rb}

@@ -1,9 +1,9 @@
 require_relative 'spec_helper'
 require_relative 'test_job'
 
-class CommandSpec < Minitest::Unit::TestCase
+class CommandSpec < Minitest::Test
   describe 'command' do
-    
+
     it "starts workers" do  
       i = 0  
       while i < 10 do

data/spec/database.yml

@@ -1,3 +1,4 @@
 test:
   adapter: sqlite3
-  database: "spec/db/test.sqlite3"
+  database: "spec/db/test.sqlite3"
+  pool: 10

data/spec/launcher_spec.rb

@@ -1,7 +1,7 @@
 require_relative 'spec_helper'
 require_relative 'test_job'
 
-class LauncherSpec < Minitest::Unit::TestCase
+class LauncherSpec < Minitest::Test
   describe 'launcher' do
     
     it "starts working jobs" do    

data/spec/manager_spec.rb

@@ -1,8 +1,12 @@
 require_relative 'spec_helper'
 
-class ManagerSpec < Minitest::Unit::TestCase
+class ManagerSpec < Minitest::Test
   describe 'manager' do
   
+    before :all do
+      DelayedJobCelluloid::Worker.exit_on_complete = true
+    end
+    
     it "creates N worker instances" do
       mgr = DelayedJobCelluloid::Manager.new({}, 3)
       assert_equal mgr.ready.size, 3
@@ -79,8 +83,9 @@ class ManagerSpec < Minitest::Unit::TestCase
       
       mgr.worker_died(worker, "test")
     
-      assert_equal 0, mgr.ready.size
-      assert_equal 1, mgr.busy.size
+      assert_equal false, mgr.stopped?
+      assert_equal 1, mgr.ready.size
+      assert_equal 0, mgr.busy.size
       worker.verify
     end
   

data/spec/spec_helper.rb

@@ -4,11 +4,11 @@ require 'delayed_job_active_record'
 require 'minitest/autorun'
 require 'minitest/spec'
 require 'minitest/pride'
-require 'rails'
 require 'active_record'
 require 'sqlite3'
 require 'logger'
 require 'celluloid/autostart'
+require 'rails'
 
 ROOT = File.join(File.dirname(__FILE__), '..')
 RAILS_ROOT = ROOT

data/spec/worker_spec.rb

@@ -3,8 +3,8 @@ require_relative 'test_job'
 
 DelayedJobCelluloid::Worker.backend = :active_record
 
-class WorkerSpec < Minitest::Unit::TestCase
-    
+class WorkerSpec < Minitest::Test
+
   describe "Worker" do
 
     before :all do
@@ -12,27 +12,27 @@ class WorkerSpec < Minitest::Unit::TestCase
     end
 
     it "runs jobs" do
-      
+
       manager = Minitest::Mock.new
       async = Minitest::Mock.new
-      100.times {async.expect(:work, nil, [Celluloid::ActorProxy])}
-      100.times {async.expect(:worker_done, nil, [Celluloid::ActorProxy])}
+      100.times {async.expect(:work, nil, [DelayedJobCelluloid::Worker])}
+      100.times {async.expect(:worker_done, nil, [DelayedJobCelluloid::Worker])}
       async.expect(:real_thread, async, [nil, Thread])
       100.times {manager.expect(:async, async, [])}
-      
-    
+
+
       worker = DelayedJobCelluloid::Worker.new({},manager)
       test = TestJob.new
       test.delay.test_this
-      
+
       assert_equal 1, Delayed::Job.count
-      
+
       worker.start
 
       assert_equal 0, Delayed::Job.count
       manager.verify
       async.verify
     end
-    
+
   end
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: delayed_job_celluloid
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Tom Mooney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-01 00:00:00.000000000 Z
+date: 2014-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: delayed_job
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: celluloid
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.16.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.16.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -56,44 +56,72 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: delayed_job_active_record
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 5.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 5.5.0
 description: run delayed_job workers in multiple threads within a single process
 email:
 - tmooney3979@gmail.com
@@ -115,7 +143,7 @@ files:
 - lib/delayed_job_celluloid/worker.rb
 - lib/generators/delayed_job_celluloid/delayed_job_celluloid_generator.rb
 - lib/generators/delayed_job_celluloid/templates/script
-- spec/command_test.rb
+- spec/command_spec.rb
 - spec/database.yml
 - spec/launcher_spec.rb
 - spec/manager_spec.rb
@@ -132,22 +160,22 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.1.11
 signing_key: 
 specification_version: 4
 summary: multi-threaded delayed_job workers!
 test_files:
-- spec/command_test.rb
+- spec/command_spec.rb
 - spec/database.yml
 - spec/launcher_spec.rb
 - spec/manager_spec.rb