delayed_job_master 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ac6f45642e4c4848deea50d7819bfef1f0d17b8c
-  data.tar.gz: ac6ba6a5386a2f3d2cb72f495ce5c4bc06f5e266
+SHA256:
+  metadata.gz: 548efefc42b3ecd02bcb85da66da4a8e001376057a54ee7a636adf5177346923
+  data.tar.gz: 6c83c6a641d334f262bb191f68649c90cfa083ede76ace8e960c9b7a1ce99b9d
 SHA512:
-  metadata.gz: 74016c0784861d3f3b2c43b8ff6bc2cd2dfe2252918f4848d752073218aea50b53bb372307e823c9f9b3cf8dc596d5617a0b1d0740e1d12873e289e0fcc074c3
-  data.tar.gz: 9ca2c253636b19f8810ba693ea9bc5c8ac6d2b8278a33aec12af5548cb17712f1f90cd0d94e8025daac1c539ffb805b84ee497ce86cdcd95cb61636b7ce1cdcb
+  metadata.gz: 81b40df17053df90e59c84c3e53bbc854ba0e623cc8185811725673867532f68525a65a86d732914f117f56788ebee607466d469e546f3be4332e2935ff17b60
+  data.tar.gz: c77e25e5aa04b6f4c11c3e7c630c3c8dcd45f691c5088cac3a8f07bb6980317419c1b5a45149c2154c4982991593f67f06bebef32dc27915e4d4958a05590981

data/.travis.yml

@@ -4,11 +4,11 @@ services:
 rvm:
   - 2.3
   - 2.4
+  - 2.5
+  - 2.6
 gemfile:
   - gemfiles/active_record.gemfile
   - gemfiles/mongoid.gemfile
-before_install:
-  - gem install bundler -v 1.15.3
 before_script:
   - cd spec/dummy
   - bundle exec rake db:migrate RAILS_ENV=test

data/CHANGELOG.md

@@ -0,0 +1,16 @@
+# CHANGELOG
+
+## 1.2.0
+
+* Add destroy_failed_jobs for worker.
+* Get pid and worker instance in after_fork callback.
+* Change config class for convenience.
+
+## 1.1.0
+
+* Add monitor callback to verify database connection.
+* Refactoring.
+
+## 1.0.0
+
+* First release.

data/Gemfile

@@ -1,4 +1,3 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in delayed_job_master.gemspec
 gemspec

data/README.md

@@ -80,6 +80,7 @@ add_worker do |worker|
   # worker.max_run_time 4.hours
   # worker.min_priority 1
   # worker.max_priority 10
+  # worker.destroy_failed_jobs true
 end
 
 # worker2
@@ -89,11 +90,11 @@ add_worker do |worker|
   worker.count 2
 end
 
-before_fork do |master, worker_info|
+before_fork do |master, worker|
   Delayed::Worker.before_fork if defined?(Delayed::Worker)
 end
 
-after_fork do |master, worker_info|
+after_fork do |master, worker|
   Delayed::Worker.after_fork if defined?(Delayed::Worker)
 end
 

data/gemfiles/active_record.gemfile

@@ -1,5 +1,7 @@
 source 'https://rubygems.org'
 
 gem "delayed_job_active_record", "~> 4.1.0"
+gem "rails", "~> 5.2.0"
+gem "sqlite3", "~> 1.3.6"
 
 gemspec path: "../"

data/gemfiles/mongoid.gemfile

@@ -1,5 +1,7 @@
 source 'https://rubygems.org'
 
 gem "delayed_job_mongoid", "~> 2.3.0"
+gem "rails", "~> 5.2.0"
+gem "sqlite3", "~> 1.3.6"
 
 gemspec path: "../"

data/lib/delayed/master.rb

@@ -1,31 +1,29 @@
 require 'fileutils'
 require 'logger'
-require 'ostruct'
-require_relative 'util/file_reopener'
 require_relative 'master/version'
 require_relative 'master/command'
 require_relative 'master/callback'
-require_relative 'master/worker_info'
+require_relative 'master/worker'
 require_relative 'master/worker_pool'
 require_relative 'master/signal_handler'
+require_relative 'master/util/file_reopener'
 
 module Delayed
   class Master
-    attr_reader :config, :logger, :worker_infos
+    attr_reader :config, :logger, :workers
 
     def initialize(argv)
-      config = Command.new(argv).config
-      @config = OpenStruct.new(config).freeze
+      @config = Command.new(argv).config
       @logger = setup_logger(@config.log_file, @config.log_level)
-      @worker_infos = []
+      @workers = []
 
       @signal_handler = SignalHandler.new(self)
-      @worker_pool = WorkerPool.new(self, config)
+      @worker_pool = WorkerPool.new(self, @config)
     end
 
     def run
       load_app
-      show_worker_configs
+      show_config
       daemonize if @config.daemon
 
       create_pid_file
@@ -66,7 +64,7 @@ module Delayed
     def reopen_files
       @signal_handler.dispatch('USR1')
       @logger.info "reopening files..."
-      Delayed::Util::FileReopener.reopen
+      Util::FileReopener.reopen
       @logger.info "reopened"
     end
 
@@ -106,9 +104,9 @@ module Delayed
       Process.daemon(true)
     end
 
-    def show_worker_configs
-      @config.worker_configs.each do |config|
-        puts "#{config[:count]} worker for '#{config[:queues].join(',')}' under #{config[:control]} control"
+    def show_config
+      @config.workers.each do |setting|
+        puts "#{setting.count} worker for '#{setting.queues.join(',')}' under #{setting.control} control"
       end
     end
   end

data/lib/delayed/master/callback.rb

@@ -1,12 +1,12 @@
 module Delayed
   class Master
     class Callback
-      def initialize(config = {})
-        @config = config.select { |k, _| Delayed::Master::DSL::CALLBACK_CONFIGS.include?(k) }
+      def initialize(config)
+        @callbacks = config.callbacks
       end
 
       def run(name, *args)
-        @config[name].call(*args) if @config[name]
+        @callbacks[name].call(*args) if @callbacks[name]
       end
     end
   end

data/lib/delayed/master/command.rb

@@ -1,5 +1,5 @@
 require 'optparse'
-require_relative 'dsl'
+require_relative 'config'
 
 module Delayed
   class Master
@@ -7,7 +7,7 @@ module Delayed
       attr_reader :config
 
       def initialize(args)
-        @config = {}
+        @config = Config.new
 
         OptionParser.new { |opt|
           opt.banner = <<-EOS
@@ -20,10 +20,10 @@ module Delayed
             exit
           end
           opt.on('-D', '--daemon', 'Start master as a daemon') do |boolean|
-            @config[:daemon] = boolean
+            @config.daemon(boolean)
           end
           opt.on('-c', '--config=FILE', 'Specify config file') do |file|
-            @config.merge!(DSL.new(file).config)
+            @config.read(file)
           end
         }.parse(args)
       end

data/lib/delayed/master/config.rb

@@ -0,0 +1,84 @@
+module Delayed
+  class Master
+    class Config
+      SIMPLE_CONFIGS   = [:working_directory, :log_file, :log_level, :pid_file, :monitor_wait, :daemon]
+      CALLBACK_CONFIGS = [:before_fork, :after_fork, :before_monitor, :after_monitor]
+
+      attr_reader :data, :workers
+
+      def initialize(file = nil)
+        @data = {}
+        @workers = []
+        read(file) if file
+      end
+
+      def read(file)
+        instance_eval(File.read(file))
+      end
+
+      def add_worker
+        worker = WorkerSetting.new(control: :static, count: 1)
+        yield worker
+        worker.exit_on_complete(true) if worker.control == :dynamic
+        @workers << worker
+      end
+
+      def callbacks
+        @data.select { |k, _| CALLBACK_CONFIGS.include?(k) }
+      end
+
+      SIMPLE_CONFIGS.each do |key|
+        define_method(key) do |value = nil|
+          if value
+            @data[key] = value
+          else
+            @data[key]
+          end
+        end
+      end
+
+      CALLBACK_CONFIGS.each do |key|
+        define_method(key) do |&block|
+          if block
+            @data[key] = block
+          else
+            @data[key]
+          end
+        end
+      end
+
+      class WorkerSetting
+        SIMPLE_CONFIGS = [:control, :count, :max_memory,
+                          :min_priority, :max_priority, :sleep_delay, :read_ahead, :exit_on_complete,
+                          :max_attempts, :max_run_time, :destroy_failed_jobs]
+        ARRAY_CONFIGS  = [:queues]
+
+        attr_reader :data
+
+        def initialize(default = {})
+          @data = default
+        end
+
+        SIMPLE_CONFIGS.each do |key|
+          define_method(key) do |value = nil|
+            if value
+              @data[key] = value
+            else
+              @data[key]
+            end
+          end
+        end
+
+        ARRAY_CONFIGS.each do |key|
+          define_method(key) do |value = nil|
+            if value
+              @data[key] = Array(value)
+            else
+              @data[key]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/job_counter/active_record.rb

@@ -4,11 +4,11 @@ module Delayed
   class Master
     class JobCounter
       class << self
-        def count(config)
-          jobs = ready_to_run(config[:max_run_time] || Delayed::Worker::DEFAULT_MAX_RUN_TIME)
-          jobs.where!("priority >= ?", config[:min_priority]) if config[:min_priority]
-          jobs.where!("priority <= ?", config[:max_priority]) if config[:max_priority]
-          jobs.where!(queue: config[:queues]) if config[:queues].any?
+        def count(setting)
+          jobs = ready_to_run(setting.max_run_time || Delayed::Worker::DEFAULT_MAX_RUN_TIME)
+          jobs.where!("priority >= ?", setting.min_priority) if setting.min_priority
+          jobs.where!("priority <= ?", setting.max_priority) if setting.max_priority
+          jobs.where!(queue: setting.queues) if setting.queues.any?
           jobs.count
         end
 

data/lib/delayed/master/job_counter/mongoid.rb

@@ -4,12 +4,12 @@ module Delayed
   class Master
     class JobCounter
       class << self
-        def count(config)
+        def count(setting)
           right_now = Delayed::Job.db_time_now
-          jobs = reservation_criteria(right_now, config[:max_run_time] || Delayed::Worker::DEFAULT_MAX_RUN_TIME)
-          jobs = jobs.gte(priority: config[:min_priority].to_i) if config[:min_priority]
-          jobs = jobs.lte(priority: config[:max_priority].to_i) if config[:max_priority]
-          jobs = jobs.any_in(queue: config[:queues]) if config[:queues].any?
+          jobs = reservation_criteria(right_now, setting.max_run_time || Delayed::Worker::DEFAULT_MAX_RUN_TIME)
+          jobs = jobs.gte(priority: setting.min_priority.to_i) if setting.min_priority
+          jobs = jobs.lte(priority: setting.max_priority.to_i) if setting.max_priority
+          jobs = jobs.any_in(queue: setting.queues) if setting.queues.any?
           jobs.count
         end
 

data/lib/delayed/master/signal_handler.rb

@@ -4,7 +4,7 @@ module Delayed
       def initialize(master)
         @master = master
         @logger = master.logger
-        @worker_infos = master.worker_infos
+        @workers = master.workers
       end
 
       def register
@@ -28,9 +28,9 @@ module Delayed
       end
 
       def dispatch(signal)
-        @worker_infos.each do |worker_info|
-          next unless worker_info.pid
-          dispatch_to(signal, worker_info.pid)
+        @workers.each do |worker|
+          next unless worker.pid
+          dispatch_to(signal, worker.pid)
         end
       end
 

data/lib/delayed/master/util/file_reopener.rb

@@ -0,0 +1,18 @@
+module Delayed
+  class Master
+    class Util
+      class FileReopener
+        class << self
+          def reopen
+            ObjectSpace.each_object(File) do |file|
+              next if file.closed? || !file.sync
+              file.reopen file.path 
+              file.sync = true
+              file.flush
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/version.rb

@@ -1,5 +1,5 @@
 module Delayed
   class Master
-    VERSION = "1.1.0"
+    VERSION = "1.2.0"
   end
 end

data/lib/delayed/master/worker.rb

@@ -0,0 +1,19 @@
+module Delayed
+  class Master
+    class Worker
+      attr_reader :index, :setting
+      attr_accessor :pid, :instance
+
+      def initialize(index, setting)
+        @index = index
+        @setting = setting
+      end
+
+      def title
+        titles = ["delayed_job.#{@index}"]
+        titles << "(#{@setting.queues.join(',')})" if @setting.queues
+        titles.join(' ')
+      end
+    end
+  end
+end

data/lib/delayed/master/worker_pool.rb

@@ -1,27 +1,27 @@
 module Delayed
   class Master
     class WorkerPool
-      def initialize(master, config = {})
+      def initialize(master, config)
         @master = master
-        @logger = master.logger
-        @worker_infos = master.worker_infos
+        @config = config
 
-        @config = OpenStruct.new(config).freeze
-        @static_worker_configs, @dynamic_worker_configs = @config.worker_configs.partition { |wc| wc[:control] == :static }
+        @logger = master.logger
+        @workers = master.workers
 
+        @static_settings, @dynamic_settings = config.workers.partition { |conf| conf.control == :static }
         @callback = Delayed::Master::Callback.new(config)
       end
 
       def init
-        @static_worker_configs.each_with_index do |config, i|
-          worker_info = Delayed::Master::WorkerInfo.new(i, config)
-          @worker_infos << worker_info
-          fork_worker(worker_info)
-          @logger.info "started worker #{worker_info.pid}"
+        @static_settings.each_with_index do |setting, i|
+          worker = Delayed::Master::Worker.new(i, setting)
+          @workers << worker
+          fork_worker(worker)
+          @logger.info "started worker #{worker.pid}"
         end
 
         @prepared = true
-        debug_worker_infos
+        print_workers
       end
 
       def monitor_while(&block)
@@ -41,28 +41,31 @@ module Delayed
 
       private
 
-      def fork_worker(worker_info)
-        @callback.run(:before_fork, @master, worker_info)
-        worker_info.pid = fork do
-          @callback.run(:after_fork, @master, worker_info)
-          $0 = worker_info.title
-          worker = create_new_worker(worker_info)
-          worker.start
+      def fork_worker(worker)
+        @callback.run(:before_fork, @master, worker)
+        worker.pid = fork do
+          worker.pid = Process.pid
+          worker.instance = create_instance(worker)
+          @callback.run(:after_fork, @master, worker)
+          $0 = worker.title
+          worker.instance.start
         end
       end
 
-      def create_new_worker(worker_info)
-        worker = Delayed::Worker.new(worker_info.config)
-        [:max_run_time, :max_attempts].each do |key|
-          value = worker_info.config[key]
-          Delayed::Worker.send("#{key}=", value) if value
+      def create_instance(worker)
+        instance = Delayed::Worker.new(worker.setting.data)
+        [:max_run_time, :max_attempts, :destroy_failed_jobs].each do |key|
+          if (value = worker.setting.send(key))
+            Delayed::Worker.send("#{key}=", value)
+          end
         end
         [:max_memory].each do |key|
-          value = worker_info.config[key]
-          worker.send("#{key}=", value) if value
+          if (value = worker.setting.send(key))
+            instance.send("#{key}=", value)
+          end
         end
-        worker.master_logger = @logger
-        worker
+        instance.master_logger = @logger
+        instance
       end
 
       def monitor
@@ -76,14 +79,14 @@ module Delayed
       def check_pid
         pid = wait_pid
         return unless pid
-        worker_info = @worker_infos.detect { |wi| wi.pid == pid }
-        return unless worker_info
+        worker = @workers.detect { |worker| worker.pid == pid }
+        return unless worker
 
-        case worker_info.config[:control]
+        case worker.setting.control
         when :static
-          fork_alt_worker(worker_info)
+          fork_alt_worker(worker)
         when :dynamic
-          @worker_infos.delete(worker_info)
+          @workers.delete(worker)
         end
       end
 
@@ -94,43 +97,43 @@ module Delayed
       end
 
       def check_dynamic_worker
-        @dynamic_worker_configs.each do |worker_config|
-          current_count = @worker_infos.count { |wi| wi.config[:queues] == worker_config[:queues] }
-          remaining_count = worker_config[:count] - current_count
-          if remaining_count > 0 && (job_count = count_job_for_worker(worker_config)) > 0
+        @dynamic_settings.each do |setting|
+          current_count = @workers.count { |worker| worker.setting.queues == setting.queues }
+          remaining_count = setting.count - current_count
+          if remaining_count > 0 && (job_count = count_job(setting)) > 0
             [remaining_count, job_count].min.times do
-              fork_dynamic_worker(worker_config)
+              fork_dynamic_worker(setting)
             end
           end
         end
       end
 
-      def count_job_for_worker(worker_config)
-        Delayed::Master::JobCounter.count(worker_config)
+      def count_job(setting)
+        Delayed::Master::JobCounter.count(setting)
       end
 
-      def fork_dynamic_worker(worker_config)
-        worker_info = Delayed::Master::WorkerInfo.new(@worker_infos.size, worker_config)
-        @worker_infos << worker_info
+      def fork_dynamic_worker(setting)
+        worker = Delayed::Master::Worker.new(@workers.size, setting)
+        @workers << worker
 
         @logger.info "forking dynamic worker..."
-        fork_worker(worker_info)
-        @logger.info "forked worker #{worker_info.pid}"
+        fork_worker(worker)
+        @logger.info "forked worker #{worker.pid}"
 
-        debug_worker_infos
+        print_workers
       end
 
-      def fork_alt_worker(worker_info)
-        @logger.info "worker #{worker_info.pid} seems to be killed, forking alternative worker..."
-        fork_worker(worker_info)
-        @logger.info "forked worker #{worker_info.pid}"
+      def fork_alt_worker(worker)
+        @logger.info "worker #{worker.pid} seems to be killed, forking alternative worker..."
+        fork_worker(worker)
+        @logger.info "forked worker #{worker.pid}"
 
-        debug_worker_infos
+        print_workers
       end
 
-      def debug_worker_infos
-        @worker_infos.each do |worker_info|
-          @logger.debug "#{worker_info.pid}: #{worker_info.title}"
+      def print_workers
+        @workers.each do |worker|
+          @logger.debug "#{worker.pid}: #{worker.title}"
         end
       end
     end

data/lib/delayed/worker/plugins/signal_handler.rb

@@ -7,7 +7,7 @@ module Delayed
             trap('USR1') do
               Thread.new do
                 master_logger.info "reopening files..."
-                Delayed::Util::FileReopener.reopen
+                Delayed::Master::Util::FileReopener.reopen
                 master_logger.info "reopened"
               end
             end

data/lib/generators/delayed_job_master/templates/config.rb

@@ -34,6 +34,7 @@ add_worker do |worker|
   # worker.max_run_time 4.hours
   # worker.min_priority 1
   # worker.max_priority 10
+  # worker.destroy_failed_jobs true
 end
 
 # worker2
@@ -43,11 +44,11 @@ add_worker do |worker|
   worker.count 2
 end
 
-before_fork do |master, worker_info|
+before_fork do |master, worker|
   Delayed::Worker.before_fork if defined?(Delayed::Worker)
 end
 
-after_fork do |master, worker_info|
+after_fork do |master, worker|
   Delayed::Worker.after_fork if defined?(Delayed::Worker)
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: delayed_job_master
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-20 00:00:00.000000000 Z
+date: 2019-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: delayed_job
@@ -146,6 +146,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -157,15 +158,15 @@ files:
 - lib/delayed/master.rb
 - lib/delayed/master/callback.rb
 - lib/delayed/master/command.rb
-- lib/delayed/master/dsl.rb
+- lib/delayed/master/config.rb
 - lib/delayed/master/job_counter.rb
 - lib/delayed/master/job_counter/active_record.rb
 - lib/delayed/master/job_counter/mongoid.rb
 - lib/delayed/master/signal_handler.rb
+- lib/delayed/master/util/file_reopener.rb
 - lib/delayed/master/version.rb
-- lib/delayed/master/worker_info.rb
+- lib/delayed/master/worker.rb
 - lib/delayed/master/worker_pool.rb
-- lib/delayed/util/file_reopener.rb
 - lib/delayed/worker/extension.rb
 - lib/delayed/worker/plugins/memory_checker.rb
 - lib/delayed/worker/plugins/signal_handler.rb
@@ -192,8 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A simple delayed_job master process to control multiple workers

data/lib/delayed/master/dsl.rb

@@ -1,60 +0,0 @@
-module Delayed
-  class Master
-    class DSL
-      SIMPLE_CONFIGS   = [:working_directory, :log_file, :log_level, :pid_file, :monitor_wait]
-      CALLBACK_CONFIGS = [:before_fork, :after_fork, :before_monitor, :after_monitor]
-
-      attr_reader :config
-
-      def initialize(config_file)
-        @config = { worker_configs: [] }
-        instance_eval(File.read(config_file))
-        @config
-      end
-
-      def add_worker
-        setting = WorkerSetting.new(control: :static, count: 1)
-        yield setting
-        setting.config[:exit_on_complete] = true if setting.config[:control] == :dynamic
-        @config[:worker_configs] << setting.config
-      end
-
-      SIMPLE_CONFIGS.each do |key|
-        define_method(key) do |value|
-          @config[key] = value
-        end
-      end
-
-      CALLBACK_CONFIGS.each do |key|
-        define_method(key) do |&block|
-          @config[key] = block
-        end
-      end
-
-      class WorkerSetting
-        SIMPLE_CONFIGS = [:control, :count, :max_memory,
-                          :min_priority, :max_priority, :sleep_delay, :read_ahead, :exit_on_complete,
-                          :max_attempts, :max_run_time]
-        ARRAY_CONFIGS  = [:queues]
-
-        attr_reader :config
-
-        def initialize(default = {})
-          @config = default
-        end
-
-        SIMPLE_CONFIGS.each do |key|
-          define_method(key) do |value|
-            @config[key] = value
-          end
-        end
-
-        ARRAY_CONFIGS.each do |key|
-          define_method(key) do |value|
-            @config[key] = Array(value)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/delayed/master/worker_info.rb

@@ -1,19 +0,0 @@
-module Delayed
-  class Master
-    class WorkerInfo
-      attr_reader :index, :config
-      attr_accessor :pid
-
-      def initialize(index, config = {})
-        @index = index
-        @config = config
-      end
-
-      def title
-        titles = ["delayed_job.#{@index}"]
-        titles << "(#{@config[:queues].join(',')})" if @config[:queues]
-        titles.join(' ')
-      end
-    end
-  end
-end

data/lib/delayed/util/file_reopener.rb

@@ -1,16 +0,0 @@
-module Delayed
-  class Util
-    class FileReopener
-      class << self
-        def reopen
-          ObjectSpace.each_object(File) do |file|
-            next if file.closed? || !file.sync
-            file.reopen file.path 
-            file.sync = true
-            file.flush
-          end
-        end
-      end
-    end
-  end
-end