delayed_job_master 2.0.3 → 3.0.0

data/lib/delayed/master/core.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'logger'
+require_relative 'safe_array'
+require_relative 'command'
+require_relative 'worker'
+require_relative 'database'
+require_relative 'callbacks'
+require_relative 'monitoring'
+require_relative 'job_checker'
+require_relative 'job_listener'
+require_relative 'signaler'
+require_relative 'file_reopener'
+
+module Delayed
+  module Master
+    class Core
+      attr_reader :config, :logger, :databases, :callbacks, :workers
+      attr_reader :monitoring, :job_checker, :job_listener
+
+      def initialize(argv)
+        @config = Command.new(argv).config
+        @logger = setup_logger(@config.log_file, @config.log_level)
+        @workers = SafeArray.new
+
+        @databases = Database.all(@config.databases)
+        @callbacks = Callbacks.new(@config)
+        @monitoring = Monitoring.new(self)
+        @job_checker = JobChecker.new(self)
+        @job_listener = JobListener.klass.new(self)
+        @signaler = Signaler.new(self)
+      end
+
+      def run
+        print_config
+        daemonize if @config.daemon
+
+        @logger.info { "started master #{Process.pid}".tap { |msg| puts msg } }
+
+        handle_pid_file do
+          @signaler.register
+          @prepared = true
+
+          start
+          wait
+          shutdown
+        end
+
+        @logger.info { "shut down master" }
+      end
+
+      def start
+        @job_checker.start
+        @job_listener.start
+        @monitoring.start
+      end
+
+      def wait
+        @job_checker.wait
+        @job_listener.wait
+        @monitoring.wait
+      end
+
+      def shutdown
+        @job_checker.shutdown
+        @job_listener.shutdown
+        @monitoring.shutdown
+      end
+
+      def prepared?
+        @prepared
+      end
+
+      def quit
+        @signaler.dispatch(:KILL)
+        shutdown
+        @workers.clear
+        @stop = true
+      end
+
+      def stop
+        @signaler.dispatch(:TERM)
+        shutdown
+        @workers.clear
+        @stop = true
+      end
+
+      def graceful_stop
+        @signaler.dispatch(:TERM)
+        @stop = true
+      end
+
+      def stop?
+        @stop == true
+      end
+
+      def reopen_files
+        @signaler.dispatch(:USR1)
+        @logger.info { "reopening files..." }
+        FileReopener.reopen
+        @logger.info { "reopened" }
+      end
+
+      def restart
+        @signaler.dispatch(:USR2)
+        @logger.info { "restarting master..." }
+        exec(*([$0] + ARGV))
+      end
+
+      private
+
+      def setup_logger(log_file, log_level)
+        FileUtils.mkdir_p(File.dirname(log_file)) if log_file.is_a?(String)
+        logger = Logger.new(log_file)
+        logger.level = log_level
+        logger
+      end
+
+      def print_config
+        @config.abstract_texts.each do |text|
+          @logger.info { text }
+        end
+      end
+
+      def daemonize
+        Process.daemon(true)
+      end
+
+      def handle_pid_file
+        create_pid_file
+        yield
+        remove_pid_file
+      end
+
+      def create_pid_file
+        FileUtils.mkdir_p(File.dirname(@config.pid_file))
+        File.write(@config.pid_file, Process.pid)
+      end
+
+      def remove_pid_file
+        File.delete(@config.pid_file) if File.exist?(@config.pid_file)
+      end
+    end
+  end
+end

data/lib/delayed/master/database.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class Database
+      class_attribute :model_cache
+      self.model_cache = {}
+
+      attr_accessor :spec_name
+
+      def initialize(spec_name)
+        @spec_name = spec_name
+      end
+
+      def model
+        cache_model do
+          define_model
+        end
+      end
+
+      def with_connection
+        model.connection_pool.with_connection do |connection|
+          yield connection
+        end
+      end
+
+      private
+
+      def cache_model
+        self.class.model_cache[@spec_name] ||= yield
+      end
+
+      def define_model
+        model = Class.new(Delayed::Job)
+        model_name = "DelayedJob#{@spec_name.capitalize}"
+        unless Delayed::Master.const_defined?(model_name)
+          Delayed::Master.const_set(model_name, model)
+          Delayed::Master.const_get(model_name).establish_connection(@spec_name)
+        end
+        Delayed::Master.const_get(model_name)
+      end
+
+      class << self
+        def all(spec_names = nil)
+          spec_names = spec_names.presence || spec_names_with_delayed_job_table
+          spec_names.map { |spec_name| new(spec_name) }
+        end
+
+        private
+
+        def spec_names_with_delayed_job_table
+          @spec_names_with_delayed_job_table ||= spec_names_without_replica.select do |spec_name|
+            exist_delayed_job_table?(spec_name)
+          end
+        end
+
+        def spec_names_without_replica
+          configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env)
+          configs.reject(&:replica?).map do |c|
+            c.respond_to?(:name) ? c.name.to_sym : c.spec_name.to_sym
+          end
+        end
+
+        def exist_delayed_job_table?(spec_name)
+          new(spec_name).with_connection do |connection|
+            connection.tables.include?('delayed_jobs')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/file_reopener.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class FileReopener
+      class << self
+        def reopen
+          ObjectSpace.each_object(File) do |file|
+            next if file.closed? || !file.sync
+            file.reopen file.path, 'a+'
+            file.sync = true
+            file.flush
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/forker.rb

@@ -1,42 +1,53 @@
+# frozen_string_literal: true
+
+require_relative 'file_reopener'
+
 module Delayed
-  class Master
+  module Master
     class Forker
       def initialize(master)
         @master = master
         @config = master.config
+        @callbacks = master.callbacks
       end
 
-      def new_worker(worker)
-        @master.logger.info "forking #{worker.name}..."
-        fork_worker(worker)
-        @master.logger.info "forked #{worker.name} with pid #{worker.pid}"
-
-        @master.workers << worker
+      def call(worker)
+        around_fork(worker) do
+          @callbacks.run(:before_fork, @master, worker)
+          worker.pid = fork do
+            @callbacks.run(:after_fork, @master, worker)
+            after_fork_at_child(worker)
+            worker.pid = Process.pid
+            worker.instance = create_instance(worker)
+            $0 = worker.process_title
+            worker.instance.start
+          end
+        end
       end
 
       private
 
-      def fork_worker(worker)
-        @config.run_callback(:before_fork, @master, worker)
-        worker.pid = fork do
-          worker.pid = Process.pid
-          worker.instance = create_instance(worker)
-          @config.run_callback(:after_fork, @master, worker)
-          $0 = worker.process_title
-          worker.instance.start
-        end
+      def around_fork(worker)
+        @master.logger.info { "forking #{worker.name}..." }
+        yield
+        @master.logger.info { "forked #{worker.name} with pid #{worker.pid}" }
+      end
+
+      def after_fork_at_child(worker)
+        FileReopener.reopen
       end
 
       def create_instance(worker)
-        require_relative 'worker_extension'
+        require_relative 'worker/extension'
 
-        instance = Delayed::Worker.new(worker.setting.data)
+        instance = Delayed::Worker.new
         [:max_run_time, :max_attempts, :destroy_failed_jobs].each do |key|
           if (value = worker.setting.send(key))
             Delayed::Worker.send("#{key}=", value)
           end
         end
-        [:max_memory].each do |key|
+        [:min_priority, :max_priority, :sleep_delay, :read_ahead, :exit_on_complete, :queues,
+         :max_threads, :max_memory].each do |key|
           if (value = worker.setting.send(key))
             instance.send("#{key}=", value)
           end

data/lib/delayed/master/job_checker.rb

@@ -1,83 +1,127 @@
-require_relative 'job_counter'
-require_relative 'database_detector'
+# frozen_string_literal: true
+
+require_relative 'database'
+require_relative 'forker'
+require_relative 'job_finder'
+require_relative 'sleep'
 
 module Delayed
-  class Master
+  module Master
     class JobChecker
+      include Sleep
+
       def initialize(master)
         @master = master
         @config = master.config
-        @spec_names = target_spec_names
+        @databases = master.databases
+        @callbacks = master.callbacks
+        @queues = @databases.map { |database| [database, Queue.new] }.to_h
+        @threads = []
+      end
 
-        define_models
-        extend_after_fork_callback
+      def start
+        @threads << start_scheduler_thread
+        @threads += @databases.map do |database|
+          start_checker_thread(database)
+        end
       end
 
-      def check
-        workers = []
-        mon = Monitor.new
+      def start_scheduler_thread
+        Thread.new do
+          loop_with_sleep @config.polling_interval do |i|
+            if @master.stop?
+              stop
+              break
+            elsif i == 0
+              schedule(@databases)
+            end
+          end
+        end
+      end
 
-        threads = @spec_names.map do |spec_name|
-          Thread.new(spec_name) do |spec_name|
-            find_jobs_in_db(spec_name) do |setting|
-              mon.synchronize do
-                workers << Worker.new(index: @master.workers.size + workers.size, database: spec_name, setting: setting)
+      def start_checker_thread(database)
+        Thread.new(database) do |database|
+          loop do
+            if @queues[database].pop == :stop
+              break
+            else
+              @callbacks.call(:polling, @master, database) do
+                check(database)
               end
             end
           end
         end
-
-        threads.each(&:join)
-
-        workers
       end
 
-      private
+      def stop
+        @databases.each do |database|
+          queue = @queues[database]
+          queue.clear
+          queue.push(:stop)
+        end
+      end
 
-      def define_models
-        @spec_names.each do |spec_name|
-          klass = Class.new(Delayed::Job)
-          klass_name = "DelayedJob#{spec_name.capitalize}"
-          unless Delayed::Master.const_defined?(klass_name)
-            Delayed::Master.const_set(klass_name, klass)
-            Delayed::Master.const_get(klass_name).establish_connection(spec_name)
-          end
+      def schedule(databases)
+        Array(databases).each do |database|
+          queue = @queues[database]
+          queue.push(database) if queue.size == 0
         end
       end
 
-      def model_for(spec_name)
-        Delayed::Master.const_get("DelayedJob#{spec_name.capitalize}")
+      def wait
+        @threads.each(&:join)
       end
 
-      def extend_after_fork_callback
-        prc = @config.after_fork
-        @config.after_fork do |master, worker|
-          prc.call(master, worker)
-          ActiveRecord::Base.establish_connection(worker.database) if worker.database
+      def shutdown
+        @threads.each(&:kill)
+      end
+
+      private
+
+      def check(database)
+        free_settings = detect_free_settings(database)
+        return if free_settings.blank?
+
+        @master.logger.debug { "checking jobs @#{database.spec_name}..." }
+        settings = check_jobs(database, free_settings)
+        if settings.present?
+          @master.logger.info { "found jobs for #{settings.uniq.map(&:worker_info).join(', ')}" }
+          fork_workers(database, settings)
         end
+      rescue => e
+        @master.logger.warn { "#{e.class}: #{e.message}" }
+        @master.logger.debug { e.backtrace.join("\n") }
       end
 
-      def target_spec_names
-        if @config.databases.nil? || @config.databases.empty?
-          DatabaseDetector.new.call
-        else
-          @config.databases
+      def detect_free_settings(database)
+        @config.worker_settings.each_with_object([]) do |setting, array|
+          used_count = @master.workers.count { |worker| worker.setting.queues == setting.queues }
+          free_count = setting.max_processes - used_count
+          array << [setting, free_count] if free_count > 0
         end
       end
 
-      def find_jobs_in_db(spec_name)
-        counter = JobCounter.new(model_for(spec_name))
+      def check_jobs(database, settings)
+        finder = JobFinder.new(database.model)
 
-        @config.worker_settings.each do |setting|
-          count = @master.workers.count { |worker| worker.setting.queues == setting.queues }
-          slot = setting.count - count
-          if slot > 0 && (job_count = counter.count(setting)) > 0
-            [slot, job_count].min.times do
-              yield setting
+        settings.each_with_object([]) do |(setting, free_count), array|
+          job_ids = finder.call(setting, free_count)
+          if job_ids.size > 0
+            [free_count, job_ids.size].min.times do
+              array << setting
             end
           end
         end
       end
+
+      def fork_workers(database, settings)
+        settings.each do |setting|
+          worker = Worker.new(database: database, setting: setting)
+          Forker.new(@master).call(worker)
+          @master.workers << worker
+          @master.monitoring.schedule(worker)
+        end
+      end
     end
   end
 end

data/lib/delayed/master/job_finder.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# JobFinder runs SQL query which is almost same as delayed_job_active_record.
+# See https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/delayed/backend/active_record.rb
+module Delayed
+  module Master
+    class JobFinder
+      def initialize(model)
+        @model = model
+      end
+
+      def call(setting, limit)
+        scope(setting).limit(limit).pluck(:id)
+      end
+
+      def count(setting)
+        scope(setting).count
+      end
+
+      private
+
+      def scope(setting)
+        @model.ready_to_run(nil, setting.max_run_time || Delayed::Worker::DEFAULT_MAX_RUN_TIME).tap do |jobs|
+          jobs.where!("priority >= ?", setting.min_priority) if setting.min_priority
+          jobs.where!("priority <= ?", setting.max_priority) if setting.max_priority
+          jobs.where!(queue: setting.queues) if setting.queues.present?
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/job_listener.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class JobListener
+      def initialize(master)
+      end
+
+      def start
+      end
+
+      def wait
+      end
+
+      def shutdown
+      end
+
+      class << self
+        def klass
+          case DelayedJobMaster.config.listener
+          when :postgresql
+            require_relative 'postgresql/job_listener'
+            Postgresql::JobListener
+          else
+            self
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/monitoring.rb

@@ -1,58 +1,59 @@
-require_relative 'forker'
-require_relative 'job_checker' if defined?(Delayed::Backend::ActiveRecord)
+# frozen_string_literal: true
+
+require_relative 'safe_array'
+require_relative 'sleep'
 
 module Delayed
-  class Master
+  module Master
     class Monitoring
+      include Sleep
+
       def initialize(master)
         @master = master
         @config = master.config
-        @forker = Forker.new(master)
-        @job_checker = JobChecker.new(master)
+        @callbacks = master.callbacks
+        @threads = SafeArray.new
       end
 
-      def monitor_while(&block)
-        loop do
-          break if block.call
-          monitor do
-            check_terminated
-            check_queued_jobs
+      def start
+        @threads << Thread.new do
+          loop_with_sleep @config.monitor_interval do |i|
+            if @master.stop?
+              break
+            elsif i == 0
+              @callbacks.call(:monitor, @master) {}
+            end
           end
-          sleep @config.monitor_wait.to_i
         end
       end
 
-      private
-
-      def monitor
-        @config.run_callback(:before_monitor, @master)
-        yield
-        @config.run_callback(:after_monitor, @master)
-      rescue Exception => e
-        @master.logger.warn "#{e.class}: #{e.message} at #{__FILE__}: #{__LINE__}"
+      def schedule(worker)
+        @threads << Thread.new do
+          wait_pid(worker)
+          @threads.delete(Thread.current)
+        end
       end
 
-      def check_terminated
-        if (pid = terminated_pid)
-          @master.logger.debug "found terminated pid: #{pid}"
-          @master.workers.reject! { |worker| worker.pid == pid }
-        end
+      def wait
+        @threads.each(&:join)
       end
 
-      def terminated_pid
-        Process.waitpid(-1, Process::WNOHANG)
-      rescue Errno::ECHILD
-        nil
+      def shutdown
+        @threads.each(&:kill)
       end
 
-      def check_queued_jobs
-        @master.logger.debug "checking jobs..."
+      private
 
-        new_workers = @job_checker.check
-        new_workers.each do |worker|
-          @master.logger.info "found jobs for #{worker.info}"
-          @forker.new_worker(worker)
-        end
+      def wait_pid(worker)
+        Process.waitpid(worker.pid)
+        @master.logger.debug { "found terminated pid: #{worker.pid}" }
+        @master.workers.delete(worker)
+      rescue Errno::ECHILD
+        @master.logger.warn { "failed to waitpid: #{worker.pid}" }
+        @master.workers.delete(worker)
+      rescue => e
+        @master.logger.warn { "#{e.class}: #{e.message}" }
+        @master.logger.debug { e.backtrace.join("\n") }
       end
     end
   end