exekutor 0.1.0 → 0.1.1

data/lib/exekutor/internal/cli/info.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative "application_loader"
 require "terminal-table"
 
@@ -19,96 +21,128 @@ module Exekutor
           load_application(options[:environment], print_message: !quiet?)
 
           ActiveSupport.on_load(:active_record, yield: true) do
-            # Use system time zone
-            Time.zone = Time.new.zone
+            clear_application_loading_message
+            print_time_zone_warning if different_time_zone? && !quiet?
 
             hosts = Exekutor::Info::Worker.distinct.pluck(:hostname)
-            job_info = Exekutor::Job.pending.order(:queue).group(:queue)
-                                    .pluck(:queue, Arel.sql("COUNT(*)"), Arel.sql("MIN(scheduled_at)"))
-
-            clear_application_loading_message unless quiet?
-            puts Rainbow("Workers").bright.blue
-            if hosts.present?
-              total_workers = 0
-              hosts.each do |host|
-                table = Terminal::Table.new
-                table.title = host if hosts.many?
-                table.headings = ["id", "Status", "Last heartbeat"]
-                worker_count = 0
-                Exekutor::Info::Worker.where(hostname: host).each do |worker|
-                  worker_count += 1
-                  table << [
-                    worker.id.split("-").first << "…",
-                    worker.status,
-                    if worker.last_heartbeat_at.nil?
-                      if !worker.running?
-                        "N/A"
-                      elsif worker.created_at < 10.minutes.ago
-                        Rainbow("None").red
-                      else
-                        "None"
-                      end
-                    elsif worker.last_heartbeat_at > 2.minutes.ago
-                      worker.last_heartbeat_at.strftime "%R"
-                    elsif worker.last_heartbeat_at > 10.minutes.ago
-                      Rainbow(worker.last_heartbeat_at.strftime("%R")).yellow
-                    else
-                      Rainbow(worker.last_heartbeat_at.strftime("%D %R")).red
-                    end
-                  ]
-                  # TODO switch / flag to print threads and queues
-                end
-                total_workers += worker_count
-                table.add_separator
-                table.add_row [(hosts.many? ? "Subtotal" : "Total"), { value: worker_count, alignment: :right, colspan: 2 }]
-                puts table
-              end
-
-              if hosts.many?
-                puts Terminal::Table.new rows: [
-                  ["Total hosts", hosts.size],
-                  ["Total workers", total_workers]
-                ]
-              end
-            else
-              message = Rainbow("There are no active workers")
-              message = message.red if job_info.present?
-              puts message
+            job_info = pending_jobs_per_queue
+
+            print_workers(hosts, job_info.present?, options)
+            puts
+            print_jobs(job_info)
+          end
+        end
+
+        private
+
+        def pending_jobs_per_queue
+          Exekutor::Job.pending.order(:queue).group(:queue)
+                       .pluck(:queue, Arel.sql("COUNT(*)"), Arel.sql("MIN(scheduled_at)"))
+        end
+
+        def print_jobs(job_info)
+          puts Rainbow("Jobs").bright.blue
+          if job_info.present?
+            puts create_job_info_table(job_info)
+          else
+            puts Rainbow("No pending jobs").green
+          end
+        end
+
+        def create_job_info_table(job_info)
+          Terminal::Table.new(headings: ["Queue", "Pending jobs", "Next job scheduled at"]).tap do |table|
+            total_count = 0
+            job_info.each do |queue, count, min_scheduled_at|
+              table << [queue, { value: count, alignment: :right }, format_scheduled_at(min_scheduled_at)]
+              total_count += count
+            end
+            if job_info.many?
+              table.add_separator
+              table << ["Total", { value: total_count, alignment: :right, colspan: 2 }]
+            end
+          end
+        end
+
+        def format_scheduled_at(min_scheduled_at)
+          if min_scheduled_at.nil?
+            "N/A"
+          elsif min_scheduled_at < 30.minutes.ago
+            Rainbow(min_scheduled_at.strftime("%D %R")).red
+          elsif min_scheduled_at < 1.minute.ago
+            Rainbow(min_scheduled_at.strftime("%D %R")).yellow
+          else
+            min_scheduled_at.strftime("%D %R")
+          end
+        end
+
+        def print_workers(hosts, has_pending_jobs, options)
+          puts Rainbow("Workers").bright.blue
+          if hosts.present?
+            total_workers = 0
+            hosts.each do |host|
+              total_workers += print_host_info(host, options.merge(many_hosts: hosts.many?))
             end
 
-            puts " "
-            puts "#{Rainbow("Jobs").bright.blue}"
-            if job_info.present?
-              table = Terminal::Table.new
-              table.headings = ["Queue", "Pending jobs", "Next job scheduled at"]
-              total_count = 0
-              job_info.each do |queue, count, min_scheduled_at|
-                table << [
-                  queue, count,
-                  if min_scheduled_at.nil?
-                    "N/A"
-                  elsif min_scheduled_at < 30.minutes.ago
-                    Rainbow(min_scheduled_at.strftime("%D %R")).red
-                  elsif min_scheduled_at < 1.minute.ago
-                    Rainbow(min_scheduled_at.strftime("%D %R")).yellow
-                  else
-                    min_scheduled_at.strftime("%D %R")
-                  end
-                ]
-                total_count += count
-              end
-              if job_info.many?
-                table.add_separator
-                table.add_row ["Total", { value: total_count, alignment: :right, colspan: 2 }]
-              end
-              puts table
-            else
-              puts Rainbow("No pending jobs").green
+            if hosts.many?
+              puts Terminal::Table.new rows: [
+                ["Total hosts", hosts.size],
+                ["Total workers", total_workers]
+              ]
             end
+          else
+            message = Rainbow("There are no active workers")
+            message = message.red if has_pending_jobs
+            puts message
           end
         end
 
-        private
+        def print_host_info(host, options)
+          many_hosts = options[:many_hosts]
+          table = Terminal::Table.new headings: ["id", "Status", "Last heartbeat"]
+          table.title = host if many_hosts
+          worker_count = 0
+          Exekutor::Info::Worker.where(hostname: host).each do |worker|
+            worker_count += 1
+            table << worker_info_row(worker)
+          end
+          table.add_separator
+          table.add_row [(many_hosts ? "Subtotal" : "Total"),
+                         { value: worker_count, alignment: :right, colspan: 2 }]
+          puts table
+          worker_count
+        end
+
+        def worker_info_row(worker)
+          [
+            worker.id.split("-").first << "…",
+            worker.status,
+            worker_heartbeat_column(worker)
+          ]
+        end
+
+        def worker_heartbeat_column(worker)
+          last_heartbeat_at = worker.last_heartbeat_at
+          if last_heartbeat_at
+            colorize_heartbeat(last_heartbeat_at)
+          elsif !worker.running?
+            "N/A"
+          elsif worker.started_at < 10.minutes.ago
+            Rainbow("None").red
+          else
+            "None"
+          end
+        end
+
+        def colorize_heartbeat(timestamp)
+          case Time.now - timestamp
+          when (10.minutes)..nil
+            Rainbow(timestamp.strftime("%D %R")).red
+          when (2.minutes)..(10.minutes)
+            Rainbow(timestamp.strftime("%R")).yellow
+          else
+            timestamp.strftime "%R"
+          end
+        end
 
         # @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
         def quiet?
@@ -119,8 +153,7 @@ module Exekutor
         def verbose?
           !quiet? && !!@global_options[:verbose]
         end
-
       end
     end
   end
-end
+end

data/lib/exekutor/internal/cli/manager.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative "application_loader"
 require_relative "default_option_value"
 require_relative "daemon"
@@ -24,101 +26,23 @@ module Exekutor
         # @option options [Integer] :poll_interval The interval in seconds for job polling
         # @return [Void]
         def start(options)
+          Process.setproctitle "Exekutor worker (Initializing…) [#{$PROGRAM_NAME}]"
           daemonize(restarting: options[:restart]) if options[:daemonize]
 
           load_application(options[:environment])
 
-          config_files = if options[:configfile].is_a? DefaultConfigFileValue
-                           options[:configfile].to_a(@global_options[:identifier])
-                         else
-                           options[:configfile]&.map { |path| File.expand_path(path, Rails.root) }
-                         end
-
-          worker_options = DEFAULT_CONFIGURATION.dup
-
-          config_files&.each do |path|
-            puts "Loading config file: #{path}" if verbose?
-            config = begin
-                       YAML.safe_load(File.read(path), symbolize_names: true)
-                     rescue => e
-                       raise Error, "Cannot read config file: #{path} (#{e.to_s})"
-                     end
-            unless config.keys == [:exekutor]
-              raise Error, "Config should have an `exekutor` root node: #{path} (Found: #{config.keys.join(', ')})"
-            end
-
-            # Remove worker specific options before calling Exekutor.config.set
-            worker_options.merge! config[:exekutor].extract!(:queue, :status_server_port)
-
-            begin
-              Exekutor.config.set **config[:exekutor]
-            rescue => e
-              raise Error, "Cannot load config file: #{path} (#{e.to_s})"
-            end
-          end
-
-          worker_options.merge! Exekutor.config.worker_options
-          worker_options.merge! @global_options.slice(:identifier)
-          if verbose?
-            worker_options[:verbose] = true
-          elsif quiet?
-            worker_options[:quiet] = true
-          end
-          if options[:threads] && !options[:threads].is_a?(DefaultOptionValue)
-            min, max = if options[:threads].is_a?(Integer)
-                         [options[:threads], options[:threads]]
-                       else
-                         options[:threads].to_s.split(":")
-                       end
-            if max.nil?
-              options[:min_threads] = options[:max_threads] = Integer(min)
-            else
-              options[:min_threads] = Integer(min)
-              options[:max_threads] = Integer(max)
-            end
-          end
-          worker_options.merge!(
-            options.slice(:queue, :min_threads, :max_threads, :poll_interval)
-                   .reject { |_, value| value.is_a? DefaultOptionValue }
-                   .transform_keys(poll_interval: :polling_interval)
-          )
-
-          worker_options[:queue] = nil if worker_options[:queue] == ["*"]
-
-          # TODO health check server
-
           # Specify `yield: true` to prevent running in the context of the loaded module
           ActiveSupport.on_load(:exekutor, yield: true) do
-            ActiveSupport.on_load(:active_record, yield: true) do
-              worker = Worker.new(worker_options)
-              %w[INT TERM QUIT].each do |signal|
-                ::Kernel.trap(signal) { ::Thread.new { worker.stop } }
-              end
+            worker_options = worker_options(options[:configfile], cli_worker_overrides(options))
 
-              Process.setproctitle "Exekutor worker #{worker.id} [#{Rails.root}]"
-              if worker_options[:set_db_connection_name]
-                Internal::BaseRecord.connection.class.set_callback(:checkout, :after) do
-                  Internal::DatabaseConnection.set_application_name raw_connection, worker.id
-                end
-                Internal::BaseRecord.connection_pool.connections.each do |conn|
-                  Internal::DatabaseConnection.set_application_name conn.raw_connection, worker.id
-                end
-              end
-
-              ActiveSupport.on_load(:active_job, yield: true) do
-                puts "Worker #{worker.id} started (Use `#{Rainbow("ctrl + c").magenta}` to stop)" unless quiet?
-                puts "#{worker_options.pretty_inspect}" if verbose?
-                begin
-                  worker.start
-                  worker.join
-                ensure
-                  worker.stop if worker.running?
-                end
-              end
+            ActiveSupport.on_load(:active_record, yield: true) do
+              start_and_join_worker(worker_options, options[:daemonize])
             end
           end
         end
 
+        # Stops a daemonized worker
+        # @return [Void]
         def stop(options)
           daemon = Daemon.new(pidfile: pidfile)
           pid = daemon.pid
@@ -136,23 +60,12 @@ module Exekutor
           end
 
           Process.kill("INT", pid)
-          sleep(0.3)
-          wait_until = if options[:shutdown_timeout].nil? || options[:shutdown_timeout] == DEFAULT_FOREVER
-                         nil
-                       else
-                         Time.now + options[:shutdown_timeout]
-                       end
-          while daemon.status?(:running, :not_owned)
-            puts "Waiting for worker to finish…" unless quiet?
-            if wait_until && wait_until > Time.now
-              Process.kill("TERM", pid)
-              break
-            end
-            sleep 0.1
-          end
+          wait_for_process_end(daemon, pid, shutdown_timeout(options))
           puts "Worker (PID: #{pid}) stopped." unless quiet?
         end
 
+        # Restarts a daemonized worker
+        # @return [Void]
         def restart(stop_options, start_options)
           stop stop_options.merge(restart: true)
           start start_options.merge(restart: true, daemonize: true)
@@ -160,6 +73,135 @@ module Exekutor
 
         private
 
+        def worker_options(config_file, cli_overrides)
+          worker_options = DEFAULT_CONFIGURATION.dup
+
+          load_config_files(config_file, worker_options)
+
+          worker_options.merge! Exekutor.config.worker_options
+          worker_options.merge! @global_options.slice(:identifier)
+          worker_options.merge! cli_overrides
+
+          if quiet?
+            worker_options[:quiet] = true
+          elsif verbose?
+            worker_options[:verbose] = true
+          end
+
+          worker_options[:queue] = nil if Array.wrap(worker_options[:queue]) == ["*"]
+          worker_options
+        end
+
+        def cli_worker_overrides(cli_options)
+          worker_options = cli_options.slice(:queue, :poll_interval)
+                                      .reject { |_, value| value.is_a? DefaultOptionValue }
+                                      .transform_keys(poll_interval: :polling_interval)
+
+          min_threads, max_threads = parse_thread_limitations(cli_options[:threads])
+          if min_threads
+            worker_options[:min_threads] = min_threads
+            worker_options[:max_threads] = max_threads || min_threads
+          end
+
+          worker_options
+        end
+
+        def parse_thread_limitations(threads)
+          return if threads.blank? || threads.is_a?(DefaultOptionValue)
+
+          if threads.is_a?(Integer)
+            [threads, threads]
+          else
+            threads.to_s.split(":").map { |s| Integer(s) }
+          end
+        end
+
+        def load_config_files(config_files, worker_options)
+          config_files = if config_files.is_a? DefaultConfigFileValue
+                           config_files.to_a(@global_options[:identifier])
+                         else
+                           config_files&.map { |path| File.expand_path(path, Rails.root) }
+                         end
+
+          config_files&.each { |path| load_config_file(path, worker_options) }
+        end
+
+        def load_config_file(path, worker_options)
+          puts "Loading config file: #{path}" if verbose?
+          config = begin
+                     YAML.safe_load(File.read(path), symbolize_names: true)
+                   rescue StandardError => e
+                     raise Error, "Cannot read config file: #{path} (#{e})"
+                   end
+          unless config.keys == [:exekutor]
+            raise Error, "Config should have an `exekutor` root node: #{path} (Found: #{config.keys.join(", ")})"
+          end
+
+          # Remove worker specific options before calling Exekutor.config.set
+          worker_options.merge! config[:exekutor].extract!(:queue, :status_server_port)
+
+          begin
+            Exekutor.config.set(**config[:exekutor])
+          rescue StandardError => e
+            raise Error, "Cannot load config file: #{path} (#{e})"
+          end
+        end
+
+        def start_and_join_worker(worker_options, is_daemonized)
+          worker = Worker.new(worker_options)
+          %w[INT TERM QUIT].each do |signal|
+            ::Kernel.trap(signal) { ::Thread.new { worker.stop } }
+          end
+
+          Process.setproctitle "Exekutor worker #{worker.id} [#{Rails.root}]"
+          set_db_connection_name(worker.id) if worker_options[:set_db_connection_name]
+
+          ActiveSupport.on_load(:active_job, yield: true) do
+            worker.start
+            print_startup_message(worker, worker_options) unless quiet? || is_daemonized
+            worker.join
+          ensure
+            worker.stop if worker.running?
+          end
+        end
+
+        def print_startup_message(worker, worker_options)
+          puts "Worker #{worker.id} started (Use `#{Rainbow("ctrl + c").magenta}` to stop)"
+          puts worker_options.pretty_inspect if verbose?
+        end
+
+        # rubocop:disable Naming/AccessorMethodName
+        def set_db_connection_name(worker_id)
+          # rubocop:enable Naming/AccessorMethodName
+          Internal::BaseRecord.connection.class.set_callback(:checkout, :after) do
+            Internal::DatabaseConnection.set_application_name raw_connection, worker_id
+          end
+          Internal::BaseRecord.connection_pool.connections.each do |conn|
+            Internal::DatabaseConnection.set_application_name conn.raw_connection, worker_id
+          end
+        end
+
+        def wait_for_process_end(daemon, pid, shutdown_timeout)
+          wait_until = (Time.now.to_f + shutdown_timeout if shutdown_timeout)
+          sleep 0.1
+          while daemon.status?(:running, :not_owned)
+            if wait_until && wait_until > Time.now.to_f
+              puts "Sending TERM signal" unless quiet?
+              Process.kill("TERM", pid) if pid
+              break
+            end
+            sleep 0.1
+          end
+        end
+
+        def shutdown_timeout(options)
+          if options[:shutdown_timeout].nil? || options[:shutdown_timeout] == DEFAULT_FOREVER
+            nil
+          else
+            options[:shutdown_timeout]
+          end
+        end
+
         # @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
         def quiet?
           !!@global_options[:quiet]
@@ -175,6 +217,8 @@ module Exekutor
           @global_options[:identifier]
         end
 
+        # rubocop:disable Style/FormatStringToken
+
         # @return [String] The path to the pidfile
         def pidfile
           pidfile = @global_options[:pidfile] || DEFAULT_PIDFILE
@@ -187,37 +231,43 @@ module Exekutor
           end
         end
 
-        # Daemonizes the current process. Do this before loading your application to prevent deadlocks.
+        # Daemonizes the current process.
         # @return [Void]
         def daemonize(restarting: false)
           daemonizer = Daemon.new(pidfile: pidfile)
           daemonizer.validate!
-          unless quiet?
-            if restarting
-              puts "Restarting worker as a daemon…"
-            else
-              stop_options = if @global_options[:pidfile] && @global_options[:pidfile] != DEFAULT_PIDFILE
-                               "--pid #{pidfile} "
-                             elsif identifier
-                               "--id #{identifier} "
-                             end
+          print_daemonize_message(restarting) unless quiet?
 
-              puts "Running worker as a daemon… (Use `#{Rainbow("exekutor #{stop_options}stop").magenta}` to stop)"
-            end
-          end
           daemonizer.daemonize
         rescue Daemon::Error => e
           puts Rainbow(e.message).red
           raise GLI::CustomExit.new(nil, 1)
         end
 
+        def print_daemonize_message(restarting)
+          if restarting
+            puts "Restarting worker as a daemon…"
+          else
+            stop_options = if @global_options[:pidfile] && @global_options[:pidfile] != DEFAULT_PIDFILE
+                             "--pid #{pidfile} "
+                           elsif identifier
+                             "--id #{identifier} "
+                           end
+
+            puts "Running worker as a daemon… (Use `#{Rainbow("exekutor #{stop_options}stop").magenta}` to stop)"
+          end
+        end
+
+        # The default value for the pid file
         class DefaultPidFileValue < DefaultOptionValue
           def initialize
             super("tmp/pids/exekutor[.%{identifier}].pid")
           end
 
+          # @param identifier [nil,String] the worker identifier
+          # @return [String] the path to the default pidfile of the worker with the specified identifier
           def for_identifier(identifier)
-            if identifier.nil? || identifier.length.zero?
+            if identifier.nil? || identifier.empty? # rubocop:disable Rails/Blank – Rails is not loaded here
               "tmp/pids/exekutor.pid"
             else
               "tmp/pids/exekutor.#{identifier}.pid"
@@ -225,36 +275,53 @@ module Exekutor
           end
         end
 
+        # The default value for the config file
         class DefaultConfigFileValue < DefaultOptionValue
           def initialize
-            super('"config/exekutor.yml", overridden by "config/exekutor.%{identifier}.yml" if an identifier is specified')
+            super <<~DESC
+              "config/exekutor.yml", overridden by "config/exekutor.%{identifier}.yml" if an identifier is specified
+            DESC
           end
 
+          # @param identifier [nil,String] the worker identifier
+          # @return [Array<String>] the paths to the configfiles to load
           def to_a(identifier = nil)
             files = []
-            files << %w[config/exekutor.yml config/exekutor.yaml]
-                       .lazy.map { |path| Rails.root.join(path) }
-                       .find { |path| File.exists? path }
+            %w[config/exekutor.yml config/exekutor.yaml].each do |path|
+              path = Rails.root.join(path)
+              if File.exist? path
+                files.append path
+                break
+              end
+            end
             if identifier.present?
-              files << %W[config/exekutor.#{identifier}.yml config/exekutor.#{identifier}.yaml]
-                         .lazy.map { |path| Rails.root.join(path) }
-                         .find { |path| File.exists? path }
+              %W[config/exekutor.#{identifier}.yml config/exekutor.#{identifier}.yaml].each do |path|
+                path = Rails.root.join(path)
+                if File.exist? path
+                  files.append path
+                  break
+                end
+              end
             end
-            files.compact
+            files
           end
         end
 
+        # rubocop:enable Style/FormatStringToken
+
         DEFAULT_PIDFILE = DefaultPidFileValue.new.freeze
         DEFAULT_CONFIG_FILES = DefaultConfigFileValue.new.freeze
 
-        DEFAULT_THREADS = DefaultOptionValue.new("Minimum: 1, Maximum: Active record pool size minus 1, with a minimum of 1").freeze
+        DEFAULT_THREADS = DefaultOptionValue.new(
+          "Minimum: 1, Maximum: Active record pool size minus 1, with a minimum of 1"
+        ).freeze
         DEFAULT_QUEUE = DefaultOptionValue.new("All queues").freeze
         DEFAULT_FOREVER = DefaultOptionValue.new("Forever").freeze
 
-        DEFAULT_CONFIGURATION = { set_db_connection_name: true }
+        DEFAULT_CONFIGURATION = { set_db_connection_name: true }.freeze
 
         class Error < StandardError; end
       end
     end
   end
-end
+end