exekutor 0.1.0

data/lib/exekutor/internal/cli/daemon.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Exekutor
+  # @private
+  module Internal
+    module CLI
+      # Manages daemonization of the current process.
+      # @private
+      class Daemon
+        # The path of the generated pidfile.
+        # @return [String]
+        attr_reader :pidfile
+
+        # @param pidfile [String] Pidfile path
+        def initialize(pidfile:)
+          @pidfile = pidfile
+        end
+
+        # Daemonizes the current process and writes out a pidfile.
+        # @return [void]
+        def daemonize
+          validate!
+          ::Process.daemon true
+          write_pid
+        end
+
+        # The process ID for this daemon, if known
+        # @return [Integer,nil] The process ID
+        # @raise [Error] if the pid-file is corrupt
+        def pid
+          return nil unless ::File.exist? pidfile
+
+          pid = ::File.read(pidfile)
+          if pid.to_i.positive?
+            pid.to_i
+          else
+            raise Error, "Corrupt PID-file. Check #{pidfile}"
+          end
+        end
+
+        # The process status for this daemon. Possible states are:
+        # - +:running+ when the daemon is running;
+        # - +:not_running+ when the daemon is not running;
+        # - +:dead+ when the daemon is dead. (Ie. the PID is known, but the process is gone);
+        # - +:not_owned+ when the daemon cannot be accessed.
+        # @return [:running, :not_running, :dead, :not_owned] the status
+        def status
+          pid = self.pid
+          return :not_running if pid.nil?
+
+          # If sig is 0, then no signal is sent, but error checking is still performed; this can be used to check for the
+          # existence of a process ID or process group ID.
+          ::Process.kill(0, pid)
+          :running
+        rescue Errno::ESRCH
+          :dead
+        rescue Errno::EPERM
+          :not_owned
+        end
+
+        # Checks whether {#status} matches any of the given statuses.
+        # @param statuses [Symbol...] The statuses to check for.
+        # @return [Boolean] whether the status matches
+        # @see #status
+        def status?(*statuses)
+          statuses.include? self.status
+        end
+
+        # Raises an {Error} if a daemon is already running. Deletes the pidfile is the process is dead.
+        # @return [void]
+        # @raise [Error] when the daemon is running
+        def validate!
+          case self.status
+          when :running, :not_owned
+            raise Error, "A worker is already running. Check #{pidfile}"
+          else
+            delete_pid
+          end
+          nil
+        end
+
+        private
+
+        # Writes the current process ID to the pidfile. The pidfile will be deleted upon exit.
+        # @return [void]
+        # @see #pidfile
+        # @raise [Error] is the daemon is already running
+        def write_pid
+          File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(::Process.pid.to_s) }
+          at_exit { delete_pid }
+        rescue Errno::EEXIST
+          validate!
+          retry
+        end
+
+        # Deletes the pidfile
+        # @return [void]
+        # @see #pidfile
+        def delete_pid
+          File.delete(pidfile) if File.exist?(pidfile)
+        end
+
+        # Raised when spawning a daemon process fails
+        class Error < StandardError; end
+      end
+    end
+  end
+end

data/lib/exekutor/internal/cli/default_option_value.rb

@@ -0,0 +1,29 @@
+module Exekutor
+  # @private
+  module Internal
+    module CLI
+      # Used as a default value for CLI flags.
+      # @private
+      class DefaultOptionValue
+        def initialize(description = nil, value: nil)
+          @description = description || value&.to_s || "none"
+          @value = value
+        end
+
+        # The value to display in the CLI help message
+        def to_s
+          @description
+        end
+
+        # The actual value, if set. If the value responds to +call+, it will be called
+        def value
+          if @value.respond_to? :call
+            @value.call
+          else
+            @value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/exekutor/internal/cli/info.rb

@@ -0,0 +1,126 @@
+require_relative "application_loader"
+require "terminal-table"
+
+module Exekutor
+  # @private
+  module Internal
+    module CLI
+      # Prints info for the CLI
+      # @private
+      class Info
+        include ApplicationLoader
+
+        def initialize(options)
+          @global_options = options
+        end
+
+        # Prints Exekutor info to STDOUT
+        def print(options)
+          load_application(options[:environment], print_message: !quiet?)
+
+          ActiveSupport.on_load(:active_record, yield: true) do
+            # Use system time zone
+            Time.zone = Time.new.zone
+
+            hosts = Exekutor::Info::Worker.distinct.pluck(:hostname)
+            job_info = Exekutor::Job.pending.order(:queue).group(:queue)
+                                    .pluck(:queue, Arel.sql("COUNT(*)"), Arel.sql("MIN(scheduled_at)"))
+
+            clear_application_loading_message unless quiet?
+            puts Rainbow("Workers").bright.blue
+            if hosts.present?
+              total_workers = 0
+              hosts.each do |host|
+                table = Terminal::Table.new
+                table.title = host if hosts.many?
+                table.headings = ["id", "Status", "Last heartbeat"]
+                worker_count = 0
+                Exekutor::Info::Worker.where(hostname: host).each do |worker|
+                  worker_count += 1
+                  table << [
+                    worker.id.split("-").first << "…",
+                    worker.status,
+                    if worker.last_heartbeat_at.nil?
+                      if !worker.running?
+                        "N/A"
+                      elsif worker.created_at < 10.minutes.ago
+                        Rainbow("None").red
+                      else
+                        "None"
+                      end
+                    elsif worker.last_heartbeat_at > 2.minutes.ago
+                      worker.last_heartbeat_at.strftime "%R"
+                    elsif worker.last_heartbeat_at > 10.minutes.ago
+                      Rainbow(worker.last_heartbeat_at.strftime("%R")).yellow
+                    else
+                      Rainbow(worker.last_heartbeat_at.strftime("%D %R")).red
+                    end
+                  ]
+                  # TODO switch / flag to print threads and queues
+                end
+                total_workers += worker_count
+                table.add_separator
+                table.add_row [(hosts.many? ? "Subtotal" : "Total"), { value: worker_count, alignment: :right, colspan: 2 }]
+                puts table
+              end
+
+              if hosts.many?
+                puts Terminal::Table.new rows: [
+                  ["Total hosts", hosts.size],
+                  ["Total workers", total_workers]
+                ]
+              end
+            else
+              message = Rainbow("There are no active workers")
+              message = message.red if job_info.present?
+              puts message
+            end
+
+            puts " "
+            puts "#{Rainbow("Jobs").bright.blue}"
+            if job_info.present?
+              table = Terminal::Table.new
+              table.headings = ["Queue", "Pending jobs", "Next job scheduled at"]
+              total_count = 0
+              job_info.each do |queue, count, min_scheduled_at|
+                table << [
+                  queue, count,
+                  if min_scheduled_at.nil?
+                    "N/A"
+                  elsif min_scheduled_at < 30.minutes.ago
+                    Rainbow(min_scheduled_at.strftime("%D %R")).red
+                  elsif min_scheduled_at < 1.minute.ago
+                    Rainbow(min_scheduled_at.strftime("%D %R")).yellow
+                  else
+                    min_scheduled_at.strftime("%D %R")
+                  end
+                ]
+                total_count += count
+              end
+              if job_info.many?
+                table.add_separator
+                table.add_row ["Total", { value: total_count, alignment: :right, colspan: 2 }]
+              end
+              puts table
+            else
+              puts Rainbow("No pending jobs").green
+            end
+          end
+        end
+
+        private
+
+        # @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
+        def quiet?
+          !!@global_options[:quiet]
+        end
+
+        # @return [Boolean] Whether verbose mode is enabled. Always returns false if quiet mode is enabled.
+        def verbose?
+          !quiet? && !!@global_options[:verbose]
+        end
+
+      end
+    end
+  end
+end

data/lib/exekutor/internal/cli/manager.rb

@@ -0,0 +1,260 @@
+require_relative "application_loader"
+require_relative "default_option_value"
+require_relative "daemon"
+
+module Exekutor
+  # @private
+  module Internal
+    module CLI
+      # Manager for the CLI
+      # @private
+      class Manager
+        include ApplicationLoader
+
+        def initialize(options)
+          @global_options = options
+        end
+
+        # Starts a new worker
+        # @option options [Boolean] :restart Whether the worker is being restarted
+        # @option options [Boolean] :daemonize Whether the worker should be daemonized
+        # @option options [String] :environment The Rails environment to load
+        # @option options [String] :queue The queue(s) to watch
+        # @option options [String] :threads The number of threads to use for job execution
+        # @option options [Integer] :poll_interval The interval in seconds for job polling
+        # @return [Void]
+        def start(options)
+          daemonize(restarting: options[:restart]) if options[:daemonize]
+
+          load_application(options[:environment])
+
+          config_files = if options[:configfile].is_a? DefaultConfigFileValue
+                           options[:configfile].to_a(@global_options[:identifier])
+                         else
+                           options[:configfile]&.map { |path| File.expand_path(path, Rails.root) }
+                         end
+
+          worker_options = DEFAULT_CONFIGURATION.dup
+
+          config_files&.each do |path|
+            puts "Loading config file: #{path}" if verbose?
+            config = begin
+                       YAML.safe_load(File.read(path), symbolize_names: true)
+                     rescue => e
+                       raise Error, "Cannot read config file: #{path} (#{e.to_s})"
+                     end
+            unless config.keys == [:exekutor]
+              raise Error, "Config should have an `exekutor` root node: #{path} (Found: #{config.keys.join(', ')})"
+            end
+
+            # Remove worker specific options before calling Exekutor.config.set
+            worker_options.merge! config[:exekutor].extract!(:queue, :status_server_port)
+
+            begin
+              Exekutor.config.set **config[:exekutor]
+            rescue => e
+              raise Error, "Cannot load config file: #{path} (#{e.to_s})"
+            end
+          end
+
+          worker_options.merge! Exekutor.config.worker_options
+          worker_options.merge! @global_options.slice(:identifier)
+          if verbose?
+            worker_options[:verbose] = true
+          elsif quiet?
+            worker_options[:quiet] = true
+          end
+          if options[:threads] && !options[:threads].is_a?(DefaultOptionValue)
+            min, max = if options[:threads].is_a?(Integer)
+                         [options[:threads], options[:threads]]
+                       else
+                         options[:threads].to_s.split(":")
+                       end
+            if max.nil?
+              options[:min_threads] = options[:max_threads] = Integer(min)
+            else
+              options[:min_threads] = Integer(min)
+              options[:max_threads] = Integer(max)
+            end
+          end
+          worker_options.merge!(
+            options.slice(:queue, :min_threads, :max_threads, :poll_interval)
+                   .reject { |_, value| value.is_a? DefaultOptionValue }
+                   .transform_keys(poll_interval: :polling_interval)
+          )
+
+          worker_options[:queue] = nil if worker_options[:queue] == ["*"]
+
+          # TODO health check server
+
+          # Specify `yield: true` to prevent running in the context of the loaded module
+          ActiveSupport.on_load(:exekutor, yield: true) do
+            ActiveSupport.on_load(:active_record, yield: true) do
+              worker = Worker.new(worker_options)
+              %w[INT TERM QUIT].each do |signal|
+                ::Kernel.trap(signal) { ::Thread.new { worker.stop } }
+              end
+
+              Process.setproctitle "Exekutor worker #{worker.id} [#{Rails.root}]"
+              if worker_options[:set_db_connection_name]
+                Internal::BaseRecord.connection.class.set_callback(:checkout, :after) do
+                  Internal::DatabaseConnection.set_application_name raw_connection, worker.id
+                end
+                Internal::BaseRecord.connection_pool.connections.each do |conn|
+                  Internal::DatabaseConnection.set_application_name conn.raw_connection, worker.id
+                end
+              end
+
+              ActiveSupport.on_load(:active_job, yield: true) do
+                puts "Worker #{worker.id} started (Use `#{Rainbow("ctrl + c").magenta}` to stop)" unless quiet?
+                puts "#{worker_options.pretty_inspect}" if verbose?
+                begin
+                  worker.start
+                  worker.join
+                ensure
+                  worker.stop if worker.running?
+                end
+              end
+            end
+          end
+        end
+
+        def stop(options)
+          daemon = Daemon.new(pidfile: pidfile)
+          pid = daemon.pid
+          if pid.nil?
+            unless quiet?
+              if options[:restart]
+                puts "Executor was not running"
+              else
+                puts "Executor is not running (pidfile not found at #{daemon.pidfile})"
+              end
+            end
+            return
+          elsif daemon.status? :not_running, :dead
+            return
+          end
+
+          Process.kill("INT", pid)
+          sleep(0.3)
+          wait_until = if options[:shutdown_timeout].nil? || options[:shutdown_timeout] == DEFAULT_FOREVER
+                         nil
+                       else
+                         Time.now + options[:shutdown_timeout]
+                       end
+          while daemon.status?(:running, :not_owned)
+            puts "Waiting for worker to finish…" unless quiet?
+            if wait_until && wait_until > Time.now
+              Process.kill("TERM", pid)
+              break
+            end
+            sleep 0.1
+          end
+          puts "Worker (PID: #{pid}) stopped." unless quiet?
+        end
+
+        def restart(stop_options, start_options)
+          stop stop_options.merge(restart: true)
+          start start_options.merge(restart: true, daemonize: true)
+        end
+
+        private
+
+        # @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
+        def quiet?
+          !!@global_options[:quiet]
+        end
+
+        # @return [Boolean] Whether verbose mode is enabled. Always returns false if quiet mode is enabled.
+        def verbose?
+          !quiet? && !!@global_options[:verbose]
+        end
+
+        # @return [String] The identifier for this worker
+        def identifier
+          @global_options[:identifier]
+        end
+
+        # @return [String] The path to the pidfile
+        def pidfile
+          pidfile = @global_options[:pidfile] || DEFAULT_PIDFILE
+          if pidfile == DEFAULT_PIDFILE
+            pidfile.for_identifier(identifier)
+          elsif identifier && pidfile.include?("%{identifier}")
+            pidfile.sub "%{identifier}", identifier
+          else
+            pidfile
+          end
+        end
+
+        # Daemonizes the current process. Do this before loading your application to prevent deadlocks.
+        # @return [Void]
+        def daemonize(restarting: false)
+          daemonizer = Daemon.new(pidfile: pidfile)
+          daemonizer.validate!
+          unless quiet?
+            if restarting
+              puts "Restarting worker as a daemon…"
+            else
+              stop_options = if @global_options[:pidfile] && @global_options[:pidfile] != DEFAULT_PIDFILE
+                               "--pid #{pidfile} "
+                             elsif identifier
+                               "--id #{identifier} "
+                             end
+
+              puts "Running worker as a daemon… (Use `#{Rainbow("exekutor #{stop_options}stop").magenta}` to stop)"
+            end
+          end
+          daemonizer.daemonize
+        rescue Daemon::Error => e
+          puts Rainbow(e.message).red
+          raise GLI::CustomExit.new(nil, 1)
+        end
+
+        class DefaultPidFileValue < DefaultOptionValue
+          def initialize
+            super("tmp/pids/exekutor[.%{identifier}].pid")
+          end
+
+          def for_identifier(identifier)
+            if identifier.nil? || identifier.length.zero?
+              "tmp/pids/exekutor.pid"
+            else
+              "tmp/pids/exekutor.#{identifier}.pid"
+            end
+          end
+        end
+
+        class DefaultConfigFileValue < DefaultOptionValue
+          def initialize
+            super('"config/exekutor.yml", overridden by "config/exekutor.%{identifier}.yml" if an identifier is specified')
+          end
+
+          def to_a(identifier = nil)
+            files = []
+            files << %w[config/exekutor.yml config/exekutor.yaml]
+                       .lazy.map { |path| Rails.root.join(path) }
+                       .find { |path| File.exists? path }
+            if identifier.present?
+              files << %W[config/exekutor.#{identifier}.yml config/exekutor.#{identifier}.yaml]
+                         .lazy.map { |path| Rails.root.join(path) }
+                         .find { |path| File.exists? path }
+            end
+            files.compact
+          end
+        end
+
+        DEFAULT_PIDFILE = DefaultPidFileValue.new.freeze
+        DEFAULT_CONFIG_FILES = DefaultConfigFileValue.new.freeze
+
+        DEFAULT_THREADS = DefaultOptionValue.new("Minimum: 1, Maximum: Active record pool size minus 1, with a minimum of 1").freeze
+        DEFAULT_QUEUE = DefaultOptionValue.new("All queues").freeze
+        DEFAULT_FOREVER = DefaultOptionValue.new("Forever").freeze
+
+        DEFAULT_CONFIGURATION = { set_db_connection_name: true }
+
+        class Error < StandardError; end
+      end
+    end
+  end
+end

data/lib/exekutor/internal/configuration_builder.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Exekutor
+  module Internal
+    # DSL for the configuration
+    # @private
+    module ConfigurationBuilder
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :__option_names, instance_writer: false, default: []
+      end
+
+      # Indicates an unset value
+      # @private
+      DEFAULT_VALUE = Object.new.freeze
+      private_constant "DEFAULT_VALUE"
+
+      # Sets option values in bulk
+      # @return [self]
+      def set(**options)
+        invalid_options = options.keys - __option_names
+        if invalid_options.present?
+          raise error_class, "Invalid option#{"s" if invalid_options.many?}: #{invalid_options.map(&:inspect).join(", ")}"
+        end
+
+        options.each do |name, value|
+          send "#{name}=", value
+        end
+        self
+      end
+
+      module ClassMethods
+        # Defines a configuration option with the given name.
+        # @param name [Symbol] the name of the option
+        # @param required [Boolean] whether a value is required. If +true+, any +nil+ or +blank?+ value will not be allowed.
+        # @param type [Class,Array<Class>] the allowed value types. If set the value must be an instance of any of the given classes.
+        # @param enum [Array<Any>] the allowed values. If set the value must be one of the given values.
+        # @param range [Range] the allowed value range. If set the value must be included in this range.
+        # @param default [Any] the default value
+        # @param reader [Symbol] the name of the reader method
+        def define_option(name, required: false, type: nil, enum: nil, range: nil, default: DEFAULT_VALUE,
+                          reader: name)
+          __option_names << name
+          if reader
+            define_method reader do
+              if instance_variable_defined? :"@#{name}"
+                instance_variable_get :"@#{name}"
+              elsif default.respond_to? :call
+                default.call
+              elsif default != DEFAULT_VALUE
+                default
+              end
+            end
+          end
+          define_method "#{name}=" do |value|
+            validate_option_presence! name, value if required
+            validate_option_type! name, value, *type if type.present?
+            validate_option_enum! name, value, *enum if enum.present?
+            validate_option_range! name, value, range if range.present?
+            yield value if block_given?
+
+            instance_variable_set :"@#{name}", value
+            self
+          end
+        end
+      end
+
+      # Validates whether the option is present for configuration values that are required
+      # raise [StandardError] if the value is nil or blank
+      def validate_option_presence!(name, value)
+        unless value.present? || value.is_a?(FalseClass)
+          raise error_class, "##{name} cannot be #{value.nil? ? "nil" : "blank"}"
+        end
+      end
+
+      # Validates whether the value class is allowed
+      # @raise [StandardError] if the type of value is not allowed
+      def validate_option_type!(name, value, *allowed_types)
+        return if allowed_types.include?(value.class)
+
+        raise error_class, "##{name} should be an instance of #{allowed_types.to_sentence(last_word_connector: ', or ')} (Actual: #{value.class})"
+      end
+
+      # Validates whether the value is a valid enum option
+      # @raise [StandardError] if the value is not included in the allowed values
+      def validate_option_enum!(name, value, *allowed_values)
+        return if allowed_values.include?(value)
+
+        raise error_class, "##{name} should be one of #{allowed_values.map(&:inspect).to_sentence(last_word_connector: ', or ')}"
+      end
+
+      # Validates whether the value falls in the allowed range
+      # @raise [StandardError] if the value is not included in the allowed range
+      def validate_option_range!(name, value, allowed_range)
+        return if allowed_range.include?(value)
+
+        raise error_class, "##{name} should be between #{allowed_range.first} and #{allowed_range.last}#{
+          if allowed_range.respond_to?(:exclude_end?) && allowed_range.exclude_end?
+            " (exclusive)"
+          end}"
+      end
+
+      protected
+
+      # The error class to raise when an invalid option value is set
+      # @return [Class<StandardError>]
+      def error_class
+        raise "Implementing class should override #error_class"
+      end
+    end
+  end
+end

data/lib/exekutor/internal/database_connection.rb

@@ -0,0 +1,21 @@
+module Exekutor
+  # @private
+  module Internal
+    # Helper methods for the DB connection name
+    module DatabaseConnection
+      # Sets the connection name
+      def self.set_application_name(pg_conn, id, process = nil)
+        pg_conn.exec("SET application_name = #{pg_conn.escape_identifier(application_name(id, process))}")
+      end
+
+      # The connection name for the specified worker id and process
+      def self.application_name(id, process = nil)
+        "Exekutor[id: #{id}]#{" #{process}" if process}"
+      end
+
+      def self.ensure_active!(connection = BaseRecord.connection)
+        connection.reconnect! unless connection.active?
+      end
+    end
+  end
+end