exekutor 0.1.0 → 0.1.2

data/lib/exekutor/internal/configuration_builder.rb

@@ -14,46 +14,47 @@ module Exekutor
       # Indicates an unset value
       # @private
       DEFAULT_VALUE = Object.new.freeze
-      private_constant "DEFAULT_VALUE"
+      private_constant :DEFAULT_VALUE
 
       # Sets option values in bulk
       # @return [self]
       def set(**options)
         invalid_options = options.keys - __option_names
         if invalid_options.present?
-          raise error_class, "Invalid option#{"s" if invalid_options.many?}: #{invalid_options.map(&:inspect).join(", ")}"
+          raise error_class, "Invalid option#{"s" if invalid_options.many?}: #{
+            invalid_options.map(&:inspect).join(", ")}"
         end
 
         options.each do |name, value|
-          send "#{name}=", value
+          send :"#{name}=", value
         end
         self
       end
 
-      module ClassMethods
+      class_methods do # rubocop:disable Metrics/BlockLength
         # Defines a configuration option with the given name.
         # @param name [Symbol] the name of the option
-        # @param required [Boolean] whether a value is required. If +true+, any +nil+ or +blank?+ value will not be allowed.
-        # @param type [Class,Array<Class>] the allowed value types. If set the value must be an instance of any of the given classes.
+        # @param required [Boolean] whether a value is required. If +true+, any +nil+ or +blank?+ value will not be
+        #   allowed.
+        # @param type [Class,Array<Class>] the allowed value types. If set the value must be an instance of any of the
+        #   given classes.
         # @param enum [Array<Any>] the allowed values. If set the value must be one of the given values.
         # @param range [Range] the allowed value range. If set the value must be included in this range.
         # @param default [Any] the default value
         # @param reader [Symbol] the name of the reader method
+        # rubocop:disable Metrics/ParameterLists
         def define_option(name, required: false, type: nil, enum: nil, range: nil, default: DEFAULT_VALUE,
-                          reader: name)
+                          reader: name, &block)
+          # rubocop:enable Metrics/ParameterLists
           __option_names << name
-          if reader
-            define_method reader do
-              if instance_variable_defined? :"@#{name}"
-                instance_variable_get :"@#{name}"
-              elsif default.respond_to? :call
-                default.call
-              elsif default != DEFAULT_VALUE
-                default
-              end
-            end
-          end
-          define_method "#{name}=" do |value|
+          define_reader(reader, name, default) if reader
+          define_writer(name, required, type, enum, range, &block)
+        end
+
+        private
+
+        def define_writer(name, required, type, enum, range)
+          define_method :"#{name}=" do |value|
             validate_option_presence! name, value if required
             validate_option_type! name, value, *type if type.present?
             validate_option_enum! name, value, *enum if enum.present?
@@ -64,14 +65,26 @@ module Exekutor
             self
           end
         end
+
+        def define_reader(name, variable_name, default)
+          define_method name do
+            if instance_variable_defined? :"@#{variable_name}"
+              instance_variable_get :"@#{variable_name}"
+            elsif default.respond_to? :call
+              default.call
+            elsif default != DEFAULT_VALUE
+              default
+            end
+          end
+        end
       end
 
       # Validates whether the option is present for configuration values that are required
       # raise [StandardError] if the value is nil or blank
       def validate_option_presence!(name, value)
-        unless value.present? || value.is_a?(FalseClass)
-          raise error_class, "##{name} cannot be #{value.nil? ? "nil" : "blank"}"
-        end
+        return if value.present? || value.is_a?(FalseClass)
+
+        raise error_class, "##{name} cannot be #{value.nil? ? "nil" : "blank"}"
       end
 
       # Validates whether the value class is allowed
@@ -79,7 +92,10 @@ module Exekutor
       def validate_option_type!(name, value, *allowed_types)
         return if allowed_types.include?(value.class)
 
-        raise error_class, "##{name} should be an instance of #{allowed_types.to_sentence(last_word_connector: ', or ')} (Actual: #{value.class})"
+        raise error_class, "##{name} should be an instance of #{
+          allowed_types.map { |c| c == NilClass || c.nil? ? "nil" : c }
+                       .to_sentence(two_words_connector: " or ", last_word_connector: ", or ")
+        } (Actual: #{value.class})"
       end
 
       # Validates whether the value is a valid enum option
@@ -87,7 +103,10 @@ module Exekutor
       def validate_option_enum!(name, value, *allowed_values)
         return if allowed_values.include?(value)
 
-        raise error_class, "##{name} should be one of #{allowed_values.map(&:inspect).to_sentence(last_word_connector: ', or ')}"
+        raise error_class, "##{name} should be one of #{
+          allowed_values.map(&:inspect)
+                        .to_sentence(two_words_connector: " or ", last_word_connector: ", or ")
+        }"
       end
 
       # Validates whether the value falls in the allowed range
@@ -95,10 +114,10 @@ module Exekutor
       def validate_option_range!(name, value, allowed_range)
         return if allowed_range.include?(value)
 
-        raise error_class, "##{name} should be between #{allowed_range.first} and #{allowed_range.last}#{
-          if allowed_range.respond_to?(:exclude_end?) && allowed_range.exclude_end?
-            " (exclusive)"
-          end}"
+        raise error_class, <<~MSG
+            ##{name} should be between #{allowed_range.first.inspect} and #{allowed_range.last.inspect}#{
+          " (exclusive)" if allowed_range.respond_to?(:exclude_end?) && allowed_range.exclude_end?}
+        MSG
       end
 
       protected
@@ -106,8 +125,8 @@ module Exekutor
       # The error class to raise when an invalid option value is set
       # @return [Class<StandardError>]
       def error_class
-        raise "Implementing class should override #error_class"
+        raise NotImplementedError, "Implementing class should override #error_class"
       end
     end
   end
-end
+end

data/lib/exekutor/internal/database_connection.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Exekutor
   # @private
   module Internal
@@ -9,10 +11,14 @@ module Exekutor
       end
 
       # The connection name for the specified worker id and process
+      # @param id [String] the id of the worker
+      # @param process [nil,String] the process name
       def self.application_name(id, process = nil)
         "Exekutor[id: #{id}]#{" #{process}" if process}"
       end
 
+      # Reconnects the database if it is not active
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] the connection adapter to use
       def self.ensure_active!(connection = BaseRecord.connection)
         connection.reconnect! unless connection.active?
       end

data/lib/exekutor/internal/executable.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Exekutor
   # Contains internal classes
   # @private
@@ -7,6 +9,7 @@ module Exekutor
       # Possible states
       STATES = %i[pending started stopped crashed killed].freeze
 
+      # Initializes the internal variables
       def initialize
         @state = Concurrent::AtomicReference.new(:pending)
         @consecutive_errors = Concurrent::AtomicFixnum.new(0)
@@ -23,17 +26,18 @@ module Exekutor
         @state.get
       end
 
-      # Whether the state equals +:started+
+      # @return [Boolean] whether the state equals +:started+
       def running?
         @state.get == :started
       end
 
+      # @return [Concurrent::AtomicFixnum] the number of consecutive errors that have occurred
       def consecutive_errors
         @consecutive_errors
       end
 
-      # Calculates an exponential delay based on {#consecutive_errors}. The delay ranges from 10 seconds on the first error
-      # to 10 minutes from the 13th error on.
+      # Calculates an exponential delay based on {#consecutive_errors}. The delay ranges from 10 seconds on the first
+      # error to 10 minutes from the 13th error on.
       # @return [Float] The delay
       def restart_delay
         if @consecutive_errors.value > 150
@@ -41,7 +45,7 @@ module Exekutor
           Exekutor.on_fatal_error error
           raise error
         end
-        delay = (9 + @consecutive_errors.value ** 2.5)
+        delay = (9 + (@consecutive_errors.value**2.5))
         delay += delay * (rand(-5..5) / 100.0)
         delay.clamp(10.0, 600.0)
       end
@@ -50,7 +54,8 @@ module Exekutor
 
       # Changes the state to the given value if the current state matches the expected state. Does nothing otherwise.
       # @param expected_state [:pending,:started,:stopped,:crashed] the expected state
-      # @param new_state [:pending,:started,:stopped,:crashed] the state to change to if the current state matches the expected
+      # @param new_state [:pending,:started,:stopped,:crashed] the state to change to if the current state matches the
+      #   expected
       # @raise ArgumentError if an invalid state was passed
       def compare_and_set_state(expected_state, new_state)
         validate_state! new_state
@@ -59,7 +64,7 @@ module Exekutor
 
       # Updates the state to the given value
       # @raise ArgumentError if an invalid state was passed
-      def set_state(new_state)
+      def state=(new_state)
         validate_state! new_state
         @state.set new_state
       end
@@ -72,4 +77,4 @@ module Exekutor
       end
     end
   end
-end
+end

data/lib/exekutor/internal/executor.rb

@@ -15,6 +15,15 @@ module Exekutor
       define_callbacks :after_execute, freeze: true
       attr_reader :pending_job_updates
 
+      # rubocop:disable Metrics/ParameterLists
+
+      # Create a new executor
+      # @param min_threads [Integer] the minimum number of threads that should be active
+      # @param max_threads [Integer] the maximum number of threads that may be active
+      # @param max_thread_idletime [Integer] the amount of seconds a thread may be idle before being reclaimed
+      # @param delete_completed_jobs [Boolean] whether to delete jobs that complete successfully
+      # @param delete_discarded_jobs [Boolean] whether to delete jobs that are discarded
+      # @param delete_failed_jobs [Boolean] whether to delete jobs that fail
       def initialize(min_threads: 1, max_threads: default_max_threads, max_thread_idletime: 180,
                      delete_completed_jobs: false, delete_discarded_jobs: false, delete_failed_jobs: false)
         super()
@@ -31,14 +40,16 @@ module Exekutor
         }.freeze
       end
 
+      # rubocop:enable Metrics/ParameterLists
+
       # Starts the executor
       def start
-        set_state :started
+        self.state = :started
       end
 
       # Stops the executor
       def stop
-        set_state :stopped
+        self.state = :stopped
 
         @executor.shutdown
       end
@@ -54,7 +65,7 @@ module Exekutor
       # Executes the job on one of the execution threads. Releases the job if there is no thread available to execute
       # the job.
       def post(job)
-        @executor.post job, &method(:execute)
+        @executor.post(job) { |*args| execute(*args) }
         @queued_job_ids.append(job[:id])
       rescue Concurrent::RejectedExecutionError
         logger.error "Ran out of threads! Releasing job #{job[:id]}"
@@ -108,20 +119,16 @@ module Exekutor
         @active_job_ids.delete(job[:id])
       end
 
-      def _execute(job, start_time: Concurrent.monotonic_time)
+      def _execute(job, start_time: Process.clock_gettime(Process::CLOCK_MONOTONIC))
         raise Exekutor::DiscardJob, "Maximum queue time expired" if queue_time_expired?(job)
 
-        if (timeout = job[:options] && job[:options]["execution_timeout"]).present?
-          Timeout.timeout Float(timeout), JobExecutionTimeout do
-            ActiveJob::Base.execute(job[:payload])
-          end
-        else
+        with_job_execution_timeout job.dig(:options, "execution_timeout") do
           ActiveJob::Base.execute(job[:payload])
         end
 
-        on_job_completed(job, runtime: Concurrent.monotonic_time - start_time)
+        on_job_completed(job, runtime: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time)
       rescue StandardError, JobExecutionTimeout => e
-        on_job_failed(job, e, runtime: Concurrent.monotonic_time - start_time)
+        on_job_failed(job, e, runtime: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time)
       rescue Exception # rubocop:disable Lint/RescueException
         # Try to release job when an Exception occurs
         update_job job, status: "p", worker_id: nil
@@ -129,15 +136,17 @@ module Exekutor
       end
 
       def on_job_completed(job, runtime:)
-        if @options[:delete_completed_jobs]
+        next_status = "c"
+        if delete_job? next_status
           delete_job job
         else
-          update_job job, status: "c", runtime: runtime
+          update_job job, status: next_status, runtime: runtime
         end
       end
 
       def on_job_failed(job, error, runtime:)
-        discarded = [Exekutor::DiscardJob, JobExecutionTimeout].any?(&error.method(:is_a?))
+        discarded = [Exekutor::DiscardJob, JobExecutionTimeout].any? { |c| error.is_a? c }
+        next_status = discarded ? "d" : "f"
         unless discarded
           Internal::Hooks.on(:job_failure, job, error)
           log_error error, "Job failed"
@@ -147,14 +156,25 @@ module Exekutor
           # Don't consider this as a failure, try again later.
           update_job job, status: "p", worker_id: nil
 
-        elsif @options[discarded ? :delete_discarded_jobs : :delete_failed_jobs]
+        elsif delete_job? next_status
           delete_job job
 
-        else
           # Try to update the job and create a JobError record if update succeeds
-          if update_job job, status: discarded ? "d" : "f", runtime: runtime
-            JobError.create!(job_id: job[:id], error: error)
-          end
+        elsif update_job job, status: next_status, runtime: runtime
+          JobError.create(job_id: job[:id], error: error)
+        end
+      end
+
+      def delete_job?(next_status)
+        case next_status
+        when "c"
+          @options[:delete_completed_jobs]
+        when "d"
+          @options[:delete_discarded_jobs]
+        when "f"
+          @options[:delete_failed_jobs]
+        else
+          false
         end
       end
 
@@ -199,9 +219,18 @@ module Exekutor
           job[:options]["start_execution_before"].to_f <= Time.now.to_f
       end
 
+      def with_job_execution_timeout(timeout, &block)
+        if timeout
+          Timeout.timeout Float(timeout), JobExecutionTimeout, &block
+        else
+          yield
+        end
+      end
+
       def lost_db_connection?(error)
-        [ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished].any?(&error.method(:kind_of?)) &&
-          !ActiveRecord::Base.connection.active?
+        [ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished].any? do |error_class|
+          error.is_a? error_class
+        end && !ActiveRecord::Base.connection.active?
       end
 
       # The default maximum number of threads. The value is equal to the size of the DB connection pool minus 1, with

data/lib/exekutor/internal/hooks.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
+# The Exekutor namespace
 module Exekutor
   module Internal
+    # The internal implementation of the Exekutor hooks
     class Hooks
       include Internal::Callbacks
 
@@ -14,14 +18,15 @@ module Exekutor
       def register(callback = nil, &block)
         if callback
           callback = callback.new if callback.is_a? Class
-          raise 'callback must respond to #callbacks' unless callback.respond_to? :callbacks
+          raise "callback must respond to #callbacks" unless callback.respond_to? :callbacks
+
           callback.callbacks.each do |type, callbacks|
-            callbacks.each { |callback| add_callback! type, [], callback }
+            callbacks.each { |cb| add_callback! type, [], cb }
           end
-        elsif block.arity == 1
-          block.call self
-        else
+        elsif block.arity.zero?
           instance_eval(&block)
+        else
+          yield self
         end
       end
 
@@ -82,6 +87,4 @@ module Exekutor
   # @!attribute [r] hooks
   #   @return [Internal::Hooks] The hooks for exekutor.
   mattr_reader :hooks, default: Internal::Hooks.new
-
-  # TODO register_hook method instead of `hooks.register`?
-end
+end

data/lib/exekutor/internal/listener.rb

@@ -7,13 +7,16 @@ module Exekutor
   module Internal
     # Listens for jobs to be executed
     class Listener
-      include Executable, Logger
+      include Executable
+      include Logger
 
       # The PG notification channel for enqueued jobs
       JOB_ENQUEUED_CHANNEL = "exekutor::job_enqueued"
       # The PG notification channel for a worker. Must be formatted with the worker ID.
       PROVIDER_CHANNEL = "exekutor::worker::%s"
 
+      # rubocop:disable Metrics/ParameterLists
+
       # Creates a new listener
       # @param worker_id [String] the ID of the worker
       # @param queues [Array<String>] the queues to watch
@@ -21,11 +24,14 @@ module Exekutor
       # @param pool [ThreadPoolExecutor] the thread pool to use
       # @param wait_timeout [Integer] the time to listen for notifications
       # @param set_db_connection_name [Boolean] whether to set the application name on the DB connection
-      def initialize(worker_id:, queues: nil, provider:, pool:, wait_timeout: 60, set_db_connection_name: false)
+      def initialize(worker_id:, provider:, pool:, queues: nil, min_priority: nil, max_priority: nil, wait_timeout: 60,
+                     set_db_connection_name: false)
         super()
         @config = {
           worker_id: worker_id,
-          queues: queues || [],
+          queues: queues.presence,
+          min_priority: min_priority,
+          max_priority: max_priority,
           wait_timeout: wait_timeout,
           set_db_connection_name: set_db_connection_name
         }
@@ -37,6 +43,8 @@ module Exekutor
         @listening = Concurrent::AtomicBoolean.new false
       end
 
+      # rubocop:enable Metrics/ParameterLists
+
       # Starts the listener
       def start
         return false unless compare_and_set_state :pending, :started
@@ -47,11 +55,11 @@ module Exekutor
 
       # Stops the listener
       def stop
-        set_state :stopped
+        self.state = :stopped
         begin
           Exekutor::Job.connection.execute(%(NOTIFY "#{provider_channel}"))
         rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished
-          #ignored
+          # ignored
         end
       end
 
@@ -62,16 +70,33 @@ module Exekutor
         PROVIDER_CHANNEL % @config[:worker_id]
       end
 
-      # Whether this listener is listening to the given queue
-      # @return [Boolean]
+      # @return [Boolean] Whether the job matches the configured queues and priority range
+      def job_filter_match?(job_info)
+        listening_to_queue?(job_info["q"]) && listening_to_priority?(job_info["p"].to_i)
+      end
+
+      # @return [Boolean] Whether this listener is listening to the given queue
       def listening_to_queue?(queue)
         queues = @config[:queues]
-        queues.empty? || queues.include?(queue)
+        queues.nil? || queues.include?(queue)
+      end
+
+      # @return [Boolean] Whether this listener is listening to the given priority
+      def listening_to_priority?(priority)
+        minimum = @config[:min_priority]
+        maximum = @config[:max_priority]
+        (minimum.nil? || minimum <= priority) && (maximum.nil? || maximum >= priority)
       end
 
       # Starts the listener thread
-      def start_thread
-        @pool.post(&method(:run)) if running?
+      def start_thread(delay: nil)
+        return unless running?
+
+        if delay
+          Concurrent::ScheduledTask.execute(delay, executor: @pool) { run }
+        else
+          @pool.post { run }
+        end
       end
 
       # Sets up the PG notifications and listens for new jobs
@@ -79,30 +104,33 @@ module Exekutor
         return unless running? && @thread_running.make_true
 
         with_pg_connection do |connection|
-          begin
-            connection.exec(%(LISTEN "#{provider_channel}"))
-            connection.exec(%(LISTEN "#{JOB_ENQUEUED_CHANNEL}"))
-            consecutive_errors.value = 0
-            catch(:shutdown) { wait_for_jobs(connection) }
-          ensure
-            connection.exec("UNLISTEN *")
-          end
-        end
-      rescue StandardError => err
-        Exekutor.on_fatal_error err, "[Listener] Runtime error!"
-        set_state :crashed if err.is_a? UnsupportedDatabase
-
-        if running?
-          consecutive_errors.increment
-          delay = restart_delay
-          logger.info "Restarting in %0.1f seconds…" % [delay]
-          Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
+          connection.exec(%(LISTEN "#{provider_channel}"))
+          connection.exec(%(LISTEN "#{JOB_ENQUEUED_CHANNEL}"))
+          consecutive_errors.value = 0
+          catch(:shutdown) { wait_for_jobs(connection) }
+        ensure
+          connection.exec("UNLISTEN *")
         end
+      rescue StandardError => e
+        on_thread_error(e)
       ensure
         @thread_running.make_false
         @listening.make_false
       end
 
+      # Called when an error is raised in #run
+      def on_thread_error(error)
+        Exekutor.on_fatal_error error, "[Listener] Runtime error!"
+        self.state = :crashed if error.is_a? UnsupportedDatabase
+
+        return unless running?
+
+        consecutive_errors.increment
+        delay = restart_delay
+        logger.info format("Restarting in %0.1f seconds…", delay)
+        start_thread delay: delay
+      end
+
       # Listens for jobs. Blocks until the listener is stopped
       def wait_for_jobs(connection)
         while running?
@@ -112,25 +140,20 @@ module Exekutor
             next unless channel == JOB_ENQUEUED_CHANNEL
 
             job_info = begin
-                         payload.split(";").map { |el| el.split(":") }.to_h
-                       rescue
-                         logger.error "Invalid notification payload: #{payload}"
-                         next
+                         JobParser.parse(payload)
+                       rescue StandardError => e
+                         logger.error e.message
+                         nil
                        end
-            unless %w[id q t].all? { |n| job_info[n].present? }
-              logger.error "[Listener] Notification payload is missing #{%w[id q t].select { |n| job_info[n].blank? }.join(", ")}"
-              next
-            end
-            next unless listening_to_queue? job_info["q"]
-
-            scheduled_at = job_info["t"].to_f
-            @provider.update_earliest_scheduled_at(scheduled_at)
+            next unless job_info && job_filter_match?(job_info)
+
+            @provider.update_earliest_scheduled_at(job_info["t"].to_f)
           end
         end
       end
 
-      # Gets a DB connection and removes it from the pool. Sets the application name if +set_db_connection_name+ is true.
-      # Closes the connection after yielding it to the given block.
+      # Gets a DB connection and removes it from the pool. Sets the application name if +set_db_connection_name+ is
+      # true. Closes the connection after yielding it to the given block.
       # (Grabbed from PG adapter for action cable)
       # @yield yields the connection
       # @yieldparam connection [PG::Connection] the DB connection
@@ -159,6 +182,7 @@ module Exekutor
           raise UnsupportedDatabase,
                 "The raw connection of the active record connection adapter must be an instance of PG::Connection"
         end
+        true
       end
 
       # For testing purposes
@@ -166,10 +190,28 @@ module Exekutor
         @listening.true?
       end
 
+      # Parses a NOTIFY payload to a job
+      class JobParser
+        JOB_INFO_KEYS = %w[id q p t].freeze
+
+        def self.parse(payload)
+          job_info = begin
+                       payload.split(";").to_h { |el| el.split(":") }
+                     rescue StandardError
+                       raise Error, "Invalid notification payload: #{payload}"
+                     end
+          if (missing_keys = JOB_INFO_KEYS.select { |n| job_info[n].blank? }).present?
+            raise Error, "[Listener] Notification payload is missing #{missing_keys.join(", ")}"
+          end
+
+          job_info
+        end
+      end
+
       # Raised when an error occurs in the listener.
       class Error < Exekutor::Error; end
 
-      # Raised when the database connection is not an instance of PG::Connection.
+      # Raised when the database connection is not an instance of +PG::Connection+.
       class UnsupportedDatabase < Exekutor::Error; end
     end
   end