exekutor 0.1.0 → 0.1.2

data/lib/exekutor/internal/logger.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 require "rainbow"
 
+# The Exekutor namespace
 module Exekutor
   # @private
   module Internal
@@ -9,7 +12,7 @@ module Exekutor
 
       included do
         # The log tags to use when writing to the log
-        mattr_accessor :log_tags, default: [self.name.demodulize]
+        mattr_accessor :log_tags, default: [name.demodulize]
       end
 
       protected
@@ -35,25 +38,24 @@ module Exekutor
   # Prints a message to STDOUT, unless {Exekutor::Configuration#quiet?} is true
   # @private
   def self.say(*args)
-    puts(*args) unless config.quiet?
+    puts(*args) unless config.quiet? # rubocop:disable Rails/Output
   end
 
   # Prints the error in the log and to STDERR (unless {Exekutor::Configuration#quiet?} is true)
-  # @param err [Exception] the error to print
+  # @param error [Exception] the error to print
   # @param message [String] the message to print above the error
   # @return [void]
-  def self.print_error(err, message = nil)
-    @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
-    error = "#{err.class} – #{err.message}\nat #{@backtrace_cleaner.clean(err.backtrace).join("\n   ")}"
-
+  def self.print_error(error, message = nil)
+    error = strferr(error)
     unless config.quiet?
-      $stderr.puts Rainbow(message).bright.red if message
-      $stderr.puts Rainbow(error).red
+      warn Rainbow(message).bright.red if message
+      warn Rainbow(error).red
     end
-    unless ActiveSupport::Logger.logger_outputs_to?(logger, $stdout)
+    if config.quiet? || !ActiveSupport::Logger.logger_outputs_to?(logger, $stdout)
       logger.error message if message
       logger.error error
     end
+    nil
   end
 
   # Gets the logger
@@ -71,4 +73,21 @@ module Exekutor
                 ActiveSupport::TaggedLogging.new logger
               end
   end
+
+  # @return [ActiveSupport::BacktraceCleaner] A cleaner for error backtraces
+  def self.backtrace_cleaner
+    @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
+  end
+
+  # @return [String] The given error class, message, and cleaned backtrace as a string
+  def self.strferr(err)
+    raise ArgumentError, "err must not be nil" if err.nil?
+    return err if err.is_a? String
+
+    "#{err.class} – #{err.message}\nat #{
+      err.backtrace ? backtrace_cleaner.clean(err.backtrace).join("\n   ") : "unknown location"
+    }"
+  end
+
+  private_class_method :backtrace_cleaner, :strferr
 end

data/lib/exekutor/internal/provider.rb

@@ -8,31 +8,33 @@ module Exekutor
   module Internal
     # Reserves jobs and provides them to an executor
     class Provider
-      include Executable, Callbacks, Logger
+      include Logger
+      include Callbacks
+      include Executable
 
       define_callbacks :on_queue_empty, freeze: true
 
       # Represents an unknown value
       UNKNOWN = Object.new.freeze
-      private_constant "UNKNOWN"
+      private_constant :UNKNOWN
 
       MAX_WAIT_TIMEOUT = 300
-      private_constant "MAX_WAIT_TIMEOUT"
+      private_constant :MAX_WAIT_TIMEOUT
 
       # Creates a new provider
       # @param reserver [Reserver] the job reserver
       # @param executor [Executor] the job executor
       # @param pool [ThreadPoolExecutor] the thread pool to use
-      # @param polling_interval [Integer] the polling interval
+      # @param polling_interval [ActiveSupport::Duration] the polling interval
       # @param interval_jitter [Float] the polling interval jitter
-      def initialize(reserver:, executor:, pool:, polling_interval: 60,
+      def initialize(reserver:, executor:, pool:, polling_interval: 60.seconds,
                      interval_jitter: polling_interval.to_i > 1 ? polling_interval * 0.1 : 0)
         super()
         @reserver = reserver
         @executor = executor
         @pool = pool
 
-        @polling_interval = polling_interval.freeze
+        @polling_interval = polling_interval.to_i.freeze
         @interval_jitter = interval_jitter.to_f.freeze
 
         @event = Concurrent::Event.new
@@ -48,14 +50,14 @@ module Exekutor
 
         # Always poll at startup to fill up threads, use small jitter so workers started at the same time dont hit
         # the db at the same time
-        @next_poll_at.set (1 + 2 * Kernel.rand).second.from_now
+        @next_poll_at.set (1 + (2 * Kernel.rand)).seconds.from_now.to_f
         start_thread
         true
       end
 
       # Stops the provider
       def stop
-        set_state :stopped
+        self.state = :stopped
         @event.set
       end
 
@@ -63,7 +65,7 @@ module Exekutor
       def poll
         raise Exekutor::Error, "Provider is not running" unless running?
 
-        @next_poll_at.set Time.now
+        @next_poll_at.set Time.now.to_f
         @event.set
       end
 
@@ -71,51 +73,39 @@ module Exekutor
       # argument is given. Updates the timestamp for the earliest job is a timestamp is given and that timestamp is
       # before the known timestamp. Does nothing if a timestamp is given and the earliest job timestamp is not known.
       # @param scheduled_at [Time,Numeric] the time a job is scheduled at
-      # @return [Time] the timestamp for the next job, or +nil+ if the timestamp is unknown or no jobs are pending
+      # @return [Float,nil] the timestamp for the next job, or +nil+ if the timestamp is unknown or no jobs are pending
       def update_earliest_scheduled_at(scheduled_at = UNKNOWN)
-        overwrite_unknown = false
-        case scheduled_at
-        when UNKNOWN
-          # If we fetch the value from the DB, we can safely overwrite the UNKNOWN value
-          overwrite_unknown = true
-          scheduled_at = @reserver.earliest_scheduled_at
-        when Numeric
-          scheduled_at = Time.at(scheduled_at)
-        when Time
-          #  All good
-        else
+        scheduled_at = scheduled_at.to_f if scheduled_at.is_a? Time
+        unless scheduled_at == UNKNOWN || scheduled_at.is_a?(Numeric)
           raise ArgumentError, "scheduled_at must be a Time or Numeric"
         end
 
-        updated = false
-        scheduled_at = @next_job_scheduled_at.update do |current|
-          if current == UNKNOWN
-            if overwrite_unknown || scheduled_at <= Time.now
-              updated = true
-              scheduled_at
-            else
-              current
-            end
-          elsif current.nil? || scheduled_at.nil? || current > scheduled_at
-            updated = true
-            scheduled_at
-          else
-            current
-          end
+        changed = false
+        earliest_scheduled_at = @next_job_scheduled_at.update do |current|
+          earliest = ScheduledAtComparer.determine_earliest(current, scheduled_at, @reserver)
+          changed = earliest != current
+          earliest
         end
-        if scheduled_at == UNKNOWN
+
+        if earliest_scheduled_at == UNKNOWN
           nil
         else
-          @event.set if updated && scheduled_at.present?
-          scheduled_at
+          @event.set if changed && earliest_scheduled_at.present?
+          earliest_scheduled_at
         end
       end
 
       private
 
       # Starts the provision thread
-      def start_thread
-        @pool.post(&method(:run)) if running?
+      def start_thread(delay: nil)
+        return unless running?
+
+        if delay
+          Concurrent::ScheduledTask.execute(delay, executor: @pool) { run }
+        else
+          @pool.post { run }
+        end
       end
 
       # Does the provisioning of jobs to the executor. Blocks until the provider is stopped.
@@ -134,19 +124,24 @@ module Exekutor
             consecutive_errors.value = 0
           end
         end
-      rescue StandardError => err
-        Exekutor.on_fatal_error err, "[Provider] Runtime error!"
-        consecutive_errors.increment
-        if running?
-          delay = restart_delay
-          logger.info "Restarting in %0.1f seconds…" % [delay]
-          Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
-        end
+      rescue StandardError => e
+        on_thread_error(e)
       ensure
         BaseRecord.connection_pool.release_connection
         @thread_running.make_false
       end
 
+      # Called when an error is raised in #run
+      def on_thread_error(error)
+        Exekutor.on_fatal_error error, "[Provider] Runtime error!"
+        return unless running?
+
+        consecutive_errors.increment
+        delay = restart_delay
+        logger.info format("Restarting in %0.1f seconds…", delay)
+        start_thread delay: delay
+      end
+
       # Waits for any event to happen. An event could be:
       # - The listener was notified of a new job;
       # - The next job is scheduled for the current time;
@@ -157,8 +152,8 @@ module Exekutor
         return unless timeout.positive?
 
         @event.wait timeout
-      rescue StandardError => err
-        Exekutor.on_fatal_error err, "[Provider] An error occurred while waiting"
+      rescue StandardError => e
+        Exekutor.on_fatal_error e, "[Provider] An error occurred while waiting"
         sleep 0.1 if running?
       ensure
         throw :shutdown unless running?
@@ -171,21 +166,8 @@ module Exekutor
         return unless available_workers.positive?
 
         jobs = @reserver.reserve available_workers
-        unless jobs.nil?
-          begin
-            logger.debug "Reserved #{jobs.size} job(s)"
-            jobs.each(&@executor.method(:post))
-          rescue Exception # rubocop:disable Lint/RescueException
-            # Try to release all jobs before re-raising
-            begin
-              Exekutor::Job.where(id: jobs.collect { |job| job[:id] }, status: "e")
-                           .update_all(status: "p", worker_id: nil)
-            rescue # rubocop:disable Lint/RescueStandardError
-              # ignored
-            end
-            raise
-          end
-        end
+        execute_jobs jobs unless jobs.nil?
+
         if jobs.nil? || jobs.size.to_i < available_workers
           # If we ran out of work, update the earliest scheduled at
           update_earliest_scheduled_at
@@ -194,8 +176,24 @@ module Exekutor
 
         elsif @next_job_scheduled_at.get == UNKNOWN
           # If the next job timestamp is still unknown, set it to now to indicate there's still work to do
-          @next_job_scheduled_at.set Time.now
+          @next_job_scheduled_at.set Time.now.to_f
+        end
+      end
+
+      # Posts the given jobs to the executor thread pool
+      # @param jobs [Array] the jobs to execute
+      def execute_jobs(jobs)
+        logger.debug "Reserved #{jobs.size} job(s)"
+        jobs.each { |job| @executor.post(job) }
+      rescue Exception # rubocop:disable Lint/RescueException
+        # Try to release all jobs before re-raising
+        begin
+          Exekutor::Job.where(id: jobs.pluck(:id), status: "e")
+                       .update_all(status: "p", worker_id: nil)
+        rescue StandardError
+          # ignored
         end
+        raise
       end
 
       def perform_pending_job_updates
@@ -224,7 +222,7 @@ module Exekutor
         return if jobs&.size.to_i.zero?
 
         logger.info "Restarting #{jobs.size} abandoned job#{"s" if jobs.size > 1}"
-        jobs.each(&@executor.method(:post))
+        jobs.each { |job| @executor.post(job) }
       end
 
       # @return [Boolean] Whether the polling is enabled. Ie. whether a polling interval is set.
@@ -232,7 +230,7 @@ module Exekutor
         @polling_interval.present?
       end
 
-      # @return [Time,nil] the 'scheduled at' value for the next job, or nil if unknown or if there is no pending job
+      # @return [Float,nil] the 'scheduled at' value for the next job, or nil if unknown or if there is no pending job
       def next_job_scheduled_at
         at = @next_job_scheduled_at.get
         if at == UNKNOWN
@@ -243,10 +241,10 @@ module Exekutor
         end
       end
 
-      # @return [Time,nil] When the next poll is scheduled, or nil if polling is disabled
+      # @return [Float,nil] When the next poll is scheduled, or nil if polling is disabled
       def next_poll_scheduled_at
         if polling_enabled?
-          @next_poll_at.update { |planned_at| planned_at || Time.now + polling_interval }
+          @next_poll_at.update { |planned_at| planned_at || (Time.now.to_f + polling_interval) }
         else
           # noinspection RubyMismatchedReturnType
           @next_poll_at.get
@@ -262,9 +260,9 @@ module Exekutor
 
         timeout = [MAX_WAIT_TIMEOUT].tap do |timeouts|
           # noinspection RubyMismatchedArgumentType
-          timeouts.append next_job_at - Time.now if next_job_at
+          timeouts.append next_job_at - Time.now.to_f if next_job_at
           # noinspection RubyMismatchedArgumentType
-          timeouts.append next_poll_at - Time.now if next_poll_at
+          timeouts.append next_poll_at - Time.now.to_f if next_poll_at
         end.min
 
         if timeout <= 0.001
@@ -278,13 +276,13 @@ module Exekutor
       # @return [Boolean] Whether the `reserver` should be called.
       def reserve_jobs_now?
         next_poll_at = next_poll_scheduled_at
-        if next_poll_at && next_poll_at - Time.now <= 0.001
-          @next_poll_at.update { Time.now + polling_interval if polling_enabled? }
+        if next_poll_at && next_poll_at - Time.now.to_f <= 0.001
+          @next_poll_at.update { Time.now.to_f + polling_interval if polling_enabled? }
           return true
         end
 
         next_job_at = next_job_scheduled_at
-        next_job_at && next_job_at <= Time.now
+        next_job_at && next_job_at <= Time.now.to_f
       end
 
       # @return [Float] Gets the polling interval jitter
@@ -295,7 +293,7 @@ module Exekutor
       # Get the polling interval. If a jitter is configured, the interval is reduced or increased by `0.5 * jitter`.
       # @return [Float] The amount of seconds before the next poll
       def polling_interval
-        raise "Polling is disabled" unless @polling_interval.present?
+        raise "Polling is disabled" if @polling_interval.blank?
 
         @polling_interval + if polling_interval_jitter.zero?
                               0
@@ -303,6 +301,27 @@ module Exekutor
                               (Kernel.rand - 0.5) * polling_interval_jitter
                             end
       end
+
+      # Abstraction to determine earliest out of 2 scheduled at timestamps
+      class ScheduledAtComparer
+        def self.determine_earliest(current_value, other_value, reserver)
+          if other_value == UNKNOWN
+            # Fetch the value from the db
+            reserver.earliest_scheduled_at&.to_f
+          elsif current_value == UNKNOWN
+            # We don't know the value yet, don't use the given value as earliest unless work is scheduled immediately
+            other_value > Time.now.to_f ? UNKNOWN : other_value
+          elsif current_value.nil? || current_value > other_value
+            # The given value is earlier than the known value
+            other_value
+          else
+            # No changes
+            current_value
+          end
+        end
+      end
+
+      private_constant :ScheduledAtComparer
     end
   end
 end

data/lib/exekutor/internal/reserver.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Exekutor
   # @private
   module Internal
@@ -10,23 +11,23 @@ module Exekutor
       # Creates a new Reserver
       # @param worker_id [String] the id of the worker
       # @param queues [Array<String>] the queues to watch
-      def initialize(worker_id, queues)
+      def initialize(worker_id, queues: nil, min_priority: nil, max_priority: nil)
         @worker_id = worker_id
-        @queue_filter_sql = build_queue_filter_sql(queues)
+        @reserve_filter_sql = build_filter_sql(queues: queues, min_priority: min_priority, max_priority: max_priority)
         @json_serializer = Exekutor.config.load_json_serializer
       end
 
       # Reserves pending jobs
       # @param limit [Integer] the number of jobs to reserve
-      # @return [Array<Job>,nil] the reserved jobs, or nil if no jobs were reserved
+      # @return [Array<Hash>,nil] the reserved jobs, or nil if no jobs were reserved
       def reserve(limit)
         return unless limit.positive?
 
         results = Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, [@worker_id, limit], prepare: true
           UPDATE exekutor_jobs SET worker_id = $1, status = 'e' WHERE id IN (
              SELECT id FROM exekutor_jobs
-                WHERE scheduled_at <= now() AND "status"='p' #{@queue_filter_sql}
-                ORDER BY priority, scheduled_at, enqueued_at
+                WHERE scheduled_at <= now() AND "status"='p'#{" AND #{@reserve_filter_sql}" if @reserve_filter_sql}
+                ORDER BY priority#{" DESC" if Exekutor.config.inverse_priority?}, scheduled_at, enqueued_at
                 FOR UPDATE SKIP LOCKED
                 LIMIT $2
           ) RETURNING "id", "payload", "options", "scheduled_at"
@@ -36,6 +37,9 @@ module Exekutor
         parse_jobs results
       end
 
+      # Gets the jobs that are assigned to this worker and have an id that is not included in +active_job_ids+
+      # @param active_job_ids [Array<String>] The ids of the jobs that should be excluded
+      # @return [Array<Hash>] the jobs
       def get_abandoned_jobs(active_job_ids)
         jobs = Exekutor::Job.executing.where(worker_id: @worker_id)
         jobs = jobs.where.not(id: active_job_ids) if active_job_ids.present?
@@ -47,7 +51,7 @@ module Exekutor
       # @return [Time,nil] The earliest scheduled at, or nil if the queues are empty
       def earliest_scheduled_at
         jobs = Exekutor::Job.pending
-        jobs.where! @queue_filter_sql.gsub(/^\s*AND\s+/, "") unless @queue_filter_sql.nil?
+        jobs.where! @reserve_filter_sql unless @reserve_filter_sql.nil?
         jobs.minimum(:scheduled_at)
       end
 
@@ -59,7 +63,7 @@ module Exekutor
           { id: result["id"],
             payload: parse_json(result["payload"]),
             options: parse_json(result["options"]),
-            scheduled_at: result['scheduled_at'] }
+            scheduled_at: result["scheduled_at"] }
         end
       end
 
@@ -68,28 +72,71 @@ module Exekutor
         @json_serializer.load str unless str.nil?
       end
 
+      # Builds SQL filter for the given queues and priorities
+      def build_filter_sql(queues:, min_priority:, max_priority:)
+        filters = [
+          build_queue_filter_sql(queues),
+          build_priority_filter_sql(min_priority, max_priority)
+        ]
+        filters.compact!
+        filters.join(" AND ") unless filters.empty?
+      end
+
       # Builds SQL filter for the given queues
       def build_queue_filter_sql(queues)
         return nil if queues.nil? || (queues.is_a?(Array) && queues.empty?)
-        unless queues.is_a?(String) || queues.is_a?(Symbol) || queues.is_a?(Array)
-          raise ArgumentError, "queues must be nil, a String, Symbol, or an array of Strings or Symbols"
-        end
 
         queues = queues.first if queues.is_a?(Array) && queues.one?
-        if queues.is_a? Array
-          unless queues.all? { |q| (q.is_a?(String) || q.is_a?(Symbol)) && !q.blank? }
-            raise ArgumentError, "queues contains an invalid value"
-          end
+        validate_queues! queues
 
-          Exekutor::Job.sanitize_sql_for_conditions(["AND queue IN (?)", queues])
-        else
-          raise ArgumentError, "queue name cannot be empty" if queues.blank?
+        conditions = if queues.is_a? Array
+                       ["queue IN (?)", queues]
+                     else
+                       ["queue = ?", queues]
+                     end
+        Exekutor::Job.sanitize_sql_for_conditions conditions
+      end
+
+      # Builds SQL filter for the given priorities
+      def build_priority_filter_sql(minimum, maximum)
+        minimum = coerce_priority(minimum)
+        maximum = coerce_priority(maximum)
 
-          Exekutor::Job.sanitize_sql_for_conditions(["AND queue = ?", queues])
+        conditions = if minimum && maximum
+                       ["priority BETWEEN ? AND ?", minimum, maximum]
+                     elsif minimum
+                       ["priority >= ?", minimum]
+                     elsif maximum
+                       ["priority <= ?", maximum]
+                     end
+        Exekutor::Job.sanitize_sql_for_conditions conditions if conditions
+      end
+
+      # @return [Integer,nil] returns nil unless +priority+ is between 1 and 32,766
+      def coerce_priority(priority)
+        priority if priority && (1..32_766).cover?(priority)
+      end
+
+      # Raises an error if the queues value is invalid
+      # @param queues [String,Symbol,Array<String,Symbol>] the queues to validate
+      # @raise [ArgumentError] if the queue is invalid or includes an invalid value
+      def validate_queues!(queues)
+        case queues
+        when Array
+          raise ArgumentError, "queues contains an invalid value" unless queues.all? { |queue| valid_queue_name? queue }
+        when String, Symbol
+          raise ArgumentError, "queue name cannot be empty" unless valid_queue_name? queues
+        else
+          raise ArgumentError,
+                "queues must be nil, a String, Symbol, or an array of Strings or Symbols (Actual: #{queues.class})"
         end
       end
 
+      # @param queue [String,Symbol] the name of a queue
+      # @return [Boolean] whether the name is a valid queue name
+      def valid_queue_name?(queue)
+        (queue.is_a?(String) || queue.is_a?(Symbol)) && queue.present? && queue.length <= Queue::MAX_NAME_LENGTH
+      end
     end
   end
 end
-