exekutor 0.1.0

data/lib/exekutor/internal/executable.rb

@@ -0,0 +1,75 @@
+module Exekutor
+  # Contains internal classes
+  # @private
+  module Internal
+    # Mixin for an executable
+    module Executable
+      # Possible states
+      STATES = %i[pending started stopped crashed killed].freeze
+
+      def initialize
+        @state = Concurrent::AtomicReference.new(:pending)
+        @consecutive_errors = Concurrent::AtomicFixnum.new(0)
+      end
+
+      # The state of this executable. Possible values are:
+      # - +:pending+ the executable has not been started yet
+      # - +:started+ the executable has started
+      # - +:stopped+ the executable has stopped
+      # - +:crashed+ the executable has crashed
+      # - +:killed+ the executable was killed
+      # @return [:pending,:started,:stopped,:crashed,:killed] the state
+      def state
+        @state.get
+      end
+
+      # Whether the state equals +:started+
+      def running?
+        @state.get == :started
+      end
+
+      def consecutive_errors
+        @consecutive_errors
+      end
+
+      # Calculates an exponential delay based on {#consecutive_errors}. The delay ranges from 10 seconds on the first error
+      # to 10 minutes from the 13th error on.
+      # @return [Float] The delay
+      def restart_delay
+        if @consecutive_errors.value > 150
+          error = SystemExit.new "Too many consecutive errors (#{@consecutive_errors.value})"
+          Exekutor.on_fatal_error error
+          raise error
+        end
+        delay = (9 + @consecutive_errors.value ** 2.5)
+        delay += delay * (rand(-5..5) / 100.0)
+        delay.clamp(10.0, 600.0)
+      end
+
+      private
+
+      # Changes the state to the given value if the current state matches the expected state. Does nothing otherwise.
+      # @param expected_state [:pending,:started,:stopped,:crashed] the expected state
+      # @param new_state [:pending,:started,:stopped,:crashed] the state to change to if the current state matches the expected
+      # @raise ArgumentError if an invalid state was passed
+      def compare_and_set_state(expected_state, new_state)
+        validate_state! new_state
+        @state.compare_and_set expected_state, new_state
+      end
+
+      # Updates the state to the given value
+      # @raise ArgumentError if an invalid state was passed
+      def set_state(new_state)
+        validate_state! new_state
+        @state.set new_state
+      end
+
+      # Validates whether +state+ is a valid value
+      # @raise ArgumentError if an invalid state was passed
+      def validate_state!(state)
+        raise ArgumentError, "State must be a symbol (was: #{state.class.name})" unless state.is_a? Symbol
+        raise ArgumentError, "Invalid state: #{state}" unless STATES.include? state
+      end
+    end
+  end
+end

data/lib/exekutor/internal/executor.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require_relative "executable"
+require_relative "callbacks"
+
+module Exekutor
+  # @private
+  module Internal
+    # Executes jobs from a thread pool
+    class Executor
+      include Logger
+      include Callbacks
+      include Executable
+
+      define_callbacks :after_execute, freeze: true
+      attr_reader :pending_job_updates
+
+      def initialize(min_threads: 1, max_threads: default_max_threads, max_thread_idletime: 180,
+                     delete_completed_jobs: false, delete_discarded_jobs: false, delete_failed_jobs: false)
+        super()
+        @executor = ThreadPoolExecutor.new name: "exekutor-job", fallback_policy: :abort, max_queue: max_threads,
+                                           min_threads: min_threads, max_threads: max_threads,
+                                           idletime: max_thread_idletime
+        @queued_job_ids = Concurrent::Array.new
+        @active_job_ids = Concurrent::Array.new
+        @pending_job_updates = Concurrent::Hash.new
+        @options = {
+          delete_completed_jobs: delete_completed_jobs,
+          delete_discarded_jobs: delete_discarded_jobs,
+          delete_failed_jobs: delete_failed_jobs
+        }.freeze
+      end
+
+      # Starts the executor
+      def start
+        set_state :started
+      end
+
+      # Stops the executor
+      def stop
+        set_state :stopped
+
+        @executor.shutdown
+      end
+
+      # Kills the executor
+      def kill
+        Thread.new { compare_and_set_state :started, :killed }
+        @executor.kill
+
+        release_assigned_jobs
+      end
+
+      # Executes the job on one of the execution threads. Releases the job if there is no thread available to execute
+      # the job.
+      def post(job)
+        @executor.post job, &method(:execute)
+        @queued_job_ids.append(job[:id])
+      rescue Concurrent::RejectedExecutionError
+        logger.error "Ran out of threads! Releasing job #{job[:id]}"
+        update_job job, status: "p", worker_id: nil
+      end
+
+      # @return [Integer] the number of available threads to execute jobs on. Returns 0 if the executor is not running.
+      def available_threads
+        if @executor.running?
+          @executor.available_threads
+        else
+          0
+        end
+      end
+
+      # @return [Integer] the minimum number of threads to execute jobs on.
+      def minimum_threads
+        @executor.min_length
+      end
+
+      # @return [Integer] the maximum number of threads to execute jobs on.
+      def maximum_threads
+        @executor.max_length
+      end
+
+      # @return [Array<String>] The ids of the jobs that are currently being executed
+      def active_job_ids
+        @active_job_ids.dup.to_a
+      end
+
+      # Prunes the inactive threads from the pool.
+      def prune_pool
+        @executor.prune_pool
+      end
+
+      private
+
+      # Executes the given job
+      def execute(job)
+        @queued_job_ids.delete(job[:id])
+        @active_job_ids.append(job[:id])
+        Rails.application.reloader.wrap do
+          DatabaseConnection.ensure_active!
+          Internal::Hooks.run :job_execution, job do
+            _execute(job)
+            # Run internal callbacks
+            run_callbacks :after, :execute, job
+          end
+        end
+      ensure
+        @active_job_ids.delete(job[:id])
+      end
+
+      def _execute(job, start_time: Concurrent.monotonic_time)
+        raise Exekutor::DiscardJob, "Maximum queue time expired" if queue_time_expired?(job)
+
+        if (timeout = job[:options] && job[:options]["execution_timeout"]).present?
+          Timeout.timeout Float(timeout), JobExecutionTimeout do
+            ActiveJob::Base.execute(job[:payload])
+          end
+        else
+          ActiveJob::Base.execute(job[:payload])
+        end
+
+        on_job_completed(job, runtime: Concurrent.monotonic_time - start_time)
+      rescue StandardError, JobExecutionTimeout => e
+        on_job_failed(job, e, runtime: Concurrent.monotonic_time - start_time)
+      rescue Exception # rubocop:disable Lint/RescueException
+        # Try to release job when an Exception occurs
+        update_job job, status: "p", worker_id: nil
+        raise
+      end
+
+      def on_job_completed(job, runtime:)
+        if @options[:delete_completed_jobs]
+          delete_job job
+        else
+          update_job job, status: "c", runtime: runtime
+        end
+      end
+
+      def on_job_failed(job, error, runtime:)
+        discarded = [Exekutor::DiscardJob, JobExecutionTimeout].any?(&error.method(:is_a?))
+        unless discarded
+          Internal::Hooks.on(:job_failure, job, error)
+          log_error error, "Job failed"
+        end
+
+        if lost_db_connection?(error)
+          # Don't consider this as a failure, try again later.
+          update_job job, status: "p", worker_id: nil
+
+        elsif @options[discarded ? :delete_discarded_jobs : :delete_failed_jobs]
+          delete_job job
+
+        else
+          # Try to update the job and create a JobError record if update succeeds
+          if update_job job, status: discarded ? "d" : "f", runtime: runtime
+            JobError.create!(job_id: job[:id], error: error)
+          end
+        end
+      end
+
+      # Updates the active record entity for this job with the given attributes.
+      def update_job(job, **attrs)
+        Exekutor::Job.where(id: job[:id]).update_all(attrs)
+        true
+      rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished => e
+        unless Exekutor::Job.connection.active?
+          log_error e, "Could not update job"
+          # Save the update for when the connection is back
+          @pending_job_updates.merge!(job[:id] => attrs) do |_k, old, new|
+            if old == :destroy
+              old
+            else
+              old&.merge!(new) || new
+            end
+          end
+        end
+        false
+      end
+
+      def delete_job(job)
+        Exekutor::Job.destroy(job[:id])
+        true
+      rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished => e
+        unless Exekutor::Job.connection.active?
+          log_error e, "Could not delete job"
+          # Save the deletion for when the connection is back
+          @pending_job_updates[job[:id]] = :destroy
+        end
+        false
+      end
+
+      def release_assigned_jobs
+        @queued_job_ids.each { |id| update_job({ id: id }, status: "p", worker_id: nil) }
+        @active_job_ids.each { |id| update_job({ id: id }, status: "p", worker_id: nil) }
+      end
+
+      def queue_time_expired?(job)
+        job[:options] && job[:options]["start_execution_before"] &&
+          job[:options]["start_execution_before"].to_f <= Time.now.to_f
+      end
+
+      def lost_db_connection?(error)
+        [ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished].any?(&error.method(:kind_of?)) &&
+          !ActiveRecord::Base.connection.active?
+      end
+
+      # The default maximum number of threads. The value is equal to the size of the DB connection pool minus 1, with
+      # a minimum of 1.
+      def default_max_threads
+        connection_pool_size = Exekutor::Job.connection_db_config.pool
+        if connection_pool_size && connection_pool_size > 2
+          connection_pool_size - 1
+        else
+          1
+        end
+      end
+
+      # The thread pool to use for executing jobs.
+      class ThreadPoolExecutor < Concurrent::ThreadPoolExecutor
+        # Number of inactive threads available to execute tasks.
+        # https://github.com/ruby-concurrency/concurrent-ruby/issues/684#issuecomment-427594437
+        # @return [Integer]
+        def available_threads
+          synchronize do
+            if Concurrent.on_jruby?
+              @executor.getMaximumPoolSize - @executor.getActiveCount
+            else
+              workers_still_to_be_created = @max_length - @pool.length
+              workers_created_but_waiting = @ready.length
+              workers_still_to_be_created + workers_created_but_waiting
+            end
+          end
+        end
+      end
+
+      # Thrown when the job execution timeout expires. Inherits from Exception so it's less likely to be caught by
+      # rescue statements.
+      class JobExecutionTimeout < Exception # rubocop:disable Lint/InheritException
+      end
+    end
+  end
+end

data/lib/exekutor/internal/hooks.rb

@@ -0,0 +1,87 @@
+module Exekutor
+  module Internal
+    class Hooks
+      include Internal::Callbacks
+
+      define_callbacks :before_enqueue, :around_enqueue, :after_enqueue,
+                       :before_job_execution, :around_job_execution, :after_job_execution,
+                       :on_job_failure, :on_fatal_error,
+                       :before_startup, :after_startup,
+                       :before_shutdown, :after_shutdown,
+                       freeze: true
+
+      # Registers a hook to be called.
+      def register(callback = nil, &block)
+        if callback
+          callback = callback.new if callback.is_a? Class
+          raise 'callback must respond to #callbacks' unless callback.respond_to? :callbacks
+          callback.callbacks.each do |type, callbacks|
+            callbacks.each { |callback| add_callback! type, [], callback }
+          end
+        elsif block.arity == 1
+          block.call self
+        else
+          instance_eval(&block)
+        end
+      end
+
+      # @see #register
+      def <<(callback)
+        register callback
+      end
+
+      # Executes an +:on+ callback with the given type.
+      def self.on(type, *args)
+        ::Exekutor.hooks.send(:run_callbacks, :on, type, *args)
+      end
+
+      # Executes the +:before+, +:around+, and +:after+ callbacks with the given type.
+      def self.run(type, *args, &block)
+        ::Exekutor.hooks.send(:with_callbacks, type, *args, &block)
+      end
+    end
+  end
+
+  # Prints the error to STDERR and the log, and calls the :on_fatal_error hooks.
+  def self.on_fatal_error(error, message = nil)
+    Exekutor.print_error(error, message)
+    return if defined?(@calling_fatal_error_hook) && @calling_fatal_error_hook
+
+    @calling_fatal_error_hook = true
+    Internal::Hooks.on(:fatal_error, error)
+  ensure
+    @calling_fatal_error_hook = false
+  end
+
+  # Exekutor.hooks.register do
+  #    after_job_failure do |error, job|
+  #       Appsignal.add_exception error
+  #    end
+  # end
+
+  # Exekutor.hooks.after_job_failure do error
+  #    Appsignal.add_exception error
+  # end
+
+  # class ExekutorHooks < ::Exekutor::Hook
+  #   around_job_execution :instrument
+  #   after_job_failure :report_error
+  #   after_fatal_error :report_error
+  #
+  #   def instrument(job)
+  #     Appsignal.monitor_transaction … { yield }
+  #   end
+  #
+  #   def send_to_appsignal(error)
+  #       Appsignal.add_exception error
+  #   end
+  # end
+  #
+  # Exekutor.hooks.register ExekutorHooks
+
+  # @!attribute [r] hooks
+  #   @return [Internal::Hooks] The hooks for exekutor.
+  mattr_reader :hooks, default: Internal::Hooks.new
+
+  # TODO register_hook method instead of `hooks.register`?
+end

data/lib/exekutor/internal/listener.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require_relative "executable"
+
+module Exekutor
+  # @private
+  module Internal
+    # Listens for jobs to be executed
+    class Listener
+      include Executable, Logger
+
+      # The PG notification channel for enqueued jobs
+      JOB_ENQUEUED_CHANNEL = "exekutor::job_enqueued"
+      # The PG notification channel for a worker. Must be formatted with the worker ID.
+      PROVIDER_CHANNEL = "exekutor::worker::%s"
+
+      # Creates a new listener
+      # @param worker_id [String] the ID of the worker
+      # @param queues [Array<String>] the queues to watch
+      # @param provider [Provider] the job provider
+      # @param pool [ThreadPoolExecutor] the thread pool to use
+      # @param wait_timeout [Integer] the time to listen for notifications
+      # @param set_db_connection_name [Boolean] whether to set the application name on the DB connection
+      def initialize(worker_id:, queues: nil, provider:, pool:, wait_timeout: 60, set_db_connection_name: false)
+        super()
+        @config = {
+          worker_id: worker_id,
+          queues: queues || [],
+          wait_timeout: wait_timeout,
+          set_db_connection_name: set_db_connection_name
+        }
+
+        @provider = provider
+        @pool = pool
+
+        @thread_running = Concurrent::AtomicBoolean.new false
+        @listening = Concurrent::AtomicBoolean.new false
+      end
+
+      # Starts the listener
+      def start
+        return false unless compare_and_set_state :pending, :started
+
+        start_thread
+        true
+      end
+
+      # Stops the listener
+      def stop
+        set_state :stopped
+        begin
+          Exekutor::Job.connection.execute(%(NOTIFY "#{provider_channel}"))
+        rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished
+          #ignored
+        end
+      end
+
+      private
+
+      # The PG notification channel for a worker
+      def provider_channel
+        PROVIDER_CHANNEL % @config[:worker_id]
+      end
+
+      # Whether this listener is listening to the given queue
+      # @return [Boolean]
+      def listening_to_queue?(queue)
+        queues = @config[:queues]
+        queues.empty? || queues.include?(queue)
+      end
+
+      # Starts the listener thread
+      def start_thread
+        @pool.post(&method(:run)) if running?
+      end
+
+      # Sets up the PG notifications and listens for new jobs
+      def run
+        return unless running? && @thread_running.make_true
+
+        with_pg_connection do |connection|
+          begin
+            connection.exec(%(LISTEN "#{provider_channel}"))
+            connection.exec(%(LISTEN "#{JOB_ENQUEUED_CHANNEL}"))
+            consecutive_errors.value = 0
+            catch(:shutdown) { wait_for_jobs(connection) }
+          ensure
+            connection.exec("UNLISTEN *")
+          end
+        end
+      rescue StandardError => err
+        Exekutor.on_fatal_error err, "[Listener] Runtime error!"
+        set_state :crashed if err.is_a? UnsupportedDatabase
+
+        if running?
+          consecutive_errors.increment
+          delay = restart_delay
+          logger.info "Restarting in %0.1f seconds…" % [delay]
+          Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
+        end
+      ensure
+        @thread_running.make_false
+        @listening.make_false
+      end
+
+      # Listens for jobs. Blocks until the listener is stopped
+      def wait_for_jobs(connection)
+        while running?
+          @listening.make_true
+          connection.wait_for_notify(@config[:wait_timeout]) do |channel, _pid, payload|
+            throw :shutdown unless running?
+            next unless channel == JOB_ENQUEUED_CHANNEL
+
+            job_info = begin
+                         payload.split(";").map { |el| el.split(":") }.to_h
+                       rescue
+                         logger.error "Invalid notification payload: #{payload}"
+                         next
+                       end
+            unless %w[id q t].all? { |n| job_info[n].present? }
+              logger.error "[Listener] Notification payload is missing #{%w[id q t].select { |n| job_info[n].blank? }.join(", ")}"
+              next
+            end
+            next unless listening_to_queue? job_info["q"]
+
+            scheduled_at = job_info["t"].to_f
+            @provider.update_earliest_scheduled_at(scheduled_at)
+          end
+        end
+      end
+
+      # Gets a DB connection and removes it from the pool. Sets the application name if +set_db_connection_name+ is true.
+      # Closes the connection after yielding it to the given block.
+      # (Grabbed from PG adapter for action cable)
+      # @yield yields the connection
+      # @yieldparam connection [PG::Connection] the DB connection
+      def with_pg_connection # :nodoc:
+        ar_conn = Exekutor::Job.connection_pool.checkout.tap do |conn|
+          # Action Cable is taking ownership over this database connection, and
+          # will perform the necessary cleanup tasks
+          ActiveRecord::Base.connection_pool.remove(conn)
+        end
+        DatabaseConnection.ensure_active! ar_conn
+        pg_conn = ar_conn.raw_connection
+
+        verify!(pg_conn)
+        if @config[:set_db_connection_name]
+          DatabaseConnection.set_application_name pg_conn, @config[:worker_id], :listener
+        end
+        yield pg_conn
+      ensure
+        ar_conn.disconnect!
+      end
+
+      # Verifies the connection
+      # @raise [Error] if the connection is not an instance of +PG::Connection+ or is invalid.
+      def verify!(pg_conn)
+        unless pg_conn.is_a?(PG::Connection)
+          raise UnsupportedDatabase,
+                "The raw connection of the active record connection adapter must be an instance of PG::Connection"
+        end
+      end
+
+      # For testing purposes
+      def listening?
+        @listening.true?
+      end
+
+      # Raised when an error occurs in the listener.
+      class Error < Exekutor::Error; end
+
+      # Raised when the database connection is not an instance of PG::Connection.
+      class UnsupportedDatabase < Exekutor::Error; end
+    end
+  end
+end

data/lib/exekutor/internal/logger.rb

@@ -0,0 +1,74 @@
+require "rainbow"
+
+module Exekutor
+  # @private
+  module Internal
+    # Mixin to use the logger
+    module Logger
+      extend ActiveSupport::Concern
+
+      included do
+        # The log tags to use when writing to the log
+        mattr_accessor :log_tags, default: [self.name.demodulize]
+      end
+
+      protected
+
+      # Prints the error to the log
+      # @param err [Exception] the error to print
+      # @param message [String] the message to print above the error
+      # @return [void]
+      def log_error(err, message)
+        @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
+        logger.error message if message
+        logger.error "#{err.class} – #{err.message}\nat #{@backtrace_cleaner.clean(err.backtrace).join("\n   ")}"
+      end
+
+      # Gets the logger
+      # @return [ActiveSupport::TaggedLogging]
+      def logger
+        @logger ||= Exekutor.logger.tagged(log_tags.compact)
+      end
+    end
+  end
+
+  # Prints a message to STDOUT, unless {Exekutor::Configuration#quiet?} is true
+  # @private
+  def self.say(*args)
+    puts(*args) unless config.quiet?
+  end
+
+  # Prints the error in the log and to STDERR (unless {Exekutor::Configuration#quiet?} is true)
+  # @param err [Exception] the error to print
+  # @param message [String] the message to print above the error
+  # @return [void]
+  def self.print_error(err, message = nil)
+    @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
+    error = "#{err.class} – #{err.message}\nat #{@backtrace_cleaner.clean(err.backtrace).join("\n   ")}"
+
+    unless config.quiet?
+      $stderr.puts Rainbow(message).bright.red if message
+      $stderr.puts Rainbow(error).red
+    end
+    unless ActiveSupport::Logger.logger_outputs_to?(logger, $stdout)
+      logger.error message if message
+      logger.error error
+    end
+  end
+
+  # Gets the logger
+  # @return [ActiveSupport::TaggedLogging]
+  mattr_reader :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
+
+  # Sets the logger
+  # @param logger [ActiveSupport::Logger]
+  def self.logger=(logger)
+    raise ArgumentError, "logger must be a logger" unless logger.is_a? Logger
+
+    @logger = if logger.is_a? ActiveSupport::TaggedLogging
+                logger
+              else
+                ActiveSupport::TaggedLogging.new logger
+              end
+  end
+end