exekutor 0.1.0

data/lib/exekutor/internal/provider.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+require_relative "executable"
+require_relative "callbacks"
+
+module Exekutor
+  # @private
+  module Internal
+    # Reserves jobs and provides them to an executor
+    class Provider
+      include Executable, Callbacks, Logger
+
+      define_callbacks :on_queue_empty, freeze: true
+
+      # Represents an unknown value
+      UNKNOWN = Object.new.freeze
+      private_constant "UNKNOWN"
+
+      MAX_WAIT_TIMEOUT = 300
+      private_constant "MAX_WAIT_TIMEOUT"
+
+      # Creates a new provider
+      # @param reserver [Reserver] the job reserver
+      # @param executor [Executor] the job executor
+      # @param pool [ThreadPoolExecutor] the thread pool to use
+      # @param polling_interval [Integer] the polling interval
+      # @param interval_jitter [Float] the polling interval jitter
+      def initialize(reserver:, executor:, pool:, polling_interval: 60,
+                     interval_jitter: polling_interval.to_i > 1 ? polling_interval * 0.1 : 0)
+        super()
+        @reserver = reserver
+        @executor = executor
+        @pool = pool
+
+        @polling_interval = polling_interval.freeze
+        @interval_jitter = interval_jitter.to_f.freeze
+
+        @event = Concurrent::Event.new
+        @thread_running = Concurrent::AtomicBoolean.new false
+
+        @next_job_scheduled_at = Concurrent::AtomicReference.new UNKNOWN
+        @next_poll_at = Concurrent::AtomicReference.new nil
+      end
+
+      # Starts the provider.
+      def start
+        return false unless compare_and_set_state :pending, :started
+
+        # Always poll at startup to fill up threads, use small jitter so workers started at the same time dont hit
+        # the db at the same time
+        @next_poll_at.set (1 + 2 * Kernel.rand).second.from_now
+        start_thread
+        true
+      end
+
+      # Stops the provider
+      def stop
+        set_state :stopped
+        @event.set
+      end
+
+      # Makes the provider poll for jobs
+      def poll
+        raise Exekutor::Error, "Provider is not running" unless running?
+
+        @next_poll_at.set Time.now
+        @event.set
+      end
+
+      # Updates the timestamp for when the next job is scheduled. Gets the earliest scheduled_at from the DB if no
+      # argument is given. Updates the timestamp for the earliest job is a timestamp is given and that timestamp is
+      # before the known timestamp. Does nothing if a timestamp is given and the earliest job timestamp is not known.
+      # @param scheduled_at [Time,Numeric] the time a job is scheduled at
+      # @return [Time] the timestamp for the next job, or +nil+ if the timestamp is unknown or no jobs are pending
+      def update_earliest_scheduled_at(scheduled_at = UNKNOWN)
+        overwrite_unknown = false
+        case scheduled_at
+        when UNKNOWN
+          # If we fetch the value from the DB, we can safely overwrite the UNKNOWN value
+          overwrite_unknown = true
+          scheduled_at = @reserver.earliest_scheduled_at
+        when Numeric
+          scheduled_at = Time.at(scheduled_at)
+        when Time
+          #  All good
+        else
+          raise ArgumentError, "scheduled_at must be a Time or Numeric"
+        end
+
+        updated = false
+        scheduled_at = @next_job_scheduled_at.update do |current|
+          if current == UNKNOWN
+            if overwrite_unknown || scheduled_at <= Time.now
+              updated = true
+              scheduled_at
+            else
+              current
+            end
+          elsif current.nil? || scheduled_at.nil? || current > scheduled_at
+            updated = true
+            scheduled_at
+          else
+            current
+          end
+        end
+        if scheduled_at == UNKNOWN
+          nil
+        else
+          @event.set if updated && scheduled_at.present?
+          scheduled_at
+        end
+      end
+
+      private
+
+      # Starts the provision thread
+      def start_thread
+        @pool.post(&method(:run)) if running?
+      end
+
+      # Does the provisioning of jobs to the executor. Blocks until the provider is stopped.
+      def run
+        return unless running? && @thread_running.make_true
+
+        DatabaseConnection.ensure_active!
+        perform_pending_job_updates
+        restart_abandoned_jobs
+        catch(:shutdown) do
+          while running?
+            wait_for_event
+            next unless reserve_jobs_now?
+
+            reserve_and_execute_jobs
+            consecutive_errors.value = 0
+          end
+        end
+      rescue StandardError => err
+        Exekutor.on_fatal_error err, "[Provider] Runtime error!"
+        consecutive_errors.increment
+        if running?
+          delay = restart_delay
+          logger.info "Restarting in %0.1f seconds…" % [delay]
+          Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
+        end
+      ensure
+        BaseRecord.connection_pool.release_connection
+        @thread_running.make_false
+      end
+
+      # Waits for any event to happen. An event could be:
+      # - The listener was notified of a new job;
+      # - The next job is scheduled for the current time;
+      # - The polling interval;
+      # - A call to {#poll}
+      def wait_for_event
+        timeout = wait_timeout
+        return unless timeout.positive?
+
+        @event.wait timeout
+      rescue StandardError => err
+        Exekutor.on_fatal_error err, "[Provider] An error occurred while waiting"
+        sleep 0.1 if running?
+      ensure
+        throw :shutdown unless running?
+        @event.reset
+      end
+
+      # Reserves jobs and posts them to the executor
+      def reserve_and_execute_jobs
+        available_workers = @executor.available_threads
+        return unless available_workers.positive?
+
+        jobs = @reserver.reserve available_workers
+        unless jobs.nil?
+          begin
+            logger.debug "Reserved #{jobs.size} job(s)"
+            jobs.each(&@executor.method(:post))
+          rescue Exception # rubocop:disable Lint/RescueException
+            # Try to release all jobs before re-raising
+            begin
+              Exekutor::Job.where(id: jobs.collect { |job| job[:id] }, status: "e")
+                           .update_all(status: "p", worker_id: nil)
+            rescue # rubocop:disable Lint/RescueStandardError
+              # ignored
+            end
+            raise
+          end
+        end
+        if jobs.nil? || jobs.size.to_i < available_workers
+          # If we ran out of work, update the earliest scheduled at
+          update_earliest_scheduled_at
+
+          run_callbacks :on, :queue_empty if jobs.nil?
+
+        elsif @next_job_scheduled_at.get == UNKNOWN
+          # If the next job timestamp is still unknown, set it to now to indicate there's still work to do
+          @next_job_scheduled_at.set Time.now
+        end
+      end
+
+      def perform_pending_job_updates
+        updates = @executor.pending_job_updates
+        while (id, attrs = updates.shift).present?
+          begin
+            if attrs == :destroy
+              Exekutor::Job.destroy(id)
+            else
+              Exekutor::Job.where(id: id).update_all(attrs)
+            end
+          rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished
+            unless Exekutor::Job.connection.active?
+              # Connection lost again, requeue update and avoid trying further updates
+              updates[id] ||= attrs
+              return
+            end
+          end
+        end
+      end
+
+      # Restarts all jobs that have the 'executing' status but are no longer running. Releases the jobs if the
+      # execution thread pool is full.
+      def restart_abandoned_jobs
+        jobs = @reserver.get_abandoned_jobs(@executor.active_job_ids)
+        return if jobs&.size.to_i.zero?
+
+        logger.info "Restarting #{jobs.size} abandoned job#{"s" if jobs.size > 1}"
+        jobs.each(&@executor.method(:post))
+      end
+
+      # @return [Boolean] Whether the polling is enabled. Ie. whether a polling interval is set.
+      def polling_enabled?
+        @polling_interval.present?
+      end
+
+      # @return [Time,nil] the 'scheduled at' value for the next job, or nil if unknown or if there is no pending job
+      def next_job_scheduled_at
+        at = @next_job_scheduled_at.get
+        if at == UNKNOWN
+          nil
+        else
+          # noinspection RubyMismatchedReturnType
+          at
+        end
+      end
+
+      # @return [Time,nil] When the next poll is scheduled, or nil if polling is disabled
+      def next_poll_scheduled_at
+        if polling_enabled?
+          @next_poll_at.update { |planned_at| planned_at || Time.now + polling_interval }
+        else
+          # noinspection RubyMismatchedReturnType
+          @next_poll_at.get
+        end
+      end
+
+      # @return [Numeric] The timeout to wait until the next event
+      def wait_timeout
+        return MAX_WAIT_TIMEOUT if @executor.available_threads.zero?
+
+        next_job_at = next_job_scheduled_at
+        next_poll_at = next_poll_scheduled_at
+
+        timeout = [MAX_WAIT_TIMEOUT].tap do |timeouts|
+          # noinspection RubyMismatchedArgumentType
+          timeouts.append next_job_at - Time.now if next_job_at
+          # noinspection RubyMismatchedArgumentType
+          timeouts.append next_poll_at - Time.now if next_poll_at
+        end.min
+
+        if timeout <= 0.001
+          0
+        else
+          # noinspection RubyMismatchedReturnType
+          timeout
+        end
+      end
+
+      # @return [Boolean] Whether the `reserver` should be called.
+      def reserve_jobs_now?
+        next_poll_at = next_poll_scheduled_at
+        if next_poll_at && next_poll_at - Time.now <= 0.001
+          @next_poll_at.update { Time.now + polling_interval if polling_enabled? }
+          return true
+        end
+
+        next_job_at = next_job_scheduled_at
+        next_job_at && next_job_at <= Time.now
+      end
+
+      # @return [Float] Gets the polling interval jitter
+      def polling_interval_jitter
+        @interval_jitter
+      end
+
+      # Get the polling interval. If a jitter is configured, the interval is reduced or increased by `0.5 * jitter`.
+      # @return [Float] The amount of seconds before the next poll
+      def polling_interval
+        raise "Polling is disabled" unless @polling_interval.present?
+
+        @polling_interval + if polling_interval_jitter.zero?
+                              0
+                            else
+                              (Kernel.rand - 0.5) * polling_interval_jitter
+                            end
+      end
+    end
+  end
+end

data/lib/exekutor/internal/reserver.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+module Exekutor
+  # @private
+  module Internal
+    # Reserves jobs to be executed by the current worker
+    class Reserver
+      # The name to use for the SQL log message
+      ACTION_NAME = "Exekutor::Reserve"
+
+      # Creates a new Reserver
+      # @param worker_id [String] the id of the worker
+      # @param queues [Array<String>] the queues to watch
+      def initialize(worker_id, queues)
+        @worker_id = worker_id
+        @queue_filter_sql = build_queue_filter_sql(queues)
+        @json_serializer = Exekutor.config.load_json_serializer
+      end
+
+      # Reserves pending jobs
+      # @param limit [Integer] the number of jobs to reserve
+      # @return [Array<Job>,nil] the reserved jobs, or nil if no jobs were reserved
+      def reserve(limit)
+        return unless limit.positive?
+
+        results = Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, [@worker_id, limit], prepare: true
+          UPDATE exekutor_jobs SET worker_id = $1, status = 'e' WHERE id IN (
+             SELECT id FROM exekutor_jobs
+                WHERE scheduled_at <= now() AND "status"='p' #{@queue_filter_sql}
+                ORDER BY priority, scheduled_at, enqueued_at
+                FOR UPDATE SKIP LOCKED
+                LIMIT $2
+          ) RETURNING "id", "payload", "options", "scheduled_at"
+        SQL
+        return unless results&.length&.positive?
+
+        parse_jobs results
+      end
+
+      def get_abandoned_jobs(active_job_ids)
+        jobs = Exekutor::Job.executing.where(worker_id: @worker_id)
+        jobs = jobs.where.not(id: active_job_ids) if active_job_ids.present?
+        attrs = %i[id payload options scheduled_at]
+        jobs.pluck(*attrs).map { |p| attrs.zip(p).to_h }
+      end
+
+      # Gets the earliest scheduled at of all pending jobs in the watched queues
+      # @return [Time,nil] The earliest scheduled at, or nil if the queues are empty
+      def earliest_scheduled_at
+        jobs = Exekutor::Job.pending
+        jobs.where! @queue_filter_sql.gsub(/^\s*AND\s+/, "") unless @queue_filter_sql.nil?
+        jobs.minimum(:scheduled_at)
+      end
+
+      private
+
+      # Parses jobs from the SQL results
+      def parse_jobs(sql_results)
+        sql_results.map do |result|
+          { id: result["id"],
+            payload: parse_json(result["payload"]),
+            options: parse_json(result["options"]),
+            scheduled_at: result['scheduled_at'] }
+        end
+      end
+
+      # Parses JSON using the configured serializer
+      def parse_json(str)
+        @json_serializer.load str unless str.nil?
+      end
+
+      # Builds SQL filter for the given queues
+      def build_queue_filter_sql(queues)
+        return nil if queues.nil? || (queues.is_a?(Array) && queues.empty?)
+        unless queues.is_a?(String) || queues.is_a?(Symbol) || queues.is_a?(Array)
+          raise ArgumentError, "queues must be nil, a String, Symbol, or an array of Strings or Symbols"
+        end
+
+        queues = queues.first if queues.is_a?(Array) && queues.one?
+        if queues.is_a? Array
+          unless queues.all? { |q| (q.is_a?(String) || q.is_a?(Symbol)) && !q.blank? }
+            raise ArgumentError, "queues contains an invalid value"
+          end
+
+          Exekutor::Job.sanitize_sql_for_conditions(["AND queue IN (?)", queues])
+        else
+          raise ArgumentError, "queue name cannot be empty" if queues.blank?
+
+          Exekutor::Job.sanitize_sql_for_conditions(["AND queue = ?", queues])
+        end
+      end
+
+    end
+  end
+end
+

data/lib/exekutor/internal/status_server.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+module Exekutor
+  module Internal
+    # Serves a simple health check app
+    class StatusServer
+      include Internal::Logger
+      include Internal::Executable
+
+      DEFAULT_HANDLER = "webrick"
+
+      def initialize(worker:, pool:, port:, handler: DEFAULT_HANDLER, heartbeat_timeout: 30)
+        super()
+        @worker = worker
+        @pool = pool
+        @port = port
+        @handler = Rack::Handler.get(handler)
+        @heartbeat_timeout = heartbeat_timeout
+        @thread_running = Concurrent::AtomicBoolean.new false
+        @server = Concurrent::AtomicReference.new
+      end
+
+      def start
+        return false unless compare_and_set_state :pending, :started
+
+        start_thread
+      end
+
+      def running?
+        super && @thread_running.value
+      end
+
+      def stop
+        set_state :stopped
+        return unless @thread_running.value
+
+        server = @server.value
+        if server&.respond_to? :shutdown
+          server.shutdown
+        elsif server&.respond_to? :stop
+          server.stop
+        elsif server
+          Exekutor.say! "Cannot shutdown status server, #{server.class.name} does not respond to shutdown or stop"
+        end
+      end
+
+      protected
+
+      def run(worker, port)
+        return unless state == :started && @thread_running.make_true
+
+        Exekutor.say "Starting status server at 0.0.0.0:#{port}… (Timeout: #{@heartbeat_timeout} minutes)"
+        @handler.run(App.new(worker, @heartbeat_timeout), Port: port, Host: "0.0.0.0", Silent: true,
+                     Logger: ::Logger.new(File.open(File::NULL, "w")), AccessLog: []) do |server|
+          @server.set server
+        end
+      rescue StandardError => err
+        Exekutor.on_fatal_error err, "[HealthServer] Runtime error!"
+        if running?
+          logger.info "Restarting in 10 seconds…"
+          Concurrent::ScheduledTask.execute(10.0, executor: @pool, &method(:start_thread))
+        end
+      ensure
+        @thread_running.make_false
+      end
+
+      # The Rack-app for the health-check server
+      class App
+
+        def initialize(worker, heartbeat_timeout)
+          @worker = worker
+          @heartbeat_timeout = heartbeat_timeout
+        end
+
+        def flatlined?
+          last_heartbeat = @worker.last_heartbeat
+          last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.minutes.ago
+        end
+
+        def call(env)
+          case Rack::Request.new(env).path
+          when "/"
+            [200, {}, [
+              <<~RESPONSE
+                [Exekutor]
+                 - Use GET /ready to check whether the worker is running and connected to the DB
+                 - Use GET /live to check whether the worker is running and is not hanging
+                 - Use GET /threads to check thread usage
+              RESPONSE
+            ]]
+          when "/ready"
+            running = @worker.running?
+            if running
+              Exekutor::Job.connection_pool.with_connection do |connection|
+                running = connection.active?
+              end
+            end
+            running = false if running && flatlined?
+            [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
+              "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}"
+            ]]
+          when "/live"
+            running = @worker.running?
+            last_heartbeat = if running
+                               @worker.last_heartbeat
+                             end
+            if running && (last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.minutes.ago)
+              running = false
+            end
+            [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
+              "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}; Heartbeat: #{last_heartbeat&.iso8601 || "null"}"
+            ]]
+          when "/threads"
+            if @worker.running?
+              info = @worker.thread_stats
+              [(info ? 200 : 503), { "Content-Type" => "application/json" }, [info.to_json]]
+            else
+              [503, {"Content-Type" => "application/json"}, [{ error: "Worker not running" }.to_json]]
+            end
+          else
+            [404, {}, ["Not found"]]
+          end
+        end
+      end
+
+      private
+
+      def start_thread
+        @pool.post(@worker, @port, &method(:run)) if state == :started
+      end
+    end
+  end
+end

data/lib/exekutor/job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "internal/base_record"
+
+module Exekutor
+  # Active record instance for a job
+  class Job < Internal::BaseRecord
+    self.implicit_order_column = :enqueued_at
+
+    belongs_to :worker, optional: true, class_name: "Info::Worker"
+    has_many :execution_errors, class_name: "JobError"
+
+    enum status: { pending: "p", executing: "e", completed: "c", failed: "f", discarded: "d" }
+
+    # Sets the status to pending and clears the assigned worker
+    def release!
+      update! status: "p", worker_id: nil
+    end
+
+    # Sets the status to pending, clears the assigned worker, and schedules execution at the indicated time.
+    # @param at [Time] when the job should be executed
+    def reschedule!(at: Time.current)
+      update! status: "p", scheduled_at: at, worker_id: nil, runtime: nil
+    end
+
+    # Sets the status to discarded.
+    def discard!
+      update! status: "d"
+    end
+  end
+end

data/lib/exekutor/job_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "internal/base_record"
+
+module Exekutor
+  # Active record instance for errors raised by jobs
+  class JobError < Internal::BaseRecord
+    self.implicit_order_column = :created_at
+    belongs_to :job
+  end
+end

data/lib/exekutor/job_options.rb

@@ -0,0 +1,95 @@
+module Exekutor
+  # Mixin which defines custom job options for Exekutor. This module should be included in your job class.
+  # You can define the following options after including this module:
+  #
+  # ==== Queue timeout
+  #   MyJob.set(queue_timeout: 1.hour).perform_later
+  # How long the job is allowed to be in the queue. If a job is not performed before this timeout, it will be discarded.
+  # The value should be a +ActiveSupport::Duration+.
+  #
+  # ==== Execution timeout
+  #   MyJob.set(execution_timeout: 1.minute).perform_later
+  # How long the job is allowed to run. If a job is taking longer than this timeout, it will be killed and discarded.
+  # The value should be a +ActiveSupport::Duration+. Be aware that +Timeout::timeout+ is used internally for this, which
+  # can raise an error at any line of code in your application. <em>Use with caution</em>
+  #
+  # == Usage
+  # === +#set+
+  # You can specify the options per job when enqueueing the job using +#set+.
+  #   MyJob.set(option_name: @option_value).perform_later
+  #
+  # === +#exekutor_options+
+  # You can also specify options that apply to all instances of a job by calling {#exekutor_options}.
+  #  class MyOtherJob < ActiveJob::Base
+  #     include Exekutor::JobOptions
+  #     exekutor_options execution_timeout: 10.seconds
+  #   end
+  #
+  # *NB* These options only work for jobs that are scheduled with +#perform_later+, the options are ignored when you
+  # perform the job immediately using +#perform_now+.
+  module JobOptions
+    extend ActiveSupport::Concern
+
+    # @private
+    VALID_EXEKUTOR_OPTIONS = %i[queue_timeout execution_timeout].freeze
+    private_constant "VALID_EXEKUTOR_OPTIONS"
+
+    # @return [Hash<Symbol, Object>] the exekutor options for this job
+    attr_reader :exekutor_options
+
+    # @!visibility private
+    def enqueue(options = {})
+      # :nodoc:
+      @exekutor_options = self.class.exekutor_job_options || {}
+      job_options = options&.slice(*VALID_EXEKUTOR_OPTIONS)
+      if job_options
+        self.class.validate_exekutor_options! job_options
+        @exekutor_options = @exekutor_options.merge job_options
+      end
+      super(options)
+    end
+
+    class_methods do
+      # Sets the exekutor options that apply to all instances of this job. These options can be overwritten with +#set+.
+      # @param options [Hash<Symbol, Object>] the exekutor options
+      # @option options [ActiveSupport::Duration] :queue_timeout The queue timeout
+      # @option options [ActiveSupport::Duration] :execution_timeout The execution timeout
+      # @return [void]
+      def exekutor_options(options)
+        validate_exekutor_options! options
+        @exekutor_job_options = options
+      end
+
+      # Gets the exekutor options that apply to all instances of this job. These options may be overwritten by +#set+.
+      # @return [Hash<Symbol, Object>] the options
+      def exekutor_job_options
+        @exekutor_job_options if defined?(@exekutor_job_options)
+      end
+
+      # Validates the exekutor job options passed to {#exekutor_options} and +#set+
+      # @param options [Hash<Symbol, Object>] the options to validate
+      # @raise [InvalidOption] if any of the options are invalid
+      # @private
+      # @return [void]
+      def validate_exekutor_options!(options)
+        return unless options.present?
+
+        invalid_options = options.keys - VALID_EXEKUTOR_OPTIONS
+        if invalid_options.present?
+          raise InvalidOption, "Invalid option#{"s" if invalid_options.many?}: " \
+            "#{invalid_options.map(&:inspect).join(", ")}. " \
+            "Valid options are: #{VALID_EXEKUTOR_OPTIONS.map(&:inspect).join(", ")}"
+        end
+        if options[:queue_timeout]
+          raise InvalidOption, ":queue_timeout must be an interval" unless options[:queue_timeout].is_a? ActiveSupport::Duration
+        end
+        if options[:execution_timeout]
+          raise InvalidOption, ":execution_timeout must be an interval" unless options[:execution_timeout].is_a? ActiveSupport::Duration
+        end
+      end
+    end
+
+    # Raised when invalid options are given
+    class InvalidOption < ::Exekutor::Error; end
+  end
+end