exekutor 0.1.0 → 0.1.2

data/lib/exekutor/internal/status_server.rb

@@ -1,14 +1,29 @@
 # frozen_string_literal: true
+
 module Exekutor
   module Internal
-    # Serves a simple health check app
+    # Serves a simple health check app. The app provides 4 endpoints:
+    # - +/+, which lists the other endpoints;
+    # - +/ready+, which indicates whether the worker is ready to start work;
+    # - +/live+, which indicates whether the worker is ready and whether the worker is still alive;
+    # - +/threads+, which indicated the thread usage of the worker.
+    #
+    # Please note that this server uses +webrick+ by default, which is no longer a default gem from ruby 3.0 onwards.
+    #
+    # === Example requests
+    #    $ curl localhost:9000/ready
+    #    [OK] ID: f1a2ee6a-cdac-459c-a4b8-de7c6a8bbae6; State: started
+    #    $ curl localhost:9000/live
+    #    [OK] ID: f1a2ee6a-cdac-459c-a4b8-de7c6a8bbae6; State: started; Heartbeat: 2023-04-05T16:27:00Z
+    #    $ curl localhost:9000/threads
+    #    {"minimum":1,"maximum":10,"available":4,"usage_percent":60.0}
     class StatusServer
       include Internal::Logger
       include Internal::Executable
 
       DEFAULT_HANDLER = "webrick"
 
-      def initialize(worker:, pool:, port:, handler: DEFAULT_HANDLER, heartbeat_timeout: 30)
+      def initialize(worker:, pool:, port:, handler: DEFAULT_HANDLER, heartbeat_timeout: 30.minutes)
         super()
         @worker = worker
         @pool = pool
@@ -19,45 +34,50 @@ module Exekutor
         @server = Concurrent::AtomicReference.new
       end
 
+      # Starts the web server
       def start
         return false unless compare_and_set_state :pending, :started
 
         start_thread
       end
 
+      # @return [Boolean] whether the web server is active
       def running?
         super && @thread_running.value
       end
 
+      # Stops the web server
       def stop
-        set_state :stopped
+        self.state = :stopped
         return unless @thread_running.value
 
         server = @server.value
-        if server&.respond_to? :shutdown
-          server.shutdown
-        elsif server&.respond_to? :stop
-          server.stop
+        shutdown_method = %i[shutdown stop].find { |method| server.respond_to? method }
+        if shutdown_method
+          server.send(shutdown_method)
+          Exekutor.say "Status server stopped"
         elsif server
-          Exekutor.say! "Cannot shutdown status server, #{server.class.name} does not respond to shutdown or stop"
+          Exekutor.print_error "Cannot shutdown status server, " \
+                               "#{server.class.name} does not respond to `shutdown` or `stop`"
         end
       end
 
       protected
 
+      # Runs the web server, should be called from a separate thread
       def run(worker, port)
         return unless state == :started && @thread_running.make_true
 
-        Exekutor.say "Starting status server at 0.0.0.0:#{port}… (Timeout: #{@heartbeat_timeout} minutes)"
+        Exekutor.say "Starting status server at 0.0.0.0:#{port}… (Timeout: #{@heartbeat_timeout.inspect})"
         @handler.run(App.new(worker, @heartbeat_timeout), Port: port, Host: "0.0.0.0", Silent: true,
                      Logger: ::Logger.new(File.open(File::NULL, "w")), AccessLog: []) do |server|
           @server.set server
         end
-      rescue StandardError => err
-        Exekutor.on_fatal_error err, "[HealthServer] Runtime error!"
+      rescue StandardError => e
+        Exekutor.on_fatal_error e, "[Status server] Runtime error!"
         if running?
           logger.info "Restarting in 10 seconds…"
-          Concurrent::ScheduledTask.execute(10.0, executor: @pool, &method(:start_thread))
+          Concurrent::ScheduledTask.execute(10.0, executor: @pool) { start_thread }
         end
       ensure
         @thread_running.make_false
@@ -65,68 +85,81 @@ module Exekutor
 
       # The Rack-app for the health-check server
       class App
-
         def initialize(worker, heartbeat_timeout)
           @worker = worker
           @heartbeat_timeout = heartbeat_timeout
         end
 
-        def flatlined?
-          last_heartbeat = @worker.last_heartbeat
-          last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.minutes.ago
-        end
-
         def call(env)
           case Rack::Request.new(env).path
           when "/"
-            [200, {}, [
-              <<~RESPONSE
-                [Exekutor]
-                 - Use GET /ready to check whether the worker is running and connected to the DB
-                 - Use GET /live to check whether the worker is running and is not hanging
-                 - Use GET /threads to check thread usage
-              RESPONSE
-            ]]
+            render_root
           when "/ready"
-            running = @worker.running?
-            if running
-              Exekutor::Job.connection_pool.with_connection do |connection|
-                running = connection.active?
-              end
-            end
-            running = false if running && flatlined?
-            [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
-              "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}"
-            ]]
+            render_ready
           when "/live"
-            running = @worker.running?
-            last_heartbeat = if running
-                               @worker.last_heartbeat
-                             end
-            if running && (last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.minutes.ago)
-              running = false
-            end
-            [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
-              "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}; Heartbeat: #{last_heartbeat&.iso8601 || "null"}"
-            ]]
+            render_live
           when "/threads"
-            if @worker.running?
-              info = @worker.thread_stats
-              [(info ? 200 : 503), { "Content-Type" => "application/json" }, [info.to_json]]
-            else
-              [503, {"Content-Type" => "application/json"}, [{ error: "Worker not running" }.to_json]]
-            end
+            render_threads
           else
             [404, {}, ["Not found"]]
           end
         end
+
+        private
+
+        def flatlined?(last_heartbeat = @worker.last_heartbeat)
+          last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.ago
+        end
+
+        def render_threads
+          if @worker.running?
+            info = @worker.thread_stats
+            [(info ? 200 : 503), { "Content-Type" => "application/json" }, [info.to_json]]
+          else
+            [503, { "Content-Type" => "application/json" }, [{ error: "Worker not running" }.to_json]]
+          end
+        end
+
+        def render_live
+          running = @worker.running?
+          last_heartbeat = (@worker.last_heartbeat if running)
+          running = false if flatlined?(last_heartbeat)
+          [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
+            "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}; " \
+            "Heartbeat: #{last_heartbeat&.iso8601 || "null"}"
+          ]]
+        end
+
+        def render_ready
+          running = @worker.running?
+          if running
+            Exekutor::Job.connection_pool.with_connection do |connection|
+              running = connection.active?
+            end
+          end
+          running = false if running && flatlined?
+          [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
+            "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}"
+          ]]
+        end
+
+        def render_root
+          [200, {}, [
+            <<~RESPONSE
+              [Exekutor]
+               - Use GET /ready to check whether the worker is running and connected to the DB
+               - Use GET /live to check whether the worker is running and is not hanging
+               - Use GET /threads to check thread usage
+            RESPONSE
+          ]]
+        end
       end
 
       private
 
       def start_thread
-        @pool.post(@worker, @port, &method(:run)) if state == :started
+        @pool.post(@worker, @port) { |*args| run(*args) } if state == :started
       end
     end
   end
-end
+end

data/lib/exekutor/job.rb

@@ -28,4 +28,4 @@ module Exekutor
       update! status: "d"
     end
   end
-end
+end

data/lib/exekutor/job_error.rb

@@ -8,4 +8,4 @@ module Exekutor
     self.implicit_order_column = :created_at
     belongs_to :job
   end
-end
+end

data/lib/exekutor/job_options.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Exekutor
   # Mixin which defines custom job options for Exekutor. This module should be included in your job class.
   # You can define the following options after including this module:
@@ -32,7 +34,7 @@ module Exekutor
 
     # @private
     VALID_EXEKUTOR_OPTIONS = %i[queue_timeout execution_timeout].freeze
-    private_constant "VALID_EXEKUTOR_OPTIONS"
+    private_constant :VALID_EXEKUTOR_OPTIONS
 
     # @return [Hash<Symbol, Object>] the exekutor options for this job
     attr_reader :exekutor_options
@@ -67,28 +69,35 @@ module Exekutor
       end
 
       # Validates the exekutor job options passed to {#exekutor_options} and +#set+
-      # @param options [Hash<Symbol, Object>] the options to validate
+      # @param options [Hash<Symbol, Any>] the options to validate
       # @raise [InvalidOption] if any of the options are invalid
       # @private
       # @return [void]
       def validate_exekutor_options!(options)
-        return unless options.present?
+        return true if options.blank?
 
-        invalid_options = options.keys - VALID_EXEKUTOR_OPTIONS
-        if invalid_options.present?
+        if (invalid_options = options.keys - VALID_EXEKUTOR_OPTIONS).present?
           raise InvalidOption, "Invalid option#{"s" if invalid_options.many?}: " \
-            "#{invalid_options.map(&:inspect).join(", ")}. " \
-            "Valid options are: #{VALID_EXEKUTOR_OPTIONS.map(&:inspect).join(", ")}"
-        end
-        if options[:queue_timeout]
-          raise InvalidOption, ":queue_timeout must be an interval" unless options[:queue_timeout].is_a? ActiveSupport::Duration
-        end
-        if options[:execution_timeout]
-          raise InvalidOption, ":execution_timeout must be an interval" unless options[:execution_timeout].is_a? ActiveSupport::Duration
+                               "#{invalid_options.map(&:inspect).join(", ")}. " \
+                               "Valid options are: #{VALID_EXEKUTOR_OPTIONS.map(&:inspect).join(", ")}"
         end
+        JobOptions.validate_option_type! options, :queue_timeout, ::ActiveSupport::Duration
+        JobOptions.validate_option_type! options, :execution_timeout, ::ActiveSupport::Duration
+        true
       end
     end
 
+    # Validates the type of an option
+    # @param options [Hash<Symbol, Any>] the options
+    # @param name [Symbol] the name of the option to validate
+    # @param valid_type [Class] the valid type for the value
+    # @raise [InvalidOption] if the configured value is not an instance of +valid_type+
+    def self.validate_option_type!(options, name, valid_type)
+      return if options[name].nil? || options[name].is_a?(valid_type)
+
+      raise InvalidOption, ":#{name} must be an instance of #{valid_type.name} (given: #{options[name].class.name})"
+    end
+
     # Raised when invalid options are given
     class InvalidOption < ::Exekutor::Error; end
   end

data/lib/exekutor/plugins/appsignal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 raise Exekutor::Plugins::LoadError, "Appsignal not found, is the gem loaded?" unless defined? Appsignal
 
 module Exekutor
@@ -21,14 +23,14 @@ module Exekutor
 
         ::Appsignal.monitor_transaction(
           "perform_job.exekutor",
-          class: payload['job_class'],
+          class: payload["job_class"],
           method: "perform",
           params: params,
           metadata: {
-            id: payload['job_id'],
-            queue: payload['queue_name'],
-            priority: payload.fetch('priority', Exekutor.config.default_queue_priority),
-            attempts: payload.fetch('attempts', 0)
+            id: payload["job_id"],
+            queue: payload["queue_name"],
+            priority: payload.fetch("priority", Exekutor.config.default_queue_priority),
+            attempts: payload.fetch("attempts", 0)
           },
           queue_start: job[:scheduled_at]
         ) do

data/lib/exekutor/plugins.rb

@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
+# The Exekutor namespace
 module Exekutor
   module Plugins
+    # Raised when a plugin cannot be loaded
     class LoadError < ::LoadError; end
   end
 
   def self.load_plugin(name)
-    if File.exist? File.join(__dir__, "plugins/#{name}.rb")
-      require_relative "plugins/#{name}"
-    else
+    unless File.exist? File.join(__dir__, "plugins/#{name}.rb")
       raise Plugins::LoadError, "The #{name} plugin does not exist. Have you spelled it correctly?"
     end
+
+    require_relative "plugins/#{name}"
   end
-end
+end

data/lib/exekutor/queue.rb

@@ -6,7 +6,7 @@ module Exekutor
     # Used when logging the SQL queries
     # @private
     ACTION_NAME = "Exekutor::Enqueue"
-    private_constant "ACTION_NAME"
+    private_constant :ACTION_NAME
 
     # Valid range for job priority
     # @private
@@ -17,18 +17,18 @@ module Exekutor
     MAX_NAME_LENGTH = 63
 
     # Adds a job to the queue, scheduled to perform immediately
-    # @param jobs [Array<ActiveJob::Base>] the jobs to enqueue
-    # @return [void]
-    def push(*jobs)
-      create_records(jobs)
+    # @param jobs [ActiveJob::Base,Array<ActiveJob::Base>] the jobs to enqueue
+    # @return [Integer] the number of enqueued jobs
+    def push(jobs)
+      create_records Array.wrap(jobs)
     end
 
     # Adds a job to the queue, scheduled to be performed at the indicated time
-    # @param jobs [Array<ActiveJob::Base>] the jobs to enqueue
+    # @param jobs [ActiveJob::Base,Array<ActiveJob::Base>] the jobs to enqueue
     # @param timestamp [Time,Date,Integer,Float] when the job should be performed
-    # @return [void]
-    def schedule_at(*jobs, timestamp)
-      create_records(jobs, scheduled_at: timestamp)
+    # @return [Integer] the number of enqueued jobs
+    def schedule_at(jobs, timestamp)
+      create_records(Array.wrap(jobs), scheduled_at: timestamp)
     end
 
     private
@@ -38,45 +38,83 @@ module Exekutor
     # @param scheduled_at [Time,Date,Integer,Float] when the job should be performed
     # @return [void]
     def create_records(jobs, scheduled_at: nil)
-      unless jobs.is_a?(Array) && jobs.all? { |job| job.is_a?(ActiveJob::Base) }
-        raise ArgumentError, "jobs must be an array with ActiveJob items"
+      raise ArgumentError, "jobs must be an array with ActiveJob items (Actual: #{jobs.class})" unless jobs.is_a?(Array)
+
+      unless jobs.all?(ActiveJob::Base)
+        raise ArgumentError, "jobs must be an array with ActiveJob items (Found: #{
+          jobs.map { |job| job.class unless job.is_a? ActiveJob::Base }.compact.join(", ")
+        })"
       end
 
+      scheduled_at = parse_scheduled_at(scheduled_at)
+      json_serializer = Exekutor.config.load_json_serializer
+
+      inserted_records = nil
+      Internal::Hooks.run :enqueue, jobs do
+        inserted_records = insert_job_records(jobs, scheduled_at, json_serializer)
+      end
+      inserted_records
+    end
+
+    # Converts the given value to an epoch timestamp. Returns the current epoch timestamp if the given value is nil
+    # @param scheduled_at [nil,Numeric,Time,Date] The timestamp to convert to an epoch timestamp
+    # @return [Float,Integer] The epoch equivalent of +scheduled_at+
+    def parse_scheduled_at(scheduled_at)
       if scheduled_at.nil?
-        scheduled_at = Time.now.to_i
+        Time.now.to_i
       else
         case scheduled_at
         when Integer, Float
           raise ArgumentError, "scheduled_at must be a valid epoch" unless scheduled_at.positive?
+
+          scheduled_at
         when Time
-          scheduled_at = scheduled_at.to_f
+          scheduled_at.to_f
         when Date
-          scheduled_at = scheduled_at.at_beginning_of_day.to_f
+          scheduled_at.at_beginning_of_day.to_f
         else
           raise ArgumentError, "scheduled_at must be an epoch, time, or date"
         end
       end
+    end
 
-      json_serializer = Exekutor.config.load_json_serializer
-
-      Internal::Hooks.run :enqueue, jobs do
-        if jobs.one?
-          Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, job_sql_binds(jobs.first, scheduled_at, json_serializer), prepare: true
-            INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES ($1, $2, to_timestamp($3), $4, $5, $6) RETURNING id;
-          SQL
-        else
-          insert_statements = jobs.map do |job|
-            Exekutor::Job.sanitize_sql_for_assignment(
-              ["(?, ?, to_timestamp(?), ?, ?::jsonb, ?::jsonb)", *job_sql_binds(job, scheduled_at, json_serializer)]
-            )
-          end
-          Exekutor::Job.connection.insert <<~SQL, ACTION_NAME
+    # Fires off an INSERT INTO query for the given jobs
+    # @param jobs [Array<ActiveJob::Base>] the jobs to insert
+    # @param scheduled_at [Integer,Float] the scheduled execution time for the jobs as an epoch timestamp
+    # @param json_serializer [#dump] the serializer to use to convert hashes into JSON
+    def insert_job_records(jobs, scheduled_at, json_serializer)
+      if jobs.one?
+        insert_singular_job(jobs.first, json_serializer, scheduled_at)
+      else
+        insert_statements = jobs.map do |job|
+          Exekutor::Job.sanitize_sql_for_assignment(
+            ["(?, ?, to_timestamp(?), ?, ?::jsonb, ?::jsonb)", *job_sql_binds(job, scheduled_at, json_serializer)]
+          )
+        end
+        begin
+          pg_result = Exekutor::Job.connection.execute <<~SQL, ACTION_NAME
             INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES #{insert_statements.join(",")}
           SQL
+          inserted_records = pg_result.cmd_tuples
+        ensure
+          pg_result.clear
         end
+        inserted_records
       end
     end
 
+    # Fires off an INSERT INTO query for the given job using a prepared statement
+    # @param job [ActiveJob::Base] the job to insert
+    # @param scheduled_at [Integer,Float] the scheduled execution time for the jobs as an epoch timestamp
+    # @param json_serializer [#dump] the serializer to use to convert hashes into JSON
+    def insert_singular_job(job, json_serializer, scheduled_at)
+      sql_binds = job_sql_binds(job, scheduled_at, json_serializer)
+      ar_result = Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, sql_binds, prepare: true
+        INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES ($1, $2, to_timestamp($3), $4, $5, $6) RETURNING id;
+      SQL
+      ar_result.length
+    end
+
     # Converts the specified job to SQL bind parameters to insert it into the database
     # @param job [ActiveJob::Base] the job to insert
     # @param scheduled_at [Float] the epoch timestamp for when the job should be executed
@@ -86,7 +124,8 @@ module Exekutor
       if job.queue_name.blank?
         raise Error, "The queue must be set"
       elsif job.queue_name && job.queue_name.length > Queue::MAX_NAME_LENGTH
-        raise Error, "The queue name \"#{job.queue_name}\" is too long, the limit is #{Queue::MAX_NAME_LENGTH} characters"
+        raise Error,
+              "The queue name \"#{job.queue_name}\" is too long, the limit is #{Queue::MAX_NAME_LENGTH} characters"
       end
 
       options = exekutor_options job
@@ -106,7 +145,7 @@ module Exekutor
     def exekutor_options(job)
       return nil unless job.respond_to?(:exekutor_options)
 
-      options = job.exekutor_options.stringify_keys
+      options = job.exekutor_options&.stringify_keys
       if options && options["queue_timeout"]
         options["start_execution_before"] = Time.now.to_f + options.delete("queue_timeout").to_f
       end

data/lib/exekutor/version.rb

@@ -2,5 +2,5 @@
 
 module Exekutor
   # The current version of Exekutor
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end