exekutor 0.1.0 → 0.1.1

data/lib/exekutor/plugins/appsignal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 raise Exekutor::Plugins::LoadError, "Appsignal not found, is the gem loaded?" unless defined? Appsignal
 
 module Exekutor
@@ -21,14 +23,14 @@ module Exekutor
 
         ::Appsignal.monitor_transaction(
           "perform_job.exekutor",
-          class: payload['job_class'],
+          class: payload["job_class"],
           method: "perform",
           params: params,
           metadata: {
-            id: payload['job_id'],
-            queue: payload['queue_name'],
-            priority: payload.fetch('priority', Exekutor.config.default_queue_priority),
-            attempts: payload.fetch('attempts', 0)
+            id: payload["job_id"],
+            queue: payload["queue_name"],
+            priority: payload.fetch("priority", Exekutor.config.default_queue_priority),
+            attempts: payload.fetch("attempts", 0)
           },
           queue_start: job[:scheduled_at]
         ) do

data/lib/exekutor/plugins.rb

@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
+# The Exekutor namespace
 module Exekutor
   module Plugins
+    # Raised when a plugin cannot be loaded
     class LoadError < ::LoadError; end
   end
 
   def self.load_plugin(name)
-    if File.exist? File.join(__dir__, "plugins/#{name}.rb")
-      require_relative "plugins/#{name}"
-    else
+    unless File.exist? File.join(__dir__, "plugins/#{name}.rb")
       raise Plugins::LoadError, "The #{name} plugin does not exist. Have you spelled it correctly?"
     end
+
+    require_relative "plugins/#{name}"
   end
-end
+end

data/lib/exekutor/queue.rb

@@ -6,7 +6,7 @@ module Exekutor
     # Used when logging the SQL queries
     # @private
     ACTION_NAME = "Exekutor::Enqueue"
-    private_constant "ACTION_NAME"
+    private_constant :ACTION_NAME
 
     # Valid range for job priority
     # @private
@@ -38,42 +38,59 @@ module Exekutor
     # @param scheduled_at [Time,Date,Integer,Float] when the job should be performed
     # @return [void]
     def create_records(jobs, scheduled_at: nil)
-      unless jobs.is_a?(Array) && jobs.all? { |job| job.is_a?(ActiveJob::Base) }
+      unless jobs.is_a?(Array) && jobs.all?(ActiveJob::Base)
         raise ArgumentError, "jobs must be an array with ActiveJob items"
       end
 
+      scheduled_at = parse_scheduled_at(scheduled_at)
+      json_serializer = Exekutor.config.load_json_serializer
+
+      Internal::Hooks.run :enqueue, jobs do
+        insert_job_records(jobs, scheduled_at, json_serializer)
+      end
+    end
+
+    # Converts the given value to an epoch timestamp. Returns the current epoch timestamp if the given value is nil
+    # @param scheduled_at [nil,Numeric,Time,Date] The timestamp to convert to an epoch timestamp
+    # @return [Float,Integer] The epoch equivalent of +scheduled_at+
+    def parse_scheduled_at(scheduled_at)
       if scheduled_at.nil?
-        scheduled_at = Time.now.to_i
+        Time.now.to_i
       else
         case scheduled_at
         when Integer, Float
           raise ArgumentError, "scheduled_at must be a valid epoch" unless scheduled_at.positive?
+
+          scheduled_at
         when Time
-          scheduled_at = scheduled_at.to_f
+          scheduled_at.to_f
         when Date
-          scheduled_at = scheduled_at.at_beginning_of_day.to_f
+          scheduled_at.at_beginning_of_day.to_f
         else
           raise ArgumentError, "scheduled_at must be an epoch, time, or date"
         end
       end
+    end
 
-      json_serializer = Exekutor.config.load_json_serializer
-
-      Internal::Hooks.run :enqueue, jobs do
-        if jobs.one?
-          Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, job_sql_binds(jobs.first, scheduled_at, json_serializer), prepare: true
-            INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES ($1, $2, to_timestamp($3), $4, $5, $6) RETURNING id;
-          SQL
-        else
-          insert_statements = jobs.map do |job|
-            Exekutor::Job.sanitize_sql_for_assignment(
-              ["(?, ?, to_timestamp(?), ?, ?::jsonb, ?::jsonb)", *job_sql_binds(job, scheduled_at, json_serializer)]
-            )
-          end
-          Exekutor::Job.connection.insert <<~SQL, ACTION_NAME
-            INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES #{insert_statements.join(",")}
-          SQL
+    # Fires off an INSERT INTO query for the given jobs
+    # @param jobs [Array<ActiveJob::Base>] the jobs to insert
+    # @param scheduled_at [Integer,Float] the scheduled execution time for the jobs as an epoch timestamp
+    # @param json_serializer [#dump] the serializer to use to convert hashes into JSON
+    def insert_job_records(jobs, scheduled_at, json_serializer)
+      if jobs.one?
+        sql_binds = job_sql_binds(jobs.first, scheduled_at, json_serializer)
+        Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, sql_binds, prepare: true
+          INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES ($1, $2, to_timestamp($3), $4, $5, $6) RETURNING id;
+        SQL
+      else
+        insert_statements = jobs.map do |job|
+          Exekutor::Job.sanitize_sql_for_assignment(
+            ["(?, ?, to_timestamp(?), ?, ?::jsonb, ?::jsonb)", *job_sql_binds(job, scheduled_at, json_serializer)]
+          )
         end
+        Exekutor::Job.connection.insert <<~SQL, ACTION_NAME
+          INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES #{insert_statements.join(",")}
+        SQL
       end
     end
 
@@ -86,7 +103,8 @@ module Exekutor
       if job.queue_name.blank?
         raise Error, "The queue must be set"
       elsif job.queue_name && job.queue_name.length > Queue::MAX_NAME_LENGTH
-        raise Error, "The queue name \"#{job.queue_name}\" is too long, the limit is #{Queue::MAX_NAME_LENGTH} characters"
+        raise Error,
+              "The queue name \"#{job.queue_name}\" is too long, the limit is #{Queue::MAX_NAME_LENGTH} characters"
       end
 
       options = exekutor_options job

data/lib/exekutor/version.rb

@@ -2,5 +2,5 @@
 
 module Exekutor
   # The current version of Exekutor
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/exekutor/worker.rb

@@ -23,64 +23,43 @@ module Exekutor
     # @option config [Array<String>] :queues the queues to work on
     # @option config [Integer] :min_threads the minimum number of execution threads that should be active
     # @option config [Integer] :max_threads the maximum number of execution threads that may be active
-    # @option config [Integer] :max_thread_idletime the maximum number of seconds a thread may be idle before being stopped
+    # @option config [Integer] :max_thread_idletime the maximum number of seconds a thread may be idle before being
+    #   stopped
     # @option config [Integer] :polling_interval the polling interval in seconds
     # @option config [Float] :poling_jitter the polling jitter
     # @option config [Boolean] :set_db_connection_name whether the DB connection name should be set
-    # @option config [Integer,Boolean] :wait_for_termination how long the worker should wait on jobs to be completed before exiting
+    # @option config [Integer,Boolean] :wait_for_termination how long the worker should wait on jobs to be completed
+    #   before exiting
     # @option config [Integer] :status_server_port the port to run the status server on
     # @option config [String] :status_server_handler The name of the rack handler to use for the status server
-    # @option config [Integer] :healthcheck_timeout The timeout of a worker in minutes before the healthcheck server deems it as down
+    # @option config [Integer] :healthcheck_timeout The timeout of a worker in minutes before the healthcheck server
+    #   deems it as down
     def initialize(config = {})
       super()
       @config = config
       @record = create_record!
 
-      @reserver = Internal::Reserver.new @record.id, config[:queues]
-      @executor = Internal::Executor.new(**config.slice(:min_threads, :max_threads, :max_thread_idletime,
-                                                        :delete_completed_jobs, :delete_discarded_jobs,
-                                                        :delete_failed_jobs))
+      provider_pool = create_provider_pool(config)
 
-      provider_threads = 1
-      provider_threads += 1 if config.fetch(:enable_listener, true)
-      provider_threads += 1 if config[:status_server_port].to_i > 0
-
-      provider_pool = Concurrent::FixedThreadPool.new provider_threads, max_queue: provider_threads,
-                                                      name: "exekutor-provider"
-
-      @provider = Internal::Provider.new reserver: @reserver, executor: @executor, pool: provider_pool,
-                                         **provider_options(config)
+      @executor = create_executor(config)
+      @provider = create_provider(config, @executor, provider_pool)
 
       @executables = [@executor, @provider]
-      if config.fetch(:enable_listener, true)
-        listener = Internal::Listener.new worker_id: @record.id, provider: @provider, pool: provider_pool,
-                                          **listener_options(config)
-        @executables << listener
-      end
-      if config[:status_server_port].to_i > 0
-        server = Internal::StatusServer.new worker: self, pool: provider_pool, **status_server_options(config)
-        @executables << server
-      end
+      create_listener(provider_pool, config) if config.fetch(:enable_listener, true)
+      create_status_server(provider_pool, config) if config[:status_server_port].to_i.positive?
       @executables.freeze
-
-      @executor.after_execute(@record) do |_job, worker_info|
-        worker_info.heartbeat! rescue nil
-        @provider.poll if @provider.running?
-      end
-      @provider.on_queue_empty(@record) do |worker_info|
-        worker_info.heartbeat! rescue nil
-        @executor.prune_pool
-      end
     end
 
     # Starts the worker. Does nothing if the worker has already started.
     # @return [Boolean] whether the worker was started
     def start
       return false unless compare_and_set_state(:pending, :started)
+
       Internal::Hooks.run :startup, self do
         @executables.each(&:start)
         @record.update(status: "r")
       end
+
       true
     end
 
@@ -89,22 +68,21 @@ module Exekutor
     # @return true
     def stop
       Internal::Hooks.run :shutdown, self do
-        set_state :stopped
+        self.state = :stopped
         unless @record.destroyed?
           begin
             @record.update(status: "s")
-          rescue
-            #ignored
+          rescue StandardError
+            # ignored
           end
         end
         @executables.reverse_each(&:stop)
-
-        wait_for_termination @config[:wait_for_termination] if @config[:wait_for_termination]
+        wait_for_termination @config[:wait_for_termination]
 
         begin
           @record.destroy
-        rescue
-          #ignored
+        rescue StandardError
+          # ignored
         end
         @stop_event&.set if defined?(@stop_event)
       end
@@ -121,8 +99,8 @@ module Exekutor
       @executor.kill
       begin
         @record.destroy
-      rescue
-        #ignored
+      rescue StandardError
+        # ignored
       end
       true
     end
@@ -138,7 +116,7 @@ module Exekutor
 
     # Reserves and executes jobs.
     def reserve_jobs
-      @provider.poll
+      @provider.poll if @provider&.running?
     end
 
     # The worker ID.
@@ -146,24 +124,76 @@ module Exekutor
       @record.id
     end
 
+    # @return [Time,nil] The timestamp of the last heartbeat. The timestamp is truncated to whole minutes.
     def last_heartbeat
       @record.last_heartbeat_at
     end
 
+    # Returns the thread usage for this worker. The resulting hash will contain the following key-value pairs:
+    # - +:minimum+, (Integer) the minimum number of threads that should be active;
+    # - +:maximum+, (Integer) the maximum number of threads may should be active;
+    # - +:available+, (Integer) the number of threads that are available to execute new jobs;
+    # - +:usage_percent+, (Float, 0-100) the percentage of workers that are currently busy executing jobs.
+    # @return [Hash] the thread usage
     def thread_stats
       available = @executor.available_threads
+      usage_percent = (((100 - (available * 100.0 / @executor.maximum_threads))).round(2) if @executor.running?)
       {
         minimum: @executor.minimum_threads,
         maximum: @executor.maximum_threads,
         available: available,
-        usage_percent: if @executor.running?
-                         ((1 - (available.to_f / @executor.maximum_threads)) * 100).round(2)
-                       end
+        usage_percent: usage_percent
       }
     end
 
     private
 
+    def create_provider_pool(config)
+      provider_threads = 1
+      provider_threads += 1 if config.fetch(:enable_listener, true)
+      provider_threads += 1 if config[:status_server_port].to_i.positive?
+
+      Concurrent::FixedThreadPool.new provider_threads, max_queue: provider_threads, name: "exekutor-provider"
+    end
+
+    def create_executor(worker_options)
+      executor = Internal::Executor.new(**executor_options(worker_options))
+
+      executor.after_execute(@record) do |_job, worker_info|
+        begin
+          worker_info.heartbeat!
+        rescue StandardError
+          # ignored
+        end
+        reserve_jobs
+      end
+
+      executor
+    end
+
+    def executor_options(worker_options)
+      worker_options.slice(:min_threads, :max_threads, :max_thread_idletime,
+                           :delete_completed_jobs, :delete_discarded_jobs,
+                           :delete_failed_jobs)
+    end
+
+    def create_provider(worker_options, executor, thread_pool)
+      @reserver = Internal::Reserver.new @record.id, worker_options[:queues]
+      provider = Internal::Provider.new reserver: @reserver, executor: executor, pool: thread_pool,
+                                        **provider_options(worker_options)
+
+      provider.on_queue_empty(@record, executor) do |worker_info, thr_executor|
+        begin
+          worker_info.heartbeat!
+        rescue StandardError
+          # ignored
+        end
+        thr_executor.prune_pool
+      end
+
+      provider
+    end
+
     def provider_options(worker_options)
       worker_options.slice(:polling_interval, :polling_jitter).transform_keys do |key|
         case key
@@ -175,10 +205,21 @@ module Exekutor
       end
     end
 
+    def create_listener(provider_pool, config)
+      listener = Internal::Listener.new worker_id: @record.id, provider: @provider, pool: provider_pool,
+                                        **listener_options(config)
+      @executables << listener
+    end
+
     def listener_options(worker_options)
       worker_options.slice(:queues, :set_db_connection_name)
     end
 
+    def create_status_server(provider_pool, config)
+      server = Internal::StatusServer.new worker: self, pool: provider_pool, **status_server_options(config)
+      @executables << server
+    end
+
     def status_server_options(worker_options)
       worker_options.slice(:status_server_port, :status_server_handler, :healthcheck_timeout).transform_keys do |key|
         case key

data/lib/exekutor.rb

@@ -2,8 +2,8 @@
 
 require_relative "exekutor/version"
 
+# The Exekutor namespace
 module Exekutor
-
   # Base error class
   class Error < StandardError; end
 
@@ -45,5 +45,5 @@ ActiveSupport.on_load(:active_record) do
   end
 end
 
-Exekutor.private_constant "Internal"
+Exekutor.private_constant :Internal
 ActiveSupport.run_load_hooks(:exekutor, Exekutor)

data/lib/generators/exekutor/configuration_generator.rb

@@ -1,18 +1,22 @@
 # frozen_string_literal: true
-require 'rails/generators'
+
+require "rails/generators"
 
 module Exekutor
+  # Generates a YAML configuration file
   class ConfigurationGenerator < Rails::Generators::Base
-    desc 'Create YAML configuration for Exekutor'
+    desc "Create YAML configuration for Exekutor"
 
-    class_option :identifier, type: :string, aliases: %i(--id), desc: "The worker identifier"
+    class_option :identifier, type: :string, aliases: %i[--id], desc: "The worker identifier"
 
+    # Creates the configuration file at +config/exekutor.yml+. Uses the current worker configuration as the base.
     def create_configuration_file
       config = { queues: %w[queues to watch] }.merge(Exekutor.config.worker_options)
       config[:status_port] = 8765
       config[:set_db_connection_name] = true
       config[:wait_for_termination] = 120
-      create_file "config/exekutor#{".#{options[:identifier]}" if options[:identifier]}.yml", { "exekutor" => config.stringify_keys }.to_yaml
+      create_file "config/exekutor#{".#{options[:identifier]}" if options[:identifier]}.yml",
+                  { "exekutor" => config.stringify_keys }.to_yaml
     end
   end
-end
+end

data/lib/generators/exekutor/install_generator.rb

@@ -1,29 +1,27 @@
 # frozen_string_literal: true
-require 'rails/generators'
-require 'rails/generators/active_record'
+
+require "rails/generators"
+require "rails/generators/active_record"
 
 module Exekutor
+  # Generates the initializer and migrations
   class InstallGenerator < Rails::Generators::Base
     include ActiveRecord::Generators::Migration
-    desc 'Create migrations for Exekutor'
+    desc "Create migrations for Exekutor"
 
-    TEMPLATE_DIR = File.join(__dir__, 'templates/install')
+    TEMPLATE_DIR = File.join(__dir__, "templates/install")
     source_paths << TEMPLATE_DIR
 
+    # Creates the initializer file at +config/initializers/exekutor.rb+
     def create_initializer_file
-      template 'initializers/exekutor.rb.erb', 'config/initializers/exekutor.rb'
+      template "initializers/exekutor.rb.erb", "config/initializers/exekutor.rb"
     end
 
+    # Creates the migration file in the migrations folder
     def create_migration_file
-      migration_template 'migrations/create_exekutor_schema.rb.erb', File.join(db_migrate_path, 'create_exekutor_schema.rb')
-      if defined? Fx
-        %w(job_notifier requeue_orphaned_jobs).each do |function|
-          copy_file "functions/#{function}.sql", Fx::Definition.new(name: function, version: 1).full_path
-        end
-        %w(notify_workers requeue_orphaned_jobs).each do |trigger|
-          copy_file "triggers/#{trigger}.sql", Fx::Definition.new(name: trigger, version: 1, type: "trigger").full_path
-        end
-      end
+      migration_template "migrations/create_exekutor_schema.rb.erb",
+                         File.join(db_migrate_path, "create_exekutor_schema.rb")
+      create_fx_files
     end
 
     protected
@@ -39,5 +37,18 @@ module Exekutor
     def trigger_sql(name)
       File.read File.join(TEMPLATE_DIR, "triggers/#{name}.sql")
     end
+
+    private
+
+    def create_fx_files
+      return unless defined?(Fx)
+
+      %w[job_notifier requeue_orphaned_jobs].each do |function|
+        copy_file "functions/#{function}.sql", Fx::Definition.new(name: function, version: 1).full_path
+      end
+      %w[notify_workers requeue_orphaned_jobs].each do |trigger|
+        copy_file "triggers/#{trigger}.sql", Fx::Definition.new(name: trigger, version: 1, type: "trigger").full_path
+      end
+    end
   end
-end
+end

data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb

@@ -7,20 +7,20 @@ class CreateExekutorSchema < ActiveRecord::Migration[<%= migration_version %>]
 
       t.jsonb :info, null: false
 
-      t.datetime :created_at, null: false, default: -> { 'now()' }
-      t.datetime :last_heartbeat_at, null: false, default: -> { 'now()' }
+      t.datetime :started_at, null: false, default: -> { "now()" }
+      t.datetime :last_heartbeat_at, null: false, default: -> { "now()" }
 
-      t.column :status, :char, null: false, default: 'i'
+      t.column :status, :char, null: false, default: "i"
 
       t.index [:hostname, :pid], unique: true
     end
 
     create_table :exekutor_jobs, id: :uuid do |t|
       # Worker options
-      t.string :queue, null: false, default: 'default', limit: 200, index: true
-      t.integer :priority, null: false, default: 16383, limit: 2
-      t.datetime :enqueued_at, null: false, default: -> { 'now()' }
-      t.datetime :scheduled_at, null: false, default: -> { 'now()' }
+      t.string :queue, null: false, default: "default", limit: 200, index: true
+      t.integer :priority, null: false, default: 16_383, limit: 2
+      t.datetime :enqueued_at, null: false, default: -> { "now()" }
+      t.datetime :scheduled_at, null: false, default: -> { "now()" }
 
       # Job options
       t.uuid :active_job_id, null: false, index: true
@@ -28,16 +28,17 @@ class CreateExekutorSchema < ActiveRecord::Migration[<%= migration_version %>]
       t.jsonb :options
 
       # Execution options
-      t.column :status, :char, index: true, null: false, default: 'p'
+      t.column :status, :char, index: true, null: false, default: "p"
       t.float :runtime
       t.references :worker, type: :uuid, foreign_key: { to_table: :exekutor_workers, on_delete: :nullify }
 
-      t.index [:priority, :scheduled_at, :enqueued_at], where: %Q{"status"='p'}, name: :index_exekutor_jobs_on_dequeue_order
+      t.index [:priority, :scheduled_at, :enqueued_at], where: %q("status"='p'),
+              name: :index_exekutor_jobs_on_dequeue_order
     end
 
     create_table :exekutor_job_errors, id: :uuid do |t|
       t.references :job, type: :uuid, null: false, foreign_key: { to_table: :exekutor_jobs, on_delete: :cascade }
-      t.datetime :created_at, null: false, default: -> { 'now()' }
+      t.datetime :created_at, null: false, default: -> { "now()" }
       t.jsonb :error, null: false
     end
       <% if defined? Fx %>