exekutor 0.1.0

data/lib/exekutor/plugins/appsignal.rb

@@ -0,0 +1,46 @@
+raise Exekutor::Plugins::LoadError, "Appsignal not found, is the gem loaded?" unless defined? Appsignal
+
+module Exekutor
+  module Plugins
+    # Hooks to send job execution info and raised errors to Appsignal
+    class Appsignal
+      include Hook
+      before_shutdown { ::Appsignal.stop("exekutor") }
+
+      around_job_execution :invoke_with_instrumentation
+
+      on_job_failure { |_job, error| report_error error }
+      on_fatal_error :report_error
+
+      def invoke_with_instrumentation(job)
+        payload = job[:payload]
+        params = ::Appsignal::Utils::HashSanitizer.sanitize(
+          payload.fetch("arguments", {}),
+          ::Appsignal.config[:filter_parameters]
+        )
+
+        ::Appsignal.monitor_transaction(
+          "perform_job.exekutor",
+          class: payload['job_class'],
+          method: "perform",
+          params: params,
+          metadata: {
+            id: payload['job_id'],
+            queue: payload['queue_name'],
+            priority: payload.fetch('priority', Exekutor.config.default_queue_priority),
+            attempts: payload.fetch('attempts', 0)
+          },
+          queue_start: job[:scheduled_at]
+        ) do
+          yield job
+        end
+      end
+
+      def report_error(error)
+        ::Appsignal.add_exception(error)
+      end
+    end
+  end
+end
+
+Exekutor.hooks.register Exekutor::Plugins::Appsignal

data/lib/exekutor/plugins.rb

@@ -0,0 +1,13 @@
+module Exekutor
+  module Plugins
+    class LoadError < ::LoadError; end
+  end
+
+  def self.load_plugin(name)
+    if File.exist? File.join(__dir__, "plugins/#{name}.rb")
+      require_relative "plugins/#{name}"
+    else
+      raise Plugins::LoadError, "The #{name} plugin does not exist. Have you spelled it correctly?"
+    end
+  end
+end

data/lib/exekutor/queue.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Exekutor
+  # The job queue
+  class Queue
+    # Used when logging the SQL queries
+    # @private
+    ACTION_NAME = "Exekutor::Enqueue"
+    private_constant "ACTION_NAME"
+
+    # Valid range for job priority
+    # @private
+    VALID_PRIORITIES = (1..32_767).freeze
+
+    # Maximum length for the queue name
+    # @private
+    MAX_NAME_LENGTH = 63
+
+    # Adds a job to the queue, scheduled to perform immediately
+    # @param jobs [Array<ActiveJob::Base>] the jobs to enqueue
+    # @return [void]
+    def push(*jobs)
+      create_records(jobs)
+    end
+
+    # Adds a job to the queue, scheduled to be performed at the indicated time
+    # @param jobs [Array<ActiveJob::Base>] the jobs to enqueue
+    # @param timestamp [Time,Date,Integer,Float] when the job should be performed
+    # @return [void]
+    def schedule_at(*jobs, timestamp)
+      create_records(jobs, scheduled_at: timestamp)
+    end
+
+    private
+
+    # Creates {Exekutor::Job} records for the specified jobs, scheduled at the indicated time
+    # @param jobs [Array<ActiveJob::Base>] the jobs to enqueue
+    # @param scheduled_at [Time,Date,Integer,Float] when the job should be performed
+    # @return [void]
+    def create_records(jobs, scheduled_at: nil)
+      unless jobs.is_a?(Array) && jobs.all? { |job| job.is_a?(ActiveJob::Base) }
+        raise ArgumentError, "jobs must be an array with ActiveJob items"
+      end
+
+      if scheduled_at.nil?
+        scheduled_at = Time.now.to_i
+      else
+        case scheduled_at
+        when Integer, Float
+          raise ArgumentError, "scheduled_at must be a valid epoch" unless scheduled_at.positive?
+        when Time
+          scheduled_at = scheduled_at.to_f
+        when Date
+          scheduled_at = scheduled_at.at_beginning_of_day.to_f
+        else
+          raise ArgumentError, "scheduled_at must be an epoch, time, or date"
+        end
+      end
+
+      json_serializer = Exekutor.config.load_json_serializer
+
+      Internal::Hooks.run :enqueue, jobs do
+        if jobs.one?
+          Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, job_sql_binds(jobs.first, scheduled_at, json_serializer), prepare: true
+            INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES ($1, $2, to_timestamp($3), $4, $5, $6) RETURNING id;
+          SQL
+        else
+          insert_statements = jobs.map do |job|
+            Exekutor::Job.sanitize_sql_for_assignment(
+              ["(?, ?, to_timestamp(?), ?, ?::jsonb, ?::jsonb)", *job_sql_binds(job, scheduled_at, json_serializer)]
+            )
+          end
+          Exekutor::Job.connection.insert <<~SQL, ACTION_NAME
+            INSERT INTO exekutor_jobs ("queue", "priority", "scheduled_at", "active_job_id", "payload", "options") VALUES #{insert_statements.join(",")}
+          SQL
+        end
+      end
+    end
+
+    # Converts the specified job to SQL bind parameters to insert it into the database
+    # @param job [ActiveJob::Base] the job to insert
+    # @param scheduled_at [Float] the epoch timestamp for when the job should be executed
+    # @param json_serializer [#dump] the serializer to use to convert hashes into JSON
+    # @return [Array] the SQL bind parameters for inserting the specified job
+    def job_sql_binds(job, scheduled_at, json_serializer)
+      if job.queue_name.blank?
+        raise Error, "The queue must be set"
+      elsif job.queue_name && job.queue_name.length > Queue::MAX_NAME_LENGTH
+        raise Error, "The queue name \"#{job.queue_name}\" is too long, the limit is #{Queue::MAX_NAME_LENGTH} characters"
+      end
+
+      options = exekutor_options job
+      [
+        job.queue_name.presence,
+        job_priority(job),
+        scheduled_at,
+        job.job_id,
+        json_serializer.dump(job.serialize),
+        options.present? ? json_serializer.dump(options) : nil
+      ]
+    end
+
+    # Get the exekutor options for the specified job.
+    # @param job [ActiveJob::Base] the job to get the options for
+    # @return [Hash<String,Object>] the exekutor options
+    def exekutor_options(job)
+      return nil unless job.respond_to?(:exekutor_options)
+
+      options = job.exekutor_options.stringify_keys
+      if options && options["queue_timeout"]
+        options["start_execution_before"] = Time.now.to_f + options.delete("queue_timeout").to_f
+      end
+      options["execution_timeout"] = options["execution_timeout"].to_f if options && options["execution_timeout"]
+
+      options
+    end
+
+    # Get the priority for the specified job.
+    # @param job [ActiveJob::Base] the job to get the priority for
+    # @return [Integer] the priority
+    def job_priority(job)
+      priority = job.priority
+      if priority.is_a? Integer
+        unless VALID_PRIORITIES.include? priority
+          raise Error, <<~MESSAGE
+            Job priority must be between #{VALID_PRIORITIES.begin} and #{VALID_PRIORITIES.end} (actual: #{priority})
+          MESSAGE
+        end
+
+        priority
+      elsif priority.nil?
+        Exekutor.config.default_queue_priority
+      else
+        raise Error, "Job priority must be an Integer or nil"
+      end
+    end
+
+    # Default error for queueing problems
+    class Error < Exekutor::Error; end
+  end
+end

data/lib/exekutor/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Exekutor
+  # The current version of Exekutor
+  VERSION = "0.1.0"
+end

data/lib/exekutor/worker.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require_relative "internal/executable"
+
+module Exekutor
+  # The job worker
+  class Worker
+    include Internal::Executable
+
+    attr_reader :record
+
+    # Creates a new worker with the specified config and immediately starts it
+    # @see #initialize
+    #
+    # @return The worker
+    def self.start(config = {})
+      new(config).tap(&:start)
+    end
+
+    # Creates a new worker with the specified config
+    # @param config [Hash] The worker configuration
+    # @option config [String] :identifier the identifier for this worker
+    # @option config [Array<String>] :queues the queues to work on
+    # @option config [Integer] :min_threads the minimum number of execution threads that should be active
+    # @option config [Integer] :max_threads the maximum number of execution threads that may be active
+    # @option config [Integer] :max_thread_idletime the maximum number of seconds a thread may be idle before being stopped
+    # @option config [Integer] :polling_interval the polling interval in seconds
+    # @option config [Float] :poling_jitter the polling jitter
+    # @option config [Boolean] :set_db_connection_name whether the DB connection name should be set
+    # @option config [Integer,Boolean] :wait_for_termination how long the worker should wait on jobs to be completed before exiting
+    # @option config [Integer] :status_server_port the port to run the status server on
+    # @option config [String] :status_server_handler The name of the rack handler to use for the status server
+    # @option config [Integer] :healthcheck_timeout The timeout of a worker in minutes before the healthcheck server deems it as down
+    def initialize(config = {})
+      super()
+      @config = config
+      @record = create_record!
+
+      @reserver = Internal::Reserver.new @record.id, config[:queues]
+      @executor = Internal::Executor.new(**config.slice(:min_threads, :max_threads, :max_thread_idletime,
+                                                        :delete_completed_jobs, :delete_discarded_jobs,
+                                                        :delete_failed_jobs))
+
+      provider_threads = 1
+      provider_threads += 1 if config.fetch(:enable_listener, true)
+      provider_threads += 1 if config[:status_server_port].to_i > 0
+
+      provider_pool = Concurrent::FixedThreadPool.new provider_threads, max_queue: provider_threads,
+                                                      name: "exekutor-provider"
+
+      @provider = Internal::Provider.new reserver: @reserver, executor: @executor, pool: provider_pool,
+                                         **provider_options(config)
+
+      @executables = [@executor, @provider]
+      if config.fetch(:enable_listener, true)
+        listener = Internal::Listener.new worker_id: @record.id, provider: @provider, pool: provider_pool,
+                                          **listener_options(config)
+        @executables << listener
+      end
+      if config[:status_server_port].to_i > 0
+        server = Internal::StatusServer.new worker: self, pool: provider_pool, **status_server_options(config)
+        @executables << server
+      end
+      @executables.freeze
+
+      @executor.after_execute(@record) do |_job, worker_info|
+        worker_info.heartbeat! rescue nil
+        @provider.poll if @provider.running?
+      end
+      @provider.on_queue_empty(@record) do |worker_info|
+        worker_info.heartbeat! rescue nil
+        @executor.prune_pool
+      end
+    end
+
+    # Starts the worker. Does nothing if the worker has already started.
+    # @return [Boolean] whether the worker was started
+    def start
+      return false unless compare_and_set_state(:pending, :started)
+      Internal::Hooks.run :startup, self do
+        @executables.each(&:start)
+        @record.update(status: "r")
+      end
+      true
+    end
+
+    # Stops the worker. If +wait_for_termination+ is set, this method blocks until the execution thread is terminated
+    # or killed.
+    # @return true
+    def stop
+      Internal::Hooks.run :shutdown, self do
+        set_state :stopped
+        unless @record.destroyed?
+          begin
+            @record.update(status: "s")
+          rescue
+            #ignored
+          end
+        end
+        @executables.reverse_each(&:stop)
+
+        wait_for_termination @config[:wait_for_termination] if @config[:wait_for_termination]
+
+        begin
+          @record.destroy
+        rescue
+          #ignored
+        end
+        @stop_event&.set if defined?(@stop_event)
+      end
+      true
+    end
+
+    # Kills the worker. Does not wait for any jobs to be completed.
+    # @return true
+    def kill
+      Thread.new do
+        @executables.reverse_each(&:stop)
+        @stop_event&.set if defined?(@stop_event)
+      end
+      @executor.kill
+      begin
+        @record.destroy
+      rescue
+        #ignored
+      end
+      true
+    end
+
+    # Blocks until the worker is stopped.
+    def join
+      @stop_event = Concurrent::Event.new
+      Kernel.loop do
+        @stop_event.wait 10
+        break unless running?
+      end
+    end
+
+    # Reserves and executes jobs.
+    def reserve_jobs
+      @provider.poll
+    end
+
+    # The worker ID.
+    def id
+      @record.id
+    end
+
+    def last_heartbeat
+      @record.last_heartbeat_at
+    end
+
+    def thread_stats
+      available = @executor.available_threads
+      {
+        minimum: @executor.minimum_threads,
+        maximum: @executor.maximum_threads,
+        available: available,
+        usage_percent: if @executor.running?
+                         ((1 - (available.to_f / @executor.maximum_threads)) * 100).round(2)
+                       end
+      }
+    end
+
+    private
+
+    def provider_options(worker_options)
+      worker_options.slice(:polling_interval, :polling_jitter).transform_keys do |key|
+        case key
+        when :polling_jitter
+          :interval_jitter
+        else
+          key
+        end
+      end
+    end
+
+    def listener_options(worker_options)
+      worker_options.slice(:queues, :set_db_connection_name)
+    end
+
+    def status_server_options(worker_options)
+      worker_options.slice(:status_server_port, :status_server_handler, :healthcheck_timeout).transform_keys do |key|
+        case key
+        when :healthcheck_timeout
+          :heartbeat_timeout
+        else
+          key.to_s.gsub(/^status_server_/, "").to_sym
+        end
+      end
+    end
+
+    # Waits for the execution threads to finish. Does nothing if +timeout+ is falsey. If +timeout+ is zero, the
+    # execution threads are killed immediately. If +timeout+ is a positive +Numeric+, waits for the indicated amount of
+    # seconds to let the execution threads finish and kills the threads if the timeout is exceeded. Otherwise; waits
+    # for the execution threads to finish indefinitely.
+    # @param timeout The time to wait.
+    def wait_for_termination(timeout)
+      if timeout.is_a?(Numeric) && timeout.zero?
+        @executor.kill
+      elsif timeout.is_a?(Numeric) && timeout.positive?
+        @executor.kill unless @executor.wait_for_termination timeout
+      elsif timeout
+        @executor.wait_for_termination
+      end
+    end
+
+    # Creates the active record entry for this worker.
+    def create_record!
+      info = {}
+      info.merge!(@config.slice(:identifier, :max_threads, :queues, :polling_interval))
+      Info::Worker.create!({
+                             hostname: Socket.gethostname,
+                             pid: Process.pid,
+                             info: info.compact
+                           })
+    end
+  end
+end

data/lib/exekutor.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "exekutor/version"
+
+module Exekutor
+
+  # Base error class
+  class Error < StandardError; end
+
+  # Error that can be raised during job execution causing the job to be discarded
+  class DiscardJob < StandardError; end
+end
+
+require_relative "exekutor/queue"
+
+require_relative "exekutor/plugins"
+require_relative "exekutor/configuration"
+require_relative "exekutor/job_options"
+
+require_relative "exekutor/internal/database_connection"
+require_relative "exekutor/internal/logger"
+
+require_relative "exekutor/internal/executor"
+require_relative "exekutor/internal/reserver"
+require_relative "exekutor/internal/provider"
+require_relative "exekutor/internal/listener"
+require_relative "exekutor/internal/hooks"
+
+require_relative "exekutor/asynchronous"
+require_relative "exekutor/cleanup"
+require_relative "exekutor/internal/status_server"
+require_relative "exekutor/hook"
+require_relative "exekutor/worker"
+
+ActiveSupport.on_load(:active_job) do
+  require_relative "active_job/queue_adapters/exekutor_adapter"
+end
+
+ActiveSupport.on_load(:active_record) do
+  # Wait until the Rails app is initialized so Exekutor.config.base_record_class can be set.
+  ActiveSupport.on_load(:after_initialize) do
+    require_relative "exekutor/info/worker"
+    require_relative "exekutor/job"
+    require_relative "exekutor/job_error"
+  end
+end
+
+Exekutor.private_constant "Internal"
+ActiveSupport.run_load_hooks(:exekutor, Exekutor)

data/lib/generators/exekutor/configuration_generator.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+require 'rails/generators'
+
+module Exekutor
+  class ConfigurationGenerator < Rails::Generators::Base
+    desc 'Create YAML configuration for Exekutor'
+
+    class_option :identifier, type: :string, aliases: %i(--id), desc: "The worker identifier"
+
+    def create_configuration_file
+      config = { queues: %w[queues to watch] }.merge(Exekutor.config.worker_options)
+      config[:status_port] = 8765
+      config[:set_db_connection_name] = true
+      config[:wait_for_termination] = 120
+      create_file "config/exekutor#{".#{options[:identifier]}" if options[:identifier]}.yml", { "exekutor" => config.stringify_keys }.to_yaml
+    end
+  end
+end

data/lib/generators/exekutor/install_generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module Exekutor
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+    desc 'Create migrations for Exekutor'
+
+    TEMPLATE_DIR = File.join(__dir__, 'templates/install')
+    source_paths << TEMPLATE_DIR
+
+    def create_initializer_file
+      template 'initializers/exekutor.rb.erb', 'config/initializers/exekutor.rb'
+    end
+
+    def create_migration_file
+      migration_template 'migrations/create_exekutor_schema.rb.erb', File.join(db_migrate_path, 'create_exekutor_schema.rb')
+      if defined? Fx
+        %w(job_notifier requeue_orphaned_jobs).each do |function|
+          copy_file "functions/#{function}.sql", Fx::Definition.new(name: function, version: 1).full_path
+        end
+        %w(notify_workers requeue_orphaned_jobs).each do |trigger|
+          copy_file "triggers/#{trigger}.sql", Fx::Definition.new(name: trigger, version: 1, type: "trigger").full_path
+        end
+      end
+    end
+
+    protected
+
+    def migration_version
+      ActiveRecord::VERSION::STRING.to_f
+    end
+
+    def function_sql(name)
+      File.read File.join(TEMPLATE_DIR, "functions/#{name}.sql")
+    end
+
+    def trigger_sql(name)
+      File.read File.join(TEMPLATE_DIR, "triggers/#{name}.sql")
+    end
+  end
+end

data/lib/generators/exekutor/templates/install/functions/job_notifier.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE FUNCTION exekutor_job_notifier() RETURNS TRIGGER AS $$
+  BEGIN
+          PERFORM pg_notify('exekutor::job_enqueued',
+                            CONCAT('id:', NEW.id,';q:', NEW.queue,';t:', extract ('epoch' from NEW.scheduled_at)));
+  RETURN NULL;
+  END;
+      $$ LANGUAGE plpgsql

data/lib/generators/exekutor/templates/install/functions/requeue_orphaned_jobs.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE FUNCTION requeue_orphaned_jobs() RETURNS TRIGGER AS $$ BEGIN
+  UPDATE exekutor_jobs
+  SET status = 'p'
+  WHERE worker_id = OLD.id
+    AND status = 'e';
+  RETURN OLD; END;
+      $$ LANGUAGE plpgsql

data/lib/generators/exekutor/templates/install/initializers/exekutor.rb.erb

@@ -0,0 +1,14 @@
+Exekutor.configure do
+  # Check the README for all configuration options
+
+  # config.logger = Logger.new("log/exekutor.log")
+  <%= default_config = Exekutor::Configuration.new
+      Exekutor::Configuration.__option_names
+                             .without(:logger, :base_record_class_name, :json_serializer, :polling_jitter,
+                                      :max_execution_thread_idletime, :status_server_handler)
+                             .select {|name| default_config.respond_to?(name) }
+                             .map {|name|  [name, default_config.send(name)] }
+                             .filter {|(_name, value)| value.present?}
+                             .map {|(name, value)| "# config.#{name} = #{ value.inspect}" }
+                             .join("\n  ") %>
+end

data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+class CreateExekutorSchema < ActiveRecord::Migration[<%= migration_version %>]
+  def change
+    create_table :exekutor_workers, id: :uuid do |t|
+      t.string :hostname, null: false, limit: 255
+      t.integer :pid, null: false
+
+      t.jsonb :info, null: false
+
+      t.datetime :created_at, null: false, default: -> { 'now()' }
+      t.datetime :last_heartbeat_at, null: false, default: -> { 'now()' }
+
+      t.column :status, :char, null: false, default: 'i'
+
+      t.index [:hostname, :pid], unique: true
+    end
+
+    create_table :exekutor_jobs, id: :uuid do |t|
+      # Worker options
+      t.string :queue, null: false, default: 'default', limit: 200, index: true
+      t.integer :priority, null: false, default: 16383, limit: 2
+      t.datetime :enqueued_at, null: false, default: -> { 'now()' }
+      t.datetime :scheduled_at, null: false, default: -> { 'now()' }
+
+      # Job options
+      t.uuid :active_job_id, null: false, index: true
+      t.jsonb :payload, null: false
+      t.jsonb :options
+
+      # Execution options
+      t.column :status, :char, index: true, null: false, default: 'p'
+      t.float :runtime
+      t.references :worker, type: :uuid, foreign_key: { to_table: :exekutor_workers, on_delete: :nullify }
+
+      t.index [:priority, :scheduled_at, :enqueued_at], where: %Q{"status"='p'}, name: :index_exekutor_jobs_on_dequeue_order
+    end
+
+    create_table :exekutor_job_errors, id: :uuid do |t|
+      t.references :job, type: :uuid, null: false, foreign_key: { to_table: :exekutor_jobs, on_delete: :cascade }
+      t.datetime :created_at, null: false, default: -> { 'now()' }
+      t.jsonb :error, null: false
+    end
+      <% if defined? Fx %>
+    create_function :job_notifier
+    create_trigger :notify_workers
+
+    create_function :requeue_orphaned_jobs
+    create_trigger :requeue_orphaned_jobs
+      <% else %>
+    reversible do |direction|
+      direction.up do
+        execute <<~SQL
+          <%= function_sql "job_notifier" %>
+        SQL
+        execute <<~SQL
+          <%= trigger_sql "notify_workers" %>
+        SQL
+
+        execute <<~SQL
+          <%= function_sql "requeue_orphaned_jobs" %>
+        SQL
+        execute <<~SQL
+          <%= trigger_sql "requeue_orphaned_jobs" %>
+        SQL
+      end
+      direction.down do
+        execute <<~SQL
+          DROP TRIGGER requeue_orphaned_jobs ON exekutor_workers
+        SQL
+        execute <<~SQL
+          DROP FUNCTION requeue_orphaned_jobs
+        SQL
+        execute <<~SQL
+          DROP TRIGGER notify_exekutor_workers ON exekutor_jobs
+        SQL
+        execute <<~SQL
+          DROP FUNCTION exekutor_job_notifier
+        SQL
+      end
+    end
+    <% end %>
+  end
+end

data/lib/generators/exekutor/templates/install/triggers/notify_workers.sql

@@ -0,0 +1,6 @@
+CREATE TRIGGER notify_exekutor_workers
+  AFTER INSERT OR UPDATE OF queue, scheduled_at, status
+                  ON exekutor_jobs
+                    FOR EACH ROW
+                    WHEN (NEW.status = 'p')
+                    EXECUTE FUNCTION exekutor_job_notifier()

data/lib/generators/exekutor/templates/install/triggers/requeue_orphaned_jobs.sql

@@ -0,0 +1,5 @@
+CREATE TRIGGER requeue_orphaned_jobs
+  BEFORE DELETE
+  ON exekutor_workers
+  FOR EACH ROW
+  EXECUTE FUNCTION requeue_orphaned_jobs()