solid_queue 0.1.1

data/lib/solid_queue/app_executor.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module AppExecutor
+    def wrap_in_app_executor(&block)
+      if SolidQueue.app_executor
+        SolidQueue.app_executor.wrap(&block)
+      else
+        yield
+      end
+    end
+
+    def handle_thread_error(error)
+      SolidQueue.logger.error("[SolidQueue] #{error}")
+
+      if SolidQueue.on_thread_error
+        SolidQueue.on_thread_error.call(error)
+      end
+    end
+  end
+end

data/lib/solid_queue/configuration.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Configuration
+    WORKER_DEFAULTS = {
+      queues: "*",
+      threads: 5,
+      processes: 1,
+      polling_interval: 0.1
+    }
+
+    DISPATCHER_DEFAULTS = {
+      batch_size: 500,
+      polling_interval: 1,
+      concurrency_maintenance_interval: 600
+    }
+
+    def initialize(mode: :work, load_from: nil)
+      @mode = mode
+      @raw_config = config_from(load_from)
+    end
+
+    def processes
+      case mode
+      when :dispatch then dispatchers
+      when :work     then workers
+      when :all      then dispatchers + workers
+      else           raise "Invalid mode #{mode}"
+      end
+    end
+
+    def workers
+      if mode.in? %i[ work all]
+        workers_options.flat_map do |worker_options|
+          processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          processes.times.collect { SolidQueue::Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
+        end
+      else
+        []
+      end
+    end
+
+    def dispatchers
+      if mode.in? %i[ dispatch all]
+        dispatchers_options.flat_map do |dispatcher_options|
+          SolidQueue::Dispatcher.new(**dispatcher_options)
+        end
+      end
+    end
+
+    def max_number_of_threads
+      # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
+      workers_options.map { |options| options[:threads] }.max + 2
+    end
+
+    private
+      attr_reader :raw_config, :mode
+
+      DEFAULT_CONFIG_FILE_PATH = "config/solid_queue.yml"
+
+      def config_from(file_or_hash, env: Rails.env)
+        config = load_config_from(file_or_hash)
+        config[env.to_sym] ? config[env.to_sym] : config
+      end
+
+      def workers_options
+        @workers_options ||= (raw_config[:workers] || [ WORKER_DEFAULTS ])
+          .map { |options| options.dup.symbolize_keys }
+      end
+
+      def dispatchers_options
+        @dispatchers_options ||= (raw_config[:dispatchers] || [ DISPATCHER_DEFAULTS ])
+          .map { |options| options.dup.symbolize_keys }
+      end
+
+      def load_config_from(file_or_hash)
+        case file_or_hash
+        when Pathname then load_config_file file_or_hash
+        when String   then load_config_file Pathname.new(file_or_hash)
+        when NilClass then load_config_file default_config_file
+        when Hash     then file_or_hash.dup
+        else          raise "Solid Queue cannot be initialized with #{file_or_hash.inspect}"
+        end
+      end
+
+      def load_config_file(file)
+        if file.exist?
+          ActiveSupport::ConfigurationFile.parse(file).deep_symbolize_keys
+        else
+          raise "Configuration file not found in #{file}"
+        end
+      end
+
+      def default_config_file
+        path_to_file = ENV["SOLID_QUEUE_CONFIG"] || DEFAULT_CONFIG_FILE_PATH
+
+        Rails.root.join(path_to_file).tap do |config_file|
+          raise "Configuration for Solid Queue not found in #{config_file}" unless config_file.exist?
+        end
+      end
+  end
+end

data/lib/solid_queue/dispatcher.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Dispatcher < Processes::Base
+    include Processes::Runnable, Processes::Poller
+
+    attr_accessor :batch_size, :concurrency_maintenance_interval
+
+    set_callback :boot, :after, :launch_concurrency_maintenance
+    set_callback :shutdown, :before, :stop_concurrency_maintenance
+
+    def initialize(**options)
+      options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
+
+      @batch_size = options[:batch_size]
+      @polling_interval = options[:polling_interval]
+      @concurrency_maintenance_interval = options[:concurrency_maintenance_interval]
+    end
+
+    private
+      def run
+        batch = dispatch_next_batch
+
+        unless batch.size > 0
+          procline "waiting"
+          interruptible_sleep(polling_interval)
+        end
+      end
+
+      def dispatch_next_batch
+        with_polling_volume do
+          SolidQueue::ScheduledExecution.dispatch_next_batch(batch_size)
+        end
+      end
+
+      def launch_concurrency_maintenance
+        @concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: concurrency_maintenance_interval) do
+          expire_semaphores
+          unblock_blocked_executions
+        end
+
+        @concurrency_maintenance_task.add_observer do |_, _, error|
+          handle_thread_error(error) if error
+        end
+
+        @concurrency_maintenance_task.execute
+      end
+
+      def stop_concurrency_maintenance
+        @concurrency_maintenance_task.shutdown
+      end
+
+      def expire_semaphores
+        wrap_in_app_executor do
+          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
+        end
+      end
+
+      def unblock_blocked_executions
+        wrap_in_app_executor do
+          BlockedExecution.unblock(batch_size)
+        end
+      end
+
+      def initial_jitter
+        Kernel.rand(0...polling_interval)
+      end
+
+      def metadata
+        super.merge(batch_size: batch_size)
+      end
+  end
+end

data/lib/solid_queue/engine.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Engine < ::Rails::Engine
+    isolate_namespace SolidQueue
+
+    rake_tasks do
+      load "solid_queue/tasks.rb"
+    end
+
+    config.solid_queue = ActiveSupport::OrderedOptions.new
+
+    initializer "solid_queue.config" do
+      config.solid_queue.each do |name, value|
+        SolidQueue.public_send("#{name}=", value)
+      end
+    end
+
+    initializer "solid_queue.app_executor", before: :run_prepare_callbacks do |app|
+      config.solid_queue.app_executor    ||= app.executor
+      config.solid_queue.on_thread_error ||= -> (exception) { Rails.error.report(exception, handled: false) }
+
+      SolidQueue.app_executor = config.solid_queue.app_executor
+      SolidQueue.on_thread_error = config.solid_queue.on_thread_error
+    end
+
+    initializer "solid_queue.logger" do |app|
+      ActiveSupport.on_load(:solid_queue) do
+        self.logger = app.logger
+      end
+    end
+
+    initializer "solid_queue.active_job.extensions" do
+      ActiveSupport.on_load :active_job do
+        include ActiveJob::ConcurrencyControls
+      end
+    end
+  end
+end

data/lib/solid_queue/pool.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Pool
+    include AppExecutor
+
+    attr_reader :size
+
+    delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor
+
+    def initialize(size, on_idle: nil)
+      @size = size
+      @on_idle = on_idle
+      @available_threads = Concurrent::AtomicFixnum.new(size)
+      @mutex = Mutex.new
+    end
+
+    def post(execution)
+      available_threads.decrement
+
+      future = Concurrent::Future.new(args: [ execution ], executor: executor) do |thread_execution|
+        wrap_in_app_executor do
+          thread_execution.perform
+        ensure
+          available_threads.increment
+          mutex.synchronize { on_idle.try(:call) if idle? }
+        end
+      end
+
+      future.add_observer do |_, _, error|
+        handle_thread_error(error) if error
+      end
+
+      future.execute
+    end
+
+    def idle_threads
+      available_threads.value
+    end
+
+    def idle?
+      idle_threads > 0
+    end
+
+    private
+      attr_reader :available_threads, :on_idle, :mutex
+
+      DEFAULT_OPTIONS = {
+        min_threads: 0,
+        idletime: 60,
+        fallback_policy: :abort
+      }
+
+      def executor
+        @executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size)
+      end
+  end
+end

data/lib/solid_queue/processes/base.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Processes
+    class Base
+      include ActiveSupport::Callbacks
+      define_callbacks :boot, :shutdown
+
+      include AppExecutor, Registrable, Interruptible, Procline
+
+      private
+        def observe_initial_delay
+          interruptible_sleep(initial_jitter)
+        end
+
+        def boot
+        end
+
+        def shutdown
+        end
+
+        def initial_jitter
+          0
+        end
+    end
+  end
+end

data/lib/solid_queue/processes/interruptible.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Interruptible
+    def wake_up
+      interrupt
+    end
+
+    private
+      SELF_PIPE_BLOCK_SIZE = 11
+
+      def interrupt
+        self_pipe[:writer].write_nonblock( "." )
+      rescue Errno::EAGAIN, Errno::EINTR
+        # Ignore writes that would block and retry
+        # if another signal arrived while writing
+        retry
+      end
+
+      def interruptible_sleep(time)
+        if time > 0 && self_pipe[:reader].wait_readable(time)
+          loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) }
+        end
+      rescue Errno::EAGAIN, Errno::EINTR
+      end
+
+      # Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html)
+      def self_pipe
+        @self_pipe ||= create_self_pipe
+      end
+
+      def create_self_pipe
+        reader, writer = IO.pipe
+        { reader: reader, writer: writer }
+      end
+  end
+end

data/lib/solid_queue/processes/pidfile.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  class Pidfile
+    def initialize(path)
+      @path = path
+      @pid = ::Process.pid
+    end
+
+    def setup
+      check_status
+      write_file
+      set_at_exit_hook
+    end
+
+    def delete
+      delete_file
+    end
+
+    private
+      attr_reader :path, :pid
+
+      def check_status
+        if ::File.exist?(path)
+          existing_pid = ::File.open(path).read.strip.to_i
+          existing_pid > 0 && ::Process.kill(0, existing_pid)
+
+          already_running!
+        else
+          FileUtils.mkdir_p File.dirname(path)
+        end
+      rescue Errno::ESRCH => e
+        # Process is dead, ignore, just delete the file
+        delete
+      rescue Errno::EPERM
+        already_running!
+      end
+
+      def write_file
+        ::File.open(path, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |file| file.write(pid.to_s) }
+      rescue Errno::EEXIST
+        check_status
+        retry
+      end
+
+      def set_at_exit_hook
+        at_exit { delete if ::Process.pid == pid }
+      end
+
+      def delete_file
+        ::File.delete(path) if ::File.exist?(path)
+      end
+
+      def already_running!
+        abort "A Solid Queue supervisor is already running. Check #{path}"
+      end
+  end
+end

data/lib/solid_queue/processes/poller.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Poller
+    extend ActiveSupport::Concern
+
+    included do
+      attr_accessor :polling_interval
+    end
+
+    private
+      def with_polling_volume
+        if SolidQueue.silence_polling?
+          ActiveRecord::Base.logger.silence { yield }
+        else
+          yield
+        end
+      end
+
+      def metadata
+        super.merge(polling_interval: polling_interval)
+      end
+  end
+end

data/lib/solid_queue/processes/procline.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Procline
+    # Sets the procline ($0)
+    # solid-queue-supervisor(0.1.0): <string>
+    def procline(string)
+      $0 = "solid-queue-#{self.class.name.demodulize.downcase}(#{SolidQueue::VERSION}): #{string}"
+    end
+  end
+end

data/lib/solid_queue/processes/registrable.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Registrable
+    extend ActiveSupport::Concern
+
+    included do
+      set_callback :boot, :after, :register
+      set_callback :boot, :after, :launch_heartbeat
+
+      set_callback :shutdown, :before, :stop_heartbeat
+      set_callback :shutdown, :after, :deregister
+    end
+
+    def inspect
+      "#{kind}(pid=#{process_pid}, hostname=#{hostname}, metadata=#{metadata})"
+    end
+    alias to_s inspect
+
+    private
+      attr_accessor :process
+
+      def register
+        @process = SolidQueue::Process.register \
+          kind: self.class.name.demodulize,
+          pid: process_pid,
+          hostname: hostname,
+          supervisor: try(:supervisor),
+          metadata: metadata
+      end
+
+      def deregister
+        process.deregister if registered?
+      end
+
+      def registered?
+        process&.persisted?
+      end
+
+      def launch_heartbeat
+        @heartbeat_task = Concurrent::TimerTask.new(execution_interval: SolidQueue.process_heartbeat_interval) { heartbeat }
+        @heartbeat_task.execute
+      end
+
+      def stop_heartbeat
+        @heartbeat_task&.shutdown
+      end
+
+      def heartbeat
+        process.heartbeat
+      end
+
+      def kind
+        self.class.name.demodulize
+      end
+
+      def hostname
+        @hostname ||= Socket.gethostname
+      end
+
+      def process_pid
+        @pid ||= ::Process.pid
+      end
+
+      def metadata
+        {}
+      end
+  end
+end

data/lib/solid_queue/processes/runnable.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Runnable
+    include Supervised
+
+    def start
+      @stopping = false
+
+      observe_initial_delay
+      run_callbacks(:boot) { boot }
+
+      start_loop
+    end
+
+    def stop
+      @stopping = true
+      @thread&.join
+    end
+
+  private
+    attr_writer :mode
+
+    DEFAULT_MODE = :async
+
+    def mode
+      (@mode || DEFAULT_MODE).to_s.inquiry
+    end
+
+    def boot
+      register_signal_handlers if supervised?
+      SolidQueue.logger.info("[SolidQueue] Starting #{self}")
+    end
+
+    def start_loop
+      if mode.async?
+        @thread = Thread.new { do_start_loop }
+      else
+        do_start_loop
+      end
+    end
+
+    def do_start_loop
+      loop do
+        break if shutting_down?
+
+        run
+      end
+    ensure
+      run_callbacks(:shutdown) { shutdown }
+    end
+
+    def shutting_down?
+      stopping? || supervisor_went_away? || finished?
+    end
+
+    def run
+      raise NotImplementedError
+    end
+
+    def stopping?
+      @stopping
+    end
+
+    def finished?
+      running_inline? && all_work_completed?
+    end
+
+    def all_work_completed?
+      false
+    end
+
+    def running_inline?
+      mode.inline?
+    end
+  end
+end

data/lib/solid_queue/processes/signals.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  class GracefulTerminationRequested < Interrupt; end
+  class ImmediateTerminationRequested < Interrupt; end
+
+  module Signals
+    extend ActiveSupport::Concern
+
+    private
+      SIGNALS = %i[ QUIT INT TERM ]
+
+      def register_signal_handlers
+        SIGNALS.each do |signal|
+          trap(signal) do
+            signal_queue << signal
+            interrupt
+          end
+        end
+      end
+
+      def restore_default_signal_handlers
+        SIGNALS.each do |signal|
+          trap(signal, :DEFAULT)
+        end
+      end
+
+      def process_signal_queue
+        while signal = signal_queue.shift
+          handle_signal(signal)
+        end
+      end
+
+      def handle_signal(signal)
+        case signal
+        when :TERM, :INT
+          request_graceful_termination
+        when :QUIT
+          request_immediate_termination
+        else
+          SolidQueue.logger.warn "Received unhandled signal #{signal}"
+        end
+      end
+
+      def request_graceful_termination
+        raise GracefulTerminationRequested
+      end
+
+      def request_immediate_termination
+        raise ImmediateTerminationRequested
+      end
+
+      def signal_processes(pids, signal)
+        pids.each do |pid|
+          signal_process pid, signal
+        end
+      end
+
+      def signal_process(pid, signal)
+        ::Process.kill signal, pid
+      rescue Errno::ESRCH
+        # Ignore, process died before
+      end
+
+      def signal_queue
+        @signal_queue ||= []
+      end
+  end
+end