solid_queue 0.3.3 → 0.4.0

data/lib/solid_queue/dispatcher.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
 module SolidQueue
-  class Dispatcher < Processes::Base
-    include Processes::Poller
-
+  class Dispatcher < Processes::Poller
     attr_accessor :batch_size, :concurrency_maintenance, :recurring_schedule
 
     after_boot :start_concurrency_maintenance, :load_recurring_schedule
@@ -13,10 +11,11 @@ module SolidQueue
       options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
 
       @batch_size = options[:batch_size]
-      @polling_interval = options[:polling_interval]
 
       @concurrency_maintenance = ConcurrencyMaintenance.new(options[:concurrency_maintenance_interval], options[:batch_size]) if options[:concurrency_maintenance]
       @recurring_schedule = RecurringSchedule.new(options[:recurring_tasks])
+
+      super(**options)
     end
 
     def metadata
@@ -51,6 +50,10 @@ module SolidQueue
         recurring_schedule.unload_tasks
       end
 
+      def all_work_completed?
+        SolidQueue::ScheduledExecution.none? && recurring_schedule.empty?
+      end
+
       def set_procline
         procline "waiting"
       end

data/lib/solid_queue/log_subscriber.rb

@@ -7,6 +7,10 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     debug formatted_event(event, action: "Dispatch scheduled jobs", **event.payload.slice(:batch_size, :size))
   end
 
+  def claim(event)
+    debug formatted_event(event, action: "Claim jobs", **event.payload.slice(:process_id, :job_ids, :claimed_job_ids, :size))
+  end
+
   def release_many_claimed(event)
     debug formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
   end
@@ -40,13 +44,16 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def enqueue_recurring_task(event)
-    attributes = event.payload.slice(:task, :at, :active_job_id)
+    attributes = event.payload.slice(:task, :active_job_id, :enqueue_error)
+    attributes[:at] = event.payload[:at]&.iso8601
 
-    if event.payload[:other_adapter]
+    if attributes[:active_job_id].nil? && event.payload[:skipped].nil?
+      error formatted_event(event, action: "Error enqueuing recurring task", **attributes)
+    elsif event.payload[:other_adapter]
       debug formatted_event(event, action: "Enqueued recurring task outside Solid Queue", **attributes)
     else
-      action = attributes[:active_job_id].present? ? "Enqueued recurring task" : "Skipped recurring task – already dispatched"
-      info formatted_event(event, action: action, **attributes)
+      action = event.payload[:skipped].present? ? "Skipped recurring task – already dispatched" : "Enqueued recurring task"
+      debug formatted_event(event, action: action, **attributes)
     end
   end
 
@@ -55,7 +62,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
 
     attributes = {
       pid: process.pid,
-      hostname: process.hostname
+      hostname: process.hostname,
+      process_id: process.process_id
     }.merge(process.metadata)
 
     info formatted_event(event, action: "Started #{process.kind}", **attributes)
@@ -66,15 +74,16 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
 
     attributes = {
       pid: process.pid,
-      hostname: process.hostname
+      hostname: process.hostname,
+      process_id: process.process_id
     }.merge(process.metadata)
 
-    info formatted_event(event, action: "Shut down #{process.kind}", **attributes)
+    info formatted_event(event, action: "Shutdown #{process.kind}", **attributes)
   end
 
   def register_process(event)
     process_kind = event.payload[:kind]
-    attributes = event.payload.slice(:pid, :hostname)
+    attributes = event.payload.slice(:pid, :hostname, :process_id)
 
     if error = event.payload[:error]
       warn formatted_event(event, action: "Error registering #{process_kind}", **attributes.merge(error: formatted_error(error)))
@@ -90,9 +99,9 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
       process_id: process.id,
       pid: process.pid,
       hostname: process.hostname,
-      last_heartbeat_at: process.last_heartbeat_at,
-      claimed_size: process.claimed_executions.size,
-      pruned: event.payload
+      last_heartbeat_at: process.last_heartbeat_at.iso8601,
+      claimed_size: event.payload[:claimed_size],
+      pruned: event.payload[:pruned]
     }
 
     if error = event.payload[:error]
@@ -111,7 +120,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def graceful_termination(event)
-    attributes = event.payload.slice(:supervisor_pid, :supervised_pids)
+    attributes = event.payload.slice(:process_id, :supervisor_pid, :supervised_processes)
 
     if event.payload[:shutdown_timeout_exceeded]
       warn formatted_event(event, action: "Supervisor wasn't terminated gracefully - shutdown timeout exceeded", **attributes)
@@ -121,7 +130,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def immediate_termination(event)
-    info formatted_event(event, action: "Supervisor terminated immediately", **event.payload.slice(:supervisor_pid, :supervised_pids))
+    info formatted_event(event, action: "Supervisor terminated immediately", **event.payload.slice(:process_id, :supervisor_pid, :supervised_processes))
   end
 
   def unhandled_signal_error(event)

data/lib/solid_queue/processes/callbacks.rb

@@ -8,12 +8,5 @@ module SolidQueue::Processes
       extend ActiveModel::Callbacks
       define_model_callbacks :boot, :shutdown
     end
-
-    private
-      def boot
-      end
-
-      def shutdown
-      end
   end
 end

data/lib/solid_queue/processes/poller.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 module SolidQueue::Processes
-  module Poller
-    extend ActiveSupport::Concern
-
+  class Poller < Base
     include Runnable
 
-    included do
-      attr_accessor :polling_interval
+    attr_accessor :polling_interval
+
+    def initialize(polling_interval:, **options)
+      @polling_interval = polling_interval
     end
 
     def metadata
@@ -16,11 +16,7 @@ module SolidQueue::Processes
 
     private
       def run
-        if mode.async?
-          @thread = Thread.new { start_loop }
-        else
-          start_loop
-        end
+        start_loop
       end
 
       def start_loop
@@ -43,8 +39,11 @@ module SolidQueue::Processes
         raise NotImplementedError
       end
 
+      def shutdown
+      end
+
       def with_polling_volume
-        if SolidQueue.silence_polling?
+        if SolidQueue.silence_polling? && ActiveRecord::Base.logger
           ActiveRecord::Base.logger.silence { yield }
         else
           yield

data/lib/solid_queue/processes/procline.rb

@@ -5,7 +5,7 @@ module SolidQueue::Processes
     # Sets the procline ($0)
     # solid-queue-supervisor(0.1.0): <string>
     def procline(string)
-      $0 = "solid-queue-#{self.class.name.demodulize.downcase}(#{SolidQueue::VERSION}): #{string}"
+      $0 = "solid-queue-#{self.class.name.demodulize.underscore.dasherize}(#{SolidQueue::VERSION}): #{string}"
     end
   end
 end

data/lib/solid_queue/processes/registrable.rb

@@ -11,12 +11,16 @@ module SolidQueue::Processes
       after_shutdown :deregister
     end
 
+    def process_id
+      process&.id
+    end
+
     private
       attr_accessor :process
 
       def register
         @process = SolidQueue::Process.register \
-          kind: self.class.name.demodulize,
+          kind: kind,
           pid: pid,
           hostname: hostname,
           supervisor: try(:supervisor),
@@ -36,6 +40,10 @@ module SolidQueue::Processes
           wrap_in_app_executor { heartbeat }
         end
 
+        @heartbeat_task.add_observer do |_, _, error|
+          handle_thread_error(error) if error
+        end
+
         @heartbeat_task.execute
       end
 

data/lib/solid_queue/processes/runnable.rb

@@ -7,20 +7,32 @@ module SolidQueue::Processes
     attr_writer :mode
 
     def start
-      @stopping = false
+      @stopped = false
 
       SolidQueue.instrument(:start_process, process: self) do
         run_callbacks(:boot) { boot }
       end
 
-      run
+      if running_async?
+        @thread = create_thread { run }
+      else
+        run
+      end
     end
 
     def stop
-      @stopping = true
+      @stopped = true
       @thread&.join
     end
 
+    def name
+      @name ||= [ kind.downcase, SecureRandom.hex(6) ].join("-")
+    end
+
+    def alive?
+      !running_async? || @thread.alive?
+    end
+
     private
       DEFAULT_MODE = :async
 
@@ -29,22 +41,22 @@ module SolidQueue::Processes
       end
 
       def boot
-        if supervised?
+        if running_as_fork?
           register_signal_handlers
           set_procline
         end
       end
 
       def shutting_down?
-        stopping? || supervisor_went_away? || finished?
+        stopped? || (running_as_fork? && supervisor_went_away?) || finished?
       end
 
       def run
         raise NotImplementedError
       end
 
-      def stopping?
-        @stopping
+      def stopped?
+        @stopped
       end
 
       def finished?
@@ -61,5 +73,24 @@ module SolidQueue::Processes
       def running_inline?
         mode.inline?
       end
+
+      def running_async?
+        mode.async?
+      end
+
+      def running_as_fork?
+        mode.fork?
+      end
+
+
+      def create_thread(&block)
+        Thread.new do
+          Thread.current.name = name
+          block.call
+        rescue Exception => exception
+          handle_thread_error(exception)
+          raise
+        end
+      end
   end
 end

data/lib/solid_queue/processes/supervised.rb

@@ -9,7 +9,6 @@ module SolidQueue::Processes
     end
 
     def supervised_by(process)
-      self.mode = :supervised
       @supervisor = process
     end
 
@@ -19,11 +18,11 @@ module SolidQueue::Processes
       end
 
       def supervisor_went_away?
-        supervised? && supervisor&.pid != ::Process.ppid
+        supervised? && supervisor.pid != ::Process.ppid
       end
 
       def supervised?
-        mode.supervised?
+        supervisor.present?
       end
 
       def register_signal_handlers

data/lib/solid_queue/supervisor/async_supervisor.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor::AsyncSupervisor < Supervisor
+    def initialize(*)
+      super
+      @threads = Concurrent::Map.new
+    end
+
+    def kind
+      "Supervisor(async)"
+    end
+
+    def stop
+      super
+      stop_threads
+      threads.clear
+
+      shutdown
+    end
+
+    private
+      attr_reader :threads
+
+      def start_process(configured_process)
+        configured_process.supervised_by process
+        configured_process.start
+
+        threads[configured_process.name] = configured_process
+      end
+
+      def stop_threads
+        stop_threads = threads.values.map do |thr|
+          Thread.new { thr.stop }
+        end
+
+        stop_threads.each { |thr| thr.join(SolidQueue.shutdown_timeout) }
+      end
+
+      def all_threads_terminated?
+        threads.values.none?(&:alive?)
+      end
+  end
+end

data/lib/solid_queue/supervisor/fork_supervisor.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor::ForkSupervisor < Supervisor
+    include Signals, Pidfiled
+
+    def initialize(*)
+      super
+      @forks = {}
+    end
+
+    def kind
+      "Supervisor(fork)"
+    end
+
+    private
+      attr_reader :forks
+
+      def supervise
+        loop do
+          break if stopped?
+
+          procline "supervising #{forks.keys.join(", ")}"
+          process_signal_queue
+
+          unless stopped?
+            reap_and_replace_terminated_forks
+            interruptible_sleep(1.second)
+          end
+        end
+      ensure
+        shutdown
+      end
+
+      def start_process(configured_process)
+        configured_process.supervised_by process
+        configured_process.mode = :fork
+
+        pid = fork do
+          configured_process.start
+        end
+
+        forks[pid] = configured_process
+      end
+
+      def terminate_gracefully
+        SolidQueue.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do |payload|
+          term_forks
+
+          Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
+            reap_terminated_forks
+          end
+
+          unless all_forks_terminated?
+            payload[:shutdown_timeout_exceeded] = true
+            terminate_immediately
+          end
+        end
+      end
+
+      def terminate_immediately
+        SolidQueue.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do
+          quit_forks
+        end
+      end
+
+      def term_forks
+        signal_processes(forks.keys, :TERM)
+      end
+
+      def quit_forks
+        signal_processes(forks.keys, :QUIT)
+      end
+
+      def reap_and_replace_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          replace_fork(pid, status)
+        end
+      end
+
+      def reap_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          forks.delete(pid)
+        end
+      rescue SystemCallError
+        # All children already reaped
+      end
+
+      def replace_fork(pid, status)
+        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
+          if supervised_fork = forks.delete(pid)
+            payload[:fork] = supervised_fork
+            start_process(supervised_fork)
+          end
+        end
+      end
+
+      def all_forks_terminated?
+        forks.empty?
+      end
+  end
+end

data/lib/solid_queue/supervisor/maintenance.rb

@@ -0,0 +1,34 @@
+module SolidQueue
+  module Supervisor::Maintenance
+    extend ActiveSupport::Concern
+
+    included do
+      after_boot :release_orphaned_executions
+    end
+
+    private
+      def launch_maintenance_task
+        @maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) do
+          prune_dead_processes
+        end
+
+        @maintenance_task.add_observer do |_, _, error|
+          handle_thread_error(error) if error
+        end
+
+        @maintenance_task.execute
+      end
+
+      def stop_maintenance_task
+        @maintenance_task&.shutdown
+      end
+
+      def prune_dead_processes
+        wrap_in_app_executor { SolidQueue::Process.prune }
+      end
+
+      def release_orphaned_executions
+        wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
+      end
+  end
+end

data/lib/solid_queue/{processes → supervisor}/pidfile.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-module SolidQueue::Processes
-  class Pidfile
+module SolidQueue
+  class Supervisor::Pidfile
     def initialize(path)
       @path = path
       @pid = ::Process.pid

data/lib/solid_queue/supervisor/pidfiled.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor
+    module Pidfiled
+      extend ActiveSupport::Concern
+
+      included do
+        before_boot :setup_pidfile
+        after_shutdown :delete_pidfile
+      end
+
+      private
+        def setup_pidfile
+          if path = SolidQueue.supervisor_pidfile
+            @pidfile = Pidfile.new(path).tap(&:setup)
+          end
+        end
+
+        def delete_pidfile
+          @pidfile&.delete
+        end
+    end
+  end
+end

data/lib/solid_queue/supervisor/signals.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor
+    module Signals
+      extend ActiveSupport::Concern
+
+      included do
+        before_boot :register_signal_handlers
+        after_shutdown :restore_default_signal_handlers
+      end
+
+      private
+        SIGNALS = %i[ QUIT INT TERM ]
+
+        def register_signal_handlers
+          SIGNALS.each do |signal|
+            trap(signal) do
+              signal_queue << signal
+              interrupt
+            end
+          end
+        end
+
+        def restore_default_signal_handlers
+          SIGNALS.each do |signal|
+            trap(signal, :DEFAULT)
+          end
+        end
+
+        def process_signal_queue
+          while signal = signal_queue.shift
+            handle_signal(signal)
+          end
+        end
+
+        def handle_signal(signal)
+          case signal
+          when :TERM, :INT
+            stop
+            terminate_gracefully
+          when :QUIT
+            stop
+            terminate_immediately
+          else
+            SolidQueue.instrument :unhandled_signal_error, signal: signal
+          end
+        end
+
+        def signal_processes(pids, signal)
+          pids.each do |pid|
+            signal_process pid, signal
+          end
+        end
+
+        def signal_process(pid, signal)
+          ::Process.kill signal, pid
+        rescue Errno::ESRCH
+          # Ignore, process died before
+        end
+
+        def signal_queue
+          @signal_queue ||= []
+        end
+    end
+  end
+end