solid_queue 0.3.4 → 0.4.0

data/lib/solid_queue/processes/supervised.rb

@@ -9,7 +9,6 @@ module SolidQueue::Processes
     end
 
     def supervised_by(process)
-      self.mode = :supervised
       @supervisor = process
     end
 
@@ -19,11 +18,11 @@ module SolidQueue::Processes
       end
 
       def supervisor_went_away?
-        supervised? && supervisor&.pid != ::Process.ppid
+        supervised? && supervisor.pid != ::Process.ppid
       end
 
       def supervised?
-        mode.supervised?
+        supervisor.present?
       end
 
       def register_signal_handlers

data/lib/solid_queue/supervisor/async_supervisor.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor::AsyncSupervisor < Supervisor
+    def initialize(*)
+      super
+      @threads = Concurrent::Map.new
+    end
+
+    def kind
+      "Supervisor(async)"
+    end
+
+    def stop
+      super
+      stop_threads
+      threads.clear
+
+      shutdown
+    end
+
+    private
+      attr_reader :threads
+
+      def start_process(configured_process)
+        configured_process.supervised_by process
+        configured_process.start
+
+        threads[configured_process.name] = configured_process
+      end
+
+      def stop_threads
+        stop_threads = threads.values.map do |thr|
+          Thread.new { thr.stop }
+        end
+
+        stop_threads.each { |thr| thr.join(SolidQueue.shutdown_timeout) }
+      end
+
+      def all_threads_terminated?
+        threads.values.none?(&:alive?)
+      end
+  end
+end

data/lib/solid_queue/supervisor/fork_supervisor.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor::ForkSupervisor < Supervisor
+    include Signals, Pidfiled
+
+    def initialize(*)
+      super
+      @forks = {}
+    end
+
+    def kind
+      "Supervisor(fork)"
+    end
+
+    private
+      attr_reader :forks
+
+      def supervise
+        loop do
+          break if stopped?
+
+          procline "supervising #{forks.keys.join(", ")}"
+          process_signal_queue
+
+          unless stopped?
+            reap_and_replace_terminated_forks
+            interruptible_sleep(1.second)
+          end
+        end
+      ensure
+        shutdown
+      end
+
+      def start_process(configured_process)
+        configured_process.supervised_by process
+        configured_process.mode = :fork
+
+        pid = fork do
+          configured_process.start
+        end
+
+        forks[pid] = configured_process
+      end
+
+      def terminate_gracefully
+        SolidQueue.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do |payload|
+          term_forks
+
+          Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
+            reap_terminated_forks
+          end
+
+          unless all_forks_terminated?
+            payload[:shutdown_timeout_exceeded] = true
+            terminate_immediately
+          end
+        end
+      end
+
+      def terminate_immediately
+        SolidQueue.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do
+          quit_forks
+        end
+      end
+
+      def term_forks
+        signal_processes(forks.keys, :TERM)
+      end
+
+      def quit_forks
+        signal_processes(forks.keys, :QUIT)
+      end
+
+      def reap_and_replace_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          replace_fork(pid, status)
+        end
+      end
+
+      def reap_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          forks.delete(pid)
+        end
+      rescue SystemCallError
+        # All children already reaped
+      end
+
+      def replace_fork(pid, status)
+        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
+          if supervised_fork = forks.delete(pid)
+            payload[:fork] = supervised_fork
+            start_process(supervised_fork)
+          end
+        end
+      end
+
+      def all_forks_terminated?
+        forks.empty?
+      end
+  end
+end

data/lib/solid_queue/supervisor/maintenance.rb

@@ -0,0 +1,34 @@
+module SolidQueue
+  module Supervisor::Maintenance
+    extend ActiveSupport::Concern
+
+    included do
+      after_boot :release_orphaned_executions
+    end
+
+    private
+      def launch_maintenance_task
+        @maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) do
+          prune_dead_processes
+        end
+
+        @maintenance_task.add_observer do |_, _, error|
+          handle_thread_error(error) if error
+        end
+
+        @maintenance_task.execute
+      end
+
+      def stop_maintenance_task
+        @maintenance_task&.shutdown
+      end
+
+      def prune_dead_processes
+        wrap_in_app_executor { SolidQueue::Process.prune }
+      end
+
+      def release_orphaned_executions
+        wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
+      end
+  end
+end

data/lib/solid_queue/{processes → supervisor}/pidfile.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-module SolidQueue::Processes
-  class Pidfile
+module SolidQueue
+  class Supervisor::Pidfile
     def initialize(path)
       @path = path
       @pid = ::Process.pid

data/lib/solid_queue/supervisor/pidfiled.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor
+    module Pidfiled
+      extend ActiveSupport::Concern
+
+      included do
+        before_boot :setup_pidfile
+        after_shutdown :delete_pidfile
+      end
+
+      private
+        def setup_pidfile
+          if path = SolidQueue.supervisor_pidfile
+            @pidfile = Pidfile.new(path).tap(&:setup)
+          end
+        end
+
+        def delete_pidfile
+          @pidfile&.delete
+        end
+    end
+  end
+end

data/lib/solid_queue/supervisor/signals.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Supervisor
+    module Signals
+      extend ActiveSupport::Concern
+
+      included do
+        before_boot :register_signal_handlers
+        after_shutdown :restore_default_signal_handlers
+      end
+
+      private
+        SIGNALS = %i[ QUIT INT TERM ]
+
+        def register_signal_handlers
+          SIGNALS.each do |signal|
+            trap(signal) do
+              signal_queue << signal
+              interrupt
+            end
+          end
+        end
+
+        def restore_default_signal_handlers
+          SIGNALS.each do |signal|
+            trap(signal, :DEFAULT)
+          end
+        end
+
+        def process_signal_queue
+          while signal = signal_queue.shift
+            handle_signal(signal)
+          end
+        end
+
+        def handle_signal(signal)
+          case signal
+          when :TERM, :INT
+            stop
+            terminate_gracefully
+          when :QUIT
+            stop
+            terminate_immediately
+          else
+            SolidQueue.instrument :unhandled_signal_error, signal: signal
+          end
+        end
+
+        def signal_processes(pids, signal)
+          pids.each do |pid|
+            signal_process pid, signal
+          end
+        end
+
+        def signal_process(pid, signal)
+          ::Process.kill signal, pid
+        rescue Errno::ESRCH
+          # Ignore, process died before
+        end
+
+        def signal_queue
+          @signal_queue ||= []
+        end
+    end
+  end
+end

data/lib/solid_queue/supervisor.rb

@@ -2,190 +2,72 @@
 
 module SolidQueue
   class Supervisor < Processes::Base
-    include Processes::Signals
+    include Maintenance
 
     class << self
-      def start(mode: :work, load_configuration_from: nil)
+      def start(mode: :fork, load_configuration_from: nil)
         SolidQueue.supervisor = true
         configuration = Configuration.new(mode: mode, load_from: load_configuration_from)
 
-        new(*configuration.processes).start
+        klass = mode == :fork ? ForkSupervisor : AsyncSupervisor
+        klass.new(configuration).tap(&:start)
       end
     end
 
-    def initialize(*configured_processes)
-      @configured_processes = Array(configured_processes)
-      @forks = {}
+    def initialize(configuration)
+      @configuration = configuration
     end
 
     def start
-      run_callbacks(:boot) { boot }
+      boot
 
-      start_forks
+      start_processes
       launch_maintenance_task
 
       supervise
-    rescue Processes::GracefulTerminationRequested
-      graceful_termination
-    rescue Processes::ImmediateTerminationRequested
-      immediate_termination
-    ensure
-      run_callbacks(:shutdown) { shutdown }
+    end
+
+    def stop
+      @stopped = true
     end
 
     private
-      attr_reader :configured_processes, :forks
+      attr_reader :configuration
 
       def boot
-        sync_std_streams
-        setup_pidfile
-        register_signal_handlers
-      end
-
-      def supervise
-        loop do
-          procline "supervising #{forks.keys.join(", ")}"
-
-          process_signal_queue
-          reap_and_replace_terminated_forks
-          interruptible_sleep(1.second)
-        end
-      end
-
-      def sync_std_streams
-        STDOUT.sync = STDERR.sync = true
-      end
-
-      def setup_pidfile
-        @pidfile = if SolidQueue.supervisor_pidfile
-          Processes::Pidfile.new(SolidQueue.supervisor_pidfile).tap(&:setup)
-        end
-      end
-
-      def start_forks
-        configured_processes.each { |configured_process| start_fork(configured_process) }
-      end
-
-      def launch_maintenance_task
-        @maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) do
-          prune_dead_processes
-          release_orphaned_executions
-        end
-        @maintenance_task.execute
-      end
-
-      def shutdown
-        stop_process_prune
-        restore_default_signal_handlers
-        delete_pidfile
-      end
-
-      def graceful_termination
-        SolidQueue.instrument(:graceful_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do |payload|
-          term_forks
-
-          wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
-            reap_terminated_forks
-          end
-
-          unless all_forks_terminated?
-            payload[:shutdown_timeout_exceeded] = true
-            immediate_termination
+        SolidQueue.instrument(:start_process, process: self) do
+          run_callbacks(:boot) do
+            @stopped = false
+            sync_std_streams
           end
         end
       end
 
-      def immediate_termination
-        SolidQueue.instrument(:immediate_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do
-          quit_forks
-        end
-      end
-
-      def term_forks
-        signal_processes(forks.keys, :TERM)
+      def start_processes
+        configuration.processes.each { |configured_process| start_process(configured_process) }
       end
 
-      def quit_forks
-        signal_processes(forks.keys, :QUIT)
+      def stopped?
+        @stopped
       end
 
-      def stop_process_prune
-        @maintenance_task&.shutdown
-      end
-
-      def delete_pidfile
-        @pidfile&.delete
-      end
-
-      def prune_dead_processes
-        wrap_in_app_executor { SolidQueue::Process.prune }
-      end
-
-      def release_orphaned_executions
-        wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
-      end
-
-      def start_fork(configured_process)
-        configured_process.supervised_by process
-
-        pid = fork do
-          configured_process.start
-        end
-
-        forks[pid] = configured_process
-      end
-
-      def reap_and_replace_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          replace_fork(pid, status)
-        end
-      end
-
-      def reap_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          forks.delete(pid)
-        end
-      rescue SystemCallError
-        # All children already reaped
-      end
-
-      def replace_fork(pid, status)
-        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
-          if supervised_fork = forks.delete(pid)
-            payload[:fork] = supervised_fork
-            start_fork(supervised_fork)
-          end
-        end
+      def supervise
       end
 
-      def all_forks_terminated?
-        forks.empty?
+      def start_process(configured_process)
+        raise NotImplementedError
       end
 
-      def wait_until(timeout, condition, &block)
-        if timeout > 0
-          deadline = monotonic_time_now + timeout
-
-          while monotonic_time_now < deadline && !condition.call
-            sleep 0.1
-            block.call
-          end
-        else
-          while !condition.call
-            sleep 0.5
-            block.call
+      def shutdown
+        SolidQueue.instrument(:shutdown_process, process: self) do
+          run_callbacks(:shutdown) do
+            stop_maintenance_task
           end
         end
       end
 
-      def monotonic_time_now
-        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      def sync_std_streams
+        STDOUT.sync = STDERR.sync = true
       end
   end
 end

data/lib/solid_queue/tasks.rb

@@ -1,16 +1,6 @@
 namespace :solid_queue do
   desc "start solid_queue supervisor to dispatch and process jobs"
   task start: :environment do
-    SolidQueue::Supervisor.start(mode: :all)
-  end
-
-  desc "start solid_queue supervisor to process jobs"
-  task work: :environment do
-    SolidQueue::Supervisor.start(mode: :work)
-  end
-
-  desc "start solid_queue dispatcher to enqueue scheduled jobs"
-  task dispatch: :environment do
-    SolidQueue::Supervisor.start(mode: :dispatch)
+    SolidQueue::Supervisor.start
   end
 end

data/lib/solid_queue/timer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Timer
+    extend self
+
+    def wait_until(timeout, condition, &block)
+      if timeout > 0
+        deadline = monotonic_time_now + timeout
+
+        while monotonic_time_now < deadline && !condition.call
+          sleep 0.1
+          block.call
+        end
+      else
+        while !condition.call
+          sleep 0.5
+          block.call
+        end
+      end
+    end
+
+    private
+      def monotonic_time_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+  end
+end

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.3.4"
+  VERSION = "0.4.0"
 end

data/lib/solid_queue/worker.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
 module SolidQueue
-  class Worker < Processes::Base
-    include Processes::Poller
-
+  class Worker < Processes::Poller
     attr_accessor :queues, :pool
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
 
-      @polling_interval = options[:polling_interval]
       @queues = Array(options[:queues])
       @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
+
+      super(**options)
     end
 
     def metadata
@@ -31,7 +30,7 @@ module SolidQueue
 
       def claim_executions
         with_polling_volume do
-          SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process.id)
+          SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process_id)
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-19 00:00:00.000000000 Z
+date: 2024-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -203,6 +203,7 @@ files:
 - app/models/solid_queue/job/schedulable.rb
 - app/models/solid_queue/pause.rb
 - app/models/solid_queue/process.rb
+- app/models/solid_queue/process/executor.rb
 - app/models/solid_queue/process/prunable.rb
 - app/models/solid_queue/queue.rb
 - app/models/solid_queue/queue_selector.rb
@@ -234,15 +235,20 @@ files:
 - lib/solid_queue/processes/base.rb
 - lib/solid_queue/processes/callbacks.rb
 - lib/solid_queue/processes/interruptible.rb
-- lib/solid_queue/processes/pidfile.rb
 - lib/solid_queue/processes/poller.rb
 - lib/solid_queue/processes/procline.rb
 - lib/solid_queue/processes/registrable.rb
 - lib/solid_queue/processes/runnable.rb
-- lib/solid_queue/processes/signals.rb
 - lib/solid_queue/processes/supervised.rb
 - lib/solid_queue/supervisor.rb
+- lib/solid_queue/supervisor/async_supervisor.rb
+- lib/solid_queue/supervisor/fork_supervisor.rb
+- lib/solid_queue/supervisor/maintenance.rb
+- lib/solid_queue/supervisor/pidfile.rb
+- lib/solid_queue/supervisor/pidfiled.rb
+- lib/solid_queue/supervisor/signals.rb
 - lib/solid_queue/tasks.rb
+- lib/solid_queue/timer.rb
 - lib/solid_queue/version.rb
 - lib/solid_queue/worker.rb
 homepage: https://github.com/rails/solid_queue
@@ -251,7 +257,9 @@ licenses:
 metadata:
   homepage_uri: https://github.com/rails/solid_queue
   source_code_uri: https://github.com/rails/solid_queue
-post_install_message:
+post_install_message: |
+  Upgrading to Solid Queue 0.4.x? There are some breaking changes about how Solid Queue is started. Check
+  https://github.com/rails/solid_cache/blob/main/UPGRADING.md for upgrade instructions.
 rdoc_options: []
 require_paths:
 - lib
@@ -266,7 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: Database-backed Active Job backend.