solid_queue 0.3.3 → 0.4.0

data/lib/solid_queue/supervisor.rb

@@ -2,182 +2,72 @@
 
 module SolidQueue
   class Supervisor < Processes::Base
-    include Processes::Signals
+    include Maintenance
 
     class << self
-      def start(mode: :work, load_configuration_from: nil)
+      def start(mode: :fork, load_configuration_from: nil)
         SolidQueue.supervisor = true
         configuration = Configuration.new(mode: mode, load_from: load_configuration_from)
 
-        new(*configuration.processes).start
+        klass = mode == :fork ? ForkSupervisor : AsyncSupervisor
+        klass.new(configuration).tap(&:start)
       end
     end
 
-    def initialize(*configured_processes)
-      @configured_processes = Array(configured_processes)
-      @forks = {}
+    def initialize(configuration)
+      @configuration = configuration
     end
 
     def start
-      run_callbacks(:boot) { boot }
+      boot
+
+      start_processes
+      launch_maintenance_task
 
-      start_forks
-      launch_process_prune
       supervise
-    rescue Processes::GracefulTerminationRequested
-      graceful_termination
-    rescue Processes::ImmediateTerminationRequested
-      immediate_termination
-    ensure
-      run_callbacks(:shutdown) { shutdown }
+    end
+
+    def stop
+      @stopped = true
     end
 
     private
-      attr_reader :configured_processes, :forks
+      attr_reader :configuration
 
       def boot
-        sync_std_streams
-        setup_pidfile
-        register_signal_handlers
-      end
-
-      def supervise
-        loop do
-          procline "supervising #{forks.keys.join(", ")}"
-
-          process_signal_queue
-          reap_and_replace_terminated_forks
-          interruptible_sleep(1.second)
-        end
-      end
-
-      def sync_std_streams
-        STDOUT.sync = STDERR.sync = true
-      end
-
-      def setup_pidfile
-        @pidfile = if SolidQueue.supervisor_pidfile
-          Processes::Pidfile.new(SolidQueue.supervisor_pidfile).tap(&:setup)
-        end
-      end
-
-      def start_forks
-        configured_processes.each { |configured_process| start_fork(configured_process) }
-      end
-
-      def launch_process_prune
-        @prune_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) { prune_dead_processes }
-        @prune_task.execute
-      end
-
-      def shutdown
-        stop_process_prune
-        restore_default_signal_handlers
-        delete_pidfile
-      end
-
-      def graceful_termination
-        SolidQueue.instrument(:graceful_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do |payload|
-          term_forks
-
-          wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
-            reap_terminated_forks
-          end
-
-          unless all_forks_terminated?
-            payload[:shutdown_timeout_exceeded] = true
-            immediate_termination
+        SolidQueue.instrument(:start_process, process: self) do
+          run_callbacks(:boot) do
+            @stopped = false
+            sync_std_streams
           end
         end
       end
 
-      def immediate_termination
-        SolidQueue.instrument(:immediate_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do
-          quit_forks
-        end
+      def start_processes
+        configuration.processes.each { |configured_process| start_process(configured_process) }
       end
 
-      def term_forks
-        signal_processes(forks.keys, :TERM)
+      def stopped?
+        @stopped
       end
 
-      def quit_forks
-        signal_processes(forks.keys, :QUIT)
-      end
-
-      def stop_process_prune
-        @prune_task&.shutdown
-      end
-
-      def delete_pidfile
-        @pidfile&.delete
-      end
-
-      def prune_dead_processes
-        wrap_in_app_executor { SolidQueue::Process.prune }
-      end
-
-      def start_fork(configured_process)
-        configured_process.supervised_by process
-
-        pid = fork do
-          configured_process.start
-        end
-
-        forks[pid] = configured_process
-      end
-
-      def reap_and_replace_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          replace_fork(pid, status)
-        end
-      end
-
-      def reap_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          forks.delete(pid)
-        end
-      rescue SystemCallError
-        # All children already reaped
-      end
-
-      def replace_fork(pid, status)
-        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
-          if supervised_fork = forks.delete(pid)
-            payload[:fork] = supervised_fork
-            start_fork(supervised_fork)
-          end
-        end
+      def supervise
       end
 
-      def all_forks_terminated?
-        forks.empty?
+      def start_process(configured_process)
+        raise NotImplementedError
       end
 
-      def wait_until(timeout, condition, &block)
-        if timeout > 0
-          deadline = monotonic_time_now + timeout
-
-          while monotonic_time_now < deadline && !condition.call
-            sleep 0.1
-            block.call
-          end
-        else
-          while !condition.call
-            sleep 0.5
-            block.call
+      def shutdown
+        SolidQueue.instrument(:shutdown_process, process: self) do
+          run_callbacks(:shutdown) do
+            stop_maintenance_task
           end
         end
       end
 
-      def monotonic_time_now
-        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      def sync_std_streams
+        STDOUT.sync = STDERR.sync = true
       end
   end
 end

data/lib/solid_queue/tasks.rb

@@ -1,16 +1,6 @@
 namespace :solid_queue do
   desc "start solid_queue supervisor to dispatch and process jobs"
   task start: :environment do
-    SolidQueue::Supervisor.start(mode: :all)
-  end
-
-  desc "start solid_queue supervisor to process jobs"
-  task work: :environment do
-    SolidQueue::Supervisor.start(mode: :work)
-  end
-
-  desc "start solid_queue dispatcher to enqueue scheduled jobs"
-  task dispatch: :environment do
-    SolidQueue::Supervisor.start(mode: :dispatch)
+    SolidQueue::Supervisor.start
   end
 end

data/lib/solid_queue/timer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Timer
+    extend self
+
+    def wait_until(timeout, condition, &block)
+      if timeout > 0
+        deadline = monotonic_time_now + timeout
+
+        while monotonic_time_now < deadline && !condition.call
+          sleep 0.1
+          block.call
+        end
+      else
+        while !condition.call
+          sleep 0.5
+          block.call
+        end
+      end
+    end
+
+    private
+      def monotonic_time_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+  end
+end

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

data/lib/solid_queue/worker.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
 module SolidQueue
-  class Worker < Processes::Base
-    include Processes::Poller
-
+  class Worker < Processes::Poller
     attr_accessor :queues, :pool
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
 
-      @polling_interval = options[:polling_interval]
       @queues = Array(options[:queues])
       @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
+
+      super(**options)
     end
 
     def metadata
@@ -31,7 +30,7 @@ module SolidQueue
 
       def claim_executions
         with_polling_volume do
-          SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process.id)
+          SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process_id)
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-11 00:00:00.000000000 Z
+date: 2024-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -203,6 +203,7 @@ files:
 - app/models/solid_queue/job/schedulable.rb
 - app/models/solid_queue/pause.rb
 - app/models/solid_queue/process.rb
+- app/models/solid_queue/process/executor.rb
 - app/models/solid_queue/process/prunable.rb
 - app/models/solid_queue/queue.rb
 - app/models/solid_queue/queue_selector.rb
@@ -234,15 +235,20 @@ files:
 - lib/solid_queue/processes/base.rb
 - lib/solid_queue/processes/callbacks.rb
 - lib/solid_queue/processes/interruptible.rb
-- lib/solid_queue/processes/pidfile.rb
 - lib/solid_queue/processes/poller.rb
 - lib/solid_queue/processes/procline.rb
 - lib/solid_queue/processes/registrable.rb
 - lib/solid_queue/processes/runnable.rb
-- lib/solid_queue/processes/signals.rb
 - lib/solid_queue/processes/supervised.rb
 - lib/solid_queue/supervisor.rb
+- lib/solid_queue/supervisor/async_supervisor.rb
+- lib/solid_queue/supervisor/fork_supervisor.rb
+- lib/solid_queue/supervisor/maintenance.rb
+- lib/solid_queue/supervisor/pidfile.rb
+- lib/solid_queue/supervisor/pidfiled.rb
+- lib/solid_queue/supervisor/signals.rb
 - lib/solid_queue/tasks.rb
+- lib/solid_queue/timer.rb
 - lib/solid_queue/version.rb
 - lib/solid_queue/worker.rb
 homepage: https://github.com/rails/solid_queue
@@ -251,7 +257,9 @@ licenses:
 metadata:
   homepage_uri: https://github.com/rails/solid_queue
   source_code_uri: https://github.com/rails/solid_queue
-post_install_message:
+post_install_message: |
+  Upgrading to Solid Queue 0.4.x? There are some breaking changes about how Solid Queue is started. Check
+  https://github.com/rails/solid_cache/blob/main/UPGRADING.md for upgrade instructions.
 rdoc_options: []
 require_paths:
 - lib

data/lib/solid_queue/processes/signals.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-module SolidQueue::Processes
-  class GracefulTerminationRequested < Interrupt; end
-  class ImmediateTerminationRequested < Interrupt; end
-
-  module Signals
-    extend ActiveSupport::Concern
-
-    private
-      SIGNALS = %i[ QUIT INT TERM ]
-
-      def register_signal_handlers
-        SIGNALS.each do |signal|
-          trap(signal) do
-            signal_queue << signal
-            interrupt
-          end
-        end
-      end
-
-      def restore_default_signal_handlers
-        SIGNALS.each do |signal|
-          trap(signal, :DEFAULT)
-        end
-      end
-
-      def process_signal_queue
-        while signal = signal_queue.shift
-          handle_signal(signal)
-        end
-      end
-
-      def handle_signal(signal)
-        case signal
-        when :TERM, :INT
-          request_graceful_termination
-        when :QUIT
-          request_immediate_termination
-        else
-          SolidQueue.instrument :unhandled_signal_error, signal: signal
-        end
-      end
-
-      def request_graceful_termination
-        raise GracefulTerminationRequested
-      end
-
-      def request_immediate_termination
-        raise ImmediateTerminationRequested
-      end
-
-      def signal_processes(pids, signal)
-        pids.each do |pid|
-          signal_process pid, signal
-        end
-      end
-
-      def signal_process(pid, signal)
-        ::Process.kill signal, pid
-      rescue Errno::ESRCH
-        # Ignore, process died before
-      end
-
-      def signal_queue
-        @signal_queue ||= []
-      end
-  end
-end