solid_queue 1.2.1 → 1.4.0

data/lib/puma/plugin/solid_queue.rb

@@ -1,5 +1,13 @@
 require "puma/plugin"
 
+module Puma
+  class DSL
+    def solid_queue_mode(mode = :fork)
+      @options[:solid_queue_mode] = mode.to_sym
+    end
+  end
+end
+
 Puma::Plugin.create do
   attr_reader :puma_pid, :solid_queue_pid, :log_writer, :solid_queue_supervisor
 
@@ -7,26 +15,78 @@ Puma::Plugin.create do
     @log_writer = launcher.log_writer
     @puma_pid = $$
 
-    in_background do
-      monitor_solid_queue
+    if launcher.options[:solid_queue_mode] == :async
+      start_async(launcher)
+    else
+      start_forked(launcher)
     end
+  end
+
+  private
+    def start_forked(launcher)
+      in_background do
+        monitor_solid_queue
+      end
+
+      if Gem::Version.new(Puma::Const::VERSION) < Gem::Version.new("7")
+        launcher.events.on_booted do
+          @solid_queue_pid = fork do
+            Thread.new { monitor_puma }
+            SolidQueue::Supervisor.start(mode: :fork)
+          end
+        end
 
-    launcher.events.on_booted do
-      @solid_queue_pid = fork do
-        Thread.new { monitor_puma }
-        SolidQueue::Supervisor.start
+        launcher.events.on_stopped { stop_solid_queue_fork }
+        launcher.events.on_restart { stop_solid_queue_fork }
+      else
+        launcher.events.after_booted do
+          @solid_queue_pid = fork do
+            Thread.new { monitor_puma }
+            start_solid_queue(mode: :fork)
+          end
+        end
+
+        launcher.events.after_stopped { stop_solid_queue_fork }
+        launcher.events.before_restart { stop_solid_queue_fork }
       end
     end
 
-    launcher.events.on_stopped { stop_solid_queue }
-    launcher.events.on_restart { stop_solid_queue }
-  end
+    def start_async(launcher)
+      if Gem::Version.new(Puma::Const::VERSION) < Gem::Version.new("7")
+        launcher.events.on_booted do
+          start_solid_queue(mode: :async, standalone: false)
+        end
+
+        launcher.events.on_stopped { solid_queue_supervisor&.stop }
+
+        launcher.events.on_restart do
+          solid_queue_supervisor&.stop
+          start_solid_queue(mode: :async, standalone: false)
+        end
+      else
+        launcher.events.after_booted do
+          start_solid_queue(mode: :async, standalone: false)
+        end
+
+        launcher.events.after_stopped { solid_queue_supervisor&.stop }
+
+        launcher.events.before_restart do
+          solid_queue_supervisor&.stop
+          start_solid_queue(mode: :async, standalone: false)
+        end
+      end
+    end
+
+    def start_solid_queue(**options)
+      @solid_queue_supervisor = SolidQueue::Supervisor.start(**options)
+    end
+
+    def stop_solid_queue_fork
+      return unless solid_queue_pid
 
-  private
-    def stop_solid_queue
       Process.waitpid(solid_queue_pid, Process::WNOHANG)
       log "Stopping Solid Queue..."
-      Process.kill(:INT, solid_queue_pid) if solid_queue_pid
+      Process.kill(:INT, solid_queue_pid)
       Process.wait(solid_queue_pid)
     rescue Errno::ECHILD, Errno::ESRCH
     end
@@ -36,7 +96,7 @@ Puma::Plugin.create do
     end
 
     def monitor_solid_queue
-      monitor(:solid_queue_dead?, "Detected Solid Queue has gone away, stopping Puma...")
+      monitor(:solid_queue_fork_dead?, "Detected Solid Queue has gone away, stopping Puma...")
     end
 
     def monitor(process_dead, message)
@@ -50,7 +110,7 @@ Puma::Plugin.create do
       end
     end
 
-    def solid_queue_dead?
+    def solid_queue_fork_dead?
       if solid_queue_started?
         Process.waitpid(solid_queue_pid, Process::WNOHANG)
       end

data/lib/solid_queue/app_executor.rb

@@ -17,5 +17,15 @@ module SolidQueue
         SolidQueue.on_thread_error.call(error)
       end
     end
+
+    def create_thread(&block)
+      Thread.new do
+        Thread.current.name = name
+        block.call
+      rescue Exception => exception
+        handle_thread_error(exception)
+        raise
+      end
+    end
   end
 end

data/lib/solid_queue/async_supervisor.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class AsyncSupervisor < Supervisor
+    after_shutdown :terminate_gracefully, unless: :standalone?
+
+    def stop
+      super
+      @thread&.join
+    end
+
+    private
+      def supervise
+        if standalone? then super
+        else
+          @thread = create_thread { super }
+        end
+      end
+
+      def check_and_replace_terminated_processes
+        terminated_threads = process_instances.select { |thread_id, instance| !instance.alive? }
+        terminated_threads.each { |thread_id, _| replace_thread(thread_id) }
+      end
+
+      def replace_thread(thread_id)
+        SolidQueue.instrument(:replace_thread, supervisor_pid: ::Process.pid) do |payload|
+          if (instance = process_instances.delete(thread_id))
+            payload[:thread] = instance
+
+            error = Processes::ThreadTerminatedError.new(instance.name)
+            release_claimed_jobs_by(instance, with_error: error)
+
+            start_process(configured_processes.delete(thread_id))
+          end
+        end
+      end
+
+      def perform_graceful_termination
+        process_instances.values.each(&:stop)
+
+        Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_processes_terminated? })
+      end
+
+      def perform_immediate_termination
+        exit!
+      end
+
+      def all_processes_terminated?
+        process_instances.values.none?(&:alive?)
+      end
+  end
+end

data/lib/solid_queue/cli.rb

@@ -8,11 +8,15 @@ module SolidQueue
       desc: "Path to config file (default: #{Configuration::DEFAULT_CONFIG_FILE_PATH}).",
       banner: "SOLID_QUEUE_CONFIG"
 
+    class_option :mode, type: :string, enum: %w[ fork async ],
+      desc: "Whether to fork processes for workers and dispatchers (fork) or to run these in the same process as the supervisor (async) (default: fork).",
+      banner: "SOLID_QUEUE_SUPERVISOR_MODE"
+
     class_option :recurring_schedule_file, type: :string,
       desc: "Path to recurring schedule definition (default: #{Configuration::DEFAULT_RECURRING_SCHEDULE_FILE_PATH}).",
       banner: "SOLID_QUEUE_RECURRING_SCHEDULE"
 
-    class_option :skip_recurring, type: :boolean, default: false,
+    class_option :skip_recurring, type: :boolean,
       desc: "Whether to skip recurring tasks scheduling",
       banner: "SOLID_QUEUE_SKIP_RECURRING"
 

data/lib/solid_queue/configuration.rb

@@ -28,6 +28,11 @@ module SolidQueue
       concurrency_maintenance_interval: 600
     }
 
+    SCHEDULER_DEFAULTS = {
+      polling_interval: 5,
+      dynamic_tasks_enabled: false
+    }
+
     DEFAULT_CONFIG_FILE_PATH = "config/queue.yml"
     DEFAULT_RECURRING_SCHEDULE_FILE_PATH = "config/recurring.yml"
 
@@ -56,6 +61,14 @@ module SolidQueue
       end
     end
 
+    def mode
+      @options[:mode].to_s.inquiry
+    end
+
+    def standalone?
+      mode.fork? || @options[:standalone]
+    end
+
     private
       attr_reader :options
 
@@ -84,6 +97,8 @@ module SolidQueue
 
       def default_options
         {
+          mode: ENV["SOLID_QUEUE_SUPERVISOR_MODE"] || :fork,
+          standalone: true,
           config_file: Rails.root.join(ENV["SOLID_QUEUE_CONFIG"] || DEFAULT_CONFIG_FILE_PATH),
           recurring_schedule_file: Rails.root.join(ENV["SOLID_QUEUE_RECURRING_SCHEDULE"] || DEFAULT_RECURRING_SCHEDULE_FILE_PATH),
           only_work: false,
@@ -110,7 +125,12 @@ module SolidQueue
 
       def workers
         workers_options.flat_map do |worker_options|
-          processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          processes = if mode.fork?
+            worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          else
+            1
+          end
+
           processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
         end
       end
@@ -122,8 +142,10 @@ module SolidQueue
       end
 
       def schedulers
-        if !skip_recurring_tasks? && recurring_tasks.any?
-          [ Process.new(:scheduler, recurring_tasks: recurring_tasks) ]
+        return [] if skip_recurring_tasks?
+
+        if recurring_tasks.any? || dynamic_recurring_tasks_enabled?
+          [ Process.new(:scheduler, { recurring_tasks: recurring_tasks, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ]
         else
           []
         end
@@ -139,17 +161,29 @@ module SolidQueue
           .map { |options| options.dup.symbolize_keys }
       end
 
+      def scheduler_options
+        @scheduler_options ||= processes_config.fetch(:scheduler, {}).dup.symbolize_keys
+      end
+
+      def dynamic_recurring_tasks_enabled?
+        scheduler_options.fetch(:dynamic_tasks_enabled, SCHEDULER_DEFAULTS[:dynamic_tasks_enabled])
+      end
+
       def recurring_tasks
         @recurring_tasks ||= recurring_tasks_config.map do |id, options|
-          RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule)
+          RecurringTask.from_configuration(id, **options.merge(static: true)) if options&.has_key?(:schedule)
         end.compact
       end
 
       def processes_config
         @processes_config ||= config_from \
-          options.slice(:workers, :dispatchers).presence || options[:config_file],
-          keys: [ :workers, :dispatchers ],
-          fallback: { workers: [ WORKER_DEFAULTS ], dispatchers: [ DISPATCHER_DEFAULTS ] }
+          options.slice(:workers, :dispatchers, :scheduler).presence || options[:config_file],
+          keys: [ :workers, :dispatchers, :scheduler ],
+          fallback: {
+            workers: [ WORKER_DEFAULTS ],
+            dispatchers: [ DISPATCHER_DEFAULTS ],
+            scheduler: SCHEDULER_DEFAULTS
+          }
       end
 
       def recurring_tasks_config
@@ -158,7 +192,6 @@ module SolidQueue
         end
       end
 
-
       def config_from(file_or_hash, keys: [], fallback: {}, env: Rails.env)
         load_config_from(file_or_hash).then do |config|
           config = config[env.to_sym] ? config[env.to_sym] : config
@@ -188,6 +221,7 @@ module SolidQueue
         if file.exist?
           ActiveSupport::ConfigurationFile.parse(file).deep_symbolize_keys
         else
+          puts "[solid_queue] WARNING: Provided configuration file '#{file}' does not exist. Falling back to default configuration."
           {}
         end
       end

data/lib/solid_queue/dispatcher.rb

@@ -3,6 +3,7 @@
 module SolidQueue
   class Dispatcher < Processes::Poller
     include LifecycleHooks
+
     attr_reader :batch_size
 
     after_boot :run_start_hooks

data/lib/solid_queue/fork_supervisor.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class ForkSupervisor < Supervisor
+    private
+
+    def perform_graceful_termination
+      term_forks
+
+      Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_processes_terminated? }) do
+        reap_terminated_forks
+      end
+    end
+
+    def perform_immediate_termination
+      quit_forks
+    end
+
+    def term_forks
+      signal_processes(process_instances.keys, :TERM)
+    end
+
+    def quit_forks
+      signal_processes(process_instances.keys, :QUIT)
+    end
+
+    def check_and_replace_terminated_processes
+      loop do
+        pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+        break unless pid
+
+        replace_fork(pid, status)
+      end
+    end
+
+    def reap_terminated_forks
+      loop do
+        pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+        break unless pid
+
+        if (terminated_fork = process_instances.delete(pid)) && (!status.exited? || status.exitstatus.to_i > 0)
+          error = Processes::ProcessExitError.new(status)
+          release_claimed_jobs_by(terminated_fork, with_error: error)
+        end
+
+        configured_processes.delete(pid)
+      end
+    rescue SystemCallError
+      # All children already reaped
+    end
+
+    def replace_fork(pid, status)
+      SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
+        if terminated_fork = process_instances.delete(pid)
+          payload[:fork] = terminated_fork
+          error = Processes::ProcessExitError.new(status)
+          release_claimed_jobs_by(terminated_fork, with_error: error)
+
+          start_process(configured_processes.delete(pid))
+        end
+      end
+    end
+
+    def all_processes_terminated?
+      process_instances.empty?
+    end
+  end
+end

data/lib/solid_queue/processes/registrable.rb

@@ -18,17 +18,19 @@ module SolidQueue::Processes
       attr_accessor :process
 
       def register
-        @process = SolidQueue::Process.register \
-          kind: kind,
-          name: name,
-          pid: pid,
-          hostname: hostname,
-          supervisor: try(:supervisor),
-          metadata: metadata.compact
+        wrap_in_app_executor do
+          @process = SolidQueue::Process.register \
+            kind: kind,
+            name: name,
+            pid: pid,
+            hostname: hostname,
+            supervisor: try(:supervisor),
+            metadata: metadata.compact
+        end
       end
 
       def deregister
-        process&.deregister
+        wrap_in_app_executor { process&.deregister }
       end
 
       def registered?
@@ -52,10 +54,14 @@ module SolidQueue::Processes
       end
 
       def heartbeat
-        process.heartbeat
+        process&.heartbeat
       rescue ActiveRecord::RecordNotFound
         self.process = nil
         wake_up
       end
+
+      def reload_metadata
+        wrap_in_app_executor { process&.update(metadata: metadata.compact) }
+      end
   end
 end

data/lib/solid_queue/processes/runnable.rb

@@ -7,20 +7,26 @@ module SolidQueue::Processes
     attr_writer :mode
 
     def start
-      boot
-
-      if running_async?
-        @thread = create_thread { run }
-      else
+      run_in_mode do
+        boot
         run
       end
     end
 
     def stop
       super
-
       wake_up
-      @thread&.join
+
+      # When not supervised, block until the thread terminates for backward
+      # compatibility with code that expects stop to be synchronous.
+      # When supervised, the supervisor controls the shutdown timeout.
+      unless supervised?
+        @thread&.join
+      end
+    end
+
+    def alive?
+      !running_async? || @thread&.alive?
     end
 
     private
@@ -30,6 +36,18 @@ module SolidQueue::Processes
         (@mode || DEFAULT_MODE).to_s.inquiry
       end
 
+      def run_in_mode(&block)
+        case
+        when running_as_fork?
+          fork(&block)
+        when running_async?
+          @thread = create_thread(&block)
+          @thread.object_id
+        else
+          block.call
+        end
+      end
+
       def boot
         SolidQueue.instrument(:start_process, process: self) do
           run_callbacks(:boot) do
@@ -74,16 +92,5 @@ module SolidQueue::Processes
       def running_as_fork?
         mode.fork?
       end
-
-
-      def create_thread(&block)
-        Thread.new do
-          Thread.current.name = name
-          block.call
-        rescue Exception => exception
-          handle_thread_error(exception)
-          raise
-        end
-      end
   end
 end

data/lib/solid_queue/processes/thread_terminated_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Processes
+    class ThreadTerminatedError < RuntimeError
+      def initialize(name)
+        super("Thread #{name} terminated unexpectedly")
+      end
+    end
+  end
+end

data/lib/solid_queue/scheduler/recurring_schedule.rb

@@ -4,21 +4,28 @@ module SolidQueue
   class Scheduler::RecurringSchedule
     include AppExecutor
 
-    attr_reader :configured_tasks, :scheduled_tasks
+    attr_reader :scheduled_tasks
+
+    def initialize(static_tasks, dynamic_tasks_enabled: false)
+      @static_tasks = Array(static_tasks).map { |task| RecurringTask.wrap(task) }.select(&:valid?)
+      @dynamic_tasks_enabled = dynamic_tasks_enabled
 
-    def initialize(tasks)
-      @configured_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?)
       @scheduled_tasks = Concurrent::Hash.new
     end
 
+    def configured_tasks
+      static_tasks + dynamic_tasks
+    end
+
     def empty?
-      configured_tasks.empty?
+      scheduled_tasks.empty? && dynamic_tasks.empty?
     end
 
     def schedule_tasks
       wrap_in_app_executor do
-        persist_tasks
-        reload_tasks
+        persist_static_tasks
+        reload_static_tasks
+        reload_dynamic_tasks
       end
 
       configured_tasks.each do |task|
@@ -39,14 +46,57 @@ module SolidQueue
       configured_tasks.map(&:key)
     end
 
+    def reschedule_dynamic_tasks
+      wrap_in_app_executor do
+        reload_dynamic_tasks
+        schedule_created_dynamic_tasks
+        unschedule_deleted_dynamic_tasks
+      end
+    end
+
     private
-      def persist_tasks
-        SolidQueue::RecurringTask.static.where.not(key: task_keys).delete_all
-        SolidQueue::RecurringTask.create_or_update_all configured_tasks
+      attr_reader :static_tasks
+
+      def static_task_keys
+        static_tasks.map(&:key)
+      end
+
+      def dynamic_tasks
+        @dynamic_tasks ||= load_dynamic_tasks
+      end
+
+      def dynamic_tasks_enabled?
+        @dynamic_tasks_enabled
+      end
+
+      def schedule_created_dynamic_tasks
+        RecurringTask.dynamic.where.not(key: scheduled_tasks.keys).each do |task|
+          schedule_task(task)
+        end
+      end
+
+      def unschedule_deleted_dynamic_tasks
+        (scheduled_tasks.keys - RecurringTask.pluck(:key)).each do |key|
+          scheduled_tasks[key].cancel
+          scheduled_tasks.delete(key)
+        end
+      end
+
+      def persist_static_tasks
+        RecurringTask.static.where.not(key: static_task_keys).delete_all
+        RecurringTask.create_or_update_all static_tasks
+      end
+
+      def reload_static_tasks
+        @static_tasks = RecurringTask.static.where(key: static_task_keys).to_a
+      end
+
+      def reload_dynamic_tasks
+        @dynamic_tasks = load_dynamic_tasks
       end
 
-      def reload_tasks
-        @configured_tasks = SolidQueue::RecurringTask.where(key: task_keys)
+      def load_dynamic_tasks
+        dynamic_tasks_enabled? ? RecurringTask.dynamic.to_a : []
       end
 
       def schedule(task)

data/lib/solid_queue/scheduler.rb

@@ -5,7 +5,7 @@ module SolidQueue
     include Processes::Runnable
     include LifecycleHooks
 
-    attr_reader :recurring_schedule
+    attr_reader :recurring_schedule, :polling_interval
 
     after_boot :run_start_hooks
     after_boot :schedule_recurring_tasks
@@ -14,7 +14,10 @@ module SolidQueue
     after_shutdown :run_exit_hooks
 
     def initialize(recurring_tasks:, **options)
-      @recurring_schedule = RecurringSchedule.new(recurring_tasks)
+      options = options.dup.with_defaults(SolidQueue::Configuration::SCHEDULER_DEFAULTS)
+      @dynamic_tasks_enabled = options[:dynamic_tasks_enabled]
+      @polling_interval = options[:polling_interval]
+      @recurring_schedule = RecurringSchedule.new(recurring_tasks, dynamic_tasks_enabled: @dynamic_tasks_enabled)
 
       super(**options)
     end
@@ -24,13 +27,16 @@ module SolidQueue
     end
 
     private
-      SLEEP_INTERVAL = 60 # Right now it doesn't matter, can be set to 1 in the future for dynamic tasks
+
+      STATIC_SLEEP_INTERVAL = 60
 
       def run
         loop do
           break if shutting_down?
 
-          interruptible_sleep(SLEEP_INTERVAL)
+          reload_dynamic_schedule if dynamic_tasks_enabled?
+
+          interruptible_sleep(sleep_interval)
         end
       ensure
         SolidQueue.instrument(:shutdown_process, process: self) do
@@ -46,10 +52,23 @@ module SolidQueue
         recurring_schedule.unschedule_tasks
       end
 
+      def reload_dynamic_schedule
+        recurring_schedule.reschedule_dynamic_tasks
+        reload_metadata
+      end
+
+      def dynamic_tasks_enabled?
+        @dynamic_tasks_enabled
+      end
+
       def all_work_completed?
         recurring_schedule.empty?
       end
 
+      def sleep_interval
+        dynamic_tasks_enabled? ? polling_interval : STATIC_SLEEP_INTERVAL
+      end
+
       def set_procline
         procline "scheduling #{recurring_schedule.task_keys.join(",")}"
       end