solid_queue 1.2.4 → 1.4.0

data/lib/solid_queue/configuration.rb

@@ -28,6 +28,11 @@ module SolidQueue
       concurrency_maintenance_interval: 600
     }
 
+    SCHEDULER_DEFAULTS = {
+      polling_interval: 5,
+      dynamic_tasks_enabled: false
+    }
+
     DEFAULT_CONFIG_FILE_PATH = "config/queue.yml"
     DEFAULT_RECURRING_SCHEDULE_FILE_PATH = "config/recurring.yml"
 
@@ -56,6 +61,14 @@ module SolidQueue
       end
     end
 
+    def mode
+      @options[:mode].to_s.inquiry
+    end
+
+    def standalone?
+      mode.fork? || @options[:standalone]
+    end
+
     private
       attr_reader :options
 
@@ -84,6 +97,8 @@ module SolidQueue
 
       def default_options
         {
+          mode: ENV["SOLID_QUEUE_SUPERVISOR_MODE"] || :fork,
+          standalone: true,
           config_file: Rails.root.join(ENV["SOLID_QUEUE_CONFIG"] || DEFAULT_CONFIG_FILE_PATH),
           recurring_schedule_file: Rails.root.join(ENV["SOLID_QUEUE_RECURRING_SCHEDULE"] || DEFAULT_RECURRING_SCHEDULE_FILE_PATH),
           only_work: false,
@@ -110,7 +125,12 @@ module SolidQueue
 
       def workers
         workers_options.flat_map do |worker_options|
-          processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          processes = if mode.fork?
+            worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          else
+            1
+          end
+
           processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
         end
       end
@@ -122,8 +142,10 @@ module SolidQueue
       end
 
       def schedulers
-        if !skip_recurring_tasks? && recurring_tasks.any?
-          [ Process.new(:scheduler, recurring_tasks: recurring_tasks) ]
+        return [] if skip_recurring_tasks?
+
+        if recurring_tasks.any? || dynamic_recurring_tasks_enabled?
+          [ Process.new(:scheduler, { recurring_tasks: recurring_tasks, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ]
         else
           []
         end
@@ -139,17 +161,29 @@ module SolidQueue
           .map { |options| options.dup.symbolize_keys }
       end
 
+      def scheduler_options
+        @scheduler_options ||= processes_config.fetch(:scheduler, {}).dup.symbolize_keys
+      end
+
+      def dynamic_recurring_tasks_enabled?
+        scheduler_options.fetch(:dynamic_tasks_enabled, SCHEDULER_DEFAULTS[:dynamic_tasks_enabled])
+      end
+
       def recurring_tasks
         @recurring_tasks ||= recurring_tasks_config.map do |id, options|
-          RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule)
+          RecurringTask.from_configuration(id, **options.merge(static: true)) if options&.has_key?(:schedule)
         end.compact
       end
 
       def processes_config
         @processes_config ||= config_from \
-          options.slice(:workers, :dispatchers).presence || options[:config_file],
-          keys: [ :workers, :dispatchers ],
-          fallback: { workers: [ WORKER_DEFAULTS ], dispatchers: [ DISPATCHER_DEFAULTS ] }
+          options.slice(:workers, :dispatchers, :scheduler).presence || options[:config_file],
+          keys: [ :workers, :dispatchers, :scheduler ],
+          fallback: {
+            workers: [ WORKER_DEFAULTS ],
+            dispatchers: [ DISPATCHER_DEFAULTS ],
+            scheduler: SCHEDULER_DEFAULTS
+          }
       end
 
       def recurring_tasks_config
@@ -158,7 +192,6 @@ module SolidQueue
         end
       end
 
-
       def config_from(file_or_hash, keys: [], fallback: {}, env: Rails.env)
         load_config_from(file_or_hash).then do |config|
           config = config[env.to_sym] ? config[env.to_sym] : config

data/lib/solid_queue/dispatcher.rb

@@ -3,6 +3,7 @@
 module SolidQueue
   class Dispatcher < Processes::Poller
     include LifecycleHooks
+
     attr_reader :batch_size
 
     after_boot :run_start_hooks

data/lib/solid_queue/fork_supervisor.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class ForkSupervisor < Supervisor
+    private
+
+    def perform_graceful_termination
+      term_forks
+
+      Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_processes_terminated? }) do
+        reap_terminated_forks
+      end
+    end
+
+    def perform_immediate_termination
+      quit_forks
+    end
+
+    def term_forks
+      signal_processes(process_instances.keys, :TERM)
+    end
+
+    def quit_forks
+      signal_processes(process_instances.keys, :QUIT)
+    end
+
+    def check_and_replace_terminated_processes
+      loop do
+        pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+        break unless pid
+
+        replace_fork(pid, status)
+      end
+    end
+
+    def reap_terminated_forks
+      loop do
+        pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+        break unless pid
+
+        if (terminated_fork = process_instances.delete(pid)) && (!status.exited? || status.exitstatus.to_i > 0)
+          error = Processes::ProcessExitError.new(status)
+          release_claimed_jobs_by(terminated_fork, with_error: error)
+        end
+
+        configured_processes.delete(pid)
+      end
+    rescue SystemCallError
+      # All children already reaped
+    end
+
+    def replace_fork(pid, status)
+      SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
+        if terminated_fork = process_instances.delete(pid)
+          payload[:fork] = terminated_fork
+          error = Processes::ProcessExitError.new(status)
+          release_claimed_jobs_by(terminated_fork, with_error: error)
+
+          start_process(configured_processes.delete(pid))
+        end
+      end
+    end
+
+    def all_processes_terminated?
+      process_instances.empty?
+    end
+  end
+end

data/lib/solid_queue/processes/registrable.rb

@@ -54,10 +54,14 @@ module SolidQueue::Processes
       end
 
       def heartbeat
-        process.heartbeat
+        process&.heartbeat
       rescue ActiveRecord::RecordNotFound
         self.process = nil
         wake_up
       end
+
+      def reload_metadata
+        wrap_in_app_executor { process&.update(metadata: metadata.compact) }
+      end
   end
 end

data/lib/solid_queue/processes/runnable.rb

@@ -7,20 +7,26 @@ module SolidQueue::Processes
     attr_writer :mode
 
     def start
-      boot
-
-      if running_async?
-        @thread = create_thread { run }
-      else
+      run_in_mode do
+        boot
         run
       end
     end
 
     def stop
       super
-
       wake_up
-      @thread&.join
+
+      # When not supervised, block until the thread terminates for backward
+      # compatibility with code that expects stop to be synchronous.
+      # When supervised, the supervisor controls the shutdown timeout.
+      unless supervised?
+        @thread&.join
+      end
+    end
+
+    def alive?
+      !running_async? || @thread&.alive?
     end
 
     private
@@ -30,6 +36,18 @@ module SolidQueue::Processes
         (@mode || DEFAULT_MODE).to_s.inquiry
       end
 
+      def run_in_mode(&block)
+        case
+        when running_as_fork?
+          fork(&block)
+        when running_async?
+          @thread = create_thread(&block)
+          @thread.object_id
+        else
+          block.call
+        end
+      end
+
       def boot
         SolidQueue.instrument(:start_process, process: self) do
           run_callbacks(:boot) do
@@ -74,16 +92,5 @@ module SolidQueue::Processes
       def running_as_fork?
         mode.fork?
       end
-
-
-      def create_thread(&block)
-        Thread.new do
-          Thread.current.name = name
-          block.call
-        rescue Exception => exception
-          handle_thread_error(exception)
-          raise
-        end
-      end
   end
 end

data/lib/solid_queue/processes/thread_terminated_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Processes
+    class ThreadTerminatedError < RuntimeError
+      def initialize(name)
+        super("Thread #{name} terminated unexpectedly")
+      end
+    end
+  end
+end

data/lib/solid_queue/scheduler/recurring_schedule.rb

@@ -4,21 +4,28 @@ module SolidQueue
   class Scheduler::RecurringSchedule
     include AppExecutor
 
-    attr_reader :configured_tasks, :scheduled_tasks
+    attr_reader :scheduled_tasks
+
+    def initialize(static_tasks, dynamic_tasks_enabled: false)
+      @static_tasks = Array(static_tasks).map { |task| RecurringTask.wrap(task) }.select(&:valid?)
+      @dynamic_tasks_enabled = dynamic_tasks_enabled
 
-    def initialize(tasks)
-      @configured_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?)
       @scheduled_tasks = Concurrent::Hash.new
     end
 
+    def configured_tasks
+      static_tasks + dynamic_tasks
+    end
+
     def empty?
-      configured_tasks.empty?
+      scheduled_tasks.empty? && dynamic_tasks.empty?
     end
 
     def schedule_tasks
       wrap_in_app_executor do
-        persist_tasks
-        reload_tasks
+        persist_static_tasks
+        reload_static_tasks
+        reload_dynamic_tasks
       end
 
       configured_tasks.each do |task|
@@ -39,14 +46,57 @@ module SolidQueue
       configured_tasks.map(&:key)
     end
 
+    def reschedule_dynamic_tasks
+      wrap_in_app_executor do
+        reload_dynamic_tasks
+        schedule_created_dynamic_tasks
+        unschedule_deleted_dynamic_tasks
+      end
+    end
+
     private
-      def persist_tasks
-        SolidQueue::RecurringTask.static.where.not(key: task_keys).delete_all
-        SolidQueue::RecurringTask.create_or_update_all configured_tasks
+      attr_reader :static_tasks
+
+      def static_task_keys
+        static_tasks.map(&:key)
+      end
+
+      def dynamic_tasks
+        @dynamic_tasks ||= load_dynamic_tasks
+      end
+
+      def dynamic_tasks_enabled?
+        @dynamic_tasks_enabled
+      end
+
+      def schedule_created_dynamic_tasks
+        RecurringTask.dynamic.where.not(key: scheduled_tasks.keys).each do |task|
+          schedule_task(task)
+        end
+      end
+
+      def unschedule_deleted_dynamic_tasks
+        (scheduled_tasks.keys - RecurringTask.pluck(:key)).each do |key|
+          scheduled_tasks[key].cancel
+          scheduled_tasks.delete(key)
+        end
+      end
+
+      def persist_static_tasks
+        RecurringTask.static.where.not(key: static_task_keys).delete_all
+        RecurringTask.create_or_update_all static_tasks
+      end
+
+      def reload_static_tasks
+        @static_tasks = RecurringTask.static.where(key: static_task_keys).to_a
+      end
+
+      def reload_dynamic_tasks
+        @dynamic_tasks = load_dynamic_tasks
       end
 
-      def reload_tasks
-        @configured_tasks = SolidQueue::RecurringTask.where(key: task_keys).to_a
+      def load_dynamic_tasks
+        dynamic_tasks_enabled? ? RecurringTask.dynamic.to_a : []
       end
 
       def schedule(task)

data/lib/solid_queue/scheduler.rb

@@ -5,7 +5,7 @@ module SolidQueue
     include Processes::Runnable
     include LifecycleHooks
 
-    attr_reader :recurring_schedule
+    attr_reader :recurring_schedule, :polling_interval
 
     after_boot :run_start_hooks
     after_boot :schedule_recurring_tasks
@@ -14,7 +14,10 @@ module SolidQueue
     after_shutdown :run_exit_hooks
 
     def initialize(recurring_tasks:, **options)
-      @recurring_schedule = RecurringSchedule.new(recurring_tasks)
+      options = options.dup.with_defaults(SolidQueue::Configuration::SCHEDULER_DEFAULTS)
+      @dynamic_tasks_enabled = options[:dynamic_tasks_enabled]
+      @polling_interval = options[:polling_interval]
+      @recurring_schedule = RecurringSchedule.new(recurring_tasks, dynamic_tasks_enabled: @dynamic_tasks_enabled)
 
       super(**options)
     end
@@ -24,13 +27,16 @@ module SolidQueue
     end
 
     private
-      SLEEP_INTERVAL = 60 # Right now it doesn't matter, can be set to 1 in the future for dynamic tasks
+
+      STATIC_SLEEP_INTERVAL = 60
 
       def run
         loop do
           break if shutting_down?
 
-          interruptible_sleep(SLEEP_INTERVAL)
+          reload_dynamic_schedule if dynamic_tasks_enabled?
+
+          interruptible_sleep(sleep_interval)
         end
       ensure
         SolidQueue.instrument(:shutdown_process, process: self) do
@@ -46,10 +52,23 @@ module SolidQueue
         recurring_schedule.unschedule_tasks
       end
 
+      def reload_dynamic_schedule
+        recurring_schedule.reschedule_dynamic_tasks
+        reload_metadata
+      end
+
+      def dynamic_tasks_enabled?
+        @dynamic_tasks_enabled
+      end
+
       def all_work_completed?
         recurring_schedule.empty?
       end
 
+      def sleep_interval
+        dynamic_tasks_enabled? ? polling_interval : STATIC_SLEEP_INTERVAL
+      end
+
       def set_procline
         procline "scheduling #{recurring_schedule.task_keys.join(",")}"
       end

data/lib/solid_queue/supervisor/maintenance.rb

@@ -32,5 +32,16 @@ module SolidQueue
           ClaimedExecution.orphaned.fail_all_with(Processes::ProcessMissingError.new)
         end
       end
+
+      # When a supervised process crashes or exits we need to mark all the
+      # executions it had claimed as failed so that they can be retried
+      # by some other worker.
+      def release_claimed_jobs_by(terminated_process, with_error:)
+        wrap_in_app_executor do
+          if registered_process = SolidQueue::Process.find_by(name: terminated_process.name)
+            registered_process.fail_all_claimed_executions_with(with_error)
+          end
+        end
+      end
   end
 end

data/lib/solid_queue/supervisor/signals.rb

@@ -6,8 +6,8 @@ module SolidQueue
       extend ActiveSupport::Concern
 
       included do
-        before_boot :register_signal_handlers
-        after_shutdown :restore_default_signal_handlers
+        before_boot :register_signal_handlers, if: :standalone?
+        after_shutdown :restore_default_signal_handlers, if: :standalone?
       end
 
       private

data/lib/solid_queue/supervisor.rb

@@ -13,17 +13,21 @@ module SolidQueue
         configuration = Configuration.new(**options)
 
         if configuration.valid?
-          new(configuration).tap(&:start)
+          klass = configuration.mode.fork? ? ForkSupervisor : AsyncSupervisor
+          klass.new(configuration).tap(&:start)
         else
           abort configuration.errors.full_messages.join("\n") + "\nExiting..."
         end
       end
     end
 
+    delegate :mode, :standalone?, to: :configuration
+
     def initialize(configuration)
       @configuration = configuration
-      @forks = {}
+
       @configured_processes = {}
+      @process_instances = {}
 
       super
     end
@@ -43,8 +47,12 @@ module SolidQueue
       run_stop_hooks
     end
 
+    def kind
+      "Supervisor(#{mode})"
+    end
+
     private
-      attr_reader :configuration, :forks, :configured_processes
+      attr_reader :configuration, :configured_processes, :process_instances
 
       def boot
         SolidQueue.instrument(:start_process, process: self) do
@@ -62,11 +70,13 @@ module SolidQueue
         loop do
           break if stopped?
 
-          set_procline
-          process_signal_queue
+          if standalone?
+            set_procline
+            process_signal_queue
+          end
 
           unless stopped?
-            reap_and_replace_terminated_forks
+            check_and_replace_terminated_processes
             interruptible_sleep(1.second)
           end
         end
@@ -77,30 +87,23 @@ module SolidQueue
       def start_process(configured_process)
         process_instance = configured_process.instantiate.tap do |instance|
           instance.supervised_by process
-          instance.mode = :fork
+          instance.mode = mode
         end
 
-        pid = fork do
-          process_instance.start
-        end
+        process_id = process_instance.start
 
-        configured_processes[pid] = configured_process
-        forks[pid] = process_instance
+        configured_processes[process_id] = configured_process
+        process_instances[process_id] = process_instance
       end
 
-      def set_procline
-        procline "supervising #{supervised_processes.join(", ")}"
+      def check_and_replace_terminated_processes
       end
 
       def terminate_gracefully
-        SolidQueue.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: supervised_processes) do |payload|
-          term_forks
-
-          Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
-            reap_terminated_forks
-          end
+        SolidQueue.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: configured_processes.keys) do |payload|
+          perform_graceful_termination
 
-          unless all_forks_terminated?
+          unless all_processes_terminated?
             payload[:shutdown_timeout_exceeded] = true
             terminate_immediately
           end
@@ -108,84 +111,37 @@ module SolidQueue
       end
 
       def terminate_immediately
-        SolidQueue.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: supervised_processes) do
-          quit_forks
-        end
-      end
-
-      def shutdown
-        SolidQueue.instrument(:shutdown_process, process: self) do
-          run_callbacks(:shutdown) do
-            stop_maintenance_task
-          end
+        SolidQueue.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: configured_processes.keys) do
+          perform_immediate_termination
         end
       end
 
-      def sync_std_streams
-        STDOUT.sync = STDERR.sync = true
+      def perform_graceful_termination
+        raise NotImplementedError
       end
 
-      def supervised_processes
-        forks.keys
+      def perform_immediate_termination
+        raise NotImplementedError
       end
 
-      def term_forks
-        signal_processes(forks.keys, :TERM)
+      def all_processes_terminated?
+        raise NotImplementedError
       end
 
-      def quit_forks
-        signal_processes(forks.keys, :QUIT)
-      end
-
-      def reap_and_replace_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          replace_fork(pid, status)
-        end
-      end
-
-      def reap_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          if (terminated_fork = forks.delete(pid)) && (!status.exited? || status.exitstatus > 0)
-            handle_claimed_jobs_by(terminated_fork, status)
-          end
-
-          configured_processes.delete(pid)
-        end
-      rescue SystemCallError
-        # All children already reaped
-      end
-
-      def replace_fork(pid, status)
-        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
-          if terminated_fork = forks.delete(pid)
-            payload[:fork] = terminated_fork
-            handle_claimed_jobs_by(terminated_fork, status)
-
-            start_process(configured_processes.delete(pid))
+      def shutdown
+        SolidQueue.instrument(:shutdown_process, process: self) do
+          run_callbacks(:shutdown) do
+            stop_maintenance_task
           end
         end
       end
 
-      # When a supervised fork crashes or exits we need to mark all the
-      # executions it had claimed as failed so that they can be retried
-      # by some other worker.
-      def handle_claimed_jobs_by(terminated_fork, status)
-        wrap_in_app_executor do
-          if registered_process = SolidQueue::Process.find_by(name: terminated_fork.name)
-            error = Processes::ProcessExitError.new(status)
-            registered_process.fail_all_claimed_executions_with(error)
-          end
-        end
+      def set_procline
+        procline "supervising #{configured_processes.keys.join(", ")}"
       end
 
-      def all_forks_terminated?
-        forks.empty?
+      def sync_std_streams
+        STDOUT.sync = STDERR.sync = true
       end
   end
 end

data/lib/solid_queue/timer.rb

@@ -4,18 +4,18 @@ module SolidQueue
   module Timer
     extend self
 
-    def wait_until(timeout, condition, &block)
+    def wait_until(timeout, condition)
       if timeout > 0
         deadline = monotonic_time_now + timeout
 
         while monotonic_time_now < deadline && !condition.call
           sleep 0.1
-          block.call
+          yield if block_given?
         end
       else
         while !condition.call
           sleep 0.5
-          block.call
+          yield if block_given?
         end
       end
     end

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "1.2.4"
+  VERSION = "1.4.0"
 end