solid_queue 1.4.0 → 1.6.0

data/lib/solid_queue/configuration.rb

@@ -3,10 +3,14 @@
 module SolidQueue
   class Configuration
     include ActiveModel::Model
+    include ActiveModel::Validations::Callbacks
 
-    validate :ensure_configured_processes
-    validate :ensure_valid_recurring_tasks
-    validate :ensure_correctly_sized_thread_pool
+    validate :ensure_configured_processes, :ensure_valid_recurring_tasks
+    validate :ensure_valid_worker_execution_options
+    validate :ensure_fiber_workers_have_required_dependency, :ensure_fiber_workers_use_supported_isolation_level
+    validate :warn_about_incorrectly_sized_database_pool, :warn_about_missing_config_files
+
+    before_validation { warnings.clear }
 
     class Process < Struct.new(:kind, :attributes)
       def instantiate
@@ -35,38 +39,48 @@ module SolidQueue
 
     DEFAULT_CONFIG_FILE_PATH = "config/queue.yml"
     DEFAULT_RECURRING_SCHEDULE_FILE_PATH = "config/recurring.yml"
+    FIBER_QUERY_SCOPED_CONNECTIONS_VERSION = Gem::Version.new("7.2.0")
 
     def initialize(**options)
       @options = options.with_defaults(default_options)
     end
 
     def configured_processes
-      if only_work? then workers
+      if only_work?
+        workers
+      elsif only_recurring?
+        schedulers
       else
         dispatchers + workers + schedulers
       end
     end
 
-    def error_messages
-      if configured_processes.none?
-        "No workers or processed configured. Exiting..."
-      else
-        error_messages = invalid_tasks.map do |task|
-            all_messages = task.errors.full_messages.map { |msg| "\t#{msg}" }.join("\n")
-            "#{task.key}:\n#{all_messages}"
-          end
-          .join("\n")
+    def mode
+      options[:mode].to_s.inquiry
+    end
 
-        "Invalid processes configured:\n#{error_messages}"
-      end
+    def standalone?
+      mode.fork? || options[:standalone]
     end
 
-    def mode
-      @options[:mode].to_s.inquiry
+    def warnings
+      @warnings ||= ActiveModel::Errors.new(self)
     end
 
-    def standalone?
-      mode.fork? || @options[:standalone]
+    def check
+      if valid?
+        warnings.full_messages.each { |warning| $stderr.puts warning }
+        $stdout.puts "Solid Queue configuration is valid."
+
+        true
+      else
+        $stderr.puts "Solid Queue configuration is invalid:"
+        (warnings.full_messages + errors.full_messages).each do |message|
+          message.each_line { |line| $stderr.puts "  #{line.chomp}" }
+        end
+
+        false
+      end
     end
 
     private
@@ -88,10 +102,53 @@ module SolidQueue
         end
       end
 
-      def ensure_correctly_sized_thread_pool
-        if (db_pool_size = SolidQueue::Record.connection_pool&.size) && db_pool_size < estimated_number_of_threads
-          errors.add(:base, "Solid Queue is configured to use #{estimated_number_of_threads} threads but the " +
-            "database connection pool is #{db_pool_size}. Increase it in `config/database.yml`")
+      def warn_about_incorrectly_sized_database_pool
+        db_pool_size = SolidQueue::Record.connection_pool&.size
+
+        if db_pool_size && db_pool_size < estimated_database_pool_size
+          warnings.add(:base, "Warning: Solid Queue needs at least #{estimated_database_pool_size} database connections " \
+            "for the configured workers but the database connection pool is #{db_pool_size}. Increase it in `config/database.yml`")
+        end
+      rescue ActiveRecord::ActiveRecordError
+        # No usable database connection. Skip the pool-size warning in that case.
+      end
+
+      def warn_about_missing_config_files
+        files = [ options[:config_file] ]
+        files << options[:recurring_schedule_file] unless skip_recurring_tasks?
+
+        files.compact.each do |file|
+          unless Pathname.new(file).exist?
+            warnings.add(:base, "Warning: provided configuration file '#{file}' does not exist. Falling back to default configuration.")
+          end
+        end
+      end
+
+      def ensure_valid_worker_execution_options
+        workers_options.each do |options|
+          if options.key?(:threads) && options.key?(:fibers)
+            errors.add(:base, "Workers can specify either `threads` or `fibers`, but not both.")
+          end
+        end
+      end
+
+      def ensure_fiber_workers_have_required_dependency
+        return unless workers_options.any? { |options| fiber_worker?(options) }
+
+        require "async"
+        require "async/semaphore"
+      rescue LoadError
+        errors.add(:base, "Fiber workers require the `async` gem. " \
+          "Add `gem \"async\"` to your Gemfile to configure workers with `fibers`.")
+      end
+
+      def ensure_fiber_workers_use_supported_isolation_level
+        return unless workers_options.any? { |options| fiber_worker?(options) }
+
+        unless ActiveSupport::IsolatedExecutionState.isolation_level == :fiber
+          errors.add(:base, "Fiber workers require fiber-scoped isolated execution state. " \
+            "Set `config.active_support.isolation_level = :fiber` in your Rails configuration " \
+            "(or `ActiveSupport::IsolatedExecutionState.isolation_level = :fiber` outside Rails).")
         end
       end
 
@@ -103,6 +160,7 @@ module SolidQueue
           recurring_schedule_file: Rails.root.join(ENV["SOLID_QUEUE_RECURRING_SCHEDULE"] || DEFAULT_RECURRING_SCHEDULE_FILE_PATH),
           only_work: false,
           only_dispatch: false,
+          only_recurring: ActiveModel::Type::Boolean.new.cast(ENV["SOLID_QUEUE_ONLY_RECURRING"]),
           skip_recurring: ActiveModel::Type::Boolean.new.cast(ENV["SOLID_QUEUE_SKIP_RECURRING"])
         }
       end
@@ -119,6 +177,10 @@ module SolidQueue
         options[:only_dispatch]
       end
 
+      def only_recurring?
+        options[:only_recurring]
+      end
+
       def skip_recurring_tasks?
         options[:skip_recurring] || only_work?
       end
@@ -131,7 +193,8 @@ module SolidQueue
             1
           end
 
-          processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
+          defaults = worker_defaults_for(worker_options)
+          processes.times.map { Process.new(:worker, worker_options.with_defaults(defaults)) }
         end
       end
 
@@ -221,15 +284,46 @@ module SolidQueue
         if file.exist?
           ActiveSupport::ConfigurationFile.parse(file).deep_symbolize_keys
         else
-          puts "[solid_queue] WARNING: Provided configuration file '#{file}' does not exist. Falling back to default configuration."
           {}
         end
       end
 
-      def estimated_number_of_threads
-        # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
-        thread_count = workers_options.map { |options| options.fetch(:threads, WORKER_DEFAULTS[:threads]) }.max
-        (thread_count || 1) + 2
+      def estimated_database_pool_size
+        worker_pool_size = workers_options.map { |options| estimated_database_pool_size_for_worker(options) }.max
+        worker_pool_size || 1
+      end
+
+      def estimated_database_pool_size_for_worker(options)
+        # Connections used to execute jobs + 1 for the worker's polling thread + 1 for the heartbeat task
+        estimated_execution_connections_for_worker(options) + 2
+      end
+
+      def worker_capacity(options)
+        options[:fibers] || options[:threads] || WORKER_DEFAULTS[:threads]
+      end
+
+      def estimated_execution_connections_for_worker(options)
+        fiber_worker?(options) ? fiber_execution_connections_for_worker(options) : worker_capacity(options)
+      end
+
+      def fiber_execution_connections_for_worker(options)
+        fiber_jobs_release_connections_between_queries? ? 1 : worker_capacity(options)
+      end
+
+      def fiber_jobs_release_connections_between_queries?
+        ActiveRecord.gem_version >= FIBER_QUERY_SCOPED_CONNECTIONS_VERSION
+      end
+
+      def fiber_worker?(options)
+        options.key?(:fibers)
+      end
+
+      def worker_defaults_for(options)
+        if fiber_worker?(options)
+          WORKER_DEFAULTS.except(:threads)
+        else
+          WORKER_DEFAULTS
+        end
       end
   end
 end

data/lib/solid_queue/engine.rb

@@ -16,6 +16,12 @@ module SolidQueue
       end
     end
 
+    initializer "solid_queue.time_zone" do |app|
+      unless config.solid_queue.key?(:time_zone)
+        SolidQueue.time_zone = app.config.time_zone
+      end
+    end
+
     initializer "solid_queue.app_executor", before: :run_prepare_callbacks do |app|
       config.solid_queue.app_executor    ||= app.executor
       config.solid_queue.on_thread_error ||= ->(exception) { Rails.error.report(exception, handled: false) }
@@ -37,5 +43,9 @@ module SolidQueue
         include ActiveJob::ConcurrencyControls
       end
     end
+
+    initializer "solid_queue.deprecator" do |app|
+      app.deprecators[:solid_queue] = SolidQueue.deprecator
+    end
   end
 end

data/lib/solid_queue/fiber_pool.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class FiberPool < Pool
+    def initialize(size, on_idle: nil)
+      super
+
+      @state_mutex = Mutex.new
+      @shutdown = false
+      @fatal_error = nil
+      @boot_queue = Thread::Queue.new
+      @pending_executions = Thread::Queue.new
+      @reactor_thread = nil
+    end
+
+    def post(execution)
+      raise_if_fatal_error!
+      raise RuntimeError, "Execution pool is shutting down" if shutdown?
+
+      super
+    end
+
+    def available_capacity
+      raise_if_fatal_error!
+      super
+    end
+
+    def shutdown
+      state_mutex.synchronize do
+        next false if @shutdown
+
+        @shutdown = true
+      end.tap do |shut_down|
+        # Wake the reactor: already-queued executions are drained before the
+        # blocked pop in +wait_for_executions+ returns nil
+        pending_executions.close if shut_down
+      end
+    end
+
+    def shutdown?
+      state_mutex.synchronize { @shutdown }
+    end
+
+    def wait_for_termination(timeout)
+      reactor_thread&.join(timeout)
+    end
+
+    private
+      attr_reader :boot_queue, :pending_executions, :reactor_thread, :state_mutex
+
+      def name
+        @name ||= "solid_queue-fiber-pool-#{object_id}"
+      end
+
+      def schedule(execution)
+        start_reactor_if_needed
+        pending_executions << execution
+      end
+
+      # The reactor thread is started lazily, when the first execution is posted,
+      # so that the pool can be safely built before forking: in the default fork
+      # supervisor mode, workers are instantiated in the supervisor process, and
+      # a thread started there wouldn't survive the fork. The async gem is also
+      # required lazily here, so that setups without fiber workers never load it.
+      def start_reactor_if_needed
+        @reactor_thread ||= begin
+          require "async"
+          require "async/semaphore"
+
+          start_reactor.tap do
+            boot_result = boot_queue.pop
+            raise boot_result if boot_result.is_a?(Exception)
+          end
+        end
+      end
+
+      def start_reactor
+        create_thread do
+          Async do |task|
+            semaphore = Async::Semaphore.new(size, parent: task)
+            boot_queue << :ready
+
+            # The reactor exits when all in-flight execution fibers, children
+            # of this task, have finished
+            wait_for_executions(semaphore)
+          end
+        rescue Exception => error
+          register_fatal_error(error)
+          raise
+        end
+      end
+
+      def wait_for_executions(semaphore)
+        # Thread::Queue#pop is fiber-scheduler-aware: it suspends this fiber, letting
+        # execution fibers run, and wakes the reactor when the poller thread pushes new
+        # work or closes the queue on shutdown, after which it drains any remaining
+        # executions and returns nil
+        while execution = pending_executions.pop
+          semaphore.async(execution) do |_execution_task, scheduled_execution|
+            perform_execution(scheduled_execution)
+          end
+        end
+      end
+
+      def perform_execution(execution)
+        wrap_in_app_executor { execution.perform }
+      rescue Async::Stop => error
+        handle_thread_error(error)
+        register_fatal_error(error)
+      rescue Exception => error
+        handle_thread_error(error)
+      ensure
+        restore_capacity
+      end
+
+      def register_fatal_error(error)
+        state_mutex.synchronize do
+          @fatal_error ||= error
+        end
+
+        boot_queue << error if boot_queue.empty?
+        on_idle&.call
+      end
+
+      def raise_if_fatal_error!
+        error = state_mutex.synchronize { @fatal_error }
+        raise error if error
+      end
+  end
+end

data/lib/solid_queue/fork_supervisor.rb

@@ -31,6 +31,21 @@ module SolidQueue
 
         replace_fork(pid, status)
       end
+
+      check_boot_timeouts
+    end
+
+    def check_boot_timeouts
+      process_instances.each do |pid, instance|
+        terminate_unready_process(pid) if instance.boot_timed_out?
+      end
+    end
+
+    def terminate_unready_process(pid)
+      SolidQueue.instrument(:fork_boot_timeout, process: process_instances[pid], pid: pid) do
+        # A child stuck in boot cannot reach its run loop to stop gracefully
+        signal_process(pid, :KILL)
+      end
     end
 
     def reap_terminated_forks
@@ -38,9 +53,13 @@ module SolidQueue
         pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
         break unless pid
 
-        if (terminated_fork = process_instances.delete(pid)) && (!status.exited? || status.exitstatus.to_i > 0)
-          error = Processes::ProcessExitError.new(status)
-          release_claimed_jobs_by(terminated_fork, with_error: error)
+        if terminated_fork = process_instances.delete(pid)
+          terminated_fork.mark_as_reaped
+
+          if !status.exited? || status.exitstatus.to_i > 0
+            error = Processes::ProcessExitError.new(status)
+            release_claimed_jobs_by(terminated_fork, with_error: error)
+          end
         end
 
         configured_processes.delete(pid)
@@ -52,6 +71,7 @@ module SolidQueue
     def replace_fork(pid, status)
       SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
         if terminated_fork = process_instances.delete(pid)
+          terminated_fork.mark_as_reaped
           payload[:fork] = terminated_fork
           error = Processes::ProcessExitError.new(status)
           release_claimed_jobs_by(terminated_fork, with_error: error)

data/lib/solid_queue/log_subscriber.rb

@@ -161,6 +161,11 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     end
   end
 
+  def fork_boot_timeout(event)
+    process = event.payload[:process]
+    warn formatted_event(event, action: "Terminate #{process.kind} that failed to boot in time", **event.payload.slice(:pid).merge(hostname: process.hostname, name: process.name))
+  end
+
   private
     def formatted_event(event, action:, **attributes)
       "SolidQueue-#{SolidQueue::VERSION} #{action} (#{event.duration.round(1)}ms)  #{formatted_attributes(**attributes)}"

data/lib/solid_queue/pool.rb

@@ -4,51 +4,72 @@ module SolidQueue
   class Pool
     include AppExecutor
 
-    attr_reader :size
+    def self.build(type:, size:, on_idle: nil)
+      SolidQueue.const_get("#{type.to_s.camelize}Pool").new(size, on_idle: on_idle)
+    end
 
-    delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor
+    attr_reader :size
 
     def initialize(size, on_idle: nil)
       @size = size
       @on_idle = on_idle
-      @available_threads = Concurrent::AtomicFixnum.new(size)
+      @available_capacity = size
       @mutex = Mutex.new
     end
 
+    def type
+      self.class.name.demodulize.delete_suffix("Pool").underscore.to_sym
+    end
+
     def post(execution)
-      available_threads.decrement
-
-      Concurrent::Promises.future_on(executor, execution) do |thread_execution|
-        wrap_in_app_executor do
-          thread_execution.perform
-        ensure
-          available_threads.increment
-          mutex.synchronize { on_idle.try(:call) if idle? }
-        end
-      end.on_rejection! do |e|
-        handle_thread_error(e)
+      reserve_capacity!
+
+      begin
+        schedule(execution)
+      rescue Exception
+        restore_capacity
+        raise
       end
     end
 
-    def idle_threads
-      available_threads.value
+    def available_capacity
+      mutex.synchronize { @available_capacity }
     end
 
     def idle?
-      idle_threads > 0
+      available_capacity.positive?
     end
 
     private
-      attr_reader :available_threads, :on_idle, :mutex
+      attr_reader :mutex, :on_idle
+
+      def schedule(execution)
+        raise NotImplementedError
+      end
+
+      def perform_execution(execution)
+        wrap_in_app_executor { execution.perform }
+      rescue Exception => error
+        handle_thread_error(error)
+      ensure
+        restore_capacity
+      end
+
+      def reserve_capacity!
+        mutex.synchronize do
+          raise RuntimeError, "Execution pool is at capacity" if @available_capacity <= 0
 
-      DEFAULT_OPTIONS = {
-        min_threads: 0,
-        idletime: 60,
-        fallback_policy: :abort
-      }
+          @available_capacity -= 1
+        end
+      end
+
+      def restore_capacity
+        should_notify = mutex.synchronize do
+          @available_capacity += 1
+          @available_capacity.positive?
+        end
 
-      def executor
-        @executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size)
+        on_idle&.call if should_notify
       end
   end
 end

data/lib/solid_queue/processes/runnable.rb

@@ -4,7 +4,9 @@ module SolidQueue::Processes
   module Runnable
     include Supervised
 
-    attr_writer :mode
+    def mode=(value)
+      @mode = (value || DEFAULT_MODE).to_s.inquiry
+    end
 
     def start
       run_in_mode do
@@ -29,21 +31,32 @@ module SolidQueue::Processes
       !running_async? || @thread&.alive?
     end
 
+    def boot_timed_out?
+      @boot_guard.timed_out?
+    end
+
+    def mark_as_reaped
+      @boot_guard.close
+    end
+
     private
       DEFAULT_MODE = :async
 
       def mode
-        (@mode || DEFAULT_MODE).to_s.inquiry
+        @mode ||= DEFAULT_MODE.to_s.inquiry
       end
 
       def run_in_mode(&block)
         case
         when running_as_fork?
-          fork(&block)
+          @boot_guard = BootGuards::ForkGuard.new
+          fork(&block).tap { @boot_guard.start }
         when running_async?
+          @boot_guard = BootGuards::NullGuard.new
           @thread = create_thread(&block)
           @thread.object_id
         else
+          @boot_guard = BootGuards::NullGuard.new
           block.call
         end
       end
@@ -57,6 +70,8 @@ module SolidQueue::Processes
             end
           end
         end
+
+        @boot_guard.complete
       end
 
       def shutting_down?
@@ -93,4 +108,76 @@ module SolidQueue::Processes
         mode.fork?
       end
   end
+
+  module BootGuards
+    # Tracks a process that shares memory with its supervisor, whose boot time
+    # doesn't need monitoring.
+    class NullGuard
+      def complete
+        @completed = true
+      end
+
+      def start
+      end
+
+      def completed?
+        @completed
+      end
+
+      def timed_out?
+        false
+      end
+
+      def close
+      end
+    end
+
+    # Tracks a forked process from the moment it's started until its boot
+    # callbacks finish, over a pipe that survives forking: the forked process
+    # writes to it when it's done booting, and its supervisor reads from it to
+    # decide whether the process is taking too long to boot and needs replacing.
+    class ForkGuard
+      def initialize
+        @reader, @writer = IO.pipe
+        @created_at = SolidQueue::Timer.monotonic_time_now
+      end
+
+      # Runs in the forked process when it has finished booting
+      def complete
+        reader.close
+        writer.write(".")
+      rescue Errno::EPIPE
+        # The supervisor stopped waiting while this process finished booting
+      ensure
+        writer.close
+      end
+
+      # Runs in the parent process right after forking
+      def start
+        writer.close
+      end
+
+      # A byte means boot completed; EOF means the process exited before
+      # finishing its boot, and will be replaced when it's reaped
+      def completed?
+        @completed ||= begin
+          completed = reader.read_nonblock(1, exception: false) != :wait_readable
+          reader.close if completed
+          completed
+        end
+      end
+
+      def timed_out?
+        !completed? && SolidQueue::Timer.monotonic_time_now - created_at >= SolidQueue.fork_boot_timeout
+      end
+
+      def close
+        reader.close unless reader.closed?
+        writer.close unless writer.closed?
+      end
+
+      private
+        attr_reader :reader, :writer, :created_at
+    end
+  end
 end

data/lib/solid_queue/scheduler/recurring_schedule.rb

@@ -33,8 +33,8 @@ module SolidQueue
       end
     end
 
-    def schedule_task(task)
-      scheduled_tasks[task.key] = schedule(task)
+    def schedule_task(task, run_at: task.next_time)
+      scheduled_tasks[task.key] = schedule(task, run_at: run_at)
     end
 
     def unschedule_tasks
@@ -99,9 +99,11 @@ module SolidQueue
         dynamic_tasks_enabled? ? RecurringTask.dynamic.to_a : []
       end
 
-      def schedule(task)
-        scheduled_task = Concurrent::ScheduledTask.new(task.delay_from_now, args: [ self, task, task.next_time ]) do |thread_schedule, thread_task, thread_task_run_at|
-          thread_schedule.schedule_task(thread_task)
+      def schedule(task, run_at: task.next_time)
+        delay = [ (run_at - Time.current).to_f, 0.1 ].max
+
+        scheduled_task = Concurrent::ScheduledTask.new(delay, args: [ self, task, run_at ]) do |thread_schedule, thread_task, thread_task_run_at|
+          thread_schedule.schedule_task(thread_task, run_at: thread_task.next_time_after(thread_task_run_at))
 
           wrap_in_app_executor do
             thread_task.enqueue(at: thread_task_run_at)

data/lib/solid_queue/supervisor/signals.rb

@@ -11,7 +11,7 @@ module SolidQueue
       end
 
       private
-        SIGNALS = %i[ QUIT INT TERM ]
+        SIGNALS = Gem.win_platform? ? %i[ INT TERM ] : %i[ QUIT INT TERM ]
 
         def register_signal_handlers
           SIGNALS.each do |signal|