solid_queue 1.1.0 → 1.2.1

data/lib/active_job/concurrency_controls.rb

@@ -5,6 +5,7 @@ module ActiveJob
     extend ActiveSupport::Concern
 
     DEFAULT_CONCURRENCY_GROUP = ->(*) { self.class.name }
+    CONCURRENCY_ON_CONFLICT_BEHAVIOUR = %i[ block discard ]
 
     included do
       class_attribute :concurrency_key, instance_accessor: false
@@ -12,14 +13,16 @@ module ActiveJob
 
       class_attribute :concurrency_limit
       class_attribute :concurrency_duration, default: SolidQueue.default_concurrency_control_period
+      class_attribute :concurrency_on_conflict, default: :block
     end
 
     class_methods do
-      def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period)
+      def limits_concurrency(key:, to: 1, group: DEFAULT_CONCURRENCY_GROUP, duration: SolidQueue.default_concurrency_control_period, on_conflict: :block)
         self.concurrency_key = key
         self.concurrency_limit = to
         self.concurrency_group = group
         self.concurrency_duration = duration
+        self.concurrency_on_conflict = on_conflict.presence_in(CONCURRENCY_ON_CONFLICT_BEHAVIOUR) || :block
       end
     end
 

data/lib/active_job/queue_adapters/solid_queue_adapter.rb

@@ -7,7 +7,10 @@ module ActiveJob
     # To use it set the queue_adapter config to +:solid_queue+.
     #
     #   Rails.application.config.active_job.queue_adapter = :solid_queue
-    class SolidQueueAdapter
+    class SolidQueueAdapter < (Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR == 1 ? Object : AbstractAdapter)
+      class_attribute :stopping, default: false, instance_writer: false
+      SolidQueue.on_worker_stop { self.stopping = true }
+
       def enqueue_after_transaction_commit?
         true
       end

data/lib/generators/solid_queue/install/templates/config/recurring.yml

@@ -1,10 +1,15 @@
-# production:
+# examples:
 #   periodic_cleanup:
 #     class: CleanSoftDeletedRecordsJob
 #     queue: background
 #     args: [ 1000, { batch_size: 500 } ]
 #     schedule: every hour
-#   periodic_command:
+#   periodic_cleanup_with_command:
 #     command: "SoftDeletedRecord.due.delete_all"
 #     priority: 2
 #     schedule: at 5am every day
+
+production:
+  clear_solid_queue_finished_jobs:
+    command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
+    schedule: every hour at minute 12

data/lib/solid_queue/app_executor.rb

@@ -4,7 +4,7 @@ module SolidQueue
   module AppExecutor
     def wrap_in_app_executor(&block)
       if SolidQueue.app_executor
-        SolidQueue.app_executor.wrap(&block)
+        SolidQueue.app_executor.wrap(source: "application.solid_queue", &block)
       else
         yield
       end

data/lib/solid_queue/cli.rb

@@ -13,7 +13,8 @@ module SolidQueue
       banner: "SOLID_QUEUE_RECURRING_SCHEDULE"
 
     class_option :skip_recurring, type: :boolean, default: false,
-      desc: "Whether to skip recurring tasks scheduling"
+      desc: "Whether to skip recurring tasks scheduling",
+      banner: "SOLID_QUEUE_SKIP_RECURRING"
 
     def self.exit_on_failure?
       true

data/lib/solid_queue/configuration.rb

@@ -2,6 +2,12 @@
 
 module SolidQueue
   class Configuration
+    include ActiveModel::Model
+
+    validate :ensure_configured_processes
+    validate :ensure_valid_recurring_tasks
+    validate :ensure_correctly_sized_thread_pool
+
     class Process < Struct.new(:kind, :attributes)
       def instantiate
         "SolidQueue::#{kind.to_s.titleize}".safe_constantize.new(**attributes)
@@ -36,24 +42,60 @@ module SolidQueue
       end
     end
 
-    def max_number_of_threads
-      # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
-      workers_options.map { |options| options[:threads] }.max + 2
+    def error_messages
+      if configured_processes.none?
+        "No workers or processed configured. Exiting..."
+      else
+        error_messages = invalid_tasks.map do |task|
+            all_messages = task.errors.full_messages.map { |msg| "\t#{msg}" }.join("\n")
+            "#{task.key}:\n#{all_messages}"
+          end
+          .join("\n")
+
+        "Invalid processes configured:\n#{error_messages}"
+      end
     end
 
     private
       attr_reader :options
 
+      def ensure_configured_processes
+        unless configured_processes.any?
+          errors.add(:base, "No processes configured")
+        end
+      end
+
+      def ensure_valid_recurring_tasks
+        unless skip_recurring_tasks? || invalid_tasks.none?
+          error_messages = invalid_tasks.map do |task|
+            "- #{task.key}: #{task.errors.full_messages.join(", ")}"
+          end
+
+          errors.add(:base, "Invalid recurring tasks:\n#{error_messages.join("\n")}")
+        end
+      end
+
+      def ensure_correctly_sized_thread_pool
+        if (db_pool_size = SolidQueue::Record.connection_pool&.size) && db_pool_size < estimated_number_of_threads
+          errors.add(:base, "Solid Queue is configured to use #{estimated_number_of_threads} threads but the " +
+            "database connection pool is #{db_pool_size}. Increase it in `config/database.yml`")
+        end
+      end
+
       def default_options
         {
           config_file: Rails.root.join(ENV["SOLID_QUEUE_CONFIG"] || DEFAULT_CONFIG_FILE_PATH),
           recurring_schedule_file: Rails.root.join(ENV["SOLID_QUEUE_RECURRING_SCHEDULE"] || DEFAULT_RECURRING_SCHEDULE_FILE_PATH),
           only_work: false,
           only_dispatch: false,
-          skip_recurring: false
+          skip_recurring: ActiveModel::Type::Boolean.new.cast(ENV["SOLID_QUEUE_SKIP_RECURRING"])
         }
       end
 
+      def invalid_tasks
+        recurring_tasks.select(&:invalid?)
+      end
+
       def only_work?
         options[:only_work]
       end
@@ -99,8 +141,8 @@ module SolidQueue
 
       def recurring_tasks
         @recurring_tasks ||= recurring_tasks_config.map do |id, options|
-          RecurringTask.from_configuration(id, **options)
-        end.select(&:valid?)
+          RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule)
+        end.compact
       end
 
       def processes_config
@@ -111,7 +153,9 @@ module SolidQueue
       end
 
       def recurring_tasks_config
-        @recurring_tasks_config ||= config_from options[:recurring_schedule_file]
+        @recurring_tasks_config ||= begin
+          config_from options[:recurring_schedule_file]
+        end
       end
 
 
@@ -147,5 +191,11 @@ module SolidQueue
           {}
         end
       end
+
+      def estimated_number_of_threads
+        # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
+        thread_count = workers_options.map { |options| options.fetch(:threads, WORKER_DEFAULTS[:threads]) }.max
+        (thread_count || 1) + 2
+      end
   end
 end

data/lib/solid_queue/dispatcher.rb

@@ -2,10 +2,14 @@
 
 module SolidQueue
   class Dispatcher < Processes::Poller
-    attr_accessor :batch_size, :concurrency_maintenance
+    include LifecycleHooks
+    attr_reader :batch_size
 
+    after_boot :run_start_hooks
     after_boot :start_concurrency_maintenance
     before_shutdown :stop_concurrency_maintenance
+    before_shutdown :run_stop_hooks
+    after_shutdown :run_exit_hooks
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
@@ -22,9 +26,12 @@ module SolidQueue
     end
 
     private
+      attr_reader :concurrency_maintenance
+
       def poll
         batch = dispatch_next_batch
-        batch.size
+
+        batch.zero? ? polling_interval : 0.seconds
       end
 
       def dispatch_next_batch
@@ -37,20 +44,12 @@ module SolidQueue
         concurrency_maintenance&.start
       end
 
-      def schedule_recurring_tasks
-        recurring_schedule.schedule_tasks
-      end
-
       def stop_concurrency_maintenance
         concurrency_maintenance&.stop
       end
 
-      def unschedule_recurring_tasks
-        recurring_schedule.unschedule_tasks
-      end
-
       def all_work_completed?
-        SolidQueue::ScheduledExecution.none? && recurring_schedule.empty?
+        SolidQueue::ScheduledExecution.none?
       end
 
       def set_procline

data/lib/solid_queue/lifecycle_hooks.rb

@@ -5,7 +5,7 @@ module SolidQueue
     extend ActiveSupport::Concern
 
     included do
-      mattr_reader :lifecycle_hooks, default: { start: [], stop: [] }
+      mattr_reader :lifecycle_hooks, default: { start: [], stop: [], exit: [] }
     end
 
     class_methods do
@@ -17,7 +17,12 @@ module SolidQueue
         self.lifecycle_hooks[:stop] << block
       end
 
+      def on_exit(&block)
+        self.lifecycle_hooks[:exit] << block
+      end
+
       def clear_hooks
+        self.lifecycle_hooks[:exit] = []
         self.lifecycle_hooks[:start] = []
         self.lifecycle_hooks[:stop] = []
       end
@@ -32,9 +37,13 @@ module SolidQueue
         run_hooks_for :stop
       end
 
+      def run_exit_hooks
+        run_hooks_for :exit
+      end
+
       def run_hooks_for(event)
         self.class.lifecycle_hooks.fetch(event, []).each do |block|
-          block.call
+            block.call(self)
         rescue Exception => exception
           handle_thread_error(exception)
         end

data/lib/solid_queue/log_subscriber.rb

@@ -145,6 +145,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def replace_fork(event)
+    supervisor_pid = event.payload[:supervisor_pid]
     status = event.payload[:status]
     attributes = event.payload.slice(:pid).merge \
       status: (status.exitstatus || "no exit status set"),
@@ -155,7 +156,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
 
     if replaced_fork = event.payload[:fork]
       info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname, name: replaced_fork.name))
-    else
+    elsif supervisor_pid != 1 # Running Docker, possibly having some processes that have been reparented
       warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes)
     end
   end

data/lib/solid_queue/pool.rb

@@ -18,20 +18,16 @@ module SolidQueue
     def post(execution)
       available_threads.decrement
 
-      future = Concurrent::Future.new(args: [ execution ], executor: executor) do |thread_execution|
+      Concurrent::Promises.future_on(executor, execution) do |thread_execution|
         wrap_in_app_executor do
           thread_execution.perform
         ensure
           available_threads.increment
           mutex.synchronize { on_idle.try(:call) if idle? }
         end
+      end.on_rejection! do |e|
+        handle_thread_error(e)
       end
-
-      future.add_observer do |_, _, error|
-        handle_thread_error(error) if error
-      end
-
-      future.execute
     end
 
     def idle_threads

data/lib/solid_queue/processes/base.rb

@@ -4,7 +4,8 @@ module SolidQueue
   module Processes
     class Base
       include Callbacks # Defines callbacks needed by other concerns
-      include AppExecutor, Registrable, Interruptible, Procline
+      include AppExecutor, Registrable, Procline
+      prepend Interruptible
 
       attr_reader :name
 

data/lib/solid_queue/processes/interruptible.rb

@@ -2,28 +2,39 @@
 
 module SolidQueue::Processes
   module Interruptible
+    def initialize(...)
+      super
+      @self_pipe = create_self_pipe
+    end
+
     def wake_up
       interrupt
     end
 
     private
+      SELF_PIPE_BLOCK_SIZE = 11
+
+      attr_reader :self_pipe
 
       def interrupt
-        queue << true
+        self_pipe[:writer].write_nonblock(".")
+      rescue Errno::EAGAIN, Errno::EINTR
+        # Ignore writes that would block and retry
+        # if another signal arrived while writing
+        retry
       end
 
       def interruptible_sleep(time)
-        # Invoking from the main thread can result in a 35% slowdown (at least when running the test suite).
-        # Using some form of Async (Futures) addresses this performance issue.
-        Concurrent::Promises.future(time) do |timeout|
-          if timeout > 0 && queue.pop(timeout:)
-            queue.clear
-          end
-        end.value
+        if time > 0 && self_pipe[:reader].wait_readable(time)
+          loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) }
+        end
+      rescue Errno::EAGAIN, Errno::EINTR, IO::EWOULDBLOCKWaitReadable
       end
 
-      def queue
-        @queue ||= Queue.new
+      # Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html)
+      def create_self_pipe
+        reader, writer = IO.pipe
+        { reader: reader, writer: writer }
       end
   end
 end

data/lib/solid_queue/processes/poller.rb

@@ -25,11 +25,11 @@ module SolidQueue::Processes
         loop do
           break if shutting_down?
 
-          wrap_in_app_executor do
-            unless poll > 0
-              interruptible_sleep(polling_interval)
-            end
+          delay = wrap_in_app_executor do
+            poll
           end
+
+          interruptible_sleep(delay)
         end
       ensure
         SolidQueue.instrument(:shutdown_process, process: self) do

data/lib/solid_queue/processes/process_pruned_error.rb

@@ -4,7 +4,7 @@ module SolidQueue
   module Processes
     class ProcessPrunedError < RuntimeError
       def initialize(last_heartbeat_at)
-        super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
+        super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at})")
       end
     end
   end

data/lib/solid_queue/processes/registrable.rb

@@ -7,8 +7,7 @@ module SolidQueue::Processes
     included do
       after_boot :register, :launch_heartbeat
 
-      before_shutdown :stop_heartbeat
-      after_shutdown :deregister
+      after_shutdown :stop_heartbeat, :deregister
     end
 
     def process_id

data/lib/solid_queue/scheduler.rb

@@ -3,11 +3,15 @@
 module SolidQueue
   class Scheduler < Processes::Base
     include Processes::Runnable
+    include LifecycleHooks
 
-    attr_accessor :recurring_schedule
+    attr_reader :recurring_schedule
 
+    after_boot :run_start_hooks
     after_boot :schedule_recurring_tasks
     before_shutdown :unschedule_recurring_tasks
+    before_shutdown :run_stop_hooks
+    after_shutdown :run_exit_hooks
 
     def initialize(recurring_tasks:, **options)
       @recurring_schedule = RecurringSchedule.new(recurring_tasks)

data/lib/solid_queue/supervisor.rb

@@ -5,15 +5,17 @@ module SolidQueue
     include LifecycleHooks
     include Maintenance, Signals, Pidfiled
 
+    after_shutdown :run_exit_hooks
+
     class << self
       def start(**options)
         SolidQueue.supervisor = true
         configuration = Configuration.new(**options)
 
-        if configuration.configured_processes.any?
+        if configuration.valid?
           new(configuration).tap(&:start)
         else
-          abort "No workers or processed configured. Exiting..."
+          abort configuration.errors.full_messages.join("\n") + "\nExiting..."
         end
       end
     end
@@ -170,8 +172,11 @@ module SolidQueue
         end
       end
 
+      # When a supervised fork crashes or exits we need to mark all the
+      # executions it had claimed as failed so that they can be retried
+      # by some other worker.
       def handle_claimed_jobs_by(terminated_fork, status)
-        if registered_process = process.supervisees.find_by(name: terminated_fork.name)
+        if registered_process = SolidQueue::Process.find_by(name: terminated_fork.name)
           error = Processes::ProcessExitError.new(status)
           registered_process.fail_all_claimed_executions_with(error)
         end

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "1.1.0"
+  VERSION = "1.2.1"
 end

data/lib/solid_queue/worker.rb

@@ -6,13 +6,16 @@ module SolidQueue
 
     after_boot :run_start_hooks
     before_shutdown :run_stop_hooks
+    after_shutdown :run_exit_hooks
 
-    attr_accessor :queues, :pool
+    attr_reader :queues, :pool
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
 
-      @queues = Array(options[:queues])
+      # Ensure that the queues array is deep frozen to prevent accidental modification
+      @queues = Array(options[:queues]).map(&:freeze).freeze
+
       @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
 
       super(**options)
@@ -29,7 +32,7 @@ module SolidQueue
             pool.post(execution)
           end
 
-          executions.size
+          pool.idle? ? polling_interval : 10.minutes
         end
       end
 

data/lib/solid_queue.rb

@@ -41,14 +41,20 @@ module SolidQueue
   mattr_accessor :clear_finished_jobs_after, default: 1.day
   mattr_accessor :default_concurrency_control_period, default: 3.minutes
 
-  delegate :on_start, :on_stop, to: Supervisor
+  delegate :on_start, :on_stop, :on_exit, to: Supervisor
 
-  def on_worker_start(...)
-    Worker.on_start(...)
-  end
+  [ Dispatcher, Scheduler, Worker ].each do |process|
+    define_singleton_method(:"on_#{process.name.demodulize.downcase}_start") do |&block|
+      process.on_start(&block)
+    end
+
+    define_singleton_method(:"on_#{process.name.demodulize.downcase}_stop") do |&block|
+      process.on_stop(&block)
+    end
 
-  def on_worker_stop(...)
-    Worker.on_stop(...)
+    define_singleton_method(:"on_#{process.name.demodulize.downcase}_exit") do |&block|
+      process.on_exit(&block)
+    end
   end
 
   def supervisor?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-05 00:00:00.000000000 Z
+date: 2025-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -84,16 +84,30 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +234,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.0
 description: Database-backed Active Job backend.
 email:
 - rosa@37signals.com
@@ -307,15 +335,8 @@ metadata:
   homepage_uri: https://github.com/rails/solid_queue
   source_code_uri: https://github.com/rails/solid_queue
 post_install_message: |
-  Upgrading to Solid Queue 0.9.0? There are some breaking changes about how recurring tasks are configured.
-
-  Upgrading to Solid Queue 0.8.0 from < 0.6.0? You need to upgrade to 0.6.0 first.
-
-  Upgrading to Solid Queue 0.4.x, 0.5.x, 0.6.x or 0.7.x? There are some breaking changes about how Solid Queue is started,
-  configuration and new migrations.
-
-  --> Check https://github.com/rails/solid_queue/blob/main/UPGRADING.md
-  for upgrade instructions.
+  Upgrading from Solid Queue < 1.0? Check details on breaking changes and upgrade instructions
+  --> https://github.com/rails/solid_queue/blob/main/UPGRADING.md
 rdoc_options: []
 require_paths:
 - lib
@@ -323,7 +344,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="