solid_queue 1.1.2 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17a05d42c432fb64bc429ef13665309e61fa0221d0c7e7534e1ae688772a8ed4
-  data.tar.gz: 7173644a027051dfcd4911b0611995bac6686c9a78cd104836c5b3fbccf6d9dc
+  metadata.gz: 778d9495c79b9b8af00416fd1980250fe6c3213043cbfd31eba5eb5e2795ad13
+  data.tar.gz: 14f929171d65334300648a5f80e59b9f8245034119cfd436d37dae545210c97b
 SHA512:
-  metadata.gz: fb0400861a8946176b1ed8d63cc5504743696ec456c637b459c59489897fcce388e4defba72ecaf061e70362965792474037aef4bcdee48bb071d9f22002611e
-  data.tar.gz: f7b4f080b7b20b716950ff80ce5eff7bb1cc68c153bb65e2ef797044222dfcf52a5717e9a640817a946a54713ca164fb20ed7acbbc90bfec1effe57015c3d872
+  metadata.gz: cbbd8113e179c49ffcfe707aa38c96f31ac812822d380e7a6739daced30ee029046ec19cf2ea70c81e19c0f1a819412454f1f4eb1a4cfad7cf5fa8b0f88e3021
+  data.tar.gz: c324b1127bbad96191d3c2e268058dd1cec7c29cffe45a511b0899f3e028e86b7ad0cf0638c93100393316bde1e429ab28c828a381c0692b8558c100f296b6f6

data/README.md

@@ -9,6 +9,7 @@ Solid Queue can be used with SQL databases such as MySQL, PostgreSQL or SQLite,
 ## Table of contents
 
 - [Installation](#installation)
+  - [Usage in development and other non-production environments](#usage-in-development-and-other-non-production-environments)
   - [Single database configuration](#single-database-configuration)
   - [Incremental adoption](#incremental-adoption)
   - [High performance requirements](#high-performance-requirements)
@@ -38,6 +39,8 @@ Solid Queue is configured by default in new Rails 8 applications. But if you're
 1. `bundle add solid_queue`
 2. `bin/rails solid_queue:install`
 
+(Note: The minimum supported version of Rails is 7.1 and Ruby is 3.1.6.)
+
 This will configure Solid Queue as the production Active Job backend, create the configuration files `config/queue.yml` and `config/recurring.yml`, and create the `db/queue_schema.rb`. It'll also create a `bin/jobs` executable wrapper that you can use to start Solid Queue.
 
 Once you've done that, you will then have to add the configuration for the queue database in `config/database.yml`. If you're using SQLite, it'll look like this:
@@ -84,7 +87,7 @@ For example, if you're using SQLite in development, update `database.yml` as fol
 
 ```diff
 development:
-  primary:
++ primary:
     <<: *default
     database: storage/development.sqlite3
 +  queue:
@@ -310,7 +313,7 @@ and then remove the paused ones. Pausing in general should be something rare, us
 Do this:
 
 ```yml
-queues: background, backend
+queues: [ background, backend ]
 ```
 
 instead of this:
@@ -372,9 +375,11 @@ In Solid queue, you can hook into two different points in the supervisor's life:
 - `start`: after the supervisor has finished booting and right before it forks workers and dispatchers.
 - `stop`: after receiving a signal (`TERM`, `INT` or `QUIT`) and right before starting graceful or immediate shutdown.
 
-And into two different points in a worker's life:
-- `worker_start`: after the worker has finished booting and right before it starts the polling loop.
-- `worker_stop`: after receiving a signal (`TERM`, `INT` or `QUIT`) and right before starting graceful or immediate shutdown (which is just `exit!`).
+And into two different points in the worker's, dispatcher's and scheduler's life:
+- `(worker|dispatcher|scheduler)_start`: after the worker/dispatcher/scheduler has finished booting and right before it starts the polling loop or loading the recurring schedule.
+- `(worker|dispatcher|scheduler)_stop`: after receiving a signal (`TERM`, `INT` or `QUIT`) and right before starting graceful or immediate shutdown (which is just `exit!`).
+
+Each of these hooks has an instance of the supervisor/worker/dispatcher/scheduler yielded to the block so that you may read its configuration for logging or metrics reporting purposes.
 
 You can use the following methods with a block to do this:
 ```ruby
@@ -383,12 +388,30 @@ SolidQueue.on_stop
 
 SolidQueue.on_worker_start
 SolidQueue.on_worker_stop
+
+SolidQueue.on_dispatcher_start
+SolidQueue.on_dispatcher_stop
+
+SolidQueue.on_scheduler_start
+SolidQueue.on_scheduler_stop
 ```
 
 For example:
 ```ruby
-SolidQueue.on_start { start_metrics_server }
-SolidQueue.on_stop { stop_metrics_server }
+SolidQueue.on_start do |supervisor|
+  MyMetricsReporter.process_name = supervisor.name
+
+  start_metrics_server
+end
+
+SolidQueue.on_stop do |_supervisor|
+  stop_metrics_server
+end
+
+SolidQueue.on_worker_start do |worker|
+  MyMetricsReporter.process_name = worker.name
+  MyMetricsReporter.queues = worker.queues.join(',')
+end
 ```
 
 These can be called several times to add multiple hooks, but it needs to happen before Solid Queue is started. An initializer would be a good place to do this.

data/app/models/solid_queue/claimed_execution.rb

@@ -92,7 +92,7 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
 
   private
     def execute
-      ActiveJob::Base.execute(job.arguments)
+      ActiveJob::Base.execute(job.arguments.merge("provider_job_id" => job.id))
       Result.new(true, nil)
     rescue Exception => e
       Result.new(false, e)

data/app/models/solid_queue/scheduled_execution.rb

@@ -14,7 +14,7 @@ module SolidQueue
       def dispatch_next_batch(batch_size)
         transaction do
           job_ids = next_batch(batch_size).non_blocking_lock.pluck(:job_id)
-          if job_ids.empty? then []
+          if job_ids.empty? then 0
           else
             SolidQueue.instrument(:dispatch_scheduled, batch_size: batch_size) do |payload|
               payload[:size] = dispatch_jobs(job_ids)

data/lib/solid_queue/configuration.rb

@@ -141,7 +141,7 @@ module SolidQueue
 
       def recurring_tasks
         @recurring_tasks ||= recurring_tasks_config.map do |id, options|
-          RecurringTask.from_configuration(id, **options) if options.has_key?(:schedule)
+          RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule)
         end.compact
       end
 
@@ -153,7 +153,9 @@ module SolidQueue
       end
 
       def recurring_tasks_config
-        @recurring_tasks_config ||= config_from options[:recurring_schedule_file]
+        @recurring_tasks_config ||= begin
+          config_from options[:recurring_schedule_file]
+        end
       end
 
 

data/lib/solid_queue/dispatcher.rb

@@ -2,10 +2,14 @@
 
 module SolidQueue
   class Dispatcher < Processes::Poller
-    attr_accessor :batch_size, :concurrency_maintenance
+    include LifecycleHooks
+    attr_reader :batch_size
 
+    after_boot :run_start_hooks
     after_boot :start_concurrency_maintenance
     before_shutdown :stop_concurrency_maintenance
+    before_shutdown :run_stop_hooks
+    after_shutdown :run_exit_hooks
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
@@ -22,10 +26,12 @@ module SolidQueue
     end
 
     private
+      attr_reader :concurrency_maintenance
+
       def poll
         batch = dispatch_next_batch
 
-        batch.size.zero? ? polling_interval : 0.seconds
+        batch.zero? ? polling_interval : 0.seconds
       end
 
       def dispatch_next_batch
@@ -38,20 +44,12 @@ module SolidQueue
         concurrency_maintenance&.start
       end
 
-      def schedule_recurring_tasks
-        recurring_schedule.schedule_tasks
-      end
-
       def stop_concurrency_maintenance
         concurrency_maintenance&.stop
       end
 
-      def unschedule_recurring_tasks
-        recurring_schedule.unschedule_tasks
-      end
-
       def all_work_completed?
-        SolidQueue::ScheduledExecution.none? && recurring_schedule.empty?
+        SolidQueue::ScheduledExecution.none?
       end
 
       def set_procline

data/lib/solid_queue/engine.rb

@@ -37,5 +37,13 @@ module SolidQueue
         include ActiveJob::ConcurrencyControls
       end
     end
+
+    initializer "solid_queue.include_interruptible_concern" do
+      if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2")
+        SolidQueue::Processes::Base.include SolidQueue::Processes::Interruptible
+      else
+        SolidQueue::Processes::Base.include SolidQueue::Processes::OgInterruptible
+      end
+    end
   end
 end

data/lib/solid_queue/lifecycle_hooks.rb

@@ -5,7 +5,7 @@ module SolidQueue
     extend ActiveSupport::Concern
 
     included do
-      mattr_reader :lifecycle_hooks, default: { start: [], stop: [] }
+      mattr_reader :lifecycle_hooks, default: { start: [], stop: [], exit: [] }
     end
 
     class_methods do
@@ -17,7 +17,12 @@ module SolidQueue
         self.lifecycle_hooks[:stop] << block
       end
 
+      def on_exit(&block)
+        self.lifecycle_hooks[:exit] << block
+      end
+
       def clear_hooks
+        self.lifecycle_hooks[:exit] = []
         self.lifecycle_hooks[:start] = []
         self.lifecycle_hooks[:stop] = []
       end
@@ -32,9 +37,13 @@ module SolidQueue
         run_hooks_for :stop
       end
 
+      def run_exit_hooks
+        run_hooks_for :exit
+      end
+
       def run_hooks_for(event)
         self.class.lifecycle_hooks.fetch(event, []).each do |block|
-          block.call
+            block.call(self)
         rescue Exception => exception
           handle_thread_error(exception)
         end

data/lib/solid_queue/pool.rb

@@ -18,20 +18,16 @@ module SolidQueue
     def post(execution)
       available_threads.decrement
 
-      future = Concurrent::Future.new(args: [ execution ], executor: executor) do |thread_execution|
+      Concurrent::Promises.future_on(executor, execution) do |thread_execution|
         wrap_in_app_executor do
           thread_execution.perform
         ensure
           available_threads.increment
           mutex.synchronize { on_idle.try(:call) if idle? }
         end
+      end.on_rejection! do |e|
+        handle_thread_error(e)
       end
-
-      future.add_observer do |_, _, error|
-        handle_thread_error(error) if error
-      end
-
-      future.execute
     end
 
     def idle_threads

data/lib/solid_queue/processes/base.rb

@@ -4,7 +4,7 @@ module SolidQueue
   module Processes
     class Base
       include Callbacks # Defines callbacks needed by other concerns
-      include AppExecutor, Registrable, Interruptible, Procline
+      include AppExecutor, Registrable, Procline
 
       attr_reader :name
 

data/lib/solid_queue/processes/interruptible.rb

@@ -2,6 +2,8 @@
 
 module SolidQueue::Processes
   module Interruptible
+    include SolidQueue::AppExecutor
+
     def wake_up
       interrupt
     end
@@ -13,17 +15,19 @@ module SolidQueue::Processes
       end
 
       # Sleeps for 'time'.  Can be interrupted asynchronously and return early via wake_up.
-      # @param time [Numeric] the time to sleep. 0 returns immediately.
-      # @return [true, nil]
-      # * returns `true` if an interrupt was requested via #wake_up between the
-      #   last call to `interruptible_sleep` and now, resulting in an early return.
-      # * returns `nil` if it slept the full `time` and was not interrupted.
+      # @param time [Numeric, Duration] the time to sleep. 0 returns immediately.
       def interruptible_sleep(time)
         # Invoking this from the main thread may result in significant slowdown.
         # Utilizing asynchronous execution (Futures) addresses this performance issue.
         Concurrent::Promises.future(time) do |timeout|
-          queue.pop(timeout:).tap { queue.clear }
+          queue.clear unless queue.pop(timeout:).nil?
+        end.on_rejection! do |e|
+          wrapped_exception = RuntimeError.new("Interruptible#interruptible_sleep - #{e.class}: #{e.message}")
+          wrapped_exception.set_backtrace(e.backtrace)
+          handle_thread_error(wrapped_exception)
         end.value
+
+        nil
       end
 
       def queue

data/lib/solid_queue/processes/og_interruptible.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  # The original implementation of Interruptible that works
+  # with Ruby 3.1 and earlier
+  module OgInterruptible
+    def wake_up
+      interrupt
+    end
+
+    private
+      SELF_PIPE_BLOCK_SIZE = 11
+
+      def interrupt
+        self_pipe[:writer].write_nonblock(".")
+      rescue Errno::EAGAIN, Errno::EINTR
+        # Ignore writes that would block and retry
+        # if another signal arrived while writing
+        retry
+      end
+
+      def interruptible_sleep(time)
+        if time > 0 && self_pipe[:reader].wait_readable(time)
+          loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) }
+        end
+      rescue Errno::EAGAIN, Errno::EINTR
+      end
+
+      # Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html)
+      def self_pipe
+        @self_pipe ||= create_self_pipe
+      end
+
+      def create_self_pipe
+        reader, writer = IO.pipe
+        { reader: reader, writer: writer }
+      end
+  end
+end

data/lib/solid_queue/processes/process_pruned_error.rb

@@ -4,7 +4,7 @@ module SolidQueue
   module Processes
     class ProcessPrunedError < RuntimeError
       def initialize(last_heartbeat_at)
-        super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
+        super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at})")
       end
     end
   end

data/lib/solid_queue/scheduler.rb

@@ -3,11 +3,15 @@
 module SolidQueue
   class Scheduler < Processes::Base
     include Processes::Runnable
+    include LifecycleHooks
 
-    attr_accessor :recurring_schedule
+    attr_reader :recurring_schedule
 
+    after_boot :run_start_hooks
     after_boot :schedule_recurring_tasks
     before_shutdown :unschedule_recurring_tasks
+    before_shutdown :run_stop_hooks
+    after_shutdown :run_exit_hooks
 
     def initialize(recurring_tasks:, **options)
       @recurring_schedule = RecurringSchedule.new(recurring_tasks)

data/lib/solid_queue/supervisor.rb

@@ -5,6 +5,8 @@ module SolidQueue
     include LifecycleHooks
     include Maintenance, Signals, Pidfiled
 
+    after_shutdown :run_exit_hooks
+
     class << self
       def start(**options)
         SolidQueue.supervisor = true

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "1.1.2"
+  VERSION = "1.1.4"
 end

data/lib/solid_queue/worker.rb

@@ -6,14 +6,16 @@ module SolidQueue
 
     after_boot :run_start_hooks
     before_shutdown :run_stop_hooks
+    after_shutdown :run_exit_hooks
 
-
-    attr_accessor :queues, :pool
+    attr_reader :queues, :pool
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
 
-      @queues = Array(options[:queues])
+      # Ensure that the queues array is deep frozen to prevent accidental modification
+      @queues = Array(options[:queues]).map(&:freeze).freeze
+
       @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
 
       super(**options)

data/lib/solid_queue.rb

@@ -41,14 +41,20 @@ module SolidQueue
   mattr_accessor :clear_finished_jobs_after, default: 1.day
   mattr_accessor :default_concurrency_control_period, default: 3.minutes
 
-  delegate :on_start, :on_stop, to: Supervisor
+  delegate :on_start, :on_stop, :on_exit, to: Supervisor
 
-  def on_worker_start(...)
-    Worker.on_start(...)
-  end
+  [ Dispatcher, Scheduler, Worker ].each do |process|
+    define_singleton_method(:"on_#{process.name.demodulize.downcase}_start") do |&block|
+      process.on_start(&block)
+    end
+
+    define_singleton_method(:"on_#{process.name.demodulize.downcase}_stop") do |&block|
+      process.on_stop(&block)
+    end
 
-  def on_worker_stop(...)
-    Worker.on_stop(...)
+    define_singleton_method(:"on_#{process.name.demodulize.downcase}_exit") do |&block|
+      process.on_exit(&block)
+    end
   end
 
   def supervisor?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.4
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-27 00:00:00.000000000 Z
+date: 2025-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -220,6 +220,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.6.0
 description: Database-backed Active Job backend.
 email:
 - rosa@37signals.com
@@ -281,6 +295,7 @@ files:
 - lib/solid_queue/processes/base.rb
 - lib/solid_queue/processes/callbacks.rb
 - lib/solid_queue/processes/interruptible.rb
+- lib/solid_queue/processes/og_interruptible.rb
 - lib/solid_queue/processes/poller.rb
 - lib/solid_queue/processes/process_exit_error.rb
 - lib/solid_queue/processes/process_missing_error.rb
@@ -307,15 +322,8 @@ metadata:
   homepage_uri: https://github.com/rails/solid_queue
   source_code_uri: https://github.com/rails/solid_queue
 post_install_message: |
-  Upgrading to Solid Queue 0.9.0? There are some breaking changes about how recurring tasks are configured.
-
-  Upgrading to Solid Queue 0.8.0 from < 0.6.0? You need to upgrade to 0.6.0 first.
-
-  Upgrading to Solid Queue 0.4.x, 0.5.x, 0.6.x or 0.7.x? There are some breaking changes about how Solid Queue is started,
-  configuration and new migrations.
-
-  --> Check https://github.com/rails/solid_queue/blob/main/UPGRADING.md
-  for upgrade instructions.
+  Upgrading from Solid Queue < 1.0? Check details on breaking changes and upgrade instructions
+  --> https://github.com/rails/solid_queue/blob/main/UPGRADING.md
 rdoc_options: []
 require_paths:
 - lib
@@ -323,7 +331,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="