solid_queue 1.5.1 → 1.7.0

data/lib/solid_queue/fiber_pool.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class FiberPool < Pool
+    def initialize(size, on_idle: nil)
+      super
+
+      @state_mutex = Mutex.new
+      @shutdown = false
+      @fatal_error = nil
+      @boot_queue = Thread::Queue.new
+      @pending_executions = Thread::Queue.new
+      @reactor_thread = nil
+    end
+
+    def post(execution)
+      raise_if_fatal_error!
+      raise RuntimeError, "Execution pool is shutting down" if shutdown?
+
+      super
+    end
+
+    def available_capacity
+      raise_if_fatal_error!
+      super
+    end
+
+    def shutdown
+      state_mutex.synchronize do
+        next false if @shutdown
+
+        @shutdown = true
+      end.tap do |shut_down|
+        # Wake the reactor: already-queued executions are drained before the
+        # blocked pop in +wait_for_executions+ returns nil
+        pending_executions.close if shut_down
+      end
+    end
+
+    def shutdown?
+      state_mutex.synchronize { @shutdown }
+    end
+
+    def wait_for_termination(timeout)
+      reactor_thread&.join(timeout)
+    end
+
+    private
+      attr_reader :boot_queue, :pending_executions, :reactor_thread, :state_mutex
+
+      def name
+        @name ||= "solid_queue-fiber-pool-#{object_id}"
+      end
+
+      def schedule(execution)
+        start_reactor_if_needed
+        pending_executions << execution
+      end
+
+      # The reactor thread is started lazily, when the first execution is posted,
+      # so that the pool can be safely built before forking: in the default fork
+      # supervisor mode, workers are instantiated in the supervisor process, and
+      # a thread started there wouldn't survive the fork. The async gem is also
+      # required lazily here, so that setups without fiber workers never load it.
+      def start_reactor_if_needed
+        @reactor_thread ||= begin
+          require "async"
+          require "async/semaphore"
+
+          start_reactor.tap do
+            boot_result = boot_queue.pop
+            raise boot_result if boot_result.is_a?(Exception)
+          end
+        end
+      end
+
+      def start_reactor
+        create_thread do
+          Async do |task|
+            semaphore = Async::Semaphore.new(size, parent: task)
+            boot_queue << :ready
+
+            # The reactor exits when all in-flight execution fibers, children
+            # of this task, have finished
+            wait_for_executions(semaphore)
+          end
+        rescue Exception => error
+          register_fatal_error(error)
+          raise
+        end
+      end
+
+      def wait_for_executions(semaphore)
+        # Thread::Queue#pop is fiber-scheduler-aware: it suspends this fiber, letting
+        # execution fibers run, and wakes the reactor when the poller thread pushes new
+        # work or closes the queue on shutdown, after which it drains any remaining
+        # executions and returns nil
+        while execution = pending_executions.pop
+          semaphore.async(execution) do |_execution_task, scheduled_execution|
+            perform_execution(scheduled_execution)
+          end
+        end
+      end
+
+      def perform_execution(execution)
+        wrap_in_app_executor { execution.perform }
+      rescue Async::Stop => error
+        handle_thread_error(error)
+        register_fatal_error(error)
+      rescue Exception => error
+        handle_thread_error(error)
+      ensure
+        restore_capacity
+      end
+
+      def register_fatal_error(error)
+        state_mutex.synchronize do
+          @fatal_error ||= error
+        end
+
+        boot_queue << error if boot_queue.empty?
+        on_idle&.call
+      end
+
+      def raise_if_fatal_error!
+        error = state_mutex.synchronize { @fatal_error }
+        raise error if error
+      end
+  end
+end

data/lib/solid_queue/fork_supervisor.rb

@@ -57,8 +57,7 @@ module SolidQueue
           terminated_fork.mark_as_reaped
 
           if !status.exited? || status.exitstatus.to_i > 0
-            error = Processes::ProcessExitError.new(status)
-            release_claimed_jobs_by(terminated_fork, with_error: error)
+            attempt_to_release_claimed_jobs_by(terminated_fork, status)
           end
         end
 
@@ -73,14 +72,24 @@ module SolidQueue
         if terminated_fork = process_instances.delete(pid)
           terminated_fork.mark_as_reaped
           payload[:fork] = terminated_fork
-          error = Processes::ProcessExitError.new(status)
-          release_claimed_jobs_by(terminated_fork, with_error: error)
+
+          attempt_to_release_claimed_jobs_by(terminated_fork, status)
 
           start_process(configured_processes.delete(pid))
         end
       end
     end
 
+    # The database may be unreachable — likely the same reason the fork
+    # terminated. Neither starting a replacement nor shutting down can depend
+    # on it: the jobs claimed by the terminated fork will be failed when its
+    # stale registration is pruned once the database is back.
+    def attempt_to_release_claimed_jobs_by(terminated_fork, status)
+      release_claimed_jobs_by(terminated_fork, with_error: Processes::ProcessExitError.new(status))
+    rescue StandardError => error
+      handle_thread_error(error)
+    end
+
     def all_processes_terminated?
       process_instances.empty?
     end

data/lib/solid_queue/log_subscriber.rb

@@ -16,7 +16,10 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def fail_many_claimed(event)
-    warn formatted_event(event, action: "Fail claimed jobs", **event.payload.slice(:job_ids, :process_ids))
+    attributes = event.payload.slice(:job_ids, :process_ids)
+    attributes[:error] = formatted_error(event.payload[:error]) if event.payload[:error]
+
+    warn formatted_event(event, action: "Fail claimed jobs", **attributes)
   end
 
   def release_claimed(event)
@@ -39,6 +42,18 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     debug formatted_event(event, action: "Discard job", **event.payload.slice(:job_id, :status))
   end
 
+  def finish_batch(event)
+    info formatted_event(event, action: "Finish batch", **event.payload.slice(:batch_id, :total_jobs, :completed_jobs, :failed_jobs))
+  end
+
+  def sweep_stalled_batches(event)
+    debug formatted_event(event, action: "Sweep stalled batches", **event.payload.slice(:stale_executions, :finished_batches, :started_batches))
+  end
+
+  def batch_progress_error(event)
+    error formatted_event(event, action: "Error updating batch progress", **event.payload.slice(:batch_id, :job_id), error: formatted_error(event.payload[:error]))
+  end
+
   def release_many_blocked(event)
     debug formatted_event(event, action: "Unblock jobs", **event.payload.slice(:limit, :size))
   end

data/lib/solid_queue/pool.rb

@@ -4,51 +4,72 @@ module SolidQueue
   class Pool
     include AppExecutor
 
-    attr_reader :size
+    def self.build(type:, size:, on_idle: nil)
+      SolidQueue.const_get("#{type.to_s.camelize}Pool").new(size, on_idle: on_idle)
+    end
 
-    delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor
+    attr_reader :size
 
     def initialize(size, on_idle: nil)
       @size = size
       @on_idle = on_idle
-      @available_threads = Concurrent::AtomicFixnum.new(size)
+      @available_capacity = size
       @mutex = Mutex.new
     end
 
+    def type
+      self.class.name.demodulize.delete_suffix("Pool").underscore.to_sym
+    end
+
     def post(execution)
-      available_threads.decrement
-
-      Concurrent::Promises.future_on(executor, execution) do |thread_execution|
-        wrap_in_app_executor do
-          thread_execution.perform
-        ensure
-          available_threads.increment
-          mutex.synchronize { on_idle.try(:call) if idle? }
-        end
-      end.on_rejection! do |e|
-        handle_thread_error(e)
+      reserve_capacity!
+
+      begin
+        schedule(execution)
+      rescue Exception
+        restore_capacity
+        raise
       end
     end
 
-    def idle_threads
-      available_threads.value
+    def available_capacity
+      mutex.synchronize { @available_capacity }
     end
 
     def idle?
-      idle_threads > 0
+      available_capacity.positive?
     end
 
     private
-      attr_reader :available_threads, :on_idle, :mutex
+      attr_reader :mutex, :on_idle
+
+      def schedule(execution)
+        raise NotImplementedError
+      end
+
+      def perform_execution(execution)
+        wrap_in_app_executor { execution.perform }
+      rescue Exception => error
+        handle_thread_error(error)
+      ensure
+        restore_capacity
+      end
+
+      def reserve_capacity!
+        mutex.synchronize do
+          raise RuntimeError, "Execution pool is at capacity" if @available_capacity <= 0
 
-      DEFAULT_OPTIONS = {
-        min_threads: 0,
-        idletime: 60,
-        fallback_policy: :abort
-      }
+          @available_capacity -= 1
+        end
+      end
+
+      def restore_capacity
+        should_notify = mutex.synchronize do
+          @available_capacity += 1
+          @available_capacity.positive?
+        end
 
-      def executor
-        @executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size)
+        on_idle&.call if should_notify
       end
   end
 end

data/lib/solid_queue/processes/runnable.rb

@@ -50,7 +50,7 @@ module SolidQueue::Processes
         case
         when running_as_fork?
           @boot_guard = BootGuards::ForkGuard.new
-          fork(&block).tap { @boot_guard.start }
+          create_fork(&block).tap { @boot_guard.start }
         when running_async?
           @boot_guard = BootGuards::NullGuard.new
           @thread = create_thread(&block)
@@ -64,10 +64,7 @@ module SolidQueue::Processes
       def boot
         SolidQueue.instrument(:start_process, process: self) do
           run_callbacks(:boot) do
-            if running_as_fork?
-              register_signal_handlers
-              set_procline
-            end
+            set_procline if running_as_fork?
           end
         end
 

data/lib/solid_queue/processes/supervised.rb

@@ -25,6 +25,13 @@ module SolidQueue::Processes
         supervisor.present?
       end
 
+      def create_fork(&block)
+        fork do
+          register_signal_handlers
+          block.call
+        end
+      end
+
       def register_signal_handlers
         %w[ INT TERM ].each do |signal|
           trap(signal) do

data/lib/solid_queue/supervisor/signals.rb

@@ -29,6 +29,9 @@ module SolidQueue
         end
 
         def process_signal_queue
+          # Embedded supervisors don't own their process's signals
+          return unless standalone?
+
           while signal = signal_queue.shift
             handle_signal(signal)
           end

data/lib/solid_queue/supervisor.rb

@@ -39,9 +39,13 @@ module SolidQueue
       run_start_hooks
 
       start_processes
-      launch_maintenance_task
 
-      supervise
+      if stopped?
+        shutdown
+      else
+        launch_maintenance_task
+        supervise
+      end
     end
 
     def stop
@@ -65,27 +69,33 @@ module SolidQueue
       end
 
       def start_processes
-        configuration.configured_processes.each { |configured_process| start_process(configured_process) }
+        configuration.configured_processes.each do |configured_process|
+          # Honour signals that arrive during boot or start hooks: a queued TERM
+          # stops us here, before starting children, instead of in #supervise,
+          # after all of them have been started
+          break if time_to_stop?
+
+          start_process(configured_process)
+        end
       end
 
       def supervise
-        loop do
-          break if stopped?
-
-          if standalone?
-            set_procline
-            process_signal_queue
-          end
-
-          unless stopped?
-            check_and_replace_terminated_processes
-            interruptible_sleep(1.second)
-          end
+        until time_to_stop?
+          set_procline
+          check_and_replace_terminated_processes
+          interruptible_sleep(1.second)
         end
       ensure
         shutdown
       end
 
+      # Process any signals queued while we were busy and report whether
+      # we've been asked to stop
+      def time_to_stop?
+        process_signal_queue
+        stopped?
+      end
+
       def start_process(configured_process)
         process_instance = configured_process.instantiate.tap do |instance|
           instance.supervised_by process
@@ -139,7 +149,10 @@ module SolidQueue
       end
 
       def set_procline
-        procline "supervising #{configured_processes.keys.join(", ")}"
+        # Embedded supervisors don't own their process's title
+        if standalone?
+          procline "supervising #{configured_processes.keys.join(", ")}"
+        end
       end
 
       def sync_std_streams

data/lib/solid_queue/thread_pool.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class ThreadPool < Pool
+    delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor
+
+    private
+      DEFAULT_OPTIONS = {
+        min_threads: 0,
+        idletime: 60,
+        fallback_policy: :abort
+      }
+
+      def schedule(execution)
+        Concurrent::Promises.future_on(executor, execution) do |thread_execution|
+          perform_execution(thread_execution)
+        end.on_rejection! do |error|
+          # Backstop for errors raised outside perform_execution's own rescue,
+          # such as when restoring capacity or waking up the worker
+          handle_thread_error(error)
+        end
+      end
+
+      def executor
+        @executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size)
+      end
+  end
+end

data/lib/solid_queue/version.rb

@@ -1,5 +1,5 @@
 module SolidQueue
-  VERSION = "1.5.1"
+  VERSION = "1.7.0"
 
   def self.next_major_version
     Gem::Version.new(VERSION).segments.first + 1

data/lib/solid_queue/worker.rb

@@ -11,18 +11,24 @@ module SolidQueue
     attr_reader :queues, :pool
 
     def initialize(**options)
+      execution_pool_type = options.key?(:fibers) ? :fiber : :thread
+
       options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
+      execution_pool_size = execution_pool_type == :fiber ? options[:fibers] : options[:threads]
 
       # Ensure that the queues array is deep frozen to prevent accidental modification
       @queues = Array(options[:queues]).map(&:freeze).freeze
 
-      @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
+      @pool = Pool.build \
+        type: execution_pool_type,
+        size: execution_pool_size,
+        on_idle: -> { wake_up }
 
       super(**options)
     end
 
     def metadata
-      super.merge(queues: queues.join(","), thread_pool_size: pool.size)
+      super.merge(queues: queues.join(","), pool_type: pool.type, pool_size: pool.size)
     end
 
     private
@@ -38,7 +44,7 @@ module SolidQueue
 
       def claim_executions
         with_polling_volume do
-          SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process_id)
+          SolidQueue::ReadyExecution.claim(queues, pool.available_capacity, process_id)
         end
       end
 

data/lib/solid_queue.rb

@@ -5,6 +5,7 @@ require "solid_queue/engine"
 
 require "active_job"
 require "active_job/queue_adapters"
+require "active_job/batch_id"
 
 require "active_support"
 require "active_support/core_ext/numeric/time"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-07-29 00:00:00.000000000 Z
+date: 2026-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.24'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.24'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -273,13 +287,21 @@ files:
 - README.md
 - UPGRADING.md
 - app/jobs/solid_queue/recurring_job.rb
+- app/models/solid_queue/batch.rb
+- app/models/solid_queue/batch/callbacks.rb
+- app/models/solid_queue/batch/clearable.rb
+- app/models/solid_queue/batch/status.rb
+- app/models/solid_queue/batch/sweepable.rb
+- app/models/solid_queue/batch_execution.rb
 - app/models/solid_queue/blocked_execution.rb
 - app/models/solid_queue/claimed_execution.rb
 - app/models/solid_queue/execution.rb
 - app/models/solid_queue/execution/dispatching.rb
 - app/models/solid_queue/execution/job_attributes.rb
 - app/models/solid_queue/failed_execution.rb
+- app/models/solid_queue/failed_execution/batchable.rb
 - app/models/solid_queue/job.rb
+- app/models/solid_queue/job/batchable.rb
 - app/models/solid_queue/job/clearable.rb
 - app/models/solid_queue/job/concurrency_controls.rb
 - app/models/solid_queue/job/executable.rb
@@ -301,6 +323,7 @@ files:
 - app/models/solid_queue/scheduled_execution.rb
 - app/models/solid_queue/semaphore.rb
 - config/routes.rb
+- lib/active_job/batch_id.rb
 - lib/active_job/concurrency_controls.rb
 - lib/active_job/queue_adapters/solid_queue_adapter.rb
 - lib/generators/solid_queue/install/USAGE
@@ -309,6 +332,7 @@ files:
 - lib/generators/solid_queue/install/templates/config/queue.yml
 - lib/generators/solid_queue/install/templates/config/recurring.yml
 - lib/generators/solid_queue/install/templates/db/queue_schema.rb
+- lib/generators/solid_queue/update/templates/db/add_batches_to_solid_queue.rb
 - lib/generators/solid_queue/update/update_generator.rb
 - lib/puma/plugin/solid_queue.rb
 - lib/solid_queue.rb
@@ -318,7 +342,9 @@ files:
 - lib/solid_queue/configuration.rb
 - lib/solid_queue/dispatcher.rb
 - lib/solid_queue/dispatcher/concurrency_maintenance.rb
+- lib/solid_queue/dispatcher/maintenance.rb
 - lib/solid_queue/engine.rb
+- lib/solid_queue/fiber_pool.rb
 - lib/solid_queue/fork_supervisor.rb
 - lib/solid_queue/lifecycle_hooks.rb
 - lib/solid_queue/log_subscriber.rb
@@ -343,6 +369,7 @@ files:
 - lib/solid_queue/supervisor/pidfiled.rb
 - lib/solid_queue/supervisor/signals.rb
 - lib/solid_queue/tasks.rb
+- lib/solid_queue/thread_pool.rb
 - lib/solid_queue/timer.rb
 - lib/solid_queue/version.rb
 - lib/solid_queue/worker.rb