solid_queue 1.5.1 → 1.7.0

data/app/models/solid_queue/batch_execution.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class BatchExecution < Execution
+    self.assumable_attributes_from_job = [ :batch_id ]
+
+    belongs_to :batch
+
+    scope :with_finished_jobs, -> { joins(:job).merge(SolidQueue::Job.finished) }
+    scope :with_failed_jobs, -> { joins(job: :failed_execution) }
+
+    after_commit :finish_batch, on: :destroy
+
+    class << self
+      def create_all_from_jobs(jobs)
+        jobs.select(&:batched?).group_by(&:batch_id).each do |batch_id, jobs_in_batch|
+          # Update the counter first: inserting tracking rows takes a shared FK lock on
+          # the batch row, then incrementing can deadlock concurrent MySQL adders.
+          if attempt_to_update_total_jobs(batch_id, jobs_in_batch)
+            super jobs_in_batch
+          else
+            raise Batch::AlreadyFinished, "Can't add jobs into an already finished batch"
+          end
+        end
+      end
+
+      private
+        def attempt_to_update_total_jobs(batch_id, jobs)
+          new_jobs_count = count_new_jobs_among(jobs)
+          updated = SolidQueue::Batch.where(id: batch_id).unfinished.update_all([ "total_jobs = total_jobs + ?", new_jobs_count ])
+          updated > 0
+        end
+
+        # A job that has executed before was already counted when it first joined
+        # the batch: retries keep their active_job_id and batch across re-enqueues.
+        # This might undercount jobs whose retries switch to another batch, but that
+        # should be a rare enough case. The counter is used only for report/info, so
+        # we favour simplicity here
+        def count_new_jobs_among(jobs)
+          jobs.reject { |job| job.arguments["executions"].to_i > 0 }.map(&:active_job_id).uniq.size
+        end
+    end
+
+    private
+      def finish_batch
+        # Skip the serialized callback and metadata columns on this hot path
+        if batch = Batch.select(:id, :finished_at, :enqueued_at).find_by(id: batch_id)
+          batch.finish
+        end
+      end
+  end
+end

data/app/models/solid_queue/claimed_execution.rb

@@ -48,6 +48,7 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
           payload[:process_ids] = executions.map(&:process_id).uniq
           payload[:job_ids] = executions.map(&:job_id).uniq
           payload[:size] = executions.size
+          payload[:error] = error
         end
       end
     end

data/app/models/solid_queue/failed_execution/batchable.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class FailedExecution
+    # A FailedExecution is created only after retries are exhausted, when the
+    # job stops counting as pending in its batch.
+    module Batchable
+      extend ActiveSupport::Concern
+
+      included do
+        after_create :destroy_job_batch_execution, if: -> { Batch.migrated? && job.batch_id? }
+      end
+
+      private
+        def destroy_job_batch_execution
+          job.batch_execution&.destroy!
+        rescue ActiveRecord::ActiveRecordError => e
+          SolidQueue.instrument(:batch_progress_error, batch_id: job.batch_id, job_id: job.id, error: e)
+        end
+    end
+  end
+end

data/app/models/solid_queue/failed_execution.rb

@@ -2,7 +2,7 @@
 
 module SolidQueue
   class FailedExecution < Execution
-    include Dispatching
+    include Dispatching, Batchable
 
     serialize :error, coder: JSON
 

data/app/models/solid_queue/job/batchable.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module Batchable
+      extend ActiveSupport::Concern
+
+      included do
+        belongs_to :batch, optional: true
+        has_one :batch_execution
+
+        after_create :create_batch_execution, if: :batched?
+        after_update :update_batch_progress, if: :batched?
+        before_destroy :destroy_batch_execution, if: :batched?
+      end
+
+      class_methods do
+        def batch_all(jobs)
+          BatchExecution.create_all_from_jobs(jobs) if Batch.migrated?
+        end
+      end
+
+      # Also guards against the batches schema not being installed: without
+      # its migration, jobs don't even have a batch_id.
+      def batched?
+        Batch.migrated? && batch_id?
+      end
+
+      private
+        def create_batch_execution
+          BatchExecution.create_all_from_jobs([ self ])
+        end
+
+        def update_batch_progress
+          return unless saved_change_to_finished_at? && finished_at.present?
+
+          batch_execution&.destroy!
+        rescue ActiveRecord::ActiveRecordError => e
+          SolidQueue.instrument(:batch_progress_error, batch_id: batch_id, job_id: id, error: e)
+        end
+
+        # Destroy through Active Record instead of relying on the foreign
+        # key's cascade, so destroying the tracking row retries the batch
+        # completion check.
+        def destroy_batch_execution
+          batch_execution&.destroy!
+        end
+    end
+  end
+end

data/app/models/solid_queue/job/executable.rb

@@ -18,6 +18,9 @@ module SolidQueue
 
       class_methods do
         def prepare_all_for_execution(jobs)
+          # Track before dispatch so conflict-discarded jobs count like single enqueues.
+          batch_all(jobs)
+
           due, not_yet_due = jobs.partition(&:due?)
           dispatch_all(due) + schedule_all(not_yet_due)
         end
@@ -78,7 +81,8 @@ module SolidQueue
 
       def finished!
         if SolidQueue.preserve_finished_jobs?
-          touch(:finished_at)
+          # update! rather than touch so the batch tracking callbacks run
+          update!(finished_at: Time.current)
         else
           destroy!
         end

data/app/models/solid_queue/job.rb

@@ -4,13 +4,19 @@ module SolidQueue
   class Job < Record
     class EnqueueError < StandardError; end
 
-    include Executable, Clearable, Recurrable
+    include Executable, Clearable, Recurrable, Batchable
 
     serialize :arguments, coder: JSON
 
     class << self
       def enqueue_all(active_jobs)
-        active_jobs.each { |job| job.scheduled_at ||= Time.current }
+        # Bulk enqueues bypass ActiveJob#enqueue, so batch membership is captured here
+        current_batch_id = Batch.current_batch_id
+
+        active_jobs.each do |job|
+          job.scheduled_at ||= Time.current
+          job.batch_id = current_batch_id || job.batch_id
+        end
         active_jobs_by_job_id = active_jobs.index_by(&:job_id)
 
         transaction do
@@ -63,7 +69,9 @@ module SolidQueue
             class_name: active_job.class.name,
             arguments: active_job.serialize,
             concurrency_key: active_job.concurrency_key
-          }
+          }.tap do |attributes|
+            attributes[:batch_id] = active_job.batch_id if Batch.migrated?
+          end
         end
     end
   end

data/lib/active_job/batch_id.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Inspired by active_job/core.rb docs
+# https://github.com/rails/rails/blob/1c2529b9a6ba5a1eff58be0d0373d7d9d401015b/activejob/lib/active_job/core.rb#L136
+module ActiveJob
+  module BatchId
+    extend ActiveSupport::Concern
+
+    included do
+      attr_accessor :batch_id
+      attr_accessor :callback_batch_id
+    end
+
+    # Seed membership at construction for enqueue paths deferred until after the
+    # batch context ends, notably bulk enqueue. Enqueueing inside another batch
+    # rebinds membership; enqueueing without a batch preserves it.
+    # SolidQueue::Job.enqueue_all repeats this capture because bulk enqueue
+    # bypasses #enqueue.
+    def initialize(*arguments, **kwargs)
+      super
+      self.batch_id = SolidQueue::Batch.current_batch_id if solid_queue_job?
+    end
+
+    def enqueue(options = {})
+      self.batch_id = SolidQueue::Batch.current_batch_id || batch_id if solid_queue_job?
+      super
+    end
+
+    def serialize
+      super.tap do |data|
+        data["batch_id"] = batch_id if batch_id
+        data["callback_batch_id"] = callback_batch_id if callback_batch_id
+      end
+    end
+
+    def deserialize(job_data)
+      super
+      self.batch_id = job_data["batch_id"]
+      self.callback_batch_id = job_data["callback_batch_id"]
+    end
+
+    def batch
+      batch_id_to_load = callback_batch_id || batch_id
+      return if batch_id_to_load.nil?
+      return @batch if defined?(@batch) && @loaded_batch_id == batch_id_to_load
+
+      @loaded_batch_id = batch_id_to_load
+      @batch = SolidQueue::Batch.find_by(id: batch_id_to_load)
+    end
+
+    private
+
+      def solid_queue_job?
+        self.class.queue_adapter_name == "solid_queue"
+      end
+  end
+end

data/lib/generators/solid_queue/install/templates/db/queue_schema.rb

@@ -37,7 +37,9 @@ ActiveRecord::Schema[7.1].define(version: 1) do
     t.string "concurrency_key"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.bigint "batch_id"
     t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id"
+    t.index [ "batch_id" ], name: "index_solid_queue_jobs_on_batch_id"
     t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name"
     t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at"
     t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering"
@@ -120,6 +122,35 @@ ActiveRecord::Schema[7.1].define(version: 1) do
     t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true
   end
 
+  create_table "solid_queue_batches", force: :cascade do |t|
+    t.string "active_job_batch_id"
+    t.string "description"
+    t.text "on_finish"
+    t.text "on_success"
+    t.text "on_failure"
+    t.text "metadata"
+    t.integer "total_jobs", default: 0, null: false
+    t.integer "completed_jobs", default: 0, null: false
+    t.integer "failed_jobs", default: 0, null: false
+    t.datetime "enqueued_at"
+    t.datetime "finished_at"
+    t.datetime "failed_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index [ "active_job_batch_id" ], name: "index_solid_queue_batches_on_active_job_batch_id", unique: true
+    t.index [ "finished_at" ], name: "index_solid_queue_batches_on_finished_at"
+  end
+
+  create_table "solid_queue_batch_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.bigint "batch_id", null: false
+    t.datetime "created_at", null: false
+    t.index [ "job_id" ], name: "index_solid_queue_batch_executions_on_job_id", unique: true
+    t.index [ "batch_id" ], name: "index_solid_queue_batch_executions_on_batch_id"
+  end
+
+  add_foreign_key "solid_queue_batch_executions", "solid_queue_batches", column: "batch_id", on_delete: :cascade
+  add_foreign_key "solid_queue_batch_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
   add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
   add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
   add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade

data/lib/generators/solid_queue/update/templates/db/add_batches_to_solid_queue.rb

@@ -0,0 +1,39 @@
+class AddBatchesToSolidQueue < ActiveRecord::Migration[7.1]
+  def change
+    # Fresh installs create all of this with the base schema, so skip
+    # anything that already exists
+    add_column :solid_queue_jobs, :batch_id, :bigint, if_not_exists: true
+    add_index :solid_queue_jobs, :batch_id, if_not_exists: true
+
+    create_table :solid_queue_batches, if_not_exists: true do |t|
+      t.string :active_job_batch_id
+      t.string :description
+      t.text :on_finish
+      t.text :on_success
+      t.text :on_failure
+      t.text :metadata
+      t.integer :total_jobs, default: 0, null: false
+      t.integer :completed_jobs, default: 0, null: false
+      t.integer :failed_jobs, default: 0, null: false
+      t.datetime :enqueued_at
+      t.datetime :finished_at
+      t.datetime :failed_at
+      t.datetime :created_at, null: false
+      t.datetime :updated_at, null: false
+
+      t.index :active_job_batch_id, unique: true
+      t.index :finished_at
+    end
+
+    create_table :solid_queue_batch_executions, if_not_exists: true do |t|
+      t.bigint :job_id, null: false
+      t.bigint :batch_id, null: false
+      t.datetime :created_at, null: false
+
+      t.index :job_id, unique: true
+      t.index :batch_id
+      t.foreign_key :solid_queue_batches, column: :batch_id, on_delete: :cascade
+      t.foreign_key :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    end
+  end
+end

data/lib/solid_queue/configuration.rb

@@ -6,7 +6,9 @@ module SolidQueue
     include ActiveModel::Validations::Callbacks
 
     validate :ensure_configured_processes, :ensure_valid_recurring_tasks
-    validate :warn_about_incorrectly_sized_thread_pool, :warn_about_missing_config_files
+    validate :ensure_valid_worker_execution_options
+    validate :ensure_fiber_workers_have_required_dependency, :ensure_fiber_workers_use_supported_isolation_level
+    validate :warn_about_incorrectly_sized_database_pool, :warn_about_missing_config_files
 
     before_validation { warnings.clear }
 
@@ -27,7 +29,8 @@ module SolidQueue
       batch_size: 500,
       polling_interval: 1,
       concurrency_maintenance: true,
-      concurrency_maintenance_interval: 600
+      concurrency_maintenance_interval: 600,
+      batch_maintenance: true
     }
 
     SCHEDULER_DEFAULTS = {
@@ -37,6 +40,7 @@ module SolidQueue
 
     DEFAULT_CONFIG_FILE_PATH = "config/queue.yml"
     DEFAULT_RECURRING_SCHEDULE_FILE_PATH = "config/recurring.yml"
+    FIBER_QUERY_SCOPED_CONNECTIONS_VERSION = Gem::Version.new("7.2.0")
 
     def initialize(**options)
       @options = options.with_defaults(default_options)
@@ -99,12 +103,12 @@ module SolidQueue
         end
       end
 
-      def warn_about_incorrectly_sized_thread_pool
+      def warn_about_incorrectly_sized_database_pool
         db_pool_size = SolidQueue::Record.connection_pool&.size
 
-        if db_pool_size && db_pool_size < estimated_number_of_threads
-          warnings.add(:base, "Warning: Solid Queue is configured to use #{estimated_number_of_threads} threads but the " \
-            "database connection pool is #{db_pool_size}. Increase it in `config/database.yml`")
+        if db_pool_size && db_pool_size < estimated_database_pool_size
+          warnings.add(:base, "Warning: Solid Queue needs at least #{estimated_database_pool_size} database connections " \
+            "for the configured workers but the database connection pool is #{db_pool_size}. Increase it in `config/database.yml`")
         end
       rescue ActiveRecord::ActiveRecordError
         # No usable database connection. Skip the pool-size warning in that case.
@@ -121,6 +125,34 @@ module SolidQueue
         end
       end
 
+      def ensure_valid_worker_execution_options
+        workers_options.each do |options|
+          if options.key?(:threads) && options.key?(:fibers)
+            errors.add(:base, "Workers can specify either `threads` or `fibers`, but not both.")
+          end
+        end
+      end
+
+      def ensure_fiber_workers_have_required_dependency
+        return unless workers_options.any? { |options| fiber_worker?(options) }
+
+        require "async"
+        require "async/semaphore"
+      rescue LoadError
+        errors.add(:base, "Fiber workers require the `async` gem. " \
+          "Add `gem \"async\"` to your Gemfile to configure workers with `fibers`.")
+      end
+
+      def ensure_fiber_workers_use_supported_isolation_level
+        return unless workers_options.any? { |options| fiber_worker?(options) }
+
+        unless ActiveSupport::IsolatedExecutionState.isolation_level == :fiber
+          errors.add(:base, "Fiber workers require fiber-scoped isolated execution state. " \
+            "Set `config.active_support.isolation_level = :fiber` in your Rails configuration " \
+            "(or `ActiveSupport::IsolatedExecutionState.isolation_level = :fiber` outside Rails).")
+        end
+      end
+
       def default_options
         {
           mode: ENV["SOLID_QUEUE_SUPERVISOR_MODE"] || :fork,
@@ -162,7 +194,8 @@ module SolidQueue
             1
           end
 
-          processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
+          defaults = worker_defaults_for(worker_options)
+          processes.times.map { Process.new(:worker, worker_options.with_defaults(defaults)) }
         end
       end
 
@@ -256,10 +289,42 @@ module SolidQueue
         end
       end
 
-      def estimated_number_of_threads
-        # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
-        thread_count = workers_options.map { |options| options.fetch(:threads, WORKER_DEFAULTS[:threads]) }.max
-        (thread_count || 1) + 2
+      def estimated_database_pool_size
+        worker_pool_size = workers_options.map { |options| estimated_database_pool_size_for_worker(options) }.max
+        worker_pool_size || 1
+      end
+
+      def estimated_database_pool_size_for_worker(options)
+        # Connections used to execute jobs + 1 for the worker's polling thread + 1 for the heartbeat task
+        estimated_execution_connections_for_worker(options) + 2
+      end
+
+      def worker_capacity(options)
+        options[:fibers] || options[:threads] || WORKER_DEFAULTS[:threads]
+      end
+
+      def estimated_execution_connections_for_worker(options)
+        fiber_worker?(options) ? fiber_execution_connections_for_worker(options) : worker_capacity(options)
+      end
+
+      def fiber_execution_connections_for_worker(options)
+        fiber_jobs_release_connections_between_queries? ? 1 : worker_capacity(options)
+      end
+
+      def fiber_jobs_release_connections_between_queries?
+        ActiveRecord.gem_version >= FIBER_QUERY_SCOPED_CONNECTIONS_VERSION
+      end
+
+      def fiber_worker?(options)
+        options.key?(:fibers)
+      end
+
+      def worker_defaults_for(options)
+        if fiber_worker?(options)
+          WORKER_DEFAULTS.except(:threads)
+        else
+          WORKER_DEFAULTS
+        end
       end
   end
 end

data/lib/solid_queue/dispatcher/concurrency_maintenance.rb

@@ -1,44 +1,11 @@
 # frozen_string_literal: true
 
 module SolidQueue
-  class Dispatcher::ConcurrencyMaintenance
-    include AppExecutor
-
-    attr_reader :interval, :batch_size
-
+  # Kept for compatibility: concurrency maintenance runs on the shared
+  # Dispatcher::Maintenance timer, together with batch maintenance.
+  class Dispatcher::ConcurrencyMaintenance < Dispatcher::Maintenance
     def initialize(interval, batch_size)
-      @interval = interval
-      @batch_size = batch_size
-    end
-
-    def start
-      @concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: interval) do
-        expire_semaphores
-        unblock_blocked_executions
-      end
-
-      @concurrency_maintenance_task.add_observer do |_, _, error|
-        handle_thread_error(error) if error
-      end
-
-      @concurrency_maintenance_task.execute
-    end
-
-    def stop
-      @concurrency_maintenance_task&.shutdown
+      super(interval, batch_size, concurrency: true, batches: false)
     end
-
-    private
-      def expire_semaphores
-        wrap_in_app_executor do
-          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
-        end
-      end
-
-      def unblock_blocked_executions
-        wrap_in_app_executor do
-          BlockedExecution.unblock(batch_size)
-        end
-      end
   end
 end

data/lib/solid_queue/dispatcher/maintenance.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Dispatcher::Maintenance
+    include AppExecutor
+
+    attr_reader :interval, :batch_size
+
+    def initialize(interval, batch_size, concurrency:, batches:)
+      @interval = interval
+      @batch_size = batch_size
+      @concurrency = concurrency
+      @batches = batches
+    end
+
+    def concurrency?
+      @concurrency
+    end
+
+    def batches?
+      @batches
+    end
+
+    def metadata
+      { concurrency_maintenance_interval: (interval if concurrency?), batch_maintenance: batches? }
+    end
+
+    def start
+      @maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: interval) do
+        if concurrency?
+          expire_semaphores
+          unblock_blocked_executions
+        end
+
+        sweep_stalled_batches if batches?
+      end
+
+      @maintenance_task.add_observer do |_, _, error|
+        handle_thread_error(error) if error
+      end
+
+      @maintenance_task.execute
+    end
+
+    def stop
+      @maintenance_task&.shutdown
+    end
+
+    private
+      def expire_semaphores
+        wrap_in_app_executor do
+          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
+        end
+      end
+
+      def unblock_blocked_executions
+        wrap_in_app_executor do
+          BlockedExecution.unblock(batch_size)
+        end
+      end
+
+      def sweep_stalled_batches
+        wrap_in_app_executor do
+          if Batch.migrated?
+            Batch.sweep_stalled(batch_size: batch_size)
+          else
+            warn_once_about_pending_batch_migrations
+          end
+        end
+      end
+
+      def warn_once_about_pending_batch_migrations
+        unless @warned_about_pending_migrations
+          Batch.warn_about_pending_migrations
+          @warned_about_pending_migrations = true
+        end
+      end
+  end
+end

data/lib/solid_queue/dispatcher.rb

@@ -7,8 +7,8 @@ module SolidQueue
     attr_reader :batch_size
 
     after_boot :run_start_hooks
-    after_boot :start_concurrency_maintenance
-    before_shutdown :stop_concurrency_maintenance
+    after_boot :start_maintenance
+    before_shutdown :stop_maintenance
     before_shutdown :run_stop_hooks
     after_shutdown :run_exit_hooks
 
@@ -17,17 +17,21 @@ module SolidQueue
 
       @batch_size = options[:batch_size]
 
-      @concurrency_maintenance = ConcurrencyMaintenance.new(options[:concurrency_maintenance_interval], options[:batch_size]) if options[:concurrency_maintenance]
+      # Run both maintenance routines on one timer instead of another thread.
+      if options[:concurrency_maintenance] || options[:batch_maintenance]
+        @maintenance = Maintenance.new(options[:concurrency_maintenance_interval], options[:batch_size],
+          concurrency: options[:concurrency_maintenance], batches: options[:batch_maintenance])
+      end
 
       super(**options)
     end
 
     def metadata
-      super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval)
+      super.merge(batch_size: batch_size).merge(maintenance&.metadata || {})
     end
 
     private
-      attr_reader :concurrency_maintenance
+      attr_reader :maintenance
 
       def poll
         batch = dispatch_next_batch
@@ -41,12 +45,12 @@ module SolidQueue
         end
       end
 
-      def start_concurrency_maintenance
-        concurrency_maintenance&.start
+      def start_maintenance
+        maintenance&.start
       end
 
-      def stop_concurrency_maintenance
-        concurrency_maintenance&.stop
+      def stop_maintenance
+        maintenance&.stop
       end
 
       def all_work_completed?

data/lib/solid_queue/engine.rb

@@ -41,6 +41,10 @@ module SolidQueue
     initializer "solid_queue.active_job.extensions" do
       ActiveSupport.on_load :active_job do
         include ActiveJob::ConcurrencyControls
+
+        ActiveSupport.on_load :active_record do
+          ActiveJob::Base.include ActiveJob::BatchId
+        end
       end
     end