solid_queue 1.6.0 → 1.7.0

data/app/models/solid_queue/job/batchable.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Job
+    module Batchable
+      extend ActiveSupport::Concern
+
+      included do
+        belongs_to :batch, optional: true
+        has_one :batch_execution
+
+        after_create :create_batch_execution, if: :batched?
+        after_update :update_batch_progress, if: :batched?
+        before_destroy :destroy_batch_execution, if: :batched?
+      end
+
+      class_methods do
+        def batch_all(jobs)
+          BatchExecution.create_all_from_jobs(jobs) if Batch.migrated?
+        end
+      end
+
+      # Also guards against the batches schema not being installed: without
+      # its migration, jobs don't even have a batch_id.
+      def batched?
+        Batch.migrated? && batch_id?
+      end
+
+      private
+        def create_batch_execution
+          BatchExecution.create_all_from_jobs([ self ])
+        end
+
+        def update_batch_progress
+          return unless saved_change_to_finished_at? && finished_at.present?
+
+          batch_execution&.destroy!
+        rescue ActiveRecord::ActiveRecordError => e
+          SolidQueue.instrument(:batch_progress_error, batch_id: batch_id, job_id: id, error: e)
+        end
+
+        # Destroy through Active Record instead of relying on the foreign
+        # key's cascade, so destroying the tracking row retries the batch
+        # completion check.
+        def destroy_batch_execution
+          batch_execution&.destroy!
+        end
+    end
+  end
+end

data/app/models/solid_queue/job/executable.rb

@@ -18,6 +18,9 @@ module SolidQueue
 
       class_methods do
         def prepare_all_for_execution(jobs)
+          # Track before dispatch so conflict-discarded jobs count like single enqueues.
+          batch_all(jobs)
+
           due, not_yet_due = jobs.partition(&:due?)
           dispatch_all(due) + schedule_all(not_yet_due)
         end
@@ -78,7 +81,8 @@ module SolidQueue
 
       def finished!
         if SolidQueue.preserve_finished_jobs?
-          touch(:finished_at)
+          # update! rather than touch so the batch tracking callbacks run
+          update!(finished_at: Time.current)
         else
           destroy!
         end

data/app/models/solid_queue/job.rb

@@ -4,13 +4,19 @@ module SolidQueue
   class Job < Record
     class EnqueueError < StandardError; end
 
-    include Executable, Clearable, Recurrable
+    include Executable, Clearable, Recurrable, Batchable
 
     serialize :arguments, coder: JSON
 
     class << self
       def enqueue_all(active_jobs)
-        active_jobs.each { |job| job.scheduled_at ||= Time.current }
+        # Bulk enqueues bypass ActiveJob#enqueue, so batch membership is captured here
+        current_batch_id = Batch.current_batch_id
+
+        active_jobs.each do |job|
+          job.scheduled_at ||= Time.current
+          job.batch_id = current_batch_id || job.batch_id
+        end
         active_jobs_by_job_id = active_jobs.index_by(&:job_id)
 
         transaction do
@@ -63,7 +69,9 @@ module SolidQueue
             class_name: active_job.class.name,
             arguments: active_job.serialize,
             concurrency_key: active_job.concurrency_key
-          }
+          }.tap do |attributes|
+            attributes[:batch_id] = active_job.batch_id if Batch.migrated?
+          end
         end
     end
   end

data/lib/active_job/batch_id.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Inspired by active_job/core.rb docs
+# https://github.com/rails/rails/blob/1c2529b9a6ba5a1eff58be0d0373d7d9d401015b/activejob/lib/active_job/core.rb#L136
+module ActiveJob
+  module BatchId
+    extend ActiveSupport::Concern
+
+    included do
+      attr_accessor :batch_id
+      attr_accessor :callback_batch_id
+    end
+
+    # Seed membership at construction for enqueue paths deferred until after the
+    # batch context ends, notably bulk enqueue. Enqueueing inside another batch
+    # rebinds membership; enqueueing without a batch preserves it.
+    # SolidQueue::Job.enqueue_all repeats this capture because bulk enqueue
+    # bypasses #enqueue.
+    def initialize(*arguments, **kwargs)
+      super
+      self.batch_id = SolidQueue::Batch.current_batch_id if solid_queue_job?
+    end
+
+    def enqueue(options = {})
+      self.batch_id = SolidQueue::Batch.current_batch_id || batch_id if solid_queue_job?
+      super
+    end
+
+    def serialize
+      super.tap do |data|
+        data["batch_id"] = batch_id if batch_id
+        data["callback_batch_id"] = callback_batch_id if callback_batch_id
+      end
+    end
+
+    def deserialize(job_data)
+      super
+      self.batch_id = job_data["batch_id"]
+      self.callback_batch_id = job_data["callback_batch_id"]
+    end
+
+    def batch
+      batch_id_to_load = callback_batch_id || batch_id
+      return if batch_id_to_load.nil?
+      return @batch if defined?(@batch) && @loaded_batch_id == batch_id_to_load
+
+      @loaded_batch_id = batch_id_to_load
+      @batch = SolidQueue::Batch.find_by(id: batch_id_to_load)
+    end
+
+    private
+
+      def solid_queue_job?
+        self.class.queue_adapter_name == "solid_queue"
+      end
+  end
+end

data/lib/generators/solid_queue/install/templates/db/queue_schema.rb

@@ -37,7 +37,9 @@ ActiveRecord::Schema[7.1].define(version: 1) do
     t.string "concurrency_key"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.bigint "batch_id"
     t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id"
+    t.index [ "batch_id" ], name: "index_solid_queue_jobs_on_batch_id"
     t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name"
     t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at"
     t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering"
@@ -120,6 +122,35 @@ ActiveRecord::Schema[7.1].define(version: 1) do
     t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true
   end
 
+  create_table "solid_queue_batches", force: :cascade do |t|
+    t.string "active_job_batch_id"
+    t.string "description"
+    t.text "on_finish"
+    t.text "on_success"
+    t.text "on_failure"
+    t.text "metadata"
+    t.integer "total_jobs", default: 0, null: false
+    t.integer "completed_jobs", default: 0, null: false
+    t.integer "failed_jobs", default: 0, null: false
+    t.datetime "enqueued_at"
+    t.datetime "finished_at"
+    t.datetime "failed_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index [ "active_job_batch_id" ], name: "index_solid_queue_batches_on_active_job_batch_id", unique: true
+    t.index [ "finished_at" ], name: "index_solid_queue_batches_on_finished_at"
+  end
+
+  create_table "solid_queue_batch_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.bigint "batch_id", null: false
+    t.datetime "created_at", null: false
+    t.index [ "job_id" ], name: "index_solid_queue_batch_executions_on_job_id", unique: true
+    t.index [ "batch_id" ], name: "index_solid_queue_batch_executions_on_batch_id"
+  end
+
+  add_foreign_key "solid_queue_batch_executions", "solid_queue_batches", column: "batch_id", on_delete: :cascade
+  add_foreign_key "solid_queue_batch_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
   add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
   add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
   add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade

data/lib/generators/solid_queue/update/templates/db/add_batches_to_solid_queue.rb

@@ -0,0 +1,39 @@
+class AddBatchesToSolidQueue < ActiveRecord::Migration[7.1]
+  def change
+    # Fresh installs create all of this with the base schema, so skip
+    # anything that already exists
+    add_column :solid_queue_jobs, :batch_id, :bigint, if_not_exists: true
+    add_index :solid_queue_jobs, :batch_id, if_not_exists: true
+
+    create_table :solid_queue_batches, if_not_exists: true do |t|
+      t.string :active_job_batch_id
+      t.string :description
+      t.text :on_finish
+      t.text :on_success
+      t.text :on_failure
+      t.text :metadata
+      t.integer :total_jobs, default: 0, null: false
+      t.integer :completed_jobs, default: 0, null: false
+      t.integer :failed_jobs, default: 0, null: false
+      t.datetime :enqueued_at
+      t.datetime :finished_at
+      t.datetime :failed_at
+      t.datetime :created_at, null: false
+      t.datetime :updated_at, null: false
+
+      t.index :active_job_batch_id, unique: true
+      t.index :finished_at
+    end
+
+    create_table :solid_queue_batch_executions, if_not_exists: true do |t|
+      t.bigint :job_id, null: false
+      t.bigint :batch_id, null: false
+      t.datetime :created_at, null: false
+
+      t.index :job_id, unique: true
+      t.index :batch_id
+      t.foreign_key :solid_queue_batches, column: :batch_id, on_delete: :cascade
+      t.foreign_key :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    end
+  end
+end

data/lib/solid_queue/configuration.rb

@@ -29,7 +29,8 @@ module SolidQueue
       batch_size: 500,
       polling_interval: 1,
       concurrency_maintenance: true,
-      concurrency_maintenance_interval: 600
+      concurrency_maintenance_interval: 600,
+      batch_maintenance: true
     }
 
     SCHEDULER_DEFAULTS = {

data/lib/solid_queue/dispatcher/concurrency_maintenance.rb

@@ -1,44 +1,11 @@
 # frozen_string_literal: true
 
 module SolidQueue
-  class Dispatcher::ConcurrencyMaintenance
-    include AppExecutor
-
-    attr_reader :interval, :batch_size
-
+  # Kept for compatibility: concurrency maintenance runs on the shared
+  # Dispatcher::Maintenance timer, together with batch maintenance.
+  class Dispatcher::ConcurrencyMaintenance < Dispatcher::Maintenance
     def initialize(interval, batch_size)
-      @interval = interval
-      @batch_size = batch_size
-    end
-
-    def start
-      @concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: interval) do
-        expire_semaphores
-        unblock_blocked_executions
-      end
-
-      @concurrency_maintenance_task.add_observer do |_, _, error|
-        handle_thread_error(error) if error
-      end
-
-      @concurrency_maintenance_task.execute
-    end
-
-    def stop
-      @concurrency_maintenance_task&.shutdown
+      super(interval, batch_size, concurrency: true, batches: false)
     end
-
-    private
-      def expire_semaphores
-        wrap_in_app_executor do
-          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
-        end
-      end
-
-      def unblock_blocked_executions
-        wrap_in_app_executor do
-          BlockedExecution.unblock(batch_size)
-        end
-      end
   end
 end

data/lib/solid_queue/dispatcher/maintenance.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Dispatcher::Maintenance
+    include AppExecutor
+
+    attr_reader :interval, :batch_size
+
+    def initialize(interval, batch_size, concurrency:, batches:)
+      @interval = interval
+      @batch_size = batch_size
+      @concurrency = concurrency
+      @batches = batches
+    end
+
+    def concurrency?
+      @concurrency
+    end
+
+    def batches?
+      @batches
+    end
+
+    def metadata
+      { concurrency_maintenance_interval: (interval if concurrency?), batch_maintenance: batches? }
+    end
+
+    def start
+      @maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: interval) do
+        if concurrency?
+          expire_semaphores
+          unblock_blocked_executions
+        end
+
+        sweep_stalled_batches if batches?
+      end
+
+      @maintenance_task.add_observer do |_, _, error|
+        handle_thread_error(error) if error
+      end
+
+      @maintenance_task.execute
+    end
+
+    def stop
+      @maintenance_task&.shutdown
+    end
+
+    private
+      def expire_semaphores
+        wrap_in_app_executor do
+          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
+        end
+      end
+
+      def unblock_blocked_executions
+        wrap_in_app_executor do
+          BlockedExecution.unblock(batch_size)
+        end
+      end
+
+      def sweep_stalled_batches
+        wrap_in_app_executor do
+          if Batch.migrated?
+            Batch.sweep_stalled(batch_size: batch_size)
+          else
+            warn_once_about_pending_batch_migrations
+          end
+        end
+      end
+
+      def warn_once_about_pending_batch_migrations
+        unless @warned_about_pending_migrations
+          Batch.warn_about_pending_migrations
+          @warned_about_pending_migrations = true
+        end
+      end
+  end
+end

data/lib/solid_queue/dispatcher.rb

@@ -7,8 +7,8 @@ module SolidQueue
     attr_reader :batch_size
 
     after_boot :run_start_hooks
-    after_boot :start_concurrency_maintenance
-    before_shutdown :stop_concurrency_maintenance
+    after_boot :start_maintenance
+    before_shutdown :stop_maintenance
     before_shutdown :run_stop_hooks
     after_shutdown :run_exit_hooks
 
@@ -17,17 +17,21 @@ module SolidQueue
 
       @batch_size = options[:batch_size]
 
-      @concurrency_maintenance = ConcurrencyMaintenance.new(options[:concurrency_maintenance_interval], options[:batch_size]) if options[:concurrency_maintenance]
+      # Run both maintenance routines on one timer instead of another thread.
+      if options[:concurrency_maintenance] || options[:batch_maintenance]
+        @maintenance = Maintenance.new(options[:concurrency_maintenance_interval], options[:batch_size],
+          concurrency: options[:concurrency_maintenance], batches: options[:batch_maintenance])
+      end
 
       super(**options)
     end
 
     def metadata
-      super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval)
+      super.merge(batch_size: batch_size).merge(maintenance&.metadata || {})
     end
 
     private
-      attr_reader :concurrency_maintenance
+      attr_reader :maintenance
 
       def poll
         batch = dispatch_next_batch
@@ -41,12 +45,12 @@ module SolidQueue
         end
       end
 
-      def start_concurrency_maintenance
-        concurrency_maintenance&.start
+      def start_maintenance
+        maintenance&.start
       end
 
-      def stop_concurrency_maintenance
-        concurrency_maintenance&.stop
+      def stop_maintenance
+        maintenance&.stop
       end
 
       def all_work_completed?

data/lib/solid_queue/engine.rb

@@ -41,6 +41,10 @@ module SolidQueue
     initializer "solid_queue.active_job.extensions" do
       ActiveSupport.on_load :active_job do
         include ActiveJob::ConcurrencyControls
+
+        ActiveSupport.on_load :active_record do
+          ActiveJob::Base.include ActiveJob::BatchId
+        end
       end
     end
 

data/lib/solid_queue/fork_supervisor.rb

@@ -57,8 +57,7 @@ module SolidQueue
           terminated_fork.mark_as_reaped
 
           if !status.exited? || status.exitstatus.to_i > 0
-            error = Processes::ProcessExitError.new(status)
-            release_claimed_jobs_by(terminated_fork, with_error: error)
+            attempt_to_release_claimed_jobs_by(terminated_fork, status)
           end
         end
 
@@ -73,14 +72,24 @@ module SolidQueue
         if terminated_fork = process_instances.delete(pid)
           terminated_fork.mark_as_reaped
           payload[:fork] = terminated_fork
-          error = Processes::ProcessExitError.new(status)
-          release_claimed_jobs_by(terminated_fork, with_error: error)
+
+          attempt_to_release_claimed_jobs_by(terminated_fork, status)
 
           start_process(configured_processes.delete(pid))
         end
       end
     end
 
+    # The database may be unreachable — likely the same reason the fork
+    # terminated. Neither starting a replacement nor shutting down can depend
+    # on it: the jobs claimed by the terminated fork will be failed when its
+    # stale registration is pruned once the database is back.
+    def attempt_to_release_claimed_jobs_by(terminated_fork, status)
+      release_claimed_jobs_by(terminated_fork, with_error: Processes::ProcessExitError.new(status))
+    rescue StandardError => error
+      handle_thread_error(error)
+    end
+
     def all_processes_terminated?
       process_instances.empty?
     end

data/lib/solid_queue/log_subscriber.rb

@@ -16,7 +16,10 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def fail_many_claimed(event)
-    warn formatted_event(event, action: "Fail claimed jobs", **event.payload.slice(:job_ids, :process_ids))
+    attributes = event.payload.slice(:job_ids, :process_ids)
+    attributes[:error] = formatted_error(event.payload[:error]) if event.payload[:error]
+
+    warn formatted_event(event, action: "Fail claimed jobs", **attributes)
   end
 
   def release_claimed(event)
@@ -39,6 +42,18 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     debug formatted_event(event, action: "Discard job", **event.payload.slice(:job_id, :status))
   end
 
+  def finish_batch(event)
+    info formatted_event(event, action: "Finish batch", **event.payload.slice(:batch_id, :total_jobs, :completed_jobs, :failed_jobs))
+  end
+
+  def sweep_stalled_batches(event)
+    debug formatted_event(event, action: "Sweep stalled batches", **event.payload.slice(:stale_executions, :finished_batches, :started_batches))
+  end
+
+  def batch_progress_error(event)
+    error formatted_event(event, action: "Error updating batch progress", **event.payload.slice(:batch_id, :job_id), error: formatted_error(event.payload[:error]))
+  end
+
   def release_many_blocked(event)
     debug formatted_event(event, action: "Unblock jobs", **event.payload.slice(:limit, :size))
   end

data/lib/solid_queue/processes/runnable.rb

@@ -50,7 +50,7 @@ module SolidQueue::Processes
         case
         when running_as_fork?
           @boot_guard = BootGuards::ForkGuard.new
-          fork(&block).tap { @boot_guard.start }
+          create_fork(&block).tap { @boot_guard.start }
         when running_async?
           @boot_guard = BootGuards::NullGuard.new
           @thread = create_thread(&block)
@@ -64,10 +64,7 @@ module SolidQueue::Processes
       def boot
         SolidQueue.instrument(:start_process, process: self) do
           run_callbacks(:boot) do
-            if running_as_fork?
-              register_signal_handlers
-              set_procline
-            end
+            set_procline if running_as_fork?
           end
         end
 

data/lib/solid_queue/processes/supervised.rb

@@ -25,6 +25,13 @@ module SolidQueue::Processes
         supervisor.present?
       end
 
+      def create_fork(&block)
+        fork do
+          register_signal_handlers
+          block.call
+        end
+      end
+
       def register_signal_handlers
         %w[ INT TERM ].each do |signal|
           trap(signal) do

data/lib/solid_queue/supervisor/signals.rb

@@ -29,6 +29,9 @@ module SolidQueue
         end
 
         def process_signal_queue
+          # Embedded supervisors don't own their process's signals
+          return unless standalone?
+
           while signal = signal_queue.shift
             handle_signal(signal)
           end

data/lib/solid_queue/supervisor.rb

@@ -39,9 +39,13 @@ module SolidQueue
       run_start_hooks
 
       start_processes
-      launch_maintenance_task
 
-      supervise
+      if stopped?
+        shutdown
+      else
+        launch_maintenance_task
+        supervise
+      end
     end
 
     def stop
@@ -65,27 +69,33 @@ module SolidQueue
       end
 
       def start_processes
-        configuration.configured_processes.each { |configured_process| start_process(configured_process) }
+        configuration.configured_processes.each do |configured_process|
+          # Honour signals that arrive during boot or start hooks: a queued TERM
+          # stops us here, before starting children, instead of in #supervise,
+          # after all of them have been started
+          break if time_to_stop?
+
+          start_process(configured_process)
+        end
       end
 
       def supervise
-        loop do
-          break if stopped?
-
-          if standalone?
-            set_procline
-            process_signal_queue
-          end
-
-          unless stopped?
-            check_and_replace_terminated_processes
-            interruptible_sleep(1.second)
-          end
+        until time_to_stop?
+          set_procline
+          check_and_replace_terminated_processes
+          interruptible_sleep(1.second)
         end
       ensure
         shutdown
       end
 
+      # Process any signals queued while we were busy and report whether
+      # we've been asked to stop
+      def time_to_stop?
+        process_signal_queue
+        stopped?
+      end
+
       def start_process(configured_process)
         process_instance = configured_process.instantiate.tap do |instance|
           instance.supervised_by process
@@ -139,7 +149,10 @@ module SolidQueue
       end
 
       def set_procline
-        procline "supervising #{configured_processes.keys.join(", ")}"
+        # Embedded supervisors don't own their process's title
+        if standalone?
+          procline "supervising #{configured_processes.keys.join(", ")}"
+        end
       end
 
       def sync_std_streams

data/lib/solid_queue/version.rb

@@ -1,5 +1,5 @@
 module SolidQueue
-  VERSION = "1.6.0"
+  VERSION = "1.7.0"
 
   def self.next_major_version
     Gem::Version.new(VERSION).segments.first + 1

data/lib/solid_queue.rb

@@ -5,6 +5,7 @@ require "solid_queue/engine"
 
 require "active_job"
 require "active_job/queue_adapters"
+require "active_job/batch_id"
 
 require "active_support"
 require "active_support/core_ext/numeric/time"