solid_queue 0.5.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 952d693063dae16e88a88acb2f8d77f396472cb29811e7ab2c47c06fc400cf10
-  data.tar.gz: 338ad6a0057939a54997cdef138fa5e62e974112625c43fd65240354a0ba760a
+  metadata.gz: c2734f6cbcc795345207dc0cf221bd2744f2f98449da8b0edfc8175f3bbd7869
+  data.tar.gz: c4b4c2eca7dcb93e86a2f69d220ba1b9f8817c1d52a7cd1ff137ed5ec84e50f8
 SHA512:
-  metadata.gz: b0bddd34b216770c9e0658bb620ae8801ab500074692417aeb0907c1f22f4f9e40a6233266aa69c1aa2b015dc294864d529659f0ee4adbdfdbef647d03fb4d35
-  data.tar.gz: f2323054f8fdc5ee686f738d2a8359b23fe88f00b1b376cc18e99588bacc9df3be2e3a8bb660ce31db854f0fcca91560ad4d703a737b1d05c3a1dea7817c10a2
+  metadata.gz: 9d09b58e43c4bc19ad2ab89d10072b36b53a449e9602f5c5a8fc7b637e91d8472eb70f417148657e6b146e5cd1034025afdbe6d39fa2b9fc82ce6bba3997e57f
+  data.tar.gz: 27168fd2216fbca4e9b19677a7885b23663012d8aed7bf54c25be8335bd528a733b55790e662a30815b9ba24d824390372eff24ed0b9f72947b6ff4feeec17c5

data/app/models/solid_queue/claimed_execution.rb

@@ -29,8 +29,21 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
     def release_all
       SolidQueue.instrument(:release_many_claimed) do |payload|
         includes(:job).tap do |executions|
-          payload[:size] = executions.size
           executions.each(&:release)
+
+          payload[:size] = executions.size
+        end
+      end
+    end
+
+    def fail_all_with(error)
+      SolidQueue.instrument(:fail_many_claimed) do |payload|
+        includes(:job).tap do |executions|
+          executions.each { |execution| execution.failed_with(error) }
+
+          payload[:process_ids] = executions.map(&:process_id).uniq
+          payload[:job_ids] = executions.map(&:job_id).uniq
+          payload[:size] = executions.size
         end
       end
     end
@@ -69,6 +82,13 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
     raise UndiscardableError, "Can't discard a job in progress"
   end
 
+  def failed_with(error)
+    transaction do
+      job.failed_with(error)
+      destroy!
+    end
+  end
+
   private
     def execute
       ActiveJob::Base.execute(job.arguments)
@@ -83,11 +103,4 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
         destroy!
       end
     end
-
-    def failed_with(error)
-      transaction do
-        job.failed_with(error)
-        destroy!
-      end
-    end
 end

data/app/models/solid_queue/process/executor.rb

@@ -8,7 +8,19 @@ module SolidQueue
       included do
         has_many :claimed_executions
 
-        after_destroy -> { claimed_executions.release_all }, if: :claims_executions?
+        after_destroy :release_all_claimed_executions
+      end
+
+      def fail_all_claimed_executions_with(error)
+        if claims_executions?
+          claimed_executions.fail_all_with(error)
+        end
+      end
+
+      def release_all_claimed_executions
+        if claims_executions?
+          claimed_executions.release_all
+        end
       end
 
       private

data/app/models/solid_queue/process/prunable.rb

@@ -15,11 +15,18 @@ module SolidQueue
             prunable.non_blocking_lock.find_in_batches(batch_size: 50) do |batch|
               payload[:size] += batch.size
 
-              batch.each { |process| process.deregister(pruned: true) }
+              batch.each(&:prune)
             end
           end
         end
       end
+
+      def prune
+        error = Processes::ProcessPrunedError.new(last_heartbeat_at)
+        fail_all_claimed_executions_with(error)
+
+        deregister(pruned: true)
+      end
     end
   end
 end

data/app/models/solid_queue/process.rb

@@ -4,7 +4,7 @@ class SolidQueue::Process < SolidQueue::Record
   include Executor, Prunable
 
   belongs_to :supervisor, class_name: "SolidQueue::Process", optional: true, inverse_of: :supervisees
-  has_many :supervisees, class_name: "SolidQueue::Process", inverse_of: :supervisor, foreign_key: :supervisor_id, dependent: :destroy
+  has_many :supervisees, class_name: "SolidQueue::Process", inverse_of: :supervisor, foreign_key: :supervisor_id
 
   store :metadata, coder: JSON
 
@@ -13,10 +13,10 @@ class SolidQueue::Process < SolidQueue::Record
       create!(attributes.merge(last_heartbeat_at: Time.current)).tap do |process|
         payload[:process_id] = process.id
       end
+    rescue Exception => error
+      payload[:error] = error
+      raise
     end
-  rescue Exception => error
-    SolidQueue.instrument :register_process, **attributes.merge(error: error)
-    raise
   end
 
   def heartbeat
@@ -25,12 +25,19 @@ class SolidQueue::Process < SolidQueue::Record
 
   def deregister(pruned: false)
     SolidQueue.instrument :deregister_process, process: self, pruned: pruned do |payload|
-      payload[:claimed_size] = claimed_executions.size if claims_executions?
-
       destroy!
+
+      unless supervised? || pruned
+        supervisees.each(&:deregister)
+      end
     rescue Exception => error
       payload[:error] = error
       raise
     end
   end
+
+  private
+    def supervised?
+      supervisor_id.present?
+    end
 end

data/app/models/solid_queue/recurring_execution.rb

@@ -25,7 +25,7 @@ module SolidQueue
       def record(task_key, run_at, &block)
         transaction do
           block.call.tap do |active_job|
-            if active_job
+            if active_job && active_job.successfully_enqueued?
               create_or_insert!(job_id: active_job.provider_job_id, task_key: task_key, run_at: run_at)
             end
           end

data/app/models/solid_queue/recurring_task.rb

@@ -43,7 +43,7 @@ module SolidQueue
     def enqueue(at:)
       SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload|
         active_job = if using_solid_queue_adapter?
-          perform_later_and_record(run_at: at)
+          enqueue_and_record(run_at: at)
         else
           payload[:other_adapter] = true
 
@@ -87,8 +87,15 @@ module SolidQueue
         job_class.queue_adapter_name.inquiry.solid_queue?
       end
 
-      def perform_later_and_record(run_at:)
-        RecurringExecution.record(key, run_at) { perform_later }
+      def enqueue_and_record(run_at:)
+        RecurringExecution.record(key, run_at) do
+          job_class.new(*arguments_with_kwargs).tap do |active_job|
+            active_job.run_callbacks(:enqueue) do
+              Job.enqueue(active_job)
+            end
+            active_job.successfully_enqueued = true
+          end
+        end
       end
 
       def perform_later(&block)

data/db/migrate/20240811173327_add_name_to_processes.rb

@@ -0,0 +1,5 @@
+class AddNameToProcesses < ActiveRecord::Migration[7.1]
+  def change
+    add_column :solid_queue_processes, :name, :string
+  end
+end

data/db/migrate/20240813160053_make_name_not_null.rb

@@ -0,0 +1,16 @@
+class MakeNameNotNull < ActiveRecord::Migration[7.1]
+  def up
+    SolidQueue::Process.where(name: nil).find_each do |process|
+      process.name ||= [ process.kind.downcase, SecureRandom.hex(10) ].join("-")
+      process.save!
+    end
+
+    change_column :solid_queue_processes, :name, :string, null: false
+    add_index :solid_queue_processes, [ :name, :supervisor_id ], unique: true
+  end
+
+  def down
+    remove_index :solid_queue_processes, [ :name, :supervisor_id ]
+    change_column :solid_queue_processes, :name, :string, null: true
+  end
+end

data/db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb

@@ -0,0 +1,5 @@
+class ChangeSolidQueueRecurringTasksStaticToNotNull < ActiveRecord::Migration[7.1]
+  def change
+    change_column_null :solid_queue_recurring_tasks, :static, false, true
+  end
+end

data/lib/solid_queue/configuration.rb

@@ -2,6 +2,12 @@
 
 module SolidQueue
   class Configuration
+    class Process < Struct.new(:kind, :attributes)
+      def instantiate
+        "SolidQueue::#{kind.to_s.titleize}".safe_constantize.new(**attributes)
+      end
+    end
+
     WORKER_DEFAULTS = {
       queues: "*",
       threads: 3,
@@ -17,33 +23,20 @@ module SolidQueue
       recurring_tasks: []
     }
 
+    DEFAULT_CONFIG = {
+      workers: [ WORKER_DEFAULTS ],
+      dispatchers: [ DISPATCHER_DEFAULTS ]
+    }
+
     def initialize(mode: :fork, load_from: nil)
       @mode = mode.to_s.inquiry
       @raw_config = config_from(load_from)
     end
 
-    def processes
+    def configured_processes
       dispatchers + workers
     end
 
-    def workers
-      workers_options.flat_map do |worker_options|
-        processes = if mode.fork?
-          worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
-        else
-          WORKER_DEFAULTS[:processes]
-        end
-        processes.times.map { Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
-      end
-    end
-
-    def dispatchers
-      dispatchers_options.map do |dispatcher_options|
-        recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
-        Dispatcher.new **dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
-      end
-    end
-
     def max_number_of_threads
       # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
       workers_options.map { |options| options[:threads] }.max + 2
@@ -54,23 +47,46 @@ module SolidQueue
 
       DEFAULT_CONFIG_FILE_PATH = "config/solid_queue.yml"
 
+      def workers
+        workers_options.flat_map do |worker_options|
+          processes = if mode.fork?
+            worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          else
+            WORKER_DEFAULTS[:processes]
+          end
+          processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
+        end
+      end
+
+      def dispatchers
+        dispatchers_options.map do |dispatcher_options|
+          recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
+          Process.new :dispatcher, dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
+        end
+      end
+
       def config_from(file_or_hash, env: Rails.env)
-        config = load_config_from(file_or_hash)
-        config[env.to_sym] ? config[env.to_sym] : config
+        load_config_from(file_or_hash).then do |config|
+          config = config[env.to_sym] ? config[env.to_sym] : config
+          if (config.keys & DEFAULT_CONFIG.keys).any? then config
+          else
+            DEFAULT_CONFIG
+          end
+        end
       end
 
       def workers_options
-        @workers_options ||= options_from_raw_config(:workers, WORKER_DEFAULTS)
+        @workers_options ||= options_from_raw_config(:workers)
           .map { |options| options.dup.symbolize_keys }
       end
 
       def dispatchers_options
-        @dispatchers_options ||= options_from_raw_config(:dispatchers, DISPATCHER_DEFAULTS)
+        @dispatchers_options ||= options_from_raw_config(:dispatchers)
           .map { |options| options.dup.symbolize_keys }
       end
 
-      def options_from_raw_config(key, defaults)
-        raw_config.empty? ? [ defaults ] : Array(raw_config[key])
+      def options_from_raw_config(key)
+        Array(raw_config[key])
       end
 
       def parse_recurring_tasks(tasks)

data/lib/solid_queue/log_subscriber.rb

@@ -12,11 +12,15 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def release_many_claimed(event)
-    debug formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
+    info formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
+  end
+
+  def fail_many_claimed(event)
+    warn formatted_event(event, action: "Fail claimed jobs", **event.payload.slice(:job_ids, :process_ids))
   end
 
   def release_claimed(event)
-    debug formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
+    info formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
   end
 
   def retry_all(event)
@@ -63,7 +67,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     attributes = {
       pid: process.pid,
       hostname: process.hostname,
-      process_id: process.process_id
+      process_id: process.process_id,
+      name: process.name
     }.merge(process.metadata)
 
     info formatted_event(event, action: "Started #{process.kind}", **attributes)
@@ -75,7 +80,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     attributes = {
       pid: process.pid,
       hostname: process.hostname,
-      process_id: process.process_id
+      process_id: process.process_id,
+      name: process.name
     }.merge(process.metadata)
 
     info formatted_event(event, action: "Shutdown #{process.kind}", **attributes)
@@ -83,7 +89,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
 
   def register_process(event)
     process_kind = event.payload[:kind]
-    attributes = event.payload.slice(:pid, :hostname, :process_id)
+    attributes = event.payload.slice(:pid, :hostname, :process_id, :name)
 
     if error = event.payload[:error]
       warn formatted_event(event, action: "Error registering #{process_kind}", **attributes.merge(error: formatted_error(error)))
@@ -99,6 +105,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
       process_id: process.id,
       pid: process.pid,
       hostname: process.hostname,
+      name: process.name,
       last_heartbeat_at: process.last_heartbeat_at.iso8601,
       claimed_size: event.payload[:claimed_size],
       pruned: event.payload[:pruned]
@@ -147,7 +154,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
       termsig: status.termsig
 
     if replaced_fork = event.payload[:fork]
-      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname))
+      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname, name: replaced_fork.name))
     else
       warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes)
     end

data/lib/solid_queue/processes/base.rb

@@ -6,6 +6,12 @@ module SolidQueue
       include Callbacks # Defines callbacks needed by other concerns
       include AppExecutor, Registrable, Interruptible, Procline
 
+      attr_reader :name
+
+      def initialize(*)
+        @name = generate_name
+      end
+
       def kind
         self.class.name.demodulize
       end
@@ -21,6 +27,11 @@ module SolidQueue
       def metadata
         {}
       end
+
+      private
+        def generate_name
+          [ kind.downcase, SecureRandom.hex(10) ].join("-")
+        end
     end
   end
 end

data/lib/solid_queue/processes/poller.rb

@@ -8,6 +8,8 @@ module SolidQueue::Processes
 
     def initialize(polling_interval:, **options)
       @polling_interval = polling_interval
+
+      super(**options)
     end
 
     def metadata
@@ -43,10 +45,12 @@ module SolidQueue::Processes
       end
 
       def with_polling_volume
-        if SolidQueue.silence_polling? && ActiveRecord::Base.logger
-          ActiveRecord::Base.logger.silence { yield }
-        else
-          yield
+        SolidQueue.instrument(:polling) do
+          if SolidQueue.silence_polling? && ActiveRecord::Base.logger
+            ActiveRecord::Base.logger.silence { yield }
+          else
+            yield
+          end
         end
       end
   end

data/lib/solid_queue/processes/process_exit_error.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Processes
+    class ProcessExitError < RuntimeError
+      def initialize(status)
+        message = case
+        when status.exitstatus.present? then "Process pid=#{status.pid} exited with status #{status.  exitstatus}"
+        when status.signaled? then "Process pid=#{status.pid} received unhandled signal #{status. termsig}"
+        else "Process pid=#{status.pid} exited unexpectedly"
+        end
+
+        super(message)
+      end
+    end
+  end
+end

data/lib/solid_queue/processes/process_missing_error.rb

@@ -0,0 +1,9 @@
+module SolidQueue
+  module Processes
+    class ProcessMissingError < RuntimeError
+      def initialize
+        super("The process that was running this job no longer exists")
+      end
+    end
+  end
+end

data/lib/solid_queue/processes/process_pruned_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Processes
+    class ProcessPrunedError < RuntimeError
+      def initialize(last_heartbeat_at)
+        super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
+      end
+    end
+  end
+end

data/lib/solid_queue/processes/registrable.rb

@@ -21,6 +21,7 @@ module SolidQueue::Processes
       def register
         @process = SolidQueue::Process.register \
           kind: kind,
+          name: name,
           pid: pid,
           hostname: hostname,
           supervisor: try(:supervisor),

data/lib/solid_queue/processes/runnable.rb

@@ -25,10 +25,6 @@ module SolidQueue::Processes
       @thread&.join
     end
 
-    def name
-      @name ||= [ kind.downcase, SecureRandom.hex(6) ].join("-")
-    end
-
     def alive?
       !running_async? || @thread.alive?
     end

data/lib/solid_queue/supervisor/async_supervisor.rb

@@ -23,10 +23,13 @@ module SolidQueue
       attr_reader :threads
 
       def start_process(configured_process)
-        configured_process.supervised_by process
-        configured_process.start
+        process_instance = configured_process.instantiate.tap do |instance|
+          instance.supervised_by process
+        end
+
+        process_instance.start
 
-        threads[configured_process.name] = configured_process
+        threads[process_instance.name] = process_instance
       end
 
       def stop_threads

data/lib/solid_queue/supervisor/fork_supervisor.rb

@@ -6,7 +6,9 @@ module SolidQueue
 
     def initialize(*)
       super
+
       @forks = {}
+      @configured_processes = {}
     end
 
     def kind
@@ -14,7 +16,7 @@ module SolidQueue
     end
 
     private
-      attr_reader :forks
+      attr_reader :forks, :configured_processes
 
       def supervise
         loop do
@@ -33,14 +35,17 @@ module SolidQueue
       end
 
       def start_process(configured_process)
-        configured_process.supervised_by process
-        configured_process.mode = :fork
+        process_instance = configured_process.instantiate.tap do |instance|
+          instance.supervised_by process
+          instance.mode = :fork
+        end
 
         pid = fork do
-          configured_process.start
+          process_instance.start
         end
 
-        forks[pid] = configured_process
+        configured_processes[pid] = configured_process
+        forks[pid] = process_instance
       end
 
       def terminate_gracefully
@@ -86,7 +91,11 @@ module SolidQueue
           pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
           break unless pid
 
-          forks.delete(pid)
+          if (terminated_fork = forks.delete(pid)) && !status.exited? || status.exitstatus > 0
+            handle_claimed_jobs_by(terminated_fork, status)
+          end
+
+          configured_processes.delete(pid)
         end
       rescue SystemCallError
         # All children already reaped
@@ -94,13 +103,22 @@ module SolidQueue
 
       def replace_fork(pid, status)
         SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
-          if supervised_fork = forks.delete(pid)
-            payload[:fork] = supervised_fork
-            start_process(supervised_fork)
+          if terminated_fork = forks.delete(pid)
+            payload[:fork] = terminated_fork
+            handle_claimed_jobs_by(terminated_fork, status)
+
+            start_process(configured_processes.delete(pid))
           end
         end
       end
 
+      def handle_claimed_jobs_by(terminated_fork, status)
+        if registered_process = process.supervisees.find_by(name: terminated_fork.name)
+          error = Processes::ProcessExitError.new(status)
+          registered_process.fail_all_claimed_executions_with(error)
+        end
+      end
+
       def all_forks_terminated?
         forks.empty?
       end

data/lib/solid_queue/supervisor/maintenance.rb

@@ -3,7 +3,7 @@ module SolidQueue
     extend ActiveSupport::Concern
 
     included do
-      after_boot :release_orphaned_executions
+      after_boot :fail_orphaned_executions
     end
 
     private
@@ -27,8 +27,10 @@ module SolidQueue
         wrap_in_app_executor { SolidQueue::Process.prune }
       end
 
-      def release_orphaned_executions
-        wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
+      def fail_orphaned_executions
+        wrap_in_app_executor do
+          ClaimedExecution.orphaned.fail_all_with(Processes::ProcessMissingError.new)
+        end
       end
   end
 end

data/lib/solid_queue/supervisor.rb

@@ -9,13 +9,18 @@ module SolidQueue
         SolidQueue.supervisor = true
         configuration = Configuration.new(mode: mode, load_from: load_configuration_from)
 
-        klass = mode == :fork ? ForkSupervisor : AsyncSupervisor
-        klass.new(configuration).tap(&:start)
+        if configuration.configured_processes.any?
+          klass = mode == :fork ? ForkSupervisor : AsyncSupervisor
+          klass.new(configuration).tap(&:start)
+        else
+          abort "No workers or processed configured. Exiting..."
+        end
       end
     end
 
     def initialize(configuration)
       @configuration = configuration
+      super
     end
 
     def start
@@ -44,7 +49,7 @@ module SolidQueue
       end
 
       def start_processes
-        configuration.processes.each { |configured_process| start_process(configured_process) }
+        configuration.configured_processes.each { |configured_process| start_process(configured_process) }
       end
 
       def stopped?

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.5.0"
+  VERSION = "0.6.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-14 00:00:00.000000000 Z
+date: 2024-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -220,6 +220,9 @@ files:
 - db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb
 - db/migrate/20240218110712_create_recurring_executions.rb
 - db/migrate/20240719134516_create_recurring_tasks.rb
+- db/migrate/20240811173327_add_name_to_processes.rb
+- db/migrate/20240813160053_make_name_not_null.rb
+- db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb
 - lib/active_job/concurrency_controls.rb
 - lib/active_job/queue_adapters/solid_queue_adapter.rb
 - lib/generators/solid_queue/install/USAGE
@@ -239,6 +242,9 @@ files:
 - lib/solid_queue/processes/callbacks.rb
 - lib/solid_queue/processes/interruptible.rb
 - lib/solid_queue/processes/poller.rb
+- lib/solid_queue/processes/process_exit_error.rb
+- lib/solid_queue/processes/process_missing_error.rb
+- lib/solid_queue/processes/process_pruned_error.rb
 - lib/solid_queue/processes/procline.rb
 - lib/solid_queue/processes/registrable.rb
 - lib/solid_queue/processes/runnable.rb