solid_queue 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 952d693063dae16e88a88acb2f8d77f396472cb29811e7ab2c47c06fc400cf10
-  data.tar.gz: 338ad6a0057939a54997cdef138fa5e62e974112625c43fd65240354a0ba760a
+  metadata.gz: e62ec09bbed1f2bcb96da96865c3516adbc254ad7cea234d3b6573ea001c758e
+  data.tar.gz: d78e7543194f3f6470bdb80c52ffca78c520a602d93423185c02ba82475bbda6
 SHA512:
-  metadata.gz: b0bddd34b216770c9e0658bb620ae8801ab500074692417aeb0907c1f22f4f9e40a6233266aa69c1aa2b015dc294864d529659f0ee4adbdfdbef647d03fb4d35
-  data.tar.gz: f2323054f8fdc5ee686f738d2a8359b23fe88f00b1b376cc18e99588bacc9df3be2e3a8bb660ce31db854f0fcca91560ad4d703a737b1d05c3a1dea7817c10a2
+  metadata.gz: e7cdceced9162911efdd0f2020042e45f7c73c74c506cf1d4958211b2cc34f02e96553d9910b3d3cca925f00e10960837be4872a33c3e44388e0b6c37cbcec31
+  data.tar.gz: d9a17282687ef9feaa4b357124411e9bd088bebb8c0ce6fd67ccf887e4d9cf20a431882e84b40f68ff5ca05a008258bf4d64b6ab86b675cb581be65154d1e39e

data/app/models/solid_queue/claimed_execution.rb

@@ -35,6 +35,18 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
       end
     end
 
+    def fail_all_with(error)
+      SolidQueue.instrument(:fail_many_claimed) do |payload|
+        includes(:job).tap do |executions|
+          payload[:size] = executions.size
+          payload[:process_ids] = executions.map(&:process_id).uniq
+          payload[:job_ids] = executions.map(&:job_id).uniq
+
+          executions.each { |execution| execution.failed_with(error) }
+        end
+      end
+    end
+
     def discard_all_in_batches(*)
       raise UndiscardableError, "Can't discard jobs in progress"
     end
@@ -69,6 +81,13 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
     raise UndiscardableError, "Can't discard a job in progress"
   end
 
+  def failed_with(error)
+    transaction do
+      job.failed_with(error)
+      destroy!
+    end
+  end
+
   private
     def execute
       ActiveJob::Base.execute(job.arguments)
@@ -83,11 +102,4 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
         destroy!
       end
     end
-
-    def failed_with(error)
-      transaction do
-        job.failed_with(error)
-        destroy!
-      end
-    end
 end

data/app/models/solid_queue/process/executor.rb

@@ -11,6 +11,12 @@ module SolidQueue
         after_destroy -> { claimed_executions.release_all }, if: :claims_executions?
       end
 
+      def fail_all_claimed_executions_with(error)
+        if claims_executions?
+          claimed_executions.fail_all_with(error)
+        end
+      end
+
       private
         def claims_executions?
           kind == "Worker"

data/app/models/solid_queue/process/prunable.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 module SolidQueue
+  class ProcessPrunedError < RuntimeError
+    def initialize(last_heartbeat_at)
+      super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
+    end
+  end
+
   class Process
     module Prunable
       extend ActiveSupport::Concern
@@ -15,11 +21,18 @@ module SolidQueue
             prunable.non_blocking_lock.find_in_batches(batch_size: 50) do |batch|
               payload[:size] += batch.size
 
-              batch.each { |process| process.deregister(pruned: true) }
+              batch.each(&:prune)
             end
           end
         end
       end
+
+      def prune
+        error = ProcessPrunedError.new(last_heartbeat_at)
+        fail_all_claimed_executions_with(error)
+
+        deregister(pruned: true)
+      end
     end
   end
 end

data/app/models/solid_queue/process.rb

@@ -13,10 +13,10 @@ class SolidQueue::Process < SolidQueue::Record
       create!(attributes.merge(last_heartbeat_at: Time.current)).tap do |process|
         payload[:process_id] = process.id
       end
+    rescue Exception => error
+      payload[:error] = error
+      raise
     end
-  rescue Exception => error
-    SolidQueue.instrument :register_process, **attributes.merge(error: error)
-    raise
   end
 
   def heartbeat
@@ -25,8 +25,6 @@ class SolidQueue::Process < SolidQueue::Record
 
   def deregister(pruned: false)
     SolidQueue.instrument :deregister_process, process: self, pruned: pruned do |payload|
-      payload[:claimed_size] = claimed_executions.size if claims_executions?
-
       destroy!
     rescue Exception => error
       payload[:error] = error

data/app/models/solid_queue/recurring_execution.rb

@@ -25,7 +25,7 @@ module SolidQueue
       def record(task_key, run_at, &block)
         transaction do
           block.call.tap do |active_job|
-            if active_job
+            if active_job && active_job.successfully_enqueued?
               create_or_insert!(job_id: active_job.provider_job_id, task_key: task_key, run_at: run_at)
             end
           end

data/app/models/solid_queue/recurring_task.rb

@@ -43,7 +43,7 @@ module SolidQueue
     def enqueue(at:)
       SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload|
         active_job = if using_solid_queue_adapter?
-          perform_later_and_record(run_at: at)
+          enqueue_and_record(run_at: at)
         else
           payload[:other_adapter] = true
 
@@ -87,8 +87,15 @@ module SolidQueue
         job_class.queue_adapter_name.inquiry.solid_queue?
       end
 
-      def perform_later_and_record(run_at:)
-        RecurringExecution.record(key, run_at) { perform_later }
+      def enqueue_and_record(run_at:)
+        RecurringExecution.record(key, run_at) do
+          job_class.new(*arguments_with_kwargs).tap do |active_job|
+            active_job.run_callbacks(:enqueue) do
+              Job.enqueue(active_job)
+            end
+            active_job.successfully_enqueued = true
+          end
+        end
       end
 
       def perform_later(&block)

data/db/migrate/20240811173327_add_name_to_processes.rb

@@ -0,0 +1,5 @@
+class AddNameToProcesses < ActiveRecord::Migration[7.1]
+  def change
+    add_column :solid_queue_processes, :name, :string
+  end
+end

data/db/migrate/20240813160053_make_name_not_null.rb

@@ -0,0 +1,16 @@
+class MakeNameNotNull < ActiveRecord::Migration[7.1]
+  def up
+    SolidQueue::Process.where(name: nil).find_each do |process|
+      process.name ||= [ process.kind.downcase, SecureRandom.hex(10) ].join("-")
+      process.save!
+    end
+
+    change_column :solid_queue_processes, :name, :string, null: false
+    add_index :solid_queue_processes, [ :name, :supervisor_id ], unique: true
+  end
+
+  def down
+    remove_index :solid_queue_processes, [ :name, :supervisor_id ]
+    change_column :solid_queue_processes, :name, :string, null: false
+  end
+end

data/db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb

@@ -0,0 +1,5 @@
+class ChangeSolidQueueRecurringTasksStaticToNotNull < ActiveRecord::Migration[7.1]
+  def change
+    change_column_null :solid_queue_recurring_tasks, :static, false, true
+  end
+end

data/lib/solid_queue/configuration.rb

@@ -2,6 +2,12 @@
 
 module SolidQueue
   class Configuration
+    class Process < Struct.new(:kind, :attributes)
+      def instantiate
+        "SolidQueue::#{kind.to_s.titleize}".safe_constantize.new(**attributes)
+      end
+    end
+
     WORKER_DEFAULTS = {
       queues: "*",
       threads: 3,
@@ -22,28 +28,10 @@ module SolidQueue
       @raw_config = config_from(load_from)
     end
 
-    def processes
+    def configured_processes
       dispatchers + workers
     end
 
-    def workers
-      workers_options.flat_map do |worker_options|
-        processes = if mode.fork?
-          worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
-        else
-          WORKER_DEFAULTS[:processes]
-        end
-        processes.times.map { Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
-      end
-    end
-
-    def dispatchers
-      dispatchers_options.map do |dispatcher_options|
-        recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
-        Dispatcher.new **dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
-      end
-    end
-
     def max_number_of_threads
       # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
       workers_options.map { |options| options[:threads] }.max + 2
@@ -54,6 +42,24 @@ module SolidQueue
 
       DEFAULT_CONFIG_FILE_PATH = "config/solid_queue.yml"
 
+      def workers
+        workers_options.flat_map do |worker_options|
+          processes = if mode.fork?
+            worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          else
+            WORKER_DEFAULTS[:processes]
+          end
+          processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
+        end
+      end
+
+      def dispatchers
+        dispatchers_options.map do |dispatcher_options|
+          recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
+          Process.new :dispatcher, dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
+        end
+      end
+
       def config_from(file_or_hash, env: Rails.env)
         config = load_config_from(file_or_hash)
         config[env.to_sym] ? config[env.to_sym] : config

data/lib/solid_queue/log_subscriber.rb

@@ -12,11 +12,15 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def release_many_claimed(event)
-    debug formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
+    info formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
+  end
+
+  def fail_many_claimed(event)
+    warn formatted_event(event, action: "Fail claimed jobs", **event.payload.slice(:job_ids, :process_ids))
   end
 
   def release_claimed(event)
-    debug formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
+    info formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
   end
 
   def retry_all(event)
@@ -63,7 +67,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     attributes = {
       pid: process.pid,
       hostname: process.hostname,
-      process_id: process.process_id
+      process_id: process.process_id,
+      name: process.name
     }.merge(process.metadata)
 
     info formatted_event(event, action: "Started #{process.kind}", **attributes)
@@ -75,7 +80,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     attributes = {
       pid: process.pid,
       hostname: process.hostname,
-      process_id: process.process_id
+      process_id: process.process_id,
+      name: process.name
     }.merge(process.metadata)
 
     info formatted_event(event, action: "Shutdown #{process.kind}", **attributes)
@@ -83,7 +89,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
 
   def register_process(event)
     process_kind = event.payload[:kind]
-    attributes = event.payload.slice(:pid, :hostname, :process_id)
+    attributes = event.payload.slice(:pid, :hostname, :process_id, :name)
 
     if error = event.payload[:error]
       warn formatted_event(event, action: "Error registering #{process_kind}", **attributes.merge(error: formatted_error(error)))
@@ -99,6 +105,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
       process_id: process.id,
       pid: process.pid,
       hostname: process.hostname,
+      name: process.name,
       last_heartbeat_at: process.last_heartbeat_at.iso8601,
       claimed_size: event.payload[:claimed_size],
       pruned: event.payload[:pruned]
@@ -147,7 +154,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
       termsig: status.termsig
 
     if replaced_fork = event.payload[:fork]
-      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname))
+      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname, name: replaced_fork.name))
     else
       warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes)
     end

data/lib/solid_queue/processes/base.rb

@@ -6,6 +6,12 @@ module SolidQueue
       include Callbacks # Defines callbacks needed by other concerns
       include AppExecutor, Registrable, Interruptible, Procline
 
+      attr_reader :name
+
+      def initialize(*)
+        @name = generate_name
+      end
+
       def kind
         self.class.name.demodulize
       end
@@ -21,6 +27,11 @@ module SolidQueue
       def metadata
         {}
       end
+
+      private
+        def generate_name
+          [ kind.downcase, SecureRandom.hex(10) ].join("-")
+        end
     end
   end
 end

data/lib/solid_queue/processes/poller.rb

@@ -8,6 +8,8 @@ module SolidQueue::Processes
 
     def initialize(polling_interval:, **options)
       @polling_interval = polling_interval
+
+      super(**options)
     end
 
     def metadata

data/lib/solid_queue/processes/registrable.rb

@@ -21,6 +21,7 @@ module SolidQueue::Processes
       def register
         @process = SolidQueue::Process.register \
           kind: kind,
+          name: name,
           pid: pid,
           hostname: hostname,
           supervisor: try(:supervisor),

data/lib/solid_queue/processes/runnable.rb

@@ -25,10 +25,6 @@ module SolidQueue::Processes
       @thread&.join
     end
 
-    def name
-      @name ||= [ kind.downcase, SecureRandom.hex(6) ].join("-")
-    end
-
     def alive?
       !running_async? || @thread.alive?
     end

data/lib/solid_queue/supervisor/async_supervisor.rb

@@ -23,10 +23,13 @@ module SolidQueue
       attr_reader :threads
 
       def start_process(configured_process)
-        configured_process.supervised_by process
-        configured_process.start
+        process_instance = configured_process.instantiate.tap do |instance|
+          instance.supervised_by process
+        end
+
+        process_instance.start
 
-        threads[configured_process.name] = configured_process
+        threads[process_instance.name] = process_instance
       end
 
       def stop_threads

data/lib/solid_queue/supervisor/fork_supervisor.rb

@@ -1,12 +1,26 @@
 # frozen_string_literal: true
 
 module SolidQueue
+  class ProcessExitError < RuntimeError
+    def initialize(status)
+      message = case
+      when status.exitstatus.present? then "Process pid=#{status.pid} exited with status #{status.exitstatus}"
+      when status.signaled? then "Process pid=#{status.pid} received unhandled signal #{status.termsig}"
+      else "Process pid=#{status.pid} exited unexpectedly"
+      end
+
+      super(message)
+    end
+  end
+
   class Supervisor::ForkSupervisor < Supervisor
     include Signals, Pidfiled
 
     def initialize(*)
       super
+
       @forks = {}
+      @configured_processes = {}
     end
 
     def kind
@@ -14,7 +28,7 @@ module SolidQueue
     end
 
     private
-      attr_reader :forks
+      attr_reader :forks, :configured_processes
 
       def supervise
         loop do
@@ -33,14 +47,17 @@ module SolidQueue
       end
 
       def start_process(configured_process)
-        configured_process.supervised_by process
-        configured_process.mode = :fork
+        process_instance = configured_process.instantiate.tap do |instance|
+          instance.supervised_by process
+          instance.mode = :fork
+        end
 
         pid = fork do
-          configured_process.start
+          process_instance.start
         end
 
-        forks[pid] = configured_process
+        configured_processes[pid] = configured_process
+        forks[pid] = process_instance
       end
 
       def terminate_gracefully
@@ -86,7 +103,11 @@ module SolidQueue
           pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
           break unless pid
 
-          forks.delete(pid)
+          if (terminated_fork = forks.delete(pid)) && !status.exited? || status.exitstatus > 0
+            handle_claimed_jobs_by(terminated_fork, status)
+          end
+
+          configured_processes.delete(pid)
         end
       rescue SystemCallError
         # All children already reaped
@@ -94,13 +115,22 @@ module SolidQueue
 
       def replace_fork(pid, status)
         SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
-          if supervised_fork = forks.delete(pid)
-            payload[:fork] = supervised_fork
-            start_process(supervised_fork)
+          if terminated_fork = forks.delete(pid)
+            payload[:fork] = terminated_fork
+            handle_claimed_jobs_by(terminated_fork, status)
+
+            start_process(configured_processes.delete(pid))
           end
         end
       end
 
+      def handle_claimed_jobs_by(terminated_fork, status)
+        if registered_process = process.supervisees.find_by(name: terminated_fork.name)
+          error = ProcessExitError.new(status)
+          registered_process.fail_all_claimed_executions_with(error)
+        end
+      end
+
       def all_forks_terminated?
         forks.empty?
       end

data/lib/solid_queue/supervisor/maintenance.rb

@@ -1,9 +1,15 @@
 module SolidQueue
+  class ProcessMissingError < RuntimeError
+    def initialize
+      super("The process that was running this job no longer exists")
+    end
+  end
+
   module Supervisor::Maintenance
     extend ActiveSupport::Concern
 
     included do
-      after_boot :release_orphaned_executions
+      after_boot :fail_orphaned_executions
     end
 
     private
@@ -27,8 +33,10 @@ module SolidQueue
         wrap_in_app_executor { SolidQueue::Process.prune }
       end
 
-      def release_orphaned_executions
-        wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
+      def fail_orphaned_executions
+        wrap_in_app_executor do
+          SolidQueue::ClaimedExecution.orphaned.fail_all_with(ProcessMissingError.new)
+        end
       end
   end
 end

data/lib/solid_queue/supervisor.rb

@@ -16,6 +16,7 @@ module SolidQueue
 
     def initialize(configuration)
       @configuration = configuration
+      super
     end
 
     def start
@@ -44,7 +45,7 @@ module SolidQueue
       end
 
       def start_processes
-        configuration.processes.each { |configured_process| start_process(configured_process) }
+        configuration.configured_processes.each { |configured_process| start_process(configured_process) }
       end
 
       def stopped?

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-14 00:00:00.000000000 Z
+date: 2024-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -220,6 +220,9 @@ files:
 - db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb
 - db/migrate/20240218110712_create_recurring_executions.rb
 - db/migrate/20240719134516_create_recurring_tasks.rb
+- db/migrate/20240811173327_add_name_to_processes.rb
+- db/migrate/20240813160053_make_name_not_null.rb
+- db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb
 - lib/active_job/concurrency_controls.rb
 - lib/active_job/queue_adapters/solid_queue_adapter.rb
 - lib/generators/solid_queue/install/USAGE