solid_queue 0.4.1 → 0.7.1

data/db/migrate/20240813160053_make_name_not_null.rb

@@ -0,0 +1,16 @@
+class MakeNameNotNull < ActiveRecord::Migration[7.1]
+  def up
+    SolidQueue::Process.where(name: nil).find_each do |process|
+      process.name ||= [ process.kind.downcase, SecureRandom.hex(10) ].join("-")
+      process.save!
+    end
+
+    change_column :solid_queue_processes, :name, :string, null: false
+    add_index :solid_queue_processes, [ :name, :supervisor_id ], unique: true
+  end
+
+  def down
+    remove_index :solid_queue_processes, [ :name, :supervisor_id ]
+    change_column :solid_queue_processes, :name, :string, null: true
+  end
+end

data/db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb

@@ -0,0 +1,5 @@
+class ChangeSolidQueueRecurringTasksStaticToNotNull < ActiveRecord::Migration[7.1]
+  def change
+    change_column_null :solid_queue_recurring_tasks, :static, false, true
+  end
+end

data/lib/generators/solid_queue/install/USAGE

@@ -7,3 +7,4 @@ Example:
     This will perform the following:
         Installs solid_queue migrations
         Replaces Active Job's adapter in environment configuration
+        Installs bin/jobs binstub to start the supervisor

data/lib/generators/solid_queue/install/install_generator.rb

@@ -3,19 +3,33 @@
 class SolidQueue::InstallGenerator < Rails::Generators::Base
   source_root File.expand_path("templates", __dir__)
 
-  class_option :skip_migrations, type: :boolean, default: nil, desc: "Skip migrations"
+  class_option :skip_adapter, type: :boolean, default: nil, desc: "Skip setting Solid Queue as the Active Job's adapter"
+  class_option :database, type: :string, default: nil, desc: "The database to use for migrations, if different from the primary one."
 
   def add_solid_queue
-    if (env_config = Pathname(destination_root).join("config/environments/production.rb")).exist?
-      gsub_file env_config, /(# )?config\.active_job\.queue_adapter\s+=.*/, "config.active_job.queue_adapter = :solid_queue"
+    unless options[:skip_adapter]
+      if (env_config = Pathname(destination_root).join("config/environments/production.rb")).exist?
+        say "Setting solid_queue as Active Job's queue adapter"
+        gsub_file env_config, /(# )?config\.active_job\.queue_adapter\s+=.*/, "config.active_job.queue_adapter = :solid_queue"
+      end
     end
 
-    copy_file "config.yml", "config/solid_queue.yml"
+    if File.exist?("config/solid_queue.yml")
+      say "Skipping sample configuration as config/solid_queue.yml exists"
+    else
+      say "Copying sample configuration"
+      copy_file "config.yml", "config/solid_queue.yml"
+    end
+
+    say "Copying binstub"
+    copy_file "jobs", "bin/jobs"
+    chmod "bin/jobs", 0755 & ~File.umask, verbose: false
   end
 
   def create_migrations
-    unless options[:skip_migrations]
-      rails_command "railties:install:migrations FROM=solid_queue", inline: true
-    end
+    say "Installing database migrations"
+    arguments = [ "FROM=solid_queue" ]
+    arguments << "DATABASE=#{options[:database]}" if options[:database].present?
+    rails_command "railties:install:migrations #{arguments.join(" ")}", inline: true
   end
 end

data/lib/generators/solid_queue/install/templates/jobs

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/environment"
+require "solid_queue/cli"
+
+SolidQueue::Cli.start(ARGV)

data/lib/puma/plugin/solid_queue.rb

@@ -1,13 +1,5 @@
 require "puma/plugin"
 
-module Puma
-  class DSL
-    def solid_queue_mode(mode = :fork)
-      @options[:solid_queue_mode] = mode.to_sym
-    end
-  end
-end
-
 Puma::Plugin.create do
   attr_reader :puma_pid, :solid_queue_pid, :log_writer, :solid_queue_supervisor
 
@@ -15,36 +7,22 @@ Puma::Plugin.create do
     @log_writer = launcher.log_writer
     @puma_pid = $$
 
-    if launcher.options[:solid_queue_mode] == :async
-      start_async(launcher)
-    else
-      start_forked(launcher)
+    in_background do
+      monitor_solid_queue
     end
-  end
 
-  private
-    def start_forked(launcher)
-      in_background do
-        monitor_solid_queue
+    launcher.events.on_booted do
+      @solid_queue_pid = fork do
+        Thread.new { monitor_puma }
+        SolidQueue::Supervisor.start
       end
-
-      launcher.events.on_booted do
-        @solid_queue_pid = fork do
-          Thread.new { monitor_puma }
-          SolidQueue::Supervisor.start(mode: :fork)
-        end
-      end
-
-      launcher.events.on_stopped { stop_solid_queue }
-      launcher.events.on_restart { stop_solid_queue }
     end
 
-    def start_async(launcher)
-      launcher.events.on_booted { @solid_queue_supervisor = SolidQueue::Supervisor.start(mode: :async) }
-      launcher.events.on_stopped { solid_queue_supervisor.stop }
-      launcher.events.on_restart { solid_queue_supervisor.stop; solid_queue_supervisor.start }
-    end
+    launcher.events.on_stopped { stop_solid_queue }
+    launcher.events.on_restart { stop_solid_queue }
+  end
 
+  private
     def stop_solid_queue
       Process.waitpid(solid_queue_pid, Process::WNOHANG)
       log "Stopping Solid Queue..."

data/lib/solid_queue/cli.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module SolidQueue
+  class Cli < Thor
+    class_option :config_file, type: :string, aliases: "-c", default: Configuration::DEFAULT_CONFIG_FILE_PATH, desc: "Path to config file"
+
+    def self.exit_on_failure?
+      true
+    end
+
+    desc :start, "Starts Solid Queue supervisor to dispatch and perform enqueued jobs. Default command."
+    default_command :start
+
+    def start
+      SolidQueue::Supervisor.start(load_configuration_from: options["config_file"])
+    end
+  end
+end

data/lib/solid_queue/configuration.rb

@@ -2,6 +2,12 @@
 
 module SolidQueue
   class Configuration
+    class Process < Struct.new(:kind, :attributes)
+      def instantiate
+        "SolidQueue::#{kind.to_s.titleize}".safe_constantize.new(**attributes)
+      end
+    end
+
     WORKER_DEFAULTS = {
       queues: "*",
       threads: 3,
@@ -17,65 +23,70 @@ module SolidQueue
       recurring_tasks: []
     }
 
-    def initialize(mode: :fork, load_from: nil)
-      @mode = mode.to_s.inquiry
+    DEFAULT_CONFIG = {
+      workers: [ WORKER_DEFAULTS ],
+      dispatchers: [ DISPATCHER_DEFAULTS ]
+    }
+
+    def initialize(load_from: nil)
       @raw_config = config_from(load_from)
     end
 
-    def processes
+    def configured_processes
       dispatchers + workers
     end
 
-    def workers
-      workers_options.flat_map do |worker_options|
-        processes = if mode.fork?
-          worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
-        else
-          WORKER_DEFAULTS[:processes]
-        end
-        processes.times.map { Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
-      end
-    end
-
-    def dispatchers
-      dispatchers_options.map do |dispatcher_options|
-        recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
-        Dispatcher.new **dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
-      end
-    end
-
     def max_number_of_threads
       # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
       workers_options.map { |options| options[:threads] }.max + 2
     end
 
     private
-      attr_reader :raw_config, :mode
+      attr_reader :raw_config
 
       DEFAULT_CONFIG_FILE_PATH = "config/solid_queue.yml"
 
+      def workers
+        workers_options.flat_map do |worker_options|
+          processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+          processes.times.map { Process.new(:worker, worker_options.with_defaults(WORKER_DEFAULTS)) }
+        end
+      end
+
+      def dispatchers
+        dispatchers_options.map do |dispatcher_options|
+          recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
+          Process.new :dispatcher, dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
+        end
+      end
+
       def config_from(file_or_hash, env: Rails.env)
-        config = load_config_from(file_or_hash)
-        config[env.to_sym] ? config[env.to_sym] : config
+        load_config_from(file_or_hash).then do |config|
+          config = config[env.to_sym] ? config[env.to_sym] : config
+          if (config.keys & DEFAULT_CONFIG.keys).any? then config
+          else
+            DEFAULT_CONFIG
+          end
+        end
       end
 
       def workers_options
-        @workers_options ||= options_from_raw_config(:workers, WORKER_DEFAULTS)
+        @workers_options ||= options_from_raw_config(:workers)
           .map { |options| options.dup.symbolize_keys }
       end
 
       def dispatchers_options
-        @dispatchers_options ||= options_from_raw_config(:dispatchers, DISPATCHER_DEFAULTS)
+        @dispatchers_options ||= options_from_raw_config(:dispatchers)
           .map { |options| options.dup.symbolize_keys }
       end
 
-      def options_from_raw_config(key, defaults)
-        raw_config.empty? ? [ defaults ] : Array(raw_config[key])
+      def options_from_raw_config(key)
+        Array(raw_config[key])
       end
 
       def parse_recurring_tasks(tasks)
         Array(tasks).map do |id, options|
-          Dispatcher::RecurringTask.from_configuration(id, **options)
+          RecurringTask.from_configuration(id, **options)
         end.select(&:valid?)
       end
 

data/lib/solid_queue/dispatcher/recurring_schedule.rb

@@ -7,7 +7,7 @@ module SolidQueue
     attr_reader :configured_tasks, :scheduled_tasks
 
     def initialize(tasks)
-      @configured_tasks = Array(tasks).map { |task| Dispatcher::RecurringTask.wrap(task) }
+      @configured_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?)
       @scheduled_tasks = Concurrent::Hash.new
     end
 
@@ -15,33 +15,42 @@ module SolidQueue
       configured_tasks.empty?
     end
 
-    def load_tasks
+    def schedule_tasks
+      wrap_in_app_executor do
+        persist_tasks
+        reload_tasks
+      end
+
       configured_tasks.each do |task|
-        load_task(task)
+        schedule_task(task)
       end
     end
 
-    def load_task(task)
+    def schedule_task(task)
       scheduled_tasks[task.key] = schedule(task)
     end
 
-    def unload_tasks
+    def unschedule_tasks
       scheduled_tasks.values.each(&:cancel)
       scheduled_tasks.clear
     end
 
-    def tasks
-      configured_tasks.each_with_object({}) { |task, hsh| hsh[task.key] = task.to_h }
-    end
-
-    def inspect
-      configured_tasks.map(&:to_s).join(" | ")
+    def task_keys
+      configured_tasks.map(&:key)
     end
 
     private
+      def persist_tasks
+        SolidQueue::RecurringTask.create_or_update_all configured_tasks
+      end
+
+      def reload_tasks
+        @configured_tasks = SolidQueue::RecurringTask.where(key: task_keys)
+      end
+
       def schedule(task)
         scheduled_task = Concurrent::ScheduledTask.new(task.delay_from_now, args: [ self, task, task.next_time ]) do |thread_schedule, thread_task, thread_task_run_at|
-          thread_schedule.load_task(thread_task)
+          thread_schedule.schedule_task(thread_task)
 
           wrap_in_app_executor do
             thread_task.enqueue(at: thread_task_run_at)

data/lib/solid_queue/dispatcher.rb

@@ -4,8 +4,8 @@ module SolidQueue
   class Dispatcher < Processes::Poller
     attr_accessor :batch_size, :concurrency_maintenance, :recurring_schedule
 
-    after_boot :start_concurrency_maintenance, :load_recurring_schedule
-    before_shutdown :stop_concurrency_maintenance, :unload_recurring_schedule
+    after_boot :start_concurrency_maintenance, :schedule_recurring_tasks
+    before_shutdown :stop_concurrency_maintenance, :unschedule_recurring_tasks
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
@@ -19,7 +19,7 @@ module SolidQueue
     end
 
     def metadata
-      super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.tasks.presence)
+      super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.task_keys.presence)
     end
 
     private
@@ -38,16 +38,16 @@ module SolidQueue
         concurrency_maintenance&.start
       end
 
-      def load_recurring_schedule
-        recurring_schedule.load_tasks
+      def schedule_recurring_tasks
+        recurring_schedule.schedule_tasks
       end
 
       def stop_concurrency_maintenance
         concurrency_maintenance&.stop
       end
 
-      def unload_recurring_schedule
-        recurring_schedule.unload_tasks
+      def unschedule_recurring_tasks
+        recurring_schedule.unschedule_tasks
       end
 
       def all_work_completed?
@@ -55,7 +55,7 @@ module SolidQueue
       end
 
       def set_procline
-        procline "waiting"
+        procline "dispatching every #{polling_interval.seconds} seconds"
       end
   end
 end

data/lib/solid_queue/lifecycle_hooks.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module LifecycleHooks
+    extend ActiveSupport::Concern
+
+    included do
+      mattr_reader :lifecycle_hooks, default: { start: [], stop: [] }
+    end
+
+    class_methods do
+      def on_start(&block)
+        self.lifecycle_hooks[:start] << block
+      end
+
+      def on_stop(&block)
+        self.lifecycle_hooks[:stop] << block
+      end
+
+      def clear_hooks
+        self.lifecycle_hooks[:start] = []
+        self.lifecycle_hooks[:stop] = []
+      end
+    end
+
+    private
+      def run_start_hooks
+        run_hooks_for :start
+      end
+
+      def run_stop_hooks
+        run_hooks_for :stop
+      end
+
+      def run_hooks_for(event)
+        self.class.lifecycle_hooks.fetch(event, []).each do |block|
+          block.call
+        rescue Exception => exception
+          handle_thread_error(exception)
+        end
+      end
+  end
+end

data/lib/solid_queue/log_subscriber.rb

@@ -12,11 +12,15 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
   end
 
   def release_many_claimed(event)
-    debug formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
+    info formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
+  end
+
+  def fail_many_claimed(event)
+    warn formatted_event(event, action: "Fail claimed jobs", **event.payload.slice(:job_ids, :process_ids))
   end
 
   def release_claimed(event)
-    debug formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
+    info formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
   end
 
   def retry_all(event)
@@ -63,7 +67,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     attributes = {
       pid: process.pid,
       hostname: process.hostname,
-      process_id: process.process_id
+      process_id: process.process_id,
+      name: process.name
     }.merge(process.metadata)
 
     info formatted_event(event, action: "Started #{process.kind}", **attributes)
@@ -75,7 +80,8 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
     attributes = {
       pid: process.pid,
       hostname: process.hostname,
-      process_id: process.process_id
+      process_id: process.process_id,
+      name: process.name
     }.merge(process.metadata)
 
     info formatted_event(event, action: "Shutdown #{process.kind}", **attributes)
@@ -83,7 +89,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
 
   def register_process(event)
     process_kind = event.payload[:kind]
-    attributes = event.payload.slice(:pid, :hostname, :process_id)
+    attributes = event.payload.slice(:pid, :hostname, :process_id, :name)
 
     if error = event.payload[:error]
       warn formatted_event(event, action: "Error registering #{process_kind}", **attributes.merge(error: formatted_error(error)))
@@ -99,6 +105,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
       process_id: process.id,
       pid: process.pid,
       hostname: process.hostname,
+      name: process.name,
       last_heartbeat_at: process.last_heartbeat_at.iso8601,
       claimed_size: event.payload[:claimed_size],
       pruned: event.payload[:pruned]
@@ -147,7 +154,7 @@ class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
       termsig: status.termsig
 
     if replaced_fork = event.payload[:fork]
-      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname))
+      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname, name: replaced_fork.name))
     else
       warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes)
     end

data/lib/solid_queue/processes/base.rb

@@ -6,6 +6,12 @@ module SolidQueue
       include Callbacks # Defines callbacks needed by other concerns
       include AppExecutor, Registrable, Interruptible, Procline
 
+      attr_reader :name
+
+      def initialize(*)
+        @name = generate_name
+      end
+
       def kind
         self.class.name.demodulize
       end
@@ -21,6 +27,11 @@ module SolidQueue
       def metadata
         {}
       end
+
+      private
+        def generate_name
+          [ kind.downcase, SecureRandom.hex(10) ].join("-")
+        end
     end
   end
 end

data/lib/solid_queue/processes/poller.rb

@@ -8,6 +8,8 @@ module SolidQueue::Processes
 
     def initialize(polling_interval:, **options)
       @polling_interval = polling_interval
+
+      super(**options)
     end
 
     def metadata
@@ -43,10 +45,12 @@ module SolidQueue::Processes
       end
 
       def with_polling_volume
-        if SolidQueue.silence_polling? && ActiveRecord::Base.logger
-          ActiveRecord::Base.logger.silence { yield }
-        else
-          yield
+        SolidQueue.instrument(:polling) do
+          if SolidQueue.silence_polling? && ActiveRecord::Base.logger
+            ActiveRecord::Base.logger.silence { yield }
+          else
+            yield
+          end
         end
       end
   end

data/lib/solid_queue/processes/process_exit_error.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Processes
+    class ProcessExitError < RuntimeError
+      def initialize(status)
+        message = "Process pid=#{status.pid} exited unexpectedly."
+        if status.exitstatus.present?
+          message += " Exited with status #{status.exitstatus}."
+        end
+
+        if status.signaled?
+          message += " Received unhandled signal #{status.termsig}."
+        end
+
+        super(message)
+      end
+    end
+  end
+end

data/lib/solid_queue/processes/process_missing_error.rb

@@ -0,0 +1,9 @@
+module SolidQueue
+  module Processes
+    class ProcessMissingError < RuntimeError
+      def initialize
+        super("The process that was running this job no longer exists")
+      end
+    end
+  end
+end

data/lib/solid_queue/processes/process_pruned_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module Processes
+    class ProcessPrunedError < RuntimeError
+      def initialize(last_heartbeat_at)
+        super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
+      end
+    end
+  end
+end

data/lib/solid_queue/processes/registrable.rb

@@ -21,6 +21,7 @@ module SolidQueue::Processes
       def register
         @process = SolidQueue::Process.register \
           kind: kind,
+          name: name,
           pid: pid,
           hostname: hostname,
           supervisor: try(:supervisor),

data/lib/solid_queue/processes/runnable.rb

@@ -7,11 +7,7 @@ module SolidQueue::Processes
     attr_writer :mode
 
     def start
-      @stopped = false
-
-      SolidQueue.instrument(:start_process, process: self) do
-        run_callbacks(:boot) { boot }
-      end
+      boot
 
       if running_async?
         @thread = create_thread { run }
@@ -25,14 +21,6 @@ module SolidQueue::Processes
       @thread&.join
     end
 
-    def name
-      @name ||= [ kind.downcase, SecureRandom.hex(6) ].join("-")
-    end
-
-    def alive?
-      !running_async? || @thread.alive?
-    end
-
     private
       DEFAULT_MODE = :async
 
@@ -41,9 +29,15 @@ module SolidQueue::Processes
       end
 
       def boot
-        if running_as_fork?
-          register_signal_handlers
-          set_procline
+        SolidQueue.instrument(:start_process, process: self) do
+          run_callbacks(:boot) do
+            @stopped = false
+
+            if running_as_fork?
+              register_signal_handlers
+              set_procline
+            end
+          end
         end
       end
 

data/lib/solid_queue/supervisor/maintenance.rb

@@ -3,7 +3,7 @@ module SolidQueue
     extend ActiveSupport::Concern
 
     included do
-      after_boot :release_orphaned_executions
+      after_boot :fail_orphaned_executions
     end
 
     private
@@ -27,8 +27,10 @@ module SolidQueue
         wrap_in_app_executor { SolidQueue::Process.prune }
       end
 
-      def release_orphaned_executions
-        wrap_in_app_executor { SolidQueue::ClaimedExecution.orphaned.release_all }
+      def fail_orphaned_executions
+        wrap_in_app_executor do
+          ClaimedExecution.orphaned.fail_all_with(Processes::ProcessMissingError.new)
+        end
       end
   end
 end