solid_queue 0.4.1 → 0.7.1

data/lib/solid_queue/supervisor.rb

@@ -2,24 +2,33 @@
 
 module SolidQueue
   class Supervisor < Processes::Base
-    include Maintenance
+    include LifecycleHooks
+    include Maintenance, Signals, Pidfiled
 
     class << self
-      def start(mode: :fork, load_configuration_from: nil)
+      def start(load_configuration_from: nil)
         SolidQueue.supervisor = true
-        configuration = Configuration.new(mode: mode, load_from: load_configuration_from)
+        configuration = Configuration.new(load_from: load_configuration_from)
 
-        klass = mode == :fork ? ForkSupervisor : AsyncSupervisor
-        klass.new(configuration).tap(&:start)
+        if configuration.configured_processes.any?
+          new(configuration).tap(&:start)
+        else
+          abort "No workers or processed configured. Exiting..."
+        end
       end
     end
 
     def initialize(configuration)
       @configuration = configuration
+      @forks = {}
+      @configured_processes = {}
+
+      super
     end
 
     def start
       boot
+      run_start_hooks
 
       start_processes
       launch_maintenance_task
@@ -29,10 +38,11 @@ module SolidQueue
 
     def stop
       @stopped = true
+      run_stop_hooks
     end
 
     private
-      attr_reader :configuration
+      attr_reader :configuration, :forks, :configured_processes
 
       def boot
         SolidQueue.instrument(:start_process, process: self) do
@@ -44,18 +54,66 @@ module SolidQueue
       end
 
       def start_processes
-        configuration.processes.each { |configured_process| start_process(configured_process) }
+        configuration.configured_processes.each { |configured_process| start_process(configured_process) }
+      end
+
+      def supervise
+        loop do
+          break if stopped?
+
+          set_procline
+          process_signal_queue
+
+          unless stopped?
+            reap_and_replace_terminated_forks
+            interruptible_sleep(1.second)
+          end
+        end
+      ensure
+        shutdown
+      end
+
+      def start_process(configured_process)
+        process_instance = configured_process.instantiate.tap do |instance|
+          instance.supervised_by process
+          instance.mode = :fork
+        end
+
+        pid = fork do
+          process_instance.start
+        end
+
+        configured_processes[pid] = configured_process
+        forks[pid] = process_instance
       end
 
       def stopped?
         @stopped
       end
 
-      def supervise
+      def set_procline
+        procline "supervising #{supervised_processes.join(", ")}"
       end
 
-      def start_process(configured_process)
-        raise NotImplementedError
+      def terminate_gracefully
+        SolidQueue.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: supervised_processes) do |payload|
+          term_forks
+
+          Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
+            reap_terminated_forks
+          end
+
+          unless all_forks_terminated?
+            payload[:shutdown_timeout_exceeded] = true
+            terminate_immediately
+          end
+        end
+      end
+
+      def terminate_immediately
+        SolidQueue.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: supervised_processes) do
+          quit_forks
+        end
       end
 
       def shutdown
@@ -69,5 +127,63 @@ module SolidQueue
       def sync_std_streams
         STDOUT.sync = STDERR.sync = true
       end
+
+      def supervised_processes
+        forks.keys
+      end
+
+      def term_forks
+        signal_processes(forks.keys, :TERM)
+      end
+
+      def quit_forks
+        signal_processes(forks.keys, :QUIT)
+      end
+
+      def reap_and_replace_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          replace_fork(pid, status)
+        end
+      end
+
+      def reap_terminated_forks
+        loop do
+          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+          break unless pid
+
+          if (terminated_fork = forks.delete(pid)) && (!status.exited? || status.exitstatus > 0)
+            handle_claimed_jobs_by(terminated_fork, status)
+          end
+
+          configured_processes.delete(pid)
+        end
+      rescue SystemCallError
+        # All children already reaped
+      end
+
+      def replace_fork(pid, status)
+        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
+          if terminated_fork = forks.delete(pid)
+            payload[:fork] = terminated_fork
+            handle_claimed_jobs_by(terminated_fork, status)
+
+            start_process(configured_processes.delete(pid))
+          end
+        end
+      end
+
+      def handle_claimed_jobs_by(terminated_fork, status)
+        if registered_process = process.supervisees.find_by(name: terminated_fork.name)
+          error = Processes::ProcessExitError.new(status)
+          registered_process.fail_all_claimed_executions_with(error)
+        end
+      end
+
+      def all_forks_terminated?
+        forks.empty?
+      end
   end
 end

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.4.1"
+  VERSION = "0.7.1"
 end

data/lib/solid_queue/worker.rb

@@ -2,6 +2,11 @@
 
 module SolidQueue
   class Worker < Processes::Poller
+    include LifecycleHooks
+
+    after_boot :run_start_hooks
+    before_shutdown :run_stop_hooks
+
     attr_accessor :queues, :pool
 
     def initialize(**options)

data/lib/solid_queue.rb

@@ -43,6 +43,16 @@ module SolidQueue
   mattr_accessor :clear_finished_jobs_after, default: 1.day
   mattr_accessor :default_concurrency_control_period, default: 3.minutes
 
+  delegate :on_start, :on_stop, to: Supervisor
+
+  def on_worker_start(...)
+    Worker.on_start(...)
+  end
+
+  def on_worker_stop(...)
+    Worker.on_stop(...)
+  end
+
   def supervisor?
     supervisor
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.7.1
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-05 00:00:00.000000000 Z
+date: 2024-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -188,6 +202,8 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
+- UPGRADING.md
+- app/jobs/solid_queue/recurring_job.rb
 - app/models/solid_queue/blocked_execution.rb
 - app/models/solid_queue/claimed_execution.rb
 - app/models/solid_queue/execution.rb
@@ -210,39 +226,48 @@ files:
 - app/models/solid_queue/ready_execution.rb
 - app/models/solid_queue/record.rb
 - app/models/solid_queue/recurring_execution.rb
+- app/models/solid_queue/recurring_task.rb
+- app/models/solid_queue/recurring_task/arguments.rb
 - app/models/solid_queue/scheduled_execution.rb
 - app/models/solid_queue/semaphore.rb
 - config/routes.rb
 - db/migrate/20231211200639_create_solid_queue_tables.rb
 - db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb
 - db/migrate/20240218110712_create_recurring_executions.rb
+- db/migrate/20240719134516_create_recurring_tasks.rb
+- db/migrate/20240811173327_add_name_to_processes.rb
+- db/migrate/20240813160053_make_name_not_null.rb
+- db/migrate/20240819165045_change_solid_queue_recurring_tasks_static_to_not_null.rb
 - lib/active_job/concurrency_controls.rb
 - lib/active_job/queue_adapters/solid_queue_adapter.rb
 - lib/generators/solid_queue/install/USAGE
 - lib/generators/solid_queue/install/install_generator.rb
 - lib/generators/solid_queue/install/templates/config.yml
+- lib/generators/solid_queue/install/templates/jobs
 - lib/puma/plugin/solid_queue.rb
 - lib/solid_queue.rb
 - lib/solid_queue/app_executor.rb
+- lib/solid_queue/cli.rb
 - lib/solid_queue/configuration.rb
 - lib/solid_queue/dispatcher.rb
 - lib/solid_queue/dispatcher/concurrency_maintenance.rb
 - lib/solid_queue/dispatcher/recurring_schedule.rb
-- lib/solid_queue/dispatcher/recurring_task.rb
 - lib/solid_queue/engine.rb
+- lib/solid_queue/lifecycle_hooks.rb
 - lib/solid_queue/log_subscriber.rb
 - lib/solid_queue/pool.rb
 - lib/solid_queue/processes/base.rb
 - lib/solid_queue/processes/callbacks.rb
 - lib/solid_queue/processes/interruptible.rb
 - lib/solid_queue/processes/poller.rb
+- lib/solid_queue/processes/process_exit_error.rb
+- lib/solid_queue/processes/process_missing_error.rb
+- lib/solid_queue/processes/process_pruned_error.rb
 - lib/solid_queue/processes/procline.rb
 - lib/solid_queue/processes/registrable.rb
 - lib/solid_queue/processes/runnable.rb
 - lib/solid_queue/processes/supervised.rb
 - lib/solid_queue/supervisor.rb
-- lib/solid_queue/supervisor/async_supervisor.rb
-- lib/solid_queue/supervisor/fork_supervisor.rb
 - lib/solid_queue/supervisor/maintenance.rb
 - lib/solid_queue/supervisor/pidfile.rb
 - lib/solid_queue/supervisor/pidfiled.rb
@@ -258,8 +283,9 @@ metadata:
   homepage_uri: https://github.com/rails/solid_queue
   source_code_uri: https://github.com/rails/solid_queue
 post_install_message: |
-  Upgrading to Solid Queue 0.4.x? There are some breaking changes about how Solid Queue is started. Check
-  https://github.com/rails/solid_queue/blob/main/UPGRADING.md for upgrade instructions.
+  Upgrading to Solid Queue 0.4.x, 0.5.x, 0.6.x or 0.7.x? There are some breaking changes about how Solid Queue is started,
+  configuration and new migrations. Check https://github.com/rails/solid_queue/blob/main/UPGRADING.md
+  for upgrade instructions.
 rdoc_options: []
 require_paths:
 - lib

data/lib/solid_queue/dispatcher/recurring_task.rb

@@ -1,99 +0,0 @@
-require "fugit"
-
-module SolidQueue
-  class Dispatcher::RecurringTask
-    class << self
-      def wrap(args)
-        args.is_a?(self) ? args : from_configuration(args.first, **args.second)
-      end
-
-      def from_configuration(key, **options)
-        new(key, class_name: options[:class], schedule: options[:schedule], arguments: options[:args])
-      end
-    end
-
-    attr_reader :key, :schedule, :class_name, :arguments
-
-    def initialize(key, class_name:, schedule:, arguments: nil)
-      @key = key
-      @class_name = class_name
-      @schedule = schedule
-      @arguments = Array(arguments)
-    end
-
-    def delay_from_now
-      [ (next_time - Time.current).to_f, 0 ].max
-    end
-
-    def next_time
-      parsed_schedule.next_time.utc
-    end
-
-    def enqueue(at:)
-      SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload|
-        active_job = if using_solid_queue_adapter?
-          perform_later_and_record(run_at: at)
-        else
-          payload[:other_adapter] = true
-
-          perform_later do |job|
-            unless job.successfully_enqueued?
-              payload[:enqueue_error] = job.enqueue_error&.message
-            end
-          end
-        end
-
-        payload[:active_job_id] = active_job.job_id if active_job
-      rescue RecurringExecution::AlreadyRecorded
-        payload[:skipped] = true
-      rescue Job::EnqueueError => error
-        payload[:enqueue_error] = error.message
-      end
-    end
-
-    def valid?
-      parsed_schedule.instance_of?(Fugit::Cron)
-    end
-
-    def to_s
-      "#{class_name}.perform_later(#{arguments.map(&:inspect).join(",")}) [ #{parsed_schedule.original} ]"
-    end
-
-    def to_h
-      {
-        schedule: schedule,
-        class_name: class_name,
-        arguments: arguments
-      }
-    end
-
-    private
-      def using_solid_queue_adapter?
-        job_class.queue_adapter_name.inquiry.solid_queue?
-      end
-
-      def perform_later_and_record(run_at:)
-        RecurringExecution.record(key, run_at) { perform_later }
-      end
-
-      def perform_later(&block)
-        job_class.perform_later(*arguments_with_kwargs, &block)
-      end
-
-      def arguments_with_kwargs
-        if arguments.last.is_a?(Hash)
-          arguments[0...-1] + [ Hash.ruby2_keywords_hash(arguments.last) ]
-        else
-          arguments
-        end
-      end
-
-      def parsed_schedule
-        @parsed_schedule ||= Fugit.parse(schedule)
-      end
-
-      def job_class
-        @job_class ||= class_name.safe_constantize
-      end
-  end
-end

data/lib/solid_queue/supervisor/async_supervisor.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module SolidQueue
-  class Supervisor::AsyncSupervisor < Supervisor
-    def initialize(*)
-      super
-      @threads = Concurrent::Map.new
-    end
-
-    def kind
-      "Supervisor(async)"
-    end
-
-    def stop
-      super
-      stop_threads
-      threads.clear
-
-      shutdown
-    end
-
-    private
-      attr_reader :threads
-
-      def start_process(configured_process)
-        configured_process.supervised_by process
-        configured_process.start
-
-        threads[configured_process.name] = configured_process
-      end
-
-      def stop_threads
-        stop_threads = threads.values.map do |thr|
-          Thread.new { thr.stop }
-        end
-
-        stop_threads.each { |thr| thr.join(SolidQueue.shutdown_timeout) }
-      end
-
-      def all_threads_terminated?
-        threads.values.none?(&:alive?)
-      end
-  end
-end

data/lib/solid_queue/supervisor/fork_supervisor.rb

@@ -1,108 +0,0 @@
-# frozen_string_literal: true
-
-module SolidQueue
-  class Supervisor::ForkSupervisor < Supervisor
-    include Signals, Pidfiled
-
-    def initialize(*)
-      super
-      @forks = {}
-    end
-
-    def kind
-      "Supervisor(fork)"
-    end
-
-    private
-      attr_reader :forks
-
-      def supervise
-        loop do
-          break if stopped?
-
-          procline "supervising #{forks.keys.join(", ")}"
-          process_signal_queue
-
-          unless stopped?
-            reap_and_replace_terminated_forks
-            interruptible_sleep(1.second)
-          end
-        end
-      ensure
-        shutdown
-      end
-
-      def start_process(configured_process)
-        configured_process.supervised_by process
-        configured_process.mode = :fork
-
-        pid = fork do
-          configured_process.start
-        end
-
-        forks[pid] = configured_process
-      end
-
-      def terminate_gracefully
-        SolidQueue.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do |payload|
-          term_forks
-
-          Timer.wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
-            reap_terminated_forks
-          end
-
-          unless all_forks_terminated?
-            payload[:shutdown_timeout_exceeded] = true
-            terminate_immediately
-          end
-        end
-      end
-
-      def terminate_immediately
-        SolidQueue.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: forks.keys) do
-          quit_forks
-        end
-      end
-
-      def term_forks
-        signal_processes(forks.keys, :TERM)
-      end
-
-      def quit_forks
-        signal_processes(forks.keys, :QUIT)
-      end
-
-      def reap_and_replace_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          replace_fork(pid, status)
-        end
-      end
-
-      def reap_terminated_forks
-        loop do
-          pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
-          break unless pid
-
-          forks.delete(pid)
-        end
-      rescue SystemCallError
-        # All children already reaped
-      end
-
-      def replace_fork(pid, status)
-        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
-          if supervised_fork = forks.delete(pid)
-            payload[:fork] = supervised_fork
-            start_process(supervised_fork)
-          end
-        end
-      end
-
-      def all_forks_terminated?
-        forks.empty?
-      end
-  end
-end