good_job 1.7.0 → 1.9.2

data/lib/good_job/notifier.rb

@@ -15,7 +15,7 @@ module GoodJob # :nodoc:
     # Default Postgres channel for LISTEN/NOTIFY
     CHANNEL = 'good_job'.freeze
     # Defaults for instance of Concurrent::ThreadPoolExecutor
-    POOL_OPTIONS = {
+    EXECUTOR_OPTIONS = {
       name: name,
       min_threads: 0,
       max_threads: 1,
@@ -30,13 +30,13 @@ module GoodJob # :nodoc:
     # @!attribute [r] instances
     #   @!scope class
     #   List of all instantiated Notifiers in the current process.
-    #   @return [array<GoodJob:Adapter>]
+    #   @return [Array<GoodJob::Adapter>]
     cattr_reader :instances, default: [], instance_reader: false
 
     # Send a message via Postgres NOTIFY
     # @param message [#to_json]
     def self.notify(message)
-      connection = ActiveRecord::Base.connection
+      connection = Job.connection
       connection.exec_query <<~SQL.squish
         NOTIFY #{CHANNEL}, #{connection.quote(message.to_json)}
       SQL
@@ -53,7 +53,7 @@ module GoodJob # :nodoc:
 
       self.class.instances << self
 
-      create_pool
+      create_executor
       listen
     end
 
@@ -63,34 +63,42 @@ module GoodJob # :nodoc:
       @listening.true?
     end
 
-    # Restart the notifier.
-    # When shutdown, start; or shutdown and start.
-    # @param wait [Boolean] Wait for background thread to finish
-    # @return [void]
-    def restart(wait: true)
-      shutdown(wait: wait)
-      create_pool
-      listen
-    end
+    # Tests whether the notifier is running.
+    # @return [true, false, nil]
+    delegate :running?, to: :executor, allow_nil: true
+
+    # Tests whether the scheduler is shutdown.
+    # @return [true, false, nil]
+    delegate :shutdown?, to: :executor, allow_nil: true
 
     # Shut down the notifier.
     # This stops the background LISTENing thread.
-    # If +wait+ is +true+, the notifier will wait for background thread to shutdown.
-    # If +wait+ is +false+, this method will return immediately even though threads may still be running.
     # Use {#shutdown?} to determine whether threads have stopped.
-    # @param wait [Boolean] Wait for actively executing threads to finish
+    # @param timeout [nil, Numeric] Seconds to wait for active threads.
+    #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
+    #   * +-1+, the scheduler will wait until the shutdown is complete.
+    #   * +0+, the scheduler will immediately shutdown and stop any threads.
+    #   * A positive number will wait that many seconds before stopping any remaining active threads.
     # @return [void]
-    def shutdown(wait: true)
-      return unless @pool.running?
+    def shutdown(timeout: -1)
+      return if executor.nil? || executor.shutdown?
 
-      @pool.shutdown
-      @pool.wait_for_termination if wait
+      executor.shutdown if executor.running?
+
+      if executor.shuttingdown? && timeout # rubocop:disable Style/GuardClause
+        executor_wait = timeout.negative? ? nil : timeout
+        executor.kill unless executor.wait_for_termination(executor_wait)
+      end
     end
 
-    # Tests whether the notifier is shutdown.
-    # @return [true, false, nil]
-    def shutdown?
-      !@pool.running?
+    # Restart the notifier.
+    # When shutdown, start; or shutdown and start.
+    # @param timeout [nil, Numeric] Seconds to wait; shares same values as {#shutdown}.
+    # @return [void]
+    def restart(timeout: -1)
+      shutdown(timeout: timeout) if running?
+      create_executor
+      listen
     end
 
     # Invoked on completion of ThreadPoolExecutor task
@@ -109,36 +117,36 @@ module GoodJob # :nodoc:
 
     private
 
-    def create_pool
-      @pool = Concurrent::ThreadPoolExecutor.new(POOL_OPTIONS)
+    attr_reader :executor
+
+    def create_executor
+      @executor = Concurrent::ThreadPoolExecutor.new(EXECUTOR_OPTIONS)
     end
 
     def listen
-      future = Concurrent::Future.new(args: [@recipients, @pool, @listening], executor: @pool) do |recipients, pool, listening|
+      future = Concurrent::Future.new(args: [@recipients, executor, @listening], executor: @executor) do |thr_recipients, thr_executor, thr_listening|
         with_listen_connection do |conn|
           ActiveSupport::Notifications.instrument("notifier_listen.good_job") do
             conn.async_exec("LISTEN #{CHANNEL}").clear
           end
 
           ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-            while pool.running?
-              listening.make_true
+            thr_listening.make_true
+            while thr_executor.running?
               conn.wait_for_notify(WAIT_INTERVAL) do |channel, _pid, payload|
-                listening.make_false
                 next unless channel == CHANNEL
 
                 ActiveSupport::Notifications.instrument("notifier_notified.good_job", { payload: payload })
                 parsed_payload = JSON.parse(payload, symbolize_names: true)
-                recipients.each do |recipient|
+                thr_recipients.each do |recipient|
                   target, method_name = recipient.is_a?(Array) ? recipient : [recipient, :call]
                   target.send(method_name, parsed_payload)
                 end
               end
-              listening.make_false
             end
           end
         ensure
-          listening.make_false
+          thr_listening.make_false
           ActiveSupport::Notifications.instrument("notifier_unlisten.good_job") do
             conn.async_exec("UNLISTEN *").clear
           end
@@ -150,8 +158,8 @@ module GoodJob # :nodoc:
     end
 
     def with_listen_connection
-      ar_conn = ActiveRecord::Base.connection_pool.checkout.tap do |conn|
-        ActiveRecord::Base.connection_pool.remove(conn)
+      ar_conn = Job.connection_pool.checkout.tap do |conn|
+        Job.connection_pool.remove(conn)
       end
       pg_conn = ar_conn.raw_connection
       raise AdapterCannotListenError unless pg_conn.respond_to? :wait_for_notify

data/lib/good_job/poller.rb

@@ -16,7 +16,7 @@ module GoodJob # :nodoc:
     # @!attribute [r] instances
     #   @!scope class
     #   List of all instantiated Pollers in the current process.
-    #   @return [array<GoodJob:Poller>]
+    #   @return [Array<GoodJob::Poller>]
     cattr_reader :instances, default: [], instance_reader: false
 
     # Creates GoodJob::Poller from a GoodJob::Configuration instance.
@@ -40,35 +40,43 @@ module GoodJob # :nodoc:
 
       self.class.instances << self
 
-      create_pool
+      create_timer
     end
 
-    # Shut down the poller.
-    # If +wait+ is +true+, the poller will wait for background thread to shutdown.
-    # If +wait+ is +false+, this method will return immediately even though threads may still be running.
+    # Tests whether the timer is running.
+    # @return [true, false, nil]
+    delegate :running?, to: :timer, allow_nil: true
+
+    # Tests whether the timer is shutdown.
+    # @return [true, false, nil]
+    delegate :shutdown?, to: :timer, allow_nil: true
+
+    # Shut down the notifier.
     # Use {#shutdown?} to determine whether threads have stopped.
-    # @param wait [Boolean] Wait for actively executing threads to finish
+    # @param timeout [nil, Numeric] Seconds to wait for active threads.
+    #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
+    #   * +-1+, the scheduler will wait until the shutdown is complete.
+    #   * +0+, the scheduler will immediately shutdown and stop any threads.
+    #   * A positive number will wait that many seconds before stopping any remaining active threads.
     # @return [void]
-    def shutdown(wait: true)
-      return unless @timer&.running?
+    def shutdown(timeout: -1)
+      return if timer.nil? || timer.shutdown?
 
-      @timer.shutdown
-      @timer.wait_for_termination if wait
-    end
+      timer.shutdown if timer.running?
 
-    # Tests whether the poller is shutdown.
-    # @return [true, false, nil]
-    def shutdown?
-      !@timer&.running?
+      if timer.shuttingdown? && timeout # rubocop:disable Style/GuardClause
+        timer_wait = timeout.negative? ? nil : timeout
+        timer.kill unless timer.wait_for_termination(timer_wait)
+      end
     end
 
     # Restart the poller.
     # When shutdown, start; or shutdown and start.
-    # @param wait [Boolean] Wait for background thread to finish
+    # @param timeout [nil, Numeric] Seconds to wait; shares same values as {#shutdown}.
     # @return [void]
-    def restart(wait: true)
-      shutdown(wait: wait)
-      create_pool
+    def restart(timeout: -1)
+      shutdown(timeout: timeout) if running?
+      create_timer
     end
 
     # Invoked on completion of TimerTask task.
@@ -76,12 +84,14 @@ module GoodJob # :nodoc:
     # @return [void]
     def timer_observer(time, executed_task, thread_error)
       GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
-      instrument("finished_timer_task", { result: executed_task, error: thread_error, time: time })
+      ActiveSupport::Notifications.instrument("finished_timer_task", { result: executed_task, error: thread_error, time: time })
     end
 
     private
 
-    def create_pool
+    attr_reader :timer
+
+    def create_timer
       return if @timer_options[:execution_interval] <= 0
 
       @timer = Concurrent::TimerTask.new(@timer_options) do

data/lib/good_job/scheduler.rb

@@ -16,28 +16,28 @@ module GoodJob # :nodoc:
   #
   class Scheduler
     # Defaults for instance of Concurrent::ThreadPoolExecutor
-    # The thread pool is where work is performed.
-    DEFAULT_POOL_OPTIONS = {
+    # The thread pool executor is where work is performed.
+    DEFAULT_EXECUTOR_OPTIONS = {
       name: name,
       min_threads: 0,
       max_threads: Configuration::DEFAULT_MAX_THREADS,
       auto_terminate: true,
       idletime: 60,
-      max_queue: 1, # ideally zero, but 0 == infinite
+      max_queue: Configuration::DEFAULT_MAX_THREADS,
       fallback_policy: :discard,
     }.freeze
 
     # @!attribute [r] instances
     #   @!scope class
     #   List of all instantiated Schedulers in the current process.
-    #   @return [array<GoodJob:Scheduler>]
+    #   @return [Array<GoodJob::Scheduler>]
     cattr_reader :instances, default: [], instance_reader: false
 
     # Creates GoodJob::Scheduler(s) and Performers from a GoodJob::Configuration instance.
     # @param configuration [GoodJob::Configuration]
     # @param warm_cache_on_initialize [Boolean]
     # @return [GoodJob::Scheduler, GoodJob::MultiScheduler]
-    def self.from_configuration(configuration, warm_cache_on_initialize: true)
+    def self.from_configuration(configuration, warm_cache_on_initialize: false)
       schedulers = configuration.queue_string.split(';').map do |queue_string_and_max_threads|
         queue_string, max_threads = queue_string_and_max_threads.split(':')
         max_threads = (max_threads || configuration.max_threads).to_i
@@ -61,8 +61,8 @@ module GoodJob # :nodoc:
     # @param performer [GoodJob::JobPerformer]
     # @param max_threads [Numeric, nil] number of seconds between polls for jobs
     # @param max_cache [Numeric, nil] maximum number of scheduled jobs to cache in memory
-    # @param warm_cache_on_initialize [Boolean] whether to warm the cache immediately
-    def initialize(performer, max_threads: nil, max_cache: nil, warm_cache_on_initialize: true)
+    # @param warm_cache_on_initialize [Boolean] whether to warm the cache immediately, or manually by calling +warm_cache+
+    def initialize(performer, max_threads: nil, max_cache: nil, warm_cache_on_initialize: false)
       raise ArgumentError, "Performer argument must implement #next" unless performer.respond_to?(:next)
 
       self.class.instances << self
@@ -70,63 +70,75 @@ module GoodJob # :nodoc:
       @performer = performer
 
       @max_cache = max_cache || 0
-      @pool_options = DEFAULT_POOL_OPTIONS.dup
-      @pool_options[:max_threads] = max_threads if max_threads.present?
-      @pool_options[:name] = "GoodJob::Scheduler(queues=#{@performer.name} max_threads=#{@pool_options[:max_threads]})"
+      @executor_options = DEFAULT_EXECUTOR_OPTIONS.dup
+      if max_threads.present?
+        @executor_options[:max_threads] = max_threads
+        @executor_options[:max_queue] = max_threads
+      end
+      @executor_options[:name] = "GoodJob::Scheduler(queues=#{@performer.name} max_threads=#{@executor_options[:max_threads]})"
 
-      create_pool
+      create_executor
       warm_cache if warm_cache_on_initialize
     end
 
+    # Tests whether the scheduler is running.
+    # @return [true, false, nil]
+    delegate :running?, to: :executor, allow_nil: true
+
+    # Tests whether the scheduler is shutdown.
+    # @return [true, false, nil]
+    delegate :shutdown?, to: :executor, allow_nil: true
+
     # Shut down the scheduler.
-    # This stops all threads in the pool.
-    # If +wait+ is +true+, the scheduler will wait for any active tasks to finish.
-    # If +wait+ is +false+, this method will return immediately even though threads may still be running.
+    # This stops all threads in the thread pool.
     # Use {#shutdown?} to determine whether threads have stopped.
-    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @param timeout [nil, Numeric] Seconds to wait for actively executing jobs to finish
+    #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
+    #   * +-1+, the scheduler will wait until the shutdown is complete.
+    #   * +0+, the scheduler will immediately shutdown and stop any active tasks.
+    #   * A positive number will wait that many seconds before stopping any remaining active tasks.
     # @return [void]
-    def shutdown(wait: true)
-      return unless @pool&.running?
-
-      instrument("scheduler_shutdown_start", { wait: wait })
-      instrument("scheduler_shutdown", { wait: wait }) do
-        @timer_set.shutdown
+    def shutdown(timeout: -1)
+      return if executor.nil? || executor.shutdown?
+
+      instrument("scheduler_shutdown_start", { timeout: timeout })
+      instrument("scheduler_shutdown", { timeout: timeout }) do
+        if executor.running?
+          @timer_set.shutdown
+          executor.shutdown
+        end
 
-        @pool.shutdown
-        @pool.wait_for_termination if wait
-        # TODO: Should be killed if wait is not true
+        if executor.shuttingdown? && timeout
+          executor_wait = timeout.negative? ? nil : timeout
+          executor.kill unless executor.wait_for_termination(executor_wait)
+        end
       end
     end
 
-    # Tests whether the scheduler is shutdown.
-    # @return [true, false, nil]
-    def shutdown?
-      !@pool&.running?
-    end
-
     # Restart the Scheduler.
     # When shutdown, start; or shutdown and start.
-    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @param timeout [nil, Numeric] Seconds to wait for actively executing jobs to finish; shares same values as {#shutdown}.
     # @return [void]
-    def restart(wait: true)
+    def restart(timeout: -1)
       instrument("scheduler_restart_pools") do
-        shutdown(wait: wait) unless shutdown?
-        create_pool
+        shutdown(timeout: timeout) if running?
+        create_executor
         warm_cache
       end
     end
 
     # Wakes a thread to allow the performer to execute a task.
-    # @param state [nil, Object] Contextual information for the performer. See {Performer#next?}.
+    # @param state [nil, Object] Contextual information for the performer. See {JobPerformer#next?}.
     # @return [nil, Boolean] Whether work was started.
-    #   Returns +nil+ if the scheduler is unable to take new work, for example if the thread pool is shut down or at capacity.
-    #   Returns +true+ if the performer started executing work.
-    #   Returns +false+ if the performer decides not to attempt to execute a task based on the +state+ that is passed to it.
+    #
+    #   * +nil+ if the scheduler is unable to take new work, for example if the thread pool is shut down or at capacity.
+    #   * +true+ if the performer started executing work.
+    #   * +false+ if the performer decides not to attempt to execute a task based on the +state+ that is passed to it.
     def create_thread(state = nil)
-      return nil unless @pool.running?
+      return nil unless executor.running?
 
       if state
-        return false unless @performer.next?(state)
+        return false unless performer.next?(state)
 
         if state[:scheduled_at]
           scheduled_at = if state[:scheduled_at].is_a? String
@@ -141,18 +153,12 @@ module GoodJob # :nodoc:
       delay ||= 0
       run_now = delay <= 0.01
       if run_now
-        return nil unless @pool.ready_worker_count.positive?
+        return nil unless executor.ready_worker_count.positive?
       elsif @max_cache.positive?
         return nil unless remaining_cache_count.positive?
       end
 
-      future = Concurrent::ScheduledTask.new(delay, args: [@performer], executor: @pool, timer_set: timer_set) do |performer|
-        output = nil
-        Rails.application.executor.wrap { output = performer.next }
-        output
-      end
-      future.add_observer(self, :task_observer)
-      future.execute
+      create_task(delay)
 
       run_now ? true : nil
     end
@@ -163,43 +169,69 @@ module GoodJob # :nodoc:
     def task_observer(time, output, thread_error)
       GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
       instrument("finished_job_task", { result: output, error: thread_error, time: time })
-      create_thread if output
+      create_task if output
     end
 
+    # Information about the Scheduler
+    # @return [Hash]
+    def stats
+      {
+        name: performer.name,
+        max_threads: @executor_options[:max_threads],
+        active_threads: @executor_options[:max_threads] - executor.ready_worker_count,
+        available_threads: executor.ready_worker_count,
+        max_cache: @max_cache,
+        active_cache: cache_count,
+        available_cache: remaining_cache_count,
+      }
+    end
+
+    # Preload existing runnable and future-scheduled jobs
+    # @return [void]
     def warm_cache
       return if @max_cache.zero?
 
-      @performer.next_at(
-        limit: @max_cache,
-        now_limit: @pool_options[:max_threads]
-      ).each do |scheduled_at|
-        create_thread({ scheduled_at: scheduled_at })
+      future = Concurrent::Future.new(args: [self, @performer], executor: executor) do |thr_scheduler, thr_performer|
+        Rails.application.executor.wrap do
+          thr_performer.next_at(
+            limit: @max_cache,
+            now_limit: @executor_options[:max_threads]
+          ).each do |scheduled_at|
+            thr_scheduler.create_thread({ scheduled_at: scheduled_at })
+          end
+        end
       end
-    end
 
-    def stats
-      {
-        name: @performer.name,
-        max_threads: @pool_options[:max_threads],
-        active_threads: @pool.ready_worker_count - @pool_options[:max_threads],
-        inactive_threads: @pool.ready_worker_count,
-        max_cache: @max_cache,
-        cache_count: cache_count,
-        cache_remaining: remaining_cache_count,
-      }
+      observer = lambda do |_time, _output, thread_error|
+        GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
+        create_task # If cache-warming exhausts the threads, ensure there isn't an executable task remaining
+      end
+      future.add_observer(observer, :call)
+
+      future.execute
     end
 
     private
 
-    attr_reader :timer_set
+    attr_reader :performer, :executor, :timer_set
 
-    def create_pool
-      instrument("scheduler_create_pool", { performer_name: @performer.name, max_threads: @pool_options[:max_threads] }) do
-        @timer_set = Concurrent::TimerSet.new
-        @pool = ThreadPoolExecutor.new(@pool_options)
+    def create_executor
+      instrument("scheduler_create_pool", { performer_name: performer.name, max_threads: @executor_options[:max_threads] }) do
+        @timer_set = TimerSet.new
+        @executor = ThreadPoolExecutor.new(@executor_options)
       end
     end
 
+    def create_task(delay = 0)
+      future = Concurrent::ScheduledTask.new(delay, args: [performer], executor: executor, timer_set: timer_set) do |thr_performer|
+        Rails.application.executor.wrap do
+          thr_performer.next
+        end
+      end
+      future.add_observer(self, :task_observer)
+      future.execute
+    end
+
     def instrument(name, payload = {}, &block)
       payload = payload.reverse_merge({
                                         scheduler: self,
@@ -211,7 +243,7 @@ module GoodJob # :nodoc:
     end
 
     def cache_count
-      timer_set.instance_variable_get(:@queue).length
+      timer_set.length
     end
 
     def remaining_cache_count
@@ -236,5 +268,21 @@ module GoodJob # :nodoc:
         end
       end
     end
+
+    # Custom sub-class of +Concurrent::TimerSet+ for additional behavior.
+    # @private
+    class TimerSet < Concurrent::TimerSet
+      # Number of scheduled jobs in the queue
+      # @return [Integer]
+      def length
+        @queue.length
+      end
+
+      # Clear the queue
+      # @return [void]
+      def reset
+        synchronize { @queue.clear }
+      end
+    end
   end
 end