good_job 1.5.0 → 1.9.0

data/lib/good_job/notifier.rb

@@ -15,7 +15,7 @@ module GoodJob # :nodoc:
     # Default Postgres channel for LISTEN/NOTIFY
     CHANNEL = 'good_job'.freeze
     # Defaults for instance of Concurrent::ThreadPoolExecutor
-    POOL_OPTIONS = {
+    EXECUTOR_OPTIONS = {
       name: name,
       min_threads: 0,
       max_threads: 1,
@@ -30,7 +30,7 @@ module GoodJob # :nodoc:
     # @!attribute [r] instances
     #   @!scope class
     #   List of all instantiated Notifiers in the current process.
-    #   @return [array<GoodJob:Adapter>]
+    #   @return [Array<GoodJob::Adapter>]
     cattr_reader :instances, default: [], instance_reader: false
 
     # Send a message via Postgres NOTIFY
@@ -53,7 +53,7 @@ module GoodJob # :nodoc:
 
       self.class.instances << self
 
-      create_pool
+      create_executor
       listen
     end
 
@@ -63,34 +63,43 @@ module GoodJob # :nodoc:
       @listening.true?
     end
 
-    # Restart the notifier.
-    # When shutdown, start; or shutdown and start.
-    # @param wait [Boolean] Wait for background thread to finish
-    # @return [void]
-    def restart(wait: true)
-      shutdown(wait: wait)
-      create_pool
-      listen
-    end
+    # Tests whether the notifier is running.
+    # @return [true, false, nil]
+    delegate :running?, to: :executor, allow_nil: true
+
+    # Tests whether the scheduler is shutdown.
+    # @return [true, false, nil]
+    delegate :shutdown?, to: :executor, allow_nil: true
 
     # Shut down the notifier.
     # This stops the background LISTENing thread.
-    # If +wait+ is +true+, the notifier will wait for background thread to shutdown.
-    # If +wait+ is +false+, this method will return immediately even though threads may still be running.
     # Use {#shutdown?} to determine whether threads have stopped.
-    # @param wait [Boolean] Wait for actively executing threads to finish
+    # @param timeout [nil, Numeric] Seconds to wait for active threads.
+    #
+    #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
+    #   * +-1+, the scheduler will wait until the shutdown is complete.
+    #   * +0+, the scheduler will immediately shutdown and stop any threads.
+    #   * A positive number will wait that many seconds before stopping any remaining active threads.
     # @return [void]
-    def shutdown(wait: true)
-      return unless @pool.running?
+    def shutdown(timeout: -1)
+      return if executor.nil? || executor.shutdown?
 
-      @pool.shutdown
-      @pool.wait_for_termination if wait
+      executor.shutdown if executor.running?
+
+      if executor.shuttingdown? && timeout # rubocop:disable Style/GuardClause
+        executor_wait = timeout.negative? ? nil : timeout
+        executor.kill unless executor.wait_for_termination(executor_wait)
+      end
     end
 
-    # Tests whether the notifier is shutdown.
-    # @return [true, false, nil]
-    def shutdown?
-      !@pool.running?
+    # Restart the notifier.
+    # When shutdown, start; or shutdown and start.
+    # @param timeout [nil, Numeric] Seconds to wait; shares same values as {#shutdown}.
+    # @return [void]
+    def restart(timeout: -1)
+      shutdown(timeout: timeout) if running?
+      create_executor
+      listen
     end
 
     # Invoked on completion of ThreadPoolExecutor task
@@ -109,36 +118,36 @@ module GoodJob # :nodoc:
 
     private
 
-    def create_pool
-      @pool = Concurrent::ThreadPoolExecutor.new(POOL_OPTIONS)
+    attr_reader :executor
+
+    def create_executor
+      @executor = Concurrent::ThreadPoolExecutor.new(EXECUTOR_OPTIONS)
     end
 
     def listen
-      future = Concurrent::Future.new(args: [@recipients, @pool, @listening], executor: @pool) do |recipients, pool, listening|
+      future = Concurrent::Future.new(args: [@recipients, executor, @listening], executor: @executor) do |thr_recipients, thr_executor, thr_listening|
         with_listen_connection do |conn|
           ActiveSupport::Notifications.instrument("notifier_listen.good_job") do
             conn.async_exec("LISTEN #{CHANNEL}").clear
           end
 
           ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-            while pool.running?
-              listening.make_true
+            thr_listening.make_true
+            while thr_executor.running?
               conn.wait_for_notify(WAIT_INTERVAL) do |channel, _pid, payload|
-                listening.make_false
                 next unless channel == CHANNEL
 
                 ActiveSupport::Notifications.instrument("notifier_notified.good_job", { payload: payload })
                 parsed_payload = JSON.parse(payload, symbolize_names: true)
-                recipients.each do |recipient|
+                thr_recipients.each do |recipient|
                   target, method_name = recipient.is_a?(Array) ? recipient : [recipient, :call]
                   target.send(method_name, parsed_payload)
                 end
               end
-              listening.make_false
             end
           end
         ensure
-          listening.make_false
+          thr_listening.make_false
           ActiveSupport::Notifications.instrument("notifier_unlisten.good_job") do
             conn.async_exec("UNLISTEN *").clear
           end

data/lib/good_job/poller.rb

@@ -16,7 +16,7 @@ module GoodJob # :nodoc:
     # @!attribute [r] instances
     #   @!scope class
     #   List of all instantiated Pollers in the current process.
-    #   @return [array<GoodJob:Poller>]
+    #   @return [Array<GoodJob::Poller>]
     cattr_reader :instances, default: [], instance_reader: false
 
     # Creates GoodJob::Poller from a GoodJob::Configuration instance.
@@ -40,35 +40,44 @@ module GoodJob # :nodoc:
 
       self.class.instances << self
 
-      create_pool
+      create_timer
     end
 
-    # Shut down the poller.
-    # If +wait+ is +true+, the poller will wait for background thread to shutdown.
-    # If +wait+ is +false+, this method will return immediately even though threads may still be running.
+    # Tests whether the timer is running.
+    # @return [true, false, nil]
+    delegate :running?, to: :timer, allow_nil: true
+
+    # Tests whether the timer is shutdown.
+    # @return [true, false, nil]
+    delegate :shutdown?, to: :timer, allow_nil: true
+
+    # Shut down the notifier.
     # Use {#shutdown?} to determine whether threads have stopped.
-    # @param wait [Boolean] Wait for actively executing threads to finish
+    # @param timeout [nil, Numeric] Seconds to wait for active threads.
+    #
+    #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
+    #   * +-1+, the scheduler will wait until the shutdown is complete.
+    #   * +0+, the scheduler will immediately shutdown and stop any threads.
+    #   * A positive number will wait that many seconds before stopping any remaining active threads.
     # @return [void]
-    def shutdown(wait: true)
-      return unless @timer&.running?
+    def shutdown(timeout: -1)
+      return if timer.nil? || timer.shutdown?
 
-      @timer.shutdown
-      @timer.wait_for_termination if wait
-    end
+      timer.shutdown if timer.running?
 
-    # Tests whether the poller is shutdown.
-    # @return [true, false, nil]
-    def shutdown?
-      !@timer&.running?
+      if timer.shuttingdown? && timeout # rubocop:disable Style/GuardClause
+        timer_wait = timeout.negative? ? nil : timeout
+        timer.kill unless timer.wait_for_termination(timer_wait)
+      end
     end
 
     # Restart the poller.
     # When shutdown, start; or shutdown and start.
-    # @param wait [Boolean] Wait for background thread to finish
+    # @param timeout [nil, Numeric] Seconds to wait; shares same values as {#shutdown}.
     # @return [void]
-    def restart(wait: true)
-      shutdown(wait: wait)
-      create_pool
+    def restart(timeout: -1)
+      shutdown(timeout: timeout) if running?
+      create_timer
     end
 
     # Invoked on completion of TimerTask task.
@@ -76,12 +85,14 @@ module GoodJob # :nodoc:
     # @return [void]
     def timer_observer(time, executed_task, thread_error)
       GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
-      instrument("finished_timer_task", { result: executed_task, error: thread_error, time: time })
+      ActiveSupport::Notifications.instrument("finished_timer_task", { result: executed_task, error: thread_error, time: time })
     end
 
     private
 
-    def create_pool
+    attr_reader :timer
+
+    def create_timer
       return if @timer_options[:execution_interval] <= 0
 
       @timer = Concurrent::TimerTask.new(@timer_options) do

data/lib/good_job/railtie.rb

@@ -19,5 +19,9 @@ module GoodJob
         GoodJob::CurrentExecution.error_on_discard = event.payload[:error]
       end
     end
+
+    config.after_initialize do
+      GoodJob::Scheduler.instances.each(&:warm_cache)
+    end
   end
 end

data/lib/good_job/scheduler.rb

@@ -1,5 +1,6 @@
 require "concurrent/executor/thread_pool_executor"
-require "concurrent/timer_task"
+require "concurrent/executor/timer_set"
+require "concurrent/scheduled_task"
 require "concurrent/utility/processor_counter"
 
 module GoodJob # :nodoc:
@@ -8,40 +9,46 @@ module GoodJob # :nodoc:
   # periodically checking for available tasks, executing tasks within a thread,
   # and efficiently scaling active threads.
   #
-  # Every scheduler has a single {Performer} that will execute tasks.
+  # Every scheduler has a single {JobPerformer} that will execute tasks.
   # The scheduler is responsible for calling its performer efficiently across threads managed by an instance of +Concurrent::ThreadPoolExecutor+.
   # If a performer does not have work, the thread will go to sleep.
   # The scheduler maintains an instance of +Concurrent::TimerTask+, which wakes sleeping threads and causes them to check whether the performer has new work.
   #
   class Scheduler
     # Defaults for instance of Concurrent::ThreadPoolExecutor
-    # The thread pool is where work is performed.
-    DEFAULT_POOL_OPTIONS = {
+    # The thread pool executor is where work is performed.
+    DEFAULT_EXECUTOR_OPTIONS = {
       name: name,
       min_threads: 0,
       max_threads: Configuration::DEFAULT_MAX_THREADS,
       auto_terminate: true,
       idletime: 60,
-      max_queue: 0,
+      max_queue: Configuration::DEFAULT_MAX_THREADS,
       fallback_policy: :discard,
     }.freeze
 
     # @!attribute [r] instances
     #   @!scope class
     #   List of all instantiated Schedulers in the current process.
-    #   @return [array<GoodJob:Scheduler>]
+    #   @return [Array<GoodJob::Scheduler>]
     cattr_reader :instances, default: [], instance_reader: false
 
     # Creates GoodJob::Scheduler(s) and Performers from a GoodJob::Configuration instance.
     # @param configuration [GoodJob::Configuration]
+    # @param warm_cache_on_initialize [Boolean]
     # @return [GoodJob::Scheduler, GoodJob::MultiScheduler]
-    def self.from_configuration(configuration)
+    def self.from_configuration(configuration, warm_cache_on_initialize: true)
       schedulers = configuration.queue_string.split(';').map do |queue_string_and_max_threads|
         queue_string, max_threads = queue_string_and_max_threads.split(':')
         max_threads = (max_threads || configuration.max_threads).to_i
 
         job_performer = GoodJob::JobPerformer.new(queue_string)
-        GoodJob::Scheduler.new(job_performer, max_threads: max_threads)
+        GoodJob::Scheduler.new(
+          job_performer,
+          max_threads: max_threads,
+          max_cache: configuration.max_cache,
+          warm_cache_on_initialize: warm_cache_on_initialize
+        )
       end
 
       if schedulers.size > 1
@@ -53,74 +60,108 @@ module GoodJob # :nodoc:
 
     # @param performer [GoodJob::JobPerformer]
     # @param max_threads [Numeric, nil] number of seconds between polls for jobs
-    def initialize(performer, max_threads: nil)
+    # @param max_cache [Numeric, nil] maximum number of scheduled jobs to cache in memory
+    # @param warm_cache_on_initialize [Boolean] whether to warm the cache immediately
+    def initialize(performer, max_threads: nil, max_cache: nil, warm_cache_on_initialize: true)
       raise ArgumentError, "Performer argument must implement #next" unless performer.respond_to?(:next)
 
       self.class.instances << self
 
       @performer = performer
 
-      @pool_options = DEFAULT_POOL_OPTIONS.dup
-      @pool_options[:max_threads] = max_threads if max_threads.present?
-      @pool_options[:name] = "GoodJob::Scheduler(queues=#{@performer.name} max_threads=#{@pool_options[:max_threads]})"
+      @max_cache = max_cache || 0
+      @executor_options = DEFAULT_EXECUTOR_OPTIONS.dup
+      if max_threads.present?
+        @executor_options[:max_threads] = max_threads
+        @executor_options[:max_queue] = max_threads
+      end
+      @executor_options[:name] = "GoodJob::Scheduler(queues=#{@performer.name} max_threads=#{@executor_options[:max_threads]})"
 
-      create_pool
+      create_executor
+      warm_cache if warm_cache_on_initialize
     end
 
+    # Tests whether the scheduler is running.
+    # @return [true, false, nil]
+    delegate :running?, to: :executor, allow_nil: true
+
+    # Tests whether the scheduler is shutdown.
+    # @return [true, false, nil]
+    delegate :shutdown?, to: :executor, allow_nil: true
+
     # Shut down the scheduler.
-    # This stops all threads in the pool.
-    # If +wait+ is +true+, the scheduler will wait for any active tasks to finish.
-    # If +wait+ is +false+, this method will return immediately even though threads may still be running.
+    # This stops all threads in the thread pool.
     # Use {#shutdown?} to determine whether threads have stopped.
-    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @param timeout [nil, Numeric] Seconds to wait for actively executing jobs to finish
+    #
+    #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
+    #   * +-1+, the scheduler will wait until the shutdown is complete.
+    #   * +0+, the scheduler will immediately shutdown and stop any active tasks.
+    #   * A positive number will wait that many seconds before stopping any remaining active tasks.
     # @return [void]
-    def shutdown(wait: true)
-      return unless @pool&.running?
-
-      instrument("scheduler_shutdown_start", { wait: wait })
-      instrument("scheduler_shutdown", { wait: wait }) do
-        @pool.shutdown
-        @pool.wait_for_termination if wait
-        # TODO: Should be killed if wait is not true
-      end
-    end
+    def shutdown(timeout: -1)
+      return if executor.nil? || executor.shutdown?
 
-    # Tests whether the scheduler is shutdown.
-    # @return [true, false, nil]
-    def shutdown?
-      !@pool&.running?
+      instrument("scheduler_shutdown_start", { timeout: timeout })
+      instrument("scheduler_shutdown", { timeout: timeout }) do
+        if executor.running?
+          @timer_set.shutdown
+          executor.shutdown
+        end
+
+        if executor.shuttingdown? && timeout
+          executor_wait = timeout.negative? ? nil : timeout
+          executor.kill unless executor.wait_for_termination(executor_wait)
+        end
+      end
     end
 
     # Restart the Scheduler.
     # When shutdown, start; or shutdown and start.
-    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @param timeout [nil, Numeric] Seconds to wait for actively executing jobs to finish; shares same values as {#shutdown}.
     # @return [void]
-    def restart(wait: true)
+    def restart(timeout: -1)
       instrument("scheduler_restart_pools") do
-        shutdown(wait: wait) unless shutdown?
-        create_pool
+        shutdown(timeout: timeout) if running?
+        create_executor
+        warm_cache
       end
     end
 
     # Wakes a thread to allow the performer to execute a task.
-    # @param state [nil, Object] Contextual information for the performer. See {Performer#next?}.
+    # @param state [nil, Object] Contextual information for the performer. See {JobPerformer#next?}.
     # @return [nil, Boolean] Whether work was started.
-    #   Returns +nil+ if the scheduler is unable to take new work, for example if the thread pool is shut down or at capacity.
-    #   Returns +true+ if the performer started executing work.
-    #   Returns +false+ if the performer decides not to attempt to execute a task based on the +state+ that is passed to it.
+    #
+    #   * +nil+ if the scheduler is unable to take new work, for example if the thread pool is shut down or at capacity.
+    #   * +true+ if the performer started executing work.
+    #   * +false+ if the performer decides not to attempt to execute a task based on the +state+ that is passed to it.
     def create_thread(state = nil)
-      return nil unless @pool.running? && @pool.ready_worker_count.positive?
-      return false if state && !@performer.next?(state)
+      return nil unless executor.running?
 
-      future = Concurrent::Future.new(args: [@performer], executor: @pool) do |performer|
-        output = nil
-        Rails.application.executor.wrap { output = performer.next }
-        output
+      if state
+        return false unless performer.next?(state)
+
+        if state[:scheduled_at]
+          scheduled_at = if state[:scheduled_at].is_a? String
+                           Time.zone.parse state[:scheduled_at]
+                         else
+                           state[:scheduled_at]
+                         end
+          delay = [(scheduled_at - Time.current).to_f, 0].max
+        end
+      end
+
+      delay ||= 0
+      run_now = delay <= 0.01
+      if run_now
+        return nil unless executor.ready_worker_count.positive?
+      elsif @max_cache.positive?
+        return nil unless remaining_cache_count.positive?
       end
-      future.add_observer(self, :task_observer)
-      future.execute
 
-      true
+      create_task(delay)
+
+      run_now ? true : nil
     end
 
     # Invoked on completion of ThreadPoolExecutor task
@@ -129,17 +170,57 @@ module GoodJob # :nodoc:
     def task_observer(time, output, thread_error)
       GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
       instrument("finished_job_task", { result: output, error: thread_error, time: time })
-      create_thread if output
+      create_task if output
+    end
+
+    # Information about the Scheduler
+    # @return [Hash]
+    def stats
+      {
+        name: performer.name,
+        max_threads: @executor_options[:max_threads],
+        active_threads: @executor_options[:max_threads] - executor.ready_worker_count,
+        available_threads: executor.ready_worker_count,
+        max_cache: @max_cache,
+        active_cache: cache_count,
+        available_cache: remaining_cache_count,
+      }
+    end
+
+    # Preload existing runnable and future-scheduled jobs
+    # @return [void]
+    def warm_cache
+      return if @max_cache.zero?
+
+      performer.next_at(
+        limit: @max_cache,
+        now_limit: @executor_options[:max_threads]
+      ).each do |scheduled_at|
+        create_thread({ scheduled_at: scheduled_at })
+      end
     end
 
     private
 
-    def create_pool
-      instrument("scheduler_create_pool", { performer_name: @performer.name, max_threads: @pool_options[:max_threads] }) do
-        @pool = ThreadPoolExecutor.new(@pool_options)
+    attr_reader :performer, :executor, :timer_set
+
+    def create_executor
+      instrument("scheduler_create_pool", { performer_name: performer.name, max_threads: @executor_options[:max_threads] }) do
+        @timer_set = TimerSet.new
+        @executor = ThreadPoolExecutor.new(@executor_options)
       end
     end
 
+    def create_task(delay = 0)
+      future = Concurrent::ScheduledTask.new(delay, args: [performer], executor: executor, timer_set: timer_set) do |thr_performer|
+        output = nil
+        Rails.application.executor.wrap { output = thr_performer.next }
+        output
+      end
+      future.add_observer(self, :task_observer)
+      future.execute
+    end
+
     def instrument(name, payload = {}, &block)
       payload = payload.reverse_merge({
                                         scheduler: self,
@@ -150,6 +231,14 @@ module GoodJob # :nodoc:
       ActiveSupport::Notifications.instrument("#{name}.good_job", payload, &block)
     end
 
+    def cache_count
+      timer_set.length
+    end
+
+    def remaining_cache_count
+      @max_cache - cache_count
+    end
+
     # Custom sub-class of +Concurrent::ThreadPoolExecutor+ to add additional worker status.
     # @private
     class ThreadPoolExecutor < Concurrent::ThreadPoolExecutor
@@ -168,5 +257,21 @@ module GoodJob # :nodoc:
         end
       end
     end
+
+    # Custom sub-class of +Concurrent::TimerSet+ for additional behavior.
+    # @private
+    class TimerSet < Concurrent::TimerSet
+      # Number of scheduled jobs in the queue
+      # @return [Integer]
+      def length
+        @queue.length
+      end
+
+      # Clear the queue
+      # @return [void]
+      def reset
+        synchronize { @queue.clear }
+      end
+    end
   end
 end