good_job 1.4.1 → 1.8.0

data/lib/good_job/cli.rb

@@ -15,6 +15,11 @@ module GoodJob
     # Requiring this loads the application's configuration and classes.
     RAILS_ENVIRONMENT_RB = File.expand_path("config/environment.rb")
 
+    # @!visibility private
+    def self.exit_on_failure?
+      true
+    end
+
     # @!macro thor.desc
     #   @!method $1
     #   @return [void]
@@ -27,7 +32,8 @@ module GoodJob
       See option descriptions for the matching environment variable name.
 
       == Configuring queues
-      \x5Separate multiple queues with commas; exclude queues with a leading minus;
+
+      Separate multiple queues with commas; exclude queues with a leading minus;
       separate isolated execution pools with semicolons and threads with colons.
 
     DESCRIPTION
@@ -43,10 +49,26 @@ module GoodJob
                   type: :numeric,
                   banner: 'SECONDS',
                   desc: "Interval between polls for available jobs in seconds (env var: GOOD_JOB_POLL_INTERVAL, default: 5)"
+    method_option :max_cache,
+                  type: :numeric,
+                  banner: 'COUNT',
+                  desc: "Maximum number of scheduled jobs to cache in memory (env var: GOOD_JOB_MAX_CACHE, default: 10000)"
+    method_option :shutdown_timeout,
+                  type: :numeric,
+                  banner: 'SECONDS',
+                  desc: "Number of seconds to wait for jobs to finish when shutting down before stopping the thread. (env var: GOOD_JOB_SHUTDOWN_TIMEOUT, default: -1 (forever))"
+    method_option :daemonize,
+                  type: :boolean,
+                  desc: "Run as a background daemon (default: false)"
+    method_option :pidfile,
+                  type: :string,
+                  desc: "Path to write daemonized Process ID (env var: GOOD_JOB_PIDFILE, default: tmp/pids/good_job.pid)"
     def start
       set_up_application!
       configuration = GoodJob::Configuration.new(options)
 
+      Daemon.new(pidfile: configuration.pidfile).daemonize if configuration.daemonize?
+
       notifier = GoodJob::Notifier.new
       poller = GoodJob::Poller.new(poll_interval: configuration.poll_interval)
       scheduler = GoodJob::Scheduler.from_configuration(configuration)
@@ -63,9 +85,8 @@ module GoodJob
         break if @stop_good_job_executable || scheduler.shutdown? || notifier.shutdown?
       end
 
-      notifier.shutdown
-      poller.shutdown
-      scheduler.shutdown
+      executors = [notifier, poller, scheduler]
+      GoodJob._shutdown_all(executors, timeout: configuration.shutdown_timeout)
     end
 
     default_task :start

data/lib/good_job/configuration.rb

@@ -8,20 +8,23 @@ module GoodJob
     # Default number of threads to use per {Scheduler}
     DEFAULT_MAX_THREADS = 5
     # Default number of seconds between polls for jobs
-    DEFAULT_POLL_INTERVAL = 5
+    DEFAULT_POLL_INTERVAL = 10
+    # Default number of threads to use per {Scheduler}
+    DEFAULT_MAX_CACHE = 10000
     # Default number of seconds to preserve jobs for {CLI#cleanup_preserved_jobs}
     DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO = 24 * 60 * 60
+    # Default to always wait for jobs to finish for {#shutdown}
+    DEFAULT_SHUTDOWN_TIMEOUT = -1
 
-    # @!attribute [r] options
-    #   The options that were explicitly set when initializing +Configuration+.
-    #   @return [Hash]
-    #
-    # @!attribute [r] env
-    #   The environment from which to read GoodJob's environment variables. By
-    #   default, this is the current process's environment, but it can be set
-    #   to something else in {#initialize}.
-    #   @return [Hash]
-    attr_reader :options, :env
+    # The options that were explicitly set when initializing +Configuration+.
+    # @return [Hash]
+    attr_reader :options
+
+    # The environment from which to read GoodJob's environment variables. By
+    # default, this is the current process's environment, but it can be set
+    # to something else in {#initialize}.
+    # @return [Hash]
+    attr_reader :env
 
     # @param options [Hash] Any explicitly specified configuration options to
     #   use. Keys are symbols that match the various methods on this class.
@@ -43,8 +46,12 @@ module GoodJob
     #   Value to use if none was specified in the configuration.
     # @return [Symbol]
     def execution_mode(default: :external)
-      if options[:execution_mode]
+      if defined?(GOOD_JOB_WITHIN_CLI) && GOOD_JOB_WITHIN_CLI
+        :external
+      elsif options[:execution_mode]
         options[:execution_mode]
+      elsif rails_config[:execution_mode]
+        rails_config[:execution_mode]
       elsif env['GOOD_JOB_EXECUTION_MODE'].present?
         env['GOOD_JOB_EXECUTION_MODE'].to_sym
       else
@@ -72,9 +79,10 @@ module GoodJob
     def max_threads
       (
         options[:max_threads] ||
-        env['GOOD_JOB_MAX_THREADS'] ||
-        env['RAILS_MAX_THREADS'] ||
-        DEFAULT_MAX_THREADS
+          rails_config[:max_threads] ||
+          env['GOOD_JOB_MAX_THREADS'] ||
+          env['RAILS_MAX_THREADS'] ||
+          DEFAULT_MAX_THREADS
       ).to_i
     end
 
@@ -85,6 +93,7 @@ module GoodJob
     # @return [String]
     def queue_string
       options[:queues] ||
+        rails_config[:queues] ||
         env['GOOD_JOB_QUEUES'] ||
         '*'
     end
@@ -96,20 +105,67 @@ module GoodJob
     def poll_interval
       (
         options[:poll_interval] ||
-        env['GOOD_JOB_POLL_INTERVAL'] ||
-        DEFAULT_POLL_INTERVAL
+          rails_config[:poll_interval] ||
+          env['GOOD_JOB_POLL_INTERVAL'] ||
+          DEFAULT_POLL_INTERVAL
       ).to_i
     end
 
+    # The maximum number of future-scheduled jobs to store in memory.
+    # Storing future-scheduled jobs in memory reduces execution latency
+    # at the cost of increased memory usage. 10,000 stored jobs = ~20MB.
+    # @return [Integer]
+    def max_cache
+      (
+        options[:max_cache] ||
+          rails_config[:max_cache] ||
+          env['GOOD_JOB_MAX_CACHE'] ||
+          DEFAULT_MAX_CACHE
+      ).to_i
+    end
+
+    # The number of seconds to wait for jobs to finish when shutting down
+    # before stopping the thread. +-1+ is forever.
+    # @return [Numeric]
+    def shutdown_timeout
+      (
+        options[:shutdown_timeout] ||
+          rails_config[:shutdown_timeout] ||
+          env['GOOD_JOB_SHUTDOWN_TIMEOUT'] ||
+          DEFAULT_SHUTDOWN_TIMEOUT
+      ).to_f
+    end
+
     # Number of seconds to preserve jobs when using the +good_job cleanup_preserved_jobs+ CLI command.
     # This configuration is only used when {GoodJob.preserve_job_records} is +true+.
-    # @return [Boolean]
+    # @return [Integer]
     def cleanup_preserved_jobs_before_seconds_ago
       (
         options[:before_seconds_ago] ||
-        env['GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO'] ||
-        DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO
+          rails_config[:cleanup_preserved_jobs_before_seconds_ago] ||
+          env['GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO'] ||
+          DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO
       ).to_i
     end
+
+    # Tests whether to daemonize the process.
+    # @return [Boolean]
+    def daemonize?
+      options[:daemonize] || false
+    end
+
+    # Path of the pidfile to create when running as a daemon.
+    # @return [Pathname,String]
+    def pidfile
+      options[:pidfile] ||
+        env['GOOD_JOB_PIDFILE'] ||
+        Rails.application.root.join('tmp', 'pids', 'good_job.pid')
+    end
+
+    private
+
+    def rails_config
+      Rails.application.config.good_job
+    end
   end
 end

data/lib/good_job/daemon.rb

@@ -0,0 +1,59 @@
+module GoodJob
+  #
+  # Manages daemonization of the current process.
+  #
+  class Daemon
+    # The path of the generated pidfile.
+    # @return [Pathname,String]
+    attr_reader :pidfile
+
+    # @param pidfile [Pathname,String] Pidfile path
+    def initialize(pidfile:)
+      @pidfile = pidfile
+    end
+
+    # Daemonizes the current process and writes out a pidfile.
+    def daemonize
+      check_pid
+      Process.daemon
+      write_pid
+    end
+
+    private
+
+    def write_pid
+      File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(Process.pid.to_s) }
+      at_exit { File.delete(pidfile) if File.exist?(pidfile) }
+    rescue Errno::EEXIST
+      check_pid
+      retry
+    end
+
+    def delete_pid
+      File.delete(pidfile) if File.exist?(pidfile)
+    end
+
+    def check_pid
+      case pid_status(pidfile)
+      when :running, :not_owned
+        abort "A server is already running. Check #{pidfile}"
+      when :dead
+        File.delete(pidfile)
+      end
+    end
+
+    def pid_status(pidfile)
+      return :exited unless File.exist?(pidfile)
+
+      pid = ::File.read(pidfile).to_i
+      return :dead if pid.zero?
+
+      Process.kill(0, pid) # check process status
+      :running
+    rescue Errno::ESRCH
+      :dead
+    rescue Errno::EPERM
+      :not_owned
+    end
+  end
+end

data/lib/good_job/job.rb

@@ -15,6 +15,8 @@ module GoodJob
 
     self.table_name = 'good_jobs'.freeze
 
+    attr_readonly :serialized_params
+
     # Parse a string representing a group of queues into a more readable data
     # structure.
     # @return [Hash]
@@ -72,6 +74,12 @@ module GoodJob
     # @return [ActiveRecord::Relation]
     scope :priority_ordered, -> { order('priority DESC NULLS LAST') }
 
+    # Order jobs by scheduled (unscheduled or soonest first).
+    # @!method schedule_ordered
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :schedule_ordered, -> { order(Arel.sql('COALESCE(scheduled_at, created_at) ASC')) }
+
     # Get Jobs were completed before the given timestamp. If no timestamp is
     # provided, get all jobs that have been completed. By default, GoodJob
     # deletes jobs after they are completed and this will find no jobs.
@@ -147,6 +155,23 @@ module GoodJob
       [good_job, result, error] if good_job
     end
 
+    # Fetches the scheduled execution time of the next eligible Job(s).
+    # @return [Array<(DateTime)>]
+    def self.next_scheduled_at(after: nil, limit: 100, now_limit: nil)
+      query = advisory_unlocked.unfinished.schedule_ordered
+
+      after ||= Time.current
+      after_query = query.where('scheduled_at > ?', after).or query.where(scheduled_at: nil).where('created_at > ?', after)
+      after_at = after_query.limit(limit).pluck(:scheduled_at, :created_at).map { |timestamps| timestamps.compact.first }
+
+      if now_limit&.positive?
+        now_query = query.where('scheduled_at < ?', Time.current).or query.where(scheduled_at: nil)
+        now_at = now_query.limit(now_limit).pluck(:scheduled_at, :created_at).map { |timestamps| timestamps.compact.first }
+      end
+
+      Array(now_at) + after_at
+    end
+
     # Places an ActiveJob job on a queue by creating a new {Job} record.
     # @param active_job [ActiveJob::Base]
     #   The job to enqueue.

data/lib/good_job/job_performer.rb

@@ -0,0 +1,74 @@
+require 'concurrent/delay'
+
+module GoodJob
+  #
+  # JobPerformer queries the database for jobs and performs them on behalf of a
+  # {Scheduler}. It mainly functions as glue between a {Scheduler} and the jobs
+  # it should be executing.
+  #
+  # The JobPerformer must be safe to execute across multiple threads.
+  #
+  class JobPerformer
+    # @param queue_string [String] Queues to execute jobs from
+    def initialize(queue_string)
+      @queue_string = queue_string
+
+      @job_query = Concurrent::Delay.new { GoodJob::Job.queue_string(queue_string) }
+      @parsed_queues = Concurrent::Delay.new { GoodJob::Job.queue_parser(queue_string) }
+    end
+
+    # A meaningful name to identify the performer in logs and for debugging.
+    # @return [String] The queues from which Jobs are worked
+    def name
+      @queue_string
+    end
+
+    # Perform the next eligible job
+    # @return [nil, Object] Returns job result or +nil+ if no job was found
+    def next
+      job_query.perform_with_advisory_lock
+    end
+
+    # Tests whether this performer should be used in GoodJob's current state.
+    #
+    # For example, state will be a LISTEN/NOTIFY message that is passed down
+    # from the Notifier to the Scheduler. The Scheduler is able to ask
+    # its performer "does this message relate to you?", and if not, ignore it
+    # to minimize thread wake-ups, database queries, and thundering herds.
+    #
+    # @return [Boolean] whether the performer's {#next} method should be
+    #   called in the current state.
+    def next?(state = {})
+      return true unless state[:queue_name]
+
+      if parsed_queues[:exclude]
+        parsed_queues[:exclude].exclude?(state[:queue_name])
+      elsif parsed_queues[:include]
+        parsed_queues[:include].include?(state[:queue_name])
+      else
+        true
+      end
+    end
+
+    # The Returns timestamps of when next tasks may be available.
+    # @param after [DateTime, Time, nil] future jobs scheduled after this time
+    # @param limit [Integer] number of future timestamps to return
+    # @param now_limit [Integer] number of past timestamps to return
+    # @return [Array<(Time, Timestamp)>, nil]
+    def next_at(after: nil, limit: nil, now_limit: nil)
+      job_query.next_scheduled_at(after: after, limit: limit, now_limit: now_limit)
+    end
+
+    private
+
+    attr_reader :queue_string
+
+    def job_query
+      @job_query.value
+    end
+
+    def parsed_queues
+      @parsed_queues.value
+    end
+  end
+end

data/lib/good_job/lockable.rb

@@ -92,8 +92,6 @@ module GoodJob
       # @return [ActiveRecord::Relation]
       scope :owns_advisory_locked, -> { joins_advisory_locks.where('"pg_locks"."pid" = pg_backend_pid()') }
 
-      # @!attribute [r] create_with_advisory_lock
-      # @return [Boolean]
       # Whether an advisory lock should be acquired in the same transaction
       # that created the record.
       #
@@ -107,6 +105,8 @@ module GoodJob
       #   record = MyLockableRecord.create(create_with_advisory_lock: true)
       #   record.advisory_locked?
       #   => true
+      #
+      # @return [Boolean]
       attr_accessor :create_with_advisory_lock
 
       after_create -> { advisory_lock }, if: :create_with_advisory_lock

data/lib/good_job/multi_scheduler.rb

@@ -1,16 +1,16 @@
 module GoodJob
   # Delegates the interface of a single {Scheduler} to multiple Schedulers.
   class MultiScheduler
-    # @return [array<Scheduler>] List of the scheduler delegates
+    # @return [Array<Scheduler>] List of the scheduler delegates
     attr_reader :schedulers
 
     def initialize(schedulers)
       @schedulers = schedulers
     end
 
-    # Delegates to {Scheduler#shutdown}.
-    def shutdown(wait: true)
-      schedulers.each { |s| s.shutdown(wait: wait) }
+    # Delegates to {Scheduler#running?}.
+    def running?
+      schedulers.all?(&:running?)
     end
 
     # Delegates to {Scheduler#shutdown?}.
@@ -18,9 +18,14 @@ module GoodJob
       schedulers.all?(&:shutdown?)
     end
 
+    # Delegates to {Scheduler#shutdown}.
+    def shutdown(timeout: -1)
+      GoodJob._shutdown_all(schedulers, timeout: timeout)
+    end
+
     # Delegates to {Scheduler#restart}.
-    def restart(wait: true)
-      schedulers.each { |s| s.restart(wait: wait) }
+    def restart(timeout: -1)
+      GoodJob._shutdown_all(schedulers, :restart, timeout: timeout)
     end
 
     # Delegates to {Scheduler#create_thread}.

data/lib/good_job/notifier.rb

@@ -15,7 +15,7 @@ module GoodJob # :nodoc:
     # Default Postgres channel for LISTEN/NOTIFY
     CHANNEL = 'good_job'.freeze
     # Defaults for instance of Concurrent::ThreadPoolExecutor
-    POOL_OPTIONS = {
+    EXECUTOR_OPTIONS = {
       name: name,
       min_threads: 0,
       max_threads: 1,
@@ -30,7 +30,7 @@ module GoodJob # :nodoc:
     # @!attribute [r] instances
     #   @!scope class
     #   List of all instantiated Notifiers in the current process.
-    #   @return [array<GoodJob:Adapter>]
+    #   @return [Array<GoodJob:Adapter>]
     cattr_reader :instances, default: [], instance_reader: false
 
     # Send a message via Postgres NOTIFY
@@ -53,7 +53,7 @@ module GoodJob # :nodoc:
 
       self.class.instances << self
 
-      create_pool
+      create_executor
       listen
     end
 
@@ -63,34 +63,43 @@ module GoodJob # :nodoc:
       @listening.true?
     end
 
-    # Restart the notifier.
-    # When shutdown, start; or shutdown and start.
-    # @param wait [Boolean] Wait for background thread to finish
-    # @return [void]
-    def restart(wait: true)
-      shutdown(wait: wait)
-      create_pool
-      listen
-    end
+    # Tests whether the notifier is running.
+    # @return [true, false, nil]
+    delegate :running?, to: :executor, allow_nil: true
+
+    # Tests whether the scheduler is shutdown.
+    # @return [true, false, nil]
+    delegate :shutdown?, to: :executor, allow_nil: true
 
     # Shut down the notifier.
     # This stops the background LISTENing thread.
-    # If +wait+ is +true+, the notifier will wait for background thread to shutdown.
-    # If +wait+ is +false+, this method will return immediately even though threads may still be running.
     # Use {#shutdown?} to determine whether threads have stopped.
-    # @param wait [Boolean] Wait for actively executing threads to finish
+    # @param timeout [nil, Numeric] Seconds to wait for active threads.
+    #
+    #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
+    #   * +-1+, the scheduler will wait until the shutdown is complete.
+    #   * +0+, the scheduler will immediately shutdown and stop any threads.
+    #   * A positive number will wait that many seconds before stopping any remaining active threads.
     # @return [void]
-    def shutdown(wait: true)
-      return unless @pool.running?
+    def shutdown(timeout: -1)
+      return if executor.nil? || executor.shutdown?
 
-      @pool.shutdown
-      @pool.wait_for_termination if wait
+      executor.shutdown if executor.running?
+
+      if executor.shuttingdown? && timeout # rubocop:disable Style/GuardClause
+        executor_wait = timeout.negative? ? nil : timeout
+        executor.kill unless executor.wait_for_termination(executor_wait)
+      end
     end
 
-    # Tests whether the notifier is shutdown.
-    # @return [true, false, nil]
-    def shutdown?
-      !@pool.running?
+    # Restart the notifier.
+    # When shutdown, start; or shutdown and start.
+    # @param timeout [nil, Numeric] Seconds to wait; shares same values as {#shutdown}.
+    # @return [void]
+    def restart(timeout: -1)
+      shutdown(timeout: timeout) if running?
+      create_executor
+      listen
     end
 
     # Invoked on completion of ThreadPoolExecutor task
@@ -109,36 +118,36 @@ module GoodJob # :nodoc:
 
     private
 
-    def create_pool
-      @pool = Concurrent::ThreadPoolExecutor.new(POOL_OPTIONS)
+    attr_reader :executor
+
+    def create_executor
+      @executor = Concurrent::ThreadPoolExecutor.new(EXECUTOR_OPTIONS)
     end
 
     def listen
-      future = Concurrent::Future.new(args: [@recipients, @pool, @listening], executor: @pool) do |recipients, pool, listening|
+      future = Concurrent::Future.new(args: [@recipients, executor, @listening], executor: @executor) do |thr_recipients, thr_executor, thr_listening|
         with_listen_connection do |conn|
           ActiveSupport::Notifications.instrument("notifier_listen.good_job") do
             conn.async_exec("LISTEN #{CHANNEL}").clear
           end
 
           ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
-            while pool.running?
-              listening.make_true
+            thr_listening.make_true
+            while thr_executor.running?
               conn.wait_for_notify(WAIT_INTERVAL) do |channel, _pid, payload|
-                listening.make_false
                 next unless channel == CHANNEL
 
                 ActiveSupport::Notifications.instrument("notifier_notified.good_job", { payload: payload })
                 parsed_payload = JSON.parse(payload, symbolize_names: true)
-                recipients.each do |recipient|
+                thr_recipients.each do |recipient|
                   target, method_name = recipient.is_a?(Array) ? recipient : [recipient, :call]
                   target.send(method_name, parsed_payload)
                 end
               end
-              listening.make_false
             end
           end
         ensure
-          listening.make_false
+          thr_listening.make_false
           ActiveSupport::Notifications.instrument("notifier_unlisten.good_job") do
             conn.async_exec("UNLISTEN *").clear
           end