good_job 1.5.0 → 1.9.0

data/lib/good_job/cli.rb

@@ -15,6 +15,23 @@ module GoodJob
     # Requiring this loads the application's configuration and classes.
     RAILS_ENVIRONMENT_RB = File.expand_path("config/environment.rb")
 
+    class << self
+      # Whether the CLI is running from the executable
+      # @return [Boolean, nil]
+      attr_accessor :within_exe
+      alias within_exe? within_exe
+
+      # Whether to log to STDOUT
+      # @return [Boolean, nil]
+      attr_accessor :log_to_stdout
+      alias log_to_stdout? log_to_stdout
+
+      # @!visibility private
+      def exit_on_failure?
+        true
+      end
+    end
+
     # @!macro thor.desc
     #   @!method $1
     #   @return [void]
@@ -27,7 +44,8 @@ module GoodJob
       See option descriptions for the matching environment variable name.
 
       == Configuring queues
-      \x5Separate multiple queues with commas; exclude queues with a leading minus;
+
+      Separate multiple queues with commas; exclude queues with a leading minus;
       separate isolated execution pools with semicolons and threads with colons.
 
     DESCRIPTION
@@ -43,10 +61,26 @@ module GoodJob
                   type: :numeric,
                   banner: 'SECONDS',
                   desc: "Interval between polls for available jobs in seconds (env var: GOOD_JOB_POLL_INTERVAL, default: 5)"
+    method_option :max_cache,
+                  type: :numeric,
+                  banner: 'COUNT',
+                  desc: "Maximum number of scheduled jobs to cache in memory (env var: GOOD_JOB_MAX_CACHE, default: 10000)"
+    method_option :shutdown_timeout,
+                  type: :numeric,
+                  banner: 'SECONDS',
+                  desc: "Number of seconds to wait for jobs to finish when shutting down before stopping the thread. (env var: GOOD_JOB_SHUTDOWN_TIMEOUT, default: -1 (forever))"
+    method_option :daemonize,
+                  type: :boolean,
+                  desc: "Run as a background daemon (default: false)"
+    method_option :pidfile,
+                  type: :string,
+                  desc: "Path to write daemonized Process ID (env var: GOOD_JOB_PIDFILE, default: tmp/pids/good_job.pid)"
     def start
       set_up_application!
       configuration = GoodJob::Configuration.new(options)
 
+      Daemon.new(pidfile: configuration.pidfile).daemonize if configuration.daemonize?
+
       notifier = GoodJob::Notifier.new
       poller = GoodJob::Poller.new(poll_interval: configuration.poll_interval)
       scheduler = GoodJob::Scheduler.from_configuration(configuration)
@@ -63,9 +97,8 @@ module GoodJob
         break if @stop_good_job_executable || scheduler.shutdown? || notifier.shutdown?
       end
 
-      notifier.shutdown
-      poller.shutdown
-      scheduler.shutdown
+      executors = [notifier, poller, scheduler]
+      GoodJob._shutdown_all(executors, timeout: configuration.shutdown_timeout)
     end
 
     default_task :start
@@ -115,7 +148,7 @@ module GoodJob
       # Rails or from the application can be set up here.
       def set_up_application!
         require RAILS_ENVIRONMENT_RB
-        return unless defined?(GOOD_JOB_LOG_TO_STDOUT) && GOOD_JOB_LOG_TO_STDOUT && !ActiveSupport::Logger.logger_outputs_to?(GoodJob.logger, $stdout)
+        return unless GoodJob::CLI.log_to_stdout? && !ActiveSupport::Logger.logger_outputs_to?(GoodJob.logger, $stdout)
 
         GoodJob::LogSubscriber.loggers << ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
         GoodJob::LogSubscriber.reset_logger

data/lib/good_job/configuration.rb

@@ -5,23 +5,28 @@ module GoodJob
   # set options to get the final values for each option.
   #
   class Configuration
+    # Valid execution modes.
+    EXECUTION_MODES = [:async, :async_server, :external, :inline].freeze
     # Default number of threads to use per {Scheduler}
     DEFAULT_MAX_THREADS = 5
     # Default number of seconds between polls for jobs
-    DEFAULT_POLL_INTERVAL = 5
+    DEFAULT_POLL_INTERVAL = 10
+    # Default number of threads to use per {Scheduler}
+    DEFAULT_MAX_CACHE = 10000
     # Default number of seconds to preserve jobs for {CLI#cleanup_preserved_jobs}
     DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO = 24 * 60 * 60
+    # Default to always wait for jobs to finish for {Adapter#shutdown}
+    DEFAULT_SHUTDOWN_TIMEOUT = -1
+
+    # The options that were explicitly set when initializing +Configuration+.
+    # @return [Hash]
+    attr_reader :options
 
-    # @!attribute [r] options
-    #   The options that were explicitly set when initializing +Configuration+.
-    #   @return [Hash]
-    #
-    # @!attribute [r] env
-    #   The environment from which to read GoodJob's environment variables. By
-    #   default, this is the current process's environment, but it can be set
-    #   to something else in {#initialize}.
-    #   @return [Hash]
-    attr_reader :options, :env
+    # The environment from which to read GoodJob's environment variables. By
+    # default, this is the current process's environment, but it can be set
+    # to something else in {#initialize}.
+    # @return [Hash]
+    attr_reader :env
 
     # @param options [Hash] Any explicitly specified configuration options to
     #   use. Keys are symbols that match the various methods on this class.
@@ -32,38 +37,30 @@ module GoodJob
       @env = env
     end
 
+    def validate!
+      raise ArgumentError, "GoodJob execution mode must be one of #{EXECUTION_MODES.join(', ')}. It was '#{execution_mode}' which is not valid." unless execution_mode.in?(EXECUTION_MODES)
+    end
+
     # Specifies how and where jobs should be executed. See {Adapter#initialize}
     # for more details on possible values.
-    #
-    # When running inside a Rails app, you may want to use
-    # {#rails_execution_mode}, which takes the current Rails environment into
-    # account when determining the final value.
-    #
-    # @param default [Symbol]
-    #   Value to use if none was specified in the configuration.
     # @return [Symbol]
-    def execution_mode(default: :external)
-      if options[:execution_mode]
-        options[:execution_mode]
-      elsif rails_config[:execution_mode]
-        rails_config[:execution_mode]
-      elsif env['GOOD_JOB_EXECUTION_MODE'].present?
-        env['GOOD_JOB_EXECUTION_MODE'].to_sym
-      else
-        default
-      end
-    end
+    def execution_mode
+      @_execution_mode ||= begin
+        mode = if GoodJob::CLI.within_exe?
+                 :external
+               else
+                 options[:execution_mode] ||
+                   rails_config[:execution_mode] ||
+                   env['GOOD_JOB_EXECUTION_MODE']
+               end
 
-    # Like {#execution_mode}, but takes the current Rails environment into
-    # account (e.g. in the +test+ environment, it falls back to +:inline+).
-    # @return [Symbol]
-    def rails_execution_mode
-      if execution_mode(default: nil)
-        execution_mode
-      elsif Rails.env.development? || Rails.env.test?
-        :inline
-      else
-        :external
+        if mode
+          mode.to_sym
+        elsif Rails.env.development? || Rails.env.test?
+          :inline
+        else
+          :external
+        end
       end
     end
 
@@ -74,10 +71,10 @@ module GoodJob
     def max_threads
       (
         options[:max_threads] ||
-        rails_config[:max_threads] ||
-        env['GOOD_JOB_MAX_THREADS'] ||
-        env['RAILS_MAX_THREADS'] ||
-        DEFAULT_MAX_THREADS
+          rails_config[:max_threads] ||
+          env['GOOD_JOB_MAX_THREADS'] ||
+          env['RAILS_MAX_THREADS'] ||
+          DEFAULT_MAX_THREADS
       ).to_i
     end
 
@@ -100,24 +97,63 @@ module GoodJob
     def poll_interval
       (
         options[:poll_interval] ||
-        rails_config[:poll_interval] ||
-        env['GOOD_JOB_POLL_INTERVAL'] ||
-        DEFAULT_POLL_INTERVAL
+          rails_config[:poll_interval] ||
+          env['GOOD_JOB_POLL_INTERVAL'] ||
+          DEFAULT_POLL_INTERVAL
+      ).to_i
+    end
+
+    # The maximum number of future-scheduled jobs to store in memory.
+    # Storing future-scheduled jobs in memory reduces execution latency
+    # at the cost of increased memory usage. 10,000 stored jobs = ~20MB.
+    # @return [Integer]
+    def max_cache
+      (
+        options[:max_cache] ||
+          rails_config[:max_cache] ||
+          env['GOOD_JOB_MAX_CACHE'] ||
+          DEFAULT_MAX_CACHE
       ).to_i
     end
 
+    # The number of seconds to wait for jobs to finish when shutting down
+    # before stopping the thread. +-1+ is forever.
+    # @return [Numeric]
+    def shutdown_timeout
+      (
+        options[:shutdown_timeout] ||
+          rails_config[:shutdown_timeout] ||
+          env['GOOD_JOB_SHUTDOWN_TIMEOUT'] ||
+          DEFAULT_SHUTDOWN_TIMEOUT
+      ).to_f
+    end
+
     # Number of seconds to preserve jobs when using the +good_job cleanup_preserved_jobs+ CLI command.
     # This configuration is only used when {GoodJob.preserve_job_records} is +true+.
-    # @return [Boolean]
+    # @return [Integer]
     def cleanup_preserved_jobs_before_seconds_ago
       (
         options[:before_seconds_ago] ||
-        rails_config[:cleanup_preserved_jobs_before_seconds_ago] ||
-        env['GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO'] ||
-        DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO
+          rails_config[:cleanup_preserved_jobs_before_seconds_ago] ||
+          env['GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO'] ||
+          DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO
       ).to_i
     end
 
+    # Tests whether to daemonize the process.
+    # @return [Boolean]
+    def daemonize?
+      options[:daemonize] || false
+    end
+
+    # Path of the pidfile to create when running as a daemon.
+    # @return [Pathname,String]
+    def pidfile
+      options[:pidfile] ||
+        env['GOOD_JOB_PIDFILE'] ||
+        Rails.application.root.join('tmp', 'pids', 'good_job.pid')
+    end
+
     private
 
     def rails_config

data/lib/good_job/daemon.rb

@@ -0,0 +1,59 @@
+module GoodJob
+  #
+  # Manages daemonization of the current process.
+  #
+  class Daemon
+    # The path of the generated pidfile.
+    # @return [Pathname,String]
+    attr_reader :pidfile
+
+    # @param pidfile [Pathname,String] Pidfile path
+    def initialize(pidfile:)
+      @pidfile = pidfile
+    end
+
+    # Daemonizes the current process and writes out a pidfile.
+    def daemonize
+      check_pid
+      Process.daemon
+      write_pid
+    end
+
+    private
+
+    def write_pid
+      File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(Process.pid.to_s) }
+      at_exit { File.delete(pidfile) if File.exist?(pidfile) }
+    rescue Errno::EEXIST
+      check_pid
+      retry
+    end
+
+    def delete_pid
+      File.delete(pidfile) if File.exist?(pidfile)
+    end
+
+    def check_pid
+      case pid_status(pidfile)
+      when :running, :not_owned
+        abort "A server is already running. Check #{pidfile}"
+      when :dead
+        File.delete(pidfile)
+      end
+    end
+
+    def pid_status(pidfile)
+      return :exited unless File.exist?(pidfile)
+
+      pid = ::File.read(pidfile).to_i
+      return :dead if pid.zero?
+
+      Process.kill(0, pid) # check process status
+      :running
+    rescue Errno::ESRCH
+      :dead
+    rescue Errno::EPERM
+      :not_owned
+    end
+  end
+end

data/lib/good_job/job.rb

@@ -15,6 +15,8 @@ module GoodJob
 
     self.table_name = 'good_jobs'.freeze
 
+    attr_readonly :serialized_params
+
     # Parse a string representing a group of queues into a more readable data
     # structure.
     # @return [Hash]
@@ -72,6 +74,12 @@ module GoodJob
     # @return [ActiveRecord::Relation]
     scope :priority_ordered, -> { order('priority DESC NULLS LAST') }
 
+    # Order jobs by scheduled (unscheduled or soonest first).
+    # @!method schedule_ordered
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :schedule_ordered, -> { order(Arel.sql('COALESCE(scheduled_at, created_at) ASC')) }
+
     # Get Jobs were completed before the given timestamp. If no timestamp is
     # provided, get all jobs that have been completed. By default, GoodJob
     # deletes jobs after they are completed and this will find no jobs.
@@ -147,6 +155,23 @@ module GoodJob
       [good_job, result, error] if good_job
     end
 
+    # Fetches the scheduled execution time of the next eligible Job(s).
+    # @return [Array<(DateTime)>]
+    def self.next_scheduled_at(after: nil, limit: 100, now_limit: nil)
+      query = advisory_unlocked.unfinished.schedule_ordered
+
+      after ||= Time.current
+      after_query = query.where('scheduled_at > ?', after).or query.where(scheduled_at: nil).where('created_at > ?', after)
+      after_at = after_query.limit(limit).pluck(:scheduled_at, :created_at).map { |timestamps| timestamps.compact.first }
+
+      if now_limit&.positive?
+        now_query = query.where('scheduled_at < ?', Time.current).or query.where(scheduled_at: nil)
+        now_at = now_query.limit(now_limit).pluck(:scheduled_at, :created_at).map { |timestamps| timestamps.compact.first }
+      end
+
+      Array(now_at) + after_at
+    end
+
     # Places an ActiveJob job on a queue by creating a new {Job} record.
     # @param active_job [ActiveJob::Base]
     #   The job to enqueue.

data/lib/good_job/job_performer.rb

@@ -39,6 +39,8 @@ module GoodJob
     # @return [Boolean] whether the performer's {#next} method should be
     #   called in the current state.
     def next?(state = {})
+      return true unless state[:queue_name]
+
       if parsed_queues[:exclude]
         parsed_queues[:exclude].exclude?(state[:queue_name])
       elsif parsed_queues[:include]
@@ -48,6 +50,15 @@ module GoodJob
       end
     end
 
+    # The Returns timestamps of when next tasks may be available.
+    # @param after [DateTime, Time, nil] future jobs scheduled after this time
+    # @param limit [Integer] number of future timestamps to return
+    # @param now_limit [Integer] number of past timestamps to return
+    # @return [Array<(Time, DateTime)>, nil]
+    def next_at(after: nil, limit: nil, now_limit: nil)
+      job_query.next_scheduled_at(after: after, limit: limit, now_limit: now_limit)
+    end
+
     private
 
     attr_reader :queue_string

data/lib/good_job/lockable.rb

@@ -92,8 +92,6 @@ module GoodJob
       # @return [ActiveRecord::Relation]
       scope :owns_advisory_locked, -> { joins_advisory_locks.where('"pg_locks"."pid" = pg_backend_pid()') }
 
-      # @!attribute [r] create_with_advisory_lock
-      # @return [Boolean]
       # Whether an advisory lock should be acquired in the same transaction
       # that created the record.
       #
@@ -107,6 +105,8 @@ module GoodJob
       #   record = MyLockableRecord.create(create_with_advisory_lock: true)
       #   record.advisory_locked?
       #   => true
+      #
+      # @return [Boolean]
       attr_accessor :create_with_advisory_lock
 
       after_create -> { advisory_lock }, if: :create_with_advisory_lock

data/lib/good_job/multi_scheduler.rb

@@ -1,16 +1,16 @@
 module GoodJob
   # Delegates the interface of a single {Scheduler} to multiple Schedulers.
   class MultiScheduler
-    # @return [array<Scheduler>] List of the scheduler delegates
+    # @return [Array<Scheduler>] List of the scheduler delegates
     attr_reader :schedulers
 
     def initialize(schedulers)
       @schedulers = schedulers
     end
 
-    # Delegates to {Scheduler#shutdown}.
-    def shutdown(wait: true)
-      schedulers.each { |s| s.shutdown(wait: wait) }
+    # Delegates to {Scheduler#running?}.
+    def running?
+      schedulers.all?(&:running?)
     end
 
     # Delegates to {Scheduler#shutdown?}.
@@ -18,9 +18,14 @@ module GoodJob
       schedulers.all?(&:shutdown?)
     end
 
+    # Delegates to {Scheduler#shutdown}.
+    def shutdown(timeout: -1)
+      GoodJob._shutdown_all(schedulers, timeout: timeout)
+    end
+
     # Delegates to {Scheduler#restart}.
-    def restart(wait: true)
-      schedulers.each { |s| s.restart(wait: wait) }
+    def restart(timeout: -1)
+      GoodJob._shutdown_all(schedulers, :restart, timeout: timeout)
     end
 
     # Delegates to {Scheduler#create_thread}.
@@ -39,7 +44,7 @@ module GoodJob
 
       if results.any?
         true
-      elsif results.any? { |result| result == false }
+      elsif results.any?(false)
         false
       else # rubocop:disable Style/EmptyElse
         nil