good_job 1.8.0 → 1.9.4

data/lib/good_job/adapter.rb

@@ -4,22 +4,24 @@ module GoodJob
   #
   class Adapter
     # Valid execution modes.
-    EXECUTION_MODES = [:async, :external, :inline].freeze
+    EXECUTION_MODES = [:async, :async_server, :external, :inline].freeze
 
-    # @param execution_mode [nil, Symbol] specifies how and where jobs should be executed. You can also set this with the environment variable +GOOD_JOB_EXECUTION_MODE+.
+    # @param execution_mode [Symbol, nil] specifies how and where jobs should be executed. You can also set this with the environment variable +GOOD_JOB_EXECUTION_MODE+.
     #
     #  - +:inline+ executes jobs immediately in whatever process queued them (usually the web server process). This should only be used in test and development environments.
     #  - +:external+ causes the adapter to enqueue jobs, but not execute them. When using this option (the default for production environments), you'll need to use the command-line tool to actually execute your jobs.
-    #  - +:async+ causes the adapter to execute you jobs in separate threads in whatever process queued them (usually the web process). This is akin to running the command-line tool's code inside your web server. It can be more economical for small workloads (you don't need a separate machine or environment for running your jobs), but if your web server is under heavy load or your jobs require a lot of resources, you should choose `:external` instead.
+    #  - +:async_server+ executes jobs in separate threads within the Rails webserver process (`bundle exec rails server`). It can be more economical for small workloads because you don't need a separate machine or environment for running your jobs, but if your web server is under heavy load or your jobs require a lot of resources, you should choose +:external+ instead.
+    #    When not in the Rails webserver, jobs will execute in +:external+ mode to ensure jobs are not executed within `rails console`, `rails db:migrate`, `rails assets:prepare`, etc.
+    #  - +:async+ executes jobs in any Rails process.
     #
     #  The default value depends on the Rails environment:
     #
     #  - +development+ and +test+: +:inline+
     #  - +production+ and all other environments: +:external+
     #
-    # @param max_threads [nil, Integer] sets the number of threads per scheduler to use when +execution_mode+ is set to +:async+. The +queues+ parameter can specify a number of threads for each group of queues which will override this value. You can also set this with the environment variable +GOOD_JOB_MAX_THREADS+. Defaults to +5+.
-    # @param queues [nil, String] determines which queues to execute jobs from when +execution_mode+ is set to +:async+. See {file:README.md#optimize-queues-threads-and-processes} for more details on the format of this string. You can also set this with the environment variable +GOOD_JOB_QUEUES+. Defaults to +"*"+.
-    # @param poll_interval [nil, Integer] sets the number of seconds between polls for jobs when +execution_mode+ is set to +:async+. You can also set this with the environment variable +GOOD_JOB_POLL_INTERVAL+. Defaults to +1+.
+    # @param max_threads [Integer, nil] sets the number of threads per scheduler to use when +execution_mode+ is set to +:async+. The +queues+ parameter can specify a number of threads for each group of queues which will override this value. You can also set this with the environment variable +GOOD_JOB_MAX_THREADS+. Defaults to +5+.
+    # @param queues [String, nil] determines which queues to execute jobs from when +execution_mode+ is set to +:async+. See {file:README.md#optimize-queues-threads-and-processes} for more details on the format of this string. You can also set this with the environment variable +GOOD_JOB_QUEUES+. Defaults to +"*"+.
+    # @param poll_interval [Integer, nil] sets the number of seconds between polls for jobs when +execution_mode+ is set to +:async+. You can also set this with the environment variable +GOOD_JOB_POLL_INTERVAL+. Defaults to +1+.
     def initialize(execution_mode: nil, queues: nil, max_threads: nil, poll_interval: nil)
       if caller[0..4].find { |c| c.include?("/config/application.rb") || c.include?("/config/environments/") }
         ActiveSupport::Deprecation.warn(<<~DEPRECATION)
@@ -46,8 +48,7 @@ module GoodJob
           poll_interval: poll_interval,
         }
       )
-
-      raise ArgumentError, "execution_mode: must be one of #{EXECUTION_MODES.join(', ')}." unless EXECUTION_MODES.include?(@configuration.execution_mode)
+      @configuration.validate!
 
       if execute_async? # rubocop:disable Style/GuardClause
         @notifier = GoodJob::Notifier.new
@@ -69,7 +70,7 @@ module GoodJob
     # Enqueues an ActiveJob job to be run at a specific time.
     # For use by Rails; you should generally not call this directly.
     # @param active_job [ActiveJob::Base] the job to be enqueued from +#perform_later+
-    # @param timestamp [Integer] the epoch time to perform the job
+    # @param timestamp [Integer, nil] the epoch time to perform the job
     # @return [GoodJob::Job]
     def enqueue_at(active_job, timestamp)
       good_job = GoodJob::Job.enqueue(
@@ -96,22 +97,21 @@ module GoodJob
     end
 
     # Shut down the thread pool executors.
-    # @param timeout [nil, Numeric] Seconds to wait for active threads.
-    #
+    # @param timeout [nil, Numeric, Symbol] Seconds to wait for active threads.
     #   * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
     #   * +-1+, the scheduler will wait until the shutdown is complete.
     #   * +0+, the scheduler will immediately shutdown and stop any threads.
     #   * A positive number will wait that many seconds before stopping any remaining active threads.
-    # @param wait [Boolean] Deprecated. Use +timeout:+ instead.
+    # @param wait [Boolean, nil] Deprecated. Use +timeout:+ instead.
     # @return [void]
     def shutdown(timeout: :default, wait: nil)
-      timeout = if wait.present?
+      timeout = if wait.nil?
+                  timeout
+                else
                   ActiveSupport::Deprecation.warn(
                     "Using `GoodJob::Adapter.shutdown` with `wait:` kwarg is deprecated; use `timeout:` kwarg instead e.g. GoodJob::Adapter.shutdown(timeout: #{wait ? '-1' : 'nil'})"
                   )
                   wait ? -1 : nil
-                else
-                  timeout
                 end
 
       timeout = if timeout == :default
@@ -125,18 +125,36 @@ module GoodJob
     end
 
     # Whether in +:async+ execution mode.
+    # @return [Boolean]
     def execute_async?
-      @configuration.execution_mode == :async
+      @configuration.execution_mode == :async ||
+        @configuration.execution_mode == :async_server && in_server_process?
     end
 
     # Whether in +:external+ execution mode.
+    # @return [Boolean]
     def execute_externally?
-      @configuration.execution_mode == :external
+      @configuration.execution_mode == :external ||
+        @configuration.execution_mode == :async_server && !in_server_process?
     end
 
     # Whether in +:inline+ execution mode.
+    # @return [Boolean]
     def execute_inline?
       @configuration.execution_mode == :inline
     end
+
+    private
+
+    # Whether running in a web server process.
+    # @return [Boolean, nil]
+    def in_server_process?
+      return @_in_server_process if defined? @_in_server_process
+
+      @_in_server_process = Rails.const_defined?('Server') ||
+                            caller.grep(%r{config.ru}).any? || # EXAMPLE: config.ru:3:in `block in <main>' OR config.ru:3:in `new_from_string'
+                            caller.grep(%{/rack/handler/}).any? || # EXAMPLE: iodine-0.7.44/lib/rack/handler/iodine.rb:13:in `start'
+                            (Concurrent.on_jruby? && caller.grep(%r{jruby/rack/rails_booter}).any?) # EXAMPLE: uri:classloader:/jruby/rack/rails_booter.rb:83:in `load_environment'
+    end
   end
 end

data/lib/good_job/cli.rb

@@ -15,9 +15,21 @@ module GoodJob
     # Requiring this loads the application's configuration and classes.
     RAILS_ENVIRONMENT_RB = File.expand_path("config/environment.rb")
 
-    # @!visibility private
-    def self.exit_on_failure?
-      true
+    class << self
+      # Whether the CLI is running from the executable
+      # @return [Boolean, nil]
+      attr_accessor :within_exe
+      alias within_exe? within_exe
+
+      # Whether to log to STDOUT
+      # @return [Boolean, nil]
+      attr_accessor :log_to_stdout
+      alias log_to_stdout? log_to_stdout
+
+      # @!visibility private
+      def exit_on_failure?
+        true
+      end
     end
 
     # @!macro thor.desc
@@ -71,7 +83,7 @@ module GoodJob
 
       notifier = GoodJob::Notifier.new
       poller = GoodJob::Poller.new(poll_interval: configuration.poll_interval)
-      scheduler = GoodJob::Scheduler.from_configuration(configuration)
+      scheduler = GoodJob::Scheduler.from_configuration(configuration, warm_cache_on_initialize: true)
       notifier.recipients << [scheduler, :create_thread]
       poller.recipients << [scheduler, :create_thread]
 
@@ -136,7 +148,7 @@ module GoodJob
       # Rails or from the application can be set up here.
       def set_up_application!
         require RAILS_ENVIRONMENT_RB
-        return unless defined?(GOOD_JOB_LOG_TO_STDOUT) && GOOD_JOB_LOG_TO_STDOUT && !ActiveSupport::Logger.logger_outputs_to?(GoodJob.logger, $stdout)
+        return unless GoodJob::CLI.log_to_stdout? && !ActiveSupport::Logger.logger_outputs_to?(GoodJob.logger, $stdout)
 
         GoodJob::LogSubscriber.loggers << ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
         GoodJob::LogSubscriber.reset_logger

data/lib/good_job/configuration.rb

@@ -5,6 +5,8 @@ module GoodJob
   # set options to get the final values for each option.
   #
   class Configuration
+    # Valid execution modes.
+    EXECUTION_MODES = [:async, :async_server, :external, :inline].freeze
     # Default number of threads to use per {Scheduler}
     DEFAULT_MAX_THREADS = 5
     # Default number of seconds between polls for jobs
@@ -13,7 +15,7 @@ module GoodJob
     DEFAULT_MAX_CACHE = 10000
     # Default number of seconds to preserve jobs for {CLI#cleanup_preserved_jobs}
     DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO = 24 * 60 * 60
-    # Default to always wait for jobs to finish for {#shutdown}
+    # Default to always wait for jobs to finish for {Adapter#shutdown}
     DEFAULT_SHUTDOWN_TIMEOUT = -1
 
     # The options that were explicitly set when initializing +Configuration+.
@@ -35,40 +37,30 @@ module GoodJob
       @env = env
     end
 
+    def validate!
+      raise ArgumentError, "GoodJob execution mode must be one of #{EXECUTION_MODES.join(', ')}. It was '#{execution_mode}' which is not valid." unless execution_mode.in?(EXECUTION_MODES)
+    end
+
     # Specifies how and where jobs should be executed. See {Adapter#initialize}
     # for more details on possible values.
-    #
-    # When running inside a Rails app, you may want to use
-    # {#rails_execution_mode}, which takes the current Rails environment into
-    # account when determining the final value.
-    #
-    # @param default [Symbol]
-    #   Value to use if none was specified in the configuration.
     # @return [Symbol]
-    def execution_mode(default: :external)
-      if defined?(GOOD_JOB_WITHIN_CLI) && GOOD_JOB_WITHIN_CLI
-        :external
-      elsif options[:execution_mode]
-        options[:execution_mode]
-      elsif rails_config[:execution_mode]
-        rails_config[:execution_mode]
-      elsif env['GOOD_JOB_EXECUTION_MODE'].present?
-        env['GOOD_JOB_EXECUTION_MODE'].to_sym
-      else
-        default
-      end
-    end
+    def execution_mode
+      @_execution_mode ||= begin
+        mode = if GoodJob::CLI.within_exe?
+                 :external
+               else
+                 options[:execution_mode] ||
+                   rails_config[:execution_mode] ||
+                   env['GOOD_JOB_EXECUTION_MODE']
+               end
 
-    # Like {#execution_mode}, but takes the current Rails environment into
-    # account (e.g. in the +test+ environment, it falls back to +:inline+).
-    # @return [Symbol]
-    def rails_execution_mode
-      if execution_mode(default: nil)
-        execution_mode
-      elsif Rails.env.development? || Rails.env.test?
-        :inline
-      else
-        :external
+        if mode
+          mode.to_sym
+        elsif Rails.env.development? || Rails.env.test?
+          :inline
+        else
+          :external
+        end
       end
     end
 
@@ -92,9 +84,9 @@ module GoodJob
     # on the format of this string.
     # @return [String]
     def queue_string
-      options[:queues] ||
-        rails_config[:queues] ||
-        env['GOOD_JOB_QUEUES'] ||
+      options[:queues].presence ||
+        rails_config[:queues].presence ||
+        env['GOOD_JOB_QUEUES'].presence ||
         '*'
     end
 

data/lib/good_job/current_execution.rb

@@ -3,7 +3,6 @@ require 'active_support/core_ext/module/attribute_accessors_per_thread'
 module GoodJob
   # Thread-local attributes for passing values from Instrumentation.
   # (Cannot use ActiveSupport::CurrentAttributes because ActiveJob resets it)
-
   module CurrentExecution
     # @!attribute [rw] error_on_retry
     #   @!scope class

data/lib/good_job/daemon.rb

@@ -13,6 +13,7 @@ module GoodJob
     end
 
     # Daemonizes the current process and writes out a pidfile.
+    # @return [void]
     def daemonize
       check_pid
       Process.daemon
@@ -21,6 +22,7 @@ module GoodJob
 
     private
 
+    # @return [void]
     def write_pid
       File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(Process.pid.to_s) }
       at_exit { File.delete(pidfile) if File.exist?(pidfile) }
@@ -29,10 +31,12 @@ module GoodJob
       retry
     end
 
+    # @return [void]
     def delete_pid
       File.delete(pidfile) if File.exist?(pidfile)
     end
 
+    # @return [void]
     def check_pid
       case pid_status(pidfile)
       when :running, :not_owned
@@ -42,6 +46,8 @@ module GoodJob
       end
     end
 
+    # @param pidfile [Pathname, String]
+    # @return [Symbol]
     def pid_status(pidfile)
       return :exited unless File.exist?(pidfile)
 

data/lib/good_job/execution_result.rb

@@ -0,0 +1,20 @@
+module GoodJob
+  # Stores the results of job execution
+  class ExecutionResult
+    # @return [Object, nil]
+    attr_reader :value
+    # @return [Exception, nil]
+    attr_reader :handled_error
+    # @return [Exception, nil]
+    attr_reader :unhandled_error
+
+    # @param value [Object, nil]
+    # @param handled_error [Exception, nil]
+    # @param unhandled_error [Exception, nil]
+    def initialize(value:, handled_error: nil, unhandled_error: nil)
+      @value = value
+      @handled_error = handled_error
+      @unhandled_error = unhandled_error
+    end
+  end
+end

data/lib/good_job/job.rb

@@ -1,8 +1,9 @@
 module GoodJob
-  #
-  # Represents a request to perform an +ActiveJob+ job.
-  #
-  class Job < ActiveRecord::Base
+  # ActiveRecord model that represents an +ActiveJob+ job.
+  # Parent class can be configured with +GoodJob.active_record_parent_class+.
+  # @!parse
+  #   class Job < ActiveRecord::Base; end
+  class Job < Object.const_get(GoodJob.active_record_parent_class)
     include Lockable
 
     # Raised if something attempts to execute a previously completed Job again.
@@ -19,6 +20,7 @@ module GoodJob
 
     # Parse a string representing a group of queues into a more readable data
     # structure.
+    # @param string [String] Queue string
     # @return [Hash]
     #   How to match a given queue. It can have the following keys and values:
     #   - +{ all: true }+ indicates that all queues match.
@@ -48,6 +50,14 @@ module GoodJob
       end
     end
 
+    # Get Jobs with given class name
+    # @!method with_job_class
+    # @!scope class
+    # @param string [String]
+    #   Job class name
+    # @return [ActiveRecord::Relation]
+    scope :with_job_class, ->(job_class) { where("serialized_params->>'job_class' = ?", job_class) }
+
     # Get Jobs that have not yet been completed.
     # @!method unfinished
     # @!scope class
@@ -92,6 +102,12 @@ module GoodJob
     # @return [ActiveRecord::Relation]
     scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
 
+    # Get Jobs that started but not finished yet.
+    # @!method running
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :running, -> { where.not(performed_at: nil).where(finished_at: nil) }
+
     # Get Jobs on queues that match the given queue string.
     # @!method queue_string(string)
     # @!scope class
@@ -134,29 +150,26 @@ module GoodJob
 
     # Finds the next eligible Job, acquire an advisory lock related to it, and
     # executes the job.
-    # @return [Array<(GoodJob::Job, Object, Exception)>, nil]
+    # @return [ExecutionResult, nil]
     #   If a job was executed, returns an array with the {Job} record, the
     #   return value for the job's +#perform+ method, and the exception the job
     #   raised, if any (if the job raised, then the second array entry will be
     #   +nil+). If there were no jobs to execute, returns +nil+.
     def self.perform_with_advisory_lock
-      good_job = nil
-      result = nil
-      error = nil
-
       unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
         good_job = good_jobs.first
         # TODO: Determine why some records are fetched without an advisory lock at all
         break unless good_job&.executable?
 
-        result, error = good_job.perform
+        good_job.perform
       end
-
-      [good_job, result, error] if good_job
     end
 
     # Fetches the scheduled execution time of the next eligible Job(s).
-    # @return [Array<(DateTime)>]
+    # @param after [DateTime]
+    # @param limit [Integer]
+    # @param now_limit [Integer, nil]
+    # @return [Array<DateTime>]
     def self.next_scheduled_at(after: nil, limit: 100, now_limit: nil)
       query = advisory_unlocked.unfinished.schedule_ordered
 
@@ -182,7 +195,6 @@ module GoodJob
     # @return [Job]
     #   The new {Job} instance representing the queued ActiveJob job.
     def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
-      good_job = nil
       ActiveSupport::Notifications.instrument("enqueue_job.good_job", { active_job: active_job, scheduled_at: scheduled_at, create_with_advisory_lock: create_with_advisory_lock }) do |instrument_payload|
         good_job = GoodJob::Job.new(
           queue_name: active_job.queue_name.presence || DEFAULT_QUEUE_NAME,
@@ -196,49 +208,37 @@ module GoodJob
 
         good_job.save!
         active_job.provider_job_id = good_job.id
-      end
 
-      good_job
+        good_job
+      end
     end
 
     # Execute the ActiveJob job this {Job} represents.
-    # @return [Array<(Object, Exception)>]
+    # @return [ExecutionResult]
     #   An array of the return value of the job's +#perform+ method and the
     #   exception raised by the job, if any. If the job completed successfully,
     #   the second array entry (the exception) will be +nil+ and vice versa.
     def perform
       raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
 
-      GoodJob::CurrentExecution.reset
-
       self.performed_at = Time.current
       save! if GoodJob.preserve_job_records
 
-      result, unhandled_error = execute
-
-      result_error = nil
-      if result.is_a?(Exception)
-        result_error = result
-        result = nil
-      end
-
-      job_error = unhandled_error ||
-                  result_error ||
-                  GoodJob::CurrentExecution.error_on_retry ||
-                  GoodJob::CurrentExecution.error_on_discard
+      result = execute
 
+      job_error = result.handled_error || result.unhandled_error
       self.error = "#{job_error.class}: #{job_error.message}" if job_error
 
-      if unhandled_error && GoodJob.retry_on_unhandled_error
+      if result.unhandled_error && GoodJob.retry_on_unhandled_error
         save!
-      elsif GoodJob.preserve_job_records == true || (unhandled_error && GoodJob.preserve_job_records == :on_unhandled_error)
+      elsif GoodJob.preserve_job_records == true || (result.unhandled_error && GoodJob.preserve_job_records == :on_unhandled_error)
         self.finished_at = Time.current
         save!
       else
         destroy!
       end
 
-      [result, job_error]
+      result
     end
 
     # Tests whether this job is safe to be executed by this thread.
@@ -249,16 +249,26 @@ module GoodJob
 
     private
 
+    # @return [ExecutionResult]
     def execute
       params = serialized_params.merge(
         "provider_job_id" => id
       )
 
+      GoodJob::CurrentExecution.reset
       ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self, process_id: GoodJob::CurrentExecution.process_id, thread_name: GoodJob::CurrentExecution.thread_name }) do
-        [ActiveJob::Base.execute(params), nil]
+        value = ActiveJob::Base.execute(params)
+
+        if value.is_a?(Exception)
+          handled_error = value
+          value = nil
+        end
+        handled_error ||= GoodJob::CurrentExecution.error_on_retry || GoodJob::CurrentExecution.error_on_discard
+
+        ExecutionResult.new(value: value, handled_error: handled_error)
+      rescue StandardError => e
+        ExecutionResult.new(value: nil, unhandled_error: e)
       end
-    rescue StandardError => e
-      [nil, e]
     end
   end
 end