good_job 1.2.1 → 1.2.6

data/engine/config/routes.rb

@@ -0,0 +1,4 @@
+GoodJob::Engine.routes.draw do
+  root to: 'dashboards#index'
+  resources :active_jobs, only: :show
+end

data/engine/lib/good_job/engine.rb

@@ -0,0 +1,5 @@
+module GoodJob
+  class Engine < ::Rails::Engine
+    isolate_namespace GoodJob
+  end
+end

data/lib/active_job/queue_adapters/good_job_adapter.rb

@@ -1,5 +1,6 @@
-module ActiveJob
-  module QueueAdapters
+module ActiveJob # :nodoc:
+  module QueueAdapters # :nodoc:
+    # See {GoodJob::Adapter} for details.
     class GoodJobAdapter < GoodJob::Adapter
       def initialize(execution_mode: nil, max_threads: nil, poll_interval: nil, scheduler: nil, inline: false)
         configuration = GoodJob::Configuration.new({ execution_mode: execution_mode }, env: ENV)

data/lib/generators/good_job/install_generator.rb

@@ -2,6 +2,13 @@ require 'rails/generators'
 require 'rails/generators/active_record'
 
 module GoodJob
+  #
+  # Implements the Rails generator used for setting up GoodJob in a Rails
+  # application. Run it with +bin/rails g good_job:install+ in your console.
+  #
+  # This generator is primarily dedicated to stubbing out a migration that adds
+  # a table to hold GoodJob's queued jobs in your database.
+  #
   class InstallGenerator < Rails::Generators::Base
     include Rails::Generators::Migration
 
@@ -11,6 +18,7 @@ module GoodJob
 
     source_paths << File.join(File.dirname(__FILE__), "templates")
 
+    # Generates the actual migration file and places it on disk.
     def create_migration_file
       migration_template 'migration.rb.erb', 'db/migrate/create_good_jobs.rb', migration_version: migration_version
     end

data/lib/good_job.rb

@@ -1,19 +1,18 @@
 require "rails"
-require 'good_job/railtie'
 
-require 'good_job/configuration'
-require 'good_job/log_subscriber'
-require 'good_job/lockable'
-require 'good_job/job'
-require 'good_job/scheduler'
-require 'good_job/multi_scheduler'
-require 'good_job/adapter'
-require 'good_job/pg_locks'
-require 'good_job/performer'
-require 'good_job/current_execution'
-require 'good_job/notifier'
+require "active_job"
+require "active_job/queue_adapters"
 
-require 'active_job/queue_adapters/good_job_adapter'
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect(
+  'cli' => "CLI"
+)
+loader.push_dir(File.join(__dir__, ["generators"]))
+loader.setup
+
+require "good_job/railtie"
 
 # GoodJob is a multithreaded, Postgres-based, ActiveJob backend for Ruby on Rails.
 #
@@ -21,48 +20,65 @@ require 'active_job/queue_adapters/good_job_adapter'
 module GoodJob
   # @!attribute [rw] logger
   #   @!scope class
-  #   The logger used by GoodJob
+  #   The logger used by GoodJob (default: +Rails.logger+).
+  #   Use this to redirect logs to a special location or file.
   #   @return [Logger]
-  mattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT))
+  #   @example Output GoodJob logs to a file:
+  #     GoodJob.logger = ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new("log/my_logs.log"))
+  mattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
 
   # @!attribute [rw] preserve_job_records
   #   @!scope class
-  #   Whether to preserve job records in the database after they have finished for inspection
+  #   Whether to preserve job records in the database after they have finished (default: +false+).
+  #   By default, GoodJob deletes job records after the job is completed successfully.
+  #   If you want to preserve jobs for latter inspection, set this to +true+.
+  #   If you want to preserve only jobs that finished with error for latter inspection, set this to +:on_unhandled_error+.
+  #   If +true+, you will need to clean out jobs using the +good_job cleanup_preserved_jobs+ CLI command.
   #   @return [Boolean]
   mattr_accessor :preserve_job_records, default: false
 
   # @!attribute [rw] reperform_jobs_on_standard_error
   #   @!scope class
-  #   Whether to re-perform a job when a type of +StandardError+ is raised and bubbles up to the GoodJob backend
+  #   Whether to re-perform a job when a type of +StandardError+ is raised to GoodJob (default: +true+).
+  #   If +true+, causes jobs to be re-queued and retried if they raise an instance of +StandardError+.
+  #   If +false+, jobs will be discarded or marked as finished if they raise an instance of +StandardError+.
+  #   Instances of +Exception+, like +SIGINT+, will *always* be retried, regardless of this attribute's value.
   #   @return [Boolean]
   mattr_accessor :reperform_jobs_on_standard_error, default: true
 
   # @!attribute [rw] on_thread_error
   #   @!scope class
-  #   Called when a thread raises an error
+  #   This callable will be called when an exception reaches GoodJob (default: +nil+).
+  #   It can be useful for logging errors to bug tracking services, like Sentry or Airbrake.
   #   @example Send errors to Sentry
   #     # config/initializers/good_job.rb
-  #
-  #     # With Sentry (or Bugsnag, Airbrake, Honeybadger, etc.)
   #     GoodJob.on_thread_error = -> (exception) { Raven.capture_exception(exception) }
   #   @return [#call, nil]
   mattr_accessor :on_thread_error, default: nil
 
-  # Shuts down all execution pools
+  # Stop executing jobs.
+  # GoodJob does its work in pools of background threads.
+  # When forking processes you should shut down these background threads before forking, and restart them after forking.
+  # For example, you should use +shutdown+ and +restart+ when using async execution mode with Puma.
+  # See the {file:README.md#executing-jobs-async--in-process} for more explanation and examples.
   # @param wait [Boolean] whether to wait for shutdown
   # @return [void]
   def self.shutdown(wait: true)
-    Notifier.instances.each { |adapter| adapter.shutdown(wait: wait) }
+    Notifier.instances.each { |notifier| notifier.shutdown(wait: wait) }
     Scheduler.instances.each { |scheduler| scheduler.shutdown(wait: wait) }
   end
 
-  # Tests if execution pools are shut down
-  # @return [Boolean] whether execution pools are shut down
+  # Tests whether jobs have stopped executing.
+  # @return [Boolean] whether background threads are shut down
   def self.shutdown?
     Notifier.instances.all?(&:shutdown?) && Scheduler.instances.all?(&:shutdown?)
   end
 
-  # Restarts all execution pools
+  # Stops and restarts executing jobs.
+  # GoodJob does its work in pools of background threads.
+  # When forking processes you should shut down these background threads before forking, and restart them after forking.
+  # For example, you should use +shutdown+ and +restart+ when using async execution mode with Puma.
+  # See the {file:README.md#executing-jobs-async--in-process} for more explanation and examples.
   # @return [void]
   def self.restart
     Notifier.instances.each(&:restart)

data/lib/good_job/adapter.rb

@@ -1,7 +1,28 @@
 module GoodJob
+  #
+  # ActiveJob Adapter.
+  #
   class Adapter
+    # Valid execution modes.
     EXECUTION_MODES = [:async, :external, :inline].freeze
 
+    # @param execution_mode [nil, Symbol] specifies how and where jobs should be executed. You can also set this with the environment variable +GOOD_JOB_EXECUTION_MODE+.
+    #
+    #  - +:inline+ executes jobs immediately in whatever process queued them (usually the web server process). This should only be used in test and development environments.
+    #  - +:external+ causes the adapter to enqueue jobs, but not execute them. When using this option (the default for production environments), you'll need to use the command-line tool to actually execute your jobs.
+    #  - +:async+ causes the adapter to execute you jobs in separate threads in whatever process queued them (usually the web process). This is akin to running the command-line tool's code inside your web server. It can be more economical for small workloads (you don't need a separate machine or environment for running your jobs), but if your web server is under heavy load or your jobs require a lot of resources, you should choose `:external` instead.
+    #
+    #  The default value depends on the Rails environment:
+    #
+    #  - +development+ and +test+: +:inline+
+    #  - +production+ and all other environments: +:external+
+    #
+    # @param max_threads [nil, Integer] sets the number of threads per scheduler to use when +execution_mode+ is set to +:async+. The +queues+ parameter can specify a number of threads for each group of queues which will override this value. You can also set this with the environment variable +GOOD_JOB_MAX_THREADS+. Defaults to +5+.
+    # @param queues [nil, String] determines which queues to execute jobs from when +execution_mode+ is set to +:async+. See {file:README.md#optimize-queues-threads-and-processes} for more details on the format of this string. You can also set this with the environment variable +GOOD_JOB_QUEUES+. Defaults to +"*"+.
+    # @param poll_interval [nil, Integer] sets the number of seconds between polls for jobs when +execution_mode+ is set to +:async+. You can also set this with the environment variable +GOOD_JOB_POLL_INTERVAL+. Defaults to +1+.
+    # @param scheduler [nil, Scheduler] (deprecated) a scheduler to be managed by the adapter
+    # @param notifier [nil, Notifier] (deprecated) a notifier to be managed by the adapter
+    # @param inline [nil, Boolean] (deprecated) whether to run in inline execution mode
     def initialize(execution_mode: nil, queues: nil, max_threads: nil, poll_interval: nil, scheduler: nil, notifier: nil, inline: false)
       if inline && execution_mode.nil?
         ActiveSupport::Deprecation.warn('GoodJob::Adapter#new(inline: true) is deprecated; use GoodJob::Adapter.new(execution_mode: :inline) instead')
@@ -9,10 +30,12 @@ module GoodJob
       end
 
       configuration = GoodJob::Configuration.new(
-        execution_mode: execution_mode,
-        queues: queues,
-        max_threads: max_threads,
-        poll_interval: poll_interval
+        {
+          execution_mode: execution_mode,
+          queues: queues,
+          max_threads: max_threads,
+          poll_interval: poll_interval,
+        }
       )
 
       @execution_mode = configuration.execution_mode
@@ -25,10 +48,19 @@ module GoodJob
       end
     end
 
+    # Enqueues the ActiveJob job to be performed.
+    # For use by Rails; you should generally not call this directly.
+    # @param active_job [ActiveJob::Base] the job to be enqueued from +#perform_later+
+    # @return [GoodJob::Job]
     def enqueue(active_job)
       enqueue_at(active_job, nil)
     end
 
+    # Enqueues an ActiveJob job to be run at a specific time.
+    # For use by Rails; you should generally not call this directly.
+    # @param active_job [ActiveJob::Base] the job to be enqueued from +#perform_later+
+    # @param timestamp [Integer] the epoch time to perform the job
+    # @return [GoodJob::Job]
     def enqueue_at(active_job, timestamp)
       good_job = GoodJob::Job.enqueue(
         active_job,
@@ -50,23 +82,31 @@ module GoodJob
       good_job
     end
 
+    # Gracefully stop processing jobs.
+    # Waits for termination by default.
+    # @param wait [Boolean] Whether to wait for shut down.
+    # @return [void]
     def shutdown(wait: true)
       @notifier&.shutdown(wait: wait)
       @scheduler&.shutdown(wait: wait)
     end
 
+    # Whether in +:async+ execution mode.
     def execute_async?
       @execution_mode == :async
     end
 
+    # Whether in +:external+ execution mode.
     def execute_externally?
       @execution_mode == :external
     end
 
+    # Whether in +:inline+ execution mode.
     def execute_inline?
       @execution_mode == :inline
     end
 
+    # (deprecated) Whether in +:inline+ execution mode.
     def inline?
       ActiveSupport::Deprecation.warn('GoodJob::Adapter::inline? is deprecated; use GoodJob::Adapter::execute_inline? instead')
       execute_inline?

data/lib/good_job/cli.rb

@@ -1,20 +1,48 @@
 require 'thor'
 
 module GoodJob
+  #
+  # Implements the +good_job+ command-line tool, which executes jobs and
+  # provides other utilities. The actual entry point is in +exe/good_job+, but
+  # it just sets up and calls this class.
+  #
+  # The +good_job+ command-line tool is based on Thor, a CLI framework for
+  # Ruby. For more on general usage, see http://whatisthor.com/ and
+  # https://github.com/erikhuda/thor/wiki.
+  #
   class CLI < Thor
+    # Path to the local Rails application's environment configuration.
+    # Requiring this loads the application's configuration and classes.
     RAILS_ENVIRONMENT_RB = File.expand_path("config/environment.rb")
 
-    desc :start, "Start job worker"
+    # @!macro thor.desc
+    #   @!method $1
+    #   @return [void]
+    #   The +good_job $1+ command. $2
+    desc :start, "Executes queued jobs."
+    long_desc <<~DESCRIPTION
+      Executes queued jobs.
+
+      All options can be configured with environment variables.
+      See option descriptions for the matching environment variable name.
+
+      == Configuring queues
+      \x5Separate multiple queues with commas; exclude queues with a leading minus;
+      separate isolated execution pools with semicolons and threads with colons.
+
+    DESCRIPTION
     method_option :max_threads,
                   type: :numeric,
-                  desc: "Maximum number of threads to use for working jobs (default: ActiveRecord::Base.connection_pool.size)"
+                  banner: 'COUNT',
+                  desc: "Maximum number of threads to use for working jobs. (env var: GOOD_JOB_MAX_THREADS, default: 5)"
     method_option :queues,
                   type: :string,
-                  banner: "queue1,queue2(;queue3,queue4:5;-queue1,queue2)",
-                  desc: "Queues to work from. Separate multiple queues with commas; exclude queues with a leading minus; separate isolated execution pools with semicolons and threads with colons (default: *)"
+                  banner: "QUEUE_LIST",
+                  desc: "Queues to work from. (env var: GOOD_JOB_QUEUES, default: *)"
     method_option :poll_interval,
                   type: :numeric,
-                  desc: "Interval between polls for available jobs in seconds (default: 1)"
+                  banner: 'SECONDS',
+                  desc: "Interval between polls for available jobs in seconds (env var: GOOD_JOB_POLL_INTERVAL, default: 1)"
     def start
       set_up_application!
 
@@ -39,17 +67,37 @@ module GoodJob
 
     default_task :start
 
-    desc :cleanup_preserved_jobs, "Delete preserved job records"
+    # @!macro thor.desc
+    desc :cleanup_preserved_jobs, "Deletes preserved job records."
+    long_desc <<~DESCRIPTION
+      Deletes preserved job records.
+
+      By default, GoodJob deletes job records when the job is performed and this
+      command is not necessary.
+
+      However, when `GoodJob.preserve_job_records = true`, the jobs will be
+      preserved in the database. This is useful when wanting to analyze or
+      inspect job performance.
+
+      If you are preserving job records this way, use this command regularly
+      to delete old records and preserve space in your database.
+
+    DESCRIPTION
     method_option :before_seconds_ago,
                   type: :numeric,
-                  default: 24 * 60 * 60,
-                  desc: "Delete records finished more than this many seconds ago"
-
+                  banner: 'SECONDS',
+                  desc: "Delete records finished more than this many seconds ago (env var: GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO, default: 86400)"
     def cleanup_preserved_jobs
       set_up_application!
 
-      timestamp = Time.current - options[:before_seconds_ago]
-      ActiveSupport::Notifications.instrument("cleanup_preserved_jobs.good_job", { before_seconds_ago: options[:before_seconds_ago], timestamp: timestamp }) do |payload|
+      configuration = GoodJob::Configuration.new(options)
+
+      timestamp = Time.current - configuration.cleanup_preserved_jobs_before_seconds_ago
+
+      ActiveSupport::Notifications.instrument(
+        "cleanup_preserved_jobs.good_job",
+        { before_seconds_ago: configuration.cleanup_preserved_jobs_before_seconds_ago, timestamp: timestamp }
+      ) do |payload|
         deleted_records_count = GoodJob::Job.finished(timestamp).delete_all
 
         payload[:deleted_records_count] = deleted_records_count
@@ -57,11 +105,16 @@ module GoodJob
     end
 
     no_commands do
+      # Load the current Rails application and configuration that the good_job
+      # command-line tool should be working within.
+      #
+      # GoodJob components that need access to constants, classes, etc. from
+      # Rails or from the application can be set up here.
       def set_up_application!
         require RAILS_ENVIRONMENT_RB
-        return unless defined?(GOOD_JOB_LOG_TO_STDOUT) && GOOD_JOB_LOG_TO_STDOUT && !ActiveSupport::Logger.logger_outputs_to?(GoodJob.logger, STDOUT)
+        return unless defined?(GOOD_JOB_LOG_TO_STDOUT) && GOOD_JOB_LOG_TO_STDOUT && !ActiveSupport::Logger.logger_outputs_to?(GoodJob.logger, $stdout)
 
-        GoodJob::LogSubscriber.loggers << ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT))
+        GoodJob::LogSubscriber.loggers << ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
         GoodJob::LogSubscriber.reset_logger
       end
     end

data/lib/good_job/configuration.rb

@@ -1,12 +1,47 @@
 module GoodJob
+  #
+  # +GoodJob::Configuration+ provides normalized configuration information to
+  # the rest of GoodJob. It combines environment information with explicitly
+  # set options to get the final values for each option.
+  #
   class Configuration
+    # Default number of threads to use per {Scheduler}
+    DEFAULT_MAX_THREADS = 5
+    # Default number of seconds between polls for jobs
+    DEFAULT_POLL_INTERVAL = 1
+    # Default number of seconds to preserve jobs for {CLI#cleanup_preserved_jobs}
+    DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO = 24 * 60 * 60
+
+    # @!attribute [r] options
+    #   The options that were explicitly set when initializing +Configuration+.
+    #   @return [Hash]
+    #
+    # @!attribute [r] env
+    #   The environment from which to read GoodJob's environment variables. By
+    #   default, this is the current process's environment, but it can be set
+    #   to something else in {#initialize}.
+    #   @return [Hash]
     attr_reader :options, :env
 
+    # @param options [Hash] Any explicitly specified configuration options to
+    #   use. Keys are symbols that match the various methods on this class.
+    # @param env [Hash] A +Hash+ from which to read environment variables that
+    #   might specify additional configuration values.
     def initialize(options, env: ENV)
       @options = options
       @env = env
     end
 
+    # Specifies how and where jobs should be executed. See {Adapter#initialize}
+    # for more details on possible values.
+    #
+    # When running inside a Rails app, you may want to use
+    # {#rails_execution_mode}, which takes the current Rails environment into
+    # account when determining the final value.
+    #
+    # @param default [Symbol]
+    #   Value to use if none was specified in the configuration.
+    # @return [Symbol]
     def execution_mode(default: :external)
       if options[:execution_mode]
         options[:execution_mode]
@@ -17,6 +52,9 @@ module GoodJob
       end
     end
 
+    # Like {#execution_mode}, but takes the current Rails environment into
+    # account (e.g. in the +test+ environment, it falls back to +:inline+).
+    # @return [Symbol]
     def rails_execution_mode
       if execution_mode(default: nil)
         execution_mode
@@ -29,26 +67,47 @@ module GoodJob
       end
     end
 
+    # Indicates the number of threads to use per {Scheduler}. Note that
+    # {#queue_string} may provide more specific thread counts to use with
+    # individual schedulers.
+    # @return [Integer]
     def max_threads
       (
         options[:max_threads] ||
         env['GOOD_JOB_MAX_THREADS'] ||
         env['RAILS_MAX_THREADS'] ||
-        ActiveRecord::Base.connection_pool.size
+        DEFAULT_MAX_THREADS
       ).to_i
     end
 
+    # Describes which queues to execute jobs from and how those queues should
+    # be grouped into {Scheduler} instances. See
+    # {file:README.md#optimize-queues-threads-and-processes} for more details
+    # on the format of this string.
+    # @return [String]
     def queue_string
       options[:queues] ||
         env['GOOD_JOB_QUEUES'] ||
         '*'
     end
 
+    # The number of seconds between polls for jobs. GoodJob will execute jobs
+    # on queues continuously until a queue is empty, at which point it will
+    # poll (using this interval) for new queued jobs to execute.
+    # @return [Integer]
     def poll_interval
       (
         options[:poll_interval] ||
         env['GOOD_JOB_POLL_INTERVAL'] ||
-        1
+        DEFAULT_POLL_INTERVAL
+      ).to_i
+    end
+
+    def cleanup_preserved_jobs_before_seconds_ago
+      (
+        options[:before_seconds_ago] ||
+        env['GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO'] ||
+        DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO
       ).to_i
     end
   end