good_job 1.0.3 → 1.1.4

data/exe/good_job

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
 require 'good_job/cli'
+GOOD_JOB_LOG_TO_STDOUT = true
 GoodJob::CLI.start(ARGV)

data/lib/active_job/queue_adapters/good_job_adapter.rb

@@ -1,18 +1,9 @@
 module ActiveJob
   module QueueAdapters
     class GoodJobAdapter < GoodJob::Adapter
-      def initialize(execution_mode: nil)
-        execution_mode = if execution_mode
-                           execution_mode
-                         elsif ENV['GOOD_JOB_EXECUTION_MODE'].present?
-                           ENV['GOOD_JOB_EXECUTION_MODE'].to_sym
-                         elsif Rails.env.development? || Rails.env.test?
-                           :inline
-                         else
-                           :external
-                         end
-
-        super(execution_mode: execution_mode)
+      def initialize(execution_mode: nil, max_threads: nil, poll_interval: nil, scheduler: nil, inline: false)
+        configuration = GoodJob::Configuration.new({ execution_mode: execution_mode }, env: ENV)
+        super(execution_mode: configuration.rails_execution_mode, max_threads: max_threads, poll_interval: poll_interval, scheduler: scheduler, inline: inline)
       end
     end
   end

data/lib/good_job.rb

@@ -1,20 +1,46 @@
 require "rails"
 require 'good_job/railtie'
 
-require 'good_job/logging'
+require 'good_job/configuration'
+require 'good_job/log_subscriber'
 require 'good_job/lockable'
 require 'good_job/job'
 require 'good_job/scheduler'
+require 'good_job/multi_scheduler'
 require 'good_job/adapter'
 require 'good_job/pg_locks'
 require 'good_job/performer'
+require 'good_job/current_execution'
+require 'good_job/notifier'
 
 require 'active_job/queue_adapters/good_job_adapter'
 
 module GoodJob
+  mattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT))
   mattr_accessor :preserve_job_records, default: false
   mattr_accessor :reperform_jobs_on_standard_error, default: true
-  include Logging
+  mattr_accessor :on_thread_error, default: nil
 
   ActiveSupport.run_load_hooks(:good_job, self)
+
+  # Shuts down all execution pools
+  # @param wait [Boolean] whether to wait for shutdown
+  # @return [void]
+  def self.shutdown(wait: true)
+    Notifier.instances.each { |adapter| adapter.shutdown(wait: wait) }
+    Scheduler.instances.each { |scheduler| scheduler.shutdown(wait: wait) }
+  end
+
+  # Tests if execution pools are shut down
+  # @return [Boolean] whether execution pools are shut down
+  def self.shutdown?
+    Notifier.instances.all?(&:shutdown?) && Scheduler.instances.all?(&:shutdown?)
+  end
+
+  # Restarts all execution pools
+  # @return [void]
+  def self.restart
+    Notifier.instances.each(&:restart)
+    Scheduler.instances.each(&:restart)
+  end
 end

data/lib/good_job/adapter.rb

@@ -1,17 +1,27 @@
 module GoodJob
   class Adapter
-    EXECUTION_MODES = [:inline, :external].freeze # TODO: async
+    EXECUTION_MODES = [:async, :external, :inline].freeze
 
-    def initialize(execution_mode: nil, inline: false)
-      if inline
+    def initialize(execution_mode: nil, queues: nil, max_threads: nil, poll_interval: nil, scheduler: nil, notifier: nil, inline: false)
+      if inline && execution_mode.nil?
         ActiveSupport::Deprecation.warn('GoodJob::Adapter#new(inline: true) is deprecated; use GoodJob::Adapter.new(execution_mode: :inline) instead')
-        @execution_mode = :inline
-      elsif execution_mode
-        raise ArgumentError, "execution_mode: must be one of #{EXECUTION_MODES.join(', ')}." unless EXECUTION_MODES.include?(execution_mode)
+        execution_mode = :inline
+      end
+
+      configuration = GoodJob::Configuration.new(
+        execution_mode: execution_mode,
+        queues: queues,
+        max_threads: max_threads,
+        poll_interval: poll_interval
+      )
+
+      @execution_mode = configuration.execution_mode
+      raise ArgumentError, "execution_mode: must be one of #{EXECUTION_MODES.join(', ')}." unless EXECUTION_MODES.include?(@execution_mode)
 
-        @execution_mode = execution_mode
-      else
-        @execution_mode = :external
+      if @execution_mode == :async # rubocop:disable Style/GuardClause
+        @notifier = notifier || GoodJob::Notifier.new
+        @scheduler = scheduler || GoodJob::Scheduler.from_configuration(configuration)
+        @notifier.recipients << [@scheduler, :create_thread]
       end
     end
 
@@ -34,11 +44,23 @@ module GoodJob
         end
       end
 
+      executed_locally = execute_async? && @scheduler.create_thread(queue_name: good_job.queue_name)
+      Notifier.notify(queue_name: good_job.queue_name) unless executed_locally
+
       good_job
     end
 
-    def shutdown(wait: true) # rubocop:disable Lint/UnusedMethodArgument
-      nil
+    def shutdown(wait: true)
+      @notifier&.shutdown(wait: wait)
+      @scheduler&.shutdown(wait: wait)
+    end
+
+    def execute_async?
+      @execution_mode == :async
+    end
+
+    def execute_externally?
+      @execution_mode == :external
     end
 
     def execute_inline?
@@ -49,9 +71,5 @@ module GoodJob
       ActiveSupport::Deprecation.warn('GoodJob::Adapter::inline? is deprecated; use GoodJob::Adapter::execute_inline? instead')
       execute_inline?
     end
-
-    def execute_externally?
-      @execution_mode == :external
-    end
   end
 end

data/lib/good_job/cli.rb

@@ -10,47 +10,18 @@ module GoodJob
                   desc: "Maximum number of threads to use for working jobs (default: ActiveRecord::Base.connection_pool.size)"
     method_option :queues,
                   type: :string,
-                  banner: "queue1,queue2",
-                  desc: "Queues to work from. Separate multiple queues with commas (default: *)"
+                  banner: "queue1,queue2(;queue3,queue4:5;-queue1,queue2)",
+                  desc: "Queues to work from. Separate multiple queues with commas; exclude queues with a leading minus; separate isolated execution pools with semicolons and threads with colons (default: *)"
     method_option :poll_interval,
                   type: :numeric,
                   desc: "Interval between polls for available jobs in seconds (default: 1)"
     def start
-      require RAILS_ENVIRONMENT_RB
+      set_up_application!
 
-      max_threads = (
-        options[:max_threads] ||
-        ENV['GOOD_JOB_MAX_THREADS'] ||
-        ENV['RAILS_MAX_THREADS'] ||
-        ActiveRecord::Base.connection_pool.size
-      ).to_i
-
-      queue_names = (
-        options[:queues] ||
-        ENV['GOOD_JOB_QUEUES'] ||
-        '*'
-      ).split(',').map(&:strip)
-
-      poll_interval = (
-        options[:poll_interval] ||
-        ENV['GOOD_JOB_POLL_INTERVAL']
-      ).to_i
-
-      job_query = GoodJob::Job.all.priority_ordered
-      queue_names_without_all = queue_names.reject { |q| q == '*' }
-      job_query = job_query.where(queue_name: queue_names_without_all) unless queue_names_without_all.size.zero?
-      job_performer = GoodJob::Performer.new(job_query, :perform_with_advisory_lock)
-
-      $stdout.puts "GoodJob worker starting with max_threads=#{max_threads} on queues=#{queue_names.join(',')}"
-
-      timer_options = {}
-      timer_options[:execution_interval] = poll_interval if poll_interval.positive?
-
-      pool_options = {
-        max_threads: max_threads,
-      }
-
-      scheduler = GoodJob::Scheduler.new(job_performer, timer_options: timer_options, pool_options: pool_options)
+      notifier = GoodJob::Notifier.new
+      configuration = GoodJob::Configuration.new(options)
+      scheduler = GoodJob::Scheduler.from_configuration(configuration)
+      notifier.recipients << [scheduler, :create_thread]
 
       @stop_good_job_executable = false
       %w[INT TERM].each do |signal|
@@ -59,27 +30,40 @@ module GoodJob
 
       Kernel.loop do
         sleep 0.1
-        break if @stop_good_job_executable || scheduler.shutdown?
+        break if @stop_good_job_executable || scheduler.shutdown? || notifier.shutdown?
       end
 
-      $stdout.puts "\nFinishing GoodJob's current jobs before exiting..."
+      notifier.shutdown
       scheduler.shutdown
-      $stdout.puts "GoodJob's jobs finished, exiting..."
     end
 
+    default_task :start
+
     desc :cleanup_preserved_jobs, "Delete preserved job records"
     method_option :before_seconds_ago,
                   type: :numeric,
                   default: 24 * 60 * 60,
                   desc: "Delete records finished more than this many seconds ago"
+
     def cleanup_preserved_jobs
-      require RAILS_ENVIRONMENT_RB
+      set_up_application!
 
       timestamp = Time.current - options[:before_seconds_ago]
-      result = GoodJob::Job.finished(timestamp).delete_all
-      $stdout.puts "Deleted #{result} preserved #{'job'.pluralize(result)} finished before #{timestamp}."
+      ActiveSupport::Notifications.instrument("cleanup_preserved_jobs.good_job", { before_seconds_ago: options[:before_seconds_ago], timestamp: timestamp }) do |payload|
+        deleted_records_count = GoodJob::Job.finished(timestamp).delete_all
+
+        payload[:deleted_records_count] = deleted_records_count
+      end
     end
 
-    default_task :start
+    no_commands do
+      def set_up_application!
+        require RAILS_ENVIRONMENT_RB
+        return unless defined?(GOOD_JOB_LOG_TO_STDOUT) && GOOD_JOB_LOG_TO_STDOUT && !ActiveSupport::Logger.logger_outputs_to?(GoodJob.logger, STDOUT)
+
+        GoodJob::LogSubscriber.loggers << ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT))
+        GoodJob::LogSubscriber.reset_logger
+      end
+    end
   end
 end

data/lib/good_job/configuration.rb

@@ -0,0 +1,55 @@
+module GoodJob
+  class Configuration
+    attr_reader :options, :env
+
+    def initialize(options, env: ENV)
+      @options = options
+      @env = env
+    end
+
+    def execution_mode(default: :external)
+      if options[:execution_mode]
+        options[:execution_mode]
+      elsif env['GOOD_JOB_EXECUTION_MODE'].present?
+        env['GOOD_JOB_EXECUTION_MODE'].to_sym
+      else
+        default
+      end
+    end
+
+    def rails_execution_mode
+      if execution_mode(default: nil)
+        execution_mode
+      elsif Rails.env.development?
+        :inline
+      elsif Rails.env.test?
+        :inline
+      else
+        :external
+      end
+    end
+
+    def max_threads
+      (
+        options[:max_threads] ||
+        env['GOOD_JOB_MAX_THREADS'] ||
+        env['RAILS_MAX_THREADS'] ||
+        ActiveRecord::Base.connection_pool.size
+      ).to_i
+    end
+
+    def queue_string
+      options[:queues] ||
+        env['GOOD_JOB_QUEUES'] ||
+        '*'
+    end
+
+    def poll_interval
+      (
+        options[:poll_interval] ||
+        env['GOOD_JOB_POLL_INTERVAL'] ||
+        1
+      ).to_i
+    end
+  end
+end

data/lib/good_job/current_execution.rb

@@ -0,0 +1,35 @@
+module GoodJob
+  # Thread-local attributes for passing values from Instrumentation.
+  # (Cannot use ActiveSupport::CurrentAttributes because ActiveJob resets it)
+
+  module CurrentExecution
+    # @!attribute [rw] error_on_retry
+    #   @!scope class
+    #   Error captured by retry_on
+    #   @return [Exception, nil]
+    thread_mattr_accessor :error_on_retry
+
+    # @!attribute [rw] error_on_discard
+    #   @!scope class
+    #   Error captured by discard_on
+    #   @return [Exception, nil]
+    thread_mattr_accessor :error_on_discard
+
+    # Resets attributes
+    # @return [void]
+    def self.reset
+      self.error_on_retry = nil
+      self.error_on_discard = nil
+    end
+
+    # @return [Integer] Current process ID
+    def self.process_id
+      Process.pid
+    end
+
+    # @return [String] Current thread name
+    def self.thread_name
+      (Thread.current.name || Thread.current.object_id).to_s
+    end
+  end
+end

data/lib/good_job/job.rb

@@ -9,6 +9,25 @@ module GoodJob
 
     self.table_name = 'good_jobs'.freeze
 
+    def self.queue_parser(string)
+      string = string.presence || '*'
+
+      if string.first == '-'
+        exclude_queues = true
+        string = string[1..-1]
+      end
+
+      queues = string.split(',').map(&:strip)
+
+      if queues.include?('*')
+        { all: true }
+      elsif exclude_queues
+        { exclude: queues }
+      else
+        { include: queues }
+      end
+    end
+
     scope :unfinished, (lambda do
       if column_names.include?('finished_at')
         where(finished_at: nil)
@@ -18,15 +37,26 @@ module GoodJob
       end
     end)
     scope :only_scheduled, -> { where(arel_table['scheduled_at'].lteq(Time.current)).or(where(scheduled_at: nil)) }
-    scope :priority_ordered, -> { order(priority: :desc) }
+    scope :priority_ordered, -> { order('priority DESC NULLS LAST') }
     scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
+    scope :queue_string, (lambda do |string|
+      parsed = queue_parser(string)
+
+      if parsed[:all]
+        all
+      elsif parsed[:exclude]
+        where.not(queue_name: parsed[:exclude]).or where(queue_name: nil)
+      elsif parsed[:include]
+        where(queue_name: parsed[:include])
+      end
+    end)
 
     def self.perform_with_advisory_lock
       good_job = nil
       result = nil
       error = nil
 
-      unfinished.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
+      unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
         good_job = good_jobs.first
         break unless good_job
 
@@ -43,7 +73,7 @@ module GoodJob
           queue_name: active_job.queue_name.presence || DEFAULT_QUEUE_NAME,
           priority: active_job.priority || DEFAULT_PRIORITY,
           serialized_params: active_job.serialize,
-          scheduled_at: scheduled_at || Time.current,
+          scheduled_at: scheduled_at,
           create_with_advisory_lock: create_with_advisory_lock
         )
 
@@ -59,11 +89,11 @@ module GoodJob
     def perform(destroy_after: !GoodJob.preserve_job_records, reperform_on_standard_error: GoodJob.reperform_jobs_on_standard_error)
       raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
 
+      GoodJob::CurrentExecution.reset
       result = nil
       rescued_error = nil
       error = nil
 
-      ActiveSupport::Notifications.instrument("before_perform_job.good_job", { good_job: self })
       self.performed_at = Time.current
       save! unless destroy_after
 
@@ -72,18 +102,23 @@ module GoodJob
       )
 
       begin
-        ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self }) do
+        ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self, process_id: GoodJob::CurrentExecution.process_id, thread_name: GoodJob::CurrentExecution.thread_name }) do
           result = ActiveJob::Base.execute(params)
         end
       rescue StandardError => e
         rescued_error = e
       end
 
+      retry_or_discard_error = GoodJob::CurrentExecution.error_on_retry ||
+                               GoodJob::CurrentExecution.error_on_discard
+
       if rescued_error
         error = rescued_error
       elsif result.is_a?(Exception)
         error = result
         result = nil
+      elsif retry_or_discard_error
+        error = retry_or_discard_error
       end
 
       error_message = "#{error.class}: #{error.message}" if error

data/lib/good_job/log_subscriber.rb

@@ -0,0 +1,167 @@
+module GoodJob
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def create(event)
+      good_job = event.payload[:good_job]
+
+      debug do
+        "GoodJob created job resource with id #{good_job.id}"
+      end
+    end
+
+    def timer_task_finished(event)
+      exception = event.payload[:error]
+      return unless exception
+
+      error do
+        "GoodJob error: #{exception}\n #{exception.backtrace}"
+      end
+    end
+
+    def job_finished(event)
+      exception = event.payload[:error]
+      return unless exception
+
+      error do
+        "GoodJob error: #{exception}\n #{exception.backtrace}"
+      end
+    end
+
+    def scheduler_create_pools(event)
+      max_threads = event.payload[:max_threads]
+      poll_interval = event.payload[:poll_interval]
+      performer_name = event.payload[:performer_name]
+      process_id = event.payload[:process_id]
+
+      info(tags: [process_id]) do
+        "GoodJob started scheduler with queues=#{performer_name} max_threads=#{max_threads} poll_interval=#{poll_interval}."
+      end
+    end
+
+    def scheduler_shutdown_start(event)
+      process_id = event.payload[:process_id]
+
+      info(tags: [process_id]) do
+        "GoodJob shutting down scheduler..."
+      end
+    end
+
+    def scheduler_shutdown(event)
+      process_id = event.payload[:process_id]
+
+      info(tags: [process_id]) do
+        "GoodJob scheduler is shut down."
+      end
+    end
+
+    def scheduler_restart_pools(event)
+      process_id = event.payload[:process_id]
+
+      info(tags: [process_id]) do
+        "GoodJob scheduler has restarted."
+      end
+    end
+
+    def perform_job(event)
+      good_job = event.payload[:good_job]
+      process_id = event.payload[:process_id]
+      thread_name = event.payload[:thread_name]
+
+      info(tags: [process_id, thread_name]) do
+        "Executed GoodJob #{good_job.id}"
+      end
+    end
+
+    def notifier_listen(_event)
+      info do
+        "Notifier subscribed with LISTEN"
+      end
+    end
+
+    def notifier_notified(event)
+      payload = event.payload[:payload]
+
+      debug do
+        "Notifier received payload: #{payload}"
+      end
+    end
+
+    def notifier_notify_error(event)
+      error = event.payload[:error]
+
+      error do
+        "Notifier errored: #{error}"
+      end
+    end
+
+    def notifier_unlisten(_event)
+      info do
+        "Notifier unsubscribed with UNLISTEN"
+      end
+    end
+
+    def cleanup_preserved_jobs(event)
+      timestamp = event.payload[:timestamp]
+      deleted_records_count = event.payload[:deleted_records_count]
+
+      info do
+        "GoodJob deleted #{deleted_records_count} preserved #{'job'.pluralize(deleted_records_count)} finished before #{timestamp}."
+      end
+    end
+
+    class << self
+      def loggers
+        @_loggers ||= [GoodJob.logger]
+      end
+
+      def logger
+        @_logger ||= begin
+                      logger = Logger.new(StringIO.new)
+                      loggers.each do |each_logger|
+                        logger.extend(ActiveSupport::Logger.broadcast(each_logger))
+                      end
+                      logger
+                    end
+      end
+
+      def reset_logger
+        @_logger = nil
+      end
+    end
+
+    def logger
+      GoodJob::LogSubscriber.logger
+    end
+
+    private
+
+    def tag_logger(*tags, &block)
+      tags = tags.dup.unshift("GoodJob").compact
+
+      self.class.loggers.inject(block) do |inner, each_logger|
+        if each_logger.respond_to?(:tagged)
+          tags_for_logger = if each_logger.formatter.current_tags.include?("ActiveJob")
+                              ["ActiveJob"] + tags
+                            else
+                              tags
+                            end
+
+          proc { each_logger.tagged(*tags_for_logger, &inner) }
+        else
+          inner
+        end
+      end.call
+    end
+
+    %w(info debug warn error fatal unknown).each do |level|
+      class_eval <<-METHOD, __FILE__, __LINE__ + 1
+        def #{level}(progname = nil, tags: [], &block)
+          return unless logger
+
+          tag_logger(*tags) do
+            logger.#{level}(progname, &block)
+          end
+        end
+      METHOD
+    end
+  end
+end