good_job 1.0.1 → 1.1.2

data/lib/active_job/queue_adapters/good_job_adapter.rb

@@ -1,18 +1,9 @@
 module ActiveJob
   module QueueAdapters
     class GoodJobAdapter < GoodJob::Adapter
-      def initialize(execution_mode: nil)
-        execution_mode = if execution_mode
-                           execution_mode
-                         elsif ENV['GOOD_JOB_EXECUTION_MODE'].present?
-                           ENV['GOOD_JOB_EXECUTION_MODE'].to_sym
-                         elsif Rails.env.development? || Rails.env.test?
-                           :inline
-                         else
-                           :external
-                         end
-
-        super(execution_mode: execution_mode)
+      def initialize(execution_mode: nil, max_threads: nil, poll_interval: nil, scheduler: nil, inline: false)
+        configuration = GoodJob::Configuration.new({ execution_mode: execution_mode }, env: ENV)
+        super(execution_mode: configuration.rails_execution_mode, max_threads: max_threads, poll_interval: poll_interval, scheduler: scheduler, inline: inline)
       end
     end
   end

data/lib/generators/good_job/install_generator.rb

@@ -0,0 +1,24 @@
+require 'rails/generators'
+require 'rails/generators/active_record'
+
+module GoodJob
+  class InstallGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    class << self
+      delegate :next_migration_number, to: ActiveRecord::Generators::Base
+    end
+
+    source_paths << File.join(File.dirname(__FILE__), "templates")
+
+    def create_migration_file
+      migration_template 'migration.rb.erb', 'db/migrate/create_good_jobs.rb', migration_version: migration_version
+    end
+
+    private
+
+    def migration_version
+      "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+    end
+  end
+end

data/lib/generators/good_job/templates/migration.rb.erb

@@ -0,0 +1,20 @@
+class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
+  def change
+    enable_extension 'pgcrypto'
+
+    create_table :good_jobs, id: :uuid do |t|
+      t.text :queue_name
+      t.integer :priority
+      t.jsonb :serialized_params
+      t.timestamp :scheduled_at
+      t.timestamp :performed_at
+      t.timestamp :finished_at
+      t.text :error
+
+      t.timestamps
+    end
+
+    add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)"
+    add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)"
+  end
+end

data/lib/good_job.rb

@@ -1,10 +1,12 @@
 require "rails"
 require 'good_job/railtie'
 
-require 'good_job/logging'
+require 'good_job/configuration'
+require 'good_job/log_subscriber'
 require 'good_job/lockable'
 require 'good_job/job'
 require 'good_job/scheduler'
+require 'good_job/multi_scheduler'
 require 'good_job/adapter'
 require 'good_job/pg_locks'
 require 'good_job/performer'
@@ -12,8 +14,10 @@ require 'good_job/performer'
 require 'active_job/queue_adapters/good_job_adapter'
 
 module GoodJob
+  cattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT))
   mattr_accessor :preserve_job_records, default: false
-  include Logging
+  mattr_accessor :reperform_jobs_on_standard_error, default: true
+  mattr_accessor :on_thread_error, default: nil
 
   ActiveSupport.run_load_hooks(:good_job, self)
 end

data/lib/good_job/adapter.rb

@@ -1,18 +1,26 @@
 module GoodJob
   class Adapter
-    EXECUTION_MODES = [:inline, :external].freeze # TODO: async
+    EXECUTION_MODES = [:async, :external, :inline].freeze
 
-    def initialize(execution_mode: nil, inline: false)
-      if inline
+    def initialize(execution_mode: nil, max_threads: nil, poll_interval: nil, scheduler: nil, inline: false)
+      if inline && execution_mode.nil?
         ActiveSupport::Deprecation.warn('GoodJob::Adapter#new(inline: true) is deprecated; use GoodJob::Adapter.new(execution_mode: :inline) instead')
-        @execution_mode = :inline
-      elsif execution_mode
-        raise ArgumentError, "execution_mode: must be one of #{EXECUTION_MODES.join(', ')}." unless EXECUTION_MODES.include?(execution_mode)
-
-        @execution_mode = execution_mode
-      else
-        @execution_mode = :external
+        execution_mode = :inline
       end
+
+      configuration = GoodJob::Configuration.new({
+                                                   execution_mode: execution_mode,
+                                                   max_threads: max_threads,
+                                                   poll_interval: poll_interval,
+                                                 },
+                                                 env: ENV)
+
+      raise ArgumentError, "execution_mode: must be one of #{EXECUTION_MODES.join(', ')}." unless EXECUTION_MODES.include?(configuration.execution_mode)
+
+      @execution_mode = configuration.execution_mode
+
+      @scheduler = scheduler
+      @scheduler = GoodJob::Scheduler.from_configuration(configuration) if @execution_mode == :async && @scheduler.blank?
     end
 
     def enqueue(active_job)
@@ -34,11 +42,21 @@ module GoodJob
         end
       end
 
+      @scheduler.create_thread if execute_async?
+
       good_job
     end
 
-    def shutdown(wait: true) # rubocop:disable Lint/UnusedMethodArgument
-      nil
+    def shutdown(wait: true)
+      @scheduler&.shutdown(wait: wait)
+    end
+
+    def execute_async?
+      @execution_mode == :async
+    end
+
+    def execute_externally?
+      @execution_mode == :external
     end
 
     def execute_inline?
@@ -49,9 +67,5 @@ module GoodJob
       ActiveSupport::Deprecation.warn('GoodJob::Adapter::inline? is deprecated; use GoodJob::Adapter::execute_inline? instead')
       execute_inline?
     end
-
-    def execute_externally?
-      @execution_mode == :external
-    end
   end
 end

data/lib/good_job/cli.rb

@@ -10,53 +10,16 @@ module GoodJob
                   desc: "Maximum number of threads to use for working jobs (default: ActiveRecord::Base.connection_pool.size)"
     method_option :queues,
                   type: :string,
-                  banner: "queue1,queue2",
-                  desc: "Queues to work from. Separate multiple queues with commas (default: *)"
+                  banner: "queue1,queue2(;queue3,queue4:5;-queue1,queue2)",
+                  desc: "Queues to work from. Separate multiple queues with commas; exclude queues with a leading minus; separate isolated execution pools with semicolons and threads with colons (default: *)"
     method_option :poll_interval,
                   type: :numeric,
                   desc: "Interval between polls for available jobs in seconds (default: 1)"
     def start
-      require RAILS_ENVIRONMENT_RB
+      set_up_application!
 
-      max_threads = (
-        options[:max_threads] ||
-        ENV['GOOD_JOB_MAX_THREADS'] ||
-        ENV['RAILS_MAX_THREADS'] ||
-        ActiveRecord::Base.connection_pool.size
-      ).to_i
-
-      queue_names = (
-        options[:queues] ||
-        ENV['GOOD_JOB_QUEUES'] ||
-        '*'
-      ).split(',').map(&:strip)
-
-      poll_interval = (
-        options[:poll_interval] ||
-        ENV['GOOD_JOB_POLL_INTERVAL']
-      ).to_i
-
-      job_query = GoodJob::Job.all.priority_ordered
-      queue_names_without_all = queue_names.reject { |q| q == '*' }
-      job_query = job_query.where(queue_name: queue_names_without_all) unless queue_names_without_all.size.zero?
-
-      performer_method = if GoodJob.preserve_job_records
-                           :perform_with_advisory_lock_and_preserve_job_records
-                         else
-                           :perform_with_advisory_lock_and_destroy_job_records
-                         end
-      job_performer = GoodJob::Performer.new(job_query, performer_method)
-
-      $stdout.puts "GoodJob worker starting with max_threads=#{max_threads} on queues=#{queue_names.join(',')}"
-
-      timer_options = {}
-      timer_options[:execution_interval] = poll_interval if poll_interval.positive?
-
-      pool_options = {
-        max_threads: max_threads,
-      }
-
-      scheduler = GoodJob::Scheduler.new(job_performer, timer_options: timer_options, pool_options: pool_options)
+      configuration = Configuration.new(options, env: ENV)
+      scheduler = Scheduler.from_configuration(configuration)
 
       @stop_good_job_executable = false
       %w[INT TERM].each do |signal|
@@ -68,24 +31,31 @@ module GoodJob
         break if @stop_good_job_executable || scheduler.shutdown?
       end
 
-      $stdout.puts "\nFinishing GoodJob's current jobs before exiting..."
       scheduler.shutdown
-      $stdout.puts "GoodJob's jobs finished, exiting..."
     end
 
+    default_task :start
+
     desc :cleanup_preserved_jobs, "Delete preserved job records"
     method_option :before_seconds_ago,
                   type: :numeric,
                   default: 24 * 60 * 60,
                   desc: "Delete records finished more than this many seconds ago"
     def cleanup_preserved_jobs
-      require RAILS_ENVIRONMENT_RB
+      set_up_application!
 
       timestamp = Time.current - options[:before_seconds_ago]
-      result = GoodJob::Job.finished(timestamp).delete_all
-      $stdout.puts "Deleted #{result} preserved #{'job'.pluralize(result)} finished before #{timestamp}."
+      ActiveSupport::Notifications.instrument("cleanup_preserved_jobs.good_job", { before_seconds_ago: options[:before_seconds_ago], timestamp: timestamp }) do |payload|
+        deleted_records_count = GoodJob::Job.finished(timestamp).delete_all
+
+        payload[:deleted_records_count] = deleted_records_count
+      end
     end
 
-    default_task :start
+    no_commands do
+      def set_up_application!
+        require RAILS_ENVIRONMENT_RB
+      end
+    end
   end
 end

data/lib/good_job/configuration.rb

@@ -0,0 +1,55 @@
+module GoodJob
+  class Configuration
+    attr_reader :options, :env
+
+    def initialize(options, env: ENV)
+      @options = options
+      @env = env
+    end
+
+    def execution_mode(default: :external)
+      if options[:execution_mode]
+        options[:execution_mode]
+      elsif env['GOOD_JOB_EXECUTION_MODE'].present?
+        env['GOOD_JOB_EXECUTION_MODE'].to_sym
+      else
+        default
+      end
+    end
+
+    def rails_execution_mode
+      if execution_mode(default: nil)
+        execution_mode
+      elsif Rails.env.development?
+        :inline
+      elsif Rails.env.test?
+        :inline
+      else
+        :external
+      end
+    end
+
+    def max_threads
+      (
+        options[:max_threads] ||
+        env['GOOD_JOB_MAX_THREADS'] ||
+        env['RAILS_MAX_THREADS'] ||
+        ActiveRecord::Base.connection_pool.size
+      ).to_i
+    end
+
+    def queue_string
+      options[:queues] ||
+        env['GOOD_JOB_QUEUES'] ||
+        '*'
+    end
+
+    def poll_interval
+      (
+        options[:poll_interval] ||
+        env['GOOD_JOB_POLL_INTERVAL'] ||
+        1
+      ).to_i
+    end
+  end
+end

data/lib/good_job/job.rb

@@ -18,32 +18,41 @@ module GoodJob
       end
     end)
     scope :only_scheduled, -> { where(arel_table['scheduled_at'].lteq(Time.current)).or(where(scheduled_at: nil)) }
-    scope :priority_ordered, -> { order(priority: :desc) }
+    scope :priority_ordered, -> { order('priority DESC NULLS LAST') }
     scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
+    scope :queue_string, (lambda do |string|
+      string = string.presence || '*'
 
-    def self.perform_with_advisory_lock(destroy_after: !GoodJob.preserve_job_records)
+      if string.first == '-'
+        exclude_queues = true
+        string = string[1..-1]
+      end
+
+      queue_names_without_all = string.split(',').map(&:strip).reject { |q| q == '*' }
+      return if queue_names_without_all.size.zero?
+
+      if exclude_queues
+        where.not(queue_name: queue_names_without_all).or where(queue_name: nil)
+      else
+        where(queue_name: queue_names_without_all)
+      end
+    end)
+
+    def self.perform_with_advisory_lock
       good_job = nil
       result = nil
       error = nil
 
-      unfinished.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
+      unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
         good_job = good_jobs.first
         break unless good_job
 
-        result, error = good_job.perform(destroy_after: destroy_after)
+        result, error = good_job.perform
       end
 
       [good_job, result, error] if good_job
     end
 
-    def self.perform_with_advisory_lock_and_preserve_job_records
-      perform_with_advisory_lock(destroy_after: false)
-    end
-
-    def self.perform_with_advisory_lock_and_destroy_job_records
-      perform_with_advisory_lock(destroy_after: true)
-    end
-
     def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
       good_job = nil
       ActiveSupport::Notifications.instrument("enqueue_job.good_job", { active_job: active_job, scheduled_at: scheduled_at, create_with_advisory_lock: create_with_advisory_lock }) do |instrument_payload|
@@ -64,40 +73,49 @@ module GoodJob
       good_job
     end
 
-    def perform(destroy_after: true)
+    def perform(destroy_after: !GoodJob.preserve_job_records, reperform_on_standard_error: GoodJob.reperform_jobs_on_standard_error)
       raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
 
       result = nil
+      rescued_error = nil
       error = nil
 
       ActiveSupport::Notifications.instrument("before_perform_job.good_job", { good_job: self })
       self.performed_at = Time.current
       save! unless destroy_after
 
-      ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self }) do
-        params = serialized_params.merge(
-          "provider_job_id" => id
-        )
-        begin
+      params = serialized_params.merge(
+        "provider_job_id" => id
+      )
+
+      begin
+        ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self }) do
           result = ActiveJob::Base.execute(params)
-        rescue StandardError => e
-          error = e
         end
+      rescue StandardError => e
+        rescued_error = e
       end
 
-      if error.nil? && result.is_a?(Exception)
+      if rescued_error
+        error = rescued_error
+      elsif result.is_a?(Exception)
         error = result
         result = nil
       end
 
       error_message = "#{error.class}: #{error.message}" if error
       self.error = error_message
-      self.finished_at = Time.current
 
-      if destroy_after
-        destroy!
-      else
+      if rescued_error && reperform_on_standard_error
         save!
+      else
+        self.finished_at = Time.current
+
+        if destroy_after
+          destroy!
+        else
+          save!
+        end
       end
 
       [result, error]

data/lib/good_job/log_subscriber.rb

@@ -0,0 +1,110 @@
+module GoodJob
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def create(event)
+      good_job = event.payload[:good_job]
+
+      debug do
+        "GoodJob created job resource with id #{good_job.id}"
+      end
+    end
+
+    def timer_task_finished(event)
+      exception = event.payload[:error]
+      return unless exception
+
+      error do
+        "GoodJob error: #{exception}\n #{exception.backtrace}"
+      end
+    end
+
+    def job_finished(event)
+      exception = event.payload[:error]
+      return unless exception
+
+      error do
+        "GoodJob error: #{exception}\n #{exception.backtrace}"
+      end
+    end
+
+    def scheduler_create_pools(event)
+      max_threads = event.payload[:max_threads]
+      poll_interval = event.payload[:poll_interval]
+      performer_name = event.payload[:performer_name]
+      process_id = event.payload[:process_id]
+
+      info_and_stdout(tags: [process_id]) do
+        "GoodJob started scheduler with queues=#{performer_name} max_threads=#{max_threads} poll_interval=#{poll_interval}."
+      end
+    end
+
+    def scheduler_shutdown_start(event)
+      process_id = event.payload[:process_id]
+
+      info_and_stdout(tags: [process_id]) do
+        "GoodJob shutting down scheduler..."
+      end
+    end
+
+    def scheduler_shutdown(event)
+      process_id = event.payload[:process_id]
+
+      info_and_stdout(tags: [process_id]) do
+        "GoodJob scheduler is shut down."
+      end
+    end
+
+    def scheduler_restart_pools(event)
+      process_id = event.payload[:process_id]
+
+      info_and_stdout(tags: [process_id]) do
+        "GoodJob scheduler has restarted."
+      end
+    end
+
+    def cleanup_preserved_jobs(event)
+      timestamp = event.payload[:timestamp]
+      deleted_records_count = event.payload[:deleted_records_count]
+
+      info_and_stdout do
+        "GoodJob deleted #{deleted_records_count} preserved #{'job'.pluralize(deleted_records_count)} finished before #{timestamp}."
+      end
+    end
+
+    private
+
+    def logger
+      GoodJob.logger
+    end
+
+    %w(info debug warn error fatal unknown).each do |level|
+      class_eval <<-METHOD, __FILE__, __LINE__ + 1
+        def #{level}(progname = nil, tags: [], &block)
+          return unless logger
+
+          if logger.respond_to?(:tagged)
+            tags.unshift "GoodJob" unless logger.formatter.current_tags.include?("GoodJob")
+            logger.tagged(*tags.compact) do
+              logger.#{level}(progname, &block)
+            end
+          else
+            logger.#{level}(progname, &block)
+          end
+        end
+      METHOD
+    end
+
+    def info_and_stdout(progname = nil, tags: [], &block)
+      unless ActiveSupport::Logger.logger_outputs_to?(logger, STDOUT)
+        tags_string = (['GoodJob'] + tags).map { |t| "[#{t}]" }.join(' ')
+        stdout_message = "#{tags_string} #{yield}"
+        $stdout.puts stdout_message
+      end
+
+      info(progname, tags: [], &block)
+    end
+
+    def thread_name
+      Thread.current.name || Thread.current.object_id
+    end
+  end
+end