good_job 1.3.5 → 1.6.0

data/lib/active_job/queue_adapters/good_job_adapter.rb

@@ -2,9 +2,9 @@ module ActiveJob # :nodoc:
   module QueueAdapters # :nodoc:
     # See {GoodJob::Adapter} for details.
     class GoodJobAdapter < GoodJob::Adapter
-      def initialize(execution_mode: nil, max_threads: nil, poll_interval: nil, scheduler: nil, inline: false)
-        configuration = GoodJob::Configuration.new({ execution_mode: execution_mode }, env: ENV)
-        super(execution_mode: configuration.rails_execution_mode, max_threads: max_threads, poll_interval: poll_interval, scheduler: scheduler, inline: inline)
+      def initialize(**options)
+        configuration = GoodJob::Configuration.new(options, env: ENV)
+        super(**options.merge(execution_mode: configuration.rails_execution_mode))
       end
     end
   end

data/lib/good_job.rb

@@ -1,16 +1,15 @@
 require "rails"
-
 require "active_job"
 require "active_job/queue_adapters"
 
 require "zeitwerk"
-
-loader = Zeitwerk::Loader.for_gem
-loader.inflector.inflect(
-  'cli' => "CLI"
-)
-loader.push_dir(File.join(__dir__, ["generators"]))
-loader.setup
+Zeitwerk::Loader.for_gem.tap do |loader|
+  loader.inflector.inflect({
+                             "cli" => "CLI",
+                           })
+  loader.ignore(File.join(File.dirname(__FILE__), "generators"))
+  loader.setup
+end
 
 require "good_job/railtie"
 

data/lib/good_job/adapter.rb

@@ -20,13 +20,22 @@ module GoodJob
     # @param max_threads [nil, Integer] sets the number of threads per scheduler to use when +execution_mode+ is set to +:async+. The +queues+ parameter can specify a number of threads for each group of queues which will override this value. You can also set this with the environment variable +GOOD_JOB_MAX_THREADS+. Defaults to +5+.
     # @param queues [nil, String] determines which queues to execute jobs from when +execution_mode+ is set to +:async+. See {file:README.md#optimize-queues-threads-and-processes} for more details on the format of this string. You can also set this with the environment variable +GOOD_JOB_QUEUES+. Defaults to +"*"+.
     # @param poll_interval [nil, Integer] sets the number of seconds between polls for jobs when +execution_mode+ is set to +:async+. You can also set this with the environment variable +GOOD_JOB_POLL_INTERVAL+. Defaults to +1+.
-    # @param scheduler [nil, Scheduler] (deprecated) a scheduler to be managed by the adapter
-    # @param notifier [nil, Notifier] (deprecated) a notifier to be managed by the adapter
-    # @param inline [nil, Boolean] (deprecated) whether to run in inline execution mode
-    def initialize(execution_mode: nil, queues: nil, max_threads: nil, poll_interval: nil, scheduler: nil, notifier: nil, inline: false)
-      if inline && execution_mode.nil?
-        ActiveSupport::Deprecation.warn('GoodJob::Adapter#new(inline: true) is deprecated; use GoodJob::Adapter.new(execution_mode: :inline) instead')
-        execution_mode = :inline
+    def initialize(execution_mode: nil, queues: nil, max_threads: nil, poll_interval: nil)
+      if caller[0..4].find { |c| c.include?("/config/application.rb") || c.include?("/config/environments/") }
+        ActiveSupport::Deprecation.warn(<<~DEPRECATION)
+          GoodJob no longer recommends creating a GoodJob::Adapter instance:
+
+              config.active_job.queue_adapter = GoodJob::Adapter.new...
+
+          Instead, configure GoodJob through configuration:
+
+              config.active_job.queue_adapter = :good_job
+              config.good_job.execution_mode = :#{execution_mode}
+              config.good_job.max_threads = #{max_threads}
+              config.good_job.poll_interval = #{poll_interval}
+              # etc...
+
+        DEPRECATION
       end
 
       configuration = GoodJob::Configuration.new(
@@ -42,9 +51,9 @@ module GoodJob
       raise ArgumentError, "execution_mode: must be one of #{EXECUTION_MODES.join(', ')}." unless EXECUTION_MODES.include?(@execution_mode)
 
       if @execution_mode == :async # rubocop:disable Style/GuardClause
-        @notifier = notifier || GoodJob::Notifier.new
+        @notifier = GoodJob::Notifier.new
         @poller = GoodJob::Poller.new(poll_interval: configuration.poll_interval)
-        @scheduler = scheduler || GoodJob::Scheduler.from_configuration(configuration)
+        @scheduler = GoodJob::Scheduler.from_configuration(configuration)
         @notifier.recipients << [@scheduler, :create_thread]
         @poller.recipients << [@scheduler, :create_thread]
       end
@@ -108,11 +117,5 @@ module GoodJob
     def execute_inline?
       @execution_mode == :inline
     end
-
-    # (deprecated) Whether in +:inline+ execution mode.
-    def inline?
-      ActiveSupport::Deprecation.warn('GoodJob::Adapter::inline? is deprecated; use GoodJob::Adapter::execute_inline? instead')
-      execute_inline?
-    end
   end
 end

data/lib/good_job/cli.rb

@@ -15,6 +15,11 @@ module GoodJob
     # Requiring this loads the application's configuration and classes.
     RAILS_ENVIRONMENT_RB = File.expand_path("config/environment.rb")
 
+    # @!visibility private
+    def self.exit_on_failure?
+      true
+    end
+
     # @!macro thor.desc
     #   @!method $1
     #   @return [void]
@@ -27,7 +32,8 @@ module GoodJob
       See option descriptions for the matching environment variable name.
 
       == Configuring queues
-      \x5Separate multiple queues with commas; exclude queues with a leading minus;
+
+      Separate multiple queues with commas; exclude queues with a leading minus;
       separate isolated execution pools with semicolons and threads with colons.
 
     DESCRIPTION
@@ -43,10 +49,18 @@ module GoodJob
                   type: :numeric,
                   banner: 'SECONDS',
                   desc: "Interval between polls for available jobs in seconds (env var: GOOD_JOB_POLL_INTERVAL, default: 5)"
+    method_option :daemonize,
+                  type: :boolean,
+                  desc: "Run as a background daemon (default: false)"
+    method_option :pidfile,
+                  type: :string,
+                  desc: "Path to write daemonized Process ID (env var: GOOD_JOB_PIDFILE, default: tmp/pids/good_job.pid)"
     def start
       set_up_application!
       configuration = GoodJob::Configuration.new(options)
 
+      Daemon.new(pidfile: configuration.pidfile).daemonize if configuration.daemonize?
+
       notifier = GoodJob::Notifier.new
       poller = GoodJob::Poller.new(poll_interval: configuration.poll_interval)
       scheduler = GoodJob::Scheduler.from_configuration(configuration)

data/lib/good_job/configuration.rb

@@ -12,16 +12,15 @@ module GoodJob
     # Default number of seconds to preserve jobs for {CLI#cleanup_preserved_jobs}
     DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO = 24 * 60 * 60
 
-    # @!attribute [r] options
-    #   The options that were explicitly set when initializing +Configuration+.
-    #   @return [Hash]
-    #
-    # @!attribute [r] env
-    #   The environment from which to read GoodJob's environment variables. By
-    #   default, this is the current process's environment, but it can be set
-    #   to something else in {#initialize}.
-    #   @return [Hash]
-    attr_reader :options, :env
+    # The options that were explicitly set when initializing +Configuration+.
+    # @return [Hash]
+    attr_reader :options
+
+    # The environment from which to read GoodJob's environment variables. By
+    # default, this is the current process's environment, but it can be set
+    # to something else in {#initialize}.
+    # @return [Hash]
+    attr_reader :env
 
     # @param options [Hash] Any explicitly specified configuration options to
     #   use. Keys are symbols that match the various methods on this class.
@@ -45,6 +44,8 @@ module GoodJob
     def execution_mode(default: :external)
       if options[:execution_mode]
         options[:execution_mode]
+      elsif rails_config[:execution_mode]
+        rails_config[:execution_mode]
       elsif env['GOOD_JOB_EXECUTION_MODE'].present?
         env['GOOD_JOB_EXECUTION_MODE'].to_sym
       else
@@ -58,9 +59,7 @@ module GoodJob
     def rails_execution_mode
       if execution_mode(default: nil)
         execution_mode
-      elsif Rails.env.development?
-        :inline
-      elsif Rails.env.test?
+      elsif Rails.env.development? || Rails.env.test?
         :inline
       else
         :external
@@ -74,6 +73,7 @@ module GoodJob
     def max_threads
       (
         options[:max_threads] ||
+        rails_config[:max_threads] ||
         env['GOOD_JOB_MAX_THREADS'] ||
         env['RAILS_MAX_THREADS'] ||
         DEFAULT_MAX_THREADS
@@ -87,6 +87,7 @@ module GoodJob
     # @return [String]
     def queue_string
       options[:queues] ||
+        rails_config[:queues] ||
         env['GOOD_JOB_QUEUES'] ||
         '*'
     end
@@ -98,17 +99,42 @@ module GoodJob
     def poll_interval
       (
         options[:poll_interval] ||
+        rails_config[:poll_interval] ||
         env['GOOD_JOB_POLL_INTERVAL'] ||
         DEFAULT_POLL_INTERVAL
       ).to_i
     end
 
+    # Number of seconds to preserve jobs when using the +good_job cleanup_preserved_jobs+ CLI command.
+    # This configuration is only used when {GoodJob.preserve_job_records} is +true+.
+    # @return [Integer]
     def cleanup_preserved_jobs_before_seconds_ago
       (
         options[:before_seconds_ago] ||
+        rails_config[:cleanup_preserved_jobs_before_seconds_ago] ||
         env['GOOD_JOB_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO'] ||
         DEFAULT_CLEANUP_PRESERVED_JOBS_BEFORE_SECONDS_AGO
       ).to_i
     end
+
+    # Tests whether to daemonize the process.
+    # @return [Boolean]
+    def daemonize?
+      options[:daemonize] || false
+    end
+
+    # Path of the pidfile to create when running as a daemon.
+    # @return [Pathname,String]
+    def pidfile
+      options[:pidfile] ||
+        env['GOOD_JOB_PIDFILE'] ||
+        Rails.application.root.join('tmp', 'pids', 'good_job.pid')
+    end
+
+    private
+
+    def rails_config
+      Rails.application.config.good_job
+    end
   end
 end

data/lib/good_job/daemon.rb

@@ -0,0 +1,59 @@
+module GoodJob
+  #
+  # Manages daemonization of the current process.
+  #
+  class Daemon
+    # The path of the generated pidfile.
+    # @return [Pathname,String]
+    attr_reader :pidfile
+
+    # @param pidfile [Pathname,String] Pidfile path
+    def initialize(pidfile:)
+      @pidfile = pidfile
+    end
+
+    # Daemonizes the current process and writes out a pidfile.
+    def daemonize
+      check_pid
+      Process.daemon
+      write_pid
+    end
+
+    private
+
+    def write_pid
+      File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(Process.pid.to_s) }
+      at_exit { File.delete(pidfile) if File.exist?(pidfile) }
+    rescue Errno::EEXIST
+      check_pid
+      retry
+    end
+
+    def delete_pid
+      File.delete(pidfile) if File.exist?(pidfile)
+    end
+
+    def check_pid
+      case pid_status(pidfile)
+      when :running, :not_owned
+        abort "A server is already running. Check #{pidfile}"
+      when :dead
+        File.delete(pidfile)
+      end
+    end
+
+    def pid_status(pidfile)
+      return :exited unless File.exist?(pidfile)
+
+      pid = ::File.read(pidfile).to_i
+      return :dead if pid.zero?
+
+      Process.kill(0, pid) # check process status
+      :running
+    rescue Errno::ESRCH
+      :dead
+    rescue Errno::EPERM
+      :not_owned
+    end
+  end
+end

data/lib/good_job/job.rb

@@ -139,7 +139,7 @@ module GoodJob
       unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
         good_job = good_jobs.first
         # TODO: Determine why some records are fetched without an advisory lock at all
-        break unless good_job&.owns_advisory_lock?
+        break unless good_job&.executable?
 
         result, error = good_job.perform
       end
@@ -216,6 +216,12 @@ module GoodJob
       [result, job_error]
     end
 
+    # Tests whether this job is safe to be executed by this thread.
+    # @return [Boolean]
+    def executable?
+      self.class.unscoped.unfinished.owns_advisory_locked.exists?(id: id)
+    end
+
     private
 
     def execute

data/lib/good_job/job_performer.rb

@@ -0,0 +1,63 @@
+require 'concurrent/delay'
+
+module GoodJob
+  #
+  # JobPerformer queries the database for jobs and performs them on behalf of a
+  # {Scheduler}. It mainly functions as glue between a {Scheduler} and the jobs
+  # it should be executing.
+  #
+  # The JobPerformer must be safe to execute across multiple threads.
+  #
+  class JobPerformer
+    # @param queue_string [String] Queues to execute jobs from
+    def initialize(queue_string)
+      @queue_string = queue_string
+
+      @job_query = Concurrent::Delay.new { GoodJob::Job.queue_string(queue_string) }
+      @parsed_queues = Concurrent::Delay.new { GoodJob::Job.queue_parser(queue_string) }
+    end
+
+    # A meaningful name to identify the performer in logs and for debugging.
+    # @return [String] The queues from which Jobs are worked
+    def name
+      @queue_string
+    end
+
+    # Perform the next eligible job
+    # @return [nil, Object] Returns job result or +nil+ if no job was found
+    def next
+      job_query.perform_with_advisory_lock
+    end
+
+    # Tests whether this performer should be used in GoodJob's current state.
+    #
+    # For example, state will be a LISTEN/NOTIFY message that is passed down
+    # from the Notifier to the Scheduler. The Scheduler is able to ask
+    # its performer "does this message relate to you?", and if not, ignore it
+    # to minimize thread wake-ups, database queries, and thundering herds.
+    #
+    # @return [Boolean] whether the performer's {#next} method should be
+    #   called in the current state.
+    def next?(state = {})
+      if parsed_queues[:exclude]
+        parsed_queues[:exclude].exclude?(state[:queue_name])
+      elsif parsed_queues[:include]
+        parsed_queues[:include].include?(state[:queue_name])
+      else
+        true
+      end
+    end
+
+    private
+
+    attr_reader :queue_string
+
+    def job_query
+      @job_query.value
+    end
+
+    def parsed_queues
+      @parsed_queues.value
+    end
+  end
+end

data/lib/good_job/lockable.rb

@@ -92,8 +92,6 @@ module GoodJob
       # @return [ActiveRecord::Relation]
       scope :owns_advisory_locked, -> { joins_advisory_locks.where('"pg_locks"."pid" = pg_backend_pid()') }
 
-      # @!attribute [r] create_with_advisory_lock
-      # @return [Boolean]
       # Whether an advisory lock should be acquired in the same transaction
       # that created the record.
       #
@@ -107,6 +105,8 @@ module GoodJob
       #   record = MyLockableRecord.create(create_with_advisory_lock: true)
       #   record.advisory_locked?
       #   => true
+      #
+      # @return [Boolean]
       attr_accessor :create_with_advisory_lock
 
       after_create -> { advisory_lock }, if: :create_with_advisory_lock
@@ -141,9 +141,9 @@ module GoodJob
       end
 
       def supports_cte_materialization_specifiers?
-        return @supports_cte_materialization_specifiers if defined?(@supports_cte_materialization_specifiers)
+        return @_supports_cte_materialization_specifiers if defined?(@_supports_cte_materialization_specifiers)
 
-        @supports_cte_materialization_specifiers = ActiveRecord::Base.connection.postgresql_version >= 120000
+        @_supports_cte_materialization_specifiers = ActiveRecord::Base.connection.postgresql_version >= 120000
       end
     end
 
@@ -153,10 +153,12 @@ module GoodJob
     # all remaining locks).
     # @return [Boolean] whether the lock was acquired.
     def advisory_lock
-      where_sql = <<~SQL.squish
-        pg_try_advisory_lock(('x' || substr(md5(:table_name || :id::text), 1, 16))::bit(64)::bigint)
+      query = <<~SQL.squish
+        SELECT 1 AS one
+        WHERE pg_try_advisory_lock(('x'||substr(md5($1 || $2::text), 1, 16))::bit(64)::bigint)
       SQL
-      self.class.unscoped.exists?([where_sql, { table_name: self.class.table_name, id: send(self.class.primary_key) }])
+      binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)]]
+      ActiveRecord::Base.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Lock', binds).any?
     end
 
     # Releases an advisory lock on this record if it is locked by this database
@@ -166,9 +168,10 @@ module GoodJob
     def advisory_unlock
       query = <<~SQL.squish
         SELECT 1 AS one
-        WHERE pg_advisory_unlock(('x'||substr(md5(:table_name || :id::text), 1, 16))::bit(64)::bigint)
+        WHERE pg_advisory_unlock(('x'||substr(md5($1 || $2::text), 1, 16))::bit(64)::bigint)
       SQL
-      self.class.connection.execute(sanitize_sql_for_conditions([query, { table_name: self.class.table_name, id: send(self.class.primary_key) }])).ntuples.positive?
+      binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)]]
+      self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Unlock', binds).any?
     end
 
     # Acquires an advisory lock on this record or raises
@@ -205,13 +208,32 @@ module GoodJob
     # Tests whether this record has an advisory lock on it.
     # @return [Boolean]
     def advisory_locked?
-      self.class.unscoped.advisory_locked.exists?(id: send(self.class.primary_key))
+      query = <<~SQL.squish
+        SELECT 1 AS one
+        FROM pg_locks
+        WHERE pg_locks.locktype = 'advisory'
+          AND pg_locks.objsubid = 1
+          AND pg_locks.classid = ('x' || substr(md5($1 || $2::text), 1, 16))::bit(32)::int
+          AND pg_locks.objid = (('x' || substr(md5($3 || $4::text), 1, 16))::bit(64) << 32)::bit(32)::int
+      SQL
+      binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)], [nil, self.class.table_name], [nil, send(self.class.primary_key)]]
+      self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Locked?', binds).any?
     end
 
     # Tests whether this record is locked by the current database session.
     # @return [Boolean]
     def owns_advisory_lock?
-      self.class.unscoped.owns_advisory_locked.exists?(id: send(self.class.primary_key))
+      query = <<~SQL.squish
+        SELECT 1 AS one
+        FROM pg_locks
+        WHERE pg_locks.locktype = 'advisory'
+          AND pg_locks.objsubid = 1
+          AND pg_locks.classid = ('x' || substr(md5($1 || $2::text), 1, 16))::bit(32)::int
+          AND pg_locks.objid = (('x' || substr(md5($3 || $4::text), 1, 16))::bit(64) << 32)::bit(32)::int
+          AND pg_locks.pid = pg_backend_pid()
+      SQL
+      binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)], [nil, self.class.table_name], [nil, send(self.class.primary_key)]]
+      self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Owns Advisory Lock?', binds).any?
     end
 
     # Releases all advisory locks on the record that are held by the current
@@ -227,5 +249,14 @@ module GoodJob
       # Made public in Rails 5.2
       self.class.send(:sanitize_sql_for_conditions, *args)
     end
+
+    def pg_or_jdbc_query(query)
+      if Concurrent.on_jruby?
+        # Replace $1 bind parameters with ?
+        query.gsub(/\$\d*/, '?')
+      else
+        query
+      end
+    end
   end
 end