good_job 1.0.3 → 1.1.4

data/lib/good_job/multi_scheduler.rb

@@ -0,0 +1,34 @@
+module GoodJob
+  class MultiScheduler
+    attr_reader :schedulers
+
+    def initialize(schedulers)
+      @schedulers = schedulers
+    end
+
+    def shutdown(wait: true)
+      schedulers.each { |s| s.shutdown(wait: wait) }
+    end
+
+    def shutdown?
+      schedulers.all?(&:shutdown?)
+    end
+
+    def restart(wait: true)
+      schedulers.each { |s| s.restart(wait: wait) }
+    end
+
+    def create_thread(state = nil)
+      results = []
+      any_true = schedulers.any? do |scheduler|
+        scheduler.create_thread(state).tap { |result| results << result }
+      end
+
+      if any_true
+        true
+      else
+        results.any? { |result| result == false } ? false : nil
+      end
+    end
+  end
+end

data/lib/good_job/notifier.rb

@@ -0,0 +1,116 @@
+require 'concurrent/atomic/atomic_boolean'
+
+module GoodJob # :nodoc:
+  #
+  # Wrapper for Postgres LISTEN/NOTIFY
+  #
+  class Notifier
+    CHANNEL = 'good_job'.freeze
+    POOL_OPTIONS = {
+      min_threads: 0,
+      max_threads: 1,
+      auto_terminate: true,
+      idletime: 60,
+      max_queue: 1,
+      fallback_policy: :discard,
+    }.freeze
+    WAIT_INTERVAL = 1
+
+    # @!attribute [r] instances
+    #   @!scope class
+    #   @return [array<GoodJob:Adapter>] the instances of +GoodJob::Notifier+
+    cattr_reader :instances, default: [], instance_reader: false
+
+    def self.notify(message)
+      connection = ActiveRecord::Base.connection
+      connection.exec_query <<~SQL
+        NOTIFY #{CHANNEL}, #{connection.quote(message.to_json)}
+      SQL
+    end
+
+    attr_reader :recipients
+
+    def initialize(*recipients)
+      @recipients = Concurrent::Array.new(recipients)
+      @listening = Concurrent::AtomicBoolean.new(false)
+
+      self.class.instances << self
+
+      create_pool
+      listen
+    end
+
+    def listening?
+      @listening.true?
+    end
+
+    def restart(wait: true)
+      shutdown(wait: wait)
+      create_pool
+      listen
+    end
+
+    def shutdown(wait: true)
+      return unless @pool.running?
+
+      @pool.shutdown
+      @pool.wait_for_termination if wait
+    end
+
+    def shutdown?
+      !@pool.running?
+    end
+
+    private
+
+    def create_pool
+      @pool = Concurrent::ThreadPoolExecutor.new(POOL_OPTIONS)
+    end
+
+    def listen
+      future = Concurrent::Future.new(args: [@recipients, @pool, @listening], executor: @pool) do |recipients, pool, listening|
+        Rails.application.reloader.wrap do
+          conn = ActiveRecord::Base.connection.raw_connection
+          ActiveSupport::Notifications.instrument("notifier_listen.good_job") do
+            conn.async_exec "LISTEN #{CHANNEL}"
+          end
+
+          begin
+            ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+              while pool.running?
+                listening.make_true
+                conn.wait_for_notify(WAIT_INTERVAL) do |channel, _pid, payload|
+                  listening.make_false
+                  next unless channel == CHANNEL
+
+                  ActiveSupport::Notifications.instrument("notifier_notified.good_job", { payload: payload })
+                  parsed_payload = JSON.parse(payload, symbolize_names: true)
+                  recipients.each do |recipient|
+                    target, method_name = recipient.is_a?(Array) ? recipient : [recipient, :call]
+                    target.send(method_name, parsed_payload)
+                  end
+                end
+                listening.make_false
+              end
+            end
+          rescue StandardError => e
+            ActiveSupport::Notifications.instrument("notifier_notify_error.good_job", { error: e })
+            raise
+          ensure
+            @listening.make_false
+            ActiveSupport::Notifications.instrument("notifier_unlisten.good_job") do
+              conn.async_exec "UNLISTEN *"
+            end
+          end
+        end
+      end
+
+      future.add_observer(self, :listen_observer)
+      future.execute
+    end
+
+    def listen_observer(_time, _result, _thread_error)
+      listen unless shutdown?
+    end
+  end
+end

data/lib/good_job/performer.rb

@@ -1,12 +1,22 @@
 module GoodJob
   class Performer
-    def initialize(target, method_name)
+    attr_reader :name
+
+    def initialize(target, method_name, name: nil, filter: nil)
       @target = target
       @method_name = method_name
+      @name = name
+      @filter = filter
     end
 
     def next
       @target.public_send(@method_name)
     end
+
+    def next?(state = {})
+      return true unless @filter.respond_to?(:call)
+
+      @filter.call(state)
+    end
   end
 end

data/lib/good_job/railtie.rb

@@ -2,6 +2,17 @@ module GoodJob
   class Railtie < ::Rails::Railtie
     initializer "good_job.logger" do
       ActiveSupport.on_load(:good_job) { self.logger = ::Rails.logger }
+      GoodJob::LogSubscriber.attach_to :good_job
+    end
+
+    initializer "good_job.active_job_notifications" do
+      ActiveSupport::Notifications.subscribe "enqueue_retry.active_job" do |event|
+        GoodJob::CurrentExecution.error_on_retry = event.payload[:error]
+      end
+
+      ActiveSupport::Notifications.subscribe "discard.active_job" do |event|
+        GoodJob::CurrentExecution.error_on_discard = event.payload[:error]
+      end
     end
   end
 end

data/lib/good_job/scheduler.rb

@@ -2,16 +2,25 @@ require "concurrent/executor/thread_pool_executor"
 require "concurrent/timer_task"
 require "concurrent/utility/processor_counter"
 
-module GoodJob
+module GoodJob # :nodoc:
+  #
+  # Schedulers are generic thread execution pools that are responsible for
+  # periodically checking for available execution tasks, executing tasks in a
+  # bounded thread-pool, and efficiently scaling execution threads.
+  #
+  # Schedulers are "generic" in the sense that they delegate task execution
+  # details to a "Performer" object that responds to #next.
+  #
   class Scheduler
+    # Defaults for instance of Concurrent::TimerTask
     DEFAULT_TIMER_OPTIONS = {
       execution_interval: 1,
       timeout_interval: 1,
       run_now: true,
     }.freeze
 
+    # Defaults for instance of Concurrent::ThreadPoolExecutor
     DEFAULT_POOL_OPTIONS = {
-      name: 'good_job',
       min_threads: 0,
       max_threads: Concurrent.processor_count,
       auto_terminate: true,
@@ -20,44 +29,113 @@ module GoodJob
       fallback_policy: :discard,
     }.freeze
 
+    # @!attribute [r] instances
+    #   @!scope class
+    #   All instantiated Schedulers in the current process.
+    #   @return [array<GoodJob:Scheduler>]
+    cattr_reader :instances, default: [], instance_reader: false
+
+    # Creates GoodJob::Scheduler(s) and Performers from a GoodJob::Configuration instance.
+    # @param configuration [GoodJob::Configuration]
+    # @return [GoodJob::Scheduler, GoodJob::MultiScheduler]
+    def self.from_configuration(configuration)
+      schedulers = configuration.queue_string.split(';').map do |queue_string_and_max_threads|
+        queue_string, max_threads = queue_string_and_max_threads.split(':')
+        max_threads = (max_threads || configuration.max_threads).to_i
+
+        job_query = GoodJob::Job.queue_string(queue_string)
+        parsed = GoodJob::Job.queue_parser(queue_string)
+        job_filter = proc do |state|
+          if parsed[:exclude]
+            !parsed[:exclude].include? state[:queue_name]
+          elsif parsed[:include]
+            parsed[:include].include? state[:queue_name]
+          else
+            true
+          end
+        end
+        job_performer = GoodJob::Performer.new(job_query, :perform_with_advisory_lock, name: queue_string, filter: job_filter)
+
+        timer_options = {}
+        timer_options[:execution_interval] = configuration.poll_interval
+
+        pool_options = {
+          max_threads: max_threads,
+        }
+
+        GoodJob::Scheduler.new(job_performer, timer_options: timer_options, pool_options: pool_options)
+      end
+
+      if schedulers.size > 1
+        GoodJob::MultiScheduler.new(schedulers)
+      else
+        schedulers.first
+      end
+    end
+
+    # @param performer [GoodJob::Performer]
+    # @param timer_options [Hash] Options to instantiate a Concurrent::TimerTask
+    # @param pool_options [Hash] Options to instantiate a Concurrent::ThreadPoolExecutor
     def initialize(performer, timer_options: {}, pool_options: {})
       raise ArgumentError, "Performer argument must implement #next" unless performer.respond_to?(:next)
 
+      self.class.instances << self
+
       @performer = performer
-      @pool = ThreadPoolExecutor.new(DEFAULT_POOL_OPTIONS.merge(pool_options))
-      @timer = Concurrent::TimerTask.new(DEFAULT_TIMER_OPTIONS.merge(timer_options)) do
-        create_thread
-      end
-      @timer.add_observer(self, :timer_observer)
-      @timer.execute
-    end
+      @pool_options = DEFAULT_POOL_OPTIONS.merge(pool_options)
+      @timer_options = DEFAULT_TIMER_OPTIONS.merge(timer_options)
 
-    def execute
+      @pool_options[:name] = "GoodJob::Scheduler(queues=#{@performer.name} max_threads=#{@pool_options[:max_threads]} poll_interval=#{@timer_options[:execution_interval]})"
+
+      create_pools
     end
 
+    # Shut down the Scheduler.
+    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @return [void]
     def shutdown(wait: true)
       @_shutdown = true
 
-      ActiveSupport::Notifications.instrument("scheduler_start_shutdown.good_job", { wait: wait })
-      ActiveSupport::Notifications.instrument("scheduler_shutdown.good_job", { wait: wait }) do
-        if @timer.running?
+      instrument("scheduler_shutdown_start", { wait: wait })
+      instrument("scheduler_shutdown", { wait: wait }) do
+        if @timer&.running?
           @timer.shutdown
           @timer.wait_for_termination if wait
         end
 
-        if @pool.running?
+        if @pool&.running?
           @pool.shutdown
           @pool.wait_for_termination if wait
         end
       end
     end
 
+    # True when the Scheduler is shutdown.
+    # @return [true, false, nil]
     def shutdown?
       @_shutdown
     end
 
-    def create_thread
-      return false unless @pool.ready_worker_count.positive?
+    # Restart the Scheduler. When shutdown, start; or shutdown and start.
+    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @return [void]
+    def restart(wait: true)
+      instrument("scheduler_restart_pools") do
+        shutdown(wait: wait) unless shutdown?
+        create_pools
+        @_shutdown = false
+      end
+    end
+
+    # Triggers a Performer execution, if an execution thread is available.
+    # @param state [nil, Object] Allows Performer#next? to accept or reject the execution
+    # @return [nil, Boolean] if the thread was created
+    def create_thread(state = nil)
+      return nil unless @pool.running? && @pool.ready_worker_count.positive?
+
+      if state
+        return false unless @performer.next?(state)
+      end
 
       future = Concurrent::Future.new(args: [@performer], executor: @pool) do |performer|
         output = nil
@@ -66,26 +144,63 @@ module GoodJob
       end
       future.add_observer(self, :task_observer)
       future.execute
+
+      true
     end
 
+    # Invoked on completion of TimerTask task.
+    # @!visibility private
+    # @return [void]
     def timer_observer(time, executed_task, thread_error)
-      ActiveSupport::Notifications.instrument("finished_timer_task.good_job", { result: executed_task, error: thread_error, time: time })
+      GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
+      instrument("finished_timer_task", { result: executed_task, error: thread_error, time: time })
     end
 
+    # Invoked on completion of ThreadPoolExecutor task
+    # @!visibility private
+    # @return [void]
     def task_observer(time, output, thread_error)
-      ActiveSupport::Notifications.instrument("finished_job_task.good_job", { result: output, error: thread_error, time: time })
+      GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
+      instrument("finished_job_task", { result: output, error: thread_error, time: time })
       create_thread if output
     end
 
-    class ThreadPoolExecutor < Concurrent::ThreadPoolExecutor
-      # https://github.com/ruby-concurrency/concurrent-ruby/issues/684#issuecomment-427594437
-      def ready_worker_count
-        synchronize do
-          workers_still_to_be_created = @max_length - @pool.length
-          workers_created_but_waiting = @ready.length
+    private
 
-          workers_still_to_be_created + workers_created_but_waiting
-        end
+    # @return [void]
+    def create_pools
+      instrument("scheduler_create_pools", { performer_name: @performer.name, max_threads: @pool_options[:max_threads], poll_interval: @timer_options[:execution_interval] }) do
+        @pool = ThreadPoolExecutor.new(@pool_options)
+        next unless @timer_options[:execution_interval].positive?
+
+        @timer = Concurrent::TimerTask.new(@timer_options) { create_thread }
+        @timer.add_observer(self, :timer_observer)
+        @timer.execute
+      end
+    end
+
+    def instrument(name, payload = {}, &block)
+      payload = payload.reverse_merge({
+                                        scheduler: self,
+                                        process_id: GoodJob::CurrentExecution.process_id,
+                                        thread_name: GoodJob::CurrentExecution.thread_name,
+                                      })
+
+      ActiveSupport::Notifications.instrument("#{name}.good_job", payload, &block)
+    end
+  end
+
+  # Slightly customized sub-class of Concurrent::ThreadPoolExecutor
+  class ThreadPoolExecutor < Concurrent::ThreadPoolExecutor
+    # Number of idle or potential threads available to execute tasks
+    # https://github.com/ruby-concurrency/concurrent-ruby/issues/684#issuecomment-427594437
+    # @return [Integer]
+    def ready_worker_count
+      synchronize do
+        workers_still_to_be_created = @max_length - @pool.length
+        workers_created_but_waiting = @ready.length
+
+        workers_still_to_be_created + workers_created_but_waiting
       end
     end
   end

data/lib/good_job/version.rb

@@ -1,3 +1,3 @@
 module GoodJob
-  VERSION = '1.0.3'.freeze
+  VERSION = '1.1.4'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.4
 platform: ruby
 authors:
 - Ben Sheldon
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-26 00:00:00.000000000 Z
+date: 2020-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: foreman
   requirement: !ruby/object:Gem::Requirement
@@ -137,7 +151,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: pry-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rbtrace
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -220,6 +262,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sigdump
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A multithreaded, Postgres-based ActiveJob backend for Ruby on Rails
 email:
 - bensheldon@gmail.com
@@ -241,9 +311,13 @@ files:
 - lib/good_job.rb
 - lib/good_job/adapter.rb
 - lib/good_job/cli.rb
+- lib/good_job/configuration.rb
+- lib/good_job/current_execution.rb
 - lib/good_job/job.rb
 - lib/good_job/lockable.rb
-- lib/good_job/logging.rb
+- lib/good_job/log_subscriber.rb
+- lib/good_job/multi_scheduler.rb
+- lib/good_job/notifier.rb
 - lib/good_job/performer.rb
 - lib/good_job/pg_locks.rb
 - lib/good_job/railtie.rb
@@ -258,7 +332,7 @@ metadata:
   documentation_uri: https://rdoc.info/github/bensheldon/good_job
   homepage_uri: https://github.com/bensheldon/good_job
   source_code_uri: https://github.com/bensheldon/good_job
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--title"
 - GoodJob - a multithreaded, Postgres-based ActiveJob backend for Ruby on Rails
@@ -281,7 +355,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A multithreaded, Postgres-based ActiveJob backend for Ruby on Rails
 test_files: []