good_job 1.0.2 → 1.1.3

data/lib/good_job/multi_scheduler.rb

@@ -0,0 +1,25 @@
+module GoodJob
+  class MultiScheduler
+    attr_reader :schedulers
+
+    def initialize(schedulers)
+      @schedulers = schedulers
+    end
+
+    def shutdown(wait: true)
+      schedulers.each { |s| s.shutdown(wait: wait) }
+    end
+
+    def shutdown?
+      schedulers.all?(&:shutdown?)
+    end
+
+    def restart(wait: true)
+      schedulers.each { |s| s.restart(wait: wait) }
+    end
+
+    def create_thread
+      schedulers.all?(&:create_thread)
+    end
+  end
+end

data/lib/good_job/performer.rb

@@ -1,8 +1,11 @@
 module GoodJob
   class Performer
-    def initialize(target, method_name)
+    attr_reader :name
+
+    def initialize(target, method_name, name: nil)
       @target = target
       @method_name = method_name
+      @name = name
     end
 
     def next

data/lib/good_job/railtie.rb

@@ -2,6 +2,17 @@ module GoodJob
   class Railtie < ::Rails::Railtie
     initializer "good_job.logger" do
       ActiveSupport.on_load(:good_job) { self.logger = ::Rails.logger }
+      GoodJob::LogSubscriber.attach_to :good_job
+    end
+
+    initializer "good_job.active_job_notifications" do
+      ActiveSupport::Notifications.subscribe "enqueue_retry.active_job" do |event|
+        GoodJob::CurrentExecution.error_on_retry = event.payload[:error]
+      end
+
+      ActiveSupport::Notifications.subscribe "discard.active_job" do |event|
+        GoodJob::CurrentExecution.error_on_discard = event.payload[:error]
+      end
     end
   end
 end

data/lib/good_job/scheduler.rb

@@ -2,14 +2,22 @@ require "concurrent/executor/thread_pool_executor"
 require "concurrent/timer_task"
 require "concurrent/utility/processor_counter"
 
-module GoodJob
+module GoodJob # :nodoc:
+  # Schedulers are generic thread execution pools that are responsible for
+  # periodically checking for available execution tasks, executing tasks in a
+  # bounded thread-pool, and efficiently scaling execution threads.
+  #
+  # Schedulers are "generic" in the sense that they delegate task execution
+  # details to a "Performer" object that responds to #next.
   class Scheduler
+    # Defaults for instance of Concurrent::TimerTask
     DEFAULT_TIMER_OPTIONS = {
       execution_interval: 1,
       timeout_interval: 1,
       run_now: true,
     }.freeze
 
+    # Defaults for instance of Concurrent::ThreadPoolExecutor
     DEFAULT_POOL_OPTIONS = {
       name: 'good_job',
       min_threads: 0,
@@ -20,42 +28,93 @@ module GoodJob
       fallback_policy: :discard,
     }.freeze
 
+    # @!attribute [r] instances
+    #   @!scope class
+    #   All instantiated Schedulers in the current process.
+    #   @return [array<GoodJob:Scheduler>]
+    cattr_reader :instances, default: [], instance_reader: false
+
+    # Creates GoodJob::Scheduler(s) and Performers from a GoodJob::Configuration instance.
+    # @param configuration [GoodJob::Configuration]
+    # @return [GoodJob::Scheduler, GoodJob::MultiScheduler]
+    def self.from_configuration(configuration)
+      schedulers = configuration.queue_string.split(';').map do |queue_string_and_max_threads|
+        queue_string, max_threads = queue_string_and_max_threads.split(':')
+        max_threads = (max_threads || configuration.max_threads).to_i
+
+        job_query = GoodJob::Job.queue_string(queue_string)
+        job_performer = GoodJob::Performer.new(job_query, :perform_with_advisory_lock, name: queue_string)
+
+        timer_options = {}
+        timer_options[:execution_interval] = configuration.poll_interval if configuration.poll_interval.positive?
+
+        pool_options = {
+          max_threads: max_threads,
+        }
+
+        GoodJob::Scheduler.new(job_performer, timer_options: timer_options, pool_options: pool_options)
+      end
+
+      if schedulers.size > 1
+        GoodJob::MultiScheduler.new(schedulers)
+      else
+        schedulers.first
+      end
+    end
+
+    # @param performer [GoodJob::Performer]
+    # @param timer_options [Hash] Options to instantiate a Concurrent::TimerTask
+    # @param pool_options [Hash] Options to instantiate a Concurrent::ThreadPoolExecutor
     def initialize(performer, timer_options: {}, pool_options: {})
       raise ArgumentError, "Performer argument must implement #next" unless performer.respond_to?(:next)
 
+      self.class.instances << self
+
       @performer = performer
-      @pool = ThreadPoolExecutor.new(DEFAULT_POOL_OPTIONS.merge(pool_options))
-      @timer = Concurrent::TimerTask.new(DEFAULT_TIMER_OPTIONS.merge(timer_options)) do
-        create_thread
-      end
-      @timer.add_observer(self, :timer_observer)
-      @timer.execute
-    end
+      @pool_options = DEFAULT_POOL_OPTIONS.merge(pool_options)
+      @timer_options = DEFAULT_TIMER_OPTIONS.merge(timer_options)
 
-    def execute
+      create_pools
     end
 
+    # Shut down the Scheduler.
+    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @return [void]
     def shutdown(wait: true)
       @_shutdown = true
 
-      ActiveSupport::Notifications.instrument("scheduler_start_shutdown.good_job", { wait: wait })
-      ActiveSupport::Notifications.instrument("scheduler_shutdown.good_job", { wait: wait }) do
-        if @timer.running?
+      ActiveSupport::Notifications.instrument("scheduler_shutdown_start.good_job", { wait: wait, process_id: process_id })
+      ActiveSupport::Notifications.instrument("scheduler_shutdown.good_job", { wait: wait, process_id: process_id }) do
+        if @timer&.running?
           @timer.shutdown
           @timer.wait_for_termination if wait
         end
 
-        if @pool.running?
+        if @pool&.running?
           @pool.shutdown
           @pool.wait_for_termination if wait
         end
       end
     end
 
+    # True when the Scheduler is shutdown.
+    # @return [true, false, nil]
     def shutdown?
       @_shutdown
     end
 
+    # Restart the Scheduler. When shutdown, start; or shutdown and start.
+    # @param wait [Boolean] Wait for actively executing jobs to finish
+    # @return [void]
+    def restart(wait: true)
+      ActiveSupport::Notifications.instrument("scheduler_restart_pools.good_job", { process_id: process_id }) do
+        shutdown(wait: wait) unless shutdown?
+        create_pools
+      end
+    end
+
+    # Triggers the execution the Performer, if an execution thread is available.
+    # @return [Boolean]
     def create_thread
       return false unless @pool.ready_worker_count.positive?
 
@@ -66,26 +125,62 @@ module GoodJob
       end
       future.add_observer(self, :task_observer)
       future.execute
+      true
     end
 
+    # Invoked on completion of TimerTask task.
+    # @!visibility private
+    # @return [void]
     def timer_observer(time, executed_task, thread_error)
+      GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
       ActiveSupport::Notifications.instrument("finished_timer_task.good_job", { result: executed_task, error: thread_error, time: time })
     end
 
+    # Invoked on completion of ThreadPoolExecutor task
+    # @!visibility private
+    # @return [void]
     def task_observer(time, output, thread_error)
+      GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
       ActiveSupport::Notifications.instrument("finished_job_task.good_job", { result: output, error: thread_error, time: time })
       create_thread if output
     end
 
-    class ThreadPoolExecutor < Concurrent::ThreadPoolExecutor
-      # https://github.com/ruby-concurrency/concurrent-ruby/issues/684#issuecomment-427594437
-      def ready_worker_count
-        synchronize do
-          workers_still_to_be_created = @max_length - @pool.length
-          workers_created_but_waiting = @ready.length
+    private
 
-          workers_still_to_be_created + workers_created_but_waiting
-        end
+    # @return [void]
+    def create_pools
+      ActiveSupport::Notifications.instrument("scheduler_create_pools.good_job", { performer_name: @performer.name, max_threads: @pool_options[:max_threads], poll_interval: @timer_options[:execution_interval], process_id: process_id }) do
+        @pool = ThreadPoolExecutor.new(@pool_options)
+        next unless @timer_options[:execution_interval].positive?
+
+        @timer = Concurrent::TimerTask.new(@timer_options) { create_thread }
+        @timer.add_observer(self, :timer_observer)
+        @timer.execute
+      end
+    end
+
+    # @return [Integer] Current process ID
+    def process_id
+      Process.pid
+    end
+
+    # @return [String] Current thread name
+    def thread_name
+      (Thread.current.name || Thread.current.object_id).to_s
+    end
+  end
+
+  # Slightly customized sub-class of Concurrent::ThreadPoolExecutor
+  class ThreadPoolExecutor < Concurrent::ThreadPoolExecutor
+    # Number of idle or potential threads available to execute tasks
+    # https://github.com/ruby-concurrency/concurrent-ruby/issues/684#issuecomment-427594437
+    # @return [Integer]
+    def ready_worker_count
+      synchronize do
+        workers_still_to_be_created = @max_length - @pool.length
+        workers_created_but_waiting = @ready.length
+
+        workers_still_to_be_created + workers_created_but_waiting
       end
     end
   end

data/lib/good_job/version.rb

@@ -1,3 +1,3 @@
 module GoodJob
-  VERSION = '1.0.2'.freeze
+  VERSION = '1.1.3'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Ben Sheldon
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-25 00:00:00.000000000 Z
+date: 2020-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: foreman
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +248,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A multithreaded, Postgres-based ActiveJob backend for Ruby on Rails
 email:
 - bensheldon@gmail.com
@@ -241,9 +283,12 @@ files:
 - lib/good_job.rb
 - lib/good_job/adapter.rb
 - lib/good_job/cli.rb
+- lib/good_job/configuration.rb
+- lib/good_job/current_execution.rb
 - lib/good_job/job.rb
 - lib/good_job/lockable.rb
-- lib/good_job/logging.rb
+- lib/good_job/log_subscriber.rb
+- lib/good_job/multi_scheduler.rb
 - lib/good_job/performer.rb
 - lib/good_job/pg_locks.rb
 - lib/good_job/railtie.rb
@@ -258,7 +303,7 @@ metadata:
   documentation_uri: https://rdoc.info/github/bensheldon/good_job
   homepage_uri: https://github.com/bensheldon/good_job
   source_code_uri: https://github.com/bensheldon/good_job
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--title"
 - GoodJob - a multithreaded, Postgres-based ActiveJob backend for Ruby on Rails
@@ -281,7 +326,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A multithreaded, Postgres-based ActiveJob backend for Ruby on Rails
 test_files: []

data/lib/good_job/logging.rb

@@ -1,70 +0,0 @@
-module GoodJob
-  module Logging
-    extend ActiveSupport::Concern
-
-    included do
-      cattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT))
-
-      def self.tag_logger(*tags)
-        if logger.respond_to?(:tagged)
-          tags.unshift "GoodJob" unless logger.formatter.current_tags.include?("GoodJob")
-          logger.tagged(*tags) { yield }
-        else
-          yield
-        end
-      end
-    end
-
-    class LogSubscriber < ActiveSupport::LogSubscriber
-      def create(event)
-        good_job = event.payload[:good_job]
-
-        info do
-          "Created GoodJob resource with id #{good_job.id}"
-        end
-      end
-
-      def timer_task_finished(event)
-        exception = event.payload[:error]
-        return unless exception
-
-        error do
-          "ERROR: #{exception}\n #{exception.backtrace}"
-        end
-      end
-
-      def job_finished(event)
-        exception = event.payload[:error]
-        return unless exception
-
-        error do
-          "ERROR: #{exception}\n #{exception.backtrace}"
-        end
-      end
-
-      def scheduler_start_shutdown(_event)
-        info do
-          "Shutting down scheduler..."
-        end
-      end
-
-      def scheduler_shutdown(_event)
-        info do
-          "Scheduler is shut down."
-        end
-      end
-
-      private
-
-      def logger
-        GoodJob.logger
-      end
-
-      def thread_name
-        Thread.current.name || Thread.current.object_id
-      end
-    end
-  end
-end
-
-GoodJob::Logging::LogSubscriber.attach_to :good_job