rocketjob 4.0.0 → 4.1.0

data/lib/rocket_job/supervisor.rb

@@ -0,0 +1,96 @@
+require 'rocket_job/supervisor/shutdown'
+
+module RocketJob
+  # Starts a server instance, along with the workers and ensures workers remain running until they need to shutdown.
+  class Supervisor
+    include SemanticLogger::Loggable
+    include Supervisor::Shutdown
+
+    attr_reader :server, :worker_pool
+    attr_accessor :worker_id
+
+    # Start the Supervisor, using the supplied attributes to create a new Server instance.
+    def self.run(attrs = {})
+      Thread.current.name = 'rocketjob main'
+      RocketJob.create_indexes
+      register_signal_handlers
+
+      server = Server.create!(attrs)
+      new(server).run
+    ensure
+      server&.destroy
+    end
+
+    def initialize(server)
+      @server      = server
+      @worker_pool = WorkerPool.new(server.name, server.filter)
+      @mutex       = Mutex.new
+    end
+
+    def run
+      logger.info "Using MongoDB Database: #{RocketJob::Job.collection.database.name}"
+      logger.info('Running with filter', server.filter) if server.filter
+      server.started!
+      logger.info 'Rocket Job Server started'
+
+      event_listener = Thread.new { Event.listener }
+      Subscribers::Server.subscribe(self) do
+        Subscribers::Worker.subscribe(self) do
+          Subscribers::Logger.subscribe do
+            supervise_pool
+            stop!
+          end
+        end
+      end
+    rescue ::Mongoid::Errors::DocumentNotFound
+      logger.info('Server has been destroyed. Going down hard!')
+    rescue Exception => exc
+      logger.error('RocketJob::Server is stopping due to an exception', exc)
+    ensure
+      event_listener.kill if event_listener
+      # Logs the backtrace for each running worker
+      worker_pool.log_backtraces
+      logger.info('Shutdown Complete')
+    end
+
+    def stop!
+      server.stop! if server.may_stop?
+      worker_pool.stop
+      while !worker_pool.join
+        logger.info 'Waiting for workers to finish processing ...'
+        # One or more workers still running so update heartbeat so that server reports "alive".
+        server.refresh(worker_pool.living_count)
+      end
+    end
+
+    def supervise_pool
+      stagger = true
+      while !self.class.shutdown?
+        synchronize do
+          if server.running?
+            worker_pool.prune
+            worker_pool.rebalance(server.max_workers, stagger)
+            stagger = false
+          elsif server.paused?
+            worker_pool.stop
+            sleep(0.1)
+            worker_pool.prune
+            stagger = true
+          else
+            break
+          end
+        end
+
+        self.class.wait_for_event(Config.instance.heartbeat_seconds)
+
+        break if self.class.shutdown?
+
+        synchronize { server.refresh(worker_pool.living_count) }
+      end
+    end
+
+    def synchronize(&block)
+      @mutex.synchronize(&block)
+    end
+  end
+end

data/lib/rocket_job/supervisor/shutdown.rb

@@ -0,0 +1,63 @@
+require 'active_support/concern'
+require 'concurrent'
+
+module RocketJob
+  class Supervisor
+    module Shutdown
+      extend ActiveSupport::Concern
+
+      included do
+        # Set shutdown indicator for this server process
+        def self.shutdown!
+          @shutdown.set
+          event!
+        end
+
+        # Returns [true|false] whether the shutdown indicator has been set for this server process
+        def self.shutdown?
+          @shutdown.set?
+        end
+
+        # An event has occured
+        def self.event!
+          @event.set
+        end
+
+        # Returns [true|false] whether the shutdown indicator was set before the timeout was reached
+        def self.wait_for_event(timeout = nil)
+          @event.wait(timeout)
+          @event.reset
+        end
+
+        @shutdown = Concurrent::Event.new
+        @event    = Concurrent::Event.new
+
+        # Register handlers for the various signals
+        # Term:
+        #   Perform clean shutdown
+        #
+        def self.register_signal_handlers
+          Signal.trap 'SIGTERM' do
+            Thread.new do
+              shutdown!
+              message = 'Shutdown signal (SIGTERM) received. Will shutdown as soon as active jobs/slices have completed.'
+              logger.warn(message)
+            end
+          end
+
+          Signal.trap 'INT' do
+            Thread.new do
+              shutdown!
+              message = 'Shutdown signal (INT) received. Will shutdown as soon as active jobs/slices have completed.'
+              logger.warn(message)
+            end
+          end
+        rescue StandardError
+          logger.warn 'SIGTERM handler not installed. Not able to shutdown gracefully'
+        end
+
+        private_class_method :register_signal_handlers
+      end
+    end
+  end
+end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '4.0.0'.freeze
+  VERSION = '4.1.0'.freeze
 end

data/lib/rocket_job/worker.rb

@@ -8,16 +8,18 @@ module RocketJob
   class Worker
     include SemanticLogger::Loggable
     include ActiveSupport::Callbacks
-    extend Forwardable
-
-    def_delegator :@thread, :alive?
-    def_delegator :@thread, :backtrace
-    def_delegator :@thread, :join
 
     define_callbacks :running
 
     attr_accessor :id, :re_check_seconds, :filter, :current_filter
-    attr_reader :thread, :name
+    attr_reader :thread, :name, :inline
+
+    # Raised when a worker is killed so that it shutdown immediately, yet cleanly.
+    #
+    # Note:
+    # - It is not recommended to catch this exception since it is to shutdown workers quickly.
+    class Shutdown < Interrupt
+    end
 
     def self.before_running(*filters, &blk)
       set_callback(:running, :before, *filters, &blk)
@@ -38,37 +40,46 @@ module RocketJob
                    filter: nil)
       @id               = id
       @server_name      = server_name
-      @shutdown         =
-        if defined?(Concurrent::JavaAtomicBoolean) || defined?(Concurrent::CAtomicBoolean)
-          Concurrent::AtomicBoolean.new(false)
-        else
-          false
-        end
+      @shutdown         = Concurrent::Event.new
       @name             = "#{server_name}:#{id}"
       @re_check_seconds = (re_check_seconds || 60).to_f
       @re_check_start   = Time.now
       @filter           = filter.nil? ? {} : filter.dup
       @current_filter   = @filter.dup
       @thread           = Thread.new { run } unless inline
+      @inline           = inline
     end
 
-    if defined?(Concurrent::JavaAtomicBoolean) || defined?(Concurrent::CAtomicBoolean)
-      # Tells this worker to shutdown as soon the current job/slice is complete
-      def shutdown!
-        @shutdown.make_true
-      end
+    def alive?
+      inline ? true : @thread.alive?
+    end
 
-      def shutdown?
-        @shutdown.value
-      end
-    else
-      def shutdown!
-        @shutdown = true
-      end
+    def backtrace
+      inline ? Thread.current.backtrace : @thread.backtrace
+    end
 
-      def shutdown?
-        @shutdown
-      end
+    def join(*args)
+      @thread.join(*args) unless inline
+    end
+
+    # Send each active worker the RocketJob::ShutdownException so that stops processing immediately.
+    def kill
+      return true if inline
+
+      @thread.raise(Shutdown, "Shutdown due to kill request for worker: #{name}") if @thread.alive?
+    end
+
+    def shutdown?
+      @shutdown.set?
+    end
+
+    def shutdown!
+      @shutdown.set
+    end
+
+    # Returns [true|false] whether the shutdown indicator was set
+    def wait_for_shutdown?(timeout = nil)
+      @shutdown.wait(timeout)
     end
 
     private
@@ -82,14 +93,13 @@ module RocketJob
       Thread.current.name = format('rocketjob %03i', id)
       logger.info 'Started'
       until shutdown?
+        wait = RocketJob::Config.instance.max_poll_seconds
         if process_available_jobs
           # Keeps workers staggered across the poll interval so that
           # all workers don't poll at the same time
-          sleep rand(RocketJob::Config.instance.max_poll_seconds * 1000) / 1000
-        else
-          break if shutdown?
-          sleep RocketJob::Config.instance.max_poll_seconds
+          wait = rand(wait * 1000) / 1000
         end
+        break if wait_for_shutdown?(wait)
       end
       logger.info 'Stopping'
     rescue Exception => exc

data/lib/rocket_job/worker_pool.rb

@@ -0,0 +1,103 @@
+require 'concurrent-ruby'
+require 'rocket_job/supervisor/shutdown'
+
+module RocketJob
+  class WorkerPool
+    include SemanticLogger::Loggable
+
+    attr_reader :server_name, :filter, :workers
+
+    def initialize(server_name, filter)
+      @server_name = server_name
+      @filter      = filter
+      @workers     = Concurrent::Array.new
+      @worker_id   = 0
+    end
+
+    # Find a worker in the list by its id
+    def find(id)
+      workers.find { |worker| worker.id == id }
+    end
+
+    # Add new workers to get back to the `max_workers` if not already at `max_workers`
+    #   Parameters
+    #     stagger_start
+    #       Whether to stagger when the workers poll for work the first time.
+    #       It spreads out the queue polling over the max_poll_seconds so
+    #       that not all workers poll at the same time.
+    #       The worker also responds faster than max_poll_seconds when a new job is created.
+    def rebalance(max_workers, stagger_start = false)
+      count = max_workers.to_i - living_count
+      return 0 unless count > 0
+
+      logger.info("#{'Stagger ' if stagger_start}Starting #{count} workers")
+
+      add_one
+      count -= 1
+      delay = Config.instance.max_poll_seconds.to_f / max_workers
+
+      count.times.each do
+        sleep(delay) if stagger_start
+        return -1 if Supervisor.shutdown?
+        add_one
+      end
+    end
+
+    # Returns [Integer] number of dead workers removed.
+    def prune
+      remove_count = workers.count - living_count
+      return 0 if remove_count.zero?
+
+      logger.info "Cleaned up #{remove_count} dead workers"
+      workers.delete_if { |t| !t.alive? }
+      remove_count
+    end
+
+    # Tell all workers to stop working.
+    def stop
+      workers.each(&:shutdown!)
+    end
+
+    # Kill Worker threads
+    def kill
+      workers.each(&:kill)
+    end
+
+    # Wait for all workers to stop.
+    # Return [true] if all workers stopped
+    # Return [false] on timeout
+    def join(timeout = 5)
+      while (worker = workers.first)
+        if worker.join(timeout)
+          # Worker thread is dead
+          workers.shift
+        else
+          return false
+        end
+      end
+      true
+    end
+
+    # Returns [Fixnum] number of workers (threads) that are alive
+    def living_count
+      workers.count(&:alive?)
+    end
+
+    def log_backtraces
+      workers.each { |worker| logger.backtrace(thread: worker.thread) if worker.thread && worker.alive? }
+    end
+
+    private
+
+    def add_one
+      workers << Worker.new(id: next_worker_id, server_name: server_name, filter: filter)
+    rescue StandardError => exc
+      logger.fatal('Cannot start worker', exc)
+    end
+
+    def next_worker_id
+      @worker_id += 1
+    end
+
+  end
+end

data/lib/rocketjob.rb

@@ -23,12 +23,16 @@ module RocketJob
   autoload :CLI,                'rocket_job/cli'
   autoload :Config,             'rocket_job/config'
   autoload :DirmonEntry,        'rocket_job/dirmon_entry'
+  autoload :Event,              'rocket_job/event'
   autoload :Heartbeat,          'rocket_job/heartbeat'
   autoload :Job,                'rocket_job/job'
   autoload :JobException,       'rocket_job/job_exception'
   autoload :Worker,             'rocket_job/worker'
   autoload :Performance,        'rocket_job/performance'
   autoload :Server,             'rocket_job/server'
+  autoload :Subscriber,         'rocket_job/subscriber'
+  autoload :Supervisor,         'rocket_job/supervisor'
+  autoload :WorkerPool,         'rocket_job/worker_pool'
 
   module Plugins
     module Job
@@ -69,17 +73,23 @@ module RocketJob
   end
 
   module Sliced
-    autoload :Input,               'rocket_job/sliced/input'
-    autoload :Output,              'rocket_job/sliced/output'
-    autoload :Slice,               'rocket_job/sliced/slice'
-    autoload :Slices,              'rocket_job/sliced/slices'
-    autoload :Store,               'rocket_job/sliced/store'
+    autoload :Input,            'rocket_job/sliced/input'
+    autoload :Output,           'rocket_job/sliced/output'
+    autoload :Slice,            'rocket_job/sliced/slice'
+    autoload :Slices,           'rocket_job/sliced/slices'
+    autoload :Store,            'rocket_job/sliced/store'
 
     module Writer
-      autoload :Input,             'rocket_job/sliced/writer/input'
-      autoload :Output,            'rocket_job/sliced/writer/output'
+      autoload :Input,          'rocket_job/sliced/writer/input'
+      autoload :Output,         'rocket_job/sliced/writer/output'
     end
   end
+
+  module Subscribers
+    autoload :Logger,           'rocket_job/subscribers/logger'
+    autoload :Server,           'rocket_job/subscribers/server'
+    autoload :Worker,           'rocket_job/subscribers/worker'
+  end
 end
 
 # Add Active Job adapter for Rails

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.1.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-14 00:00:00.000000000 Z
+date: 2019-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: iostreams
   requirement: !ruby/object:Gem::Requirement
@@ -115,6 +115,7 @@ files:
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
 - lib/rocket_job/dirmon_entry.rb
+- lib/rocket_job/event.rb
 - lib/rocket_job/extensions/mongo/logging.rb
 - lib/rocket_job/extensions/mongoid/clients/options.rb
 - lib/rocket_job/extensions/mongoid/contextual/mongo.rb
@@ -155,14 +156,23 @@ files:
 - lib/rocket_job/plugins/transaction.rb
 - lib/rocket_job/rocket_job.rb
 - lib/rocket_job/server.rb
+- lib/rocket_job/server/model.rb
+- lib/rocket_job/server/state_machine.rb
 - lib/rocket_job/sliced/input.rb
 - lib/rocket_job/sliced/output.rb
 - lib/rocket_job/sliced/slice.rb
 - lib/rocket_job/sliced/slices.rb
 - lib/rocket_job/sliced/writer/input.rb
 - lib/rocket_job/sliced/writer/output.rb
+- lib/rocket_job/subscriber.rb
+- lib/rocket_job/subscribers/logger.rb
+- lib/rocket_job/subscribers/server.rb
+- lib/rocket_job/subscribers/worker.rb
+- lib/rocket_job/supervisor.rb
+- lib/rocket_job/supervisor/shutdown.rb
 - lib/rocket_job/version.rb
 - lib/rocket_job/worker.rb
+- lib/rocket_job/worker_pool.rb
 - lib/rocketjob.rb
 homepage: http://rocketjob.io
 licenses:
@@ -183,8 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Ruby's missing batch system.