rocketjob 4.0.0 → 4.1.0

data/lib/rocket_job/server/model.rb

@@ -0,0 +1,138 @@
+require 'yaml'
+require 'active_support/concern'
+
+module RocketJob
+  class Server
+    # Model attributes
+    module Model
+      extend ActiveSupport::Concern
+
+      included do
+        store_in collection: 'rocket_job.servers'
+
+        # Unique Name of this server instance
+        #   Default: `host name:PID`
+        # The unique name is used on re-start to re-queue any jobs that were being processed
+        # at the time the server unexpectedly terminated, if any
+        field :name, type: String, default: -> { "#{SemanticLogger.host}:#{$$}" }
+
+        # The maximum number of workers this server should start
+        #   If set, it will override the default value in RocketJob::Config
+        field :max_workers, type: Integer, default: -> { Config.instance.max_workers }
+
+        # When this server process was started
+        field :started_at, type: Time
+
+        # Filter to apply to control which job classes this server can process
+        field :yaml_filter, type: String
+
+        # The heartbeat information for this server
+        embeds_one :heartbeat, class_name: 'RocketJob::Heartbeat'
+
+        # Current state
+        #   Internal use only. Do not set this field directly
+        field :state, type: Symbol, default: :starting
+
+        index({name: 1}, background: true, unique: true, drop_dups: true)
+
+        validates_presence_of :state, :name, :max_workers
+
+        # Requeue any jobs being worked by this server when it is destroyed
+        before_destroy :requeue_jobs
+
+        # Returns [Hash<String:Integer>] of the number of servers in each state.
+        # Note: If there are no servers in that particular state then the hash will not have a value for it.
+        #
+        # Example servers in every state:
+        #   RocketJob::Server.counts_by_state
+        #   # => {
+        #          :aborted => 1,
+        #          :completed => 37,
+        #          :failed => 1,
+        #          :paused => 3,
+        #          :queued => 4,
+        #          :running => 1,
+        #          :queued_now => 1,
+        #          :scheduled => 3
+        #        }
+        #
+        # Example no servers active:
+        #   RocketJob::Server.counts_by_state
+        #   # => {}
+        def self.counts_by_state
+          counts = {}
+          collection.aggregate([{'$group' => {_id: '$state', count: {'$sum' => 1}}}]).each do |result|
+            counts[result['_id'].to_sym] = result['count']
+          end
+          counts
+        end
+
+        # Destroy's all instances of zombie servers and requeues any jobs still "running"
+        # on those servers.
+        def self.destroy_zombies
+          count = 0
+          each do |server|
+            next unless server.zombie?
+            logger.warn "Destroying zombie server #{server.name}, and requeueing its jobs"
+            server.destroy
+            count += 1
+          end
+          count
+        end
+
+        # Scope for all zombie servers
+        def self.zombies(missed = 4)
+          dead_seconds        = Config.instance.heartbeat_seconds * missed
+          last_heartbeat_time = Time.now - dead_seconds
+          where(
+            :state.in => %i[stopping running paused],
+            '$or'     => [
+              {'heartbeat.updated_at' => {'$exists' => false}},
+              {'heartbeat.updated_at' => {'$lte' => last_heartbeat_time}}
+            ]
+          )
+        end
+      end
+
+      # Where clause filter to apply to workers looking for jobs
+      def filter
+        YAML.load(yaml_filter) if yaml_filter
+      end
+
+      def filter=(hash)
+        self.yaml_filter = hash.nil? ? nil : hash.to_yaml
+      end
+
+      # Returns [true|false] if this server has missed at least the last 4 heartbeats
+      #
+      # Possible causes for a server to miss its heartbeats:
+      # - The server process has died
+      # - The server process is "hanging"
+      # - The server is no longer able to communicate with the MongoDB Server
+      def zombie?(missed = 4)
+        return false unless running? || stopping? || paused?
+        return true if heartbeat.nil? || heartbeat.updated_at.nil?
+        dead_seconds = Config.instance.heartbeat_seconds * missed
+        (Time.now - heartbeat.updated_at) >= dead_seconds
+      end
+
+      # Updates the heartbeat and returns a refreshed server instance.
+      def refresh(worker_count)
+        SemanticLogger.silence(:info) do
+          find_and_update(
+            'heartbeat.updated_at' => Time.now,
+            'heartbeat.workers'    => worker_count
+          )
+        end
+      end
+
+      private
+
+      # Requeue any jobs assigned to this server when it is destroyed
+      def requeue_jobs
+        RocketJob::Job.requeue_dead_server(name)
+      end
+
+    end
+  end
+end

data/lib/rocket_job/server/state_machine.rb

@@ -0,0 +1,60 @@
+require 'active_support/concern'
+
+module RocketJob
+  class Server
+    # State machine for sliced jobs
+    module StateMachine
+      extend ActiveSupport::Concern
+
+      included do
+        # States
+        #   :starting -> :running -> :paused
+        #                         -> :stopping
+        aasm column: :state, whiny_persistence: true do
+          state :starting, initial: true
+          state :running
+          state :paused
+          state :stopping
+
+          event :started do
+            transitions from: :starting, to: :running
+            before do
+              self.started_at = Time.now
+              build_heartbeat(updated_at: Time.now, workers: 0)
+            end
+          end
+
+          event :pause do
+            transitions from: :running, to: :paused
+          end
+
+          event :resume do
+            transitions from: :paused, to: :running
+          end
+
+          event :stop do
+            transitions from: :running, to: :stopping
+            transitions from: :paused, to: :stopping
+            transitions from: :starting, to: :stopping
+          end
+        end
+
+        # Stop all running, paused, or starting servers
+        def self.stop_all
+          where(:state.in => %i[running paused starting]).each(&:stop!)
+        end
+
+        # Pause all running servers
+        def self.pause_all
+          running.each(&:pause!)
+        end
+
+        # Resume all paused servers
+        def self.resume_all
+          paused.each(&:resume!)
+        end
+      end
+
+    end
+  end
+end

data/lib/rocket_job/subscriber.rb

@@ -0,0 +1,79 @@
+require 'active_support/concern'
+
+module RocketJob
+  # Mix-in to publish and subscribe to events.
+  #
+  # Example:
+  # def MySubscriber
+  #   include RocketJob::Subscriber
+  #
+  #   def hello
+  #     logger.info "Hello Action Received"
+  #   end
+  #
+  #   def show(message:)
+  #     logger.info "Received: #{message}"
+  #   end
+  #
+  #   # If `message` is not supplied it defaults to "Hello World"
+  #   def show_default(message: "Hello World")
+  #     logger.info "Received: #{message}"
+  #   end
+  # end
+  #
+  # MySubscriber.subscribe
+  module Subscriber
+    extend ActiveSupport::Concern
+
+    # Test Mode
+    # Bypasses publishing the event and calls the subscribers directly
+    def self.test_mode!
+      @test_mode = true
+    end
+
+    def self.test_mode?
+      @test_mode
+    end
+
+    @test_mode = false
+
+    included do
+      include SemanticLogger::Loggable
+
+      # Name of the event published and subscribed to
+      class_attribute :event_name, instance_accessor: false
+      self.event_name = name
+
+      def self.publish(action, **parameters)
+        raise(ArgumentError, "Invalid action: #{action}") unless public_method_defined?(action)
+        if event_name == Event::ALL_EVENTS
+          raise(NotImplementedError, "Cannot publish to an all events subscriber: event_name='#{Event::ALL_EVENTS}'")
+        end
+
+        event = Event.new(name: event_name, action: action, parameters: parameters)
+        Subscriber.test_mode? ? Event.process_event(event) : event.save!
+      end
+
+      def self.subscribe(*args, &block)
+        instance = new(*args)
+        Event.subscribe(instance, &block)
+      end
+    end
+
+    def process_action(action, parameters)
+      unless public_methods.include?(action)
+        logger.warn("Ignoring unknown action: #{action}")
+        return
+      end
+
+      args = (method(action).arity == 0) || parameters.nil? ? nil : parameters.symbolize_keys
+      args ? public_send(action, **args) : public_send(action)
+    rescue StandardError => exc
+      logger.error('Exception calling subscriber. Resuming..', exc)
+    end
+
+    def process_event(name, action, parameters)
+      raise(NotImplementedError)
+    end
+  end
+end

data/lib/rocket_job/subscribers/logger.rb

@@ -0,0 +1,75 @@
+require 'socket'
+
+module RocketJob
+  module Subscribers
+    class Logger
+      include RocketJob::Subscriber
+
+      def self.host_name
+        @host_name ||= Socket.gethostname
+      end
+
+      def self.host_name=(host_name)
+        @host_name = host_name
+      end
+
+      # Change the log level
+      #
+      # Examples:
+      #   # Change the global log level to :trace on all servers.
+      #   RocketJob::Subscribers::Logger.publish(:set, level: :trace)
+      #
+      #   # Change the global log level to :trace on one server.
+      #   RocketJob::Subscribers::Logger.publish(:set, level: :trace, host_name: 'server1.company.com')
+      #
+      #   # Change the global log level to :trace for a specific process id.
+      #   RocketJob::Subscribers::Logger.publish(:set, level: :trace, host_name: 'server1.company.com', pid: 34567)
+      #
+      #   # Change the log level for a specific class to :trace.
+      #   RocketJob::Subscribers::Logger.publish(:set, level: :trace, class_name: 'RocketJob::Supervisor')
+      def set(level: :info, class_name: nil, host_name: nil, pid: nil)
+        return unless for_me?(host_name, pid)
+
+        if class_name
+          class_name.constantize.logger.level = level
+          logger.info "Changed log level to #{level} for #{class_name}"
+        else
+          SemanticLogger.default_level = level
+          logger.info "Changed global log level to #{level}"
+        end
+      end
+
+      # Dump all backtraces to the log file.
+      #
+      # Examples:
+      #   # Thread dump on all servers:
+      #   RocketJob::Subscribers::Logger.publish(:thread_dump)
+      #
+      #   # Change the global log level to :trace on one server.
+      #   RocketJob::Subscribers::Logger.publish(:thread_dump, host_name: 'server1.company.com')
+      #
+      #   # Change the global log level to :trace for a specific process id.
+      #   RocketJob::Subscribers::Logger.publish(:thread_dump, host_name: 'server1.company.com', pid: 34567)
+      def thread_dump(host_name: nil, pid: nil)
+        return unless for_me?(host_name, pid)
+
+        Thread.list.each do |thread|
+          next if thread == Thread.current
+
+          logger.backtrace(thread: thread)
+        end
+      end
+
+      private
+
+      def for_me?(host_name, pid)
+        return true if host_name.nil? && pid.nil?
+
+        return false if host_name && (host_name != self.class.host_name)
+        return false if pid && (pid != $$)
+
+        true
+      end
+    end
+  end
+end

data/lib/rocket_job/subscribers/server.rb

@@ -0,0 +1,71 @@
+module RocketJob
+  module Subscribers
+    class Server
+      include RocketJob::Subscriber
+
+      attr_reader :supervisor
+
+      def initialize(supervisor)
+        @supervisor = supervisor
+      end
+
+      def kill(server_id: nil, wait_timeout: 3)
+        return unless my_server?(server_id)
+
+        supervisor.synchronize do
+          supervisor.worker_pool.stop
+          supervisor.worker_pool.join(wait_timeout)
+          supervisor.worker_pool.kill
+        end
+
+        Supervisor.shutdown!
+        logger.info "Killed"
+      end
+
+      def pause(server_id: nil)
+        return unless my_server?(server_id)
+
+        supervisor.synchronize { supervisor.server.pause! if supervisor.server.may_pause? }
+        Supervisor.event!
+        logger.info "Paused"
+      end
+
+      def refresh(server_id: nil)
+        return unless my_server?(server_id)
+
+        Supervisor.event!
+        logger.info "Refreshed"
+      end
+
+      def resume(server_id: nil)
+        return unless my_server?(server_id)
+
+        supervisor.synchronize { supervisor.server.resume! if supervisor.server.may_resume? }
+        Supervisor.event!
+        logger.info "Resumed"
+      end
+
+      def stop(server_id: nil)
+        return unless my_server?(server_id)
+
+        Supervisor.shutdown!
+        logger.info "Shutdown"
+      end
+
+      def thread_dump(server_id: nil)
+        return unless my_server?(server_id)
+
+        logger.info "Thread dump"
+        supervisor.worker_pool.log_backtraces
+      end
+
+      private
+
+      def my_server?(server_id)
+        return true if server_id.nil?
+
+        server_id == supervisor.server.id
+      end
+    end
+  end
+end

data/lib/rocket_job/subscribers/worker.rb

@@ -0,0 +1,61 @@
+module RocketJob
+  module Subscribers
+    class Worker
+      include RocketJob::Subscriber
+
+      attr_reader :supervisor
+
+      def initialize(supervisor)
+        @supervisor = supervisor
+      end
+
+      def kill(server_id:, worker_id:, wait_timeout: 3)
+        return unless my_server?(server_id)
+
+        worker = locate_worker(worker_id)
+        return unless worker
+
+        worker.shutdown!
+        worker.join(wait_timeout)
+        worker.kill
+
+        logger.info "Killed"
+      end
+
+      def stop(server_id:, worker_id:)
+        return unless my_server?(server_id)
+
+        worker = locate_worker(worker_id)
+        return unless worker
+
+        worker.shutdown!
+        logger.info "Stopped Worker: #{worker_id}"
+      end
+
+      def thread_dump(server_id:, worker_id:)
+        return unless my_server?(server_id)
+
+        worker = locate_worker(worker_id)
+        return unless worker
+
+        logger.info "Thread dump Worker: #{worker_id}"
+        logger.backtrace(thread: worker.thread) if worker.thread && worker.alive?
+      end
+
+      private
+
+      def my_server?(server_id)
+        server_id == supervisor.server.id
+      end
+
+      def locate_worker(worker_id)
+        return unless worker_id
+
+        worker = supervisor.worker_pool.find(worker_id)
+        return unless worker&.alive?
+
+        worker
+      end
+    end
+  end
+end