rocketjob 4.0.0 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3d1d8a5bd37991ac8ad2dd5ae28b833dc4cb59d69da1a1cc281c904ab15c3eb
-  data.tar.gz: 39336f01d701f34e4f25e00e03fa5f06299ec19d1f7c0b52af5aff6ae4b4327b
+  metadata.gz: d14befce83747b072cf398356d5a5a798630d65c9aa50ec5f5de1e1f4d0e8d69
+  data.tar.gz: 17c5e295968836458ef1b998d1ffb79228ab6b5971c6339ec1bdc74d429d2511
 SHA512:
-  metadata.gz: 42266518d00516c62ccd3d0139e6fa22427f48b14454a2f06f0c04f559620b9bbf71700f38b518db03b8290cd6ab7ad7353c47cfbb579aa2f5642ef2a9b2cfb5
-  data.tar.gz: 444e848667f4a09629b18b467713fe9ae26abb22fd68ceb9048e0adb617c3bbe066ec407c9922577f9ea6bc40271a9d6c84d6064456e5e0532ddfa8f0be85a69
+  metadata.gz: 07b25f25ca1fd804e6c3d98ebcf3724acb8bcb564281f1b5ffd21bd815e8a05d7a99988721c5e3a25063da44f800ea3c998fcbecd7751a8b10755b214804feab
+  data.tar.gz: 48093ed5e152571a097e07f8e606df99fc48fcf1f8364e78b77d371e5e668acf384c972a002c078930b1af809dd7e34483ba28194fbcd52a9ae06bfb60f2dfce

data/lib/rocket_job/cli.rb

@@ -47,7 +47,7 @@ module RocketJob
       opts[:max_workers] = workers if workers
       opts[:filter]      = filter if filter
 
-      Server.run(opts)
+      Supervisor.run(opts)
     end
 
     def rails?
@@ -96,7 +96,7 @@ module RocketJob
 
       require 'rocketjob'
       begin
-        require 'rocketjob_batch'
+        require 'rocketjob_enterprise'
       rescue LoadError
         nil
       end

data/lib/rocket_job/event.rb

@@ -0,0 +1,163 @@
+require 'concurrent-ruby'
+
+module RocketJob
+  # RocketJob::Event
+  #
+  # Publish and Subscribe to events. Events are published immediately and usually consumed
+  # almost immediately by all subscriber processes.
+  class Event
+    include SemanticLogger::Loggable
+    include Plugins::Document
+    include Mongoid::Timestamps
+
+    ALL_EVENTS = '*'.freeze
+
+    # Capped collection long polling interval.
+    class_attribute :long_poll_seconds, instance_accessor: false
+    self.long_poll_seconds = 300
+
+    # Capped collection size.
+    # Only used the first time the collection is created.
+    #
+    # Default: 128MB.
+    class_attribute :capped_collection_size, instance_accessor: false
+    self.capped_collection_size = 128 * 1024 * 1024
+
+    # Mandatory Event Name
+    #   Examples:
+    #     '/rocket_job/config'
+    #     '/rocket_job/server'
+    #     '/rocket_job/worker'
+    field :name, type: String
+
+    # Event Action
+    #   Examples:
+    #     :shutdown
+    #     :pause
+    #     :updated
+    field :action, type: Symbol
+
+    # Hash Parameters to be sent with the event (event specific).
+    field :parameters, type: Hash
+
+    validates_presence_of :name
+
+    store_in collection: 'rocket_job.events'
+    index({created_at: 1}, background: true)
+
+    # Add a subscriber for its events.
+    # Returns a handle to the subscription that can be used to unsubscribe
+    # this particular subscription
+    #
+    # Example:
+    # def MySubscriber
+    #   include RocketJob::Subscriber
+    #
+    #   def hello
+    #     logger.info "Hello Action Received"
+    #   end
+    #
+    #   def show(message:)
+    #     logger.info "Received: #{message}"
+    #   end
+    # end
+    #
+    # MySubscriber.subscribe
+    def self.subscribe(subscriber)
+      if block_given?
+        begin
+          handle = add_subscriber(subscriber)
+          yield(subscriber)
+        ensure
+          unsubscribe(handle) if handle
+        end
+      else
+        add_subscriber(subscriber)
+      end
+    end
+
+    # Unsubscribes a previous subscription
+    def self.unsubscribe(handle)
+      @subscribers.each_value { |v| v.delete_if { |i| i.object_id == handle } }
+    end
+
+    # Indefinitely tail the capped collection looking for new events.
+    #   time: the start time from which to start looking for new events.
+    def self.listener(time: @load_time)
+      Thread.current.name = 'rocketjob event'
+      create_capped_collection
+
+      logger.info('Event listener started')
+      tail_capped_collection(time) { |event| process_event(event) }
+    rescue Exception => exc
+      logger.error('#listener Event listener is terminating due to unhandled exception', exc)
+      raise(exc)
+    end
+
+    # Create the capped collection only if it does not exist.
+    # Drop the collection before calling this method to re-create it.
+    def self.create_capped_collection(size: capped_collection_size)
+      if collection_exists?
+        convert_to_capped_collection(size) unless collection.capped?
+      else
+        collection.client[collection_name, {capped: true, size: size}].create
+      end
+    end
+
+    private
+
+    @load_time   = Time.now.utc
+    @subscribers = Concurrent::Map.new { Concurrent::Array.new }
+
+    def self.add_subscriber(subscriber)
+      name               = subscriber.class.event_name
+      @subscribers[name] = @subscribers[name] << subscriber
+      subscriber.object_id
+    end
+
+    def self.tail_capped_collection(time)
+      with(socket_timeout: long_poll_seconds + 10) do
+        filter = {created_at: {'$gt' => time}}
+        collection.
+          find(filter).
+          await_data.
+          cursor_type(:tailable_await).
+          max_await_time_ms(long_poll_seconds * 1000).
+          sort('$natural' => 1).
+          each do |doc|
+          event = Mongoid::Factory.from_db(Event, doc)
+          # Recovery will occur from after the last message read
+          time = event.created_at
+          yield(event)
+        end
+      end
+    rescue Mongo::Error::SocketError, Mongo::Error::SocketTimeoutError, Mongo::Error::OperationFailure, Timeout::Error => exc
+      logger.info("Creating a new cursor and trying again: #{exc.class.name} #{exc.message}")
+      retry
+    end
+
+    # Process a new event, calling registered subscribers.
+    def self.process_event(event)
+      logger.info('Event Received', event.attributes)
+
+      if @subscribers.key?(event.name)
+        @subscribers[event.name].each { |subscriber| subscriber.process_action(event.action, event.parameters) }
+      end
+
+      if @subscribers.key?(ALL_EVENTS)
+        @subscribers[ALL_EVENTS].each { |subscriber| subscriber.process_event(event.name, event.action, event.parameters) }
+      end
+    rescue StandardError => exc
+      logger.error('Unknown subscriber. Continuing..', exc)
+    end
+
+    def self.collection_exists?
+      collection.database.collection_names.include?(collection_name.to_s)
+    end
+
+    # Convert a non-capped collection to capped
+    def self.convert_to_capped_collection(size)
+      collection.database.command('convertToCapped' => collection_name.to_s, 'size' => size)
+    end
+  end
+end

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -46,13 +46,6 @@ module RocketJob
       field :queued_retention, type: Integer, user_editable: true, copy_on_restart: true
 
       def perform
-        if destroy_zombies
-          # Cleanup zombie servers
-          RocketJob::Server.destroy_zombies
-          # Requeue jobs where the worker is in the zombie state and its server has gone away
-          RocketJob::ActiveWorker.requeue_zombies
-        end
-
         RocketJob::Job.aborted.where(completed_at: {'$lte' => aborted_retention.seconds.ago}).destroy_all if aborted_retention
         if completed_retention
           RocketJob::Job.completed.where(completed_at: {'$lte' => completed_retention.seconds.ago}).destroy_all
@@ -60,6 +53,13 @@ module RocketJob
         RocketJob::Job.failed.where(completed_at: {'$lte' => failed_retention.seconds.ago}).destroy_all if failed_retention
         RocketJob::Job.paused.where(completed_at: {'$lte' => paused_retention.seconds.ago}).destroy_all if paused_retention
         RocketJob::Job.queued.where(created_at: {'$lte' => queued_retention.seconds.ago}).destroy_all if queued_retention
+
+        if destroy_zombies
+          # Cleanup zombie servers
+          RocketJob::Server.destroy_zombies
+          # Requeue jobs where the worker is in the zombie state and its server has gone away
+          RocketJob::ActiveWorker.requeue_zombies
+        end
       end
     end
   end

data/lib/rocket_job/plugins/transaction.rb

@@ -25,7 +25,7 @@ module RocketJob
     #   end
     #
     # Performance
-    # - On Ruby (MRI) an empty transaction block call takes about 1ms.
+    # - On CRuby an empty transaction block call takes about 1ms.
     # - On JRuby an empty transaction block call takes about 55ms.
     #
     # Note:

data/lib/rocket_job/rocket_job.rb

@@ -1,4 +1,11 @@
 module RocketJob
+  def self.create_indexes
+    # Ensure models with indexes are loaded into memory first
+    Job.create_indexes
+    Server.create_indexes
+    DirmonEntry.create_indexes
+  end
+
   # Whether the current process is running inside a Rocket Job server process.
   def self.server?
     @server

data/lib/rocket_job/server.rb

@@ -1,5 +1,6 @@
-require 'yaml'
-require 'concurrent'
+require 'rocket_job/server/model'
+require 'rocket_job/server/state_machine'
+
 module RocketJob
   # Server
   #
@@ -29,359 +30,7 @@ module RocketJob
     include Plugins::Document
     include Plugins::StateMachine
     include SemanticLogger::Loggable
-
-    store_in collection: 'rocket_job.servers'
-
-    # Unique Name of this server instance
-    #   Default: `host name:PID`
-    # The unique name is used on re-start to re-queue any jobs that were being processed
-    # at the time the server unexpectedly terminated, if any
-    field :name, type: String, default: -> { "#{SemanticLogger.host}:#{$$}" }
-
-    # The maximum number of workers this server should start
-    #   If set, it will override the default value in RocketJob::Config
-    field :max_workers, type: Integer, default: -> { Config.instance.max_workers }
-
-    # When this server process was started
-    field :started_at, type: Time
-
-    # Filter to apply to control which job classes this server can process
-    field :yaml_filter, type: String
-
-    # The heartbeat information for this server
-    embeds_one :heartbeat, class_name: 'RocketJob::Heartbeat'
-
-    # Current state
-    #   Internal use only. Do not set this field directly
-    field :state, type: Symbol, default: :starting
-
-    index({name: 1}, background: true, unique: true, drop_dups: true)
-
-    validates_presence_of :state, :name, :max_workers
-
-    # States
-    #   :starting -> :running -> :paused
-    #                         -> :stopping
-    aasm column: :state, whiny_persistence: true do
-      state :starting, initial: true
-      state :running
-      state :paused
-      state :stopping
-
-      event :started do
-        transitions from: :starting, to: :running
-        before do
-          self.started_at = Time.now
-        end
-      end
-
-      event :pause do
-        transitions from: :running, to: :paused
-      end
-
-      event :resume do
-        transitions from: :paused, to: :running
-      end
-
-      event :stop do
-        transitions from: :running, to: :stopping
-        transitions from: :paused, to: :stopping
-        transitions from: :starting, to: :stopping
-      end
-    end
-
-    # Requeue any jobs being worked by this server when it is destroyed
-    before_destroy :requeue_jobs
-
-    # Destroy's all instances of zombie servers and requeues any jobs still "running"
-    # on those servers.
-    def self.destroy_zombies
-      count = 0
-      each do |server|
-        next unless server.zombie?
-        logger.warn "Destroying zombie server #{server.name}, and requeueing its jobs"
-        server.destroy
-        count += 1
-      end
-      count
-    end
-
-    # Stop all running, paused, or starting servers
-    def self.stop_all
-      where(:state.in => %i[running paused starting]).each(&:stop!)
-    end
-
-    # Pause all running servers
-    def self.pause_all
-      running.each(&:pause!)
-    end
-
-    # Resume all paused servers
-    def self.resume_all
-      paused.each(&:resume!)
-    end
-
-    # Returns [Hash<String:Integer>] of the number of servers in each state.
-    # Note: If there are no servers in that particular state then the hash will not have a value for it.
-    #
-    # Example servers in every state:
-    #   RocketJob::Server.counts_by_state
-    #   # => {
-    #          :aborted => 1,
-    #          :completed => 37,
-    #          :failed => 1,
-    #          :paused => 3,
-    #          :queued => 4,
-    #          :running => 1,
-    #          :queued_now => 1,
-    #          :scheduled => 3
-    #        }
-    #
-    # Example no servers active:
-    #   RocketJob::Server.counts_by_state
-    #   # => {}
-    def self.counts_by_state
-      counts = {}
-      collection.aggregate(
-        [
-          {
-            '$group' => {
-              _id:   '$state',
-              count: {'$sum' => 1}
-            }
-          }
-        ]
-      ).each do |result|
-        counts[result['_id'].to_sym] = result['count']
-      end
-      counts
-    end
-
-    # On MRI the 'concurrent-ruby-ext' gem may not be loaded
-    if defined?(Concurrent::JavaAtomicBoolean) || defined?(Concurrent::CAtomicBoolean)
-      # Returns [true|false] whether the shutdown indicator has been set for this server process
-      def self.shutdown?
-        @shutdown.value
-      end
-
-      # Set shutdown indicator for this server process
-      def self.shutdown!
-        @shutdown.make_true
-      end
-
-      @shutdown = Concurrent::AtomicBoolean.new(false)
-    else
-      # Returns [true|false] whether the shutdown indicator has been set for this server process
-      def self.shutdown?
-        @shutdown
-      end
-
-      # Set shutdown indicator for this server process
-      def self.shutdown!
-        @shutdown = true
-      end
-
-      @shutdown = false
-    end
-
-    # Run the server process
-    # Attributes supplied are passed to #new
-    def self.run(attrs = {})
-      Thread.current.name = 'rocketjob main'
-      # Create Indexes on server startup
-      ::Mongoid::Tasks::Database.create_indexes
-      register_signal_handlers
-
-      server = create!(attrs)
-      server.send(:run)
-    ensure
-      server&.destroy
-    end
-
-    # Returns [Boolean] whether the server is shutting down
-    def shutdown?
-      self.class.shutdown? || !running?
-    end
-
-    # Scope for all zombie servers
-    def self.zombies(missed = 4)
-      dead_seconds        = Config.instance.heartbeat_seconds * missed
-      last_heartbeat_time = Time.now - dead_seconds
-      where(
-        :state.in => %i[stopping running paused],
-        '$or'     => [
-          {'heartbeat.updated_at' => {'$exists' => false}},
-          {'heartbeat.updated_at' => {'$lte' => last_heartbeat_time}}
-        ]
-      )
-    end
-
-    # Returns [true|false] if this server has missed at least the last 4 heartbeats
-    #
-    # Possible causes for a server to miss its heartbeats:
-    # - The server process has died
-    # - The server process is "hanging"
-    # - The server is no longer able to communicate with the MongoDB Server
-    def zombie?(missed = 4)
-      return false unless running? || stopping? || paused?
-      return true if heartbeat.nil? || heartbeat.updated_at.nil?
-      dead_seconds = Config.instance.heartbeat_seconds * missed
-      (Time.now - heartbeat.updated_at) >= dead_seconds
-    end
-
-    # Where clause filter to apply to workers looking for jobs
-    def filter
-      YAML.load(yaml_filter) if yaml_filter
-    end
-
-    def filter=(hash)
-      self.yaml_filter = hash.nil? ? nil : hash.to_yaml
-    end
-
-    private
-
-    # Returns [Array<Worker>] collection of workers
-    def workers
-      @workers ||= []
-    end
-
-    # Management Thread
-    def run
-      logger.info "Using MongoDB Database: #{RocketJob::Job.collection.database.name}"
-      logger.info('Running with filter', filter) if filter
-      build_heartbeat(updated_at: Time.now, workers: 0)
-      started!
-      logger.info 'Rocket Job Server started'
-
-      run_workers
-
-      logger.info 'Waiting for workers to stop'
-      # Tell each worker to shutdown cleanly
-      workers.each(&:shutdown!)
-
-      while (worker = workers.first)
-        if worker.join(5)
-          # Worker thread is dead
-          workers.shift
-        else
-          # Timeout waiting for worker to stop
-          find_and_update(
-            'heartbeat.updated_at' => Time.now,
-            'heartbeat.workers'    => worker_count
-          )
-        end
-      end
-
-      logger.info 'Shutdown'
-    rescue ::Mongoid::Errors::DocumentNotFound
-      logger.warn('Server has been destroyed. Going down hard!')
-    rescue Exception => exc
-      logger.error('RocketJob::Server is stopping due to an exception', exc)
-    ensure
-      # Logs the backtrace for each running worker
-      workers.each { |worker| logger.backtrace(thread: worker.thread) if worker.thread && worker.alive? }
-    end
-
-    def run_workers
-      stagger = true
-      while running? || paused?
-        SemanticLogger.silence(:info) do
-          find_and_update(
-            'heartbeat.updated_at' => Time.now,
-            'heartbeat.workers'    => worker_count
-          )
-        end
-        if paused?
-          workers.each(&:shutdown!)
-          stagger = true
-        end
-
-        # In case number of threads has been modified
-        adjust_workers(stagger)
-        stagger = false
-
-        # Stop server if shutdown indicator was set
-        if self.class.shutdown? && may_stop?
-          stop!
-        else
-          sleep Config.instance.heartbeat_seconds
-        end
-      end
-    end
-
-    # Returns [Fixnum] number of workers (threads) that are alive
-    def worker_count
-      workers.count(&:alive?)
-    end
-
-    def next_worker_id
-      @worker_id ||= 0
-      @worker_id += 1
-    end
-
-    # Re-adjust the number of running workers to get it up to the
-    # required number of workers
-    #   Parameters
-    #     stagger_workers
-    #       Whether to stagger when the workers poll for work the first time
-    #       It spreads out the queue polling over the max_poll_seconds so
-    #       that not all workers poll at the same time
-    #       The worker also respond faster than max_poll_seconds when a new
-    #       job is added.
-    def adjust_workers(stagger_workers = false)
-      count = worker_count
-      # Cleanup workers that have stopped
-      if count != workers.count
-        logger.info "Cleaning up #{workers.count - count} workers that went away"
-        workers.delete_if { |t| !t.alive? }
-      end
-
-      return unless running?
-
-      # Need to add more workers?
-      return unless count < max_workers
-
-      worker_count = max_workers - count
-      logger.info "Starting #{worker_count} workers"
-      worker_count.times.each do
-        sleep(Config.instance.max_poll_seconds.to_f / max_workers) if stagger_workers
-        return if shutdown?
-        # Start worker
-        begin
-          workers << Worker.new(id: next_worker_id, server_name: name, filter: filter)
-        rescue Exception => exc
-          logger.fatal('Cannot start worker', exc)
-        end
-      end
-    end
-
-    # Register handlers for the various signals
-    # Term:
-    #   Perform clean shutdown
-    #
-    def self.register_signal_handlers
-      Signal.trap 'SIGTERM' do
-        shutdown!
-        message = 'Shutdown signal (SIGTERM) received. Will shutdown as soon as active jobs/slices have completed.'
-        # Logging uses a mutex to access Queue on MRI/CRuby
-        defined?(JRuby) ? logger.warn(message) : puts(message)
-      end
-
-      Signal.trap 'INT' do
-        shutdown!
-        message = 'Shutdown signal (INT) received. Will shutdown as soon as active jobs/slices have completed.'
-        # Logging uses a mutex to access Queue on MRI/CRuby
-        defined?(JRuby) ? logger.warn(message) : puts(message)
-      end
-    rescue StandardError
-      logger.warn 'SIGTERM handler not installed. Not able to shutdown gracefully'
-    end
-
-    private_class_method :register_signal_handlers
-
-    # Requeue any jobs assigned to this server when it is destroyed
-    def requeue_jobs
-      RocketJob::Job.requeue_dead_server(name)
-    end
+    include Server::Model
+    include Server::StateMachine
   end
 end