rocketjob 2.1.3 → 3.0.0.alpha

data/lib/rocket_job/plugins/job/persistence.rb

@@ -10,18 +10,7 @@ module RocketJob
 
         included do
           # Store all job types in this collection
-          set_collection_name 'rocket_job.jobs'
-
-          # Create indexes
-          def self.create_indexes
-            # Used by find_and_modify in .rocket_job_retrieve
-            ensure_index({state: 1, priority: 1, _id: 1}, background: true)
-            # Remove outdated indexes if present
-            drop_index('state_1_run_at_1_priority_1_created_at_1_sub_state_1') rescue nil
-            drop_index('state_1_priority_1_created_at_1_sub_state_1') rescue nil
-            drop_index('state_1_priority_1_created_at_1') rescue nil
-            drop_index('created_at_1') rescue nil
-          end
+          store_in collection: 'rocket_job.jobs'
 
           # Retrieves the next job to work on in priority based order
           # and assigns it to this worker
@@ -35,40 +24,12 @@ module RocketJob
           #   skip_job_ids [Array<BSON::ObjectId>]
           #     Job ids to exclude when looking for the next job
           def self.rocket_job_retrieve(worker_name, skip_job_ids = nil)
-            run_at = [
-              {run_at: {'$exists' => false}},
-              {run_at: {'$lte' => Time.now}}
-            ]
-            update = query = nil
-            if defined?(RocketJobPro)
-              query  = {
-                '$and' => [
-                  {
-                    '$or' => [
-                      {'state' => 'queued'}, # Jobs
-                      {'state' => 'running', 'sub_state' => :processing} # Slices
-                    ]
-                  },
-                  {
-                    '$or' => run_at
-                  }
-                ]
-              }
-              update = {'$set' => {'worker_name' => worker_name, 'state' => 'running'}}
-            else
-              query  = {'state' => 'queued', '$or' => run_at}
-              update = {'$set' => {'worker_name' => worker_name, 'state' => 'running', 'started_at' => Time.now}}
-            end
+            query  = queued_now
+            update = {'$set' => {'worker_name' => worker_name, 'state' => 'running', 'started_at' => Time.now}}
 
-            query['_id'] = {'$nin' => skip_job_ids} if skip_job_ids && skip_job_ids.size > 0
+            query  = query.where(:id.nin => skip_job_ids) if skip_job_ids && skip_job_ids.size > 0
 
-            if doc = find_and_modify(
-              query:  query,
-              sort:   {priority: 1, _id: 1},
-              update: update
-            )
-              load(doc)
-            end
+            query.sort(priority: 1, _id: 1).find_one_and_update(update)
           end
 
           # Returns [Hash<String:Integer>] of the number of jobs in each state
@@ -114,7 +75,7 @@ module RocketJob
 
             # Calculate :queued_now and :scheduled if there are queued jobs
             if queued_count = counts[:queued]
-              scheduled_count = RocketJob::Job.where(state: :queued, run_at: {'$gt' => Time.now}).count
+              scheduled_count = RocketJob::Job.scheduled.count
               if scheduled_count > 0
                 queued_now_count    = queued_count - scheduled_count
                 counts[:queued_now] = queued_count - scheduled_count if queued_now_count > 0
@@ -133,7 +94,7 @@ module RocketJob
           return super unless destroy_on_complete
           begin
             super
-          rescue MongoMapper::DocumentNotFound
+          rescue Mongoid::Errors::DocumentNotFound
             unless completed?
               self.state = :completed
               rocket_job_set_completed_at
@@ -143,23 +104,6 @@ module RocketJob
           end
         end
 
-        private
-
-        # After this model is loaded, convert any hashes in the arguments list to HashWithIndifferentAccess
-        def load_from_database(*args)
-          super
-          if arguments.present?
-            self.arguments = arguments.collect { |i| i.is_a?(BSON::OrderedHash) ? i.with_indifferent_access : i }
-          end
-        end
-
-        # Apply RocketJob defaults after initializing default values
-        # but before setting attributes. after_initialize is too late
-        def initialize_default_values(except = {})
-          super
-          rocket_job_set_defaults
-        end
-
       end
     end
   end

data/lib/rocket_job/plugins/job/worker.rb

@@ -8,15 +8,15 @@ module RocketJob
       module Worker
         extend ActiveSupport::Concern
 
-        included do
+        module ClassMethods
           # Run this job later
           #
           # Saves it to the database for processing later by workers
-          def self.perform_later(*args, &block)
+          def perform_later(args, &block)
             if RocketJob::Config.inline_mode
-              perform_now(*args, &block)
+              perform_now(args, &block)
             else
-              job = new(arguments: args)
+              job = new(args)
               block.call(job) if block
               job.save!
               job
@@ -28,8 +28,8 @@ module RocketJob
           # The job is not saved to the database since it is processed entriely in memory
           # As a result before_save and before_destroy callbacks will not be called.
           # Validations are still called however prior to calling #perform
-          def self.perform_now(*args, &block)
-            job = new(arguments: args)
+          def perform_now(args, &block)
+            job = new(args)
             block.call(job) if block
             job.perform_now
             job
@@ -48,7 +48,7 @@ module RocketJob
           #
           # Note:
           #   If a job is in queued state it will be started
-          def self.rocket_job_next_job(worker_name, skip_job_ids = nil)
+          def rocket_job_next_job(worker_name, skip_job_ids = nil)
             while (job = rocket_job_retrieve(worker_name, skip_job_ids))
               case
               when job.running?
@@ -67,16 +67,13 @@ module RocketJob
             end
           end
 
-          # Requeues all jobs that were running on worker that died
-          def self.requeue_dead_worker(worker_name)
-            # TODO Need to requeue paused, failed since user may have transitioned job before it finished
-            running.each do |job|
-              job.requeue!(worker_name) if job.may_requeue?(worker_name)
+          # Requeues all jobs that were running on a server that died
+          def requeue_dead_server(server_name)
+            # Need to requeue paused, failed since user may have transitioned job before it finished
+            where(:state.in => [:running, :paused, :faled]).each do |job|
+              job.requeue!(server_name) if job.may_requeue?(server_name)
             end
           end
-
-          # Turn off embedded callbacks. Slow and not used for Jobs
-          embedded_callbacks_off
         end
 
         # Runs the job now in the current thread.
@@ -91,14 +88,9 @@ module RocketJob
         #
         # Exceptions are _not_ suppressed and should be handled by the caller.
         def perform_now
-          # Call validations
-          if respond_to?(:validate!)
-            validate!
-          elsif invalid?
-            raise(MongoMapper::DocumentNotValid, self)
-          end
-          worker = RocketJob::Worker.new(name: 'inline')
-          worker.started
+          raise(Mongoid::Errors::Validations, self) unless valid?
+
+          worker = RocketJob::Worker.new(inline: true)
           start if may_start?
           # Re-Raise exceptions
           rocket_job_work(worker, true) if running?
@@ -114,7 +106,7 @@ module RocketJob
         # The job is automatically saved only if an exception is raised in the supplied block.
         #
         # worker_name: [String]
-        #   Name of the worker on which the exception has occurred
+        #   Name of the server on which the exception has occurred
         #
         # re_raise_exceptions: [true|false]
         #   Re-raise the exception after updating the job
@@ -150,10 +142,10 @@ module RocketJob
             run_callbacks :perform do
               # Allow callbacks to fail, complete or abort the job
               if running?
-                ret = perform(*arguments)
+                ret = perform
                 if collect_output?
                   # Result must be a Hash, if not put it in a Hash
-                  self.result = (ret.is_a?(Hash) || ret.is_a?(BSON::OrderedHash)) ? ret : {result: ret}
+                  self.result = ret.is_a?(Hash) ? ret : {'result' => ret}
                 end
               end
             end
@@ -166,6 +158,12 @@ module RocketJob
           false
         end
 
+        # Returns [Hash<String:[Array<ActiveWorker>]>] All servers actively working on this job
+        def rocket_job_active_servers
+          return {} unless running?
+          {worker_name => [ActiveServer.new(worker_name, started_at, self)]}
+        end
+
       end
     end
   end

data/lib/rocket_job/plugins/processing_window.rb

@@ -20,13 +20,10 @@ module RocketJob
     # class BusinessHoursJob < RocketJob::Job
     #   include RocketJob::Plugins::ProcessingWindow
     #
-    #   # Set the default processing_window
-    #   rocket_job do |job|
-    #     # The start of the processing window
-    #     job.processing_schedule = "30 8 * * * America/New_York"
-    #     # How long the processing window is:
-    #     job.processing_duration = 12.hours
-    #   end
+    #   # The start of the processing window
+    #   self.processing_schedule = "30 8 * * * America/New_York"
+    #   # How long the processing window is:
+    #   self..processing_duration = 12.hours
     #
     #   def perform
     #     # Job will only run between 8:30am and 8:30pm Eastern
@@ -41,8 +38,8 @@ module RocketJob
       extend ActiveSupport::Concern
 
       included do
-        key :processing_schedule, String
-        key :processing_duration, Integer
+        field :processing_schedule, type: String, class_attribute: true
+        field :processing_duration, type: Integer, class_attribute: true
 
         before_create :rocket_job_processing_window_set_run_at
         before_retry :rocket_job_processing_window_set_run_at

data/lib/rocket_job/plugins/retry.rb

@@ -26,9 +26,7 @@ module RocketJob
     #   include RocketJob::Plugins::Retry
     #
     #   # Set the default retry_count
-    #   rocket_job do |job|
-    #     job.max_retries = 3
-    #   end
+    #   self.max_retries = 3
     #
     #   def perform
     #     puts "DONE"
@@ -52,13 +50,10 @@ module RocketJob
 
         # Maximum number of times to retry this job
         # 25 is approximately 3 weeks of retries
-        key :max_retries, Integer, default: 25
+        field :max_retries, type: Integer, default: 25, class_attribute: true, user_editable: true
 
         # List of times when this job failed
-        key :failed_times, Array
-
-        # Make max_retries editable in Rocket Job Mission Control
-        public_rocket_job_properties :max_retries
+        field :failed_times, type: Array, default: []
 
         validates_presence_of :max_retries
       end

data/lib/rocket_job/plugins/singleton.rb

@@ -17,7 +17,7 @@ module RocketJob
 
         # Returns [true|false] whether another instance of this job is already active
         def rocket_job_singleton_active?
-          self.class.where(state: [:running, :queued], _id: {'$ne' => id}).exists?
+          self.class.where(:state.in => [:running, :queued], :id.ne => id).exists?
         end
       end
 

data/lib/rocket_job/plugins/state_machine.rb

@@ -86,13 +86,7 @@ module RocketJob
           write_attribute(attr_name, state)
 
           begin
-            if aasm_skipping_validations(name)
-              saved = save(validate: false)
-              write_attribute(attr_name, old_value) unless saved
-              saved
-            else
-              save!
-            end
+            save!
           rescue Exception => exc
             write_attribute(attr_name, old_value)
             raise(exc)

data/lib/rocket_job/server.rb

@@ -0,0 +1,352 @@
+# encoding: UTF-8
+require 'concurrent'
+module RocketJob
+  # Server
+  #
+  # On startup a server instance will automatically register itself
+  # if not already present
+  #
+  # Starting a server in the foreground:
+  #   - Using a Rails runner:
+  #     bin/rocketjob
+  #
+  # Starting a server in the background:
+  #   - Using a Rails runner:
+  #     nohup bin/rocketjob --quiet 2>&1 1>output.log &
+  #
+  # Stopping a server:
+  #   - Stop the server via the Web UI
+  #   - Send a regular kill signal to make it shutdown once all active work is complete
+  #       kill <pid>
+  #   - Or, use the following Ruby code:
+  #     server = RocketJob::Server.where(name: 'server name').first
+  #     server.stop!
+  #
+  #   Sending the kill signal locally will result in starting the shutdown process
+  #   immediately. Via the UI or Ruby code the server can take up to 15 seconds
+  #   (the heartbeat interval) to start shutting down.
+  class Server
+    include Plugins::Document
+    include Plugins::StateMachine
+    include SemanticLogger::Loggable
+
+    # Unique Name of this server instance
+    #   Default: `host name:PID`
+    # The unique name is used on re-start to re-queue any jobs that were being processed
+    # at the time the server unexpectedly terminated, if any
+    field :name, type: String, default: -> { "#{SemanticLogger.host}:#{$$}" }
+
+    # The maximum number of workers this server should start
+    #   If set, it will override the default value in RocketJob::Config
+    field :max_workers, type: Integer, default: -> { Config.instance.max_worker_threads }
+
+    # When this server process was started
+    field :started_at, type: Time
+
+    # The heartbeat information for this server
+    embeds_one :heartbeat, class_name: 'RocketJob::Heartbeat'
+
+    # Current state
+    #   Internal use only. Do not set this field directly
+    field :state, type: Symbol, default: :starting
+
+    index({name: 1}, background: true, unique: true, drop_dups: true)
+
+    validates_presence_of :state, :name, :max_workers
+
+    # States
+    #   :starting -> :running -> :paused
+    #                         -> :stopping
+    aasm column: :state do
+      state :starting, initial: true
+      state :running
+      state :paused
+      state :stopping
+
+      event :started do
+        transitions from: :starting, to: :running
+        before do
+          self.started_at = Time.now
+        end
+      end
+
+      event :pause do
+        transitions from: :running, to: :paused
+      end
+
+      event :resume do
+        transitions from: :paused, to: :running
+      end
+
+      event :stop do
+        transitions from: :running, to: :stopping
+        transitions from: :paused, to: :stopping
+        transitions from: :starting, to: :stopping
+      end
+    end
+
+    # Requeue any jobs being worked by this server when it is destroyed
+    before_destroy :requeue_jobs
+
+    # Destroy's all instances of zombie server and requeue any jobs still "running"
+    # on those servers
+    def self.destroy_zombies
+      count = 0
+      each do |server|
+        next unless server.zombie?
+        logger.warn "Destroying zombie server #{server.name}, and requeueing its jobs"
+        server.destroy
+        count += 1
+      end
+      count
+    end
+
+    # Stop all running, paused, or starting servers
+    def self.stop_all
+      where(:state.in => [:running, :paused, :starting]).each(&:stop!)
+    end
+
+    # Pause all running servers
+    def self.pause_all
+      running.each(&:pause!)
+    end
+
+    # Resume all paused servers
+    def self.resume_all
+      paused.each(&:resume!)
+    end
+
+    # Returns [Hash<String:Integer>] of the number of servers in each state.
+    # Note: If there are no servers in that particular state then the hash will not have a value for it.
+    #
+    # Example servers in every state:
+    #   RocketJob::Server.counts_by_state
+    #   # => {
+    #          :aborted => 1,
+    #          :completed => 37,
+    #          :failed => 1,
+    #          :paused => 3,
+    #          :queued => 4,
+    #          :running => 1,
+    #          :queued_now => 1,
+    #          :scheduled => 3
+    #        }
+    #
+    # Example no servers active:
+    #   RocketJob::Server.counts_by_state
+    #   # => {}
+    def self.counts_by_state
+      counts = {}
+      collection.aggregate([
+        {
+          '$group' => {
+            _id:   '$state',
+            count: {'$sum' => 1}
+          }
+        }
+      ]
+      ).each do |result|
+        counts[result['_id'].to_sym] = result['count']
+      end
+      counts
+    end
+
+    # On MRI the 'concurrent-ruby-ext' gem may not be loaded
+    if defined?(Concurrent::JavaAtomicBoolean) || defined?(Concurrent::CAtomicBoolean)
+      # Returns [true|false] whether the shutdown indicator has been set for this server process
+      def self.shutdown?
+        @@shutdown.value
+      end
+
+      # Set shutdown indicator for this server process
+      def self.shutdown!
+        @@shutdown.make_true
+      end
+
+      @@shutdown = Concurrent::AtomicBoolean.new(false)
+    else
+      # Returns [true|false] whether the shutdown indicator has been set for this server process
+      def self.shutdown?
+        @@shutdown
+      end
+
+      # Set shutdown indicator for this server process
+      def self.shutdown!
+        @@shutdown = true
+      end
+
+      @@shutdown = false
+    end
+
+    # Run the server process
+    # Attributes supplied are passed to #new
+    def self.run(attrs = {})
+      Thread.current.name = 'rocketjob main'
+      # Create Indexes on server startup
+      Mongoid::Tasks::Database.create_indexes
+      register_signal_handlers
+
+      server = create!(attrs)
+      server.send(:run)
+
+    ensure
+      server.destroy if server
+    end
+
+    # Returns [Boolean] whether the server is shutting down
+    def shutdown?
+      self.class.shutdown? || !running?
+    end
+
+    # Returns [true|false] if this server has missed at least the last 4 heartbeats
+    #
+    # Possible causes for a server to miss its heartbeats:
+    # - The server process has died
+    # - The server process is "hanging"
+    # - The server is no longer able to communicate with the MongoDB Server
+    def zombie?(missed = 4)
+      return false unless running? || stopping?
+      return true if heartbeat.nil? || heartbeat.updated_at.nil?
+      dead_seconds = Config.instance.heartbeat_seconds * missed
+      (Time.now - heartbeat.updated_at) >= dead_seconds
+    end
+
+    private
+
+    attr_reader :workers
+
+    # Returns [Array<Worker>] collection of workers
+    def workers
+      @workers ||= []
+    end
+
+    # Management Thread
+    def run
+      logger.info "Using MongoDB Database: #{RocketJob::Job.collection.database.name}"
+      build_heartbeat(updated_at: Time.now, current_threads: 0)
+      started!
+      adjust_workers(true)
+      logger.info "RocketJob Server started with #{workers.size} workers running"
+
+      while running? || paused?
+        sleep Config.instance.heartbeat_seconds
+
+        find_and_update(
+          'heartbeat.updated_at'      => Time.now,
+          'heartbeat.current_threads' => worker_count
+        )
+
+        # In case number of threads has been modified
+        adjust_workers
+
+        # Stop server if shutdown indicator was set
+        stop! if self.class.shutdown? && may_stop?
+      end
+
+      logger.info 'Waiting for workers to stop'
+      # Tell each worker to shutdown cleanly
+      workers.each(&:shutdown!)
+
+      while worker = workers.first
+        if worker.join(5)
+          # Worker thread is dead
+          workers.shift
+        else
+          # Timeout waiting for worker to stop
+          begin
+            find_and_update(
+              'heartbeat.updated_at'      => Time.now,
+              'heartbeat.current_threads' => worker_count
+            )
+          rescue Mongoid::Errors::DocumentNotFound
+            logger.warn('Server has been destroyed. Going down hard!')
+            break
+          end
+        end
+      end
+
+      # Logs the backtrace for each running worker
+      if SemanticLogger::VERSION.to_i >= 4
+        workers.each { |thread| logger.backtrace(thread: thread) }
+      end
+      logger.info 'Shutdown'
+    rescue Exception => exc
+      logger.error('RocketJob::Server is stopping due to an exception', exc)
+    end
+
+    # Returns [Fixnum] number of workers (threads) that are alive
+    def worker_count
+      workers.count(&:alive?)
+    end
+
+    def next_worker_id
+      @worker_id ||= 0
+      @worker_id += 1
+    end
+
+    # Re-adjust the number of running workers to get it up to the
+    # required number of workers
+    #   Parameters
+    #     stagger_workers
+    #       Whether to stagger when the workers poll for work the first time
+    #       It spreads out the queue polling over the max_poll_seconds so
+    #       that not all workers poll at the same time
+    #       The worker also respond faster than max_poll_seconds when a new
+    #       job is added.
+    def adjust_workers(stagger_workers=false)
+      count = worker_count
+      # Cleanup workers that have stopped
+      if count != workers.count
+        logger.info "Cleaning up #{workers.count - count} workers that went away"
+        workers.delete_if { |t| !t.alive? }
+      end
+
+      # Need to add more workers?
+      if count < max_workers
+        worker_count = max_workers - count
+        logger.info "Starting #{worker_count} workers"
+        worker_count.times.each do
+          sleep (Config.instance.max_poll_seconds.to_f / max_workers) * (next_worker_id - 1) if stagger_workers
+          return if shutdown?
+          # Start worker
+          begin
+            workers << Worker.new(id: next_worker_id, server_name: name)
+          rescue Exception => exc
+            logger.fatal('Cannot start worker', exc)
+          end
+        end
+      end
+    end
+
+    # Register handlers for the various signals
+    # Term:
+    #   Perform clean shutdown
+    #
+    def self.register_signal_handlers
+      begin
+        Signal.trap 'SIGTERM' do
+          shutdown!
+          message = 'Shutdown signal (SIGTERM) received. Will shutdown as soon as active jobs/slices have completed.'
+          # Logging uses a mutex to access Queue on MRI/CRuby
+          defined?(JRuby) ? logger.warn(message) : puts(message)
+        end
+
+        Signal.trap 'INT' do
+          shutdown!
+          message = 'Shutdown signal (INT) received. Will shutdown as soon as active jobs/slices have completed.'
+          # Logging uses a mutex to access Queue on MRI/CRuby
+          defined?(JRuby) ? logger.warn(message) : puts(message)
+        end
+      rescue StandardError
+        logger.warn 'SIGTERM handler not installed. Not able to shutdown gracefully'
+      end
+    end
+
+    # Requeue any jobs assigned to this server when it is destroyed
+    def requeue_jobs
+      RocketJob::Job.requeue_dead_server(name)
+    end
+
+  end
+end
+