rocketjob 1.3.0 → 2.0.0.rc1

data/lib/rocket_job/job.rb

@@ -1,440 +1,16 @@
 # encoding: UTF-8
-require 'aasm'
 module RocketJob
   # The base job from which all jobs are created
   class Job
-    include MongoMapper::Document
-    include AASM
     include SemanticLogger::Loggable
-    include Concerns::Worker
-
-    # Prevent data in MongoDB from re-defining the model behavior
-    #self.static_keys = true
-
-    #
-    # User definable attributes
-    #
-    # The following attributes are set when the job is created
-    # @formatter:off
-
-    # Description for this job instance
-    key :description,             String
-
-    # Method that must be invoked to complete this job
-    key :perform_method,          Symbol, default: :perform
-
-    # Priority of this job as it relates to other jobs [1..100]
-    #   1: Highest Priority
-    #  50: Default Priority
-    # 100: Lowest Priority
-    #
-    # Example:
-    #   A job with a priority of 40 will execute before a job with priority 50
-    #
-    # In RocketJob Pro, if a SlicedJob is running and a higher priority job
-    # arrives, then the current job will complete the current slices and process
-    # the new higher priority job
-    key :priority,                Integer, default: 50
-
-    # Run this job no earlier than this time
-    key :run_at,                  Time
-
-    # If a job has not started by this time, destroy it
-    key :expires_at,              Time
-
-    # When specified a job will be re-scheduled to run at it's next scheduled interval
-    # Format is the same as cron.
-    # #TODO Future capability.
-    #key :schedule,                String
-
-    # When the job completes destroy it from both the database and the UI
-    key :destroy_on_complete,     Boolean, default: true
-
-    # Any user supplied arguments for the method invocation
-    # All keys must be UTF-8 strings. The values can be any valid BSON type:
-    #   Integer
-    #   Float
-    #   Time    (UTC)
-    #   String  (UTF-8)
-    #   Array
-    #   Hash
-    #   True
-    #   False
-    #   Symbol
-    #   nil
-    #   Regular Expression
-    #
-    # Note: Date is not supported, convert it to a UTC time
-    key :arguments,               Array
-
-    # Whether to store the results from this job
-    key :collect_output,          Boolean, default: false
-
-    # Raise or lower the log level when calling the job
-    # Can be used to reduce log noise, especially during high volume calls
-    # For debugging a single job can be logged at a low level such as :trace
-    #   Levels supported: :trace, :debug, :info, :warn, :error, :fatal
-    key :log_level,               Symbol
-
-    #
-    # Read-only attributes
-    #
-
-    # Current state, as set by AASM
-    key :state,                   Symbol, default: :queued
-
-    # When the job was created
-    key :created_at,              Time, default: -> { Time.now }
-
-    # When processing started on this job
-    key :started_at,              Time
-
-    # When the job completed processing
-    key :completed_at,            Time
-
-    # Number of times that this job has failed to process
-    key :failure_count,           Integer, default: 0
-
-    # This name of the worker that this job is being processed by, or was processed by
-    key :worker_name,             String
-
-    #
-    # Values that jobs can update during processing
-    #
-
-    # Allow a job to updates its estimated progress
-    # Any integer from 0 to 100
-    key :percent_complete,        Integer, default: 0
-
-    # Store the last exception for this job
-    one :exception,               class_name: 'RocketJob::JobException'
-
-    # Store the Hash result from this job if collect_output is true,
-    # and the job returned actually returned a Hash, otherwise nil
-    # Not applicable to SlicedJob jobs, since its output is stored in a
-    # separate collection
-    key :result,                  Hash
-
-    # Store all job types in this collection
-    set_collection_name 'rocket_job.jobs'
-
-    validates_presence_of :state, :failure_count, :created_at, :perform_method
-    validates :priority, inclusion: 1..100
-    validates :log_level, inclusion: SemanticLogger::LEVELS + [nil]
-
-    # User definable properties in Dirmon Entry
-    def self.rocket_job_properties
-      @rocket_job_properties ||= (self == RocketJob::Job ? [] : superclass.rocket_job_properties)
-    end
-
-    # Add to user definable properties in Dirmon Entry
-    def self.public_rocket_job_properties(*properties)
-      rocket_job_properties.concat(properties).uniq!
-    end
-
-    # User definable properties in Dirmon Entry
-    public_rocket_job_properties :description, :priority, :perform_method, :log_level, :arguments
-
-    # State Machine events and transitions
-    #
-    #   :queued -> :running -> :completed
-    #                       -> :paused     -> :running
-    #                                      -> :aborted
-    #                       -> :failed     -> :running
-    #                                      -> :aborted
-    #                       -> :aborted
-    #                       -> :queued (when a worker dies)
-    #           -> :aborted
-    aasm column: :state do
-      # Job has been created and is queued for processing ( Initial state )
-      state :queued, initial: true
-
-      # Job is running
-      state :running
-
-      # Job has completed processing ( End state )
-      state :completed
-
-      # Job is temporarily paused and no further processing will be completed
-      # until this job has been resumed
-      state :paused
-
-      # Job failed to process and needs to be manually re-tried or aborted
-      state :failed
-
-      # Job was aborted and cannot be resumed ( End state )
-      state :aborted
-
-      event :start, before: :before_start do
-        transitions from: :queued, to: :running
-      end
-
-      event :complete, before: :before_complete do
-        after do
-          destroy if destroy_on_complete
-        end
-        transitions from: :running, to: :completed
-      end
-
-      event :fail, before: :before_fail do
-        transitions from: :queued,  to: :failed
-        transitions from: :running, to: :failed
-        transitions from: :paused,  to: :failed
-      end
-
-      event :retry, before: :before_retry do
-        transitions from: :failed, to: :queued
-      end
-
-      event :pause, before: :before_pause do
-        transitions from: :running, to: :paused
-      end
-
-      event :resume, before: :before_resume do
-        transitions from: :paused, to: :running
-      end
-
-      event :abort, before: :before_abort do
-        transitions from: :running, to: :aborted
-        transitions from: :queued,  to: :aborted
-        transitions from: :failed,  to: :aborted
-        transitions from: :paused,  to: :aborted
-      end
-
-      event :requeue, before: :before_requeue do
-        transitions from: :running, to: :queued
-      end
-    end
-    # @formatter:on
-
-    # Create indexes
-    def self.create_indexes
-      # Used by find_and_modify in .next_job
-      ensure_index({state: 1, run_at: 1, priority: 1, created_at: 1, sub_state: 1}, background: true)
-      # Remove outdated index if present
-      drop_index('state_1_priority_1_created_at_1_sub_state_1') rescue nil
-      # Used by Mission Control
-      ensure_index [[:created_at, 1]]
-    end
-
-    # Requeues all jobs that were running on worker that died
-    def self.requeue_dead_worker(worker_name)
-      running.each { |job| job.requeue!(worker_name) }
-    end
-
-    # Pause all running jobs
-    def self.pause_all
-      running.each(&:pause!)
-    end
-
-    # Resume all paused jobs
-    def self.resume_all
-      paused.each(&:resume!)
-    end
-
-    # Returns the number of required arguments for this job
-    def self.argument_count(method = :perform)
-      instance_method(method).arity
-    end
-
-    # Override parent defaults
-    def self.rocket_job(&block)
-      @rocket_job_defaults = block
-      self
-    end
-
-    # Returns [true|false] whether to collect the results from running this batch
-    def collect_output?
-      collect_output == true
-    end
-
-    # Returns [Float] the number of seconds the job has taken
-    # - Elapsed seconds to process the job from when a worker first started working on it
-    #   until now if still running, or until it was completed
-    # - Seconds in the queue if queued
-    def seconds
-      if completed_at
-        completed_at - (started_at || created_at)
-      elsif started_at
-        Time.now - started_at
-      else
-        Time.now - created_at
-      end
-    end
-
-    # Returns a human readable duration the job has taken
-    def duration
-      RocketJob.seconds_as_duration(seconds)
-    end
-
-    # A job has expired if the expiry time has passed before it is started
-    def expired?
-      started_at.nil? && expires_at && (expires_at < Time.now)
-    end
-
-    # Returns [Hash] status of this job
-    def as_json
-      attrs = serializable_hash(methods: [:seconds, :duration])
-      attrs.delete('result') unless collect_output?
-      case
-      when running?
-        attrs.delete('completed_at')
-        attrs.delete('result')
-        attrs
-      when paused?
-        attrs.delete('completed_at')
-        attrs.delete('result')
-        # Ensure 'paused_at' appears first in the hash
-        {'paused_at' => completed_at}.merge(attrs)
-      when aborted?
-        attrs.delete('completed_at')
-        attrs.delete('result')
-        {'aborted_at' => completed_at}.merge(attrs)
-      when failed?
-        attrs.delete('completed_at')
-        attrs.delete('result')
-        {'failed_at' => completed_at}.merge(attrs)
-      else
-        attrs
-      end
-    end
-
-    def status(time_zone = 'Eastern Time (US & Canada)')
-      h = as_json
-      h.delete('seconds')
-      h.delete('perform_method') if h['perform_method'] == :perform
-      h.dup.each_pair do |k, v|
-        case
-        when v.is_a?(Time)
-          h[k] = v.in_time_zone(time_zone).to_s
-        when v.is_a?(BSON::ObjectId)
-          h[k] = v.to_s
-        end
-      end
-      h
-    end
-
-    # Patch the way MongoMapper reloads a model
-    # Only reload MongoMapper attributes, leaving other instance variables untouched
-    def reload
-      if (doc = collection.find_one(_id: id))
-        # Clear out keys that are not returned during the reload from MongoDB
-        (keys.keys - doc.keys).each { |key| send("#{key}=", nil) }
-        initialize_default_values
-        load_from_database(doc)
-        self
-      else
-        if destroy_on_complete
-          self.state = :completed
-          before_complete
-        else
-          raise(MongoMapper::DocumentNotFound, "Document match #{_id.inspect} does not exist in #{collection.name} collection")
-        end
-      end
-    end
-
-    # Set exception information for this job and fail it
-    def fail(worker_name='user', exc_or_message='Job failed through user action')
-      if exc_or_message.is_a?(Exception)
-        self.exception        = JobException.from_exception(exc_or_message)
-        exception.worker_name = worker_name
-      else
-        build_exception(
-          class_name:  'RocketJob::JobException',
-          message:     exc_or_message,
-          backtrace:   [],
-          worker_name: worker_name
-        )
-      end
-      # not available as #super
-      aasm.current_event = :fail
-      aasm_fire_event(:fail, persist: false)
-    end
-
-    def fail!(worker_name='user', exc_or_message='Job failed through user action')
-      self.fail(worker_name, exc_or_message)
-      save!
-    end
-
-    # Requeue this running job since the worker assigned to it has died
-    def requeue!(worker_name_=nil)
-      return false if worker_name_ && (worker_name != worker_name_)
-      # not available as #super
-      aasm.current_event = :requeue!
-      aasm_fire_event(:requeue, persist: true)
-    end
-
-    # Requeue this running job since the worker assigned to it has died
-    def requeue(worker_name_=nil)
-      return false if worker_name_ && (worker_name != worker_name_)
-      # not available as #super
-      aasm.current_event = :requeue
-      aasm_fire_event(:requeue, persist: false)
-    end
-
-    protected
-
-    # Before events that can be overridden by child classes
-    def before_start
-      self.started_at = Time.now
-    end
-
-    def before_complete
-      self.percent_complete = 100
-      self.completed_at     = Time.now
-      self.worker_name      = nil
-    end
-
-    def before_fail
-      self.completed_at  = Time.now
-      self.worker_name   = nil
-      self.failure_count += 1
-    end
-
-    def before_retry
-      self.completed_at = nil
-      self.exception    = nil
-    end
-
-    def before_pause
-      self.completed_at = Time.now
-      self.worker_name  = nil
-    end
-
-    def before_resume
-      self.completed_at = nil
-    end
-
-    def before_abort
-      self.completed_at = Time.now
-      self.worker_name  = nil
-    end
-
-    def before_requeue
-      self.started_at  = nil
-      self.worker_name = nil
-    end
-
-    private
-
-    # After this model is loaded, convert any hashes in the arguments list to HashWithIndifferentAccess
-    def load_from_database(*args)
-      super
-      if arguments.present?
-        self.arguments = arguments.collect { |i| i.is_a?(BSON::OrderedHash) ? i.with_indifferent_access : i }
-      end
-    end
-
-    def self.apply_defaults(job)
-      @rocket_job_defaults.call(job) if @rocket_job_defaults
-    end
-
-    # Apply RocketJob defaults after initializing default values
-    # but before setting attributes
-    def initialize_default_values(except = {})
-      super
-      self.class.apply_defaults(self)
-    end
-
+    include Plugins::Document
+    include Plugins::Job::Model
+    include Plugins::Job::Persistence
+    include Plugins::Job::Callbacks
+    include Plugins::Job::Logger
+    include Plugins::StateMachine
+    include Plugins::Job::StateMachine
+    include Plugins::Job::Worker
+    include Plugins::Job::Defaults
   end
 end

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -10,7 +10,7 @@ module RocketJob
     # * On each subsequent Dirmon run it checks the size of each file against the
     #   previous list of known files, and only if the file size has not changed
     #   the corresponding job is started for that file.
-    # * If the job implements #file_store_upload or #upload, that method is called
+    # * If the job implements #upload, that method is called
     #   and then the file is deleted, or moved to the archive_directory if supplied
 
     # * Otherwise, the file is moved to the supplied archive_directory (defaults to
@@ -18,22 +18,26 @@ module RocketJob
     #   file name of the archived file is passed into the job as it's first argument.
 
     # Note:
-    # - Jobs that do not implement #file_store_upload or #upload _must_ have a
-    #   Hash as the first argument
+    # - Jobs that do not implement #upload _must_ have a Hash as the first argument
     #
     # With RocketJob Pro, the file is automatically uploaded into the job itself
     # using the job's #upload method, after which the file is archived or deleted
     # if no archive_directory was specified in the DirmonEntry.
     #
     # To start Dirmon for the first time
+    #   RocketJob::Jobs::DirmonJob.create!
     #
+    # If another DirmonJob instance is already queued or running, then the create
+    # above will fail with:
+    #   MongoMapper::DocumentNotValid: Validation failed: State Another instance of this job is already queued or running
     #
-    # Note:
-    #   Use `DirmonJob.start` to prevent creating multiple Dirmon jobs, otherwise
-    #   it will result in multiple jobs being started
+    # Or to start DirmonJob and ignore errors if already running
+    #   RocketJob::Jobs::DirmonJob.create
     class DirmonJob < RocketJob::Job
       # Only allow one DirmonJob instance to be running at a time
-      include RocketJob::Concerns::Singleton
+      include RocketJob::Plugins::Singleton
+      # Start a new job when this one completes, fails, or aborts
+      include RocketJob::Plugins::Restart
 
       rocket_job do |job|
         job.priority = 40
@@ -43,23 +47,22 @@ module RocketJob
       key :check_seconds, Float, default: 300.0
       key :previous_file_names, Hash # Hash[file_name, size]
 
+      before_create :set_run_at
+
       # Iterate over each Dirmon entry looking for new files
       # If a new file is found, it is not processed immediately, instead
       # it is passed to the next run of this job along with the file size.
       # If the file size has not changed, the Job is kicked off.
       def perform
         check_directories
-      ensure
-        # Run again in the future, even if this run fails with an exception
-        self.class.create!(
-          previous_file_names: previous_file_names,
-          priority:            priority,
-          check_seconds:       check_seconds,
-          run_at:              Time.now + check_seconds
-        )
       end
 
-      protected
+      private
+
+      # Set a run_at when a new instance of this job is created
+      def set_run_at
+        self.run_at = Time.now + check_seconds
+      end
 
       # Checks the directories for new files, starting jobs if files have not changed
       # since the last run
@@ -70,7 +73,7 @@ module RocketJob
             # BSON Keys cannot contain periods
             key           = pathname.to_s.gsub('.', '_')
             previous_size = previous_file_names[key]
-            if (size = check_file(entry, pathname, previous_size))
+            if size = check_file(entry, pathname, previous_size)
               new_file_names[key] = size
             end
           end
@@ -91,9 +94,6 @@ module RocketJob
           # Keep for the next run
           size
         end
-      rescue Errno::ENOENT => exc
-        # File may have been deleted since the scan was performed
-        nil
       end
 
     end

data/lib/rocket_job/jobs/simple_job.rb

@@ -0,0 +1,12 @@
+module RocketJob
+  module Jobs
+
+    class SimpleJob < RocketJob::Job
+      # No operation, used for performance testing
+      def perform
+        sleep 1
+      end
+    end
+
+  end
+end

data/lib/rocket_job/plugins/document.rb

@@ -0,0 +1,69 @@
+# encoding: UTF-8
+require 'active_support/concern'
+require 'mongo'
+require 'mongo_ha'
+require 'mongo_mapper'
+
+module RocketJob
+  module Plugins
+    # Prevent more than one instance of this job class from running at a time
+    module Document
+      extend ActiveSupport::Concern
+      include MongoMapper::Document
+
+      included do
+        # Add after_initialize & after_find callbacks
+        define_model_callbacks :initialize, :find, :only => [:after]
+
+        # Prevent data in MongoDB from re-defining the model behavior
+        #self.static_keys = true
+
+        # Turn off embedded callbacks. Slow and not used for Jobs
+        embedded_callbacks_off
+      end
+
+      # Patch the way MongoMapper reloads a model
+      def reload
+        if doc = collection.find_one(:_id => id)
+          # Clear out keys that are not returned during the reload from MongoDB
+          (keys.keys - doc.keys).each { |key| send("#{key}=", nil) }
+          initialize_default_values
+          load_from_database(doc)
+          self
+        else
+          raise MongoMapper::DocumentNotFound, "Document match #{_id.inspect} does not exist in #{collection.name} collection"
+        end
+      end
+
+      # Add after_initialize callbacks
+      # TODO: Remove after new MongoMapper gem is released
+      #       Also remove define_model_callbacks above
+      def initialize(*)
+        run_callbacks(:initialize) { super }
+      end
+
+      def initialize_from_database(*)
+        run_callbacks(:initialize) do
+          run_callbacks(:find) do
+            super
+          end
+        end
+      end
+
+      private
+
+      def update_attributes_and_reload(attrs)
+        if doc = collection.find_and_modify(query: {:_id => id}, update: {'$set' => attrs})
+          # Clear out keys that are not returned during the reload from MongoDB
+          (keys.keys - doc.keys).each { |key| send("#{key}=", nil) }
+          initialize_default_values
+          load_from_database(doc)
+          self
+        else
+          raise MongoMapper::DocumentNotFound, "Document match #{_id.inspect} does not exist in #{collection.name} collection"
+        end
+      end
+
+    end
+  end
+end