rocketjob 1.3.0 → 2.0.0.rc1

data/lib/rocket_job/plugins/job/callbacks.rb

@@ -0,0 +1,92 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    module Job
+      # Define before and after callbacks
+      #
+      # Before callbacks are called in the order they are defined.
+      # After callbacks are called in the _reverse_ order to which they were defined.
+      #
+      # Example:
+      #    before_1
+      #    before_2
+      #    perform
+      #    after_2
+      #    after_1
+      #
+      # Example including around callbacks:
+      #
+      # class MyJob < RocketJob::Job
+      #   before_perform do
+      #     puts "BEFORE 1"
+      #   end
+      #
+      #   around_perform do |job, block|
+      #     puts "AROUND 1 BEFORE"
+      #     block.call
+      #     puts "AROUND 1 AFTER"
+      #   end
+      #
+      #   before_perform do
+      #     puts "BEFORE 2"
+      #   end
+      #
+      #   after_perform do
+      #     puts "AFTER 1"
+      #   end
+      #
+      #   around_perform do |job, block|
+      #     puts "AROUND 2 BEFORE"
+      #     block.call
+      #     puts "AROUND 2 AFTER"
+      #   end
+      #
+      #   after_perform do
+      #     puts "AFTER 2"
+      #   end
+      #
+      #   def perform
+      #     puts "PERFORM"
+      #     23
+      #   end
+      # end
+      #
+      # MyJob.new.perform_now
+      #
+      # Output from the example above
+      #
+      #  BEFORE 1
+      #  AROUND 1 BEFORE
+      #  BEFORE 2
+      #  AROUND 2 BEFORE
+      #  PERFORM
+      #  AFTER 2
+      #  AROUND 2 AFTER
+      #  AFTER 1
+      #  AROUND 1 AFTER
+      module Callbacks
+        extend ActiveSupport::Concern
+        include ActiveSupport::Callbacks
+
+        included do
+          define_callbacks :perform
+
+          def self.before_perform(*filters, &blk)
+            set_callback(:perform, :before, *filters, &blk)
+          end
+
+          def self.after_perform(*filters, &blk)
+            set_callback(:perform, :after, *filters, &blk)
+          end
+
+          def self.around_perform(*filters, &blk)
+            set_callback(:perform, :around, *filters, &blk)
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/job/defaults.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    module Job
+      # Allow each child job to set its own defaults
+      module Defaults
+        extend ActiveSupport::Concern
+
+        included do
+          # Copy parent job defaults
+          def self.inherited(base)
+            super
+            @rocket_job_defaults.each { |block| base.rocket_job(&block) } if @rocket_job_defaults
+          end
+
+          # Override parent defaults
+          def self.rocket_job(&block)
+            (@rocket_job_defaults ||=[]) << block
+          end
+
+          private
+
+          def self.rocket_job_defaults
+            @rocket_job_defaults
+          end
+
+          # Apply defaults after creating the model but before applying values
+          def rocket_job_set_defaults
+            if defaults = self.class.rocket_job_defaults
+              defaults.each { |block| block.call(self) }
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/job/logger.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    module Job
+      module Logger
+        extend ActiveSupport::Concern
+
+        included do
+          around_perform :rocket_job_around_logger
+        end
+
+        private
+
+        # Add logging around the perform call
+        #   - metric allows duration to be forwarded to statsd, etc.
+        #   - log_exception logs entire exception if raised
+        #   - on_exception_level changes log level from info to error on exception
+        #   - silence noisy jobs by raising log level
+        def rocket_job_around_logger(&block)
+          logger.info('Start #perform')
+          logger.benchmark_info(
+            'Completed #perform',
+            metric:             "rocketjob/#{self.class.name.underscore}/perform",
+            log_exception:      :full,
+            on_exception_level: :error,
+            silence:            log_level,
+            &block
+          )
+        end
+
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/job/model.rb

@@ -0,0 +1,288 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    module Job
+      # Prevent more than one instance of this job class from running at a time
+      module Model
+        extend ActiveSupport::Concern
+
+        included do
+          #
+          # User definable attributes
+          #
+          # The following attributes are set when the job is created
+          # @formatter:off
+
+          # Description for this job instance
+          key :description,             String
+
+          # Priority of this job as it relates to other jobs [1..100]
+          #   1: Highest Priority
+          #  50: Default Priority
+          # 100: Lowest Priority
+          #
+          # Example:
+          #   A job with a priority of 40 will execute before a job with priority 50
+          #
+          # In RocketJob Pro, if a SlicedJob is running and a higher priority job
+          # arrives, then the current job will complete the current slices and process
+          # the new higher priority job
+          key :priority,                Integer, default: 50
+
+          # Run this job no earlier than this time
+          key :run_at,                  Time
+
+          # If a job has not started by this time, destroy it
+          key :expires_at,              Time
+
+          # When the job completes destroy it from both the database and the UI
+          key :destroy_on_complete,     Boolean, default: true
+
+          # Any user supplied arguments for the method invocation
+          # All keys must be UTF-8 strings. The values can be any valid BSON type:
+          #   Integer
+          #   Float
+          #   Time    (UTC)
+          #   String  (UTF-8)
+          #   Array
+          #   Hash
+          #   True
+          #   False
+          #   Symbol
+          #   nil
+          #   Regular Expression
+          #
+          # Note: Date is not supported, convert it to a UTC time
+          key :arguments,               Array
+
+          # Whether to store the results from this job
+          key :collect_output,          Boolean, default: false
+
+          # Raise or lower the log level when calling the job
+          # Can be used to reduce log noise, especially during high volume calls
+          # For debugging a single job can be logged at a low level such as :trace
+          #   Levels supported: :trace, :debug, :info, :warn, :error, :fatal
+          key :log_level,               Symbol
+
+          #
+          # Read-only attributes
+          #
+
+          # Current state, as set by the state machine. Do not modify this value directly.
+          key :state,                   Symbol, default: :queued
+
+          # When the job was created
+          key :created_at,              Time, default: -> { Time.now }
+
+          # When processing started on this job
+          key :started_at,              Time
+
+          # When the job completed processing
+          key :completed_at,            Time
+
+          # Number of times that this job has failed to process
+          key :failure_count,           Integer, default: 0
+
+          # This name of the worker that this job is being processed by, or was processed by
+          key :worker_name,             String
+
+          #
+          # Values that jobs can update during processing
+          #
+
+          # Allow a job to updates its estimated progress
+          # Any integer from 0 to 100
+          key :percent_complete,        Integer, default: 0
+
+          # Store the last exception for this job
+          one :exception,               class_name: 'RocketJob::JobException'
+
+          # Store the Hash result from this job if collect_output is true,
+          # and the job returned actually returned a Hash, otherwise nil
+          # Not applicable to SlicedJob jobs, since its output is stored in a
+          # separate collection
+          key :result,                  Hash
+
+          # @formatter:on
+
+          # Store all job types in this collection
+          set_collection_name 'rocket_job.jobs'
+
+          validates_presence_of :state, :failure_count, :created_at
+          validates :priority, inclusion: 1..100
+          validates :log_level, inclusion: SemanticLogger::LEVELS + [nil]
+
+          # Returns [String] the singular name for this job class
+          #
+          # Example:
+          #   job = DataStudyJob.new
+          #   job.underscore_name
+          #   # => "data_study"
+          def self.underscore_name
+            @underscore_name ||= name.sub(/Job$/, '').underscore
+          end
+
+          # Allow the collective name for this job class to be overridden
+          def self.underscore_name=(underscore_name)
+            @underscore_name = underscore_name
+          end
+
+          # Returns [String] the human readable name for this job class
+          #
+          # Example:
+          #   job = DataStudyJob.new
+          #   job.human_name
+          #   # => "Data Study"
+          def self.human_name
+            @human_name ||= name.sub(/Job$/, '').titleize
+          end
+
+          # Allow the human readable job name for this job class to be overridden
+          def self.human_name=(human_name)
+            @human_name = human_name
+          end
+
+          # Returns [String] the collective name for this job class
+          #
+          # Example:
+          #   job = DataStudyJob.new
+          #   job.collective_name
+          #   # => "data_studies"
+          def self.collective_name
+            @collective_name ||= name.sub(/Job$/, '').pluralize.underscore
+          end
+
+          # Allow the collective name for this job class to be overridden
+          def self.collective_name=(collective_name)
+            @collective_name = collective_name
+          end
+
+          # Scope for jobs scheduled to run in the future
+          def self.scheduled
+            queued.where(run_at: {'$gt' => Time.now})
+          end
+
+          # Scope for queued jobs that can run now
+          # I.e. Queued jobs excluding scheduled jobs
+          def self.queued_now
+            queued.where(
+              '$or' => [
+                {run_at: {'$exists' => false}},
+                {run_at: {'$lte' => Time.now}}
+              ]
+            )
+          end
+
+          # Returns the number of required arguments for this job
+          def self.rocket_job_argument_count
+            instance_method(:perform).arity
+          end
+
+          # User definable properties in Dirmon Entry
+          def self.rocket_job_properties
+            @rocket_job_properties ||= (self == RocketJob::Job ? [] : superclass.rocket_job_properties)
+          end
+
+          # Add to user definable properties in Dirmon Entry
+          def self.public_rocket_job_properties(*properties)
+            rocket_job_properties.concat(properties).uniq!
+          end
+
+          # User definable properties in Dirmon Entry
+          public_rocket_job_properties :description, :priority, :log_level, :arguments
+        end
+
+        # Returns [true|false] whether to collect nil results from running this batch
+        def collect_nil_output?
+          collect_output? ? (collect_nil_output == true) : false
+        end
+
+        # Returns [true|false] whether to collect the results from running this batch
+        def collect_output?
+          collect_output == true
+        end
+
+        # Returns [Float] the number of seconds the job has taken
+        # - Elapsed seconds to process the job from when a worker first started working on it
+        #   until now if still running, or until it was completed
+        # - Seconds in the queue if queued
+        def seconds
+          if completed_at
+            completed_at - (started_at || created_at)
+          elsif started_at
+            Time.now - started_at
+          else
+            Time.now - created_at
+          end
+        end
+
+        # Returns a human readable duration the job has taken
+        def duration
+          RocketJob.seconds_as_duration(seconds)
+        end
+
+        # Returns [true|false] whether the job has expired
+        def expired?
+          expires_at && (expires_at < Time.now)
+        end
+
+        # Returns [true|false] whether the job is scheduled to run in the future
+        def scheduled?
+          queued? && run_at.present? && (run_at > Time.now)
+        end
+
+        # Returns [Hash] status of this job
+        def as_json
+          attrs = serializable_hash(methods: [:seconds, :duration])
+          attrs.delete('result') unless collect_output?
+          case
+          when queued?
+            attrs.delete('started_at')
+            attrs.delete('completed_at')
+            attrs.delete('result')
+            attrs
+          when running?
+            attrs.delete('completed_at')
+            attrs.delete('result')
+            attrs
+          when paused?
+            attrs.delete('completed_at')
+            attrs.delete('result')
+            # Ensure 'paused_at' appears first in the hash
+            {'paused_at' => completed_at}.merge(attrs)
+          when aborted?
+            attrs.delete('completed_at')
+            attrs.delete('result')
+            {'aborted_at' => completed_at}.merge(attrs)
+          when failed?
+            attrs.delete('completed_at')
+            attrs.delete('result')
+            {'failed_at' => completed_at}.merge(attrs)
+          else
+            attrs
+          end
+        end
+
+        # Returns [Hash] the status of this job
+        def status(time_zone = 'Eastern Time (US & Canada)')
+          h = as_json
+          h.delete('seconds')
+          h.delete('percent_complete') if completed?
+          h.delete('failure_count') unless failure_count > 0
+          h.dup.each_pair do |k, v|
+            case
+            when v.is_a?(Time)
+              h[k] = v.in_time_zone(time_zone).to_s
+            when v.is_a?(BSON::ObjectId)
+              h[k] = v.to_s
+            end
+          end
+          h
+        end
+
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/job/persistence.rb

@@ -0,0 +1,167 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    module Job
+      # Prevent more than one instance of this job class from running at a time
+      module Persistence
+        extend ActiveSupport::Concern
+
+        included do
+          # Store all job types in this collection
+          set_collection_name 'rocket_job.jobs'
+
+          # Create indexes
+          def self.create_indexes
+            # Used by find_and_modify in .rocket_job_retrieve
+            ensure_index({state: 1, priority: 1, _id: 1}, background: true)
+            # Remove outdated indexes if present
+            drop_index('state_1_run_at_1_priority_1_created_at_1_sub_state_1') rescue nil
+            drop_index('state_1_priority_1_created_at_1_sub_state_1') rescue nil
+            drop_index('state_1_priority_1_created_at_1') rescue nil
+            drop_index('created_at_1') rescue nil
+          end
+
+          # Retrieves the next job to work on in priority based order
+          # and assigns it to this worker
+          #
+          # Returns nil if no jobs are available for processing
+          #
+          # Parameters
+          #   worker_name [String]
+          #     Name of the worker that will be processing this job
+          #
+          #   skip_job_ids [Array<BSON::ObjectId>]
+          #     Job ids to exclude when looking for the next job
+          def self.rocket_job_retrieve(worker_name, skip_job_ids = nil)
+            run_at       = [
+              {run_at: {'$exists' => false}},
+              {run_at: {'$lte' => Time.now}}
+            ]
+            query        =
+              if defined?(RocketJobPro)
+                {
+                  '$and' => [
+                    {
+                      '$or' => [
+                        {'state' => 'queued'}, # Jobs
+                        {'state' => 'running', 'sub_state' => :processing} # Slices
+                      ]
+                    },
+                    {
+                      '$or' => run_at
+                    }
+                  ]
+                }
+              else
+                {
+                  'state' => 'queued',
+                  '$or'   => run_at
+                }
+              end
+
+            query['_id'] = {'$nin' => skip_job_ids} if skip_job_ids && skip_job_ids.size > 0
+
+            if doc = find_and_modify(
+              query:  query,
+              sort:   {priority: 1, _id: 1},
+              update: {'$set' => {'worker_name' => worker_name, 'state' => 'running'}}
+            )
+              load(doc)
+            end
+          end
+
+          # Returns [Hash<String:Integer>] of the number of jobs in each state
+          # Queued jobs are separated into :queued_now and :scheduled
+          #   :queued_now are jobs that are awaiting processing and can be processed now.
+          #   :scheduled are jobs scheduled to run the future.
+          #
+          # Note: If there are no jobs in that particular state then the hash will not have a value for it
+          #
+          # Example jobs in every state:
+          #   RocketJob::Job.counts_by_state
+          #   # => {
+          #          :aborted => 1,
+          #          :completed => 37,
+          #          :failed => 1,
+          #          :paused => 3,
+          #          :queued => 4,
+          #          :running => 1,
+          #          :queued_now => 1,
+          #          :scheduled => 3
+          #        }
+          #
+          # Example jobs some states:
+          #   RocketJob::Job.counts_by_state
+          #   # => {
+          #          :failed => 1,
+          #          :running => 25,
+          #          :completed => 1237
+          #        }
+          def self.counts_by_state
+            counts = {}
+            collection.aggregate([
+              {
+                '$group' => {
+                  _id:   '$state',
+                  count: {'$sum' => 1}
+                }
+              }
+            ]
+            ).each do |result|
+              counts[result['_id']] = result['count']
+            end
+
+            # Calculate :queued_now and :scheduled if there are queued jobs
+            if queued_count = counts[:queued]
+              scheduled_count = RocketJob::Job.where(state: :queued, run_at: {'$gt' => Time.now}).count
+              if scheduled_count > 0
+                queued_now_count    = queued_count - scheduled_count
+                counts[:queued_now] = queued_count - scheduled_count if queued_now_count > 0
+                counts[:scheduled]  = scheduled_count
+              else
+                counts[:queued_now] = queued_count
+              end
+            end
+            counts
+          end
+
+        end
+
+        # Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
+        def reload
+          return super unless destroy_on_complete
+          begin
+            super
+          rescue MongoMapper::DocumentNotFound
+            unless completed?
+              self.state = :completed
+              rocket_job_set_completed_at
+              rocket_job_mark_complete
+            end
+            self
+          end
+        end
+
+        private
+
+        # After this model is loaded, convert any hashes in the arguments list to HashWithIndifferentAccess
+        def load_from_database(*args)
+          super
+          if arguments.present?
+            self.arguments = arguments.collect { |i| i.is_a?(BSON::OrderedHash) ? i.with_indifferent_access : i }
+          end
+        end
+
+        # Apply RocketJob defaults after initializing default values
+        # but before setting attributes. after_initialize is too late
+        def initialize_default_values(except = {})
+          super
+          rocket_job_set_defaults
+        end
+
+      end
+    end
+  end
+end