rocketjob 2.1.3 → 3.0.0.alpha

data/lib/rocket_job/heartbeat.rb

@@ -2,46 +2,41 @@
 module RocketJob
   # Heartbeat
   #
-  # Information from the worker as at it's last heartbeat
+  # Information from the server as at it's last heartbeat
   class Heartbeat
-    include MongoMapper::EmbeddedDocument
+    include Plugins::Document
 
-    embedded_in :worker
-
-    # @formatter:off
-    # Time of the last heartbeat received from this worker
-    key :updated_at,                 Time
+    # Time of the last heartbeat received from this server
+    field :updated_at, type: Time
 
     # Number of threads running as at the last heartbeat interval
-    key :active_threads,             Integer
+    field :active_threads, type: Integer
     # Number of threads in the pool
-    #   This number should grow and shrink between 1 and :max_threads
-    key :current_threads,             Integer
+    #   This number should grow and shrink between 1 and :max_workers
+    field :current_threads, type: Integer
 
     #
-    # Process Information
+    # Process Information. Future.
     #
 
-    # Percentage utilization for the worker process alone
-    key :process_cpu,                Integer
-    # Kilo Bytes used by the worker process (Virtual & Physical)
-    key :process_mem_phys_kb,        Integer
-    key :process_mem_virt_kb,        Integer
+    # Percentage utilization for the server process alone
+    #field :process_cpu, type: Integer
+    # Kilo Bytes used by the server process (Virtual & Physical)
+    #field :process_mem_phys_kb, type: Integer
+    #field :process_mem_virt_kb, type: Integer
 
     #
-    # System Information
+    # System Information. Future.
     #
 
     # Percentage utilization for the host machine
-    key :host_cpu,                   Integer
+    #field :host_cpu, type: Integer
     # Kilo Bytes Available on the host machine (Physical)
-    key :host_mem_avail_phys_kbytes, Float
-    key :host_mem_avail_virt_kbytes, Float
+    #field :host_mem_avail_phys_kbytes, type: Float
+    #field :host_mem_avail_virt_kbytes, type: Float
 
     # If available
-    key :load_average,               Float
-    # @formatter:on
-
+    #field :load_average, type: Float
   end
 end
 

data/lib/rocket_job/job.rb

@@ -11,6 +11,5 @@ module RocketJob
     include Plugins::StateMachine
     include Plugins::Job::StateMachine
     include Plugins::Job::Worker
-    include Plugins::Job::Defaults
   end
 end

data/lib/rocket_job/job_exception.rb

@@ -1,28 +1,26 @@
 # encoding: UTF-8
 module RocketJob
-  # Heartbeat
-  #
-  # Information from the worker as at it's last heartbeat
   class JobException
-    include MongoMapper::EmbeddedDocument
+    include Plugins::Document
+
+    embedded_in :job, inverse_of: :exception
+    embedded_in :slice, inverse_of: :exception
+    embedded_in :dirmon_entry, inverse_of: :exception
 
-    # @formatter:off
     # Name of the exception class
-    key :class_name,              String
+    field :class_name, type: String
 
     # Exception message
-    key :message,                 String
+    field :message, type: String
 
     # Exception Backtrace [Array<String>]
-    key :backtrace,               Array
+    field :backtrace, type: Array, default: []
 
-    # Name of the worker on which this exception occurred
-    key :worker_name,             String
+    # Name of the server on which this exception occurred
+    field :worker_name, type: String
 
     # The record within which this exception occurred
-    key :record_number,           Integer
-
-    # @formatter:on
+    field :record_number, type: Integer
 
     # Returns [JobException] built from the supplied exception
     def self.from_exception(exc)

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -15,10 +15,11 @@ module RocketJob
 
     # * Otherwise, the file is moved to the supplied archive_directory (defaults to
     #   `_archive` in the same folder as the file itself. The absolute path and
-    #   file name of the archived file is passed into the job as it's first argument.
+    #   file name of the archived file is passed into the job as either
+    #   `upload_file_name` or `full_file_name`.
 
     # Note:
-    # - Jobs that do not implement #upload _must_ have a Hash as the first argument
+    # - Jobs that do not implement #upload _must_ have either `upload_file_name` or `full_file_name` as an attribute.
     #
     # With RocketJob Pro, the file is automatically uploaded into the job itself
     # using the job's #upload method, after which the file is archived or deleted
@@ -39,13 +40,12 @@ module RocketJob
       # Start a new job when this one completes, fails, or aborts
       include RocketJob::Plugins::Restart
 
-      rocket_job do |job|
-        job.priority = 40
-      end
+      self.priority = 40
 
       # Number of seconds between directory scans. Default 5 mins
-      key :check_seconds, Float, default: 300.0
-      key :previous_file_names, Hash # Hash[file_name, size]
+      field :check_seconds, type: Float, default: 300.0
+      # Hash[file_name, size]
+      field :previous_file_names, type: Hash, default: {}
 
       before_create :set_run_at
 
@@ -68,7 +68,7 @@ module RocketJob
       # since the last run
       def check_directories
         new_file_names = {}
-        DirmonEntry.where(state: :enabled).each do |entry|
+        DirmonEntry.enabled.each do |entry|
           entry.each do |pathname|
             # BSON Keys cannot contain periods
             key           = pathname.to_s.gsub('.', '_')

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -30,26 +30,24 @@ module RocketJob
       include RocketJob::Plugins::Cron
       include RocketJob::Plugins::Singleton
 
-      rocket_job do |job|
-        job.priority      = 50
-        job.description   = 'Cleans out historical jobs'
-        job.cron_schedule = '0 0 * * * America/New_York'
-      end
+      self.priority      = 50
+      self.description   = 'Cleans out historical jobs'
+      self.cron_schedule = '0 0 * * * America/New_York'
 
       # Retention intervals in seconds
       # Set to nil to not
-      key :aborted_retention, Integer, default: 7.days
-      key :completed_retention, Integer, default: 7.days
-      key :failed_retention, Integer, default: 14.days
-      key :paused_retention, Integer, default: 90.days
-      key :queued_retention, Integer
+      field :aborted_retention, Integer, default: 7.days
+      field :completed_retention, Integer, default: 7.days
+      field :failed_retention, Integer, default: 14.days
+      field :paused_retention, Integer, default: 90.days
+      field :queued_retention, Integer
 
       def perform
-        RocketJob::Job.where(state: :aborted, created_at: {'$lte' => aborted_retention.ago}).destroy_all if aborted_retention
-        RocketJob::Job.where(state: :completed, created_at: {'$lte' => completed_retention.ago}).destroy_all if completed_retention
-        RocketJob::Job.where(state: :failed, created_at: {'$lte' => failed_retention.ago}).destroy_all if failed_retention
-        RocketJob::Job.where(state: :paused, created_at: {'$lte' => paused_retention.ago}).destroy_all if paused_retention
-        RocketJob::Job.where(state: :queued, created_at: {'$lte' => queued_retention.ago}).destroy_all if queued_retention
+        RocketJob::Job.aborted.where(created_at: {'$lte' => aborted_retention.ago}).destroy_all if aborted_retention
+        RocketJob::Job.completed.where(created_at: {'$lte' => completed_retention.ago}).destroy_all if completed_retention
+        RocketJob::Job.failed.where(created_at: {'$lte' => failed_retention.ago}).destroy_all if failed_retention
+        RocketJob::Job.paused.where(created_at: {'$lte' => paused_retention.ago}).destroy_all if paused_retention
+        RocketJob::Job.queued.where(created_at: {'$lte' => queued_retention.ago}).destroy_all if queued_retention
       end
 
     end

data/lib/rocket_job/performance.rb

@@ -15,12 +15,12 @@ module RocketJob
     end
 
     def run_test_case(count = self.count)
-      self.worker_processes = RocketJob::Worker.count
+      self.worker_processes = RocketJob::Server.count
       raise 'Please start workers before starting the performance test' if worker_processes == 0
 
       self.worker_processes = 0
       self.worker_threads   = 0
-      RocketJob::Worker.where(state: :running).each do |worker_process|
+      RocketJob::Server.running.each do |worker_process|
         unless worker_process.zombie?
           self.worker_processes += 1
           self.worker_threads   += worker_process.heartbeat.current_threads
@@ -29,7 +29,7 @@ module RocketJob
       puts "Running: #{worker_threads} workers, distributed across #{worker_processes} processes"
 
       puts 'Waiting for workers to pause'
-      RocketJob::Worker.pause_all
+      RocketJob::Server.pause_all
       RocketJob::Jobs::SimpleJob.delete_all
       sleep 15
 
@@ -39,7 +39,7 @@ module RocketJob
       last = RocketJob::Jobs::SimpleJob.create(priority: 100, destroy_on_complete: false)
 
       puts 'Resuming workers'
-      RocketJob::Worker.resume_all
+      RocketJob::Server.resume_all
 
       while (!last.reload.completed?)
         sleep 3
@@ -63,7 +63,7 @@ module RocketJob
         o.on('-c', '--count COUNT', 'Count of jobs to enqueue') do |arg|
           self.count = arg.to_i
         end
-        o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Location of mongo.yml config file') do |arg|
+        o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Path and filename of config file. Default: config/mongoid.yml') do |arg|
           self.mongo_config = arg
         end
         o.on('-e', '--environment ENVIRONMENT', 'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') do |arg|

data/lib/rocket_job/plugins/cron.rb

@@ -40,9 +40,7 @@ module RocketJob
     #   include RocketJob::Plugins::Cron
     #
     #   # Set the default cron_schedule
-    #   rocket_job do |job|
-    #     job.cron_schedule = "* 1 * * * UTC"
-    #   end
+    #   self.cron_schedule = "* 1 * * * UTC"
     #
     #   def perform
     #     puts "DONE"
@@ -70,9 +68,7 @@ module RocketJob
     #   include RocketJob::Plugins::Singleton
     #
     #   # Set the default cron_schedule
-    #   rocket_job do |job|
-    #     job.cron_schedule = "* 1 * * * UTC"
-    #   end
+    #   self.cron_schedule = "* 1 * * * UTC"
     #
     #   def perform
     #     puts "DONE"
@@ -105,10 +101,7 @@ module RocketJob
       included do
         include Restart
 
-        key :cron_schedule, String
-
-        # Make cron_schedule editable for scheduled jobs
-        public_rocket_job_properties :cron_schedule
+        field :cron_schedule, type: String, class_attribute: true, user_editable: true
 
         before_create :rocket_job_set_run_at
 

data/lib/rocket_job/plugins/document.rb

@@ -1,57 +1,82 @@
 # encoding: UTF-8
 require 'active_support/concern'
-require 'mongo'
-require 'mongo_ha'
-begin
-  require 'active_model/serializers/xml'
-rescue LoadError
-  raise "Please add gem 'activemodel-serializers-xml' to Gemfile to support Active Model V5"
-end
-require 'mongo_mapper'
 
 module RocketJob
   module Plugins
     # Base class for storing models in MongoDB
     module Document
       extend ActiveSupport::Concern
-      include MongoMapper::Document
+      include Mongoid::Document
 
       included do
-        # Prevent data in MongoDB from re-defining the model behavior.
-        self.static_keys     = true
-
-        # Only save changes to this instance to prevent losing
-        # changes made by other processes or threads.
-        self.partial_updates = true
+        store_in client: 'rocketjob'
 
-        # Turn off embedded callbacks. Slow and not used by Jobs.
-        embedded_callbacks_off
+        class_attribute :user_editable_fields, instance_accessor: false
+        self.user_editable_fields = []
       end
 
-      # Patch the way MongoMapper reloads a model
-      def reload
-        if doc = collection.find_one(:_id => id)
-          # Clear out keys that are not returned during the reload from MongoDB
-          (keys.keys - doc.keys).each { |key| send("#{key}=", nil) }
-          initialize_default_values
-          load_from_database(doc)
-          self
-        else
-          raise MongoMapper::DocumentNotFound, "Document match #{_id.inspect} does not exist in #{collection.name} collection"
+      module ClassMethods
+        # Defines all the fields that are accessible on the Document
+        # For each field that is defined, a getter and setter will be
+        # added as an instance method to the Document.
+        #
+        # @example Define a field.
+        #   field :score, :type => Integer, :default => 0
+        #
+        # @param [ Symbol ] name The name of the field.
+        # @param [ Hash ] options The options to pass to the field.
+        #
+        # @option options [ Class ] :type The type of the field.
+        # @option options [ String ] :label The label for the field.
+        # @option options [ Object, Proc ] :default The field's default
+        # @option options [ Boolean ] :class_attribute Keep the fields default in a class_attribute
+        # @option options [ Boolean ] :user_editable Field can be edited by end users in RJMC
+        #
+        # @return [ Field ] The generated field
+        def field(name, options)
+          if options.delete(:user_editable) == true
+            self.user_editable_fields += [name.to_sym] unless user_editable_fields.include?(name.to_sym)
+          end
+          if options.delete(:class_attribute) == true
+            class_attribute(name, instance_accessor: false)
+            if options.has_key?(:default)
+              public_send("#{name}=", options[:default])
+            end
+            options[:default] = lambda { self.class.public_send(name) }
+          end
+          super(name, options)
+        end
+
+        # V2 Backward compatibility
+        # DEPRECATED
+        def key(name, type, options = {})
+          field(name, options.merge(type: type))
+        end
+
+        # Mongoid does not apply ordering, add sort
+        def first
+          all.sort('_id' => 1).first
+        end
+
+        # Mongoid does not apply ordering, add sort
+        def last
+          all.sort('_id' => -1).first
         end
       end
 
       private
 
-      def update_attributes_and_reload(attrs)
-        if doc = collection.find_and_modify(query: {:_id => id}, update: {'$set' => attrs})
+      # Apply changes to this document returning the updated document from the database.
+      # Allows other changes to be made on the server that will be loaded.
+      def find_and_update(attrs)
+        if doc = collection.find(_id: id).find_one_and_update({'$set' => attrs}, return_document: :after)
           # Clear out keys that are not returned during the reload from MongoDB
-          (keys.keys - doc.keys).each { |key| send("#{key}=", nil) }
-          initialize_default_values
-          load_from_database(doc)
+          (fields.keys + embedded_relations.keys - doc.keys).each { |key| send("#{key}=", nil) }
+          @attributes = attributes
+          apply_defaults
           self
         else
-          raise MongoMapper::DocumentNotFound, "Document match #{_id.inspect} does not exist in #{collection.name} collection"
+          raise Mongoid::Error::DocumentNotFound, "Document match #{_id.inspect} does not exist in #{collection.name} collection"
         end
       end
 

data/lib/rocket_job/plugins/job/model.rb

@@ -13,10 +13,9 @@ module RocketJob
           # User definable attributes
           #
           # The following attributes are set when the job is created
-          # @formatter:off
 
           # Description for this job instance
-          key :description,             String
+          field :description, type: String, class_attribute: true, user_editable: true
 
           # Priority of this job as it relates to other jobs [1..100]
           #   1: Highest Priority
@@ -29,64 +28,47 @@ module RocketJob
           # In RocketJob Pro, if a SlicedJob is running and a higher priority job
           # arrives, then the current job will complete the current slices and process
           # the new higher priority job
-          key :priority,                Integer, default: 50
-
-          # Run this job no earlier than this time
-          key :run_at,                  Time
-
-          # If a job has not started by this time, destroy it
-          key :expires_at,              Time
+          field :priority, type: Integer, default: 50, class_attribute: true, user_editable: true
 
           # When the job completes destroy it from both the database and the UI
-          key :destroy_on_complete,     Boolean, default: true
-
-          # Any user supplied arguments for the method invocation
-          # All keys must be UTF-8 strings. The values can be any valid BSON type:
-          #   Integer
-          #   Float
-          #   Time    (UTC)
-          #   String  (UTF-8)
-          #   Array
-          #   Hash
-          #   True
-          #   False
-          #   Symbol
-          #   nil
-          #   Regular Expression
-          #
-          # Note: Date is not supported, convert it to a UTC time
-          key :arguments,               Array
+          field :destroy_on_complete, type: Boolean, default: true, class_attribute: true
 
           # Whether to store the results from this job
-          key :collect_output,          Boolean, default: false
+          field :collect_output, type: Boolean, default: false, class_attribute: true
+
+          # Run this job no earlier than this time
+          field :run_at, type: Time
+
+          # If a job has not started by this time, destroy it
+          field :expires_at, type: Time
 
           # Raise or lower the log level when calling the job
           # Can be used to reduce log noise, especially during high volume calls
           # For debugging a single job can be logged at a low level such as :trace
           #   Levels supported: :trace, :debug, :info, :warn, :error, :fatal
-          key :log_level,               Symbol
+          field :log_level, type: Symbol, user_editable: true
 
           #
           # Read-only attributes
           #
 
           # Current state, as set by the state machine. Do not modify this value directly.
-          key :state,                   Symbol, default: :queued
+          field :state, type: Symbol, default: :queued
 
           # When the job was created
-          key :created_at,              Time, default: -> { Time.now }
+          field :created_at, type: Time, default: -> { Time.now }
 
           # When processing started on this job
-          key :started_at,              Time
+          field :started_at, type: Time
 
           # When the job completed processing
-          key :completed_at,            Time
+          field :completed_at, type: Time
 
           # Number of times that this job has failed to process
-          key :failure_count,           Integer, default: 0
+          field :failure_count, type: Integer, default: 0
 
           # This name of the worker that this job is being processed by, or was processed by
-          key :worker_name,             String
+          field :worker_name, type: String
 
           #
           # Values that jobs can update during processing
@@ -94,38 +76,37 @@ module RocketJob
 
           # Allow a job to updates its estimated progress
           # Any integer from 0 to 100
-          key :percent_complete,        Integer, default: 0
+          field :percent_complete, type: Integer, default: 0
 
           # Store the last exception for this job
-          one :exception,               class_name: 'RocketJob::JobException'
+          embeds_one :exception, class_name: 'RocketJob::JobException'
 
           # Store the Hash result from this job if collect_output is true,
           # and the job returned actually returned a Hash, otherwise nil
           # Not applicable to SlicedJob jobs, since its output is stored in a
           # separate collection
-          key :result,                  Hash
+          field :result, type: Hash
 
-          # @formatter:on
-
-          # Store all job types in this collection
-          set_collection_name 'rocket_job.jobs'
+          index({state: 1, priority: 1, _id: 1}, background: true)
 
           validates_presence_of :state, :failure_count, :created_at
           validates :priority, inclusion: 1..100
           validates :log_level, inclusion: SemanticLogger::LEVELS + [nil]
+        end
 
+        module ClassMethods
           # Returns [String] the singular name for this job class
           #
           # Example:
           #   job = DataStudyJob.new
           #   job.underscore_name
           #   # => "data_study"
-          def self.underscore_name
+          def underscore_name
             @underscore_name ||= name.sub(/Job$/, '').underscore
           end
 
           # Allow the collective name for this job class to be overridden
-          def self.underscore_name=(underscore_name)
+          def underscore_name=(underscore_name)
             @underscore_name = underscore_name
           end
 
@@ -135,12 +116,12 @@ module RocketJob
           #   job = DataStudyJob.new
           #   job.human_name
           #   # => "Data Study"
-          def self.human_name
+          def human_name
             @human_name ||= name.sub(/Job$/, '').titleize
           end
 
           # Allow the human readable job name for this job class to be overridden
-          def self.human_name=(human_name)
+          def human_name=(human_name)
             @human_name = human_name
           end
 
@@ -150,49 +131,40 @@ module RocketJob
           #   job = DataStudyJob.new
           #   job.collective_name
           #   # => "data_studies"
-          def self.collective_name
+          def collective_name
             @collective_name ||= name.sub(/Job$/, '').pluralize.underscore
           end
 
           # Allow the collective name for this job class to be overridden
-          def self.collective_name=(collective_name)
+          def collective_name=(collective_name)
             @collective_name = collective_name
           end
 
           # Scope for jobs scheduled to run in the future
-          def self.scheduled
-            queued.where(run_at: {'$gt' => Time.now})
+          def scheduled
+            queued.where(:run_at.gt => Time.now)
           end
 
           # Scope for queued jobs that can run now
           # I.e. Queued jobs excluding scheduled jobs
-          def self.queued_now
-            queued.where(
-              '$or' => [
-                {run_at: {'$exists' => false}},
-                {run_at: {'$lte' => Time.now}}
-              ]
+          def queued_now
+            queued.or(
+              {:run_at.exists => false},
+              {:run_at.lte => Time.now}
             )
           end
 
-          # Returns the number of required arguments for this job
-          def self.rocket_job_argument_count
-            instance_method(:perform).arity
+          # DEPRECATED
+          def rocket_job
+            warn 'Replace calls to .rocket_job with calls to set class instance variables. For example: self.priority = 50'
+            yield(self)
           end
 
-          # User definable properties in Dirmon Entry
-          def self.rocket_job_properties
-            @rocket_job_properties ||= (self == RocketJob::Job ? [] : superclass.rocket_job_properties)
+          # DEPRECATED
+          def public_rocket_job_properties(*args)
+            warn "Replace calls to .public_rocket_job_properties by adding `user_editable: true` option to the field declaration in #{name} for: #{args.inspect}"
+            self.user_editable_fields += args.collect(&:to_sym)
           end
-
-          # Add to user definable properties in Dirmon Entry
-          def self.public_rocket_job_properties(*properties)
-            properties.each { |property| raise("Invalid public_rocket_job_property: #{property.inspect}") unless key?(property)}
-            rocket_job_properties.concat(properties).uniq!
-          end
-
-          # User definable properties in Dirmon Entry
-          public_rocket_job_properties :description, :priority, :log_level, :arguments
         end
 
         # Returns [true|false] whether to collect nil results from running this batch