rocketjob 3.4.3 → 3.5.0

data/lib/rocket_job/extensions/mongo/logging.rb

@@ -4,22 +4,26 @@ module Mongo
   class Monitoring
     class CommandLogSubscriber
       include SemanticLogger::Loggable
-      self.logger.name = 'Mongo'
+      logger.name = 'Mongo'
 
       def started(event)
         @event_command = event.command
       end
 
       def succeeded(event)
-        logger.debug(message: prefix(event), duration: (event.duration * 1000), payload: @event_command)
+        logger.debug(message:  prefix(event),
+                     duration: (event.duration * 1000),
+                     payload:  @event_command)
       end
 
       def failed(event)
-        logger.debug(message: "#{prefix(event)} Failed: #{event.message}", duration: (event.duration * 1000), payload: @event_command)
+        logger.debug(message:  "#{prefix(event)} Failed: #{event.message}",
+                     duration: (event.duration * 1000),
+                     payload:  @event_command)
       end
 
       def prefix(event)
-        "#{event.address.to_s} | #{event.database_name}.#{event.command_name}"
+        "#{event.address} | #{event.database_name}.#{event.command_name}"
       end
     end
   end

data/lib/rocket_job/extensions/mongoid/factory.rb

@@ -1,13 +1,15 @@
 require 'mongoid/factory'
 
 module RocketJob
-  module MongoidFactory
-    def from_db(*args)
-      super(*args)
-    rescue NameError => exc
-      RocketJob::Job.instantiate(attributes, selected_fields)
+  module Mongoid
+    module Factory
+      def from_db(*args)
+        super(*args)
+      rescue NameError
+        RocketJob::Job.instantiate(attributes, selected_fields)
+      end
     end
   end
 end
 
-::Mongoid::Factory.include(RocketJob::MongoidFactory)
+::Mongoid::Factory.extend(RocketJob::Mongoid::Factory)

data/lib/rocket_job/extensions/rocket_job_adapter.rb

@@ -70,20 +70,18 @@ module ActiveJob
         job
       end
 
-      private
-
       def self.active_job_params(active_job)
         params            = {
           description:      active_job.class.name,
           data:             active_job.serialize,
           active_job_id:    active_job.job_id,
           active_job_class: active_job.class.name,
-          active_job_queue: active_job.queue_name,
+          active_job_queue: active_job.queue_name
         }
         params[:priority] = active_job.priority if active_job.respond_to?(:priority) && active_job.priority
         params
       end
-
+      private_class_method :active_job_params
     end
   end
 end

data/lib/rocket_job/heartbeat.rb

@@ -16,23 +16,22 @@ module RocketJob
     #
 
     # Percentage utilization for the server process alone
-    #field :process_cpu, type: Integer
+    # field :process_cpu, type: Integer
     # Kilo Bytes used by the server process (Virtual & Physical)
-    #field :process_mem_phys_kb, type: Integer
-    #field :process_mem_virt_kb, type: Integer
+    # field :process_mem_phys_kb, type: Integer
+    # field :process_mem_virt_kb, type: Integer
 
     #
     # System Information. Future.
     #
 
     # Percentage utilization for the host machine
-    #field :host_cpu, type: Integer
+    # field :host_cpu, type: Integer
     # Kilo Bytes Available on the host machine (Physical)
-    #field :host_mem_avail_phys_kbytes, type: Float
-    #field :host_mem_avail_virt_kbytes, type: Float
+    # field :host_mem_avail_phys_kbytes, type: Float
+    # field :host_mem_avail_virt_kbytes, type: Float
 
     # If available
-    #field :load_average, type: Float
+    # field :load_average, type: Float
   end
 end
-

data/lib/rocket_job/job_exception.rb

@@ -22,13 +22,14 @@ module RocketJob
     field :record_number, type: Integer
 
     # Returns [JobException] built from the supplied exception
-    def self.from_exception(exc)
+    def self.from_exception(exc, **args)
       new(
-        class_name: exc.class.name,
-        message:    exc.message,
-        backtrace:  exc.backtrace || []
+        args.merge(
+          class_name: exc.class.name,
+          message:    exc.message,
+          backtrace:  exc.backtrace || []
+        )
       )
     end
-
   end
 end

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -40,7 +40,7 @@ module RocketJob
       # Start a new job when this one completes, fails, or aborts
       include RocketJob::Plugins::Restart
 
-      self.priority = 40
+      self.priority = 30
 
       # Number of seconds between directory scans. Default 5 mins
       field :check_seconds, type: Float, default: 300.0, copy_on_restart: true
@@ -72,11 +72,10 @@ module RocketJob
         DirmonEntry.enabled.each do |entry|
           entry.each do |pathname|
             # BSON Keys cannot contain periods
-            key           = pathname.to_s.gsub('.', '_')
-            previous_size = previous_file_names[key]
-            if size = check_file(entry, pathname, previous_size)
-              new_file_names[key] = size
-            end
+            key                 = pathname.to_s.tr('.', '_')
+            previous_size       = previous_file_names[key]
+            size                = check_file(entry, pathname, previous_size)
+            new_file_names[key] = size if size
           end
         end
         self.previous_file_names = new_file_names
@@ -96,7 +95,6 @@ module RocketJob
           size
         end
       end
-
     end
   end
 end

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -54,12 +54,13 @@ module RocketJob
         end
 
         RocketJob::Job.aborted.where(completed_at: {'$lte' => aborted_retention.seconds.ago}).destroy_all if aborted_retention
-        RocketJob::Job.completed.where(completed_at: {'$lte' => completed_retention.seconds.ago}).destroy_all if completed_retention
+        if completed_retention
+          RocketJob::Job.completed.where(completed_at: {'$lte' => completed_retention.seconds.ago}).destroy_all
+        end
         RocketJob::Job.failed.where(completed_at: {'$lte' => failed_retention.seconds.ago}).destroy_all if failed_retention
         RocketJob::Job.paused.where(completed_at: {'$lte' => paused_retention.seconds.ago}).destroy_all if paused_retention
         RocketJob::Job.queued.where(created_at: {'$lte' => queued_retention.seconds.ago}).destroy_all if queued_retention
       end
-
     end
   end
 end

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -0,0 +1,104 @@
+# Generalized Job.
+#
+# Create or schedule a generalized job for one off fixes or cleanups.
+#
+# Example: Iterate over all rows in a table:
+#   code = <<~CODE
+#     User.unscoped.all.order('updated_at DESC').each |user|
+#       user.cleanse_attributes!
+#       user.save!
+#     end
+#   CODE
+#
+#   RocketJob::Jobs::OnDemandJob.create!(
+#     code:          code,
+#     description:   'Cleanse users'
+#   )
+#
+# Example: Test job in a console:
+#   code = <<~CODE
+#     User.unscoped.all.order('updated_at DESC').each |user|
+#       user.cleanse_attributes!
+#       user.save!
+#     end
+#   CODE
+#
+#   job = RocketJob::Jobs::OnDemandJob.new(code: code, description: 'cleanse users')
+#   job.perform_now
+#
+# Example: Pass input data:
+#   code = <<~CODE
+#     puts data['a'] * data['b']
+#   CODE
+#
+#   RocketJob::Jobs::OnDemandJob.create!(
+#     code: code,
+#     data: {'a' => 10, 'b' => 2}
+#   )
+#
+# Example: Retain output:
+#   code = <<~CODE
+#     {'value' => data['a'] * data['b']}
+#   CODE
+#
+#   RocketJob::Jobs::OnDemandJob.create!(
+#     code:           code,
+#     collect_output: true,
+#     data:           {'a' => 10, 'b' => 2}
+#   )
+#
+# Example: Schedule the job to run nightly at 2am Eastern:
+#
+#   RocketJob::Jobs::OnDemandJob.create!(
+#     cron_schedule: '0 2 * * * America/New_York',
+#     code:          code
+#   )
+#
+# Example: Change the job priority, description, etc.
+#
+#   RocketJob::Jobs::OnDemandJob.create!(
+#     code:          code,
+#     description:   'Cleanse users',
+#     priority:      30
+#   )
+#
+# Example: Automatically retry up to 5 times on failure:
+#
+#   RocketJob::Jobs::OnDemandJob.create!(
+#     retry_limit: 5
+#     code:        code
+#   )
+module RocketJob
+  module Jobs
+    class OnDemandJob < RocketJob::Job
+      include RocketJob::Plugins::Cron
+      include RocketJob::Plugins::Retry
+
+      self.priority            = 90
+      self.description         = 'Generalized Job'
+      self.destroy_on_complete = false
+      self.retry_limit         = 0
+
+      # Be sure to store key names only as Strings, not Symbols
+      field :data, type: Hash, default: {}
+      field :code, type: String
+
+      validates :code, presence: true
+      validates_each :code do |job, attr, _value|
+        begin
+          job.send(:load_code)
+        rescue Exception => exc
+          job.errors.add(attr, "Failed to parse :code, #{exc.inspect}")
+        end
+      end
+
+      before_perform :load_code
+
+      private
+
+      def load_code
+        instance_eval("def perform\n#{code}\nend", __FILE__, __LINE__)
+      end
+    end
+  end
+end

data/lib/rocket_job/jobs/simple_job.rb

@@ -1,11 +1,9 @@
 module RocketJob
   module Jobs
-
     class SimpleJob < RocketJob::Job
       # No operation, used for performance testing
       def perform
       end
     end
-
   end
 end

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -0,0 +1,100 @@
+require 'fileutils'
+begin
+  require 'iostreams'
+rescue LoadError
+  # Optional dependency
+end
+
+module RocketJob
+  module Jobs
+    # Job to upload a file into another job.
+    #
+    # Intended for use by DirmonJob to upload a file into a specified job.
+    #
+    # Can be used directly for any job class, as long as that job responds
+    # to `#upload`.
+    class UploadFileJob < RocketJob::Job
+      self.priority = 30
+
+      # Name of the job class to instantiate and upload the file into.
+      field :job_class_name, type: String, user_editable: true
+
+      # Properties to assign to the job when it is created.
+      field :properties, type: Hash, default: {}, user_editable: true
+
+      # File to upload
+      field :upload_file_name, type: String, user_editable: true
+
+      # The original Input file name.
+      # Used by #upload to extract the IOStreams when present.
+      field :original_file_name, type: String, user_editable: true
+
+      # Optionally set the job id for the downstream job
+      # Useful when for example the archived file should contain the job id for the downstream job.
+      field :job_id, type: BSON::ObjectId
+
+      validates_presence_of :upload_file_name, :job_class_name
+      validate :job_is_a_rocket_job
+      validate :job_implements_upload
+      validate :file_exists
+
+      # Create the job and upload the file into it.
+      def perform
+        job    = job_class.new(properties)
+        job.id = job_id if job_id
+        upload_file(job)
+        job.save!
+      rescue StandardError => exc
+        # Prevent partial uploads
+        job&.cleanup! if job.respond_to?(:cleanup!)
+        raise(exc)
+      end
+
+      private
+
+      def job_class
+        @job_class ||= job_class_name.constantize
+      rescue NameError
+        nil
+      end
+
+      def upload_file(job)
+        if job.respond_to?(:upload)
+          if original_file_name && defined?(IOStreams)
+            streams = IOStreams.streams_for_file_name(original_file_name)
+            job.upload(upload_file_name, streams: streams)
+          else
+            job.upload(upload_file_name)
+          end
+        elsif job.respond_to?(:upload_file_name=)
+          job.upload_file_name = upload_file_name
+        elsif job.respond_to?(:full_file_name=)
+          job.full_file_name = upload_file_name
+        else
+          raise(ArgumentError, "Model #{job_class_name} must implement '#upload', or have attribute 'upload_file_name' or 'full_file_name'")
+        end
+      end
+
+      # Validates job_class is a Rocket Job
+      def job_is_a_rocket_job
+        klass = job_class
+        return if klass.nil? || klass.ancestors&.include?(RocketJob::Job)
+        errors.add(:job_class_name, "Model #{job_class_name} must be defined and inherit from RocketJob::Job")
+      end
+
+      VALID_INSTANCE_METHODS = %i[upload upload_file_name= full_file_name=].freeze
+
+      # Validates job_class is a Rocket Job
+      def job_implements_upload
+        klass = job_class
+        return if klass.nil? || klass.instance_methods.any? { |m| VALID_INSTANCE_METHODS.include?(m) }
+        errors.add(:job_class_name, "#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
+      end
+
+      def file_exists
+        return if upload_file_name.nil? || File.exist?(upload_file_name)
+        errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
+      end
+    end
+  end
+end

data/lib/rocket_job/performance.rb

@@ -18,7 +18,9 @@ module RocketJob
     # Loads the queue with jobs to be processed once the queue is loaded.
     # Retain the first and last job for timings, all others are destroyed on completion.
     def run_test_case(count = self.count)
-      raise 'Please start servers before starting the performance test' if RocketJob::Server.where(:state.in => ['running', 'paused']).count == 0
+      if RocketJob::Server.where(:state.in => %w[running paused]).count.zero?
+        raise 'Please start servers before starting the performance test'
+      end
 
       count_running_workers
 
@@ -33,21 +35,19 @@ module RocketJob
           running += server.heartbeat.workers unless server.zombie?
         end
         puts "Waiting for #{running} workers"
-        break if running == 0
+        break if running.zero?
         sleep 1
       end
 
       puts 'Enqueuing jobs'
       first = RocketJob::Jobs::SimpleJob.create!(priority: 1, destroy_on_complete: false)
-      (count - 2).times { |i| RocketJob::Jobs::SimpleJob.create! }
+      (count - 2).times { RocketJob::Jobs::SimpleJob.create! }
       last = RocketJob::Jobs::SimpleJob.create!(priority: 100, destroy_on_complete: false)
 
       puts 'Resuming workers'
       RocketJob::Server.resume_all
 
-      while (!last.reload.completed?)
-        sleep 3
-      end
+      sleep 3 until last.reload.completed?
 
       duration = last.reload.completed_at - first.reload.started_at
       first.destroy
@@ -67,13 +67,19 @@ module RocketJob
     # Parse command line options
     def parse(argv)
       parser        = OptionParser.new do |o|
-        o.on('-c', '--count COUNT', 'Count of jobs to enqueue') do |arg|
+        o.on('-c',
+             '--count COUNT',
+             'Count of jobs to enqueue') do |arg|
           self.count = arg.to_i
         end
-        o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Path and filename of config file. Default: config/mongoid.yml') do |arg|
+        o.on('-m',
+             '--mongo MONGO_CONFIG_FILE_NAME',
+             'Path and filename of config file. Default: config/mongoid.yml') do |arg|
           self.mongo_config = arg
         end
-        o.on('-e', '--environment ENVIRONMENT', 'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') do |arg|
+        o.on('-e',
+             '--environment ENVIRONMENT',
+             'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') do |arg|
           self.environment = arg
         end
       end
@@ -95,6 +101,5 @@ module RocketJob
       end
       puts "Running: #{workers} workers, distributed across #{servers} servers"
     end
-
   end
 end