rocketjob 6.0.0.rc3 → 6.0.3

data/lib/rocket_job/category/output.rb

@@ -13,6 +13,8 @@ module RocketJob
       #   false: do not save nil values to the output categories.
       field :nils, type: ::Mongoid::Boolean, default: false
 
+      validates_inclusion_of :serializer, in: %i[none compress encrypt bz2 encrypted_bz2 bzip2]
+
       # Renders [String] the header line.
       # Returns [nil] if no header is needed.
       def render_header
@@ -20,6 +22,13 @@ module RocketJob
 
         tabular.render_header
       end
+
+      def data_store(job)
+        RocketJob::Sliced::Output.new(
+          collection_name: build_collection_name(:output, job),
+          slice_class:     serializer_class
+        )
+      end
     end
   end
 end

data/lib/rocket_job/dirmon_entry.rb

@@ -173,7 +173,7 @@ module RocketJob
       counts
     end
 
-    # Passes each filename [Pathname] found that matches the pattern into the supplied block
+    # Yields [IOStreams::Path] for each file found that matches the current pattern.
     def each
       SemanticLogger.named_tagged(dirmon_entry: id.to_s) do
         # Case insensitive filename matching

data/lib/rocket_job/job_exception.rb

@@ -23,7 +23,7 @@ module RocketJob
       new(
         args.merge(
           class_name: exc.class.name,
-          message:    exc.message,
+          message:    exc.message.to_s.encode("UTF-8", replace: ""),
           backtrace:  exc.backtrace || []
         )
       )

data/lib/rocket_job/jobs/conversion_job.rb

@@ -1,39 +1,43 @@
 # Convert to and from CSV, JSON, xlsx, and PSV files.
 #
 # Example, Convert CSV file to JSON.
-#   job = RocketJob::ConversionJob.new
-#   job.upload("data.csv")
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "data.csv"
 #   job.output_category.file_name = "data.json"
 #   job.save!
 #
 # Example, Convert JSON file to PSV and compress it with GZip.
-#   job = RocketJob::ConversionJob.new
-#   job.upload("data.json")
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "data.json"
 #   job.output_category.file_name = "data.psv.gz"
 #   job.save!
 #
 # Example, Read a CSV file that has been zipped from a remote website and the convert it to a GZipped json file.
-#   job = RocketJob::ConversionJob.new
-#   job.upload("https://example.org/file.zip")
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "https://example.org/file.zip"
 #   job.output_category.file_name = "data.json.gz"
 #   job.save!
 #
 module RocketJob
-  class ConversionJob < RocketJob::Job
-    include RocketJob::Batch
+  module Jobs
+    class ConversionJob < RocketJob::Job
+      include RocketJob::Batch
 
-    self.destroy_on_complete = false
+      self.destroy_on_complete = false
 
-    # Detects file extension for its type
-    input_category format: :auto
-    output_category format: :auto
+      # Detects file extension for its type
+      input_category format: :auto
+      output_category format: :auto
 
-    # When the job completes it will write the result to the output_category.file_name
-    after_batch :download
+      # Upload the file specified in `input_category.file_name` unless already uploaded.
+      before_batch :upload, unless: :record_count
 
-    def perform(hash)
-      # For this job return the input hash record as-is. Could be transformed here as needed.
-      hash
+      # When the job completes it will write the result to `output_category.file_name`.
+      after_batch :cleanup!, :download
+
+      def perform(hash)
+        hash
+      end
     end
   end
 end

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -30,59 +30,48 @@ module RocketJob
     #
     # If another DirmonJob instance is already queued or running, then the create
     # above will fail with:
-    #   MongoMapper::DocumentNotValid: Validation failed: State Another instance of this job is already queued or running
+    #   Validation failed: State Another instance of this job is already queued or running
     #
     # Or to start DirmonJob and ignore errors if already running
     #   RocketJob::Jobs::DirmonJob.create
     class DirmonJob < RocketJob::Job
-      # Only allow one DirmonJob instance to be running at a time
-      include RocketJob::Plugins::Singleton
-      # Start a new job when this one completes, fails, or aborts
-      include RocketJob::Plugins::Restart
+      include RocketJob::Plugins::Cron
 
-      self.priority = 30
-
-      # Number of seconds between directory scans. Default 5 mins
-      field :check_seconds, type: Float, default: 300.0, copy_on_restart: true
+      # Runs every 5 minutes by default
+      self.cron_schedule = "*/5 * * * * UTC"
+      self.description   = "Directory Monitor"
+      self.priority      = 30
 
       # Hash[file_name, size]
       field :previous_file_names, type: Hash, default: {}, copy_on_restart: true
 
-      before_create :set_run_at
-
-      # Iterate over each Dirmon entry looking for new files
-      # If a new file is found, it is not processed immediately, instead
-      # it is passed to the next run of this job along with the file size.
-      # If the file size has not changed, the Job is kicked off.
+      # Checks the directories for new files, starting jobs if files have not changed since the last run.
       def perform
         check_directories
       end
 
       private
 
-      # Set a run_at when a new instance of this job is created
-      def set_run_at
-        self.run_at = Time.now + check_seconds
-      end
-
-      # Checks the directories for new files, starting jobs if files have not changed
-      # since the last run
+      # Iterate over each Dirmon Entry looking for new files
+      # If a new file is found, it is not processed immediately, instead
+      # it is passed to the next run of this job along with the file size.
+      # If the file size has not changed, the Job is kicked off.
       def check_directories
         new_file_names = {}
-        DirmonEntry.enabled.each do |entry|
-          entry.each do |iopath|
-            # S3 files are only visible once completely uploaded.
-            unless iopath.partial_files_visible?
-              logger.info("File: #{iopath}. Starting: #{entry.job_class_name}")
-              entry.later(iopath)
+        DirmonEntry.enabled.each do |dirmon_entry|
+          dirmon_entry.each do |path|
+            # Skip file size checking since S3 files are only visible once completely uploaded.
+            unless path.partial_files_visible?
+              logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
+              dirmon_entry.later(path)
               next
             end
 
             # BSON Keys cannot contain periods
-            key           = iopath.to_s.tr(".", "_")
+            key           = path.to_s.tr(".", "_")
             previous_size = previous_file_names[key]
             # Check every few minutes for a file size change before trying to process the file.
-            size                = check_file(entry, iopath, previous_size)
+            size                = check_file(dirmon_entry, path, previous_size)
             new_file_names[key] = size if size
           end
         end
@@ -91,14 +80,14 @@ module RocketJob
 
       # Checks if a file should result in starting a job
       # Returns [Integer] file size, or nil if the file started a job
-      def check_file(entry, iopath, previous_size)
-        size = iopath.size
+      def check_file(dirmon_entry, path, previous_size)
+        size = path.size
         if previous_size && (previous_size == size)
-          logger.info("File stabilized: #{iopath}. Starting: #{entry.job_class_name}")
-          entry.later(iopath)
+          logger.info("File stabilized: #{path}. Starting: #{dirmon_entry.job_class_name}")
+          dirmon_entry.later(path)
           nil
         else
-          logger.info("Found file: #{iopath}. File size: #{size}")
+          logger.info("Found file: #{path}. File size: #{size}")
           # Keep for the next run
           size
         end

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -27,12 +27,11 @@ module RocketJob
     #   )
     class HousekeepingJob < RocketJob::Job
       include RocketJob::Plugins::Cron
-      include RocketJob::Plugins::Singleton
 
-      self.priority    = 25
-      self.description = "Cleans out historical jobs, and zombie servers."
-      # Runs every 15 minutes
-      self.cron_schedule = "*/15 * * * * UTC"
+      # Runs every 15 minutes on the 15 minute period
+      self.cron_schedule = "0,15,30,45 * * * * UTC"
+      self.description   = "Cleans out historical jobs, and zombie servers."
+      self.priority      = 25
 
       # Whether to destroy zombie servers automatically
       field :destroy_zombies, type: Mongoid::Boolean, default: true, user_editable: true, copy_on_restart: true

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -65,27 +65,29 @@ module RocketJob
   module Jobs
     class OnDemandBatchJob < RocketJob::Job
       include RocketJob::Plugins::Cron
+      include RocketJob::Plugins::Retry
       include RocketJob::Batch
       include RocketJob::Batch::Statistics
 
       self.priority            = 90
-      self.description         = "Batch Job"
+      self.description         = "On Demand Batch Job"
       self.destroy_on_complete = false
+      self.retry_limit         = 0
 
       # Code that is performed against every row / record.
-      field :code, type: String
+      field :code, type: String, user_editable: true, copy_on_restart: true
 
       # Optional code to execute before the batch is run.
       # Usually to upload data into the job.
-      field :before_code, type: String
+      field :before_code, type: String, user_editable: true, copy_on_restart: true
 
       # Optional code to execute after the batch is run.
       # Usually to upload data into the job.
-      field :after_code, type: String
+      field :after_code, type: String, user_editable: true, copy_on_restart: true
 
       # Data that is made available to the job during the perform.
       # Be sure to store key names only as Strings, not Symbols.
-      field :data, type: Hash, default: {}
+      field :data, type: Hash, default: {}, user_editable: true, copy_on_restart: true
 
       validates :code, presence: true
       validate :validate_code
@@ -143,6 +145,10 @@ module RocketJob
       rescue Exception => e
         errors.add(field, "Failed to load :#{field}, #{e.inspect}")
       end
+
+      # Allow multiple instances of this job to run with the same cron schedule
+      def rocket_job_cron_singleton_check
+      end
     end
   end
 end

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -78,8 +78,8 @@ module RocketJob
       self.retry_limit         = 0
 
       # Be sure to store key names only as Strings, not Symbols
-      field :data, type: Hash, default: {}, copy_on_restart: true
-      field :code, type: String, copy_on_restart: true
+      field :data, type: Hash, default: {}, user_editable: true, copy_on_restart: true
+      field :code, type: String, user_editable: true, copy_on_restart: true
 
       validates :code, presence: true
       validate :validate_code
@@ -97,6 +97,10 @@ module RocketJob
       rescue Exception => e
         errors.add(:code, "Failed to parse :code, #{e.inspect}")
       end
+
+      # Allow multiple instances of this job to run with the same cron schedule
+      def rocket_job_cron_singleton_check
+      end
     end
   end
 end

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -57,6 +57,10 @@ module RocketJob
 
       def upload_file(job)
         if job.respond_to?(:upload)
+          # Return the database connection for this thread back to the connection pool
+          # in case the upload takes a long time and the database connection expires.
+          ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
+
           if original_file_name
             job.upload(upload_file_name, file_name: original_file_name)
           else

data/lib/rocket_job/plugins/cron.rb

@@ -14,41 +14,81 @@ module RocketJob
       extend ActiveSupport::Concern
 
       included do
-        include Restart
-
         field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
 
+        # Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
+        # Another job instance with a different `cron_schedule` string is permitted.
+        field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        # Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
+        #
+        # `true`: Create a new scheduled instance of this job after it has started. (Default)
+        #   - Ensures that the next scheduled instance is not missed because the current instance is still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     _only_ if they were changed during an `after_start` callback.
+        #     Changes to these during other callbacks or during the `perform` will not be saved to the new scheduled
+        #     instance.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil`.
+        #
+        # `false`: Create a new scheduled instance of this job on `fail`, or `abort`.
+        #   - Prevents the next scheduled instance from running or being scheduled while the current instance is
+        #     still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     at any time until after the job has failed, or is aborted.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil` after it fails or is aborted.
+        field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
         validates_each :cron_schedule do |record, attr, value|
           record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
         end
+        validate :rocket_job_cron_singleton_check
+
         before_save :rocket_job_cron_set_run_at
 
-        private
+        after_start :rocket_job_cron_on_start
+        after_abort :rocket_job_cron_end_state
+        after_complete :rocket_job_cron_end_state
+        after_fail :rocket_job_cron_end_state
+      end
 
-        # Prevent auto restart if this job does not have a cron schedule.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_new_instance
-        def rocket_job_restart_new_instance
-          return unless cron_schedule
+      def rocket_job_cron_set_run_at
+        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
 
-          super
-        end
+        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+      end
 
-        # On failure:
-        # - create a new instance scheduled to run in the future.
-        # - clear out the `cron_schedule` so this instance will not schedule another instance to run on completion.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_abort
-        def rocket_job_restart_abort
-          return unless cron_schedule
+      private
 
-          rocket_job_restart_new_instance
-          update_attribute(:cron_schedule, nil)
+      def rocket_job_cron_on_start
+        return unless cron_schedule && cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      def rocket_job_cron_end_state
+        return unless cron_schedule && !cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      # Returns [true|false] whether another instance of this job with the same cron schedule is already active
+      def rocket_job_cron_duplicate?
+        self.class.with(read: {mode: :primary}) do |conn|
+          conn.where(:state.in => %i[queued running failed paused], :id.ne => id, cron_schedule: cron_schedule).exists?
         end
       end
 
-      def rocket_job_cron_set_run_at
-        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
+      # Prevent creation of a new job when another is running with the same cron schedule.
+      def rocket_job_cron_singleton_check
+        return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
 
-        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+        errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
       end
     end
   end

data/lib/rocket_job/plugins/job/persistence.rb

@@ -70,6 +70,29 @@ module RocketJob
           end
         end
 
+        # Create a new instance of this job, copying across only the `copy_on_restart` attributes.
+        # Copy across input and output categories to new scheduled job so that all of the
+        # settings are remembered between instance. Example: slice_size
+        def create_restart!(**overrides)
+          if expired?
+            logger.info("Job has expired. Not creating a new instance.")
+            return
+          end
+
+          job_attrs = self.class.rocket_job_restart_attributes.each_with_object({}) do |attr, attrs|
+            attrs[attr] = send(attr)
+          end
+          job_attrs.merge!(overrides)
+
+          job                   = self.class.new(job_attrs)
+          job.input_categories  = input_categories if respond_to?(:input_categories)
+          job.output_categories = output_categories if respond_to?(:output_categories)
+
+          job.save_with_retry!
+
+          logger.info("Created a new job instance: #{job.id}")
+        end
+
         # Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
         def reload
           return super unless destroy_on_complete
@@ -85,6 +108,19 @@ module RocketJob
             self
           end
         end
+
+        # Save with retry in case persistence takes a moment.
+        def save_with_retry!(retry_limit = 10, sleep_interval = 0.5)
+          count = 0
+          while count < retry_limit
+            return true if save
+
+            logger.info("Retrying to persist new scheduled instance: #{errors.messages.inspect}")
+            sleep(sleep_interval)
+            count += 1
+          end
+          save!
+        end
       end
     end
   end

data/lib/rocket_job/plugins/restart.rb

@@ -2,128 +2,21 @@ require "active_support/concern"
 
 module RocketJob
   module Plugins
-    # Automatically starts a new instance of this job anytime it fails, aborts, or completes.
-    #
-    # Notes:
-    # * Restartable jobs automatically abort if they fail. This prevents the failed job from being retried.
-    #   - To disable this behavior, add the following empty method:
-    #        def rocket_job_restart_abort
-    #        end
-    # * On destroy this job is destroyed without starting a new instance.
-    # * On Abort a new instance is created.
-    # * Include `RocketJob::Plugins::Singleton` to prevent multiple copies of a job from running at
-    #   the same time.
-    # * The job will not be restarted if:
-    #   - A validation fails after creating the new instance of this job.
-    #   - The job has expired.
-    # * Only the fields that have `copy_on_restart: true` will be passed onto the new instance of this job.
-    #
-    # Example:
-    #
-    # class RestartableJob < RocketJob::Job
-    #   include RocketJob::Plugins::Restart
-    #
-    #   # Retain the completed job under the completed tab in Rocket Job Web Interface.
-    #   self.destroy_on_complete = false
-    #
-    #   # Will be copied to the new job on restart.
-    #   field :limit, type: Integer, copy_on_restart: true
-    #
-    #   # Will _not_ be copied to the new job on restart.
-    #   field :list, type: Array, default: [1,2,3]
-    #
-    #   # Set run_at every time a new instance of the job is created.
-    #   after_initialize set_run_at, if: :new_record?
-    #
-    #   def perform
-    #     puts "The limit is #{limit}"
-    #     puts "The list is #{list}"
-    #     'DONE'
-    #   end
-    #
-    #   private
-    #
-    #   # Run this job in 30 minutes.
-    #   def set_run_at
-    #     self.run_at = 30.minutes.from_now
-    #   end
-    # end
-    #
-    # job = RestartableJob.create!(limit: 10, list: [4,5,6])
-    # job.reload.state
-    # # => :queued
-    #
-    # job.limit
-    # # => 10
-    #
-    # job.list
-    # # => [4,5,6]
-    #
-    # # Wait 30 minutes ...
-    #
-    # job.reload.state
-    # # => :completed
-    #
-    # # A new instance was automatically created.
-    # job2 = RestartableJob.last
-    # job2.state
-    # # => :queued
-    #
-    # job2.limit
-    # # => 10
-    #
-    # job2.list
-    # # => [1,2,3]
+    # @deprecated
     module Restart
       extend ActiveSupport::Concern
 
       included do
-        after_abort :rocket_job_restart_new_instance
-        after_complete :rocket_job_restart_new_instance
+        after_abort :create_restart!
+        after_complete :create_restart!
         after_fail :rocket_job_restart_abort
       end
 
       private
 
-      # Run again in the future, even if this run fails with an exception
-      def rocket_job_restart_new_instance
-        if expired?
-          logger.info("Job has expired. Not creating a new instance.")
-          return
-        end
-        job_attrs             =
-          rocket_job_restart_attributes.each_with_object({}) { |attr, attrs| attrs[attr] = send(attr) }
-        job                   = self.class.new(job_attrs)
-
-        # Copy across input and output categories to new scheduled job so that all of the
-        # settings are remembered between instance. Example: slice_size
-        job.input_categories  = input_categories if respond_to?(:input_categories)
-        job.output_categories = output_categories if respond_to?(:output_categories)
-
-        rocket_job_restart_save(job)
-      end
-
       def rocket_job_restart_abort
         new_record? ? abort : abort!
       end
-
-      # Allow Singleton to prevent the creation of a new job if one is already running
-      # Retry since the delete may not have persisted to disk yet.
-      def rocket_job_restart_save(job, retry_limit = 10, sleep_interval = 0.5)
-        count = 0
-        while count < retry_limit
-          if job.save
-            logger.info("Created a new job instance: #{job.id}")
-            return true
-          else
-            logger.info("Job already active, retrying after a short sleep")
-            sleep(sleep_interval)
-          end
-          count += 1
-        end
-        logger.error("New job instance not started: #{job.errors.messages.inspect}")
-        false
-      end
     end
   end
 end

data/lib/rocket_job/plugins/state_machine.rb

@@ -36,8 +36,8 @@ module RocketJob
           raise(ArgumentError, "Cannot supply both a method name and a block") if methods.size.positive? && block
           raise(ArgumentError, "Must supply either a method name or a block") unless methods.size.positive? || block
 
-          # TODO: Somehow get AASM to support options such as :if and :unless to be consistent with other callbacks
-          # For example:
+          # Limitation with AASM. It only supports guards on event transitions, not for callbacks.
+          # For example, AASM does not support callback options such as :if and :unless, yet Rails callbacks do.
           #    before_start :my_callback, unless: :encrypted?
           #    before_start :my_callback, if: :encrypted?
           event = aasm.state_machine.events[event_name]

data/lib/rocket_job/plugins/throttle_dependent_jobs.rb

@@ -11,17 +11,22 @@ module RocketJob
       extend ActiveSupport::Concern
 
       included do
-        class_attribute :dependent_jobs
-        self.dependent_jobs = nil
+        field :dependent_jobs, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
 
-        define_throttle :dependent_job_exists?
-        define_batch_throttle :dependent_job_exists? if respond_to?(:define_batch_throttle)
+        define_throttle :dependent_jobs_running?
+        define_batch_throttle :dependent_jobs_running? if respond_to?(:define_batch_throttle)
+      end
+
+      class_methods do
+        def depends_on_job(*jobs)
+          self.dependent_jobs = Array(jobs).collect(&:to_s)
+        end
       end
 
       private
 
       # Checks if there are any dependent jobs are running
-      def dependent_job_exists?
+      def dependent_jobs_running?
         return false if dependent_jobs.blank?
 
         jobs_count = RocketJob::Job.running.where(:_type.in => dependent_jobs).count