rocketjob 6.0.0.rc3 → 6.0.0

data/lib/rocket_job/batch/worker.rb

@@ -67,6 +67,8 @@ module RocketJob
       # Returns [Integer] the number of records processed in the slice
       #
       # Note: The slice will be removed from processing when this method completes
+      #
+      # @deprecated Please open a ticket if you need this behavior.
       def work_first_slice(&block)
         raise "#work_first_slice can only be called from within before_batch callbacks" unless sub_state == :before
 
@@ -142,19 +144,19 @@ module RocketJob
       # Perform individual slice without callbacks
       def rocket_job_perform_slice(slice, &block)
         slice.processing_record_number ||= 0
-        records                        = []
         append                         = false
 
-        # Skip processed records in this slice if it has no output categpries.
-        if slice.processing_record_number > 1
-          records = slice.records[slice.processing_record_number - 1..-1]
-          append  = true
-          logger.info("Resuming previously incomplete slice from record number #{slice.processing_record_number}")
-        else
-          # Reprocess all records in this slice.
-          slice.processing_record_number = 0
-          records                        = slice.records
-        end
+        # Skip processed records in this slice if it has no output categories.
+        records =
+          if slice.processing_record_number.to_i > 1
+            append = true
+            logger.info("Resuming previously incomplete slice from record number #{slice.processing_record_number}")
+            slice.records[slice.processing_record_number - 1..-1]
+          else
+            # Reprocess all records in this slice.
+            slice.processing_record_number = 0
+            slice.records
+          end
 
         count = 0
         RocketJob::Sliced::Writer::Output.collect(self, input_slice: slice, append: append) do |writer|
@@ -246,7 +248,7 @@ module RocketJob
         unless new_record?
           # Fail job iff no other worker has already finished it
           # Must set write concern to at least 1 since we need the nModified back
-          result = self.class.with(write: {w: 1}) do |query|
+          result   = self.class.with(write: {w: 1}) do |query|
             query.
               where(id: id, state: :running, sub_state: :processing).
               update({"$set" => {state: :failed, worker_name: worker_name}})

data/lib/rocket_job/category/base.rb

@@ -11,7 +11,6 @@ module RocketJob
 
         # Whether to compress, encrypt, or use the bzip2 serialization for data in this category.
         field :serializer, type: ::Mongoid::StringifiedSymbol, default: :compress
-        validates_inclusion_of :serializer, in: %i[none compress encrypt bzip2]
 
         # The header columns when the file does not include a header row.
         # Note:
@@ -49,10 +48,12 @@ module RocketJob
           Sliced::CompressedSlice
         when :encrypt
           Sliced::EncryptedSlice
-        when :bzip2
+        when :bzip2, :bz2
           Sliced::BZip2OutputSlice
+        when :encrypted_bz2
+          Sliced::EncryptedBZip2OutputSlice
         else
-          raise(ArgumentError, "serialize: #{serializer.inspect} must be :none, :compress, :encrypt, or :bzip2")
+          raise(ArgumentError, "serialize: #{serializer.inspect} must be :none, :compress, :encrypt, :bz2, or :encrypted_bz2")
         end
       end
 
@@ -65,14 +66,16 @@ module RocketJob
         )
       end
 
-      def reset_tabular
-        @tabular = nil
-      end
-
       # Returns [true|false] whether this category has the attributes defined for tabular to work.
       def tabular?
         format.present?
       end
+
+      def build_collection_name(direction, job)
+        collection_name = "rocket_job.#{direction}s.#{job.id}"
+        collection_name << ".#{name}" unless name == :main
+        collection_name
+      end
     end
   end
 end

data/lib/rocket_job/category/input.rb

@@ -10,6 +10,7 @@ module RocketJob
 
       # Slice size for this input collection
       field :slice_size, type: Integer, default: 100
+      validates_presence_of :slice_size
 
       #
       # The fields below only apply if the field `format` has been set:
@@ -82,7 +83,7 @@ module RocketJob
       field :header_cleanser, type: ::Mongoid::StringifiedSymbol, default: :default
       validates :header_cleanser, inclusion: %i[default none]
 
-      validates_presence_of :slice_size
+      validates_inclusion_of :serializer, in: %i[none compress encrypt]
 
       # Cleanses the header column names when `cleanse_header` is true
       def cleanse_header!
@@ -105,6 +106,65 @@ module RocketJob
           skip_unknown:     skip_unknown
         )
       end
+
+      def data_store(job)
+        RocketJob::Sliced::Input.new(
+          collection_name: build_collection_name(:input, job),
+          slice_class:     serializer_class,
+          slice_size:      slice_size
+        )
+      end
+
+      # Returns [IOStreams::Path] of file to upload.
+      # Auto-detects file format from file name when format is :auto.
+      def upload_path(stream = nil, original_file_name: nil)
+        unless stream || file_name
+          raise(ArgumentError, "Either supply a file name to upload, or set input_collection.file_name first")
+        end
+
+        path           = IOStreams.new(stream || file_name)
+        path.file_name = original_file_name if original_file_name
+        self.file_name = path.file_name
+
+        # Auto detect the format based on the upload file name if present.
+        if format == :auto
+          self.format = path.format || :csv
+          # Rebuild tabular with new values.
+          @tabular = nil
+        end
+
+        # Remove non-printable characters from tabular input formats.
+        if tabular?
+          # Cannot change the length of fixed width lines.
+          replace = format == :fixed ? " " : ""
+          path.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
+        end
+        path
+      end
+
+      # Return a lambda to extract the header row from the uploaded file.
+      def extract_header_callback(on_first)
+        return on_first unless tabular? && tabular.header?
+
+        case mode
+        when :line
+          lambda do |line|
+            tabular.parse_header(line)
+            cleanse_header!
+            self.columns = tabular.header.columns
+            # Call chained on_first if present
+            on_first&.call(line)
+          end
+        when :array
+          lambda do |row|
+            tabular.header.columns = row
+            cleanse_header!
+            self.columns = category.tabular.header.columns
+            # Call chained on_first if present
+            on_first&.call(line)
+          end
+        end
+      end
     end
   end
 end

data/lib/rocket_job/category/output.rb

@@ -13,6 +13,8 @@ module RocketJob
       #   false: do not save nil values to the output categories.
       field :nils, type: ::Mongoid::Boolean, default: false
 
+      validates_inclusion_of :serializer, in: %i[none compress encrypt bz2 encrypted_bz2 bzip2]
+
       # Renders [String] the header line.
       # Returns [nil] if no header is needed.
       def render_header
@@ -20,6 +22,13 @@ module RocketJob
 
         tabular.render_header
       end
+
+      def data_store(job)
+        RocketJob::Sliced::Output.new(
+          collection_name: build_collection_name(:output, job),
+          slice_class:     serializer_class
+        )
+      end
     end
   end
 end

data/lib/rocket_job/dirmon_entry.rb

@@ -173,7 +173,7 @@ module RocketJob
       counts
     end
 
-    # Passes each filename [Pathname] found that matches the pattern into the supplied block
+    # Yields [IOStreams::Path] for each file found that matches the current pattern.
     def each
       SemanticLogger.named_tagged(dirmon_entry: id.to_s) do
         # Case insensitive filename matching

data/lib/rocket_job/jobs/conversion_job.rb

@@ -1,39 +1,43 @@
 # Convert to and from CSV, JSON, xlsx, and PSV files.
 #
 # Example, Convert CSV file to JSON.
-#   job = RocketJob::ConversionJob.new
-#   job.upload("data.csv")
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "data.csv"
 #   job.output_category.file_name = "data.json"
 #   job.save!
 #
 # Example, Convert JSON file to PSV and compress it with GZip.
-#   job = RocketJob::ConversionJob.new
-#   job.upload("data.json")
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "data.json"
 #   job.output_category.file_name = "data.psv.gz"
 #   job.save!
 #
 # Example, Read a CSV file that has been zipped from a remote website and the convert it to a GZipped json file.
-#   job = RocketJob::ConversionJob.new
-#   job.upload("https://example.org/file.zip")
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "https://example.org/file.zip"
 #   job.output_category.file_name = "data.json.gz"
 #   job.save!
 #
 module RocketJob
-  class ConversionJob < RocketJob::Job
-    include RocketJob::Batch
+  module Jobs
+    class ConversionJob < RocketJob::Job
+      include RocketJob::Batch
 
-    self.destroy_on_complete = false
+      self.destroy_on_complete = false
 
-    # Detects file extension for its type
-    input_category format: :auto
-    output_category format: :auto
+      # Detects file extension for its type
+      input_category format: :auto
+      output_category format: :auto
 
-    # When the job completes it will write the result to the output_category.file_name
-    after_batch :download
+      # Upload the file specified in `input_category.file_name` unless already uploaded.
+      before_batch :upload, unless: :record_count
 
-    def perform(hash)
-      # For this job return the input hash record as-is. Could be transformed here as needed.
-      hash
+      # When the job completes it will write the result to `output_category.file_name`.
+      after_batch :cleanup!, :download
+
+      def perform(hash)
+        hash
+      end
     end
   end
 end

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -30,59 +30,48 @@ module RocketJob
     #
     # If another DirmonJob instance is already queued or running, then the create
     # above will fail with:
-    #   MongoMapper::DocumentNotValid: Validation failed: State Another instance of this job is already queued or running
+    #   Validation failed: State Another instance of this job is already queued or running
     #
     # Or to start DirmonJob and ignore errors if already running
     #   RocketJob::Jobs::DirmonJob.create
     class DirmonJob < RocketJob::Job
-      # Only allow one DirmonJob instance to be running at a time
-      include RocketJob::Plugins::Singleton
-      # Start a new job when this one completes, fails, or aborts
-      include RocketJob::Plugins::Restart
+      include RocketJob::Plugins::Cron
 
-      self.priority = 30
-
-      # Number of seconds between directory scans. Default 5 mins
-      field :check_seconds, type: Float, default: 300.0, copy_on_restart: true
+      # Runs every 5 minutes by default
+      self.cron_schedule = "*/5 * * * * UTC"
+      self.description   = "Directory Monitor"
+      self.priority      = 30
 
       # Hash[file_name, size]
       field :previous_file_names, type: Hash, default: {}, copy_on_restart: true
 
-      before_create :set_run_at
-
-      # Iterate over each Dirmon entry looking for new files
-      # If a new file is found, it is not processed immediately, instead
-      # it is passed to the next run of this job along with the file size.
-      # If the file size has not changed, the Job is kicked off.
+      # Checks the directories for new files, starting jobs if files have not changed since the last run.
       def perform
         check_directories
       end
 
       private
 
-      # Set a run_at when a new instance of this job is created
-      def set_run_at
-        self.run_at = Time.now + check_seconds
-      end
-
-      # Checks the directories for new files, starting jobs if files have not changed
-      # since the last run
+      # Iterate over each Dirmon Entry looking for new files
+      # If a new file is found, it is not processed immediately, instead
+      # it is passed to the next run of this job along with the file size.
+      # If the file size has not changed, the Job is kicked off.
       def check_directories
         new_file_names = {}
-        DirmonEntry.enabled.each do |entry|
-          entry.each do |iopath|
-            # S3 files are only visible once completely uploaded.
-            unless iopath.partial_files_visible?
-              logger.info("File: #{iopath}. Starting: #{entry.job_class_name}")
-              entry.later(iopath)
+        DirmonEntry.enabled.each do |dirmon_entry|
+          dirmon_entry.each do |path|
+            # Skip file size checking since S3 files are only visible once completely uploaded.
+            unless path.partial_files_visible?
+              logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
+              dirmon_entry.later(path)
               next
             end
 
             # BSON Keys cannot contain periods
-            key           = iopath.to_s.tr(".", "_")
+            key           = path.to_s.tr(".", "_")
             previous_size = previous_file_names[key]
             # Check every few minutes for a file size change before trying to process the file.
-            size                = check_file(entry, iopath, previous_size)
+            size                = check_file(dirmon_entry, path, previous_size)
             new_file_names[key] = size if size
           end
         end
@@ -91,14 +80,14 @@ module RocketJob
 
       # Checks if a file should result in starting a job
       # Returns [Integer] file size, or nil if the file started a job
-      def check_file(entry, iopath, previous_size)
-        size = iopath.size
+      def check_file(dirmon_entry, path, previous_size)
+        size = path.size
         if previous_size && (previous_size == size)
-          logger.info("File stabilized: #{iopath}. Starting: #{entry.job_class_name}")
-          entry.later(iopath)
+          logger.info("File stabilized: #{path}. Starting: #{dirmon_entry.job_class_name}")
+          dirmon_entry.later(path)
           nil
         else
-          logger.info("Found file: #{iopath}. File size: #{size}")
+          logger.info("Found file: #{path}. File size: #{size}")
           # Keep for the next run
           size
         end

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -27,12 +27,11 @@ module RocketJob
     #   )
     class HousekeepingJob < RocketJob::Job
       include RocketJob::Plugins::Cron
-      include RocketJob::Plugins::Singleton
 
-      self.priority    = 25
-      self.description = "Cleans out historical jobs, and zombie servers."
-      # Runs every 15 minutes
-      self.cron_schedule = "*/15 * * * * UTC"
+      # Runs every 15 minutes on the 15 minute period
+      self.cron_schedule = "0,15,30,45 * * * * UTC"
+      self.description   = "Cleans out historical jobs, and zombie servers."
+      self.priority      = 25
 
       # Whether to destroy zombie servers automatically
       field :destroy_zombies, type: Mongoid::Boolean, default: true, user_editable: true, copy_on_restart: true

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -65,27 +65,29 @@ module RocketJob
   module Jobs
     class OnDemandBatchJob < RocketJob::Job
       include RocketJob::Plugins::Cron
+      include RocketJob::Plugins::Retry
       include RocketJob::Batch
       include RocketJob::Batch::Statistics
 
       self.priority            = 90
-      self.description         = "Batch Job"
+      self.description         = "On Demand Batch Job"
       self.destroy_on_complete = false
+      self.retry_limit         = 0
 
       # Code that is performed against every row / record.
-      field :code, type: String
+      field :code, type: String, user_editable: true, copy_on_restart: true
 
       # Optional code to execute before the batch is run.
       # Usually to upload data into the job.
-      field :before_code, type: String
+      field :before_code, type: String, user_editable: true, copy_on_restart: true
 
       # Optional code to execute after the batch is run.
       # Usually to upload data into the job.
-      field :after_code, type: String
+      field :after_code, type: String, user_editable: true, copy_on_restart: true
 
       # Data that is made available to the job during the perform.
       # Be sure to store key names only as Strings, not Symbols.
-      field :data, type: Hash, default: {}
+      field :data, type: Hash, default: {}, user_editable: true, copy_on_restart: true
 
       validates :code, presence: true
       validate :validate_code

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -78,8 +78,8 @@ module RocketJob
       self.retry_limit         = 0
 
       # Be sure to store key names only as Strings, not Symbols
-      field :data, type: Hash, default: {}, copy_on_restart: true
-      field :code, type: String, copy_on_restart: true
+      field :data, type: Hash, default: {}, user_editable: true, copy_on_restart: true
+      field :code, type: String, user_editable: true, copy_on_restart: true
 
       validates :code, presence: true
       validate :validate_code

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -57,6 +57,10 @@ module RocketJob
 
       def upload_file(job)
         if job.respond_to?(:upload)
+          # Return the database connection for this thread back to the connection pool
+          # in case the upload takes a long time and the database connection expires.
+          ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
+
           if original_file_name
             job.upload(upload_file_name, file_name: original_file_name)
           else

data/lib/rocket_job/plugins/cron.rb

@@ -14,41 +14,81 @@ module RocketJob
       extend ActiveSupport::Concern
 
       included do
-        include Restart
-
         field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
 
+        # Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
+        # Another job instance with a different `cron_schedule` string is permitted.
+        field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        # Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
+        #
+        # `true`: Create a new scheduled instance of this job after it has started. (Default)
+        #   - Ensures that the next scheduled instance is not missed because the current instance is still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     _only_ if they were changed during an `after_start` callback.
+        #     Changes to these during other callbacks or during the `perform` will not be saved to the new scheduled
+        #     instance.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil`.
+        #
+        # `false`: Create a new scheduled instance of this job on `fail`, or `abort`.
+        #   - Prevents the next scheduled instance from running or being scheduled while the current instance is
+        #     still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     at any time until after the job has failed, or is aborted.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil` after it fails or is aborted.
+        field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
         validates_each :cron_schedule do |record, attr, value|
           record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
         end
+        validate :rocket_job_cron_singleton_check
+
         before_save :rocket_job_cron_set_run_at
 
-        private
+        after_start :rocket_job_cron_on_start
+        after_abort :rocket_job_cron_end_state
+        after_complete :rocket_job_cron_end_state
+        after_fail :rocket_job_cron_end_state
+      end
 
-        # Prevent auto restart if this job does not have a cron schedule.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_new_instance
-        def rocket_job_restart_new_instance
-          return unless cron_schedule
+      def rocket_job_cron_set_run_at
+        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
 
-          super
-        end
+        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+      end
 
-        # On failure:
-        # - create a new instance scheduled to run in the future.
-        # - clear out the `cron_schedule` so this instance will not schedule another instance to run on completion.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_abort
-        def rocket_job_restart_abort
-          return unless cron_schedule
+      private
 
-          rocket_job_restart_new_instance
-          update_attribute(:cron_schedule, nil)
+      def rocket_job_cron_on_start
+        return unless cron_schedule && cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      def rocket_job_cron_end_state
+        return unless cron_schedule && !cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      # Returns [true|false] whether another instance of this job with the same cron schedule is already active
+      def rocket_job_cron_duplicate?
+        self.class.with(read: {mode: :primary}) do |conn|
+          conn.where(:state.in => %i[queued running failed paused], :id.ne => id, cron_schedule: cron_schedule).exists?
         end
       end
 
-      def rocket_job_cron_set_run_at
-        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
+      # Prevent creation of a new job when another is running with the same cron schedule.
+      def rocket_job_cron_singleton_check
+        return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
 
-        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+        errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
       end
     end
   end