rocketjob 5.2.0 → 5.4.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6715f9b19c04ee07962197db8e635e9e5e74b1e1dbd9e3add5239d235c5dd218
-  data.tar.gz: 5c501260e0e328691f668325ff48e88ea5da86f49f6a739343b5bf24569aee75
+  metadata.gz: 55ab6ca2b3f76cdb4ddf679a2d3e88e1d6a6f3106a69349129a267b629ef4a53
+  data.tar.gz: 4e0d07878fb4265179b4a270650cc9b89ca4bca55f1d8f9a3451cb3064062c35
 SHA512:
-  metadata.gz: db9890e678c9e2cb97c6f8faf25e8115f5946017ed3dc0fc3f208a62c2765fabb2c5f7673c9850b6202c61bd312c0e535944609e21e3dbbfd514eff2a50ffa0a
-  data.tar.gz: ede6fbe558cb428c3291bba1584d32dffcc65c6889ab90753e1a79ce33713410b071e2db44bf5007119ba0a83867e2d42485f1b16bfa4070b0a36e7ca89bbf13
+  metadata.gz: fdc7ae3827d987404f431410cb81161fbfa269dfe7575a4a86a6abf362c59c6acd0d2c7e9856273ced396eeaf20e50561f1be10a7de0d5c4ae45e5648d15d083
+  data.tar.gz: 7bb9e9ac90569e78e135293efcd357a0d75037b5417f5f530f18ae3f17f44acf28d7de5b4aeef880a0315117f2992623958a4fb4c9df93ca7273369ac052e759

data/README.md

@@ -3,9 +3,9 @@
 
 Ruby's missing batch system
 
-Checkout http://rocketjob.io/
+Checkout https://rocketjob.io/
 
-![Rocket Job](http://rocketjob.io/images/rocket/rocket-icon-512x512.png)
+![Rocket Job](https://rocketjob.io/images/rocket/rocket-icon-512x512.png)
 
 ## Documentation
 

data/lib/rocket_job/batch.rb

@@ -24,6 +24,7 @@ module RocketJob
     autoload :LowerPriority, "rocket_job/batch/lower_priority"
     autoload :Performance, "rocket_job/batch/performance"
     autoload :Statistics, "rocket_job/batch/statistics"
+    autoload :ThrottleWindows, "rocket_job/batch/throttle_windows"
     autoload :Result, "rocket_job/batch/result"
     autoload :Results, "rocket_job/batch/results"
     autoload :Tabular, "rocket_job/batch/tabular"

data/lib/rocket_job/batch/io.rb

@@ -18,7 +18,7 @@ module RocketJob
           raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}"
         end
 
-        (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(rocket_job_io_slice_arguments("inputs", category))
+        (@inputs ||= {})[category] ||= RocketJob::Sliced.factory(:input, category, self)
       end
 
       # Returns [RocketJob::Sliced::Output] output collection for holding output slices
@@ -34,7 +34,7 @@ module RocketJob
           raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}"
         end
 
-        (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(rocket_job_io_slice_arguments("outputs", category))
+        (@outputs ||= {})[category] ||= RocketJob::Sliced.factory(:output, category, self)
       end
 
       # Upload the supplied file, io, IOStreams::Path, or IOStreams::Stream.
@@ -355,8 +355,18 @@ module RocketJob
 
         return output(category).download(header_line: header_line, &block) if block
 
-        IOStreams.new(stream).writer(:line, **args) do |io|
-          output(category).download(header_line: header_line) { |record| io << record }
+        output_collection = output(category)
+
+        if output_collection.binary?
+          IOStreams.new(stream).stream(:none).writer(**args) do |io|
+            raise(ArgumenError, "A `header_line` is not supported with binary output collections") if header_line
+
+            output_collection.download { |record| io << record[:binary] }
+          end
+        else
+          IOStreams.new(stream).writer(:line, **args) do |io|
+            output_collection.download(header_line: header_line) { |record| io << record }
+          end
         end
       end
 
@@ -393,21 +403,6 @@ module RocketJob
           RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
         end
       end
-
-      private
-
-      def rocket_job_io_slice_arguments(collection_type, category)
-        collection_name = "rocket_job.#{collection_type}.#{id}"
-        collection_name << ".#{category}" unless category == :main
-
-        args = {collection_name: collection_name, slice_size: slice_size}
-        if encrypt
-          args[:slice_class] = Sliced::EncryptedSlice
-        elsif compress
-          args[:slice_class] = Sliced::CompressedSlice
-        end
-        args
-      end
     end
   end
 end

data/lib/rocket_job/batch/model.rb

@@ -44,12 +44,12 @@ module RocketJob
         # Compress uploaded records.
         # The fields are not affected in any way, only the data stored in the
         # records and results collections will compressed
-        field :compress, type: Boolean, default: false, class_attribute: true
+        field :compress, type: Object, default: false, class_attribute: true
 
         # Encrypt uploaded records.
         # The fields are not affected in any way, only the data stored in the
         # records and results collections will be encrypted
-        field :encrypt, type: Boolean, default: false, class_attribute: true
+        field :encrypt, type: Object, default: false, class_attribute: true
 
         #
         # Values that jobs can also update during processing

data/lib/rocket_job/batch/tabular/input.rb

@@ -12,6 +12,7 @@ module RocketJob
         included do
           field :tabular_input_header, type: Array, class_attribute: true, user_editable: true
           field :tabular_input_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true
+          field :tabular_input_options, type: Hash, class_attribute: true
 
           # tabular_input_mode: [:line | :array | :hash]
           #   :line
@@ -53,7 +54,9 @@ module RocketJob
           input_stream = stream.nil? ? nil : IOStreams.new(stream)
 
           if stream && (tabular_input_type == :text)
-            input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: "")
+            # Cannot change the length of fixed width lines
+            replace = tabular_input_format == :fixed ? " " : ""
+            input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
           end
 
           # If an input header is not required, then we don't extract it'
@@ -96,14 +99,15 @@ module RocketJob
             allowed_columns:  tabular_input_white_list,
             required_columns: tabular_input_required,
             skip_unknown:     tabular_input_skip_unknown,
-            format:           tabular_input_format
+            format:           tabular_input_format,
+            format_options:   tabular_input_options&.deep_symbolize_keys
           )
         end
 
         def tabular_input_render
-          unless tabular_input_header.blank? && tabular_input.header?
-            @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
-          end
+          return if tabular_input_header.blank? && tabular_input.header?
+
+          @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
         end
 
         # Cleanse custom input header if supplied.

data/lib/rocket_job/batch/tabular/output.rb

@@ -12,6 +12,7 @@ module RocketJob
         included do
           field :tabular_output_header, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
           field :tabular_output_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
+          field :tabular_output_options, type: Hash, class_attribute: true
 
           validates_inclusion_of :tabular_output_format, in: IOStreams::Tabular.registered_formats
 
@@ -31,8 +32,9 @@ module RocketJob
 
         # Overrides: `RocketJob::Batch::IO#download` to add the `tabular_output_header`.
         def download(file_name_or_io = nil, category: :main, **args, &block)
-          # No header required
-          return super(file_name_or_io, category: category, **args, &block) unless tabular_output.requires_header?(category)
+          unless tabular_output.requires_header?(category)
+            return super(file_name_or_io, category: category, **args, &block)
+          end
 
           header = tabular_output.render_header(category)
           super(file_name_or_io, header_line: header, category: category, **args, &block)
@@ -43,7 +45,11 @@ module RocketJob
         # Delimited instance used for this slice, by a single worker (thread)
         def tabular_output
           @tabular_output ||= Tabular.new(
-            main: IOStreams::Tabular.new(columns: tabular_output_header, format: tabular_output_format)
+            main: IOStreams::Tabular.new(
+              columns:        tabular_output_header,
+              format:         tabular_output_format,
+              format_options: tabular_output_options&.deep_symbolize_keys
+            )
           )
         end
 

data/lib/rocket_job/batch/throttle.rb

@@ -6,7 +6,7 @@ module RocketJob
     #
     # Example:
     #   # Do not run any slices for this job when the MySQL slave delay exceeds 5 minutes.
-    #   class MyJob < RocketJob
+    #   class MyJob < RocketJob::Job
     #     include RocketJob::Batch
     #
     #     # Define a custom mysql throttle

data/lib/rocket_job/batch/throttle_running_workers.rb

@@ -5,7 +5,7 @@ module RocketJob
     # Throttle the number of slices of a specific batch job that are processed at the same time.
     #
     # Example:
-    #   class MyJob < RocketJob
+    #   class MyJob < RocketJob::Job
     #     include RocketJob::Batch
     #
     #     # Maximum number of slices to process at the same time for each running instance.

data/lib/rocket_job/batch/throttle_windows.rb

@@ -0,0 +1,72 @@
+require "active_support/concern"
+require "fugit"
+
+module RocketJob
+  module Batch
+    # For a batch job that can run over a long period of time it can be useful
+    # to prevent its slices from being processed outside a predefined processing window.
+    #
+    # This plugin supports up to 2 different processing windows.
+    #
+    # For example, do not run this job during business hours.
+    # Allow it to run from 5pm until 8am the following day Mon through Fri.
+    #
+    #   class AfterHoursJob < RocketJob::Job
+    #     include RocketJob::Batch
+    #     include RocketJob::Batch::ThrottleWindows
+    #
+    #     # Monday through Thursday the job can start processing at 5pm Eastern.
+    #     self.primary_schedule = "0 17 * * 1-4 America/New_York"
+    #     # Slices are allowed to run until 8am the following day, which is 15 hours long:
+    #     self.primary_duration = 15.hours
+    #
+    #     # The slices for this job can run all weekend long, starting Friday at 5pm Eastern.
+    #     self.secondary_schedule = "0 17 * * 5 America/New_York"
+    #     # Slices are allowed to run until 8am on Monday morning, which is 63 hours long:
+    #     self.secondary_duration = 63.hours
+    #   end
+    #
+    # Notes:
+    # * These schedules do not affect when the job is started, completed, or when `before_batch` or
+    #   `after_batch` processing is performed. It only limits when individual slices are processed.
+    module ThrottleWindows
+      extend ActiveSupport::Concern
+
+      included do
+        # Beginning of the primary schedule. In cron format, see Scheduled Jobs `cron_schedule` for examples.
+        field :primary_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
+        # Duration in seconds of the primary window.
+        field :primary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        # Beginning of the secondary schedule. In cron format, see Scheduled Jobs `cron_schedule` for examples.
+        field :secondary_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
+        # Duration in seconds of the secondary window.
+        field :secondary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        define_batch_throttle :throttle_windows_exceeded?, filter: :throttle_filter_id
+
+        validates_each :primary_schedule, :secondary_schedule do |record, attr, value|
+          record.errors.add(attr, "Invalid #{attr}: #{value.inspect}") if value && !Fugit::Cron.new(value)
+        end
+      end
+
+      private
+
+      def throttle_windows_exceeded?
+        exceeded = primary_schedule && primary_duration && throttle_outside_window?(primary_schedule, primary_duration)
+        if exceeded && secondary_schedule && secondary_duration
+          exceeded = throttle_outside_window?(secondary_schedule, secondary_duration)
+        end
+        exceeded
+      end
+
+      def throttle_outside_window?(schedule, duration)
+        cron = Fugit::Cron.new(schedule)
+        time = Time.now.utc + 1
+        # Add 1 second since right now could be the very beginning of the processing window.
+        previous_time = cron.previous_time(time).to_utc_time
+        previous_time + duration < time
+      end
+    end
+  end
+end

data/lib/rocket_job/batch/worker.rb

@@ -28,10 +28,6 @@ module RocketJob
       #
       # If an exception was thrown the entire slice of records is marked as failed.
       #
-      # If the mongo_ha gem has been loaded, then the connection to mongo is
-      # automatically re-established and the job will resume anytime a
-      # Mongo connection failure occurs.
-      #
       # Thread-safe, can be called by multiple threads at the same time
       def rocket_job_work(worker, re_raise_exceptions = false)
         raise "Job must be started before calling #rocket_job_work" unless running?
@@ -50,7 +46,7 @@ module RocketJob
               next if slice.failed?
 
               slice.fail_on_exception!(re_raise_exceptions) { rocket_job_process_slice(slice) }
-            elsif record_count && rocket_job_batch_complete?(worker.name)
+            elsif record_count && fail_on_exception!(re_raise_exceptions) { rocket_job_batch_complete?(worker.name) }
               return false
             else
               logger.debug "No more work available for this job"
@@ -114,8 +110,6 @@ module RocketJob
         servers
       end
 
-      private
-
       def rocket_job_batch_throttled?(slice, worker)
         filter = self.class.rocket_job_batch_throttles.matching_filter(self, slice)
         return false unless filter
@@ -154,7 +148,7 @@ module RocketJob
           records = slice.records
 
           # Skip records already processed, if any.
-          #slice.processing_record_number ||= 0
+          # slice.processing_record_number ||= 0
           # TODO: Must append to existing output slices before this can be enabled.
           # if !collect_output && (slice.processing_record_number > 1)
           #   records = records[slice.processing_record_number - 1..-1]

data/lib/rocket_job/event.rb

@@ -104,8 +104,6 @@ module RocketJob
       end
     end
 
-    private
-
     @load_time   = Time.now.utc
     @subscribers = Concurrent::Map.new { Concurrent::Array.new }
 

data/lib/rocket_job/extensions/mongoid/clients/options.rb

@@ -22,8 +22,6 @@ module RocketJob
         @collection_name = collection_name&.to_sym
       end
 
-      private
-
       module ClassMethods
         def with_collection(collection_name)
           all.with_collection(collection_name)

data/lib/rocket_job/extensions/mongoid/remove_warnings.rb

@@ -0,0 +1,12 @@
+require "mongoid/fields/validators/macro"
+require "semantic_logger"
+module RocketJob
+  module RemoveMongoidWarnings
+    # Remove annoying warnings about Symbols type being deprecated.
+    def validate_options(*params)
+      SemanticLogger.silence(:error) { super(*params) }
+    end
+  end
+end
+
+::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)

data/lib/rocket_job/jobs/copy_file_job.rb

@@ -25,7 +25,7 @@ module RocketJob
 
       self.destroy_on_complete = false
       # Number of times to automatically retry the copy. Set to `0` for no retry attempts.
-      self.retry_limit = 5
+      self.retry_limit = 10
 
       # File names in IOStreams URL format.
       field :source_url, type: String, user_editable: true

data/lib/rocket_job/jobs/re_encrypt/relational_job.rb

@@ -1,8 +1,5 @@
-begin
-  require "active_record"
-rescue LoadError
-  raise 'RocketJob::Jobs::ReEncrypt::RelationalJob uses ActiveRecord to obtain the database connection, please install the gem "activerecord".'
-end
+require "active_record"
+require "sync_attr"
 
 # Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
 #

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -40,7 +40,7 @@ module RocketJob
         job.id = job_id if job_id
         upload_file(job)
         job.save!
-      rescue StandardError => e
+      rescue Exception => e
         # Prevent partial uploads
         job&.cleanup! if job.respond_to?(:cleanup!)
         raise(e)

data/lib/rocket_job/plugins/cron.rb

@@ -1,4 +1,5 @@
 require "active_support/concern"
+require "fugit"
 
 module RocketJob
   module Plugins
@@ -17,7 +18,9 @@ module RocketJob
 
         field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
 
-        validate :rocket_job_cron_valid
+        validates_each :cron_schedule do |record, attr, value|
+          record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
+        end
         before_save :rocket_job_cron_set_run_at
 
         private
@@ -42,30 +45,10 @@ module RocketJob
         end
       end
 
-      # Returns [Time] the next time this job will be scheduled to run at.
-      #
-      # Parameters
-      #   time: [Time]
-      #     The next time as of this time.
-      #     Default: Time.now
-      def rocket_job_cron_next_time(time = Time.now)
-        RocketJob::Plugins::Rufus::CronLine.new(cron_schedule).next_time(time)
-      end
-
-      private
-
       def rocket_job_cron_set_run_at
-        return unless cron_schedule
-
-        self.run_at = rocket_job_cron_next_time if cron_schedule_changed? && !run_at_changed?
-      end
-
-      def rocket_job_cron_valid
-        return unless cron_schedule
+        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
 
-        RocketJob::Plugins::Rufus::CronLine.new(cron_schedule)
-      rescue ArgumentError => e
-        errors.add(:cron_schedule, e.message)
+        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
       end
     end
   end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -7,7 +7,7 @@ module RocketJob
       #
       # Example:
       #   # Do not run this job when the MySQL slave delay exceeds 5 minutes.
-      #   class MyJob < RocketJob
+      #   class MyJob < RocketJob::Job
       #     # Define a custom mysql throttle
       #     # Prevents all jobs of this class from running on the current server.
       #     define_throttle :mysql_throttle_exceeded?

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -6,7 +6,7 @@ module RocketJob
       # Throttle the number of jobs of a specific class that are processed at the same time.
       #
       # Example:
-      #   class MyJob < RocketJob
+      #   class MyJob < RocketJob::Job
       #     # Maximum number of jobs of this class to process at the same time.
       #     self.throttle_running_jobs = 25
       #

data/lib/rocket_job/plugins/job/worker.rb

@@ -73,12 +73,13 @@ module RocketJob
             if failed? || !may_fail?
               self.exception        = JobException.from_exception(e)
               exception.worker_name = worker_name
-              save! unless new_record? || destroyed?
-            elsif new_record? || destroyed?
-              fail(worker_name, e)
             else
-              fail!(worker_name, e)
+              fail(worker_name, e)
             end
+
+            # Prevent validation failures from failing the job
+            save(validate: false) unless new_record? || destroyed?
+
             raise e if re_raise_exceptions
           end
         end