rocketjob 5.3.2 → 5.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fc6fef03f96de2e2645a4419a4f4f0cdf409da64a7bb708bcddb24ae0fc6c4f
-  data.tar.gz: 4c8573924fae294676afc2114a83b522c64a3b259e679a2a343fc4b34f7bab8f
+  metadata.gz: 97910a2bbdcc67b808677e01f9ef9650935deac9f47ee415ad7665227e06e2a3
+  data.tar.gz: 9c560b2a1deea93eb4c8e9cdf6fb28f2308e8f9a85457660c292ad38a2b76137
 SHA512:
-  metadata.gz: e21ca8c550d28e77d48a6b47b3f7e0c822350e72ba557e56749277f8a2536cd669f63b86b20c39bc264b8619196f40a2aa38940cb04aedaad1a1a07daa36b666
-  data.tar.gz: cd3202dd8ecf093ba591385aab9529f8f8f2d1bf7f286252db9ab7589f49c86293d96f3e72bd5ba85eed6b344eb92f35414cc3c32ee327dcf00f722b4929d2a4
+  metadata.gz: 768538825ba34f58d915205fb1c61066ef2d86d9c3743f487db381074e30bd75c420f99f017adbb338c0dd4f202a69cf699b0b4e2a9ebc0581423fb102c85284
+  data.tar.gz: 17d494818bf341a6b3340f4c0c5d5b77ec68c05eeb26d10eccb43c81221c14acd64ffe73ea1f610b514f233c84d5cc7cf3c2a10f4d1410b9b427308d22f5ad53

data/lib/rocket_job/batch/io.rb

@@ -18,7 +18,7 @@ module RocketJob
           raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}"
         end
 
-        (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(**rocket_job_io_slice_arguments("inputs", category))
+        (@inputs ||= {})[category] ||= RocketJob::Sliced.factory(:input, category, self)
       end
 
       # Returns [RocketJob::Sliced::Output] output collection for holding output slices
@@ -34,7 +34,7 @@ module RocketJob
           raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}"
         end
 
-        (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(**rocket_job_io_slice_arguments("outputs", category))
+        (@outputs ||= {})[category] ||= RocketJob::Sliced.factory(:output, category, self)
       end
 
       # Upload the supplied file, io, IOStreams::Path, or IOStreams::Stream.
@@ -355,8 +355,18 @@ module RocketJob
 
         return output(category).download(header_line: header_line, &block) if block
 
-        IOStreams.new(stream).writer(:line, **args) do |io|
-          output(category).download(header_line: header_line) { |record| io << record }
+        output_collection = output(category)
+
+        if output_collection.binary?
+          IOStreams.new(stream).stream(:none).writer(**args) do |io|
+            raise(ArgumenError, "A `header_line` is not supported with binary output collections") if header_line
+
+            output_collection.download { |record| io << record[:binary] }
+          end
+        else
+          IOStreams.new(stream).writer(:line, **args) do |io|
+            output_collection.download(header_line: header_line) { |record| io << record }
+          end
         end
       end
 
@@ -393,21 +403,6 @@ module RocketJob
           RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
         end
       end
-
-      private
-
-      def rocket_job_io_slice_arguments(collection_type, category)
-        collection_name = "rocket_job.#{collection_type}.#{id}"
-        collection_name << ".#{category}" unless category == :main
-
-        args = {collection_name: collection_name, slice_size: slice_size}
-        if encrypt
-          args[:slice_class] = Sliced::EncryptedSlice
-        elsif compress
-          args[:slice_class] = Sliced::CompressedSlice
-        end
-        args
-      end
     end
   end
 end

data/lib/rocket_job/batch/model.rb

@@ -44,12 +44,12 @@ module RocketJob
         # Compress uploaded records.
         # The fields are not affected in any way, only the data stored in the
         # records and results collections will compressed
-        field :compress, type: Boolean, default: false, class_attribute: true
+        field :compress, type: Object, default: false, class_attribute: true
 
         # Encrypt uploaded records.
         # The fields are not affected in any way, only the data stored in the
         # records and results collections will be encrypted
-        field :encrypt, type: Boolean, default: false, class_attribute: true
+        field :encrypt, type: Object, default: false, class_attribute: true
 
         #
         # Values that jobs can also update during processing

data/lib/rocket_job/batch/tabular/input.rb

@@ -12,6 +12,7 @@ module RocketJob
         included do
           field :tabular_input_header, type: Array, class_attribute: true, user_editable: true
           field :tabular_input_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true
+          field :tabular_input_options, type: Hash, class_attribute: true
 
           # tabular_input_mode: [:line | :array | :hash]
           #   :line
@@ -53,7 +54,9 @@ module RocketJob
           input_stream = stream.nil? ? nil : IOStreams.new(stream)
 
           if stream && (tabular_input_type == :text)
-            input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: "")
+            # Cannot change the length of fixed width lines
+            replace = tabular_input_format == :fixed ? " " : ""
+            input_stream.option_or_stream(:encode, encoding: "UTF-8", cleaner: :printable, replace: replace)
           end
 
           # If an input header is not required, then we don't extract it'
@@ -96,14 +99,15 @@ module RocketJob
             allowed_columns:  tabular_input_white_list,
             required_columns: tabular_input_required,
             skip_unknown:     tabular_input_skip_unknown,
-            format:           tabular_input_format
+            format:           tabular_input_format,
+            format_options:   tabular_input_options&.deep_symbolize_keys
           )
         end
 
         def tabular_input_render
-          unless tabular_input_header.blank? && tabular_input.header?
-            @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
-          end
+          return if tabular_input_header.blank? && tabular_input.header?
+
+          @rocket_job_input = tabular_input.record_parse(@rocket_job_input)
         end
 
         # Cleanse custom input header if supplied.

data/lib/rocket_job/batch/tabular/output.rb

@@ -12,6 +12,7 @@ module RocketJob
         included do
           field :tabular_output_header, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
           field :tabular_output_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true, copy_on_restart: true
+          field :tabular_output_options, type: Hash, class_attribute: true
 
           validates_inclusion_of :tabular_output_format, in: IOStreams::Tabular.registered_formats
 
@@ -31,8 +32,9 @@ module RocketJob
 
         # Overrides: `RocketJob::Batch::IO#download` to add the `tabular_output_header`.
         def download(file_name_or_io = nil, category: :main, **args, &block)
-          # No header required
-          return super(file_name_or_io, category: category, **args, &block) unless tabular_output.requires_header?(category)
+          unless tabular_output.requires_header?(category)
+            return super(file_name_or_io, category: category, **args, &block)
+          end
 
           header = tabular_output.render_header(category)
           super(file_name_or_io, header_line: header, category: category, **args, &block)
@@ -43,7 +45,11 @@ module RocketJob
         # Delimited instance used for this slice, by a single worker (thread)
         def tabular_output
           @tabular_output ||= Tabular.new(
-            main: IOStreams::Tabular.new(columns: tabular_output_header, format: tabular_output_format)
+            main: IOStreams::Tabular.new(
+              columns:        tabular_output_header,
+              format:         tabular_output_format,
+              format_options: tabular_output_options&.deep_symbolize_keys
+            )
           )
         end
 

data/lib/rocket_job/batch/throttle_running_workers.rb

@@ -47,24 +47,19 @@ module RocketJob
 
       # Returns [Boolean] whether the throttle for this job has been exceeded
       def throttle_running_workers_exceeded?(slice)
-        return unless throttle_running_workers&.positive?
+        return false unless throttle_running_workers&.positive?
 
         input.running.with(read: {mode: :primary}) do |conn|
           conn.where(:id.ne => slice.id).count >= throttle_running_workers
         end
       end
 
-      # Returns [Boolean] whether the throttle for this job has been exceeded
-      #
-      # With a Batch job, allow a higher priority queued job to replace a running one with
-      # a lower priority.
-      def throttle_running_jobs_exceeded?
-        return unless throttle_running_jobs&.positive?
-
-        # Cannot use this class since it will include instances of parent job classes.
-        RocketJob::Job.with(read: {mode: :primary}) do |conn|
-          conn.running.where("_type" => self.class.name, :id.ne => id, :priority.lte => priority).count >= throttle_running_jobs
-        end
+      # Allows another job with a higher priority to start even though this one is running already
+      # @overrides RocketJob::Plugins::Job::ThrottleRunningJobs#throttle_running_jobs_base_query
+      def throttle_running_jobs_base_query
+        query = super
+        query[:priority.lte] = priority if throttle_running_workers&.positive?
+        query
       end
     end
   end

data/lib/rocket_job/batch/worker.rb

@@ -46,7 +46,7 @@ module RocketJob
               next if slice.failed?
 
               slice.fail_on_exception!(re_raise_exceptions) { rocket_job_process_slice(slice) }
-            elsif record_count && rocket_job_batch_complete?(worker.name)
+            elsif record_count && fail_on_exception!(re_raise_exceptions) { rocket_job_batch_complete?(worker.name) }
               return false
             else
               logger.debug "No more work available for this job"

data/lib/rocket_job/extensions/mongoid/remove_warnings.rb

@@ -0,0 +1,12 @@
+require "mongoid/fields/validators/macro"
+require "semantic_logger"
+module RocketJob
+  module RemoveMongoidWarnings
+    # Remove annoying warnings about Symbols type being deprecated.
+    def validate_options(*params)
+      SemanticLogger.silence(:error) { super(*params) }
+    end
+  end
+end
+
+::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)

data/lib/rocket_job/jobs/copy_file_job.rb

@@ -25,7 +25,7 @@ module RocketJob
 
       self.destroy_on_complete = false
       # Number of times to automatically retry the copy. Set to `0` for no retry attempts.
-      self.retry_limit = 5
+      self.retry_limit = 10
 
       # File names in IOStreams URL format.
       field :source_url, type: String, user_editable: true

data/lib/rocket_job/jobs/re_encrypt/relational_job.rb

@@ -1,8 +1,5 @@
-begin
-  require "active_record"
-rescue LoadError
-  raise 'RocketJob::Jobs::ReEncrypt::RelationalJob uses ActiveRecord to obtain the database connection, please install the gem "activerecord".'
-end
+require "active_record"
+require "sync_attr"
 
 # Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
 #

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -40,7 +40,7 @@ module RocketJob
         job.id = job_id if job_id
         upload_file(job)
         job.save!
-      rescue StandardError => e
+      rescue Exception => e
         # Prevent partial uploads
         job&.cleanup! if job.respond_to?(:cleanup!)
         raise(e)

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -29,6 +29,9 @@ module RocketJob
           class_attribute :throttle_running_jobs
           self.throttle_running_jobs = nil
 
+          # Allow jobs to be throttled by group name instance of the job class name.
+          field :throttle_group, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
+
           define_throttle :throttle_running_jobs_exceeded?
         end
 
@@ -36,13 +39,18 @@ module RocketJob
 
         # Returns [Boolean] whether the throttle for this job has been exceeded
         def throttle_running_jobs_exceeded?
-          return unless throttle_running_jobs&.positive?
+          return false unless throttle_running_jobs&.positive?
 
-          # Cannot use this class since it will include instances of parent job classes.
           RocketJob::Job.with(read: {mode: :primary}) do |conn|
-            conn.running.where("_type" => self.class.name, :id.ne => id).count >= throttle_running_jobs
+            query = throttle_running_jobs_base_query
+            throttle_group ? query["throttle_group"] = throttle_group : query["_type"] = self.class.name
+            conn.running.where(query).count >= throttle_running_jobs
           end
         end
+
+        def throttle_running_jobs_base_query
+          {:id.ne => id}
+        end
       end
     end
   end

data/lib/rocket_job/plugins/job/worker.rb

@@ -73,12 +73,13 @@ module RocketJob
             if failed? || !may_fail?
               self.exception        = JobException.from_exception(e)
               exception.worker_name = worker_name
-              save! unless new_record? || destroyed?
-            elsif new_record? || destroyed?
-              fail(worker_name, e)
             else
-              fail!(worker_name, e)
+              fail(worker_name, e)
             end
+
+            # Prevent validation failures from failing the job
+            save(validate: false) unless new_record? || destroyed?
+
             raise e if re_raise_exceptions
           end
         end

data/lib/rocket_job/sliced.rb

@@ -0,0 +1,91 @@
+module RocketJob
+  module Sliced
+    autoload :BZip2OutputSlice, "rocket_job/sliced/bzip2_output_slice"
+    autoload :CompressedSlice, "rocket_job/sliced/compressed_slice"
+    autoload :EncryptedSlice, "rocket_job/sliced/encrypted_slice"
+    autoload :Input, "rocket_job/sliced/input"
+    autoload :Output, "rocket_job/sliced/output"
+    autoload :Slice, "rocket_job/sliced/slice"
+    autoload :Slices, "rocket_job/sliced/slices"
+    autoload :Store, "rocket_job/sliced/store"
+
+    module Writer
+      autoload :Input, "rocket_job/sliced/writer/input"
+      autoload :Output, "rocket_job/sliced/writer/output"
+    end
+
+    # Returns [RocketJob::Sliced::Slices] for the relevant type and category.
+    #
+    # Supports compress and encrypt with [true|false|Hash] values.
+    # When [Hash] they must specify whether the apply to the input or output collection types.
+    #
+    # Example, compress both input and output collections:
+    #   class MyJob < RocketJob::Job
+    #     include RocketJob::Batch
+    #     self.compress = true
+    #   end
+    #
+    # Example, compress just the output collections:
+    #   class MyJob < RocketJob::Job
+    #     include RocketJob::Batch
+    #     self.compress = {output: true}
+    #   end
+    #
+    # To use the specialized BZip output compressor, and the regular compressor for the input collections:
+    #   class MyJob < RocketJob::Job
+    #     include RocketJob::Batch
+    #     self.compress = {output: :bzip2, input: true}
+    #   end
+    def self.factory(type, category, job)
+      raise(ArgumentError, "Unknown type: #{type.inspect}") unless %i[input output].include?(type)
+
+      collection_name = "rocket_job.#{type}s.#{job.id}"
+      collection_name << ".#{category}" unless category == :main
+
+      args               = {collection_name: collection_name, slice_size: job.slice_size}
+      klass              = slice_class(type, job)
+      args[:slice_class] = klass if klass
+
+      if type == :input
+        RocketJob::Sliced::Input.new(args)
+      else
+        RocketJob::Sliced::Output.new(args)
+      end
+    end
+
+    private
+
+    # Parses the encrypt and compress options to determine which slice serializer to use.
+    # `encrypt` takes priority over any `compress` option.
+    def self.slice_class(type, job)
+      encrypt  = extract_value(type, job.encrypt)
+      compress = extract_value(type, job.compress)
+
+      if encrypt
+        case encrypt
+        when true
+          EncryptedSlice
+        else
+          raise(ArgumentError, "Unknown job `encrypt` value: #{compress}") unless compress.is_a?(Slices)
+          # Returns the supplied class to use for encryption.
+          encrypt
+        end
+      elsif compress
+        case compress
+        when true
+          CompressedSlice
+        when :bzip2
+          BZip2OutputSlice
+        else
+          raise(ArgumentError, "Unknown job `compress` value: #{compress}") unless compress.is_a?(Slices)
+          # Returns the supplied class to use for compression.
+          compress
+        end
+      end
+    end
+
+    def self.extract_value(type, value)
+      value.is_a?(Hash) ? value[type] : value
+    end
+  end
+end

data/lib/rocket_job/sliced/bzip2_output_slice.rb

@@ -0,0 +1,43 @@
+module RocketJob
+  module Sliced
+    # This is a specialized output serializer that renders each output slice as a single BZip2 compressed stream.
+    # BZip2 allows multiple output streams to be written into a single BZip2 file.
+    #
+    # Notes:
+    # * The `bzip2` linux command line utility supports multiple embedded BZip2 stream,
+    #   but some other custom implementations may not. They may only read the first slice and stop.
+    # * It is only designed for use on output collections.
+    #
+    # To download the output when using this slice:
+    #
+    #   # Download the binary BZip2 streams into a single file
+    #   IOStreams.path(output_file_name).stream(:none).writer do |io|
+    #     job.download { |slice| io << slice[:binary] }
+    #   end
+    class BZip2OutputSlice < ::RocketJob::Sliced::Slice
+      # This is a specialized binary slice for creating binary data from each slice
+      # that must be downloaded as-is into output files.
+      def self.binary?
+        true
+      end
+
+      private
+
+      def parse_records
+        records = attributes.delete("records")
+
+        # Convert BSON::Binary to a string
+        @records = [{binary: records.data}]
+      end
+
+      def serialize_records
+        return [] if @records.nil? || @records.empty?
+
+        lines = records.to_a.join("\n") + "\n"
+        s = StringIO.new
+        IOStreams::Bzip2::Writer.stream(s) { |io| io.write(lines) }
+        BSON::Binary.new(s.string)
+      end
+    end
+  end
+end

data/lib/rocket_job/sliced/input.rb

@@ -5,7 +5,7 @@ module RocketJob
         # Create indexes before uploading
         create_indexes
         Writer::Input.collect(self, on_first: on_first, &block)
-      rescue StandardError => e
+      rescue Exception => e
         drop
         raise(e)
       end
@@ -73,7 +73,7 @@ module RocketJob
           count    += 1
         end
         count
-      rescue StandardError => e
+      rescue Exception => e
         drop
         raise(e)
       end
@@ -91,7 +91,7 @@ module RocketJob
           count  += 1
         end
         count
-      rescue StandardError => e
+      rescue Exception => e
         drop
         raise(e)
       end

data/lib/rocket_job/sliced/slice.rb

@@ -94,6 +94,12 @@ module RocketJob
         end
       end
 
+      # Returns whether this is a specialized binary slice for creating binary data from each slice
+      # that is then just downloaded as-is into output files.
+      def self.binary?
+        false
+      end
+
       # `records` array has special handling so that it can be modified in place instead of having
       # to replace the entire array every time. For example, when appending lines with `<<`.
       def records

data/lib/rocket_job/sliced/slices.rb

@@ -42,6 +42,12 @@ module RocketJob
         slice
       end
 
+      # Returns whether this collection contains specialized binary slices for creating binary data from each slice
+      # that is then just downloaded as-is into output files.
+      def binary?
+        slice_class.binary?
+      end
+
       # Returns output slices in the order of their id
       # which is usually the order in which they were written.
       def each

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "5.3.2".freeze
+  VERSION = "5.4.1".freeze
 end

data/lib/rocketjob.rb

@@ -13,6 +13,9 @@ require "rocket_job/extensions/mongoid/clients/options"
 require "rocket_job/extensions/mongoid/contextual/mongo"
 require "rocket_job/extensions/mongoid/factory"
 
+# Apply patches for deprecated Symbol type
+require "rocket_job/extensions/mongoid/remove_warnings"
+
 # @formatter:off
 module RocketJob
   autoload :ActiveWorker,       "rocket_job/active_worker"
@@ -26,6 +29,7 @@ module RocketJob
   autoload :Worker,             "rocket_job/worker"
   autoload :Performance,        "rocket_job/performance"
   autoload :Server,             "rocket_job/server"
+  autoload :Sliced,             "rocket_job/sliced"
   autoload :Subscriber,         "rocket_job/subscriber"
   autoload :Supervisor,         "rocket_job/supervisor"
   autoload :ThrottleDefinition, "rocket_job/throttle_definition"
@@ -45,10 +49,6 @@ module RocketJob
       autoload :Transaction,         "rocket_job/plugins/job/transaction"
       autoload :Worker,              "rocket_job/plugins/job/worker"
     end
-    module Rufus
-      autoload :CronLine,       "rocket_job/plugins/rufus/cron_line"
-      autoload :ZoTime,         "rocket_job/plugins/rufus/zo_time"
-    end
     autoload :Cron,             "rocket_job/plugins/cron"
     autoload :Document,         "rocket_job/plugins/document"
     autoload :ProcessingWindow, "rocket_job/plugins/processing_window"
@@ -71,22 +71,9 @@ module RocketJob
     autoload :SimpleJob,               "rocket_job/jobs/simple_job"
     autoload :UploadFileJob,           "rocket_job/jobs/upload_file_job"
     module ReEncrypt
-      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
-    end
-  end
-
-  module Sliced
-    autoload :CompressedSlice,  "rocket_job/sliced/compressed_slice"
-    autoload :EncryptedSlice,   "rocket_job/sliced/encrypted_slice"
-    autoload :Input,            "rocket_job/sliced/input"
-    autoload :Output,           "rocket_job/sliced/output"
-    autoload :Slice,            "rocket_job/sliced/slice"
-    autoload :Slices,           "rocket_job/sliced/slices"
-    autoload :Store,            "rocket_job/sliced/store"
-
-    module Writer
-      autoload :Input,          "rocket_job/sliced/writer/input"
-      autoload :Output,         "rocket_job/sliced/writer/output"
+      if defined?(ActiveRecord) && defined?(SyncAttr)
+        autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 5.3.2
+  version: 5.4.1
 platform: ruby
 authors:
 - Reid Morrison
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-18 00:00:00.000000000 Z
+date: 2020-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -108,8 +108,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-description: 
-email: 
+description:
+email:
 executables:
 - rocketjob
 - rocketjob_perf
@@ -148,6 +148,7 @@ files:
 - lib/rocket_job/extensions/mongoid/clients/options.rb
 - lib/rocket_job/extensions/mongoid/contextual/mongo.rb
 - lib/rocket_job/extensions/mongoid/factory.rb
+- lib/rocket_job/extensions/mongoid/remove_warnings.rb
 - lib/rocket_job/extensions/rocket_job_adapter.rb
 - lib/rocket_job/heartbeat.rb
 - lib/rocket_job/job.rb
@@ -185,6 +186,8 @@ files:
 - lib/rocket_job/server.rb
 - lib/rocket_job/server/model.rb
 - lib/rocket_job/server/state_machine.rb
+- lib/rocket_job/sliced.rb
+- lib/rocket_job/sliced/bzip2_output_slice.rb
 - lib/rocket_job/sliced/compressed_slice.rb
 - lib/rocket_job/sliced/encrypted_slice.rb
 - lib/rocket_job/sliced/input.rb
@@ -209,7 +212,7 @@ homepage: http://rocketjob.io
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -217,15 +220,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Ruby's missing batch processing system.
 test_files: []