rocketjob 6.0.0.rc1 → 6.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2794f5dc5e0ada3ffdc3da9a13fd0cb6c5713f89254d93b69d60677283bc2d64
-  data.tar.gz: 3a208b181aca760b07432348bc2e51443a9da03cc6a143be81765ca2b3c0e37a
+  metadata.gz: 6568c1307b7d42a0968df0335e3177967a8223838eceecbe0b3c0cab72c398af
+  data.tar.gz: 1109611ffce2fe4aed881f5bd9d017681546f06d62330ffc21873adbef12c179
 SHA512:
-  metadata.gz: 44816973f2f63dc300fe41e168ae485cd8b7def5e3bbab173501f6ef2935d3f65707ec4c2f9eb7cb6b43bab2d464b163f3e10216be50babf2dab5e82a7998439
-  data.tar.gz: c0b2d210a3bb3faa49f30eeaf687052ed79e9da802635c6434e374c4f1ccc3538a71a4db9391f41e18a6265106aa7fedeaf920edf4e9d11acf81c9bc632534bd
+  metadata.gz: 073adf2196d6d0cfd5c06ad8776374a1a14618b5fbd49a550e9d8df587b0a17fd95b4dad34a60d8812a2c5f4046974e22bcf616512d77442ea28145d9bd374d2
+  data.tar.gz: ee9b6d35149f7d7799071485f2e4032d47303d3452a4b7c70c5422019a66615938dba90e1e27090a4f33b635df6f565871efd3152f5a2f2a5cb03a61168b3755

data/lib/rocket_job/batch/categories.rb

@@ -82,7 +82,8 @@ module RocketJob
             category              = Category::Input.new
             self.input_categories = [category]
           else
-            raise(ArgumentError, "Unknown Input Category: #{category_name.inspect}. Registered categories: #{input_categories.collect(&:name).join(',')}")
+            raise(ArgumentError,
+                  "Unknown Input Category: #{category_name.inspect}. Registered categories: #{input_categories.collect(&:name).join(',')}")
           end
         end
         category
@@ -94,8 +95,10 @@ module RocketJob
         # .find does not work against this association
         output_categories.each { |catg| category = catg if catg.name == category_name }
         unless category
-          raise(ArgumentError, "Unknown Output Category: #{category_name.inspect}. Registered categories: #{output_categories.collect(&:name).join(',')}")
+          raise(ArgumentError,
+                "Unknown Output Category: #{category_name.inspect}. Registered categories: #{output_categories.collect(&:name).join(',')}")
         end
+
         category
       end
 

data/lib/rocket_job/batch/io.rb

@@ -158,13 +158,13 @@ module RocketJob
         raise(ArgumentError, "Either stream, or a block must be supplied") unless stream || block
 
         category = input_category(category) unless category.is_a?(Category::Input)
-        stream   ||= category.file_name
+        stream ||= category.file_name
         path     = nil
 
         if stream
-          path                  = IOStreams.new(stream)
-          path.file_name        = file_name if file_name
-          category.file_name    = path.file_name
+          path               = IOStreams.new(stream)
+          path.file_name     = file_name if file_name
+          category.file_name = path.file_name
 
           # Auto detect the format based on the upload file name if present.
           if category.format == :auto
@@ -421,8 +421,8 @@ module RocketJob
       def download(stream = nil, category: :main, header_line: nil, **args, &block)
         raise "Cannot download incomplete job: #{id}. Currently in state: #{state}-#{sub_state}" if rocket_job_processing?
 
-        category          = output_category(category) unless category.is_a?(Category::Output)
-        output_collection = output(category)
+        category           = output_category(category) unless category.is_a?(Category::Output)
+        output_collection  = output(category)
 
         # Store the output file name in the category
         category.file_name = stream if !block && (stream.is_a?(String) || stream.is_a?(IOStreams::Path))

data/lib/rocket_job/batch/performance.rb

@@ -22,7 +22,7 @@ module RocketJob
         count_running_workers
 
         puts "Loading job with #{count} records/lines"
-        job = RocketJob::Jobs::PerformanceJob.new(log_level: :warn)
+        job                           = RocketJob::Jobs::PerformanceJob.new(log_level: :warn)
         job.input_category.slice_size = slice_size
         if encrypt
           job.input_category.serializer  = :encrypt
@@ -64,7 +64,7 @@ module RocketJob
 
       # Parse command line options
       def parse(argv)
-        parser        = OptionParser.new do |o|
+        parser = OptionParser.new do |o|
           o.on("-c", "--count COUNT", "Count of records to enqueue") do |arg|
             self.count = arg.to_i
           end

data/lib/rocket_job/batch/statistics.rb

@@ -49,7 +49,7 @@ module RocketJob
           last  = paths.pop
           return unless last
 
-          last_target       = paths.inject(in_memory) do |target, sub_key|
+          last_target = paths.inject(in_memory) do |target, sub_key|
             target.key?(sub_key) ? target[sub_key] : target[sub_key] = Hash.new(0)
           end
           last_target[last] += increment
@@ -99,7 +99,7 @@ module RocketJob
 
       # Overrides RocketJob::Batch::Logger#rocket_job_batch_log_payload
       def rocket_job_batch_log_payload
-        h              = {
+        h = {
           from:  aasm.from_state,
           to:    aasm.to_state,
           event: aasm.current_event

data/lib/rocket_job/batch/throttle_running_workers.rb

@@ -53,7 +53,7 @@ module RocketJob
       # Allows another job with a higher priority to start even though this one is running already
       # @overrides RocketJob::Plugins::Job::ThrottleRunningJobs#throttle_running_jobs_base_query
       def throttle_running_jobs_base_query
-        query = super
+        query                = super
         query[:priority.lte] = priority if throttle_running_workers&.positive?
         query
       end

data/lib/rocket_job/batch/worker.rb

@@ -96,7 +96,7 @@ module RocketJob
         case sub_state
         when :before, :after
           if running? && (server_name.nil? || worker_on_server?(server_name))
-            servers << ActiveWorker.new(worker_name, started_at, self) if running?
+            servers << ActiveWorker.new(worker_name, started_at, self)
           end
         when :processing
           query = input.running
@@ -246,7 +246,7 @@ module RocketJob
         unless new_record?
           # Fail job iff no other worker has already finished it
           # Must set write concern to at least 1 since we need the nModified back
-          result   = self.class.with(write: {w: 1}) do |query|
+          result = self.class.with(write: {w: 1}) do |query|
             query.
               where(id: id, state: :running, sub_state: :processing).
               update({"$set" => {state: :failed, worker_name: worker_name}})

data/lib/rocket_job/category/base.rb

@@ -11,7 +11,7 @@ module RocketJob
 
         # Whether to compress, encrypt, or use the bzip2 serialization for data in this category.
         field :serializer, type: ::Mongoid::StringifiedSymbol, default: :compress
-        validates_inclusion_of :serializer, in: [:none, :compress, :encrypt, :bzip2]
+        validates_inclusion_of :serializer, in: %i[none compress encrypt bzip2]
 
         # The header columns when the file does not include a header row.
         # Note:

data/lib/rocket_job/cli.rb

@@ -233,7 +233,7 @@ module RocketJob
 
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
-      parser = OptionParser.new do |o|
+      parser        = OptionParser.new do |o|
         o.on("-n", "--name NAME", "Unique Name of this server (Default: host_name:PID)") do |arg|
           Config.name = arg
         end

data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb

@@ -4,8 +4,8 @@ module Mongoid
     class Mongo
       def initialize(criteria)
         @criteria = criteria
-        @klass = criteria.klass
-        @cache = criteria.options[:cache]
+        @klass    = criteria.klass
+        @cache    = criteria.options[:cache]
         # Only line changed is here, get collection name from criteria, not @klass
         # @collection = @klass.collection
         @collection = criteria.collection

data/lib/rocket_job/extensions/rocket_job_adapter.rb

@@ -55,13 +55,13 @@ module ActiveJob
     #   - Completed jobs will not appear in completed since the Active Job adapter
     #     uses the default Rocket Job `destroy_on_completion` of `false`.
     class RocketJobAdapter
-      def self.enqueue(active_job) #:nodoc:
+      def self.enqueue(active_job)
         job                        = RocketJob::Jobs::ActiveJob.create!(active_job_params(active_job))
         active_job.provider_job_id = job.id.to_s if active_job.respond_to?(:provider_job_id=)
         job
       end
 
-      def self.enqueue_at(active_job, timestamp) #:nodoc:
+      def self.enqueue_at(active_job, timestamp)
         params          = active_job_params(active_job)
         params[:run_at] = Time.at(timestamp).utc
 

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -82,7 +82,7 @@ module RocketJob
             key           = iopath.to_s.tr(".", "_")
             previous_size = previous_file_names[key]
             # Check every few minutes for a file size change before trying to process the file.
-            size = check_file(entry, iopath, previous_size)
+            size                = check_file(entry, iopath, previous_size)
             new_file_names[key] = size if size
           end
         end

data/lib/rocket_job/jobs/re_encrypt/relational_job.rb

@@ -1,6 +1,3 @@
-require "active_record"
-require "sync_attr"
-
 # Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
 #
 # Run in Rails console:
@@ -11,117 +8,126 @@ require "sync_attr"
 # * This job will find any column in the database that starts with`encrypted_`.
 # * This means that temporary or other tables not part of the application tables will also be processed.
 # * Since it automatically finds and re-encrypts any column, new columns are handled without any manual intervention.
-module RocketJob
-  module Jobs
-    module ReEncrypt
-      class RelationalJob < RocketJob::Job
-        include RocketJob::Batch
-
-        self.priority                 = 30
-        self.destroy_on_complete      = false
-        self.throttle_running_jobs    = 1
-        self.throttle_running_workers = 10
-
-        input_category slice_size: 1_000
-
-        # Name of the table being re-encrypted
-        field :table_name, type: String
-
-        # Limit the number of records to re-encrypt in test environments
-        field :limit, type: Integer
-
-        validates_presence_of :table_name
-        before_batch :upload_records
-
-        # Returns [Hash] of table names with each entry being an array
-        # of columns that start with encrypted_
-        sync_cattr_reader :encrypted_columns do
-          h = {}
-          connection.tables.each do |table|
-            columns = connection.columns(table)
-            columns.each do |column|
-              if column.name.start_with?("encrypted_")
-                add_column = column.name
-                (h[table] ||= []) << add_column if add_column
+if defined?(ActiveRecord) && defined?(SyncAttr)
+  require "active_record"
+  require "sync_attr"
+
+  module RocketJob
+    module Jobs
+      module ReEncrypt
+        class RelationalJob < RocketJob::Job
+          include RocketJob::Batch
+
+          self.priority                 = 30
+          self.destroy_on_complete      = false
+          self.throttle_running_jobs    = 1
+          self.throttle_running_workers = 10
+
+          input_category slice_size: 1_000
+
+          # Name of the table being re-encrypted
+          field :table_name, type: String
+
+          # Limit the number of records to re-encrypt in test environments
+          field :limit, type: Integer
+
+          validates_presence_of :table_name
+          before_batch :upload_records
+
+          # Returns [Hash] of table names with each entry being an array
+          # of columns that start with encrypted_
+          sync_cattr_reader :encrypted_columns do
+            h = {}
+            connection.tables.each do |table|
+              columns = connection.columns(table)
+              columns.each do |column|
+                if column.name.start_with?("encrypted_")
+                  add_column = column.name
+                  (h[table] ||= []) << add_column if add_column
+                end
               end
             end
+            h
           end
-          h
-        end
 
-        # Re-encrypt all `encrypted_` columns in the relational database.
-        # Queues a Job for each table that needs re-encryption.
-        def self.start(**args)
-          encrypted_columns.keys.collect do |table|
-            create!(table_name: table, description: table, **args)
+          # Re-encrypt all `encrypted_` columns in the relational database.
+          # Queues a Job for each table that needs re-encryption.
+          def self.start(**args)
+            encrypted_columns.keys.collect do |table|
+              create!(table_name: table, description: table, **args)
+            end
           end
-        end
 
-        # Re-encrypt all encrypted columns for the named table.
-        # Does not use AR models since we do not have models for all tables.
-        def perform(range)
-          start_id, end_id = range
+          # Re-encrypt all encrypted columns for the named table.
+          # Does not use AR models since we do not have models for all tables.
+          def perform(range)
+            start_id, end_id = range
 
-          columns = self.class.encrypted_columns[table_name]
-          unless columns&.size&.positive?
-            logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
-            return
-          end
+            columns = self.class.encrypted_columns[table_name]
+            unless columns&.size&.positive?
+              logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
+              return
+            end
 
-          logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
-          sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
-
-          # Use AR to fetch all the records
-          self.class.connection.select_rows(sql).each do |row|
-            row.unshift(nil)
-            index   = 1
-            sql     = "update #{quoted_table_name} set "
-            updates = []
-            columns.collect do |column|
-              index += 1
-              value = row[index]
-              # Prevent re-encryption
-              unless value.blank?
-                new_value = re_encrypt(value)
-                updates << "#{column} = \"#{new_value}\"" if new_value != value
+            logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
+            sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
+
+            # Use AR to fetch all the records
+            self.class.connection.select_rows(sql).each do |row|
+              row.unshift(nil)
+              index   = 1
+              sql     = "update #{quoted_table_name} set "
+              updates = []
+              columns.collect do |column|
+                index += 1
+                value = row[index]
+                # Prevent re-encryption
+                unless value.blank?
+                  new_value = re_encrypt(value)
+                  updates << "#{column} = \"#{new_value}\"" if new_value != value
+                end
+              end
+              if updates.size.positive?
+                sql << updates.join(", ")
+                sql << " where id=#{row[1]}"
+                logger.trace sql
+                self.class.connection.execute sql
+              else
+                logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
               end
-            end
-            if updates.size.positive?
-              sql << updates.join(", ")
-              sql << " where id=#{row[1]}"
-              logger.trace sql
-              self.class.connection.execute sql
-            else
-              logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
             end
           end
-        end
 
-        # Returns a database connection.
-        #
-        # Override this method to support other ways of obtaining a thread specific database connection.
-        def self.connection
-          ActiveRecord::Base.connection
-        end
+          # Returns a database connection.
+          #
+          # Override this method to support other ways of obtaining a thread specific database connection.
+          def self.connection
+            ActiveRecord::Base.connection
+          end
 
-        private
+          private
 
-        def quoted_table_name
-          @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
-        end
+          def quoted_table_name
+            @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
+          end
 
-        def re_encrypt(encrypted_value)
-          return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
+          def re_encrypt(encrypted_value)
+            return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
 
-          SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
-        end
+            SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
+          end
 
-        # Upload range to re-encrypt all rows in the specified table.
-        def upload_records
-          start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
-          last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
-          self.record_count =
-            last_id.positive? ? (input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size) : 0
+          # Upload range to re-encrypt all rows in the specified table.
+          def upload_records
+            start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
+            last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
+            self.record_count =
+              if last_id.positive?
+                input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size
+              else
+                0
+              end
+          end
         end
       end
     end

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -96,11 +96,10 @@ module RocketJob
       def file_exists
         # Only check for file existence when it is a local file
         return unless upload_file_name.is_a?(IOStreams::Paths::File)
-        if upload_file_name.to_s == ""
-          return errors.add(:upload_file_name, "Upload file name can't be blank.")
-        end
+        return errors.add(:upload_file_name, "Upload file name can't be blank.") if upload_file_name.to_s == ""
 
         return if upload_file_name.exist?
+
         errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
       rescue NotImplementedError
         nil

data/lib/rocket_job/lookup_collection.rb

@@ -22,8 +22,6 @@ module RocketJob
       find(id: id).first
     end
 
-    private
-
     # Internal class for uploading records in batches
     class BatchUploader
       attr_reader :record_count
@@ -46,7 +44,10 @@ module RocketJob
 
       def <<(record)
         raise(ArgumentError, "Record must be a Hash") unless record.is_a?(Hash)
-        raise(ArgumentError, "Record must include an :id key") unless record.key?(:id) || record.key?("id") || record.key?("_id")
+
+        unless record.key?(:id) || record.key?("id") || record.key?("_id")
+          raise(ArgumentError, "Record must include an :id key")
+        end
 
         @documents << record
         @record_count += 1

data/lib/rocket_job/plugins/job/throttle.rb

@@ -48,7 +48,7 @@ module RocketJob
           # Note: Throttles are executed in the order they are defined.
           def define_throttle(method_name, filter: :throttle_filter_class)
             # Duplicate to prevent modifying parent class throttles
-            definitions = rocket_job_throttles ? rocket_job_throttles.dup : ThrottleDefinitions.new
+            definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
             definitions.add(method_name, filter)
             self.rocket_job_throttles = definitions
           end
@@ -57,7 +57,7 @@ module RocketJob
           def undefine_throttle(method_name)
             return unless rocket_job_throttles
 
-            definitions = rocket_job_throttles.dup
+            definitions = rocket_job_throttles.deep_dup
             definitions.remove(method_name)
             self.rocket_job_throttles = definitions
           end

data/lib/rocket_job/plugins/restart.rb

@@ -91,9 +91,9 @@ module RocketJob
           logger.info("Job has expired. Not creating a new instance.")
           return
         end
-        job_attrs              =
+        job_attrs             =
           rocket_job_restart_attributes.each_with_object({}) { |attr, attrs| attrs[attr] = send(attr) }
-        job                    = self.class.new(job_attrs)
+        job                   = self.class.new(job_attrs)
 
         # Copy across input and output categories to new scheduled job so that all of the
         # settings are remembered between instance. Example: slice_size

data/lib/rocket_job/plugins/throttle_dependent_jobs.rb

@@ -0,0 +1,38 @@
+require "active_support/concern"
+module RocketJob
+  module Plugins
+    # Prevent this job from starting, or a batch slice from starting if the dependent jobs are running.
+    #
+    # Features:
+    # - Ensures dependent jobs won't run
+    # When the throttle has been exceeded all jobs of this class will be ignored until the
+    # next refresh. `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
+    module ThrottleDependentJobs
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :dependent_jobs
+        self.dependent_jobs = nil
+
+        define_throttle :dependent_job_exists?
+        define_batch_throttle :dependent_job_exists? if respond_to?(:define_batch_throttle)
+      end
+
+      private
+
+      # Checks if there are any dependent jobs are running
+      def dependent_job_exists?
+        return false if dependent_jobs.blank?
+
+        jobs_count = RocketJob::Job.running.where(:_type.in => dependent_jobs).count
+        return false if jobs_count.zero?
+
+        logger.info(
+          message: "#{jobs_count} Dependent Jobs are running from #{dependent_jobs.join(', ')}",
+          metric:  "#{self.class.name}/dependent_jobs_throttle"
+        )
+        true
+      end
+    end
+  end
+end

data/lib/rocket_job/sliced/bzip2_output_slice.rb

@@ -34,7 +34,7 @@ module RocketJob
         return [] if @records.nil? || @records.empty?
 
         lines = records.to_a.join("\n") + "\n"
-        s = StringIO.new
+        s     = StringIO.new
         IOStreams::Bzip2::Writer.stream(s) { |io| io.write(lines) }
         BSON::Binary.new(s.string)
       end

data/lib/rocket_job/sliced/writer/output.rb

@@ -65,7 +65,6 @@ module RocketJob
           (categorized_records[named_category] ||= []) << value unless value.nil? && !job.output_category(named_category).nils
         end
       end
-
     end
   end
 end

data/lib/rocket_job/throttle_definitions.rb

@@ -1,6 +1,6 @@
 module RocketJob
   class ThrottleDefinitions
-    attr_reader :throttles
+    attr_accessor :throttles
 
     def initialize
       @throttles = []
@@ -35,5 +35,11 @@ module RocketJob
       end
       nil
     end
+
+    def deep_dup
+      new_defination           = dup
+      new_defination.throttles = throttles.map(&:dup)
+      new_defination
+    end
   end
 end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.0.0.rc1".freeze
+  VERSION = "6.0.0.rc2".freeze
 end

data/lib/rocketjob.rb

@@ -68,6 +68,7 @@ module RocketJob
     autoload :Singleton,             "rocket_job/plugins/singleton"
     autoload :StateMachine,          "rocket_job/plugins/state_machine"
     autoload :Transaction,           "rocket_job/plugins/transaction"
+    autoload :ThrottleDependentJobs, "rocket_job/plugins/throttle_dependent_jobs"
   end
 
   module Jobs
@@ -82,9 +83,7 @@ module RocketJob
     autoload :UploadFileJob,         "rocket_job/jobs/upload_file_job"
 
     module ReEncrypt
-      if defined?(ActiveRecord) && defined?(SyncAttr)
-        autoload :RelationalJob,     "rocket_job/jobs/re_encrypt/relational_job"
-      end
+      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.0.0.rc1
+  version: 6.0.0.rc2
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-03 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -186,6 +186,7 @@ files:
 - lib/rocket_job/plugins/retry.rb
 - lib/rocket_job/plugins/singleton.rb
 - lib/rocket_job/plugins/state_machine.rb
+- lib/rocket_job/plugins/throttle_dependent_jobs.rb
 - lib/rocket_job/plugins/transaction.rb
 - lib/rocket_job/ractor_worker.rb
 - lib/rocket_job/railtie.rb