rocketjob 5.0.0 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 601a4455f5e51191440ce959b6325545d51fcaf531d179dfcd78a4ec016b44c6
-  data.tar.gz: 7ea8fc1b5cc632461d01e608f5c93cb33e379e14138764fc9e424d15f9c07679
+  metadata.gz: 7614d289b64dc4300632b828fe2597532cf68894685b4c45944a48aa151fc21b
+  data.tar.gz: a9449dfd5902410a232963159a4a87ddda3c29709771c6a875eb7db07b120a3f
 SHA512:
-  metadata.gz: 823280f2250aaa7e2bef010b8b7477f1db1d2f04631c9ce133b80545768e23cab98a14cbc597630988b69b02cfc115acf34a647f50f84fc35e61a708af4c2c09
-  data.tar.gz: e562aa988c120459637cd11108a4401440faa674d1a3226a8c7e77a91ec31635935349803a9c17ace9b3603c14f8a97ec2e8196ba20a79bf518a429807152dce
+  metadata.gz: 0ab97f55bdd18969e69b0314281cd0d604aa8f08f5fc3897b1d1e52855727b72df74263643dd77721721b52e08ac048d576bd5516772e895dd8241ad3d6d65bc
+  data.tar.gz: 510c76a7e44de86eb57fac61fa7c894f88c1256daf97ced09f28e8c4db3a79d225f8114357e950cf44dbb081df280b6855b80d2bd11c1b1772b65ff77784b3be

data/lib/rocket_job/batch/io.rb

@@ -16,10 +16,7 @@ module RocketJob
       def input(category = :main)
         raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}" unless input_categories.include?(category) || (category == :main)
 
-        collection_name = "rocket_job.inputs.#{id}"
-        collection_name << ".#{category}" unless category == :main
-
-        (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(collection_name: collection_name, slice_size: slice_size)
+        (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(rocket_job_io_slice_arguments("inputs", category))
       end
 
       # Returns [RocketJob::Sliced::Output] output collection for holding output slices
@@ -33,10 +30,7 @@ module RocketJob
       def output(category = :main)
         raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}" unless output_categories.include?(category) || (category == :main)
 
-        collection_name = "rocket_job.outputs.#{id}"
-        collection_name << ".#{category}" unless category == :main
-
-        (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(collection_name: collection_name, slice_size: slice_size)
+        (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(rocket_job_io_slice_arguments("outputs", category))
       end
 
       # Upload the supplied file, io, IOStreams::Path, or IOStreams::Stream.
@@ -394,6 +388,22 @@ module RocketJob
           RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
         end
       end
+
+      private
+
+      def rocket_job_io_slice_arguments(collection_type, category)
+        collection_name = "rocket_job.#{collection_type}.#{id}"
+        collection_name << ".#{category}" unless category == :main
+
+        args = {collection_name: collection_name, slice_size: slice_size}
+        if encrypt
+          args[:slice_class] = Sliced::EncryptedSlice
+        elsif compress
+          args[:slice_class] = Sliced::CompressedSlice
+        end
+        args
+      end
+
     end
   end
 end

data/lib/rocket_job/batch/model.rb

@@ -41,6 +41,16 @@ module RocketJob
         # May or may not include the fully qualified path name.
         field :upload_file_name, type: String
 
+        # Compress uploaded records.
+        # The fields are not affected in any way, only the data stored in the
+        # records and results collections will compressed
+        field :compress, type: Boolean, default: false, class_attribute: true
+
+        # Encrypt uploaded records.
+        # The fields are not affected in any way, only the data stored in the
+        # records and results collections will be encrypted
+        field :encrypt, type: Boolean, default: false, class_attribute: true
+
         #
         # Values that jobs can also update during processing
         #
@@ -73,6 +83,16 @@ module RocketJob
         end
       end
 
+      # Returns [true|false] whether the slices for this job are encrypted
+      def encrypted?
+        encrypt == true
+      end
+
+      # Returns [true|false] whether the slices for this job are compressed
+      def compressed?
+        compress == true
+      end
+
       # Returns [Integer] percent of records completed so far
       # Returns 0 if the total record count has not yet been set
       def percent_complete

data/lib/rocket_job/batch/performance.rb

@@ -23,7 +23,7 @@ module RocketJob
 
         puts "Loading job with #{count} records/lines"
         args = {log_level: :warn, slice_size: slice_size}
-        if defined?(::RocketJob::Enterprise)
+        if defined?(::RocketJob)
           args[:compress] = compress
           args[:encrypt]  = encrypt
         end

data/lib/rocket_job/config.rb

@@ -80,9 +80,6 @@ module RocketJob
       logger.debug "Reading Mongo configuration from: #{config_file}"
       ::Mongoid.load!(config_file, environment)
 
-      # Load Encryption configuration file if present
-      return unless defined?(SymmetricEncryption)
-
       config_file =
         if encryption_file_name
           Pathname.new(encryption_file_name)

data/lib/rocket_job/jobs/copy_file_job.rb

@@ -64,13 +64,13 @@ module RocketJob
       end
 
       def source_path
-        source = IOStreams.path(source_url, **decrypt_args(source_args))
+        source = IOStreams.path(source_url, **decode_args(source_args))
         apply_streams(source, source_streams)
         source
       end
 
       def target_path
-        target = IOStreams.path(target_url, **decrypt_args(target_args))
+        target = IOStreams.path(target_url, **decode_args(target_args))
         apply_streams(target, target_streams)
         target
       end
@@ -78,26 +78,29 @@ module RocketJob
       private
 
       def set_description
-        self.description = "Copying to #{target_url}"
+        self.description ||= "Copying to #{target_url}"
       end
 
       def apply_streams(path, streams)
-        streams.each_pair { |stream, args| path.stream(stream.to_sym, args.nil? ? {} : decrypt_args(args)) }
+        streams.each_pair { |stream, args| path.stream(stream.to_sym, args.nil? ? {} : decode_args(args)) }
       end
 
-      def decrypt_args(args)
+      def decode_args(args)
         return args.symbolize_keys unless defined?(SymmetricEncryption)
 
-        decrypted_args = {}
+        decoded_args = {}
         args.each_pair do |key, value|
-          if key.to_s.start_with?("encrypted_")
-            unencrypted_key                        = key.to_s.sub("encrypted_", "")
-            decrypted_args[unencrypted_key.to_sym] = SymmetricEncryption.decrypt(value)
+          if key.to_s.start_with?("encrypted_") && defined?(SymmetricEncryption)
+            original_key               = key.to_s.sub("encrypted_", "").to_sym
+            decoded_args[original_key] = SymmetricEncryption.decrypt(value)
+          elsif key.to_s.start_with?("secret_config_") && defined?(SecretConfig)
+            original_key               = key.to_s.sub("secret_config_", "").to_sym
+            decoded_args[original_key] = SecretConfig.fetch(value)
           else
-            decrypted_args[key.to_sym] = value
+            decoded_args[key.to_sym] = value
           end
         end
-        decrypted_args
+        decoded_args
       end
     end
   end

data/lib/rocket_job/jobs/re_encrypt/relational_job.rb

@@ -0,0 +1,130 @@
+begin
+  require 'active_record'
+rescue LoadError
+  raise 'RocketJob::Jobs::ReEncrypt::RelationalJob uses ActiveRecord to obtain the database connection, please install the gem "activerecord".'
+end
+
+# Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
+#
+# Run in Rails console:
+#   RocketJob::Jobs::ReEncrypt::RelationalJob.start
+#
+# Notes:
+# * Uses table names directly since models can be removed over time and the data still needs to be re-encrypted.
+# * This job will find any column in the database that starts with`encrypted_`.
+# * This means that temporary or other tables not part of the application tables will also be processed.
+# * Since it automatically finds and re-encrypts any column, new columns are handled without any manual intervention.
+module RocketJob
+  module Jobs
+    module ReEncrypt
+      class RelationalJob < RocketJob::Job
+        include RocketJob::Batch
+
+        self.slice_size              = 1000
+        self.priority                = 30
+        self.destroy_on_complete     = false
+        self.compress                = true
+        self.throttle_running_jobs   = 1
+        self.throttle_running_slices = 10
+
+        # Name of the table being re-encrypted
+        field :table_name, type: String
+
+        # Limit the number of records to re-encrypt in test environments
+        field :limit, type: Integer
+
+        validates_presence_of :table_name
+        before_batch :upload_records
+
+        # Returns [Hash] of table names with each entry being an array
+        # of columns that start with encrypted_
+        sync_cattr_reader :encrypted_columns do
+          h = {}
+          connection.tables.each do |table|
+            columns = connection.columns(table)
+            columns.each do |column|
+              if column.name.start_with?('encrypted_')
+                add_column = column.name
+                (h[table] ||= []) << add_column if add_column
+              end
+            end
+          end
+          h
+        end
+
+        # Re-encrypt all `encrypted_` columns in the relational database.
+        # Queues a Job for each table that needs re-encryption.
+        def self.start(**args)
+          encrypted_columns.keys.collect do |table|
+            create!(table_name: table, description: table, **args)
+          end
+        end
+
+        # Re-encrypt all encrypted columns for the named table.
+        # Does not use AR models since we do not have models for all tables.
+        def perform(range)
+          start_id, end_id = range
+
+          columns = self.class.encrypted_columns[table_name]
+          unless columns&.size&.positive?
+            logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
+            return
+          end
+
+          logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
+          sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
+
+          # Use AR to fetch all the records
+          self.class.connection.select_rows(sql).each do |row|
+            row     = row.unshift(nil)
+            index   = 1
+            sql     = "update #{quoted_table_name} set "
+            updates = []
+            columns.collect do |column|
+              index += 1
+              value = row[index]
+              # Prevent re-encryption
+              unless value.blank?
+                new_value = re_encrypt(value)
+                updates << "#{column} = \"#{new_value}\"" if new_value != value
+              end
+            end
+            if updates.size.positive?
+              sql << updates.join(', ')
+              sql << " where id=#{row[1]}"
+              logger.trace sql
+              self.class.connection.execute sql
+            else
+              logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
+            end
+          end
+        end
+
+        # Returns a database connection.
+        #
+        # Override this method to support other ways of obtaining a thread specific database connection.
+        def self.connection
+          ActiveRecord::Base.connection
+        end
+
+        private
+
+        def quoted_table_name
+          @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
+        end
+
+        def re_encrypt(encrypted_value)
+          return encrypted_value if (encrypted_value == '') || encrypted_value.nil?
+          SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
+        end
+
+        # Upload range to re-encrypt all rows in the specified table.
+        def upload_records
+          start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
+          last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
+          self.record_count = last_id.positive? ? (input.upload_integer_range_in_reverse_order(start_id, last_id) * slice_size) : 0
+        end
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/job/state_machine.rb

@@ -85,7 +85,7 @@ module RocketJob
           # @formatter:on
 
           # By default all jobs are not pausable / resumable
-          class_attribute(:pausable)
+          class_attribute(:pausable, instance_predicate: false)
           self.pausable = false
 
           # Define a before and after callback method for each event
@@ -112,7 +112,7 @@ module RocketJob
         end
 
         # All regular jobs can be paused or resumed whilst queued.
-        def self.pausable?
+        def pausable?
           queued? || paused? || pausable
         end
 

data/lib/rocket_job/sliced/compressed_slice.rb

@@ -0,0 +1,29 @@
+require 'zlib'
+module RocketJob
+  module Sliced
+    # Compress the records within a slice
+    class CompressedSlice < ::RocketJob::Sliced::Slice
+      private
+
+      def parse_records
+        records = attributes.delete('records')
+
+        # Convert BSON::Binary to a string
+        binary_str = records.data
+
+        str      = Zlib::Inflate.inflate(binary_str)
+        @records = Hash.from_bson(BSON::ByteBuffer.new(str))['r']
+      end
+
+      def serialize_records
+        return [] if @records.nil? || @records.empty?
+
+        # Convert slice of records into a single string
+        str = {'r' => records.to_a}.to_bson.to_s
+
+        data = Zlib::Deflate.deflate(str)
+        BSON::Binary.new(data)
+      end
+    end
+  end
+end

data/lib/rocket_job/sliced/encrypted_slice.rb

@@ -0,0 +1,35 @@
+require 'symmetric-encryption'
+module RocketJob
+  module Sliced
+    # Compress the records within a slice
+    class EncryptedSlice < ::RocketJob::Sliced::Slice
+      private
+
+      def parse_records
+        records = attributes.delete('records')
+
+        # Convert BSON::Binary to a string
+        binary_str = records.data
+
+        header = SymmetricEncryption::Header.new
+        header.parse(binary_str)
+        # Use the header that is present to decrypt the data, since its version could be different
+        str = header.cipher.binary_decrypt(binary_str, header: header)
+
+        @records = Hash.from_bson(BSON::ByteBuffer.new(str))['r']
+      end
+
+      def serialize_records
+        return [] if @records.nil? || @records.empty?
+
+        # Convert slice of records into a single string
+        str = {'r' => to_a}.to_bson.to_s
+
+        # Encrypt to binary without applying an encoding such as Base64
+        # Use a random_iv with each encryption for better security
+        data = SymmetricEncryption.cipher.binary_encrypt(str, random_iv: true, compress: true)
+        BSON::Binary.new(data)
+      end
+    end
+  end
+end

data/lib/rocket_job/subscriber.rb

@@ -68,8 +68,10 @@ module RocketJob
 
       args = (method(action).arity == 0) || parameters.nil? ? nil : parameters.symbolize_keys
       args ? public_send(action, **args) : public_send(action)
+    rescue ArgumentError => exc
+      logger.error("##{action}: Invalid Arguments. Resuming..", exc)
     rescue StandardError => exc
-      logger.error('Exception calling subscriber. Resuming..', exc)
+      logger.error("##{action}: Exception caught. Resuming..", exc)
     end
 
     def process_event(name, action, parameters)

data/lib/rocket_job/subscribers/server.rb

@@ -9,8 +9,8 @@ module RocketJob
         @supervisor = supervisor
       end
 
-      def kill(server_id: nil, wait_timeout: 3)
-        return unless my_server?(server_id)
+      def kill(server_id: nil, name: nil, wait_timeout: 3)
+        return unless my_server?(server_id, name)
 
         supervisor.synchronize do
           supervisor.worker_pool.stop
@@ -22,38 +22,38 @@ module RocketJob
         logger.info "Killed"
       end
 
-      def pause(server_id: nil)
-        return unless my_server?(server_id)
+      def pause(server_id: nil, name: nil)
+        return unless my_server?(server_id, name)
 
         supervisor.synchronize { supervisor.server.pause! if supervisor.server.may_pause? }
         Supervisor.event!
         logger.info "Paused"
       end
 
-      def refresh(server_id: nil)
-        return unless my_server?(server_id)
+      def refresh(server_id: nil, name: nil)
+        return unless my_server?(server_id, name)
 
         Supervisor.event!
         logger.info "Refreshed"
       end
 
-      def resume(server_id: nil)
-        return unless my_server?(server_id)
+      def resume(server_id: nil, name: nil)
+        return unless my_server?(server_id, name)
 
         supervisor.synchronize { supervisor.server.resume! if supervisor.server.may_resume? }
         Supervisor.event!
         logger.info "Resumed"
       end
 
-      def stop(server_id: nil)
-        return unless my_server?(server_id)
+      def stop(server_id: nil, name: nil)
+        return unless my_server?(server_id, name)
 
         Supervisor.shutdown!
         logger.info "Shutdown"
       end
 
-      def thread_dump(server_id: nil)
-        return unless my_server?(server_id)
+      def thread_dump(server_id: nil, name: nil)
+        return unless my_server?(server_id, name)
 
         logger.info "Thread dump"
         supervisor.worker_pool.log_backtraces
@@ -61,10 +61,11 @@ module RocketJob
 
       private
 
-      def my_server?(server_id)
-        return true if server_id.nil?
+      def my_server?(server_id, name)
+        return true if server_id.nil? && name.nil?
+        return true if supervisor.server.name == name
 
-        server_id == supervisor.server.id
+        server_id.to_s == supervisor.server.id.to_s
       end
     end
   end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '5.0.0'.freeze
+  VERSION = '5.1.0'.freeze
 end

data/lib/rocketjob.rb

@@ -1,5 +1,6 @@
 require 'iostreams'
 require 'semantic_logger'
+require 'symmetric-encryption'
 require 'mongoid'
 require 'rocket_job/extensions/mongo/logging'
 require 'rocket_job/version'
@@ -63,18 +64,22 @@ module RocketJob
   end
 
   module Jobs
-    autoload :ActiveJob,        'rocket_job/jobs/active_job'
-    autoload :CopyFileJob,      'rocket_job/jobs/copy_file_job'
-    autoload :DirmonJob,        'rocket_job/jobs/dirmon_job'
-    autoload :OnDemandBatchJob, 'rocket_job/jobs/on_demand_batch_job'
-    autoload :OnDemandJob,      'rocket_job/jobs/on_demand_job'
-    autoload :HousekeepingJob,  'rocket_job/jobs/housekeeping_job'
-    autoload :PerformanceJob,   'rocket_job/jobs/performance_job'
-    autoload :SimpleJob,        'rocket_job/jobs/simple_job'
-    autoload :UploadFileJob,    'rocket_job/jobs/upload_file_job'
+    autoload :ActiveJob,               'rocket_job/jobs/active_job'
+    autoload :CopyFileJob,             'rocket_job/jobs/copy_file_job'
+    autoload :DirmonJob,               'rocket_job/jobs/dirmon_job'
+    autoload :OnDemandBatchJob,        'rocket_job/jobs/on_demand_batch_job'
+    autoload :OnDemandBatchTabularJob, 'rocket_job/jobs/on_demand_batch_tabular_job'
+    autoload :OnDemandJob,             'rocket_job/jobs/on_demand_job'
+    autoload :HousekeepingJob,         'rocket_job/jobs/housekeeping_job'
+    autoload :PerformanceJob,          'rocket_job/jobs/performance_job'
+    autoload :RelationalJob,           'rocket_job/jobs/re_encrypt/relational_job'
+    autoload :SimpleJob,               'rocket_job/jobs/simple_job'
+    autoload :UploadFileJob,           'rocket_job/jobs/upload_file_job'
   end
 
   module Sliced
+    autoload :CompressedSlice,  'rocket_job/sliced/compressed_slice'
+    autoload :EncryptedSlice,   'rocket_job/sliced/encrypted_slice'
     autoload :Input,            'rocket_job/sliced/input'
     autoload :Output,           'rocket_job/sliced/output'
     autoload :Slice,            'rocket_job/sliced/slice'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 5.1.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-15 00:00:00.000000000 Z
+date: 2020-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: symmetric-encryption
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
 description: 
 email:
 - support@rocketjob.io
@@ -135,6 +149,7 @@ files:
 - lib/rocket_job/jobs/on_demand_batch_tabular_job.rb
 - lib/rocket_job/jobs/on_demand_job.rb
 - lib/rocket_job/jobs/performance_job.rb
+- lib/rocket_job/jobs/re_encrypt/relational_job.rb
 - lib/rocket_job/jobs/simple_job.rb
 - lib/rocket_job/jobs/upload_file_job.rb
 - lib/rocket_job/performance.rb
@@ -161,6 +176,8 @@ files:
 - lib/rocket_job/server.rb
 - lib/rocket_job/server/model.rb
 - lib/rocket_job/server/state_machine.rb
+- lib/rocket_job/sliced/compressed_slice.rb
+- lib/rocket_job/sliced/encrypted_slice.rb
 - lib/rocket_job/sliced/input.rb
 - lib/rocket_job/sliced/output.rb
 - lib/rocket_job/sliced/slice.rb