rocketjob 6.0.0.rc3 → 6.0.3

data/lib/rocket_job/sliced/bzip2_output_slice.rb

@@ -7,36 +7,35 @@ module RocketJob
     # * The `bzip2` linux command line utility supports multiple embedded BZip2 stream,
     #   but some other custom implementations may not. They may only read the first slice and stop.
     # * It is only designed for use on output collections.
-    #
-    # To download the output when using this slice:
-    #
-    #   # Download the binary BZip2 streams into a single file
-    #   IOStreams.path(output_file_name).stream(:none).writer do |io|
-    #     job.download { |slice| io << slice[:binary] }
-    #   end
     class BZip2OutputSlice < ::RocketJob::Sliced::Slice
-      # This is a specialized binary slice for creating binary data from each slice
+      # This is a specialized binary slice for creating BZip2 binary data from each slice
       # that must be downloaded as-is into output files.
-      def self.binary?
-        true
+      def self.binary_format
+        :bz2
+      end
+
+      # Compress the supplied records with BZip2
+      def self.to_binary(records, record_delimiter = "\n")
+        return [] if records.blank?
+
+        lines = Array(records).join(record_delimiter) + record_delimiter
+        s     = StringIO.new
+        IOStreams::Bzip2::Writer.stream(s) { |io| io.write(lines) }
+        s.string
       end
 
       private
 
+      # Returns [Hash] the BZip2 compressed binary data in binary form when reading back from Mongo.
       def parse_records
-        records = attributes.delete("records")
-
         # Convert BSON::Binary to a string
-        @records = [{binary: records.data}]
+        @records = [attributes.delete("records").data]
       end
 
+      # Returns [BSON::Binary] the records compressed using BZip2 into a string.
       def serialize_records
-        return [] if @records.nil? || @records.empty?
-
-        lines = records.to_a.join("\n") + "\n"
-        s     = StringIO.new
-        IOStreams::Bzip2::Writer.stream(s) { |io| io.write(lines) }
-        BSON::Binary.new(s.string)
+        # TODO: Make the line terminator configurable
+        BSON::Binary.new(self.class.to_binary(@records))
       end
     end
   end

data/lib/rocket_job/sliced/compressed_slice.rb

@@ -6,13 +6,10 @@ module RocketJob
       private
 
       def parse_records
-        records = attributes.delete("records")
-
         # Convert BSON::Binary to a string
-        binary_str = records.data
-
-        str      = Zlib::Inflate.inflate(binary_str)
-        @records = Hash.from_bson(BSON::ByteBuffer.new(str))["r"]
+        compressed_str   = attributes.delete("records").data
+        decompressed_str = Zlib::Inflate.inflate(compressed_str)
+        @records         = Hash.from_bson(BSON::ByteBuffer.new(decompressed_str))["r"]
       end
 
       def serialize_records

data/lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb

@@ -0,0 +1,49 @@
+module RocketJob
+  module Sliced
+    # This is a specialized output serializer that renders each output slice as a single BZip2 compressed stream.
+    # BZip2 allows multiple output streams to be written into a single BZip2 file.
+    #
+    # Notes:
+    # * The `bzip2` linux command line utility supports multiple embedded BZip2 stream,
+    #   but some other custom implementations may not. They may only read the first slice and stop.
+    # * It is only designed for use on output collections.
+    class EncryptedBZip2OutputSlice < ::RocketJob::Sliced::Slice
+      # This is a specialized binary slice for creating BZip2 binary data from each slice
+      # that must be downloaded as-is into output files.
+      def self.binary_format
+        :bz2
+      end
+
+      private
+
+      # Returns [Hash] the BZip2 compressed binary data in binary form when reading back from Mongo.
+      def parse_records
+        # Convert BSON::Binary to a string
+        encrypted_str = attributes.delete("records").data
+
+        # Decrypt string
+        header = SymmetricEncryption::Header.new
+        header.parse(encrypted_str)
+        # Use the header that is present to decrypt the data, since its version could be different
+        decrypted_str = header.cipher.binary_decrypt(encrypted_str, header: header)
+
+        @records = [decrypted_str]
+      end
+
+      # Returns [BSON::Binary] the records compressed using BZip2 into a string.
+      def serialize_records
+        return [] if @records.nil? || @records.empty?
+
+        # TODO: Make the line terminator configurable
+        lines = records.to_a.join("\n") + "\n"
+        s     = StringIO.new
+        IOStreams::Bzip2::Writer.stream(s) { |io| io.write(lines) }
+
+        # Encrypt to binary without applying an encoding such as Base64
+        # Use a random_iv with each encryption for better security
+        data = SymmetricEncryption.cipher.binary_encrypt(s.string, random_iv: true, compress: false)
+        BSON::Binary.new(data)
+      end
+    end
+  end
+end

data/lib/rocket_job/sliced/encrypted_slice.rb

@@ -6,17 +6,15 @@ module RocketJob
       private
 
       def parse_records
-        records = attributes.delete("records")
-
         # Convert BSON::Binary to a string
-        binary_str = records.data
+        encrypted_str = attributes.delete("records").data
 
         header = SymmetricEncryption::Header.new
-        header.parse(binary_str)
+        header.parse(encrypted_str)
         # Use the header that is present to decrypt the data, since its version could be different
-        str = header.cipher.binary_decrypt(binary_str, header: header)
+        decrypted_str = header.cipher.binary_decrypt(encrypted_str, header: header)
 
-        @records = Hash.from_bson(BSON::ByteBuffer.new(str))["r"]
+        @records = Hash.from_bson(BSON::ByteBuffer.new(decrypted_str))["r"]
       end
 
       def serialize_records

data/lib/rocket_job/sliced/input.rb

@@ -1,16 +1,16 @@
 module RocketJob
   module Sliced
     class Input < Slices
-      def upload(on_first: nil, &block)
+      def upload(**args, &block)
         # Create indexes before uploading
         create_indexes
-        Writer::Input.collect(self, on_first: on_first, &block)
+        Writer::Input.collect(self, **args, &block)
       rescue Exception => e
         drop
         raise(e)
       end
 
-      def upload_mongo_query(criteria, *column_names, &block)
+      def upload_mongo_query(criteria, columns: [], slice_batch_size: nil, &block)
         options = criteria.options
 
         # Without a block extract the fields from the supplied criteria
@@ -18,23 +18,21 @@ module RocketJob
           # Criteria is returning old school :fields instead of :projections
           options[:projection] = options.delete(:fields) if options.key?(:fields)
         else
-          column_names = column_names.collect(&:to_s)
-          column_names << "_id" if column_names.size.zero?
-
-          fields = options.delete(:fields) || {}
-          column_names.each { |col| fields[col] = 1 }
+          columns = columns.blank? ? ["_id"] : columns.collect(&:to_s)
+          fields  = options.delete(:fields) || {}
+          columns.each { |col| fields[col] = 1 }
           options[:projection] = fields
 
           block =
-            if column_names.size == 1
-              column = column_names.first
+            if columns.size == 1
+              column = columns.first
               ->(document) { document[column] }
             else
-              ->(document) { column_names.collect { |c| document[c] } }
+              ->(document) { columns.collect { |c| document[c] } }
             end
         end
 
-        upload do |records|
+        upload(slice_batch_size: slice_batch_size) do |records|
           # Drop down to the mongo driver level to avoid constructing a Model for each document returned
           criteria.klass.collection.find(criteria.selector, options).each do |document|
             records << block.call(document)
@@ -42,58 +40,48 @@ module RocketJob
         end
       end
 
-      def upload_arel(arel, *column_names, &block)
+      def upload_arel(arel, columns: nil, slice_batch_size: nil, &block)
         unless block
-          column_names = column_names.empty? ? [:id] : column_names.collect(&:to_sym)
+          columns = columns.blank? ? [:id] : columns.collect(&:to_sym)
 
           block =
-            if column_names.size == 1
-              column = column_names.first
-              ->(model) { model.send(column) }
+            if columns.size == 1
+              column = columns.first
+              ->(model) { model.public_send(column) }
             else
-              ->(model) { column_names.collect { |c| model.send(c) } }
+              ->(model) { columns.collect { |c| model.public_send(c) } }
             end
           # find_each requires the :id column in the query
-          selection = column_names.include?(:id) ? column_names : column_names + [:id]
+          selection = columns.include?(:id) ? columns : columns + [:id]
           arel      = arel.select(selection)
         end
 
-        upload { |records| arel.find_each { |model| records << block.call(model) } }
+        upload(slice_batch_size: slice_batch_size) { |records| arel.find_each { |model| records << block.call(model) } }
       end
 
-      def upload_integer_range(start_id, last_id)
-        # Create indexes before uploading
-        create_indexes
-        count = 0
-        while start_id <= last_id
-          end_id = start_id + slice_size - 1
-          end_id = last_id if end_id > last_id
-          create!(records: [[start_id, end_id]])
-          start_id += slice_size
-          count    += 1
+      def upload_integer_range(start_id, last_id, slice_batch_size: 1_000)
+        # Each "record" is actually a range of Integers which makes up each slice
+        upload(slice_size: 1, slice_batch_size: slice_batch_size) do |records|
+          while start_id <= last_id
+            end_id = start_id + slice_size - 1
+            end_id = last_id if end_id > last_id
+            records << [start_id, end_id]
+            start_id += slice_size
+          end
         end
-        count
-      rescue Exception => e
-        drop
-        raise(e)
       end
 
-      def upload_integer_range_in_reverse_order(start_id, last_id)
-        # Create indexes before uploading
-        create_indexes
-        end_id = last_id
-        count  = 0
-        while end_id >= start_id
-          first_id = end_id - slice_size + 1
-          first_id = start_id if first_id.negative? || (first_id < start_id)
-          create!(records: [[first_id, end_id]])
-          end_id -= slice_size
-          count  += 1
+      def upload_integer_range_in_reverse_order(start_id, last_id, slice_batch_size: 1_000)
+        # Each "record" is actually a range of Integers which makes up each slice
+        upload(slice_size: 1, slice_batch_size: slice_batch_size) do |records|
+          end_id = last_id
+          while end_id >= start_id
+            first_id = end_id - slice_size + 1
+            first_id = start_id if first_id.negative? || (first_id < start_id)
+            records << [first_id, end_id]
+            end_id -= slice_size
+          end
         end
-        count
-      rescue Exception => e
-        drop
-        raise(e)
       end
 
       # Iterate over each failed record, if any
@@ -137,11 +125,11 @@ module RocketJob
         # TODO: Will it perform faster without the id sort?
         # I.e. Just process on a FIFO basis?
         document                 = all.queued.
-                                   sort("_id" => 1).
-                                   find_one_and_update(
-                                     {"$set" => {worker_name: worker_name, state: "running", started_at: Time.now}},
-                                     return_document: :after
-                                   )
+          sort("_id" => 1).
+          find_one_and_update(
+            {"$set" => {worker_name: worker_name, state: "running", started_at: Time.now}},
+            return_document: :after
+          )
         document.collection_name = collection_name if document
         document
       end

data/lib/rocket_job/sliced/slice.rb

@@ -95,9 +95,13 @@ module RocketJob
       end
 
       # Returns whether this is a specialized binary slice for creating binary data from each slice
-      # that is then just downloaded as-is into output files.
-      def self.binary?
-        false
+      # that is downloaded without conversion into output files.
+      def self.binary_format
+      end
+
+      # For binary formats only, format the supplied records into the binary format for this slice
+      def self.to_binary(_records)
+        raise NotImplementedError
       end
 
       # `records` array has special handling so that it can be modified in place instead of having

data/lib/rocket_job/sliced/slices.rb

@@ -42,12 +42,6 @@ module RocketJob
         slice
       end
 
-      # Returns whether this collection contains specialized binary slices for creating binary data from each slice
-      # that is then just downloaded as-is into output files.
-      def binary?
-        slice_class.binary?
-      end
-
       # Returns output slices in the order of their id
       # which is usually the order in which they were written.
       def each(&block)
@@ -96,6 +90,11 @@ module RocketJob
         slice
       end
 
+      def insert_many(slices)
+        documents = slices.collect(&:as_document)
+        all.collection.insert_many(documents)
+      end
+
       # Append to an existing slice if already present
       def append(slice, input_slice)
         existing_slice = all.where(id: input_slice.id).first
@@ -111,9 +110,13 @@ module RocketJob
 
       # Index for find_and_modify only if it is not already present
       def create_indexes
-        all.collection.indexes.create_one(state: 1, _id: 1) if all.collection.indexes.none? { |i| i["name"] == "state_1__id_1" }
-      rescue Mongo::Error::OperationFailure
-        all.collection.indexes.create_one(state: 1, _id: 1)
+        missing =
+          begin
+            all.collection.indexes.none? { |i| i["name"] == "state_1__id_1" }
+          rescue Mongo::Error::OperationFailure
+            true
+          end
+        all.collection.indexes.create_one({state: 1, _id: 1}, unique: true) if missing
       end
 
       # Forward additional methods.

data/lib/rocket_job/sliced/writer/input.rb

@@ -12,43 +12,71 @@ module RocketJob
         #     Block to call on the first line only, instead of storing in the slice.
         #     Useful for extracting the header row
         #     Default: nil
-        def self.collect(input, **args)
-          writer = new(input, **args)
+        #
+        #   slice_size: [Integer]
+        #     Override the slice size when uploading for example ranges, where slice is the size
+        #     of the range itself.
+        #
+        #   slice_batch_size: [Integer]
+        #     The number of slices to batch up and to bulk load.
+        #     For smaller slices this significantly improves upload performance.
+        #     Note: If `slice_batch_size` is too high, it can exceed the maximum BSON block size.
+        def self.collect(data_store, **args)
+          writer = new(data_store, **args)
           yield(writer)
           writer.record_count
         ensure
-          writer&.close
+          writer&.flush
         end
 
-        def initialize(input, on_first: nil)
-          @on_first      = on_first
-          @batch_count   = 0
-          @record_count  = 0
-          @input         = input
-          @record_number = 1
-          @slice         = @input.new(first_record_number: @record_number)
+        def initialize(data_store, on_first: nil, slice_size: nil, slice_batch_size: nil)
+          @on_first         = on_first
+          @record_count     = 0
+          @data_store       = data_store
+          @slice_size       = slice_size || @data_store.slice_size
+          @slice_batch_size = slice_batch_size || 20
+          @batch            = []
+          @batch_count      = 0
+          new_slice
         end
 
         def <<(line)
-          @record_number += 1
           if @on_first
             @on_first.call(line)
             @on_first = nil
             return self
           end
           @slice << line
-          @batch_count  += 1
           @record_count += 1
-          if @batch_count >= @input.slice_size
-            @input.insert(@slice)
-            @batch_count = 0
-            @slice       = @input.new(first_record_number: @record_number)
+          if @slice.size >= @slice_size
+            save_slice
+            new_slice
           end
           self
         end
 
-        def close
-          @input.insert(@slice) if @slice.size.positive?
+        def flush
+          if @slice_batch_size
+            @batch << @slice if @slice.size.positive?
+            @data_store.insert_many(@batch)
+            @batch       = []
+            @batch_count = 0
+          elsif @slice.size.positive?
+            @data_store.insert(@slice)
+          end
+        end
+
+        def new_slice
+          @slice = @data_store.new(first_record_number: @record_count + 1)
+        end
+
+        def save_slice
+          return flush unless @slice_batch_size
+
+          @batch_count += 1
+          return flush if @batch_count >= @slice_batch_size
+
+          @batch << @slice
         end
       end
     end

data/lib/rocket_job/sliced.rb

@@ -2,6 +2,7 @@ module RocketJob
   module Sliced
     autoload :BZip2OutputSlice, "rocket_job/sliced/bzip2_output_slice"
     autoload :CompressedSlice, "rocket_job/sliced/compressed_slice"
+    autoload :EncryptedBZip2OutputSlice, "rocket_job/sliced/encrypted_bzip2_output_slice"
     autoload :EncryptedSlice, "rocket_job/sliced/encrypted_slice"
     autoload :Input, "rocket_job/sliced/input"
     autoload :Output, "rocket_job/sliced/output"
@@ -13,24 +14,5 @@ module RocketJob
       autoload :Input, "rocket_job/sliced/writer/input"
       autoload :Output, "rocket_job/sliced/writer/output"
     end
-
-    # Returns [RocketJob::Sliced::Slices] for the relevant direction and category.
-    def self.factory(direction, category, job)
-      collection_name = "rocket_job.#{direction}s.#{job.id}"
-      collection_name << ".#{category.name}" unless category.name == :main
-
-      case direction
-      when :input
-        RocketJob::Sliced::Input.new(
-          collection_name: collection_name,
-          slice_class:     category.serializer_class,
-          slice_size:      category.slice_size
-        )
-      when :output
-        RocketJob::Sliced::Output.new(collection_name: collection_name, slice_class: category.serializer_class)
-      else
-        raise(ArgumentError, "Unknown direction: #{direction.inspect}")
-      end
-    end
   end
 end

data/lib/rocket_job/subscribers/secret_config.rb

@@ -0,0 +1,17 @@
+module RocketJob
+  module Subscribers
+    # Cause all instances to refresh their in-memory copy
+    # of the Secret Config Registry
+    #
+    #   RocketJob::Subscribers::SecretConfig.publish(:refresh)
+    class SecretConfig
+      include RocketJob::Subscriber
+
+      def refresh
+        logger.measure_info "Refreshed Secret Config" do
+          ::SecretConfig.refresh!
+        end
+      end
+    end
+  end
+end

data/lib/rocket_job/supervisor.rb

@@ -34,6 +34,7 @@ module RocketJob
       logger.info "Rocket Job Server started"
 
       event_listener = Thread.new { Event.listener }
+      Subscribers::SecretConfig.subscribe if defined?(SecretConfig)
       Subscribers::Server.subscribe(self) do
         Subscribers::Worker.subscribe(self) do
           Subscribers::Logger.subscribe do

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.0.0.rc3".freeze
+  VERSION = "6.0.3".freeze
 end

data/lib/rocketjob.rb

@@ -63,7 +63,6 @@ module RocketJob
     autoload :Cron,                  "rocket_job/plugins/cron"
     autoload :Document,              "rocket_job/plugins/document"
     autoload :ProcessingWindow,      "rocket_job/plugins/processing_window"
-    autoload :Restart,               "rocket_job/plugins/restart"
     autoload :Retry,                 "rocket_job/plugins/retry"
     autoload :Singleton,             "rocket_job/plugins/singleton"
     autoload :StateMachine,          "rocket_job/plugins/state_machine"
@@ -73,22 +72,24 @@ module RocketJob
 
   module Jobs
     autoload :ActiveJob,             "rocket_job/jobs/active_job"
+    autoload :ConversionJob,         "rocket_job/jobs/conversion_job"
     autoload :CopyFileJob,           "rocket_job/jobs/copy_file_job"
     autoload :DirmonJob,             "rocket_job/jobs/dirmon_job"
+    autoload :HousekeepingJob,       "rocket_job/jobs/housekeeping_job"
     autoload :OnDemandBatchJob,      "rocket_job/jobs/on_demand_batch_job"
     autoload :OnDemandJob,           "rocket_job/jobs/on_demand_job"
-    autoload :HousekeepingJob,       "rocket_job/jobs/housekeeping_job"
     autoload :PerformanceJob,        "rocket_job/jobs/performance_job"
     autoload :SimpleJob,             "rocket_job/jobs/simple_job"
     autoload :UploadFileJob,         "rocket_job/jobs/upload_file_job"
 
     module ReEncrypt
-      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
+      autoload :RelationalJob,       "rocket_job/jobs/re_encrypt/relational_job"
     end
   end
 
   module Subscribers
     autoload :Logger,                "rocket_job/subscribers/logger"
+    autoload :SecretConfig,          "rocket_job/subscribers/secret_config"
     autoload :Server,                "rocket_job/subscribers/server"
     autoload :Worker,                "rocket_job/subscribers/worker"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.0.0.rc3
+  version: 6.0.3
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2021-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
@@ -84,14 +84,14 @@ dependencies:
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.7'
 - !ruby/object:Gem::Dependency
@@ -134,9 +134,6 @@ files:
 - lib/rocket_job/batch/results.rb
 - lib/rocket_job/batch/state_machine.rb
 - lib/rocket_job/batch/statistics.rb
-- lib/rocket_job/batch/tabular.rb
-- lib/rocket_job/batch/tabular/input.rb
-- lib/rocket_job/batch/tabular/output.rb
 - lib/rocket_job/batch/throttle.rb
 - lib/rocket_job/batch/throttle_running_workers.rb
 - lib/rocket_job/batch/throttle_windows.rb
@@ -198,6 +195,7 @@ files:
 - lib/rocket_job/sliced.rb
 - lib/rocket_job/sliced/bzip2_output_slice.rb
 - lib/rocket_job/sliced/compressed_slice.rb
+- lib/rocket_job/sliced/encrypted_bzip2_output_slice.rb
 - lib/rocket_job/sliced/encrypted_slice.rb
 - lib/rocket_job/sliced/input.rb
 - lib/rocket_job/sliced/output.rb
@@ -207,6 +205,7 @@ files:
 - lib/rocket_job/sliced/writer/output.rb
 - lib/rocket_job/subscriber.rb
 - lib/rocket_job/subscribers/logger.rb
+- lib/rocket_job/subscribers/secret_config.rb
 - lib/rocket_job/subscribers/server.rb
 - lib/rocket_job/subscribers/worker.rb
 - lib/rocket_job/supervisor.rb
@@ -233,11 +232,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Ruby's missing batch processing system.