rocketjob 4.3.0.beta → 4.3.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ae22e2ca14255089a3500e9294fc689847bb92525da45ba90426cae38a45378
-  data.tar.gz: fc0f6dd9c042020a01d47f4c2be35df3896db47e3cc07d875a27f057a8e64ab6
+  metadata.gz: b5544569821640636a5dc8cd7953f2c0fbadcfa90ec1c7ac029fa4fc249d1269
+  data.tar.gz: 601088d6918f605b80565d2185872b161e1ccf27d923bd6483fe49ac8d5b2586
 SHA512:
-  metadata.gz: cb612469360af546d76ea1d024e80cbdf50f40693533fd3e608927911d62c86da6ad4ba290da0186ce98b3be95b3cfad21ceed3bf22091d6e80cf2adc7b2387d
-  data.tar.gz: 4038eb8af3353d6358f3dc74c50410d1f16cf96fe716c19b29aa8843428e2a8938267b3d130c4023b7234ca45ad5a546d65836ea69775d354889e6cb50121fd6
+  metadata.gz: 9eda8d1c26f5e808f5ef2c1157c032ed490daaf697040a8987951c5ad25f979783796567b536c897317445b413f774fddcbd82ed3898a4578e3de4ece7a91ed7
+  data.tar.gz: 21744b1a7fad3e03c49367888e4662d0d00f60c671829c54cd8c7174ead64a8b9ce0642c61ae37c063508d5df3c0a7280a78e2bb1f752ac3fcd8e57343bf4a99

data/lib/rocket_job/batch/io.rb

@@ -67,32 +67,6 @@ module RocketJob
       #       Parses each line from the file into a Hash and uploads each hash for processing by workers.
       #     See IOStreams::Stream#each_line, IOStreams::Stream#each_row, and IOStreams::Stream#each_record.
       #
-      #   encoding: [String|Encoding]
-      #     Encode returned data with this encoding.
-      #     'US-ASCII':   Original 7 bit ASCII Format
-      #     'ASCII-8BIT': 8-bit ASCII Format
-      #     'UTF-8':      UTF-8 Format
-      #     Etc.
-      #     Default: 'UTF-8'
-      #     NOTE:    If a IOStreams::Path, or IOStreams::Stream was supplied then the encoding will be set
-      #              if not already set in the supplied stream.
-      #
-      #   encode_replace: [String]
-      #     The character to replace with when a character cannot be converted to the target encoding.
-      #     nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
-      #     Default: nil
-      #     NOTE:    If a IOStreams::Path, or IOStreams::Stream was supplied then the encoding will be set
-      #              if not already set in the supplied stream.
-      #
-      #   encode_cleaner: [nil|symbol|Proc]
-      #     Cleanse data read from the input stream.
-      #     nil:           No cleansing
-      #     :printable Cleanse all non-printable characters except \r and \n
-      #     Proc/lambda    Proc to call after every read to cleanse the data
-      #     Default: :printable
-      #     NOTE:    If a IOStreams::Path, or IOStreams::Stream was supplied then the encoding will be set
-      #              if not already set in the supplied stream.
-      #
       # Example:
       #   # Load plain text records from a file
       #   job.upload('hello.csv')
@@ -100,7 +74,8 @@ module RocketJob
       # Example:
       #   # Load plain text records from a file, stripping all non-printable characters,
       #   # as well as any characters that cannot be converted to UTF-8
-      #   job.upload('hello.csv', encode_cleaner: :printable, encode_replace: '')
+      #   path = IOStreams.path('hello.csv').option(:encode, cleaner: :printable, replace: '')
+      #   job.upload(path)
       #
       # Example: Zip
       #   # Since csv is not known to RocketJob it is ignored
@@ -140,15 +115,15 @@ module RocketJob
       # * If an io stream is supplied, it is read until it returns nil.
       # * Only use this method for UTF-8 data, for binary data use #input_slice or #input_records.
       # * CSV parsing is slow, so it is usually left for the workers to do.
-      def upload(stream = nil, file_name: nil, category: :main, encoding: 'UTF-8', encode_cleaner: nil, encode_replace: nil, stream_mode: :line, on_first: nil, **args, &block)
+      def upload(stream = nil, file_name: nil, category: :main, stream_mode: :line, on_first: nil, **args, &block)
         raise(ArgumentError, 'Either stream, or a block must be supplied') unless stream || block
 
         count             =
           if block
             input(category).upload(on_first: on_first, &block)
           else
-            path = build_path(stream, file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-
+            path                  = IOStreams.new(stream)
+            path.file_name        = file_name if file_name
             self.upload_file_name = path.file_name
             input(category).upload(on_first: on_first) do |io|
               path.public_send("each_#{stream_mode}".to_sym, **args) { |line| io << line }
@@ -372,16 +347,13 @@ module RocketJob
       # Notes:
       # - The records are returned in '_id' order. Usually this is the order in
       #   which the records were originally loaded.
-      def download(stream = nil, category: :main, header_line: nil, encoding: 'UTF-8', encode_cleaner: nil, encode_replace: nil, **args, &block)
+      def download(stream = nil, category: :main, header_line: nil, **args, &block)
         raise "Cannot download incomplete job: #{id}. Currently in state: #{state}-#{sub_state}" if rocket_job_processing?
 
-        if block
-          output(category).download(header_line: header_line, &block)
-        else
-          path = build_path(stream, nil, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
-          path.line_writer(**args) do |io|
-            output(category).download(header_line: header_line) { |record| io << record }
-          end
+        return output(category).download(header_line: header_line, &block) if block
+
+        IOStreams.new(stream).line_writer(**args) do |io|
+          output(category).download(header_line: header_line) { |record| io << record }
         end
       end
 
@@ -417,17 +389,6 @@ module RocketJob
           RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
         end
       end
-
-      private
-
-      def build_path(stream, file_name, encoding: nil, encode_cleaner: nil, encode_replace: nil)
-        path           = IOStreams.new(stream)
-        path.file_name = file_name if file_name
-        if (encoding || encode_cleaner || encode_replace) && !path.setting(:encode)
-          path.option_or_stream(:encode, encoding: encoding, cleaner: encode_cleaner, replace: encode_replace)
-        end
-        path
-      end
     end
   end
 end

data/lib/rocket_job/batch/tabular/input.rb

@@ -49,20 +49,20 @@ module RocketJob
         #
         # Notes:
         # - When supplying a block the header must be set manually
-        def upload(file_name_or_io = nil, **args, &block)
-          if tabular_input_type == :text
-            args[:encoding]       = 'UTF-8'
-            args[:encode_cleaner] = :printable
-            args[:encode_replace] = ''
+        def upload(stream = nil, **args, &block)
+          input_stream = stream.nil? ? nil : IOStreams.new(stream)
+
+          if stream && (tabular_input_type == :text)
+            input_stream.option_or_stream(:encode, encoding: 'UTF-8', cleaner: :printable, replace: '')
           end
 
           # If an input header is not required, then we don't extract it'
-          return super(file_name_or_io, stream_mode: tabular_input_mode, **args, &block) unless tabular_input.header?
+          return super(input_stream, stream_mode: tabular_input_mode, **args, &block) unless tabular_input.header?
 
           # If the header is already set then it is not expected in the file
           if tabular_input_header.present?
             tabular_input_cleanse_header
-            return super(file_name_or_io, stream_mode: tabular_input_mode, **args, &block)
+            return super(input_stream, stream_mode: tabular_input_mode, **args, &block)
           end
 
           case tabular_input_mode
@@ -72,16 +72,16 @@ module RocketJob
               tabular_input_cleanse_header
               self.tabular_input_header = tabular_input.header.columns
             end
-            super(file_name_or_io, on_first: parse_header, stream_mode: tabular_input_mode, **args, &block)
+            super(input_stream, on_first: parse_header, stream_mode: tabular_input_mode, **args, &block)
           when :row
             set_header = -> (row) do
               tabular_input.header.columns = row
               tabular_input_cleanse_header
               self.tabular_input_header = tabular_input.header.columns
             end
-            super(file_name_or_io, on_first: set_header, stream_mode: tabular_input_mode, **args, &block)
+            super(input_stream, on_first: set_header, stream_mode: tabular_input_mode, **args, &block)
           when :record
-            super(file_name_or_io, stream_mode: tabular_input_mode, **args, &block)
+            super(input_stream, stream_mode: tabular_input_mode, **args, &block)
           else
             raise(ArgumentError, "Invalid tabular_input_mode: #{stream_mode.inspect}")
           end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '4.3.0.beta'.freeze
+  VERSION = '4.3.0.beta2'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 4.3.0.beta
+  version: 4.3.0.beta2
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-22 00:00:00.000000000 Z
+date: 2019-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta
+        version: 1.0.0.beta2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta
+        version: 1.0.0.beta2
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement