iostreams 1.1.0 → 1.1.1

data/lib/io_streams/tabular/header.rb

@@ -91,7 +91,7 @@ module IOStreams
         case row
         when Array
           unless columns
-            raise(IOStreams::Errors::InvalidHeader, 'Missing mandatory header when trying to convert a row into a hash')
+            raise(IOStreams::Errors::InvalidHeader, "Missing mandatory header when trying to convert a row into a hash")
           end
 
           array_to_hash(row)
@@ -142,9 +142,9 @@ module IOStreams
 
       def cleanse_column(name)
         cleansed = name.to_s.strip.downcase
-        cleansed.gsub!(/\s+/, '_')
-        cleansed.gsub!(/-+/, '_')
-        cleansed.gsub!(/\W+/, '')
+        cleansed.gsub!(/\s+/, "_")
+        cleansed.gsub!(/-+/, "_")
+        cleansed.gsub!(/\W+/, "")
         cleansed
       end
     end

data/lib/io_streams/tabular/parser/array.rb

@@ -1,4 +1,4 @@
-require 'json'
+require "json"
 module IOStreams
   class Tabular
     module Parser
@@ -15,9 +15,7 @@ module IOStreams
 
         # Returns Array
         def parse(row)
-          unless row.is_a?(::Array)
-            raise(IOStreams::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}")
-          end
+          raise(IOStreams::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)
 
           row
         end

data/lib/io_streams/tabular/parser/csv.rb

@@ -1,4 +1,4 @@
-require 'csv'
+require "csv"
 module IOStreams
   class Tabular
     module Parser
@@ -25,9 +25,7 @@ module IOStreams
         def parse(row)
           return row if row.is_a?(::Array)
 
-          unless row.is_a?(String)
-            raise(IOStreams::Errors::TypeMismatch, "Format is :csv. Invalid input: #{row.class.name}")
-          end
+          raise(IOStreams::Errors::TypeMismatch, "Format is :csv. Invalid input: #{row.class.name}") unless row.is_a?(String)
 
           parse_line(row)
         end
@@ -50,7 +48,7 @@ module IOStreams
           end
 
           def render_array(array)
-            CSV.generate_line(array, encoding: 'UTF-8', row_sep: '')
+            CSV.generate_line(array, encoding: "UTF-8", row_sep: "")
           end
         else
           def parse_line(line)

data/lib/io_streams/tabular/parser/fixed.rb

@@ -23,7 +23,7 @@ module IOStreams
         def render(row, header)
           hash = header.to_hash(row)
 
-          result = ''
+          result = ""
           fixed_layout.each do |map|
             # A nil value is considered an empty string
             value = hash[map.key].to_s
@@ -42,8 +42,8 @@ module IOStreams
           hash  = {}
           index = 0
           fixed_layout.each do |map|
-            value         = line[index..(index + map.size - 1)]
-            index         += map.size
+            value = line[index..(index + map.size - 1)]
+            index += map.size
             hash[map.key] = value.to_s.strip
           end
           hash
@@ -60,6 +60,7 @@ module IOStreams
             size = map[:size]
             key  = map[:key]
             raise(ArgumentError, "Missing required :key and :size in: #{map.inspect}") unless size && key
+
             FixedLayout.new(key, size)
           end
         end

data/lib/io_streams/tabular/parser/hash.rb

@@ -1,12 +1,10 @@
-require 'json'
+require "json"
 module IOStreams
   class Tabular
     module Parser
       class Hash < Base
         def parse(row)
-          unless row.is_a?(::Hash)
-            raise(IOStreams::Errors::TypeMismatch, "Format is :hash. Invalid input: #{row.class.name}")
-          end
+          raise(IOStreams::Errors::TypeMismatch, "Format is :hash. Invalid input: #{row.class.name}") unless row.is_a?(::Hash)
 
           row
         end

data/lib/io_streams/tabular/parser/json.rb

@@ -1,4 +1,4 @@
-require 'json'
+require "json"
 module IOStreams
   class Tabular
     module Parser
@@ -7,9 +7,7 @@ module IOStreams
         def parse(row)
           return row if row.is_a?(::Hash)
 
-          unless row.is_a?(String)
-            raise(IOStreams::Errors::TypeMismatch, "Format is :json. Invalid input: #{row.class.name}")
-          end
+          raise(IOStreams::Errors::TypeMismatch, "Format is :json. Invalid input: #{row.class.name}") unless row.is_a?(String)
 
           JSON.parse(row)
         end

data/lib/io_streams/tabular/parser/psv.rb

@@ -12,27 +12,25 @@ module IOStreams
             raise(IOStreams::Errors::InvalidHeader, "Format is :psv. Invalid input header: #{row.class.name}")
           end
 
-          row.split('|')
+          row.split("|")
         end
 
         # Returns [Array] the parsed PSV line
         def parse(row)
           return row if row.is_a?(::Array)
 
-          unless row.is_a?(String)
-            raise(IOStreams::Errors::TypeMismatch, "Format is :psv. Invalid input: #{row.class.name}")
-          end
+          raise(IOStreams::Errors::TypeMismatch, "Format is :psv. Invalid input: #{row.class.name}") unless row.is_a?(String)
 
-          row.split('|')
+          row.split("|")
         end
 
         # Return the supplied array as a single line JSON string.
         def render(row, header)
           array          = header.to_array(row)
           cleansed_array = array.collect do |i|
-            i.is_a?(String) ? i.tr('|', ':') : i
+            i.is_a?(String) ? i.tr("|", ":") : i
           end
-          cleansed_array.join('|')
+          cleansed_array.join("|")
         end
       end
     end

data/lib/io_streams/tabular/utility/csv_row.rb

@@ -1,4 +1,4 @@
-require 'csv'
+require "csv"
 module IOStreams
   class Tabular
     module Utility
@@ -11,11 +11,11 @@ module IOStreams
       #   the file is broken apart based on line feeds during the upload process and
       #   is then processed by each worker on a line by line basis.
       class CSVRow < ::CSV
-        UTF8_ENCODING = Encoding.find('UTF-8').freeze
+        UTF8_ENCODING = Encoding.find("UTF-8").freeze
 
         def initialize(encoding = UTF8_ENCODING)
-          @io = StringIO.new(''.force_encoding(encoding))
-          super(@io, row_sep: '')
+          @io = StringIO.new("".force_encoding(encoding))
+          super(@io, row_sep: "")
         end
 
         # Parse a single line of CSV data
@@ -39,9 +39,7 @@ module IOStreams
               if part[-1] == @quote_char && part.count(@quote_char).odd?
                 # extended column ends
                 csv.last << part[0..-2]
-                if csv.last =~ @parsers[:stray_quote]
-                  raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}"
-                end
+                raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}" if csv.last =~ @parsers[:stray_quote]
 
                 csv.last.gsub!(@quote_char * 2, @quote_char)
                 in_extended_col = false
@@ -59,9 +57,7 @@ module IOStreams
               else
                 # regular quoted column
                 csv << part[1..-2]
-                if csv.last =~ @parsers[:stray_quote]
-                  raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}"
-                end
+                raise MalformedCSVError, "Missing or stray quote in line #{lineno + 1}" if csv.last =~ @parsers[:stray_quote]
 
                 csv.last.gsub!(@quote_char * 2, @quote_char)
               end
@@ -82,11 +78,9 @@ module IOStreams
           # column.
           csv[-1][-1] = @row_sep if in_extended_col
 
-          if in_extended_col
-            raise MalformedCSVError, "Unclosed quoted field on line #{lineno + 1}."
-          end
+          raise MalformedCSVError, "Unclosed quoted field on line #{lineno + 1}." if in_extended_col
 
-          @lineno     += 1
+          @lineno += 1
 
           # save fields unconverted fields, if needed...
           unconverted = csv.dup if @unconverted_fields
@@ -97,9 +91,7 @@ module IOStreams
           csv         = parse_headers(csv) if @use_headers
 
           # inject unconverted fields and accessor, if requested...
-          if @unconverted_fields && (!csv.respond_to? :unconverted_fields)
-            add_unconverted_fields(csv, unconverted)
-          end
+          add_unconverted_fields(csv, unconverted) if @unconverted_fields && (!csv.respond_to? :unconverted_fields)
 
           csv
         end

data/lib/io_streams/utils.rb

@@ -1,4 +1,4 @@
-require 'uri'
+require "uri"
 module IOStreams
   module Utils
     MAX_TEMP_FILE_NAME_ATTEMPTS = 5
@@ -24,7 +24,7 @@ module IOStreams
     # Yields the path to a temporary file_name.
     #
     # File is deleted upon completion if present.
-    def self.temp_file_name(basename, extension = '')
+    def self.temp_file_name(basename, extension = "")
       result = nil
       ::Dir::Tmpname.create([basename, extension], IOStreams.temp_dir, max_try: MAX_TEMP_FILE_NAME_ATTEMPTS) do |tmpname|
         begin
@@ -40,7 +40,7 @@ module IOStreams
       attr_reader :scheme, :hostname, :path, :user, :password, :port, :query
 
       def initialize(url)
-        url       = url.gsub(' ', '%20')
+        url       = url.gsub(" ", "%20")
         uri       = ::URI.parse(url)
         @scheme   = uri.scheme
         @hostname = uri.hostname

data/lib/io_streams/utils/reliable_http.rb

@@ -0,0 +1,98 @@
+require "net/http"
+require "uri"
+module IOStreams
+  module Utils
+    class ReliableHTTP
+      attr_reader :username, :password, :max_redirects, :url
+
+      # Reliable HTTP implementation with support for:
+      # * HTTP Redirects
+      # * Basic authentication
+      # * Raises an exception anytime the HTTP call is not successful.
+      # * TODO: Automatic retries with a logarithmic backoff strategy.
+      #
+      # Parameters:
+      #   url: [String]
+      #      URI of the file to download.
+      #     Example:
+      #       https://www5.fdic.gov/idasp/Offices2.zip
+      #       http://hostname/path/file_name
+      #
+      #     Full url showing all the optional elements that can be set via the url:
+      #       https://username:password@hostname/path/file_name
+      #
+      #   username: [String]
+      #     When supplied, basic authentication is used with the username and password.
+      #
+      #   password: [String]
+      #     Password to use use with basic authentication when the username is supplied.
+      #
+      #   max_redirects: [Integer]
+      #     Maximum number of http redirects to follow.
+      def initialize(url, username: nil, password: nil, max_redirects: 10)
+        uri = URI.parse(url)
+        unless %w[http https].include?(uri.scheme)
+          raise(ArgumentError, "Invalid URL. Required Format: 'http://<host_name>/<file_name>', or 'https://<host_name>/<file_name>'")
+        end
+
+        @username      = username || uri.user
+        @password      = password || uri.password
+        @max_redirects = max_redirects
+        @url           = url
+      end
+
+      # Read a file using an http get.
+      #
+      # For example:
+      #   IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').reader {|file| puts file.read}
+      #
+      # Read the file without unzipping and streaming the first file in the zip:
+      #   IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').stream(:none).reader {|file| puts file.read}
+      #
+      # Notes:
+      # * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.
+      def post(&block)
+        handle_redirects(Net::HTTP::Post, url, max_redirects, &block)
+      end
+
+      def get(&block)
+        handle_redirects(Net::HTTP::Get, url, max_redirects, &block)
+      end
+
+      private
+
+      def handle_redirects(request_class, uri, max_redirects, &block)
+        uri    = URI.parse(uri) unless uri.is_a?(URI)
+        result = nil
+        raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if max_redirects < 1
+
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
+          request = request_class.new(uri)
+          request.basic_auth(username, password) if username
+
+          http.request(request) do |response|
+            raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound)
+
+            if response.is_a?(Net::HTTPUnauthorized)
+              raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
+            end
+
+            if response.is_a?(Net::HTTPRedirection)
+              new_uri = response["location"]
+              return handle_redirects(request_class, new_uri, max_redirects: max_redirects - 1, &block)
+            end
+
+            unless response.is_a?(Net::HTTPSuccess)
+              raise(IOStreams::Errors::CommunicationsFailure, "Invalid response code: #{response.code}")
+            end
+
+            yield(response) if block_given?
+
+            result = response
+          end
+        end
+        result
+      end
+    end
+  end
+end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.1.0".freeze
+  VERSION = "1.1.1".freeze
 end

data/lib/io_streams/writer.rb

@@ -5,7 +5,7 @@ module IOStreams
     def self.stream(output_stream, original_file_name: nil, **args, &block)
       Utils.temp_file_name("iostreams_writer") do |file_name|
         file(file_name, original_file_name: original_file_name, **args, &block)
-        ::File.open(file_name, 'rb') { |source| ::IO.copy_stream(source, output_stream) }
+        ::File.open(file_name, "rb") { |source| ::IO.copy_stream(source, output_stream) }
       end
     end
 

data/lib/io_streams/xlsx/reader.rb

@@ -1,4 +1,4 @@
-require 'csv'
+require "csv"
 
 module IOStreams
   module Xlsx
@@ -6,15 +6,15 @@ module IOStreams
       # Convert a xlsx, or xlsm file into CSV format.
       def self.file(file_name, original_file_name: file_name, &block)
         # Stream into a temp file as csv
-        Utils.temp_file_name('iostreams_csv') do |temp_file_name|
-          ::File.open(temp_file_name, 'wb') { |io| new(file_name).each { |lines| io << lines.to_csv } }
-          ::File.open(temp_file_name, 'rb', &block)
+        Utils.temp_file_name("iostreams_csv") do |temp_file_name|
+          ::File.open(temp_file_name, "wb") { |io| new(file_name).each { |lines| io << lines.to_csv } }
+          ::File.open(temp_file_name, "rb", &block)
         end
       end
 
       def initialize(file_name)
         begin
-          require 'creek' unless defined?(Creek::Book)
+          require "creek" unless defined?(Creek::Book)
         rescue LoadError => e
           raise(LoadError, "Please install the 'creek' gem for xlsx streaming support. #{e.message}")
         end

data/lib/io_streams/zip/reader.rb

@@ -48,7 +48,7 @@ module IOStreams
         # The input stream from the first file found in the zip file is passed
         # to the supplied block
         def self.file(file_name, entry_file_name: nil, &block)
-          Utils.load_soft_dependency('rubyzip v1.x', 'Read Zip', 'zip') unless defined?(::Zip)
+          Utils.load_soft_dependency("rubyzip v1.x", "Read Zip", "zip") unless defined?(::Zip)
 
           ::Zip::File.open(file_name) do |zip_file|
             if entry_file_name

data/lib/io_streams/zip/writer.rb

@@ -19,9 +19,9 @@ module IOStreams
         if entry_file_name.nil? && original_file_name && (original_file_name =~ /\.(zip)\z/i)
           entry_file_name = original_file_name.to_s[0..-5]
         end
-        entry_file_name ||= 'file'
+        entry_file_name ||= "file"
 
-        Utils.load_soft_dependency('zip_tricks', 'Zip') unless defined?(ZipTricks::Streamer)
+        Utils.load_soft_dependency("zip_tricks", "Zip") unless defined?(ZipTricks::Streamer)
 
         ZipTricks::Streamer.open(output_stream) { |zip| zip.write_deflated_file(entry_file_name, &block) }
       end

data/lib/iostreams.rb

@@ -1,59 +1,59 @@
-require 'io_streams/version'
+require "io_streams/version"
 # @formatter:off
 module IOStreams
-  autoload :Builder,  'io_streams/builder'
-  autoload :Errors,   'io_streams/errors'
-  autoload :Path,     'io_streams/path'
-  autoload :Pgp,      'io_streams/pgp'
-  autoload :Reader,   'io_streams/reader'
-  autoload :Stream,   'io_streams/stream'
-  autoload :Tabular,  'io_streams/tabular'
-  autoload :Utils,    'io_streams/utils'
-  autoload :Writer,   'io_streams/writer'
+  autoload :Builder,  "io_streams/builder"
+  autoload :Errors,   "io_streams/errors"
+  autoload :Path,     "io_streams/path"
+  autoload :Pgp,      "io_streams/pgp"
+  autoload :Reader,   "io_streams/reader"
+  autoload :Stream,   "io_streams/stream"
+  autoload :Tabular,  "io_streams/tabular"
+  autoload :Utils,    "io_streams/utils"
+  autoload :Writer,   "io_streams/writer"
 
   module Paths
-    autoload :File,    'io_streams/paths/file'
-    autoload :HTTP,    'io_streams/paths/http'
-    autoload :Matcher, 'io_streams/paths/matcher'
-    autoload :S3,      'io_streams/paths/s3'
-    autoload :SFTP,    'io_streams/paths/sftp'
+    autoload :File,    "io_streams/paths/file"
+    autoload :HTTP,    "io_streams/paths/http"
+    autoload :Matcher, "io_streams/paths/matcher"
+    autoload :S3,      "io_streams/paths/s3"
+    autoload :SFTP,    "io_streams/paths/sftp"
   end
 
   module Bzip2
-    autoload :Reader, 'io_streams/bzip2/reader'
-    autoload :Writer, 'io_streams/bzip2/writer'
+    autoload :Reader, "io_streams/bzip2/reader"
+    autoload :Writer, "io_streams/bzip2/writer"
   end
   module Encode
-    autoload :Reader, 'io_streams/encode/reader'
-    autoload :Writer, 'io_streams/encode/writer'
+    autoload :Reader, "io_streams/encode/reader"
+    autoload :Writer, "io_streams/encode/writer"
   end
   module Gzip
-    autoload :Reader, 'io_streams/gzip/reader'
-    autoload :Writer, 'io_streams/gzip/writer'
+    autoload :Reader, "io_streams/gzip/reader"
+    autoload :Writer, "io_streams/gzip/writer"
   end
   module Line
-    autoload :Reader, 'io_streams/line/reader'
-    autoload :Writer, 'io_streams/line/writer'
+    autoload :Reader, "io_streams/line/reader"
+    autoload :Writer, "io_streams/line/writer"
   end
   module Record
-    autoload :Reader, 'io_streams/record/reader'
-    autoload :Writer, 'io_streams/record/writer'
+    autoload :Reader, "io_streams/record/reader"
+    autoload :Writer, "io_streams/record/writer"
   end
   module Row
-    autoload :Reader, 'io_streams/row/reader'
-    autoload :Writer, 'io_streams/row/writer'
+    autoload :Reader, "io_streams/row/reader"
+    autoload :Writer, "io_streams/row/writer"
   end
   module SymmetricEncryption
-    autoload :Reader, 'io_streams/symmetric_encryption/reader'
-    autoload :Writer, 'io_streams/symmetric_encryption/writer'
+    autoload :Reader, "io_streams/symmetric_encryption/reader"
+    autoload :Writer, "io_streams/symmetric_encryption/writer"
   end
   module Xlsx
-    autoload :Reader, 'io_streams/xlsx/reader'
+    autoload :Reader, "io_streams/xlsx/reader"
   end
   module Zip
-    autoload :Reader, 'io_streams/zip/reader'
-    autoload :Writer, 'io_streams/zip/writer'
+    autoload :Reader, "io_streams/zip/reader"
+    autoload :Writer, "io_streams/zip/writer"
   end
 end
-require 'io_streams/deprecated'
-require 'io_streams/io_streams'
+require "io_streams/deprecated"
+require "io_streams/io_streams"