iostreams 0.15.0 → 0.16.0

data/lib/io_streams/row/writer.rb

@@ -13,12 +13,14 @@ module IOStreams
     #
     class Writer
       # Write a record as a Hash at a time to a file or stream.
-      def self.open(file_name_or_io, delimiter: $/, encoding: UTF8_ENCODING, strip_non_printable: false, **args)
+      def self.open(file_name_or_io, delimiter: $/, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args)
         if file_name_or_io.is_a?(String)
           IOStreams.line_writer(file_name_or_io,
-                                delimiter:           delimiter,
-                                encoding:            encoding,
-                                strip_non_printable: strip_non_printable) do |io|
+                                delimiter:      delimiter,
+                                encoding:       encoding,
+                                encode_cleaner: encode_cleaner,
+                                encode_replace: encode_replace
+          ) do |io|
             yield new(io, **args)
           end
         else
@@ -43,20 +45,20 @@ module IOStreams
         @delimited = delimited
 
         # Render header line when `columns` is supplied.
-        delimited << @tabular.render(columns) if columns && @tabular.requires_header?
+        delimited << @tabular.render_header if columns && @tabular.requires_header?
       end
 
       # Supply a hash or an array to render
       def <<(array)
         raise(ArgumentError, 'Must supply an Array') unless array.is_a?(Array)
-        # If header (columns) was not supplied as an argument, assume first line is the header.
-        tabular.header.columns = array if tabular.requires_header?
-        delimited << tabular.render(array)
+        if @tabular.header?
+          # If header (columns) was not supplied as an argument, assume first line is the header.
+          @tabular.header.columns = array
+          @delimited << @tabular.render_header
+        else
+          @delimited << @tabular.render(array)
+        end
       end
-
-      private
-
-      attr_reader :tabular, :delimited
     end
   end
 end

data/lib/io_streams/tabular.rb

@@ -28,7 +28,6 @@ module IOStreams
   #   tabular.render({"third"=>"3", "first_field"=>"1" })
   #   # => "1,,3"
   class Tabular
-    autoload :Errors, 'io_streams/tabular/errors'
     autoload :Header, 'io_streams/tabular/header'
 
     module Parser
@@ -54,7 +53,7 @@ module IOStreams
     #     :csv, :hash, :array, :json, :psv, :fixed
     #
     #   For all other parameters, see Tabular::Header.new
-    def initialize(format: nil, file_name: nil, **args)
+    def initialize(format: nil, file_name: nil, format_options: nil, **args)
       @header = Header.new(**args)
       klass   =
         if file_name && format.nil?
@@ -62,19 +61,24 @@ module IOStreams
         else
           self.class.parser_class(format)
         end
-      @parser = klass.new
+      @parser = format_options ? klass.new(format_options) : klass.new
     end
 
-    # Returns [true|false] whether a header row needs to be read first.
-    def requires_header?
+    # Returns [true|false] whether a header is still required in order to parse or render the current format.
+    def header?
       parser.requires_header? && IOStreams.blank?(header.columns)
     end
 
+    # Returns [true|false] whether a header row show be rendered on output.
+    def requires_header?
+      parser.requires_header?
+    end
+
     # Returns [Array] the header row/line after parsing and cleansing.
     # Returns `nil` if the row/line is blank, or a header is not required for the supplied format (:json, :hash).
     #
     # Notes:
-    # * Call `parse_header?` first to determine if the header should be parsed first.
+    # * Call `header?` first to determine if the header should be parsed first.
     # * The header columns are set after parsing the row, but the header is not cleansed.
     def parse_header(line)
       return if IOStreams.blank?(line) || !parser.requires_header?
@@ -104,60 +108,76 @@ module IOStreams
       parser.render(row, header)
     end
 
+    # Returns [String] the header rendered for the output format
+    # Return nil if no header is required.
+    def render_header
+      return unless requires_header?
+
+      if IOStreams.blank?(header.columns)
+        raise(Errors::MissingHeader, "Header columns must be set before attempting to render a header for format: #{format.inspect}")
+      end
+
+      parser.render(header.columns, header)
+    end
+
     # Returns [Array<String>] the cleansed columns
     def cleanse_header!
       header.cleanse!
       header.columns
     end
 
-    # Register a file extension and the reader and writer classes to use to format it
+    # Register a format and the parser class for it.
     #
     # Example:
-    #   # MyXls::Reader and MyXls::Writer must implement .open
-    #   register_extension(:xls, MyXls::Reader, MyXls::Writer)
-    def self.register_extension(extension, parser)
-      raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
-      @extensions[extension.nil? ? nil : extension.to_sym] = parser
+    #   register_format(:csv, IOStreams::Tabular::Parser::Csv)
+    def self.register_format(format, parser)
+      raise(ArgumentError, "Invalid format #{format.inspect}") unless format.nil? || format.to_s =~ /\A\w+\Z/
+      @formats[format.nil? ? nil : format.to_sym] = parser
     end
 
-    # De-Register a file extension
+    # De-Register a file format
     #
-    # Returns [Symbol] the extension removed, or nil if the extension was not registered
+    # Returns [Symbol] the format removed, or nil if the format was not registered
     #
     # Example:
     #   register_extension(:xls)
-    def self.deregister_extension(extension)
-      raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
-      @extensions.delete(extension.to_sym)
+    def self.deregister_format(format)
+      raise(ArgumentError, "Invalid format #{format.inspect}") unless format.to_s =~ /\A\w+\Z/
+      @formats.delete(format.to_sym)
+    end
+
+    # Returns [Array<Symbol>] the list of registered formats
+    def self.registered_formats
+      @formats.keys
     end
 
     private
 
     # A registry to hold formats for processing files during upload or download
-    @extensions = {}
+    @formats = {}
 
     def self.parser_class(format)
-      @extensions[format.nil? ? nil : format.to_sym] || raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
+      @formats[format.nil? ? nil : format.to_sym] || raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
     end
 
     # Returns the parser to use with tabular for the supplied file_name
     def self.parser_class_for_file_name(file_name)
-      extension = nil
+      format = nil
       file_name.to_s.split('.').reverse_each do |ext|
-        if @extensions.include?(ext.to_sym)
-          extension = ext.to_sym
+        if @formats.include?(ext.to_sym)
+          format = ext.to_sym
           break
         end
       end
-      parser_class(extension)
+      parser_class(format)
     end
 
-    register_extension(nil, IOStreams::Tabular::Parser::Csv)
-    register_extension(:array, IOStreams::Tabular::Parser::Array)
-    register_extension(:csv, IOStreams::Tabular::Parser::Csv)
-    register_extension(:fixed, IOStreams::Tabular::Parser::Fixed)
-    register_extension(:hash, IOStreams::Tabular::Parser::Hash)
-    register_extension(:json, IOStreams::Tabular::Parser::Json)
-    register_extension(:psv, IOStreams::Tabular::Parser::Psv)
+    register_format(nil, IOStreams::Tabular::Parser::Csv)
+    register_format(:array, IOStreams::Tabular::Parser::Array)
+    register_format(:csv, IOStreams::Tabular::Parser::Csv)
+    register_format(:fixed, IOStreams::Tabular::Parser::Fixed)
+    register_format(:hash, IOStreams::Tabular::Parser::Hash)
+    register_format(:json, IOStreams::Tabular::Parser::Json)
+    register_format(:psv, IOStreams::Tabular::Parser::Psv)
   end
 end

data/lib/io_streams/tabular/header.rb

@@ -61,17 +61,17 @@ module IOStreams
         end
 
         if !skip_unknown && !ignored_columns.empty?
-          raise(IOStreams::Tabular::Errors::InvalidHeader, "Unknown columns after cleansing: #{ignored_columns.join(',')}")
+          raise(IOStreams::Errors::InvalidHeader, "Unknown columns after cleansing: #{ignored_columns.join(',')}")
         end
 
         if ignored_columns.size == columns.size
-          raise(IOStreams::Tabular::Errors::InvalidHeader, "All columns are unknown after cleansing: #{ignored_columns.join(',')}")
+          raise(IOStreams::Errors::InvalidHeader, "All columns are unknown after cleansing: #{ignored_columns.join(',')}")
         end
 
         if required_columns
           missing_columns = required_columns - columns
           unless missing_columns.empty?
-            raise(IOStreams::Tabular::Errors::InvalidHeader, "Missing columns after cleansing: #{missing_columns.join(',')}")
+            raise(IOStreams::Errors::InvalidHeader, "Missing columns after cleansing: #{missing_columns.join(',')}")
           end
         end
 
@@ -90,12 +90,12 @@ module IOStreams
 
         case row
         when Array
-          raise(Tabular::Errors::InvalidHeader, "Missing mandatory header when trying to convert a row into a hash") unless columns
+          raise(IOStreams::Errors::InvalidHeader, "Missing mandatory header when trying to convert a row into a hash") unless columns
           array_to_hash(row)
         when Hash
           cleanse && columns ? cleanse_hash(row) : row
         else
-          raise(Tabular::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to a Hash")
+          raise(IOStreams::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to a Hash")
         end
       end
 
@@ -104,7 +104,7 @@ module IOStreams
           row = cleanse_hash(row) if cleanse
           row = columns.collect { |column| row[column] }
         end
-        raise(Tabular::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to an Array without the header columns being set.") unless row.is_a?(Array)
+        raise(IOStreams::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to an Array without the header columns being set.") unless row.is_a?(Array)
         row
       end
 

data/lib/io_streams/tabular/parser/array.rb

@@ -6,13 +6,13 @@ module IOStreams
         # Returns [Array<String>] the header row.
         # Returns nil if the row is blank.
         def parse_header(row)
-          raise(Tabular::Errors::InvalidHeader, "Format is :array. Invalid input header: #{row.class.name}") unless row.is_a?(::Array)
+          raise(IOStreams::Errors::InvalidHeader, "Format is :array. Invalid input header: #{row.class.name}") unless row.is_a?(::Array)
           row
         end
 
         # Returns Array
         def parse(row)
-          raise(Tabular::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)
+          raise(IOStreams::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)
           row
         end
 

data/lib/io_streams/tabular/parser/csv.rb

@@ -11,14 +11,18 @@ module IOStreams
         # Returns [Array<String>] the header row.
         # Returns nil if the row is blank.
         def parse_header(row)
-          raise(Tabular::Errors::InvalidHeader, "Format is :csv. Invalid input header: #{row.class.name}") unless row.is_a?(String)
+          return row if row.is_a?(::Array)
+
+          raise(IOStreams::Errors::InvalidHeader, "Format is :csv. Invalid input header: #{row.class.name}") unless row.is_a?(String)
 
           csv_parser.parse(row)
         end
 
         # Returns [Array] the parsed CSV line
         def parse(row)
-          raise(Tabular::Errors::TypeMismatch, "Format is :csv. Invalid input: #{row.class.name}") unless row.is_a?(String)
+          return row if row.is_a?(::Array)
+
+          raise(IOStreams::Errors::TypeMismatch, "Format is :csv. Invalid input: #{row.class.name}") unless row.is_a?(String)
 
           csv_parser.parse(row)
         end

data/lib/io_streams/tabular/parser/fixed.rb

@@ -3,83 +3,64 @@ module IOStreams
     module Parser
       # Parsing and rendering fixed length data
       class Fixed < Base
-        attr_reader :encoding, :encoding_options, :fixed_format
+        attr_reader :fixed_layout
 
         # Returns [IOStreams::Tabular::Parser]
         #
-        # Arguments:
-        #   format: [Array<Hash>]
+        # Parameters:
+        #   layout: [Array<Hash>]
         #     [
         #       {key: 'name',    size: 23 },
         #       {key: 'address', size: 40 },
         #       {key: 'zip',     size: 5 }
         #     ]
-        #
-        #   encoding: [String|Encoding]
-        #     nil: Don't perform any encoding conversion
-        #     'ASCII': ASCII Format
-        #     'UTF-8': UTF-8 Format
-        #     Etc.
-        #     Default: nil
-        #
-        #   replacement: [String]
-        #     The character to replace with when a character cannot be converted to the target encoding.
-        #     nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
-        #     Default: nil
-        def initialize(format:, encoding: nil, replacement: nil)
-          @encoding         = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
-          @encoding_options = replacement.nil? ? {} : {invalid: :replace, undef: :replace, replace: replacement}
-          @fixed_format     = parse_format(format)
+        def initialize(layout:)
+          @fixed_layout = parse_layout(layout)
         end
 
-        # Returns [String] fixed format values extracted from the supplied hash.
+        # Returns [String] fixed layout values extracted from the supplied hash.
         # String will be encoded to `encoding`
         def render(row, header)
           hash = header.to_hash(row)
 
-          result = encoding.nil? ? '' : ''.encode(encoding)
-          fixed_format.each do |map|
+          result = ''
+          fixed_layout.each do |map|
             # A nil value is considered an empty string
             value = hash[map.key].to_s
-            result <<
-              if encoding
-                format("%-#{map.size}.#{map.size}s".encode(encoding), value.encode(encoding, encoding_options))
-              else
-                format("%-#{map.size}.#{map.size}s", value)
-              end
+            result << format("%-#{map.size}.#{map.size}s", value)
           end
           result
         end
 
-        # Returns [Hash<Symbol, String>] fixed format values extracted from the supplied line.
+        # Returns [Hash<Symbol, String>] fixed layout values extracted from the supplied line.
         # String will be encoded to `encoding`
         def parse(line)
           unless line.is_a?(String)
-            raise(Tabular::Errors::TypeMismatch, "Format is :fixed. Invalid parse input: #{line.class.name}")
+            raise(IOStreams::Errors::TypeMismatch, "Format is :fixed. Invalid parse input: #{line.class.name}")
           end
 
           hash  = {}
           index = 0
-          fixed_format.each do |map|
+          fixed_layout.each do |map|
             value         = line[index..(index + map.size - 1)]
             index         += map.size
-            hash[map.key] = encoding.nil? ? value.strip : value.strip.encode(encoding, encoding_options)
+            hash[map.key] = value.to_s.strip
           end
           hash
         end
 
         private
 
-        FixedFormat = Struct.new(:key, :size)
+        FixedLayout = Struct.new(:key, :size)
 
-        # Returns [Array<FixedFormat>] the format for this fixed width file.
+        # Returns [Array<FixedLayout>] the layout for this fixed width file.
         # Also validates values
-        def parse_format(format)
-          format.collect do |map|
+        def parse_layout(layout)
+          layout.collect do |map|
             size = map[:size]
             key  = map[:key]
             raise(ArgumentError, "Missing required :key and :size in: #{map.inspect}") unless size && key
-            FixedFormat.new(key, size)
+            FixedLayout.new(key, size)
           end
         end
       end

data/lib/io_streams/tabular/parser/hash.rb

@@ -4,7 +4,7 @@ module IOStreams
     module Parser
       class Hash < Base
         def parse(row)
-          raise(Tabular::Errors::TypeMismatch, "Format is :hash. Invalid input: #{row.class.name}") unless row.is_a?(::Hash)
+          raise(IOStreams::Errors::TypeMismatch, "Format is :hash. Invalid input: #{row.class.name}") unless row.is_a?(::Hash)
           row
         end
 

data/lib/io_streams/tabular/parser/json.rb

@@ -5,7 +5,9 @@ module IOStreams
       # For parsing a single line of JSON at a time
       class Json < Base
         def parse(row)
-          raise(Tabular::Errors::TypeMismatch, "Format is :json. Invalid input: #{row.class.name}") unless row.is_a?(String)
+          return row if row.is_a?(::Hash)
+
+          raise(IOStreams::Errors::TypeMismatch, "Format is :json. Invalid input: #{row.class.name}") unless row.is_a?(String)
 
           JSON.parse(row)
         end

data/lib/io_streams/tabular/parser/psv.rb

@@ -6,8 +6,10 @@ module IOStreams
         # Returns [Array<String>] the header row.
         # Returns nil if the row is blank.
         def parse_header(row)
+          return row if row.is_a?(::Array)
+
           unless row.is_a?(String)
-            raise(Tabular::Errors::InvalidHeader, "Format is :psv. Invalid input header: #{row.class.name}")
+            raise(IOStreams::Errors::InvalidHeader, "Format is :psv. Invalid input header: #{row.class.name}")
           end
 
           row.split('|')
@@ -15,7 +17,9 @@ module IOStreams
 
         # Returns [Array] the parsed PSV line
         def parse(row)
-          raise(Tabular::Errors::TypeMismatch, "Format is :psv. Invalid input: #{row.class.name}") unless row.is_a?(String)
+          return row if row.is_a?(::Array)
+
+          raise(IOStreams::Errors::TypeMismatch, "Format is :psv. Invalid input: #{row.class.name}") unless row.is_a?(String)
 
           row.split('|')
         end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = '0.15.0'
+  VERSION = '0.16.0'
 end

data/lib/io_streams/xlsx/reader.rb

@@ -3,50 +3,40 @@ require 'csv'
 module IOStreams
   module Xlsx
     class Reader
-      attr_reader :worksheet
-
-      # Read from a xlsx, or xlsm file or stream.
-      #
-      # Example:
-      #   IOStreams::Xlsx::Reader.open('spreadsheet.xlsx') do |spreadsheet_stream|
-      #     spreadsheet_stream.each_line do |line|
-      #       puts line
-      #     end
-      #   end
-      def self.open(file_name_or_io, buffer_size: 65536, &block)
-        begin
-          require 'creek' unless defined?(Creek::Book)
-        rescue LoadError => e
-          raise(LoadError, "Please install the 'creek' gem for xlsx streaming support. #{e.message}")
-        end
-
-        if IOStreams.reader_stream?(file_name_or_io)
-          temp_file = Tempfile.new('rocket_job_xlsx')
-          file_name = temp_file.to_path
-
-          ::File.open(file_name, 'wb') do |file|
-            IOStreams.copy(file_name_or_io, file, buffer_size: buffer_size)
-          end
-        else
+      # Convert a xlsx, or xlsm file or stream into CSV format.
+      def self.open(file_name_or_io, _ = nil)
+        if file_name_or_io.is_a?(String)
           file_name = file_name_or_io
+        else
+          temp_file = Tempfile.new('iostreams_xlsx')
+          IOStreams.copy(file_name_or_io, temp_file)
+          file_name = temp_file.to_path
         end
 
-        block.call(self.new(Creek::Book.new(file_name, check_file_extension: false)))
+        csv_temp_file = Tempfile.new('iostreams_csv')
+        new(file_name).each { |lines| csv_temp_file << lines.to_csv }
+        csv_temp_file.rewind
+        yield csv_temp_file
       ensure
         temp_file.delete if temp_file
+        csv_temp_file.delete if csv_temp_file
       end
 
-      def initialize(workbook)
+      def initialize(file_name)
+        begin
+          require 'creek' unless defined?(Creek::Book)
+        rescue LoadError => e
+          raise(LoadError, "Please install the 'creek' gem for xlsx streaming support. #{e.message}")
+        end
+
+        workbook   = Creek::Book.new(file_name, check_file_extension: false)
         @worksheet = workbook.sheets[0]
       end
 
       # Returns each [Array] row from the spreadsheet
-      def each(&block)
-        worksheet.rows.each { |row| block.call(row.values) }
+      def each
+        @worksheet.rows.each { |row| yield row.values }
       end
-
-      alias_method :each_line, :each
-
     end
   end
 end