iostreams 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c04b1faf6890f6e1ec98965626068918c08f03b3f548f6a8e6a5c5a96da81e8b
-  data.tar.gz: 4a322fe6134797ce698f2d9bc65999b5e187d7dfc23354383965aaa0d4b85f98
+  metadata.gz: 9392af579f83fb26f1ea2cfeefae497362c1232ff1226adb4220cc54805b93c2
+  data.tar.gz: 8db9dd03113c32ebe44b54610509dc86626f44deb6900fae65ac7cdd34a3abee
 SHA512:
-  metadata.gz: eb2284f07be4b270e24cf4135b3fdd9b9e8a125f73dc65c2b581a756ca2a69ee96b6f14ac398b33c264cc9d450ac13e8a37dc70fca94d04945e231687c71c594
-  data.tar.gz: a6fbb3a182e137a252f5dac842e4a327f5176db427dd8f62da183b77329545c91515ae93740897c71daf3cca37feeb1b21356645426a629074385f16201b0136
+  metadata.gz: 2810231db04f7a98086bb47a7657adb7888dc3f16e03f48e9824170bf36a8df65ca006cfbcaca7360e42f609f5444be189fa24ba1ae4610c5c727f3c18c6bb85
+  data.tar.gz: 2e247fb89ad090d96cd6e0b20aaf9afd0b71cbbef7dc6cb7e84bd60182c51c6ce9cb461dfcdb4d35d8e5b0cdfb8822667cb35f42a8e54ed6d67b4575d02bffdb

data/lib/io_streams/bzip2/reader.rb

@@ -2,7 +2,7 @@ module IOStreams
   module Bzip2
     class Reader
       # Read from a Bzip2 file or stream, decompressing the contents as it is read
-      def self.open(file_name_or_io, _=nil, &block)
+      def self.open(file_name_or_io, **args, &block)
         begin
           require 'rbzip2' unless defined?(RBzip2)
         rescue LoadError => e

data/lib/io_streams/bzip2/writer.rb

@@ -2,7 +2,7 @@ module IOStreams
   module Bzip2
     class Writer
       # Write to a file / stream, compressing with Bzip2
-      def self.open(file_name_or_io, _=nil, &block)
+      def self.open(file_name_or_io, **args, &block)
         begin
           require 'rbzip2' unless defined?(RBzip2)
         rescue LoadError => e

data/lib/io_streams/encode/reader.rb

@@ -0,0 +1,102 @@
+module IOStreams
+  module Encode
+    class Reader
+      attr_reader :encoding, :cleaner
+
+      NOT_PRINTABLE = Regexp.compile(/[^[:print:]|\r|\n]/).freeze
+      # Builtin strip options to apply after encoding the read data.
+      CLEANSE_RULES = {
+        # Strips all non printable characters
+        printable: -> (data) { data.gsub!(NOT_PRINTABLE, '') || data }
+      }
+
+      # Read a line at a time from a file or stream
+      def self.open(file_name_or_io, **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams::File::Reader.open(file_name_or_io) { |io| yield new(io, **args) }
+        else
+          yield new(file_name_or_io, **args)
+        end
+      end
+
+      # Apply encoding conversion when reading a stream.
+      #
+      # Parameters
+      #   input_stream
+      #     The input stream that implements #read
+      #
+      #   encoding: [String|Encoding]
+      #     Encode returned data with this encoding.
+      #     'US-ASCII':   Original 7 bit ASCII Format
+      #     'ASCII-8BIT': 8-bit ASCII Format
+      #     'UTF-8':      UTF-8 Format
+      #     Etc.
+      #     Default: 'UTF-8'
+      #
+      #   encode_replace: [String]
+      #     The character to replace with when a character cannot be converted to the target encoding.
+      #     nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
+      #     Default: nil
+      #
+      #   encode_cleaner: [nil|symbol|Proc]
+      #     Cleanse data read from the input stream.
+      #     nil:           No cleansing
+      #     :printable Cleanse all non-printable characters except \r and \n
+      #     Proc/lambda    Proc to call after every read to cleanse the data
+      #     Default: nil
+      def initialize(input_stream, encoding: 'UTF-8', encode_cleaner: nil, encode_replace: nil)
+        @input_stream = input_stream
+        @cleaner      = self.class.extract_cleaner(encode_cleaner)
+
+        @encoding         = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
+        @encoding_options = encode_replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: encode_replace}
+
+        # More efficient read buffering only supported when the input stream `#read` method supports it.
+        if encode_replace.nil? && !@input_stream.method(:read).arity.between?(0, 1)
+          @read_cache_buffer = ''.encode(@encoding)
+        else
+          @read_cache_buffer = nil
+        end
+      end
+
+      # Returns [String] data returned from the input stream.
+      # Returns [nil] if end of file and no further data was read.
+      def read(size = nil)
+        block =
+          if @read_cache_buffer
+            begin
+              @input_stream.read(size, @read_cache_buffer)
+            rescue ArgumentError
+              # Handle arity of -1 when just 0..1
+              @read_cache_buffer = nil
+              @input_stream.read(size)
+            end
+          else
+            @input_stream.read(size)
+          end
+
+        # EOF reached?
+        return unless block
+
+        block = block.encode(@encoding, @encoding_options) unless block.encoding == @encoding
+        block = @cleaner.call(block) if @cleaner
+        block
+      end
+
+      private
+
+      def self.extract_cleaner(cleaner)
+        return if cleaner.nil?
+
+        case cleaner
+        when Symbol
+          proc = CLEANSE_RULES[cleaner]
+          raise(ArgumentError, "Invalid cleansing rule #{cleaner.inspect}") unless proc
+          proc
+        when Proc
+          cleaner
+        end
+      end
+    end
+  end
+end

data/lib/io_streams/encode/writer.rb

@@ -0,0 +1,78 @@
+module IOStreams
+  module Encode
+    class Writer
+      attr_reader :encoding, :cleaner
+
+      # Write a line at a time to a file or stream
+      def self.open(file_name_or_io, **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams::File::Writer.open(file_name_or_io) { |io| yield new(io, **args) }
+        else
+          yield new(file_name_or_io, **args)
+        end
+      end
+
+      # A delimited stream writer that will write to the supplied output stream
+      # Written data is encoded prior to writing.
+      #
+      # Parameters
+      #   output_stream
+      #     The output stream that implements #write
+      #
+      #   encoding: [String|Encoding]
+      #     Encode returned data with this encoding.
+      #     'US-ASCII':   Original 7 bit ASCII Format
+      #     'ASCII-8BIT': 8-bit ASCII Format
+      #     'UTF-8':      UTF-8 Format
+      #     Etc.
+      #     Default: 'UTF-8'
+      #
+      #   encode_replace: [String]
+      #     The character to replace with when a character cannot be converted to the target encoding.
+      #     nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
+      #     Default: nil
+      #
+      #   encode_cleaner: [nil|symbol|Proc]
+      #     Cleanse data read from the input stream.
+      #     nil:           No cleansing
+      #     :printable Cleanse all non-printable characters except \r and \n
+      #     Proc/lambda    Proc to call after every read to cleanse the data
+      #     Default: nil
+      def initialize(output_stream, encoding: 'UTF-8', encode_cleaner: nil, encode_replace: nil)
+        @output_stream = output_stream
+        @cleaner       = ::IOStreams::Encode::Reader.send(:extract_cleaner, encode_cleaner)
+
+        @encoding         = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
+        @encoding_options = encode_replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: encode_replace}
+      end
+
+      # Write a line to the output stream
+      #
+      # Example:
+      #   IOStreams.writer('a.txt', encoding: 'UTF-8') do |stream|
+      #     stream << 'first line' << 'second line'
+      #   end
+      def <<(record)
+        write(record)
+        self
+      end
+
+      # Write a line to the output stream followed by the delimiter.
+      # Returns [Integer] the number of bytes written.
+      #
+      # Example:
+      #   IOStreams.writer('a.txt', encoding: 'UTF-8') do |stream|
+      #     count = stream.write('first line')
+      #     puts "Wrote #{count} bytes to the output file, including the delimiter"
+      #   end
+      def write(data)
+        return 0 if data.nil?
+
+        data  = data.to_s
+        block = data.encoding == @encoding ? data : data.encode(@encoding, @encoding_options)
+        block = @cleaner.call(block) if @cleaner
+        @output_stream.write(block)
+      end
+    end
+  end
+end

data/lib/io_streams/errors.rb

@@ -0,0 +1,19 @@
+module IOStreams
+  module Errors
+    class Error < StandardError;
+    end
+
+    class InvalidHeader < Error;
+    end
+
+    class MissingHeader < Error;
+    end
+
+    class TypeMismatch < Error;
+    end
+
+    # When the specified delimiter is not found in the supplied stream / file
+    class DelimiterNotFound < Error;
+    end
+  end
+end

data/lib/io_streams/file/reader.rb

@@ -4,7 +4,7 @@ module IOStreams
       # Read from a named file
       # TODO: Add support for mode (text / binary)
       # TODO: Add encoding support: external_encoding, internal_encoding
-      def self.open(file_name, _=nil, &block)
+      def self.open(file_name, **args, &block)
         raise(ArgumentError, 'File name must be a string') unless file_name.is_a?(String)
 
         ::File.open(file_name, 'rb', &block)

data/lib/io_streams/file/writer.rb

@@ -2,9 +2,7 @@ module IOStreams
   module File
     class Writer
       # Write to a named file
-      # TODO: Add support for mode (text / binary), permissions, buffering, append
-      # TODO: Add encoding support: external_encoding, internal_encoding
-      def self.open(file_name, _=nil, &block)
+      def self.open(file_name, **args, &block)
         raise(ArgumentError, 'File name must be a string') unless file_name.is_a?(String)
 
         ::File.open(file_name, 'wb', &block)

data/lib/io_streams/gzip/reader.rb

@@ -2,7 +2,7 @@ module IOStreams
   module Gzip
     class Reader
       # Read from a gzip file or stream, decompressing the contents as it is read
-      def self.open(file_name_or_io, _=nil, &block)
+      def self.open(file_name_or_io, **args, &block)
         unless IOStreams.reader_stream?(file_name_or_io)
           ::Zlib::GzipReader.open(file_name_or_io, &block)
         else

data/lib/io_streams/gzip/writer.rb

@@ -2,7 +2,7 @@ module IOStreams
   module Gzip
     class Writer
       # Write to a file / stream, compressing with GZip
-      def self.open(file_name_or_io, _=nil, &block)
+      def self.open(file_name_or_io, **args, &block)
         unless IOStreams.writer_stream?(file_name_or_io)
           Zlib::GzipWriter.open(file_name_or_io, &block)
         else

data/lib/io_streams/io_streams.rb

@@ -1,4 +1,9 @@
 require 'concurrent'
+# Load Symmetric Encryption if present so that its reader and writer can be registered
+begin
+  require 'symmetric-encryption'
+rescue LoadError
+end
 
 # Streaming library for Ruby
 #
@@ -60,20 +65,20 @@ module IOStreams
   # Note:
   # * Passes the file_name_or_io as-is into the block if it is already a reader stream AND
   #   no streams are passed in.
-  def self.reader(file_name_or_io, streams: nil, file_name: nil, &block)
-    stream(:reader, file_name_or_io, streams: streams, file_name: file_name, &block)
+  def self.reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
+    stream(:reader, file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, &block)
   end
 
   # Iterate over a file / stream returning one line at a time.
-  def self.each_line(file_name_or_io, **args, &block)
-    line_reader(file_name_or_io, **args) do |line_stream|
+  def self.each_line(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+    line_reader(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, **args) do |line_stream|
       line_stream.each(&block)
     end
   end
 
   # Iterate over a file / stream returning one line at a time.
-  def self.each_row(file_name_or_io, **args, &block)
-    row_reader(file_name_or_io, **args) do |row_stream|
+  def self.each_row(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+    row_reader(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, **args) do |row_stream|
       row_stream.each(&block)
     end
   end
@@ -90,8 +95,8 @@ module IOStreams
   #   IOStreams.each_record(file_name) do |hash|
   #     p hash
   #   end
-  def self.each_record(file_name_or_io, **args, &block)
-    record_reader(file_name_or_io, **args) do |record_stream|
+  def self.each_record(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+    record_reader(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, **args) do |record_stream|
       record_stream.each(&block)
     end
   end
@@ -148,32 +153,32 @@ module IOStreams
   # Note:
   # * Passes the file_name_or_io as-is into the block if it is already a writer stream AND
   #   no streams are passed in.
-  def self.writer(file_name_or_io, streams: nil, file_name: nil, &block)
-    stream(:writer, file_name_or_io, streams: streams, file_name: file_name, &block)
+  def self.writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
+    stream(:writer, file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace, &block)
   end
 
-  def self.line_writer(file_name_or_io, streams: nil, file_name: nil, **args, &block)
+  def self.line_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
     return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Line::Writer) || file_name_or_io.is_a?(Array)
 
-    writer(file_name_or_io, streams: streams, file_name: file_name) do |io|
+    writer(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace) do |io|
       IOStreams::Line::Writer.open(io, **args, &block)
     end
   end
 
-  def self.row_writer(file_name_or_io, streams: nil, file_name: nil, **args, &block)
+  def self.row_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
     return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Row::Writer)
 
-    line_writer(file_name_or_io, streams: streams, file_name: file_name) do |io|
+    line_writer(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace) do |io|
       file_name = file_name_or_io if file_name.nil? && file_name_or_io.is_a?(String)
 
       IOStreams::Row::Writer.open(io, file_name: file_name, **args, &block)
     end
   end
 
-  def self.record_writer(file_name_or_io, streams: nil, file_name: nil, **args, &block)
+  def self.record_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
     return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Record::Writer)
 
-    line_writer(file_name_or_io, streams: streams, file_name: file_name) do |io|
+    line_writer(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace) do |io|
       file_name = file_name_or_io if file_name.nil? && file_name_or_io.is_a?(String)
 
       IOStreams::Record::Writer.open(io, file_name: file_name, **args, &block)
@@ -185,7 +190,6 @@ module IOStreams
   #
   # Example: Copy between 2 files
   #   IOStreams.copy('a.csv', 'b.csv')
-  #   # TODO: The above will convert the csv file to a Hash and then back to write it to the target file.
   #
   # Example: Read content from a Xlsx file and write it out in CSV form.
   #   IOStreams.copy('a.xlsx', 'b.csv')
@@ -311,23 +315,23 @@ module IOStreams
   end
 
   # Iterate over a file / stream returning each record/line one at a time.
-  def self.line_reader(file_name_or_io, streams: nil, file_name: nil, **args, &block)
+  def self.line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
     return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Line::Reader) ||
-      file_name_or_io.is_a?(IOStreams::Xlsx::Reader) ||
       file_name_or_io.is_a?(Array)
 
-    reader(file_name_or_io, streams: streams, file_name: file_name) do |io|
+    reader(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace) do |io|
       IOStreams::Line::Reader.open(io, **args, &block)
     end
   end
 
-  # Iterate over a file / stream returning each line as a hash, one at a time.
+  # Iterate over a file / stream returning each line as an array, one at a time.
   def self.row_reader(file_name_or_io,
     streams: nil,
     delimiter: nil,
-    encoding: IOStreams::UTF8_ENCODING,
-    strip_non_printable: false,
     file_name: nil,
+    encoding: nil,
+    encode_cleaner: nil,
+    encode_replace: nil,
     **args,
     &block)
 
@@ -335,12 +339,13 @@ module IOStreams
 
     line_reader(
       file_name_or_io,
-      streams:             streams,
-      delimiter:           delimiter,
-      encoding:            encoding,
-      strip_non_printable: strip_non_printable,
-      file_name:           file_name) do |io|
-
+      streams:        streams,
+      delimiter:      delimiter,
+      file_name:      file_name,
+      encoding:       encoding,
+      encode_cleaner: encode_cleaner,
+      encode_replace: encode_replace
+    ) do |io|
       file_name = file_name_or_io if file_name.nil? && file_name_or_io.is_a?(String)
       IOStreams::Row::Reader.open(io, file_name: file_name, **args, &block)
     end
@@ -350,21 +355,24 @@ module IOStreams
   def self.record_reader(file_name_or_io,
     streams: nil,
     delimiter: nil,
-    encoding: IOStreams::UTF8_ENCODING,
-    strip_non_printable: false,
     file_name: nil,
+    encoding: nil,
+    encode_cleaner: nil,
+    encode_replace: nil,
     **args,
     &block)
 
-    return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Record::Reader) || file_name_or_io.is_a?(IOStreams::Xlsx::Reader)
+    return yield(file_name_or_io) if file_name_or_io.is_a?(IOStreams::Record::Reader)
 
     line_reader(
       file_name_or_io,
-      streams:             streams,
-      delimiter:           delimiter,
-      encoding:            encoding,
-      strip_non_printable: strip_non_printable,
-      file_name:           file_name) do |io|
+      streams:        streams,
+      delimiter:      delimiter,
+      file_name:      file_name,
+      encoding:       encoding,
+      encode_cleaner: encode_cleaner,
+      encode_replace: encode_replace
+    ) do |io|
 
       file_name = file_name_or_io if file_name.nil? && file_name_or_io.is_a?(String)
       IOStreams::Record::Reader.open(io, file_name: file_name, **args, &block)
@@ -414,7 +422,7 @@ module IOStreams
   StreamStruct = Struct.new(:klass, :options)
 
   # Returns a reader or writer stream
-  def self.stream(type, file_name_or_io, streams:, file_name:, &block)
+  def self.stream(type, file_name_or_io, streams:, file_name:, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
     # TODO: Add support for different schemes, such as file://, s3://, sftp://
 
     streams = streams_for_file_name(file_name) if streams.nil? && file_name
@@ -427,6 +435,17 @@ module IOStreams
     end
 
     stream_structs = streams_for(type, streams)
+
+    # Add encoding stream if any of its options are present
+    if encoding || encode_cleaner || encode_replace
+      klass                    = type == :reader ? IOStreams::Encode::Reader : IOStreams::Encode::Writer
+      options                  = {}
+      options[:encoding]       = encoding if encoding
+      options[:encode_cleaner] = encode_cleaner if encode_cleaner
+      options[:encode_replace] = encode_replace if encode_replace
+      stream_structs.unshift(StreamStruct.new(klass, options))
+    end
+
     if stream_structs.size == 1
       stream_struct = stream_structs.first
       stream_struct.klass.open(file_name_or_io, stream_struct.options, &block)