iostreams 0.14.0 → 0.15.0

data/lib/io_streams/{delimited → line}/reader.rb

@@ -1,16 +1,14 @@
 module IOStreams
-  module Delimited
+  module Line
     class Reader
-      attr_accessor :delimiter, :buffer_size, :encoding, :strip_non_printable
+      attr_reader :delimiter, :buffer_size, :encoding, :strip_non_printable
 
-      # Read from a file or stream
-      def self.open(file_name_or_io, delimiter: nil, buffer_size: 65536, encoding: UTF8_ENCODING, strip_non_printable: false)
-        if IOStreams.reader_stream?(file_name_or_io)
-          yield new(file_name_or_io, delimiter: delimiter, buffer_size: buffer_size, encoding: encoding, strip_non_printable: strip_non_printable)
+      # Read a line at a time from a file or stream
+      def self.open(file_name_or_io, **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams::File::Reader.open(file_name_or_io) { |io| yield new(io, **args) }
         else
-          ::File.open(file_name_or_io, 'rb') do |io|
-            yield new(io, delimiter: delimiter, buffer_size: buffer_size, encoding: encoding, strip_non_printable: strip_non_printable)
-          end
+          yield new(file_name_or_io, **args)
         end
       end
 
@@ -47,6 +45,12 @@ module IOStreams
       #     Force encoding to this encoding for all data being read
       #     Default: UTF8_ENCODING
       #     Set to nil to disable encoding
+      #
+      # TODO:
+      # - Skip Comment lines. RegExp?
+      # - Skip "empty" / "blank" lines. RegExp?
+      # - Extract header line(s) / first non-comment, non-blank line
+      # - Embedded newline support, RegExp? or Proc?
       def initialize(input_stream, delimiter: nil, buffer_size: 65536, encoding: UTF8_ENCODING, strip_non_printable: false)
         @input_stream        = input_stream
         @delimiter           = delimiter
@@ -54,11 +58,12 @@ module IOStreams
         @encoding            = encoding
         @strip_non_printable = strip_non_printable
 
-        @delimiter.force_encoding(UTF8_ENCODING) if @delimiter && @encoding
+        @delimiter.encode(UTF8_ENCODING) if @delimiter && @encoding
         @buffer = ''
       end
 
-      # Returns each line at a time to to the supplied block
+      # Iterate over every line in the file/stream passing each line to supplied block in turn.
+      # Returns [Integer] the number of lines read from the file/stream.
       def each(&block)
         partial = nil
         loop do
@@ -83,32 +88,17 @@ module IOStreams
         end
       end
 
-      alias_method :each_line, :each
-
-      # Reads length bytes from the I/O stream.
-      # Not recommended, but available if someone calls #read on this delimited reader
-      def read(length = nil, outbuf = nil)
-        if length
-          while (@buffer.size < length) && (read_chunk > 0)
-          end
-          data = @buffer.slice!(0, length)
-          outbuf << data if outbuf
-          data
-        else
-          while read_chunk > 0
-          end
-          @buffer
-        end
-      end
-
-      ##########################################################################
       private
 
+      attr_reader :buffer
+      attr_writer :delimiter
+
       NOT_PRINTABLE = Regexp.compile(/[^[:print:]|\r|\n]/)
 
       # Returns [Integer] the number of bytes read into the internal buffer
       # Returns 0 on EOF
       def read_chunk
+        # TODO: read into existing buffer
         chunk = @input_stream.read(@buffer_size)
         # EOF reached?
         return 0 unless chunk

data/lib/io_streams/line/writer.rb

@@ -0,0 +1,81 @@
+module IOStreams
+  module Line
+    class Writer
+      attr_reader :delimiter, :encoding, :strip_non_printable
+
+      # Write a line at a time to a file or stream
+      def self.open(file_name_or_io, **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams::File::Writer.open(file_name_or_io) { |io| yield new(io, **args) }
+        else
+          yield new(file_name_or_io, **args)
+        end
+      end
+
+      NOT_PRINTABLE = Regexp.compile(/[^[:print:]]/)
+
+      # A delimited stream writer that will write to the supplied output stream
+      #
+      # The output stream should be binary with no text conversions performed
+      # since `strip_non_printable` will be applied to the binary stream before
+      # converting to UTF-8
+      #
+      # Parameters
+      #   output_stream
+      #     The output stream that implements #write
+      #
+      #   delimiter: [String]
+      #     Add the specified delimiter after every record when writing it
+      #     to the output stream
+      #     Default: OS Specific. Linux: "\n"
+      #
+      #   encoding:
+      #     Encode data before writing to the output stream.
+      #     Default: UTF8_ENCODING
+      #     Set to nil to disable encoding
+      #
+      #   strip_non_printable: [true|false]
+      #     Strip all non-printable characters before writing to the file / stream.
+      #     Default: false
+      #
+      # TODO: Support replacement character for invalid characters
+      def initialize(output_stream, delimiter: $/, encoding: UTF8_ENCODING, strip_non_printable: false)
+        @output_stream       = output_stream
+        @delimiter           = delimiter.encode(encoding) if delimiter && encoding
+        @encoding            = encoding
+        @strip_non_printable = strip_non_printable
+      end
+
+      # Write a line to the output stream
+      #
+      # Example:
+      #   IOStreams.line_writer('a.txt') do |stream|
+      #     stream << 'first line' << 'second line'
+      #   end
+      def <<(record)
+        write(record)
+        self
+      end
+
+      # Write a line to the output stream followed by the delimiter.
+      # Returns [Integer] the number of bytes written.
+      #
+      # Example:
+      #   IOStreams.line_writer('a.txt') do |stream|
+      #     count = stream.write('first line')
+      #     puts "Wrote #{count} bytes to the output file, including the delimiter"
+      #   end
+      def write(record)
+        chunk = record.to_s
+        chunk.gsub!(NOT_PRINTABLE, '') if strip_non_printable
+        count = output_stream.write((encoding ? chunk.encode(encoding) : chunk))
+        count += output_stream.write(delimiter) if delimiter
+        count
+      end
+
+      private
+
+      attr_reader :output_stream
+    end
+  end
+end

data/lib/io_streams/pgp.rb

@@ -232,31 +232,21 @@ module IOStreams
       end
     end
 
-    # Returns [String] containing all the keys for the supplied email address.
+    # Returns [String] containing all the public keys for the supplied email address.
     #
     # email: [String] Email address for requested key.
     #
     # ascii: [true|false]
     #   Whether to export as ASCII text instead of binary format
     #   Default: true
-    #
-    # private: [true|false]
-    #   Whether to export the private key
-    #   Default: false
-    #
-    # passphrase: [String]
-    #   In order to export a private key the passphrase for the key must be supplied.
-    #   Otherwise a `Inappropriate ioctl for device` error will be returned.
-    def self.export(email:, passphrase: nil, ascii: true, private: false)
+    def self.export(email:, ascii: true)
       version_check
-      raise(ArgumentError, "Missing keyword: passphrase when private: true") if private && passphrase.nil?
 
       armor    = ascii ? '--armor' : nil
-      cmd      = private ? '--export-secret-keys' : '--export'
       loopback = pgp_version.to_f >= 2.1 ? '--pinentry-mode loopback' : ''
-      command  = "#{executable} #{loopback} --no-tty --passphrase-fd 0 --batch #{armor} #{cmd} #{email}"
+      command  = "#{executable} #{loopback} --no-tty --passphrase-fd 0 --batch #{armor} --export #{email}"
 
-      out, err, status = Open3.capture3(command, binmode: true, stdin_data: "#{passphrase}\n")
+      out, err, status = Open3.capture3(command, binmode: true)
       logger.debug { "IOStreams::Pgp.export: #{command}\n#{err}" } if logger
       if status.success? && out.length > 0
         out

data/lib/io_streams/record/reader.rb

@@ -0,0 +1,55 @@
+module IOStreams
+  module Record
+    # Converts each line of an input stream into hash for every row
+    class Reader
+      include Enumerable
+
+      # Read a record as a Hash at a time from a file or stream.
+      def self.open(file_name_or_io, delimiter: nil, buffer_size: 65536, encoding: UTF8_ENCODING, strip_non_printable: false, **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams.line_reader(file_name_or_io,
+                                delimiter:           delimiter,
+                                buffer_size:         buffer_size,
+                                encoding:            encoding,
+                                strip_non_printable: strip_non_printable) do |io|
+            yield new(io, file_name: file_name_or_io, **args)
+          end
+        else
+          yield new(file_name_or_io, **args)
+        end
+      end
+
+      # Create a Tabular reader to return the stream as Hash records
+      # Parse a delimited data source.
+      #
+      # Parameters
+      #   delimited: [#each]
+      #     Anything that returns one line / record at a time when #each is called on it.
+      #
+      #   format: [Symbol]
+      #     :csv, :hash, :array, :json, :psv, :fixed
+      #
+      #   For all other parameters, see Tabular::Header.new
+      def initialize(delimited, cleanse_header: true, **args)
+        @tabular        = IOStreams::Tabular.new(**args)
+        @delimited      = delimited
+        @cleanse_header = cleanse_header
+      end
+
+      def each
+        delimited.each do |line|
+          if tabular.requires_header?
+            tabular.parse_header(line)
+            tabular.cleanse_header! if cleanse_header
+          else
+            yield tabular.record_parse(line)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :tabular, :delimited, :cleanse_header
+    end
+  end
+end

data/lib/io_streams/record/writer.rb

@@ -0,0 +1,63 @@
+module IOStreams
+  module Record
+    # Example, implied header from first record:
+    #   IOStreams.record_writer do |stream|
+    #     stream << {name: 'Jack', address: 'Somewhere', zipcode: 12345}
+    #     stream << {name: 'Joe', address: 'Lost', zipcode: 32443, age: 23}
+    #   end
+    #
+    # Output:
+    #   name, add
+    #
+    class Writer
+      # Write a record as a Hash at a time to a file or stream.
+      def self.open(file_name_or_io, delimiter: $/, encoding: UTF8_ENCODING, strip_non_printable: false, **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams.line_writer(file_name_or_io,
+                                delimiter:           delimiter,
+                                encoding:            encoding,
+                                strip_non_printable: strip_non_printable) do |io|
+            yield new(io, file_name: file_name_or_io, **args)
+          end
+        else
+          yield new(file_name_or_io, **args)
+        end
+      end
+
+      # Create a Tabular writer that takes individual
+      # Parse a delimited data source.
+      #
+      # Parameters
+      #   delimited: [#<<]
+      #     Anything that accepts a line / record at a time when #<< is called on it.
+      #
+      #   format: [Symbol]
+      #     :csv, :hash, :array, :json, :psv, :fixed
+      #
+      #   For all other parameters, see Tabular::Header.new
+      #
+      #     columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
+      def initialize(delimited, columns: nil, **args)
+        @tabular   = IOStreams::Tabular.new(columns: columns, **args)
+        @delimited = delimited
+
+        # Render header line when `columns` is supplied.
+        delimited << @tabular.render(columns) if columns && @tabular.requires_header?
+      end
+
+      def <<(hash)
+        raise(ArgumentError, 'Must supply a Hash') unless hash.is_a?(Hash)
+        if tabular.requires_header?
+          columns                = hash.keys
+          tabular.header.columns = columns
+          delimited << tabular.render(columns)
+        end
+        delimited << tabular.render(hash)
+      end
+
+      private
+
+      attr_reader :tabular, :delimited, :cleanse_header
+    end
+  end
+end

data/lib/io_streams/row/reader.rb

@@ -0,0 +1,60 @@
+module IOStreams
+  module Row
+    # Converts each line of an input stream into an array for every line
+    class Reader
+      # Read a line as an Array at a time from a file or stream.
+      def self.open(file_name_or_io,
+        delimiter: nil,
+        buffer_size: 65_536,
+        encoding: UTF8_ENCODING,
+        strip_non_printable: false,
+        file_name: nil,
+        **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams.line_reader(file_name_or_io,
+                                delimiter:           delimiter,
+                                buffer_size:         buffer_size,
+                                encoding:            encoding,
+                                file_name:           file_name,
+                                strip_non_printable: strip_non_printable) do |io|
+            yield new(io, file_name: file_name, **args)
+          end
+        else
+          yield new(file_name_or_io, **args)
+        end
+      end
+
+      # Create a Tabular reader to return the stream rows as arrays.
+      #
+      # Parameters
+      #   delimited: [#each]
+      #     Anything that returns one line / record at a time when #each is called on it.
+      #
+      #   format: [Symbol]
+      #     :csv, :hash, :array, :json, :psv, :fixed
+      #
+      #   For all other parameters, see Tabular::Header.new
+      def initialize(delimited, cleanse_header: true, **args)
+        @tabular        = IOStreams::Tabular.new(**args)
+        @delimited      = delimited
+        @cleanse_header = cleanse_header
+      end
+
+      def each
+        delimited.each do |line|
+          if tabular.requires_header?
+            columns = tabular.parse_header(line)
+            tabular.cleanse_header! if cleanse_header
+            yield columns
+          else
+            yield tabular.row_parse(line)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :tabular, :delimited, :cleanse_header
+    end
+  end
+end

data/lib/io_streams/row/writer.rb

@@ -0,0 +1,62 @@
+require 'csv'
+module IOStreams
+  module Row
+    # Example:
+    #   IOStreams.row_writer do |stream|
+    #     stream << ['name', 'address', 'zipcode']
+    #     stream << ['Jack', 'Somewhere', 12345]
+    #     stream << ['Joe', 'Lost', 32443]
+    #   end
+    #
+    # Output:
+    #   ...
+    #
+    class Writer
+      # Write a record as a Hash at a time to a file or stream.
+      def self.open(file_name_or_io, delimiter: $/, encoding: UTF8_ENCODING, strip_non_printable: false, **args)
+        if file_name_or_io.is_a?(String)
+          IOStreams.line_writer(file_name_or_io,
+                                delimiter:           delimiter,
+                                encoding:            encoding,
+                                strip_non_printable: strip_non_printable) do |io|
+            yield new(io, **args)
+          end
+        else
+          yield new(file_name_or_io, **args)
+        end
+      end
+
+      # Create a Tabular writer that takes individual rows as arrays.
+      #
+      # Parameters
+      #   delimited: [#<<]
+      #     Anything that accepts a line / record at a time when #<< is called on it.
+      #
+      #   format: [Symbol]
+      #     :csv, :hash, :array, :json, :psv, :fixed
+      #
+      #   For all other parameters, see Tabular::Header.new
+      #
+      #     columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
+      def initialize(delimited, columns: nil, **args)
+        @tabular   = IOStreams::Tabular.new(columns: columns, **args)
+        @delimited = delimited
+
+        # Render header line when `columns` is supplied.
+        delimited << @tabular.render(columns) if columns && @tabular.requires_header?
+      end
+
+      # Supply a hash or an array to render
+      def <<(array)
+        raise(ArgumentError, 'Must supply an Array') unless array.is_a?(Array)
+        # If header (columns) was not supplied as an argument, assume first line is the header.
+        tabular.header.columns = array if tabular.requires_header?
+        delimited << tabular.render(array)
+      end
+
+      private
+
+      attr_reader :tabular, :delimited
+    end
+  end
+end