iostreams 1.5.0 → 1.7.0

data/lib/io_streams/pgp.rb

@@ -48,8 +48,8 @@ module IOStreams
     # See `man gpg` for the remaining options
     def self.generate_key(name:,
                           email:,
-                          comment: nil,
                           passphrase:,
+                          comment: nil,
                           key_type: "RSA",
                           key_length: 4096,
                           subkey_type: "RSA",
@@ -261,7 +261,7 @@ module IOStreams
 
       import(key: key)
       set_trust(email: email, key_id: key_id)
-      email
+      email || key_id
     end
 
     # Set the trust level for an existing key.
@@ -291,10 +291,8 @@ module IOStreams
       version_check
       Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
         output = out.read.chomp
-        unless waith_thr.value.success?
-          unless output =~ /(public key not found|No public key)/i
-            raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
-          end
+        if !waith_thr.value.success? && !(output !~ /(public key not found|No public key)/i)
+          raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
         end
 
         output.each_line do |line|
@@ -336,9 +334,11 @@ module IOStreams
             match[1]
           end
         else
-          return [] if err =~ /(key not found|No (public|secret) key)/i
+          if err !~ /(key not found|No (public|secret) key)/i
+            raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email || key_id}: #{err}#{out}")
+          end
 
-          raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email || key_id}: #{err}#{out}")
+          []
         end
       end
     end
@@ -350,7 +350,7 @@ module IOStreams
     end
 
     def self.version_check
-      return unless pgp_version.to_f >= 2.3
+      return unless pgp_version.to_f >= 2.4
 
       raise(
         Pgp::UnsupportedVersion,
@@ -382,10 +382,10 @@ module IOStreams
             key_length: match[3].to_s.to_i,
             key_type:   match[2],
             date:       (begin
-                           Date.parse(match[4].to_s)
-                         rescue StandardError
-                           match[4]
-                         end)
+              Date.parse(match[4].to_s)
+            rescue StandardError
+              match[4]
+            end)
           }
         elsif (match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?}))
           # Matches: pub  2048R/C7F9D9CB 2016-10-26
@@ -396,10 +396,10 @@ module IOStreams
             key_type:   match[3],
             key_id:     match[4],
             date:       (begin
-                           Date.parse(match[5].to_s)
-                         rescue StandardError
-                           match[5]
-                         end)
+              Date.parse(match[5].to_s)
+            rescue StandardError
+              match[5]
+            end)
           }
           # Prior to gpg v2.0.30
           if match[7]

data/lib/io_streams/pgp/writer.rb

@@ -19,8 +19,7 @@ module IOStreams
 
         private
 
-        attr_reader :default_signer_passphrase
-        attr_reader :default_signer
+        attr_reader :default_signer_passphrase, :default_signer
 
         @default_signer_passphrase = nil
         @default_signer            = nil

data/lib/io_streams/stream.rb

@@ -151,6 +151,9 @@ module IOStreams
     #     Whether to apply the stream conversions during the copy.
     #     Default: true
     #
+    #   :mode [:line, :array, :hash]
+    #     When convert is `true` then use this mode to convert the contents of the file.
+    #
     # Examples:
     #
     # # Copy and convert streams based on file extensions
@@ -162,11 +165,17 @@ module IOStreams
     # # Advanced copy with custom stream conversions on source and target.
     # source = IOStreams.path("source_file").stream(encoding: "BINARY")
     # IOStreams.path("target_file.pgp").option(:pgp, passphrase: "hello").copy_from(source)
-    def copy_from(source, convert: true)
+    def copy_from(source, convert: true, mode: nil, **args)
       if convert
         stream = IOStreams.new(source)
-        writer do |target|
-          stream.reader { |src| IO.copy_stream(src, target) }
+        if mode
+          writer(mode, **args) do |target|
+            stream.each(mode) { |row| target << row }
+          end
+        else
+          writer(**args) do |target|
+            stream.reader { |src| IO.copy_stream(src, target) }
+          end
         end
       else
         stream = source.is_a?(Stream) ? source.dup : IOStreams.new(source)
@@ -176,9 +185,9 @@ module IOStreams
       end
     end
 
-    def copy_to(target, convert: true)
+    def copy_to(target, **args)
       target = IOStreams.new(target)
-      target.copy_from(self, convert: convert)
+      target.copy_from(self, **args)
     end
 
     # Set/get the original file_name
@@ -191,11 +200,41 @@ module IOStreams
       end
     end
 
-    # Set/get the original file_name
+    # Set the original file_name
     def file_name=(file_name)
       builder.file_name = file_name
     end
 
+    # Set/get the tabular format_options
+    def format(format = :none)
+      if format == :none
+        builder.format
+      else
+        builder.format = format
+        self
+      end
+    end
+
+    # Set the tabular format
+    def format=(format)
+      builder.format = format
+    end
+
+    # Set/get the tabular format options
+    def format_options(format_options = :none)
+      if format_options == :none
+        builder.format_options
+      else
+        builder.format_options = format_options
+        self
+      end
+    end
+
+    # Set the tabular format_options
+    def format_options=(format_options)
+      builder.format_options = format_options
+    end
+
     # Returns [String] the last component of this path.
     # Returns `nil` if no `file_name` was set.
     #
@@ -293,14 +332,26 @@ module IOStreams
     # Iterate over a file / stream returning each line as an array, one at a time.
     def row_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Row::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Row::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
     # Iterate over a file / stream returning each line as a hash, one at a time.
     def record_reader(delimiter: nil, embedded_within: nil, **args)
       line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
-        yield IOStreams::Record::Reader.new(io, original_file_name: builder.file_name, **args)
+        yield IOStreams::Record::Reader.new(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args
+        )
       end
     end
 
@@ -320,7 +371,14 @@ module IOStreams
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
 
       line_writer(delimiter: delimiter) do |io|
-        IOStreams::Row::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Row::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args,
+          &block
+        )
       end
     end
 
@@ -328,7 +386,14 @@ module IOStreams
       return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
 
       line_writer(delimiter: delimiter) do |io|
-        IOStreams::Record::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
+        IOStreams::Record::Writer.stream(
+          io,
+          original_file_name: builder.file_name,
+          format:             builder.format,
+          format_options:     builder.format_options,
+          **args,
+          &block
+        )
       end
     end
   end

data/lib/io_streams/tabular.rb

@@ -52,7 +52,7 @@ module IOStreams
     #   format: [Symbol]
     #     :csv, :hash, :array, :json, :psv, :fixed
     #
-    #   file_name: [String]
+    #   file_name: [IOStreams::Path | String]
     #     When `:format` is not supplied the file name can be used to infer the required format.
     #     Optional. Default: nil
     #
@@ -81,14 +81,19 @@ module IOStreams
     #       #as_hash will skip these additional columns entirely as if they were not in the file at all.
     #     false:
     #       Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
-    def initialize(format: nil, file_name: nil, format_options: nil, **args)
+    #
+    #   default_format: [Symbol]
+    #     When the format is not supplied, and the format cannot be inferred from the supplied file name
+    #     then this default format will be used.
+    #     Default: :csv
+    #     Set to nil to force it to raise an exception when the format is undefined.
+    def initialize(format: nil, file_name: nil, format_options: nil, default_format: :csv, **args)
       @header = Header.new(**args)
-      klass   =
-        if file_name && format.nil?
-          self.class.parser_class_for_file_name(file_name)
-        else
-          self.class.parser_class(format)
-        end
+      @format = file_name && format.nil? ? self.class.format_from_file_name(file_name) : format
+      @format ||= default_format
+      raise(UnknownFormat, "The format cannot be inferred from the file name: #{file_name}") unless @format
+
+      klass   = self.class.parser_class(@format)
       @parser = format_options ? klass.new(**format_options) : klass.new
     end
 
@@ -162,9 +167,9 @@ module IOStreams
     # Example:
     #   register_format(:csv, IOStreams::Tabular::Parser::Csv)
     def self.register_format(format, parser)
-      raise(ArgumentError, "Invalid format #{format.inspect}") unless format.nil? || format.to_s =~ /\A\w+\Z/
+      raise(ArgumentError, "Invalid format #{format.inspect}") unless format.to_s =~ /\A\w+\Z/
 
-      @formats[format.nil? ? nil : format.to_sym] = parser
+      @formats[format.to_sym] = parser
     end
 
     # De-Register a file format
@@ -187,23 +192,18 @@ module IOStreams
     # A registry to hold formats for processing files during upload or download
     @formats = {}
 
-    def self.parser_class(format)
-      @formats[format.nil? ? nil : format.to_sym] || raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
+    # Returns the registered format that will be used for the supplied file name.
+    def self.format_from_file_name(file_name)
+      file_name.to_s.split(".").reverse_each { |ext| return ext.to_sym if @formats.include?(ext.to_sym) }
+      nil
     end
 
-    # Returns the parser to use with tabular for the supplied file_name
-    def self.parser_class_for_file_name(file_name)
-      format = nil
-      file_name.to_s.split(".").reverse_each do |ext|
-        if @formats.include?(ext.to_sym)
-          format = ext.to_sym
-          break
-        end
-      end
-      parser_class(format)
+    # Returns the parser class for the registered format.
+    def self.parser_class(format)
+      @formats[format.nil? ? nil : format.to_sym] ||
+        raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
     end
 
-    register_format(nil, IOStreams::Tabular::Parser::Csv)
     register_format(:array, IOStreams::Tabular::Parser::Array)
     register_format(:csv, IOStreams::Tabular::Parser::Csv)
     register_format(:fixed, IOStreams::Tabular::Parser::Fixed)

data/lib/io_streams/tabular/parser/csv.rb

@@ -5,8 +5,10 @@ module IOStreams
       class Csv < Base
         attr_reader :csv_parser
 
-        def initialize
-          @csv_parser = Utility::CSVRow.new unless RUBY_VERSION.to_f >= 2.6
+        unless RUBY_VERSION.to_f >= 2.6
+          def initialize
+            @csv_parser = Utility::CSVRow.new
+          end
         end
 
         # Returns [Array<String>] the header row.

data/lib/io_streams/tabular/parser/fixed.rb

@@ -146,7 +146,7 @@ module IOStreams
 
           attr_reader :key, :size, :type, :decimals
 
-          def initialize(key: nil, size:, type: :string, decimals: 2)
+          def initialize(size:, key: nil, type: :string, decimals: 2)
             @key      = key
             @size     = size == :remainder ? -1 : size.to_i
             @type     = type.to_sym

data/lib/io_streams/tabular/utility/csv_row.rb

@@ -6,10 +6,7 @@ module IOStreams
       # 2 to 3 times better performance than CSV.parse_line and considerably less
       # garbage collection required.
       #
-      # Note:
-      #   This parser does not support line feeds embedded in quoted fields since
-      #   the file is broken apart based on line feeds during the upload process and
-      #   is then processed by each worker on a line by line basis.
+      # Note: Only used prior to Ruby 2.6
       class CSVRow < ::CSV
         UTF8_ENCODING = Encoding.find("UTF-8").freeze
 

data/lib/io_streams/utils.rb

@@ -28,11 +28,9 @@ module IOStreams
     def self.temp_file_name(basename, extension = "")
       result = nil
       ::Dir::Tmpname.create([basename, extension], IOStreams.temp_dir, max_try: MAX_TEMP_FILE_NAME_ATTEMPTS) do |tmpname|
-        begin
-          result = yield(tmpname)
-        ensure
-          ::File.unlink(tmpname) if ::File.exist?(tmpname)
-        end
+        result = yield(tmpname)
+      ensure
+        ::File.unlink(tmpname) if ::File.exist?(tmpname)
       end
       result
     end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = "1.5.0".freeze
+  VERSION = "1.7.0".freeze
 end

data/lib/iostreams.rb

@@ -23,33 +23,41 @@ module IOStreams
     autoload :Reader, "io_streams/bzip2/reader"
     autoload :Writer, "io_streams/bzip2/writer"
   end
+
   module Encode
     autoload :Reader, "io_streams/encode/reader"
     autoload :Writer, "io_streams/encode/writer"
   end
+
   module Gzip
     autoload :Reader, "io_streams/gzip/reader"
     autoload :Writer, "io_streams/gzip/writer"
   end
+
   module Line
     autoload :Reader, "io_streams/line/reader"
     autoload :Writer, "io_streams/line/writer"
   end
+
   module Record
     autoload :Reader, "io_streams/record/reader"
     autoload :Writer, "io_streams/record/writer"
   end
+
   module Row
     autoload :Reader, "io_streams/row/reader"
     autoload :Writer, "io_streams/row/writer"
   end
+
   module SymmetricEncryption
     autoload :Reader, "io_streams/symmetric_encryption/reader"
     autoload :Writer, "io_streams/symmetric_encryption/writer"
   end
+
   module Xlsx
     autoload :Reader, "io_streams/xlsx/reader"
   end
+
   module Zip
     autoload :Reader, "io_streams/zip/reader"
     autoload :Writer, "io_streams/zip/writer"

data/test/builder_test.rb

@@ -41,6 +41,35 @@ class BuilderTest < Minitest::Test
       end
     end
 
+    describe "#format" do
+      it "detects the format from the file name" do
+        streams = IOStreams::Builder.new("abc.json")
+        assert_equal :json, streams.format
+      end
+
+      it "is nil if the file name has no meaningful format" do
+        assert_nil streams.format
+      end
+
+      it "returns set format with no file_name" do
+        streams        = IOStreams::Builder.new
+        streams.format = :csv
+        assert_equal :csv, streams.format
+      end
+
+      it "returns set format with file_name" do
+        streams        = IOStreams::Builder.new("abc.json")
+        streams.format = :csv
+        assert_equal :csv, streams.format
+      end
+
+      it "validates bad format" do
+        assert_raises ArgumentError do
+          streams.format = :blah
+        end
+      end
+    end
+
     describe "#stream" do
       it "adds one stream" do
         streams.stream(:pgp, passphrase: "unlock-me")