iostreams 0.20.3 → 1.0.0.beta

data/lib/io_streams/pgp/reader.rb

@@ -2,34 +2,31 @@ require 'open3'
 
 module IOStreams
   module Pgp
-    class Reader
+    class Reader < IOStreams::Reader
       # Passphrase to use to open the private key to decrypt the received file
-      def self.default_passphrase=(default_passphrase)
-        @default_passphrase = default_passphrase
+      class << self
+        attr_writer :default_passphrase
+
+        private
+
+        attr_reader :default_passphrase
+
+        @default_passphrase = nil
       end
 
-      # Read from a PGP / GPG file or stream, decompressing the contents as it is read
-      # file_name_or_io: [String|IO]
+      # Read from a PGP / GPG file , decompressing the contents as it is read.
+      #
+      # file_name: [String]
       #   Name of file to read from
-      #   Or, the IO stream to receive the decrypted contents
+      #
       # passphrase: [String]
       #   Pass phrase for private key to decrypt the file with
-      def self.open(file_name_or_io, passphrase: self.default_passphrase, binary: true, &block)
+      def self.file(file_name, passphrase: self.default_passphrase)
         raise(ArgumentError, 'Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase') unless passphrase
 
-        return read_file(file_name_or_io, passphrase: passphrase, binary: binary, &block) if file_name_or_io.is_a?(String)
-
-        # PGP can only work against a file, not a stream, so create temp file.
-        IOStreams::File::Path.temp_file_name('iostreams_pgp') do |temp_file_name|
-          IOStreams.copy(file_name_or_io, temp_file_name, target_options: {streams: []})
-          read_file(temp_file_name, passphrase: passphrase, binary: binary, &block)
-        end
-      end
-
-      def self.read_file(file_name, passphrase: self.default_passphrase, binary: true)
         loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ? '--pinentry-mode loopback' : ''
         command  = "#{IOStreams::Pgp.executable} #{loopback} --batch --no-tty --yes --decrypt --passphrase-fd 0 #{file_name}"
-        IOStreams::Pgp.logger.debug { "IOStreams::Pgp::Reader.open: #{command}" } if IOStreams::Pgp.logger
+        IOStreams::Pgp.logger&.debug { "IOStreams::Pgp::Reader.open: #{command}" }
 
         # Read decrypted contents from stdout
         Open3.popen3(command) do |stdin, stdout, stderr, waith_thr|
@@ -37,24 +34,19 @@ module IOStreams
           stdin.close
           result =
             begin
-              stdout.binmode if binary
+              stdout.binmode
               yield(stdout)
             rescue Errno::EPIPE
               # Ignore broken pipe because gpg terminates early due to an error
               raise(Pgp::Failure, "GPG Failed reading from encrypted file: #{file_name}: #{stderr.read.chomp}")
             end
-          raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name}: #{stderr.read.chomp}") unless waith_thr.value.success?
+          unless waith_thr.value.success?
+            raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name}: #{stderr.read.chomp}")
+          end
+
           result
         end
       end
-
-      private
-
-      @default_passphrase = nil
-
-      def self.default_passphrase
-        @default_passphrase
-      end
     end
   end
 end

data/lib/io_streams/pgp/writer.rb

@@ -2,24 +2,29 @@ require 'open3'
 
 module IOStreams
   module Pgp
-    class Writer
-      # Sign all encrypted files with this users key.
-      # Default: Do not sign encyrpted files.
-      def self.default_signer=(default_signer)
-        @default_signer = default_signer
-      end
+    class Writer < IOStreams::Writer
+      class << self
+        # Sign all encrypted files with this users key.
+        # Default: Do not sign encrypted files.
+        attr_writer :default_signer
+
+        # Passphrase to use to open the private key when signing the file.
+        # Default: None.
+        attr_writer :default_signer_passphrase
+
+        private
 
-      # Passphrase to use to open the private key when signing the file.
-      # Default: None.
-      def self.default_signer_passphrase=(default_signer_passphrase)
-        @default_signer_passphrase = default_signer_passphrase
+        attr_reader :default_signer_passphrase
+        attr_reader :default_signer
+
+        @default_signer_passphrase = nil
+        @default_signer            = nil
       end
 
-      # Write to a PGP / GPG file or stream, encrypting the contents as it is written
+      # Write to a PGP / GPG file, encrypting the contents as it is written.
       #
-      # file_name_or_io: [String|IO]
+      # file_name: [String]
       #   Name of file to write to.
-      #   Or, the IO stream to write the encrypted contents to.
       #
       # recipient: [String]
       #   Email of user for which to encypt the file.
@@ -32,10 +37,6 @@ module IOStreams
       #   Passphrase to use to open the private key when signing the file.
       #   Default: default_signer_passphrase
       #
-      # binary: [true|false]
-      #   Whether to write binary data.
-      #   Default: true
-      #
       # compression: [:none|:zip|:zlib|:bzip2]
       #   Note: Standard PGP only supports :zip.
       #   :zlib is better than zip.
@@ -45,22 +46,12 @@ module IOStreams
       # compress_level: [Integer]
       #   Compression level
       #   Default: 6
-      def self.open(file_name_or_io, recipient:, signer: default_signer, signer_passphrase: default_signer_passphrase, binary: true, compression: :zip, compress_level: 6, &block)
-        compress_level = 0 if compression == :none
-
-        if file_name_or_io.is_a?(String)
-          IOStreams::File::Path.mkpath(file_name_or_io)
-          return write_file(file_name_or_io, recipient: recipient, signer: signer, signer_passphrase: signer_passphrase, binary: binary, compression: compression, compress_level: compress_level, &block)
-        end
+      def self.file(file_name, recipient: nil, import_and_trust_key: nil, signer: default_signer, signer_passphrase: default_signer_passphrase, compression: :zip, compress_level: 6, original_file_name: nil)
+        raise(ArgumentError, "Requires either :recipient or :import_and_trust_key") unless recipient || import_and_trust_key
 
-        # PGP can only work against a file, not a stream, so create temp file.
-        IOStreams::File::Path.temp_file_name('iostreams_pgp') do |temp_file_name|
-          write_file(temp_file_name, recipient: recipient, signer: signer, signer_passphrase: signer_passphrase, binary: binary, compression: compression, compress_level: compress_level, &block)
-          IOStreams.copy(temp_file_name, file_name_or_io, source_options: {streams: []})
-        end
-      end
+        recipient      = IOStreams::Pgp.import_and_trust(key: import_and_trust_key) if import_and_trust_key
+        compress_level = 0 if compression == :none
 
-      def self.write_file(file_name, recipient:, signer: default_signer, signer_passphrase: default_signer_passphrase, binary: true, compression: :zip, compress_level: 6)
         # Write to stdin, with encrypted contents being written to the file
         command = "#{IOStreams::Pgp.executable} --batch --no-tty --yes --encrypt"
         command << " --sign --local-user \"#{signer}\"" if signer
@@ -72,11 +63,11 @@ module IOStreams
         command << " --compress-algo #{compression}" unless compression == :none
         command << " --recipient \"#{recipient}\" -o \"#{file_name}\""
 
-        IOStreams::Pgp.logger.debug { "IOStreams::Pgp::Writer.open: #{command}" } if IOStreams::Pgp.logger
+        IOStreams::Pgp.logger&.debug { "IOStreams::Pgp::Writer.open: #{command}" }
 
         Open3.popen2e(command) do |stdin, out, waith_thr|
           begin
-            stdin.binmode if binary
+            stdin.binmode
             yield(stdin)
             stdin.close
           rescue Errno::EPIPE
@@ -91,19 +82,6 @@ module IOStreams
         end
       end
 
-      private
-
-      @default_signer_passphrase = nil
-      @default_signer            = nil
-
-      def self.default_signer_passphrase
-        @default_signer_passphrase
-      end
-
-      def self.default_signer
-        @default_signer
-      end
-
     end
   end
 end

data/lib/io_streams/reader.rb

@@ -0,0 +1,28 @@
+module IOStreams
+  class Reader
+    # When a Reader does not support streams, we copy the stream to a local temp file
+    # and then pass that filename in for this reader.
+    def self.stream(input_stream, **args, &block)
+      Utils.temp_file_name("iostreams_reader") do |file_name|
+        ::File.open(file_name, 'wb') { |target| ::IO.copy_stream(input_stream, target) }
+        file(file_name, **args, &block)
+      end
+    end
+
+    # When a Writer supports streams, also allow it to simply support a file
+    def self.file(file_name, original_file_name: file_name, **args, &block)
+      ::File.open(file_name, 'rb') { |file| stream(file, original_file_name: original_file_name, **args, &block) }
+    end
+
+    # For processing by either a file name or an open IO stream.
+    def self.open(file_name_or_io, **args, &block)
+      file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
+    end
+
+    attr_reader :input_stream
+
+    def initialize(input_stream)
+      @input_stream = input_stream
+    end
+  end
+end

data/lib/io_streams/record/reader.rb

@@ -1,23 +1,23 @@
 module IOStreams
   module Record
     # Converts each line of an input stream into hash for every row
-    class Reader
+    class Reader < IOStreams::Reader
       include Enumerable
 
-      # Read a record as a Hash at a time from a file or stream.
-      def self.open(file_name_or_io, delimiter: nil, buffer_size: 65536, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args)
-        if file_name_or_io.is_a?(String)
-          IOStreams.line_reader(file_name_or_io,
-                                delimiter:      delimiter,
-                                buffer_size:    buffer_size,
-                                encoding:       encoding,
-                                encode_cleaner: encode_cleaner,
-                                encode_replace: encode_replace
-          ) do |io|
-            yield new(io, file_name: file_name_or_io, **args)
-          end
-        else
-          yield new(file_name_or_io, **args)
+      # Read a record at a time from a line stream
+      # Note:
+      # - The supplied stream _must_ already be a line stream, or a stream that responds to :each
+      def self.stream(line_reader, original_file_name: nil, **args, &block)
+        # Pass-through if already a record reader
+        return block.call(line_reader) if line_reader.is_a?(self.class)
+
+        yield new(line_reader, **args)
+      end
+
+      # When reading from a file also add the line reader stream
+      def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
+        IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+          yield new(io, **args)
         end
       end
 
@@ -33,8 +33,12 @@ module IOStreams
       #
       #   For all other parameters, see Tabular::Header.new
       def initialize(line_reader, cleanse_header: true, **args)
+        unless line_reader.respond_to?(:each)
+          raise(ArgumentError, "Stream must be a IOStreams::Line::Reader or implement #each")
+        end
+
         @tabular        = IOStreams::Tabular.new(**args)
-        @line_reader      = line_reader
+        @line_reader    = line_reader
         @cleanse_header = cleanse_header
       end
 

data/lib/io_streams/record/writer.rb

@@ -1,28 +1,25 @@
 module IOStreams
   module Record
     # Example, implied header from first record:
-    #   IOStreams.record_writer do |stream|
+    #   IOStreams.path('file.csv').record_writer do |stream|
     #     stream << {name: 'Jack', address: 'Somewhere', zipcode: 12345}
     #     stream << {name: 'Joe', address: 'Lost', zipcode: 32443, age: 23}
     #   end
-    #
-    # Output:
-    #   name, add
-    #
-    class Writer
-      # Write a record as a Hash at a time to a file or stream.
-      def self.open(file_name_or_io, delimiter: $/, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args)
-        if file_name_or_io.is_a?(String)
-          IOStreams.line_writer(file_name_or_io,
-                                delimiter:      delimiter,
-                                encoding:       encoding,
-                                encode_cleaner: encode_cleaner,
-                                encode_replace: encode_replace
-          ) do |io|
-            yield new(io, file_name: file_name_or_io, **args)
-          end
-        else
-          yield new(file_name_or_io, **args)
+    class Writer < IOStreams::Writer
+      # Write a record as a Hash at a time to a stream.
+      # Note:
+      # - The supplied stream _must_ already be a line stream, or a stream that responds to :<<
+      def self.stream(line_writer, original_file_name: nil, **args, &block)
+        # Pass-through if already a row writer
+        return block.call(line_writer) if line_writer.is_a?(self.class)
+
+        yield new(line_writer, **args)
+      end
+
+      # When writing to a file also add the line writer stream
+      def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
+        IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+          yield new(io, **args, &block)
         end
       end
 
@@ -37,24 +34,27 @@ module IOStreams
       #     :csv, :hash, :array, :json, :psv, :fixed
       #
       #   For all other parameters, see Tabular::Header.new
-      #
-      #     columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
-      def initialize(delimited, columns: nil, **args)
-        @tabular   = IOStreams::Tabular.new(columns: columns, **args)
-        @delimited = delimited
+      def initialize(line_writer, columns: nil, **args)
+        unless line_writer.respond_to?(:<<)
+          raise(ArgumentError, 'Stream must be a IOStreams::Line::Writer or implement #<<')
+        end
+
+        @tabular     = IOStreams::Tabular.new(columns: columns, **args)
+        @line_writer = line_writer
 
         # Render header line when `columns` is supplied.
-        @delimited << @tabular.render_header if columns && @tabular.requires_header?
+        @line_writer << @tabular.render_header if columns && @tabular.requires_header?
       end
 
       def <<(hash)
-        raise(ArgumentError, 'Must supply a Hash') unless hash.is_a?(Hash)
+        raise(ArgumentError, '#<< only accepts a Hash argument') unless hash.is_a?(Hash)
+
         if @tabular.header?
           # Extract header from the keys from the first row when not supplied above.
           @tabular.header.columns = hash.keys
-          @delimited << @tabular.render_header
+          @line_writer << @tabular.render_header
         end
-        @delimited << @tabular.render(hash)
+        @line_writer << @tabular.render(hash)
       end
     end
   end

data/lib/io_streams/row/reader.rb

@@ -1,29 +1,21 @@
 module IOStreams
   module Row
     # Converts each line of an input stream into an array for every line
-    class Reader
-      # Read a line as an Array at a time from a file or stream.
-      def self.open(file_name_or_io,
-        delimiter: nil,
-        buffer_size: 65_536,
-        file_name: nil,
-        encoding: nil,
-        encode_cleaner: nil,
-        encode_replace: nil,
-        **args)
-        if file_name_or_io.is_a?(String)
-          IOStreams.line_reader(file_name_or_io,
-                                delimiter:      delimiter,
-                                buffer_size:    buffer_size,
-                                file_name:      file_name,
-                                encoding:       encoding,
-                                encode_cleaner: encode_cleaner,
-                                encode_replace: encode_replace
-          ) do |io|
-            yield new(io, file_name: file_name, **args)
-          end
-        else
-          yield new(file_name_or_io, **args)
+    class Reader < IOStreams::Reader
+      # Read a line as an Array at a time from a stream.
+      # Note:
+      # - The supplied stream _must_ already be a line stream, or a stream that responds to :each
+      def self.stream(line_reader, original_file_name: nil, **args, &block)
+        # Pass-through if already a row reader
+        return block.call(line_reader) if line_reader.is_a?(self.class)
+
+        yield new(line_reader, **args)
+      end
+
+      # When reading from a file also add the line reader stream
+      def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
+        IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+          yield new(io, **args)
         end
       end
 
@@ -37,14 +29,18 @@ module IOStreams
       #     :csv, :hash, :array, :json, :psv, :fixed
       #
       #   For all other parameters, see Tabular::Header.new
-      def initialize(delimited, cleanse_header: true, **args)
+      def initialize(line_reader, cleanse_header: true, **args)
+        unless line_reader.respond_to?(:each)
+          raise(ArgumentError, "Stream must be a IOStreams::Line::Reader or implement #each")
+        end
+
         @tabular        = IOStreams::Tabular.new(**args)
-        @delimited      = delimited
+        @line_reader    = line_reader
         @cleanse_header = cleanse_header
       end
 
       def each
-        @delimited.each do |line|
+        @line_reader.each do |line|
           if @tabular.header?
             columns = @tabular.parse_header(line)
             @tabular.cleanse_header! if @cleanse_header

data/lib/io_streams/row/writer.rb

@@ -2,61 +2,62 @@ require 'csv'
 module IOStreams
   module Row
     # Example:
-    #   IOStreams.row_writer do |stream|
+    #   IOStreams.path("file.csv").row_writer do |stream|
     #     stream << ['name', 'address', 'zipcode']
     #     stream << ['Jack', 'Somewhere', 12345]
     #     stream << ['Joe', 'Lost', 32443]
     #   end
-    #
-    # Output:
-    #   ...
-    #
-    class Writer
-      # Write a record as a Hash at a time to a file or stream.
-      def self.open(file_name_or_io, delimiter: $/, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args)
-        if file_name_or_io.is_a?(String)
-          IOStreams.line_writer(file_name_or_io,
-                                delimiter:      delimiter,
-                                encoding:       encoding,
-                                encode_cleaner: encode_cleaner,
-                                encode_replace: encode_replace
-          ) do |io|
-            yield new(io, **args)
-          end
-        else
-          yield new(file_name_or_io, **args)
+    class Writer < IOStreams::Writer
+      # Write a record from an Array at a time to a stream.
+      #
+      # Note:
+      # - The supplied stream _must_ already be a line stream, or a stream that responds to :<<
+      def self.stream(line_writer, original_file_name: nil, **args)
+        # Pass-through if already a row writer
+        return block.call(line_writer) if line_writer.is_a?(self.class)
+
+        yield new(line_writer, **args)
+      end
+
+      # When writing to a file also add the line writer stream
+      def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
+        IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
+          yield new(io, **args, &block)
         end
       end
 
       # Create a Tabular writer that takes individual rows as arrays.
       #
       # Parameters
-      #   delimited: [#<<]
+      #   line_writer: [#<<]
       #     Anything that accepts a line / record at a time when #<< is called on it.
       #
       #   format: [Symbol]
       #     :csv, :hash, :array, :json, :psv, :fixed
       #
       #   For all other parameters, see Tabular::Header.new
-      #
-      #     columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
-      def initialize(delimited, columns: nil, **args)
-        @tabular   = IOStreams::Tabular.new(columns: columns, **args)
-        @delimited = delimited
+      def initialize(line_writer, columns: nil, **args)
+        unless line_writer.respond_to?(:<<)
+          raise(ArgumentError, 'Stream must be a IOStreams::Line::Writer or implement #<<')
+        end
+
+        @tabular     = IOStreams::Tabular.new(columns: columns, **args)
+        @line_writer = line_writer
 
         # Render header line when `columns` is supplied.
-        delimited << @tabular.render_header if columns && @tabular.requires_header?
+        line_writer << @tabular.render_header if columns && @tabular.requires_header?
       end
 
       # Supply a hash or an array to render
       def <<(array)
         raise(ArgumentError, 'Must supply an Array') unless array.is_a?(Array)
+
         if @tabular.header?
           # If header (columns) was not supplied as an argument, assume first line is the header.
           @tabular.header.columns = array
-          @delimited << @tabular.render_header
+          @line_writer << @tabular.render_header
         else
-          @delimited << @tabular.render(array)
+          @line_writer << @tabular.render(array)
         end
       end
     end