iostreams 0.9.1 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c229181098f8b4aedf3ce542477dc44554f94f4
-  data.tar.gz: 866c35496433dbd255865b322a61bd50272b5ea5
+  metadata.gz: e653b5b68990dd8c8c6442c160a109a072e33a14
+  data.tar.gz: f30e9b16508e24ed1040f36bf16192cc2c5e7e1a
 SHA512:
-  metadata.gz: d91a975d92b3ab08d2d0185a927f6403ccdd362b63dd07871f7322219d07ea088e84ff401c9bd6ec3cb8227e29a7d7702aec6d35dbe863a2764ebda83e5e3dc9
-  data.tar.gz: 7204cf47a99466b5818b462f3cffbc303f129e277793c3fda1ec8ef1c5f697145bcda34d69ea859d674789cb3f500afcd29fcacad76963c173bdd9768757f057
+  metadata.gz: 63d58edaa9e53bc878bfca723772f0373791cea7411fe5be74b37db6e56d5a79003f06ccd0a95962a2e6d754c55f0a8af6031a68c0a70aef71ed83328e50000b
+  data.tar.gz: 28be995afd5b6627758599c6c50d2e21293bef90e1df9babb413a82152953237e5cf25191bb8d15a7aedf782c162e635a4d822878dde183a40c61a2f621ad37d

data/Rakefile

@@ -1,6 +1,4 @@
-require 'rake/clean'
 require 'rake/testtask'
-
 require_relative 'lib/io_streams/version'
 
 task :gem do
@@ -14,14 +12,10 @@ task :publish => :gem do
   system "rm iostreams-#{IOStreams::VERSION}.gem"
 end
 
-desc 'Run Test Suite'
-task :test do
-  Rake::TestTask.new(:functional) do |t|
-    t.test_files = FileList['test/**/*_test.rb']
-    t.verbose    = true
-  end
-
-  Rake::Task['functional'].invoke
+Rake::TestTask.new(:test) do |t|
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+  t.warning = true
 end
 
 task :default => :test

data/lib/io_streams/delimited/reader.rb

@@ -4,12 +4,12 @@ module IOStreams
       attr_accessor :delimiter, :buffer_size, :encoding, :strip_non_printable
 
       # Read from a file or stream
-      def self.open(file_name_or_io, options={}, &block)
+      def self.open(file_name_or_io, delimiter: nil, buffer_size: 65536, encoding: UTF8_ENCODING, strip_non_printable: false)
         if IOStreams.reader_stream?(file_name_or_io)
-          block.call(new(file_name_or_io, options))
+          yield new(file_name_or_io, delimiter: delimiter, buffer_size: buffer_size, encoding: encoding, strip_non_printable: strip_non_printable)
         else
           ::File.open(file_name_or_io, 'rb') do |io|
-            block.call(new(io, options))
+            yield new(io, delimiter: delimiter, buffer_size: buffer_size, encoding: encoding, strip_non_printable: strip_non_printable)
           end
         end
       end
@@ -24,38 +24,35 @@ module IOStreams
       #   input_stream
       #     The input stream that implements #read
       #
-      #   options
-      #     :delimiter[String]
-      #       Line / Record delimiter to use to break the stream up into records
-      #         Any string to break the stream up by
-      #         The records when saved will not include this delimiter
-      #       Default: nil
-      #         Automatically detect line endings and break up by line
-      #         Searches for the first "\r\n" or "\n" and then uses that as the
-      #         delimiter for all subsequent records
+      #   delimiter: [String]
+      #     Line / Record delimiter to use to break the stream up into records
+      #       Any string to break the stream up by
+      #       The records when saved will not include this delimiter
+      #     Default: nil
+      #       Automatically detect line endings and break up by line
+      #       Searches for the first "\r\n" or "\n" and then uses that as the
+      #       delimiter for all subsequent records
       #
-      #     :buffer_size [Integer]
-      #       Maximum size of the buffer into which to read the stream into for
-      #       processing.
-      #       Must be large enough to hold the entire first line and its delimiter(s)
-      #       Default: 65536 ( 64K )
+      #   buffer_size: [Integer]
+      #     Maximum size of the buffer into which to read the stream into for
+      #     processing.
+      #     Must be large enough to hold the entire first line and its delimiter(s)
+      #     Default: 65536 ( 64K )
       #
-      #     :strip_non_printable [true|false]
-      #       Strip all non-printable characters read from the file
-      #       Default: false
+      #   strip_non_printable: [true|false]
+      #     Strip all non-printable characters read from the file
+      #     Default: false
       #
-      #     :encoding
-      #       Force encoding to this encoding for all data being read
-      #       Default: UTF8_ENCODING
-      #       Set to nil to disable encoding
-      def initialize(input_stream, options={})
+      #   encoding:
+      #     Force encoding to this encoding for all data being read
+      #     Default: UTF8_ENCODING
+      #     Set to nil to disable encoding
+      def initialize(input_stream, delimiter: nil, buffer_size: 65536, encoding: UTF8_ENCODING, strip_non_printable: false)
         @input_stream        = input_stream
-        options              = options.dup
-        @delimiter           = options.delete(:delimiter)
-        @buffer_size         = options.delete(:buffer_size) || 65536
-        @encoding            = options.has_key?(:encoding) ? options.delete(:encoding) : UTF8_ENCODING
-        @strip_non_printable = options.delete(:strip_non_printable) || false
-        raise ArgumentError.new("Unknown IOStreams::Delimited::Reader#initialize options: #{options.inspect}") if options.size > 0
+        @delimiter           = delimiter
+        @buffer_size         = buffer_size
+        @encoding            = encoding
+        @strip_non_printable = strip_non_printable
 
         @delimiter.force_encoding(UTF8_ENCODING) if @delimiter && @encoding
         @buffer = ''

data/lib/io_streams/delimited/writer.rb

@@ -4,12 +4,12 @@ module IOStreams
       attr_accessor :delimiter
 
       # Write delimited records/lines to a file or stream
-      def self.open(file_name_or_io, options={}, &block)
+      def self.open(file_name_or_io, delimiter: $/, encoding: UTF8_ENCODING, strip_non_printable: false)
         if IOStreams.writer_stream?(file_name_or_io)
-          block.call(new(file_name_or_io, options))
+          yield new(file_name_or_io, delimiter: delimiter, encoding: encoding, strip_non_printable: strip_non_printable)
         else
           ::File.open(file_name_or_io, 'wb') do |io|
-            block.call(new(io, options))
+            yield new(io, delimiter: delimiter, encoding: encoding, strip_non_printable: strip_non_printable)
           end
         end
       end
@@ -26,28 +26,24 @@ module IOStreams
       #   output_stream
       #     The output stream that implements #write
       #
-      #   options
-      #     delimiter: [String]
-      #       Add the specified delimiter after every record when writing it
-      #       to the output stream
-      #       Default: OS Specific. Linux: "\n"
+      #   delimiter: [String]
+      #     Add the specified delimiter after every record when writing it
+      #     to the output stream
+      #     Default: OS Specific. Linux: "\n"
       #
-      #     :encoding
-      #       Force encoding to this encoding for all data being read
-      #       Default: UTF8_ENCODING
-      #       Set to nil to disable encoding
+      #   encoding:
+      #     Force encoding to this encoding for all data being read
+      #     Default: UTF8_ENCODING
+      #     Set to nil to disable encoding
       #
-      #     :strip_non_printable [true|false]
-      #       Strip all non-printable characters read from the file
-      #       Default: false
-      def initialize(output_stream, options={})
+      #   strip_non_printable: [true|false]
+      #     Strip all non-printable characters read from the file
+      #     Default: false
+      def initialize(output_stream, delimiter: $/, encoding: UTF8_ENCODING, strip_non_printable: false)
         @output_stream       = output_stream
-        options              = options.dup
-        @delimiter           = options.has_key?(:delimiter) ? options.delete(:delimiter) : $/.dup
-        @encoding            = options.has_key?(:encoding) ? options.delete(:encoding) : UTF8_ENCODING
-        @strip_non_printable = options.delete(:strip_non_printable)
-        @strip_non_printable = @strip_non_printable.nil? && (@encoding == UTF8_ENCODING)
-        raise ArgumentError.new("Unknown IOStreams::Delimited::Writer#initialize options: #{options.inspect}") if options.size > 0
+        @delimiter           = delimiter.dup
+        @encoding            = encoding
+        @strip_non_printable = strip_non_printable
         @delimiter.force_encoding(UTF8_ENCODING) if @delimiter
       end
 

data/lib/io_streams/io_streams.rb

@@ -51,7 +51,7 @@ module IOStreams
   #   # MyXls::Reader and MyXls::Writer must implement .open
   #   register_extension(:xls, MyXls::Reader, MyXls::Writer)
   def self.register_extension(extension, reader_class, writer_class)
-    raise "Invalid extension #{extension.inspect}" unless extension.to_s =~ /\A\w+\Z/
+    raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
     @@extensions[extension.to_sym] = Extension.new(reader_class, writer_class)
   end
 
@@ -62,7 +62,7 @@ module IOStreams
   # Example:
   #   register_extension(:xls)
   def self.deregister_extension(extension)
-    raise "Invalid extension #{extension.inspect}" unless extension.to_s =~ /\A\w+\Z/
+    raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
     @@extensions.delete(extension.to_sym)
   end
 
@@ -239,7 +239,7 @@ module IOStreams
       stream_struct.klass.open(file_name_or_io, stream_struct.options, &block)
     else
       # Daisy chain multiple streams together
-      last = stream_structs.inject(block) { |inner, stream_struct| -> io { stream_struct.klass.open(io, stream_struct.options, &inner) } }
+      last = stream_structs.inject(block) { |inner, ss| -> io { ss.klass.open(io, ss.options, &inner) } }
       last.call(file_name_or_io)
     end
   end
@@ -283,5 +283,7 @@ module IOStreams
   register_extension(:delimited, IOStreams::Delimited::Reader, IOStreams::Delimited::Writer)
   register_extension(:xlsx,      IOStreams::Xlsx::Reader,      nil)
   register_extension(:xlsm,      IOStreams::Xlsx::Reader,      nil)
+  register_extension(:pgp,       IOStreams::Pgp::Reader,       IOStreams::Pgp::Writer)
+  register_extension(:gpg,       IOStreams::Pgp::Reader,       IOStreams::Pgp::Writer)
   #register_extension(:csv,       IOStreams::CSV::Reader,       IOStreams::CSV::Writer)
 end

data/lib/io_streams/pgp.rb

@@ -0,0 +1,213 @@
+require 'open3'
+module IOStreams
+  # Read/Write PGP/GPG file or stream.
+  #
+  # Example Setup:
+  #
+  #   1. Install OpenPGP
+  #      Mac OSX (homebrew) : `brew install gpg2`
+  #      Redhat Linux: `rpm install gpg2`
+  #
+  #   2. # Generate senders private and public key
+  #      IOStreams::Pgp.generate_key(name: 'Sender', email: 'sender@example.org', passphrase: 'sender_passphrase')
+  #
+  #   3. # Generate receivers private and public key
+  #      IOStreams::Pgp.generate_key(name: 'Receiver', email: 'receiver@example.org', passphrase: 'receiver_passphrase')
+  #
+  # Example 1:
+  #
+  #   # Generate encrypted file for a specific recipient and sign it with senders credentials
+  #   data = %w(this is some data that should be encrypted using pgp)
+  #   IOStreams::Pgp::Writer.open('secure.gpg', recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |output|
+  #     data.each { |word| output.puts(word) }
+  #   end
+  #
+  #   # Decrypt the file sent to `receiver@example.org` using its private key
+  #   # Recipient must also have the senders public key to verify the signature
+  #   IOStreams::Pgp::Reader.open('secure.gpg', passphrase: 'receiver_passphrase') do |stream|
+  #     while !stream.eof?
+  #       ap stream.read(10)
+  #       puts
+  #     end
+  #   end
+  #
+  # Example 2:
+  #
+  #   # Default user and passphrase to sign the output file:
+  #   IOStreams::Pgp::Writer.default_signer            = 'sender@example.org'
+  #   IOStreams::Pgp::Writer.default_signer_passphrase = 'sender_passphrase'
+  #
+  #   # Default passphrase for decrypting recipients files.
+  #   # Note: Usually this would be the senders passphrase, but in this example
+  #   #       it is decrypting the file intended for the recipient.
+  #   IOStreams::Pgp::Reader.default_passphrase = 'receiver_passphrase'
+  #
+  #   # Generate encrypted file for a specific recipient and sign it with senders credentials
+  #   data = %w(this is some data that should be encrypted using pgp)
+  #   IOStreams.writer('secure.gpg', pgp: {recipient: 'receiver@example.org'}) do |output|
+  #     data.each { |word| output.puts(word) }
+  #   end
+  #
+  #   # Decrypt the file sent to `receiver@example.org` using its private key
+  #   # Recipient must also have the senders public key to verify the signature
+  #   IOStreams.reader('secure.gpg') do |stream|
+  #     while data = stream.read(10)
+  #       ap data
+  #     end
+  #   end
+  #
+  # FAQ:
+  # - If you get not trusted errors
+  #    gpg --edit-key sender@example.org
+  #      Select highest level: 5
+  #
+  # Delete test keys:
+  #   IOStreams::Pgp.delete_keys(email: 'sender@example.org', secret: true)
+  #   IOStreams::Pgp.delete_keys(email: 'receiver@example.org', secret: true)
+  #
+  # Limitations
+  # - Designed for processing larger files since a process is spawned for each file processed.
+  # - For small in memory files or individual emails, use the 'opengpgme' library.
+  #
+  # Compression Performance:
+  #   Running tests on an Early 2015 Macbook Pro Dual Core with Ruby v2.3.1
+  #
+  #   Input file: test.log 3.6GB
+  #     :none:  size: 3.6GB  write:  52s  read:  45s
+  #     :zip:   size: 411MB  write:  75s  read:  31s
+  #     :zlib:  size: 241MB  write:  66s  read:  23s  ( 756KB Memory )
+  #     :bzip2: size: 129MB  write: 430s  read: 130s  ( 5MB Memory )
+  module Pgp
+    autoload :Reader, 'io_streams/pgp/reader'
+    autoload :Writer, 'io_streams/pgp/writer'
+
+    class Failure < StandardError
+    end
+
+    # Generate a new ultimate trusted local public and private key
+    # Returns [String] the key id for the generated key
+    # Raises an exception if it fails to generate the key
+    #
+    # name: [String]
+    #   Name of who owns the key, such as organization
+    #
+    # email: [String]
+    #   Email address for the key
+    #
+    # comment: [String]
+    #   Optional comment to add to the generated key
+    #
+    # passphrase [String]
+    #   Optional passphrase to secure the key with.
+    #   Highly Recommended.
+    #   To generate a good passphrase:
+    #     `SecureRandom.urlsafe_base64(128)`
+    #
+    # See `man gpg` for the remaining options
+    def self.generate_key(name:, email:, comment: nil, passphrase: nil, key_type: 'RSA', key_length: 4096, subkey_type: 'RSA', subkey_length: key_length, expire_date: nil)
+      Open3.popen2e('gpg --batch --gen-key') do |stdin, out, waith_thr|
+        stdin.puts "Key-Type: #{key_type}" if key_type
+        stdin.puts "Key-Length: #{key_length}" if key_length
+        stdin.puts "Subkey-Type: #{subkey_type}" if subkey_type
+        stdin.puts "Subkey-Length: #{subkey_length}" if subkey_length
+        stdin.puts "Name-Real: #{name}" if name
+        stdin.puts "Name-Comment: #{comment}" if comment
+        stdin.puts "Name-Email: #{email}" if email
+        stdin.puts "Expire-Date: #{expire_date}" if expire_date
+        stdin.puts "Passphrase: #{passphrase}" if passphrase
+        stdin.puts '%commit'
+        stdin.close
+        if waith_thr.value.success?
+          key_id = nil
+          out.each_line do |line|
+            if (line = line.chomp) =~ /^gpg: key ([0-9A-F]+) marked as ultimately trusted/
+              key_id = $1.to_i(16)
+            end
+          end
+          key_id
+        else
+          raise(Pgp::Failure, "GPG Failed to generate key: #{out.read.chomp}")
+        end
+      end
+    end
+
+    # Delete a secret and public keys using its email
+    # Returns false if no key was found
+    # Raises an exception if it fails to delete the key
+    #
+    # email: [String] Email address for the key
+    #
+    # public: [true|false]
+    #   Whether to delete the public key
+    #   Default: true
+    #
+    # secret: [true|false]
+    #   Whether to delete the secret key
+    #   Default: false
+    def self.delete_keys(email:, public: true, secret: false)
+      cmd = "for i in `gpg --with-colons --fingerprint #{email} | grep \"^fpr\" | cut -d: -f10`; do\n"
+      cmd << "gpg --batch --delete-secret-keys \"$i\" ;\n" if secret
+      cmd << "gpg --batch --delete-keys \"$i\" ;\n" if public
+      cmd << 'done'
+      Open3.popen2e(cmd) do |stdin, out, waith_thr|
+        output = out.read.chomp
+        if waith_thr.value.success?
+          return false if output =~ /(public key not found|No public key)/i
+          raise(Pgp::Failure, "GPG Failed to delete keys for #{email}: #{output}") if output.include?('error')
+          true
+        else
+          raise(Pgp::Failure, "GPG Failed calling gpg to delete secret keys for #{email}: #{output}")
+        end
+      end
+    end
+
+    def self.has_key?(email:)
+      Open3.popen2e("gpg --list-keys --with-colons #{email}") do |stdin, out, waith_thr|
+        output = out.read.chomp
+        if waith_thr.value.success?
+          output.each_line do |line|
+            return true if line.match(/\Auid.*::([^\:]*):\Z/)
+          end
+          false
+        else
+          return false if output =~ /(public key not found|No public key)/i
+          raise(Pgp::Failure, "GPG Failed calling gpg to list keys for #{email}: #{output}")
+        end
+      end
+    end
+
+    # Returns [String] the key for the supplied email address
+    #
+    # email: [String] Email address for requested key
+    #
+    # ascii: [true|false]
+    #   Whether to export as ASCII text instead of binary format
+    #   Default: true
+    #
+    # secret: [true|false]
+    #   Whether to export the private key
+    #   Default: false
+    def self.export(email:, ascii: true, secret: false)
+      armor            = ascii ? ' --armor' : nil
+      cmd              = secret ? '--export-secret-keys' : '--export'
+      out, err, status = Open3.capture3("gpg#{armor} #{cmd} #{email}", binmode: true)
+      if status.success? && out.length > 0
+        out
+      else
+        raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err} #{out}")
+      end
+    end
+
+    # Imports the supplied public/private key
+    # Returns [String] the output returned from the import command
+    def self.import(key)
+      out, err, status = Open3.capture3('gpg --import', binmode: true, stdin_data: key)
+      if status.success? && out.length > 0
+        out
+      else
+        raise(Pgp::Failure, "GPG Failed importing key: #{err} #{out}")
+      end
+    end
+
+  end
+end

data/lib/io_streams/pgp/reader.rb

@@ -0,0 +1,50 @@
+require 'open3'
+
+module IOStreams
+  module Pgp
+    class Reader
+      # Passphrase to use to open the private key to decrypt the received file
+      def self.default_passphrase=(default_passphrase)
+        @default_passphrase = default_passphrase
+      end
+
+      # Read from a PGP / GPG file or stream, decompressing the contents as it is read
+      # file_name_or_io: [String|IO]
+      #   Name of file to read from
+      #   Or, the IO stream to receive the decrypted contents
+      # passphrase: [String]
+      #   Pass phrase for private key to decrypt the file with
+      def self.open(file_name_or_io, passphrase: self.default_passphrase, binary: true)
+        raise(ArgumentError, 'Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase') unless passphrase
+
+        if IOStreams.reader_stream?(file_name_or_io)
+          raise(NotImplementedError, 'Can only PGP Decrypt directly from a file name. Input streams are not yet supported.')
+        else
+          # Read decrypted contents from stdout
+          Open3.popen3("gpg --batch --no-tty --yes --decrypt --passphrase-fd 0 #{file_name_or_io}") do |stdin, stdout, stderr, waith_thr|
+            stdin.puts(passphrase) if passphrase
+            stdin.close
+            result =
+              begin
+                stdout.binmode if binary
+                yield(stdout)
+              rescue Errno::EPIPE
+                # Ignore broken pipe because gpg terminates early due to an error
+                raise(Pgp::Failure, "GPG Failed reading from encrypted file: #{file_name_or_io}: #{stderr.read.chomp}")
+              end
+            raise(Pgp::Failure, "GPG Failed to decrypt file: #{file_name_or_io}: #{stderr.read.chomp}") unless waith_thr.value.success?
+            result
+          end
+        end
+      end
+
+      private
+
+      @default_passphrase = nil
+
+      def self.default_passphrase
+        @default_passphrase
+      end
+    end
+  end
+end

data/lib/io_streams/pgp/writer.rb

@@ -0,0 +1,93 @@
+require 'open3'
+
+module IOStreams
+  module Pgp
+    class Writer
+      # Sign all encrypted files with this users key.
+      # Default: Do not sign encyrpted files.
+      def self.default_signer=(default_signer)
+        @default_signer = default_signer
+      end
+
+      # Passphrase to use to open the private key when signing the file.
+      # Default: None.
+      def self.default_signer_passphrase=(default_signer_passphrase)
+        @default_signer_passphrase = default_signer_passphrase
+      end
+
+      # Write to a PGP / GPG file or stream, encrypting the contents as it is written
+      #
+      # file_name_or_io: [String|IO]
+      #   Name of file to write to.
+      #   Or, the IO stream to write the encrypted contents to.
+      #
+      # recipient: [String]
+      #   Email of user for which to encypt the file.
+      #
+      # signer: [String]
+      #   Name of user with which to sign the encypted file.
+      #   Default: default_signer or do not sign.
+      #
+      # signer_passphrase: [String]
+      #   Passphrase to use to open the private key when signing the file.
+      #   Default: default_signer_passphrase
+      #
+      # binary: [true|false]
+      #   Whether to write binary data.
+      #   Default: true
+      #
+      # compression: [:none|:zip|:zlib|:bzip2]
+      #   Note: Standard PGP only supports :zip.
+      #   :zlib is better than zip.
+      #   :bzip2 is best, but uses a lot of memory.
+      #   Default: :zip
+      #
+      # compress_level: [Integer]
+      #   Compression level
+      #   Default: 6
+      def self.open(file_name_or_io, recipient:, signer: default_signer, signer_passphrase: default_signer_passphrase, binary: true, compression: :zip, compress_level: 6)
+        compress_level = 0 if compression == :none
+        if IOStreams.writer_stream?(file_name_or_io)
+          raise(NotImplementedError, 'Can only PGP Encrypt directly to a file name. Output to streams are not yet supported.')
+        else
+          # Write to stdin, with encrypted contents being written to the file
+          cmd = "gpg --batch --no-tty --yes --encrypt"
+          cmd << " --sign --local-user \"#{signer}\"" if signer
+          cmd << " --passphrase \"#{signer_passphrase}\"" if signer_passphrase
+          cmd << " -z #{compress_level}" if compress_level != 6
+          cmd << " --compress-algo #{compression}" unless compression == :none
+          cmd << " --recipient \"#{recipient}\" -o \"#{file_name_or_io}\""
+          Open3.popen2e(cmd) do |stdin, out, waith_thr|
+            begin
+              stdin.binmode if binary
+              yield(stdin)
+              stdin.close
+            rescue Errno::EPIPE
+              # Ignore broken pipe because gpg terminates early due to an error
+              ::File.delete(file_name_or_io)
+              raise(Pgp::Failure, "GPG Failed writing to encrypted file: #{file_name_or_io}: #{out.read.chomp}")
+            end
+            unless waith_thr.value.success?
+              ::File.delete(file_name_or_io) if ::File.exist?(file_name_or_io)
+              raise(Pgp::Failure, "GPG Failed to create encrypted file: #{file_name_or_io}: #{out.read.chomp}")
+            end
+          end
+        end
+      end
+
+      private
+
+      @default_signer_passphrase = nil
+      @default_signer            = nil
+
+      def self.default_signer_passphrase
+        @default_signer_passphrase
+      end
+
+      def self.default_signer
+        @default_signer
+      end
+
+    end
+  end
+end

data/lib/io_streams/sftp/reader.rb

@@ -0,0 +1,67 @@
+module IOStreams
+  # Example:
+  #   IOStreams::SFTP::Reader.open(
+  #     'file.txt',
+  #     user:     'jbloggs',
+  #     password: 'secret',
+  #     host:     'example.org'
+  #   ) do |input|
+  #     puts input.read
+  #   end
+  module SFTP
+    class Reader
+      include SemanticLogger::Loggable if defined?(SemanticLogger)
+
+      # Stream to a remote file over sftp.
+      #
+      # file_name: [String]
+      #   Name of file to write to.
+      #
+      # user: [String]
+      #   Name of user to login with.
+      #
+      # password: [String]
+      #   Password for the user.
+      #
+      # host: [String]
+      #   Name of the host to connect to.
+      #
+      # port: [Integer]
+      #   Port to connect to at the above host.
+      #
+      # binary [true|false]
+      #   Whether to write in binary mode
+      #   Default: true
+      #
+      # options: [Hash]
+      #   Any options supported by Net::SSH.start
+      #
+      # Note:
+      # - Net::SFTP::StatusException means the file could not be read
+      def self.open(file_name, user:, password:, host:, port: 22, binary: true, options: {}, &block)
+        raise(NotImplementedError, 'Can only SFTP directly to a file name, not another stream.') if IOStreams.writer_stream?(file_name)
+
+        begin
+          require 'net/sftp' unless defined?(Net::SFTP)
+        rescue LoadError => e
+          raise(LoadError, "Please install the 'net-sftp' gem for SFTP streaming support. #{e.message}")
+        end
+
+        options                = options.dup
+        options[:logger]       ||= self.logger if defined?(SemanticLogger)
+        options[:port]         ||= 22
+        options[:max_pkt_size] ||= 65536
+        options[:password]     = password
+        options[:port]         = port
+        mode                   = binary ? 'rb' : 'r'
+
+        result = nil
+        Net::SFTP.start(host, user, options) do |sftp|
+          result = sftp.file.open(file_name, mode, &block)
+        end
+        result
+      end
+
+    end
+  end
+end

data/lib/io_streams/sftp/writer.rb

@@ -0,0 +1,68 @@
+module IOStreams
+  # Example:
+  #   IOStreams::SFTP::Writer.open('file.txt',
+  #     user:     'jbloggs',
+  #     password: 'secret',
+  #     host:     'example.org',
+  #     options: {compression: false}
+  #   ) do |output|
+  #       output.write('Hello World')
+  #   end
+  module SFTP
+    class Writer
+      include SemanticLogger::Loggable if defined?(SemanticLogger)
+
+      # Stream to a remote file over sftp.
+      #
+      # file_name: [String]
+      #   Name of file to write to.
+      #
+      # user: [String]
+      #   Name of user to login with.
+      #
+      # password: [String]
+      #   Password for the user.
+      #
+      # host: [String]
+      #   Name of the host to connect to.
+      #
+      # port: [Integer]
+      #   Port to connect to at the above host.
+      #
+      # mkdir [true|false]
+      #   Whether to create the output directory on the target system before writing the file.
+      #   The path is created recursively if any portions of the path that are missing.
+      #   Default: false
+      #
+      # binary [true|false]
+      #   Whether to write in binary mode
+      #   Default: true
+      #
+      # options: [Hash]
+      #   Any options supported by Net::SSH.start
+      def self.open(file_name, user:, password:, host:, port: 22, mkdir: false, binary: true, options: {}, &block)
+        raise(NotImplementedError, 'Can only SFTP directly to a file name, not another stream.') if IOStreams.writer_stream?(file_name)
+
+        begin
+          require 'net/sftp' unless defined?(Net::SFTP)
+        rescue LoadError => e
+          raise(LoadError, "Please install the 'net-sftp' gem for SFTP streaming support. #{e.message}")
+        end
+
+        options                = options.dup
+        options[:logger]       ||= logger if defined?(SemanticLogger)
+        options[:port]         ||= 22
+        options[:max_pkt_size] ||= 65536
+        options[:password]     = password
+        options[:port]         = port
+        mode                   = binary ? 'wb' : 'w'
+
+        Net::SFTP.start(host, user, options) do |sftp|
+          sftp.session.exec!("mkdir -p '#{File.dirname(file_name)}'") if mkdir
+          sftp.file.open(file_name, mode, & block)
+        end
+      end
+
+    end
+  end
+end

data/lib/io_streams/streams.rb

@@ -92,7 +92,7 @@ module IOStreams
     #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv')
     #   => [ :file ]
     def streams_for_file_name(file_name)
-      raise ArgumentError.new("RocketJob Cannot detect file format when uploading to stream: #{file_name.inspect}") if file_name.respond_to?(:read)
+      raise ArgumentError.new("RocketJob Cannot detect file format when uploading to stream: #{file_name.inspect}") if reader_stream?(file_name)
 
       parts      = file_name.split('.')
       extensions = []

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams #:nodoc
-  VERSION = '0.9.1'
+  VERSION = '0.10.0'
 end

data/lib/io_streams/xlsx/reader.rb

@@ -13,17 +13,13 @@ module IOStreams
       #       puts line
       #     end
       #   end
-      def self.open(file_name_or_io, options={}, &block)
+      def self.open(file_name_or_io, buffer_size: 65536, &block)
         begin
           require 'creek' unless defined?(Creek::Book)
         rescue LoadError => e
           raise(LoadError, "Please install the 'creek' gem for xlsx streaming support. #{e.message}")
         end
 
-        options     = options.dup
-        buffer_size = options.delete(:buffer_size) || 65536
-        raise(ArgumentError, "Unknown IOStreams::Xlsx::Reader option: #{options.inspect}") if options.size > 0
-
         if IOStreams.reader_stream?(file_name_or_io)
           temp_file = Tempfile.new('rocket_job_xlsx')
           file_name = temp_file.to_path

data/lib/io_streams/zip/reader.rb

@@ -12,11 +12,7 @@ module IOStreams
       #       puts data
       #     end
       #   end
-      def self.open(file_name_or_io, options={}, &block)
-        options     = options.dup
-        buffer_size = options.delete(:buffer_size) || 65536
-        raise(ArgumentError, "Unknown IOStreams::Zip::Reader option: #{options.inspect}") if options.size > 0
-
+      def self.open(file_name_or_io, buffer_size: 65536, &block)
         if !defined?(JRuby) && !defined?(::Zip)
           # MRI needs Ruby Zip, since it only has native support for GZip
           begin

data/lib/io_streams/zip/writer.rb

@@ -7,8 +7,7 @@ module IOStreams
       #   file_name_or_io [String]
       #     Full path and filename for the output zip file
       #
-      # Options
-      #   :file_name [String]
+      #   zip_file_name: [String]
       #     Name of the file within the Zip Stream
       #
       # The stream supplied to the block only responds to #write
@@ -22,14 +21,9 @@ module IOStreams
       # Notes:
       # - Since Zip cannot write to streams, if a stream is supplied, a temp file
       #   is automatically created under the covers
-      def self.open(file_name_or_io, options={}, &block)
-        options       = options.dup
-        zip_file_name = options.delete(:file_name) || options.delete(:zip_file_name)
-        buffer_size   = options.delete(:buffer_size) || 65536
-        raise(ArgumentError, "Unknown IOStreams::Zip::Writer option: #{options.inspect}") if options.size > 0
-
+      def self.open(file_name_or_io, zip_file_name: nil, buffer_size: 65536, &block)
         # Default the name of the file within the zip to the supplied file_name without the zip extension
-        zip_file_name = file_name_or_io.to_s[0..-5] if zip_file_name.nil? && !file_name_or_io.respond_to?(:write) && (file_name_or_io =~ /\.(zip)\z/)
+        zip_file_name = file_name_or_io.to_s[0..-5] if zip_file_name.nil? && !IOStreams.writer_stream?(file_name_or_io) && (file_name_or_io =~ /\.(zip)\z/)
         zip_file_name ||= 'file'
 
         if !defined?(JRuby) && !defined?(::Zip)

data/lib/iostreams.rb

@@ -12,6 +12,11 @@ module IOStreams
     autoload :Reader, 'io_streams/gzip/reader'
     autoload :Writer, 'io_streams/gzip/writer'
   end
+  autoload :Pgp,      'io_streams/pgp'
+  module SFTP
+    autoload :Reader, 'io_streams/sftp/reader'
+    autoload :Writer, 'io_streams/sftp/writer'
+  end
   module Zip
     autoload :Reader, 'io_streams/zip/reader'
     autoload :Writer, 'io_streams/zip/writer'

data/test/pgp_reader_test.rb

@@ -0,0 +1,44 @@
+require_relative 'test_helper'
+
+module Streams
+  class PgpReaderTest < Minitest::Test
+    describe IOStreams::Pgp::Reader do
+      before do
+        file_name  = File.join(File.dirname(__FILE__), 'files', 'text.txt')
+        @data      = File.read(file_name)
+        @temp_file = Tempfile.new('iostreams')
+        @file_name = @temp_file.to_path
+      end
+
+      after do
+        @temp_file.delete if @temp_file
+      end
+
+      describe '.open' do
+        it 'reads encrypted file' do
+          IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org') do |io|
+            io.write(@data)
+          end
+
+          result = IOStreams::Pgp::Reader.open(@file_name, passphrase: 'receiver_passphrase') { |file| file.read }
+          assert_equal @data, result
+        end
+
+        it 'fails with bad passphrase' do
+          assert_raises IOStreams::Pgp::Failure do
+            IOStreams::Pgp::Reader.open(@file_name, passphrase: 'BAD') { |file| file.read }
+          end
+        end
+
+        it 'fails with stream input' do
+          io = StringIO.new
+          assert_raises NotImplementedError do
+            IOStreams::Pgp::Reader.open(io, passphrase: 'BAD') { |file| file.read }
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/test/pgp_writer_test.rb

@@ -0,0 +1,91 @@
+require_relative 'test_helper'
+
+module Streams
+  class PgpWriterTest < Minitest::Test
+    describe IOStreams::Pgp::Writer do
+      before do
+        file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
+        @data     = File.read(file_name)
+
+        @temp_file = Tempfile.new('iostreams')
+        @file_name = @temp_file.to_path
+      end
+
+      after do
+        @temp_file.delete if @temp_file
+      end
+
+      describe '.open' do
+        it 'writes encrypted text file' do
+          IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', binary: false) do |io|
+            io.write(@data)
+          end
+
+          result = IOStreams::Pgp::Reader.open(@file_name, passphrase: 'receiver_passphrase', binary: false) { |file| file.read }
+          assert_equal @data, result
+        end
+
+        it 'writes encrypted binary file' do
+          binary_file_name = File.join(File.dirname(__FILE__), 'files', 'spreadsheet.xlsx')
+          binary_data      = File.open(binary_file_name, 'rb') { |file| file.read }
+
+          File.open(binary_file_name, 'rb') do |input|
+            IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org') do |output|
+              IOStreams.copy(input, output, 65535)
+            end
+          end
+
+          result = IOStreams::Pgp::Reader.open(@file_name, passphrase: 'receiver_passphrase') { |file| file.read }
+          assert_equal binary_data, result
+        end
+
+        it 'writes and signs encrypted file' do
+          IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
+            io.write(@data)
+          end
+
+          result = IOStreams::Pgp::Reader.open(@file_name, passphrase: 'receiver_passphrase') { |file| file.read }
+          assert_equal @data, result
+        end
+
+        it 'fails with bad signer passphrase' do
+          assert_raises IOStreams::Pgp::Failure do
+            IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', signer: 'sender@example.org', signer_passphrase: 'BAD') do |io|
+              io.write(@data)
+            end
+          end
+        end
+
+        it 'fails with bad recipient' do
+          assert_raises IOStreams::Pgp::Failure do
+            IOStreams::Pgp::Writer.open(@file_name, recipient: 'BAD@example.org', signer: 'sender@example.org', signer_passphrase: 'sender_passphrase') do |io|
+              io.write(@data)
+              # Allow process to terminate
+              sleep 1
+              io.write(@data)
+            end
+          end
+        end
+
+        it 'fails with bad signer' do
+          assert_raises IOStreams::Pgp::Failure do
+            IOStreams::Pgp::Writer.open(@file_name, recipient: 'receiver@example.org', signer: 'BAD@example.org', signer_passphrase: 'sender_passphrase') do |io|
+              io.write(@data)
+            end
+          end
+        end
+
+        it 'fails with stream output' do
+          io = StringIO.new
+          assert_raises NotImplementedError do
+            IOStreams::Pgp::Writer.open(io, recipient: 'receiver@example.org') do |io|
+              io.write(@data)
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/test/test_helper.rb

@@ -17,3 +17,12 @@ SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(
   encoding:    :base64strict
 )
 
+# Test PGP Keys
+unless IOStreams::Pgp.has_key?(email: 'sender@example.org')
+  puts 'Generating test PGP key: sender@example.org'
+  IOStreams::Pgp.generate_key(name: 'Sender', email: 'sender@example.org', passphrase: 'sender_passphrase', key_length: 2048)
+end
+unless IOStreams::Pgp.has_key?(email: 'receiver@example.org')
+  puts 'Generating test PGP key: receiver@example.org'
+  IOStreams::Pgp.generate_key(name: 'Receiver', email: 'receiver@example.org', passphrase: 'receiver_passphrase', key_length: 2048)
+end

data/test/zip_writer_test.rb

@@ -33,7 +33,7 @@ module Streams
           result = nil
           begin
             zin    = ::Zip::InputStream.new(io)
-            entry  = zin.get_next_entry
+            zin.get_next_entry
             result = zin.read
           ensure
             zin.close if zin

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.10.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-20 00:00:00.000000000 Z
+date: 2016-09-27 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: symmetric-encryption
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -56,6 +42,11 @@ files:
 - lib/io_streams/gzip/reader.rb
 - lib/io_streams/gzip/writer.rb
 - lib/io_streams/io_streams.rb
+- lib/io_streams/pgp.rb
+- lib/io_streams/pgp/reader.rb
+- lib/io_streams/pgp/writer.rb
+- lib/io_streams/sftp/reader.rb
+- lib/io_streams/sftp/writer.rb
 - lib/io_streams/streams.rb
 - lib/io_streams/version.rb
 - lib/io_streams/xlsx/reader.rb
@@ -76,11 +67,13 @@ files:
 - test/files/text.zip
 - test/gzip_reader_test.rb
 - test/gzip_writer_test.rb
+- test/pgp_reader_test.rb
+- test/pgp_writer_test.rb
 - test/test_helper.rb
 - test/xlsx_reader_test.rb
 - test/zip_reader_test.rb
 - test/zip_writer_test.rb
-homepage: https://github.com/rocketjob/streams
+homepage: https://github.com/rocketjob/iostreams
 licenses:
 - Apache-2.0
 metadata: {}
@@ -92,7 +85,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -100,10 +93,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Ruby Input and Output streaming with support for Zip, Gzip, and Encryption.
+summary: Ruby file streaming. Supports Text, Zip, Gzip, Xlsx, csv, PGP / GPG and Symmetric
+  Encryption.
 test_files:
 - test/csv_reader_test.rb
 - test/csv_writer_test.rb
@@ -119,6 +113,8 @@ test_files:
 - test/files/text.zip
 - test/gzip_reader_test.rb
 - test/gzip_writer_test.rb
+- test/pgp_reader_test.rb
+- test/pgp_writer_test.rb
 - test/test_helper.rb
 - test/xlsx_reader_test.rb
 - test/zip_reader_test.rb