iostreams 0.17.3 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5c25f9652a6f8e21948dbbf18121093520dff467ea887959f9d9719311046e8
-  data.tar.gz: 1cb168e667ade3af6b85d81661fcfe9d76c1df62a30e1e4bf520a6b9c463d448
+  metadata.gz: e9c9876f8e66281fabd3a26188f1e51d23a12620de461147c9125286d58950cb
+  data.tar.gz: 81f7b6b198f4dfa37b5349de57df74d2004e7984fd176172ac61cce2c7dac9d8
 SHA512:
-  metadata.gz: d86ecd8059751b9080853ab3a6f96f6241af6fb8c610798a0f401b647c08469c1b1f828c08acc723669994be3debc7b2e5e76826146fb70a64594d6197ef00db
-  data.tar.gz: 190fd0f23a41c3f8e30f2982c474601268b0e7e435f51b344ae1925ebef78fc9b03d62514d478d01c689eae74ceaff86b42258150d09ec8eefa16f1bcd3912c2
+  metadata.gz: 91c03cfc1c218d235b2bda1ab304ebc78a8d02dbb9902372a2b5c6b5dd0afb53fcee99ed335cc9361ff2e9db9213cced9fad901a9ecf48bd56f53043792e6096
+  data.tar.gz: 9773ffd737484920543182b8b6a32c4704cc42e83fb2be014e86447eedf184458aa6a570b129b3fa55fc44148c25862730c9af937578469467e4f54f8628ef60

data/lib/io_streams/bzip2/writer.rb

@@ -9,7 +9,7 @@ module IOStreams
           raise(LoadError, "Please install the 'rbzip2' gem for Bzip2 streaming support. #{e.message}")
         end
 
-        if IOStreams.reader_stream?(file_name_or_io)
+        if IOStreams.writer_stream?(file_name_or_io)
           begin
             io = RBzip2.default_adapter::Compressor.new(file_name_or_io)
             block.call(io)
@@ -17,13 +17,12 @@ module IOStreams
             io.close
           end
         else
-          ::File.open(file_name_or_io, 'wb') do |file|
+          IOStreams::File::Writer.open(file_name_or_io) do |file|
             io = RBzip2.default_adapter::Compressor.new(file)
             block.call(io)
             io.close
           end
         end
-
       end
     end
   end

data/lib/io_streams/file/reader.rb

@@ -2,8 +2,6 @@ module IOStreams
   module File
     class Reader
       # Read from a named file
-      # TODO: Add support for mode (text / binary)
-      # TODO: Add encoding support: external_encoding, internal_encoding
       def self.open(file_name, **args, &block)
         raise(ArgumentError, 'File name must be a string') unless file_name.is_a?(String)
 

data/lib/io_streams/file/writer.rb

@@ -2,10 +2,20 @@ module IOStreams
   module File
     class Writer
       # Write to a named file
+      #
+      # Note:
+      #   If an exception is raised whilst the file is being written to the file is removed to
+      #   prevent incomplete / partial files from being created.
       def self.open(file_name, **args, &block)
         raise(ArgumentError, 'File name must be a string') unless file_name.is_a?(String)
 
-        ::File.open(file_name, 'wb', &block)
+        IOStreams.mkpath(file_name)
+        begin
+          ::File.open(file_name, 'wb', &block)
+        rescue StandardError => e
+          File.unlink(file_name) if File.exist?(file_name)
+          raise(e)
+        end
       end
     end
   end

data/lib/io_streams/gzip/writer.rb

@@ -4,6 +4,7 @@ module IOStreams
       # Write to a file / stream, compressing with GZip
       def self.open(file_name_or_io, **args, &block)
         unless IOStreams.writer_stream?(file_name_or_io)
+          IOStreams.mkpath(file_name_or_io)
           Zlib::GzipWriter.open(file_name_or_io, &block)
         else
           begin

data/lib/io_streams/io_streams.rb

@@ -1,9 +1,5 @@
 require 'concurrent'
-# Load Symmetric Encryption if present so that its reader and writer can be registered
-begin
-  require 'symmetric-encryption'
-rescue LoadError
-end
+require 'fileutils'
 
 # Streaming library for Ruby
 #
@@ -473,6 +469,12 @@ module IOStreams
     end
   end
 
+  # Used by writers that can write directly to file to create the target path
+  def self.mkpath(file_name)
+    path = ::File.dirname(file_name)
+    FileUtils.mkdir_p(path) unless ::File.exist?(path)
+  end
+
   private
 
   # A registry to hold formats for processing files during upload or download
@@ -484,7 +486,7 @@ module IOStreams
 
   # Returns a reader or writer stream
   def self.stream(type, file_name_or_io, streams:, file_name:, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
-    # TODO: Add support for different schemes, such as file://, s3://, sftp://
+    raise(ArgumentError, 'IOStreams call is missing mandatory block') if block.nil?
 
     streams = streams_for_file_name(file_name) if streams.nil? && file_name
 
@@ -565,6 +567,7 @@ module IOStreams
 
   # Register File extensions
   register_extension(:bz2, IOStreams::Bzip2::Reader, IOStreams::Bzip2::Writer)
+  register_extension(:enc, IOStreams::SymmetricEncryption::Reader, IOStreams::SymmetricEncryption::Writer)
   register_extension(:gz, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
   register_extension(:gzip, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
   register_extension(:zip, IOStreams::Zip::Reader, IOStreams::Zip::Writer)
@@ -573,12 +576,6 @@ module IOStreams
   register_extension(:xlsx, IOStreams::Xlsx::Reader, nil)
   register_extension(:xlsm, IOStreams::Xlsx::Reader, nil)
 
-  # Use Symmetric Encryption to encrypt of decrypt files with the `enc` extension
-  # when the gem `symmetric-encryption` has been loaded.
-  if defined?(SymmetricEncryption)
-    register_extension(:enc, SymmetricEncryption::Reader, SymmetricEncryption::Writer)
-  end
-
   # Support URI schemes
   #
   # Examples:

data/lib/io_streams/pgp/writer.rb

@@ -45,38 +45,39 @@ module IOStreams
       # compress_level: [Integer]
       #   Compression level
       #   Default: 6
-      def self.open(file_name_or_io, recipient:, signer: default_signer, signer_passphrase: default_signer_passphrase, binary: true, compression: :zip, compress_level: 6)
+      def self.open(file_name, recipient:, signer: default_signer, signer_passphrase: default_signer_passphrase, binary: true, compression: :zip, compress_level: 6)
         compress_level = 0 if compression == :none
-        if IOStreams.writer_stream?(file_name_or_io)
+        if IOStreams.writer_stream?(file_name)
           raise(NotImplementedError, 'Can only PGP Encrypt directly to a file name. Output to streams are not yet supported.')
-        else
-          # Write to stdin, with encrypted contents being written to the file
-          command = "#{IOStreams::Pgp.executable} --batch --no-tty --yes --encrypt"
-          command << " --sign --local-user \"#{signer}\"" if signer
-          if signer_passphrase
-            command << " --pinentry-mode loopback" if IOStreams::Pgp.pgp_version.to_f >= 2.1
-            command << " --passphrase \"#{signer_passphrase}\""
-          end
-          command << " -z #{compress_level}" if compress_level != 6
-          command << " --compress-algo #{compression}" unless compression == :none
-          command << " --recipient \"#{recipient}\" -o \"#{file_name_or_io}\""
+        end
+        IOStreams.mkpath(file_name)
 
-          IOStreams::Pgp.logger.debug { "IOStreams::Pgp::Writer.open: #{command}" } if IOStreams::Pgp.logger
+        # Write to stdin, with encrypted contents being written to the file
+        command = "#{IOStreams::Pgp.executable} --batch --no-tty --yes --encrypt"
+        command << " --sign --local-user \"#{signer}\"" if signer
+        if signer_passphrase
+          command << " --pinentry-mode loopback" if IOStreams::Pgp.pgp_version.to_f >= 2.1
+          command << " --passphrase \"#{signer_passphrase}\""
+        end
+        command << " -z #{compress_level}" if compress_level != 6
+        command << " --compress-algo #{compression}" unless compression == :none
+        command << " --recipient \"#{recipient}\" -o \"#{file_name}\""
 
-          Open3.popen2e(command) do |stdin, out, waith_thr|
-            begin
-              stdin.binmode if binary
-              yield(stdin)
-              stdin.close
-            rescue Errno::EPIPE
-              # Ignore broken pipe because gpg terminates early due to an error
-              ::File.delete(file_name_or_io) if ::File.exist?(file_name_or_io)
-              raise(Pgp::Failure, "GPG Failed writing to encrypted file: #{file_name_or_io}: #{out.read.chomp}")
-            end
-            unless waith_thr.value.success?
-              ::File.delete(file_name_or_io) if ::File.exist?(file_name_or_io)
-              raise(Pgp::Failure, "GPG Failed to create encrypted file: #{file_name_or_io}: #{out.read.chomp}")
-            end
+        IOStreams::Pgp.logger.debug { "IOStreams::Pgp::Writer.open: #{command}" } if IOStreams::Pgp.logger
+
+        Open3.popen2e(command) do |stdin, out, waith_thr|
+          begin
+            stdin.binmode if binary
+            yield(stdin)
+            stdin.close
+          rescue Errno::EPIPE
+            # Ignore broken pipe because gpg terminates early due to an error
+            ::File.delete(file_name) if ::File.exist?(file_name)
+            raise(Pgp::Failure, "GPG Failed writing to encrypted file: #{file_name}: #{out.read.chomp}")
+          end
+          unless waith_thr.value.success?
+            ::File.delete(file_name) if ::File.exist?(file_name)
+            raise(Pgp::Failure, "GPG Failed to create encrypted file: #{file_name}: #{out.read.chomp}")
           end
         end
       end

data/lib/io_streams/s3/writer.rb

@@ -67,7 +67,7 @@ module IOStreams
       #   aborted and this error is raised.  The raised error has a `#errors`
       #   method that returns the failures that caused the upload to be
       #   aborted.
-      def self.open(uri, region: nil, **args, &block)
+      def self.open(uri, region: nil, **args)
         raise(ArgumentError, 'file_name must be a URI string') unless uri.is_a?(String)
 
         IOStreams::S3.load_dependencies
@@ -75,7 +75,10 @@ module IOStreams
         options = IOStreams::S3.parse_uri(uri)
         s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
         object  = s3.bucket(options[:bucket]).object(options[:key])
-        object.upload_stream(args, &block)
+        object.upload_stream(args) do |s3|
+          s3.binmode
+          yield(s3)
+        end
       end
     end
   end

data/lib/io_streams/symmetric_encryption/reader.rb

@@ -0,0 +1,22 @@
+module IOStreams
+  module SymmetricEncryption
+    class Reader
+      # read from a file/stream using Symmetric Encryption
+      def self.open(file_name_or_io, **args, &block)
+        begin
+          require 'symmetric-encryption' unless defined?(SymmetricEncryption)
+        rescue LoadError => e
+          raise(LoadError, "Please install the 'symmetric-encryption' gem for .enc streaming support. #{e.message}")
+        end
+
+        if IOStreams.reader_stream?(file_name_or_io)
+          ::SymmetricEncryption::Reader.open(file_name_or_io, **args, &block)
+        else
+          IOStreams::File::Reader.open(file_name_or_io) do |file|
+            ::SymmetricEncryption::Reader.open(file, **args, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/io_streams/symmetric_encryption/writer.rb

@@ -0,0 +1,25 @@
+module IOStreams
+  module SymmetricEncryption
+    class Writer
+      # Write to file/stream using Symmetric Encryption
+      def self.open(file_name_or_io, compress: nil, **args, &block)
+        begin
+          require 'symmetric-encryption' unless defined?(SymmetricEncryption)
+        rescue LoadError => e
+          raise(LoadError, "Please install the 'symmetric-encryption' gem for .enc streaming support. #{e.message}")
+        end
+
+        if IOStreams.writer_stream?(file_name_or_io)
+          compress = true if compress.nil?
+          ::SymmetricEncryption::Writer.open(file_name_or_io, compress: compress, **args, &block)
+        else
+          compress = !IOStreams.compressed?(file_name_or_io) if compress.nil?
+
+          IOStreams::File::Writer.open(file_name_or_io) do |file|
+            ::SymmetricEncryption::Writer.open(file, compress: compress, **args, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/io_streams/version.rb

@@ -1,3 +1,3 @@
 module IOStreams
-  VERSION = '0.17.3'
+  VERSION = '0.18.0'
 end

data/lib/io_streams/zip/writer.rb

@@ -69,6 +69,7 @@ module IOStreams
 
       else
         def self.write_file(file_name, zip_file_name, &block)
+          IOStreams.mkpath(file_name)
           zos = ::Zip::OutputStream.new(file_name)
           zos.put_next_entry(zip_file_name)
           block.call(zos)

data/lib/iostreams.rb

@@ -42,6 +42,10 @@ module IOStreams
     autoload :Reader, 'io_streams/row/reader'
     autoload :Writer, 'io_streams/row/writer'
   end
+  module SymmetricEncryption
+    autoload :Reader, 'io_streams/symmetric_encryption/reader'
+    autoload :Writer, 'io_streams/symmetric_encryption/writer'
+  end
   module Xlsx
     autoload :Reader, 'io_streams/xlsx/reader'
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iostreams
 version: !ruby/object:Gem::Version
-  version: 0.17.3
+  version: 0.18.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-22 00:00:00.000000000 Z
+date: 2019-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -58,7 +58,8 @@ files:
 - lib/io_streams/s3/writer.rb
 - lib/io_streams/sftp/reader.rb
 - lib/io_streams/sftp/writer.rb
-- lib/io_streams/streams.rb
+- lib/io_streams/symmetric_encryption/reader.rb
+- lib/io_streams/symmetric_encryption/writer.rb
 - lib/io_streams/tabular.rb
 - lib/io_streams/tabular/header.rb
 - lib/io_streams/tabular/parser/array.rb

data/lib/io_streams/streams.rb

@@ -1,109 +0,0 @@
-module IOStreams
-  # Contains behavior for streams
-  #
-  # When a file is being read the streams are processed from right to left.
-  # When writing a file streams are processed left to right.
-  # For example:
-  #   file.gz.enc ==> [:gz, :enc]
-  #     Read:  Unencrypt, then Gunzip
-  #     Write: GZip, then Encrypt
-  class Streams
-
-    # Returns [Streams] collection of streams to process against the file
-    #
-    def self.streams_for_file_name(file_name)
-
-    end
-
-    # Create a processing stream given:
-    # - No stream. Defaults to :file
-    # - A single String implies a file_name and the streams will be created based on the file_name
-    # - One or more symbols or hashes for a stream
-    # - One or more arrays for streams
-    def initialize(*args)
-      if args.size == 0
-        @streams = [:file]
-      elsif args.size == 1
-        stream = args.first
-        if stream
-          @stream = stream.is_a?(String) ? streams_for_file_name(stream) : Array(stream)
-        else
-          @streams = [:file]
-        end
-      else
-        @streams = streams
-      end
-      @streams.flatten!
-    end
-
-    def delimited?
-
-    end
-
-    def delete(stream)
-
-    end
-
-    # Add another stream for processing
-    def <<(stream)
-
-    end
-
-    # Add a stream for processing
-    def unshift(stream)
-
-    end
-
-    private
-
-    # Return the Stream klass for the specified hash or symbol
-    # Parameters
-    #   stream [Hash|Symbol]
-    def stream_for(stream)
-      if stream.is_a?(Symbol)
-        registered_klass(stream, {})
-      else
-        registered_klass(@stream.first)
-      end
-    end
-
-    # Returns [Array] the formats required to process the file by looking at
-    # its extension(s)
-    #
-    # Extensions supported:
-    #   .zip       Zip File                                   [ :zip ]
-    #   .gz, .gzip GZip File                                  [ :gzip ]
-    #   .enc       File Encrypted using symmetric encryption  [ :enc ]
-    #   other      All other extensions will be returned as:  [ :file ]
-    #
-    # When a file is encrypted, it may also be compressed:
-    #   .zip.enc  [ :zip, :enc ]
-    #   .gz.enc   [ :gz,  :enc ]
-    #
-    # Example Zip file:
-    #   IOStreams.streams_for_file_name('myfile.zip')
-    #   => [ :zip ]
-    #
-    # Example Encrypted Gzip file:
-    #   IOStreams.streams_for_file_name('myfile.csv.gz.enc')
-    #   => [ :gz, :enc ]
-    #
-    # Example plain text / binary file:
-    #   IOStreams.streams_for_file_name('myfile.csv')
-    #   => [ :file ]
-    def streams_for_file_name(file_name)
-      raise ArgumentError.new("Cannot auto-detect streams when already a stream: #{file_name.inspect}") if reader_stream?(file_name)
-
-      parts      = file_name.split('.')
-      extensions = []
-      while extension = parts.pop
-        break unless @extensions[extension.to_sym]
-        extensions.unshift(extension.to_sym)
-      end
-      extensions << :file if extensions.size == 0
-      extensions
-    end
-
-
-  end
-end