iostreams 0.20.3 → 1.0.0.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92d4f1b2dd506273b8e9cde6990775d416cfe8ab9fd6f0dd03811b134bacec73
-  data.tar.gz: 30f4b7dd513c9a6b806df6b478c9fc1997b30436f6ceff99788178d1972cae31
+  metadata.gz: 158fef0e14167719cd27b7b1cb0445bfb08403a3afc4490fad74a0e0fc9ab67b
+  data.tar.gz: b1f3db75a4773591507706befa66122204ffda1642136c79427671a4bf240605
 SHA512:
-  metadata.gz: 81709e604218a4d2d764e3d237a48293bc35beffc50c17a1a2ba524f7b4ac8311d94b605f476f9e958eb259f87c2d4b16cc734b71ed7b6f66276ca4f262ccbcd
-  data.tar.gz: 2d3a4e9e9bd11018d6eee4baecdd80bede728f4aca4c0e9b050c6045cda7e4e75f6a6dc6595021bc30a6419bfbb39cbe020ec0885d913c2c8531f731e9e87f4e
+  metadata.gz: 556a898a859c9447822cb0d3aeea58ce6d839536d43fe3ef46fdf94fb1794d50d596da713fd6eef4203ef84ed94790ad8d69581d54c6a167bc9cea57e4d6b976
+  data.tar.gz: 7c38374f8ec37252a3b5428472235cf44a3a6379b5190776d766f447f339f16f3f85b69713c77592601466f82eac998c227de8136cecc64f82081e00f540314b

data/lib/io_streams/bzip2/reader.rb

@@ -1,28 +1,16 @@
 module IOStreams
   module Bzip2
-    class Reader
-      # Read from a Bzip2 file or stream, decompressing the contents as it is read
-      def self.open(file_name_or_io, **args, &block)
-        begin
-          require 'rbzip2' unless defined?(RBzip2)
-        rescue LoadError => e
-          raise(LoadError, "Please install the 'rbzip2' gem for Bzip2 streaming support. #{e.message}")
-        end
+    class Reader < IOStreams::Reader
+      # Read from a Bzip2 stream, decompressing the contents as it is read
+      def self.stream(input_stream, **_args)
+        Utils.load_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
 
-        if IOStreams.reader_stream?(file_name_or_io)
-          begin
-            io = RBzip2.default_adapter::Decompressor.new(file_name_or_io)
-            block.call(io)
-          ensure
-            io.close if io && (io.respond_to?(:closed?) && !io.closed?)
-          end
-        else
-          ::File.open(file_name_or_io, 'rb') do |file|
-            io = RBzip2.default_adapter::Decompressor.new(file)
-            block.call(io)
-          end
+        begin
+          io = RBzip2.default_adapter::Decompressor.new(input_stream)
+          yield io
+        ensure
+          io&.close
         end
-
       end
     end
   end

data/lib/io_streams/bzip2/writer.rb

@@ -1,27 +1,15 @@
 module IOStreams
   module Bzip2
-    class Writer
-      # Write to a file / stream, compressing with Bzip2
-      def self.open(file_name_or_io, **args, &block)
-        begin
-          require 'rbzip2' unless defined?(RBzip2)
-        rescue LoadError => e
-          raise(LoadError, "Please install the 'rbzip2' gem for Bzip2 streaming support. #{e.message}")
-        end
+    class Writer < IOStreams::Writer
+      # Write to a stream, compressing with Bzip2
+      def self.stream(input_stream, original_file_name: nil, **_args)
+        Utils.load_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
 
-        if IOStreams.writer_stream?(file_name_or_io)
-          begin
-            io = RBzip2.default_adapter::Compressor.new(file_name_or_io)
-            block.call(io)
-          ensure
-            io.close
-          end
-        else
-          IOStreams::File::Writer.open(file_name_or_io) do |file|
-            io = RBzip2.default_adapter::Compressor.new(file)
-            block.call(io)
-            io.close
-          end
+        begin
+          io = RBzip2.default_adapter::Compressor.new(input_stream)
+          yield io
+        ensure
+          io&.close
         end
       end
     end

data/lib/io_streams/deprecated.rb

@@ -0,0 +1,217 @@
+module IOStreams
+  UTF8_ENCODING   = Encoding.find('UTF-8').freeze
+  BINARY_ENCODING = Encoding.find('BINARY').freeze
+
+  # Deprecated IOStreams from v0.x. Do not use, will be removed soon.
+  module Deprecated
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+
+      # DEPRECATED. Use `#path` or `#io`
+      # Examples:
+      #   IOStreams.path("data.zip").reader { |f| f.read(100) }
+      #
+      #   IOStreams.path(file_name).option(:encode, encoding: "BINARY").reader { |f| f.read(100) }
+      #
+      #   io_stream = StringIO.new("Hello World")
+      #   IOStreams.stream(io_stream).reader { |f| f.read(100) }
+      def reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.reader(&block)
+      end
+
+      # DEPRECATED
+      def each_line(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.each_line(**args, &block)
+      end
+
+      # DEPRECATED
+      def each_row(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.each_row(**args, &block)
+      end
+
+      # DEPRECATED
+      def each_record(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.each_record(**args, &block)
+      end
+
+      # DEPRECATED. Use `#path` or `#io`
+      # Examples:
+      #   IOStreams.path("data.zip").writer { |f| f.write("Hello World") }
+      #
+      #   IOStreams.path(file_name).option(:encode, encoding: "BINARY").writer { |f| f.write("Hello World") }
+      #
+      #   io_stream = StringIO.new("Hello World")
+      #   IOStreams.stream(io_stream).writer { |f| f.write("Hello World") }
+      def writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.writer(&block)
+      end
+
+      # DEPRECATED
+      def line_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.line_writer(**args, &block)
+      end
+
+      # DEPRECATED
+      def row_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.row_writer(**args, &block)
+      end
+
+      # DEPRECATED
+      def record_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.record_writer(**args, &block)
+      end
+
+      # Copies the source file/stream to the target file/stream.
+      # Returns [Integer] the number of bytes copied
+      #
+      # Example: Copy between 2 files
+      #   IOStreams.copy('a.csv', 'b.csv')
+      #
+      # Example: Read content from a Xlsx file and write it out in CSV form.
+      #   IOStreams.copy('a.xlsx', 'b.csv')
+      #
+      # Example:
+      #   # Read content from a JSON file and write it out in CSV form.
+      #   #
+      #   # The output header for the CSV file is extracted from the first row in the JSON file.
+      #   # If the first JSON row does not contain all the column names then they will be ignored
+      #   # for the rest of the file.
+      #   IOStreams.copy('a.json', 'b.csv')
+      #
+      # Example:
+      #   # Read a PSV file and write out a CSV file from it.
+      #   IOStreams.copy('a.psv', 'b.csv')
+      #
+      # Example:
+      #   # Copy between 2 files, encrypting the target file with Symmetric Encryption
+      #   # Since the target file_name already includes `.enc` in the filename, it is automatically
+      #   # encrypted.
+      #   IOStreams.copy('a.csv', 'b.csv.enc')
+      #
+      # Example:
+      #   # Copy between 2 files, encrypting the target file with Symmetric Encryption
+      #   # Since the target file_name does not include `.enc` in the filename, to encrypt it
+      #   # the encryption stream is added.
+      #   IOStreams.copy('a.csv', 'b', target_options: [:enc])
+      #
+      # Example:
+      #   # Copy between 2 files, encrypting the target file with Symmetric Encryption
+      #   # Since the target file_name does not include `.enc` in the filename, to encrypt it
+      #   # the encryption stream is added, along with the optional compression option.
+      #   IOStreams.copy('a.csv', 'b', target_options: [enc: { compress: true }])
+      #
+      # Example:
+      #   # Create a pgp encrypted file.
+      #   # For PGP Encryption the recipients email address is required.
+      #   IOStreams.copy('a.xlsx', 'b.csv.pgp', target_options: [:csv, pgp: { recipient_email: 'user@nospam.org' }])
+      #
+      # Example: Copy between 2 existing streams
+      #   IOStreams.reader('a.csv') do |source_stream|
+      #     IOStreams.writer('b.csv.enc') do |target_stream|
+      #       IOStreams.copy(source_stream, target_stream)
+      #     end
+      #   end
+      #
+      # Example:
+      #   # Copy between 2 csv files, reducing the number of columns present and encrypting the
+      #   # target file with Symmetric Encryption
+      #   output_headers = %w[name address]
+      #   IOStreams.copy(
+      #     'a.csv',
+      #     'b.csv.enc',
+      #     target_options: [csv:{headers: output_headers}, enc: {compress: true}]
+      #   )
+      #
+      # Example:
+      #   # Copy a locally encrypted file to AWS S3.
+      #   # Decrypts the file, then compresses it with gzip as it is being streamed into S3.
+      #   # Useful for when the entire bucket is encrypted on S3.
+      #   IOStreams.copy('a.csv.enc', 's3://my_bucket/b.csv.gz')
+      def copy(source_file_name_or_io, target_file_name_or_io, buffer_size: nil, source_options: {}, target_options: {})
+        # TODO: prevent stream conversions when reader and writer streams are the same!
+        reader(source_file_name_or_io, **source_options) do |source_stream|
+          writer(target_file_name_or_io, **target_options) do |target_stream|
+            IO.copy_stream(source_stream, target_stream)
+          end
+        end
+      end
+
+      # DEPRECATED
+      def reader_stream?(file_name_or_io)
+        file_name_or_io.respond_to?(:read)
+      end
+
+      # DEPRECATED
+      def writer_stream?(file_name_or_io)
+        file_name_or_io.respond_to?(:write)
+      end
+
+      # DEPRECATED. Use Path#compressed?
+      def compressed?(file_name)
+        Path.new(file_name).compressed?
+      end
+
+      # DEPRECATED. Use Path#encrypted?
+      def encrypted?(file_name)
+        Path.new(file_name).encrypted?
+      end
+
+      # DEPRECATED
+      def line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.line_reader(**args, &block)
+      end
+
+      # DEPRECATED
+      def row_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.line_reader(**args, &block)
+      end
+
+      # DEPRECATED
+      def record_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
+        path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
+        path.record_reader(**args, &block)
+      end
+
+      private
+
+      def build_path(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil)
+        path = new(file_name_or_io)
+        path.file_name(file_name) if file_name
+
+        apply_old_style_streams(path, streams) if streams
+
+        if encoding || encode_cleaner || encode_replace
+          path.option_or_stream(:encode, encoding: encoding, cleaner: encode_cleaner, replace: encode_replace)
+        end
+
+        path
+      end
+
+      # Applies old form streams to the path
+      def apply_old_style_streams(path, streams)
+        if streams.is_a?(Symbol)
+          path.stream(streams)
+        elsif streams.is_a?(Array)
+          streams.each { |stream| apply_old_style_streams(path, stream) }
+        elsif streams.is_a?(Hash)
+          streams.each_pair { |stream, options| path.stream(stream, options) }
+        else
+          raise ArgumentError, "Invalid old style stream supplied: #{params.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/io_streams/encode/reader.rb

@@ -1,22 +1,18 @@
 module IOStreams
   module Encode
-    class Reader
+    class Reader < IOStreams::Reader
       attr_reader :encoding, :cleaner
 
       NOT_PRINTABLE = Regexp.compile(/[^[:print:]|\r|\n]/).freeze
       # Builtin strip options to apply after encoding the read data.
       CLEANSE_RULES = {
         # Strips all non printable characters
-        printable: -> (data) { data.gsub!(NOT_PRINTABLE, '') || data }
-      }
+        printable: ->(data) { data.gsub!(NOT_PRINTABLE, '') || data }
+      }.freeze
 
       # Read a line at a time from a file or stream
-      def self.open(file_name_or_io, **args)
-        if file_name_or_io.is_a?(String)
-          IOStreams::File::Reader.open(file_name_or_io) { |io| yield new(io, **args) }
-        else
-          yield new(file_name_or_io, **args)
-        end
+      def self.stream(input_stream, original_file_name: nil, **args)
+        yield new(input_stream, **args)
       end
 
       # Apply encoding conversion when reading a stream.
@@ -33,26 +29,26 @@ module IOStreams
       #     Etc.
       #     Default: 'UTF-8'
       #
-      #   encode_replace: [String]
+      #   replace: [String]
       #     The character to replace with when a character cannot be converted to the target encoding.
       #     nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
       #     Default: nil
       #
-      #   encode_cleaner: [nil|symbol|Proc]
+      #   cleaner: [nil|symbol|Proc]
       #     Cleanse data read from the input stream.
       #     nil:           No cleansing
       #     :printable Cleanse all non-printable characters except \r and \n
       #     Proc/lambda    Proc to call after every read to cleanse the data
       #     Default: nil
-      def initialize(input_stream, encoding: 'UTF-8', encode_cleaner: nil, encode_replace: nil)
-        @input_stream = input_stream
-        @cleaner      = self.class.extract_cleaner(encode_cleaner)
+      def initialize(input_stream, encoding: 'UTF-8', cleaner: nil, replace: nil)
+        super(input_stream)
 
+        @cleaner          = self.class.extract_cleaner(cleaner)
         @encoding         = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
-        @encoding_options = encode_replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: encode_replace}
+        @encoding_options = replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: replace}
 
         # More efficient read buffering only supported when the input stream `#read` method supports it.
-        if encode_replace.nil? && !@input_stream.method(:read).arity.between?(0, 1)
+        if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1)
           @read_cache_buffer = ''.encode(@encoding)
         else
           @read_cache_buffer = nil

data/lib/io_streams/encode/writer.rb

@@ -1,15 +1,11 @@
 module IOStreams
   module Encode
-    class Writer
+    class Writer < IOStreams::Writer
       attr_reader :encoding, :cleaner
 
       # Write a line at a time to a file or stream
-      def self.open(file_name_or_io, **args)
-        if file_name_or_io.is_a?(String)
-          IOStreams::File::Writer.open(file_name_or_io) { |io| yield new(io, **args) }
-        else
-          yield new(file_name_or_io, **args)
-        end
+      def self.stream(input_stream, original_file_name: nil, **args)
+        yield new(input_stream, **args)
       end
 
       # A delimited stream writer that will write to the supplied output stream
@@ -27,23 +23,23 @@ module IOStreams
       #     Etc.
       #     Default: 'UTF-8'
       #
-      #   encode_replace: [String]
+      #   replace: [String]
       #     The character to replace with when a character cannot be converted to the target encoding.
       #     nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
       #     Default: nil
       #
-      #   encode_cleaner: [nil|symbol|Proc]
+      #   cleaner: [nil|symbol|Proc]
       #     Cleanse data read from the input stream.
       #     nil:           No cleansing
       #     :printable Cleanse all non-printable characters except \r and \n
       #     Proc/lambda    Proc to call after every read to cleanse the data
       #     Default: nil
-      def initialize(output_stream, encoding: 'UTF-8', encode_cleaner: nil, encode_replace: nil)
-        @output_stream = output_stream
-        @cleaner       = ::IOStreams::Encode::Reader.send(:extract_cleaner, encode_cleaner)
+      def initialize(output_stream, encoding: 'UTF-8', cleaner: nil, replace: nil)
+        super(output_stream)
 
+        @cleaner          = ::IOStreams::Encode::Reader.send(:extract_cleaner, cleaner)
         @encoding         = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
-        @encoding_options = encode_replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: encode_replace}
+        @encoding_options = replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: replace}
       end
 
       # Write a line to the output stream

data/lib/io_streams/errors.rb

@@ -1,22 +1,22 @@
 module IOStreams
   module Errors
-    class Error < StandardError;
+    class Error < StandardError
     end
 
-    class InvalidHeader < Error;
+    class InvalidHeader < Error
     end
 
-    class MissingHeader < Error;
+    class MissingHeader < Error
     end
 
-    class TypeMismatch < Error;
+    class TypeMismatch < Error
     end
 
-    class CommunicationsFailure < Error;
+    class CommunicationsFailure < Error
     end
 
     # When the specified delimiter is not found in the supplied stream / file
-    class DelimiterNotFound < Error;
+    class DelimiterNotFound < Error
     end
   end
 end

data/lib/io_streams/gzip/reader.rb

@@ -1,20 +1,13 @@
 module IOStreams
   module Gzip
-    class Reader
-      # Read from a gzip file or stream, decompressing the contents as it is read
-      def self.open(file_name_or_io, **args, &block)
-        unless IOStreams.reader_stream?(file_name_or_io)
-          ::Zlib::GzipReader.open(file_name_or_io, &block)
-        else
-          begin
-            io = ::Zlib::GzipReader.new(file_name_or_io)
-            block.call(io)
-          ensure
-            io.close if io && (io.respond_to?(:closed?) && !io.closed?)
-          end
-        end
+    class Reader < IOStreams::Reader
+      # Read from a gzip stream, decompressing the contents as it is read
+      def self.stream(input_stream, original_file_name: nil, &block)
+        io = ::Zlib::GzipReader.new(input_stream)
+        block.call(io)
+      ensure
+        io&.close
       end
-
     end
   end
 end