iostreams 0.7.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dbaf3d597b1fc3ad00a9c371f932236332d3a1b4
+  data.tar.gz: ee333efa4fe7737d77dfdaeff353e4811fa4f298
+SHA512:
+  metadata.gz: 801f73a031c8fbadd38bf7939fa90dd6fca5360fef7c8f77b08c65ed72bdb142deaecccf8c9a3dd2c7f47af4725397635959cdb14a87e4af603967ced96709a2
+  data.tar.gz: 7abef1b3814194b1fa69d05e5c007aaca2b386329fb2d4af3a228dcedccd465501c5a2036f510b7e3602c96ad54b030fe7385dbb096a566e3a26ead686d3f167

data/README.md

@@ -0,0 +1,56 @@
+# iostreams
+
+Ruby Input and Output streaming with support for Zip, Gzip, and Encryption.
+
+## Status
+
+Alpha - Feedback on the API is welcome. API will change.
+
+## Introduction
+
+`iostreams` allows files to be read and written in a streaming fashion to reduce
+memory overhead. It supports reading and writing of Zip, GZip and encrypted files.
+
+These streams can be chained together just like piped programs in linux.
+This allows one stream to read the file, another stream to decrypt the file and
+then a third stream to decompress the result.
+
+The objective is that all of these streaming processes are performed used streaming
+so that only portions of the file are loaded into memory at a time.
+Where possible each stream never goes to disk, which for example could expose
+un-encrypted data.
+
+## Notes
+
+* Due to the nature of Zip, both its Reader and Writer methods will create
+  a temp file when reading from or writing to a stream.
+  Recommended to use Gzip over Zip since it can be streamed.
+
+## Meta
+
+* Code: `git clone git://github.com/rocketjob/iostreams.git`
+* Home: <https://github.com/rocketjob/iostreams>
+* Issues: <http://github.com/rocketjob/iostreams/issues>
+* Gems: <http://rubygems.org/gems/iostreams>
+
+This project uses [Semantic Versioning](http://semver.org/).
+
+## Author
+
+[Reid Morrison](https://github.com/reidmorrison)
+
+## License
+
+Copyright 2015 Reid Morrison
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,28 @@
+require 'rake/clean'
+require 'rake/testtask'
+
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require 'io_streams/version'
+
+task :gem do
+  system "gem build iostreams.gemspec"
+end
+
+task :publish => :gem do
+  system "git tag -a v#{IOStreams::VERSION} -m 'Tagging #{IOStreams::VERSION}'"
+  system "git push --tags"
+  system "gem push iostreams-#{IOStreams::VERSION}.gem"
+  system "rm iostreams-#{IOStreams::VERSION}.gem"
+end
+
+desc "Run Test Suite"
+task :test do
+  Rake::TestTask.new(:functional) do |t|
+    t.test_files = FileList['test/**/*_test.rb']
+    t.verbose    = true
+  end
+
+  Rake::Task['functional'].invoke
+end
+
+task :default => :test

data/lib/io_streams/file/reader.rb

@@ -0,0 +1,15 @@
+module IOStreams
+  module File
+    class Reader
+      # Read from a file or stream
+      def self.open(file_name_or_io, _=nil, &block)
+        unless file_name_or_io.respond_to?(:read)
+          ::File.open(file_name_or_io, 'rb', &block)
+        else
+          block.call(file_name_or_io)
+        end
+      end
+
+    end
+  end
+end

data/lib/io_streams/file/writer.rb

@@ -0,0 +1,15 @@
+module IOStreams
+  module File
+    class Writer
+      # Write to a file or stream
+      def self.open(file_name_or_io, _=nil, &block)
+        unless file_name_or_io.respond_to?(:write)
+          ::File.open(file_name_or_io, 'wb', &block)
+        else
+          block.call(file_name_or_io)
+        end
+      end
+
+    end
+  end
+end

data/lib/io_streams/gzip/reader.rb

@@ -0,0 +1,20 @@
+module IOStreams
+  module Gzip
+    class Reader
+      # Read from a gzip file or stream, decompressing the contents as it is read
+      def self.open(file_name_or_io, _=nil, &block)
+        unless file_name_or_io.respond_to?(:read)
+          ::Zlib::GzipReader.open(file_name_or_io, &block)
+        else
+          begin
+            io = ::Zlib::GzipReader.new(file_name_or_io)
+            block.call(io)
+          ensure
+            io.close if io && (io.respond_to?(:closed?) && !io.closed?)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/io_streams/gzip/writer.rb

@@ -0,0 +1,20 @@
+module IOStreams
+  module Gzip
+    class Writer
+      # Write to a file / stream, compressing with GZip
+      def self.open(file_name_or_io, _=nil, &block)
+        unless file_name_or_io.respond_to?(:write)
+          Zlib::GzipWriter.open(file_name_or_io, &block)
+        else
+          begin
+            io = Zlib::GzipWriter.new(file_name_or_io)
+            block.call(io)
+          ensure
+            io.close if io && (io.respond_to?(:closed?) && !io.closed?)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/io_streams/io_streams.rb

@@ -0,0 +1,243 @@
+require 'thread_safe'
+module IOStreams
+  # A registry to hold formats for processing files during upload or download
+  @@extensions = ThreadSafe::Hash.new
+
+  # Returns [Array] the formats required to process the file by looking at
+  # its extension(s)
+  #
+  # Extensions supported:
+  #   .zip       Zip File                                   [ :zip ]
+  #   .gz, .gzip GZip File                                  [ :gzip ]
+  #   .enc       File Encrypted using symmetric encryption  [ :enc ]
+  #   other      All other extensions will be returned as:  [ :file ]
+  #
+  # When a file is encrypted, it may also be compressed:
+  #   .zip.enc  [ :zip, :enc ]
+  #   .gz.enc   [ :gz,  :enc ]
+  #
+  # Example Zip file:
+  #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.zip')
+  #   => [ :zip ]
+  #
+  # Example Encrypted Gzip file:
+  #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv.gz.enc')
+  #   => [ :gz, :enc ]
+  #
+  # Example plain text / binary file:
+  #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv')
+  #   => [ :file ]
+  def self.streams_for_file_name(file_name)
+    raise ArgumentError.new("File name cannot be nil") if file_name.nil?
+    raise ArgumentError.new("RocketJob Cannot detect file format when uploading to stream: #{file_name.inspect}") if file_name.respond_to?(:read)
+    parts = file_name.split('.')
+    extensions = []
+    while extension = parts.pop
+      break unless @@extensions[extension.to_sym]
+      extensions.unshift(extension.to_sym)
+    end
+    extensions << :file if extensions.size == 0
+    extensions
+  end
+
+  Extension = Struct.new(:reader_class, :writer_class)
+
+  # Register a file extension and the reader and writer classes to use to format it
+  #
+  # Example:
+  #   # MyXls::Reader and MyXls::Writer must implement .open
+  #   register_extension(:xls, MyXls::Reader, MyXls::Writer)
+  def self.register_extension(extension, reader_class, writer_class)
+    raise "Invalid extension #{extension.inspect}" unless extension.to_s =~ /\A\w+\Z/
+    @@extensions[extension.to_sym] = Extension.new(reader_class, writer_class)
+  end
+
+  # De-Register a file extension
+  #
+  # Returns [Symbol] the extension removed, or nil if the extension was not registered
+  #
+  # Example:
+  #   register_extension(:xls)
+  def self.deregister_extension(extension)
+    raise "Invalid extension #{extension.inspect}" unless extension.to_s =~ /\A\w+\Z/
+    @@extensions.delete(extension.to_sym)
+  end
+
+  # Returns a Reader for reading a file / stream
+  #
+  # Parameters
+  #   file_name_or_io [String|IO]
+  #     The file_name of the file to write to, or an IO Stream that implements
+  #     #read.
+  #
+  #   streams [Symbol|Array]
+  #     The formats/streams that be used to convert the data whilst it is
+  #     being read.
+  #     When nil, the file_name will be inspected to try and determine what
+  #     streams should be applied.
+  #     Default: nil
+  #
+  # Stream types / extensions supported:
+  #   .zip       Zip File                                   [ :zip ]
+  #   .gz, .gzip GZip File                                  [ :gzip ]
+  #   .enc       File Encrypted using symmetric encryption  [ :enc ]
+  #   other      All other extensions will be returned as:  [ :file ]
+  #
+  # When a file is encrypted, it may also be compressed:
+  #   .zip.enc  [ :zip, :enc ]
+  #   .gz.enc   [ :gz,  :enc ]
+  #
+  # Example: Zip
+  #   IOStreams.reader('myfile.zip') do |stream|
+  #     puts stream.read
+  #   end
+  #
+  # Example: Encrypted Zip
+  #   IOStreams.reader('myfile.zip.enc') do |stream|
+  #     puts stream.read
+  #   end
+  #
+  # Example: Explicitly set the streams
+  #   IOStreams.reader('myfile.zip.enc', [:zip, :enc]) do |stream|
+  #     puts stream.read
+  #   end
+  #
+  # Example: Supply custom options
+  #   # Encrypt the file and get Symmetric Encryption to also compress it
+  #   IOStreams.reader('myfile.csv.enc', [:enc]) do |stream|
+  #     puts stream.read
+  #   end
+  def self.reader(file_name_or_io, streams=nil, &block)
+    stream(:reader, file_name_or_io, streams, &block)
+  end
+
+  # Returns a Writer for writing to a file / stream
+  #
+  # Parameters
+  #   file_name_or_io [String|IO]
+  #     The file_name of the file to write to, or an IO Stream that implements
+  #     #write.
+  #
+  #   streams [Symbol|Array]
+  #     The formats/streams that be used to convert the data whilst it is
+  #     being written.
+  #     When nil, the file_name will be inspected to try and determine what
+  #     streams should be applied.
+  #     Default: nil
+  #
+  # Stream types / extensions supported:
+  #   .zip       Zip File                                   [ :zip ]
+  #   .gz, .gzip GZip File                                  [ :gzip ]
+  #   .enc       File Encrypted using symmetric encryption  [ :enc ]
+  #   other      All other extensions will be returned as:  [ :file ]
+  #
+  # When a file is encrypted, it may also be compressed:
+  #   .zip.enc  [ :zip, :enc ]
+  #   .gz.enc   [ :gz,  :enc ]
+  #
+  # Example: Zip
+  #   IOStreams.writer('myfile.zip') do |stream|
+  #     stream.write(data)
+  #   end
+  #
+  # Example: Encrypted Zip
+  #   IOStreams.writer('myfile.zip.enc') do |stream|
+  #     stream.write(data)
+  #   end
+  #
+  # Example: Explicitly set the streams
+  #   IOStreams.writer('myfile.zip.enc', [:zip, :enc]) do |stream|
+  #     stream.write(data)
+  #   end
+  #
+  # Example: Supply custom options
+  #   IOStreams.writer('myfile.csv.enc', [enc: { compress: true }]) do |stream|
+  #     stream.write(data)
+  #   end
+  #
+  # Example: Set internal filename when creating a zip file
+  #   IOStreams.writer('myfile.csv.zip', zip: { zip_file_name: 'myfile.csv' }) do |stream|
+  #     stream.write(data)
+  #   end
+  def self.writer(file_name_or_io, streams=nil, &block)
+    stream(:writer, file_name_or_io, streams, &block)
+  end
+
+  # Copies the source stream to the target stream
+  # Returns [Integer] the number of bytes copied
+  #
+  # Example:
+  #   IOStreams.reader('a.csv') do |source_stream|
+  #     IOStreams.writer('b.csv.enc') do |target_stream|
+  #       IOStreams.copy(source_stream, target_stream)
+  #     end
+  #   end
+  def self.copy(source_stream, target_stream, buffer_size=65536)
+    bytes = 0
+    while data = source_stream.read(buffer_size)
+      break if data.size == 0
+      bytes += data.size
+      target_stream.write(data)
+    end
+    bytes
+  end
+
+  ##########################################################################
+  private
+
+  # Struct to hold the Stream and options if any
+  StreamStruct = Struct.new(:klass, :options)
+
+  # Returns a reader or writer stream
+  def self.stream(type, file_name_or_io, streams=nil, &block)
+    unless streams
+      respond_to = type == :reader ? :read : :write
+      streams = file_name_or_io.respond_to?(respond_to) ? [ :file ] : streams_for_file_name(file_name_or_io)
+    end
+    stream_structs = streams_for(type, streams)
+    if stream_structs.size == 1
+      stream_struct = stream_structs.first
+      stream_struct.klass.open(file_name_or_io, stream_struct.options, &block)
+    else
+      # Daisy chain multiple streams together
+      last = stream_structs.inject(block){ |inner, stream_struct| -> io { stream_struct.klass.open(io, stream_struct.options, &inner) } }
+      last.call(file_name_or_io)
+    end
+  end
+
+  # type: :reader or :writer
+  def self.streams_for(type, params)
+    if params.is_a?(Symbol)
+      [ stream_struct_for_stream(type, params) ]
+    elsif params.is_a?(Array)
+      a = []
+      params.each do |stream|
+        if stream.is_a?(Hash)
+          stream.each_pair { |stream_sym, options| a << stream_struct_for_stream(type, stream_sym, options) }
+        else
+          a << stream_struct_for_stream(type, stream)
+        end
+      end
+      a
+    elsif params.is_a?(Hash)
+      a = []
+      params.each_pair { |stream, options| a << stream_struct_for_stream(type, stream, options) }
+      a
+    else
+      raise ArgumentError, "Invalid params supplied: #{params.inspect}"
+    end
+  end
+
+  def self.stream_struct_for_stream(type, stream, options={})
+    ext   = @@extensions[stream.to_sym] || raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
+    klass = ext.send("#{type}_class")
+    StreamStruct.new(klass, options)
+  end
+
+  # Register File extensions
+  register_extension(:enc,  SymmetricEncryption::Reader, SymmetricEncryption::Writer) if defined?(SymmetricEncryption)
+  register_extension(:file, IOStreams::File::Reader,         IOStreams::File::Writer)
+  register_extension(:gz,   IOStreams::Gzip::Reader,         IOStreams::Gzip::Writer)
+  register_extension(:gzip, IOStreams::Gzip::Reader,         IOStreams::Gzip::Writer)
+  register_extension(:zip,  IOStreams::Zip::Reader,          IOStreams::Zip::Writer)
+end

data/lib/io_streams/version.rb

@@ -0,0 +1,3 @@
+module IOStreams #:nodoc
+  VERSION = "0.7.0"
+end

data/lib/io_streams/zip/reader.rb

@@ -0,0 +1,82 @@
+module IOStreams
+  module Zip
+    class Reader
+      # Read from a zip file or stream, decompressing the contents as it is read
+      # The input stream from the first file found in the zip file is passed
+      # to the supplied block
+      #
+      # Example:
+      #   IOStreams::ZipReader.open('abc.zip') do |io_stream|
+      #     # Read 256 bytes at a time
+      #     while data = io_stream.read(256)
+      #       puts data
+      #     end
+      #   end
+      def self.open(file_name_or_io, options={}, &block)
+        options       = options.dup
+        buffer_size   = options.delete(:buffer_size) || 65536
+        raise(ArgumentError, "Unknown IOStreams::Zip::Reader option: #{options.inspect}") if options.size > 0
+
+        # File name supplied
+        return read_file(file_name_or_io, &block) unless file_name_or_io.respond_to?(:read)
+
+        # Stream supplied
+        begin
+          # Since ZIP cannot be streamed, download un-zipped data to a local file before streaming
+          temp_file = Tempfile.new('rocket_job')
+          file_name = temp_file.to_path
+
+          # Stream zip stream into temp file
+          ::File.open(file_name, 'wb') do |file|
+            IOStreams.copy(file_name_or_io, file, buffer_size)
+          end
+
+          read_file(file_name, &block)
+        ensure
+          temp_file.delete if temp_file
+        end
+      end
+
+      if defined?(JRuby)
+        # Java has built-in support for Zip files
+        def self.read_file(file_name, &block)
+          fin = Java::JavaIo::FileInputStream.new(file_name)
+          zin = Java::JavaUtilZip::ZipInputStream.new(fin)
+          zin.get_next_entry
+          block.call(zin.to_io)
+        ensure
+          zin.close if zin
+          fin.close if fin
+        end
+
+      else
+        # MRI needs Ruby Zip, since it only has native support for GZip
+        begin
+          require 'zip'
+        rescue LoadError => exc
+          puts "Please install gem rubyzip so that RocketJob can read Zip files in Ruby MRI"
+          raise(exc)
+        end
+
+        # Read from a zip file or stream, decompressing the contents as it is read
+        # The input stream from the first file found in the zip file is passed
+        # to the supplied block
+        def self.read_file(file_name, &block)
+          begin
+            zin = ::Zip::InputStream.new(file_name)
+            zin.get_next_entry
+            block.call(zin)
+          ensure
+            begin
+              zin.close if zin
+            rescue IOError
+              # Ignore file already closed errors since Zip::InputStream
+              # does not have a #closed? method
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/io_streams/zip/writer.rb

@@ -0,0 +1,86 @@
+module IOStreams
+  module Zip
+    class Writer
+      # Write a single file in Zip format to the supplied output file name
+      #
+      # Parameters
+      #   zip_file_name [String]
+      #     Full path and filename for the output zip file
+      #
+      #   file_name [String]
+      #     Name of the file within the Zip Stream
+      #
+      # The stream supplied to the block only responds to #write
+      #
+      # Example:
+      #   IOStreams::ZipWriter.open('myfile.zip', zip_file_name: 'myfile.txt') do |io_stream|
+      #     io_stream.write("hello world\n")
+      #     io_stream.write("and more\n")
+      #   end
+      #
+      # Notes:
+      # - Since Zip cannot write to streams, if a stream is supplied, a temp file
+      #   is automatically created under the covers
+      def self.open(file_name_or_io, options={}, &block)
+        options       = options.dup
+        zip_file_name = options.delete(:zip_file_name)
+        buffer_size   = options.delete(:buffer_size) || 65536
+        raise(ArgumentError, "Unknown IOStreams::Zip::Writer option: #{options.inspect}") if options.size > 0
+
+        # Default the name of the file within the zip to the supplied file_name without the zip extension
+        zip_file_name = file_name_or_io.to_s[0..-5] if zip_file_name.nil? && !file_name_or_io.respond_to?(:write) && (file_name_or_io =~ /\.(zip)\z/)
+        zip_file_name ||= 'file'
+
+        # File name supplied
+        return write_file(file_name_or_io, zip_file_name, &block) unless file_name_or_io.respond_to?(:write)
+
+        # Stream supplied
+        begin
+          # Since ZIP cannot be streamed, download to a local file before streaming
+          temp_file = Tempfile.new('rocket_job')
+          write_file(temp_file.to_path, zip_file_name, &block)
+
+          # Stream temp file into output stream
+          IOStreams.copy(temp_file, file_name_or_io, buffer_size)
+        ensure
+          temp_file.delete if temp_file
+        end
+      end
+
+      private
+
+      if defined?(JRuby)
+
+        def self.write_file(file_name, zip_file_name, &block)
+          out  = Java::JavaIo::FileOutputStream.new(file_name)
+          zout = Java::JavaUtilZip::ZipOutputStream.new(out)
+          zout.put_next_entry(Java::JavaUtilZip::ZipEntry.new(zip_file_name))
+          io = zout.to_io
+          block.call(io)
+        ensure
+          io.close if io && !io.closed?
+          out.close if out
+        end
+
+      else
+        # MRI needs Ruby Zip, since it only has native support for GZip
+        begin
+          require 'zip'
+        rescue LoadError => exc
+          puts "Please install gem rubyzip so that RocketJob can read Zip files in Ruby MRI"
+          raise(exc)
+        end
+
+        def self.write_file(file_name, zip_file_name, &block)
+          zos = ::Zip::OutputStream.new(file_name)
+          zos.put_next_entry(zip_file_name)
+          block.call(zos)
+        ensure
+          zos.close if zos
+        end
+
+      end
+
+    end
+  end
+end

data/lib/iostreams.rb

@@ -0,0 +1,16 @@
+require 'io_streams/version'
+module IOStreams
+  module File
+    autoload :Reader, 'io_streams/file/reader'
+    autoload :Writer, 'io_streams/file/writer'
+  end
+  module Gzip
+    autoload :Reader, 'io_streams/gzip/reader'
+    autoload :Writer, 'io_streams/gzip/writer'
+  end
+  module Zip
+    autoload :Reader,  'io_streams/zip/reader'
+    autoload :Writer,  'io_streams/zip/writer'
+  end
+end
+require 'io_streams/io_streams'

data/test/file_reader_test.rb

@@ -0,0 +1,31 @@
+require_relative 'test_helper'
+
+# Unit Test for IOStreams::File
+module Streams
+  class FileReaderTest < Minitest::Test
+    context IOStreams::File::Reader do
+      setup do
+        @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
+        @data      = File.read(@file_name)
+      end
+
+      context '.open' do
+        should 'file' do
+          result = IOStreams::File::Reader.open(@file_name) do |io|
+            io.read
+          end
+          assert_equal @data, result
+        end
+        should 'stream' do
+          result = File.open(@file_name) do |file|
+            IOStreams::File::Reader.open(file) do |io|
+              io.read
+            end
+          end
+          assert_equal @data, result
+        end
+      end
+
+    end
+  end
+end

data/test/file_writer_test.rb

@@ -0,0 +1,33 @@
+require_relative 'test_helper'
+
+# Unit Test for IOStreams::File
+module Streams
+  class FileWriterTest < Minitest::Test
+    context IOStreams::File::Writer do
+      setup do
+        @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
+        @data      = File.read(@file_name)
+      end
+
+      context '.open' do
+        should 'file' do
+          temp_file = Tempfile.new('rocket_job')
+          file_name = temp_file.to_path
+          IOStreams::File::Writer.open(file_name) do |io|
+            io.write(@data)
+          end
+          result = File.read(file_name)
+          assert_equal @data, result
+        end
+        should 'stream' do
+          io_string = StringIO.new
+          IOStreams::File::Writer.open(io_string) do |io|
+            io.write(@data)
+          end
+          assert_equal @data, io_string.string
+        end
+      end
+
+    end
+  end
+end

data/test/files/text.txt

@@ -0,0 +1,3 @@
+Hello World
+Line2
+Line3

data/test/gzip_reader_test.rb

@@ -0,0 +1,32 @@
+require_relative 'test_helper'
+
+# Unit Test for IOStreams::Gzip
+module Streams
+  class GzipReaderTest < Minitest::Test
+    context IOStreams::Gzip::Reader do
+      setup do
+        @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
+        @gzip_data  = File.open(@file_name, 'rb') { |f| f.read }
+        @data       = Zlib::GzipReader.open(@file_name) {|gz| gz.read }
+      end
+
+      context '.open' do
+        should 'file' do
+          result = IOStreams::Gzip::Reader.open(@file_name) do |io|
+            io.read
+          end
+          assert_equal @data, result
+        end
+        should 'stream' do
+          result = File.open(@file_name) do |file|
+            IOStreams::Gzip::Reader.open(file) do |io|
+              io.read
+            end
+          end
+          assert_equal @data, result
+        end
+      end
+
+    end
+  end
+end

data/test/gzip_writer_test.rb

@@ -0,0 +1,38 @@
+require_relative 'test_helper'
+
+# Unit Test for IOStreams::Gzip
+module Streams
+  class GzipWriterTest < Minitest::Test
+    context IOStreams::Gzip::Writer do
+      setup do
+        @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
+        @data = Zlib::GzipReader.open(@file_name) {|gz| gz.read }
+      end
+
+      context '.open' do
+        should 'file' do
+          temp_file = Tempfile.new('rocket_job')
+          file_name = temp_file.to_path
+          IOStreams::Gzip::Writer.open(file_name) do |io|
+            io.write(@data)
+          end
+          result = Zlib::GzipReader.open(file_name) {|gz| gz.read }
+          temp_file.delete
+          assert_equal @data, result
+        end
+        should 'stream' do
+          io_string = StringIO.new(''.force_encoding('ASCII-8BIT'))
+          IOStreams::Gzip::Writer.open(io_string) do |io|
+            io.write(@data)
+          end
+          io = StringIO.new(io_string.string)
+          gz = Zlib::GzipReader.new(io)
+          data = gz.read
+          gz.close
+          assert_equal @data, data
+        end
+      end
+
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'yaml'
+require 'minitest/autorun'
+require 'minitest/reporters'
+require 'shoulda/context'
+require 'iostreams'
+require 'awesome_print'
+require 'symmetric-encryption'
+
+Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
+
+# Test cipher
+SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(
+  cipher_name: 'aes-128-cbc',
+  key:         '1234567890ABCDEF1234567890ABCDEF',
+  iv:          '1234567890ABCDEF',
+  encoding:    :base64strict
+)
+

data/test/zip_reader_test.rb

@@ -0,0 +1,33 @@
+require_relative 'test_helper'
+require 'zip'
+
+# Unit Test for IOStreams::Zip
+module Streams
+  class ZipReaderTest < Minitest::Test
+    context IOStreams::Zip::Reader do
+      setup do
+        @file_name = File.join(File.dirname(__FILE__), 'files', 'text.zip')
+        @zip_data  = File.open(@file_name, 'rb') { |f| f.read }
+        @data      = Zip::File.open(@file_name) { |zip_file| zip_file.first.get_input_stream.read }
+      end
+
+      context '.open' do
+        should 'file' do
+          result = IOStreams::Zip::Reader.open(@file_name) do |io|
+            io.read
+          end
+          assert_equal @data, result
+        end
+        should 'stream' do
+          result = File.open(@file_name) do |file|
+            IOStreams::Zip::Reader.open(file) do |io|
+              io.read
+            end
+          end
+          assert_equal @data, result
+        end
+      end
+
+    end
+  end
+end

data/test/zip_writer_test.rb

@@ -0,0 +1,47 @@
+require_relative 'test_helper'
+require 'zip'
+
+# Unit Test for IOStreams::Zip
+module Streams
+  class ZipWriterTest < Minitest::Test
+    context IOStreams::Zip::Writer do
+      setup do
+        file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
+        @data      = File.read(file_name)
+      end
+
+      context '.open' do
+        should 'file' do
+          temp_file = Tempfile.new('rocket_job')
+          file_name = temp_file.to_path
+          IOStreams::Zip::Writer.open(file_name, zip_file_name: 'text.txt') do |io|
+            io.write(@data)
+          end
+          result = Zip::File.open(file_name) do |zip_file|
+            zip_file.first.get_input_stream.read
+          end
+          temp_file.delete
+          assert_equal @data, result
+        end
+
+        should 'stream' do
+          io_string = StringIO.new(''.force_encoding('ASCII-8BIT'))
+          IOStreams::Zip::Writer.open(io_string) do |io|
+            io.write(@data)
+          end
+          io = StringIO.new(io_string.string)
+          result = nil
+          begin
+            zin = ::Zip::InputStream.new(io)
+            entry = zin.get_next_entry
+            result = zin.read
+          ensure
+            zin.close if zin
+          end
+          assert_equal @data, result
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: iostreams
+version: !ruby/object:Gem::Version
+  version: 0.7.0
+platform: ruby
+authors:
+- Reid Morrison
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: symmetric-encryption
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: thread_safe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- reidmo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/io_streams/file/reader.rb
+- lib/io_streams/file/writer.rb
+- lib/io_streams/gzip/reader.rb
+- lib/io_streams/gzip/writer.rb
+- lib/io_streams/io_streams.rb
+- lib/io_streams/version.rb
+- lib/io_streams/zip/reader.rb
+- lib/io_streams/zip/writer.rb
+- lib/iostreams.rb
+- test/file_reader_test.rb
+- test/file_writer_test.rb
+- test/files/text.txt
+- test/files/text.txt.gz
+- test/files/text.txt.gz.zip
+- test/files/text.zip
+- test/gzip_reader_test.rb
+- test/gzip_writer_test.rb
+- test/test_helper.rb
+- test/zip_reader_test.rb
+- test/zip_writer_test.rb
+homepage: https://github.com/rocketjob/streams
+licenses:
+- Apache License V2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Ruby Input and Output streaming with support for Zip, Gzip, and Encryption.
+test_files:
+- test/file_reader_test.rb
+- test/file_writer_test.rb
+- test/files/text.txt
+- test/files/text.txt.gz
+- test/files/text.txt.gz.zip
+- test/files/text.zip
+- test/gzip_reader_test.rb
+- test/gzip_writer_test.rb
+- test/test_helper.rb
+- test/zip_reader_test.rb
+- test/zip_writer_test.rb