s3io 0.0.2 → 1.0.0

data/README.md

@@ -25,12 +25,13 @@ Or install it yourself as:
 ## Usage
 
 Once wrapped, S3 objects behave the way you'd expect from an ordinary IO object.
+It can read:
 
     require 'aws-sdk'
     require 's3io'
 
     s3_object = S3.buckets['some-bucket'].objects['path/to/object']
-    io = S3io.new(s3_object)
+    io = S3io.open(s3_object, 'r')
 
     first_100_bytes = io.read(100) # reading first 100 bytes
 
@@ -44,9 +45,22 @@ Once wrapped, S3 objects behave the way you'd expect from an ordinary IO object.
 
     puts io.read # and print everything from that byte to the end
 
+
+It can write:
+
+    require 'aws-sdk'
+    require 's3io'
+
+    s3_object = S3.buckets['some-bucket'].objects['path/to/object']
+
+    S3io.open(s3_object, 'w') do |s3io|
+      io.write 'abc'
+      io.write 'def'
+    end
+
 ## To do
 
-* Write support
+* Code documentation
 
 ## Contributing
 

data/lib/s3io.rb

@@ -1,5 +1,7 @@
 require "s3io/version"
 require "s3io/wrapper"
+require "s3io/read_wrapper"
+require "s3io/write_wrapper"
 
 require "aws-sdk"
 
@@ -11,7 +13,7 @@ module S3io
   # @param [Hash] options options hash
   # @option options [Integer] :line_buffer_size size of the buffer that is used for reading contents of S3 object when iterating over its lines
   # @return [S3io::Wrapper] a wrapped S3 object
-  def self.new(s3object, options = {})
-    Wrapper.new(s3object, options)
-  end
+  #def self.new(s3object, options = {})
+  #  Wrapper.new(s3object, options)
+  #end
 end

data/lib/s3io/read_wrapper.rb

@@ -0,0 +1,86 @@
+module S3io
+
+  def self.reader(s3object, options = {}, &block)
+    open(s3object, 'r', options, &block)
+  end
+
+  class ReadWrapper < Wrapper
+
+    # Default buffer size for line parser in bytes
+    LINE_BUFFER_SIZE = 5 * 1024 * 1024 # MiB
+
+    include Enumerable
+
+    # Current byte position in S3 object
+    attr_accessor :pos
+
+    def initialize(s3object, options = {})
+      super(s3object)
+
+      @options = {
+        :line_buffer_size => (options[:line_buffer_size] || LINE_BUFFER_SIZE)
+      }
+    end
+
+    # Reads data from S3 object.
+    #
+    # @param [Integer] bytes number of bytes to read
+    def read(bytes = nil)
+      content_length = @s3object.content_length
+
+      return '' if (@pos >= content_length) || (bytes == 0)
+
+      bytes ||= content_length
+
+      upper_bound = @pos + bytes - 1
+      upper_bound = (content_length - 1) if upper_bound >= content_length
+
+      data = @s3object.read :range => @pos..upper_bound
+      @pos = upper_bound + 1
+
+      return data
+    end
+
+    def eof?
+      @pos >= @s3object.content_length
+    end
+
+    # Rewinds position to the very beginning of S3 object.
+    def rewind
+      @pos = 0
+    end
+
+    # Iterates over S3 object lines.
+    #
+    # @param [String] separator line separator string
+    def each(separator = $/)
+      return enum_for(:each, separator) unless block_given?
+
+      line = ''
+      newline_pos = nil
+
+      # Either trying to parse the remainder or reading some more data
+      while newline_pos || !(buffer = read(@options[:line_buffer_size])).empty?
+        prev_newline_pos = newline_pos || 0
+        newline_pos = buffer.index(separator, prev_newline_pos)
+
+        if newline_pos
+          line << buffer[prev_newline_pos..newline_pos]
+          newline_pos += 1
+          yield line
+          line = ''
+        else
+          line << buffer[prev_newline_pos..-1]
+        end
+      end
+
+      # Flush the remainder if body doesn't end with separator
+      yield line unless line.empty?
+
+      return self
+    end
+    alias lines each
+    alias each_line each
+
+  end
+end

data/lib/s3io/version.rb

@@ -1,3 +1,3 @@
 module S3io
-  VERSION = "0.0.2"
+  VERSION = "1.0.0"
 end

data/lib/s3io/wrapper.rb

@@ -1,88 +1,44 @@
 module S3io
-  # This class wraps an AWS S3 object in order to provide IO-like API.
-  # S3 objects wrapped this way can be used in methods that would otherwise expect an instance of File, StringIO etc.
-  class Wrapper
 
-    # Default buffer size for line parser in bytes
-    LINE_BUFFER_SIZE = 5 * 1024 * 1024 # MiB
+  def self.open(s3object, mode_string = 'r', options = {}, &block)
+    wrapper_class = case mode_string
+    when 'r'
+      ReadWrapper
+    when 'w'
+      WriteWrapper
+    else
+      fail "S3IO only supports 'r' or 'w' as access modes"
+    end
 
-    include Enumerable
+    wrapper = wrapper_class.new(s3object, options)
 
-    # Current byte position in S3 object
-    attr_accessor :pos
+    if block_given?
+      result = yield wrapper if block_given?
+      wrapper.close
+
+      return result
+    else
+      return wrapper
+    end
+  end
 
-    # Options that were passed during initialization
-    attr_reader :options
+  # This class wraps an AWS S3 object in order to provide IO-like API.
+  # S3 objects wrapped this way can be used in methods that would otherwise expect an instance of File, StringIO etc.
+  class Wrapper
 
     # Wraps an AWS::S3::S3Object into IO-like object.
     #
     # @param [AWS::S3::S3Object] s3object an object to wrap
-    # @param [Hash] options options hash
-    # @option options [Integer] :line_buffer_size size of the buffer that is used for reading contents of S3 object when iterating over its lines
-    def initialize(s3object, options = {})
+    def initialize(s3object)
       @s3object = s3object
-      @options = {
-        :line_buffer_size => (options[:line_buffer_size] || LINE_BUFFER_SIZE)
-      }
-
       @pos = 0
     end
 
-    # Reads data from S3 object.
-    #
-    # @param [Integer] bytes number of bytes to read
-    def read(bytes = nil)
-      content_length = @s3object.content_length
-
-      return '' if (@pos >= content_length) || (bytes == 0)
-
-      bytes ||= content_length
-
-      upper_bound = @pos + bytes - 1
-      upper_bound = (content_length - 1) if upper_bound >= content_length
-
-      data = @s3object.read :range => @pos..upper_bound
-      @pos = upper_bound + 1
-
-      return data
-    end
-
-    # Rewinds position to the very beginning of S3 object.
-    def rewind
+    def close
+      @s3object = nil
       @pos = 0
-    end
 
-    # Iterates over S3 object lines.
-    #
-    # @param [String] separator line separator string
-    def each(separator = $/)
-      return enum_for(:each, separator) unless block_given?
-
-      line = ''
-      newline_pos = nil
-
-      # Either trying to parse the remainder or reading some more data
-      while newline_pos || !(buffer = read(@options[:line_buffer_size])).empty?
-        prev_newline_pos = newline_pos || 0
-        newline_pos = buffer.index(separator, prev_newline_pos)
-
-        if newline_pos
-          line << buffer[prev_newline_pos..newline_pos]
-          newline_pos += 1
-          yield line
-          line = ''
-        else
-          line << buffer[prev_newline_pos..-1]
-        end
-      end
-
-      # Flush the remainder if body doesn't end with separator
-      yield line unless line.empty?
-
-      return self
+      return nil
     end
-    alias lines each
-    alias each_line each
-
   end
 end

data/lib/s3io/write_wrapper.rb

@@ -0,0 +1,64 @@
+module S3io
+
+  def self.writer(s3object, options = {}, &block)
+    open(s3object, 'w', options, &block)
+  end
+
+  class WriteWrapper < Wrapper
+
+    # Default maximum file size
+    MAX_FILE_SIZE = 1 * 1024 * 1024 * 1024 * 1024 # TiB
+
+    # S3 only supports up to 10K multipart chunks
+    MAX_NUM_CHUNKS = 10000
+
+    # Minimum chunk size in S3 is 5MiB
+    MIN_CHUNK_SIZE = 5 * 1024 * 1024 # MiB
+
+    # Number of bytes written to S3
+    attr_reader :pos
+
+    def initialize(s3object, options = {})
+      super(s3object)
+
+      @options = {
+        :max_file_size => (options[:max_file_size] || MAX_FILE_SIZE),
+        :multipart_upload_options => (options[:multipart_upload_options] || {})
+      }
+
+      @min_chunk_size = [(@options[:max_file_size].to_f / MAX_NUM_CHUNKS).ceil, MIN_CHUNK_SIZE].max
+      @multipart_upload = @s3object.multipart_upload(@options[:multipart_upload_options])
+      @write_buffer = ''
+    end
+
+    def write(data)
+      fail "S3 Object is already closed" unless @s3object
+
+      data_str = data.to_s
+      data_size = data_str.size
+
+      @write_buffer << data_str
+      @pos += data_size
+      self.flush if @write_buffer.size >= @min_chunk_size
+
+      return data_size
+    end
+
+    def flush
+      return if @write_buffer.empty?
+
+      @multipart_upload.add_part(@write_buffer)
+      @write_buffer.replace '' # String#clear isn't available on 1.8.x
+
+      return nil
+    end
+
+    def close
+      self.flush
+      @multipart_upload.close
+      @multipart_upload = nil
+
+      super
+    end
+  end
+end

data/test/{test_s3io_wrapper.rb → test_s3io_read_wrapper.rb}

@@ -3,7 +3,7 @@ require 's3io'
 require 'stringio'
 
 # Emulate S3Object in a way that allows us to check if its methods are being called properly
-class S3ObjectMock
+class S3ObjectReadMock
   def initialize(body = '')
     @body = body
   end
@@ -20,35 +20,35 @@ class S3ObjectMock
   end
 end
 
-class S3ioWrapperTest < Test::Unit::TestCase
+class S3ioReadWrapperTest < Test::Unit::TestCase
   S3_TEST_DATA = File.read('test/s3_test_data.csv')
 
   def setup
-    @s3object = S3ObjectMock.new(S3_TEST_DATA)
+    @s3object = S3ObjectReadMock.new(S3_TEST_DATA)
   end
 
-  def test_s3io_wrapper_full_read
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_full_read
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     assert_equal(S3_TEST_DATA, wrapper.read)
   end
 
-  def test_s3io_wrapper_zero_read
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_zero_read
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     assert_equal('', wrapper.read(0))
     assert_equal(0, wrapper.pos)
   end
 
-  def test_s3io_wrapper_partial_read
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_partial_read
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     assert_equal(S3_TEST_DATA[0..99], wrapper.read(100))
     assert_equal(S3_TEST_DATA[100..100], wrapper.read(1))
   end
 
-  def test_s3io_wrapper_each
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_each
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     lines = []
     wrapper.each do |line|
@@ -58,15 +58,15 @@ class S3ioWrapperTest < Test::Unit::TestCase
     assert_equal(S3_TEST_DATA.lines.to_a, lines)
   end
 
-  def test_s3io_wrapper_each_enum
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_each_enum
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     assert_equal(S3_TEST_DATA.lines.to_a,
                  wrapper.lines.to_a)
   end
 
-  def test_s3io_wrapper_pos
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_pos
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     assert_equal(0, wrapper.pos)
 
@@ -76,8 +76,8 @@ class S3ioWrapperTest < Test::Unit::TestCase
     assert_equal(S3_TEST_DATA[77, 32], wrapper.read(32))
   end
 
-  def test_s3io_wrapper_pos_beyond
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_pos_beyond
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     pos_beyond = @s3object.content_length + 100
     wrapper.pos = pos_beyond
@@ -86,8 +86,8 @@ class S3ioWrapperTest < Test::Unit::TestCase
     assert_equal(pos_beyond, wrapper.pos)
   end
 
-  def test_s3io_wrapper_rewind
-    wrapper = S3io::Wrapper.new(@s3object)
+  def test_rewind
+    wrapper = S3io::ReadWrapper.new(@s3object)
 
     wrapper.lines do |line|
       # iterate through all the lines
@@ -99,8 +99,31 @@ class S3ioWrapperTest < Test::Unit::TestCase
     assert_equal(S3_TEST_DATA[0..100], wrapper.read(101))
   end
 
-  def test_s3io_wrapper_line_buffer_size
-    wrapper = S3io::Wrapper.new(@s3object, :line_buffer_size => 25)
+  def test_lines_custom_separator
+    wrapper = S3io::ReadWrapper.new(@s3object)
+
+    assert_equal(S3_TEST_DATA.lines(",").to_a, wrapper.lines(",").to_a)
+  end
+
+  def test_eof
+    wrapper = S3io::ReadWrapper.new(@s3object)
+
+    assert_equal(false, wrapper.eof?)
+
+    wrapper.read(10)
+    assert_equal(false, wrapper.eof?)
+
+    wrapper.read
+    assert_equal(true, wrapper.eof?)
+
+    wrapper.rewind
+    assert_equal(false, wrapper.eof?)
+  end
+
+  # Custom options for wrapper
+
+  def test_line_buffer_size
+    wrapper = S3io::ReadWrapper.new(@s3object, :line_buffer_size => 25)
 
     wrapper.lines.each_with_index do |line, index|
       break if index == 1 # skip two buffered reads
@@ -110,14 +133,8 @@ class S3ioWrapperTest < Test::Unit::TestCase
     assert_equal(S3_TEST_DATA[50..-1], wrapper.read)
   end
 
-  def test_s3io_wrapper_lines_custom_separator
-    wrapper = S3io::Wrapper.new(@s3object)
-
-    assert_equal(S3_TEST_DATA.lines(",").to_a, wrapper.lines(",").to_a)
-  end
-
-  def test_s3io_wrapper_empty_each
-    wrapper = S3io::Wrapper.new(S3ObjectMock.new(''))
+  def test_empty_each
+    wrapper = S3io::ReadWrapper.new(S3ObjectReadMock.new(''))
 
     wrapper.each do |line|
       assert_equal(false, :never_gets_called)

data/test/test_s3io_write_wrapper.rb

@@ -0,0 +1,91 @@
+require 'test/unit'
+require 's3io'
+require 'stringio'
+
+class S3ObjectWriteMock
+  attr_reader :body
+  attr_accessor :uploaded
+  attr_accessor :multipart_upload_options
+
+  class MultiPartUploadMock
+    attr_reader :options
+
+    def initialize(body, options, mock)
+      @body = body
+      @mock = mock
+      @mock.multipart_upload_options = options
+    end
+
+    def add_part(part)
+      @body << part
+    end
+
+    def close
+      @mock.uploaded = @body
+    end
+  end
+
+  def initialize
+    @body = ''
+  end
+
+  def multipart_upload(options = {})
+    MultiPartUploadMock.new(@body, options, self)
+  end
+end
+
+class S3ioWriteWrapperTest < Test::Unit::TestCase
+  def setup
+    @s3object = S3ObjectWriteMock.new
+  end
+
+  def test_full_write
+    wrapper = S3io::WriteWrapper.new(@s3object)
+
+    test_string = 'This is a test.'
+
+    wrapper.write(test_string)
+    assert_equal(test_string.size, wrapper.pos)
+    assert_equal({}, @s3object.multipart_upload_options)
+
+    wrapper.close
+    assert_equal(test_string, @s3object.uploaded)
+    assert_equal(0, wrapper.pos)
+  end
+
+  def test_multipart_write
+    wrapper = S3io::WriteWrapper.new(@s3object, :max_file_size => 1)
+
+    chunk1 = 'z' * (S3io::WriteWrapper::MIN_CHUNK_SIZE + 100)
+    chunk2 = 'y' * (S3io::WriteWrapper::MIN_CHUNK_SIZE + 300)
+
+    full_body = chunk1 + chunk2
+
+    wrapper.write(chunk1)
+    assert_equal(chunk1, @s3object.body)
+    assert_equal(nil, @s3object.uploaded)
+
+    wrapper.write(chunk2)
+    assert_equal(full_body, @s3object.body)
+    assert_equal(nil, @s3object.uploaded)
+
+    wrapper.close
+    assert_equal(full_body, @s3object.uploaded)
+  end
+
+  def test_flush
+    wrapper = S3io::WriteWrapper.new(@s3object, :multipart_upload_options => {:metadata => {:yes => 'no'}})
+    test_string = '42'
+
+    assert_equal('', @s3object.body)
+    assert_equal({:metadata => {:yes => 'no'}}, @s3object.multipart_upload_options)
+
+    wrapper.write(test_string)
+    wrapper.flush
+    assert_equal(test_string, @s3object.body)
+
+    wrapper.close
+    assert_equal(test_string, @s3object.body)
+    assert_equal(test_string, @s3object.uploaded)
+end
+end

metadata

@@ -1,39 +1,46 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: s3io
-version: !ruby/object:Gem::Version
-  version: 0.0.2
+version: !ruby/object:Gem::Version 
+  hash: 23
   prerelease: 
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
 platform: ruby
-authors:
+authors: 
 - Arthur Pirogovski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-14 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2013-01-22 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: aws-sdk
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
 description: An IO-compatible wrapper for S3
-email:
+email: 
 - arthur@flyingtealeaf.com
 executables: []
+
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - .travis.yml
 - Gemfile
@@ -41,41 +48,49 @@ files:
 - README.md
 - Rakefile
 - lib/s3io.rb
+- lib/s3io/read_wrapper.rb
 - lib/s3io/version.rb
 - lib/s3io/wrapper.rb
+- lib/s3io/write_wrapper.rb
 - s3io.gemspec
 - test/s3_test_data.csv
-- test/test_s3io.rb
-- test/test_s3io_wrapper.rb
+- test/test_s3io_read_wrapper.rb
+- test/test_s3io_write_wrapper.rb
 homepage: http://github.com/fiksu/s3io
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: Amazon's official AWS SDK provides an API for S3 that isn't compatible with
-  Ruby's standard IO class and its derivatives. This gem provides a thin wrapper around
-  AWS SDK that makes it possible to access objects stored on S3 as if they were instances
-  of File or StringIO classes.
-test_files:
+summary: Amazon's official AWS SDK provides an API for S3 that isn't compatible with Ruby's standard IO class and its derivatives. This gem provides a thin wrapper around AWS SDK that makes it possible to access objects stored on S3 as if they were instances of File or StringIO classes.
+test_files: 
 - test/s3_test_data.csv
-- test/test_s3io.rb
-- test/test_s3io_wrapper.rb
+- test/test_s3io_read_wrapper.rb
+- test/test_s3io_write_wrapper.rb
 has_rdoc: 

data/test/test_s3io.rb

@@ -1,16 +0,0 @@
-require 'test/unit'
-require 's3io'
-
-class S3ioTest < Test::Unit::TestCase
-  def test_s3io_new
-    s3object = Object.new
-    wrapper = S3io.new(s3object)
-    assert_equal(S3io::Wrapper, wrapper.class)
-  end
-
-  def test_s3io_new_with_options
-    s3object = Object.new
-    wrapper = S3io.new(s3object, :line_buffer_size => 128)
-    assert_equal(128, wrapper.options[:line_buffer_size])
-  end
-end