s3io 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+*.swp
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+rvm:
+  - ree
+  - ruby-head
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - jruby-18mode # JRuby in 1.8 mode
+  - jruby-19mode # JRuby in 1.9 mode
+  - rbx-18mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in s3io.gemspec
+gemspec
+
+gem "rake"
+
+group :development do
+  platforms :ruby do
+    gem "yard"
+    gem "redcarpet"
+  end
+end

data/LICENSE.txt

@@ -0,0 +1,30 @@
+Copyright (c) 2012, Fiksu, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ o Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+ o Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+
+ o Fiksu, Inc. nor the names of its contributors may be used to
+   endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,57 @@
+# S3io
+
+[![Build Status](https://travis-ci.org/fiksu/s3io.png)](https://travis-ci.org/fiksu/s3io)
+
+An IO-compatible wrapper for S3.
+
+Amazon's official AWS SDK provides an API for S3 that isn't compatible with Ruby's standard IO class and its derivatives. This gem provides a thin wrapper around AWS SDK that makes it possible to access objects stored on S3 as if they were instances of File or StringIO classes.
+
+Currently only reads are supported with writes support coming soon.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 's3io'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install s3io
+
+## Usage
+
+Once wrapped, S3 objects behave the way you'd expect from an ordinary IO object.
+
+    require 'aws-sdk'
+    require 's3io'
+
+    s3_object = S3.buckets['some-bucket'].objects['path/to/object']
+    io = S3io.new(s3object)
+
+    first_100_bytes = io.read(100) # reading first 100 bytes
+
+    io.rewind # back to the first byte
+
+    io.lines do |line|
+      puts line # Just printing lines one by one
+    end
+
+    io.pos = 42 # seek byte 42
+
+    puts io.read # and print everything from that byte to the end
+
+## To do
+
+* Write support
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => [:test]

data/lib/s3io/version.rb

@@ -0,0 +1,3 @@
+module S3io
+  VERSION = "0.0.1"
+end

data/lib/s3io/wrapper.rb

@@ -0,0 +1,88 @@
+module S3io
+  # This class wraps an AWS S3 object in order to provide IO-like API.
+  # S3 objects wrapped this way can be used in methods that would otherwise expect an instance of File, StringIO etc.
+  class Wrapper
+
+    # Default buffer size for line parser in bytes
+    LINE_BUFFER_SIZE = 5 * 1024 * 1024 # MiB
+
+    include Enumerable
+
+    # Current byte position in S3 object
+    attr_accessor :pos
+
+    # Options that were passed during initialization
+    attr_reader :options
+
+    # Wraps an AWS::S3::S3Object into IO-like object.
+    #
+    # @param [AWS::S3::S3Object] s3object an object to wrap
+    # @param [Hash] options options hash
+    # @option options [Integer] :line_buffer_size size of the buffer that is used for reading contents of S3 object when iterating over its lines
+    def initialize(s3object, options = {})
+      @s3object = s3object
+      @options = {
+        :line_buffer_size => (options[:line_buffer_size] || LINE_BUFFER_SIZE)
+      }
+
+      @pos = 0
+    end
+
+    # Reads data from S3 object.
+    #
+    # @param [Integer] bytes number of bytes to read
+    def read(bytes = nil)
+      content_length = @s3object.content_length
+
+      return '' if (@pos >= content_length) || (bytes == 0)
+
+      bytes ||= content_length
+
+      upper_bound = @pos + bytes - 1
+      upper_bound = (content_length - 1) if upper_bound >= content_length
+
+      data = @s3object.read :range => @pos..upper_bound
+      @pos = upper_bound + 1
+
+      return data
+    end
+
+    # Rewinds position to the very beginning of S3 object.
+    def rewind
+      @pos = 0
+    end
+
+    # Iterates over S3 object lines.
+    #
+    # @param [String] separator line separator string
+    def each(separator = $/)
+      return enum_for(:each, separator) unless block_given?
+
+      line = ''
+      newline_pos = nil
+
+      # Either trying to parse the remainder or reading some more data
+      while newline_pos || !(buffer = read(@options[:line_buffer_size])).empty?
+        prev_newline_pos = newline_pos || 0
+        newline_pos = buffer.index(separator, prev_newline_pos)
+
+        if newline_pos
+          line << buffer[prev_newline_pos..newline_pos]
+          newline_pos += 1
+          yield line
+          line = ''
+        else
+          line << buffer[prev_newline_pos..-1]
+        end
+      end
+
+      # Flush the remainder if body doesn't end with separator
+      yield line unless line.empty?
+
+      return self
+    end
+    alias lines each
+    alias each_line each
+
+  end
+end

data/lib/s3io.rb

@@ -0,0 +1,17 @@
+require "s3io/version"
+require "s3io/wrapper"
+
+require "aws-sdk"
+
+# A top-level module that provides an S3 wrapper class.
+module S3io
+  # A shortcut for wrapping an S3 object
+  #
+  # @param [AWS::S3::S3Object] s3object an object to wrap
+  # @param [Hash] options options hash
+  # @option options [Integer] :line_buffer_size size of the buffer that is used for reading contents of S3 object when iterating over its lines
+  # @return [S3io::Wrapper] a wrapped S3 object
+  def self.new(s3object, options = {})
+    Wrapper.new(s3object, options)
+  end
+end

data/s3io.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 's3io/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "s3io"
+  gem.version       = S3io::VERSION
+  gem.authors       = ["Arthur Pirogovski"]
+  gem.email         = ["arthur@flyingtealeaf.com"]
+  gem.description   = %q{An IO-compatible wrapper for S3}
+  gem.summary       = %q{Amazon's official AWS SDK provides an API for S3 that isn't compatible with Ruby's standard IO class and its derivatives. This gem provides a thin wrapper around AWS SDK that makes it possible to access objects stored on S3 as if they were instances of File or StringIO classes.}
+  gem.homepage      = "http://github.com/fiksu/s3io"
+
+  gem.add_dependency('aws-sdk')
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/test/s3_test_data.csv

@@ -0,0 +1,102 @@
+,,,,
+Natalie,French,CO,Elmira,N7D 6Z8
+Conan,Gross,YT,Jeffersonville,16555
+Noel,Torres,Virginia,Roswell,09380
+Veda,Serrano,British Columbia,Pullman,C2R 3U1
+Ross,Gilmore,Prince Edward Island,Los Alamitos,13816
+Melissa,Conley,PA,Bradford,42959
+Cheyenne,Graves,AK,Flint,18721
+Kelsey,Harding,Maryland,Pasadena,J8Z 8A9
+Harding,Webb,PE,Bowie,70957
+Derek,Nunez,Ontario,Temple City,X1P 7A3
+Oliver,Williams,MO,Eau Claire,81691
+Dacey,Olsen,Saskatchewan,Lebanon,H7E 3G8
+George,Moss,NL,Vallejo,R8W 8O7
+Wallace,Robbins,Newfoundland and Labrador,Belleville,63676
+Tiger,Morris,KS,Little Falls,85473
+Ebony,Franks,NY,Saginaw,X8B 5E6
+Fallon,Singleton,Yukon,Commerce,R9A 2K7
+Lester,Palmer,Prince Edward Island,Laurel,44859
+Zena,Decker,GA,Winston-Salem,X2C 8D2
+Diana,Vega,North Carolina,Covina,J2O 8Z0
+Ocean,Weber,YT,Lebanon,O4G 6G3
+Wade,Key,QC,Guthrie,57605
+May,Cleveland,NU,Olean,X8Y 3A5
+Kalia,Hensley,Alberta,Attleboro,85352
+Jeremy,Brown,ON,Minneapolis,89868
+Blaze,Mooney,Arizona,Rolling Hills Estates,C8H 9J6
+Kaden,Morse,SC,Bowie,50748
+Harper,Hansen,AB,Valparaiso,V4I 6P2
+Ainsley,Edwards,ON,Connellsville,I2W 4Y3
+Damon,Frazier,NM,Houma,02838
+Jerome,Mcbride,Prince Edward Island,Phoenix,30148
+Josephine,Herring,Wyoming,Barrow,74176
+Alexa,Bowman,NS,Yukon,K7U 7S9
+Colin,Middleton,ON,Suffolk,K5V 8Q5
+Kylee,Murphy,KY,Hutchinson,P8A 4N7
+Emily,Powell,IL,Princeton,Y2P 8A2
+Gwendolyn,Matthews,AB,Elsmere,S8G 7F7
+Kareem,Lee,Saskatchewan,Brookfield,20798
+Portia,Carroll,Newfoundland and Labrador,Brockton,25318
+Brianna,Hall,DE,Charleston,S3Q 9I3
+Tanya,Colon,Alberta,Pascagoula,23877
+Winifred,Michael,NL,Hornell,K4S 6C6
+Zephr,Jones,NM,Caguas,26964
+Nicole,Gallegos,Pennsylvania,New Madrid,96343
+Amelia,Kent,AB,New Bedford,V7Y 1J8
+Olivia,Schwartz,AB,Wisconsin Dells,V9L 4J8
+Priscilla,Hawkins,PE,Orange,54645
+Andrew,Gates,Saskatchewan,Rosemead,02306
+Sean,Hicks,Nunavut,Eau Claire,S9D 3K8
+Quentin,Alford,Oregon,Pomona,72996
+Alana,Morin,NT,Auburn,N3F 4J4
+Shay,Contreras,Saskatchewan,Santa Fe,X7B 2Q0
+Steven,Morin,TX,Grand Island,08123
+Keely,Padilla,AL,Fort Dodge,P6C 3S3
+TaShya,Reid,New Hampshire,Franklin,Y2C 8J3
+Brock,Tate,New Brunswick,Idaho Springs,N1X 6H6
+Valentine,Casey,SD,Fond du Lac,U3I 8C6
+Cassady,Cunningham,MB,San Mateo,33629
+Tasha,Stone,West Virginia,Avalon,01425
+Edan,Hammond,SC,Blythe,T9Q 2F2
+Howard,Burgess,GA,Vincennes,55979
+Desiree,Drake,Ontario,Duquesne,I3C 3W2
+Hyacinth,Byrd,WV,Lake Charles,L4B 1J8
+Zia,Moran,New Brunswick,Lynwood,38040
+Sydnee,Parrish,Alberta,Fort Dodge,77918
+Wyoming,Lowery,Yukon,LaGrange,G9Y 6V9
+Anthony,Underwood,PE,Janesville,L9X 2X0
+Allen,Stevens,Ontario,Bell Gardens,O3G 5X0
+Ivory,White,NB,South El Monte,22714
+Nasim,Cotton,NU,Nashua,48071
+Laura,Olson,AB,Hastings,42464
+Dahlia,Sheppard,Saskatchewan,St. Marys,84133
+Dora,Carey,NC,Lakewood,18460
+Vernon,Fulton,MB,Covington,O1R 5N8
+Sacha,Snider,WI,West Palm Beach,28013
+Merritt,Contreras,YT,Fallon,32049
+Kendall,Warner,Newfoundland and Labrador,Bowling Green,10748
+Rudyard,Leonard,NS,Las Vegas,67390
+Iris,Hayden,YT,Cedar Falls,62673
+Jasper,Cooper,MD,Kona,29787
+Aurora,Sparks,NS,Natchez,87459
+Bianca,Ford,MO,Dickinson,66786
+James,Roach,Nunavut,Troy,55532
+Yeo,Sykes,Nevada,Peoria,N9Y 2Q7
+Anjolie,Dudley,QC,Pullman,36382
+Neville,Campbell,Indiana,West Hartford,40449
+Madeson,Atkins,TN,Sutter Creek,96232
+Lynn,Delacruz,ND,Rolling Hills Estates,67917
+Hadassah,May,LA,Suffolk,E7T 2J0
+Kaseem,Mcpherson,PE,Salem,87566
+Phelan,Gates,Alaska,Huntington,76853
+Amaya,Sullivan,NU,Moultrie,L4Y 3C5
+Uma,Riggs,NB,Schenectady,65323
+Scarlet,Mullen,PE,Butler,U2T 2C0
+Elijah,Mcpherson,HI,La Verne,69112
+Jolie,Massey,BC,Worland,I3W 5A0
+Leo,Bond,MB,Fairfax,O8X 5Z0
+Remedios,Jimenez,North Carolina,Seattle,71966
+Wynne,Nixon,AB,Hanahan,36015
+Kyra,Chandler,NL,Edina,00572
+

data/test/test_s3io.rb

@@ -0,0 +1,16 @@
+require 'test/unit'
+require 's3io'
+
+class S3ioTest < Test::Unit::TestCase
+  def test_s3io_new
+    s3object = Object.new
+    wrapper = S3io.new(s3object)
+    assert_equal(S3io::Wrapper, wrapper.class)
+  end
+
+  def test_s3io_new_with_options
+    s3object = Object.new
+    wrapper = S3io.new(s3object, :line_buffer_size => 128)
+    assert_equal(128, wrapper.options[:line_buffer_size])
+  end
+end

data/test/test_s3io_wrapper.rb

@@ -0,0 +1,128 @@
+require 'test/unit'
+require 's3io'
+require 'stringio'
+
+# Emulate S3Object in a way that allows us to check if its methods are being called properly
+class S3ObjectMock
+  def initialize(body = '')
+    @body = body
+  end
+
+  def read(options = {})
+    range = options[:range]
+    fail "The mock should be called with a :range option" unless range
+
+    return @body[range]
+  end
+
+  def content_length
+    @body.size
+  end
+end
+
+class S3ioWrapperTest < Test::Unit::TestCase
+  S3_TEST_DATA = File.read('test/s3_test_data.csv')
+
+  def setup
+    @s3object = S3ObjectMock.new(S3_TEST_DATA)
+  end
+
+  def test_s3io_wrapper_full_read
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    assert_equal(S3_TEST_DATA, wrapper.read)
+  end
+
+  def test_s3io_wrapper_zero_read
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    assert_equal('', wrapper.read(0))
+    assert_equal(0, wrapper.pos)
+  end
+
+  def test_s3io_wrapper_partial_read
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    assert_equal(S3_TEST_DATA[0..99], wrapper.read(100))
+    assert_equal(S3_TEST_DATA[100..100], wrapper.read(1))
+  end
+
+  def test_s3io_wrapper_each
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    lines = []
+    wrapper.each do |line|
+      lines << line
+    end
+
+    assert_equal(S3_TEST_DATA.lines.to_a, lines)
+  end
+
+  def test_s3io_wrapper_each_enum
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    assert_equal(S3_TEST_DATA.lines.to_a,
+                 wrapper.lines.to_a)
+  end
+
+  def test_s3io_wrapper_pos
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    assert_equal(0, wrapper.pos)
+
+    wrapper.pos = 77
+
+    assert_equal(77, wrapper.pos)
+    assert_equal(S3_TEST_DATA[77, 32], wrapper.read(32))
+  end
+
+  def test_s3io_wrapper_pos_beyond
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    pos_beyond = @s3object.content_length + 100
+    wrapper.pos = pos_beyond
+
+    assert_equal('', wrapper.read)
+    assert_equal(pos_beyond, wrapper.pos)
+  end
+
+  def test_s3io_wrapper_rewind
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    wrapper.lines do |line|
+      # iterate through all the lines
+    end
+
+    wrapper.rewind
+    assert_equal(0, wrapper.pos)
+
+    assert_equal(S3_TEST_DATA[0..100], wrapper.read(101))
+  end
+
+  def test_s3io_wrapper_line_buffer_size
+    wrapper = S3io::Wrapper.new(@s3object, :line_buffer_size => 25)
+
+    wrapper.lines.each_with_index do |line, index|
+      break if index == 1 # skip two buffered reads
+    end
+
+    assert_equal(50, wrapper.pos)
+    assert_equal(S3_TEST_DATA[50..-1], wrapper.read)
+  end
+
+  def test_s3io_wrapper_lines_custom_separator
+    wrapper = S3io::Wrapper.new(@s3object)
+
+    assert_equal(S3_TEST_DATA.lines(",").to_a, wrapper.lines(",").to_a)
+  end
+
+  def test_s3io_wrapper_empty_each
+    wrapper = S3io::Wrapper.new(S3ObjectMock.new(''))
+
+    wrapper.each do |line|
+      assert_equal(false, :never_gets_called)
+    end
+
+    assert_equal('', wrapper.read)
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: s3io
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Arthur Pirogovski
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-12-13 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: aws-sdk
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: An IO-compatible wrapper for S3
+email: 
+- arthur@flyingtealeaf.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/s3io.rb
+- lib/s3io/version.rb
+- lib/s3io/wrapper.rb
+- s3io.gemspec
+- test/s3_test_data.csv
+- test/test_s3io.rb
+- test/test_s3io_wrapper.rb
+homepage: http://github.com/fiksu/s3io
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Amazon's official AWS SDK provides an API for S3 that isn't compatible with Ruby's standard IO class and its derivatives. This gem provides a thin wrapper around AWS SDK that makes it possible to access objects stored on S3 as if they were instances of File or StringIO classes.
+test_files: 
+- test/s3_test_data.csv
+- test/test_s3io.rb
+- test/test_s3io_wrapper.rb
+has_rdoc: