uber-s3 0.1.1

data/README.md

@@ -0,0 +1,122 @@
+# Uber-S3
+
+A simple, but very fast, S3 client written in Ruby supporting
+synchronous and asynchronous HTTP communication.
+
+## Examples
+
+```ruby
+require 'uber-s3'
+
+##########################################################################
+# Connecting to S3
+# adapter can be :net_http or :em_http_fibered
+s3 = UberS3.new({
+  :access_key         => 'abc',
+  :secret_access_key  => 'def',
+  :bucket             => 'funbucket',
+  :persistent         => true,
+  :adapter            => :em_http_fibered
+})
+
+
+
+##########################################################################
+# Saving objects
+s3.store('/test.txt', 'Look ma no hands')
+
+o = s3.object('/test.txt')
+o.value = 'Look ma no hands'
+o.save
+
+# or..
+
+o = UberS3::Object.new(client.bucket, '/test.txt', 'heyo')
+o.save # => true
+
+
+
+##########################################################################
+# Reading objects
+s3['/test.txt'].class       # => UberS3::Object
+s3['/test.txt'].value       # => 'heyo'
+s3.get('/test.txt').value   # => 'heyo'
+
+s3.exists?('/anotherone')   # => false
+
+
+
+##########################################################################
+# Object access control
+
+o.access = :private
+o.access = :public_read
+# etc.
+
+# Valid options:
+# :private, :public_read, :public_read_write, :authenticated_read
+
+# See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/index.html?RESTAccessPolicy.html
+# NOTE: default object access level is :private
+
+
+
+##########################################################################
+# Deleting objects
+o.delete # => true
+
+
+##########################################################################
+# Save optional parameters
+# See http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?RESTObjectPUT.html
+
+options = { :access => :public_read, :content_type => 'text/plain' }
+o = UberS3::Object.new(client.bucket, '/test.txt', 'heyo', options)
+o.save
+
+# or..
+
+o = s3.object('/test.txt')
+o.value = 'Look ma no hands'
+o.access = :public_read
+o.content_type = 'text/plain'
+o.save
+
+# List of parameter methods:
+# :access               -- Object access control
+# :cache_control        -- http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
+# :content_disposition  -- http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1
+# :content_encoding     -- http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
+# :content_md5          -- End-to-end integrity check
+# :content_type         -- http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17
+# :expires              -- Number of milliseconds before expiration
+# :storage_class        -- Amazon S3's storage levels (redundancy for price)
+
+
+##########################################################################
+# Iterating objects in a bucket
+s3.objects('/path').each {|obj| puts obj } 
+
+```
+
+## Ruby version notes
+
+* Tested on MRI 1.9.2 (net_http / em_http_fibered adapters) -- recommended
+* Ruby 1.8.7 works for net/http clients, em_http_fibered adapter requires fibers (duh)
+* JRuby in 1.9 mode + em_http_fibered adapter should work, but held back because of issues with EM
+
+## Other notes
+
+* If Nokogiri is available, it will use it instead of REXML
+
+## TODO
+
+* Refactor UberS3::Object class, consider putting the operations/headers into separate classes
+* Better exception handling and reporting
+* Query string authentication -- neat feature for providing temporary public access to a private object
+* Object versioning support
+
+## S3 API Docs
+
+- S3 REST API: http://docs.amazonwebservices.com/AmazonS3/latest/API/
+- S3 Request Authorization: http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html

data/lib/uber-s3.rb

@@ -0,0 +1,35 @@
+require 'cgi'
+require 'time'
+require 'openssl'
+require 'forwardable'
+require 'base64'
+require 'digest/md5'
+
+class UberS3
+  extend Forwardable
+  
+  attr_accessor :connection, :bucket  
+  def_delegators :@bucket, :store, :set, :object, :get, :[], :exists?, :objects
+  
+  def initialize(options={})
+    self.connection = Connection.open(self, options)
+    self.bucket     = options[:bucket]
+  end
+  
+  def inspect
+    "#<UberS3 client v#{UberS3::VERSION}>"
+  end
+  
+  def bucket=(bucket)
+    @bucket = bucket.is_a?(Bucket) ? bucket : Bucket.new(self, bucket)
+  end
+end
+
+require 'uber-s3/version'
+require 'uber-s3/exceptions'
+require 'uber-s3/connection'
+require 'uber-s3/authorization'
+require 'uber-s3/bucket'
+require 'uber-s3/object'
+
+require 'uber-s3/util/xml_document'

data/lib/uber-s3/authorization.rb

@@ -0,0 +1,29 @@
+class UberS3
+  module Authorization
+    
+    def sign(client, verb, path, headers={})
+      req_verb                  = verb.to_s.upcase
+      req_content_md5           = headers['Content-MD5']
+      req_content_type          = headers['Content-Type']
+      req_date                  = headers['Date']
+      req_canonical_resource    = "/#{client.bucket}/#{path}".split('?').first
+      req_canonical_amz_headers = ''
+
+      headers.keys.select {|k| k =~ /^x-amz-acl/ }.each do |amz_key|
+        req_canonical_amz_headers << amz_key+':'+headers[amz_key]+"\n"
+      end
+      
+      canonical_string_to_sign = "#{req_verb}\n"+
+                                 "#{req_content_md5}\n"+
+                                 "#{req_content_type}\n"+
+                                 "#{req_date}\n"+
+                                 "#{req_canonical_amz_headers}"+
+                                 "#{req_canonical_resource}"
+      
+      digest = OpenSSL::Digest::Digest.new('sha1')
+      [OpenSSL::HMAC.digest(digest, client.connection.secret_access_key, canonical_string_to_sign)].pack("m").strip
+    end
+    
+    extend self    
+  end
+end

data/lib/uber-s3/bucket.rb

@@ -0,0 +1,99 @@
+class UberS3
+  class Bucket
+    
+    attr_accessor :client, :name
+    
+    def initialize(client, name)
+      self.client = client
+      self.name   = name
+    end
+
+    def to_s
+      name.to_s
+    end
+    
+    def connection
+      client.connection
+    end
+
+    def store(key, value, options={})
+      Object.new(self, key, value, options).save
+    end
+    alias_method :set, :store
+
+    def object(key)
+      Object.new(self, key)
+    end
+
+    def get(key)
+      object(key).fetch
+    end
+    alias_method :[], :get
+
+    def exists?(key)
+      object(key).exists?
+    end
+    
+    def objects(key, options={})
+      ObjectList.new(self, key, options)
+    end
+    
+    
+    class ObjectList
+      attr_accessor :bucket, :key, :options, :objects
+      
+      def initialize(bucket, key, options={})
+        self.bucket   = bucket
+        self.key      = key.gsub(/^\//, '')
+        self.options  = options
+        self.objects  = []
+      end
+      
+      def fetch(marker=nil)
+        @objects = []
+
+        response = bucket.connection.get("/?prefix=#{CGI.escape(key)}&marker=#{marker}")
+        # response = bucket.connection.get("/?prefix=#{CGI.escape(key)}&marker=#{marker}&max-keys=2")
+        
+        if response[:status] != 200
+          raise UberS3Error
+        else
+          @objects = parse_contents(response[:body])
+        end
+      end
+      
+      def parse_contents(xml)
+        objects = []
+        doc = Util::XmlDocument.new(xml)
+        
+        # TODO: can use more error checking on the xml stuff
+        
+        @is_truncated = doc.xpath('//ListBucketResult/IsTruncated').first.text == "true"
+        contents = doc.xpath('//ListBucketResult/Contents')
+        
+        contents.each do |content|
+          h = {}
+          content.elements.each {|el| h[el.name] = el.text }
+          objects << ::UberS3::Object.new(bucket, h['Key'], nil, { :size => h['Size'].to_i })
+        end if contents.any?
+
+        objects
+      end
+      
+      def each(&block)
+        loop do
+          marker = objects.last.key rescue nil
+          fetch(marker)
+          
+          objects.each {|obj| block.call(obj) }
+          break if @is_truncated == false
+        end
+      end
+      
+      def to_a
+        fetch
+      end
+    end
+    
+  end
+end

data/lib/uber-s3/connection.rb

@@ -0,0 +1,62 @@
+class UberS3
+  module Connection
+    
+    def self.open(client, options={})
+      adapter = options.delete(:adapter) || :net_http      
+      
+      begin
+        require "uber-s3/connection/#{adapter}"
+        klass = instance_eval(adapter.to_s.split('_').map {|x| x.capitalize}.join(""))
+      rescue LoadError
+        raise "Cannot load #{adapter} adapter class"
+      end
+      
+      klass.new(client, options)
+    end
+    
+    
+    class Adapter
+    
+      attr_accessor :client, :access_key, :secret_access_key, :persistent
+    
+      def initialize(client, options={})
+        self.client             = client
+        self.access_key         = options[:access_key]
+        self.secret_access_key  = options[:secret_access_key]
+        self.persistent         = options[:persistent] || true
+      end
+    
+      [:get, :post, :put, :delete, :head].each do |verb|
+        define_method(verb) do |*args|
+          path, headers, body = args
+          path = path.gsub(/^\//, '')
+          headers ||= {}
+
+          # Default headers
+          headers['Date'] = Time.now.httpdate if !headers.keys.include?('Date')
+          headers['User-Agent'] ||= "UberS3 v#{UberS3::VERSION}"
+          headers['Connection'] = (persistent ? 'keep-alive' : 'close')
+          
+          if body
+            headers['Content-Type']   ||= 'application/octet-stream'
+            headers['Content-Length'] ||= body.bytesize.to_s
+          end
+          
+          # Authorize the request          
+          signature = Authorization.sign(client, verb, path, headers)
+          headers['Authorization'] = "AWS #{access_key}:#{signature}"
+          
+          # Make the request
+          url = "http://#{client.bucket}.s3.amazonaws.com/#{path}"
+          request(verb, url, headers, body)
+        end
+      end
+
+      def request(verb, url, headers={}, body=nil)
+        raise "Abstract method"
+      end
+
+    end
+    
+  end  
+end

data/lib/uber-s3/connection/em_http.rb

@@ -0,0 +1,13 @@
+require 'eventmachine'
+require 'em-http'
+
+module UberS3::Connection
+  class EmHttp < Adapter
+    
+    # NOTE: this will be very difficult to support
+    # with our interface.. will need lots of work
+    # perhaps will limit async capabilities to
+    # fibered mode
+    
+  end
+end

data/lib/uber-s3/connection/em_http_fibered.rb

@@ -0,0 +1,24 @@
+require 'eventmachine'
+require 'em-http'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+
+module UberS3::Connection
+  class EmHttpFibered < Adapter
+    
+    def request(verb, url, headers={}, body=nil)
+      params = {}
+      params[:head] = headers
+      params[:body] = body if body
+    
+      r = EM::HttpRequest.new(url).send(verb, params)
+      
+      {
+        :status => r.response_header.status,
+        :header => r.response_header,
+        :body   => r.response
+      }
+    end
+    
+  end
+end

data/lib/uber-s3/connection/net_http.rb

@@ -0,0 +1,23 @@
+require 'net/http'
+
+module UberS3::Connection
+  class NetHttp < Adapter
+    
+    def request(verb, url, headers={}, body=nil)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      
+      req_klass = instance_eval("Net::HTTP::"+verb.to_s.capitalize)
+      req = req_klass.new(uri.to_s, headers)
+            
+      r = http.request(req, body)
+      
+      {
+        :status => r.code.to_i,
+        :header => r.header.to_hash,
+        :body   => r.body
+      }
+    end
+    
+  end
+end

data/lib/uber-s3/exceptions.rb

@@ -0,0 +1,7 @@
+class UberS3
+  
+  class UberS3Error < StandardError; end
+  
+  
+  
+end

data/lib/uber-s3/object.rb

@@ -0,0 +1,130 @@
+class UberS3
+  class Object
+
+    attr_accessor :bucket, :key, :value, :errors
+    # attr_accessor :etag, :last_modified
+    
+    attr_accessor :access,
+                  :cache_control,
+                  :content_disposition,
+                  :content_encoding,
+                  :size,
+                  :content_md5,
+                  :content_type,
+                  :expires,
+                  :storage_class
+    
+    def initialize(bucket, key, value=nil, options={})
+      self.bucket = bucket
+      self.key    = key
+      self.value  = value
+      
+      options.each {|k,v| self.send((k.to_s+'=').to_sym, v) }
+    end
+    
+    def to_s
+      "#<#{self.class} @key=\"#{self.key}\">"
+    end
+    
+    def exists?
+      bucket.connection.head(key)[:status] == 200
+    end
+    
+    def fetch
+      self.value = bucket.connection.get(key)[:body]
+      self
+    end
+    
+    def save
+      headers = {}
+      
+      # Standard pass through values
+      headers['Cache-Control']        = cache_control
+      headers['Content-Disposition']  = content_disposition
+      headers['Content-Encoding']     = content_encoding
+      headers['Content-Length']       = size.to_s
+      headers['Content-MD5']          = content_md5
+      headers['Content-Type']         = content_type
+      headers['Expires']              = expires
+            
+      headers.each {|k,v| headers.delete(k) if v.nil? || v.empty? }
+
+      # Content MD5 integrity check
+      if !content_md5.nil?
+        # Our object expects a md5 hex digest
+        md5_digest = content_md5.unpack('a2'*16).collect {|i| i.hex.chr }.join
+        headers['Content-MD5'] = Base64.encode64(md5_digest).strip
+      end
+
+      # ACL
+      if !access.nil?
+        headers['x-amz-acl'] = access.to_s.gsub('_', '-')
+      end
+      
+      # Storage class
+      if !storage_class.nil?
+        headers['x-amz-storage-class'] = storage_class.to_s.upcase
+      end
+      
+      # Let's do it
+      response = bucket.connection.put(key, headers, value)
+      
+      if response[:status] != 200
+        self.errors = response[:body]    
+      else
+        self.errors = nil
+      end
+      
+      response[:status] == 200
+    end
+    
+    def delete
+      bucket.connection.delete(key)[:status] == 204
+    end
+    
+    def value
+      fetch if !@value
+      @value
+    end
+    
+    def persisted?
+      # TODO
+    end
+    
+    def url
+      # TODO
+    end
+    
+    def key=(key)
+      @key = key.gsub(/^\//,'')
+    end
+    
+    ##########################
+    
+    def value=(value)
+      self.size = value.to_s.bytesize
+      @value = value
+    end
+    
+    def access=(access)
+      valid_values = [:private, :public_read, :public_read_write, :authenticated_read]
+      
+      if valid_values.include?(access)
+        @access = access
+      else
+        raise "Invalid access value"
+      end
+    end
+    
+    def storage_class=(storage_class)
+      valid_values = [:standard, :reduced_redundancy]
+      
+      if valid_values.include?(storage_class)
+        @storage_class = storage_class
+      else
+        raise "Invalid storage_class value"
+      end
+    end
+    
+  end
+end

data/lib/uber-s3/operation.rb

@@ -0,0 +1,22 @@
+class UberS3
+  module Operation
+    
+    # Load and include all operations
+    Dir[File.dirname(__FILE__)+'/operation/*'].each do |group|
+      Dir[group+'/*.rb'].each do |op|
+      
+        group_klass_name = File.basename(group).split('_').map {|x| x.capitalize}.join("")
+        op_klass_name    = File.basename(op).gsub(/.rb$/i, '').split('_').map {|x| x.capitalize}.join("")
+        
+        require op
+        op_klass = instance_eval(group_klass_name+'::'+op_klass_name)
+        group_klass = instance_eval(group_klass_name)
+      
+        group_klass.send :include, op_klass
+      end
+    end
+        
+  end
+end
+
+

data/lib/uber-s3/operation/object/access_policy.rb

@@ -0,0 +1,20 @@
+module UberS3::Operation::Object
+  module AccessPolicy
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+        attr_accessor :access
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/cache_control.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module CacheControl
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/content_disposition.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module ContentDisposition
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/content_encoding.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module ContentEncoding
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/content_md5.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module ContentMd5
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/content_type.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module ContentType
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/expires.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module Expires
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/meta.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module Meta
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/operation/object/storage_class.rb

@@ -0,0 +1,19 @@
+module UberS3::Operation::Object
+  module StorageClass
+    
+    def self.included(base)
+      base.send :extend,  ClassMethods
+      base.send :include, InstanceMethods
+      
+      base.instance_eval do
+      end
+    end
+
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+    end
+    
+  end
+end

data/lib/uber-s3/util/xml_document.rb

@@ -0,0 +1,48 @@
+begin
+  require 'nokogiri'
+rescue LoadError
+  require 'rexml/document'
+end
+
+module UberS3::Util
+  class XmlDocument
+    
+    def initialize(xml)
+      if defined?(Nokogiri)
+        @parser = NokogiriParser.new(xml)
+      else
+        @parser = RexmlParser.new(xml)
+      end
+    end
+    
+    def method_missing(sym, *args, &block)
+      @parser.send sym, *args, &block
+    end
+        
+    class RexmlParser
+      attr_accessor :doc
+      
+      def initialize(xml)
+        self.doc = REXML::Document.new(xml)
+      end
+      
+      def xpath(path)
+        REXML::XPath.match(doc.root, path)
+      end
+    end
+    
+    class NokogiriParser
+      attr_accessor :doc
+      
+      def initialize(xml)
+        self.doc = Nokogiri::XML(xml)
+        doc.remove_namespaces!
+      end
+      
+      def xpath(path)
+        doc.xpath(path).to_a
+      end
+    end
+    
+  end
+end

data/lib/uber-s3/version.rb

@@ -0,0 +1,3 @@
+class UberS3
+  VERSION = '0.1.1'
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification 
+name: uber-s3
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Peter Kieltyka
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-01 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.6.0
+  type: :development
+  version_requirements: *id002
+description: A simple & very fast S3 client supporting sync and async communication
+email: 
+- peter@nulayer.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- lib/uber-s3/authorization.rb
+- lib/uber-s3/bucket.rb
+- lib/uber-s3/connection/em_http.rb
+- lib/uber-s3/connection/em_http_fibered.rb
+- lib/uber-s3/connection/net_http.rb
+- lib/uber-s3/connection.rb
+- lib/uber-s3/exceptions.rb
+- lib/uber-s3/object.rb
+- lib/uber-s3/operation/object/access_policy.rb
+- lib/uber-s3/operation/object/cache_control.rb
+- lib/uber-s3/operation/object/content_disposition.rb
+- lib/uber-s3/operation/object/content_encoding.rb
+- lib/uber-s3/operation/object/content_md5.rb
+- lib/uber-s3/operation/object/content_type.rb
+- lib/uber-s3/operation/object/expires.rb
+- lib/uber-s3/operation/object/meta.rb
+- lib/uber-s3/operation/object/storage_class.rb
+- lib/uber-s3/operation.rb
+- lib/uber-s3/util/xml_document.rb
+- lib/uber-s3/version.rb
+- lib/uber-s3.rb
+has_rdoc: true
+homepage: http://github.com/nulayer/uber-s3
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: A simple & very fast S3 client supporting sync and async communication
+test_files: []
+