aws-s3 0.2.1 → 0.3.0

data/INSTALL

@@ -30,6 +30,10 @@ If you go the svn route, be sure that you have all the dependencies installed. T
   
 == Dependencies
 
+AWS::S3 requires Ruby 1.8.4 or greater. 
+
+It also has the following dependencies:
+
   sudo gem i xml-simple -ry
   sudo gem i builder    -ry
   sudo gem i mime-types -ry

data/README

@@ -93,11 +93,7 @@ be found in the documentation for Bucket.find.
 To add an object to a bucket you specify the name of the object, its value, and the bucket to put it in.
 
   file = 'black-flowers.mp3'
-  S3Object.store(
-    file, 
-    File.open(file), 
-    'jukebox'
-  )
+  S3Object.store(file, open(file), 'jukebox')
 
 You'll see your file has been added to it:
 
@@ -129,11 +125,7 @@ bucket.
 
 You can store an object on S3 by specifying a key, its data and the name of the bucket you want to put it in:
 
-  S3Object.store(
-    'headshot.jpg', 
-    File.open('headshot.jpg'), 
-    'photos'
-  )
+  S3Object.store('me.jpg', open('headshot.jpg'), 'photos')
 
 The content type of the object will be inferred by its extension. If the appropriate content type can not be inferred, S3 defaults
 to <tt>binary/octect-stream</tt>.
@@ -143,7 +135,7 @@ If you want to override this, you can explicitly indicate what content type the
   file = 'black-flowers.m4a'
   S3Object.store(
     file,
-    File.open(file),
+    open(file),
     'jukebox',
     :content_type => 'audio/mp4a-latm'
   )
@@ -165,7 +157,7 @@ You can fetch just the object's data directly:
 
 Or stream it by passing a block to <tt>stream</tt>:
 
-  File.open('song.mp3', 'w') do |file|
+  open('song.mp3', 'w') do |file|
     S3Object.stream('song.mp3', 'jukebox') do |chunk|
       file.write chunk
     end
@@ -179,6 +171,9 @@ tell the object to reload its <tt>value</tt>:
 
 Other functionality includes:
 
+  # Check if an object exists?
+  S3Object.exists? 'headshot.jpg', 'photos'
+
   # Copying an object
   S3Object.copy 'headshot.jpg', 'headshot2.jpg', 'photos'
 
@@ -247,14 +242,10 @@ If <tt>data</tt> is an I/O stream it will be read in segments and written to the
 may be desirable for very large files so they are not read into memory all at once.
 
   # Non streamed upload
-  S3Object.store('simple-text-file.txt', 
-                 'hello world!', 
-                 'marcel')
+  S3Object.store('greeting.txt', 'hello world!', 'marcel')
 
   # Streamed upload
-  S3Object.store('roots.mpeg', 
-                 File.open('roots.mpeg'), 
-                 'marcel')
+  S3Object.store('roots.mpeg', open('roots.mpeg'), 'marcel')
 
 
 == Setting the current bucket
@@ -269,11 +260,8 @@ a subclass of Bucket or S3Object and telling it what bucket to use:
 
 For all methods that take a bucket name as an argument, the current bucket will be used if the bucket name argument is omitted.
 
-  other_song = '/Users/marcel/baby-please-come-home.mp3'
-  JukeBoxSong.store(
-    File.basename(other_song), 
-    File.open(other_song)
-  )
+  other_song = 'baby-please-come-home.mp3'
+  JukeBoxSong.store(other_song, open(other_song))
 
 This time we didn't have to explicitly pass in the bucket name, as the JukeBoxSong class knows that it will
 always use the 'jukebox' bucket. 
@@ -477,12 +465,25 @@ Unless you specify otherwise, logs will be written to the bucket you want to log
 
 Now instead of logging right into the jukebox bucket, the logs will go into the bucket called jukebox-logs.
 
-Once logs have accumulated, you can access them using the <tt>log</tt> method:
+Once logs have accumulated, you can access them using the <tt>logs</tt> method:
 
   pp Bucket.logs('jukebox')
-  [#<AWS::S3::S3Object '/jukebox-logs/log-2006-11-14-07-15-24-2061C35880A310A1'>,
-   #<AWS::S3::S3Object '/jukebox-logs/log-2006-11-14-08-15-27-D8EEF536EC09E6B3'>,
-   #<AWS::S3::S3Object '/jukebox-logs/log-2006-11-14-08-15-29-355812B2B15BD789'>]
+  [#<AWS::S3::Logging::Log '/jukebox-logs/log-2006-11-14-07-15-24-2061C35880A310A1'>,
+   #<AWS::S3::Logging::Log '/jukebox-logs/log-2006-11-14-08-15-27-D8EEF536EC09E6B3'>,
+   #<AWS::S3::Logging::Log '/jukebox-logs/log-2006-11-14-08-15-29-355812B2B15BD789'>]
+
+Each log has a <tt>lines</tt> method that gives you information about each request in that log. All the fields are available 
+as named methods. More information is available in Logging::Log::Line.
+
+  logs = Bucket.logs('jukebox')
+  log  = logs.first
+  line = log.lines.first
+  line.operation
+  # => 'REST.GET.LOGGING_STATUS'
+  line.request_uri
+  # => 'GET /jukebox?logging HTTP/1.1'
+  line.remote_ip
+  # => "67.165.183.125"
 
 Disabling logging is just as simple as enabling it:
 

data/Rakefile

@@ -46,7 +46,7 @@ namespace :doc do
       strip_comments[info.comment]
     end
     
-    File.open('README', 'w') do |file|
+    open('README', 'w') do |file|
       file.write ERB.new(IO.read('README.erb')).result(binding)
     end
   end
@@ -120,7 +120,7 @@ namespace :dist do
     changelog = IO.read('CHANGELOG')
     changelog.sub!(/^trunk:/, "#{spec.version}:")
     
-    File.open('CHANGELOG', 'w') do |file|
+    open('CHANGELOG', 'w') do |file|
       file.write "trunk:\n\n#{changelog}"
     end
   end
@@ -187,6 +187,19 @@ task :stats do
 end
 
 namespace :test do
+  find_file = lambda do |name|
+    file_name = lambda {|path| File.join(path, "#{name}.rb")}
+    root = $:.detect do |path|
+      File.exist?(file_name[path])
+    end
+    file_name[root] if root
+  end
+  
+  TEST_LOADER = find_file['rake/rake_test_loader']
+  multiruby   = lambda do |glob|
+    system 'multiruby', TEST_LOADER, *Dir.glob(glob)
+  end
+    
   desc 'Check test coverage'
   task :coverage do
     system("rcov --sort coverage #{File.join(library_root, 'test/*_test.rb')}")
@@ -209,6 +222,23 @@ namespace :test do
     show_test_coverage_results
   end
   
+  desc 'Run local tests against multiple versions of Ruby'
+  task :version_audit do
+    multiruby['test/*_test.rb']
+  end
+  
+  namespace :version_audit do
+    desc 'Run remote tests against multiple versions of Ruby'
+    task :remote do
+      multiruby['test/remote/*_test.rb']
+    end
+    
+    desc 'Run all tests against multiple versions of Ruby'
+    task :all do
+      multiruby['test/**/*_test.rb']
+    end
+  end
+  
   def show_test_coverage_results
     system("open #{File.join(library_root, 'coverage/index.html')}") if PLATFORM['darwin']
   end
@@ -260,7 +290,6 @@ namespace :site do
   require 'rdoc/markup/simple_markup/to_html'
   
   readme    = lambda { IO.read('README')[/^== Getting started\n(.*)/m, 1] }
-  ruby_code = /^\s*\n(\s+\S+.*?)\n\s*\n/m
 
   readme_to_html = lambda do
     handler = SM::ToHtml.new
@@ -277,7 +306,7 @@ namespace :site do
   
   desc 'Regenerate the public website page'
   task :build => 'doc:readme' do
-    File.open('site/public/index.html', 'w') do |file|
+    open('site/public/index.html', 'w') do |file|
       erb_data = {}
       erb_data[:readme] = readme_to_html.call
       file.write ERB.new(IO.read('site/index.erb')).result(binding)

data/lib/aws/s3.rb

@@ -6,6 +6,7 @@ require 'digest/sha1'
 require 'net/https'
 require 'time'
 require 'date'
+require 'open-uri'
 
 $:.unshift(File.dirname(__FILE__))
 require 's3/extensions'

data/lib/aws/s3/acl.rb

@@ -131,7 +131,7 @@ module AWS
         def to_xml
           Builder.new(owner, grants).to_s
         end
-        
+
         private
         
           def owner?
@@ -170,6 +170,11 @@ module AWS
                 super
               end
             end
+            
+            # Two grant lists are equal if they have identical grants both in terms of permission and grantee.
+            def ==(grants)
+              size == grants.size && all? {|grant| grants.include?(grant)}
+            end
           end
         
         class Builder < XmlGenerator #:nodoc:
@@ -313,7 +318,7 @@ module AWS
         end
         
         def inspect #:nodoc:
-          "#<#{self.class}:##{object_id} #{self}>"
+          "#<%s:0x%s %s>" % [self.class, object_id, self]
         end
         
         def to_s #:nodoc:
@@ -440,7 +445,7 @@ module AWS
         end
         
         def inspect #:nodoc:
-          "#<#{self.class}:##{object_id} #{type_representation || '(xsi not set yet)'}>"
+          "#<%s:0x%s %s>" [self.class, object_id, type_representation || '(type not set yet)']
         end
         
         private

data/lib/aws/s3/base.rb

@@ -66,7 +66,7 @@ module AWS #:nodoc:
         def request(verb, path, options = {}, body = nil, attempts = 0, &block)
           Service.response = nil
           process_options!(options, verb)
-          response = response_class.new(connection.request(verb, URI.escape(path), options, body, &block))
+          response = response_class.new(connection.request(verb, path, options, body, &block))
           Service.response = response
 
           Error::Response.new(response.response).error.raise if response.error?
@@ -74,7 +74,8 @@ module AWS #:nodoc:
         # Once in a while, a request to S3 returns an internal error. A glitch in the matrix I presume. Since these 
         # errors are few and far between the request method will rescue InternalErrors the first three times they encouter them
         # and will retry the request again. Most of the time the second attempt will work.
-        rescue InternalError, Timeout::Error
+        rescue *retry_exceptions
+          body.rewind if body.respond_to?(:rewind)        
           attempts == 3 ? raise : (attempts += 1; retry)
         end
 
@@ -111,11 +112,8 @@ module AWS #:nodoc:
         # 
         # For all methods that take a bucket name as an argument, the current bucket will be used if the bucket name argument is omitted.
         #
-        #   other_song = '/Users/marcel/baby-please-come-home.mp3'
-        #   JukeBoxSong.store(
-        #     File.basename(other_song), 
-        #     File.open(other_song)
-        #   )
+        #   other_song = 'baby-please-come-home.mp3'
+        #   JukeBoxSong.store(other_song, open(other_song))
         # 
         # This time we didn't have to explicitly pass in the bucket name, as the JukeBoxSong class knows that it will
         # always use the 'jukebox' bucket. 
@@ -176,6 +174,10 @@ module AWS #:nodoc:
           def bucket_name(name)
             name || current_bucket
           end
+
+          def retry_exceptions
+            [InternalError, Timeout::Error, Errno::EPIPE, Errno::EINVAL, RequestTimeout]
+          end
           
           class RequestOptions < Hash #:nodoc:
             attr_reader :options, :verb
@@ -228,4 +230,4 @@ module AWS #:nodoc:
         end
     end
   end
-end
+end

data/lib/aws/s3/bucket.rb

@@ -35,11 +35,7 @@ module AWS
     # To add an object to a bucket you specify the name of the object, its value, and the bucket to put it in.
     # 
     #   file = 'black-flowers.mp3'
-    #   S3Object.store(
-    #     file, 
-    #     File.open(file), 
-    #     'jukebox'
-    #   )
+    #   S3Object.store(file, open(file), 'jukebox')
     #
     # You'll see your file has been added to it:
     # 

data/lib/aws/s3/connection.rb

@@ -5,6 +5,11 @@ module AWS
         def connect(options = {})
           new(options)
         end
+        
+        def prepare_path(path)
+          path = path.remove_extended unless path.utf8?
+          URI.escape(path)
+        end
       end
       
       attr_reader :access_key_id, :secret_access_key, :http, :options
@@ -19,8 +24,11 @@ module AWS
       end
           
       def request(verb, path, headers = {}, body = nil, &block)
-        http.start do
+        requester = Proc.new do 
+          path    = self.class.prepare_path(path)
           request = request_method(verb).new(path, headers)
+          ensure_content_type!(request)
+          add_user_agent!(request)
           authenticate!(request)
           if body
             if body.respond_to?(:read)                                                                
@@ -32,10 +40,17 @@ module AWS
           end
           http.request(request, &block)
         end
+        
+        if persistent?
+          http.start unless http.started?
+          requester.call
+        else
+          http.start(&requester)
+        end
       end
       
       def url_for(path, options = {})
-        path         = URI.escape(path)
+        path         = self.class.prepare_path(path)
         request      = request_method(:get).new(path, {})
         query_string = query_string_authentication(request, options)
         "#{protocol(options)}#{http.address}#{path}?#{query_string}"
@@ -45,11 +60,15 @@ module AWS
         http.address[/^([^.]+).#{DEFAULT_HOST}$/, 1]
       end
       
+      def persistent?
+        options[:persistent]
+      end
+      
       def protocol(options = {})
         (options[:use_ssl] || http.use_ssl?) ? 'https://' : 'http://'
       end
       
-      private      
+      private
         def extract_keys!
           missing_keys = []
           extract_key = Proc.new {|key| options[key] || (missing_keys.push(key); nil)}
@@ -60,16 +79,25 @@ module AWS
         
         def connect
           extract_keys!
-          @http         = Net::HTTP.new(options[:server], options[:port])
-          @http.use_ssl = !options[:use_ssl].nil? || options[:port] == 443
+          @http             = Net::HTTP.new(options[:server], options[:port])
+          @http.use_ssl     = !options[:use_ssl].nil? || options[:port] == 443
+          @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
           @http
         end
         
+        def ensure_content_type!(request)
+          request['Content-Type'] ||= 'binary/octet-stream'
+        end
+        
         # Just do Header authentication for now
         def authenticate!(request)
           request['Authorization'] = Authentication::Header.new(request, access_key_id, secret_access_key)
         end
         
+        def add_user_agent!(request)
+          request['User-Agent'] ||= "AWS::S3/#{Version}"
+        end
+        
         def query_string_authentication(request, options = {})
           Authentication::QueryString.new(request, access_key_id, secret_access_key, options)
         end
@@ -121,6 +149,9 @@ module AWS
           # argument is set.
           # * <tt>:use_ssl</tt> - Whether requests should be made over SSL. If set to true, the <tt>:port</tt> argument
           # will be implicitly set to 443, unless specified otherwise. Defaults to false.
+          # * <tt>:persistent</tt> - Whether to use a persistent connection to the server. Having this on provides around a two fold 
+          # performance increase but for long running processes some firewalls may find the long lived connection suspicious and close the connection.
+          # If you run into connection errors, try setting <tt>:persistent</tt> to false. Defaults to true.
           def establish_connection!(options = {})
             # After you've already established the default connection, just specify 
             # the difference for subsequent connections
@@ -147,13 +178,16 @@ module AWS
           
           # Removes the connection for the current class. If there is no connection for the current class, the default
           # connection will be removed.
-          def disconnect
-            connections.delete(connection_name) || connections.delete(default_connection)
+          def disconnect(name = connection_name)
+            name       = default_connection unless connections.has_key?(name)
+            connection = connections[name]
+            connection.http.finish if connection.persistent?
+            connections.delete(name)
           end
           
           # Clears *all* connections, from all classes, with prejudice. 
           def disconnect!
-            connections.clear
+            connections.each_key {|connection| disconnect(connection)}
           end
 
           private
@@ -174,7 +208,7 @@ module AWS
       class Options < Hash #:nodoc:
         class << self
           def valid_options
-            [:access_key_id, :secret_access_key, :server, :port, :use_ssl]
+            [:access_key_id, :secret_access_key, :server, :port, :use_ssl, :persistent]
           end
         end
         
@@ -183,12 +217,17 @@ module AWS
           super()
           @options = options
           validate!
+          extract_persistent!
           extract_server!
           extract_port!
           extract_remainder!
         end
         
         private
+          def extract_persistent!
+            self[:persistent] = options.has_key?(:persitent) ? options[:persitent] : true
+          end
+          
           def extract_server!
             self[:server] = options.delete(:server) || DEFAULT_HOST
           end