aws-s3 0.2.1 → 0.3.0

data/lib/aws/s3/exceptions.rb

@@ -28,6 +28,9 @@ module AWS
     class NoSuchKey < ResponseError
     end
     
+    class RequestTimeout < ResponseError
+    end
+    
     # Abstract super class for all invalid options.
     class InvalidOption < S3Exception
     end

data/lib/aws/s3/extensions.rb

@@ -2,12 +2,14 @@
 
 class Hash
   def to_query_string(include_question_mark = true)
-    return '' if empty?
     query_string = ''
-    query_string << '?' if include_question_mark
-    query_string << inject([]) do |parameters, (key, value)|
-      parameters << [key, value].map {|element| element.to_s}.join('=')
-    end.join('&')
+    unless empty?
+      query_string << '?' if include_question_mark
+      query_string << inject([]) do |params, (key, value)| 
+        params << "#{key}=#{value}" 
+      end.join('&')
+    end
+    query_string
   end
   
   def to_normalized_options
@@ -25,7 +27,7 @@ end
 
 class String
   def previous!
-    self[-1] = self[-1] - 1
+    self[-1] -= 1
     self
   end
   
@@ -34,7 +36,7 @@ class String
   end
   
   def to_header
-    downcase.gsub('_', '-')
+    downcase.tr('_', '-')
   end
   
   # ActiveSupport adds an underscore method to String so let's just use that one if
@@ -43,7 +45,51 @@ class String
     gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
     gsub(/([a-z\d])([A-Z])/,'\1_\2').
     downcase
-  end unless public_instance_methods.include? 'underscore'
+  end unless public_method_defined? :underscore
+
+  def utf8?
+    scan(/[^\x00-\xa0]/u) { |s| s.unpack('U') }
+    true
+  rescue ArgumentError
+    false
+  end
+  
+  # All paths in in S3 have to be valid unicode so this takes care of 
+  # cleaning up any strings that aren't valid utf-8 according to String#utf8?
+  def remove_extended!
+    gsub!(/[\x80-\xFF]/) { "%02X" % $&[0] }
+  end
+  
+  def remove_extended
+    dup.remove_extended!
+  end
+end
+
+class CoercibleString < String
+  class << self
+    def coerce(string)
+      new(string).coerce
+    end
+  end
+  
+  def coerce
+    case self
+    when 'true':          true
+    when 'false':         false
+    when  /^\d+$/:        Integer(self)
+    when datetime_format: Time.parse(self)
+    else
+      self
+    end
+  end
+  
+  private
+    # Lame hack since Date._parse is so accepting. S3 dates are of the form: '2006-10-29T23:14:47.000Z'
+    # so unless the string looks like that, don't even try, otherwise it might convert an object's
+    # key from something like '03 1-2-3-Apple-Tree.mp3' to Sat Feb 03 00:00:00 CST 2001.
+    def datetime_format
+      /^\d{4}-\d{2}-\d{2}\w\d{2}:\d{2}:\d{2}/
+    end
 end
 
 class Symbol
@@ -214,6 +260,50 @@ module SelectiveAttributeProxy
   end
 end
 
+# When streaming data up, Net::HTTPGenericRequest hard codes a chunk size of 1k. For large files this
+# is an unfortunately low chunk size, so here we make it use a much larger default size and move it into a method
+# so that the implementation of send_request_with_body_stream doesn't need to be changed to change the chunk size (at least not anymore
+# than I've already had to...).
+module Net
+  class HTTPGenericRequest
+    def send_request_with_body_stream(sock, ver, path, f)
+      raise ArgumentError, "Content-Length not given and Transfer-Encoding is not `chunked'" unless content_length() or chunked?
+      unless content_type()
+        warn 'net/http: warning: Content-Type did not set; using application/x-www-form-urlencoded' if $VERBOSE
+        set_content_type 'application/x-www-form-urlencoded'
+      end
+      write_header sock, ver, path
+      if chunked?
+        while s = f.read(chunk_size)
+          sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
+        end
+        sock.write "0\r\n\r\n"
+      else
+        while s = f.read(chunk_size)
+          sock.write s
+        end
+      end
+    end
+    
+    def chunk_size
+      1048576 # 1 megabyte
+    end
+  end
+  
+  # Net::HTTP before 1.8.4 doesn't have the use_ssl? method or the Delete request type
+  class HTTP
+    def use_ssl?
+      @use_ssl
+    end unless public_method_defined? :use_ssl?
+    
+    class Delete < HTTPRequest
+      METHOD = 'DELETE'
+      REQUEST_HAS_BODY = false
+      RESPONSE_HAS_BODY = true
+    end unless const_defined? :Delete
+  end
+end
+
 class XmlGenerator < String #:nodoc:
   attr_reader :xml
   def initialize
@@ -222,4 +312,4 @@ class XmlGenerator < String #:nodoc:
     build
   end
 end
-#:startdoc:
+#:startdoc:

data/lib/aws/s3/logging.rb

@@ -17,12 +17,25 @@ module AWS
     # 
     # Now instead of logging right into the jukebox bucket, the logs will go into the bucket called jukebox-logs.
     # 
-    # Once logs have accumulated, you can access them using the <tt>log</tt> method:
+    # Once logs have accumulated, you can access them using the <tt>logs</tt> method:
     # 
     #   pp Bucket.logs('jukebox')
-    #   [#<AWS::S3::S3Object '/jukebox-logs/log-2006-11-14-07-15-24-2061C35880A310A1'>,
-    #    #<AWS::S3::S3Object '/jukebox-logs/log-2006-11-14-08-15-27-D8EEF536EC09E6B3'>,
-    #    #<AWS::S3::S3Object '/jukebox-logs/log-2006-11-14-08-15-29-355812B2B15BD789'>]
+    #   [#<AWS::S3::Logging::Log '/jukebox-logs/log-2006-11-14-07-15-24-2061C35880A310A1'>,
+    #    #<AWS::S3::Logging::Log '/jukebox-logs/log-2006-11-14-08-15-27-D8EEF536EC09E6B3'>,
+    #    #<AWS::S3::Logging::Log '/jukebox-logs/log-2006-11-14-08-15-29-355812B2B15BD789'>]
+    #
+    # Each log has a <tt>lines</tt> method that gives you information about each request in that log. All the fields are available 
+    # as named methods. More information is available in Logging::Log::Line.
+    #
+    #   logs = Bucket.logs('jukebox')
+    #   log  = logs.first
+    #   line = log.lines.first
+    #   line.operation
+    #   # => 'REST.GET.LOGGING_STATUS'
+    #   line.request_uri
+    #   # => 'GET /jukebox?logging HTTP/1.1'
+    #   line.remote_ip
+    #   # => "67.165.183.125"
     #
     # Disabling logging is just as simple as enabling it:
     #
@@ -70,7 +83,128 @@ module AWS
             end
           end
         end
+      
+      # A bucket log exposes requests made on the given bucket. Lines of the log represent a single request. The lines of a log
+      # can be accessed with the lines method. 
+      #
+      #   log = Bucket.logs_for('marcel').first
+      #   log.lines
+      # 
+      # More information about the logged requests can be found in the documentation for Log::Line.
+      class Log
+        def initialize(log_object) #:nodoc:
+          @log = log_object
+        end
+        
+        # Returns the lines for the log. Each line is wrapped in a Log::Line.
+        def lines
+          log.value.map {|line| Line.new(line)}
+        end
+        memoized :lines
+        
+        def inspect #:nodoc:
+          "#<%s:0x%s '%s'>" % [self.class.name, object_id, log.path]
+        end
         
+        private
+          attr_reader :log
+          
+        # Each line of a log exposes the raw line, but it also has method accessors for all the fields of the logged request.
+        #
+        # The list of supported log line fields are listed in the S3 documentation: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/LogFormat.html
+        #
+        #   line = log.lines.first
+        #   line.remote_ip
+        #   # => '72.21.206.5'
+        #
+        # If a certain field does not apply to a given request (for example, the <tt>key</tt> field does not apply to a bucket request),
+        # or if it was unknown or unavailable, it will return <tt>nil</tt>.
+        #
+        #   line.operation
+        #   # => 'REST.GET.BUCKET'
+        #   line.key
+        #   # => nil
+        class Line < String
+          DATE          = /\[([^\]]+)\]/
+          QUOTED_STRING = /"([^"]+)"/
+          REST          = /(\S+)/
+          LINE_SCANNER  = /#{DATE}|#{QUOTED_STRING}|#{REST}/
+          
+          cattr_accessor :decorators
+          @@decorators = Hash.new {|hash, key| hash[key] = lambda {|entry| CoercibleString.coerce(entry)}}
+          cattr_reader   :fields
+          @@fields     = []
+          
+          class << self
+            def field(name, offset, type = nil, &block) #:nodoc:
+              decorators[name] = block if block_given?
+              fields << name
+              class_eval(<<-EVAL, __FILE__, __LINE__)
+                def #{name}
+                  value = parts[#{offset} - 1]
+                  if value == '-'
+                    nil
+                  else
+                    self.class.decorators[:#{name}].call(value)
+                  end
+                end
+                memoized :#{name}
+              EVAL
+            end
+            
+            # Time.parse doesn't like %d/%B/%Y:%H:%M:%S %z so we have to transform it unfortunately
+            def typecast_time(datetime) #:nodoc:
+              month = datetime[/[a-z]+/i]
+              datetime.sub!(%r|^(\w{2})/(\w{3})|, '\2/\1')
+              datetime.sub!(month, Date::ABBR_MONTHS[month.downcase].to_s)
+              datetime.sub!(':', ' ')
+              Time.parse(datetime)
+            end
+          end
+          
+          def initialize(line) #:nodoc:
+            super(line)
+            @parts = parse
+          end
+          
+          field(:owner,            1)  {|entry| Owner.new('id' => entry) }
+          field :bucket,           2
+          field(:time,             3)  {|entry| typecast_time(entry)}
+          field :remote_ip,        4
+          field(:requestor,        5)  {|entry| Owner.new('id' => entry) }
+          field :request_id,       6
+          field :operation,        7
+          field :key,              8
+          field :request_uri,      9
+          field :http_status,      10
+          field :error_code,       11
+          field :bytes_sent,       12
+          field :object_size,      13
+          field :total_time,       14
+          field :turn_around_time, 15
+          field :referrer,         16
+          field :user_agent,       17
+          
+          # Returns all fields of the line in a hash of the form <tt>:field_name => :field_value</tt>.
+          #
+          #   line.attributes.values_at(:bucket, :key)
+          #   # => ['marcel', 'kiss.jpg']
+          def attributes
+            self.class.fields.inject({}) do |attribute_hash, field|
+              attribute_hash[field] = send(field)
+              attribute_hash
+            end
+          end
+          
+          private
+            attr_reader :parts
+            
+            def parse
+              scan(LINE_SCANNER).flatten.compact
+            end
+        end
+      end
+      
       module Management #:nodoc:
         def self.included(klass) #:nodoc:
           klass.extend(ClassMethods)
@@ -120,11 +254,20 @@ module AWS
           # Returns the collection of logs for the bucket named <tt>name</tt>.
           #
           #   Bucket.logs_for 'marcel'
-          def logs_for(name = nil)
-            name = bucket_name(name)
+          #
+          # Accepts the same options as Bucket.find, such as <tt>:max_keys</tt> and <tt>:marker</tt>.
+          def logs_for(name = nil, options = {})
+            if name.is_a?(Hash)
+              options = name
+              name    = nil
+            end
+            
+            name           = bucket_name(name)
             logging_status = logging_status_for(name)
             return [] unless logging_status.logging_enabled?
-            objects(logging_status.target_bucket, :prefix => logging_status.target_prefix)
+            objects(logging_status.target_bucket, options.merge(:prefix => logging_status.target_prefix)).map do |log_object|
+              Log.new(log_object)
+            end
           end
           alias_method :logs, :logs_for
         end
@@ -154,8 +297,8 @@ module AWS
           self.class.logging_enabled_for?(name)
         end
         
-        def logs
-          self.class.logs_for(name)
+        def logs(options = {})
+          self.class.logs_for(name, options)
         end
       end
     end

data/lib/aws/s3/object.rb

@@ -5,11 +5,7 @@ module AWS
     #
     # You can store an object on S3 by specifying a key, its data and the name of the bucket you want to put it in:
     #
-    #   S3Object.store(
-    #     'headshot.jpg', 
-    #     File.open('headshot.jpg'), 
-    #     'photos'
-    #   )
+    #   S3Object.store('me.jpg', open('headshot.jpg'), 'photos')
     #
     # The content type of the object will be inferred by its extension. If the appropriate content type can not be inferred, S3 defaults
     # to <tt>binary/octect-stream</tt>.
@@ -19,7 +15,7 @@ module AWS
     #   file = 'black-flowers.m4a'
     #   S3Object.store(
     #     file,
-    #     File.open(file),
+    #     open(file),
     #     'jukebox',
     #     :content_type => 'audio/mp4a-latm'
     #   )
@@ -41,7 +37,7 @@ module AWS
     #
     # Or stream it by passing a block to <tt>stream</tt>:
     # 
-    #   File.open('song.mp3', 'w') do |file|
+    #   open('song.mp3', 'w') do |file|
     #     S3Object.stream('song.mp3', 'jukebox') do |chunk|
     #       file.write chunk
     #     end
@@ -55,6 +51,9 @@ module AWS
     #
     # Other functionality includes:
     #
+    #   # Check if an object exists?
+    #   S3Object.exists? 'headshot.jpg', 'photos'
+    #
     #   # Copying an object
     #   S3Object.copy 'headshot.jpg', 'headshot2.jpg', 'photos'
     #
@@ -166,6 +165,10 @@ module AWS
           # the desired object's key, which functionally makes the key ordered one degree higher than the desired object key according to 
           # alphabetic ordering. This is a hack, but it should work around 99% of the time. I can't think of a scenario where it would return
           # something incorrect.
+          
+          # We need to ensure the key doesn't have extended characters but not uri escape it before doing the lookup and comparing since if the object exists, 
+          # the key on S3 will have been normalized
+          key    = key.remove_extended unless key.utf8?
           bucket = Bucket.find(bucket_name(bucket), :marker => key.previous, :max_keys => 1)
           # If our heuristic failed, trigger a NoSuchKey exception
           if (object = bucket.objects.first) && object.key == key
@@ -178,9 +181,10 @@ module AWS
         # Makes a copy of the object with <tt>key</tt> to <tt>copy_name</tt>.
         def copy(key, copy_key, bucket = nil, options = {})
           bucket          = bucket_name(bucket)
-          original        = find(key, bucket)
+          original        = open(url_for(key, bucket))
           default_options = {:content_type => original.content_type}
-          store(copy_key, original.value, bucket, default_options.merge(options)).success?
+          store(copy_key, original, bucket, default_options.merge(options))
+          acl(copy_key, bucket, acl(key, bucket))
         end
         
         # Rename the object with key <tt>from</tt> to have key in <tt>to</tt>.
@@ -189,17 +193,25 @@ module AWS
           delete(from, bucket)
         end
         
-        # Fetch information about the key with <tt>name</tt> from <tt>bucket</tt>. Information includes content type, content length,
+        # Fetch information about the object with <tt>key</tt> from <tt>bucket</tt>. Information includes content type, content length,
         # last modified time, and others.
         #
         # If the specified key does not exist, NoSuchKey is raised.
         def about(key, bucket = nil, options = {})
           response = head(path!(bucket, key, options), options)
-          if response.client_error?
-            raise NoSuchKey.new("No such key `#{key}'", bucket)
-          else
-            About.new(response.headers)
-          end
+          raise NoSuchKey.new("No such key `#{key}'", bucket) if response.code == 404
+          About.new(response.headers)
+        end
+        
+        # Checks if the object with <tt>key</tt> in <tt>bucket</tt> exists.
+        #
+        #   S3Object.exists? 'kiss.jpg', 'marcel'
+        #   # => true
+        def exists?(key, bucket = nil)
+          about(key, bucket)
+          true
+        rescue NoSuchKey
+          false
         end
       
         # Delete object with <tt>key</tt> from <tt>bucket</tt>.
@@ -214,14 +226,10 @@ module AWS
         # may be desirable for very large files so they are not read into memory all at once.
         # 
         #   # Non streamed upload
-        #   S3Object.store('simple-text-file.txt', 
-        #                  'hello world!', 
-        #                  'marcel')
+        #   S3Object.store('greeting.txt', 'hello world!', 'marcel')
         #
         #   # Streamed upload
-        #   S3Object.store('roots.mpeg', 
-        #                  File.open('roots.mpeg'), 
-        #                  'marcel')
+        #   S3Object.store('roots.mpeg', open('roots.mpeg'), 'marcel')
         def store(key, data, bucket = nil, options = {})
           validate_key!(key)
           # Must build path before infering content type in case bucket is being used for options
@@ -577,7 +585,7 @@ module AWS
         
       # Don't dump binary data :)
       def inspect #:nodoc:
-        "#<AWS::S3::S3Object:0x#{object_id} '#{path}'>"
+        "#<%s:0x%s '%s'>" % [self.class, object_id, path]
       end
       
       private