s33r 0.4.1 → 0.4.2

data/lib/s33r/core.rb

@@ -3,8 +3,8 @@ require 'time'
 require 'net/http'
 require 'net/https'
 require 'openssl'
-require 'xml/libxml'
 require 'parsedate'
+require File.join(File.dirname(__FILE__), 'libxml_loader')
 
 # Module to handle S3 operations which don't require an internet connection,
 # i.e. data validation and request-building operations;
@@ -157,12 +157,17 @@ module S33r
     headers
   end
 
-  # Add metadata headers, correctly prefixing them first.
+  # Add metadata headers, correctly prefixing them first,
+  # e.g. you might do metadata_headers({}, {'myname' => 'elliot', 'myage' => 36})
+  # to add two headers to the request:
+  # 
+  #   x-amz-meta-myname: elliot
+  #   x-amz-meta-myage: 36
   # 
   # Returns headers with the metadata headers appended.
   def metadata_headers(headers, metadata={})
     unless metadata.empty?
-      metadata.each { |key, value| headers[METADATA_PREFIX + key] = value }
+      metadata.each { |key, value| headers[METADATA_PREFIX + key] = value.to_s }
     end
     headers
   end
@@ -285,7 +290,7 @@ module S33r
   # Remove the namespace declaration from S3 XML response bodies (libxml
   # isn't fond of it).
   def S33r.remove_namespace(xml_in)
-    namespace = RESPONSE_NAMESPACE_URI.gsub('/', '\/')
+    namespace = S33r::RESPONSE_NAMESPACE_URI.gsub('/', '\/')
     xml_in.gsub(/ xmlns="#{namespace}"/, '')
   end
 end

data/lib/s33r/libxml_extensions.rb

@@ -1,3 +1,5 @@
+require File.join(File.dirname(__FILE__), 'libxml_loader')
+
 # Convenience methods for libxml classes.
 module XML
   # Find first element matching +xpath+ and return its content

data/lib/s33r/libxml_loader.rb

@@ -0,0 +1,6 @@
+begin
+  require 'xml/libxml'
+rescue LoadError
+  require 'rubygems'
+  require_gem 'libxml-ruby'
+end

data/lib/s33r/logging.rb

@@ -1,13 +1,13 @@
 require 'rubygems'
-require 'xml/libxml'
 require_gem 'builder'
+require File.join(File.dirname(__FILE__), 'libxml_loader')
 
 module S33r
   # For manipulating logging directives on resources
   # (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/LoggingHowTo.html).
   # 
-  # Calling LoggingResource.new (no arguments) will generate a blank instance
-  # which can be used to remove logging from a resource.
+  # Creating a LoggingResource instance using new and no arguments will generate a "blank" instance;
+  # this can be put to the ?logging URL for a resource to remove logging from it.
   class LoggingResource
     attr_reader :log_target, :log_prefix
   

data/lib/s33r/named_bucket.rb

@@ -2,6 +2,8 @@
 #-- specify a prefix and/or delimiter
 
 require File.join(File.dirname(__FILE__), 'client')
+require File.join(File.dirname(__FILE__), 's3_obj')
+require File.join(File.dirname(__FILE__), 's33r_exception')
 
 module S33r
   # Wraps the S33r::Client class to make it more convenient for use with a single bucket.
@@ -33,10 +35,7 @@ module S33r
         a :default_bucket option"
       end
 
-      # holds a BucketListing instance
-      @listing = nil
-
-      # all content should be created as public-read
+      # all content inside the bucket should be created as public-read
       @public_contents = (true == options[:public_contents])
       @client_headers.merge!(canned_acl_header('public-read')) if @public_contents
       
@@ -58,14 +57,31 @@ module S33r
       @strict
     end
     
-    # Get a single object from a bucket as a blob.
-    def [](key)
-      get_resource(@name, key).body
+    # Get a single object from a bucket as an S3Object.
+    # 
+    # To get a bare object (with no content):
+    #   
+    #   bucket['key']
+    #   
+    # To get the object and load its content:
+    # 
+    #   bucket['key', :load]
+    def [](key, eager=false)
+      obj = listing.contents[key]
+      obj.named_bucket = self
+      obj.load if :load == eager
+      obj
+    end
+    
+    # Get a raw response for a key inside the bucket.
+    def get_raw(key, headers={})
+      get_resource(@name, key, headers)
     end
 
     # Get a BucketListing instance for the content of this bucket.
+    # Uses the Client.list_bucket method to get the listing.
     def listing
-      list_bucket(@name)
+      list_bucket(@name)[1]
     end
     
     # Does this bucket exist?
@@ -84,9 +100,12 @@ module S33r
       listing.pretty
     end
     
-    # List content of the bucket, and attach each item to this bucket as it is yielded.
-    def each_item
-      listing.contents.each_value { |item| item.named_bucket = self; yield item }
+    # List content of the bucket, and attach each item to this NamedBucket
+    # instance as it is yielded (to enable easier manipulation directly from the S3Object).
+    # Note that the objects are incomplete, as the data associated with them has not been
+    # "got" from S3 yet.
+    def each_object
+      listing.contents.each_value { |obj| obj.named_bucket = self; yield obj }
     end
     
     # Does the given key exist in the bucket?
@@ -114,9 +133,8 @@ module S33r
     # NB S3 doesn't discriminate between successfully deleting a key
     # and trying to delete a non-existent key (both return a 204).
     # If you want to test for existence first, use key_exists?.
-    def delete_resource(resource_key, headers={})
-      super(@name, resource_key, headers)
-      listing
+    def delete(key, headers={})
+      delete_resource(@name, key, headers)
     end
     
     # Generate an authenticated URL (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/) 
@@ -127,4 +145,4 @@ module S33r
       super(@aws_access_key, @aws_secret_access_key, @name, resource_key, expires)
     end
   end
-end
+end

data/lib/s33r/s33r_exception.rb

@@ -41,6 +41,10 @@ module S33r
     # (i.e. if LogDelivery permissions not set - see S33r::S3ACL::ACLDoc.add_log_target_grants)
     class BucketNotLogTargetable < Exception
     end
+    
+    # Raised if you try to do a put with empty body
+    class TryingToPutEmptyResource < Exception
+    end
 
   end
 end

data/lib/s33r/s33r_http.rb

@@ -2,7 +2,7 @@ require 'net/http'
 require File.join(File.dirname(__FILE__), 's33r_exception')
 
 # Addtions to the Net::HTTP classes
-
+#
 # Adds some convenience functions for checking response status.
 class Net::HTTPResponse
   attr_accessor :success

data/lib/s33r/s3_acl.rb

@@ -1,7 +1,8 @@
 require 'rubygems'
-require 'xml/libxml'
 require_gem 'builder'
-require File.join(File.dirname(__FILE__), 's33r_exception')
+base = File.dirname(__FILE__)
+require File.join(base, 'libxml_loader')
+require File.join(base, 's33r_exception')
 
 module S33r
   # S3 ACL handling.

data/lib/s33r/s3_obj.rb

@@ -0,0 +1,186 @@
+require 'date'
+
+# Representation of an object stored in a bucket.
+module S33r
+  class S3Object
+    attr_accessor :key, :last_modified, :etag, :size, :owner, :storage_class, :value, :named_bucket, 
+    :content_type, :mime_type
+    
+    # Metadata set by x-amz-meta- style headers. Note that the bit after x-amz-meta- 
+    # is stored for each key, rather than the full key.
+    attr_accessor :meta
+
+    def initialize(key, metadata={}, amz_meta={}, value=nil)
+      @key = key
+      @meta = amz_meta
+      @value = value
+      set_properties(metadata) unless metadata.empty?
+    end
+    
+    # Set the properties of the object from some metadata name-value pairs.
+    # 
+    # +metadata+ is a hash of properties and their values, used to set the 
+    # corresponding properties on the object.
+    # 
+    # +value+ is the data associated with the object on S3.
+    def set_properties(metadata)
+      # required properties
+      @etag = metadata[:etag].gsub("\"", "") if metadata[:etag]
+      @last_modified = DateTime.parse(metadata[:last_modified]) if metadata[:last_modified]
+      @size = metadata[:size].to_i if metadata[:size]
+
+      # only set if creating object from XML (not available otherwise)
+      @owner = metadata[:owner]
+      
+      # only set if creating object from HTTP response
+      @content_type = metadata[:content_type]
+    end
+    
+    # To create an object which reads the content in from a file;
+    # this is not very efficient - it's actually better to use NamedBucket.put_file,
+    # as this will stream out of a file to S3, rather than load the file in
+    # memory first.
+    def self.from_file(key, filename)
+      mime_type = guess_mime_type(filename)
+      content_type = mime_type.simplified
+      value = File.open(filename).read
+      self.new(key, { :content_type => content_type }, {}, value)
+    end
+
+    # Set properties of the object from an XML string.
+    # 
+    # +xml_str+ should be a string representing a full XML document,
+    # containing a <Contents> element as its root element.
+    def self.from_xml_string(xml_str)
+      self.from_xml_node(XML.get_xml_doc(xml_str))
+    end
+    
+    # Create a new instance from an XML document.
+    def self.from_xml_node(doc)
+      metadata = self.parse_xml_node(doc)
+      self.new(metadata[:key], metadata)
+    end
+
+    # Get properties of the object from an XML document, e.g. as returned in a bucket listing.
+    # 
+    # +doc+: XML::Document instance to parse to get properties for this object.
+    # 
+    # Returns the metadata relating to the object, as stored on S3.
+    #-- TODO: include amz-meta elements
+    def self.parse_xml_node(doc)
+      metadata = {}
+      metadata[:key] = doc.xget('Key')
+      metadata[:last_modified] = doc.xget('LastModified')
+      metadata[:etag] = doc.xget('ETag')
+      metadata[:size] = doc.xget('Size')
+      
+      # Build representation of the owner.
+      user_xml_doc = doc.find('Owner').to_a.first
+      metadata[:owner] = S3ACL::CanonicalUser.from_xml(user_xml_doc)
+      
+      metadata
+    end
+    
+    # Create a new instance from a HTTP response.
+    # This is useful if you do a GET for a resource key and
+    # want to convert the response into an object; NB the response
+    # doesn't necessarily contain all the metadata you might want - you need to 
+    # do a HEAD for that.
+    # 
+    # +key+ is the key for the resource (not part of the response).
+    # 
+    # Note that if the resp returns nil, a blank object is created.
+    def self.from_response(key, resp)
+      result = self.parse_response(resp)
+      if result
+        metadata, amz_meta, value = result
+      else
+        metadata = {}
+        amz_meta = {}
+        value = nil
+      end
+      self.new(key, metadata, amz_meta, value)
+    end
+    
+    # Parse the response returned by GET on a resource key
+    # within a bucket.
+    # 
+    # +resp+ is a Net::HTTPResponse instance.
+    # 
+    # Returns an array [+metadata+, +response.body+]; or nil if the object 
+    # doesn't exist.
+    def self.parse_response(resp)
+      resp_headers = resp.to_hash
+      
+      # If there's no etag, there's no content in the resource.
+      if resp_headers['etag']
+        metadata = {}
+        metadata[:last_modified] = resp_headers['last-modified'][0]
+        metadata[:etag] = resp_headers['etag'][0]
+        metadata[:size] = resp_headers['content-length'][0]
+        metadata[:content_type] = resp_headers['content-type'][0]
+        
+        # x-amz-meta- response headers.
+        interesting_header = Regexp.new(METADATA_PREFIX)
+        amz_meta = {}
+        resp.each_header do |key, value|
+          amz_meta[key.gsub(interesting_header, '')] = value if interesting_header =~ key
+        end
+      
+        # The actual content of the S3 object.
+        value = resp.body
+      
+        [metadata, amz_meta, value]
+      else
+        nil
+      end
+    end
+    
+    # Load content into this object from S3; will perform 
+    # an HTTP request to "refresh" the object (providing the object 
+    # has an association with a bucket it can use for piggybacking).
+    # 
+    # Returns false if value cannot be retrieved.
+    def load
+      if @named_bucket and @named_bucket.key_exists?(@key)
+        resp = @named_bucket.get_raw(@key)
+        if resp.ok?
+          @value = resp.body
+          @content_type = resp.to_hash['content-type']
+          return true
+        else
+          return false
+        end
+      else
+        return false
+      end
+    end
+    
+    # Remove this object from its associated NamedBucket.
+    #
+    # Returns false if this object is not associated with a bucket;
+    # otherwise returns the response from S3.
+    def delete
+      if @named_bucket.nil?
+        return false
+      else
+        @named_bucket.delete_resource(@key)
+      end
+    end
+    
+    # Save this object back into its bucket.
+    # 
+    # Only works if the object has an associated NamedBucket;
+    # returns false if it doesn't.
+    def save
+      if @named_bucket.nil?
+        return false
+      else
+        headers = {}
+        headers["Content-Type"] = @content_type || ''
+        headers = metadata_headers(headers, meta)
+        @named_bucket.put_stream(@value, @key, headers)
+      end
+    end
+  end
+end

data/test/cases/spec_bucket_listing.rb

@@ -12,6 +12,8 @@ context 'S33r bucket listing' do
     @with_empty_bucket_listing_xml = File.open(xml_file3) { |f| f.read }
     xml_file4 = File.join(base, '../files/bucket_listing_broken.xml')
     @with_broken_bucket_listing_xml = File.open(xml_file4) { |f| f.read }
+    xml_file5 = File.join(base, '../files/suspect_bucket_listing.xml')
+    @with_suspect_bl_xml = File.open(xml_file5) { |f| f.read }
     
     @bucket_listing = BucketListing.new(@with_bucket_listing_xml)
     @bucket_properties = %w(name prefix marker max_keys is_truncated)
@@ -85,38 +87,12 @@ context 'S33r bucket listing' do
   specify 'should provide easy access to <CommonPrefixes> elements as a hash' do
     todo
   end
-end
-
-context 'S3 object' do
-  setup do
-    @s3_object_xml = File.open(File.join(base, '../files/s3_object.xml')).read
-    @s3obj = S3Object.new
-    @s3obj.set_from_xml_string(@s3_object_xml)
-  end
-
-  specify 'can be initialised from XML fragment with correct data types' do
-    @s3obj.key.should.equal '/home/ell/dir1/four.txt'
-    d = @s3obj.last_modified
-    [d.year, d.month, d.day, d.hour, d.min, d.sec].should.equal [2006, 8, 19, 22, 53, 29]
-    @s3obj.etag.should.equal '24ce59274b89287b3960c184153ac24b'
-    @s3obj.size.should.equal 14
-  end
-
-  specify 'should treat the owner as an object in his/her own right' do
-    [@s3obj.owner.user_id, @s3obj.owner.display_name].should.equal \
-    ['56efddfead5aa65da942f156fb2b294f44d78fd932d701331edc5fba19620fd4', 'elliotsmith3']
-    @s3obj.owner.should_be_instance_of S3ACL::CanonicalUser
-  end
-
-  specify 'can be associated with a NamedBucket' do
-    todo
-  end
-
-  specify 'can be saved by proxing through the NamedBucket it is associated with' do
-    todo
-  end
-
-  specify 'cannot be saved unless associated with a NamedBucket' do
-    todo
+  
+  # attempting to fix ListBucketResult errors reported by Alex Payne
+  specify 'should handle suspect bucket listing' do
+    puts @with_suspect_bl_xml
+    bl = BucketListing.new(@with_suspect_bl_xml)
+    bl.contents.size.should_be 1
+    bl.contents.keys.should.include 'orly.jpg'
   end
 end

data/test/cases/spec_s3_object.rb

@@ -0,0 +1,35 @@
+base = File.dirname(__FILE__)
+require base + '/../test_setup'
+
+context 'S3 object' do
+  setup do
+    @s3_object_xml = File.open(File.join(base, '../files/s3_object.xml')).read
+    @s3obj = S3Object.from_xml_string(@s3_object_xml)
+  end
+
+  specify 'can be initialised from XML fragment with correct data types' do
+    @s3obj.key.should.equal '/home/ell/dir1/four.txt'
+    d = @s3obj.last_modified
+    [d.year, d.month, d.day, d.hour, d.min, d.sec].should.equal [2006, 8, 19, 22, 53, 29]
+    @s3obj.etag.should.equal '24ce59274b89287b3960c184153ac24b'
+    @s3obj.size.should.equal 14
+  end
+
+  specify 'should treat the owner as an object in his/her own right' do
+    [@s3obj.owner.user_id, @s3obj.owner.display_name].should.equal \
+    ['56efddfead5aa65da942f156fb2b294f44d78fd932d701331edc5fba19620fd4', 'elliotsmith3']
+    @s3obj.owner.should_be_instance_of S3ACL::CanonicalUser
+  end
+
+  specify 'can be associated with a NamedBucket' do
+    todo
+  end
+
+  specify 'can be saved by proxing through the NamedBucket it is associated with' do
+    todo
+  end
+
+  specify 'cannot be saved unless associated with a NamedBucket' do
+    todo
+  end
+end