s3lib 0.1.0

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rubygems'
+require 'rake/testtask'
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+  # t.verbose = true
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "s3lib"
+    gemspec.summary = "An Amazon S3 interface library used as an example in The S3 Cookbook (http://thes3cookbook.com)"
+    gemspec.email = 'scott@scottpatten.ca'
+    gemspec.homepage = 'http://thes3cookbook.com'
+    gemspec.description = "This library forms the basis for building a library to talk to Amazon S3 using Ruby.  It is used as an example of how to build an Amazon S3 interface in The S3 Cookbook (http://thes3cookbook.com)"
+    gemspec.authors = ["Scott Patten"]
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:major: 0
+:build: 
+:minor: 1
+:patch: 0

data/bin/s3lib

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'yaml'
+
+# Set the S3 keys to the right account if the account name was given in the command line
+unless ARGV.empty?
+  s3_keys = YAML::load_file(File.join(ENV['HOME'], '.s3_keys.yml'))
+  if s3_keys.has_key?(ARGV[0])
+    keys = s3_keys[ARGV[0]]
+    ENV['AMAZON_ACCESS_KEY_ID'] = keys['amazon_access_key_id']
+    ENV['AMAZON_SECRET_ACCESS_KEY'] = keys['amazon_secret_access_key']
+  end
+end
+
+exec("irb -r s3lib")

data/bin/s3sh_as

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+
+# Set the S3 keys to the right account if the account name was given in the command line
+unless ARGV.empty?
+  s3_keys = YAML::load_file(File.join(ENV['HOME'], '.s3_keys.yml'))
+  if s3_keys.has_key?(ARGV[0])
+    keys = s3_keys[ARGV[0]]
+    ENV['AMAZON_ACCESS_KEY_ID'] = keys['amazon_access_key_id']
+    ENV['AMAZON_SECRET_ACCESS_KEY'] = keys['amazon_secret_access_key']
+  end
+end
+
+exec("s3sh")

data/github-test.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+require 'yaml'
+ 
+if ARGV.size < 1
+  puts "Usage: github-test.rb my-project.gemspec"
+  exit
+end
+ 
+require 'rubygems/specification'
+data = File.read(ARGV[0])
+spec = nil
+ 
+if data !~ %r{!ruby/object:Gem::Specification}
+  Thread.new { spec = eval("$SAFE = 3\n#{data}") }.join
+else
+  spec = YAML.load(data)
+end
+ 
+spec.validate
+ 
+puts spec
+puts "OK"

data/lib/acl.rb

@@ -0,0 +1,134 @@
+module S3Lib
+  
+  class Acl
+    
+    attr_reader :xml, :parent, :url
+  
+    def initialize(parent_or_url)
+      if parent_or_url.respond_to?(:url)
+        @parent = parent_or_url
+        @url = @parent.url.sub(/\/\Z/,'') + '?acl'
+      else
+        @url = parent_or_url.sub(/\/\Z/,'').sub(/\?acl/, '') + '?acl'
+      end
+    end
+        
+    def grants(params = {})
+      refresh_grants if params[:refresh]
+      @grants || get_grants
+    end
+    
+    def clear_grants
+      @grants = []
+      set_grants
+    end
+    
+    # permission must be one of :read, :write, :read_acl, :write_acl or :full_control
+    # The grantee Hash should look like this:
+    # {:type => :canonical|:email|:all_s3|:public, 
+    #  :grantee => canonical_user_id | email_address}
+    #
+    # The :grantee element of the hash is only required (and meaningful) 
+    # for :canonical and :email Grants   
+    #
+    # Some examples:
+    # Add public read access to an object:
+    # add_grant(:read, :type => :public)
+    #
+    # Give write access to a user with email of 'test@example.com'
+    # add_grant(:write, :type => :email, :grantee => 'test@example.com')
+    #
+    # Give full control to user with canonical ID of 1234567890
+    # add_grant(:full_control, :type => :canonical, :grantee => 1234567890)
+    #
+    # Note that you still have to PUT your changes to the server.
+    # Do this using set_grants or use the bang version of add_grant
+    #
+    # add_grant(:read, :type => :public)
+    # set_grants()
+    #
+    # add_grant!(:read, :type => :public) 
+    def add_grant(permission, grantee)
+      grants.push(S3Lib::Grant.new(permission, grantee))      
+    end
+        
+    # Add a grant and PUT it to the server right away
+    def add_grant!(permission, grantee)
+      add_grant(permission, grantee)
+      set_grants
+    end    
+    
+    def remove_grant(grant_num)
+      grants.delete_at(grant_num)
+      refresh_grants
+    end
+    
+    def refresh_grants
+      get_grants
+    end
+    
+    def set_grants
+      Acl.acl_request(:put, @url, :body => to_xml)
+      refresh_grants
+    end
+    
+    def owner
+      get_grants unless @xml
+      @xml.elements['Owner'].elements['ID'].text
+    end
+    
+    def to_xml
+      builder = Builder::XmlMarkup.new(:indent => 2)
+      xml = builder.AccessControlPolicy('xmlns' => 'http://s3.amazonaws.com/doc/2006-03-01/') do
+        builder.Owner do
+          builder.ID(owner)
+        end
+        builder.AccessControlList do
+          grants.each do |grant|
+            builder << grant.to_xml
+          end
+        end
+      end
+    end
+    
+    def inspect
+      grants.collect do |grant|
+        grant.inspect
+      end.join("\n")
+    end
+    
+    private 
+        
+    def get_grants
+      response = Acl.acl_request(:get, @url)
+      @xml = REXML::Document.new(response).root
+      @grants = REXML::XPath.match(@xml, '//Grant').collect do |grant|
+        grantee = grant.elements['Grantee']
+        permission = grant.elements['Permission'].text
+        S3Lib::Grant.new(permission, grantee)
+      end
+    end    
+    
+    def self.acl_request(verb, url, options = {})
+      begin
+        if verb == :put
+          options = {'content-type' => 'text/xml'}.merge(options) # Make sure content-type is set for :put
+        end
+        response = S3Lib.request(verb, url, options)
+      rescue S3Lib::S3ResponseError => error
+        puts "Error of type #{error.amazon_error_type}"
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'NotSignedUp': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by somebody else", error.io, error.s3requester)
+        when 'AccessDenied': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by someone else.", error.io, error.s3requester)
+        when 'MalformedACLError': raise S3Lib::MalformedACLError.new("Your ACL was malformed.", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      response
+    end    
+  
+  end
+  
+end

data/lib/acl_access.rb

@@ -0,0 +1,20 @@
+module S3Lib
+  module AclAccess
+    
+    def refresh_acl
+      get_acl
+    end    
+    
+    def acl(params = {})
+      refresh_acl if params[:refresh]      
+      @acl || get_acl
+    end    
+        
+    private
+    
+    def get_acl
+      @acl = Acl.new(self)
+    end
+    
+  end
+end

data/lib/acl_creating_a_grant_recipe.rb

@@ -0,0 +1,95 @@
+module S3Lib
+  
+  class Acl
+    
+    attr_reader :xml, :url
+  
+    def initialize(url)
+      @url = url.sub(/\/\Z/,'').sub(/\?acl/, '') + '?acl'
+    end
+        
+    def grants(params = {})
+      refresh_grants if params[:refresh]
+      @grants || get_grants
+    end
+    
+    # permission must be one of :read, :write, :read_acl, :write_acl or :full_control
+    # The grantee Hash should look like this:
+    # {:type => :canonical|:email|:all_s3|:public, 
+    #  :grantee => canonical_user_id | email_address}
+    #
+    # The :grantee element of the hash is only required (and meaningful) 
+    # for :canonical and :email Grants
+    def add_grant(permission, grantee)
+      grants.push(S3Lib::Grant.new(permission, grantee))      
+    end
+    
+    # Add a grant and PUT it to the server right away
+    def add_grant!(permission, grantee)
+      add_grant(permission, grantee)
+      set_grants
+    end
+    
+    def refresh_grants
+      get_grants
+    end
+    
+    def set_grants
+      Acl.acl_request(:put, @url, :body => to_xml)
+      refresh_grants
+    end
+    
+    def owner
+      get_grants unless @xml
+      @xml.elements['Owner'].elements['ID'].text
+    end
+    
+    def to_xml
+      builder = Builder::XmlMarkup.new(:indent => 2)
+      xml = builder.AccessControlPolicy('xmlns' => 'http://s3.amazonaws.com/doc/2006-03-01/') do
+        builder.Owner do
+          builder.ID(owner)
+        end
+        builder.AccessControlList do
+          grants.each do |grant|
+            builder << grant.to_xml
+          end
+        end
+      end
+    end
+    
+    private 
+        
+    def get_grants
+      response = Acl.acl_request(:get, @url)
+      @xml = REXML::Document.new(response).root
+      @grants = REXML::XPath.match(@xml, '//Grant').collect do |grant|
+        grantee = grant.elements['Grantee']
+        permission = grant.elements['Permission'].text
+        S3Lib::Grant.new(permission, grantee)
+      end
+    end    
+    
+    def self.acl_request(verb, url, options = {})
+      begin
+        if verb == :put
+          options = {'content-type' => 'text/xml'}.merge(options) # Make sure content-type is set for :put
+        end
+        response = S3Lib.request(verb, url, options)
+      rescue S3Lib::S3ResponseError => error
+        puts "Error of type #{error.amazon_error_type}"
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'NotSignedUp': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by somebody else", error.io, error.s3requester)
+        when 'AccessDenied': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by someone else.", error.io, error.s3requester)
+        when 'MalformedACLError': raise S3Lib::MalformedACLError.new("Your ACL was malformed.", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      response
+    end    
+  
+  end
+  
+end

data/lib/acl_reading_acl_recipe.rb

@@ -0,0 +1,59 @@
+module S3Lib
+  require 'rexml/document'
+  require File.join(File.dirname(__FILE__), 'grant_reading_acl_recipe.rb')  
+  require File.join(File.dirname(__FILE__), 's3_authenticator.rb')    
+  
+  class Acl
+    
+    attr_reader :xml, :url
+  
+    def initialize(url)
+      @url = url.sub(/\/\Z/,'').sub(/\?acl/, '') + '?acl'
+    end
+        
+    def grants(params = {})
+      refresh_grants if params[:refresh]
+      @grants || get_grants
+    end
+    
+    def clear_grants
+      @grants = []
+      set_grants
+    end
+    
+    def refresh_grants
+      get_grants
+    end
+    
+    private 
+    
+    def get_grants
+      response = Acl.acl_request(:get, @url)
+      @xml = REXML::Document.new(response).root
+      @grants = REXML::XPath.match(@xml, '//Grant').collect do |grant|
+        grantee = grant.elements['Grantee']
+        permission = grant.elements['Permission'].text
+        S3Lib::Grant.new(permission, grantee)
+      end
+    end    
+    
+    def self.acl_request(verb, url, options = {})
+      begin
+        response = S3Lib.request(verb, url, options)
+      rescue S3Lib::S3ResponseError => error
+        puts "Error of type #{error.amazon_error_type}"
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'NotSignedUp': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by somebody else", error.io, error.s3requester)
+        when 'AccessDenied': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by someone else.", error.io, error.s3requester)
+        when 'MalformedACLError': raise S3Lib::MalformedACLError.new("Your ACL was malformed.", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      response
+    end    
+  
+  end
+  
+end

data/lib/acl_refreshing_cached_grants_recipe.rb

@@ -0,0 +1,54 @@
+module S3Lib
+  require 'rexml/document'
+  require File.join(File.dirname(__FILE__), 'grant_reading_acl_recipe.rb')  
+  require File.join(File.dirname(__FILE__), 's3_authenticator.rb')    
+  
+  class Acl
+    
+    attr_reader :xml, :url
+  
+    def initialize(url)
+      @url = url.sub(/\/\Z/,'').sub(/\?acl/, '') + '?acl'
+    end
+        
+    def grants(params = {})
+      refresh_grants if params[:refresh]
+      @grants || get_grants
+    end
+    
+    def refresh_grants
+      get_grants
+    end
+    
+    private 
+    
+    def get_grants
+      response = Acl.acl_request(:get, @url)
+      @xml = REXML::Document.new(response).root
+      @grants = REXML::XPath.match(@xml, '//Grant').collect do |grant|
+        grantee = grant.elements['Grantee']
+        permission = grant.elements['Permission'].text
+        S3Lib::Grant.new(permission, grantee)
+      end
+    end    
+    
+    def self.acl_request(verb, url, options = {})
+      begin
+        response = S3Lib.request(verb, url, options)
+      rescue S3Lib::S3ResponseError => error
+        puts "Error of type #{error.amazon_error_type}"
+        case error.amazon_error_type
+        when 'NoSuchBucket': raise S3Lib::BucketNotFoundError.new("The bucket '#{bucket}' does not exist.", error.io, error.s3requester)
+        when 'NotSignedUp': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by somebody else", error.io, error.s3requester)
+        when 'AccessDenied': raise S3Lib::NotYourBucketError.new("The bucket '#{bucket}' is owned by someone else.", error.io, error.s3requester)
+        when 'MalformedACLError': raise S3Lib::MalformedACLError.new("Your ACL was malformed.", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      response
+    end    
+  
+  end
+  
+end

data/lib/bucket.rb

@@ -0,0 +1,116 @@
+# bucket.rb
+module S3Lib
+    
+  class Bucket
+    
+    attr_reader :name, :xml, :prefix, :marker, :max_keys
+    
+    def self.create(name, params = {})
+      params['x-amz-acl'] = params.delete(:access) if params[:access] # translate from :access to 'x-amz-acl'
+      response = self.bucket_request(:put, name, params)
+      response.status[0] == "200" ? true : false
+    end
+    
+    # passing :force => true will cause the bucket to be deleted even if it is not empty.
+    def self.delete(name, params = {})
+      if params.delete(:force)
+        self.delete_all(name, params)
+      end
+      response = self.bucket_request(:delete, name, params)
+    end
+    
+    def delete(params = {})      
+      self.class.delete(@name, @params.merge(params))
+    end
+    
+    def self.delete_all(name, params = {})
+      bucket = Bucket.find(name, params)
+      bucket.delete_all
+    end
+    
+    def delete_all
+      objects.each do |object|
+        object.delete
+      end
+    end
+
+    def self.find(name, params = {})
+      response = self.bucket_request(:get, name, params)
+      doc = REXML::Document.new(response).root
+      Bucket.new(doc, params)
+    end
+    
+    def initialize(doc, params = {})
+      @xml = doc
+      @params = params
+      # The XML has to have a name, but Service::buckets doesn't return
+      # MaxKeys, Prefix or Marker elements
+      @name = @xml.elements['Name'].text 
+      @max_keys = @xml.elements['MaxKeys'].text.to_i if @xml.elements['MaxKeys']
+      @prefix = @xml.elements['Prefix'].text if @xml.elements['Prefix']
+      @marker = @xml.elements['Marker'].text if @xml.elements['Marker']
+    end
+    
+    def is_truncated?
+      @xml.elements['IsTruncated'].text == 'true'
+    end
+    
+    def objects(params = {})
+      refresh if params[:refresh]
+      @objects || get_objects
+    end
+        
+    def refresh
+      refreshed_bucket = Bucket.find(@name, @params)
+      @xml = refreshed_bucket.xml
+      @objects = nil
+    end
+    
+    def refresh_acl
+      get_acl
+    end
+    
+    def url
+      @name
+    end
+    
+    # access an object in the bucket by key name
+    def [](key)
+      objects.detect{|object| object.key == key}
+    end
+    
+    def acl(params = {})
+      refresh_acl if params[:refresh]      
+      @acl || get_acl
+    end
+    
+    private
+        
+    def self.bucket_request(verb, name, params = {})
+      begin
+        response = S3Lib.request(verb, name, params)  
+      rescue S3Lib::S3ResponseError => error
+        case error.amazon_error_type
+        when "NoSuchBucket": raise S3Lib::BucketNotFoundError.new("The bucket '#{name}' does not exist.", error.io, error.s3requester)
+        when "NotSignedUp": raise S3Lib::NotYourBucketError.new("The bucket '#{name}' is owned by someone else.", error.io, error.s3requester)
+        when "BucketNotEmpty": raise S3Lib::BucketNotEmptyError.new("The bucket '#{name}' is not empty, so you can't delete it.\nTry using Bucket.delete_all('#{name}') first, or Bucket.delete('#{name}', :force => true).", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+    end    
+    
+    def get_objects
+      @objects = REXML::XPath.match(@xml, '//Contents').collect do |object|
+        key = object.elements['Key'].text
+        S3Lib::S3Object.new(self, key, :lazy_load => true)
+      end
+    end
+    
+    def get_acl
+      @acl = Acl.new(self)
+    end
+    
+  end
+  
+end

data/lib/bucket_before_refactoring.rb

@@ -0,0 +1,120 @@
+# bucket.rb
+require File.join(File.dirname(__FILE__), 's3_authenticator')
+require 'rexml/document'
+
+module S3Lib
+  
+  class NotYourBucketError < S3Lib::S3ResponseError
+  end
+  
+  class BucketNotFoundError < S3Lib::S3ResponseError
+  end
+  
+  class BucketNotEmptyError < S3Lib::S3ResponseError
+  end  
+  
+  class Bucket
+    
+    attr_reader :name, :xml, :prefix, :marker, :max_keys
+    
+    def self.create(name, params = {})
+      params['x-amz-acl'] = params.delete(:access) if params[:access] # translate from :access to 'x-amz-acl'
+      begin
+        response = S3Lib.request(:put, name, params)
+      rescue OpenURI::HTTPError => error
+        if error.amazon_error_type == "BucketAlreadyExists"
+          S3Lib::NotYourBucketError.new("The bucket '#{name}' is already owned by somebody else", error.io, error.s3requester)
+        else
+          raise # re-raise the exception if it's not a BucketAlreadyExists error
+        end
+      end    
+      response.status[0] == "200" ? true : false
+    end
+    
+    # passing :force => true will cause the bucket to be deleted even if it is not empty.
+    def self.delete(name, params = {})
+      if params.delete(:force)
+        self.delete_all(name, params)
+      end
+      begin
+        response = S3Lib.request(:delete, name, params)  
+      rescue S3Lib::S3ResponseError => error
+        case error.amazon_error_type
+        when "NoSuchBucket": raise S3Lib::BucketNotFoundError.new("The bucket '#{name}' does not exist.", error.io, error.s3requester)
+        when "NotSignedUp": raise S3Lib::NotYourBucketError.new("The bucket '#{name}' is not owned by you.", error.io, error.s3requester)
+        when "BucketNotEmpty": raise S3Lib::BucketNotEmptyError.new("The bucket '#{name}' is not empty, so you can't delete it.\nTry using Bucket.delete_all('#{name}') first, or Bucket.delete('#{name}', :force => true).", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end            
+    end
+    
+    def delete(params = {})
+      self.class.delete(@name, @params.merge(params))
+    end
+    
+    def self.delete_all(name, params = {})
+      bucket = Bucket.find(name, params)
+      bucket.delete_all
+    end
+    
+    def delete_all
+      objects.each do |object|
+        object.delete
+      end
+    end
+    
+    # Errors for find
+    # Trying to find a bucket that doesn't exist will raise a NoSuchBucket error
+    # Trying to find a bucket that you don't have access to will raise a NotSignedUp error
+    def self.find(name, params = {})
+     begin
+        response = S3Lib.request(:get, name)
+      rescue S3Lib::S3ResponseError => error
+        case error.amazon_error_type
+        when "NoSuchBucket": raise S3Lib::BucketNotFoundError.new("The bucket '#{name}' does not exist.", error.io, error.s3requester)
+        when "NotSignedUp": raise S3Lib::NotYourBucketError.new("The bucket '#{name}' is not owned by you", error.io, error.s3requester)
+        else # Re-raise the error if it's not one of the above
+          raise
+        end
+      end
+      doc = REXML::Document.new(response)
+      Bucket.new(doc, params)
+    end
+    
+    def initialize(doc, params = {})
+      @xml = doc.root
+      @params = params
+      @name = @xml.elements['Name'].text
+      @max_keys = @xml.elements['MaxKeys'].text.to_i
+      @prefix = @xml.elements['Prefix'].text
+      @marker = @xml.elements['Marker'].text
+    end
+    
+    def is_truncated?
+      @xml.elements['IsTruncated'].text == 'true'
+    end
+    
+    def objects(params = {})
+      refresh if params[:refresh]
+      @objects || get_objects
+    end
+        
+    def refresh
+      refreshed_bucket = Bucket.find(@name, @params)
+      @xml = refreshed_bucket.xml
+      @objects = nil
+    end
+    
+    private
+    
+    def get_objects
+      @objects = REXML::XPath.match(@xml, '//Contents').collect do |object|
+        key = object.elements['Key'].text
+        S3Lib::S3Object.new(self, key, :lazy_load => true)
+      end
+    end
+    
+  end
+  
+end