s3lib 0.1.0

data/test/full_test.rb

@@ -0,0 +1,84 @@
+require 'test/unit'
+require File.join(File.dirname(__FILE__), '../lib/s3_authenticator')
+
+class S3AuthenticatorTest < Test::Unit::TestCase
+  
+  def setup
+    # The id and secret key are non-working credentials from the S3 Developer's Guide 
+    # See http://developer.amazonwebservices.com/connect/entry.jspa?externalID=123&categoryID=48
+    ENV['AMAZON_ACCESS_KEY_ID'] = '0PN6J17HBGXHT7JJ3X82'
+    ENV['AMAZON_SECRET_ACCESS_KEY'] = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
+    @s3_test = S3Lib::AuthenticatedRequest.new
+  end
+
+  # See http://developer.amazonwebservices.com/connect/entry.jspa?externalID=123&categoryID=48
+  def test_dg_sample_one
+    @s3_test.make_authenticated_request(:get, '/photos/puppy.jpg', {'Host' => 'johnsmith.s3.amazonaws.com',
+                                                                    'Date' => 'Tue, 27 Mar 2007 19:36:42 +0000'})
+    expected_canonical_string = "GET\n\n\nTue, 27 Mar 2007 19:36:42 +0000\n/johnsmith/photos/puppy.jpg"
+    assert_equal expected_canonical_string, @s3_test.canonical_string
+    assert_equal "AWS 0PN6J17HBGXHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=", @s3_test.authorization_string    
+  end
+
+  # See http://developer.amazonwebservices.com/connect/entry.jspa?externalID=123&categoryID=48
+  def test_dg_sample_two
+    @s3_test.make_authenticated_request(:put, '/photos/puppy.jpg', {'Content-Type' => 'image/jpeg',
+                                                                    'Content-Length' => '94328',
+                                                                    'Host' => 'johnsmith.s3.amazonaws.com',
+                                                                    'Date' => 'Tue, 27 Mar 2007 21:15:45 +0000'})
+    expected_canonical_string = "PUT\n\nimage/jpeg\nTue, 27 Mar 2007 21:15:45 +0000\n/johnsmith/photos/puppy.jpg"
+    assert_equal expected_canonical_string, @s3_test.canonical_string  
+    assert_equal "AWS 0PN6J17HBGXHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=", @s3_test.authorization_string  
+  end
+
+  def test_dg_sample_three
+    @s3_test.make_authenticated_request(:get, '', {'prefix' => 'photos',
+                                                   'max-keys' => '50',
+                                                   'marker' => 'puppy',
+                                                   'host' => 'johnsmith.s3.amazonaws.com',
+                                                   'date' => 'Tue, 27 Mar 2007 19:42:41 +0000'})
+    assert_equal "GET\n\n\nTue, 27 Mar 2007 19:42:41 +0000\n/johnsmith/", @s3_test.canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=', @s3_test.authorization_string
+  end 
+
+  def test_dg_sample_four
+    @s3_test.make_authenticated_request(:get, '?acl', {'host' => 'johnsmith.s3.amazonaws.com', 
+                                                       'date' => 'Tue, 27 Mar 2007 19:44:46 +0000'})
+  
+    assert_equal "GET\n\n\nTue, 27 Mar 2007 19:44:46 +0000\n/johnsmith/?acl", @s3_test.canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=', @s3_test.authorization_string
+  
+  end
+
+  def test_dg_sample_five
+    @s3_test.make_authenticated_request(:delete, '/johnsmith/photos/puppy.jpg', 
+                                                  {'User-Agent' => 'dotnet',
+                                                   'host' => 's3.amazonaws.com',
+                                                   'date' => 'Tue, 27 Mar 2007 21:20:27 +0000',
+                                                   'x-amz-date' => 'Tue, 27 Mar 2007 21:20:26 +0000' })                                                   
+    assert_equal "DELETE\n\n\n\nx-amz-date:Tue, 27 Mar 2007 21:20:26 +0000\n/johnsmith/photos/puppy.jpg", @s3_test.canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=', @s3_test.authorization_string
+  end   
+
+  def test_dg_sample_six
+    @s3_test.make_authenticated_request(:put,    '/db-backup.dat.gz', 
+                                                  {'User-Agent' => 'curl/7.15.5',
+                                                   'host' => 'static.johnsmith.net:8080',
+                                                   'date' => 'Tue, 27 Mar 2007 21:06:08 +0000',
+                                                   'x-amz-acl' => 'public-read',
+                                                   'content-type' => 'application/x-download',
+                                                   'Content-MD5' => '4gJE4saaMU4BqNR0kLY+lw==',
+                                                   'X-Amz-Meta-ReviewedBy' => ['joe@johnsmith.net', 'jane@johnsmith.net'],
+                                                   'X-Amz-Meta-FileChecksum' => '0x02661779',
+                                                   'X-Amz-Meta-ChecksumAlgorithm' => 'crc32',
+                                                   'Content-Disposition' => 'attachment; filename=database.dat',
+                                                   'Content-Encoding' => 'gzip',
+                                                   'Content-Length' => '5913339' })
+    expected_canonical_string =  "PUT\n4gJE4saaMU4BqNR0kLY+lw==\napplication/x-download\nTue, 27 Mar 2007 21:06:08 +0000\n" + 
+                                 "x-amz-acl:public-read\nx-amz-meta-checksumalgorithm:crc32\nx-amz-meta-filechecksum:0x02661779\n" +
+                                 "x-amz-meta-reviewedby:joe@johnsmith.net,jane@johnsmith.net\n/static.johnsmith.net/db-backup.dat.gz"
+    assert_equal expected_canonical_string, @s3_test.canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=', @s3_test.authorization_string
+  end
+  
+end

data/test/s3_authenticator_test.rb

@@ -0,0 +1,291 @@
+require 'test/unit'
+require File.join(File.dirname(__FILE__), '../lib/s3_authenticator')
+
+# Make private methods and attributes public so that you can test them
+module S3Lib
+  class AuthenticatedRequest
+    
+    @@test_mode = false
+  
+    attr_reader :bucket, :headers  
+  
+    def initialize()
+    end
+  
+    def public_canonicalized_resource
+      canonicalized_resource
+    end
+  
+    def public_canonicalized_headers
+      canonicalized_headers
+    end  
+  
+    def public_canonicalized_positional_headers
+      canonicalized_positional_headers
+    end
+  
+    def public_canonicalized_amazon_headers
+      canonicalized_amazon_headers
+    end
+  
+    def public_canonical_string
+      canonical_string
+    end
+  
+    def public_authorization_string
+      authorization_string
+    end
+    
+    def self.test_mode
+      @@test_mode = true
+    end
+    
+    def make_authenticated_request(verb, request_path, headers = {})
+      @verb = verb
+      @request_path = request_path.gsub(/^\//,'') # Strip off the leading '/'
+    
+      @amazon_id = ENV['AMAZON_ACCESS_KEY_ID']
+      @amazon_secret = ENV['AMAZON_SECRET_ACCESS_KEY']    
+    
+      @headers = headers.downcase_keys.join_values    
+      get_bucket_list_params          
+      get_bucket_name
+      fix_date
+      
+      if @@test_mode
+        test_open(uri_with_bucket_list_params, @headers.merge(:method => @verb, 'Authorization' => authorization_string))
+      else
+        req = open(uri_with_bucket_list_params, @headers.merge(:method => @verb, 'Authorization' => authorization_string))
+      end
+    end
+
+    # Over-ride RestOpenURI#open
+    def test_open(uri, headers)
+      {:uri => uri, :headers => headers}
+    end
+  
+  end  
+end
+
+
+class S3AuthenticatorTest < Test::Unit::TestCase
+  
+  def setup
+    # The id and secret key are non-working credentials from the S3 Developer's Guide 
+    # See http://developer.amazonwebservices.com/connect/entry.jspa?externalID=123&categoryID=48
+    ENV['AMAZON_ACCESS_KEY_ID'] = '0PN6J17HBGXHT7JJ3X82'
+    ENV['AMAZON_SECRET_ACCESS_KEY'] = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
+    S3Lib::AuthenticatedRequest.test_mode    
+    @s3_test = S3Lib::AuthenticatedRequest.new
+    
+  end
+  
+  def test_http_verb_is_uppercase
+    @s3_test.make_authenticated_request(:get, '/', {'host' => 's3.amazonaws.com'})
+    assert_match /^GET\n/, @s3_test.canonical_string
+  end
+  
+  def test_bucket_name_is_correct_with_virtual_hosting
+    @s3_test.make_authenticated_request(:get, '/', {'host' => 'some_bucket.s3.amazonaws.com'})
+    assert_equal 'some_bucket/', @s3_test.bucket
+  end
+  
+  def test_bucket_name_is_correct_with_cname_virtual_hosting
+    @s3_test.make_authenticated_request(:get, '/', {'host' => 'some_bucket.example.com'})
+    assert_equal 'some_bucket.example.com/', @s3_test.bucket    
+  end
+  
+  def test_bucket_name_is_lowercase_with_virtual_hosting
+    @s3_test.make_authenticated_request(:get, '/', {'host' => 'Some_Bucket.s3.amazonaws.com'})
+    assert_equal 'some_bucket/', @s3_test.bucket    
+  end
+  
+  def test_bucket_name_is_lowercase_with_cname_virtual_hosting
+    @s3_test.make_authenticated_request(:get, '/', {'host' => 'Some_Bucket.example.com'})
+    assert_equal 'some_bucket.example.com/', @s3_test.bucket    
+  end  
+  
+  def test_forward_slash_is_always_added
+    @s3_test.make_authenticated_request(:get, '')
+    assert_match /^\//, @s3_test.public_canonicalized_resource
+  end
+  
+  def test_positional_headers_with_all_headers
+    @s3_test.make_authenticated_request(:get, '', {'content-type' => 'some crazy content type', 
+                                                   'date' => 'December 25th, 2007', 
+                                                   'content-md5' => 'whee'})
+    assert_equal "whee\nsome crazy content type\nDecember 25th, 2007\n", @s3_test.public_canonicalized_positional_headers
+  end
+  
+  def test_positional_headers_with_only_date_header
+    @s3_test.make_authenticated_request(:get, '', {'date' => 'December 25th, 2007'})
+    assert_equal "\n\nDecember 25th, 2007\n", @s3_test.public_canonicalized_positional_headers
+  end
+  
+  def test_positional_headers_with_no_headers_should_have_date_defined
+    @s3_test.make_authenticated_request(:get, '' )
+    date = @s3_test.headers['date']
+    assert_equal "\n\n#{date}\n", @s3_test.public_canonicalized_positional_headers      
+  end
+  
+  def test_amazon_headers_should_remove_non_amazon_headers
+    @s3_test.make_authenticated_request(:get, '', {'content-type' => 'content', 
+                                                   'some-other-header' => 'other',
+                                                   'x-amz-meta-one' => 'one',
+                                                   'x-amz-meta-two' => 'two'})
+    headers = @s3_test.public_canonicalized_amazon_headers
+    assert_no_match /other/, headers
+    assert_no_match /content/, headers
+  end
+  
+  def test_amazon_headers_should_keep_amazon_headers
+    @s3_test.make_authenticated_request(:get, '', {'content-type' => 'content', 
+                                                   'some-other-header' => 'other',
+                                                   'x-amz-meta-one' => 'one',
+                                                   'x-amz-meta-two' => 'two'})
+    headers = @s3_test.public_canonicalized_amazon_headers    
+    assert_match /x-amz-meta-one/, headers
+    assert_match /x-amz-meta-two/, headers
+  end
+  
+  def test_amazon_headers_should_be_lowercase
+    @s3_test.make_authenticated_request(:get, '', {'content-type' => 'content', 
+                                                   'some-other-header' => 'other',
+                                                   'X-amz-meta-one' => 'one',
+                                                   'x-Amz-meta-two' => 'two'})
+    headers = @s3_test.public_canonicalized_amazon_headers    
+    assert_match /x-amz-meta-one/, headers
+    assert_match /x-amz-meta-two/, headers
+  end
+  
+  def test_amazon_headers_should_be_alphabetized
+    @s3_test.make_authenticated_request(:get, '', {'content-type' => 'content', 
+                                                   'some-other-header' => 'other',
+                                                   'X-amz-meta-one' => 'one',
+                                                   'x-Amz-meta-two' => 'two',
+                                                   'x-amz-meta-zed' => 'zed',
+                                                   'x-amz-meta-alpha' => 'alpha'})
+    headers = @s3_test.public_canonicalized_amazon_headers    
+    assert_match /alpha.*one.*two.*zed/m, headers # /m on the reg-exp makes .* include newlines
+  end
+  
+  def test_xamzdate_should_override_date_header
+    @s3_test.make_authenticated_request(:get, '', {'date' => 'December 15, 2005', 'x-amz-date' => 'Tue, 27 Mar 2007 21:20:26 +0000'})
+    headers = @s3_test.public_canonicalized_headers    
+    assert_match /2007/, headers
+    assert_no_match /2005/, headers
+  end
+
+  def test_xamzdate_should_override_capitalized_date_header
+    @s3_test.make_authenticated_request(:get, '', {'date' => 'December 15, 2005', 'X-amz-date' => 'Tue, 27 Mar 2007 21:20:26 +0000'})
+    headers = @s3_test.public_canonicalized_headers    
+    assert_match /2007/, headers
+    assert_no_match /2005/, headers
+  end
+  
+  def test_leading_spaces_get_stripped_from_header_values
+    @s3_test.make_authenticated_request(:get, '', {'x-amz-meta-one' => ' one with a leading space',
+                                                   'x-Amz-meta-two' => ' two with a leading and trailing space '})
+    headers = @s3_test.public_canonicalized_amazon_headers 
+    assert_match /x-amz-meta-one:one with a leading space/, headers
+    assert_match /x-amz-meta-two:two with a leading and trailing space /, headers 
+  end
+  
+  def test_values_as_arrays_should_be_joined_as_commas
+     @s3_test.make_authenticated_request(:get, '', {'x-amz-mult' => ['a', 'b', 'c']})
+     
+     headers = @s3_test.public_canonicalized_amazon_headers
+     assert_match /a,b,c/, headers
+  end
+  
+  def test_date_should_be_added_if_not_passed_in
+    @s3_test.make_authenticated_request(:get, '')
+    assert @s3_test.headers.has_key?('date')
+  end
+  
+  # See http://developer.amazonwebservices.com/connect/entry.jspa?externalID=123&categoryID=48
+  def test_dg_sample_one
+    @s3_test.make_authenticated_request(:get, '/photos/puppy.jpg', {'Host' => 'johnsmith.s3.amazonaws.com',
+                                                                    'Date' => 'Tue, 27 Mar 2007 19:36:42 +0000'})
+    assert_equal 'johnsmith/', @s3_test.bucket
+    expected_canonical_string = "GET\n\n\nTue, 27 Mar 2007 19:36:42 +0000\n/johnsmith/photos/puppy.jpg"
+    assert_equal expected_canonical_string, @s3_test.public_canonical_string
+    assert_equal "AWS 0PN6J17HBGXHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=", @s3_test.public_authorization_string    
+  end
+  
+  # See http://developer.amazonwebservices.com/connect/entry.jspa?externalID=123&categoryID=48
+  def test_dg_sample_two
+    @s3_test.make_authenticated_request(:put, '/photos/puppy.jpg', {'Content-Type' => 'image/jpeg',
+                                                                    'Content-Length' => '94328',
+                                                                    'Host' => 'johnsmith.s3.amazonaws.com',
+                                                                    'Date' => 'Tue, 27 Mar 2007 21:15:45 +0000'})
+    assert_equal 'johnsmith/', @s3_test.bucket
+    expected_canonical_string = "PUT\n\nimage/jpeg\nTue, 27 Mar 2007 21:15:45 +0000\n/johnsmith/photos/puppy.jpg"
+    assert_equal expected_canonical_string, @s3_test.public_canonical_string  
+    assert_equal "AWS 0PN6J17HBGXHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=", @s3_test.public_authorization_string  
+  end
+  
+  def test_dg_sample_three
+    @s3_test.make_authenticated_request(:get, '', {'prefix' => 'photos',
+                                                   'max-keys' => '50',
+                                                   'marker' => 'puppy',
+                                                   'host' => 'johnsmith.s3.amazonaws.com',
+                                                   'date' => 'Tue, 27 Mar 2007 19:42:41 +0000'})
+    assert_equal 'johnsmith/', @s3_test.bucket                                                   
+    assert_equal "GET\n\n\nTue, 27 Mar 2007 19:42:41 +0000\n/johnsmith/", @s3_test.public_canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=', @s3_test.public_authorization_string
+  end 
+  
+  def test_dg_sample_four
+    @s3_test.make_authenticated_request(:get, '?acl', {'host' => 'johnsmith.s3.amazonaws.com', 
+                                                       'date' => 'Tue, 27 Mar 2007 19:44:46 +0000'})
+    
+    assert_equal "GET\n\n\nTue, 27 Mar 2007 19:44:46 +0000\n/johnsmith/?acl", @s3_test.public_canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=', @s3_test.public_authorization_string
+    
+  end
+  
+  def test_dg_sample_five
+    @s3_test.make_authenticated_request(:delete, '/johnsmith/photos/puppy.jpg', 
+                                                  {'User-Agent' => 'dotnet',
+                                                   'host' => 's3.amazonaws.com',
+                                                   'date' => 'Tue, 27 Mar 2007 21:20:27 +0000',
+                                                   'x-amz-date' => 'Tue, 27 Mar 2007 21:20:26 +0000' })                                                   
+    assert_equal "DELETE\n\n\n\nx-amz-date:Tue, 27 Mar 2007 21:20:26 +0000\n/johnsmith/photos/puppy.jpg", @s3_test.public_canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=', @s3_test.public_authorization_string
+  end   
+  
+  def test_dg_sample_six
+    @s3_test.make_authenticated_request(:put,    '/db-backup.dat.gz', 
+                                                  {'User-Agent' => 'curl/7.15.5',
+                                                   'host' => 'static.johnsmith.net:8080',
+                                                   'date' => 'Tue, 27 Mar 2007 21:06:08 +0000',
+                                                   'x-amz-acl' => 'public-read',
+                                                   'content-type' => 'application/x-download',
+                                                   'Content-MD5' => '4gJE4saaMU4BqNR0kLY+lw==',
+                                                   'X-Amz-Meta-ReviewedBy' => ['joe@johnsmith.net', 'jane@johnsmith.net'],
+                                                   'X-Amz-Meta-FileChecksum' => '0x02661779',
+                                                   'X-Amz-Meta-ChecksumAlgorithm' => 'crc32',
+                                                   'Content-Disposition' => 'attachment; filename=database.dat',
+                                                   'Content-Encoding' => 'gzip',
+                                                   'Content-Length' => '5913339' })
+    assert_equal 'static.johnsmith.net/', @s3_test.bucket                                              
+    expected_canonical_string =  "PUT\n4gJE4saaMU4BqNR0kLY+lw==\napplication/x-download\nTue, 27 Mar 2007 21:06:08 +0000\n" + 
+                                 "x-amz-acl:public-read\nx-amz-meta-checksumalgorithm:crc32\nx-amz-meta-filechecksum:0x02661779\n" +
+                                 "x-amz-meta-reviewedby:joe@johnsmith.net,jane@johnsmith.net\n/static.johnsmith.net/db-backup.dat.gz"
+    assert_equal expected_canonical_string, @s3_test.public_canonical_string
+    assert_equal 'AWS 0PN6J17HBGXHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=', @s3_test.public_authorization_string
+  end  
+  
+  def test_url_has_http_on_it
+    request = @s3_test.make_authenticated_request(:get, '')
+    assert_match /^http:\/\//, request[:uri]
+  end
+  
+  def test_url_is_sane
+    request = @s3_test.make_authenticated_request(:get, 'photos')
+    assert_equal 'http://s3.amazonaws.com/photos', request[:uri]
+  end
+
+  
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification 
+name: s3lib
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Scott Patten
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-11-11 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: This library forms the basis for building a library to talk to Amazon S3 using Ruby.  It is used as an example of how to build an Amazon S3 interface in The S3 Cookbook (http://thes3cookbook.com)
+email: scott@scottpatten.ca
+executables: 
+- s3lib
+- s3sh_as
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- VERSION.yml
+- bin/s3lib
+- bin/s3sh_as
+- github-test.rb
+- lib/acl.rb
+- lib/acl_access.rb
+- lib/acl_creating_a_grant_recipe.rb
+- lib/acl_reading_acl_recipe.rb
+- lib/acl_refreshing_cached_grants_recipe.rb
+- lib/bucket.rb
+- lib/bucket_before_refactoring.rb
+- lib/bucket_create.rb
+- lib/bucket_find.rb
+- lib/bucket_with_acl_mixin.rb
+- lib/error_handling.rb
+- lib/grant.rb
+- lib/grant_creating_a_grant_recipe.rb
+- lib/grant_reading_acl_recipe.rb
+- lib/object.rb
+- lib/object_from_bucket_test.rb
+- lib/object_take1.rb
+- lib/object_with_acl_mixin.rb
+- lib/put_with_curl_test.rb
+- lib/s3_authenticator.rb
+- lib/s3_authenticator_dev.rb
+- lib/s3_authenticator_dev_private.rb
+- lib/s3_errors.rb
+- lib/s3lib.rb
+- lib/s3lib_with_mixin.rb
+- lib/service.rb
+- lib/service_dev.rb
+- s3lib.gemspec
+- sample_usage.rb
+- test/acl_test.rb
+- test/amazon_headers_test.rb
+- test/canonical_resource_test.rb
+- test/canonical_string_tests.rb
+- test/first_test.rb
+- test/first_test_private.rb
+- test/full_test.rb
+- test/s3_authenticator_test.rb
+has_rdoc: true
+homepage: http://thes3cookbook.com
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: An Amazon S3 interface library used as an example in The S3 Cookbook (http://thes3cookbook.com)
+test_files: []
+