right_aws_api 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 164b81b2c4e7845571d6616ce59b22a3baa7788a
-  data.tar.gz: e7cbd22f3857d650a026b29699e22cc9718d1821
+  metadata.gz: 64f50bf6419f2dab98f45aaf99cdba9cb1239e4f
+  data.tar.gz: 02dcede70b66b401f4a2fb1227881623da8a2979
 SHA512:
-  metadata.gz: 4b970d6a9794c29953c16e63018f7a97522476f66a7357ea5be7e0bd6d3d110b880615d86531a45a70b7ebe41d96d3bf18697074d2a7fc5844b1fc16caf51089
-  data.tar.gz: d0846816165f048ebfa43d026d144c03c35158f4006c750a7d0c14117a1f50d4031a3d46422b34321935d182e074f121bc426e84be043b3ba76371ed03b59ffa
+  metadata.gz: 8624a2807021be070b88d46783d923df81961ddd802517e61eb74eb2358d28d995c03fe57df35a4ef711c76e39e67de8ed8c954ed66f14725315201ebc644c55
+  data.tar.gz: 5464bb232f2fa074ec5a26d8fd04df4c19fd3a7b71ff026ab14d273f791ba9a6ce0e333326d4c8fbf5b73cbd780252b85b140e7566314ccda47530c7454a3a3c

data/HISTORY

@@ -1,11 +1,13 @@
-== 2015-01-16
-  - v0.2.2
-    Added an ability to not use DNS like buckets in AWS S3
-    Some minor bugs were fixed in singature V4
+== current
 
-== 2014-10-30
-  - v0.2.1
-    Added support for AWS v4 signature and make it the default
+== 2015-01-16, v0.2.3
+  - Added an ability to not use DNS like buckets in AWS S3
+  - Added support for S3 folders creation
 
-== 2013-06-28
-  - pre-release candidate created
+== 2015-01-16, v0.2.2
+  - Some minor bugs were fixed in singature V4
+
+== 2014-10-30, v0.2.1
+  - Added support for AWS v4 signature and make it the default
+
+== 2013-06-28, pre-release

data/README.md

@@ -66,6 +66,8 @@ call and you know what params it accepts - just call the method with those param
 ```ruby
   require "right_aws_api"
 
+  ec2 = RightScale::CloudApi::AWS::EC2::Manager.new(key, secret, endpoint)
+
   # Get a list of your instances
   ec2.DescribeInstances
 
@@ -156,6 +158,110 @@ call and you know what params it accepts - just call the method with those param
   sqs.DeleteQueue('myCoolQueue')
 ```
 
+### Options
+
+There is a way to provide extra options when you instantiate a new manager:
+
+```ruby
+  options = { :key => value }
+  ec2 = RightScale::CloudApi::AWS::EC2::Manager.new(key, secret, endpoint, options)
+  sqs = RightScale::CloudApi::AWS::SQS::Manager.new(key, secret, account_number, endpoint, options)
+  s3  = RightScale::CloudApi::AWS::S3::Manager::new(key, secret, endpoint, options)
+  etc
+```
+
+The options allow you to tweak the managers behavoir. Here is a list of the options that make sense for AWS services:
+
+Name                     | Type    | Default                     | Description
+-------------------------| --------| ----------------------------|--------------
+:abort_on_timeout        | Boolean | false                       | When set to +true+ the gem does not perform a retry call when there is a connection timeout. This may help you to deal with request idempotence issue. Lets say you make a create call and you get back a timeout. It is possible that AWS created a new resource but just failed to report it properly. It is better to stop here rather than keep retrying creating more and more resources.
+:api_version             | String  | see service Manager         | The required cloud API version if it is different from the default one.
+:cache                   | Boolean | false                       | Cache cloud responses when possible so that we don't parse them again if cloud response does not change (see cloud specific ApiManager definition).
+:cloud                   | Hash    | {}                          | A set of cloud or service specific options. There is an only option for AWS S3 so far: :no_dns_buckets => false/true.
+:connection_open_timeout | Integer | up to NetHttpPersistent gem | Connection open timeout (in seconds).
+:connection_read_timeout | Integer | up to NetHttpPersistent gem | Connection read timeout (in seconds).
+:connection_retry_count  | Integer | 3                           | Max number of establish connection retry attempts before it gives up.
+:connection_retry_delay  | Float   | 0.5                         | Initial retry backoff delay in seconds. The value is doubled on every retry attempt.
+:connection_verify_mode  | Integer | OpenSSL::SSL::VERIFY_PEER   | Try OpenSSL::SSL::VERIFY_NONE is there is an SSL sertificate issue with th eremote end. This may happen when working with DNS-line S3 buckets.
+:headers                 | Hash    | {}                          | A set of request headers to be added to every API request.
+:logger                  | Logger  | -> STDOUT                   | Current logger. When nil is given it logs to '/dev/nul'.
+:log_filter_patterns     | Array   | see DEFAULT_LOG_FILTERS     | A set of log filters that define what to log (see {RightScale::CloudApi::CloudApiLogger}).
+:params                  | Hash    | {}                          | A set of URL params to be added to every API request.
+:raw_response            | Boolean | false                       | By default the gem parses all XML and JSON responses and returns them as ruby Hashes. Sometimes it is not what one would want (Amazon S3 GetObject for example). Setting this option to +true+ forces the gem to return the body of the response as it is.
+
+For more options see https://github.com/rightscale/right_cloud_api_base/blob/master/lib/base/api_manager.rb
+
+### Response
+
+It is easy to get the last HTTP request and response data:
+
+```ruby
+  s3.PutObject('Bucket' => 'a.b.c.d.1.com', 'Object' => '13', :body => 'hahaha') #=> ''
+
+  s3.request.verb    # => 'put'
+  s3.request.path    # => '/13'
+  s3.request.headers # =>
+    {
+      "content-type"         => ["binary/octet-stream"],
+      "content-length"       => [6],
+      "content-md5"          => ["EBpuyfk4iF3wpE8gRY0utA=="],
+      "x-amz-content-sha256" => ["23453452345234523452345"],
+      "x-amz-date"           => ["20150123T225348Z"],
+      "x-amz-expires"        => [3600],
+      "host"                 => ["a.b.c.d.1.com.s3.amazonaws.com"],
+      "authorization"        => ["AWS4-HMAC-SHA256 Credential=000/20150123/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-expires, Signature=111"]}
+  s3.request.body    #=> "hahaha"
+
+  s3.response.code    #=> '200'
+  s3.response.headers #=>
+    {
+      "x-amz-id-2"       => ["wB/XOQ+dfgdfgdfgwgfdg"],
+      "x-amz-request-id" => ["FEC7DEE7C51ACAB3"],
+      "date"             => ["Fri, 23 Jan 2015 22:53:49 GMT"],
+      "etag"             => ["\"101a6ec9f938885df0a44f20458d2eb4\""],
+      "content-length"   => ["0"],
+      "server"           => ["AmazonS3"]}
+  s3.response.body    #=> ''
+
+```
+
+Furthermore every response value may return its HTTP response code and HTTP headers through 'metadata' method:
+
+```ruby
+response1 = s3.GetObject('Bucket' => 'a.b.c.d.1.com', 'Object' => '13')
+response2 = s3.GetObject('Bucket' => 'a.b.c.d.1.com', 'Object' => '14')
+
+response1 #=> "hahaha"
+response1.metadata #=>
+  { :code    => '200',
+    :headers =>
+    {"x-amz-id-2"       => ["sdasfdasd="],
+     "x-amz-request-id" => ["B2AB70B2D081BF2B"],
+     "date"             => ["Fri, 23 Jan 2015 23:02:36 GMT"],
+     "last-modified"    => ["Fri, 23 Jan 2015 22:53:49 GMT"],
+     "etag"             => ["\"2345435234523452345324\""],
+     "accept-ranges"    => ["bytes"],
+     "content-type"     => ["binary/octet-stream"],
+     "content-length"   => ["6"],
+     "server"           => ["AmazonS3"]}}
+
+response2 #=> "hohohohoho"
+response2.metadata #=>
+{ :code    => '200',
+  :headers =>
+  {"x-amz-id-2"       => ["asdasdf="],
+   "x-amz-request-id" => ["50DBC360C934C548"],
+   "date"             => ["Fri, 23 Jan 2015 23:02:36 GMT"],
+   "last-modified"    => ["Fri, 23 Jan 2015 23:01:55 GMT"],
+   "etag"             => ["\"2345345243523452345234\""],
+   "accept-ranges"    => ["bytes"],
+   "content-type"     => ["binary/octet-stream"],
+   "content-length"   => ["10"],
+   "server"           => ["AmazonS3"]}}
+```
+
+
+
 ## Dependencies
 
 This gem depends on a base gem which is shared across all RightScale cloud libraries:

data/lib/cloud/aws/s3/manager.rb

@@ -38,13 +38,35 @@ module RightScale
 
         # Amazon Simple Storage Service (S3) compatible manager (thread safe).
         #
+        # There are 2 ways of using S3 API manager: _HTTP verb calls_ or _helper methods_
+        #
+        # ### HTTP verbs
+        #
+        # The manager supports following HTTP verbs: get, post, put, head and delete,
+        # and all of them share the same syntax:
+        #
+        # ```ruby
+        #  s3.post(path, :params => Hash, :headers => Hash, :body => 'foo-bar', :options => Hash)
+        # ```
+        #
+        # ### Helper methods
+        #
+        # Eventhough the _HTTP verbs_ are powerfull they are not too handy.
+        # If you like to call API actions by their name you may find our helper methods usefull.
+        #
+        # In most cases these methods use the same names and they take the same parameters
+        # that Amazon uses in their docs.
+        #
+        # You can find use case examples for both the verbs and the methods below.
+        # The helper method definitions can be found here: https://github.com/rightscale/right_aws_api/blob/master/lib/cloud/aws/s3/wrappers/default.rb
+        #
         # @example
         #  require "right_aws_api"
         #
         #  s3 = RightScale::CloudApi::AWS::S3::Manager.new(key, secret, 'https://s3.amazonaws.com')
         #
         # @example
-        #  # -- Using HTTP verb methods --
+        #  # -- Using HTTP verbs --
         #
         #  # List all buckets
         #  s3.get
@@ -75,11 +97,13 @@ module RightScale
         #
         #  # Put object
         #  # Do not forget to provide a proper 'content-type' header because the default
-        #  # one is set to 'application/octet-stream'
+        #  # one is set to 'binary/octet-stream'
         #  s3.put('devs-us-east/boot1.jpg',
         #          :body    => 'This is my object DATA. WooHoo!!!',
         #          :headers => {'content-type' => 'text/plain'})
         #
+        #  # Create a folder
+        #  s3.put('devs-us-east/logs/')
         #
         # @example
         #   # A simple example of a multi-thread file download:
@@ -111,7 +135,6 @@ module RightScale
         #     end
         #   end
         #
-        #
         # @example
         #  # -- Using helper methods --
         #
@@ -135,12 +158,15 @@ module RightScale
         #               :params => { 'response-content-type' => 'image/jpeg'})
         #
         #  # Put object
-        #  # P.S. 'content-type' is 'application/octet-stream' by default
+        #  # P.S. 'content-type' is 'binary/octet-stream' by default
         #  s3.PutObject('Bucket' => 'devs-us-east',
         #               'Object' => 'boot1.jpg',
         #               :body    => file_content,
         #               :headers => {'content-type' => 'image/jpeg'})
         #
+        #  # Create a folder
+        #  s3.PutObject('Bucket' => 'devs-us-east', 'Object' => 'logs/', :body => '')
+        #
         # @example
         #
         #  # List all buckets
@@ -152,9 +178,9 @@ module RightScale
         #            "CreationDate"=>"2011-05-25T20:46:28.000Z"},
         #           {"Name"=>"CR_right_test",
         #            "CreationDate"=>"2011-06-08T20:46:32.000Z"},
-        #           {"Name"=>"DarrylTest", 
+        #           {"Name"=>"DarrylTest",
         #            "CreationDate"=>"2011-06-03T03:43:08.000Z"},
-        #           {"Name"=>"RightScalePeter", 
+        #           {"Name"=>"RightScalePeter",
         #            "CreationDate"=>"2008-10-28T03:59:20.000Z"}]},
         #       "Owner"=>
         #        {"DisplayName"=>"fghsfg",

data/lib/cloud/aws/s3/routines/request_signer.rb

@@ -120,8 +120,12 @@ module RightScale
             # Amazon wants them to be escaped before we sign the request.
             # P.S. but do not escape "/" (signature v4 does not like this)
             #
-            object = $2.to_s.split('/').map{|i| Utils::AWS::amz_escape(i)}.join('/')
-            [ $1, object ]
+            new_bucket = $1
+            pre_object = $2.to_s
+            object     = pre_object.split('/').map{|i| Utils::AWS::amz_escape(i)}.join('/')
+            # Preserve '/' if it is the last symbol of the object because it is an operation on a folder
+            object    += '/' if object.length > 0 && pre_object[/\/$/]
+            [ new_bucket, object ]
           end
 
 
@@ -183,7 +187,7 @@ module RightScale
             #      So we need to include it into our signature to avoid the error below:
             #      'The request signature we calculated does not match the signature you provided.
             #       Check your key and signing method.'
-            headers.set_if_blank('content-type', 'application/octet-stream')
+            headers.set_if_blank('content-type', 'binary/octet-stream')
             headers
           end
 
@@ -210,10 +214,15 @@ module RightScale
           # @return [Boolean]
           #
           def is_dns_bucket?(bucket)
-            return false if @data[:options][:cloud][:no_dns_buckets]
+            return false if no_dns_buckets?
             Utils::AWS::is_dns_bucket?(bucket)
           end
 
+
+          def no_dns_buckets?
+            !!@data[:options][:cloud][:no_dns_buckets]
+          end
+
         end
       end
     end

data/lib/cloud/aws/s3/wrappers/default.rb

@@ -253,7 +253,7 @@ module RightScale
 
               base.query_api_pattern 'PutObject', :put, '{:Bucket}/{:Object}',
                 :body    => Utils::MUST_BE_SET,
-                :headers => { 'content-type' => 'application/octet-stream' }
+                :headers => { 'content-type' => 'binary/octet-stream' }
 
 
               base.query_api_pattern 'PutObjectAcl', :put, '{:Bucket}/{:Object}',
@@ -286,7 +286,7 @@ module RightScale
               base.query_api_pattern 'UploadPart', :post, '{:Bucket}/{:Object}',
                 :params  => { 'partNumber' => :PartNumber,
                               'uploadId'   => :UploadId },
-                :headers => { 'content-type' => 'application/octet-stream' }
+                :headers => { 'content-type' => 'binary/octet-stream' }
 
 
               base.query_api_pattern 'UploadPartCopy', :put, '{:DestinationBucket}/{:DestinationObject}',

data/lib/right_aws_api_version.rb

@@ -33,7 +33,7 @@ module RightScale
       # Gem version namespace
       module VERSION
         # Current version
-        STRING = '0.2.2'
+        STRING = '0.2.3'
       end
     end
   end

data/spec/cloud/aws/s3/routines/request_signer_spec.rb

@@ -37,6 +37,10 @@ describe RightScale::CloudApi::AWS::S3::RequestSigner do
 
 
   context '#compute_host' do
+    before :each do
+      allow(subject).to receive(:no_dns_buckets?).and_return(false)
+    end
+
     context 'DNS bucket' do
       before :each do
         bucket  = 'foo-bar-bucket'
@@ -88,9 +92,9 @@ describe RightScale::CloudApi::AWS::S3::RequestSigner do
     end
 
     context 'content-type' do
-      it 'defaults content-type to application/octet-stream' do
+      it 'defaults content-type to binary/octet-stream' do
         result      = subject.compute_headers!(@headers, @body, @host)
-        expectation = ['application/octet-stream']
+        expectation = ['binary/octet-stream']
         expect(result['content-type']).to eq(expectation)
       end
     end
@@ -101,6 +105,7 @@ describe RightScale::CloudApi::AWS::S3::RequestSigner do
     before :each do
       @path   = 'foo-bar'
       @object = 'foo%2Fbar%2F%D0%B1%D0%B0%D0%BD%D0%B0%D0%BD%D0%B0.jpg'
+      allow(subject).to receive(:no_dns_buckets?).and_return(false)
     end
 
     it 'works for DNS bucket' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: right_aws_api
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - RightScale, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-17 00:00:00.000000000 Z
+date: 2015-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: right_cloud_api_base