right_aws 2.1.0 → 3.0.0

data/lib/s3/right_s3_interface.rb

@@ -38,6 +38,17 @@ module RightAws
     ONE_YEAR_IN_SECONDS    = 365 * 24 * 60 * 60
     AMAZON_HEADER_PREFIX   = 'x-amz-'
     AMAZON_METADATA_PREFIX = 'x-amz-meta-'
+    S3_REQUEST_PARAMETERS = [ 'acl',
+        'location',
+        'logging', # this one is beta, no support for now
+        'response-content-type',
+        'response-content-language',
+        'response-expires',
+        'response-cache-control',
+        'response-content-disposition',
+        'response-content-encoding',
+        'torrent' ].sort
+
 
     @@bench = AwsBenchmarkingBlock.new
     def self.bench_xml
@@ -107,13 +118,20 @@ module RightAws
       s3_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value|
         out_string << (key[/^#{AMAZON_HEADER_PREFIX}/o] ? "#{key}:#{value}\n" : "#{value}\n")
       end
-        # ignore everything after the question mark...
+      # ignore everything after the question mark by default...
       out_string << path.gsub(/\?.*$/, '')
-       # ...unless there is an acl or torrent parameter
-      out_string << '?acl'      if path[/[&?]acl($|&|=)/]
-      out_string << '?torrent'  if path[/[&?]torrent($|&|=)/]
-      out_string << '?location' if path[/[&?]location($|&|=)/]
-      out_string << '?logging'  if path[/[&?]logging($|&|=)/]  # this one is beta, no support for now
+      # ... unless there is a parameter that we care about.
+      S3_REQUEST_PARAMETERS.each do |parameter|
+        if path[/[&?]#{parameter}(=[^&]*)?($|&)/]
+          if $1
+            value = CGI::unescape($1)
+          else
+            value = ''
+          end
+          out_string << (out_string[/[?]/] ? "&#{parameter}#{value}" : "?#{parameter}#{value}")
+        end
+      end
+
       out_string
     end
 
@@ -135,6 +153,7 @@ module RightAws
       # extract bucket name and check it's dns compartibility
       headers[:url].to_s[%r{^([a-z0-9._-]*)(/[^?]*)?(\?.+)?}i]
       bucket_name, key_path, params_list = $1, $2, $3
+      key_path = key_path.gsub( '%2F', '/' ) if key_path
       # select request model
       if !param(:no_subdomains) && is_dns_bucket?(bucket_name)
         # fix a path
@@ -836,6 +855,7 @@ module RightAws
     def generate_link(method, headers={}, expires=nil) #:nodoc:
         # calculate request data
       server, path, path_to_sign = fetch_request_params(headers)
+      path_to_sign = CGI.unescape(path_to_sign)
         # expiration time
       expires ||= DEFAULT_EXPIRES_AFTER
       expires   = Time.now.utc + expires if expires.is_a?(Fixnum) && (expires < ONE_YEAR_IN_SECONDS)
@@ -844,7 +864,7 @@ module RightAws
       headers.each{ |key, value| headers.delete(key) if (value.nil? || key.is_a?(Symbol)) }
         #generate auth strings
       auth_string = canonical_string(method, path_to_sign, headers, expires)
-      signature   = CGI::escape(Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new("sha1"), @aws_secret_access_key, auth_string)).strip)
+      signature   = CGI::escape(AwsUtils::sign( @aws_secret_access_key, auth_string))
         # path building
       addon = "Signature=#{signature}&Expires=#{expires}&AWSAccessKeyId=#{@aws_access_key_id}"
       path += path[/\?/] ? "&#{addon}" : "?#{addon}"
@@ -916,8 +936,20 @@ module RightAws
       #  s3.get_link('my_awesome_bucket',key) #=> https://s3.amazonaws.com:443/my_awesome_bucket/asia%2Fcustomers?Signature=QAO...
       #
       # see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/VirtualHosting.html
-    def get_link(bucket, key, expires=nil, headers={})
-      generate_link('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"), expires)
+      #
+      # To specify +response+-* parameters, define them in the response_params hash:
+      #
+      #  s3.get_link('my_awesome_bucket',key,nil,{},{ "response-content-disposition" => "attachment; filename=caf�.png", "response-content-type" => "image/png"})
+      #
+      #    #=> https://s3.amazonaws.com:443/my_awesome_bucket/asia%2Fcustomers?response-content-disposition=attachment%3B%20filename%3Dcaf%25C3%25A9.png&response-content-type=image%2Fpng&Signature=wio...
+      #
+    def get_link(bucket, key, expires=nil, headers={}, response_params={})
+      if response_params.size > 0
+        response_params = '?' + response_params.map { |k, v| "#{k}=#{CGI::escape(v).gsub(/[+]/, '%20')}" }.join('&')
+      else
+        response_params = ''
+      end
+      generate_link('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}#{response_params}"), expires)
     rescue
       on_exception
     end
@@ -1171,7 +1203,7 @@ module RightAws
       def headers_to_string(headers)
         result = {}
         headers.each do |key, value|
-          value       = value.to_s if value.is_a?(Array) && value.size<2
+          value       = value.first if value.is_a?(Array) && value.size<2
           result[key] = value
         end
         result

data/lib/sdb/right_sdb_interface.rb

@@ -33,7 +33,7 @@ module RightAws
     DEFAULT_PORT      = 443
     DEFAULT_PROTOCOL  = 'https'
     DEFAULT_PATH      = '/'
-    API_VERSION       = '2007-11-07'
+    API_VERSION       = '2009-04-15'
     DEFAULT_NIL_REPRESENTATION = 'nil'
 
     @@bench = AwsBenchmarkingBlock.new
@@ -399,12 +399,21 @@ module RightAws
     #                                                     :box_usage  => "0.0000093222",
     #                                                     :request_id => "81273d21-001-1111-b3f9-512d91d29ac8" }
     #
+    #  # request all attributes using a consistent read
+    #  # see:  http://docs.amazonwebservices.com/AmazonSimpleDB/latest/DeveloperGuide/index.html?ConsistencySummary.html
+    #  sdb.get_attributes('family', 'toys', nil, true) # => { :attributes => {"cat"    => ["clew", "Jons_socks", "mouse"] },
+    #                                                              "Silvia" => ["beetle", "rolling_pin", "kids"],
+    #                                                              "Jon"    => ["vacuum_cleaner", "hammer", "spade"]},
+    #                                              :box_usage  => "0.0000093222",
+    #                                              :request_id => "81273d21-000-1111-b3f9-512d91d29ac8" }
+    # 
     # see: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/SDB_API_GetAttributes.html
     #
-    def get_attributes(domain_name, item_name, attribute_name=nil)
-      link = generate_request("GetAttributes", 'DomainName'    => domain_name,
-                                               'ItemName'      => item_name,
-                                               'AttributeName' => attribute_name )
+    def get_attributes(domain_name, item_name, attribute_name=nil, consistent_read=nil)
+      link = generate_request("GetAttributes", 'DomainName'     => domain_name,
+                                               'ItemName'       => item_name,
+                                               'AttributeName'  => attribute_name,
+                                               'ConsistentRead' => consistent_read )
       res = request_info(link, QSdbGetAttributesParser.new)
       res[:attributes].each_value do |values|
         values.collect! { |e| sdb_to_ruby(e) }

data/lib/sns/right_sns_interface.rb

@@ -0,0 +1,286 @@
+#
+# Copyright (c) 2007-2008 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+module RightAws
+  class SnsInterface < RightAwsBase
+    include RightAwsBaseInterface
+
+    DEFAULT_HOST        = 'sns.us-east-1.amazonaws.com'
+    DEFAULT_PORT        = 443
+    DEFAULT_PROTOCOL    = 'https'
+    DEFAULT_SERVICE     = '/'
+    REQUEST_TTL         = 30
+
+    # Apparently boilerplate stuff
+    @@bench = AwsBenchmarkingBlock.new
+    def self.bench_xml
+      @@bench.xml
+    end
+    def self.bench_service
+      @@bench.service
+    end
+
+    def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={})
+      if params[:region]
+        server = "sns.#{params[:region]}.amazonaws.com"
+        params.delete(:region)
+      else
+        server = DEFAULT_HOST
+      end
+      init({  :name               => 'SNS',
+              :default_host       => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).host    : server,
+              :default_port       => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).port    : DEFAULT_PORT,
+              :default_service    => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).path    : DEFAULT_SERVICE,
+              :default_protocol   => ENV['SNS_URL'] ? URI.parse(ENV['SNS_URL']).scheme  : DEFAULT_PROTOCOL},
+           aws_access_key_id     || ENV['AWS_ACCESS_KEY_ID'],
+           aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'],
+           params)
+    end
+
+    # TODO: RJG - Seems like generate_request and generate_rest_request could be in a sub class?
+    # Generates a request hash for the sns API
+    def generate_request(action, params={})  # :nodoc:
+        # Sometimes we need to use queue uri (delete queue etc)
+        # In that case we will use Symbol key: 'param[:queue_url]'
+      service = params[:sns_url] ? URI(params[:sns_url]).path : '/'
+        # remove unset(=optional) and symbolyc keys
+      params.each{ |key, value| params.delete(key) if (value.nil? || key.is_a?(Symbol)) }
+        # prepare output hash
+      service_hash = { "Action"           => action,
+                       "Expires"          => (Time.now + REQUEST_TTL).utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+                       "AWSAccessKeyId"   => @aws_access_key_id }
+                       #"Version"          => API_VERSION }
+      service_hash.update(params)
+      service_params = signed_service_params(@aws_secret_access_key, service_hash, :get, @params[:server], service)
+      request        = Net::HTTP::Get.new("#{AwsUtils::URLencode(service)}?#{service_params}")
+        # prepare output hash
+      { :request  => request,
+        :server   => @params[:server],
+        :port     => @params[:port],
+        :protocol => @params[:protocol] }
+    end
+
+    # Generates a request hash for the REST API
+    def generate_rest_request(method, param) # :nodoc:
+      sns_uri = param[:sns_url] ? URI(param[:sns_url]).path : '/'
+      message   = param[:message]                # extract message body if nesessary
+        # remove unset(=optional) and symbolyc keys
+      param.each{ |key, value| param.delete(key) if (value.nil? || key.is_a?(Symbol)) }
+        # created request
+      param_to_str = param.to_a.collect{|key,val| key.to_s + "=" + CGI::escape(val.to_s) }.join("&")
+      param_to_str = "?#{param_to_str}" unless param_to_str.right_blank?
+      request = "Net::HTTP::#{method.capitalize}".right_constantize.new("#{sns_uri}#{param_to_str}")
+      request.body = message if message
+        # set main headers
+      request['content-md5']  = ''
+      request['Content-Type'] = 'text/plain'
+      request['Date']         = Time.now.httpdate
+        # generate authorization string
+      auth_string = "#{method.upcase}\n#{request['content-md5']}\n#{request['Content-Type']}\n#{request['Date']}\n#{CGI::unescape(sns_uri)}"
+      signature   = AwsUtils::sign(@aws_secret_access_key, auth_string)
+        # set other headers
+      request['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}"
+      #request['AWS-Version']   = API_VERSION
+        # prepare output hash
+      { :request  => request,
+        :server   => @params[:server],
+        :port     => @params[:port],
+        :protocol => @params[:protocol] }
+    end
+
+    # Sends request to Amazon and parses the response
+      # Raises AwsError if any banana happened
+    def request_info(request, parser) # :nodoc:
+      request_info_impl(:sns_connection, @@bench, request, parser)
+    end
+
+    def create_topic(topic_name)
+      req_hash = generate_request('CreateTopic', 'Name' => topic_name)
+      request_info(req_hash, SnsCreateTopicParser.new)
+    end
+
+    def list_topics()
+      req_hash = generate_request('ListTopics')
+      request_info(req_hash, SnsListTopicsParser.new)
+    end
+
+    def delete_topic(topic_arn)
+      req_hash = generate_request('DeleteTopic', 'TopicArn' => topic_arn)
+      request_info(req_hash, RightHttp2xxParser.new)
+    end
+
+    def subscribe(topic_arn, protocol, endpoint)
+      req_hash = generate_request('Subscribe', 'TopicArn' => topic_arn, 'Protocol' => protocol, 'Endpoint' => endpoint)
+      request_info(req_hash, SnsSubscribeParser.new)
+    end
+
+    def unsubscribe(subscription_arn)
+      req_hash = generate_request('Unsubscribe', 'SubscriptionArn' => subscription_arn)
+      request_info(req_hash, RightHttp2xxParser.new)
+    end
+
+    def publish(topic_arn, message, subject)
+      req_hash = generate_request('Publish', 'TopicArn' => topic_arn, 'Message' => message, 'Subject' => subject)
+      request_info(req_hash, SnsPublishParser.new)
+    end
+
+    def set_topic_attribute(topic_arn, attribute_name, attribute_value)
+      if attribute_name != 'Policy' && attribute_name != 'DisplayName'
+        raise(ArgumentError, "The only values accepted for the attribute_name parameter are (Policy, DisplayName)")
+      end
+      req_hash = generate_request('SetTopicAttributes', 'TopicArn' => topic_arn, 'AttributeName' => attribute_name, 'AttributeValue' => attribute_value)
+      request_info(req_hash, RightHttp2xxParser.new)
+    end
+
+    def get_topic_attributes(topic_arn)
+      req_hash = generate_request('GetTopicAttributes', 'TopicArn' => topic_arn)
+      request_info(req_hash, SnsGetTopicAttributesParser.new)
+    end
+
+    # Calls either the ListSubscriptions or ListSubscriptionsByTopic depending on whether or not the topic_arn parameter is provided.
+    def list_subscriptions(topic_arn = nil)
+      req_hash = topic_arn ? generate_request('ListSubscriptionsByTopic', 'TopicArn' => topic_arn) : generate_request('ListSubscriptions')
+      request_info(req_hash, SnsListSubscriptionsParser.new)
+    end
+
+    def confirm_subscription(topic_arn, token, authenticate_on_unsubscribe=false)
+      req_hash = generate_request('ConfirmSubscription', 'AuthenticateOnUnsubscribe' => authenticate_on_unsubscribe.to_s, 'Token' => token, 'TopicArn' => topic_arn)
+      request_info(req_hash, SnsConfirmSubscriptionParser.new)
+    end
+
+    def add_permission(topic_arn, label, acct_action_hash_ary)
+      n_hash = {
+        'TopicArn'  => topic_arn,
+        'Label'     => label
+      }
+
+      acct_action_hash_ary.each_with_index do |hash_val, idx|
+        n_hash["AWSAccountId.member.#{idx+1}"]  = hash_val[:aws_account_id]
+        n_hash["ActionName.member.#{idx+1}"]    = hash_val[:action]
+      end
+
+      req_hash = generate_request('AddPermission', n_hash)
+      request_info(req_hash, RightHttp2xxParser.new)
+    end
+
+    def remove_permission(topic_arn, label)
+      req_hash = generate_request('RemovePermission', 'TopicArn' => topic_arn, 'Label' => label)
+      request_info(req_hash, RightHttp2xxParser.new)
+    end
+
+    class SnsCreateTopicParser < RightAWSParser # :nodoc:
+      def reset
+        @result  = ''
+        @request_id = ''
+      end
+      def tagend(name)
+        case name
+          when 'RequestId'  then @result_id = @text
+          when 'TopicArn'   then @result = @text
+        end
+      end
+    end
+
+    class SnsListTopicsParser < RightAWSParser # :nodoc:
+      def reset
+        @result     = []
+        @request_id = ''
+      end
+      def tagstart(name, attributes)
+        @current_key = {} if name == 'member'
+      end
+      def tagend(name)
+        case name
+          when 'RequestId'  then @result_id         = @text
+          when 'TopicArn'   then @current_key[:arn] = @text
+          when 'member'     then @result << @current_key
+        end
+      end
+    end
+
+    class SnsSubscribeParser < RightAWSParser # :nodoc:
+      def reset
+        @result = ''
+      end
+      def tagend(name)
+        case name
+          when 'SubscriptionArn' then @result = @text
+        end
+      end
+    end
+
+    class SnsPublishParser < RightAWSParser # :nodoc:
+      def reset
+        @result = ''
+      end
+      def tagend(name)
+        case name
+          when 'MessageId' then @result = @text
+        end
+      end
+    end
+
+    class SnsGetTopicAttributesParser < RightAWSParser # :nodoc:
+      def reset
+        @result = {}
+      end
+      def tagend(name)
+        case name
+          when 'key' then @current_attr = @text
+          when 'value' then @result[@current_attr] = @text
+        end
+      end
+    end
+
+    class SnsListSubscriptionsParser < RightAWSParser # :nodoc:
+      def reset
+        @result = []
+      end
+      def tagstart(name, attributes)
+        @current_key = {} if name == 'member'
+      end
+      def tagend(name)
+        case name
+          when 'TopicArn'         then @current_key[:topic_arn]         = @text
+          when 'Protocol'         then @current_key[:protocol]          = @text
+          when 'SubscriptionArn'  then @current_key[:subscription_arn]  = @text
+          when 'Owner'            then @current_key[:owner]             = @text
+          when 'Endpoint'         then @current_key[:endpoint]          = @text
+          when 'member'     then @result << @current_key
+        end
+      end
+    end
+
+    class SnsConfirmSubscriptionParser < RightAWSParser # :nodoc:
+      def reset
+        @result = ''
+      end
+      def tagend(name)
+        case name
+          when 'SubscriptionArn' then @result = @text
+        end
+      end
+    end
+
+  end
+end

data/test/README.mdown

@@ -0,0 +1,39 @@
+# Notes and tips for developers
+
+## Setting up credentials for testing
+
+Before you can run any tests, you need to set up credentials (API key and secret) that
+will be used when talking to AWS.  The credentials are loaded in `test/test_credentials.rb`
+and are expected to be found in `~/.rightscale/testcredentials.rb` and look like this:
+
+    TestCredentials.aws_access_key_id= 'AAAAAAAAAAAAAAAAAAAA'
+    TestCredentials.aws_secret_access_key= 'asdfasdfsadf'
+    TestCredentials.account_number= '???'
+
+If you prefer to store your secret key in the OS X keychain, you can do this:
+
+    def secret_access_key_from_keychain (key_id)
+      dump = `security -q find-generic-password -a "#{key_id}" -g 2>&1`
+      dump[/password: "(.*)"/, 1]
+    end
+
+    TestCredentials.aws_access_key_id= 'AAAAAAAAAAAAAAAAAAAA'
+    TestCredentials.aws_secret_access_key= secret_access_key_from_keychain(TestCredentials.aws_access_key_id)
+    TestCredentials.account_number= '???'
+
+## Running tests
+
+There is no test suite that runs all tests.  Each module is tested separately.  E.g.,
+to run the Load Balancer tests, run `rake testelb`.  Run `rake -T` for a full list.
+
+Some tests need to launch services on AWS to have something to test.  This means two things:
+
+1. Running all the tests will cost you money.
+2. You will need to shut down some services separately once you are done, or things
+    will keep running and cost you money.
+
+As an example, the ELB and Route 53 tests need a load balancer for testing.  Starting a load balancer
+for every test would make every test case cost as much as running the LB for one hour, so it makes
+more sense to leave it running until it's no longer needed.
+
+The ELB tests contain instructions for shutting down the load balancer.