fog 1.7.0 → 1.8.0

data/lib/fog/aws/requests/dns/change_resource_record_sets.rb

@@ -198,6 +198,7 @@ module Fog
         @elb_hosted_zone_mapping ||= {
           "ap-northeast-1" => "Z2YN17T5R711GT",
           "ap-southeast-1" => "Z1WI8VXHPB1R38",
+          "ap-southeast-2" => "Z2999QAZ9SRTIC",
           "eu-west-1"      => "Z3NF1Z3NOM5OY2",
           "sa-east-1"      => "Z2ES78Y61JGQKS",
           "us-east-1"      => "Z3DZXE0Q79N41H",

data/lib/fog/aws/requests/storage/delete_multiple_objects.rb

@@ -0,0 +1,172 @@
+module Fog
+  module Storage
+    class AWS
+      class Real
+
+        require 'fog/aws/parsers/storage/delete_multiple_objects'
+
+        # Delete multiple objects from S3
+        #
+        # ==== Parameters
+        # * bucket_name<~String> - Name of bucket containing object to delete
+        # * object_names<~Array> - Array of object names to delete
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * body<~Hash>:
+        #     * 'DeleteResult'<~Array>:
+        #       * 'Deleted'<~Hash>:
+        #         * 'Key'<~String> - Name of the object that was deleted
+        #         * 'VersionId'<~String> - ID for the versioned onject in case of a versioned delete
+        #         * 'DeleteMarker'<~Boolean> - Indicates if the request accessed a delete marker
+        #         * 'DeleteMarkerVersionId'<~String> - Version ID of the delete marker accessed
+        #       * 'Error'<~Hash>:
+        #         * 'Key'<~String> - Name of the object that failed to be deleted
+        #         * 'VersionId'<~String> - ID of the versioned object that was attempted to be deleted
+        #         * 'Code'<~String> - Status code for the result of the failed delete
+        #         * 'Message'<~String> - Error description
+        #
+        # ==== See Also
+        # http://docs.amazonwebservices.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+
+        # bucket_name -- name of the bucket to use
+        # object_names -- filename
+        # For versioned deletes, options should include a version_ids hash, which
+        # maps from filename to an array of versions.
+        # The semantics are that for each (object_name, version) tuple, the
+        # caller must insert the object_name and an associated version (if
+        # desired), so for n versions, the object must be inserted n times.
+        def delete_multiple_objects(bucket_name, object_names, options = {})
+          data = "<Delete>"
+          data << "<Quiet>true</Quiet>" if options.delete(:quiet)
+          version_ids = options.delete('versionId')
+          object_names.each do |object_name|
+            data << "<Object>"
+            data << "<Key>#{CGI.escape(object_name)}</Key>"
+            object_version = version_ids.nil? ? nil : version_ids[object_name]
+            if object_version
+              data << "<VersionId>#{CGI.escape(object_version)}</VersionId>"
+            end
+            data << "</Object>"
+          end
+          data << "</Delete>"
+
+          headers = options
+          headers['Content-Length'] = data.length
+          headers['Content-MD5'] = Base64.encode64(Digest::MD5.digest(data)).
+                                   gsub("\n", '')
+
+          request({
+            :body       => data,
+            :expects    => 200,
+            :headers    => headers,
+            :host       => "#{bucket_name}.#{@host}",
+            :method     => 'POST',
+            :parser     => Fog::Parsers::Storage::AWS::DeleteMultipleObjects.new,
+            :query      => {'delete' => nil}
+          })
+        end
+
+      end
+
+      class Mock # :nodoc:all
+
+        def delete_multiple_objects(bucket_name, object_names, options = {})
+          response = Excon::Response.new
+          if bucket = self.data[:buckets][bucket_name]
+            response.status = 200
+            response.body = { 'DeleteResult' => [] }
+            version_ids = options.delete('versionId')
+            object_names.each do |object_name|
+              object_version = version_ids.nil? ? nil : version_ids[object_name]
+              response.body['DeleteResult'] << delete_object_helper(bucket,
+                                                                object_name,
+                                                                object_version)
+            end
+          else
+            response.status = 404
+            raise(Excon::Errors.status_error({:expects => 200}, response))
+          end
+          response
+        end
+
+        private
+
+        def delete_object_helper(bucket, object_name, version_id)
+          response = { 'Deleted' => {} }
+          if bucket[:versioning]
+            bucket[:objects][object_name] ||= []
+
+            if version_id
+              version = bucket[:objects][object_name].find { |object| object['VersionId'] == version_id}
+
+              # S3 special cases the 'null' value to not error out if no such version exists.
+              if version || (version_id == 'null')
+                bucket[:objects][object_name].delete(version)
+                bucket[:objects].delete(object_name) if bucket[:objects][object_name].empty?
+
+                response['Deleted'] = { 'Key' => object_name,
+                                        'VersionId' => version_id,
+                                        'DeleteMarker' => 'true',
+                                        'DeleteMarkerVersionId' => version_id
+                                      }
+              else
+                response = delete_error_body(object_name,
+                                             version_id,
+                                             'InvalidVersion',
+                                             'Invalid version ID specified')
+              end
+            else
+              delete_marker = {
+                :delete_marker    => true,
+                'Key'             => object_name,
+                'VersionId'       => bucket[:versioning] == 'Enabled' ? Fog::Mock.random_base64(32) : 'null',
+                'Last-Modified'   => Fog::Time.now.to_date_header
+              }
+
+              # When versioning is suspended, a delete marker is placed if the last object ID is not the value 'null',
+              # otherwise the last object is replaced.
+              if bucket[:versioning] == 'Suspended' && bucket[:objects][object_name].first['VersionId'] == 'null'
+                bucket[:objects][object_name].shift
+              end
+
+              bucket[:objects][object_name].unshift(delete_marker)
+
+              response['Deleted'] = { 'Key' => object_name,
+                                      'VersionId' => delete_marker['VersionId'],
+                                      'DeleteMarkerVersionId' =>
+                                          delete_marker['VersionId'],
+                                      'DeleteMarker' => 'true',
+                                    }
+            end
+          else
+            if version_id && version_id != 'null'
+              response = delete_error_body(object_name,
+                                           version_id,
+                                           'InvalidVersion',
+                                           'Invalid version ID specified')
+              response = invalid_version_id_payload(version_id)
+            else
+              bucket[:objects].delete(object_name)
+              response['Deleted'] = { 'Key' => object_name }
+            end
+          end
+          response
+        end
+
+        def delete_error_body(key, version_id, message, code)
+          {
+            'Error' => {
+              'Code'      => code,
+              'Message'   => message,
+              'VersionId' => version_id,
+              'Key'       => key,
+            }
+          }
+        end
+
+      end
+    end
+  end
+end
+

data/lib/fog/aws/signaturev4.rb

@@ -70,4 +70,4 @@ DATA
 
     end
   end
-end
+end

data/lib/fog/aws/sqs.rb

@@ -40,7 +40,7 @@ module Fog
           setup_credentials(options)
           @region = options[:region] || 'us-east-1'
 
-          unless ['ap-northeast-1', 'ap-southeast-1', 'eu-west-1', 'us-east-1', 'us-west-1', 'us-west-2', 'sa-east-1'].include?(@region)
+          unless ['ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-1', 'us-east-1', 'us-west-1', 'us-west-2', 'sa-east-1'].include?(@region)
             raise ArgumentError, "Unknown region: #{@region.inspect}"
           end
         end

data/lib/fog/aws/storage.rb

@@ -26,6 +26,7 @@ module Fog
       request :delete_bucket_policy
       request :delete_bucket_website
       request :delete_object
+      request :delete_multiple_objects
       request :get_bucket
       request :get_bucket_acl
       request :get_bucket_lifecycle
@@ -337,6 +338,7 @@ DATA
           for key in (params[:query] || {}).keys.sort
             if %w{
               acl
+              delete
               lifecycle
               location
               logging

data/lib/fog/bluebox/requests/compute/create_block.rb

@@ -30,14 +30,12 @@ module Fog
             'location' => location_id
           }
 
-          body = URI.encode options.map {|k,v| "#{k}=#{v}"}.join('&')
-
           request(
             :expects  => 200,
             :method   => 'POST',
             :path     => '/api/blocks.json',
             :query    => query,
-            :body     => URI.encode(body)
+            :body     => options.map {|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}.join('&')
           )
         end
 

data/lib/fog/brightbox/compute.rb

@@ -1,5 +1,6 @@
 require 'fog/brightbox'
 require 'fog/compute'
+require 'fog/brightbox/oauth2'
 
 module Fog
   module Compute
@@ -16,6 +17,12 @@ module Fog
       # User credentials (still requires client details)
       recognizes :brightbox_username, :brightbox_password, :brightbox_account
 
+      # Cached tokens
+      recognizes :brightbox_access_token, :brightbox_refresh_token
+
+      # Automatic token management
+      recognizes :brightbox_token_management
+
       # Excon connection settings
       recognizes :persistent
 
@@ -106,6 +113,7 @@ module Fog
       request :remove_nodes_load_balancer
       request :remove_servers_server_group
       request :reset_ftp_password_account
+      request :reset_ftp_password_scoped_account
       request :reset_secret_api_client
       request :reset_secret_application
       request :shutdown_server
@@ -126,6 +134,139 @@ module Fog
       request :update_user
 
       module Shared
+        include Fog::Brightbox::OAuth2
+
+        # Creates a new instance of the Brightbox Compute service
+        #
+        # @note If you open a connection using just a refresh token when it
+        #   expires the service will no longer be able to authenticate.
+        #
+        # @param [Hash] options
+        # @option options [String] :brightbox_api_url   Override the default (or configured) API endpoint
+        # @option options [String] :brightbox_auth_url  Override the default (or configured) API authentication endpoint
+        # @option options [String] :brightbox_client_id Client identifier to authenticate with (overrides configured)
+        # @option options [String] :brightbox_secret    Client secret to authenticate with (overrides configured)
+        # @option options [String] :brightbox_username  Email or user identifier for user based authentication
+        # @option options [String] :brightbox_password  Password for user based authentication
+        # @option options [String] :brightbox_account   Account identifier to scope this connection to
+        # @option options [String] :connection_options  Settings to pass to underlying {Fog::Connection}
+        # @option options [Boolean] :persistent         Sets a persistent HTTP {Fog::Connection}
+        # @option options [String] :brightbox_access_token  Sets the OAuth access token to use rather than requesting a new token
+        # @option options [String] :brightbox_refresh_token Sets the refresh token to use when requesting a newer access token
+        # @option options [String] :brightbox_token_management Overide the existing behaviour to request access tokens if expired (default is `true`)
+        #
+        def initialize(options)
+          # Currently authentication and api endpoints are the same but may change
+          @auth_url            = options[:brightbox_auth_url]  || Fog.credentials[:brightbox_auth_url] || API_URL
+          @auth_connection     = Fog::Connection.new(@auth_url)
+
+          @api_url             = options[:brightbox_api_url]   || Fog.credentials[:brightbox_api_url]  || API_URL
+          @connection_options  = options[:connection_options]  || {}
+          @persistent          = options[:persistent]          || false
+          @connection          = Fog::Connection.new(@api_url, @persistent, @connection_options)
+
+          # Authentication options
+          client_id            = options[:brightbox_client_id] || Fog.credentials[:brightbox_client_id]
+          client_secret        = options[:brightbox_secret]    || Fog.credentials[:brightbox_secret]
+
+          username             = options[:brightbox_username]  || Fog.credentials[:brightbox_username]
+          password             = options[:brightbox_password]  || Fog.credentials[:brightbox_password]
+          @configured_account  = options[:brightbox_account]   || Fog.credentials[:brightbox_account]
+          # Request account can be changed at anytime and changes behaviour of future requests
+          @scoped_account      = @configured_account
+
+          credential_options   = {:username => username, :password => password}
+          @credentials         = CredentialSet.new(client_id, client_secret, credential_options)
+
+          # If existing tokens have been cached, allow continued use of them in the service
+          @credentials.update_tokens(options[:brightbox_access_token], options[:brightbox_refresh_token])
+
+          @token_management    = options.fetch(:brightbox_token_management, true)
+        end
+
+        # Sets the scoped account for future requests
+        # @param [String] scoped_account Identifier of the account to scope request to
+        def scoped_account=(scoped_account)
+          @scoped_account = scoped_account
+        end
+
+        # This returns the account identifier that the request should be scoped by
+        # based on the options passed to the request and current configuration
+        #
+        # @param [String] options_account Any identifier passed into the request
+        #
+        # @return [String, nil] The account identifier to scope the request to or nil
+        def scoped_account(options_account = nil)
+          [options_account, @scoped_account].compact.first
+        end
+
+        # Resets the scoped account back to intially configured one
+        def scoped_account_reset
+          @scoped_account = @configured_account
+        end
+
+        # Returns the scoped account being used for requests
+        #
+        # * For API clients this is the owning account
+        # * For User applications this is the account specified by either +account_id+
+        #   option on a connection or the +brightbox_account+ setting in your configuration
+        #
+        # @return [Fog::Compute::Brightbox::Account]
+        #
+        def account
+          Fog::Compute::Brightbox::Account.new(get_scoped_account).tap do |acc|
+            # Connection is more like the compute 'service'
+            acc.connection = self
+          end
+        end
+
+        # Returns true if authentication is being performed as a user
+        # @return [Boolean]
+        def authenticating_as_user?
+          @credentials.user_details?
+        end
+
+        # Returns true if an access token is set
+        # @return [Boolean]
+        def access_token_available?
+          !! @credentials.access_token
+        end
+
+        # Returns the current access token or nil
+        # @return [String,nil]
+        def access_token
+          @credentials.access_token
+        end
+
+        # Returns the current refresh token or nil
+        # @return [String,nil]
+        def refresh_token
+          @credentials.refresh_token
+        end
+
+        # Requests a new access token
+        #
+        # @return [String] New access token
+        def get_access_token
+          begin
+            get_access_token!
+          rescue Excon::Errors::Unauthorized, Excon::Errors::BadRequest
+            @credentials.update_tokens(nil, nil)
+          end
+          @credentials.access_token
+        end
+
+        # Requests a new access token and raises if there is a problem
+        #
+        # @return [String] New access token
+        # @raise [Excon::Errors::BadRequest] The credentials are expired or incorrect
+        #
+        def get_access_token!
+          response = request_access_token(@auth_connection, @credentials)
+          update_credentials_from_response(@credentials, response)
+          @credentials.access_token
+        end
+
         # Returns an identifier for the default image for use
         #
         # Currently tries to find the latest version Ubuntu LTS (i686) widening
@@ -139,119 +280,133 @@ module Fog
           return @default_image_id unless @default_image_id.nil?
           @default_image_id = Fog.credentials[:brightbox_default_image] || select_default_image
         end
+
+      private
+
+        # This makes a request of the API based on the configured setting for
+        # token management.
+        #
+        # @param [Hash] options Excon compatible options
+        # @see https://github.com/geemus/excon/blob/master/lib/excon/connection.rb
+        #
+        # @return [Hash] Data of response body
+        #
+        def make_request(options)
+          if @token_management
+            managed_token_request(options)
+          else
+            authenticated_request(options)
+          end
+        end
+
+        # This request checks for access tokens and will ask for a new one if
+        # it receives Unauthorized from the API before repeating the request
+        #
+        # @param [Hash] options Excon compatible options
+        #
+        # @return [Excon::Response]
+        def managed_token_request(options)
+          begin
+            get_access_token unless access_token_available?
+            response = authenticated_request(options)
+          rescue Excon::Errors::Unauthorized
+            get_access_token
+            response = authenticated_request(options)
+          end
+        end
+
+        # This request makes an authenticated request of the API using currently
+        # setup credentials.
+        #
+        # @param [Hash] options Excon compatible options
+        #
+        # @return [Excon::Response]
+        def authenticated_request(options)
+          headers = options[:headers] || {}
+          headers.merge!("Authorization" => "OAuth #{@credentials.access_token}", "Content-Type" => "application/json")
+          options[:headers] = headers
+          # TODO This is just a wrapper around a call to Excon::Connection#request
+          #   so can be extracted from Compute by passing in the connection,
+          #   credentials and options
+          @connection.request(options)
+        end
       end
 
+      # The Mock Service allows you to run a fake instance of the Service
+      # which makes no real connections.
+      #
+      # @todo Implement
+      #
       class Mock
         include Shared
 
-        def initialize(options)
-          @brightbox_client_id = options[:brightbox_client_id] || Fog.credentials[:brightbox_client_id]
-          @brightbox_secret = options[:brightbox_secret] || Fog.credentials[:brightbox_secret]
+        def request(method, path, expected_responses, parameters = {})
+          _request
         end
 
-        def request(options)
-          raise "Not implemented"
+        def request_access_token(connection, credentials)
+          _request
         end
 
       private
+
+        def _request
+          raise Fog::Errors::MockNotImplemented
+        end
+
         def select_default_image
           "img-mockd"
         end
       end
 
+      # The Real Service actually makes real connections to the Brightbox
+      # service.
+      #
       class Real
         include Shared
 
-        def initialize(options)
-          # Currently authentication and api endpoints are the same but may change
-          @auth_url             = options[:brightbox_auth_url] || Fog.credentials[:brightbox_auth_url] || API_URL
-          @api_url              = options[:brightbox_api_url] || Fog.credentials[:brightbox_api_url] || API_URL
-          @connection_options   = options[:connection_options] || {}
-          @brightbox_client_id  = options[:brightbox_client_id] || Fog.credentials[:brightbox_client_id]
-          @brightbox_secret     = options[:brightbox_secret] || Fog.credentials[:brightbox_secret]
-          @brightbox_username   = options[:brightbox_username] || Fog.credentials[:brightbox_username]
-          @brightbox_password   = options[:brightbox_password] || Fog.credentials[:brightbox_password]
-          @brightbox_account    = options[:brightbox_account] || Fog.credentials[:brightbox_account]
-          @persistent           = options[:persistent] || false
-          @connection = Fog::Connection.new(@api_url, @persistent, @connection_options)
-        end
-
-        def request(method, url, expected_responses, options = {})
+        # Makes an API request to the given path using passed options or those
+        # set with the service setup
+        #
+        # @todo Standard Fog behaviour is to return the Excon::Response but
+        #   this was unintentionally changed to be the Hash version of the
+        #   data in the body. This loses access to some details and should
+        #   be corrected in a backwards compatible manner
+        #
+        # @param [String] method HTTP method to use for the request
+        # @param [String] path   The absolute path for the request
+        # @param [Array<Fixnum>] expected_responses HTTP response codes that have been successful
+        # @param [Hash]  parameters Keys and values for JSON
+        # @option parameters [String] :account_id The scoping account if required
+        #
+        # @return [Hash]
+        def request(method, path, expected_responses, parameters = {})
           request_options = {
             :method   => method.to_s.upcase,
-            :path     => url,
+            :path     => path,
             :expects  => expected_responses
           }
-          options[:account_id] = @brightbox_account if options[:account_id].nil? && @brightbox_account
-          request_options[:body] = Fog::JSON.encode(options) unless options.empty?
-          make_request(request_options)
-        end
-
-        # Returns the scoped account being used for requests
-        #
-        # API Clients:: This is the owning account
-        # User Apps::   This is the account specified by either +account_id+
-        #               option on a connection or the +brightbox_account+
-        #               setting in your configuration
-        #
-        # === Returns:
-        #
-        # <tt>Fog::Compute::Brightbox::Account</tt>
-        #
-        def account
-          Fog::Compute::Brightbox::Account.new(get_scoped_account)
-        end
 
-      private
-        def get_oauth_token(options = {})
-          auth_url = options[:brightbox_auth_url] || @auth_url
-
-          connection = Fog::Connection.new(auth_url)
-          authentication_body_hash = if @brightbox_username && @brightbox_password
-            {
-              'client_id' => @brightbox_client_id,
-              'grant_type' => 'password',
-              'username' => @brightbox_username,
-              'password' => @brightbox_password
-            }
-          else
-            {'client_id' => @brightbox_client_id, 'grant_type' => 'none'}
+          # Select the account to scope for this request
+          account = scoped_account(parameters.fetch(:account_id, nil))
+          if account
+            request_options[:query] = { :account_id => account }
           end
-          @authentication_body = Fog::JSON.encode(authentication_body_hash)
-
-          response = connection.request({
-            :path => "/token",
-            :expects  => 200,
-            :headers  => {
-              'Authorization' => "Basic " + Base64.encode64("#{@brightbox_client_id}:#{@brightbox_secret}").chomp,
-              'Content-Type' => 'application/json'
-            },
-            :method   => 'POST',
-            :body     => @authentication_body
-          })
-          @oauth_token = Fog::JSON.decode(response.body)["access_token"]
-          return @oauth_token
-        end
 
-        def make_request(params)
-          begin
-            get_oauth_token if @oauth_token.nil?
-            response = authenticated_request(params)
-          rescue Excon::Errors::Unauthorized
-            get_oauth_token
-            response = authenticated_request(params)
-          end
+          request_options[:body] = Fog::JSON.encode(parameters) unless parameters.empty?
+
+          response = make_request(request_options)
+
+          # FIXME We should revert to returning the Excon::Request after a suitable
+          # configuration option is in place to switch back to this incorrect behaviour
           unless response.body.empty?
-            response = Fog::JSON.decode(response.body)
+            Fog::JSON.decode(response.body)
+          else
+            response
           end
         end
 
-        def authenticated_request(options)
-          headers = options[:headers] || {}
-          headers.merge!("Authorization" => "OAuth #{@oauth_token}", "Content-Type" => "application/json")
-          options[:headers] = headers
-          @connection.request(options)
-        end
+      private
 
         # Queries the API and tries to select the most suitable official Image
         # to use if the user chooses not to select their own.