fog-maestrodev 1.15.0.20130927082724 → 1.18.0.20131111203459

data/lib/fog/hp/requests/storage/post_container.rb

@@ -0,0 +1,38 @@
+module Fog
+  module Storage
+    class HP
+      class Real
+
+        # Create or update metadata for an existing container
+        #
+        # ==== Parameters
+        # * container<~String> - Name for existing container, should be < 256 bytes and must not contain '/'
+        # * headers<~Hash> - Hash of metadata name/value pairs
+        def post_container(container, headers = {})
+          response = request(
+            :expects  => 204,
+            :headers  => headers,
+            :method   => 'POST',
+            :path     => Fog::HP.escape(container)
+          )
+          response
+        end
+
+      end
+
+      class Mock # :nodoc:all
+
+        def post_container(container_name, headers = {})
+          if self.data[:containers][container_name].nil?
+            raise Fog::Storage::HP::NotFound
+          end
+          response = Excon::Response.new
+          response.status = 204
+          response
+        end
+
+      end
+
+    end
+  end
+end

data/lib/fog/hp/requests/storage/post_object.rb

@@ -0,0 +1,56 @@
+module Fog
+  module Storage
+    class HP
+      class Real
+
+        # Create or update metadata for an existing object
+        #
+        # ==== Parameters
+        # * container<~String> - Name of the existing container, should be < 256 bytes and must not contain '/'
+        # * object<~String> - Name of the existing object
+        # * headers<~Hash> - Hash of metadata name/value pairs
+        def post_object(container, object, headers = {})
+          response = request(
+            :expects  => 202,
+            :headers  => headers,
+            :method   => 'POST',
+            :path     => "#{Fog::HP.escape(container)}/#{Fog::HP.escape(object)}"
+          )
+          response
+        end
+
+      end
+
+      class Mock # :nodoc:all
+
+        def post_object(container_name, object_name, headers = {})
+          response = Excon::Response.new
+          if container = self.data[:containers][container_name]
+            response.status = 202
+            object = container[:objects][object_name]
+
+            for key, value in headers
+              case key
+              when 'Content-Disposition', 'Content-Encoding', 'X-Delete-At', 'X-Delete-After', /^X-Object-Meta-/
+                object[key] = value unless value.nil?
+              end
+            end
+
+            container[:objects][object_name] = object
+            response.headers = {
+              'Content-Length'  => object['Content-Length'],
+              'Content-Type'    => object['Content-Type'],
+              'ETag'            => object['ETag'],
+              'Date'            => object['Date']
+            }
+          else
+            raise Fog::Storage::HP::NotFound
+          end
+          response
+        end
+
+      end
+
+    end
+  end
+end

data/lib/fog/hp/simple_http_instrumentor.rb

@@ -0,0 +1,59 @@
+module Excon
+  class SimpleHttpInstrumentor
+    class << self
+      attr_accessor :events
+
+      def instrument(name, params = {}, &block)
+        params = params.dup
+        if params.has_key?(:headers) && params[:headers].has_key?('Authorization')
+          params[:headers] = params[:headers].dup
+          params[:headers]['Authorization'] = REDACTED
+        end
+        if params.has_key?(:password)
+          params[:password] = REDACTED
+        end
+        $stderr.puts("--- #{name} ---")
+        if name.include?('.request')
+          query = ''
+          tmp_query = ''
+          if params.has_key?(:query) && !params[:query].nil?
+            params[:query].each do |key, value|
+              tmp_query += "#{key}=#{value}&"
+            end
+            if !tmp_query.nil?
+              query = "?#{tmp_query}"
+              query.chomp!('&')
+            end
+          end
+          $stderr.puts("#{params[:method]} #{params[:path]}#{query} HTTP/1.1" )
+          $stderr.puts("User-Agent: #{params[:headers]['User-Agent']}")
+          $stderr.puts("Host: #{params[:host]} Port: #{params[:port]}")
+          $stderr.puts("Accept: #{params[:headers]['Accept']}")
+          $stderr.puts("X-Auth-Token: #{params[:headers]['X-Auth-Token']}")
+          $stderr.puts("Body: #{params[:body]}")
+        elsif name.include?('.response')
+          $stderr.puts("HTTP/1.1 #{params[:status]}")
+          $stderr.puts("Content-Length: #{params[:headers]['Content-Length']}")
+          $stderr.puts("Content-Type: #{params[:headers]['Content-Type']}")
+          $stderr.puts("Date: #{params[:headers]['Date']}")
+          params[:headers].each do |key, value|
+            if !['Content-Length', 'Content-Type', 'Date'].include?(key)
+              $stderr.puts("#{key}: #{value}")
+            end
+          end
+          $stderr.puts("Date: #{params[:headers]['Date']}\n")
+          $stderr.puts("Body: #{params[:body]}")
+        elsif name.include?('.retry')
+          $stderr.puts("#{params.inspect}")
+        elsif name.include?('.error')
+          $stderr.puts("#{params.inspect}")
+        end
+
+        if block_given?
+          yield
+        end
+      end
+
+    end
+  end
+end

data/lib/fog/hp/storage.rb

@@ -6,11 +6,11 @@ module Fog
     class HP < Fog::Service
 
       requires    :hp_secret_key, :hp_tenant_id, :hp_avl_zone
-      recognizes  :hp_auth_uri, :hp_cdn_ssl, :hp_cdn_uri
+      recognizes  :hp_auth_uri, :hp_cdn_ssl, :hp_cdn_uri, :credentials, :hp_service_type
       recognizes  :persistent, :connection_options
       recognizes  :hp_use_upass_auth_style, :hp_auth_version, :user_agent
       recognizes  :hp_access_key, :hp_account_id  # :hp_account_id is deprecated use hp_access_key instead
-      
+
       # :os_account_meta_temp_url_key is an OpenStack specific setting used to generate temporary urls.
       recognizes  :os_account_meta_temp_url_key
 
@@ -41,6 +41,8 @@ module Fog
       request :head_object
       request :head_shared_container
       request :head_shared_object
+      request :post_container
+      request :post_object
       request :put_container
       request :put_object
       request :put_shared_object
@@ -57,6 +59,7 @@ module Fog
               :hp_cdn_uri     => @hp_cdn_uri,
               :hp_tenant_id   => @hp_tenant_id,
               :hp_avl_zone    => @hp_avl_zone,
+              :credentials    => @credentials,
               :connection_options => @connection_options
             )
             if @cdn.enabled?
@@ -146,9 +149,66 @@ module Fog
           write_h = write_header.split(',') unless write_header.nil?
           return read_h, write_h
         end
+        
+        # Get an expiring object https url
+        #
+        # ==== Parameters
+        # * container<~String> - Name of container containing object
+        # * object<~String> - Name of object to get expiring url for
+        # * expires<~Time> - An expiry time for this url
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * body<~String> - url for object
+        def get_object_https_url(container, object, expires, options = {})
+          create_temp_url(container, object, expires, "GET", options.merge(:scheme => "https"))
+        end
+        
+        # Get an expiring object http url
+        #
+        # ==== Parameters
+        # * container<~String> - Name of container containing object
+        # * object<~String> - Name of object to get expiring url for
+        # * expires<~Time> - An expiry time for this url
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * body<~String> - url for object
+        def get_object_http_url(container, object, expires, options = {})
+          create_temp_url(container, object, expires, "GET", options.merge(:scheme => "http"))
+        end
 
+        # Get an object http url expiring in the given amount of seconds
+        #
+        # ==== Parameters
+        # * container<~String> - Name of container containing object
+        # * object<~String> - Name of object to get expiring url for
+        # * expires_secs<~Integer> - the amount of seconds from now until the url expires
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * body<~String> - url for object
         def generate_object_temp_url(container, object, expires_secs, method)
-          return unless (container && object && expires_secs && method)
+          expiration_time = (Time.now + expires_secs.to_i).to_i
+          create_temp_url(container, object, expiration_time, method, {})
+        end
+        
+        # creates a temporary url 
+        #
+        # ==== Parameters
+        # * container<~String> - Name of container containing object
+        # * object<~String> - Name of object to get expiring url for
+        # * expires<~Time> - An expiry time for this url
+        # * method<~String> - The method to use for accessing the object (GET, PUT, HEAD)
+        # * scheme<~String> - The scheme to use (http, https)
+        # * options<~Hash> - An optional options hash
+        #
+        # ==== Returns
+        # * response<~Excon::Response>:
+        #   * body<~String> - url for object
+        def create_temp_url(container, object, expires, method, options = {})
+          raise ArgumentError, "Insufficient parameters specified." unless (container && object && expires && method)
+          raise ArgumentError, "Storage must my instantiated with the :os_account_meta_temp_url_key option" if @os_account_meta_temp_url_key.nil?
 
           # POST not allowed
           allowed_methods = %w{GET PUT HEAD}
@@ -156,23 +216,17 @@ module Fog
             raise ArgumentError.new("Invalid method '#{method}' specified. Valid methods are: #{allowed_methods.join(', ')}")
           end
 
-          expires = (Time.now + expires_secs.to_i).to_i
-
-          # split up the storage uri
-          uri = URI.parse(@hp_storage_uri)
-          host   = uri.host
-          path   = uri.path
-          port   = uri.port
-          scheme = uri.scheme
-
+          expires        = expires.to_i
+          scheme = options[:scheme] || @scheme
+          
           # do not encode before signature generation, encode after
-          sig_path = "#{path}/#{container}/#{object}"
-          encoded_path = "#{path}/#{Fog::HP.escape(container)}/#{Fog::HP.escape(object)}"
+          sig_path = "#{@path}/#{container}/#{object}"
+          encoded_path = "#{@path}/#{Fog::HP.escape(container)}/#{Fog::HP.escape(object)}"
 
           string_to_sign = "#{method}\n#{expires}\n#{sig_path}"
-                    
-          signature = nil          
-          
+
+          signature = nil
+
           # HP uses a different strategy to create the signature that is passed to swift than OpenStack.
           # As the HP provider is broadly used by OpenStack users the OpenStack strategy is applied when
           # the @os_account_meta_temp_url_key is given.
@@ -190,11 +244,10 @@ module Fog
             signature     = @hp_tenant_id.to_s + ":" + @hp_access_key.to_s + ":" + signed_string
             signature     = Fog::HP.escape(signature)
           end
-
+          
           # generate the temp url using the signature and expiry
-          "#{scheme}://#{host}:#{port}#{encoded_path}?temp_url_sig=#{signature}&temp_url_expires=#{expires}"
+          "#{scheme}://#{@host}#{encoded_path}?temp_url_sig=#{signature}&temp_url_expires=#{expires}"          
         end
-
       end
 
       class Mock
@@ -246,6 +299,7 @@ module Fog
 
       class Real
         include Utils
+        attr_reader :credentials
         attr_reader :hp_cdn_ssl
 
         def initialize(options={})
@@ -263,9 +317,12 @@ module Fog
           @hp_cdn_ssl    = options[:hp_cdn_ssl]
           @connection_options = options[:connection_options] || {}
           ### Set an option to use the style of authentication desired; :v1 or :v2 (default)
+          ### A symbol is required, we should ensure that the value is loaded as a symbol
           auth_version = options[:hp_auth_version] || :v2
+          auth_version = auth_version.to_s.downcase.to_sym
+          
           ### Pass the service name for object storage to the authentication call
-          options[:hp_service_type] = "Object Storage"
+          options[:hp_service_type] ||= "Object Storage"
           @hp_tenant_id = options[:hp_tenant_id]
           @hp_avl_zone  = options[:hp_avl_zone]
           @os_account_meta_temp_url_key = options[:os_account_meta_temp_url_key]
@@ -277,6 +334,7 @@ module Fog
             # the CS service catalog returns the cdn endpoint
             @hp_storage_uri = credentials[:endpoint_url]
             @hp_cdn_uri  = credentials[:cdn_endpoint_url]
+            @credentials = credentials
           else
             # Call the legacy v1.0/v1.1 authentication
             credentials = Fog::HP.authenticate_v1(options, @connection_options)
@@ -309,7 +367,6 @@ module Fog
                 'Accept'       => 'application/json',
                 'X-Auth-Token' => @auth_token
               }.merge!(params[:headers] || {}),
-              :host     => @host,
               :path     => "#{@path}/#{params[:path]}",
             }), &block)
           rescue Excon::Errors::HTTPStatusError => error
@@ -335,7 +392,6 @@ module Fog
                 'Accept'       => 'application/json',
                 'X-Auth-Token' => @auth_token
               }.merge!(params[:headers] || {}),
-              :host     => @host,
               :path     => "#{params[:path]}",
             }), &block)
           rescue Excon::Errors::HTTPStatusError => error

data/lib/fog/hp.rb

@@ -1,11 +1,12 @@
 require 'fog/core'
+require 'fog/hp/simple_http_instrumentor'
 
 module Fog
   module HP
 
     # define a specific version for the HP Provider
     unless const_defined?(:VERSION)
-      VERSION = '0.0.20'
+      VERSION = '0.0.22'
     end
 
     extend Fog::Provider
@@ -55,10 +56,14 @@ module Fog
       end
     end
 
+    service(:block_storage, 'hp/block_storage', 'BlockStorage')
+    service(:block_storage_v2, 'hp/block_storage_v2', 'BlockStorageV2')
     service(:cdn,     'hp/cdn',     'CDN')
     service(:compute, 'hp/compute', 'Compute')
+    service(:dns,     'hp/dns',     'DNS')
+    service(:lb,      'hp/lb',      'LB')
+    service(:network, 'hp/network', 'Network')
     service(:storage, 'hp/storage', 'Storage')
-    service(:block_storage, 'hp/block_storage', 'BlockStorage')
 
     # legacy swauth 1.0/1.1 style authentication
     def self.authenticate_v1(options, connection_options = {})
@@ -85,8 +90,6 @@ module Fog
           'X-Auth-Key'  => @hp_secret_key,
           'X-Auth-User' => @hp_access_key
         },
-        :host     => @host,
-        :port     => @port,
         :method   => 'GET',
         :path     => @auth_path
       })
@@ -101,8 +104,43 @@ module Fog
       }
     end
 
+    def self.service_catalog(options, connection_options = {})
+      creds = authenticate_v2(options, connection_options)
+      return {} if creds.nil?
+      return {} if creds[:service_catalog].nil?
+      return creds[:service_catalog]
+    end
+
     # keystone based control services style authentication
     def self.authenticate_v2(options, connection_options = {})
+      unless options[:credentials].nil?
+        expires = true
+        begin
+          expire = DateTime.parse(options[:credentials][:expires])
+          expires = false if expire > DateTime.now
+        rescue
+        end
+        if expires
+          options = options.clone
+          options.delete(:credentials)
+        else
+          service_catalog = options[:credentials][:service_catalog]
+          type  = options[:hp_service_type]
+          zone   = options[:hp_avl_zone]
+          begin
+            creds = options[:credentials].clone
+            creds[:endpoint_url] = get_endpoint_url(service_catalog, type, zone)
+            begin
+              creds[:cdn_endpoint_url] = get_endpoint_url(service_catalog, "CDN", zone)
+            rescue
+            end
+            return creds
+          rescue
+          end
+          options = options.clone
+          options.delete(:credentials)
+        end
+      end
       hp_auth_uri = options[:hp_auth_uri] || "https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens"
       # append /tokens if missing from auth uri
       @hp_auth_uri = hp_auth_uri.include?('tokens')? hp_auth_uri : hp_auth_uri + "tokens"
@@ -162,8 +200,6 @@ module Fog
           :headers => {
               'Content-Type' => 'application/json'
           },
-          :host => @host,
-          :port => @port,
           :method => 'POST',
           :body => Fog::JSON.encode(request_body),
           :path => @auth_path
@@ -174,18 +210,22 @@ module Fog
 
       ### fish out auth_token and endpoint for the service
       auth_token = body['access']['token']['id']
-      endpoint_url = get_endpoint_from_catalog(body['access']['serviceCatalog'], @hp_service_type, @hp_avl_zone)
-      # If service is Storage, then get the CDN endpoint as well. 'Name' is unique instead of 'Type'
-      if @hp_service_type == "Object Storage"
-        cdn_endpoint_url = get_endpoint_from_catalog(body['access']['serviceCatalog'], "CDN", @hp_avl_zone)
+      expires = body['access']['token']['expires']
+      service_catalog = get_service_catalog(body['access']['serviceCatalog'])
+      endpoint_url = get_endpoint_url(service_catalog, @hp_service_type, @hp_avl_zone)
+      begin
+        cdn_endpoint_url = get_endpoint_url(service_catalog, "CDN", @hp_avl_zone)
+      rescue
       end
 
-      return {
+      creds = {
         :auth_token => auth_token,
+        :expires => expires,
+        :service_catalog => service_catalog,
         :endpoint_url => endpoint_url,
         :cdn_endpoint_url => cdn_endpoint_url
       }
-
+      return creds
     end
 
     # CGI.escape, but without special treatment on spaces
@@ -195,22 +235,50 @@ module Fog
       end
     end
 
+    # converts any attributes hash from aliased keys to original attribute keys
+    def self.convert_aliased_attributes_to_original(model, attributes)
+      original_attributes = {}
+      attributes.each do |k, v|
+        if orig_key = model.aliases.invert[k]
+          original_attributes[orig_key] = v
+        else
+          original_attributes[k] = v
+        end
+      end
+      original_attributes
+    end
+
     private
 
-    def self.get_endpoint_from_catalog(service_catalog, service_type, avl_zone)
-      raise "Unable to parse service catalog." unless service_catalog
-      service_item = service_catalog.detect do |s|
-        # 'Name' is unique instead of 'Type'
-        s["name"] == service_type
+    def self.get_service_catalog(body)
+      raise "Unable to parse service catalog." unless body
+      service_catalog = {}
+      body.each do |s|
+        name = s["name"]
+        next if name.nil?
+        name = name.to_sym
+        next if s['endpoints'].nil?
+        service_catalog[name] = {}
+        s['endpoints'].each do |ep|
+          next if ep['region'].nil?
+          next if ep['publicURL'].nil?
+          next if ep['publicURL'].empty?
+          service_catalog[name][ep['region'].to_sym] = ep['publicURL']
+        end
       end
-      if service_item and service_item['endpoints']
-        endpoint = service_item['endpoints'].detect do |ep|
-          ep['region'] == avl_zone
+      return service_catalog
+    end
+
+    def self.get_endpoint_url(service_catalog, service_type, avl_zone)
+      return nil if service_type.nil?
+      service_type = service_type.to_sym
+      avl_zone = avl_zone.to_sym
+      unless service_catalog[service_type].nil?
+        unless service_catalog[service_type][avl_zone].nil?
+          return service_catalog[service_type][avl_zone]
         end
-        endpoint_url = endpoint['publicURL'] if endpoint
-        raise "Unable to retrieve endpoint service url for availability zone '#{avl_zone}' from service catalog. " if endpoint_url.nil?
-        return endpoint_url
       end
+      raise "Unable to retrieve endpoint service url for availability zone '#{avl_zone}' from service catalog. "
     end
 
     def self.set_user_agent_header(conn_opts, base_str, client_str)
@@ -261,6 +329,23 @@ module Fog
         ip.join('.')
       end
 
+      def self.uuid
+        # pattern of 8-4-4-4-12 hexadecimal digits
+        uuid = []
+        [8,4,4,4,12].each do |x|
+          uuid << Fog::Mock.random_hex(x)
+        end
+        uuid.join('-')
+      end
+
+      def self.mac_address
+        mac_add = []
+        6.times do
+          mac_add << Fog::Mock.random_hex(2)
+        end
+        mac_add.join(':')
+      end
+
     end
 
   end

data/lib/fog/internet_archive/models/storage/file.rb

@@ -152,7 +152,7 @@ module Fog
 
 
         # Get publicly acessible url via http GET.
-        # 
+        #
         #     required attributes: directory, key
         # 
         # @return [String] public url

data/lib/fog/internet_archive/requests/storage/head_object.rb

@@ -13,7 +13,7 @@ module Fog
         # @option options [String] If-None-Match Returns object only if its etag differs from this value, otherwise returns 304 (Not Modified)
         # @option options [Time]   If-Unmodified-Since Returns object only if it has not been modified since this time, otherwise returns 412 (Precodition Failed).
         # @option options [String] Range Range of object to download
-        # 
+        #
         # @return [Excon::Response] response: 
         #   * body [String] Contents of object
         #   * headers [Hash]: 
@@ -58,4 +58,4 @@ module Fog
       end
     end
   end
-end
+end

data/lib/fog/linode/compute.rb

@@ -95,7 +95,7 @@ module Fog
           params[:query] ||= {}
           params[:query].merge!(:api_key => @linode_api_key)
 
-          response = @connection.request(params.merge!({:host => @host}))
+          response = @connection.request(params)
 
           unless response.body.empty?
             response.body = Fog::JSON.decode(response.body)

data/lib/fog/linode/dns.rb

@@ -70,7 +70,7 @@ module Fog
           params[:query] ||= {}
           params[:query].merge!(:api_key => @linode_api_key)
 
-          response = @connection.request(params.merge!({:host => @host}))
+          response = @connection.request(params)
 
           unless response.body.empty?
             response.body = Fog::JSON.decode(response.body)

data/lib/fog/local/models/storage/directories.rb

@@ -10,10 +10,14 @@ module Fog
         model Fog::Storage::Local::Directory
 
         def all
-          data = Dir.entries(service.local_root).select do |entry|
-            entry[0...1] != '.' && ::File.directory?(service.path_to(entry))
-          end.map do |entry|
-            {:key => entry}
+          data = if ::File.directory?(service.local_root)
+            Dir.entries(service.local_root).select do |entry|
+              entry[0...1] != '.' && ::File.directory?(service.path_to(entry))
+            end.map do |entry|
+              {:key => entry}
+            end
+          else
+            []
           end
           load(data)
         end

data/lib/fog/openstack/compute.rb

@@ -342,7 +342,6 @@ module Fog
                 'Accept' => 'application/json',
                 'X-Auth-Token' => @auth_token
               }.merge!(params[:headers] || {}),
-              :host     => @host,
               :path     => "#{@path}/#{@tenant_id}/#{params[:path]}",
               :query    => params[:query]
             }))
@@ -416,7 +415,7 @@ module Fog
 
           @port   = uri.port
           @scheme = uri.scheme
-           
+
           # Not all implementations have identity service in the catalog
           if @openstack_identity_public_endpoint || @openstack_management_url
             @identity_connection = Fog::Connection.new(