fog 1.31.0 → 1.37.0

data/lib/fog/openstack/core.rb

@@ -40,6 +40,8 @@ module Fog
           new_error
         end
       end
+
+      class InterfaceNotImplemented < Fog::Errors::Error; end
     end
 
     service(:compute ,      'Compute')
@@ -53,6 +55,123 @@ module Fog
     service(:baremetal,     'Baremetal')
     service(:planning,      'Planning')
 
+    module Core
+      attr_accessor :auth_token
+      attr_reader :auth_token_expiration
+      attr_reader :current_user
+      attr_reader :current_user_id
+      attr_reader :current_tenant
+      attr_reader :openstack_domain_name
+      attr_reader :openstack_user_domain
+      attr_reader :openstack_project_domain
+      attr_reader :openstack_domain_id
+      attr_reader :openstack_user_domain_id
+      attr_reader :openstack_project_domain_id
+      attr_reader :openstack_identity_prefix
+
+      def initialize_identity options
+        # Create @openstack_* instance variables from all :openstack_* options
+        options.select{|x|x.to_s.start_with? 'openstack'}.each do |openstack_param, value|
+          instance_variable_set "@#{openstack_param}".to_sym, value
+        end
+
+        @auth_token        ||= options[:openstack_auth_token]
+        @openstack_identity_public_endpoint = options[:openstack_identity_endpoint]
+
+        @openstack_auth_uri    = URI.parse(options[:openstack_auth_url])
+        @openstack_must_reauthenticate  = false
+        @openstack_endpoint_type = options[:openstack_endpoint_type] || 'publicURL'
+
+        unless @auth_token
+          missing_credentials = Array.new
+
+          missing_credentials << :openstack_api_key unless @openstack_api_key
+          unless @openstack_username || @openstack_userid
+            missing_credentials << 'openstack_username or openstack_userid'
+          end
+          raise ArgumentError, "Missing required arguments: #{missing_credentials.join(', ')}" unless missing_credentials.empty?
+        end
+
+        @current_user = options[:current_user]
+        @current_user_id = options[:current_user_id]
+        @current_tenant = options[:current_tenant]
+
+      end
+
+      def credentials
+        options =  {
+          :provider                    => 'openstack',
+          :openstack_auth_url          => @openstack_auth_uri.to_s,
+          :openstack_auth_token        => @auth_token,
+          :openstack_identity_endpoint => @openstack_identity_public_endpoint,
+          :current_user                => @current_user,
+          :current_user_id             => @current_user_id,
+          :current_tenant              => @current_tenant,
+          :unscoped_token              => @unscoped_token}
+        openstack_options.merge options
+      end
+
+      def reload
+        @connection.reset
+      end
+
+      private
+
+      def openstack_options
+        options={}
+        # Create a hash of (:openstack_*, value) of all the @openstack_* instance variables
+        self.instance_variables.select{|x|x.to_s.start_with? '@openstack'}.each do |openstack_param|
+          option_name = openstack_param.to_s[1..-1]
+          options[option_name.to_sym] = instance_variable_get openstack_param
+        end
+        options
+      end
+
+      def authenticate
+        if !@openstack_management_url || @openstack_must_reauthenticate
+
+          options = openstack_options
+
+          options[:openstack_auth_token] = @openstack_must_reauthenticate ? nil : @openstack_auth_token
+
+          credentials = Fog::OpenStack.authenticate(options, @connection_options)
+
+          @current_user = credentials[:user]
+          @current_user_id = credentials[:current_user_id]
+          @current_tenant = credentials[:tenant]
+
+          @openstack_must_reauthenticate = false
+          @auth_token = credentials[:token]
+          @openstack_management_url = credentials[:server_management_url]
+          @unscoped_token = credentials[:unscoped_token]
+        else
+          @auth_token = @openstack_auth_token
+        end
+        @openstack_management_uri = URI.parse(@openstack_management_url)
+
+        @host   = @openstack_management_uri.host
+        @path   = @openstack_management_uri.path
+        @path.sub!(/\/$/, '')
+        @port   = @openstack_management_uri.port
+        @scheme = @openstack_management_uri.scheme
+
+        # Not all implementations have identity service in the catalog
+        if @openstack_identity_public_endpoint || @openstack_management_url
+          @identity_connection = Fog::Core::Connection.new(
+              @openstack_identity_public_endpoint || @openstack_management_url,
+              false, @connection_options)
+        end
+
+        true
+      end
+    end
+
+    @@token_cache = {}
+
+    def self.clear_token_cache
+      @@token_cache.clear
+    end
+
     def self.authenticate(options, connection_options = {})
       case options[:openstack_auth_uri].path
       when /v1(\.\d+)?/
@@ -149,9 +268,9 @@ module Fog
         raise Fog::Errors::NotFound, message
       end
 
-      if service['endpoints'].count > 1
-        regions = service["endpoints"].map{ |e| e['region'] }.uniq.join(',')
-        raise Fog::Errors::NotFound.new("Multiple regions available choose one of these '#{regions}'")
+      regions = service["endpoints"].map{ |e| e['region'] }.uniq
+      if regions.count > 1
+        raise Fog::Errors::NotFound.new("Multiple regions available choose one of these '#{regions.join(',')}'")
       end
 
       identity_service = get_service(body, identity_service_type) if identity_service_type
@@ -175,8 +294,7 @@ module Fog
 
     # Keystone Style Auth
     def self.authenticate_v3(options, connection_options = {})
-      uri                   = options[:openstack_auth_uri]
-      tenant_name           = options[:openstack_tenant]
+      uri = options[:openstack_auth_uri]
       project_name          = options[:openstack_project_name]
       service_type          = options[:openstack_service_type]
       service_name          = options[:openstack_service_name]
@@ -184,11 +302,11 @@ module Fog
       endpoint_type         = map_endpoint_type(options[:openstack_endpoint_type] || 'publicURL')
       openstack_region      = options[:openstack_region]
 
-      body, token_headers = retrieve_tokens_v3(options, connection_options)
+      token, body = retrieve_tokens_v3 options, connection_options
 
-      service = get_service(body, service_type, service_name)
+      service = get_service_v3(body, service_type, service_name, openstack_region, options)
 
-      options[:unscoped_token] = token_headers['X-Subject-Token']
+      options[:unscoped_token] = token
 
       unless service
         unless project_name
@@ -196,8 +314,8 @@ module Fog
               :expects => [200],
               :headers => {'Content-Type' => 'application/json',
                            'Accept' => 'application/json',
-                           'X-Auth-Token' => token_headers['X-Subject-Token']},
-              :method  => 'GET'
+                           'X-Auth-Token' => token},
+              :method => 'GET'
           }
           user_id = body['token']['user']['id']
           response = Fog::Core::Connection.new(
@@ -205,34 +323,34 @@ module Fog
 
           projects_body = Fog::JSON.decode(response.body)
           if projects_body['projects'].empty?
-            raise Fog::Errors::NotFound.new('No Project Found')
+            options[:openstack_domain_id] = body['token']['user']['domain']['id']
           else
             options[:openstack_project_name] = projects_body['projects'].first['name']
-            options[:openstack_domain_name] = body['token']['user']['domain']['name']
+            options[:openstack_domain_id] = projects_body['projects'].first['domain_id']
           end
         end
 
-        tenant_name = options[:openstack_project_name]
+        token, body = retrieve_tokens_v3(options, connection_options)
+        service = get_service_v3(body, service_type, service_name, openstack_region)
+      end
 
-        unless tenant_name
-          response = Fog::Core::Connection.new(
-            "#{uri.scheme}://#{uri.host}:#{uri.port}/v3/projects", false, connection_options).request({
-            :expects => [200, 204],
-            :headers => {'Content-Type' => 'application/json',
-                         'Accept' => 'application/json',
-                         'X-Auth-Token' => token_headers['X-Subject-Token']},
-            :method  => 'GET'
-          })
-          body = Fog::JSON.decode(response.body)
-          if body['projects'].empty?
-            raise Fog::Errors::NotFound.new('No Tenant Found')
-          else
-            options[:openstack_tenant] = body['projects'].first['name']
-          end
+      unless service
+        available_services = body['token']['catalog'].map { |service|
+          service['type']
+        }.sort.join ', '
 
-        end
-        body, token_headers = retrieve_tokens_v3(options, connection_options)
-        service = get_service(body, service_type, service_name)
+        available_regions = body['token']['catalog'].map { |service|
+          service['endpoints'].map { |endpoint|
+            endpoint['region']
+          }.uniq
+        }.uniq.sort.join ', '
+
+        missing = service_type.join ', '
+
+        message = "Could not find service #{missing}#{(' in region '+openstack_region) if openstack_region}."+
+            " Have #{available_services}#{(' in regions '+available_regions) if openstack_region}"
+
+        raise Fog::Errors::NotFound, message
       end
 
       service['endpoints'] = service['endpoints'].select do |endpoint|
@@ -243,48 +361,34 @@ module Fog
         raise Fog::Errors::NotFound.new("No endpoints available for region '#{openstack_region}'")
       end if openstack_region
 
-      unless service
-        available = body['token']['catalog'].map { |endpoint|
-          endpoint['type']
-        }.sort.join ', '
-
-        missing = service_type.join ', '
-
-        message = "Could not find service #{missing}.  Have #{available}"
-
-        raise Fog::Errors::NotFound, message
+      regions = service["endpoints"].map { |e| e['region'] }.uniq
+      if regions.count > 1
+        raise Fog::Errors::NotFound.new("Multiple regions available choose one of these '#{regions.join(',')}'")
       end
 
-      admin = false
-      body['token']['roles'].each do |r|
-        admin = true if r["name"] == "admin"
-      end
+      identity_service = get_service_v3(body, identity_service_type, nil, nil, :openstack_endpoint_path_matches => /\/v3/) if identity_service_type
 
-      regions = service["endpoints"].map{ |e| e['region'] }.uniq
-      if regions.count > 1 and not admin
-        raise Fog::Errors::NotFound.new("Multiple regions available choose one of these '#{regions.join(',')}'")
-      end
+      management_url = service['endpoints'].find { |e| e['interface']==endpoint_type }['url']
+      identity_url = identity_service['endpoints'].find { |e| e['interface']=='public' }['url'] if identity_service
 
-      identity_service = get_service(body, identity_service_type) if identity_service_type
-      tenant = body['token']['project']['name'] if body['token']['project']
-      tenant = body['token']['user']['project']['name'] unless body['token']['project']
-      user = body['token']['user']['name']
+      if body['token']['project']
+        tenant = body['token']['project']['name']
+      else
+        tenant = body['token']['user']['project']['name'] if body['token']['user']['project']
+      end
 
-      management_url = service['endpoints'].find{|s| s["interface"][endpoint_type]}["url"]
-      identity_url   = identity_service['endpoints'].find{|s| s["interface"]["public"]}["url"] if identity_service
-      {
-        :user                     => user,
-        :tenant                   => tenant,
-        :identity_public_endpoint => identity_url,
-        :server_management_url    => management_url,
-        :token                    => token_headers['X-Subject-Token'],
-        :expires                  => body['token']['expires_at'],
-        :current_user_id          => body['token']['user']['id'],
-        :unscoped_token           => options[:unscoped_token]
+      return {
+          :user                     => body['token']['user']['name'],
+          :tenant                   => tenant,
+          :identity_public_endpoint => identity_url,
+          :server_management_url    => management_url,
+          :token                    => token,
+          :expires                  => body['token']['expires_at'],
+          :current_user_id          => body['token']['user']['id'],
+          :unscoped_token           => options[:unscoped_token]
       }
     end
 
-
     def self.get_service(body, service_type=[], service_name=nil)
       if not body['access'].nil?
         body['access']['serviceCatalog'].find do |s|
@@ -313,7 +417,7 @@ module Fog
       auth_token  = options[:openstack_auth_token] || options[:unscoped_token]
       uri         = options[:openstack_auth_uri]
 
-      connection = Fog::Core::Connection.new(uri.to_s, false, connection_options)
+      identity_v2_connection = Fog::Core::Connection.new(uri.to_s, false, connection_options)
       request_body = {:auth => Hash.new}
 
       if auth_token
@@ -328,7 +432,7 @@ module Fog
       end
       request_body[:auth][:tenantName] = tenant_name if tenant_name
 
-      response = connection.request({
+      response = identity_v2_connection.request({
         :expects  => [200, 204],
         :headers  => {'Content-Type' => 'application/json'},
         :body     => Fog::JSON.encode(request_body),
@@ -340,87 +444,132 @@ module Fog
     end
 
     def self.retrieve_tokens_v3(options, connection_options = {})
-      api_key     = options[:openstack_api_key].to_s
-      username    = options[:openstack_username].to_s
-      if options[:openstack_tenant_id]
-        tenant_id   = options[:openstack_tenant_id].to_s
-      else
-        tenant_name = options[:openstack_tenant].to_s
-      end
-      auth_token  = options[:openstack_auth_token] || options[:unscoped_token]
-      uri         = options[:openstack_auth_uri]
-      userdomain  = options[:openstack_user_domain] || options[:openstack_domain] || 'Default'
-      project_domain  = options[:openstack_project_domain]  || options[:openstack_domain] || 'Default'
+
+      api_key      = options[:openstack_api_key].to_s
+      username     = options[:openstack_username].to_s
+      userid       = options[:openstack_userid]
+      domain_id    = options[:openstack_domain_id]
+      domain_name  = options[:openstack_domain_name]
+      project_domain = options[:openstack_project_domain]
+      project_domain_id = options[:openstack_project_domain_id]
+      user_domain  = options[:openstack_user_domain]
+      user_domain_id  = options[:openstack_user_domain_id]
+      project_name = options[:openstack_project_name]
+      project_id   = options[:openstack_project_id]
+      auth_token   = options[:openstack_auth_token] || options[:unscoped_token]
+      uri          = options[:openstack_auth_uri]
 
       connection = Fog::Core::Connection.new(uri.to_s, false, connection_options)
-      request_body = {:auth => Hash.new}
+      request_body = {:auth => {}}
+
+      scope = {}
+
+      if project_name || project_id
+        scope[:project] = if project_id.nil? then
+                            if project_domain || project_domain_id
+                              {:name => project_name, :domain => project_domain_id.nil? ? {:name => project_domain} : {:id => project_domain_id}}
+                            else
+                              {:name => project_name, :domain => domain_id.nil? ? {:name => domain_name} : {:id => domain_id}}
+                            end
+                          else
+                            {:id => project_id}
+                          end
+      elsif domain_name || domain_id
+        scope[:domain] = domain_id.nil? ? {:name => domain_name} : {:id => domain_id}
+      else
+        # unscoped token
+      end
 
       if auth_token
-        request_body[:auth][:token] = {
-          :id => auth_token
+        request_body[:auth][:identity] = {
+            :methods => %w{token},
+            :token => {
+                :id => auth_token
+            }
         }
       else
         request_body[:auth][:identity] = {
-          :methods => ["password"],
-          :password => {
-            :user => {
-              :name => username,
-              :password => api_key
+            :methods => %w{password},
+            :password => {
+                :user => {
+                    :password => api_key
+                }
             }
-          }
         }
-        unless tenant_id.nil?
-          request_body[:auth][:scope] = {
-            :project => {
-              :id => tenant_id
-            }
-          }
+
+        if userid
+          request_body[:auth][:identity][:password][:user][:id] = userid
         else
-          request_body[:auth][:scope] = {
-            :project => {
-              :name => tenant_name
-            }
-          }
-        end
-        unless userdomain.to_s.empty?
-          request_body[:auth][:identity][:password][:user][:domain] = {
-                  :name => userdomain
-          }
-        end
-        unless project_domain.to_s.empty?
-          request_body[:auth][:scope][:project][:domain] = {
-                :name => project_domain
-          }
+          if user_domain || user_domain_id
+            request_body[:auth][:identity][:password][:user].merge! :domain => user_domain_id.nil? ? {:name => user_domain} : {:id => user_domain_id}
+          elsif domain_name || domain_id
+            request_body[:auth][:identity][:password][:user].merge! :domain => domain_id.nil? ? {:name => domain_name} : {:id => domain_id}
+          end
+          request_body[:auth][:identity][:password][:user][:name] = username
         end
+
       end
+      request_body[:auth][:scope] = scope unless scope.empty?
 
-      response = connection.request({
-        :expects  => [201],
-        :headers  => {'Content-Type' => 'application/json'},
-        :body     => Fog::JSON.encode(request_body),
-        :method   => 'POST',
-        :path     => (uri.path and not uri.path.empty?) ? uri.path : 'v3'
-      })
+      path     = (uri.path and not uri.path.empty?) ? uri.path : 'v3'
 
-      return Fog::JSON.decode(response.body), response.headers
+      response, expires = @@token_cache[{body: request_body, path: path}]
+
+      unless response && expires > Time.now
+        response = connection.request({   :expects => [201],
+                                          :headers => {'Content-Type' => 'application/json'},
+                                          :body    => Fog::JSON.encode(request_body),
+                                          :method  => 'POST',
+                                          :path    => path
+                                      })
+        @@token_cache[{body: request_body, path: path}] = response, Time.now + 30 # 30-second TTL, enough for most requests
+      end
+
+      [response.headers["X-Subject-Token"], Fog::JSON.decode(response.body)]
+    end
+
+    def self.get_service_v3(hash, service_type=[], service_name=nil, region=nil, options={})
+
+      # Find all services matching any of the types in service_type, filtered by service_name if it's non-nil
+      services = hash['token']['catalog'].find_all do |s|
+        if service_name.nil? or service_name.empty?
+          service_type.include?(s['type'])
+        else
+          service_type.include?(s['type']) and s['name'] == service_name
+        end
+      end if hash['token']['catalog']
+
+      # Filter the found services by region (if specified) and whether the endpoint path matches the given regex (e.g. /\/v3/)
+      services.find do |s|
+        s['endpoints'].any? { |ep| endpoint_region?(ep, region) && endpoint_path_match?(ep, options[:openstack_endpoint_path_matches])}
+      end if services
+
+    end
+
+    def self.endpoint_region?(endpoint, region)
+      region.nil? || endpoint['region'] == region
+    end
+
+    def self.endpoint_path_match?(endpoint, match_regex)
+      match_regex.nil? || URI(endpoint['url']).path =~ match_regex
     end
 
     def self.get_supported_version(supported_versions, uri, auth_token, connection_options = {})
       connection = Fog::Core::Connection.new("#{uri.scheme}://#{uri.host}:#{uri.port}", false, connection_options)
       response = connection.request({
-        :expects => [200, 204, 300],
-        :headers => {'Content-Type' => 'application/json',
-                     'Accept' => 'application/json',
-                     'X-Auth-Token' => auth_token},
-        :method  => 'GET'
-      })
+                                        :expects => [200, 204, 300],
+                                        :headers => {'Content-Type' => 'application/json',
+                                                     'Accept' => 'application/json',
+                                                     'X-Auth-Token' => auth_token},
+                                        :method => 'GET'
+                                    })
 
       body = Fog::JSON.decode(response.body)
       version = nil
       unless body['versions'].empty?
         supported_version = body['versions'].find do |x|
           x["id"].match(supported_versions) &&
-          (x["status"] == "CURRENT" || x["status"] == "SUPPORTED")
+              (x["status"] == "CURRENT" || x["status"] == "SUPPORTED")
         end
         version = supported_version["id"] if supported_version
       end
@@ -432,8 +581,39 @@ module Fog
       version
     end
 
+    def self.get_supported_version_path(supported_versions, uri, auth_token, connection_options = {})
+      # Find a version in the path (e.g. the v1 in /xyz/v1/tenantid/abc) and get the path up until that version (e.g. /xyz))
+      path_components = uri.path.split '/'
+      version_component_index = path_components.index{|comp| comp.match(/v[0-9].?[0-9]?/) }
+      versionless_path = (path_components.take(version_component_index).join '/' if version_component_index) || ''
+      connection = Fog::Core::Connection.new("#{uri.scheme}://#{uri.host}:#{uri.port}#{versionless_path}", false, connection_options)
+      response = connection.request({
+                                        :expects => [200, 204, 300],
+                                        :headers => {'Content-Type' => 'application/json',
+                                                     'Accept' => 'application/json',
+                                                     'X-Auth-Token' => auth_token},
+                                        :method => 'GET'
+                                    })
+
+      body = Fog::JSON.decode(response.body)
+      path = nil
+      unless body['versions'].empty?
+        supported_version = body['versions'].find do |x|
+          x["id"].match(supported_versions) &&
+              (x["status"] == "CURRENT" || x["status"] == "SUPPORTED")
+        end
+        path = URI.parse(supported_version['links'].first['href']).path if supported_version
+      end
+      if path.nil?
+        raise Fog::OpenStack::Errors::ServiceUnavailable.new(
+                  "OpenStack service only supports API versions #{supported_versions.inspect}")
+      end
+
+      path.chomp '/'
+    end
+
     # CGI.escape, but without special treatment on spaces
-    def self.escape(str,extra_exclude_chars = '')
+    def self.escape(str, extra_exclude_chars = '')
       str.gsub(/([^a-zA-Z0-9_.-#{extra_exclude_chars}]+)/) do
         '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
       end

data/lib/fog/openstack/docs/compute.md

@@ -297,7 +297,7 @@ This returns a collection of `Fog::Compute::OpenStack::Flavor` models:
 
 To retrieve individual flavor:
 
-	service.flavor.get 1
+	service.flavors.get 1
 
 This returns a `Fog::Compute::OpenStack::Flavor` instance:
 

data/lib/fog/openstack/identity.rb

@@ -7,63 +7,21 @@ module Fog
       # Fog::Identity::OpenStack.new() will return a Fog::Identity::OpenStack::V2 or a Fog::Identity::OpenStack::V3,
       #  depending on whether the auth URL is for an OpenStack Identity V2 or V3 API endpoint
       def self.new(args = {})
+        @openstack_auth_uri = URI.parse(args[:openstack_auth_url]) if args[:openstack_auth_url]
         if self.inspect == 'Fog::Identity::OpenStack'
-          if args[:openstack_auth_url]
-            @openstack_auth_uri = URI.parse(args[:openstack_auth_url])
-            if @openstack_auth_uri.path =~ /\/v3/
-              service = Fog::Identity::OpenStack::V3.new(args)
-            end
-          end
-          service ||= Fog::Identity::OpenStack::V2.new(args)
+          service = (is_v3? args) ? Fog::Identity::OpenStack::V3.new(args) : Fog::Identity::OpenStack::V2.new(args)
         else
           service = Fog::Service.new(args)
         end
         service
       end
 
-      module Common
-
-        def authenticate
-          if !@openstack_management_url || @openstack_must_reauthenticate
-            options = {
-                :openstack_api_key => @openstack_api_key,
-                :openstack_username => @openstack_username,
-                :openstack_auth_token => @openstack_must_reauthenticate ? nil : @openstack_auth_token,
-                :openstack_auth_uri => @openstack_auth_uri,
-                :openstack_tenant => @openstack_tenant,
-                :openstack_service_type => @openstack_service_type,
-                :openstack_service_name => @openstack_service_name,
-                :openstack_endpoint_type => @openstack_endpoint_type,
-                :openstack_region => @openstack_region
-            }
-
-            credentials = Fog::OpenStack.authenticate(options, @connection_options)
-
-            @current_user = credentials[:user]
-            @current_tenant = credentials[:tenant]
-
-            @openstack_must_reauthenticate = false
-            @auth_token = credentials[:token]
-            @openstack_management_url = credentials[:server_management_url]
-            @openstack_current_user_id = credentials[:current_user_id]
-            @unscoped_token = credentials[:unscoped_token]
-            uri = URI.parse(@openstack_management_url)
-          else
-            @auth_token = @openstack_auth_token
-            uri = URI.parse(@openstack_management_url)
-          end
-
-          @host = uri.host
-          @path = uri.path
-          @path.sub!(/\/$/, '')
-          @port = uri.port
-          @scheme = uri.scheme
-          true
-        end
+      private
 
+      def self.is_v3?(args)
+        @openstack_auth_uri && @openstack_auth_uri.path =~ /\/v3/
       end
-    end
-
 
+    end
   end
 end