right_link 5.9.0

data/lib/clouds/metadata_sources/config_drive_metadata_source.rb

@@ -0,0 +1,129 @@
+#
+# Copyright (c) 2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require File.normalize_path(File.expand_path('../file_metadata_source', __FILE__))
+
+module RightScale
+  module MetadataSources
+    class ConfigDriveMetadataSource < FileMetadataSource
+
+      class ConfigDriveError < Exception; end
+
+      DEFAULT_CONFIG_DRIVE_MOUNTPOINT = "/mnt/configdrive"
+
+      attr_accessor :config_drive_label, :config_drive_mountpoint, :config_drive_uuid, :config_drive_filesystem
+
+      def initialize(options)
+        super(options)
+
+        @config_drive_mountpoint        = options[:config_drive_mountpoint] || DEFAULT_CONFIG_DRIVE_MOUNTPOINT
+        @config_drive_uuid              = options[:config_drive_uuid]
+        @config_drive_filesystem        = options[:config_drive_filesystem]
+        @config_drive_label             = options[:config_drive_label]
+
+        if @config_drive_uuid.nil? & @config_drive_label.nil? & @config_drive_filesystem.nil?
+          raise ArgumentError, "at least one of the following is required [options[:config_drive_label], options[:config_drive_filesystem],options[:config_drive_uuid]]"
+        end
+      end
+
+      # Queries for metadata using the given path.
+      #
+      # === Parameters
+      # path(String):: metadata path
+      #
+      # === Return
+      # metadata(String):: query result or empty
+      #
+      # === Raises
+      # QueryFailed:: on any failure to query
+      def query(path)
+        mount_config_drive
+
+        super(path)
+      end
+
+      # Mounts the configuration drive based on the provided parameters
+      #
+      # === Parameters
+      #
+      # === Return
+      # always true
+      #
+      # === Raises
+      # ConfigDriveError:: on failure to find a config drive
+      # SystemCallError:: on failure to create the mountpoint
+      # ArgumentError:: on invalid parameters
+      # VolumeError:: on a failure to mount the device
+      # ParserError:: on failure to parse volume list
+      def mount_config_drive
+        # These two conditions are available on *nix and windows
+        conditions = {}
+        conditions[:label] = @config_drive_label if @config_drive_label
+        conditions[:filesystem] = @config_drive_filesystem if @config_drive_filesystem
+
+        if ::RightScale::Platform.linux? && @config_drive_uuid
+          conditions[:uuid] = @config_drive_uuid
+        end
+
+        timeout   = 60 * 10
+        starttime = Time.now.to_i
+        backoff   = [2,5,10]
+        idx       = -1
+
+        begin
+          device_ary = []
+          if ::RightScale::Platform.windows?
+            # Check once, in case the metadata drive is onlined already
+            device_ary = ::RightScale::Platform.volume_manager.volumes(conditions)
+            ::RightScale::Platform.volume_manager.disks({:status => "Offline"}).each do |disk|
+              ::RightScale::Platform.volume_manager.online_disk(disk[:index])
+              # Per the interwebs, you cannot run disk part commands back to back without a wait.
+              sleep(15)
+              device_ary = ::RightScale::Platform.volume_manager.volumes(conditions)
+              break if device_ary.length > 0
+              ::RightScale::Platform.volume_manager.offline_disk(disk[:index])
+            end unless device_ary.length > 0
+          else
+            device_ary = ::RightScale::Platform.volume_manager.volumes(conditions)
+          end
+          idx = idx + 1 unless idx == 2
+          break if (Time.now.to_i - starttime) > timeout || device_ary.length > 0
+          @logger.warn("Configuration drive device was not found.  Trying again in #{backoff[idx % backoff.length]} seconds.")
+          Kernel.sleep(backoff[idx])
+        end while device_ary.length == 0
+
+        # REVIEW: Raise or log and exit?
+        raise ConfigDriveError.new("Configuration drive device found. Conditions: #{conditions.inspect}") if device_ary.length == 0
+
+        FileUtils.mkdir_p(@config_drive_mountpoint) unless File.directory? @config_drive_mountpoint
+
+        if ::RightScale::Platform.linux?
+          ::RightScale::Platform.volume_manager.mount_volume(device_ary[0], @config_drive_mountpoint)
+        elsif ::RightScale::Platform.windows?
+          ::RightScale::Platform.volume_manager.assign_device(device_ary[0][:index], @config_drive_mountpoint, {:idempotent => true, :clear_readonly => false, :remove_all => true})
+        end
+        return true
+      end
+
+    end # ConfigDriveMetadataSource
+  end # MetadataSources
+end # RightScale

data/lib/clouds/metadata_sources/file_metadata_source.rb

@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  module MetadataSources
+
+    # Provides metadata by reading a dictionary file on disk.
+    class FileMetadataSource < MetadataSource
+
+      attr_accessor :cloud_metadata_source_file_path, :user_metadata_source_file_path
+
+      def initialize(options)
+        super(options)
+        raise ArgumentError.new("options[:cloud_metadata_root_path] is required") unless @cloud_metadata_root_path = options[:cloud_metadata_root_path]
+        raise ArgumentError.new("options[:user_metadata_root_path] is required") unless @user_metadata_root_path = options[:user_metadata_root_path]
+
+        @cloud_metadata_source_file_path = options[:cloud_metadata_source_file_path]
+        @user_metadata_source_file_path = options[:user_metadata_source_file_path]
+      end
+
+      # Queries for metadata using the given path.
+      #
+      # === Parameters
+      # path(String):: metadata path
+      #
+      # === Return
+      # metadata(String):: query result or empty
+      #
+      # === Raises
+      # QueryFailed:: on any failure to query
+      def query(path)
+        result = ""
+        if path == @cloud_metadata_root_path
+          result = File.read(@cloud_metadata_source_file_path) if @cloud_metadata_source_file_path
+        elsif path == @user_metadata_root_path
+          result = File.read(@user_metadata_source_file_path) if @user_metadata_source_file_path
+        else
+          raise QueryFailed.new("Unknown path: #{path}")
+        end
+        result
+      rescue Exception => e
+        raise QueryFailed.new(e.message)
+      end
+
+      # Nothing to do.
+      def finish
+        true
+      end
+
+    end  # FileMetadataSource
+
+  end  # MetadataSources
+
+end  # RightScale

data/lib/clouds/metadata_sources/http_metadata_source.rb

@@ -0,0 +1,277 @@
+#
+# Copyright (c) 2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'right_http_connection'
+require 'uri'
+
+module RightScale
+
+  module MetadataSources
+
+    # Provides metadata via a single http connection which is kept alive in the
+    # case of a tree of metadata.
+    class HttpMetadataSource < MetadataSource
+
+      attr_accessor :host, :port
+
+      def initialize(options)
+        super(options)
+        raise ArgumentError, "options[:hosts] is required" unless @hosts = options[:hosts]
+        @host, @port = self.class.select_metadata_server(@hosts)
+        @connections = {}
+      end
+
+      # Queries for metadata using the given path.
+      #
+      # === Parameters
+      # path(String):: metadata path
+      #
+      # === Return
+      # metadata(String):: query result or empty
+      #
+      # === Raises
+      # QueryFailed:: on any failure to query
+      def query(path)
+        http_path = "http://#{@host}:#{@port}/#{path}"
+        attempts = 1
+        while true
+          begin
+            logger.debug("Querying \"#{http_path}\"...")
+            # get.
+            result = http_get(http_path)
+            if result
+              logger.debug("Successfully retrieved from: \"#{http_path}\"  Result: #{path} = #{result}")
+              return result
+            end
+
+            # retry, if allowed.
+            if snooze(attempts)
+              logger.info("Retrying \"#{http_path}\"...")
+            else
+              logger.error("Could not retrieve metadata from \"#{http_path}\"; retry limit exceeded.")
+              return ""
+            end
+          rescue Exception => e
+            logger.error("#{Time.now().to_s()}: Exception occurred while attempting to retrieve metadata from \"#{http_path}\"; Exception:#{e.message}")
+            finish # Reset the connections.
+            unless snooze(attempts)
+              logger.error("Trace:#{e.backtrace.join("\n")}")
+              return ""
+            end
+          end
+          attempts += 1
+        end
+      end
+
+      # Closes any http connections left open after fetching metadata.
+      def finish
+        @connections.each_value do |connection|
+          begin
+            connection.finish
+          rescue Exception => e
+            logger.error("Failed to close metadata http connection: #{e.backtrace.join("\n")}")
+          end
+        end
+        @connections = {}
+      end
+
+      # selects a host/port by attempting to dig one or more well-known DNS
+      # names or IP addresses.
+      #
+      # === Parameters
+      # hosts(Array):: array of hosts in the form [{:host => <dns name or ip address>, :port => <port or nil>}+]
+      #
+      # === Return
+      # result(Array):: pair in form of [<selected host ip address>, <selected port>]
+      def self.select_metadata_server(hosts)
+        # note that .each works for strings (by newline) and arrays.
+        last_exception = nil
+        hosts.each do |host_data|
+          begin
+            # resolve metadata server hostname.
+            addrs = Socket.gethostbyname(host_data[:host])[3..-1]
+
+            # select only IPv4 addresses
+            addrs = addrs.select { |x| x.length == 4 }
+
+            # choose a random IPv4 address
+            raw_ip = addrs[rand(addrs.size)]
+
+            # transform binary IP address into string representation
+            ip = []
+            raw_ip.each_byte { |x| ip << x.to_s }
+            return [ip.join('.'), host_data[:port] || 80]
+          rescue Exception => e
+            last_exception = e
+          end
+        end
+        raise last_exception
+      end
+
+      protected
+
+      # Some time definitions
+      SECOND = 1
+      MINUTE = 60 * SECOND
+      HOUR = 60 * MINUTE
+
+      # Time to yield before retries
+      RETRY_DELAY = 2 * SECOND
+      RETRY_DELAY_FACTOR = 1
+
+      # Total amount of time to retry
+      RETRY_MAX_TOTAL_TIME = 1 * HOUR
+
+      RETRY_MAX_ATTEMPTS = RETRY_MAX_TOTAL_TIME / RETRY_DELAY
+
+      # ensures we are not infinitely redirected.
+      MAX_REDIRECT_HISTORY = 16
+
+      # Simple sleep algorithm.  Returns true if processing
+      # should continue or false if the maximum number of attempts has
+      # been exceeded.
+      #
+      # === Parameters
+      # attempts(int):: number of attempts
+      #
+      # === Return
+      # result(Boolean):: true to continue, false to give up
+      def snooze(attempts)
+        if attempts >= RETRY_MAX_ATTEMPTS
+          logger.debug("Exceeded retry limit of #{RETRY_MAX_ATTEMPTS}.")
+          false
+        else
+          sleep(RETRY_DELAY * RETRY_DELAY_FACTOR)
+          true
+        end
+      end
+
+      # Performs an HTTP get request with built-in retries and redirection based
+      # on HTTP responses.
+      #
+      # === Parameters
+      # attempts(int):: number of attempts
+      #
+      # === Return
+      # result(String):: body of response or nil
+      def http_get(path, keep_alive = true)
+        uri = safe_parse_http_uri(path)
+        history = []
+        loop do
+          logger.debug("http_get(#{uri})")
+
+          # keep history of live connections for more efficient redirection.
+          host = uri.host
+          connection = @connections[host] ||= Rightscale::HttpConnection.new(:logger => logger, :exception => QueryFailed)
+
+          # prepare request. ensure path not empty due to Net::HTTP limitation.
+          #
+          # note that the default for Net::HTTP is to close the connection after
+          # each request (contrary to expected conventions). we must be explicit
+          # about keep-alive if we want that behavior.
+          request = Net::HTTP::Get.new(uri.path)
+          request['Connection'] = keep_alive ? 'keep-alive' : 'close'
+
+          # get.
+          response = connection.request(:protocol => uri.scheme, :server => uri.host, :port => uri.port, :request => request)
+          return response.body if response.kind_of?(Net::HTTPSuccess)
+          if response.kind_of?(Net::HTTPServerError)
+            logger.debug("Request failed but can retry; #{response.class.name}")
+            return nil
+          elsif response.kind_of?(Net::HTTPRedirection)
+            # keep history of redirects.
+            history << uri.to_s
+            location = response['Location']
+            uri = safe_parse_http_uri(location)
+            if uri.absolute?
+              if history.include?(uri.to_s)
+                logger.error("Circular redirection to #{location.inspect} detected; giving up")
+                return nil
+              elsif history.size >= MAX_REDIRECT_HISTORY
+                logger.error("Unbounded redirection to #{location.inspect} detected; giving up")
+                return nil
+              else
+                # redirect and continue in loop.
+                logger.debug("Request redirected to #{location.inspect}: #{response.class.name}")
+              end
+            else
+              # can't redirect without an absolute location.
+              logger.error("Unable to redirect to metadata server location #{location.inspect}: #{response.class.name}")
+              return nil
+            end
+          else
+            # not retryable.
+            #
+            # log an error and return empty string.
+            #
+            # note that the EC2 metadata server is known to give malformed
+            # responses on rare occasions, but the right_http_connection will
+            # consider these to be 'bananas' and retry automatically (up to a
+            # pre-defined limit).
+            logger.error("Request for metadata failed: #{response.class.name}")
+            return ""
+          end
+        end
+      end
+
+      # Handles some cases which raise exceptions in the URI class.
+      #
+      # === Parameters
+      # path(String):: URI to parse
+      #
+      # === Return
+      # uri(URI):: parsed URI
+      #
+      # === Raise
+      # URI::InvalidURIError:: on invalid URI
+      def safe_parse_http_uri(path)
+        raise ArgumentError.new("URI path cannot be empty") if path.to_s.empty?
+        begin
+          uri = URI.parse(path)
+        rescue URI::InvalidURIError => e
+          # URI raises an exception for paths like "<IP>:<port>"
+          # (e.g. "127.0.0.1:123") unless they also have scheme (e.g. http)
+          # prefix.
+          raise e if path.start_with?("http://") || path.start_with?("https://")
+          uri = URI.parse("http://" + path)
+          uri = URI.parse("https://" + path) if uri.port == 443
+          path = uri.to_s
+        end
+
+        # supply any missing default values to make URI as complete as possible.
+        if uri.scheme.nil? || uri.host.nil?
+          scheme = (uri.port == 443) ? 'https' : 'http'
+          uri = URI.parse("#{scheme}://#{path}")
+          path = uri.to_s
+        end
+        if uri.path.to_s.empty?
+          uri = URI.parse("#{path}/")
+          path = uri.to_s
+        end
+        return uri
+      end
+
+    end  # HttpMetadataSource
+
+  end  # MetadataSources
+
+end  # RightScale

data/lib/clouds/metadata_sources/selective_metadata_source.rb

@@ -0,0 +1,122 @@
+#
+# Copyright (c) 2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  module MetadataSources
+
+    # Provides metadata by attempting to get metadata from one or more listed
+    # sources; first responding successfully wins.
+    class SelectiveMetadataSource < MetadataSource
+
+      attr_accessor :metadata_sources
+
+      def initialize(options)
+        raise ArgumentError, "options[:metadata_sources] is required" unless metadata_source_types = options[:metadata_source_types]
+        raise ArgumentError, "options[:cloud] is required" unless @cloud = options[:cloud]
+        @select_metadata_override = options[:select_metadata_override]
+
+        # keep types but create selective sources on demand in case not all are used.
+        @metadata_sources = []
+        metadata_source_types.each do |metadata_source_type|
+          @metadata_sources << { :type => metadata_source_type, :source => nil }
+        end
+      end
+
+      # Queries for metadata using the given path.
+      #
+      # === Parameters
+      # path(String):: metadata path
+      #
+      # === Return
+      # metadata(String):: query result
+      #
+      # === Raises
+      # QueryFailed:: on any failure to query
+      def query(path)
+        merged_metadata = ""
+        last_query_failed = nil
+        @metadata_sources.each do |metadata_source|
+          type = metadata_source[:type]
+          unless source = metadata_source[:source]
+            # note that sources are special in that they ignore cloud vs. user
+            # specialization unlike other dependency types.
+            kind = :cloud_metadata
+            source = @cloud.create_dependency_type(kind, :metadata_source, type)
+            metadata_source[:source] = source
+          end
+          begin
+            query_result = source.query(path)
+            selected = select_metadata(path, type, query_result, merged_metadata)
+            merged_metadata = selected[:merged_metadata]
+            last_query_failed = nil  # reset last failed query
+            break unless selected[:query_next_metadata]
+          rescue QueryFailed => e
+            # temporarily ignore failed query in case next source query succeeds
+            last_query_failed = e
+          end
+        end
+        raise last_query_failed if last_query_failed
+        return merged_metadata
+      rescue Exception => e
+        raise QueryFailed.new(e.message)
+      end
+
+      # Selects metadata by determining if the metadata condition has been
+      # satisfied. Supports merging of metadata (potentially in different
+      # formats) from different sources.
+      #
+      # === Parameters
+      # path(String):: metadata path
+      # metadata_source_type(String):: metadata source type
+      # query_result(String):: raw metadata from query
+      # previous_metadata(String):: previously merged metadata or empty
+      #
+      # === Returns
+      # result[:query_next_metadata](Boolean):: when true indicates that the next metadata source should also be queried.
+      # result[:merged_metadata](String):: result of merging any previous metadata with current query result.
+      def select_metadata(path, metadata_source_type, query_result, previous_metadata)
+        return @select_metadata_override.call(self, path, metadata_source_type, query_result, previous_metadata) if @select_metadata_override
+        return {:query_next_metadata => query_result.strip.empty?, :merged_metadata => query_result}
+      end
+
+      # Attempts to finish all child metadata sources.
+      def finish
+        last_exception = nil
+        @metadata_sources.each do |metadata_source|
+          begin
+            source = metadata_source[:source]
+            source.finish if source
+          rescue Exception => e
+            last_exception = e
+          ensure
+            metadata_source[:source] = nil
+          end
+        end
+        raise last_exception if last_exception
+      end
+
+    end  # SelectiveMetadataSource
+
+  end  # MetadataSources
+
+end  # RightScale