right_cloud_api_base 0.1.0

data/lib/base/parsers/sax.rb

@@ -0,0 +1,200 @@
+#--
+# Copyright (c) 2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'xml/libxml'
+
+module RightScale
+
+  module CloudApi
+    module Parser
+
+      class Sax
+
+        UTF_8_STR = "UTF-8"
+        TEXT_MARK = "@@text"
+
+        def self.parse(input, options = {})
+          # Parse the xml text
+          # http://libxml.rubyforge.org/rdoc/
+          xml_context          = ::XML::Parser::Context.string(input)
+          xml_context.encoding = ::XML::Encoding::UTF_8 if options[:encoding] == UTF_8_STR
+          sax_parser           = ::XML::SaxParser.new(xml_context)
+          sax_parser.callbacks = new(options)
+          sax_parser.parse
+          sax_parser.callbacks.result
+        end
+
+
+        def initialize(options = {})
+          @tag            = {}
+          @path           = []
+          @str_path       = []
+          @options        = options
+          @cached_strings = {}
+        end
+
+
+        def result
+          @cached_strings.clear
+          @tag
+        end
+
+
+        def cache_string(name)
+          unless @cached_strings[name]
+            name = name.freeze
+            @cached_strings[name] = name 
+          end
+          @cached_strings[name]
+        end
+
+
+        # Callbacks
+
+        def on_error(msg)
+          fail msg
+        end
+
+
+        def on_start_element_ns(name, attr_hash, prefix, uri, namespaces)
+          name = cache_string(name)
+          # Push parent tag
+          @path << @tag
+          # Create a new tag
+          if @tag[name]
+            @tag[name] = [ @tag[name] ] unless @tag[name].is_a?(Array)
+            @tag[name] << {}
+            @tag = @tag[name].last
+          else
+            @tag[name] = {}
+            @tag = @tag[name]
+          end
+          # Put attributes
+          current_namespaces = Array(namespaces.keys)
+          current_namespaces << nil if current_namespaces._blank?
+          attr_hash.each do |key, value|
+            current_namespaces.each do |namespace|
+              namespace = namespace ? "#{namespace}:" : ''
+              namespace_and_key = cache_string("@#{namespace}#{key}")
+              @tag[namespace_and_key] = value
+            end
+          end
+          # Put namespaces
+          namespaces.each do |key, value|
+            namespace       = cache_string(key ? "@xmlns:#{key}" : '@xmlns')
+            @tag[namespace] = value
+          end
+        end
+
+
+        def on_characters(chars)
+          # Ignore lines that contains white spaces only
+          return if chars[/\A\s*\z/m]
+          # Put Text
+          if  @options[:encoding] == UTF_8_STR
+            # setting the encoding in context doesn't work(open issue with libxml-ruby).
+            # force encode as a work around.
+            # TODO remove the force encoding when issue in libxml is fixed
+            chars = chars.force_encoding(UTF_8_STR) if chars.respond_to?(:force_encoding)
+          end
+          name = cache_string(TEXT_MARK)
+          (@tag[name] ||= '') << chars
+        end
+
+
+        def on_comment(msg)
+          # Put Comments
+          name = cache_string('@@comment')
+          (@tag[name] ||= '') << msg
+        end
+
+
+        def on_end_element_ns(name, prefix, uri)
+          name = cache_string(name)
+          # Finalize tag's text
+          if @tag.key?(TEXT_MARK) && @tag[TEXT_MARK].empty?
+            # Delete text if it is blank
+            @tag.delete(TEXT_MARK)
+          elsif @tag.keys.count == 0
+            # Set tag value to nil then the tag is blank
+            @tag = nil
+          elsif @tag.keys == [TEXT_MARK]
+            # Set tag value to string if it has no any other data
+            @tag = @tag[TEXT_MARK]
+          end
+          # Make sure we saved the changes
+          if @path.last[name].is_a?(Array)
+            # If it is an Array then update the very last item
+            @path.last[name][-1] = @tag
+          else
+            # Otherwise just replace the tag
+            @path.last[name] = @tag
+          end
+          # Pop parent tag
+          @tag = @path.pop
+        end
+
+
+        def on_start_document
+        end
+
+
+        def on_reference (name)
+        end
+
+
+        def on_processing_instruction(target, data)
+        end
+
+
+        def on_cdata_block(cdata)
+        end
+
+
+        def on_has_internal_subset()
+        end
+
+
+        def on_internal_subset(name, external_id, system_id)
+        end
+
+
+        def on_is_standalone ()
+        end
+
+
+        def on_has_external_subset ()
+        end
+
+
+        def on_external_subset (name, external_id, system_id)
+        end
+
+
+        def on_end_document
+        end
+      end
+      
+    end
+  end
+end

data/lib/base/routines/cache_validator.rb

@@ -0,0 +1,184 @@
+#--
+# Copyright (c) 2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScale
+  module CloudApi
+
+    # The routine processes cache validations (when caching is enabled).
+    #
+    # It takes a response from a cloud and tries to find a pre-defined caching pattern that would
+    # fit to this response and its request. If there is a pattern it extracts a previous response
+    # from the cache and compares it to the current one.
+    #
+    # If both the responses match it raises RightScale::CloudApi::CacheHit exception.
+    #
+    # The main point of the caching - it is performed before parsing a response. So if we get a 10M
+    # XML from Amazon it will take seconds to parse it but if the response did not change there is
+    # need to parse it.
+    #
+    # @example
+    #   ec2 = RightScale::CloudApi::AWS::EC2.new(key, secret_key, :cache => true)
+    #   ec2.DescribeInstances #=> a list of instances
+    #   ec2.DescribeInstances(:options => {:cache => false}) #=> the same list of instances
+    #   ec2.DescribeInstances #=> exception if the response did not change
+    #
+    # The caching setting is per cloud specific ApiManager. For some of them it is on by default so
+    # you need to look at the ApiManager definition.
+    #
+    class CacheValidator < Routine
+
+      class Error < CloudApi::Error
+      end
+
+      # Logs a message.
+      #
+      # @param [String] message Some text.
+      #
+      def log(message)
+        cloud_api_logger.log( "#{message}", :cache_validator)
+      end
+
+      module ClassMethods
+        CACHE_PATTERN_KEYS = [ :verb, :verb!, :path, :path!, :request, :request!, :code, :code!, :response, :response!, :key, :if, :sign ]
+
+        def self.extended(base)
+          unless base.respond_to?(:options) && base.options.is_a?(Hash)
+            raise Error::new("CacheValidator routine assumes class being extended responds to :options and returns a hash") 
+          end
+        end
+
+        # Adds new cache patters.
+        # Patterns are analyzed in order of their definnition. If one pattern hits
+        # the rest are not analyzed.
+        # 
+        # @param [Hash] cache_pattern A hash of pattern keys.
+        # @option cache_pattern [Proc] :key A method that calculates a kache key name.
+        # @option cache_pattern [Proc] :sign A method that modifies the response before calculating md5.
+        #
+        # @see file:lib/base/helper/utils.rb self.pattern_matches? for the other options.
+        # 
+        # @example:
+        #  cache_pattern :verb  => /get|post/,
+        #                :path  => /Action=Describe/,
+        #                :if    => Proc::new{ |o| (o[:params].keys - %w{Action Version AWSAccessKeyId})._blank? },
+        #                :key   => Proc::new{ |o| o[:params]['Action'] },
+        #                :sign  => Proc::new{ |o| o[:response].body.to_s.sub(%r{<requestId>.+?</requestId>}i,'') }
+        #
+        def cache_pattern(cache_pattern)
+          fail Error::new("Pattern should be a Hash and should not be blank") if !cache_pattern.is_a?(Hash) || cache_pattern._blank?
+          fail Error::new("Key field not found in cache pattern definition #{cache_pattern.inspect}") unless cache_pattern.keys.include?(:key)
+          unsupported_keys = cache_pattern.keys - CACHE_PATTERN_KEYS
+          fail Error::new("Unsupported keys #{unsupported_keys.inspect} in cache pattern definition #{cache_pattern.inspect}") unless unsupported_keys._blank?
+          (options[:cache_patterns] ||= []) << cache_pattern
+        end
+      end
+
+      # The main entry point.
+      #
+      def process
+        # Do nothing if caching is off
+        return nil unless data[:options][:cache]
+        # There is nothing to cache if we stream things
+        return nil if data[:response][:instance].is_io?
+
+        cache_patterns = data[:options][:cache_patterns] || []
+        opts = { :relative_path => data[:request][:relative_path],
+                 :request       => data[:request][:instance],
+                 :response      => data[:response][:instance],
+                 :verb          => data[:request][:verb],
+                 :params        => data[:request][:orig_params].dup }
+
+        # Walk through all the cache patterns and find the first that matches
+        cache_patterns.each do |pattern|
+          # Try on the next pattern unless the current one matches.
+          next unless Utils::pattern_matches?(pattern, opts)
+          # Process the matching pattern.
+          log("Request matches to cache pattern: #{pattern.inspect}")
+          # Build a cache key and get a text to be signed
+          cache_key, text_to_sign = build_cache_key(pattern, opts)
+          cache_record = {
+            :timestamp => Time::now.utc,
+            :md5       => Digest::MD5::hexdigest(text_to_sign).to_s,
+            :hits      => 0
+          }
+          log("Processing cache record: #{cache_key} => #{cache_record.inspect}")
+          # Save current cache key for later use (by other Routines)
+          data[:vars][:cache] ||= {}
+          data[:vars][:cache][:key]    = cache_key
+          data[:vars][:cache][:record] = cache_record
+          # Get the cache storage
+          storage = (data[:vars][:system][:storage][:cache] ||= {} )
+          unless storage[cache_key]
+            # Create a new record unless exists.
+            storage[cache_key] = cache_record
+            log("New cache record created")
+          else
+            # If the record is already there but the response changed the replace the old record.
+            unless storage[cache_key][:md5] == cache_record[:md5]
+              storage[cache_key] = cache_record
+              log("Missed. Record is replaced")
+            else
+              # Raise if cache hits.
+              storage[cache_key][:hits] += 1
+              message = "Cache hit: #{cache_key.inspect} has not changed since " +
+                        "#{storage[cache_key][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
+                        "hits: #{storage[cache_key][:hits]}."
+              log(message)
+              fail CacheHit::new("CacheValidator: #{message}")
+            end
+          end
+          break
+        end
+        true
+      end
+
+    private
+
+      # Builds the cached record key and body.
+      #
+      # @param [Hash] pattern The pattern that matched to the current response.
+      # @option options [Hash] opts A set of options that will be passed to :key and :sign procs.
+      #
+      # @return [Array] An array: [key, body]
+      #
+      # @raise [RightScale::CloudApi::CacheValidator::Error] Unless :key proc id set.
+      # @raise [RightScale::CloudApi::CacheValidator::Error] Unless :sign proc returns a valid body.
+      #
+      # @example
+      #  build_cache_key(pattern) #=>
+      #    ["DescribeVolumes", "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<DescribeVolumesResponse ... </DescribeVolumesResponse>"]
+      #
+      def build_cache_key(pattern, opts)
+        key = pattern[:key].is_a?(Proc) ? pattern[:key].call(opts) : pattern[:key]
+        fail Error::new("Cannot build cache key using pattern #{pattern.inspect}") unless key
+
+        body_to_sign = opts[:response].body.to_s if opts[:response].body
+        body_to_sign = pattern[:sign].call(opts) if pattern[:sign]
+        fail Error::new("Could not create body to sign using pattern #{pattern.inspect}") unless body_to_sign
+
+        [key, body_to_sign]
+      end
+
+    end
+  end
+end

data/lib/base/routines/connection_proxies/net_http_persistent_proxy.rb

@@ -0,0 +1,194 @@
+#--
+# Copyright (c) 2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScale
+  module CloudApi
+    class ConnectionProxy
+      
+      class NetHttpPersistentProxy
+        class Error < CloudApi::Error
+        end
+
+
+        # Known timeout errors
+        TIMEOUT_ERRORS = /Timeout|ETIMEDOUT/
+
+        # Other re-triable errors
+        OTHER_ERRORS   = /SocketError|EOFError/
+
+
+        def log(message)
+          @data[:options][:cloud_api_logger].log(message, :connection_proxy, :warn)
+        end
+        
+        # Performs an HTTP request.
+        #
+        # @param [Hash] data The API request +data+ storage.
+        #   See {RightScale::CloudApi::ApiManager.initialize_api_request_options} code for its explanation.
+        #
+        # P.S. Options not supported by Net::HTTP::Persistent:
+        # :connection_retry_count, :connection_retry_delay, :cloud_api_logger
+        #
+        def request(data)
+          require "net/http/persistent"
+
+          @data = data
+          @data[:response] = {}
+          uri = @data[:connection][:uri]
+          
+          # Create a connection
+          connection = Net::HTTP::Persistent.new('right_cloud_api_gem')
+
+          # Create a fake HTTP request
+          fake = @data[:request][:instance]
+          http_request = "Net::HTTP::#{fake.verb._camelize}"._constantize::new(fake.path)
+          if fake.is_io?
+            http_request.body_stream = fake.body
+          else
+            http_request.body = fake.body
+          end
+          fake.headers.each{|header, value| http_request[header] = value }
+          fake.raw = http_request
+
+          # Register a callback to close current connection
+          @data[:callbacks][:close_current_connection] = Proc::new do |reason|
+            connection.shutdown
+            log "Current connection closed: #{reason}"
+          end
+          
+          # Set all required options
+          # P.S. :connection_retry_count, :http_connection_retry_delay are not supported by this proxy
+          #
+          http_request['user-agent'] ||= @data[:options][:connection_user_agent] if @data[:options].has_key?(:connection_user_agent)
+          connection.ca_file      = @data[:options][:connection_ca_file]         if @data[:options].has_key?(:connection_ca_file)
+          connection.read_timeout = @data[:options][:connection_read_timeout]    if @data[:options].has_key?(:connection_read_timeout)
+          connection.open_timeout = @data[:options][:connection_open_timeout]    if @data[:options].has_key?(:connection_open_timeout)
+          connection.cert         = OpenSSL::X509::Certificate.new(@data[:credentials][:cert]) if @data[:credentials].has_key?(:cert)
+          connection.key          = OpenSSL::PKey::RSA.new(@data[:credentials][:key])          if @data[:credentials].has_key?(:key)
+
+          # Make a request
+          begin
+            make_request_with_retries(connection, uri, http_request)
+          rescue => e
+            connection.shutdown
+            fail(ConnectionError, e.message)
+          end
+        end
+
+
+        # Makes request with low level retries.
+        #
+        # Net::HTTP::Persistent does not fully support retries logic that we used to have.
+        # To deal with this we disable Net::HTTP::Persistent's retries and handle them in our code.
+        #
+        # @param [Net::HTTP::Persistent] connection
+        # @param [URI] uri
+        # @param [Net::HTTPRequest] http_request
+        #
+        # @return [void]
+        #
+        def make_request_with_retries(connection, uri, http_request)
+          disable_net_http_persistent_retries(connection)
+          # Initialize retry vars:
+          connection_retry_count = @data[:options][:connection_retry_count] || 3
+          connection_retry_delay = @data[:options][:connection_retry_delay] || 0.5
+          retries_performed      = 0
+          # If block is given - pass there all the chunks of a response and then stop
+          # (don't do any parsing, analysis, etc)
+          block = @data[:vars][:system][:block]
+          begin
+            if block
+              # Response.body is a Net::ReadAdapter instance - it can't be read as a string.
+              # WEB: On its own, Net::HTTP causes response.body to be a Net::ReadAdapter when you make a request with a block
+              # that calls read_body on the response.
+              connection.request(uri, http_request) do |response|
+                # If we are at the point when we have started reading from the remote end
+                # then there is no low level retry is allowed. Otherwise we would need to reset the
+                # IO pointer, etc.
+                connection_retry_count = 0
+                # Set IO response
+                set_http_response(response)
+                response.read_body(&block)
+              end
+            else
+              # Set text response
+              response = connection.request(uri, http_request)
+              set_http_response(response)
+            end
+            nil
+          rescue => e
+            # Fail if it is an unknown error
+            fail(e) if !(e.message[TIMEOUT_ERRORS] || e.message[OTHER_ERRORS])
+            # Fail if it is a Timeout and timeouts are banned
+            fail(e) if e.message[TIMEOUT_ERRORS] && !!@data[:options][:abort_on_timeout]
+            # Fail if there are no retries left...
+            fail(e) if (connection_retry_count -= 1) < 0
+            # ... otherwise sleep a bit and retry.
+            retries_performed += 1
+            log("#{self.class.name}: Performing retry ##{retries_performed} caused by: #{e.class.name}: #{e.message}")
+            sleep(connection_retry_delay) unless connection_retry_delay._blank?
+            connection_retry_delay *= 2
+            retry
+          end
+        end
+
+
+        # Saves HTTP Response into data hash.
+        #
+        # @param [Net::HTTPResponse] response
+        #
+        # @return [void]
+        #
+        def set_http_response(response)
+          @data[:response][:instance] = HTTPResponse.new(
+            response.code,
+            response.body.is_a?(IO) ? nil : response.body,
+            response.to_hash,
+            response
+          )
+          nil
+        end
+
+
+        # Net::HTTP::Persistent believes that it can retry on any GET call what is not true for
+        # Query like API clouds (Amazon, CloudStack, Euca, etc).
+        # The solutions is to monkeypatch  Net::HTTP::Persistent#can_retry? so that is returns
+        # Net::HTTP::Persistent#retry_change_requests.
+        #
+        # @param [Net::HTTP::Persistent] connection
+        #
+        # @return [void]
+        #
+        def disable_net_http_persistent_retries(connection)
+          connection.retry_change_requests = false
+          # Monkey patch this connection instance only.
+          def connection.can_retry?(*args)
+            false
+          end
+          nil
+        end
+
+      end 
+    end
+  end
+end