aws 2.3.34 → 2.4.0

data/lib/awsbase/benchmark_fix.rb

@@ -24,10 +24,10 @@
 
 
 # A hack because there's a bug in add! in Benchmark::Tms
-module Benchmark  #:nodoc:
+module Benchmark #:nodoc:
   class Tms #:nodoc:
     def add!(&blk)
-      t = Benchmark::measure(&blk)
+      t       = Benchmark::measure(&blk)
       @utime  = utime + t.utime
       @stime  = stime + t.stime
       @cutime = cutime + t.cutime

data/lib/awsbase/errors.rb

@@ -0,0 +1,288 @@
+module Aws
+
+# Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
+# web services raise this type of error.
+# Attribute inherited by RuntimeError:
+#  message    - the text of the error, generally as returned by AWS in its XML response.
+  class AwsError < RuntimeError
+
+    # either an array of errors where each item is itself an array of [code, message]),
+    # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
+    attr_reader :errors
+
+    # Request id (if exists)
+    attr_reader :request_id
+
+    # Response HTTP error code
+    attr_reader :http_code
+
+    # Raw request text data to AWS
+    attr_reader :request_data
+
+    attr_reader :response
+
+    def initialize(errors=nil, http_code=nil, request_id=nil, request_data=nil, response=nil)
+      @errors       = errors
+      @request_id   = request_id
+      @http_code    = http_code
+      @request_data = request_data
+      @response     = response
+      msg           = @errors.is_a?(Array) ? @errors.map { |code, msg| "#{code}: #{msg}" }.join("; ") : @errors.to_s
+      msg += "\nREQUEST=#{@request_data} " unless @request_data.nil?
+      msg += "\nREQUEST ID=#{@request_id} " unless @request_id.nil?
+      super(msg)
+    end
+
+    # Does any of the error messages include the regexp +pattern+?
+    # Used to determine whether to retry request.
+    def include?(pattern)
+      if @errors.is_a?(Array)
+        @errors.each { |code, msg| return true if code =~ pattern }
+      else
+        return true if @errors_str =~ pattern
+      end
+      false
+    end
+
+    # Generic handler for AwsErrors. +aws+ is the Aws::S3, Aws::EC2, or Aws::SQS
+    # object that caused the exception (it must provide last_request and last_response). Supported
+    # boolean options are:
+    # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
+    # * <tt>:puts</tt> do a "puts" of the error
+    # * <tt>:raise</tt> re-raise the error after logging
+    def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
+      # Only log & notify if not user error
+      if !options[:raise] || system_error?($!)
+        error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
+        puts error_text if options[:puts]
+        # Log the error
+        if options[:log]
+          request   = aws.last_request ? aws.last_request.path : '-none-'
+          response  = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
+          @response = response
+          aws.logger.error error_text
+          aws.logger.error "Request was:  #{request}"
+          aws.logger.error "Response was: #{response}"
+        end
+      end
+      raise if options[:raise] # re-raise an exception
+      return nil
+    end
+
+    # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
+    # Used to force logging.
+    def self.system_error?(e)
+      !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
+    end
+
+  end
+
+# Simplified version
+  class AwsError2 < RuntimeError
+    # Request id (if exists)
+    attr_reader :request_id
+
+    # Response HTTP error code
+    attr_reader :http_code
+
+    # Raw request text data to AWS
+    attr_reader :request_data
+
+    attr_reader :response
+
+    attr_reader :errors
+
+    def initialize(http_code=nil, request_id=nil, request_data=nil, response=nil)
+
+      @request_id   = request_id
+      @http_code    = http_code
+      @request_data = request_data
+      @response     = response
+#            puts '@response=' + @response.inspect
+
+      if @response
+        ref = XmlSimple.xml_in(@response, {"ForceArray"=>false})
+#                puts "refxml=" + ref.inspect
+        msg = "#{ref['Error']['Code']}: #{ref['Error']['Message']}"
+      else
+        msg = "#{@http_code}: REQUEST(#{@request_data})"
+      end
+      msg += "\nREQUEST ID=#{@request_id} " unless @request_id.nil?
+      super(msg)
+    end
+
+
+  end
+
+
+  class AWSErrorHandler
+    # 0-100 (%)
+    DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
+
+    @@reiteration_start_delay        = 0.2
+
+    def self.reiteration_start_delay
+      @@reiteration_start_delay
+    end
+
+    def self.reiteration_start_delay=(reiteration_start_delay)
+      @@reiteration_start_delay = reiteration_start_delay
+    end
+
+    @@reiteration_time = 5
+
+    def self.reiteration_time
+      @@reiteration_time
+    end
+
+    def self.reiteration_time=(reiteration_time)
+      @@reiteration_time = reiteration_time
+    end
+
+    @@close_on_error = true
+
+    def self.close_on_error
+      @@close_on_error
+    end
+
+    def self.close_on_error=(close_on_error)
+      @@close_on_error = close_on_error
+    end
+
+    @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
+
+    def self.close_on_4xx_probability
+      @@close_on_4xx_probability
+    end
+
+    def self.close_on_4xx_probability=(close_on_4xx_probability)
+      @@close_on_4xx_probability = close_on_4xx_probability
+    end
+
+    # params:
+    #  :reiteration_time
+    #  :errors_list
+    #  :close_on_error           = true | false
+    #  :close_on_4xx_probability = 1-100
+    def initialize(aws, parser, params={}) #:nodoc:
+      @aws                      = aws # Link to RightEc2 | RightSqs | RightS3 instance
+      @parser                   = parser # parser to parse Amazon response
+      @started_at               = Time.now
+      @stop_at                  = @started_at + (params[:reiteration_time] || @@reiteration_time)
+      @errors_list              = params[:errors_list] || []
+      @reiteration_delay        = @@reiteration_start_delay
+      @retries                  = 0
+      # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
+      @close_on_error           = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
+      @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
+    end
+
+    # Returns false if
+    def check(request, options={}) #:nodoc:
+      result            = false
+      error_found       = false
+      redirect_detected = false
+      error_match       = nil
+      last_errors_text  = ''
+      response          = @aws.last_response
+      # log error
+      request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
+      # is this a redirect?
+      # yes!
+      if response.is_a?(Net::HTTPRedirection)
+        redirect_detected = true
+      else
+        # no, it's an error ...
+        @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
+        @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
+      end
+      # Check response body: if it is an Amazon XML document or not:
+      if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
+        @aws.class.bench_xml.add! do
+          error_parser = RightErrorResponseParser.new
+          error_parser.parse(response)
+          @aws.last_errors     = error_parser.errors
+          @aws.last_request_id = error_parser.requestID
+          last_errors_text     = @aws.last_errors.flatten.join("\n")
+          # on redirect :
+          if redirect_detected
+            location = response['location']
+            # ... log information and ...
+            @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
+            @aws.logger.info("##### New location: #{location} #####")
+            # ... fix the connection data
+            request[:server]   = URI.parse(location).host
+            request[:protocol] = URI.parse(location).scheme
+            request[:port]     = URI.parse(location).port
+          end
+        end
+      else # ... it is not a xml document(probably just a html page?)
+        @aws.last_errors     = [[response.code, "#{response.message} (#{request_text_data})"]]
+        @aws.last_request_id = '-undefined-'
+        last_errors_text     = response.message
+      end
+      # now - check the error
+      unless redirect_detected
+        @errors_list.each do |error_to_find|
+          if last_errors_text[/#{error_to_find}/i]
+            error_found = true
+            error_match = error_to_find
+            @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
+            break
+          end
+        end
+      end
+      # check the time has gone from the first error come
+      if redirect_detected || error_found
+        # Close the connection to the server and recreate a new one.
+        # It may have a chance that one server is a semi-down and reconnection
+        # will help us to connect to the other server
+        if !redirect_detected && @close_on_error
+          @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
+        end
+# puts 'OPTIONS3=' + options.inspect
+        if options[:retries].nil? || @retries < options[:retries]
+          if (Time.now < @stop_at)
+            @retries += 1
+            unless redirect_detected
+              @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
+              sleep @reiteration_delay
+              @reiteration_delay *= 2
+
+              # Always make sure that the fp is set to point to the beginning(?)
+              # of the File/IO. TODO: it assumes that offset is 0, which is bad.
+              if (request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
+                begin
+                  request[:request].body_stream.pos = 0
+                rescue Exception => e
+                  @logger.warn("Retry may fail due to unable to reset the file pointer" +
+                                   " -- #{self.class.name} : #{e.inspect}")
+                end
+              end
+            else
+              @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect.  ####")
+            end
+            result = @aws.request_info(request, @parser, options)
+          else
+            @aws.logger.warn("##### Ooops, time is over... ####")
+          end
+        else
+          @aws.logger.info("##### Stopped retrying because retries=#{@retries} and max=#{options[:retries]}  ####")
+        end
+        # aha, this is unhandled error:
+      elsif @close_on_error
+        # Is this a 5xx error ?
+        if @aws.last_response.code.to_s[/^5\d\d$/]
+          @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
+          # Is this a 4xx error ?
+        elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
+          @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
+                                     "probability: #{@close_on_4xx_probability}%"
+        end
+      end
+      result
+    end
+
+  end
+
+end

data/lib/awsbase/parsers.rb

@@ -0,0 +1,227 @@
+module Aws
+
+#-----------------------------------------------------------------
+
+  class RightSaxParserCallback #:nodoc:
+    def self.include_callback
+      include XML::SaxParser::Callbacks
+    end
+
+    def initialize(right_aws_parser)
+      @right_aws_parser = right_aws_parser
+    end
+
+    def on_start_element(name, attr_hash)
+      @right_aws_parser.tag_start(name, attr_hash)
+    end
+
+    def on_characters(chars)
+      @right_aws_parser.text(chars)
+    end
+
+    def on_end_element(name)
+      @right_aws_parser.tag_end(name)
+    end
+
+    def on_start_document;
+    end
+
+    def on_comment(msg)
+      ;
+    end
+
+    def on_processing_instruction(target, data)
+      ;
+    end
+
+    def on_cdata_block(cdata)
+      ;
+    end
+
+    def on_end_document;
+    end
+  end
+
+  class AwsParser #:nodoc:
+    # default parsing library
+    DEFAULT_XML_LIBRARY  = 'rexml'
+    # a list of supported parsers
+    @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
+
+    @@xml_lib            = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
+    def self.xml_lib
+      @@xml_lib
+    end
+
+    def self.xml_lib=(new_lib_name)
+      @@xml_lib = new_lib_name
+    end
+
+    attr_accessor :result
+    attr_reader :xmlpath
+    attr_accessor :xml_lib
+
+    def initialize(params={})
+      @xmlpath = ''
+      @result  = false
+      @text    = ''
+      @xml_lib = params[:xml_lib] || @@xml_lib
+      @logger  = params[:logger]
+      reset
+    end
+
+    def tag_start(name, attributes)
+      @text = ''
+      tagstart(name, attributes)
+      @xmlpath += @xmlpath.empty? ? name : "/#{name}"
+    end
+
+    def tag_end(name)
+      if @xmlpath =~ /^(.*?)\/?#{name}$/
+        @xmlpath = $1
+      end
+      tagend(name)
+    end
+
+    def text(text)
+      @text += text
+      tagtext(text)
+    end
+
+    # Parser method.
+    # Params:
+    #   xml_text         - xml message text(String) or Net:HTTPxxx instance (response)
+    #   params[:xml_lib] - library name: 'rexml' | 'libxml'
+    def parse(xml_text, params={})
+      # Get response body
+      unless xml_text.is_a?(String)
+        xml_text = xml_text.body.respond_to?(:force_encoding) ? xml_text.body.force_encoding("UTF-8") : xml_text.body
+      end
+
+      @xml_lib = params[:xml_lib] || @xml_lib
+      # check that we had no problems with this library otherwise use default
+      @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
+      # load xml library
+      if @xml_lib=='libxml' && !defined?(XML::SaxParser)
+        begin
+          require 'xml/libxml'
+          # is it new ? - Setup SaxParserCallback
+          if XML::Parser::VERSION >= '0.5.1.0'
+            RightSaxParserCallback.include_callback
+          end
+        rescue LoadError => e
+          @@supported_xml_libs.delete(@xml_lib)
+          @xml_lib = DEFAULT_XML_LIBRARY
+          if @logger
+            @logger.error e.inspect
+            @logger.error e.backtrace
+            @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
+          end
+        end
+      end
+      # Parse the xml text
+      case @xml_lib
+        when 'libxml'
+          xml = XML::SaxParser.string(xml_text)
+          # check libxml-ruby version
+          if XML::Parser::VERSION >= '0.5.1.0'
+            xml.callbacks = RightSaxParserCallback.new(self)
+          else
+            xml.on_start_element { |name, attr_hash| self.tag_start(name, attr_hash) }
+            xml.on_characters { |text| self.text(text) }
+            xml.on_end_element { |name| self.tag_end(name) }
+          end
+          xml.parse
+        else
+          REXML::Document.parse_stream(xml_text, self)
+      end
+    end
+
+    # Parser must have a lots of methods
+    # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
+    # We dont need most of them in AwsParser and method_missing helps us
+    # to skip their definition
+    def method_missing(method, *params)
+      # if the method is one of known - just skip it ...
+      return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
+                 :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
+                 :doctype].include?(method)
+      # ... else - call super to raise an exception
+      super(method, params)
+    end
+
+    # the functions to be overriden by children (if nessesery)
+    def reset;
+    end
+
+    def tagstart(name, attributes)
+      ;
+    end
+
+    def tagend(name)
+      ;
+    end
+
+    def tagtext(text)
+      ;
+    end
+  end
+
+#-----------------------------------------------------------------
+#      PARSERS: Errors
+#-----------------------------------------------------------------
+
+#<Error>
+#  <Code>TemporaryRedirect</Code>
+#  <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
+#  <RequestId>FD8D5026D1C5ABA3</RequestId>
+#  <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
+#  <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
+#  <Bucket>bucket-for-k</Bucket>
+#</Error>
+
+  class RightErrorResponseParser < AwsParser #:nodoc:
+    attr_accessor :errors # array of hashes: error/message
+    attr_accessor :requestID
+#    attr_accessor :endpoint, :host_id, :bucket
+    def parse(response)
+      super
+    end
+
+    def tagend(name)
+      case name
+        when 'RequestID';
+          @requestID = @text
+        when 'Code';
+          @code = @text
+        when 'Message';
+          @message = @text
+#       when 'Endpoint'  ; @endpoint  = @text
+#       when 'HostId'    ; @host_id   = @text
+#       when 'Bucket'    ; @bucket    = @text
+        when 'Error';
+          @errors << [@code, @message]
+      end
+    end
+
+    def reset
+      @errors = []
+    end
+  end
+
+# Dummy parser - does nothing
+# Returns the original params back
+  class RightDummyParser # :nodoc:
+    attr_accessor :result
+
+    def parse(response, params={})
+      @result = [response, params]
+    end
+  end
+
+  class RightHttp2xxParser < AwsParser # :nodoc:
+    def parse(response)
+      @result = response.is_a?(Net::HTTPSuccess)
+    end
+  end
+end