ec2_amitools 1.0.2

data/lib/ec2/common/headers.rb

@@ -0,0 +1,95 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+module EC2
+  module Common
+    class Headers
+      MANDATORY = ['content-md5', 'content-type', 'date'] # Order matters.
+      X_AMZ_PREFIX = 'x-amz'
+      X_AMZ_SECURITY_TOKEN = 'x-amz-security-token'
+      
+      def initialize(verb)
+        raise ArgumentError.new('invalid verb') if verb.to_s.empty?
+        @headers = {}
+        @verb = verb
+      end
+      
+      #---------------------------------------------------------------------
+      # Add a Header key-value pair.
+      def add(name, value)
+        raise ArgumentError.new("name '#{name.inspect}' must be a String") unless name.is_a? String
+        raise ArgumentError.new("value '#{value.inspect}' (#{name}) must be a String") unless value.is_a? String
+        @headers[name.downcase.strip] = value.strip
+      end
+
+      #-----------------------------------------------------------------------
+      # Sign the headers using HMAC SHA1.
+      def sign(creds, aws_secret_access_key, url, bucket)
+        aws_access_key_id = creds
+        delegation_token = nil
+        if (creds.is_a?(Hash))
+            aws_access_key_id = creds['aws_access_key_id']
+            aws_secret_access_key = creds['aws_secret_access_key']
+            delegation_token = creds['aws_delegation_token']
+        end
+
+        @headers['date'] = Time.now.httpdate          # add HTTP Date header
+        @headers['content-type'] ||= 'application/x-www-form-urlencoded'
+
+        # Build the data to sign.
+        data = @verb + "\n"
+
+        # Deal with mandatory headers first:
+        MANDATORY.each do |name|
+          data += @headers.fetch(name, "") + "\n"
+        end
+ 
+        unless delegation_token.nil?
+            @headers[X_AMZ_SECURITY_TOKEN] = delegation_token
+        end
+        # Add mandatory headers and those that start with the x-amz prefix.
+        @headers.sort.each do |name, value|
+          # Headers that start with x-amz must have both their name and value 
+          # added.
+          if name =~ /^#{X_AMZ_PREFIX}/
+            data += name + ":" + value +"\n"
+          end
+        end
+ 
+        uri = URI.parse(url)
+        # Ignore everything in the URL after the question mark unless, by the
+        # S3 protocol, it signifies an acl, torrent, logging or location parameter
+        if uri.host.start_with? bucket
+          data << "/#{bucket}"
+        end
+        data << if uri.path.empty?
+          "/"
+        else
+          uri.path
+        end
+        ['acl', 'logging', 'torrent', 'location'].each do |item|
+          regex = Regexp.new("[&?]#{item}($|&|=)")
+          data << '?' + item if regex.match(url)
+        end
+
+        # Sign headers and then put signature back into headers.
+        signature = Base64.encode64(Crypto::hmac_sha1(aws_secret_access_key, data))
+        signature.chomp!
+        @headers['Authorization'] = "AWS #{aws_access_key_id}:#{signature}"
+      end
+
+      #-----------------------------------------------------------------------
+      # Return the headers as a map from header name to header value.
+      def get
+        return @headers.clone
+      end
+    end
+  end
+end

data/lib/ec2/common/headersv4.rb

@@ -0,0 +1,173 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# A class to contain headers and sign them with signature version 4
+# This code is a shameless copy of our SDK's signature version 4 code base
+#
+
+require 'cgi'
+require 'uri'
+require 'time'
+require 'base64'
+require 'tempfile'
+require 'digest/md5'
+require 'openssl'
+require 'digest'
+require 'digest/sha2'
+
+module EC2
+  module Common
+    class HeadersV4
+      # Create an HttpHeaderV4,
+      # values = {
+      #   :host -> http host
+      #   :hexdigest_body -> hexdigest of the http request's body
+      #   :region -> region of the endpoint
+      #   :service -> the service that should recieve this request
+      #   :http_method -> http method
+      #   :path -> URI
+      #   :querystring -> Everything after ? in URI
+      #   :access_key_id -> access key
+      #   :secret_access_key -> secret access kry
+      #   [optional] :fixed_datetime -> Fix the datetime using DateTime object, do not use Time.now
+      # }
+      # For more info see:
+      # http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
+      def initialize values, headers={}
+        @headers = headers
+        @host = values[:host]
+        @hexdigest_body = values[:hexdigest_body]
+        @region = values[:region]
+        @service = values[:service]
+        @http_method = values[:http_method]
+        @path = values[:path]
+        @querystring = values[:querystring]
+        @access_key_id = values[:access_key_id]
+        @secret_access_key = values[:secret_access_key]
+        @fixed_datetime = values[:fixed_datetime]
+      end
+
+      def add_authorization!
+        datetime = get_datetime
+        @headers['host'] = @host
+        @headers['x-amz-date'] = datetime
+        @headers['x-amz-content-sha256'] ||= @hexdigest_body || EC2::Common::HeadersV4::hexdigest('')
+        @headers['authorization'] = authorization(datetime)
+        @headers
+      end
+
+      def authorization datetime
+        parts = []
+        parts << "AWS4-HMAC-SHA256 Credential=#{credential(datetime)}"
+        parts << "SignedHeaders=#{signed_headers}"
+        parts << "Signature=#{signature(datetime)}"
+        parts.join(', ')
+      end
+
+      def signature datetime
+        k_secret = @secret_access_key
+        k_date = hmac("AWS4" + k_secret, datetime[0,8])
+        k_region = hmac(k_date, @region)
+        k_service = hmac(k_region, @service)
+        k_credentials = hmac(k_service, 'aws4_request')
+        hexhmac(k_credentials, string_to_sign(datetime))
+      end
+
+      def string_to_sign datetime
+        parts = []
+        parts << 'AWS4-HMAC-SHA256'
+        parts << datetime
+        parts << credential_string(datetime)
+        parts << EC2::Common::HeadersV4::hexdigest(canonical_request)
+        parts.join("\n")
+      end
+
+      def credential datetime
+        "#{@access_key_id}/#{credential_string(datetime)}"
+      end
+
+      def credential_string datetime
+        parts = []
+        parts << datetime[0,8]
+        parts << @region
+        parts << @service
+        parts << 'aws4_request'
+        parts.join("/")
+      end
+
+      def canonical_request
+        parts = []
+        parts << @http_method
+        parts << CGI::unescape(@path)
+        parts << canonical_querystring
+        parts << canonical_headers + "\n"
+        parts << signed_headers
+        parts << @headers['x-amz-content-sha256']
+        parts.join("\n")
+      end
+
+      def signed_headers
+        to_sign = @headers.keys.map{|k| k.to_s.downcase}
+        to_sign.delete('authorization')
+        to_sign.sort.join(";")
+      end
+
+      def canonical_querystring
+        CGI::parse(@querystring).sort_by{|k,v| CGI::escape(k)}.map do |v|
+          value = v[1][0] || ""
+          "#{CGI::escape(v[0])}=#{CGI::escape(value)}"
+        end.join('&')
+      end
+
+      def canonical_headers
+        headers = []
+        @headers.each_pair do |k,v|
+          k_lower = k.downcase
+          headers << [k_lower,v] unless k_lower == 'authorization'
+        end
+        headers = headers.sort_by{|k,v| k}
+        headers.map{|k,v| "#{k}:#{canonical_header_values(v)}"}.join("\n")
+      end
+
+      def canonical_header_values values
+        values = [values] unless values.is_a?(Array)
+        values.map{|v|v.to_s}.join(',').gsub(/\s+/, ' ').strip
+      end
+
+      def hmac key, value
+        OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new('sha256'), key, value)
+      end
+
+      def hexhmac key, value
+        OpenSSL::HMAC.hexdigest(OpenSSL::Digest::Digest.new('sha256'), key, value)
+      end
+
+      def get_datetime
+        return @fixed_datetime.strftime("%Y%m%dT%H%M%SZ") if @fixed_datetime != nil
+        Time.now.utc.strftime("%Y%m%dT%H%M%SZ")
+      end
+
+      # Returns a SHA256 hexdigest of value.
+      # Should be used to hexdigest body of http request.
+      def self.hexdigest value, chunk_size = 1024 * 1024
+        digest = Digest::SHA256.new
+        if value.respond_to?(:read)
+          chunk = nil
+          digest.update(chunk) while chunk = value.read(chunk_size)
+          value.rewind
+        else
+          digest.update(value)
+        end
+        digest.hexdigest
+      end
+    end
+  end
+end

data/lib/ec2/common/http.rb

@@ -0,0 +1,333 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+# ---------------------------------------------------------------------------
+# Module that provides http functionality.
+# ---------------------------------------------------------------------------
+require 'uri'
+require 'set'
+require 'time'
+require 'base64'
+require 'tmpdir'
+require 'tempfile'
+require 'fileutils'
+
+require 'ec2/common/constants'
+require 'ec2/common/curl'
+require 'ec2/common/signature'
+require 'ec2/common/headers'
+require 'ec2/amitools/crypto'
+
+module EC2
+  module Common
+    module HTTP
+      DEFAULT_SIGV = EC2::Common::SIGV4
+      DEFAULT_REGION = 'us-east-1'
+
+      class Response < EC2::Common::Curl::Response
+        attr_reader :body
+        def initialize(code, type, body=nil)
+          super code, type
+          @body = body
+        end
+      end
+    
+      #-----------------------------------------------------------------------      
+      # Errors.
+      class Error < RuntimeError
+        attr_reader :code
+        def initialize(msg, code = nil)          
+          super(msg)
+          @code = code || 1
+        end
+        class PathInvalid < Error; end
+        class Write < Error;       end
+        class BadDigest < Error
+          def initialize(file, expected, obtained)
+            super("Digest for file '#{file}' #{obtained} differs from expected digest #{digest}")
+          end
+        end
+        class Transfer < Error;    end
+        class Retrieve < Transfer; end
+        class Redirect < Error
+          attr_accessor :code, :endpoint
+          def initialize(code, endpoint)
+            super("Redirected (#{code}) to new endpoint: #{endpoint}")
+            @code = code
+            @endpoint = endpoint
+          end
+        end
+      end
+      
+      
+      #-----------------------------------------------------------------------      
+      # Invoke curl with arguments given and process results of output file
+      def HTTP::invoke(url, arguments, outfile, debug=false)
+        begin
+          raise ArgumentError.new(outfile) unless File.exists? outfile
+          result = EC2::Common::Curl.execute("#{arguments.join(' ')} '#{url}'", debug)
+          if result.success?
+            if result.response.success?
+              return result.response
+            else
+              synopsis= 'Server.Error(' + result.response.code.to_s + '): '
+              message = result.stderr + ' '
+              if result.response.type == 'application/xml'                
+                require 'rexml/document'
+                doc = REXML::Document.new(IO.read(outfile))
+                if doc.root
+                  if result.response.redirect?
+                    endpoint = REXML::XPath.first(doc, '/Error/Endpoint').text
+                    raise Error::Redirect.new(result.response.code, endpoint)
+                  end
+                  content = REXML::XPath.first(doc, '/Error/Code')
+                  unless content.nil? or content.text.empty?
+                    synopsis= 'Server.'+ content.text + '(' + 
+                      result.response.code.to_s + '): '
+                  end
+                  content = REXML::XPath.first(doc, '/Error/Message')
+                  message = content.text unless content.nil?
+                end
+              else
+                if result.response.type =~ /text/
+                  message << IO.read(outfile)
+                end
+              end
+              raise Error::Transfer.new(synopsis + message, result.response.code)
+            end
+          else
+            synopsis= 'Curl.Error(' + result.status.to_s + '): '
+            message = result.stderr.split("\n").map { |line|            
+              if (m = /^curl:\s+(?:\(\d{1,2}\)\s+)*(.*)$/.match(line))
+                (m.captures[0] == "try 'curl --help' for more information") ? 
+                  '' : m.captures[0]
+              else
+                line.strip
+              end
+            }.join("\n")
+            output = result.stdout.chomp
+            if debug and not output.empty?
+              message << "\nCurl.Output: " + output.gsub("\n", "\nCurl.Output: ")
+            end
+            raise Error::Transfer.new(synopsis + message.strip + '.', result.status)
+          end
+        rescue EC2::Common::Curl::Error => e
+          raise Error::Transfer.new(e.message, e.code)
+        end
+      end
+      
+      #-----------------------------------------------------------------------      
+      # Delete the file at the specified url.
+      def HTTP::delete(url, bucket, options={}, user=nil, pass=nil, debug=false, sigv=DEFAULT_SIGV, region=DEFAULT_REGION)
+        raise ArgumentError.new('Bad options in HTTP::delete') unless options.is_a? Hash
+        begin
+          output = Tempfile.new('ec2-delete-response')
+          output.close
+          
+          arguments = ['-X DELETE']
+          arguments << get_arguments(url, bucket, 'DELETE', options, user, pass, sigv, region)
+          arguments << '-o ' + output.path
+          
+          response = HTTP::invoke(url, arguments, output.path, debug)
+          return EC2::Common::HTTP::Response.new(response.code, response.type)
+        ensure
+          output.close(true)
+          GC.start
+        end
+      end
+
+
+      #-----------------------------------------------------------------------      
+      # Put the file at the specified path to the specified url. The content of
+      # the options hash will be passed as HTTP headers. If the username and
+      # password options are specified, then the headers will be signed.
+      def HTTP::put(url, bucket, path, options={}, user=nil, pass=nil, debug=false, sigv=DEFAULT_SIGV, region=DEFAULT_REGION)
+        path ||= "/dev/null"
+        raise Error::PathInvalid.new(path) unless path and File::exist?(path)
+        raise ArgumentError.new('Bad options in HTTP::put') unless options.is_a? Hash
+        
+        begin
+          output = Tempfile.new('ec2-put-response')
+          output.close
+
+          arguments = [get_arguments(url, bucket, 'PUT', options, user, pass, sigv, region, open(path))]
+          arguments << '-T ' + path
+          arguments << '-o ' + output.path
+
+          response = HTTP::invoke(url, arguments, output.path, debug)
+          return EC2::Common::HTTP::Response.new(response.code, response.type)
+        ensure
+          output.close(true)
+          GC.start
+        end
+      end
+  
+      #-----------------------------------------------------------------------      
+      # Create a bucket using the specified url and constraints (either a file 
+      # with location string, or the string itself). The content of the 
+      # options hash will be passed as HTTP headers. If the username and 
+      # password options are specified, then the headers will be signed.
+      def HTTP::putdir(url, bucket, constraint, options={}, user=nil, pass=nil, debug=false, sigv=DEFAULT_SIGV, region=DEFAULT_REGION)
+        if constraint and File::exists?(constraint)
+          putdir_file(url, bucket, constraint, options, user, pass, debug)
+        else
+          putdir_binary_data(url, bucket, constraint, options, user, pass, debug, sigv, region)
+        end
+      end
+
+      def HTTP::putdir_file(url, bucket, path, options={}, user=nil, pass=nil, debug=false)
+        raise Error::PathInvalid.new(path) unless path and File::exist?(path)
+        raise ArgumentError.new('Bad options in HTTP::putdir_file') unless options.is_a? Hash
+
+        begin
+          output = Tempfile.new('ec2-put-response')
+          output.close
+          arguments = []
+
+          headers = EC2::Common::Headers.new('PUT')
+          options.each do |name, value| headers.add(name, value) end
+          headers.sign(user, pass, url, bucket) if user and pass
+          
+          arguments << headers.get.map { |name, value| "-H \"#{name}:#{value}\""}.join(' ')
+          arguments << '-T - '
+          arguments << '-o ' + output.path
+          arguments << " < #{path}"
+
+          response = HTTP::invoke(url, arguments, output.path, debug)
+          return EC2::Common::HTTP::Response.new(response.code, response.type)
+        ensure
+          output.close(true)
+          GC.start
+        end
+      end
+
+      def HTTP::putdir_binary_data(url, bucket, binary_data, options={}, user=nil, pass=nil, debug=false, sigv=DEFAULT_SIGV, region=DEFAULT_REGION)
+        raise ArgumentError.new('Bad options in HTTP::putdir_binary_data') unless options.is_a? Hash
+        
+        begin
+          output = Tempfile.new('ec2-put-response')
+          output.close
+          
+          arguments = ["-X PUT"]
+          arguments << "--data-binary \"#{binary_data}\""
+          arguments << get_arguments(url, bucket, 'PUT', options, user, pass, sigv, region, binary_data)
+          arguments << '-o ' + output.path
+
+          response = HTTP::invoke(url, arguments, output.path, debug)
+          return EC2::Common::HTTP::Response.new(response.code, response.type)          
+        ensure
+          output.close(true)
+          GC.start
+        end
+      end
+  
+      #-----------------------------------------------------------------------      
+      # Save the file at specified url, to the local file at specified path.
+      # The local file will be created if necessary, or overwritten already
+      # existing. If specified, the expected digest is compare to that of the
+      # retrieved file which gets deleted if the calculated digest does not meet
+      # expectations. If no path is specified, and the response is a 200 OK, the
+      # content of the response will be returned as a String
+      def HTTP::get(url, bucket, path=nil, options={}, user=nil, pass=nil, 
+                    size=nil, digest=nil, debug=false, sigv=DEFAULT_SIGV, region=DEFAULT_REGION)
+        raise ArgumentError.new('Bad options in HTTP::get') unless options.is_a? Hash
+        buffer = nil
+        if path.nil?
+          buffer = Tempfile.new('ec2-get-response')
+          buffer.close
+          path = buffer.path
+        else
+          directory = File.dirname(path)
+          FileUtils.mkdir_p(directory) unless File.exist?(directory)
+        end
+       
+        arguments = [get_arguments(url, bucket, 'GET', options, user, pass, sigv, region)]
+        arguments << "--max-filesize #{size}" if size
+        arguments << '-o ' + path
+        
+        begin
+          FileUtils.touch path
+          response = HTTP::invoke(url, arguments, path, debug)
+          body = nil
+          if response.success?
+            if digest
+              obtained = IO.popen("openssl sha1 #{path}") { |io| io.readline.split(/\s+/).last.strip }
+              unless digest == obtained                
+                File.delete(path) if File.exists?(path) and not buffer.is_a? Tempfile
+                raise Error::BadDigest.new(path, digest, obtained)
+              end
+            end
+            if buffer.is_a? Tempfile
+              buffer.open; 
+              body = buffer.read
+            end
+          else
+            File.delete(path) if File.exist?(path) and not buffer.is_a? Tempfile
+          end
+          return EC2::Common::HTTP::Response.new(response.code, response.type, body)
+        rescue Error::Transfer => e
+          File::delete(path) if File::exist?(path) and not buffer.is_a? Tempfile
+          raise Error::Retrieve.new(e.message, e.code)
+        ensure
+          if buffer.is_a? Tempfile
+            buffer.close
+            buffer.unlink if File.exists? path
+            GC.start
+          end
+        end
+      end
+      
+  
+      #-----------------------------------------------------------------------      
+      # Get the HEAD response for the specified url.
+      def HTTP::head(url, bucket, options={}, user=nil, pass=nil, debug=false, sigv=DEFAULT_SIGV, region=DEFAULT_REGION)
+        raise ArgumentError.new('Bad options in HTTP::head') unless options.is_a? Hash
+        begin
+          output = Tempfile.new('ec2-head-response')
+          output.close
+          
+          arguments = ['--head']
+          arguments << get_arguments(url, bucket, 'HEAD', options, user, pass, sigv, region)
+          arguments << '-o ' + output.path
+          
+          response = HTTP::invoke(url, arguments, output.path, debug)
+          return EC2::Common::HTTP::Response.new(response.code, response.type)
+          
+        rescue Error::Transfer => e
+          raise Error::Retrieve.new(e.message, e.code)
+        ensure
+          output.close(true)
+          GC.start
+        end
+      end
+
+      private
+      def HTTP::get_arguments(url, bucket, http_method, options, user, pass, sigv, region, file_path=nil)
+        headers = if user and pass
+          if sigv == EC2::Common::SIGV2
+            EC2::Common::Signature::curl_args_sigv2(url, bucket,
+                                                    http_method,
+                                                    options,
+                                                    user, pass)
+          elsif sigv == EC2::Common::SIGV4
+            EC2::Common::Signature::curl_args_sigv4(url, region, bucket,
+                                                    http_method,
+                                                    options,
+                                                    user, pass,
+                                                    file_path)
+          end
+        else
+          options
+        end
+        headers.map { |name, value| "-H \"#{name}:#{value}\""}.join(' ')
+      end
+    end
+  end
+end