cprobert-s3sync 1.3.6

data/lib/s3sync/S3.rb

@@ -0,0 +1,714 @@
+#!/usr/bin/env ruby
+
+#  This software code is made available "AS IS" without warranties of any
+#  kind.  You may copy, display, modify and redistribute the software
+#  code either by itself or as incorporated into your code; provided that
+#  you do not remove any proprietary notices.  Your use of this software
+#  code is at your own risk and you waive any claim against Amazon
+#  Digital Services, Inc. or its affiliates with respect to your use of
+#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
+#  affiliates.
+
+require 'base64'
+require 'cgi'
+require 'openssl'
+require 'digest/sha1'
+require 'net/https'
+require 'rexml/document'
+require 'time'
+
+# this wasn't added until v 1.8.3
+if (RUBY_VERSION < '1.8.3')
+  class Net::HTTP::Delete < Net::HTTPRequest
+    METHOD = 'DELETE'
+    REQUEST_HAS_BODY = false
+    RESPONSE_HAS_BODY = true
+  end
+end
+
+# this module has two big classes: AWSAuthConnection and
+# QueryStringAuthGenerator.  both use identical apis, but the first actually
+# performs the operation, while the second simply outputs urls with the
+# appropriate authentication query string parameters, which could be used
+# in another tool (such as your web browser for GETs).
+module S3
+  DEFAULT_HOST = 's3.amazonaws.com'
+  PORTS_BY_SECURITY = { true => 443, false => 80 }
+  METADATA_PREFIX = 'x-amz-meta-'
+  AMAZON_HEADER_PREFIX = 'x-amz-'
+
+  # builds the canonical string for signing.
+  def S3.canonical_string(method, bucket="", path="", path_args={}, headers={}, expires=nil)
+    interesting_headers = {}
+    headers.each do |key, value|
+      lk = key.downcase
+      if (lk == 'content-md5' or
+          lk == 'content-type' or
+          lk == 'date' or
+          lk =~ /^#{AMAZON_HEADER_PREFIX}/o)
+        interesting_headers[lk] = value.join.strip
+      end
+    end
+
+    # these fields get empty strings if they don't exist.
+    interesting_headers['content-type'] ||= ''
+    interesting_headers['content-md5'] ||= ''
+
+    # just in case someone used this.  it's not necessary in this lib.
+    if interesting_headers.has_key? 'x-amz-date'
+      interesting_headers['date'] = ''
+    end
+
+    # if you're using expires for query string auth, then it trumps date
+    # (and x-amz-date)
+    if not expires.nil?
+      interesting_headers['date'] = expires
+    end
+
+    buf = "#{method}\n"
+    interesting_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value|
+      if key =~ /^#{AMAZON_HEADER_PREFIX}/o
+        buf << "#{key}:#{value}\n"
+      else
+        buf << "#{value}\n"
+      end
+    end
+
+    # build the path using the bucket and key
+    if not bucket.empty?
+      buf << "/#{bucket}"
+    end
+    # append the key (it might be empty string) 
+    # append a slash regardless
+    buf << "/#{path}"
+
+    # if there is an acl, logging, or torrent parameter
+    # add them to the string
+    if path_args.has_key?('acl')
+      buf << '?acl'
+    elsif path_args.has_key?('torrent')
+      buf << '?torrent'
+    elsif path_args.has_key?('location')
+      buf << '?location'
+    elsif path_args.has_key?('logging')
+      buf << '?logging'
+    end
+
+    return buf
+  end
+
+  # encodes the given string with the aws_secret_access_key, by taking the
+  # hmac-sha1 sum, and then base64 encoding it.  optionally, it will also
+  # url encode the result of that to protect the string if it's going to
+  # be used as a query string parameter.
+  def S3.encode(aws_secret_access_key, str, urlencode=false)
+    digest = OpenSSL::Digest::Digest.new('sha1')
+    b64_hmac =
+      Base64.encode64(
+                      OpenSSL::HMAC.digest(digest, aws_secret_access_key, str)).strip
+
+    if urlencode
+      return CGI::escape(b64_hmac)
+    else
+      return b64_hmac
+    end
+  end
+
+  # build the path_argument string
+  def S3.path_args_hash_to_string(path_args={})
+    arg_string = ''
+    path_args.each { |k, v|
+      arg_string << k
+      if not v.nil?
+        arg_string << "=#{CGI::escape(v)}"
+      end
+      arg_string << '&'
+    }
+    return arg_string
+  end
+
+
+  # uses Net::HTTP to interface with S3.  note that this interface should only
+  # be used for smaller objects, as it does not stream the data.  if you were
+  # to download a 1gb file, it would require 1gb of memory.  also, this class
+  # creates a new http connection each time.  it would be greatly improved with
+  # some connection pooling.
+  class AWSAuthConnection
+    attr_accessor :calling_format
+
+    def initialize(aws_access_key_id, aws_secret_access_key, is_secure=true,
+                   server=DEFAULT_HOST, port=PORTS_BY_SECURITY[is_secure],
+                   calling_format=CallingFormat::REGULAR)
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @server = server
+      @is_secure = is_secure
+      @calling_format = calling_format
+      @port = port
+    end
+
+    def create_bucket(bucket, headers={})
+      return Response.new(make_request('PUT', bucket, '', {}, headers))
+    end
+
+    # takes options :prefix, :marker, :max_keys, and :delimiter
+    def list_bucket(bucket, options={}, headers={})
+      path_args = {}
+      options.each { |k, v|
+        path_args[k] = v.to_s
+      }
+
+      return ListBucketResponse.new(make_request('GET', bucket, '', path_args, headers))
+    end
+
+    def delete_bucket(bucket, headers={})
+      return Response.new(make_request('DELETE', bucket, '', {}, headers))
+    end
+
+    def put(bucket, key, object=nil, headers={})
+      if object == nil
+        req = make_request('PUT', bucket, CGI::escape(key), {}, headers)
+      else
+        if not object.instance_of? S3Object
+          object = S3Object.new(object)
+        end
+        req = make_request('PUT', bucket, CGI::escape(key), {}, headers, object.data, object.metadata)
+      end
+
+      return Response.new(req)
+    end
+
+    def get(bucket, key, headers={})
+      return GetResponse.new(make_request('GET', bucket, CGI::escape(key), {}, headers))
+    end
+
+    def delete(bucket, key, headers={})
+      return Response.new(make_request('DELETE', bucket, CGI::escape(key), {}, headers))
+    end
+
+    def head(bucket, key, headers={})
+      return GetResponse.new(make_request('HEAD', bucket, CGI::escape(key), {}, headers))
+    end
+
+    def get_bucket_logging(bucket, headers={})
+      return GetResponse.new(make_request('GET', bucket, '', {'logging' => nil}, headers))
+    end
+
+    def put_bucket_logging(bucket, logging_xml_doc, headers={})
+      return Response.new(make_request('PUT', bucket, '', {'logging' => nil}, headers, logging_xml_doc))
+    end
+
+    def get_bucket_acl(bucket, headers={})
+      return get_acl(bucket, '', headers)
+    end
+
+    # returns an xml document representing the access control list.
+    # this could be parsed into an object.
+    def get_acl(bucket, key, headers={})
+      return GetResponse.new(make_request('GET', bucket, CGI::escape(key), {'acl' => nil}, headers))
+    end
+
+    def put_bucket_acl(bucket, acl_xml_doc, headers={})
+      return put_acl(bucket, '', acl_xml_doc, headers)
+    end
+
+    # sets the access control policy for the given resource.  acl_xml_doc must
+    # be a string in the acl xml format.
+    def put_acl(bucket, key, acl_xml_doc, headers={})
+      return Response.new(
+                          make_request('PUT', bucket, CGI::escape(key), {'acl' => nil}, headers, acl_xml_doc, {})
+                          )
+    end
+
+    def list_all_my_buckets(headers={})
+      return ListAllMyBucketsResponse.new(make_request('GET', '', '', {}, headers))
+    end
+
+    private
+    def make_request(method, bucket='', key='', path_args={}, headers={}, data='', metadata={})
+
+      # build the domain based on the calling format
+      server = ''
+      if bucket.empty?
+        # for a bucketless request (i.e. list all buckets)
+        # revert to regular domain case since this operation
+        # does not make sense for vanity domains
+        server = @server
+      elsif @calling_format == CallingFormat::SUBDOMAIN
+        server = "#{bucket}.#{@server}" 
+      elsif @calling_format == CallingFormat::VANITY
+        server = bucket 
+      else
+        server = @server
+      end
+
+      # build the path based on the calling format
+      path = ''
+      if (not bucket.empty?) and (@calling_format == CallingFormat::REGULAR)
+        path << "/#{bucket}"
+      end
+      # add the slash after the bucket regardless
+      # the key will be appended if it is non-empty
+      path << "/#{key}"
+
+      # build the path_argument string
+      # add the ? in all cases since 
+      # signature and credentials follow path args
+      path << '?'
+      path << S3.path_args_hash_to_string(path_args) 
+
+      http = Net::HTTP.new(server, @port)
+      http.use_ssl = @is_secure
+      http.start do
+        req = method_to_request_class(method).new("#{path}")
+
+        set_headers(req, headers)
+        set_headers(req, metadata, METADATA_PREFIX)
+
+        set_aws_auth_header(req, @aws_access_key_id, @aws_secret_access_key, bucket, key, path_args)
+        if req.request_body_permitted?
+          return http.request(req, data)
+        else
+          return http.request(req)
+        end
+      end
+    end
+
+    def method_to_request_class(method)
+      case method
+      when 'GET'
+        return Net::HTTP::Get
+      when 'PUT'
+        return Net::HTTP::Put
+      when 'DELETE'
+        return Net::HTTP::Delete
+      when 'HEAD'
+        return Net::HTTP::Head
+      else
+        raise "Unsupported method #{method}"
+      end
+    end
+
+    # set the Authorization header using AWS signed header authentication
+    def set_aws_auth_header(request, aws_access_key_id, aws_secret_access_key, bucket='', key='', path_args={})
+      # we want to fix the date here if it's not already been done.
+      request['Date'] ||= Time.now.httpdate
+
+      # ruby will automatically add a random content-type on some verbs, so
+      # here we add a dummy one to 'supress' it.  change this logic if having
+      # an empty content-type header becomes semantically meaningful for any
+      # other verb.
+      request['Content-Type'] ||= ''
+
+      canonical_string =
+        S3.canonical_string(request.method, bucket, key, path_args, request.to_hash, nil)
+      encoded_canonical = S3.encode(aws_secret_access_key, canonical_string)
+
+      request['Authorization'] = "AWS #{aws_access_key_id}:#{encoded_canonical}"
+    end
+
+    def set_headers(request, headers, prefix='')
+      headers.each do |key, value|
+        request[prefix + key] = value
+      end
+    end
+  end
+
+
+  # This interface mirrors the AWSAuthConnection class above, but instead
+  # of performing the operations, this class simply returns a url that can
+  # be used to perform the operation with the query string authentication
+  # parameters set.
+  class QueryStringAuthGenerator
+    attr_accessor :calling_format
+    attr_accessor :expires
+    attr_accessor :expires_in
+    attr_reader :server
+    attr_reader :port
+
+    # by default, expire in 1 minute
+    DEFAULT_EXPIRES_IN = 60
+
+    def initialize(aws_access_key_id, aws_secret_access_key, is_secure=true, 
+                   server=DEFAULT_HOST, port=PORTS_BY_SECURITY[is_secure], 
+                   format=CallingFormat::REGULAR)
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @protocol = is_secure ? 'https' : 'http'
+      @server = server
+      @port = port
+      @calling_format = format 
+      # by default expire
+      @expires_in = DEFAULT_EXPIRES_IN
+    end
+
+    # set the expires value to be a fixed time.  the argument can
+    # be either a Time object or else seconds since epoch.
+    def expires=(value)
+      @expires = value
+      @expires_in = nil
+    end
+
+    # set the expires value to expire at some point in the future
+    # relative to when the url is generated.  value is in seconds.
+    def expires_in=(value)
+      @expires_in = value
+      @expires = nil
+    end
+
+    def create_bucket(bucket, headers={})
+      return generate_url('PUT', bucket, '', {}, headers)
+    end
+
+    # takes options :prefix, :marker, :max_keys, and :delimiter
+    def list_bucket(bucket, options={}, headers={})
+      path_args = {}
+      options.each { |k, v|
+        path_args[k] = v.to_s
+      }
+      return generate_url('GET', bucket, '', path_args, headers)
+    end
+
+    def delete_bucket(bucket, headers={})
+      return generate_url('DELETE', bucket, '', {}, headers)
+    end
+
+    # don't really care what object data is.  it's just for conformance with the
+    # other interface.  If this doesn't work, check tcpdump to see if the client is
+    # putting a Content-Type header on the wire.
+    def put(bucket, key, object=nil, headers={})
+      object = S3Object.new(object) if not object.instance_of? S3Object
+      return generate_url('PUT', bucket, CGI::escape(key), {}, merge_meta(headers, object))
+    end
+
+    def get(bucket, key, headers={})
+      return generate_url('GET', bucket, CGI::escape(key), {}, headers)
+    end
+
+    def delete(bucket, key, headers={})
+      return generate_url('DELETE', bucket, CGI::escape(key), {}, headers)
+    end
+
+    def get_bucket_logging(bucket, headers={})
+      return generate_url('GET', bucket, '', {'logging' => nil}, headers)
+    end
+
+    def put_bucket_logging(bucket, logging_xml_doc, headers={})
+      return generate_url('PUT', bucket, '', {'logging' => nil}, headers)
+    end
+
+    def get_acl(bucket, key='', headers={})
+      return generate_url('GET', bucket, CGI::escape(key), {'acl' => nil}, headers)
+    end
+
+    def get_bucket_acl(bucket, headers={})
+      return get_acl(bucket, '', headers)
+    end
+
+    # don't really care what acl_xml_doc is.
+    # again, check the wire for Content-Type if this fails.
+    def put_acl(bucket, key, acl_xml_doc, headers={})
+      return generate_url('PUT', bucket, CGI::escape(key), {'acl' => nil}, headers)
+    end
+
+    def put_bucket_acl(bucket, acl_xml_doc, headers={})
+      return put_acl(bucket, '', acl_xml_doc, headers)
+    end
+
+    def list_all_my_buckets(headers={})
+      return generate_url('GET', '', '', {}, headers)
+    end
+
+
+    private
+    # generate a url with the appropriate query string authentication
+    # parameters set.
+    def generate_url(method, bucket="", key="", path_args={}, headers={})
+      expires = 0
+      if not @expires_in.nil?
+        expires = Time.now.to_i + @expires_in
+      elsif not @expires.nil?
+        expires = @expires
+      else
+        raise "invalid expires state"
+      end
+
+      canonical_string =
+        S3::canonical_string(method, bucket, key, path_args, headers, expires)
+      encoded_canonical =
+        S3::encode(@aws_secret_access_key, canonical_string)
+
+      url = CallingFormat.build_url_base(@protocol, @server, @port, bucket, @calling_format)
+
+      path_args["Signature"] = encoded_canonical.to_s
+      path_args["Expires"] = expires.to_s
+      path_args["AWSAccessKeyId"] = @aws_access_key_id.to_s
+      arg_string = S3.path_args_hash_to_string(path_args) 
+
+      return "#{url}/#{key}?#{arg_string}"
+    end
+
+    def merge_meta(headers, object)
+      final_headers = headers.clone
+      if not object.nil? and not object.metadata.nil?
+        object.metadata.each do |k, v|
+          final_headers[METADATA_PREFIX + k] = v
+        end
+      end
+      return final_headers
+    end
+  end
+
+  class S3Object
+    attr_accessor :data
+    attr_accessor :metadata
+    def initialize(data, metadata={})
+      @data, @metadata = data, metadata
+    end
+  end
+
+  # class for storing calling format constants
+  module CallingFormat
+    REGULAR   = 0 # http://s3.amazonaws.com/bucket/key
+    SUBDOMAIN = 1 # http://bucket.s3.amazonaws.com/key
+    VANITY    = 2  # http://<vanity_domain>/key  -- vanity_domain resolves to s3.amazonaws.com
+
+    # build the url based on the calling format, and bucket
+    def CallingFormat.build_url_base(protocol, server, port, bucket, format)
+      build_url_base = "#{protocol}://"
+      if bucket.empty?
+        build_url_base << "#{server}:#{port}"
+      elsif format == SUBDOMAIN
+        build_url_base << "#{bucket}.#{server}:#{port}"
+      elsif format == VANITY
+        build_url_base << "#{bucket}:#{port}"
+      else
+        build_url_base << "#{server}:#{port}/#{bucket}"
+      end
+      return build_url_base
+    end
+  end
+
+  class Owner
+    attr_accessor :id
+    attr_accessor :display_name
+  end
+
+  class ListEntry
+    attr_accessor :key
+    attr_accessor :last_modified
+    attr_accessor :etag
+    attr_accessor :size
+    attr_accessor :storage_class
+    attr_accessor :owner
+  end
+
+  class ListProperties
+    attr_accessor :name
+    attr_accessor :prefix
+    attr_accessor :marker
+    attr_accessor :max_keys
+    attr_accessor :delimiter
+    attr_accessor :is_truncated
+    attr_accessor :next_marker
+  end
+
+  class CommonPrefixEntry
+    attr_accessor :prefix
+  end
+
+  # Parses the list bucket output into a list of ListEntry objects, and
+  # a list of CommonPrefixEntry objects if applicable.
+  class ListBucketParser
+    attr_reader :properties
+    attr_reader :entries
+    attr_reader :common_prefixes
+
+    def initialize
+      reset
+    end
+
+    def tag_start(name, attributes)
+      if name == 'ListBucketResult'
+        @properties = ListProperties.new
+      elsif name == 'Contents'
+        @curr_entry = ListEntry.new
+      elsif name == 'Owner'
+        @curr_entry.owner = Owner.new
+      elsif name == 'CommonPrefixes'
+        @common_prefix_entry = CommonPrefixEntry.new
+      end
+    end
+
+    # we have one, add him to the entries list
+    def tag_end(name)
+      # this prefix is the one we echo back from the request
+      if name == 'Name'
+        @properties.name = @curr_text
+      elsif name == 'Prefix' and @is_echoed_prefix
+        @properties.prefix = @curr_text
+        @is_echoed_prefix = nil
+      elsif name == 'Marker'
+        @properties.marker = @curr_text
+      elsif name == 'MaxKeys'
+        @properties.max_keys = @curr_text.to_i
+      elsif name == 'Delimiter'
+        @properties.delimiter = @curr_text
+      elsif name == 'IsTruncated'
+        @properties.is_truncated = @curr_text == 'true'
+      elsif name == 'NextMarker'
+        @properties.next_marker = @curr_text
+      elsif name == 'Contents'
+        @entries << @curr_entry
+      elsif name == 'Key'
+        @curr_entry.key = @curr_text
+      elsif name == 'LastModified'
+        @curr_entry.last_modified = @curr_text
+      elsif name == 'ETag'
+        @curr_entry.etag = @curr_text
+      elsif name == 'Size'
+        @curr_entry.size = @curr_text.to_i
+      elsif name == 'StorageClass'
+        @curr_entry.storage_class = @curr_text
+      elsif name == 'ID'
+        @curr_entry.owner.id = @curr_text
+      elsif name == 'DisplayName'
+        @curr_entry.owner.display_name = @curr_text
+      elsif name == 'CommonPrefixes'
+        @common_prefixes << @common_prefix_entry
+      elsif name == 'Prefix'
+        # this is the common prefix for keys that match up to the delimiter
+        @common_prefix_entry.prefix = @curr_text
+      end
+      @curr_text = ''
+    end
+
+    def text(text)
+      @curr_text += text
+    end
+
+    def xmldecl(version, encoding, standalone)
+      # ignore
+    end
+
+    # get ready for another parse
+    def reset
+      @is_echoed_prefix = true;
+      @entries = []
+      @curr_entry = nil
+      @common_prefixes = []
+      @common_prefix_entry = nil
+      @curr_text = ''
+    end
+  end
+
+  class Bucket
+    attr_accessor :name
+    attr_accessor :creation_date
+  end
+
+  class ListAllMyBucketsParser
+    attr_reader :entries
+
+    def initialize
+      reset
+    end
+
+    def tag_start(name, attributes)
+      if name == 'Bucket'
+        @curr_bucket = Bucket.new
+      end
+    end
+
+    # we have one, add him to the entries list
+    def tag_end(name)
+      if name == 'Bucket'
+        @entries << @curr_bucket
+      elsif name == 'Name'
+        @curr_bucket.name = @curr_text
+      elsif name == 'CreationDate'
+        @curr_bucket.creation_date = @curr_text
+      end
+      @curr_text = ''
+    end
+
+    def text(text)
+      @curr_text += text
+    end
+
+    def xmldecl(version, encoding, standalone)
+      # ignore
+    end
+
+    # get ready for another parse
+    def reset
+      @entries = []
+      @owner = nil
+      @curr_bucket = nil
+      @curr_text = ''
+    end
+  end
+
+  class Response
+    attr_reader :http_response
+    def initialize(response)
+      @http_response = response
+    end
+  end
+
+  class GetResponse < Response
+    attr_reader :object
+    def initialize(response)
+      super(response)
+      metadata = get_aws_metadata(response)
+      data = response.body
+      @object = S3Object.new(data, metadata)
+    end
+
+    # parses the request headers and pulls out the s3 metadata into a hash
+    def get_aws_metadata(response)
+      metadata = {}
+      response.each do |key, value|
+        if key =~ /^#{METADATA_PREFIX}(.*)$/oi
+          metadata[$1] = value
+        else
+          metadata[key] = value
+        end
+      end
+      return metadata
+    end
+  end
+
+  class ListBucketResponse < Response
+    attr_reader :properties
+    attr_reader :entries
+    attr_reader :common_prefix_entries
+
+    def initialize(response)
+      super(response)
+      if response.is_a? Net::HTTPSuccess
+        parser = ListBucketParser.new
+        REXML::Document.parse_stream(response.body, parser)
+        @properties = parser.properties
+        @entries = parser.entries
+        @common_prefix_entries = parser.common_prefixes
+      else
+        @entries = []
+      end
+    end
+  end
+
+  class ListAllMyBucketsResponse < Response
+    attr_reader :entries
+    def initialize(response)
+      super(response)
+      if response.is_a? Net::HTTPSuccess
+        parser = ListAllMyBucketsParser.new
+        REXML::Document.parse_stream(response.body, parser)
+        @entries = parser.entries
+      else
+        @entries = []
+      end
+    end
+  end
+end