radosgw-s3 0.1

data/lib/s3/objects_extension.rb

@@ -0,0 +1,37 @@
+module S3
+  module ObjectsExtension
+    # Builds the object in the bucket with given key
+    def build(key)
+      Object.send(:new, proxy_owner, :key => key)
+    end
+
+    # Finds first object with given name or raises the exception if
+    # not found
+    def find_first(name)
+      object = build(name)
+      object.retrieve
+    end
+    alias :find :find_first
+
+    # Finds the objects in the bucket.
+    #
+    # ==== Options
+    # * <tt>:prefix</tt> - Limits the response to keys which begin
+    #   with the indicated prefix
+    # * <tt>:marker</tt> - Indicates where in the bucket to begin
+    #   listing
+    # * <tt>:max_keys</tt> - The maximum number of keys you'd like
+    #   to see
+    # * <tt>:delimiter</tt> - Causes keys that contain the same
+    #   string between the prefix and the first occurrence of the
+    #   delimiter to be rolled up into a single result element
+    def find_all(options = {})
+      proxy_owner.send(:list_bucket, options)
+    end
+
+    # Destroys all keys in the bucket
+    def destroy_all
+      proxy_target.each { |object| object.destroy }
+    end
+  end
+end

data/lib/s3/parser.rb

@@ -0,0 +1,90 @@
+module S3
+  module Parser
+    include REXML
+
+    def rexml_document(xml)
+      xml.force_encoding(::Encoding::UTF_8) if xml.respond_to? :force_encoding
+      Document.new(xml)
+    end
+
+    def parse_list_all_my_buckets_result(xml)
+      names = []
+      rexml_document(xml).elements.each("ListAllMyBucketsResult/Buckets/Bucket/Name") { |e| names << e.text }
+      names
+    end
+
+    def parse_location_constraint(xml)
+      rexml_document(xml).elements["LocationConstraint"].text
+    end
+
+    def parse_list_bucket_result(xml)
+      objects_attributes = []
+      rexml_document(xml).elements.each("ListBucketResult/Contents") do |e|
+        object_attributes = {}
+        object_attributes[:key] = e.elements["Key"].text
+        object_attributes[:etag] = e.elements["ETag"].text
+        object_attributes[:last_modified] = e.elements["LastModified"].text
+        object_attributes[:size] = e.elements["Size"].text
+        objects_attributes << object_attributes
+      end
+      objects_attributes
+    end
+
+    def parse_copy_object_result(xml)
+      object_attributes = {}
+      document = rexml_document(xml)
+      object_attributes[:etag] = document.elements["CopyObjectResult/ETag"].text
+      object_attributes[:last_modified] = document.elements["CopyObjectResult/LastModified"].text
+      object_attributes
+    end
+
+    def parse_error(xml)
+      document = rexml_document(xml)
+      code = document.elements["Error/Code"].text
+      message = document.elements["Error/Message"].text
+      [code, message]
+    end
+
+    def parse_is_truncated xml
+      rexml_document(xml).elements["ListBucketResult/IsTruncated"].text =='true'
+    end
+
+
+    # Parse acl response and return hash with grantee and their permissions
+    def parse_acl(xml)
+      grants = {}
+      rexml_document(xml).elements.each("AccessControlPolicy/AccessControlList/Grant") do |grant|
+        grants.merge!(extract_grantee(grant))
+      end
+      grants
+    end
+
+    private
+
+     def convert_uri_to_group_name(uri)
+        case uri
+        when "http://acs.amazonaws.com/groups/global/AllUsers"
+          return "Everyone"
+        when "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
+          return "Authenticated Users"
+        when "http://acs.amazonaws.com/groups/s3/LogDelivery"
+          return "Log Delivery"
+        else
+          return uri
+        end
+      end
+
+     def extract_grantee(grant)
+        grants  = {}
+        grant.each_element_with_attribute("xsi:type", "Group") do |grantee|
+          group_name = convert_uri_to_group_name(grantee.get_text("URI").value)
+          grants[group_name] = grant.get_text("Permission").value
+        end
+        grant.each_element_with_attribute("xsi:type", "CanonicalUser") do |grantee|
+          user_name = grantee.get_text("DisplayName").value
+          grants[user_name] = grant.get_text("Permission").value
+        end
+        grants
+     end
+  end
+end

data/lib/s3/request.rb

@@ -0,0 +1,31 @@
+module S3
+  # Class responsible for sending chunked requests
+  # properly. Net::HTTPGenericRequest has hardcoded chunk_size, so we
+  # inherit the class and override chunk_size.
+  class Request < Net::HTTPGenericRequest
+    def initialize(chunk_size, m, reqbody, resbody, path, initheader = nil)
+      @chunk_size = chunk_size
+      super(m, reqbody, resbody, path, initheader)
+    end
+
+    private
+
+    def send_request_with_body_stream(sock, ver, path, f)
+      unless content_length() or chunked?
+        raise ArgumentError, "Content-Length not given and Transfer-Encoding is not `chunked'"
+      end
+      supply_default_content_type
+      write_header sock, ver, path
+      if chunked?
+        while s = f.read(@chunk_size)
+          sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
+        end
+        sock.write "0\r\n\r\n"
+      else
+        while s = f.read(@chunk_size)
+          sock.write s
+        end
+      end
+    end
+  end
+end

data/lib/s3/service.rb

@@ -0,0 +1,98 @@
+module S3
+  class Service
+    include Parser
+    include Proxies
+
+    attr_reader :access_key_id, :secret_access_key, :use_ssl, :use_vhost, :proxy, :host
+
+    # Compares service to other, by <tt>access_key_id</tt> and
+    # <tt>secret_access_key</tt>
+    def ==(other)
+      self.access_key_id == other.access_key_id and self.secret_access_key == other.secret_access_key
+    end
+
+    # Creates new service.
+    #
+    # ==== Options
+    # * <tt>:access_key_id</tt> - Access key id (REQUIRED)
+    # * <tt>:secret_access_key</tt> - Secret access key (REQUIRED)
+    # * <tt>:use_ssl</tt> - Use https or http protocol (false by
+    #   default)
+    # * <tt>:use_vhost</tt> - Use bucket.s3.amazonaws.com or s3.amazonaws.com/bucket (true by
+    #   default)
+    # * <tt>:debug</tt> - Display debug information on the STDOUT
+    #   (false by default)
+    # * <tt>:timeout</tt> - Timeout to use by the Net::HTTP object
+    #   (60 by default)
+    def initialize(options)
+      # The keys for these required options might exist in the options hash, but
+      # they might be set to something like `nil`. If this is the case, we want
+      # to fail early.
+      raise ArgumentError, "Missing :access_key_id." if !options[:access_key_id]
+      raise ArgumentError, "Missing :secret_access_key." if !options[:secret_access_key]
+
+      @access_key_id = options.fetch(:access_key_id)
+      @secret_access_key = options.fetch(:secret_access_key)
+      @host = options.fetch(:host)
+      @use_ssl = options.fetch(:use_ssl, false)
+      @use_vhost = options.fetch(:use_vhost, true)
+      @timeout = options.fetch(:timeout, 60)
+      @debug = options.fetch(:debug, false)
+
+      raise ArgumentError, "Missing proxy settings. Must specify at least :host." if options[:proxy] && !options[:proxy][:host]
+      @proxy = options.fetch(:proxy, nil)
+    end
+
+    # Returns all buckets in the service and caches the result (see
+    # +reload+)
+    def buckets
+      Proxy.new(lambda { list_all_my_buckets }, :owner => self, :extend => BucketsExtension)
+    end
+
+    # Returns the bucket with the given name. Does not check whether the
+    # bucket exists. But also does not issue any HTTP requests, so it's
+    # much faster than buckets.find
+    def bucket(name)
+      Bucket.send(:new, self, name)
+    end
+
+    # Returns "http://" or "https://", depends on <tt>:use_ssl</tt>
+    # value from initializer
+    def protocol
+      use_ssl ? "https://" : "http://"
+    end
+
+    # Returns 443 or 80, depends on <tt>:use_ssl</tt> value from
+    # initializer
+    def port
+      use_ssl ? 443 : 80
+    end
+
+    def inspect #:nodoc:
+      "#<#{self.class}:#@access_key_id>"
+    end
+
+    private
+
+    def list_all_my_buckets
+      response = service_request(:get)
+      names = parse_list_all_my_buckets_result(response.body)
+      names.map { |name| Bucket.send(:new, self, name) }
+    end
+
+    def service_request(method, options = {})
+      connection.request(method, options.merge(:path => "/#{options[:path]}"))
+    end
+
+    def connection
+      return @connection if defined?(@connection)
+      @connection = Connection.new(:access_key_id => @access_key_id,
+                               :secret_access_key => @secret_access_key,
+                               :host => @host,
+                               :use_ssl => @use_ssl,
+                               :timeout => @timeout,
+                               :debug => @debug,
+                               :proxy => @proxy)
+    end
+  end
+end

data/lib/s3/signature.rb

@@ -0,0 +1,261 @@
+module S3
+
+  # Class responsible for generating signatures to requests.
+  #
+  # Implements algorithm defined by Amazon Web Services to sign
+  # request with secret private credentials
+  #
+  # === See
+  # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?RESTAuthentication.html
+
+  class Signature
+
+    # Generates signature for given parameters
+    #
+    # ==== Options
+    # * <tt>:host</tt> - Hostname
+    # * <tt>:request</tt> - Net::HTTPRequest object with correct
+    #   headers
+    # * <tt>:access_key_id</tt> - Access key id
+    # * <tt>:secret_access_key</tt> - Secret access key
+    #
+    # ==== Returns
+    # Generated signature string for given hostname and request
+    def self.generate(options)
+      request = options[:request]
+      access_key_id = options[:access_key_id]
+
+      options.merge!(:headers => request,
+                     :method => request.method,
+                     :resource => request.path)
+
+      signature = canonicalized_signature(options)
+
+      "AWS #{access_key_id}:#{signature}"
+    end
+
+    # Generates temporary URL signature for given resource
+    #
+    # ==== Options
+    # * <tt>:bucket</tt> - Bucket in which the resource resides
+    # * <tt>:resource</tt> - Path to the resouce you want to create
+    #   a temporary link to
+    # * <tt>:secret_access_key</tt> - Secret access key
+    # * <tt>:expires_at</tt> - Unix time stamp of when the resouce
+    #   link will expire
+    # * <tt>:method</tt> - HTTP request method you want to use on
+    #   the resource, defaults to GET
+    # * <tt>:headers</tt> - Any additional HTTP headers you intend
+    #   to use when requesting the resource
+    # * <tt>:add_bucket_to_host</tt> - Use in case of virtual-host style,
+    #   defaults to false
+    def self.generate_temporary_url_signature(options)
+      bucket = options[:bucket]
+      resource = options[:resource]
+      secret_access_key = options[:secret_access_key]
+      expires = options[:expires_at]
+
+      headers = options[:headers] || {}
+      headers.merge!("date" => expires.to_i.to_s)
+
+      resource = "/#{URI.escape(resource, /[^#{URI::REGEXP::PATTERN::UNRESERVED}\/]/)}"
+      resource = "/#{bucket}" + resource unless options[:add_bucket_to_host]
+
+      options.merge!(:resource => resource,
+                     :method => options[:method] || :get,
+                     :headers => headers)
+      signature = canonicalized_signature(options)
+
+      CGI.escape(signature)
+    end
+
+    # Generates temporary URL for given resource
+    #
+    # ==== Options
+    # * <tt>:bucket</tt> - Bucket in which the resource resides
+    # * <tt>:resource</tt> - Path to the resouce you want to create
+    #   a temporary link to
+    # * <tt>:access_key</tt> - Access key
+    # * <tt>:secret_access_key</tt> - Secret access key
+    # * <tt>:expires_at</tt> - Unix time stamp of when the resouce
+    #   link will expire
+    # * <tt>:method</tt> - HTTP request method you want to use on
+    #   the resource, defaults to GET
+    # * <tt>:headers</tt> - Any additional HTTP headers you intend
+    #   to use when requesting the resource
+    # * <tt>:add_bucket_to_host</tt> - Use in case of virtual-host style,
+    #   defaults to false
+    def self.generate_temporary_url(options)
+      bucket = options[:bucket]
+      resource = options[:resource]
+      access_key = options[:access_key]
+      expires = options[:expires_at].to_i
+      host = options[:host]
+
+      if options[:add_bucket_to_host]
+        host = bucket + '.' + host
+        url  = "http://#{host}/#{resource}"
+      else
+        url = "http://#{host}/#{bucket}/#{resource}"
+      end
+
+      options[:host] = host
+      signature = generate_temporary_url_signature(options)
+
+      url << "?AWSAccessKeyId=#{access_key}"
+      url << "&Expires=#{expires}"
+      url << "&Signature=#{signature}"
+    end
+
+    private
+
+    def self.canonicalized_signature(options)
+      headers = options[:headers] || {}
+      host = options[:host] || ""
+      resource = options[:resource]
+      access_key_id = options[:access_key_id]
+      secret_access_key = options[:secret_access_key]
+
+      http_verb = options[:method].to_s.upcase
+      content_md5 = headers["content-md5"] || ""
+      content_type = headers["content-type"] || ""
+      date = headers["x-amz-date"].nil? ? headers["date"] : ""
+      canonicalized_resource = canonicalized_resource(host, resource)
+      canonicalized_amz_headers = canonicalized_amz_headers(headers)
+
+      string_to_sign = ""
+      string_to_sign << http_verb
+      string_to_sign << "\n"
+      string_to_sign << content_md5
+      string_to_sign << "\n"
+      string_to_sign << content_type
+      string_to_sign << "\n"
+      string_to_sign << date
+      string_to_sign << "\n"
+      string_to_sign << canonicalized_amz_headers
+      string_to_sign << canonicalized_resource
+
+      digest = OpenSSL::Digest.new("sha1")
+      hmac = OpenSSL::HMAC.digest(digest, secret_access_key, string_to_sign)
+      base64 = Base64.encode64(hmac)
+      base64.chomp
+    end
+
+    # Helper method for extracting header fields from Net::HTTPRequest
+    # and preparing them for singing in #generate method
+    #
+    # ==== Parameters
+    # * <tt>request</tt> - Net::HTTPRequest object with header fields
+    #   filled in
+    #
+    # ==== Returns
+    # String containing interesting header fields in suitable order
+    # and form
+    def self.canonicalized_amz_headers(request)
+      headers = []
+
+      # 1. Convert each HTTP header name to lower-case. For example,
+      # "X-Amz-Date" becomes "x-amz-date".
+      request.each { |key, value| headers << [key.downcase, value] if key =~ /\Ax-amz-/io }
+      #=> [["c", 0], ["a", 1], ["a", 2], ["b", 3]]
+
+      # 2. Sort the collection of headers lexicographically by header
+      # name.
+      headers.sort!
+      #=> [["a", 1], ["a", 2], ["b", 3], ["c", 0]]
+
+      # 3. Combine header fields with the same name into one
+      # "header-name:comma-separated-value-list" pair as prescribed by
+      # RFC 2616, section 4.2, without any white-space between
+      # values. For example, the two metadata headers
+      # "x-amz-meta-username: fred" and "x-amz-meta-username: barney"
+      # would be combined into the single header "x-amz-meta-username:
+      # fred,barney".
+      combined_headers = headers.inject([]) do |new_headers, header|
+        existing_header = new_headers.find { |h| h.first == header.first }
+        if existing_header
+          existing_header.last << ",#{header.last}"
+        else
+          new_headers << header
+        end
+      end
+      #=> [["a", "1,2"], ["b", "3"], ["c", "0"]]
+
+      # 4. "Un-fold" long headers that span multiple lines (as allowed
+      # by RFC 2616, section 4.2) by replacing the folding white-space
+      # (including new-line) by a single space.
+      unfolded_headers = combined_headers.map do |header|
+        key = header.first
+        value = header.last
+        value.gsub!(/\s+/, " ")
+        [key, value]
+      end
+
+      # 5. Trim any white-space around the colon in the header. For
+      # example, the header "x-amz-meta-username: fred,barney" would
+      # become "x-amz-meta-username:fred,barney"
+      joined_headers = unfolded_headers.map do |header|
+        key = header.first.strip
+        value = header.last.strip
+        "#{key}:#{value}"
+      end
+
+      # 6. Finally, append a new-line (U+000A) to each canonicalized
+      # header in the resulting list. Construct the
+      # CanonicalizedResource element by concatenating all headers in
+      # this list into a single string.
+      joined_headers << "" unless joined_headers.empty?
+      joined_headers.join("\n")
+    end
+
+    # Helper methods for extracting caninocalized resource address
+    #
+    # ==== Parameters
+    # * <tt>host</tt> - Hostname
+    # * <tt>request</tt> - Net::HTTPRequest object with header fields
+    #   filled in
+    #
+    # ==== Returns
+    # String containing extracted canonicalized resource
+    def self.canonicalized_resource(host, resource)
+      # 1. Start with the empty string ("").
+      string = ""
+
+      # 2. If the request specifies a bucket using the HTTP Host
+      # header (virtual hosted-style), append the bucket name preceded
+      # by a "/" (e.g., "/bucketname"). For path-style requests and
+      # requests that don't address a bucket, do nothing. For more
+      # information on virtual hosted-style requests, see Virtual
+      # Hosting of Buckets.
+      bucket_name = host.sub(/\.?#{host}\Z/, "")
+      string << "/#{bucket_name}" unless bucket_name.empty?
+
+      # 3. Append the path part of the un-decoded HTTP Request-URI,
+      # up-to but not including the query string.
+      uri = URI.parse(resource)
+      string << uri.path
+
+      # 4. If the request addresses a sub-resource, like ?location,
+      # ?acl, or ?torrent, append the sub-resource including question
+      # mark.
+      sub_resources = [
+        "acl",
+        "location",
+        "logging",
+        "notification",
+        "partNumber",
+        "policy",
+        "requestPayment",
+        "torrent",
+        "uploadId",
+        "uploads",
+        "versionId",
+        "versioning",
+        "versions",
+        "website"
+      ]
+      string << "?#{$1}" if uri.query =~ /&?(#{sub_resources.join("|")})(?:&|=|\Z)/
+      string
+    end
+  end
+end