s33r 0.4.2 → 0.5

data/lib/s33r/core.rb

@@ -1,296 +0,0 @@
-require 'base64'
-require 'time'
-require 'net/http'
-require 'net/https'
-require 'openssl'
-require 'parsedate'
-require File.join(File.dirname(__FILE__), 'libxml_loader')
-
-# Module to handle S3 operations which don't require an internet connection,
-# i.e. data validation and request-building operations;
-# also holds all the constants relating to S3.
-# 
-# Parts of this code are heavily based on Amazon's code. Here's their license:
-#
-#  This software code is made available "AS IS" without warranties of any
-#  kind.  You may copy, display, modify and redistribute the software
-#  code either by itself or as incorporated into your code; provided that
-#  you do not remove any proprietary notices.  Your use of this software
-#  code is at your own risk and you waive any claim against Amazon
-#  Digital Services, Inc. or its affiliates with respect to your use of
-#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
-#  affiliates.
-module S33r
-  HOST = 's3.amazonaws.com'
-  PORT = 443
-  NON_SSL_PORT = 80
-  METADATA_PREFIX = 'x-amz-meta-'
-  # Size of each chunk (in bytes) to be sent per request when putting files (1Mb).
-  DEFAULT_CHUNK_SIZE = 1048576
-  AWS_HEADER_PREFIX = 'x-amz-'
-  AWS_AUTH_HEADER_VALUE = "AWS %s:%s"
-  INTERESTING_HEADERS = ['content-md5', 'content-type', 'date']
-  # Headers which must be included with every request to S3.
-  REQUIRED_HEADERS = ['Content-Type', 'Date']
-  # Canned ACLs made available by S3.
-  CANNED_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
-  # HTTP methods which S3 will respond to.
-  METHOD_VERBS = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE']
-  # Maximum number which can be passed in max-keys parameter when GETting bucket list.
-  BUCKET_LIST_MAX_MAX_KEYS = 1000
-  # Default number of seconds an authenticated URL will last for (15 minutes).
-  DEFAULT_EXPIRY_SECS = 60 * 15
-  # The namespace used for response body XML documents.
-  RESPONSE_NAMESPACE_URI = "http://s3.amazonaws.com/doc/2006-03-01/"
-  
-  # Permissions which can be set within a <Grant>
-  # (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingPermissions.html).
-  # 
-  # NB I've missed out the WRITE_ACP permission as this is functionally
-  # equivalent to FULL_CONTROL.
-  PERMISSIONS = { 
-    :read => 'READ',  # permission to read
-    :write => 'WRITE',  # permission to write
-    :read_acl => 'READ_ACP',  # permission to read ACL settings
-    :all => 'FULL_CONTROL'  # do anything
-  }
-  
-  # Used for generating ACL XML documents.
-  NAMESPACE = 'xsi'
-  NAMESPACE_URI = 'http://www.w3.org/2001/XMLSchema-instance'
-  GRANTEE_TYPES = {
-    :amazon_customer => 'AmazonCustomerByEmail', 
-    :canonical_user => 'CanonicalUser',
-    :group => 'Group'
-  }
-  S3_GROUP_TYPES = {
-    :all_users => 'global/AllUsers',
-    :authenticated_users => 'global/AuthenticatedUsers',
-    :log_delivery => 's3/LogDelivery'
-  }
-  GROUP_ACL_URI_BASE = 'http://acs.amazonaws.com/groups/'
-
-  # Build canonical string for signing;
-  # modified (slightly) from the Amazon sample code.
-  def generate_canonical_string(method, path, headers={}, expires=nil)
-    interesting_headers = {}
-    headers.each do |key, value|
-      lk = key.downcase
-      if (INTERESTING_HEADERS.include?(lk) or lk =~ /^#{AWS_HEADER_PREFIX}/o)
-        interesting_headers[lk] = value
-      end
-    end
-
-    # These fields get empty strings if they don't exist.
-    interesting_headers['content-type'] ||= ''
-    interesting_headers['content-md5'] ||= ''
-
-    # If you're using expires for query string auth, then it trumps date.
-    if not expires.nil?
-      interesting_headers['date'] = expires
-    end
-
-    buf = "#{method}\n"
-    interesting_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value|
-      if key =~ /^#{AWS_HEADER_PREFIX}/o
-        buf << "#{key}:#{value}\n"
-      else
-        buf << "#{value}\n"
-      end
-    end
-
-    # Ignore everything after the question mark...
-    buf << path.gsub(/\?.*$/, '')
-
-    # ...unless there is an acl, logging or torrent parameter
-    if path =~ /[&?]acl($|&|=)/
-      buf << '?acl'
-    elsif path =~ /[&?]torrent($|&|=)/
-      buf << '?torrent'
-    elsif path =~ /[&?]logging($|&|=)/
-      buf << '?logging'
-    end
-
-    buf
-  end
-
-  # Get the value for the AWS authentication header.
-  def generate_auth_header_value(method, path, headers, aws_access_key, aws_secret_access_key)
-    raise S33rException::MethodNotAvailable, "Method %s not available" % method if !METHOD_VERBS.include?(method)
-
-    # check the headers needed for authentication have been set
-    missing_headers = REQUIRED_HEADERS - headers.keys
-    if !(missing_headers.empty?)
-      raise S33rException::MissingRequiredHeaders,
-        "Headers required for AWS auth value are missing: " + missing_headers.join(', ')
-    end
-
-    # get the AWS header
-    canonical_string = generate_canonical_string(method, path, headers)
-    signature = generate_signature(aws_secret_access_key, canonical_string)
-    AWS_AUTH_HEADER_VALUE % [aws_access_key, signature]
-  end
-
-  # Encode the given string with the aws_secret_access_key, by taking the
-  # hmac sha1 sum, and then base64 encoding it.
-  def generate_signature(aws_secret_access_key, str)
-    digest = OpenSSL::HMAC::digest(OpenSSL::Digest::Digest.new("SHA1"), aws_secret_access_key, str)
-    Base64.encode64(digest).strip
-  end
-
-  # Build the headers required with every S3 request (Date and Content-Type); 
-  # options hash can contain extra header settings;
-  # +:date+ and +:content_type+ are required headers, and set to defaults if not supplied
-  def add_default_headers(headers, options={})
-    # set the default headers required by AWS
-    missing_headers = REQUIRED_HEADERS - headers.keys
-
-    if missing_headers.include?('Content-Type')
-      headers['Content-Type'] = options[:content_type] || ''
-    end
-
-    if missing_headers.include?('Date')
-      date = options[:date] || Time.now
-      headers['Date'] = date.httpdate
-    end
-
-    headers
-  end
-
-  # Add metadata headers, correctly prefixing them first,
-  # e.g. you might do metadata_headers({}, {'myname' => 'elliot', 'myage' => 36})
-  # to add two headers to the request:
-  # 
-  #   x-amz-meta-myname: elliot
-  #   x-amz-meta-myage: 36
-  # 
-  # Returns headers with the metadata headers appended.
-  def metadata_headers(headers, metadata={})
-    unless metadata.empty?
-      metadata.each { |key, value| headers[METADATA_PREFIX + key] = value.to_s }
-    end
-    headers
-  end
-
-  # Add a canned ACL setter header.
-  def canned_acl_header(canned_acl, headers={})
-    unless canned_acl.nil?
-      unless CANNED_ACLS.include?(canned_acl)
-        raise S33rException::UnsupportedCannedACL, "The canned ACL #{canned_acl} is not supported"
-      end
-      headers[AWS_HEADER_PREFIX + 'acl'] = canned_acl
-    end
-    headers
-  end
-
-  # Guess a file's mime type.
-  # If the mime_type for a file cannot be guessed, "text/plain" is used.
-  def guess_mime_type(file_name)
-    mime_type = MIME::Types.type_for(file_name)[0]
-    mime_type ||= MIME::Types['text/plain'][0]
-    mime_type
-  end
-
-  # Ensure that a bucket_name is well-formed (no leading or trailing slash).
-  def bucket_name_valid?(bucket_name)
-    if ('/' == bucket_name[0,1] || '/' == bucket_name[-1,1])
-      raise S33rException::MalformedBucketName, "Bucket name cannot have a leading or trailing slash"
-    end
-  end
-
-  # Convert a hash of name/value pairs to querystring variables.
-  # Name for a variable can be a string or symbol.
-  def generate_querystring(pairs={})
-    str = ''
-    if pairs.size > 0
-      str += "?" + pairs.map { |key, value| "#{key}=#{CGI::escape(value.to_s)}" }.join('&')
-    end
-    str
-  end
-  
-  # Return the location of the ACL for a resource.
-  def s3_acl_path(bucket_name, resource_key)
-    s3_path(bucket_name, resource_key) + "?acl"
-  end
-  
-  # Return the location of the logging definition for a resource.
-  def s3_logging_path(bucket_name, resource_key)
-    s3_path(bucket_name, resource_key) + "?logging"
-  end
-  
-  # Prepend scheme and HOST to path.
-  def s3_url(path)
-    "http://" + HOST + path
-  end
-  
-  # Public readable URL for a bucket and resource.
-  def s3_public_url(bucket_name, resource_key='')
-    path = s3_path(bucket_name, resource_key)
-    s3_url(path)
-  end
-  
-  # Returns the path for this bucket and key.
-  def s3_path(bucket_name, resource_key)
-    path = '/' + bucket_name
-    path += '/' + resource_key unless '' == resource_key or resource_key.nil?
-    path
-  end
-  
-  # Generate a get-able URL for an S3 resource key which passes authentication in querystring.
-  # 
-  # int +expires+: when the URL expires (seconds since the epoch); NB you can use S33r.parse_expiry 
-  # to generate a suitable value from a string.
-  def s3_authenticated_url(aws_access_key, aws_secret_access_key, bucket_name, resource_key, 
-  expires=nil)
-    path = s3_path(bucket_name, resource_key)
-    expires = S33r.parse_expiry(expires)
-    
-    canonical_string = generate_canonical_string('GET', path, {}, expires)
-    signature = generate_signature(aws_secret_access_key, canonical_string)
-    
-    querystring = generate_querystring({ 'Signature' => signature, 'Expires' => expires,
-    'AWSAccessKeyId' => aws_access_key })
-    
-    return s3_url(path) + querystring
-  end
-  
-  # Turn keys in a hash +hsh+ into symbols.
-  # 
-  # Returns a hash with 'symbolised' keys.
-  def S33r.keys_to_symbols(hsh)
-    symbolised = hsh.inject({}) do |symbolised, key_value|
-      symbolised.merge({key_value[0].to_sym => key_value[1]})
-    end
-    symbolised
-  end
-  
-  # Parse an expiry date to seconds since the epoch.
-  # 
-  # +expires+ can be set to 'forever' to get a time 20 years in the future;
-  # 'default' to use the current time + DEFAULT_EXPIRY_SECS;
-  # or to a specific date (parseable by ParseDate); or to an integer
-  # representing seconds since the epoch.
-  # 
-  # TODO: testing for this
-  def S33r.parse_expiry(expires)
-    base_expires = Time.now.to_i
-    if 'forever' == expires
-      # 20 years (same as forever in computer terms)
-      expires = base_expires + (60 * 60 * 24 * 365.25 * 20).to_i
-    elsif 'default' == expires
-      # default to DEFAULT_EXPIRY_SECS seconds from now if expires not set
-      expires = base_expires + DEFAULT_EXPIRY_SECS
-    elsif expires.is_a?(String)
-      datetime_parts = ParseDate.parsedate(expires)
-      expires = Time.gm(*datetime_parts).to_i
-    end
-    expires
-  end
-  
-  # Remove the namespace declaration from S3 XML response bodies (libxml
-  # isn't fond of it).
-  def S33r.remove_namespace(xml_in)
-    namespace = S33r::RESPONSE_NAMESPACE_URI.gsub('/', '\/')
-    xml_in.gsub(/ xmlns="#{namespace}"/, '')
-  end
-end

data/lib/s33r/logging.rb

@@ -1,43 +0,0 @@
-require 'rubygems'
-require_gem 'builder'
-require File.join(File.dirname(__FILE__), 'libxml_loader')
-
-module S33r
-  # For manipulating logging directives on resources
-  # (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/LoggingHowTo.html).
-  # 
-  # Creating a LoggingResource instance using new and no arguments will generate a "blank" instance;
-  # this can be put to the ?logging URL for a resource to remove logging from it.
-  class LoggingResource
-    attr_reader :log_target, :log_prefix
-  
-    def initialize(log_target=nil, log_prefix=nil)
-      @log_target = log_target
-      @log_prefix = log_prefix
-    end
-    
-    # Generate a BucketLoggingStatus XML document for putting to the ?logging
-    # URL for a resource.
-    # 
-    #-- TODO: test generates correct XML
-    def to_xml
-      xml_str = ""
-      xml = Builder::XmlMarkup.new(:target => xml_str, :indent => 0)
-      
-      xml.instruct!
-      
-      # BucketLoggingStatus XML.
-      xml.BucketLoggingStatus({"xmlns" => RESPONSE_NAMESPACE_URI}) {
-        unless @log_target.nil? and @log_prefix.nil?
-          xml.LoggingEnabled {
-            xml.TargetBucket @log_target
-            xml.TargetPrefix @log_prefix
-          }
-        end
-      }
-      
-      xml_str
-    end
-    
-  end
-end

data/lib/s33r/named_bucket.rb

@@ -1,148 +0,0 @@
-#-- TODO: manipulation of bucket_listing through options hash, e.g. fetch particular page,
-#-- specify a prefix and/or delimiter
-
-require File.join(File.dirname(__FILE__), 'client')
-require File.join(File.dirname(__FILE__), 's3_obj')
-require File.join(File.dirname(__FILE__), 's33r_exception')
-
-module S33r
-  # Wraps the S33r::Client class to make it more convenient for use with a single bucket.
-  class NamedBucket < Client
-    attr_accessor :name, :strict, :public_contents, :dump_requests
-    
-    # Initialize an instance from a config_file. The config. file can include a separate
-    # +options+ section specifying options specific to NamedBucket instances (see the initialize method
-    # for more details).
-    # Other options are as for S33r::Client.init.
-    def NamedBucket.init(config_file)
-      aws_access_key, aws_secret_access_key, options = super.class.load_config(config_file)
-      NamedBucket.new(aws_access_key, aws_secret_access_key, options)
-    end
-  
-    # Initialize a NamedBucket instance.
-    # 
-    # +options+ is a hash of options for this instance:
-    # * <tt>:default_bucket => 'xxxx'</tt>: name of the bucket this client is attached to.
-    # * <tt>:public_contents => true</tt>: all items put into bucket are made public (can be overridden per request).
-    # * <tt>:strict => true</tt>: check whether the bucket exists before attempting to initialize; initialization \
-    # fails if the bucket does not exist
-    def initialize(aws_access_key, aws_secret_access_key, options={}, &block)    
-      super(aws_access_key, aws_secret_access_key, options)
-      
-      @name = options[:default_bucket]
-      if @name.nil?
-        raise S33rException::MissingBucketName, "NamedBucket cannot be initialised without specifying\
-        a :default_bucket option"
-      end
-
-      # all content inside the bucket should be created as public-read
-      @public_contents = (true == options[:public_contents])
-      @client_headers.merge!(canned_acl_header('public-read')) if @public_contents
-      
-      @strict = (true == options[:strict])
-      if @strict && !bucket_exists?(@name)
-        raise S33rException::MissingResource, "Non-existent bucket #{@bucket_name} specified"
-      end
-      
-      yield self if block_given?
-    end
-    
-    # Are all objects added to this bucket made public by default?
-    def public_contents?
-      @public_contents
-    end
-    
-    # Is this a strict bucket (i.e. the target bucket must exist on S3)?
-    def strict?
-      @strict
-    end
-    
-    # Get a single object from a bucket as an S3Object.
-    # 
-    # To get a bare object (with no content):
-    #   
-    #   bucket['key']
-    #   
-    # To get the object and load its content:
-    # 
-    #   bucket['key', :load]
-    def [](key, eager=false)
-      obj = listing.contents[key]
-      obj.named_bucket = self
-      obj.load if :load == eager
-      obj
-    end
-    
-    # Get a raw response for a key inside the bucket.
-    def get_raw(key, headers={})
-      get_resource(@name, key, headers)
-    end
-
-    # Get a BucketListing instance for the content of this bucket.
-    # Uses the Client.list_bucket method to get the listing.
-    def listing
-      list_bucket(@name)[1]
-    end
-    
-    # Does this bucket exist?
-    # Returns true if the bucket this NamedBucket is mapped to exists.
-    def exists?
-      bucket_exists?(@name)
-    end
-    
-    # Delete the bucket.
-    def destroy(headers={}, options={})
-      delete_bucket(@name, headers, options)
-    end
-    
-    # Get a pretty list of the keys in the bucket.
-    def keys
-      listing.pretty
-    end
-    
-    # List content of the bucket, and attach each item to this NamedBucket
-    # instance as it is yielded (to enable easier manipulation directly from the S3Object).
-    # Note that the objects are incomplete, as the data associated with them has not been
-    # "got" from S3 yet.
-    def each_object
-      listing.contents.each_value { |obj| obj.named_bucket = self; yield obj }
-    end
-    
-    # Does the given key exist in the bucket?
-    # Returns boolean
-    def key_exists?(key)
-      resource_exists?(@name, key)
-    end
-
-    # Put a string into a key inside the bucket.
-    def put_text(string, resource_key, headers={})
-      super(string, @name, resource_key, headers)
-    end
-
-    # Put a file into the bucket.
-    def put_file(filename, resource_key=nil, headers={}, options={})
-      super(filename, @name, resource_key, headers, options)
-    end
-    
-    # Put a generic stream (e.g. from a file handle) into the bucket.
-    def put_stream(data, resource_key, headers={})
-      put_resource(@name, resource_key, data, headers)
-    end
-    
-    # Delete an object from the bucket.
-    # NB S3 doesn't discriminate between successfully deleting a key
-    # and trying to delete a non-existent key (both return a 204).
-    # If you want to test for existence first, use key_exists?.
-    def delete(key, headers={})
-      delete_resource(@name, key, headers)
-    end
-    
-    # Generate an authenticated URL (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/) 
-    # for an object inside this bucket.
-    # 
-    # +expires+: time in secs since the epoch when the link should become invalid.
-    def s3_authenticated_url(resource_key, expires=(Time.now.to_i + DEFAULT_EXPIRY_SECS))
-      super(@aws_access_key, @aws_secret_access_key, @name, resource_key, expires)
-    end
-  end
-end