mss-sdk 1.0.0

data/lib/mss/s3/config.rb

@@ -0,0 +1,60 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+MSS::Core::Configuration.module_eval do
+
+  add_service 'S3', 's3', 's3'
+
+  add_option :s3_force_path_style, false, :boolean => true
+
+  add_option :s3_multipart_threshold, 16 * 1024 * 1024
+
+  add_option :s3_multipart_min_part_size, 5 * 1024 * 1024
+
+  add_option :s3_multipart_max_parts, 10000
+
+  add_option :s3_server_side_encryption, nil
+
+  add_option :s3_encryption_key, nil
+
+  add_option :s3_encryption_materials_location, :metadata
+
+  add_option :s3_encryption_matdesc, '{}'
+
+  add_option :s3_storage_class, 'STANDARD'
+
+  add_option :s3_cache_object_attributes, false, :boolean => true
+
+  add_option :s3_signature_version do |config, value|
+    v3_regions = %w(
+      us-east-1
+      us-west-1
+      us-west-2
+      ap-northeast-1
+      ap-southeast-1
+      ap-southeast-2
+      sa-east-1
+      eu-west-1
+      us-gov-west-1
+    )
+    if value
+      value
+    elsif config.s3 && config.s3[:signature_version]
+      config.s3[:signature_version]
+    elsif v3_regions.include?(config.s3_region)
+      :v3
+    else
+      :v4
+    end
+  end
+
+end

data/lib/mss/s3/cors_rule.rb

@@ -0,0 +1,107 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+
+    # Represents a single CORS rule for an S3 {Bucket}.
+    #
+    # @example
+    #
+    #   rule = bucket.cors.first
+    #   rule.allowed_methods #=> ['GET', 'HEAD']
+    #   rule.allowed_origins #=> ['*']
+    #
+    # @see CORSRuleCollection
+    class CORSRule
+
+      # @param [Hash] options A hash of the rule details.
+      #
+      # @option options [String] :id A unique identifier for the rule. The ID
+      #   value can be up to 255 characters long. The IDs help you find
+      #   a rule in the configuration.
+      #
+      # @option options [required,Array<String>] :allowed_methods A list of HTTP
+      #   methods that you want to allow the origin to execute.
+      #   Each rule must identify at least one method.
+      #
+      # @option options [required,Array<String>] :allowed_origins A list of
+      #   origins you want to allow cross-domain requests from. This can
+      #   contain at most one * wild character.
+      #
+      # @option options [Array<String>] :allowed_headers A list of headers
+      #   allowed in a pre-flight OPTIONS request via the
+      #   Access-Control-Request-Headers header. Each header name
+      #   specified in the Access-Control-Request-Headers header must
+      #   have a corresponding entry in the rule.
+      #
+      #   Amazon S3 will send only the allowed headers in a response
+      #   that were requested. This can contain at most one '*' wild
+      #   character.
+      #
+      # @option options [Array<String>] :max_age_seconds The time in
+      #   seconds that your browser is to cache the pre-flight response for
+      #   the specified resource.
+      #
+      # @option options [Array<String>] :expose_headers One or more headers in
+      #    the response that you want customers to be able to access
+      #    from their applications (for example, from a JavaScript
+      #    XMLHttpRequest object).
+      #
+      def initialize options = {}
+        @id = options[:id]
+        @allowed_methods = options[:allowed_methods] || []
+        @allowed_origins = options[:allowed_origins] || []
+        @allowed_headers = options[:allowed_headers] || []
+        @max_age_seconds = options[:max_age_seconds]
+        @expose_headers = options[:expose_headers] || []
+      end
+
+      # @return [String,nil] A user supplied unique identifier for this role.
+      #   Set this when you set or add roles via {CORSRuleCollection}.
+      attr_reader :id
+
+      # @return [Array<String>] A list of HTTP methods (GET, POST, etc) this
+      #   role authorizes.
+      attr_reader :allowed_methods
+
+      # @return [Array<String>] The list of origins allowed to make
+      #   cross-domain requests to the bucket.
+      attr_reader :allowed_origins
+
+      # @return [Array<String>] A list of headers allowed in the pre-flight
+      #   OPTIONS request.
+      attr_reader :allowed_headers
+
+      # @return [Integer,nil] The time in seconds the browser may cache the
+      #   pre-flight response.
+      attr_reader :max_age_seconds
+
+      # @return [Array<String>] The headers that may be accessed cross-domain.
+      attr_reader :expose_headers
+
+      # @return [Hash]
+      def to_h
+        h = {}
+        h[:id] = id if id
+        h[:allowed_methods] = allowed_methods
+        h[:allowed_origins] = allowed_origins
+        h[:allowed_headers] = allowed_headers unless allowed_headers.empty?
+        h[:max_age_seconds] = max_age_seconds if max_age_seconds
+        h[:expose_headers] = expose_headers unless expose_headers.empty?
+        h
+      end
+
+    end
+
+  end
+end

data/lib/mss/s3/cors_rule_collection.rb

@@ -0,0 +1,193 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module MSS
+  class S3
+
+    # Manages the CORS rules for a single bucket.
+    #
+    # ## Getting Rules
+    #
+    # To get the CORS rules for a bucket, use the {Bucket#cors} method.  This
+    # returns a CORSRuleCollection for the bucket.  The collection is
+    # enumerable.
+    #
+    #     # enumerating all rules for a buck
+    #     bucket.cors.each do |rule|
+    #       # rule is a CORSRule object
+    #     end
+    #
+    # ## Setting Rules
+    #
+    # You can set the rules for a bucket (replacing all existing rules) using
+    # the {#set} method.
+    #
+    #     # accepts a list of one or more rules
+    #     bucket.rules.set(rule1, rule2)
+    #
+    #     # rules can be an array of rules
+    #     bucket.rules.set(rules)
+    #
+    #     # passing an empty list or array removes all rules
+    #     bucket.rules.set([])
+    #     bucket.rules.clear # does the same thing
+    #
+    # Each rule can be a Hash, a {CORSRule} or another {CORSRuleCollection}.
+    # See {Client#put_bucket_cors} for a list of keys for a rule hash.
+    #
+    # ## Adding Rules
+    #
+    # Adding rules is the same as setting rules.  Rules you add will be
+    # appended to the end of the existing list of rules.
+    #
+    #     # add one or more rules
+    #     bucket.rules.add(rules)
+    #
+    # ## Deleting Rules
+    #
+    # To remove a rule, use the {#delete_if} method.
+    #
+    #     # delete rules that allow access from any domain
+    #     bucket.cors.delete_if{|rule| rule.allowed_origins.include?('*')
+    class CORSRuleCollection
+
+      include Core::Collection::Simple
+
+      # @param [Bucket] bucket
+      # @param [Hash] options
+      def initialize bucket, options = {}
+        @bucket = bucket
+        super
+      end
+
+      # @return [Bucket]
+      attr_reader :bucket
+
+      # Replaces the CORS rules attached to this bucket.  You can pass
+      # one or more rules as an array or a list.
+      #
+      #     # replace all exisitng rules with a single rule
+      #     bucket.cors.set(
+      #       :allowed_methods => %w(GET),
+      #       :allowed_origins => %w(http://*.mydomain.com),
+      #       :max_age_seconds => 3600)
+      #
+      # If you pass an empty array, all of the rules will be removed from
+      # the bucket.
+      #
+      #      # these two lines are equivilent
+      #      bucket.cors.clear
+      #      bucket.cors.set([])
+      #
+      # @param [Hash,CORSRule,CORSRuleCollection] rules A list or array
+      #   of one or more rules to set.  Each rule may be a Hash, a CORSRule
+      #   or a CORSRuleCollection.
+      #
+      # @return [nil]
+      #
+      def set *rules
+
+        raise ArgumentError, 'expected one or more rules' if rules.empty?
+
+        if rules == [[]]
+          self.clear
+        else
+          rules = rule_hashes(rules)
+          client.put_bucket_cors(:bucket_name => bucket.name, :rules => rules)
+        end
+
+        nil
+
+      end
+
+      # Add one or more CORS rules to this bucket.
+      #
+      #     # adding a single rule as a hash
+      #     bucket.cors.add(
+      #       :allowed_methods => %w(GET HEAD),
+      #       :allowed_origins => %w(*),
+      #       :max_age_seconds => 3600)
+      #
+      # You can add multiple rules in a single call:
+      #
+      #     # each rule may be a hash, CORSRule or a CORSRuleCollection,
+      #     bucket.cors.add(rules)
+      #
+      #     # alternatively you can pass a list of rules
+      #     bucket.cors.add(rule1, rule2, ...)
+      #
+      # @param (see #set)
+      # @return (see #set)
+      def add *rules
+        self.set(self, *rules)
+      end
+      alias_method :create, :add
+
+      # Deletes every rule for which the block evaluates to `true`.
+      #
+      # @example Remove all rules that are open to the 'world'
+      #
+      #   bucket.cors.delete_if{|rule| rule.allowed_origins.include?('*') }
+      #
+      # @yield [rule]
+      # @yieldparam [CORSRule] rule
+      # @yieldreturn [Boolean] Return `true` for each rule you want to delete.
+      # @return (see #set)
+      def delete_if &block
+        rules = []
+        self.each do |rule|
+          rules << rule unless yield(rule)
+        end
+        self.set(rules)
+      end
+
+      # Removes all CORS rules attached to this bucket.
+      #
+      # @example
+      #
+      #   bucket.cors.count #=> 3
+      #   bucket.cors.clear
+      #   bucket.cors.count #=> 0
+      #
+      # @return [nil]
+      def clear
+        client.delete_bucket_cors(:bucket_name => bucket.name)
+        nil
+      end
+
+      protected
+
+      def _each_item options
+        resp = client.get_bucket_cors(options.merge(:bucket_name => bucket.name))
+        resp.data[:rules].each do |rule|
+          yield(CORSRule.new(rule))
+        end
+      rescue MSS::S3::Errors::NoSuchCORSConfiguration
+        # no cors rules exist for this bucket, nothing to yield
+      end
+
+      def rule_hashes rule
+        case rule
+        when Hash then rule
+        when CORSRule then rule.to_h
+        when CORSRuleCollection then rule.map(&:to_h)
+        when Array then rule.map{|r| rule_hashes(r) }.flatten
+        else
+          msg = "Expected one or more CORSRule, CORSRuleCollection or hash"
+          msg << ", got #{rule.class.name}"
+          raise ArgumentError, msg
+        end
+      end
+
+    end
+  end
+end

data/lib/mss/s3/data_options.rb

@@ -0,0 +1,190 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'pathname'
+
+module MSS
+  class S3
+
+    # Used by S3#S3Object and S3::Client to accept options with
+    # data that should be uploaded (streamed).
+    # @api private
+    module DataOptions
+
+      protected
+
+      # @return [Hash] Returns a hash of options with a :data option that
+      #   responds to #read and #eof?.
+      def compute_write_options *args, &block
+
+        options = convert_args_to_options_hash(*args)
+
+        validate_data!(options, &block)
+
+        rename_file_to_data(options)
+
+        convert_data_to_io_obj(options, &block)
+
+        try_to_determine_content_length(options)
+
+        options
+
+      end
+
+      # Converts an argument list into a single hash of options.  Treats
+      # non-hash arguments in the first position as a data option.
+      def convert_args_to_options_hash *args
+        case args.count
+        when 0 then {}
+        when 1 then args[0].is_a?(Hash) ? args[0] : { :data => args[0] }
+        when 2 then args[1].merge(:data => args[0])
+        else
+          msg = "expected 0, 1 or 2 arguments, got #{args.count}"
+          raise ArgumentError, msg
+        end
+      end
+
+      # Moves options[:file] to options[:data].  If this option is a string
+      # then it is treated as a file path and is converted to an open file.
+      def rename_file_to_data options
+        if file = options.delete(:file)
+          options[:data] = file.is_a?(String) ? open_file(file) : file
+        end
+      end
+
+      # Converts the :data option to an IO-like object.  This allows us
+      # to always perform streaming uploads.
+      def convert_data_to_io_obj options, &block
+
+        data = options.delete(:data)
+
+        if block_given?
+          options[:data] = IOProxy.new(block)
+        elsif data.is_a?(String)
+          data = data.dup if data.frozen?
+          data.force_encoding("BINARY") if data.respond_to?(:force_encoding)
+          options[:data] = StringIO.new(data)
+        elsif data.is_a?(Pathname)
+          options[:data] = open_file(data.to_s)
+        elsif io_like?(data)
+          options[:data] = data
+        else
+          msg = "invalid :data option, expected a String, Pathname or "
+          msg << "an object that responds to #read and #eof?"
+          raise ArgumentError, msg
+        end
+
+      end
+
+      # Attempts to determine the content length of the :data option.
+      # This is only done when a content length is not already provided.
+      def try_to_determine_content_length options
+        unless options[:content_length]
+
+          data = options[:data]
+
+          length = case
+            when data.respond_to?(:path) && data.path then File.size(data.path)
+            when data.respond_to?(:bytesize) then data.bytesize
+            when data.respond_to?(:size)     then data.size
+            when data.respond_to?(:length)   then data.length
+            else nil
+          end
+
+          options[:content_length] = length if length
+
+        end
+      end
+
+      def validate_data! options, &block
+
+        data = options[:data]
+        file = options[:file]
+
+        raise ArgumentError, 'Object data passed multiple ways.' if
+          [data, file, block].compact.count > 1
+
+        data = file if file
+
+        return if block_given?
+        return if data.kind_of?(String)
+        return if data.kind_of?(Pathname)
+        return if io_like?(data)
+
+        msg = ":data must be provided as a String, Pathname, File, or "
+        msg << "an object that responds to #read and #eof?"
+        raise ArgumentError, msg
+
+      end
+
+      # @return [Boolean] Returns `true` if the object responds to
+      #   `#read` and `#eof?`.
+      def io_like? io
+        io.respond_to?(:read) and io.respond_to?(:eof?)
+      end
+
+      # @param [String] path Path to a file on disk.
+      # @return [File] Given a path string, returns an open File.
+      def open_file path
+        Core::ManagedFile.open(path)
+      end
+
+      # A utility class that turns a block (with 2 args) into an
+      # IO object that responds to #read and #eof.
+      # @api private
+      class IOProxy
+
+        def initialize write_block
+          unless write_block.arity == 2
+            msg = "a write block must accept 2 yield params: a buffer and "
+            msg << "a number of bytes to write"
+            raise ArgumentError, msg
+          end
+          @write_block = write_block
+          @eof = false
+        end
+
+        def read bytes = nil, output_buffer = nil
+          data = if bytes
+            (@eof) ? nil : read_chunk(bytes)
+          else
+            (@eof) ? ""  : read_all
+          end
+          output_buffer ? output_buffer.replace(data || '') : data
+        end
+
+        def eof?
+          @eof
+        end
+
+        protected
+
+        def read_chunk bytes
+          buffer = StringIO.new
+          @write_block.call(buffer, bytes)
+          buffer.rewind
+          @eof = true if buffer.size < bytes
+          buffer.read
+        end
+
+        def read_all
+          buffer = StringIO.new
+          buffer << read_chunk(1024 * 1024 * 5) until @eof
+          buffer.rewind
+          buffer.read
+        end
+
+      end
+
+    end
+  end
+end

data/lib/mss/s3/encryption_utils.rb

@@ -0,0 +1,145 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'openssl'
+
+module MSS
+  class S3
+    # @api private
+    module EncryptionUtils
+
+      protected
+
+      UNSAFE_MSG = "Unsafe encryption, data is longer than key length"
+
+      # @param [OpenSSL::PKey::RSA, String] key Key used to encrypt.
+      #
+      # @param [String] data Data to be encrypted.
+      #
+      # @note Use check_encryption_materials before this method to check
+      #   formatting of keys.
+      # @note This should not be used for data longer than the key length as
+      #   it will not be cryptographically safe.
+      #
+      # @return [String] Returns the data encrypted with the key given.
+      def encrypt data, key
+        rsa = OpenSSL::PKey::RSA
+        data_cipher_size = get_cipher_size(data.length)
+
+        # Encrypting data key
+        case key
+        when rsa # Asymmetric encryption
+          warn UNSAFE_MSG if key.public_key.n.num_bits < data_cipher_size
+          key.public_encrypt(data)
+        when String             # Symmetric encryption
+          warn UNSAFE_MSG if get_cipher_size(key.length) < data_cipher_size
+          cipher = get_aes_cipher(:encrypt, :ECB, key)
+          cipher.update(data) + cipher.final
+        end
+      end
+
+      # @param [OpenSSL::PKey::RSA, String] key Key used to encrypt.
+      #
+      # @param [String] data Data to be encrypted.
+      #
+      # @note Use check_encryption_materials before this method to check
+      #   formatting of keys
+      #
+      # @return [String] Returns the data decrypted with the key given.
+      def decrypt data, key
+        rsa = OpenSSL::PKey::RSA
+        begin
+          case key
+          when rsa # Asymmetric Decryption
+              key.private_decrypt(data)
+          when String             # Symmetric Decryption
+              cipher = get_aes_cipher(:decrypt, :ECB, key)
+              cipher.update(data) + cipher.final
+          end
+        rescue OpenSSL::Cipher::CipherError
+          raise RuntimeError, "decryption failed, incorrect key?"
+        end
+      end
+
+      # Checks for any formatting problems for keys and initialization vectors
+      #   supported with EncryptionUtils.
+      def check_encryption_materials mode, key
+        rsa = OpenSSL::PKey::RSA
+        case key
+        when rsa
+          unless key.private? or mode == :encrypt
+            msg = "invalid key, #{rsa} requires a private key"
+            raise ArgumentError, msg
+          end
+        when String # no problem
+        else
+          msg = "invalid key, must be an #{rsa} or a cipher key string"
+          raise ArgumentError, msg
+        end
+      end
+
+      # @param [OpenSSL::Cipher] cipher The cipher with configured key and iv.
+      #
+      # @yield [String, String] key_iv_pair A randomly generated key, iv pair
+      #   for use with the given cipher.  Sets the key and iv on the cipher.
+      def generate_aes_key cipher, &block
+        key_iv_pair = [cipher.random_key, cipher.random_iv]
+        yield(key_iv_pair) if block_given?
+      end
+
+      # @param [Symbol] mode The encryption/decryption mode.  Valid inputs are
+      #   :encrypt or :decrypt
+      #
+      # @param [String] key Key for the cipher.
+      #
+      # @param [String] iv IV for the cipher.
+      #
+      # @return [OpenSSL::Cipher] Will return a configured `OpenSSL::Cipher`.
+      def get_aes_cipher mode, block_mode, key = nil, iv = nil
+
+        # If no key given, default to 256 bit
+        cipher_size = (key) ? get_cipher_size(key.length) : 256
+
+        cipher = OpenSSL::Cipher.new("AES-#{cipher_size}-#{block_mode}")
+
+        (mode == :encrypt) ? cipher.encrypt : cipher.decrypt
+        cipher.key = key if key
+        cipher.iv  = iv  if iv
+        cipher
+      end
+
+      # @param  [Integer] size Size of data given.
+      # @return [Integer] Returns the AES encrypted size based on a given size.
+      def get_encrypted_size size
+        # The next multiple of 16
+        ((size / 16) + 1) * 16
+      end
+      module_function :get_encrypted_size
+
+      private
+
+      # @param  [Fixnum] key_length Length of the key given.
+      # @return [Fixnum] Returns the cipher size based on the key length.
+      def get_cipher_size(key_length)
+        case key_length
+        when 32 then 256
+        when 24 then 192
+        when 16 then 128
+        else
+          msg = "invalid key, symmetric key required to be 16, 24, or 32 bytes "
+          msg << "in length, saw length #{key_length}"
+          raise ArgumentError, msg
+        end
+      end
+    end
+  end
+end