aws-sdk-v1 1.52.0

data/lib/aws/s3/data_options.rb

@@ -0,0 +1,191 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'pathname'
+
+module AWS
+  class S3
+
+    # Used by S3#S3Object and S3::Client to accept options with
+    # data that should be uploaded (streamed).
+    # @api private
+    module DataOptions
+
+      protected
+
+      # @return [Hash] Returns a hash of options with a :data option that
+      #   responds to #read and #eof?.
+      def compute_write_options *args, &block
+
+        options = convert_args_to_options_hash(*args)
+
+        validate_data!(options, &block)
+
+        rename_file_to_data(options)
+
+        convert_data_to_io_obj(options, &block)
+
+        try_to_determine_content_length(options)
+
+        options
+
+      end
+
+      # Converts an argument list into a single hash of options.  Treats
+      # non-hash arguments in the first position as a data option.
+      def convert_args_to_options_hash *args
+        case args.count
+        when 0 then {}
+        when 1 then args[0].is_a?(Hash) ? args[0] : { :data => args[0] }
+        when 2 then args[1].merge(:data => args[0])
+        else
+          msg = "expected 0, 1 or 2 arguments, got #{args.count}"
+          raise ArgumentError, msg
+        end
+      end
+
+      # Moves options[:file] to options[:data].  If this option is a string
+      # then it is treated as a file path and is converted to an open file.
+      def rename_file_to_data options
+        if file = options.delete(:file)
+          options[:data] = file.is_a?(String) ? open_file(file) : file
+        end
+      end
+
+      # Converts the :data option to an IO-like object.  This allows us
+      # to always perform streaming uploads.
+      def convert_data_to_io_obj options, &block
+
+        data = options.delete(:data)
+
+        if block_given?
+          options[:data] = IOProxy.new(block)
+        elsif data.is_a?(String)
+          data = data.dup if data.frozen?
+          data.force_encoding("BINARY") if data.respond_to?(:force_encoding)
+          options[:data] = StringIO.new(data)
+        elsif data.is_a?(Pathname)
+          options[:data] = open_file(data.to_s)
+        elsif io_like?(data)
+          options[:data] = data
+        else
+          msg = "invalid :data option, expected a String, Pathname or "
+          msg << "an object that responds to #read and #eof?"
+          raise ArgumentError, msg
+        end
+
+      end
+
+      # Attempts to determine the content length of the :data option.
+      # This is only done when a content length is not already provided.
+      def try_to_determine_content_length options
+        unless options[:content_length]
+
+          data = options[:data]
+
+          length = case
+            when data.respond_to?(:path) && data.path then File.size(data.path)
+            when data.respond_to?(:bytesize) then data.bytesize
+            when data.respond_to?(:size)     then data.size
+            when data.respond_to?(:length)   then data.length
+            else nil
+          end
+
+          options[:content_length] = length if length
+
+        end
+      end
+
+      def validate_data! options, &block
+
+        data = options[:data]
+        file = options[:file]
+
+        raise ArgumentError, 'Object data passed multiple ways.' if
+          [data, file, block].compact.count > 1
+
+        data = file if file
+
+        return if block_given?
+        return if data.kind_of?(String)
+        return if data.kind_of?(Pathname)
+        return if io_like?(data)
+
+        msg = ":data must be provided as a String, Pathname, File, or "
+        msg << "an object that responds to #read and #eof?"
+        raise ArgumentError, msg
+
+      end
+
+      # @return [Boolean] Returns `true` if the object responds to
+      #   `#read` and `#eof?`.
+      def io_like? io
+        io.respond_to?(:read) and io.respond_to?(:eof?)
+      end
+
+      # @param [String] path Path to a file on disk.
+      # @return [File] Given a path string, returns an open File.
+      def open_file path
+        Core::ManagedFile.open(path)
+      end
+
+      # A utility class that turns a block (with 2 args) into an
+      # IO object that responds to #read and #eof.
+      # @api private
+      class IOProxy
+
+        def initialize write_block
+          unless write_block.arity == 2
+            msg = "a write block must accept 2 yield params: a buffer and "
+            msg << "a number of bytes to write"
+            raise ArgumentError, msg
+          end
+          @write_block = write_block
+          @eof = false
+        end
+
+        def read bytes = nil, output_buffer = nil
+          data = if bytes
+            (@eof) ? nil : read_chunk(bytes)
+          else
+            (@eof) ? ""  : read_all
+          end
+          output_buffer ? output_buffer.replace(data || '') : data
+        end
+
+        def eof?
+          @eof
+        end
+
+        protected
+
+        def read_chunk bytes
+          buffer = StringIO.new
+          @write_block.call(buffer, bytes)
+          buffer.rewind
+          @eof = true if buffer.size < bytes
+          buffer.read
+        end
+
+        def read_all
+          buffer = StringIO.new
+          buffer << read_chunk(1024 * 1024 * 5) until @eof
+          buffer.rewind
+          buffer.read
+        end
+
+      end
+
+    end
+  end
+end

data/lib/aws/s3/encryption_utils.rb

@@ -0,0 +1,146 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'openssl'
+
+module AWS
+  class S3
+    # @api private
+    module EncryptionUtils
+
+      protected
+
+      UNSAFE_MSG = "Unsafe encryption, data is longer than key length"
+
+      # @param [OpenSSL::PKey::RSA, String] key Key used to encrypt.
+      #
+      # @param [String] data Data to be encrypted.
+      #
+      # @note Use check_encryption_materials before this method to check
+      #   formatting of keys.
+      # @note This should not be used for data longer than the key length as
+      #   it will not be cryptographically safe.
+      #
+      # @return [String] Returns the data encrypted with the key given.
+      def encrypt data, key
+        rsa = OpenSSL::PKey::RSA
+        data_cipher_size = get_cipher_size(data.length)
+
+        # Encrypting data key
+        case key
+        when rsa # Asymmetric encryption
+          warn UNSAFE_MSG if key.public_key.n.num_bits < data_cipher_size
+          key.public_encrypt(data)
+        when String             # Symmetric encryption
+          warn UNSAFE_MSG if get_cipher_size(key.length) < data_cipher_size
+          cipher = get_aes_cipher(:encrypt, :ECB, key)
+          cipher.update(data) + cipher.final
+        end
+      end
+
+      # @param [OpenSSL::PKey::RSA, String] key Key used to encrypt.
+      #
+      # @param [String] data Data to be encrypted.
+      #
+      # @note Use check_encryption_materials before this method to check
+      #   formatting of keys
+      #
+      # @return [String] Returns the data decrypted with the key given.
+      def decrypt data, key
+        rsa = OpenSSL::PKey::RSA
+        begin
+          case key
+          when rsa # Asymmetric Decryption
+              key.private_decrypt(data)
+          when String             # Symmetric Decryption
+              cipher = get_aes_cipher(:decrypt, :ECB, key)
+              cipher.update(data) + cipher.final
+          end
+        rescue OpenSSL::Cipher::CipherError
+          raise RuntimeError, "decryption failed, incorrect key?"
+        end
+      end
+
+      # Checks for any formatting problems for keys and initialization vectors
+      #   supported with EncryptionUtils.
+      def check_encryption_materials mode, key
+        rsa = OpenSSL::PKey::RSA
+        case key
+        when rsa
+          unless key.private? or mode == :encrypt
+            msg = "invalid key, #{rsa} requires a private key"
+            raise ArgumentError, msg
+          end
+        when String # no problem
+        else
+          msg = "invalid key, must be an #{rsa} or a cipher key string"
+          raise ArgumentError, msg
+        end
+      end
+
+      # @param [OpenSSL::Cipher] cipher The cipher with configured key and iv.
+      #
+      # @yield [String, String] key_iv_pair A randomly generated key, iv pair
+      #   for use with the given cipher.  Sets the key and iv on the cipher.
+      def generate_aes_key cipher, &block
+        key_iv_pair = [cipher.random_key, cipher.random_iv]
+        yield(key_iv_pair) if block_given?
+      end
+
+      # @param [Symbol] mode The encryption/decryption mode.  Valid inputs are
+      #   :encrypt or :decrypt
+      #
+      # @param [String] key Key for the cipher.
+      #
+      # @param [String] iv IV for the cipher.
+      #
+      # @return [OpenSSL::Cipher] Will return a configured `OpenSSL::Cipher`.
+      def get_aes_cipher mode, block_mode, key = nil, iv = nil
+
+        # If no key given, default to 256 bit
+        cipher_size = (key) ? get_cipher_size(key.length) : 256
+
+        cipher = OpenSSL::Cipher.new("AES-#{cipher_size}-#{block_mode}")
+
+        (mode == :encrypt) ? cipher.encrypt : cipher.decrypt
+        cipher.key = key if key
+        cipher.iv  = iv  if iv
+        cipher
+      end
+
+      # @param  [Integer] size Size of data given.
+      # @return [Integer] Returns the AES encrypted size based on a given size.
+      def get_encrypted_size size
+        # The next multiple of 16
+        ((size / 16) + 1) * 16
+      end
+      module_function :get_encrypted_size
+
+      private
+
+      # @param  [Fixnum] key_length Length of the key given.
+      # @return [Fixnum] Returns the cipher size based on the key length.
+      def get_cipher_size(key_length)
+        case key_length
+        when 32 then 256
+        when 24 then 192
+        when 16 then 128
+        else
+          msg = "invalid key, symmetric key required to be 16, 24, or 32 bytes "
+          msg << "in length, saw length #{key_length}"
+          raise ArgumentError, msg
+        end
+      end
+    end
+  end
+end

data/lib/aws/s3/errors.rb

@@ -0,0 +1,94 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  class S3
+
+    # This module contains exception classes for each of the error
+    # types that S3 can return.  You can use these classes to rescue
+    # specific errors, for example:
+    #
+    #     begin
+    #       S3.new.buckets.mybucket.
+    #         objects.myobj.write("HELLO")
+    #     rescue S3::Errors::NoSuchBucket => e
+    #       S3.new.buckets.create("mybucket")
+    #       retry
+    #     end
+    #
+    # All errors raised as a result of error responses from the
+    # service are instances of either {ClientError} or {ServerError}.
+    module Errors
+
+      # @api private
+      GRAMMAR = Core::XML::Grammar.customize
+
+      extend Core::LazyErrorClasses
+
+      class BatchDeleteError < StandardError
+
+        def initialize error_counts
+          @error_counts = error_counts
+          total = error_counts.values.inject(0) {|sum,count| sum + count }
+          super("Failed to delete #{total} objects")
+        end
+
+        # @return [Hash] Returns a hash of error codes and how many
+        #   objects failed with that code.
+        attr_reader :error_counts
+
+      end
+
+      # This error is special, because S3 does not (and must not
+      # according to RFC 2616) return a body with the HTTP response.
+      # The interface is the same as for any other client error.
+      class NotModified < AWS::Errors::Base
+
+        include AWS::Errors::ClientError
+
+        def initialize(req, resp)
+          super(req, resp, "NotModified", "Not Modified")
+        end
+
+      end
+
+      # This error is special, because S3 does not return a body with
+      # the HTTP response.  The interface is the same as for any other
+      # client error.
+      class NoSuchKey < AWS::Errors::Base
+
+        include AWS::Errors::ClientError
+
+        def initialize(req, resp, code = nil, message = nil)
+          super(req, resp, "NoSuchKey", "No Such Key")
+        end
+
+      end
+
+      # This error is special, because S3 must first retrieve the client
+      #   side encryption key in it's encrypted form before finding if the
+      #   key is incorrect.
+      class IncorrectClientSideEncryptionKey < AWS::Errors::Base
+
+        include AWS::Errors::ClientError
+
+        def initialize(msg)
+          super("",
+                "",
+                "IncorrectClientSideEncryptionKey",
+                msg)
+        end
+      end
+    end
+  end
+end

data/lib/aws/s3/multipart_upload.rb

@@ -0,0 +1,353 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'thread'
+
+module AWS
+  class S3
+
+    # Represents a multipart upload to an S3 object.  See
+    # {S3Object#multipart_upload} for a convenient way to initiate a
+    # multipart upload.
+    # 
+    # Note: After you initiate multipart upload and upload one or more 
+    # parts, you must either complete or abort multipart upload in order 
+    # to stop getting charged for storage of the uploaded parts. Only 
+    # after you either complete or abort multipart upload, Amazon S3 
+    # frees up the parts storage and stops charging you for the parts 
+    # storage.
+    class MultipartUpload
+
+      include Core::Model
+
+      class EmptyUploadError < StandardError; end
+      
+      # @api private
+      def initialize(object, id, options = {})
+        @id = id
+        @object = object
+
+        super
+
+        @completed_parts = {}
+        @increment_mutex = Mutex.new
+        @completed_mutex = Mutex.new
+        @last_part = 0
+      end
+
+      def bucket
+        object.bucket
+      end
+
+      def inspect
+        "<#{self.class}:#{object.bucket.name}/#{object.key}:#{id}>"
+      end
+      # @return [String] Returns the upload id.
+      attr_reader :id
+
+      alias_method :upload_id, :id
+
+      # @return [S3Object] Returns the object this upload is intended for.
+      attr_reader :object
+
+      # @return [Boolean] Returns true if both multipart uploads
+      #   represent the same object and upload.
+      def ==(other)
+        other.kind_of?(MultipartUpload) and
+          other.object == object and
+          other.id == id
+      end
+
+      alias_method :eql?, :==
+
+      # @return [Boolean] True if the upload exists.
+      def exists?
+        client.list_parts(base_opts)
+      rescue Errors::NoSuchUpload => e
+        false
+      else
+        true
+      end
+
+      # @return The upload initiator.  This object will have `:id`
+      #   and `:display_name` methods; if the initiator is an IAM
+      #   user, the `:id` method will return the ARN of the user, and
+      #   if the initiator is an AWS account, this method will return
+      #   the same data as {#owner}.
+      def initiator
+        client.list_parts(base_opts).initiator
+      end
+
+      # @return The upload owner.  This object will have `:id`
+      #   and `:display_name` methods.
+      def owner
+        client.list_parts(base_opts).owner
+      end
+
+      # @return [Symbol] The class of storage used to store the
+      #   uploaded object.  Possible values:
+      #
+      #   * `:standard`
+      #   * `:reduced_redundancy?`
+      def storage_class
+        client.list_parts(base_opts).storage_class.downcase.to_sym
+      end
+
+      # @return [Boolean] True if the uploaded object will be stored
+      #   with reduced redundancy.
+      def reduced_redundancy?
+        storage_class == :reduced_redundancy
+      end
+
+      # Aborts the upload.  After a multipart upload is aborted, no
+      # additional parts can be uploaded using that upload ID. The
+      # storage consumed by any previously uploaded parts will be
+      # freed. However, if any part uploads are currently in
+      # progress, those part uploads might or might not succeed. As
+      # a result, it might be necessary to abort a given multipart
+      # upload multiple times in order to completely free all
+      # storage consumed by all parts.
+      # @return [nil]
+      def abort
+        unless aborted?
+          client.abort_multipart_upload(base_opts)
+          @aborted = true
+        end
+        nil
+      end
+      alias_method :delete, :abort
+      alias_method :cancel, :abort
+
+      # @return [Boolean] True if the upload has been aborted.
+      # @see #abort
+      def aborted?
+        @aborted
+      end
+
+      # Uploads a part.
+      #
+      # @overload add_part(data, options = {})
+      #
+      #   @param data The data to upload.
+      #     Valid values include:
+      #
+      #       * A string
+      #       * A Pathname object
+      #       * Any object responding to `read` and `eof?`; the object
+      #         must support the following access methods:
+      #
+      #             read                     # all at once
+      #             read(length) until eof?  # in chunks
+      #
+      #         If you specify data this way, you must also include
+      #         the `:content_length` option.
+      #
+      #   @param [Hash] options Additional options for the upload.
+      #
+      #   @option options [Integer] :content_length If provided,
+      #     this option must match the total number of bytes written
+      #     to S3 during the operation.  This option is required if
+      #     `:data` is an IO-like object without a `size` method.
+      #
+      # @overload add_part(options)
+      #
+      #   @param [Hash] options Options for the upload.  Either
+      #     `:data` or `:file` is required.
+      #
+      #   @option options :data The data to upload.  Valid values
+      #     include:
+      #
+      #       * A string
+      #       * A Pathname object
+      #       * Any object responding to `read` and `eof?`; the object
+      #         must support the following access methods:
+      #
+      #              read                     # all at once
+      #              read(length) until eof?  # in chunks
+      #
+      #         If you specify data this way, you must also include
+      #         the `:content_length` option.
+      #
+      #   @option options [String] :file Can be specified instead of
+      #     `:data`; its value specifies the path of a file to
+      #     upload.
+      #
+      #   @option options [Integer] :content_length If provided,
+      #     this option must match the total number of bytes written
+      #     to S3 during the operation.  This option is required if
+      #     `:data` is an IO-like object without a `size` method.
+      #
+      #   @option options [Integer] :part_number The part number.
+      def add_part(data_or_options, options = {})
+        if data_or_options.kind_of?(Hash)
+          part_options = base_opts.merge(data_or_options)
+        else
+          part_options = base_opts.merge(:data => data_or_options)
+        end
+        part_options.merge!(options)
+
+        unless part_options[:part_number]
+          @increment_mutex.synchronize do
+            part_options[:part_number] = (@last_part += 1)
+          end
+        end
+        part_number = part_options[:part_number]
+
+        resp = client.upload_part(part_options)
+        @completed_mutex.synchronize do
+          @completed_parts[part_number] = {
+            :part_number => part_number,
+            :etag => resp.etag
+          }
+        end
+        UploadedPart.new(self, part_number)
+      end
+
+      # Copies a part.
+      #
+      #   @param [string] copy_source Full S3 name of source, ie bucket/key
+      #
+      #   @param [Hash] options Additional options for the copy.
+      #
+      #   @option options [Integer] :part_number The part number.
+      #
+      #   @option options [Integer] :copy_source_range Range of bytes to copy, ie bytes=0-45687
+      def copy_part(copy_source, options = {})
+        part_options = base_opts.merge(options)
+        part_options.merge!(:copy_source => copy_source)
+
+        unless part_options[:part_number]
+          @increment_mutex.synchronize do
+            part_options[:part_number] = (@last_part += 1)
+          end
+        end
+        part_number = part_options[:part_number]
+
+        resp = client.copy_part(part_options)
+        @completed_mutex.synchronize do
+          @completed_parts[part_number] = {
+            :part_number => part_number,
+            :etag => resp[:etag]
+          }
+        end
+        UploadedPart.new(self, part_number)
+      end
+
+      # Completes the upload by assembling previously uploaded
+      # parts.
+      #
+      # @return [S3Object, ObjectVersion] If the bucket has versioning
+      #   enabled, returns the {ObjectVersion} representing the
+      #   version that was uploaded.  If versioning is disabled,
+      #   returns the object.
+      def complete(*parts)
+        parts = parts.flatten
+        case parts.first
+        when :remote_parts
+          complete_opts = get_complete_opts
+        when :local_parts, nil
+          complete_opts = base_opts.merge(:parts => completed_parts)
+        else
+          part_numbers = parts.map do |part|
+            case part
+            when Integer
+              part
+            when UploadedPart
+              raise ArgumentError.new("cannot complete an upload with parts "+
+                                      "from a different upload") unless
+                part.upload == self
+
+              part.part_number
+            else
+              raise ArgumentError.new("expected number or UploadedPart")
+            end
+          end
+          complete_opts = get_complete_opts(part_numbers)
+        end
+
+        raise EmptyUploadError.new("Unable to complete an empty upload.") if complete_opts[:parts].empty?
+
+        resp = client.complete_multipart_upload(complete_opts)
+        if resp.data[:version_id]
+          ObjectVersion.new(object, resp.data[:version_id])
+        else
+          object
+        end
+      end
+
+      # Completes the upload or aborts it if no parts have been
+      # uploaded yet.  Does nothing if the upload has already been
+      # aborted.
+      #
+      # @return [S3Object, ObjectVersion] If the bucket has versioning
+      #   enabled, returns the {ObjectVersion} representing the
+      #   version that was uploaded.  If versioning is disabled,
+      #   returns the object.  If no upload was attempted (e.g. if it
+      #   was aborted or if no parts were uploaded), returns `nil`.
+      def close
+        if aborted?
+          nil
+        elsif completed_parts.empty?
+          abort
+        else
+          complete
+        end
+      end
+
+      # @return [UploadedPartCollection] A collection representing
+      #   the parts that have been uploaded to S3 for this upload.
+      def parts
+        UploadedPartCollection.new(self)
+      end
+
+      # @api private
+      def completed_parts
+        @completed_parts.values.
+          sort { |a, b| a[:part_number] <=> b[:part_number] }
+      end
+
+      # @api private
+      def inspect
+        "<#{self.class}:#{object.bucket.name}/#{object.key}:#{id}>"
+      end
+
+      private
+
+      def get_complete_opts(part_numbers = nil)
+        parts = []
+        self.parts.each do |part|
+          parts << { :part_number => part.part_number, :etag => part.etag }
+        end
+
+        if part_numbers
+          parts.reject! do |part|
+            !part_numbers.include?(part[:part_number])
+          end
+        end
+
+        base_opts.merge(:parts => parts)
+      end
+
+      def base_opts
+        opts = {
+          :bucket_name => object.bucket.name,
+          :key => object.key
+        }
+        opts[:upload_id] = upload_id if upload_id
+        opts
+      end
+
+    end
+
+  end
+end