aws-sdk-resources 2.0.1.pre → 2.0.2.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56e7beb4eb1355b479f0a15f4c1a6c39e041b6f3
-  data.tar.gz: f84dd430391e20dba1493278bfadc5f375e2846f
+  metadata.gz: d8d73ea1b21204559a8fb036e60f6d380268a743
+  data.tar.gz: b92f963d53c46d4dc64aaab930a5acea6bcba3b3
 SHA512:
-  metadata.gz: b6386e64a750440869b42e43da646ef0ef1d6498a1d90ab1a8c8964a1e211a6d805bf253517b67b63d2a7716cdb8bee9cb36f2351320cd0aa1abbcb266de6ad9
-  data.tar.gz: 24f80ab77c37d0003aacc7f778a04f8ca993a5b507b1df73eef1c83359b1eafb2c490dc78654fce9a1986654a6355cd2f8ead129b92302e82d8ceec60d9e00a7
+  metadata.gz: cf73777ea642f80159f357fc2b0e6e1c935637c7688e650ba8949205a242517997a88e84527d02c78264caf37abf19cc92e69f3b510d42a5fca945fc9ff4dca0
+  data.tar.gz: ad3dd237c4c1ef3068794bfa1d2ec347f07e44880321a748f610f18037aa82cd57203984d808e431f8106ed4844ea61626cf1b78bd020eaf87da4158d1bb558d

data/lib/aws-sdk-resources.rb

@@ -31,6 +31,11 @@ module Aws
       else raise ArgumentError, "invalid resource definition #{definition}"
     end
     definition.apply(svc_module)
-  end
 
+    begin
+      require "aws-sdk-resources/#{_.downcase}"
+    rescue LoadError
+    end
+
+  end
 end

data/lib/aws-sdk-resources/resource.rb

@@ -4,11 +4,10 @@ module Aws
 
       extend OperationMethods
 
-      # @overload initialize(options = {})
       # @overload initialize(*identifiers, options = {})
-      # @param options Options except `:data` and identifier options are
-      #   used to construct a {Client} unless `:client` is given.
-      # @option options [Client] :client
+      #   @param [Hash] options Options except `:data` and identifier options are
+      #     used to construct a {Client} unless `:client` is given.
+      #   @option options [Client] :client
       def initialize(*args)
         options = args.last.is_a?(Hash) ? args.pop.dup : {}
         @identifiers = extract_identifiers(args, options)
@@ -62,7 +61,9 @@ module Aws
         if options[:client]
           options[:client]
         else
-          self.class.client_class.new(options)
+          self.class.client_class.new(options.merge(
+            user_agent_suffix: "resources"
+          ))
         end
       end
 

data/lib/aws-sdk-resources/s3.rb

@@ -0,0 +1,41 @@
+module Aws
+  module S3
+
+    autoload :FilePart, 'aws-sdk-resources/s3/file_part'
+    autoload :FileUploader, 'aws-sdk-resources/s3/file_uploader'
+    autoload :MultipartFileUploader, 'aws-sdk-resources/s3/multipart_file_uploader'
+    autoload :MultipartUploadError, 'aws-sdk-resources/s3/multipart_upload_error'
+
+    class Object
+
+      # Uploads a file from disk to the current object in S3.
+      #
+      # @example
+      #
+      #     obj.upload_file('/path/to/file')
+      #
+      # @param [String,Pathname,File,Tempfile] source A file or path to a file
+      #   on the local file system that should be uploaded to this object.
+      #
+      # @option options [Integer] :multipart_threshold (15728640) Files larger
+      #   than `:multipart_threshold` are uploaded using the S3 multipart APIs.
+      #   Default threshold is 15MB.
+      #
+      # @raise [MultipartUploadError] If an object is being uploaded in
+      #   parts, and the upload can not be completed, then the upload is
+      #   aborted and this error is raised.  The raised error has a `#errors`
+      #   method that returns the failures that caused the upload to be
+      #   aborted.
+      #
+      # @return [void]
+      #
+      def upload_file(source, options = {})
+        uploader = FileUploader.new(
+          multipart_threshold: options.delete(:multipart_threshold),
+          client: client)
+        uploader.upload(source, options.merge(bucket: bucket_name, key: key))
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-resources/s3/file_part.rb

@@ -0,0 +1,74 @@
+module Aws
+  module S3
+
+    # A utility class that provides an IO-like interface to a portion of
+    # a file on disk.
+    # @api private
+    class FilePart
+
+      # @option options [required,String,Pathname,File,Tempfile] :source
+      # @option options [required,Integer] :offset The file part will read
+      #   starting at this byte offset.
+      # @option options [required,Integer] :size The maximum number of bytes to
+      #   read from the `:offset`.
+      def initialize(options = {})
+        @source = options[:source]
+        @first_byte = options[:offset]
+        @last_byte = @first_byte + options[:size]
+        @size = options[:size]
+      end
+
+      # @return [String,Pathname,File,Tempfile]
+      attr_reader :source
+
+      # @return [Integer]
+      attr_reader :first_byte
+
+      # @return [Integer]
+      attr_reader :last_byte
+
+      # @return [Integer]
+      attr_reader :size
+
+      def read(bytes = nil, output_buffer = nil)
+        open_file unless @file
+        read_from_file(bytes, output_buffer)
+      end
+
+      def rewind
+        if @file
+          @file.seek(@first_byte)
+          @position = @first_byte
+        end
+        0
+      end
+
+      def close
+        @file.close if @file
+      end
+
+      private
+
+      def open_file
+        @file = File.open(@source, 'rb')
+        rewind
+      end
+
+      def read_from_file(bytes, output_buffer)
+        if bytes
+          data = @file.read([remaining_bytes, bytes].min)
+          data = nil if data == ''
+        else
+          data = @file.read(remaining_bytes)
+        end
+        @position += data ? data.bytesize : 0
+        output_buffer ? output_buffer.replace(data || '') : data
+      end
+
+      def remaining_bytes
+        @last_byte - @position
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-resources/s3/file_uploader.rb

@@ -0,0 +1,58 @@
+require 'pathname'
+
+module Aws
+  module S3
+    # @api private
+    class FileUploader
+
+      FIFTEEN_MEGABYTES = 15 * 1024 * 1024
+
+      # @option options [Client] :client
+      # @option options [Integer] :multipart_threshold Files greater than
+      #   `:multipart_threshold` bytes are uploaded using S3 multipart APIs.
+      def initialize(options = {})
+        @options = options
+        @client = options[:client] || Client.new
+        @multipart_threshold = options[:multipart_threshold] || FIFTEEN_MEGABYTES
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @return [Integer] Files larger than this in bytes are uploaded
+      #   using a {MultipartFileUploader}.
+      attr_reader :multipart_threshold
+
+      # @param [String,Pathname,File,Tempfile] source
+      # @option options [requried,String] :bucket
+      # @option options [requried,String] :key
+      # @return [void]
+      def upload(source, options = {})
+        if File.size(source) >= multipart_threshold
+          MultipartFileUploader.new(@options).upload(source, options)
+        else
+          put_object(source, options)
+        end
+      end
+
+      private
+
+      def put_object(source, options)
+        open_file(source) do |file|
+          @client.put_object(options.merge(body: file))
+        end
+      end
+
+      def open_file(source)
+        if String === source || Pathname === source
+          file = File.open(source, 'rb')
+          yield(file)
+          file.close
+        else
+          yield(source)
+        end
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-resources/s3/multipart_file_uploader.rb

@@ -0,0 +1,163 @@
+require 'pathname'
+require 'thread'
+
+module Aws
+  module S3
+    # @api private
+    class MultipartFileUploader
+
+      MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB
+
+      FILE_TOO_SMALL = "unable to multipart upload files smaller than 5MB"
+
+      MAX_PARTS = 10_000
+
+      THREAD_COUNT = 10
+
+      # @option options [Client] :client
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @param [String,Pathname,File,Tempfile] source
+      # @option options [requried,String] :bucket
+      # @option options [requried,String] :key
+      # @return [void]
+      def upload(source, options = {})
+        if File.size(source) < MIN_PART_SIZE
+          raise ArgumentError, FILE_TOO_SMALL
+        else
+          upload_id = initiate_upload(options)
+          parts = upload_parts(upload_id, source, options)
+          complete_upload(upload_id, parts, options)
+        end
+      end
+
+      private
+
+      def initiate_upload(options)
+        @client.create_multipart_upload(options).upload_id
+      end
+
+      def complete_upload(upload_id, parts, options)
+        @client.complete_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id,
+          multipart_upload: { parts: parts })
+      end
+
+      def upload_parts(upload_id, source, options)
+        pending = PartList.new(compute_parts(upload_id, source, options))
+        completed = PartList.new
+        errors = upload_in_threads(pending, completed)
+        if errors.empty?
+          completed.to_a.sort_by { |part| part[:part_number] }
+        else
+          abort_upload(upload_id, options, errors)
+        end
+      end
+
+      def abort_upload(upload_id, options, errors)
+        @client.abort_multipart_upload(
+          bucket: options[:bucket],
+          key: options[:key],
+          upload_id: upload_id
+        )
+        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
+        raise MultipartUploadError.new(msg, errors)
+      rescue MultipartUploadError => error
+        raise error
+      rescue => error
+        msg = "failed to abort multipart upload: #{error.message}"
+        raise MultipartUploadError.new(msg, errors + [error])
+      end
+
+      def compute_parts(upload_id, source, options)
+        size = File.size(source)
+        default_part_size = compute_default_part_size(size)
+        offset = 0
+        part_number = 1
+        parts = []
+        while offset < size
+          parts << options.merge(
+            upload_id: upload_id,
+            part_number: part_number,
+            body: FilePart.new(
+              source: source,
+              offset: offset,
+              size: part_size(size, default_part_size, offset)
+            )
+          )
+          part_number += 1
+          offset += default_part_size
+        end
+        parts
+      end
+
+      def upload_in_threads(pending, completed)
+        threads = []
+        THREAD_COUNT.times do
+          thread = Thread.new do
+            begin
+              while part = pending.shift
+                resp = @client.upload_part(part)
+                part[:body].close
+                completed.push(etag: resp.etag, part_number: part[:part_number])
+              end
+              nil
+            rescue => error
+              # keep other threads from uploading other parts
+              pending.clear!
+              error
+            end
+          end
+          thread.abort_on_exception = true
+          threads << thread
+        end
+        errors = threads.map(&:value).compact
+      end
+
+      def compute_default_part_size(source_size)
+        [(source_size.to_f / MAX_PARTS).ceil, MIN_PART_SIZE].max.to_i
+      end
+
+      def part_size(total_size, part_size, offset)
+        if offset + part_size > total_size
+          total_size - offset
+        else
+          part_size
+        end
+      end
+
+      # @api private
+      class PartList
+
+        def initialize(parts = [])
+          @parts = parts
+          @mutex = Mutex.new
+        end
+
+        def push(part)
+          @mutex.synchronize { @parts.push(part) }
+        end
+
+        def shift
+          @mutex.synchronize { @parts.shift }
+        end
+
+        def clear!
+          @mutex.synchronize { @parts.clear }
+        end
+
+        def to_a
+          @mutex.synchronize { @parts.dup }
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/s3/multipart_upload_error.rb

@@ -0,0 +1,16 @@
+module Aws
+  module S3
+    class MultipartUploadError < StandardError
+
+      def initialize(message, errors)
+        @errors = errors
+        super(message)
+      end
+
+      # @return [Array<StandardError>] The list of errors encountered
+      #   when uploading or aborting the upload.
+      attr_reader :errors
+
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-resources
 version: !ruby/object:Gem::Version
-  version: 2.0.1.pre
+  version: 2.0.2.pre
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-29 00:00:00.000000000 Z
+date: 2014-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 2.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: 2.0.2
 description: Provides resource-oriented abstractions for AWS.
 email: 
 executables: []
@@ -51,6 +51,11 @@ files:
 - lib/aws-sdk-resources/request.rb
 - lib/aws-sdk-resources/request_params.rb
 - lib/aws-sdk-resources/resource.rb
+- lib/aws-sdk-resources/s3/file_part.rb
+- lib/aws-sdk-resources/s3/file_uploader.rb
+- lib/aws-sdk-resources/s3/multipart_file_uploader.rb
+- lib/aws-sdk-resources/s3/multipart_upload_error.rb
+- lib/aws-sdk-resources/s3.rb
 - lib/aws-sdk-resources/source.rb
 - lib/aws-sdk-resources/validator/context.rb
 - lib/aws-sdk-resources/validator/identifier_validator.rb
@@ -84,4 +89,3 @@ signing_key:
 specification_version: 4
 summary: AWS SDK for Ruby - Resources
 test_files: []
-has_rdoc: