blobstore_client 1.3149.0 → 1.3153.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c55bf108ea4027d249543dd7c6716d324d34174
-  data.tar.gz: 5903b15e591b39f2e3b0751a248c26645552f0e8
+  metadata.gz: 10c163e30bf30a08b4909ba6e12c7081efb1f991
+  data.tar.gz: ec59e8b8134b97a754eed2ec8bf6d94b5e07c123
 SHA512:
-  metadata.gz: d6ba85dfc804a3bdb6f78959ba88810c7e535ec118114792944cdbf903b4026c9a75c75b2132cc8804d841b9b446d16dd2e8d65f8e4f296fd6745f2afb909c1c
-  data.tar.gz: 9a5a5aeeea9675e3882972d5aef8ede458b060e4148ac36140a075011ed9df99209b58b1a802d3390f2a8a391d21b823d1c99490a66d8ae72b48fabe0c6d3a1b
+  metadata.gz: e0c59b863d0ec822b33133035ac5a9734963bfd11e3b1a55f08f66ad85f5fd3bdd7bbeec58203500fe4a6ef7c2aba9f801290cdd01c3428a0329f36787badd3e
+  data.tar.gz: b1c4581c5cc14372db13fae84d410788eb24226441c8162f48df2120020ad1f8c4d07eba2ab57a5cf16a4f671971c5afc78ac7e2f76c10efe4d862e720a547c7

data/lib/blobstore_client/s3_blobstore_client.rb

@@ -1,7 +1,7 @@
 require 'openssl'
 require 'digest/sha1'
 require 'base64'
-require 'aws'
+require 'aws-sdk-resources'
 require 'securerandom'
 
 module Bosh
@@ -9,14 +9,15 @@ module Bosh
     class S3BlobstoreClient < BaseClient
 
       ENDPOINT = 'https://s3.amazonaws.com'
-      DEFAULT_CIPHER_NAME = 'aes-128-cbc'
+      DEFAULT_REGION = 'us-east-1'
+      # hack to get the v2 AWS SDK to behave with S3-compatible blobstores
+      BLANK_REGION = ' '
 
-      attr_reader :bucket_name, :encryption_key, :simple
+      attr_reader :simple
 
-      # Blobstore client for S3 with optional object encryption
+      # Blobstore client for S3
       # @param [Hash] options S3connection options
       # @option options [Symbol] bucket_name
-      # @option options [Symbol, optional] encryption_key optional encryption
       #   key that is applied before the object is sent to S3
       # @option options [Symbol, optional] access_key_id
       # @option options [Symbol, optional] secret_access_key
@@ -25,27 +26,23 @@ module Bosh
       #   simple_blobstore_client
       def initialize(options)
         super(options)
-        @bucket_name    = @options[:bucket_name]
-        @encryption_key = @options[:encryption_key]
 
-        aws_options = {
+        @aws_options = build_aws_options({
+          bucket_name: @options[:bucket_name],
           use_ssl: @options.fetch(:use_ssl, true),
-          s3_port: @options.fetch(:port, 443),
-          s3_endpoint: @options.fetch(:host, URI.parse(S3BlobstoreClient::ENDPOINT).host),
+          host: @options[:host],
+          port: @options[:port],
+          region: @options.fetch(:region, DEFAULT_REGION),
           s3_force_path_style: @options.fetch(:s3_force_path_style, false),
-          ssl_verify_peer: @options.fetch(:ssl_verify_peer, true),
-          s3_multipart_threshold: @options.fetch(:s3_multipart_threshold, 16_777_216),
-        }
-
-        aws_options.merge!(aws_credentials)
+          ssl_verify_peer:  @options.fetch(:ssl_verify_peer, true),
+          credentials_source: @options.fetch(:credentials_source, 'static'),
+          access_key_id: @options[:access_key_id],
+          secret_access_key: @options[:secret_access_key]
+        })
 
         # using S3 without credentials is a special case:
         # it is really the simple blobstore client with a bucket name
         if read_only?
-          if @encryption_key
-            raise BlobstoreError, "can't use read-only with an encryption key"
-          end
-
           unless @options[:bucket_name] || @options[:bucket]
             raise BlobstoreError, 'bucket name required'
           end
@@ -53,12 +50,10 @@ module Bosh
           @options[:bucket] ||= @options[:bucket_name]
           @options[:endpoint] ||= S3BlobstoreClient::ENDPOINT
           @simple = SimpleBlobstoreClient.new(@options)
-        else
-          @s3 = AWS::S3.new(aws_options)
         end
 
-      rescue AWS::Errors::Base => e
-        raise BlobstoreError, "Failed to initialize S3 blobstore: #{e.message}"
+      rescue Aws::S3::Errors::ServiceError => e
+        raise BlobstoreError, "Failed to initialize S3 blobstore: #{e.code} : #{e.message}"
       end
 
       # @param [File] file file to store in S3
@@ -67,17 +62,13 @@ module Bosh
 
         object_id ||= generate_object_id
 
-        file = encrypt_file(file) if @encryption_key
-
         # in Ruby 1.8 File doesn't respond to :path
         path = file.respond_to?(:path) ? file.path : file
         store_in_s3(path, full_oid_path(object_id))
 
         object_id
-      rescue AWS::Errors::Base => e
-        raise BlobstoreError, "Failed to create object, S3 response error: #{e.message}"
-      ensure
-        FileUtils.rm(file) if @encryption_key
+      rescue Aws::S3::Errors::ServiceError => e
+        raise BlobstoreError, "Failed to create object, S3 response error code #{e.code}: #{e.message}"
       end
 
       # @param [String] object_id object id to retrieve
@@ -86,81 +77,59 @@ module Bosh
         object_id = full_oid_path(object_id)
         return @simple.get_file(object_id, file) if @simple
 
-        if @encryption_key
-          cipher = OpenSSL::Cipher::Cipher.new(DEFAULT_CIPHER_NAME)
-          cipher.decrypt
-          cipher.key = Digest::SHA1.digest(encryption_key)[0..(cipher.key_len - 1)]
+        s3_object = Aws::S3::Object.new({:key => object_id}.merge(@aws_options))
+        s3_object.get do |chunk|
+          file.write(chunk)
         end
 
-        object = get_object_from_s3(object_id)
-        object.read do |chunk|
-          if @encryption_key
-            file.write(cipher.update(chunk))
-          else
-            file.write(chunk)
-          end
-        end
-
-        file.write(cipher.final) if @encryption_key
-
-      rescue AWS::S3::Errors::NoSuchKey => e
+      rescue Aws::S3::Errors::NoSuchKey => e
         raise NotFound, "S3 object '#{object_id}' not found"
-      rescue AWS::Errors::Base => e
-        raise BlobstoreError, "Failed to find object '#{object_id}', S3 response error: #{e.message}"
+      rescue Aws::S3::Errors::ServiceError => e
+        raise BlobstoreError, "Failed to find object '#{object_id}', S3 response error code #{e.code}: #{e.message}"
       end
 
       # @param [String] object_id object id to delete
       def delete_object(object_id)
         raise BlobstoreError, 'unsupported action' if @simple
         object_id = full_oid_path(object_id)
-        object = get_object_from_s3(object_id)
-        raise NotFound, "Object '#{object_id}' is not found" unless object.exists?
 
-        object.delete
-      rescue AWS::Errors::Base => e
-        raise BlobstoreError, "Failed to delete object '#{object_id}', S3 response error: #{e.message}"
+        s3_object = Aws::S3::Object.new({:key => object_id}.merge(@aws_options))
+        # TODO: don't blow up if we are cannot find an object we are trying to
+        # delete anyway
+        raise NotFound, "Object '#{object_id}' is not found" unless s3_object.exists?
+
+        s3_object.delete
+      rescue Aws::S3::Errors::ServiceError => e
+        raise BlobstoreError, "Failed to delete object '#{object_id}', S3 response error code #{e.code}: #{e.message}"
       end
 
       def object_exists?(object_id)
         object_id = full_oid_path(object_id)
         return simple.exists?(object_id) if simple
 
-        get_object_from_s3(object_id).exists?
+        # Hack to get the Aws SDK to redirect to the correct region on
+        # subsequent requests
+        unless @region_configured
+          s3 = Aws::S3::Client.new(@aws_options.reject{|k| k == :bucket_name})
+          s3.list_objects({bucket: @aws_options[:bucket_name]})
+          @region_configured = true
+        end
+
+        Aws::S3::Object.new({:key => object_id}.merge(@aws_options)).exists?
       end
 
       protected
 
-      # @param [String] oid object id
-      # @return [AWS::S3::S3Object] S3 object
-      def get_object_from_s3(oid)
-        @s3.buckets[bucket_name].objects[oid]
-      end
-
       # @param [String] path path to file which will be stored in S3
       # @param [String] oid object id
       # @return [void]
       def store_in_s3(path, oid)
-        s3_object = get_object_from_s3(oid)
-        raise BlobstoreError, "object id #{oid} is already in use" if s3_object.exists?
-        File.open(path, 'r') do |temp_file|
-          s3_object.write(temp_file, content_type: "application/octet-stream")
-        end
-      end
-
-      def encrypt_file(file)
-        cipher = OpenSSL::Cipher::Cipher.new(DEFAULT_CIPHER_NAME)
-        cipher.encrypt
-        cipher.key = Digest::SHA1.digest(encryption_key)[0..(cipher.key_len - 1)]
+        raise BlobstoreError, "object id #{oid} is already in use" if object_exists?(oid)
 
-        path = temp_path
-        File.open(path, 'w') do |temp_file|
-          while (block = file.read(32768))
-            temp_file.write(cipher.update(block))
-          end
-          temp_file.write(cipher.final)
-        end
-
-        path
+        s3_object = Aws::S3::Object.new({:key => oid}.merge(@aws_options))
+        multipart_threshold = @options.fetch(:s3_multipart_threshold, 16_777_216)
+        s3_object.upload_file(path, {content_type: "application/octet-stream", multipart_threshold: multipart_threshold})
+        nil
       end
 
       def read_only?
@@ -174,19 +143,24 @@ module Bosh
          @options[:folder] ?  @options[:folder] + '/' + object_id : object_id
       end
 
-      def aws_credentials
+      def use_v4_signing?(region)
+        (region == 'eu-central-1' ||
+         region == 'cn-north-1')
+      end
+
+      def aws_credentials(credentials_source, access_key_id, secret_access_key)
         creds = {}
         # credentials_source could be static (default) or env_or_profile
         # static credentials must be included in aws_properties
-        # env_or_profile credentials will use the AWS DefaultCredentialsProvider
-        # to find AWS credentials in environment variables or EC2 instance profiles
-        case @options.fetch(:credentials_source, 'static')
+        # env_or_profile credentials will use the Aws DefaultCredentialsProvider
+        # to find Aws credentials in environment variables or EC2 instance profiles
+        case credentials_source
           when 'static'
-            creds[:access_key_id]     = @options[:access_key_id]
-            creds[:secret_access_key] = @options[:secret_access_key]
+            creds[:access_key_id]     = access_key_id
+            creds[:secret_access_key] = secret_access_key
 
           when 'env_or_profile'
-            if !@options[:access_key_id].nil? || !@options[:secret_access_key].nil?
+            if !access_key_id.nil? || !secret_access_key.nil?
               raise BlobstoreError, "can't use access_key_id or secret_access_key with env_or_profile credentials_source"
             end
           else
@@ -194,6 +168,30 @@ module Bosh
         end
         return creds
       end
+
+      def build_aws_options(options)
+        aws_options = {
+          bucket_name: options[:bucket_name],
+          region: options[:region],
+          force_path_style: options[:s3_force_path_style],
+          ssl_verify_peer: options[:ssl_verify_peer],
+        }
+
+        unless options[:host].nil?
+          host = options[:host]
+          protocol = options[:use_ssl] ? 'https' : 'http'
+          uri = options[:port].nil? ? host : "#{host}:#{options[:port]}"
+          aws_options[:endpoint] = "#{protocol}://#{uri}"
+          aws_options[:region] = BLANK_REGION
+        end
+
+        aws_options[:signature_version] = 's3' unless use_v4_signing?(options[:region])
+
+        creds = aws_credentials(options[:credentials_source], options[:access_key_id], options[:secret_access_key])
+        aws_options.merge!(creds)
+
+        aws_options
+      end
     end
   end
 end

data/lib/blobstore_client/version.rb

@@ -1,7 +1,7 @@
 module Bosh
   module Blobstore
     class Client
-      VERSION = '1.3149.0'
+      VERSION = '1.3153.0'
     end
   end
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: blobstore_client
 version: !ruby/object:Gem::Version
-  version: 1.3149.0
+  version: 1.3153.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-07 00:00:00.000000000 Z
+date: 2015-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-resources
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.60.2
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.60.2
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3149.0
+        version: 1.3153.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3149.0
+        version: 1.3153.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement