blobstore_client 1.3215.4.0 → 1.3232.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30b0f1640de3d9c8079f33f6c7f52d7892b27b24
-  data.tar.gz: 600739dc68d5676bf9995c1048b96d60ec26a345
+  metadata.gz: a06a98a43ef258f0524a1499be7e8b0debcbaa98
+  data.tar.gz: eef981f753d46a86b6f2b4d97f268e84536c1825
 SHA512:
-  metadata.gz: 384fc07ad7e960890338dbe0f9fbfe6e5352234c5783f9c22b17bdd94256a82bfe8870a0a1189e63b8ace1fdfa1d45d0f932b3f3661d9dab004eb13e52ada148
-  data.tar.gz: 1685b461e46162ad8d87834f26e7cd96b056099b3136ecc840620d547297370b53998cedb01bd2ec8a4e3a2dbadb500f9ec485180e2051f308432c35d3109eee
+  metadata.gz: 2e7ac4e932a04a0b00ebd603fda7cd4faa3f8e68434470acbdab809c56bbbba5f8e37877f20fdd4383b13002c4b85518836e2c598ad593c45d8ec60c0c1c527e
+  data.tar.gz: 563a9ef2d01c7255513d3098450ee02d9bc9d0ee8d957da13a6af8421efe87ffdd2c6d7d86bda03104cf4d1161b22c30b0882d25ade946fc323dee4d1f077c36

data/lib/blobstore_client.rb

@@ -11,3 +11,4 @@ Bosh::Blobstore.autoload(:S3BlobstoreClient, 'blobstore_client/s3_blobstore_clie
 Bosh::Blobstore.autoload(:SimpleBlobstoreClient, 'blobstore_client/simple_blobstore_client')
 Bosh::Blobstore.autoload(:LocalClient, 'blobstore_client/local_client')
 Bosh::Blobstore.autoload(:DavBlobstoreClient, 'blobstore_client/dav_blobstore_client')
+Bosh::Blobstore.autoload(:S3cliBlobstoreClient, 'blobstore_client/s3cli_blobstore_client')

data/lib/blobstore_client/client.rb

@@ -1,7 +1,7 @@
 module Bosh
   module Blobstore
     class Client
-      PROVIDER_NAMES = %w[dav simple s3 local]
+      PROVIDER_NAMES = %w[dav simple s3 local s3cli]
 
       def self.create(blobstore_provider, options = {})
         unless PROVIDER_NAMES.include?(blobstore_provider)

data/lib/blobstore_client/s3cli_blobstore_client.rb

@@ -0,0 +1,133 @@
+require 'openssl'
+require 'digest/sha1'
+require 'base64'
+require 'securerandom'
+require 'open3'
+require 'json'
+
+module Bosh
+  module Blobstore
+    class S3cliBlobstoreClient < BaseClient
+
+      # Blobstore client for S3, using s3cli Go version
+      # @param [Hash] options S3connection options
+      # @option options [Symbol] bucket_name
+      #   key that is applied before the object is sent to S3
+      # @option options [Symbol, optional] access_key_id
+      # @option options [Symbol, optional] secret_access_key
+      # @option options [Symbol] s3cli_path
+      #   path to s3cli binary
+      # @option options [Symbol, optional] s3cli_config_path
+      #   path to store configuration files
+      # @note If access_key_id and secret_access_key are not present, the
+      #   blobstore client operates in read only mode
+      def initialize(options)
+        super(options)
+
+        @s3cli_path = @options.fetch(:s3cli_path)
+        unless Kernel.system("#{@s3cli_path} --v", out: "/dev/null", err: "/dev/null")
+          raise BlobstoreError, "Cannot find s3cli executable. Please specify s3cli_path parameter"
+        end
+
+        @s3cli_options = {
+          bucket_name: @options[:bucket_name],
+          use_ssl: @options.fetch(:use_ssl, true),
+          host: @options[:host],
+          port: @options[:port],
+          region: @options[:region],
+          ssl_verify_peer:  @options.fetch(:ssl_verify_peer, true),
+          credentials_source: @options.fetch(:credentials_source, 'none'),
+          access_key_id: @options[:access_key_id],
+          secret_access_key: @options[:secret_access_key],
+          signature_version: @options[:signature_version]
+        }
+
+        @s3cli_options.reject! {|k,v| v.nil?}
+
+        if  @options[:access_key_id].nil? &&
+            @options[:secret_access_key].nil?
+              @options[:credentials_source] = 'none'
+        end
+
+        @config_file = write_config_file(@options.fetch(:s3cli_config_path, nil))
+      end
+
+      # @param [File] file file to store in S3
+      def create_file(object_id, file)
+        object_id ||= generate_object_id
+        # in Ruby 1.8 File doesn't respond to :path
+        path = file.respond_to?(:path) ? file.path : file
+
+        store_in_s3(path, full_oid_path(object_id))
+
+        object_id
+      end
+
+      # @param [String] object_id object id to retrieve
+      # @param [File] file file to store the retrived object in
+      def get_file(object_id, file)
+        begin
+        out, err, status = Open3.capture3("#{@s3cli_path} -c #{@config_file} get #{object_id} #{file.path}")
+        rescue Exception => e
+          raise BlobstoreError, e.inspect
+        end
+        raise BlobstoreError, "Failed to download S3 object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success?
+      end
+
+      # @param [String] object_id object id to delete
+      def delete_object(object_id)
+        begin
+          out, err, status = Open3.capture3("#{@s3cli_path} -c #{@config_file} delete #{object_id}")
+        rescue Exception => e
+          raise BlobstoreError, e.inspect
+        end
+        raise BlobstoreError, "Failed to delete S3 object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success?
+      end
+
+      def object_exists?(object_id)
+        begin
+          out, err, status = Open3.capture3("#{@s3cli_path} -c #{@config_file} exists #{object_id}")
+          if status.exitstatus == 0
+            return true
+          end
+          if status.exitstatus == 3
+            return false
+          end
+        rescue Exception => e
+          raise BlobstoreError, e.inspect
+        end
+        raise BlobstoreError, "Failed to check existence of S3 object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success?
+      end
+
+      protected
+
+      # @param [String] path path to file which will be stored in S3
+      # @param [String] oid object id
+      # @return [void]
+      def store_in_s3(path, oid)
+        begin
+        out, err, status = Open3.capture3("#{@s3cli_path} -c #{@config_file} put #{path} #{oid}")
+        rescue Exception => e
+          raise BlobstoreError, e.inspect
+        end
+        raise BlobstoreError, "Failed to create S3 object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success?
+      end
+
+      def full_oid_path(object_id)
+         @options[:folder] ?  @options[:folder] + '/' + object_id : object_id
+      end
+
+      def write_config_file(config_file_dir = nil)
+        config_file_dir = Dir::tmpdir unless config_file_dir
+        Dir.mkdir(config_file_dir) unless File.exists?(config_file_dir)
+        random_name = "s3_blobstore_config-#{SecureRandom.uuid}"
+        config_file = File.join(config_file_dir, random_name)
+        config_data = JSON.dump(@s3cli_options)
+
+        File.write(config_file, config_data)
+        config_file
+      end
+
+    end
+  end
+end

data/lib/blobstore_client/version.rb

@@ -1,7 +1,7 @@
 module Bosh
   module Blobstore
     class Client
-      VERSION = '1.3215.4.0'
+      VERSION = '1.3232.0'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blobstore_client
 version: !ruby/object:Gem::Version
-  version: 1.3215.4.0
+  version: 1.3232.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-14 00:00:00.000000000 Z
+date: 2016-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-resources
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3215.4.0
+        version: 1.3232.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3215.4.0
+        version: 1.3232.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -155,6 +155,7 @@ files:
 - lib/blobstore_client/null_blobstore_client.rb
 - lib/blobstore_client/retryable_blobstore_client.rb
 - lib/blobstore_client/s3_blobstore_client.rb
+- lib/blobstore_client/s3cli_blobstore_client.rb
 - lib/blobstore_client/sha1_verifiable_blobstore_client.rb
 - lib/blobstore_client/simple_blobstore_client.rb
 - lib/blobstore_client/version.rb