blobstore_client 0.3.13 → 0.4.0

data/Rakefile

@@ -35,7 +35,6 @@ if defined?(RSpec)
   namespace :spec do
     desc "Run Unit Tests"
     rspec_task = RSpec::Core::RakeTask.new(:unit) do |t|
-      t.gemfile = "Gemfile"
       t.pattern = "spec/unit/**/*_spec.rb"
       t.rspec_opts = %w(--format progress --colour)
     end

data/lib/blobstore_client.rb

@@ -2,6 +2,7 @@
 
 module Bosh; module Blobstore; end; end
 
+require "common/common"
 require "blobstore_client/version"
 require "blobstore_client/errors"
 
@@ -23,15 +24,18 @@ module Bosh
         "local" => LocalClient
       }
 
-      def self.create(provider, options = {})
-        p = PROVIDER_MAP[provider]
-        if p
-          p.new(options)
-        else
+      def self.create(blobstore_provider, options = {})
+        provider = PROVIDER_MAP[blobstore_provider]
+
+        unless provider
           providers = PROVIDER_MAP.keys.sort.join(", ")
-          raise "Invalid client provider, available providers are: #{providers}"
+          raise BlobstoreError,
+            "Invalid client provider, available providers are: #{providers}"
         end
+
+        provider.new(options)
       end
+
     end
   end
 end

data/lib/blobstore_client/base.rb

@@ -7,7 +7,7 @@ module Bosh
     class BaseClient < Client
 
       def initialize(options)
-        @options = symbolize_keys(options)
+        @options = Bosh::Common.symbolize_keys(options)
       end
 
       def symbolize_keys(hash)
@@ -34,7 +34,7 @@ module Bosh
               File.open(path, "w") do |file|
                 file.write(contents)
               end
-              create_file(File.open(path, "r"))
+              return create_file(File.open(path, "r"))
             rescue BlobstoreError => e
               raise e
             rescue Exception => e
@@ -72,9 +72,10 @@ module Bosh
       def temp_path
         path = File.join(Dir::tmpdir, "temp-path-#{UUIDTools::UUID.random_create}")
         begin
-          yield path
+          yield path if block_given?
+          path
         ensure
-          FileUtils.rm_f(path)
+          FileUtils.rm_f(path) if block_given?
         end
       end
 

data/lib/blobstore_client/s3_blobstore_client.rb

@@ -11,10 +11,21 @@ module Bosh
 
     class S3BlobstoreClient < BaseClient
 
+      ENDPOINT = "https://s3.amazonaws.com"
       DEFAULT_CIPHER_NAME = "aes-128-cbc"
 
       attr_reader :bucket_name, :encryption_key
 
+      # Blobstore client for S3 with optional object encryption
+      # @param [Hash] options S3connection options
+      # @option options [Symbol] bucket_name
+      # @option options [Symbol, optional] encryption_key optional encryption
+      #   key that is applied before the object is sent to S3
+      # @option options [Symbol, optional] access_key_id
+      # @option options [Symbol, optional] secret_access_key
+      # @note If access_key_id and secret_access_key are not present, the
+      #   blobstore client operates in read only mode as a
+      #   simple_blobstore_client
       def initialize(options)
         super(options)
         @bucket_name    = @options[:bucket_name]
@@ -27,21 +38,45 @@ module Bosh
           :port              => 443
         }
 
-        AWS::S3::Base.establish_connection!(aws_options)
+        # using S3 without credentials is a special case:
+        # it is really the simple blobstore client with a bucket name
+        if read_only?
+          unless @options[:bucket_name] || @options[:bucket]
+            raise BlobstoreError, "bucket name required"
+          end
+          @options[:bucket] ||= @options[:bucket_name]
+          @options[:endpoint] ||= S3BlobstoreClient::ENDPOINT
+          @simple = SimpleBlobstoreClient.new(@options)
+        else
+          AWS::S3::Base.establish_connection!(aws_options)
+        end
+
       rescue AWS::S3::S3Exception => e
         raise BlobstoreError, "Failed to initialize S3 blobstore: #{e.message}"
       end
 
       def create_file(file)
+        raise BlobstoreError, "unsupported action" if @simple
+
         object_id = generate_object_id
-        temp_path do |path|
-          File.open(path, "w") do |temp_file|
-            encrypt_stream(file, temp_file)
+
+        if @encryption_key
+          temp_path do |path|
+            File.open(path, "w") do |temp_file|
+              encrypt_stream(file, temp_file)
+            end
+            File.open(path, "r") do |temp_file|
+              AWS::S3::S3Object.store(object_id, temp_file, bucket_name)
+            end
           end
-          File.open(path, "r") do |temp_file|
+        elsif file.is_a?(String)
+          File.open(file, "r") do |temp_file|
             AWS::S3::S3Object.store(object_id, temp_file, bucket_name)
           end
+        else # Ruby 1.8 passes a File
+          AWS::S3::S3Object.store(object_id, file, bucket_name)
         end
+
         object_id
       rescue AWS::S3::S3Exception => e
         raise BlobstoreError,
@@ -49,6 +84,8 @@ module Bosh
       end
 
       def get_file(object_id, file)
+        return @simple.get_file(object_id, file) if @simple
+
         object = AWS::S3::S3Object.find(object_id, bucket_name)
         from = lambda { |callback|
           object.value { |segment|
@@ -59,7 +96,12 @@ module Bosh
             end
           }
         }
-        decrypt_stream(from, file)
+        if @encryption_key
+          decrypt_stream(from, file)
+        else
+          to_stream = write_stream(file)
+          read_stream(from) { |segment| to_stream.call(segment) }
+        end
       rescue AWS::S3::NoSuchKey => e
         raise NotFound, "S3 object '#{object_id}' not found"
       rescue AWS::S3::S3Exception => e
@@ -68,6 +110,8 @@ module Bosh
       end
 
       def delete(object_id)
+        raise BlobstoreError, "unsupported action" if @simple
+
         AWS::S3::S3Object.delete(object_id, bucket_name)
       rescue AWS::S3::S3Exception => e
         raise BlobstoreError,
@@ -122,6 +166,10 @@ module Bosh
         end
       end
 
+      def read_only?
+        @options[:access_key_id].nil? && @options[:secret_access_key].nil?
+      end
+
     end
   end
 end

data/lib/blobstore_client/simple_blobstore_client.rb

@@ -10,6 +10,7 @@ module Bosh
         super(options)
         @client = HTTPClient.new
         @endpoint = @options[:endpoint]
+        @bucket = @options[:bucket] || "resources"
         @headers = {}
         user = @options[:user]
         password = @options[:password]
@@ -20,7 +21,7 @@ module Bosh
       end
 
       def url(id=nil)
-        ["#{@endpoint}/resources", id].compact.join("/")
+        ["#{@endpoint}/#{@bucket}", id].compact.join("/")
       end
 
       def create_file(file)

data/lib/blobstore_client/version.rb

@@ -3,7 +3,7 @@
 module Bosh
   module Blobstore
     class Client
-      VERSION = "0.3.13"
+      VERSION = "0.4.0"
     end
   end
 end

data/spec/unit/blobstore_client_spec.rb

@@ -25,6 +25,11 @@ describe Bosh::Blobstore::Client do
     bs.should be_instance_of Bosh::Blobstore::S3BlobstoreClient
   end
 
+  it "should pick S3 provider when S3 is used without credentials" do
+    bs = Bosh::Blobstore::Client.create('s3', {:bucket_name => "foo"})
+    bs.should be_instance_of Bosh::Blobstore::S3BlobstoreClient
+  end
+
   it "should raise an exception on an unknown client" do
     lambda {
       bs = Bosh::Blobstore::Client.create('foobar', {})

data/spec/unit/s3_blobstore_client_spec.rb

@@ -15,6 +15,34 @@ describe Bosh::Blobstore::S3BlobstoreClient do
     Bosh::Blobstore::S3BlobstoreClient.new(options)
   end
 
+  describe "read only mode" do
+    it "does not establish S3 connection on creation" do
+      AWS::S3::Base.should_not_receive(:establish_connection!)
+      @client = s3_blobstore("bucket_name" => "test")
+    end
+
+    it "should raise an error on deletion" do
+      @client = s3_blobstore("bucket_name" => "test")
+      lambda {
+        @client.delete("id")
+      }.should raise_error "unsupported action"
+    end
+
+    it "should raise an error on creation" do
+      @client = s3_blobstore("bucket_name" => "test")
+      lambda {
+        @client.create("id")
+      }.should raise_error "unsupported action"
+    end
+
+    it "should fetch objects" do
+      simple = mock("simple", :to_ary => nil, :get_file => %w[foo id])
+      Bosh::Blobstore::SimpleBlobstoreClient.should_receive(:new).and_return(simple)
+      @client = s3_blobstore("bucket_name" => "test")
+      @client.get_file("foo", "id")
+    end
+  end
+
   describe "options" do
 
     it "establishes S3 connection on creation" do
@@ -73,6 +101,17 @@ describe Bosh::Blobstore::S3BlobstoreClient do
         @client.create("some content").should eql("object_id")
       end
 
+      it "should not encrypt when encryption key is missing" do
+        client = s3_blobstore(:bucket_name       => "test",
+                              :access_key_id     => "KEY",
+                              :secret_access_key => "SECRET")
+        client.should_receive(:generate_object_id).and_return("object_id")
+        client.should_not_receive(:encrypt_stream)
+
+        AWS::S3::S3Object.should_receive(:store)
+        client.create("some content").should eql("object_id")
+      end
+
       it "should raise an exception when there is an error creating an object" do
         encrypted_file = nil
         @client.should_receive(:generate_object_id).and_return("object_id")
@@ -114,6 +153,19 @@ describe Bosh::Blobstore::S3BlobstoreClient do
         @client.get("object_id").should == "stuff"
       end
 
+      it "should not decrypt when encryption key is missing" do
+        client = s3_blobstore(:bucket_name       => "test",
+                              :access_key_id     => "KEY",
+                              :secret_access_key => "SECRET")
+
+        mock_s3_object = mock("s3_object")
+        mock_s3_object.stub!(:value).and_yield("stuff")
+        AWS::S3::S3Object.should_receive(:find).with("object_id", "test").and_return(mock_s3_object)
+        client.should_not_receive(:decrypt_stream)
+
+        client.get("object_id").should == "stuff"
+      end
+
       it "should raise an exception when there is an error fetching an object" do
         AWS::S3::S3Object.should_receive(:find).with("object_id", "test").and_raise(AWS::S3::S3Exception.new("Epic Fail"))
         lambda {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: blobstore_client
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-23 00:00:00.000000000Z
+date: 2012-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-s3
-  requirement: &70295903087360 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 0.6.2
   type: :runtime
   prerelease: false
-  version_requirements: *70295903087360
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.2
 - !ruby/object:Gem::Dependency
   name: httpclient
-  requirement: &70295903085860 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '2.2'
   type: :runtime
   prerelease: false
-  version_requirements: *70295903085860
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70295903084300 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +53,15 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70295903084300
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: ruby-atmos-pure
-  requirement: &70295903076320 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,18 +69,44 @@ dependencies:
         version: 1.0.5
   type: :runtime
   prerelease: false
-  version_requirements: *70295903076320
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: uuidtools
-  requirement: &70295903075580 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 2.1.2
+- !ruby/object:Gem::Dependency
+  name: bosh_common
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *70295903075580
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
 description: BOSH blobstore client
 email: support@vmware.com
 executables: []
@@ -104,7 +145,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2357219804209612013
+      hash: 4134087797262691141
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -113,10 +154,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2357219804209612013
+      hash: 4134087797262691141
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: BOSH blobstore client