blobstore_client 0.3.13

data/README

@@ -0,0 +1 @@
+Blobstore client for VMware AppCloud Outer Shell.

data/Rakefile

@@ -0,0 +1,50 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+$:.unshift(File.expand_path("../../rake", __FILE__))
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __FILE__)
+
+require "rubygems"
+require "bundler"
+Bundler.setup(:default, :test)
+
+require "rake"
+begin
+  require "rspec/core/rake_task"
+rescue LoadError
+end
+
+require "bundler_task"
+require "ci_task"
+
+gem_helper = Bundler::GemHelper.new(Dir.pwd)
+
+desc "Build Blobstore Client gem into the pkg directory"
+task "build" do
+  gem_helper.build_gem
+end
+
+desc "Build and install Blobstore Client into system gems"
+task "install" do
+  gem_helper.install_gem
+end
+
+BundlerTask.new
+
+if defined?(RSpec)
+  namespace :spec do
+    desc "Run Unit Tests"
+    rspec_task = RSpec::Core::RakeTask.new(:unit) do |t|
+      t.gemfile = "Gemfile"
+      t.pattern = "spec/unit/**/*_spec.rb"
+      t.rspec_opts = %w(--format progress --colour)
+    end
+
+    CiTask.new do |task|
+      task.rspec_task = rspec_task
+    end
+  end
+
+  desc "Install dependencies and run tests"
+  task :spec => %w(spec:unit)
+end

data/lib/blobstore_client.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh; module Blobstore; end; end
+
+require "blobstore_client/version"
+require "blobstore_client/errors"
+
+require "blobstore_client/client"
+require "blobstore_client/base"
+require "blobstore_client/simple_blobstore_client"
+require "blobstore_client/s3_blobstore_client"
+require "blobstore_client/local_client"
+require "blobstore_client/atmos_blobstore_client"
+
+module Bosh
+  module Blobstore
+    class Client
+
+      PROVIDER_MAP = {
+        "simple" => SimpleBlobstoreClient,
+        "s3" => S3BlobstoreClient,
+        "atmos" => AtmosBlobstoreClient,
+        "local" => LocalClient
+      }
+
+      def self.create(provider, options = {})
+        p = PROVIDER_MAP[provider]
+        if p
+          p.new(options)
+        else
+          providers = PROVIDER_MAP.keys.sort.join(", ")
+          raise "Invalid client provider, available providers are: #{providers}"
+        end
+      end
+    end
+  end
+end

data/lib/blobstore_client/atmos_blobstore_client.rb

@@ -0,0 +1,94 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "atmos"
+require "uri"
+require "multi_json"
+
+module Bosh
+  module Blobstore
+    class AtmosBlobstoreClient < BaseClient
+      SHARE_URL_EXP = "1893484800" # expires on 2030 Jan-1
+
+      def initialize(options)
+        super(options)
+        @atmos_options = {
+          :url => @options[:url],
+          :uid => @options[:uid],
+          :secret => @options[:secret]
+        }
+        @tag = @options[:tag]
+        @http_client = HTTPClient.new
+        # TODO: Remove this line once we get the proper certificate for atmos
+        @http_client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      def atmos_server
+        unless @atmos_options[:secret]
+          raise "Atmos password is missing (read-only mode)"
+        end
+        @atmos ||= Atmos::Store.new(@atmos_options)
+      end
+
+      def create_file(file)
+        obj_conf = {:data => file, :length => File.size(file.path)}
+        obj_conf[:listable_metadata] = {@tag => true} if @tag
+        object_id = atmos_server.create(obj_conf).aoid
+        encode_object_id(object_id)
+      end
+
+      def get_file(object_id, file)
+        object_info = decode_object_id(object_id)
+        oid = object_info["oid"]
+        sig = object_info["sig"]
+
+        url = @atmos_options[:url] + "/rest/objects/#{oid}?uid=" +
+              URI::escape(@atmos_options[:uid]) +
+              "&expires=#{SHARE_URL_EXP}&signature=#{URI::escape(sig)}"
+
+        response = @http_client.get(url) do |block|
+          file.write(block)
+        end
+
+        if response.status != 200
+          raise BlobstoreError, "Could not fetch object, %s/%s" %
+            [response.status, response.content]
+        end
+      end
+
+      def delete(object_id)
+        object_info = decode_object_id(object_id)
+        oid = object_info["oid"]
+        atmos_server.get(:id => oid).delete
+      rescue Atmos::Exceptions::NoSuchObjectException => e
+        raise NotFound, "Atmos object '#{object_id}' not found"
+      end
+
+      private
+
+      def decode_object_id(object_id)
+        begin
+          object_info = MultiJson.decode(Base64.decode64(URI::unescape(object_id)))
+        rescue MultiJson::DecodeError => e
+          raise BlobstoreError, "Failed to parse object_id. " +
+            "Please try updating the release"
+        end
+
+        if !object_info.kind_of?(Hash) || object_info["oid"].nil? ||
+            object_info["sig"].nil?
+          raise BlobstoreError, "Invalid object_id (#{object_id})"
+        end
+        object_info
+      end
+
+      def encode_object_id(object_id)
+        hash_string = "GET" + "\n" + "/rest/objects/" + object_id + "\n" +
+                      @atmos_options[:uid] + "\n" + SHARE_URL_EXP
+        secret = Base64.decode64(@atmos_options[:secret])
+        sig = HMAC::SHA1.digest(secret, hash_string)
+        signature = Base64.encode64(sig.to_s).chomp
+        json = MultiJson.encode({:oid => object_id, :sig => signature})
+        URI::escape(Base64.encode64(json))
+      end
+    end
+  end
+end

data/lib/blobstore_client/base.rb

@@ -0,0 +1,83 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "tmpdir"
+
+module Bosh
+  module Blobstore
+    class BaseClient < Client
+
+      def initialize(options)
+        @options = symbolize_keys(options)
+      end
+
+      def symbolize_keys(hash)
+        hash.inject({}) do |h, (key, value)|
+          h[key.to_sym] = value
+          h
+        end
+      end
+
+      def create_file(file)
+
+      end
+
+      def get_file(id, file)
+
+      end
+
+      def create(contents)
+        if contents.kind_of?(File)
+          create_file(contents)
+        else
+          temp_path do |path|
+            begin
+              File.open(path, "w") do |file|
+                file.write(contents)
+              end
+              create_file(File.open(path, "r"))
+            rescue BlobstoreError => e
+              raise e
+            rescue Exception => e
+              raise BlobstoreError,
+                "Failed to create object, underlying error: %s %s" %
+                [e.message, e.backtrace.join("\n")]
+            end
+          end
+        end
+      end
+
+      def get(id, file = nil)
+        if file
+          get_file(id, file)
+        else
+          result = nil
+          temp_path do |path|
+            begin
+              File.open(path, "w") { |file| get_file(id, file) }
+              result = File.open(path, "r") { |file| file.read }
+            rescue BlobstoreError => e
+              raise e
+            rescue Exception => e
+              raise BlobstoreError,
+                "Failed to create object, underlying error: %s %s" %
+                [e.message, e.backtrace.join("\n")]
+            end
+          end
+          result
+        end
+      end
+
+      protected
+
+      def temp_path
+        path = File.join(Dir::tmpdir, "temp-path-#{UUIDTools::UUID.random_create}")
+        begin
+          yield path
+        ensure
+          FileUtils.rm_f(path)
+        end
+      end
+
+    end
+  end
+end

data/lib/blobstore_client/client.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh
+  module Blobstore
+    class Client
+      def create(contents)
+      end
+
+      def get(id, file = nil)
+      end
+
+      def delete(id)
+      end
+    end
+  end
+end
+
+
+

data/lib/blobstore_client/errors.rb

@@ -0,0 +1,10 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh
+  module Blobstore
+
+    class BlobstoreError < StandardError; end
+    class NotFound < BlobstoreError; end
+
+  end
+end

data/lib/blobstore_client/local_client.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh
+  module Blobstore
+    class LocalClient < BaseClient
+      CHUNK_SIZE = 1024*1024
+
+      def initialize(options)
+        super(options)
+        @blobstore_path = @options[:blobstore_path]
+        raise "No blobstore path given" if @blobstore_path.nil?
+        FileUtils.mkdir_p(@blobstore_path) unless File.directory?(@blobstore_path)
+      end
+
+      def create_file(file)
+        id = UUIDTools::UUID.random_create.to_s
+        dst = File.join(@blobstore_path, id)
+        File.open(dst, 'w') do |fh|
+          until file.eof?
+            fh.write(file.read(CHUNK_SIZE))
+          end
+        end
+        id
+      end
+
+      def get_file(id, file)
+        src = File.join(@blobstore_path, id)
+
+        begin
+          File.open(src, 'r') do |src_fh|
+            until src_fh.eof?
+              file.write(src_fh.read(CHUNK_SIZE))
+            end
+          end
+        end
+      rescue Errno::ENOENT
+        raise NotFound, "Blobstore object '#{id}' not found"
+      end
+
+      def delete(id)
+        file = File.join(@blobstore_path, id)
+        FileUtils.rm(file)
+      rescue Errno::ENOENT
+        raise NotFound, "Blobstore object '#{id}' not found"
+      end
+
+    end
+  end
+end

data/lib/blobstore_client/s3_blobstore_client.rb

@@ -0,0 +1,127 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "openssl"
+require "digest/sha1"
+require "base64"
+require "aws/s3"
+require "uuidtools"
+
+module Bosh
+  module Blobstore
+
+    class S3BlobstoreClient < BaseClient
+
+      DEFAULT_CIPHER_NAME = "aes-128-cbc"
+
+      attr_reader :bucket_name, :encryption_key
+
+      def initialize(options)
+        super(options)
+        @bucket_name    = @options[:bucket_name]
+        @encryption_key = @options[:encryption_key]
+
+        aws_options = {
+          :access_key_id     => @options[:access_key_id],
+          :secret_access_key => @options[:secret_access_key],
+          :use_ssl           => true,
+          :port              => 443
+        }
+
+        AWS::S3::Base.establish_connection!(aws_options)
+      rescue AWS::S3::S3Exception => e
+        raise BlobstoreError, "Failed to initialize S3 blobstore: #{e.message}"
+      end
+
+      def create_file(file)
+        object_id = generate_object_id
+        temp_path do |path|
+          File.open(path, "w") do |temp_file|
+            encrypt_stream(file, temp_file)
+          end
+          File.open(path, "r") do |temp_file|
+            AWS::S3::S3Object.store(object_id, temp_file, bucket_name)
+          end
+        end
+        object_id
+      rescue AWS::S3::S3Exception => e
+        raise BlobstoreError,
+          "Failed to create object, S3 response error: #{e.message}"
+      end
+
+      def get_file(object_id, file)
+        object = AWS::S3::S3Object.find(object_id, bucket_name)
+        from = lambda { |callback|
+          object.value { |segment|
+            # Looks like the aws code calls this block even if segment is empty.
+            # Ideally it should be fixed upstream in the aws gem.
+            unless segment.empty?
+              callback.call(segment)
+            end
+          }
+        }
+        decrypt_stream(from, file)
+      rescue AWS::S3::NoSuchKey => e
+        raise NotFound, "S3 object '#{object_id}' not found"
+      rescue AWS::S3::S3Exception => e
+        raise BlobstoreError,
+          "Failed to find object '#{object_id}', S3 response error: #{e.message}"
+      end
+
+      def delete(object_id)
+        AWS::S3::S3Object.delete(object_id, bucket_name)
+      rescue AWS::S3::S3Exception => e
+        raise BlobstoreError,
+          "Failed to delete object '#{object_id}', S3 response error: #{e.message}"
+      end
+
+      protected
+
+      def generate_object_id
+        UUIDTools::UUID.random_create.to_s
+      end
+
+      def encrypt_stream(from, to)
+        cipher = OpenSSL::Cipher::Cipher.new(DEFAULT_CIPHER_NAME)
+        cipher.encrypt
+        cipher.key = Digest::SHA1.digest(encryption_key)[0..cipher.key_len-1]
+
+        to_stream = write_stream(to)
+        read_stream(from) { |segment| to_stream.call(cipher.update(segment)) }
+        to_stream.call(cipher.final)
+      rescue StandardError => e
+        raise BlobstoreError, "Encryption error: #{e}"
+      end
+
+      def decrypt_stream(from, to)
+        cipher = OpenSSL::Cipher::Cipher.new(DEFAULT_CIPHER_NAME)
+        cipher.decrypt
+        cipher.key = Digest::SHA1.digest(encryption_key)[0..cipher.key_len-1]
+
+        to_stream = write_stream(to)
+        read_stream(from) { |segment| to_stream.call(cipher.update(segment)) }
+        to_stream.call(cipher.final)
+      rescue StandardError => e
+        raise BlobstoreError, "Decryption error: #{e}"
+      end
+
+      def read_stream(stream, &block)
+        if stream.respond_to?(:read)
+          while contents = stream.read(32768)
+            block.call(contents)
+          end
+        elsif stream.kind_of?(Proc)
+          stream.call(block)
+        end
+      end
+
+      def write_stream(stream)
+        if stream.respond_to?(:write)
+          lambda { |contents| stream.write(contents)}
+        elsif stream.kind_of?(Proc)
+          stream
+        end
+      end
+
+    end
+  end
+end

data/lib/blobstore_client/simple_blobstore_client.rb

@@ -0,0 +1,54 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "httpclient"
+
+module Bosh
+  module Blobstore
+    class SimpleBlobstoreClient < BaseClient
+
+      def initialize(options)
+        super(options)
+        @client = HTTPClient.new
+        @endpoint = @options[:endpoint]
+        @headers = {}
+        user = @options[:user]
+        password = @options[:password]
+        if user && password
+          @headers["Authorization"] = "Basic " +
+            Base64.encode64("#{user}:#{password}").strip
+        end
+      end
+
+      def url(id=nil)
+        ["#{@endpoint}/resources", id].compact.join("/")
+      end
+
+      def create_file(file)
+        response = @client.post(url, {:content => file}, @headers)
+        if response.status != 200
+          raise BlobstoreError,
+            "Could not create object, #{response.status}/#{response.content}"
+        end
+        response.content
+      end
+
+      def get_file(id, file)
+        response = @client.get(url(id), {}, @headers) do |block|
+          file.write(block)
+        end
+
+        if response.status != 200
+          raise BlobstoreError,
+            "Could not fetch object, #{response.status}/#{response.content}"
+        end
+      end
+
+      def delete(id)
+        response = @client.delete(url(id), @headers)
+        if response.status != 204
+          raise "Could not delete object, #{response.status}/#{response.content}"
+        end
+      end
+    end
+  end
+end

data/lib/blobstore_client/version.rb

@@ -0,0 +1,9 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh
+  module Blobstore
+    class Client
+      VERSION = "0.3.13"
+    end
+  end
+end

data/spec/assets/file

@@ -0,0 +1 @@
+foobar

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require "rubygems"
+require "bundler"
+Bundler.setup(:default, :test)
+
+require "rspec"
+require "blobstore_client"

data/spec/unit/atmos_blobstore_client_spec.rb

@@ -0,0 +1,72 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Bosh::Blobstore::AtmosBlobstoreClient do
+
+  before(:each) do
+    @atmos = mock("atmos")
+    Atmos::Store.stub!(:new).and_return(@atmos)
+    atmos_opt = {:url => "http://localhost",
+                 :uid => "uid",
+                 :secret => "secret"}
+
+    @http_client = mock("http-client")
+    ssl_opt = mock("ssl-opt")
+    ssl_opt.stub!(:verify_mode=)
+    @http_client.stub!(:ssl_config).and_return(ssl_opt)
+
+    HTTPClient.stub!(:new).and_return(@http_client)
+    @client = Bosh::Blobstore::AtmosBlobstoreClient.new(atmos_opt)
+  end
+
+  it "should create an object" do
+    data = "some content"
+    object = mock("object")
+
+    @atmos.should_receive(:create).with {|opt|
+      opt[:data].read.should eql data
+      opt[:length].should eql data.length
+    }.and_return(object)
+
+    object.should_receive(:aoid).and_return("test-key")
+
+    object_id = @client.create(data)
+    object_info = MultiJson.decode(Base64.decode64(URI::unescape(object_id)))
+    object_info["oid"].should eql("test-key")
+    object_info["sig"].should_not be_nil
+  end
+
+  it "should delete an object" do
+    object = mock("object")
+    @atmos.should_receive(:get).with(:id => "test-key").and_return(object)
+    object.should_receive(:delete)
+    id = URI::escape(Base64.encode64(MultiJson.encode({:oid => "test-key", :sig => "sig"})))
+    @client.delete(id)
+  end
+
+  it "should fetch an object" do
+    url = "http://localhost/rest/objects/test-key?uid=uid&expires=1893484800&signature=sig"
+    response = mock("response")
+    response.stub!(:status).and_return(200)
+    @http_client.should_receive(:get).with(url).and_yield("some-content").and_return(response)
+    id = URI::escape(Base64.encode64(MultiJson.encode({:oid => "test-key", :sig => "sig"})))
+    @client.get(id).should eql("some-content")
+  end
+
+  it "should refuse to create object without the password" do
+    lambda {
+      no_pass_client = Bosh::Blobstore::AtmosBlobstoreClient.new(:url => "http://localhost", :uid => "uid")
+      no_pass_client.create("foo")
+    }.should raise_error(Bosh::Blobstore::BlobstoreError)
+  end
+
+  it "should be able to read without password" do
+    no_pass_client = Bosh::Blobstore::AtmosBlobstoreClient.new(:url => "http://localhost", :uid => "uid")
+
+    url = "http://localhost/rest/objects/test-key?uid=uid&expires=1893484800&signature=sig"
+    response = mock("response")
+    response.stub!(:status).and_return(200)
+    @http_client.should_receive(:get).with(url).and_yield("some-content").and_return(response)
+    id = URI::escape(Base64.encode64(MultiJson.encode({:oid => "test-key", :sig => "sig"})))
+    no_pass_client.get(id).should eql("some-content")
+  end
+end

data/spec/unit/blobstore_client_spec.rb

@@ -0,0 +1,34 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Bosh::Blobstore::Client do
+
+  it "should have a local provider" do
+    Dir.mktmpdir do |tmp|
+      bs = Bosh::Blobstore::Client.create('local', {:blobstore_path => tmp})
+      bs.should be_instance_of Bosh::Blobstore::LocalClient
+    end
+  end
+
+  it "should have an simple provider" do
+    bs = Bosh::Blobstore::Client.create('simple', {})
+    bs.should be_instance_of Bosh::Blobstore::SimpleBlobstoreClient
+  end
+
+  it "should have an atmos provider" do
+    bs = Bosh::Blobstore::Client.create('atmos', {})
+    bs.should be_instance_of Bosh::Blobstore::AtmosBlobstoreClient
+  end
+
+  it "should have an s3 provider" do
+    options = {:access_key_id => "foo", :secret_access_key => "bar"}
+    bs = Bosh::Blobstore::Client.create('s3', options)
+    bs.should be_instance_of Bosh::Blobstore::S3BlobstoreClient
+  end
+
+  it "should raise an exception on an unknown client" do
+    lambda {
+      bs = Bosh::Blobstore::Client.create('foobar', {})
+    }.should raise_error /^Invalid client provider/
+  end
+
+end

data/spec/unit/local_client_spec.rb

@@ -0,0 +1,78 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Bosh::Blobstore::LocalClient do
+
+  before(:each) do
+    @tmp = Dir.mktmpdir
+    @options = {"blobstore_path" => @tmp}
+  end
+
+  after(:each) do
+    FileUtils.rm_rf(@tmp)
+  end
+
+  it "should require blobstore_path option" do
+    lambda {
+      client = Bosh::Blobstore::LocalClient.new({})
+    }.should raise_error
+  end
+
+  it "should create blobstore_path direcory if it doesn't exist'" do
+    dir = File.join(@tmp, "blobstore")
+    client = Bosh::Blobstore::LocalClient.new({"blobstore_path" => dir})
+    File.directory?(dir).should be_true
+  end
+
+  describe "operations" do
+
+    describe "get" do
+      it "should retrive the correct contents" do
+        File.open(File.join(@tmp, 'foo'), 'w') do |fh|
+          fh.puts("bar")
+        end
+
+        client = Bosh::Blobstore::LocalClient.new(@options)
+        client.get("foo").should == "bar\n"
+      end
+    end
+
+    describe "create" do
+      it "should store a file" do
+        test_file = File.join(File.dirname(__FILE__), "../assets/file")
+        client = Bosh::Blobstore::LocalClient.new(@options)
+        fh = File.open(test_file)
+        id = client.create(fh)
+        fh.close
+        original = File.new(test_file).readlines
+        stored = File.new(File.join(@tmp, id)).readlines
+        stored.should == original
+      end
+
+      it "should store a string" do
+        client = Bosh::Blobstore::LocalClient.new(@options)
+        string = "foobar"
+        id = client.create(string)
+        stored = File.new(File.join(@tmp, id)).readlines
+        stored.should == [string]
+      end
+    end
+
+    describe "delete" do
+      it "should delete an id" do
+        client = Bosh::Blobstore::LocalClient.new(@options)
+        string = "foobar"
+        id = client.create(string)
+        client.delete(id)
+        File.exist?(File.join(@tmp, id)).should_not be_true
+      end
+
+      it "should raise NotFound error when trying to delete a missing id" do
+        client = Bosh::Blobstore::LocalClient.new(@options)
+        lambda {
+          client.delete("missing")
+          }.should raise_error Bosh::Blobstore::NotFound
+      end
+    end
+
+  end
+end

data/spec/unit/s3_blobstore_client_spec.rb

@@ -0,0 +1,230 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Bosh::Blobstore::S3BlobstoreClient do
+
+  before(:each) do
+    @aws_mock_options = {
+      :access_key_id     => "KEY",
+      :secret_access_key => "SECRET",
+      :use_ssl           => true,
+      :port              => 443
+    }
+  end
+
+  def s3_blobstore(options)
+    Bosh::Blobstore::S3BlobstoreClient.new(options)
+  end
+
+  describe "options" do
+
+    it "establishes S3 connection on creation" do
+      AWS::S3::Base.should_receive(:establish_connection!).with(@aws_mock_options)
+
+      @client = s3_blobstore("encryption_key"    => "bla",
+                             "bucket_name"       => "test",
+                             "access_key_id"     => "KEY",
+                             "secret_access_key" => "SECRET")
+
+      @client.encryption_key.should == "bla"
+      @client.bucket_name.should == "test"
+    end
+
+    it "supports Symbol option keys too" do
+      AWS::S3::Base.should_receive(:establish_connection!).with(@aws_mock_options)
+
+      @client = s3_blobstore(:encryption_key    => "bla",
+                             :bucket_name       => "test",
+                             :access_key_id     => "KEY",
+                             :secret_access_key => "SECRET")
+
+      @client.encryption_key.should == "bla"
+      @client.bucket_name.should == "test"
+    end
+  end
+
+  describe "operations" do
+
+    before :each do
+      @client = s3_blobstore(:encryption_key    => "bla",
+                             :bucket_name       => "test",
+                             :access_key_id     => "KEY",
+                             :secret_access_key => "SECRET")
+    end
+
+    describe "create" do
+
+      it "should create an object" do
+        encrypted_file = nil
+        @client.should_receive(:generate_object_id).and_return("object_id")
+        @client.should_receive(:encrypt_stream).with { |from_file, _|
+          from_file.read.should eql("some content")
+          true
+        }.and_return {|_, to_file|
+          encrypted_file = to_file
+          nil
+        }
+
+        AWS::S3::S3Object.should_receive(:store).with { |key, data, bucket|
+          key.should eql("object_id")
+          data.path.should eql(encrypted_file.path)
+          bucket.should eql("test")
+          true
+        }
+        @client.create("some content").should eql("object_id")
+      end
+
+      it "should raise an exception when there is an error creating an object" do
+        encrypted_file = nil
+        @client.should_receive(:generate_object_id).and_return("object_id")
+        @client.should_receive(:encrypt_stream).with { |from_file, _|
+          from_file.read.should eql("some content")
+          true
+        }.and_return {|_, to_file|
+          encrypted_file = to_file
+          nil
+        }
+
+        AWS::S3::S3Object.should_receive(:store).with { |key, data, bucket|
+          key.should eql("object_id")
+          data.path.should eql(encrypted_file.path)
+          bucket.should eql("test")
+          true
+        }.and_raise(AWS::S3::S3Exception.new("Epic Fail"))
+        lambda {
+          @client.create("some content")
+        }.should raise_error(Bosh::Blobstore::BlobstoreError, "Failed to create object, S3 response error: Epic Fail")
+      end
+
+    end
+
+    describe "fetch" do
+
+      it "should fetch an object" do
+        mock_s3_object = mock("s3_object")
+        mock_s3_object.stub!(:value).and_yield("ENCRYPTED")
+        AWS::S3::S3Object.should_receive(:find).with("object_id", "test").and_return(mock_s3_object)
+        @client.should_receive(:decrypt_stream).with { |from, _|
+          encrypted = ""
+          from.call(lambda {|segment| encrypted << segment})
+          encrypted.should eql("ENCRYPTED")
+          true
+        }.and_return {|_, to|
+          to.write("stuff")
+        }
+        @client.get("object_id").should == "stuff"
+      end
+
+      it "should raise an exception when there is an error fetching an object" do
+        AWS::S3::S3Object.should_receive(:find).with("object_id", "test").and_raise(AWS::S3::S3Exception.new("Epic Fail"))
+        lambda {
+          @client.get("object_id")
+        }.should raise_error(Bosh::Blobstore::BlobstoreError, "Failed to find object 'object_id', S3 response error: Epic Fail")
+      end
+
+      it "should raise more specific NotFound exception when object is not found" do
+        AWS::S3::S3Object.should_receive(:find).with("object_id", "test").and_raise(AWS::S3::NoSuchKey.new("NO KEY", "test"))
+        lambda {
+          @client.get("object_id")
+        }.should raise_error(Bosh::Blobstore::BlobstoreError, "S3 object 'object_id' not found")
+      end
+
+    end
+
+    describe "delete" do
+
+      it "should delete an object" do
+        AWS::S3::S3Object.should_receive(:delete).with("object_id", "test")
+        @client.delete("object_id")
+      end
+
+      it "should raise an exception when there is an error deleting an object" do
+        AWS::S3::S3Object.should_receive(:delete).with("object_id", "test").and_raise(AWS::S3::S3Exception.new("Epic Fail"))
+        lambda {
+          @client.delete("object_id")
+        }.should raise_error(Bosh::Blobstore::BlobstoreError, "Failed to delete object 'object_id', S3 response error: Epic Fail")
+      end
+
+    end
+
+    describe "encryption" do
+
+      before :each do
+        @from_path = File.join(Dir::tmpdir, "from-#{UUIDTools::UUID.random_create}")
+        @to_path = File.join(Dir::tmpdir, "to-#{UUIDTools::UUID.random_create}")
+      end
+
+      after :each do
+        FileUtils.rm_f(@from_path)
+        FileUtils.rm_f(@to_path)
+      end
+
+      it "encrypt/decrypt works as long as key is the same" do
+        File.open(@from_path, "w") { |f| f.write("clear text") }
+        File.open(@from_path, "r") do |from|
+          File.open(@to_path, "w") do |to|
+            @client.send(:encrypt_stream, from, to)
+          end
+        end
+
+        Base64.encode64(File.read(@to_path)).should eql("XCUKDXXzjh43DmNylgVpQQ==\n")
+
+        File.open(@from_path, "w") { |f| f.write(File.read(@to_path)) }
+        File.open(@from_path, "r") do |from|
+          File.open(@to_path, "w") do |to|
+            @client.send(:decrypt_stream, from, to)
+          end
+        end
+
+        File.read(@to_path).should eql("clear text")
+      end
+
+      it "encrypt/decrypt doesn't have padding issues for very small inputs" do
+        File.open(@from_path, "w") { |f| f.write("c") }
+        File.open(@from_path, "r") do |from|
+          File.open(@to_path, "w") do |to|
+            @client.send(:encrypt_stream, from, to)
+          end
+        end
+
+        Base64.encode64(File.read(@to_path)).should eql("S1ZnX5gPfm/rQbRCcShHSg==\n")
+
+        File.open(@from_path, "w") { |f| f.write(File.read(@to_path)) }
+        File.open(@from_path, "r") do |from|
+          File.open(@to_path, "w") do |to|
+            @client.send(:decrypt_stream, from, to)
+          end
+        end
+
+        File.read(@to_path).should eql("c")
+      end
+
+      it "should raise an exception if incorrect encryption key is used" do
+        File.open(@from_path, "w") { |f| f.write("clear text") }
+        File.open(@from_path, "r") do |from|
+          File.open(@to_path, "w") do |to|
+            @client.send(:encrypt_stream, from, to)
+          end
+        end
+
+        Base64.encode64(File.read(@to_path)).should eql("XCUKDXXzjh43DmNylgVpQQ==\n")
+
+        client2 = s3_blobstore(:encryption_key    => "zzz",
+                               :bucket_name       => "test",
+                               :access_key_id     => "KEY",
+                               :secret_access_key => "SECRET")
+
+        File.open(@from_path, "w") { |f| f.write(File.read(@to_path)) }
+        File.open(@from_path, "r") do |from|
+          File.open(@to_path, "w") do |to|
+            lambda {
+              client2.send(:decrypt_stream, from, to)
+            }.should raise_error(Bosh::Blobstore::BlobstoreError, "Decryption error: bad decrypt")
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/unit/simple_blobstore_client_spec.rb

@@ -0,0 +1,106 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Bosh::Blobstore::SimpleBlobstoreClient do
+
+  before(:each) do
+    @httpclient = mock("httpclient")
+    HTTPClient.stub!(:new).and_return(@httpclient)
+  end
+
+  describe "options" do
+
+    it "should set up authentication when present" do
+      response = mock("response")
+      response.stub!(:status).and_return(200)
+      response.stub!(:content).and_return("content_id")
+
+      @httpclient.should_receive(:get).with("http://localhost/resources/foo", {},
+                                            {"Authorization"=>"Basic am9objpzbWl0aA=="}).and_return(response)
+      @client = Bosh::Blobstore::SimpleBlobstoreClient.new({"endpoint" => "http://localhost",
+                                                            "user" => "john",
+                                                            "password" => "smith"})
+      @client.get("foo")
+    end
+
+  end
+
+  describe "operations" do
+
+    it "should create an object" do
+      response = mock("response")
+      response.stub!(:status).and_return(200)
+      response.stub!(:content).and_return("content_id")
+      @httpclient.should_receive(:post).with { |*args|
+        uri, body, _ = args
+        uri.should eql("http://localhost/resources")
+        body.should be_kind_of(Hash)
+        body[:content].should be_kind_of(File)
+        body[:content].read.should eql("some object")
+        true
+      }.and_return(response)
+
+      @client = Bosh::Blobstore::SimpleBlobstoreClient.new({"endpoint" => "http://localhost"})
+      @client.create("some object").should eql("content_id")
+    end
+
+    it "should raise an exception when there is an error creating an object" do
+      response = mock("response")
+      response.stub!(:status).and_return(500)
+
+      @httpclient.should_receive(:post).with { |*args|
+        uri, body, _ = args
+        uri.should eql("http://localhost/resources")
+        body.should be_kind_of(Hash)
+        body[:content].should be_kind_of(File)
+        body[:content].read.should eql("some object")
+        true
+      }.and_return(response)
+
+      @client = Bosh::Blobstore::SimpleBlobstoreClient.new({"endpoint" => "http://localhost"})
+      lambda {@client.create("some object")}.should raise_error
+    end
+
+    it "should fetch an object" do
+      response = mock("response")
+      response.stub!(:status).and_return(200)
+      @httpclient.should_receive(:get).with("http://localhost/resources/some object", {}, {}).
+          and_yield("content_id").
+          and_return(response)
+
+      @client = Bosh::Blobstore::SimpleBlobstoreClient.new({"endpoint" => "http://localhost"})
+      @client.get("some object").should eql("content_id")
+    end
+
+    it "should raise an exception when there is an error fetching an object" do
+      response = mock("response")
+      response.stub!(:status).and_return(500)
+      response.stub!(:content).and_return("error message")
+      @httpclient.should_receive(:get).with("http://localhost/resources/some object", {}, {}).and_return(response)
+
+      @client = Bosh::Blobstore::SimpleBlobstoreClient.new({"endpoint" => "http://localhost"})
+      lambda {@client.get("some object")}.should raise_error
+    end
+
+    it "should delete an object" do
+      response = mock("response")
+      response.stub!(:status).and_return(204)
+      response.stub!(:content).and_return("")
+      @httpclient.should_receive(:delete).with("http://localhost/resources/some object", {}).and_return(response)
+
+      @client = Bosh::Blobstore::SimpleBlobstoreClient.new({"endpoint" => "http://localhost"})
+      @client.delete("some object")
+    end
+
+    it "should raise an exception when there is an error deleting an object" do
+      response = mock("response")
+      response.stub!(:status).and_return(404)
+      response.stub!(:content).and_return("")
+      @httpclient.should_receive(:delete).with("http://localhost/resources/some object", {}).and_return(response)
+
+      @client = Bosh::Blobstore::SimpleBlobstoreClient.new({"endpoint" => "http://localhost"})
+      lambda {@client.delete("some object")}.should raise_error
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: blobstore_client
+version: !ruby/object:Gem::Version
+  version: 0.3.13
+  prerelease: 
+platform: ruby
+authors:
+- VMware
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-23 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-s3
+  requirement: &70295903087360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295903087360
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: &70295903085860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295903085860
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: &70295903084300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295903084300
+- !ruby/object:Gem::Dependency
+  name: ruby-atmos-pure
+  requirement: &70295903076320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295903076320
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: &70295903075580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295903075580
+description: BOSH blobstore client
+email: support@vmware.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/blobstore_client.rb
+- lib/blobstore_client/atmos_blobstore_client.rb
+- lib/blobstore_client/base.rb
+- lib/blobstore_client/client.rb
+- lib/blobstore_client/errors.rb
+- lib/blobstore_client/local_client.rb
+- lib/blobstore_client/s3_blobstore_client.rb
+- lib/blobstore_client/simple_blobstore_client.rb
+- lib/blobstore_client/version.rb
+- README
+- Rakefile
+- spec/assets/file
+- spec/spec_helper.rb
+- spec/unit/atmos_blobstore_client_spec.rb
+- spec/unit/blobstore_client_spec.rb
+- spec/unit/local_client_spec.rb
+- spec/unit/s3_blobstore_client_spec.rb
+- spec/unit/simple_blobstore_client_spec.rb
+homepage: http://www.vmware.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2357219804209612013
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2357219804209612013
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: BOSH blobstore client
+test_files:
+- spec/assets/file
+- spec/spec_helper.rb
+- spec/unit/atmos_blobstore_client_spec.rb
+- spec/unit/blobstore_client_spec.rb
+- spec/unit/local_client_spec.rb
+- spec/unit/s3_blobstore_client_spec.rb
+- spec/unit/simple_blobstore_client_spec.rb