blobstore_client 0.4.0 → 0.5.0

data/bin/blobstore_client_console

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+#
+# Copyright (c) 2009-2012 VMware, Inc.
+#
+# Usage example:
+#
+# => Debugger enabled
+# => Welcome to BOSH blobstore client console
+# You can use 'bsc' to access blobstore client methods
+# irb(main):001:0> oid = bsc.create("test data content")
+# => "eyJvaWQiOiJlNGQ5MTUzMy1iOTZiLVlYjc1YzQ1NTAi%0ALCJwdXJsIjpudWxsfQ==%0A"
+# irb(main):002:0> bsc.get(oid)
+# => "test data content"
+# irb(main):003:0> bsc.delete(oid)
+# => true
+
+gemfile = File.expand_path("../../Gemfile", __FILE__)
+
+if File.exists?(gemfile)
+  ENV["BUNDLE_GEMFILE"] = gemfile
+  require "rubygems"
+  require "bundler/setup"
+end
+
+$:.unshift(File.expand_path("../../lib", __FILE__))
+require "blobstore_client"
+require "irb"
+require "irb/completion"
+require "ostruct"
+require "optparse"
+
+@provider = nil
+config_file = nil
+
+opts_parser = OptionParser.new do |opts|
+  opts.on("-p", "--provider PROVIDER") { |p| @provider = p }
+  opts.on("-c", "--config FILE") { |file| config_file = file }
+end
+opts_parser.parse!
+
+unless @provider && config_file
+  puts opts_parser
+  exit(1)
+end
+
+@config = YAML.load_file(config_file)
+
+module ConsoleHelpers
+  def bsc
+    @bsc ||= Bosh::Blobstore::Client.create(@provider, @config)
+  end
+end
+
+include ConsoleHelpers
+
+begin
+  require 'ruby-debug'
+  puts "=> Debugger enabled"
+rescue LoadError
+  puts "=> ruby-debug not found, debugger disabled"
+end
+
+puts "=> Welcome to BOSH blobstore client console"
+puts "You can use 'bsc' to access blobstore client methods"
+
+IRB.start

data/lib/blobstore_client/swift_blobstore_client.rb

@@ -0,0 +1,160 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "base64"
+require "fog"
+require "multi_json"
+require "uri"
+require "uuidtools"
+
+module Bosh
+  module Blobstore
+
+    class SwiftBlobstoreClient < BaseClient
+
+      # Blobstore client for Swift
+      # @param [Hash] options Swift BlobStore options
+      # @option options [Symbol] container_name
+      # @option options [Symbol] swift_provider
+      def initialize(options)
+        super(options)
+        @http_client = HTTPClient.new
+      end
+
+      def container
+        return @container if @container
+
+        validate_options(@options)
+
+        swift_provider = @options[:swift_provider]
+        swift_options = {:provider => swift_provider}
+        swift_options.merge!(@options[swift_provider.to_sym])
+        swift = Fog::Storage.new(swift_options)
+
+        container_name = @options[:container_name]
+        @container = swift.directories.get(container_name)
+        if @container.nil?
+          raise NotFound, "Swift container '#{container_name}' not found"
+        end
+        @container
+      end
+
+      def create_file(file)
+        object_id = generate_object_id
+        object = container.files.create(:key => object_id,
+                                        :body => file,
+                                        :public => true)
+        encode_object_id(object_id, object.public_url)
+      rescue Exception => e
+        raise BlobstoreError, "Failed to create object: #{e.message}"
+      end
+
+      def get_file(object_id, file)
+        object_info = decode_object_id(object_id)
+        if object_info["purl"]
+          response = @http_client.get(object_info["purl"]) do |block|
+            file.write(block)
+          end
+          if response.status != 200
+            raise BlobstoreError, "Could not fetch object, %s/%s" %
+                  [response.status, response.content]
+          end
+        else
+          object = container.files.get(object_info["oid"]) do |block|
+            file.write(block)
+          end
+          if object.nil?
+            raise NotFound, "Swift object '#{object_id}' not found"
+          end
+        end
+      rescue Exception => e
+        raise BlobstoreError,
+              "Failed to find object '#{object_id}': #{e.message}"
+      end
+
+      def delete(object_id)
+        object_info = decode_object_id(object_id)
+        object = container.files.get(object_info["oid"])
+        if object.nil?
+          raise NotFound, "Swift object '#{object_id}' not found"
+        else
+          object.destroy
+        end
+      rescue Exception => e
+        raise BlobstoreError,
+              "Failed to delete object '#{object_id}': #{e.message}"
+      end
+
+      private
+
+      def generate_object_id
+        UUIDTools::UUID.random_create.to_s
+      end
+
+      def encode_object_id(object_id, public_url = nil)
+        json = MultiJson.encode({:oid => object_id, :purl => public_url})
+        URI::escape(Base64.encode64(json))
+      end
+
+      def decode_object_id(object_id)
+        begin
+          object_info = MultiJson.decode(Base64.decode64(URI::unescape(object_id)))
+        rescue MultiJson::DecodeError => e
+          raise BlobstoreError, "Failed to parse object_id: #{e.message}"
+        end
+
+        if !object_info.kind_of?(Hash) || object_info["oid"].nil?
+          raise BlobstoreError, "Invalid object_id: #{object_info.inspect}"
+        end
+        object_info
+      end
+
+      def validate_options(options)
+        unless options.is_a?(Hash)
+          raise "Invalid options format, Hash expected, #{options.class} given"
+        end
+        unless options.has_key?(:container_name)
+          raise "Swift container name is missing"
+        end
+        unless options.has_key?(:swift_provider)
+          raise "Swift provider is missing"
+        end
+        case options[:swift_provider]
+          when "hp"
+            unless options.has_key?(:hp)
+              raise "HP options are missing"
+            end
+            unless options[:hp].is_a?(Hash)
+              raise "Invalid HP options, Hash expected,
+                    #{options[:hp].class} given"
+            end
+            unless options[:hp].has_key?(:hp_account_id)
+              raise "HP account ID is missing"
+            end
+            unless options[:hp].has_key?(:hp_secret_key)
+              raise "HP secret key is missing"
+            end
+            unless options[:hp].has_key?(:hp_tenant_id)
+              raise "HP tenant ID is missing"
+            end
+          when "rackspace"
+            unless options.has_key?(:rackspace)
+              raise "Rackspace options are missing"
+            end
+            unless options[:rackspace].is_a?(Hash)
+              raise "Invalid Rackspace options, Hash expected,
+                    #{options[:rackspace].class} given"
+            end
+            unless options[:rackspace].has_key?(:rackspace_username)
+              raise "Rackspace username is missing"
+            end
+            unless options[:rackspace].has_key?(:rackspace_api_key)
+              raise "Rackspace API key is missing"
+            end
+          else
+            raise "Swift provider #{options[:swift_provider]} not supported"
+        end
+      end
+
+    end
+  end
+end

data/lib/blobstore_client/version.rb

@@ -3,7 +3,7 @@
 module Bosh
   module Blobstore
     class Client
-      VERSION = "0.4.0"
+      VERSION = "0.5.0"
     end
   end
 end

data/lib/blobstore_client.rb

@@ -10,6 +10,7 @@ require "blobstore_client/client"
 require "blobstore_client/base"
 require "blobstore_client/simple_blobstore_client"
 require "blobstore_client/s3_blobstore_client"
+require "blobstore_client/swift_blobstore_client"
 require "blobstore_client/local_client"
 require "blobstore_client/atmos_blobstore_client"
 
@@ -20,6 +21,7 @@ module Bosh
       PROVIDER_MAP = {
         "simple" => SimpleBlobstoreClient,
         "s3" => S3BlobstoreClient,
+        "swift" => SwiftBlobstoreClient,
         "atmos" => AtmosBlobstoreClient,
         "local" => LocalClient
       }

data/spec/unit/blobstore_client_spec.rb

@@ -30,6 +30,11 @@ describe Bosh::Blobstore::Client do
     bs.should be_instance_of Bosh::Blobstore::S3BlobstoreClient
   end
 
+  it "should have an swift provider" do
+    bs = Bosh::Blobstore::Client.create('swift', {})
+    bs.should be_instance_of Bosh::Blobstore::SwiftBlobstoreClient
+  end
+
   it "should raise an exception on an unknown client" do
     lambda {
       bs = Bosh::Blobstore::Client.create('foobar', {})

data/spec/unit/swift_blobstore_client_spec.rb

@@ -0,0 +1,315 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Bosh::Blobstore::SwiftBlobstoreClient do
+
+  def swift_options(container_name, swift_provider, credentials)
+    if credentials
+      options = {
+                  "rackspace" => {
+                    "rackspace_username" => "username",
+                    "rackspace_api_key" => "api_key"
+                  },
+                  "hp" => {
+                    "hp_account_id" => "account_id",
+                    "hp_secret_key" => "secret_key",
+                    "hp_tenant_id"  => "tenant_id"
+                  }
+                }
+    else
+      options = {}
+    end
+    options["container_name"] = container_name if container_name
+    options["swift_provider"] = swift_provider if swift_provider
+    options
+  end
+
+  def swift_blobstore(options)
+    Bosh::Blobstore::SwiftBlobstoreClient.new(options)
+  end
+
+  before(:each) do
+    @swift = mock("swift")
+    Fog::Storage.stub!(:new).and_return(@swift)
+    @http_client = mock("http-client")
+    HTTPClient.stub!(:new).and_return(@http_client)
+  end
+
+  describe "on HP Cloud Storage" do
+
+    describe "with credentials" do
+
+      before(:each) do
+        @client = swift_blobstore(swift_options("test-container",
+                                                "hp",
+                                                true))
+      end
+
+      it "should create an object" do
+        data = "some content"
+        directories = double("directories")
+        container = double("container")
+        files = double("files")
+        object = double("object")
+
+        @client.should_receive(:generate_object_id).and_return("object_id")
+        @swift.stub(:directories).and_return(directories)
+        directories.should_receive(:get).with("test-container") \
+                   .and_return(container)
+        container.should_receive(:files).and_return(files)
+        files.should_receive(:create).with { |opt|
+          opt[:key].should eql "object_id"
+          #opt[:body].should eql data
+          opt[:public].should eql true
+        }.and_return(object)
+        object.should_receive(:public_url).and_return("public-url")
+
+        object_id = @client.create(data)
+        object_info = MultiJson.decode(Base64.decode64(
+                                         URI::unescape(object_id)))
+        object_info["oid"].should eql("object_id")
+        object_info["purl"].should eql("public-url")
+      end
+
+      it "should fetch an object without a public url" do
+        data = "some content"
+        directories = double("directories")
+        container = double("container")
+        files = double("files")
+        object = double("object")
+
+        @swift.stub(:directories).and_return(directories)
+        directories.should_receive(:get).with("test-container") \
+                   .and_return(container)
+        container.should_receive(:files).and_return(files)
+        files.should_receive(:get).with("object_id").and_yield(data) \
+             .and_return(object)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        @client.get(oid).should eql(data)
+      end
+
+      it "should fetch an object with a public url" do
+        data = "some content"
+        response = mock("response")
+
+        @http_client.should_receive(:get).with("public-url") \
+                    .and_yield(data).and_return(response)
+        response.stub!(:status).and_return(200)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id",
+                                             :purl => "public-url"})))
+        @client.get(oid).should eql(data)
+      end
+
+      it "should delete an object" do
+        directories = double("directories")
+        container = double("container")
+        files = double("files")
+        object = double("object")
+
+        @swift.stub(:directories).and_return(directories)
+        directories.should_receive(:get).with("test-container") \
+                   .and_return(container)
+        container.should_receive(:files).and_return(files)
+        files.should_receive(:get).with("object_id").and_return(object)
+        object.should_receive(:destroy)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        @client.delete(oid)
+      end
+
+    end
+
+    describe "without credentials" do
+
+      before(:each) do
+        @client = swift_blobstore(swift_options("test-container",
+                                                "hp",
+                                                false))
+      end
+
+      it "should refuse to create an object" do
+        data = "some content"
+
+        lambda {
+          object_id = @client.create(data)
+        }.should raise_error(Bosh::Blobstore::BlobstoreError)
+      end
+
+      it "should refuse to fetch an object without a public url" do
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        lambda {
+          @client.get(oid)
+        }.should raise_error(Bosh::Blobstore::BlobstoreError)
+      end
+
+      it "should fetch an object with a public url" do
+        data = "some content"
+        response = mock("response")
+
+        @http_client.should_receive(:get).with("public-url") \
+                    .and_yield(data).and_return(response)
+        response.stub!(:status).and_return(200)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id",
+                                             :purl => "public-url"})))
+        @client.get(oid).should eql(data)
+      end
+
+      it "should refuse to delete an object" do
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        lambda {
+          @client.delete(oid)
+        }.should raise_error(Bosh::Blobstore::BlobstoreError)
+      end
+
+    end
+
+  end
+
+  describe "on Rackspace Cloud Files" do
+
+    describe "with credentials" do
+
+      before(:each) do
+        @client = swift_blobstore(swift_options("test-container",
+                                                "rackspace",
+                                                true))
+      end
+
+      it "should create an object" do
+        data = "some content"
+        directories = double("directories")
+        container = double("container")
+        files = double("files")
+        object = double("object")
+
+        @client.should_receive(:generate_object_id).and_return("object_id")
+        @swift.stub(:directories).and_return(directories)
+        directories.should_receive(:get).with("test-container") \
+                   .and_return(container)
+        container.should_receive(:files).and_return(files)
+        files.should_receive(:create).with { |opt|
+          opt[:key].should eql "object_id"
+          #opt[:body].should eql data
+          opt[:public].should eql true
+        }.and_return(object)
+        object.should_receive(:public_url).and_return("public-url")
+
+        object_id = @client.create(data)
+        object_info = MultiJson.decode(Base64.decode64(
+                                         URI::unescape(object_id)))
+        object_info["oid"].should eql("object_id")
+        object_info["purl"].should eql("public-url")
+      end
+
+      it "should fetch an object without a public url" do
+        data = "some content"
+        directories = double("directories")
+        container = double("container")
+        files = double("files")
+        object = double("object")
+
+        @swift.stub(:directories).and_return(directories)
+        directories.should_receive(:get).with("test-container") \
+                   .and_return(container)
+        container.should_receive(:files).and_return(files)
+        files.should_receive(:get).with("object_id").and_yield(data) \
+             .and_return(object)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        @client.get(oid).should eql(data)
+      end
+
+      it "should fetch an object with a public url" do
+        data = "some content"
+        response = mock("response")
+
+        @http_client.should_receive(:get).with("public-url") \
+                    .and_yield(data).and_return(response)
+        response.stub!(:status).and_return(200)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id",
+                                             :purl => "public-url"})))
+        @client.get(oid).should eql(data)
+      end
+
+      it "should delete an object" do
+        directories = double("directories")
+        container = double("container")
+        files = double("files")
+        object = double("object")
+
+        @swift.stub(:directories).and_return(directories)
+        directories.should_receive(:get).with("test-container") \
+                   .and_return(container)
+        container.should_receive(:files).and_return(files)
+        files.should_receive(:get).with("object_id").and_return(object)
+        object.should_receive(:destroy)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        @client.delete(oid)
+      end
+
+    end
+
+    describe "without credentials" do
+
+      before(:each) do
+        @client = swift_blobstore(swift_options("test-container",
+                                                "rackspace",
+                                                false))
+      end
+
+      it "should refuse to create an object" do
+        data = "some content"
+
+        lambda {
+          object_id = @client.create(data)
+        }.should raise_error(Bosh::Blobstore::BlobstoreError)
+      end
+
+      it "should refuse to fetch an object without a public url" do
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        lambda {
+          @client.get(oid)
+        }.should raise_error(Bosh::Blobstore::BlobstoreError)
+      end
+
+      it "should fetch an object with a public url" do
+        data = "some content"
+        response = mock("response")
+
+        @http_client.should_receive(:get).with("public-url") \
+                    .and_yield(data).and_return(response)
+        response.stub!(:status).and_return(200)
+
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id",
+                                             :purl => "public-url"})))
+        @client.get(oid).should eql(data)
+      end
+
+      it "should refuse to delete an object" do
+        oid = URI::escape(Base64.encode64(MultiJson.encode(
+                                            {:oid => "object_id"})))
+        lambda {
+          @client.delete(oid)
+        }.should raise_error(Bosh::Blobstore::BlobstoreError)
+      end
+
+    end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: blobstore_client
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-16 00:00:00.000000000 Z
+date: 2012-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-s3
@@ -27,6 +27,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.2
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -109,10 +125,12 @@ dependencies:
         version: '0.5'
 description: BOSH blobstore client
 email: support@vmware.com
-executables: []
+executables:
+- blobstore_client_console
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/blobstore_client_console
 - lib/blobstore_client.rb
 - lib/blobstore_client/atmos_blobstore_client.rb
 - lib/blobstore_client/base.rb
@@ -121,6 +139,7 @@ files:
 - lib/blobstore_client/local_client.rb
 - lib/blobstore_client/s3_blobstore_client.rb
 - lib/blobstore_client/simple_blobstore_client.rb
+- lib/blobstore_client/swift_blobstore_client.rb
 - lib/blobstore_client/version.rb
 - README
 - Rakefile
@@ -131,6 +150,7 @@ files:
 - spec/unit/local_client_spec.rb
 - spec/unit/s3_blobstore_client_spec.rb
 - spec/unit/simple_blobstore_client_spec.rb
+- spec/unit/swift_blobstore_client_spec.rb
 homepage: http://www.vmware.com
 licenses: []
 post_install_message: 
@@ -145,7 +165,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4134087797262691141
+      hash: -2537561053731340493
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -154,7 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4134087797262691141
+      hash: -2537561053731340493
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -169,3 +189,4 @@ test_files:
 - spec/unit/local_client_spec.rb
 - spec/unit/s3_blobstore_client_spec.rb
 - spec/unit/simple_blobstore_client_spec.rb
+- spec/unit/swift_blobstore_client_spec.rb