blobstore_client 0.5.0 → 1.5.0.pre.1113

data/README.md

@@ -0,0 +1,127 @@
+# Ruby client for Blobstores
+Copyright (c) 2009-2013 VMware, Inc.
+
+Lets BOSH access multiple blobstores using a unified API.
+
+## Usage
+
+    bin/blobstore_client_console [<options>]
+        -p, --provider PROVIDER Bosh Blobstore provider
+        -c, --config FILE       Bosh Blobstore configuration file
+
+## Console
+
+To explore the client API for accessing a blobstore, try creating and using a local blobstore:
+
+```
+$ gem install blobstore_client
+$ blobstore_client_console -p local -c config/local.yml.example
+=> Welcome to BOSH blobstore client console
+You can use 'bsc' to access blobstore client methods
+> bsc.create("this is a test blob")
+=> "ef00746b-21ec-4473-a888-bf257cb7ea21" 
+> bsc.get("ef00746b-21ec-4473-a888-bf257cb7ea21")
+=> "this is a test blob"
+> bsc.exists?("ef00746b-21ec-4473-a888-bf257cb7ea21")
+=> true
+> Dir['/tmp/local_blobstore/**']
+=> ["/tmp/local_blobstore/ef00746b-21ec-4473-a888-bf257cb7ea21"]
+> bsc.delete("ef00746b-21ec-4473-a888-bf257cb7ea21")
+=> true
+```
+
+## Configuration
+
+These options are passed to the Bosh Blobstore client when it is instantiated.
+
+### Local
+
+These are the options for the Blobstore client when provider is `local`:
+
+* `blobstore_path` (required)
+  Path for the blobstore
+
+### Simple
+
+These are the options for the Blobstore client when provider is `simple`:
+
+* `endpoint` (required)
+  Blobstore endpoint
+* `user` (optional)
+  Blobstore User
+* `password` (optional)
+  Blobstore Password
+* `bucket` (optional, by default `resources`)
+  Name of the bucket
+
+### Amazon S3
+
+These are the options for the Blobstore client when provider is `s3`:
+
+* `bucket_name` (required)
+  Name of the S3 bucket
+* `encryption_key` (optional)
+  Encryption_key that is applied before the object is sent to S3
+* `access_key_id` (optional, if not present, the blobstore client operates in read only mode)
+  S3 Access Key
+* `secret_access_key` (optional, if not present, the blobstore client operates in read only mode)
+  S3 Secret Access Key
+
+### Atmos
+
+These are the options for the Blobstore client when provider is `atmos`:
+
+* `url` (required)
+  Atmos URL
+* `uid` (required)
+  Atmos UID
+* `secret` (required)
+  Atmos password
+
+### OpenStack Swift provider
+
+These are the options for the Blobstore client when provider is `swift`:
+
+* `container_name` (required)
+  Name of the container
+* `swift_provider` (required)
+  OpenStack Swift provider (supported: `hp`, `openstack` and `rackspace`)
+
+#### HP Object Storage
+
+These are the options for the Blobstore client when `swift_provider` is `hp`:
+
+* `hp_access_key` (required)
+  HP Object Storage Access Key
+* `hp_secret_key` (required)
+  HP Object Storage Secret Key
+* `hp_tenant_id` (required)
+  HP Object Storage Project ID
+* `hp_avl_zone` (required)
+  HP Object Storage Availability Zone (`region-a.geo-1` or `region-b.geo-1`)
+
+#### OpenStack Object Storage
+
+These are the options for the Blobstore client when `swift_provider` is `openstack`:
+
+* `openstack_auth_url` (required)
+  URL of the OpenStack Identity endpoint to connect to
+* `openstack_username` (required)
+  OpenStack user name
+* `openstack_api_key` (required)
+  OpenStack API key
+* `openstack_tenant` (required)
+  OpenStack tenant name
+* `openstack_region` (optional)
+  OpenStack region
+
+#### Rackspace Cloud Files
+
+These are the options for the Blobstore client when `swift_provider` is `rackspace`:
+
+* `rackspace_username` (required)
+  Rackspace Cloud Files Username
+* `rackspace_api_key` (required)
+  Rackspace Cloud Files API Key
+* `rackspace_region` (optional)
+  Rackspace Cloud Files Region (`dfw` or `ord`)

data/bin/blobstore_client_console

@@ -14,36 +14,37 @@
 # irb(main):003:0> bsc.delete(oid)
 # => true
 
-gemfile = File.expand_path("../../Gemfile", __FILE__)
-
-if File.exists?(gemfile)
-  ENV["BUNDLE_GEMFILE"] = gemfile
-  require "rubygems"
-  require "bundler/setup"
-end
-
-$:.unshift(File.expand_path("../../lib", __FILE__))
-require "blobstore_client"
-require "irb"
-require "irb/completion"
-require "ostruct"
-require "optparse"
+require 'blobstore_client'
+require 'irb'
+require 'irb/completion'
+require 'ostruct'
+require 'optparse'
+require 'psych'
 
 @provider = nil
 config_file = nil
 
 opts_parser = OptionParser.new do |opts|
-  opts.on("-p", "--provider PROVIDER") { |p| @provider = p }
-  opts.on("-c", "--config FILE") { |file| config_file = file }
+  opts.on('-p', '--provider PROVIDER') { |p| @provider = p }
+  opts.on('-c', '--config FILE') { |file| config_file = file }
 end
 opts_parser.parse!
 
 unless @provider && config_file
   puts opts_parser
+  puts "\nExample config file:"
+  puts <<-YAML
+---
+endpoint: http://1.2.3.4:25250
+user: agent
+password: agent
+bucket: resources
+
+  YAML
   exit(1)
 end
 
-@config = YAML.load_file(config_file)
+@config = Psych.load_file(config_file)
 
 module ConsoleHelpers
   def bsc
@@ -55,12 +56,12 @@ include ConsoleHelpers
 
 begin
   require 'ruby-debug'
-  puts "=> Debugger enabled"
+  puts '=> Debugger enabled'
 rescue LoadError
-  puts "=> ruby-debug not found, debugger disabled"
+  puts '=> ruby-debug not found, debugger disabled'
 end
 
-puts "=> Welcome to BOSH blobstore client console"
+puts '=> Welcome to BOSH blobstore client console'
 puts "You can use 'bsc' to access blobstore client methods"
 
 IRB.start

data/config/local.yml.example

@@ -0,0 +1,2 @@
+---
+blobstore_path: /tmp/local_blobstore

data/lib/blobstore_client.rb

@@ -1,43 +1,14 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-module Bosh; module Blobstore; end; end
-
-require "common/common"
-require "blobstore_client/version"
-require "blobstore_client/errors"
-
-require "blobstore_client/client"
-require "blobstore_client/base"
-require "blobstore_client/simple_blobstore_client"
-require "blobstore_client/s3_blobstore_client"
-require "blobstore_client/swift_blobstore_client"
-require "blobstore_client/local_client"
-require "blobstore_client/atmos_blobstore_client"
-
-module Bosh
-  module Blobstore
-    class Client
-
-      PROVIDER_MAP = {
-        "simple" => SimpleBlobstoreClient,
-        "s3" => S3BlobstoreClient,
-        "swift" => SwiftBlobstoreClient,
-        "atmos" => AtmosBlobstoreClient,
-        "local" => LocalClient
-      }
-
-      def self.create(blobstore_provider, options = {})
-        provider = PROVIDER_MAP[blobstore_provider]
-
-        unless provider
-          providers = PROVIDER_MAP.keys.sort.join(", ")
-          raise BlobstoreError,
-            "Invalid client provider, available providers are: #{providers}"
-        end
-
-        provider.new(options)
-      end
-
-    end
-  end
-end
+require 'common/common'
+require 'blobstore_client/version'
+require 'blobstore_client/errors'
+require 'blobstore_client/client'
+
+Bosh::Blobstore.autoload(:BaseClient, 'blobstore_client/base')
+Bosh::Blobstore.autoload(:S3BlobstoreClient, 'blobstore_client/s3_blobstore_client')
+Bosh::Blobstore.autoload(:SimpleBlobstoreClient, 'blobstore_client/simple_blobstore_client')
+Bosh::Blobstore.autoload(:SwiftBlobstoreClient, 'blobstore_client/swift_blobstore_client')
+Bosh::Blobstore.autoload(:AtmosBlobstoreClient, 'blobstore_client/atmos_blobstore_client')
+Bosh::Blobstore.autoload(:LocalClient, 'blobstore_client/local_client')
+Bosh::Blobstore.autoload(:DavBlobstoreClient, 'blobstore_client/dav_blobstore_client')

data/lib/blobstore_client/atmos_blobstore_client.rb

@@ -1,93 +1,110 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-require "atmos"
-require "uri"
-require "multi_json"
+require 'atmos'
+require 'uri'
+require 'multi_json'
+require 'httpclient'
 
 module Bosh
   module Blobstore
     class AtmosBlobstoreClient < BaseClient
-      SHARE_URL_EXP = "1893484800" # expires on 2030 Jan-1
+      SHARE_URL_EXP = '1893484800' # expires on 2030 Jan-1
 
       def initialize(options)
         super(options)
         @atmos_options = {
-          :url => @options[:url],
-          :uid => @options[:uid],
-          :secret => @options[:secret]
+          url: @options[:url],
+          uid: @options[:uid],
+          secret: @options[:secret]
         }
         @tag = @options[:tag]
-        @http_client = HTTPClient.new
-        # TODO: Remove this line once we get the proper certificate for atmos
+
+        # Add proxy if ENV has the variable
+        proxy = case URI.parse(@atmos_options[:url] || '').scheme
+                  when 'https'
+                    ENV['HTTPS_PROXY'] || ENV['https_proxy']
+                  when 'http'
+                    ENV['HTTP_PROXY'] || ENV['http_proxy']
+                  else
+                    nil
+                end
+        if proxy
+          @atmos_options[:proxy] = proxy
+          @http_client = HTTPClient.new(proxy: proxy)
+        end
+        @http_client ||= HTTPClient.new
         @http_client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
 
       def atmos_server
-        unless @atmos_options[:secret]
-          raise "Atmos password is missing (read-only mode)"
-        end
+        raise 'Atmos password is missing (read-only mode)' unless @atmos_options[:secret]
+
         @atmos ||= Atmos::Store.new(@atmos_options)
       end
 
-      def create_file(file)
-        obj_conf = {:data => file, :length => File.size(file.path)}
-        obj_conf[:listable_metadata] = {@tag => true} if @tag
+      protected
+
+      def create_file(id, file)
+        raise BlobstoreError, 'Atmos does not support supplying the object id' if id
+        obj_conf = { data: file, length: File.size(file.path) }
+        obj_conf[:listable_metadata] = { @tag => true } if @tag
         object_id = atmos_server.create(obj_conf).aoid
         encode_object_id(object_id)
       end
 
       def get_file(object_id, file)
         object_info = decode_object_id(object_id)
-        oid = object_info["oid"]
-        sig = object_info["sig"]
+        oid = object_info['oid']
+        sig = object_info['sig']
 
         url = @atmos_options[:url] + "/rest/objects/#{oid}?uid=" +
-              URI::escape(@atmos_options[:uid]) +
-              "&expires=#{SHARE_URL_EXP}&signature=#{URI::escape(sig)}"
+          URI.escape(@atmos_options[:uid]) +
+          "&expires=#{SHARE_URL_EXP}&signature=#{URI.escape(sig)}"
 
         response = @http_client.get(url) do |block|
           file.write(block)
         end
 
         if response.status != 200
-          raise BlobstoreError, "Could not fetch object, %s/%s" %
-            [response.status, response.content]
+          raise BlobstoreError, sprintf('Could not fetch object, %s/%s', [response.status, response.content])
         end
       end
 
-      def delete(object_id)
+      def delete_object(object_id)
         object_info = decode_object_id(object_id)
-        oid = object_info["oid"]
-        atmos_server.get(:id => oid).delete
-      rescue Atmos::Exceptions::NoSuchObjectException => e
+        oid = object_info['oid']
+        atmos_server.get(id: oid).delete
+      rescue Atmos::Exceptions::NoSuchObjectException
         raise NotFound, "Atmos object '#{object_id}' not found"
       end
 
+      def object_exists?(object_id)
+        atmos_server.get(id: object_id).exists?
+      end
+
       private
 
       def decode_object_id(object_id)
         begin
-          object_info = MultiJson.decode(Base64.decode64(URI::unescape(object_id)))
-        rescue MultiJson::DecodeError => e
-          raise BlobstoreError, "Failed to parse object_id. " +
-            "Please try updating the release"
+          object_info = MultiJson.decode(Base64.decode64(URI.unescape(object_id)))
+        rescue MultiJson::DecodeError
+          raise BlobstoreError, 'Failed to parse object_id. Please try updating the release'
         end
 
-        if !object_info.kind_of?(Hash) || object_info["oid"].nil? ||
-            object_info["sig"].nil?
+        if !object_info.kind_of?(Hash) || object_info['oid'].nil? ||
+          object_info['sig'].nil?
           raise BlobstoreError, "Invalid object_id (#{object_id})"
         end
         object_info
       end
 
       def encode_object_id(object_id)
-        hash_string = "GET" + "\n" + "/rest/objects/" + object_id + "\n" +
-                      @atmos_options[:uid] + "\n" + SHARE_URL_EXP
+        hash_string = "GET\n/rest/objects/#{object_id}\n#{@atmos_options[:uid]}\n#{SHARE_URL_EXP}"
         secret = Base64.decode64(@atmos_options[:secret])
         sig = HMAC::SHA1.digest(secret, hash_string)
         signature = Base64.encode64(sig.to_s).chomp
-        json = MultiJson.encode({:oid => object_id, :sig => signature})
-        URI::escape(Base64.encode64(json))
+        json = MultiJson.encode({ oid: object_id, sig: signature })
+        URI.escape(Base64.encode64(json))
       end
     end
   end

data/lib/blobstore_client/base.rb

@@ -1,76 +1,107 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-require "tmpdir"
+require 'tmpdir'
+require 'securerandom'
 
 module Bosh
   module Blobstore
     class BaseClient < Client
 
+      # @param [Hash] options blobstore specific options
       def initialize(options)
         @options = Bosh::Common.symbolize_keys(options)
       end
 
-      def symbolize_keys(hash)
-        hash.inject({}) do |h, (key, value)|
-          h[key.to_sym] = value
-          h
-        end
-      end
-
-      def create_file(file)
-
-      end
-
-      def get_file(id, file)
-
-      end
-
-      def create(contents)
+      # Saves a file or a string to the blobstore.
+      # if it is a String, it writes it to a temp file
+      # then calls create_file() with the (temp) file
+      # @overload create(contents, id=nil)
+      #   @param [String] contents contents to upload
+      #   @param [String] id suggested object id, if nil a uuid is generated
+      # @overload create(file, id=nil)
+      #   @param [File] file file to upload
+      #   @param [String] id suggested object id, if nil a uuid is generated
+      # @return [String] object id of the created blobstore object
+      def create(contents, id = nil)
         if contents.kind_of?(File)
-          create_file(contents)
+          create_file(id, contents)
         else
           temp_path do |path|
-            begin
-              File.open(path, "w") do |file|
-                file.write(contents)
-              end
-              return create_file(File.open(path, "r"))
-            rescue BlobstoreError => e
-              raise e
-            rescue Exception => e
-              raise BlobstoreError,
-                "Failed to create object, underlying error: %s %s" %
-                [e.message, e.backtrace.join("\n")]
+            File.open(path, 'w') do |file|
+              file.write(contents)
             end
+            return create_file(id, File.open(path, 'r'))
           end
         end
+      rescue BlobstoreError => e
+        raise e
+      rescue Exception => e
+        raise BlobstoreError,
+              sprintf('Failed to create object, underlying error: %s %s', e.inspect, e.backtrace.join("\n"))
       end
 
+      # Get an object from the blobstore.
+      # @param [String] id object id
+      # @param [File] file where to store the fetched object
+      # @return [String] the object contents if the file parameter is nil
       def get(id, file = nil)
         if file
           get_file(id, file)
+          file.flush
         else
           result = nil
           temp_path do |path|
-            begin
-              File.open(path, "w") { |file| get_file(id, file) }
-              result = File.open(path, "r") { |file| file.read }
-            rescue BlobstoreError => e
-              raise e
-            rescue Exception => e
-              raise BlobstoreError,
-                "Failed to create object, underlying error: %s %s" %
-                [e.message, e.backtrace.join("\n")]
-            end
+            File.open(path, 'w') { |f| get_file(id, f) }
+            result = File.open(path, 'r') { |f| f.read }
           end
           result
         end
+      rescue BlobstoreError => e
+        raise e
+      rescue Exception => e
+        raise BlobstoreError,
+              sprintf('Failed to fetch object, underlying error: %s %s', e.inspect, e.backtrace.join("\n"))
+      end
+
+      # @return [void]
+      def delete(oid)
+        delete_object(oid)
+      end
+
+      # @return [Boolean]
+      def exists?(oid)
+        object_exists?(oid)
       end
 
       protected
 
+      # @return [String] the id
+      def create_file(id, file)
+        # needs to be implemented in each subclass
+        not_supported
+      end
+
+      def get_file(id, file)
+        # needs to be implemented in each subclass
+        not_supported
+      end
+
+      def delete_object(oid)
+        # needs to be implemented in each subclass
+        not_supported
+      end
+
+      def object_exists?(oid)
+        # needs to be implemented in each subclass
+        not_supported
+      end
+
+      def generate_object_id
+        SecureRandom.uuid
+      end
+
       def temp_path
-        path = File.join(Dir::tmpdir, "temp-path-#{UUIDTools::UUID.random_create}")
+        path = File.join(Dir.tmpdir, "temp-path-#{SecureRandom.uuid}")
         begin
           yield path if block_given?
           path
@@ -79,6 +110,11 @@ module Bosh
         end
       end
 
+      private
+
+      def not_supported
+        raise NotImplemented, 'not supported by this blobstore'
+      end
     end
   end
 end