blobstore_client 0.5.0 → 1.5.0.pre.1113

data/lib/blobstore_client/client.rb

@@ -3,17 +3,22 @@
 module Bosh
   module Blobstore
     class Client
-      def create(contents)
-      end
+      PROVIDER_NAMES = %w[dav simple s3 swift atmos local]
+
+      def self.create(blobstore_provider, options = {})
+        unless PROVIDER_NAMES.include? blobstore_provider
+          raise BlobstoreError, "Invalid client provider, available providers are: #{PROVIDER_NAMES}"
+        end
 
-      def get(id, file = nil)
+        blobstore_client_constantize(blobstore_provider).new(options)
       end
 
-      def delete(id)
+      private
+
+      def self.blobstore_client_constantize(base_string)
+        class_string = base_string.capitalize + (base_string == 'local' ? '' : 'Blobstore') + 'Client'
+        Bosh::Blobstore.const_get class_string
       end
     end
   end
 end
-
-
-

data/lib/blobstore_client/dav_blobstore_client.rb

@@ -0,0 +1,67 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require 'base64'
+require 'httpclient'
+require 'digest/sha1'
+
+module Bosh
+  module Blobstore
+    class DavBlobstoreClient < BaseClient
+
+      def initialize(options)
+        super(options)
+        @client = HTTPClient.new
+        @endpoint = @options[:endpoint]
+        # @bucket = @options[:bucket] || "resources" # dav (or simple) doesn't support buckets
+        @headers = {}
+        user = @options[:user]
+        password = @options[:password]
+        if user && password
+          @headers['Authorization'] = 'Basic ' +
+            Base64.encode64("#{user}:#{password}").strip
+        end
+      end
+
+      def url(id)
+        prefix = Digest::SHA1.hexdigest(id)[0, 2]
+
+        [@endpoint, prefix, id].compact.join('/')
+      end
+
+      def create_file(id, file)
+        id ||= generate_object_id
+
+        response = @client.put(url(id), file, @headers)
+
+        raise BlobstoreError, "Could not create object, #{response.status}/#{response.content}" if response.status != 201
+
+        id
+      end
+
+      def get_file(id, file)
+        response = @client.get(url(id), {}, @headers) do |block|
+          file.write(block)
+        end
+
+        raise BlobstoreError, "Could not fetch object, #{response.status}/#{response.content}" if response.status != 200
+      end
+
+      def delete_object(id)
+        response = @client.delete(url(id), @headers)
+
+        raise BlobstoreError, "Could not delete object, #{response.status}/#{response.content}" if response.status != 204
+      end
+
+      def object_exists?(id)
+        response = @client.head(url(id), header: @headers)
+        if response.status == 200
+          true
+        elsif response.status == 404
+          false
+        else
+          raise BlobstoreError, "Could not get object existence, #{response.status}/#{response.content}"
+        end
+      end
+    end
+  end
+end

data/lib/blobstore_client/errors.rb

@@ -5,6 +5,7 @@ module Bosh
 
     class BlobstoreError < StandardError; end
     class NotFound < BlobstoreError; end
+    class NotImplemented < BlobstoreError; end
 
   end
 end

data/lib/blobstore_client/local_client.rb

@@ -3,47 +3,56 @@
 module Bosh
   module Blobstore
     class LocalClient < BaseClient
-      CHUNK_SIZE = 1024*1024
+      CHUNK_SIZE = 1024 * 1024
 
       def initialize(options)
         super(options)
-        @blobstore_path = @options[:blobstore_path]
-        raise "No blobstore path given" if @blobstore_path.nil?
+        @blobstore_path = URI(@options[:blobstore_path]).path
+        raise "No blobstore path given in options #{@options}" if @blobstore_path.nil?
         FileUtils.mkdir_p(@blobstore_path) unless File.directory?(@blobstore_path)
       end
 
-      def create_file(file)
-        id = UUIDTools::UUID.random_create.to_s
-        dst = File.join(@blobstore_path, id)
+      protected
+
+      def create_file(id, file)
+        id ||= generate_object_id
+        dst = object_file_path(id)
+        raise BlobstoreError, "object id #{id} is already in use" if File.exist?(dst)
         File.open(dst, 'w') do |fh|
-          until file.eof?
-            fh.write(file.read(CHUNK_SIZE))
-          end
+          fh.write(file.read(CHUNK_SIZE)) until file.eof?
         end
         id
       end
 
       def get_file(id, file)
-        src = File.join(@blobstore_path, id)
+        src = object_file_path(id)
 
         begin
           File.open(src, 'r') do |src_fh|
-            until src_fh.eof?
-              file.write(src_fh.read(CHUNK_SIZE))
-            end
+            file.write(src_fh.read(CHUNK_SIZE)) until src_fh.eof?
           end
         end
       rescue Errno::ENOENT
         raise NotFound, "Blobstore object '#{id}' not found"
       end
 
-      def delete(id)
-        file = File.join(@blobstore_path, id)
+      def delete_object(id)
+        file = object_file_path(id)
         FileUtils.rm(file)
       rescue Errno::ENOENT
         raise NotFound, "Blobstore object '#{id}' not found"
       end
 
+      def object_exists?(oid)
+        File.exists?(object_file_path(oid))
+      end
+
+      private
+
+      def object_file_path(oid)
+        File.join(@blobstore_path, oid)
+      end
+
     end
   end
 end

data/lib/blobstore_client/s3_blobstore_client.rb

@@ -1,20 +1,20 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-require "openssl"
-require "digest/sha1"
-require "base64"
-require "aws/s3"
-require "uuidtools"
+require 'openssl'
+require 'digest/sha1'
+require 'base64'
+require 'aws'
+require 'securerandom'
 
 module Bosh
   module Blobstore
 
     class S3BlobstoreClient < BaseClient
 
-      ENDPOINT = "https://s3.amazonaws.com"
-      DEFAULT_CIPHER_NAME = "aes-128-cbc"
+      ENDPOINT = 'https://s3.amazonaws.com'
+      DEFAULT_CIPHER_NAME = 'aes-128-cbc'
 
-      attr_reader :bucket_name, :encryption_key
+      attr_reader :bucket_name, :encryption_key, :simple
 
       # Blobstore client for S3 with optional object encryption
       # @param [Hash] options S3connection options
@@ -32,144 +32,145 @@ module Bosh
         @encryption_key = @options[:encryption_key]
 
         aws_options = {
-          :access_key_id     => @options[:access_key_id],
-          :secret_access_key => @options[:secret_access_key],
-          :use_ssl           => true,
-          :port              => 443
+          access_key_id: @options[:access_key_id],
+          secret_access_key: @options[:secret_access_key],
+          use_ssl: true,
+          port: 443,
+          s3_endpoint: URI.parse(@options[:endpoint] || S3BlobstoreClient::ENDPOINT).host,
         }
 
         # using S3 without credentials is a special case:
         # it is really the simple blobstore client with a bucket name
         if read_only?
+          if @encryption_key
+            raise BlobstoreError, "can't use read-only with an encryption key"
+          end
+
           unless @options[:bucket_name] || @options[:bucket]
-            raise BlobstoreError, "bucket name required"
+            raise BlobstoreError, 'bucket name required'
           end
+
           @options[:bucket] ||= @options[:bucket_name]
           @options[:endpoint] ||= S3BlobstoreClient::ENDPOINT
           @simple = SimpleBlobstoreClient.new(@options)
         else
-          AWS::S3::Base.establish_connection!(aws_options)
+          @s3 = AWS::S3.new(aws_options)
         end
 
-      rescue AWS::S3::S3Exception => e
+      rescue AWS::Errors::Base => e
         raise BlobstoreError, "Failed to initialize S3 blobstore: #{e.message}"
       end
 
-      def create_file(file)
-        raise BlobstoreError, "unsupported action" if @simple
+      # @param [File] file file to store in S3
+      def create_file(object_id, file)
+        raise BlobstoreError, 'unsupported action' if @simple
 
-        object_id = generate_object_id
+        object_id ||= generate_object_id
 
-        if @encryption_key
-          temp_path do |path|
-            File.open(path, "w") do |temp_file|
-              encrypt_stream(file, temp_file)
-            end
-            File.open(path, "r") do |temp_file|
-              AWS::S3::S3Object.store(object_id, temp_file, bucket_name)
-            end
-          end
-        elsif file.is_a?(String)
-          File.open(file, "r") do |temp_file|
-            AWS::S3::S3Object.store(object_id, temp_file, bucket_name)
-          end
-        else # Ruby 1.8 passes a File
-          AWS::S3::S3Object.store(object_id, file, bucket_name)
-        end
+        file = encrypt_file(file) if @encryption_key
+
+        # in Ruby 1.8 File doesn't respond to :path
+        path = file.respond_to?(:path) ? file.path : file
+        store_in_s3(path, full_oid_path(object_id))
 
         object_id
-      rescue AWS::S3::S3Exception => e
-        raise BlobstoreError,
-          "Failed to create object, S3 response error: #{e.message}"
+      rescue AWS::Errors::Base => e
+        raise BlobstoreError, "Failed to create object, S3 response error: #{e.message}"
+      ensure
+        FileUtils.rm(file) if @encryption_key
       end
 
+      # @param [String] object_id object id to retrieve
+      # @param [File] file file to store the retrived object in
       def get_file(object_id, file)
+
+        object_id = full_oid_path(object_id)
         return @simple.get_file(object_id, file) if @simple
 
-        object = AWS::S3::S3Object.find(object_id, bucket_name)
-        from = lambda { |callback|
-          object.value { |segment|
-            # Looks like the aws code calls this block even if segment is empty.
-            # Ideally it should be fixed upstream in the aws gem.
-            unless segment.empty?
-              callback.call(segment)
-            end
-          }
-        }
         if @encryption_key
-          decrypt_stream(from, file)
-        else
-          to_stream = write_stream(file)
-          read_stream(from) { |segment| to_stream.call(segment) }
+          cipher = OpenSSL::Cipher::Cipher.new(DEFAULT_CIPHER_NAME)
+          cipher.decrypt
+          cipher.key = Digest::SHA1.digest(encryption_key)[0..(cipher.key_len - 1)]
+        end
+
+        object = get_object_from_s3(object_id)
+        object.read do |chunk|
+          if @encryption_key
+            file.write(cipher.update(chunk))
+          else
+            file.write(chunk)
+          end
         end
-      rescue AWS::S3::NoSuchKey => e
+        file.write(cipher.final) if @encryption_key
+
+      rescue AWS::S3::Errors::NoSuchKey => e
         raise NotFound, "S3 object '#{object_id}' not found"
-      rescue AWS::S3::S3Exception => e
-        raise BlobstoreError,
-          "Failed to find object '#{object_id}', S3 response error: #{e.message}"
+      rescue AWS::Errors::Base => e
+        raise BlobstoreError, "Failed to find object '#{object_id}', S3 response error: #{e.message}"
+      end
+
+      # @param [String] object_id object id to delete
+      def delete_object(object_id)
+        raise BlobstoreError, 'unsupported action' if @simple
+        object_id = full_oid_path(object_id)
+        object = get_object_from_s3(object_id)
+        unless object.exists?
+          raise BlobstoreError, "no such object: #{object_id}"
+        end
+        object.delete
+      rescue AWS::Errors::Base => e
+        raise BlobstoreError, "Failed to delete object '#{object_id}', S3 response error: #{e.message}"
       end
 
-      def delete(object_id)
-        raise BlobstoreError, "unsupported action" if @simple
+      def object_exists?(object_id)
+        object_id = full_oid_path(object_id)
+        return simple.exists?(object_id) if simple
 
-        AWS::S3::S3Object.delete(object_id, bucket_name)
-      rescue AWS::S3::S3Exception => e
-        raise BlobstoreError,
-          "Failed to delete object '#{object_id}', S3 response error: #{e.message}"
+        get_object_from_s3(object_id).exists?
       end
 
       protected
 
-      def generate_object_id
-        UUIDTools::UUID.random_create.to_s
+      # @param [String] oid object id
+      # @return [AWS::S3::S3Object] S3 object
+      def get_object_from_s3(oid)
+        @s3.buckets[bucket_name].objects[oid]
       end
 
-      def encrypt_stream(from, to)
-        cipher = OpenSSL::Cipher::Cipher.new(DEFAULT_CIPHER_NAME)
-        cipher.encrypt
-        cipher.key = Digest::SHA1.digest(encryption_key)[0..cipher.key_len-1]
-
-        to_stream = write_stream(to)
-        read_stream(from) { |segment| to_stream.call(cipher.update(segment)) }
-        to_stream.call(cipher.final)
-      rescue StandardError => e
-        raise BlobstoreError, "Encryption error: #{e}"
+      # @param [String] path path to file which will be stored in S3
+      # @param [String] oid object id
+      # @return [void]
+      def store_in_s3(path, oid)
+        s3_object = get_object_from_s3(oid)
+        raise BlobstoreError, "object id #{oid} is already in use" if s3_object.exists?
+        File.open(path, 'r') do |temp_file|
+          s3_object.write(temp_file)
+        end
       end
 
-      def decrypt_stream(from, to)
+      def encrypt_file(file)
         cipher = OpenSSL::Cipher::Cipher.new(DEFAULT_CIPHER_NAME)
-        cipher.decrypt
-        cipher.key = Digest::SHA1.digest(encryption_key)[0..cipher.key_len-1]
-
-        to_stream = write_stream(to)
-        read_stream(from) { |segment| to_stream.call(cipher.update(segment)) }
-        to_stream.call(cipher.final)
-      rescue StandardError => e
-        raise BlobstoreError, "Decryption error: #{e}"
-      end
+        cipher.encrypt
+        cipher.key = Digest::SHA1.digest(encryption_key)[0..(cipher.key_len - 1)]
 
-      def read_stream(stream, &block)
-        if stream.respond_to?(:read)
-          while contents = stream.read(32768)
-            block.call(contents)
+        path = temp_path
+        File.open(path, 'w') do |temp_file|
+          while (block = file.read(32768))
+            temp_file.write(cipher.update(block))
           end
-        elsif stream.kind_of?(Proc)
-          stream.call(block)
+          temp_file.write(cipher.final)
         end
-      end
 
-      def write_stream(stream)
-        if stream.respond_to?(:write)
-          lambda { |contents| stream.write(contents)}
-        elsif stream.kind_of?(Proc)
-          stream
-        end
+        path
       end
 
       def read_only?
         @options[:access_key_id].nil? && @options[:secret_access_key].nil?
       end
 
+      def full_oid_path(object_id)
+         @options[:folder] ?  @options[:folder] + '/' + object_id : object_id
+      end
     end
   end
 end

data/lib/blobstore_client/simple_blobstore_client.rb

@@ -1,6 +1,7 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-require "httpclient"
+require 'base64'
+require 'httpclient'
 
 module Bosh
   module Blobstore
@@ -10,25 +11,25 @@ module Bosh
         super(options)
         @client = HTTPClient.new
         @endpoint = @options[:endpoint]
-        @bucket = @options[:bucket] || "resources"
+        @bucket = @options[:bucket] || 'resources'
         @headers = {}
         user = @options[:user]
         password = @options[:password]
         if user && password
-          @headers["Authorization"] = "Basic " +
+          @headers['Authorization'] = 'Basic ' +
             Base64.encode64("#{user}:#{password}").strip
         end
       end
 
-      def url(id=nil)
-        ["#{@endpoint}/#{@bucket}", id].compact.join("/")
+      def url(id = nil)
+        ["#{@endpoint}/#{@bucket}", id].compact.join('/')
       end
 
-      def create_file(file)
-        response = @client.post(url, {:content => file}, @headers)
+      def create_file(id, file)
+        response = @client.post(url(id), { content: file }, @headers)
         if response.status != 200
           raise BlobstoreError,
-            "Could not create object, #{response.status}/#{response.content}"
+                "Could not create object, #{response.status}/#{response.content}"
         end
         response.content
       end
@@ -40,14 +41,26 @@ module Bosh
 
         if response.status != 200
           raise BlobstoreError,
-            "Could not fetch object, #{response.status}/#{response.content}"
+                "Could not fetch object, #{response.status}/#{response.content}"
         end
       end
 
-      def delete(id)
+      def delete_object(id)
         response = @client.delete(url(id), @headers)
         if response.status != 204
-          raise "Could not delete object, #{response.status}/#{response.content}"
+          raise BlobstoreError,
+                "Could not delete object, #{response.status}/#{response.content}"
+        end
+      end
+
+      def object_exists?(id)
+        response = @client.head(url(id), header: @headers)
+        if response.status == 200
+          true
+        elsif response.status == 404
+          false
+        else
+          raise BlobstoreError, "Could not get object existence, #{response.status}/#{response.content}"
         end
       end
     end