blobstore_client 0.5.0 → 1.5.0.pre.1113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md ADDED
@@ -0,0 +1,127 @@
1
+ # Ruby client for Blobstores
2
+ Copyright (c) 2009-2013 VMware, Inc.
3
+
4
+ Lets BOSH access multiple blobstores using a unified API.
5
+
6
+ ## Usage
7
+
8
+ bin/blobstore_client_console [<options>]
9
+ -p, --provider PROVIDER Bosh Blobstore provider
10
+ -c, --config FILE Bosh Blobstore configuration file
11
+
12
+ ## Console
13
+
14
+ To explore the client API for accessing a blobstore, try creating and using a local blobstore:
15
+
16
+ ```
17
+ $ gem install blobstore_client
18
+ $ blobstore_client_console -p local -c config/local.yml.example
19
+ => Welcome to BOSH blobstore client console
20
+ You can use 'bsc' to access blobstore client methods
21
+ > bsc.create("this is a test blob")
22
+ => "ef00746b-21ec-4473-a888-bf257cb7ea21"
23
+ > bsc.get("ef00746b-21ec-4473-a888-bf257cb7ea21")
24
+ => "this is a test blob"
25
+ > bsc.exists?("ef00746b-21ec-4473-a888-bf257cb7ea21")
26
+ => true
27
+ > Dir['/tmp/local_blobstore/**']
28
+ => ["/tmp/local_blobstore/ef00746b-21ec-4473-a888-bf257cb7ea21"]
29
+ > bsc.delete("ef00746b-21ec-4473-a888-bf257cb7ea21")
30
+ => true
31
+ ```
32
+
33
+ ## Configuration
34
+
35
+ These options are passed to the Bosh Blobstore client when it is instantiated.
36
+
37
+ ### Local
38
+
39
+ These are the options for the Blobstore client when provider is `local`:
40
+
41
+ * `blobstore_path` (required)
42
+ Path for the blobstore
43
+
44
+ ### Simple
45
+
46
+ These are the options for the Blobstore client when provider is `simple`:
47
+
48
+ * `endpoint` (required)
49
+ Blobstore endpoint
50
+ * `user` (optional)
51
+ Blobstore User
52
+ * `password` (optional)
53
+ Blobstore Password
54
+ * `bucket` (optional, by default `resources`)
55
+ Name of the bucket
56
+
57
+ ### Amazon S3
58
+
59
+ These are the options for the Blobstore client when provider is `s3`:
60
+
61
+ * `bucket_name` (required)
62
+ Name of the S3 bucket
63
+ * `encryption_key` (optional)
64
+ Encryption_key that is applied before the object is sent to S3
65
+ * `access_key_id` (optional, if not present, the blobstore client operates in read only mode)
66
+ S3 Access Key
67
+ * `secret_access_key` (optional, if not present, the blobstore client operates in read only mode)
68
+ S3 Secret Access Key
69
+
70
+ ### Atmos
71
+
72
+ These are the options for the Blobstore client when provider is `atmos`:
73
+
74
+ * `url` (required)
75
+ Atmos URL
76
+ * `uid` (required)
77
+ Atmos UID
78
+ * `secret` (required)
79
+ Atmos password
80
+
81
+ ### OpenStack Swift provider
82
+
83
+ These are the options for the Blobstore client when provider is `swift`:
84
+
85
+ * `container_name` (required)
86
+ Name of the container
87
+ * `swift_provider` (required)
88
+ OpenStack Swift provider (supported: `hp`, `openstack` and `rackspace`)
89
+
90
+ #### HP Object Storage
91
+
92
+ These are the options for the Blobstore client when `swift_provider` is `hp`:
93
+
94
+ * `hp_access_key` (required)
95
+ HP Object Storage Access Key
96
+ * `hp_secret_key` (required)
97
+ HP Object Storage Secret Key
98
+ * `hp_tenant_id` (required)
99
+ HP Object Storage Project ID
100
+ * `hp_avl_zone` (required)
101
+ HP Object Storage Availability Zone (`region-a.geo-1` or `region-b.geo-1`)
102
+
103
+ #### OpenStack Object Storage
104
+
105
+ These are the options for the Blobstore client when `swift_provider` is `openstack`:
106
+
107
+ * `openstack_auth_url` (required)
108
+ URL of the OpenStack Identity endpoint to connect to
109
+ * `openstack_username` (required)
110
+ OpenStack user name
111
+ * `openstack_api_key` (required)
112
+ OpenStack API key
113
+ * `openstack_tenant` (required)
114
+ OpenStack tenant name
115
+ * `openstack_region` (optional)
116
+ OpenStack region
117
+
118
+ #### Rackspace Cloud Files
119
+
120
+ These are the options for the Blobstore client when `swift_provider` is `rackspace`:
121
+
122
+ * `rackspace_username` (required)
123
+ Rackspace Cloud Files Username
124
+ * `rackspace_api_key` (required)
125
+ Rackspace Cloud Files API Key
126
+ * `rackspace_region` (optional)
127
+ Rackspace Cloud Files Region (`dfw` or `ord`)
@@ -14,36 +14,37 @@
14
14
  # irb(main):003:0> bsc.delete(oid)
15
15
  # => true
16
16
 
17
- gemfile = File.expand_path("../../Gemfile", __FILE__)
18
-
19
- if File.exists?(gemfile)
20
- ENV["BUNDLE_GEMFILE"] = gemfile
21
- require "rubygems"
22
- require "bundler/setup"
23
- end
24
-
25
- $:.unshift(File.expand_path("../../lib", __FILE__))
26
- require "blobstore_client"
27
- require "irb"
28
- require "irb/completion"
29
- require "ostruct"
30
- require "optparse"
17
+ require 'blobstore_client'
18
+ require 'irb'
19
+ require 'irb/completion'
20
+ require 'ostruct'
21
+ require 'optparse'
22
+ require 'psych'
31
23
 
32
24
  @provider = nil
33
25
  config_file = nil
34
26
 
35
27
  opts_parser = OptionParser.new do |opts|
36
- opts.on("-p", "--provider PROVIDER") { |p| @provider = p }
37
- opts.on("-c", "--config FILE") { |file| config_file = file }
28
+ opts.on('-p', '--provider PROVIDER') { |p| @provider = p }
29
+ opts.on('-c', '--config FILE') { |file| config_file = file }
38
30
  end
39
31
  opts_parser.parse!
40
32
 
41
33
  unless @provider && config_file
42
34
  puts opts_parser
35
+ puts "\nExample config file:"
36
+ puts <<-YAML
37
+ ---
38
+ endpoint: http://1.2.3.4:25250
39
+ user: agent
40
+ password: agent
41
+ bucket: resources
42
+
43
+ YAML
43
44
  exit(1)
44
45
  end
45
46
 
46
- @config = YAML.load_file(config_file)
47
+ @config = Psych.load_file(config_file)
47
48
 
48
49
  module ConsoleHelpers
49
50
  def bsc
@@ -55,12 +56,12 @@ include ConsoleHelpers
55
56
 
56
57
  begin
57
58
  require 'ruby-debug'
58
- puts "=> Debugger enabled"
59
+ puts '=> Debugger enabled'
59
60
  rescue LoadError
60
- puts "=> ruby-debug not found, debugger disabled"
61
+ puts '=> ruby-debug not found, debugger disabled'
61
62
  end
62
63
 
63
- puts "=> Welcome to BOSH blobstore client console"
64
+ puts '=> Welcome to BOSH blobstore client console'
64
65
  puts "You can use 'bsc' to access blobstore client methods"
65
66
 
66
67
  IRB.start
@@ -0,0 +1,2 @@
1
+ ---
2
+ blobstore_path: /tmp/local_blobstore
@@ -1,43 +1,14 @@
1
1
  # Copyright (c) 2009-2012 VMware, Inc.
2
2
 
3
- module Bosh; module Blobstore; end; end
4
-
5
- require "common/common"
6
- require "blobstore_client/version"
7
- require "blobstore_client/errors"
8
-
9
- require "blobstore_client/client"
10
- require "blobstore_client/base"
11
- require "blobstore_client/simple_blobstore_client"
12
- require "blobstore_client/s3_blobstore_client"
13
- require "blobstore_client/swift_blobstore_client"
14
- require "blobstore_client/local_client"
15
- require "blobstore_client/atmos_blobstore_client"
16
-
17
- module Bosh
18
- module Blobstore
19
- class Client
20
-
21
- PROVIDER_MAP = {
22
- "simple" => SimpleBlobstoreClient,
23
- "s3" => S3BlobstoreClient,
24
- "swift" => SwiftBlobstoreClient,
25
- "atmos" => AtmosBlobstoreClient,
26
- "local" => LocalClient
27
- }
28
-
29
- def self.create(blobstore_provider, options = {})
30
- provider = PROVIDER_MAP[blobstore_provider]
31
-
32
- unless provider
33
- providers = PROVIDER_MAP.keys.sort.join(", ")
34
- raise BlobstoreError,
35
- "Invalid client provider, available providers are: #{providers}"
36
- end
37
-
38
- provider.new(options)
39
- end
40
-
41
- end
42
- end
43
- end
3
+ require 'common/common'
4
+ require 'blobstore_client/version'
5
+ require 'blobstore_client/errors'
6
+ require 'blobstore_client/client'
7
+
8
+ Bosh::Blobstore.autoload(:BaseClient, 'blobstore_client/base')
9
+ Bosh::Blobstore.autoload(:S3BlobstoreClient, 'blobstore_client/s3_blobstore_client')
10
+ Bosh::Blobstore.autoload(:SimpleBlobstoreClient, 'blobstore_client/simple_blobstore_client')
11
+ Bosh::Blobstore.autoload(:SwiftBlobstoreClient, 'blobstore_client/swift_blobstore_client')
12
+ Bosh::Blobstore.autoload(:AtmosBlobstoreClient, 'blobstore_client/atmos_blobstore_client')
13
+ Bosh::Blobstore.autoload(:LocalClient, 'blobstore_client/local_client')
14
+ Bosh::Blobstore.autoload(:DavBlobstoreClient, 'blobstore_client/dav_blobstore_client')
@@ -1,93 +1,110 @@
1
1
  # Copyright (c) 2009-2012 VMware, Inc.
2
2
 
3
- require "atmos"
4
- require "uri"
5
- require "multi_json"
3
+ require 'atmos'
4
+ require 'uri'
5
+ require 'multi_json'
6
+ require 'httpclient'
6
7
 
7
8
  module Bosh
8
9
  module Blobstore
9
10
  class AtmosBlobstoreClient < BaseClient
10
- SHARE_URL_EXP = "1893484800" # expires on 2030 Jan-1
11
+ SHARE_URL_EXP = '1893484800' # expires on 2030 Jan-1
11
12
 
12
13
  def initialize(options)
13
14
  super(options)
14
15
  @atmos_options = {
15
- :url => @options[:url],
16
- :uid => @options[:uid],
17
- :secret => @options[:secret]
16
+ url: @options[:url],
17
+ uid: @options[:uid],
18
+ secret: @options[:secret]
18
19
  }
19
20
  @tag = @options[:tag]
20
- @http_client = HTTPClient.new
21
- # TODO: Remove this line once we get the proper certificate for atmos
21
+
22
+ # Add proxy if ENV has the variable
23
+ proxy = case URI.parse(@atmos_options[:url] || '').scheme
24
+ when 'https'
25
+ ENV['HTTPS_PROXY'] || ENV['https_proxy']
26
+ when 'http'
27
+ ENV['HTTP_PROXY'] || ENV['http_proxy']
28
+ else
29
+ nil
30
+ end
31
+ if proxy
32
+ @atmos_options[:proxy] = proxy
33
+ @http_client = HTTPClient.new(proxy: proxy)
34
+ end
35
+ @http_client ||= HTTPClient.new
22
36
  @http_client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
23
37
  end
24
38
 
25
39
  def atmos_server
26
- unless @atmos_options[:secret]
27
- raise "Atmos password is missing (read-only mode)"
28
- end
40
+ raise 'Atmos password is missing (read-only mode)' unless @atmos_options[:secret]
41
+
29
42
  @atmos ||= Atmos::Store.new(@atmos_options)
30
43
  end
31
44
 
32
- def create_file(file)
33
- obj_conf = {:data => file, :length => File.size(file.path)}
34
- obj_conf[:listable_metadata] = {@tag => true} if @tag
45
+ protected
46
+
47
+ def create_file(id, file)
48
+ raise BlobstoreError, 'Atmos does not support supplying the object id' if id
49
+ obj_conf = { data: file, length: File.size(file.path) }
50
+ obj_conf[:listable_metadata] = { @tag => true } if @tag
35
51
  object_id = atmos_server.create(obj_conf).aoid
36
52
  encode_object_id(object_id)
37
53
  end
38
54
 
39
55
  def get_file(object_id, file)
40
56
  object_info = decode_object_id(object_id)
41
- oid = object_info["oid"]
42
- sig = object_info["sig"]
57
+ oid = object_info['oid']
58
+ sig = object_info['sig']
43
59
 
44
60
  url = @atmos_options[:url] + "/rest/objects/#{oid}?uid=" +
45
- URI::escape(@atmos_options[:uid]) +
46
- "&expires=#{SHARE_URL_EXP}&signature=#{URI::escape(sig)}"
61
+ URI.escape(@atmos_options[:uid]) +
62
+ "&expires=#{SHARE_URL_EXP}&signature=#{URI.escape(sig)}"
47
63
 
48
64
  response = @http_client.get(url) do |block|
49
65
  file.write(block)
50
66
  end
51
67
 
52
68
  if response.status != 200
53
- raise BlobstoreError, "Could not fetch object, %s/%s" %
54
- [response.status, response.content]
69
+ raise BlobstoreError, sprintf('Could not fetch object, %s/%s', [response.status, response.content])
55
70
  end
56
71
  end
57
72
 
58
- def delete(object_id)
73
+ def delete_object(object_id)
59
74
  object_info = decode_object_id(object_id)
60
- oid = object_info["oid"]
61
- atmos_server.get(:id => oid).delete
62
- rescue Atmos::Exceptions::NoSuchObjectException => e
75
+ oid = object_info['oid']
76
+ atmos_server.get(id: oid).delete
77
+ rescue Atmos::Exceptions::NoSuchObjectException
63
78
  raise NotFound, "Atmos object '#{object_id}' not found"
64
79
  end
65
80
 
81
+ def object_exists?(object_id)
82
+ atmos_server.get(id: object_id).exists?
83
+ end
84
+
66
85
  private
67
86
 
68
87
  def decode_object_id(object_id)
69
88
  begin
70
- object_info = MultiJson.decode(Base64.decode64(URI::unescape(object_id)))
71
- rescue MultiJson::DecodeError => e
72
- raise BlobstoreError, "Failed to parse object_id. " +
73
- "Please try updating the release"
89
+ object_info = MultiJson.decode(Base64.decode64(URI.unescape(object_id)))
90
+ rescue MultiJson::DecodeError
91
+ raise BlobstoreError, 'Failed to parse object_id. Please try updating the release'
74
92
  end
75
93
 
76
- if !object_info.kind_of?(Hash) || object_info["oid"].nil? ||
77
- object_info["sig"].nil?
94
+ if !object_info.kind_of?(Hash) || object_info['oid'].nil? ||
95
+ object_info['sig'].nil?
78
96
  raise BlobstoreError, "Invalid object_id (#{object_id})"
79
97
  end
80
98
  object_info
81
99
  end
82
100
 
83
101
  def encode_object_id(object_id)
84
- hash_string = "GET" + "\n" + "/rest/objects/" + object_id + "\n" +
85
- @atmos_options[:uid] + "\n" + SHARE_URL_EXP
102
+ hash_string = "GET\n/rest/objects/#{object_id}\n#{@atmos_options[:uid]}\n#{SHARE_URL_EXP}"
86
103
  secret = Base64.decode64(@atmos_options[:secret])
87
104
  sig = HMAC::SHA1.digest(secret, hash_string)
88
105
  signature = Base64.encode64(sig.to_s).chomp
89
- json = MultiJson.encode({:oid => object_id, :sig => signature})
90
- URI::escape(Base64.encode64(json))
106
+ json = MultiJson.encode({ oid: object_id, sig: signature })
107
+ URI.escape(Base64.encode64(json))
91
108
  end
92
109
  end
93
110
  end
@@ -1,76 +1,107 @@
1
1
  # Copyright (c) 2009-2012 VMware, Inc.
2
2
 
3
- require "tmpdir"
3
+ require 'tmpdir'
4
+ require 'securerandom'
4
5
 
5
6
  module Bosh
6
7
  module Blobstore
7
8
  class BaseClient < Client
8
9
 
10
+ # @param [Hash] options blobstore specific options
9
11
  def initialize(options)
10
12
  @options = Bosh::Common.symbolize_keys(options)
11
13
  end
12
14
 
13
- def symbolize_keys(hash)
14
- hash.inject({}) do |h, (key, value)|
15
- h[key.to_sym] = value
16
- h
17
- end
18
- end
19
-
20
- def create_file(file)
21
-
22
- end
23
-
24
- def get_file(id, file)
25
-
26
- end
27
-
28
- def create(contents)
15
+ # Saves a file or a string to the blobstore.
16
+ # if it is a String, it writes it to a temp file
17
+ # then calls create_file() with the (temp) file
18
+ # @overload create(contents, id=nil)
19
+ # @param [String] contents contents to upload
20
+ # @param [String] id suggested object id, if nil a uuid is generated
21
+ # @overload create(file, id=nil)
22
+ # @param [File] file file to upload
23
+ # @param [String] id suggested object id, if nil a uuid is generated
24
+ # @return [String] object id of the created blobstore object
25
+ def create(contents, id = nil)
29
26
  if contents.kind_of?(File)
30
- create_file(contents)
27
+ create_file(id, contents)
31
28
  else
32
29
  temp_path do |path|
33
- begin
34
- File.open(path, "w") do |file|
35
- file.write(contents)
36
- end
37
- return create_file(File.open(path, "r"))
38
- rescue BlobstoreError => e
39
- raise e
40
- rescue Exception => e
41
- raise BlobstoreError,
42
- "Failed to create object, underlying error: %s %s" %
43
- [e.message, e.backtrace.join("\n")]
30
+ File.open(path, 'w') do |file|
31
+ file.write(contents)
44
32
  end
33
+ return create_file(id, File.open(path, 'r'))
45
34
  end
46
35
  end
36
+ rescue BlobstoreError => e
37
+ raise e
38
+ rescue Exception => e
39
+ raise BlobstoreError,
40
+ sprintf('Failed to create object, underlying error: %s %s', e.inspect, e.backtrace.join("\n"))
47
41
  end
48
42
 
43
+ # Get an object from the blobstore.
44
+ # @param [String] id object id
45
+ # @param [File] file where to store the fetched object
46
+ # @return [String] the object contents if the file parameter is nil
49
47
  def get(id, file = nil)
50
48
  if file
51
49
  get_file(id, file)
50
+ file.flush
52
51
  else
53
52
  result = nil
54
53
  temp_path do |path|
55
- begin
56
- File.open(path, "w") { |file| get_file(id, file) }
57
- result = File.open(path, "r") { |file| file.read }
58
- rescue BlobstoreError => e
59
- raise e
60
- rescue Exception => e
61
- raise BlobstoreError,
62
- "Failed to create object, underlying error: %s %s" %
63
- [e.message, e.backtrace.join("\n")]
64
- end
54
+ File.open(path, 'w') { |f| get_file(id, f) }
55
+ result = File.open(path, 'r') { |f| f.read }
65
56
  end
66
57
  result
67
58
  end
59
+ rescue BlobstoreError => e
60
+ raise e
61
+ rescue Exception => e
62
+ raise BlobstoreError,
63
+ sprintf('Failed to fetch object, underlying error: %s %s', e.inspect, e.backtrace.join("\n"))
64
+ end
65
+
66
+ # @return [void]
67
+ def delete(oid)
68
+ delete_object(oid)
69
+ end
70
+
71
+ # @return [Boolean]
72
+ def exists?(oid)
73
+ object_exists?(oid)
68
74
  end
69
75
 
70
76
  protected
71
77
 
78
+ # @return [String] the id
79
+ def create_file(id, file)
80
+ # needs to be implemented in each subclass
81
+ not_supported
82
+ end
83
+
84
+ def get_file(id, file)
85
+ # needs to be implemented in each subclass
86
+ not_supported
87
+ end
88
+
89
+ def delete_object(oid)
90
+ # needs to be implemented in each subclass
91
+ not_supported
92
+ end
93
+
94
+ def object_exists?(oid)
95
+ # needs to be implemented in each subclass
96
+ not_supported
97
+ end
98
+
99
+ def generate_object_id
100
+ SecureRandom.uuid
101
+ end
102
+
72
103
  def temp_path
73
- path = File.join(Dir::tmpdir, "temp-path-#{UUIDTools::UUID.random_create}")
104
+ path = File.join(Dir.tmpdir, "temp-path-#{SecureRandom.uuid}")
74
105
  begin
75
106
  yield path if block_given?
76
107
  path
@@ -79,6 +110,11 @@ module Bosh
79
110
  end
80
111
  end
81
112
 
113
+ private
114
+
115
+ def not_supported
116
+ raise NotImplemented, 'not supported by this blobstore'
117
+ end
82
118
  end
83
119
  end
84
120
  end