gitlab-fog-azure-rm 0.7.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -1
- data/CHANGELOG.md +38 -0
- data/gitlab-fog-azure-rm.gemspec +3 -2
- data/lib/fog/azurerm/models/storage/file.rb +1 -6
- data/lib/fog/azurerm/requests/storage/create_block_blob.rb +3 -31
- data/lib/fog/azurerm/requests/storage/delete_blob_https_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_blob_http_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_blob_https_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_blob_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_container_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/list_blobs.rb +1 -1
- data/lib/fog/azurerm/requests/storage/list_containers.rb +1 -1
- data/lib/fog/azurerm/requests/storage/put_blob_https_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb +1 -1
- data/lib/fog/azurerm/storage.rb +21 -10
- data/lib/fog/azurerm/utilities/general.rb +5 -0
- data/lib/fog/azurerm/version.rb +1 -1
- data/rakefile +2 -0
- data/test/models/storage/test_file.rb +2 -4
- data/test/requests/storage/test_create_block_blob.rb +0 -9
- data/test/requests/storage/test_get_blob_https_url.rb +23 -1
- data/test/requests/storage/test_get_blob_url.rb +6 -1
- data/test/requests/storage/test_list_blobs.rb +2 -2
- data/test/requests/storage/test_list_containers.rb +2 -2
- data/test/requests/storage/test_wait_blob_copy_operation_to_finish.rb +1 -1
- data/test/test_helper.rb +8 -0
- metadata +22 -10
- data/lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb +0 -110
- data/test/requests/storage/test_multipart_save_block_blob.rb +0 -105
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6a0941da3e69509575eca6b720890940f33e8617e32bbb37121e19c9e1087e33
|
4
|
+
data.tar.gz: '09b4913d90bcbba439e9501df86dcc5e4d3d012a57941e2bc7033e4ac7d97806'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6df48a1e91089a7c7af40addf64716b712daee65ea431edc4cffc98f1912c85538c5547eddced2bf512bdee1fbcca1cdd43eb2eb76022055a837f07bdad54d8c
|
7
|
+
data.tar.gz: b4696c93d25c11b5344410a09cd8142eae252350030000c3f6d041b85f69273f2196fa1715e1428ce9105471d150f1d3b036834140813d148b5ff54883481c24
|
data/.rubocop_todo.yml
CHANGED
@@ -132,7 +132,6 @@ Naming/RescuedExceptionsVariableName:
|
|
132
132
|
# SupportedStyles: separated, grouped
|
133
133
|
Style/AccessorGrouping:
|
134
134
|
Exclude:
|
135
|
-
- 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb'
|
136
135
|
- 'lib/fog/azurerm/requests/storage/save_page_blob.rb'
|
137
136
|
- 'test/test_helper.rb'
|
138
137
|
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,41 @@
|
|
1
|
+
## Unreleased
|
2
|
+
|
3
|
+
## 1.0.1
|
4
|
+
|
5
|
+
- Fix URL generation for files with UTF-8 characters !21
|
6
|
+
|
7
|
+
## 1.0.0
|
8
|
+
|
9
|
+
- Fix multipart uploads not working and drop multipart_save_block_blob !19
|
10
|
+
|
11
|
+
## 0.9.0
|
12
|
+
|
13
|
+
- Fix TimeoutError deprecation !17
|
14
|
+
- Update Azure storage libraries !16
|
15
|
+
|
16
|
+
## 0.8.1
|
17
|
+
|
18
|
+
- Fix `azure_storage_domain` not getting propagated properly !14
|
19
|
+
|
20
|
+
## 0.8.0
|
21
|
+
|
22
|
+
- Add `azure_storage_domain` for specifying custom domain !11
|
23
|
+
|
24
|
+
## 0.7.0
|
25
|
+
|
26
|
+
- Loosen mime-types version dependency !9
|
27
|
+
- Drop azure_mgmt_storage and other dependencies !8
|
28
|
+
|
29
|
+
## 0.6.0
|
30
|
+
|
31
|
+
- Fix get_object_url !7
|
32
|
+
- Add get_object_url method !6
|
33
|
+
- Add support for copy_object !5
|
34
|
+
- Add support for DELETE blob !4
|
35
|
+
- Add support for PUT blobs !3
|
36
|
+
- Add and update Rubocop files !2
|
37
|
+
- Delete all code except blob storage-related functionality !1
|
38
|
+
|
1
39
|
## 0.5.2
|
2
40
|
|
3
41
|
**Added**
|
data/gitlab-fog-azure-rm.gemspec
CHANGED
@@ -20,9 +20,10 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_development_dependency 'codeclimate-test-reporter' , '~> 1.0.0'
|
21
21
|
spec.add_development_dependency 'minitest', '~> 5.8.4'
|
22
22
|
spec.add_development_dependency 'rake', '~> 13.0'
|
23
|
-
spec.add_development_dependency 'rubocop', '~> 0.89'
|
23
|
+
spec.add_development_dependency 'rubocop', '~> 0.89.1'
|
24
24
|
spec.add_development_dependency 'simplecov'
|
25
|
-
spec.add_dependency 'azure-storage', '~>
|
25
|
+
spec.add_dependency 'azure-storage-blob', '~> 2.0'
|
26
|
+
spec.add_dependency 'azure-storage-common', '~> 2.0'
|
26
27
|
spec.add_dependency 'fog-core', '= 2.1.0'
|
27
28
|
spec.add_dependency 'fog-json', '~> 1.2.0'
|
28
29
|
spec.add_dependency 'mime-types'
|
@@ -248,12 +248,7 @@ module Fog
|
|
248
248
|
# Upload blob
|
249
249
|
def save_blob(options)
|
250
250
|
if options[:blob_type].nil? || options[:blob_type] == 'BlockBlob'
|
251
|
-
|
252
|
-
service.create_block_blob(directory.key, key, body, options)
|
253
|
-
else
|
254
|
-
service.multipart_save_block_blob(directory.key, key, body, options)
|
255
|
-
service.get_blob_properties(directory.key, key)
|
256
|
-
end
|
251
|
+
service.create_block_blob(directory.key, key, body, options)
|
257
252
|
else
|
258
253
|
service.save_page_blob(directory.key, key, body, options)
|
259
254
|
service.get_blob_properties(directory.key, key)
|
@@ -8,37 +8,9 @@ module Fog
|
|
8
8
|
msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
|
9
9
|
Fog::Logger.debug msg
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
elsif body.respond_to?(:read)
|
15
|
-
if body.respond_to?(:rewind)
|
16
|
-
begin
|
17
|
-
body.rewind
|
18
|
-
rescue
|
19
|
-
nil
|
20
|
-
end
|
21
|
-
end
|
22
|
-
data = body.read
|
23
|
-
else
|
24
|
-
data = Fog::Storage.parse_data(body)
|
25
|
-
options[:content_length] = data[:headers]['Content-Length']
|
26
|
-
options[:content_type] = data[:headers]['Content-Type']
|
27
|
-
data = data[:body]
|
28
|
-
end
|
29
|
-
|
30
|
-
raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
|
31
|
-
blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
|
32
|
-
rescue Azure::Core::Http::HTTPError => ex
|
33
|
-
raise_azure_exception(ex, msg)
|
34
|
-
end
|
35
|
-
|
36
|
-
if data.nil?
|
37
|
-
Fog::Logger.debug "Create a block blob #{blob_name} successfully."
|
38
|
-
else
|
39
|
-
Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
|
40
|
-
end
|
41
|
-
blob
|
11
|
+
@blob_client.create_block_blob(container_name, blob_name, body, options)
|
12
|
+
rescue Azure::Core::Http::HTTPError => ex
|
13
|
+
raise_azure_exception(ex, msg)
|
42
14
|
end
|
43
15
|
end
|
44
16
|
|
@@ -23,7 +23,7 @@ module Fog
|
|
23
23
|
protocol: 'https'
|
24
24
|
}
|
25
25
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
26
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
27
|
"#{uri}?#{token}"
|
28
28
|
end
|
29
29
|
end
|
@@ -22,7 +22,7 @@ module Fog
|
|
22
22
|
expiry: expires.utc.iso8601
|
23
23
|
}
|
24
24
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
25
|
-
uri = @blob_client.generate_uri(relative_path)
|
25
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
26
26
|
url = "#{uri}?#{token}"
|
27
27
|
url.gsub('https:', 'http:')
|
28
28
|
end
|
@@ -23,7 +23,7 @@ module Fog
|
|
23
23
|
protocol: 'https'
|
24
24
|
}
|
25
25
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
26
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
27
|
"#{uri}?#{token}"
|
28
28
|
end
|
29
29
|
end
|
@@ -5,7 +5,7 @@ module Fog
|
|
5
5
|
class Real
|
6
6
|
# Get a public blob url from Azure blob storage
|
7
7
|
def get_blob_url(container_name, blob_name, options = {})
|
8
|
-
uri = @blob_client.generate_uri("#{container_name}/#{blob_name}")
|
8
|
+
uri = @blob_client.generate_uri("#{container_name}/#{blob_name}", {}, { encode: true })
|
9
9
|
|
10
10
|
if options[:scheme] == 'http'
|
11
11
|
uri.to_s.gsub('https:', 'http:')
|
@@ -11,7 +11,7 @@ module Fog
|
|
11
11
|
#
|
12
12
|
def get_container_url(container_name, options = {})
|
13
13
|
query = { 'comp' => 'list', 'restype' => 'container' }
|
14
|
-
uri = @blob_client.generate_uri(container_name, query)
|
14
|
+
uri = @blob_client.generate_uri(container_name, query, { encode: true })
|
15
15
|
|
16
16
|
if options[:scheme] == 'http'
|
17
17
|
uri.to_s.gsub('https:', 'http:')
|
@@ -21,7 +21,7 @@ module Fog
|
|
21
21
|
Fog::Logger.debug msg
|
22
22
|
temp = @blob_client.list_blobs(container_name, options)
|
23
23
|
# Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
|
24
|
-
raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)
|
24
|
+
raise temp unless temp.instance_of?(Azure::Storage::Common::Service::EnumerationResults)
|
25
25
|
|
26
26
|
blobs += temp unless temp.empty?
|
27
27
|
break if temp.continuation_token.nil? || temp.continuation_token.empty?
|
@@ -16,7 +16,7 @@ module Fog
|
|
16
16
|
Fog::Logger.debug msg
|
17
17
|
temp = @blob_client.list_containers(options)
|
18
18
|
# Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
|
19
|
-
raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)
|
19
|
+
raise temp unless temp.instance_of?(Azure::Storage::Common::Service::EnumerationResults)
|
20
20
|
|
21
21
|
containers += temp unless temp.empty?
|
22
22
|
break if temp.continuation_token.nil? || temp.continuation_token.empty?
|
@@ -23,7 +23,7 @@ module Fog
|
|
23
23
|
protocol: 'https'
|
24
24
|
}
|
25
25
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
26
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
27
|
"#{uri}?#{token}"
|
28
28
|
end
|
29
29
|
end
|
@@ -18,7 +18,7 @@ module Fog
|
|
18
18
|
break if copy_status != COPY_STATUS[:PENDING]
|
19
19
|
|
20
20
|
elapse_time = Time.new - start_time
|
21
|
-
raise
|
21
|
+
raise Timeout::Error.new("The copy operation cannot be finished in #{timeout} seconds") if !timeout.nil? && elapse_time >= timeout
|
22
22
|
|
23
23
|
copied_bytes, total_bytes = blob_props[:copy_progress].split('/').map(&:to_i)
|
24
24
|
interval = copied_bytes.zero? ? 5 : (total_bytes - copied_bytes).to_f / copied_bytes * elapse_time
|
data/lib/fog/azurerm/storage.rb
CHANGED
@@ -12,6 +12,8 @@ module Fog
|
|
12
12
|
# Recognizes when creating data client
|
13
13
|
recognizes :azure_storage_account_name
|
14
14
|
recognizes :azure_storage_access_key
|
15
|
+
recognizes :azure_storage_domain
|
16
|
+
|
15
17
|
recognizes :debug
|
16
18
|
|
17
19
|
request_path 'fog/azurerm/requests/storage'
|
@@ -55,7 +57,6 @@ module Fog
|
|
55
57
|
request :put_blob_pages
|
56
58
|
request :wait_blob_copy_operation_to_finish
|
57
59
|
request :save_page_blob
|
58
|
-
request :multipart_save_block_blob
|
59
60
|
|
60
61
|
model_path 'fog/azurerm/models/storage'
|
61
62
|
model :directory
|
@@ -67,7 +68,8 @@ module Fog
|
|
67
68
|
class Mock
|
68
69
|
def initialize(_options = {})
|
69
70
|
begin
|
70
|
-
require 'azure/storage'
|
71
|
+
require 'azure/storage/common'
|
72
|
+
require 'azure/storage/blob'
|
71
73
|
rescue LoadError => e
|
72
74
|
retry if require('rubygems')
|
73
75
|
raise e.message
|
@@ -79,7 +81,8 @@ module Fog
|
|
79
81
|
class Real
|
80
82
|
def initialize(options)
|
81
83
|
begin
|
82
|
-
require 'azure/storage'
|
84
|
+
require 'azure/storage/common'
|
85
|
+
require 'azure/storage/blob'
|
83
86
|
require 'securerandom'
|
84
87
|
@debug = ENV['DEBUG'] || options[:debug]
|
85
88
|
require 'azure/core/http/debug_filter' if @debug
|
@@ -104,15 +107,23 @@ module Fog
|
|
104
107
|
|
105
108
|
@azure_storage_account_name = options[:azure_storage_account_name]
|
106
109
|
@azure_storage_access_key = options[:azure_storage_access_key]
|
110
|
+
@azure_storage_domain = options[:azure_storage_domain]
|
111
|
+
|
112
|
+
domain =
|
113
|
+
if @azure_storage_domain.nil? || @azure_storage_domain.empty?
|
114
|
+
get_blob_endpoint(@azure_storage_account_name, true, @environment)
|
115
|
+
else
|
116
|
+
get_blob_endpoint_with_domain(@azure_storage_account_name, true, @azure_storage_domain)
|
117
|
+
end
|
107
118
|
|
108
|
-
azure_client = Azure::Storage::Client.create(storage_account_name: @azure_storage_account_name,
|
109
|
-
|
110
|
-
|
111
|
-
@blob_client = azure_client
|
112
|
-
@blob_client.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
|
119
|
+
azure_client = Azure::Storage::Common::Client.create(storage_account_name: @azure_storage_account_name,
|
120
|
+
storage_access_key: @azure_storage_access_key,
|
121
|
+
storage_blob_host: domain)
|
122
|
+
@blob_client = Azure::Storage::Blob::BlobService.new(client: azure_client)
|
123
|
+
@blob_client.with_filter(Azure::Storage::Common::Core::Filter::ExponentialRetryPolicyFilter.new)
|
113
124
|
@blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
|
114
|
-
@signature_client = Azure::Storage::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
|
115
|
-
|
125
|
+
@signature_client = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
|
126
|
+
@azure_storage_access_key)
|
116
127
|
end
|
117
128
|
end
|
118
129
|
end
|
@@ -125,6 +125,11 @@ def get_blob_endpoint(storage_account_name, enable_https = false, environment =
|
|
125
125
|
"#{protocol}://#{storage_account_name}.blob#{storage_endpoint_suffix(environment)}"
|
126
126
|
end
|
127
127
|
|
128
|
+
def get_blob_endpoint_with_domain(storage_account_name, enable_https = false, domain = 'blob.core.windows.net')
|
129
|
+
protocol = enable_https ? 'https' : 'http'
|
130
|
+
"#{protocol}://#{storage_account_name}.#{domain}"
|
131
|
+
end
|
132
|
+
|
128
133
|
def current_time
|
129
134
|
time = Time.now.to_f.to_s
|
130
135
|
time.split(/\W+/).join
|
data/lib/fog/azurerm/version.rb
CHANGED
data/rakefile
CHANGED
@@ -76,10 +76,8 @@ class TestFile < Minitest::Test
|
|
76
76
|
def test_save_method_with_large_block_blob_success
|
77
77
|
@file.body = 'd' * (32 * 1024 * 1024 + 1) # SINGLE_BLOB_PUT_THRESHOLD is 32 * 1024 * 1024
|
78
78
|
|
79
|
-
@service.stub :
|
80
|
-
@
|
81
|
-
assert @file.save
|
82
|
-
end
|
79
|
+
@service.stub :create_block_blob, @raw_cloud_blob do
|
80
|
+
assert @file.save
|
83
81
|
end
|
84
82
|
end
|
85
83
|
|
@@ -52,15 +52,6 @@ class TestCreateBlockBlob < Minitest::Test
|
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
-
def test_create_block_blob_exceed_max_body_size
|
56
|
-
data = []
|
57
|
-
data.stub :size, 64 * 1024 * 1024 + 1 do
|
58
|
-
assert_raises(ArgumentError) do
|
59
|
-
@service.create_block_blob('test_container', 'test_blob', data)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
55
|
def test_create_block_blob_http_exception
|
65
56
|
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
66
57
|
@blob_client.stub :create_block_blob, http_exception do
|
@@ -17,7 +17,13 @@ class TestGetBlobHttpsUrl < Minitest::Test
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def test_get_blob_https_url_success
|
20
|
-
|
20
|
+
mock_generate_uri = Minitest::Mock.new
|
21
|
+
|
22
|
+
2.times do
|
23
|
+
mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
|
24
|
+
end
|
25
|
+
|
26
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
21
27
|
@signature_client.stub :generate_service_sas_token, @token do
|
22
28
|
assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600)
|
23
29
|
assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600)
|
@@ -25,6 +31,22 @@ class TestGetBlobHttpsUrl < Minitest::Test
|
|
25
31
|
end
|
26
32
|
end
|
27
33
|
|
34
|
+
def test_get_blob_https_url_with_domain_success
|
35
|
+
service = Fog::Storage::AzureRM.new(storage_account_credentials_with_domain)
|
36
|
+
blob_client = service.instance_variable_get(:@blob_client)
|
37
|
+
signature_client = service.instance_variable_get(:@signature_client)
|
38
|
+
|
39
|
+
url = 'https://mockaccount.test.example.com/test_container/test_blob'
|
40
|
+
token = ApiStub::Requests::Storage::File.blob_url_token
|
41
|
+
|
42
|
+
blob_client.stub :generate_uri, url do
|
43
|
+
signature_client.stub :generate_service_sas_token, token do
|
44
|
+
assert_equal "#{url}?#{token}", service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600)
|
45
|
+
assert_equal "#{url}?#{token}", service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
28
50
|
def test_get_blob_https_url_mock
|
29
51
|
assert_equal "#{@url}?#{@token}", @mock_service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600)
|
30
52
|
assert_equal "#{@url}?#{@token}", @mock_service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600)
|
@@ -12,9 +12,14 @@ class TestGetBlobUrl < Minitest::Test
|
|
12
12
|
def test_get_blob_url_success
|
13
13
|
service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
14
14
|
blob_client = service.instance_variable_get(:@blob_client)
|
15
|
+
|
16
|
+
mock_generate_uri = Minitest::Mock.new
|
15
17
|
url = ApiStub::Requests::Storage::File.blob_https_url
|
18
|
+
2.times do
|
19
|
+
mock_generate_uri.expect(:call, url, ['test_container/test_blob', {}, { encode: true }])
|
20
|
+
end
|
16
21
|
|
17
|
-
blob_client.stub :generate_uri,
|
22
|
+
blob_client.stub :generate_uri, mock_generate_uri do
|
18
23
|
assert_equal url, service.get_blob_url('test_container', 'test_blob')
|
19
24
|
|
20
25
|
options = { scheme: 'http' }
|
@@ -13,11 +13,11 @@ class TestListBlobs < Minitest::Test
|
|
13
13
|
@blob_client = @service.instance_variable_get(:@blob_client)
|
14
14
|
|
15
15
|
@blob_list = ApiStub::Requests::Storage::File.blob_list
|
16
|
-
@blobs1 = Azure::Service::EnumerationResults.new
|
16
|
+
@blobs1 = Azure::Storage::Common::Service::EnumerationResults.new
|
17
17
|
@blobs1.continuation_token = 'marker'
|
18
18
|
@blobs1.push(@blob_list[0])
|
19
19
|
@blobs1.push(@blob_list[1])
|
20
|
-
@blobs2 = Azure::Service::EnumerationResults.new
|
20
|
+
@blobs2 = Azure::Storage::Common::Service::EnumerationResults.new
|
21
21
|
@blobs2.push(@blob_list[2])
|
22
22
|
@blobs2.push(@blob_list[3])
|
23
23
|
end
|
@@ -13,11 +13,11 @@ class TestListContainers < Minitest::Test
|
|
13
13
|
@blob_client = @service.instance_variable_get(:@blob_client)
|
14
14
|
|
15
15
|
@containers = ApiStub::Requests::Storage::Directory.container_list
|
16
|
-
@containers1 = Azure::Service::EnumerationResults.new
|
16
|
+
@containers1 = Azure::Storage::Common::Service::EnumerationResults.new
|
17
17
|
@containers1.continuation_token = 'marker'
|
18
18
|
@containers1.push(@containers[0])
|
19
19
|
@containers1.push(@containers[1])
|
20
|
-
@containers2 = Azure::Service::EnumerationResults.new
|
20
|
+
@containers2 = Azure::Storage::Common::Service::EnumerationResults.new
|
21
21
|
@containers2.push(@containers[2])
|
22
22
|
end
|
23
23
|
|
@@ -106,7 +106,7 @@ class TestWaitBlobCopyOperationToFinish < Minitest::Test
|
|
106
106
|
|
107
107
|
@service.stub :get_blob_properties, multiple_values do
|
108
108
|
@service.stub :delete_blob, true do
|
109
|
-
assert_raises(
|
109
|
+
assert_raises(Timeout::Error) do
|
110
110
|
@service.wait_blob_copy_operation_to_finish('test_container', 'test_blob', copy_id, copy_status, 2)
|
111
111
|
end
|
112
112
|
end
|
data/test/test_helper.rb
CHANGED
@@ -36,6 +36,14 @@ def storage_account_credentials
|
|
36
36
|
}
|
37
37
|
end
|
38
38
|
|
39
|
+
def storage_account_credentials_with_domain
|
40
|
+
storage_account_credentials.merge(
|
41
|
+
{
|
42
|
+
azure_storage_domain: 'test.example.com'
|
43
|
+
}
|
44
|
+
)
|
45
|
+
end
|
46
|
+
|
39
47
|
# Mock Class for Blob
|
40
48
|
class MockBlob
|
41
49
|
def initialize
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-fog-azure-rm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shaffan Chaudhry
|
@@ -18,7 +18,7 @@ authors:
|
|
18
18
|
autorequire:
|
19
19
|
bindir: bin
|
20
20
|
cert_chain: []
|
21
|
-
date:
|
21
|
+
date: 2021-02-22 00:00:00.000000000 Z
|
22
22
|
dependencies:
|
23
23
|
- !ruby/object:Gem::Dependency
|
24
24
|
name: codeclimate-test-reporter
|
@@ -68,14 +68,14 @@ dependencies:
|
|
68
68
|
requirements:
|
69
69
|
- - "~>"
|
70
70
|
- !ruby/object:Gem::Version
|
71
|
-
version:
|
71
|
+
version: 0.89.1
|
72
72
|
type: :development
|
73
73
|
prerelease: false
|
74
74
|
version_requirements: !ruby/object:Gem::Requirement
|
75
75
|
requirements:
|
76
76
|
- - "~>"
|
77
77
|
- !ruby/object:Gem::Version
|
78
|
-
version:
|
78
|
+
version: 0.89.1
|
79
79
|
- !ruby/object:Gem::Dependency
|
80
80
|
name: simplecov
|
81
81
|
requirement: !ruby/object:Gem::Requirement
|
@@ -91,19 +91,33 @@ dependencies:
|
|
91
91
|
- !ruby/object:Gem::Version
|
92
92
|
version: '0'
|
93
93
|
- !ruby/object:Gem::Dependency
|
94
|
-
name: azure-storage
|
94
|
+
name: azure-storage-blob
|
95
95
|
requirement: !ruby/object:Gem::Requirement
|
96
96
|
requirements:
|
97
97
|
- - "~>"
|
98
98
|
- !ruby/object:Gem::Version
|
99
|
-
version:
|
99
|
+
version: '2.0'
|
100
100
|
type: :runtime
|
101
101
|
prerelease: false
|
102
102
|
version_requirements: !ruby/object:Gem::Requirement
|
103
103
|
requirements:
|
104
104
|
- - "~>"
|
105
105
|
- !ruby/object:Gem::Version
|
106
|
-
version:
|
106
|
+
version: '2.0'
|
107
|
+
- !ruby/object:Gem::Dependency
|
108
|
+
name: azure-storage-common
|
109
|
+
requirement: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - "~>"
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '2.0'
|
114
|
+
type: :runtime
|
115
|
+
prerelease: false
|
116
|
+
version_requirements: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - "~>"
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '2.0'
|
107
121
|
- !ruby/object:Gem::Dependency
|
108
122
|
name: fog-core
|
109
123
|
requirement: !ruby/object:Gem::Requirement
|
@@ -235,7 +249,6 @@ files:
|
|
235
249
|
- lib/fog/azurerm/requests/storage/get_object_url.rb
|
236
250
|
- lib/fog/azurerm/requests/storage/list_blobs.rb
|
237
251
|
- lib/fog/azurerm/requests/storage/list_containers.rb
|
238
|
-
- lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb
|
239
252
|
- lib/fog/azurerm/requests/storage/put_blob_block.rb
|
240
253
|
- lib/fog/azurerm/requests/storage/put_blob_https_url.rb
|
241
254
|
- lib/fog/azurerm/requests/storage/put_blob_metadata.rb
|
@@ -291,7 +304,6 @@ files:
|
|
291
304
|
- test/requests/storage/test_get_container_url.rb
|
292
305
|
- test/requests/storage/test_list_blobs.rb
|
293
306
|
- test/requests/storage/test_list_containers.rb
|
294
|
-
- test/requests/storage/test_multipart_save_block_blob.rb
|
295
307
|
- test/requests/storage/test_put_blob_block.rb
|
296
308
|
- test/requests/storage/test_put_blob_https_url.rb
|
297
309
|
- test/requests/storage/test_put_blob_metadata.rb
|
@@ -326,7 +338,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
326
338
|
- !ruby/object:Gem::Version
|
327
339
|
version: '0'
|
328
340
|
requirements: []
|
329
|
-
rubygems_version: 3.
|
341
|
+
rubygems_version: 3.1.4
|
330
342
|
signing_key:
|
331
343
|
specification_version: 4
|
332
344
|
summary: Module for the 'fog' gem to support Azure Blob Storage with CarrierWave and
|
@@ -1,110 +0,0 @@
|
|
1
|
-
module Fog
|
2
|
-
module Storage
|
3
|
-
class AzureRM
|
4
|
-
# This class provides the actual implementation for service calls.
|
5
|
-
class Real
|
6
|
-
# This class is used to store chunk data for block blob before uploading.
|
7
|
-
class BlockChunk
|
8
|
-
attr_reader :id # For debug
|
9
|
-
attr_reader :block_id
|
10
|
-
attr_reader :data
|
11
|
-
|
12
|
-
def initialize(id, block_id, data)
|
13
|
-
@id = id
|
14
|
-
@block_id = block_id
|
15
|
-
@data = data
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
# This class is a stream to read chunk data.
|
20
|
-
class BlockFileStream
|
21
|
-
attr_reader :blocks
|
22
|
-
|
23
|
-
def initialize(body)
|
24
|
-
if body.respond_to?(:read)
|
25
|
-
if body.respond_to?(:rewind)
|
26
|
-
begin
|
27
|
-
body.rewind
|
28
|
-
rescue => ex
|
29
|
-
Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
|
30
|
-
end
|
31
|
-
end
|
32
|
-
@stream = body
|
33
|
-
else
|
34
|
-
@stream = StringIO.new(body)
|
35
|
-
end
|
36
|
-
@mutex = Mutex.new
|
37
|
-
@blocks = []
|
38
|
-
end
|
39
|
-
|
40
|
-
def read(size)
|
41
|
-
block_id = Base64.strict_encode64(random_string(32))
|
42
|
-
data = nil
|
43
|
-
id = 0
|
44
|
-
@mutex.synchronize do
|
45
|
-
data = @stream.read(size)
|
46
|
-
return nil if data.nil?
|
47
|
-
@blocks << [block_id]
|
48
|
-
id = @blocks.size
|
49
|
-
end
|
50
|
-
BlockChunk.new(id, block_id, data)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def multipart_save_block_blob(container_name, blob_name, body, options)
|
55
|
-
threads_num = options.delete(:worker_thread_num)
|
56
|
-
threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
|
57
|
-
|
58
|
-
begin
|
59
|
-
# Initiate the upload
|
60
|
-
Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
|
61
|
-
content_md5 = options.delete(:content_md5)
|
62
|
-
create_block_blob(container_name, blob_name, nil, options)
|
63
|
-
|
64
|
-
# Uploading parts
|
65
|
-
Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
|
66
|
-
iostream = BlockFileStream.new(body)
|
67
|
-
|
68
|
-
threads = []
|
69
|
-
threads_num.times do |id|
|
70
|
-
thread = Thread.new do
|
71
|
-
Fog::Logger.debug "Created upload thread #{id}."
|
72
|
-
while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
|
73
|
-
Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
|
74
|
-
put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
|
75
|
-
end
|
76
|
-
Fog::Logger.debug "Upload thread #{id} finished."
|
77
|
-
end
|
78
|
-
thread.abort_on_exception = true
|
79
|
-
threads << thread
|
80
|
-
end
|
81
|
-
|
82
|
-
threads.each(&:join)
|
83
|
-
# Complete the upload
|
84
|
-
options[:content_md5] = content_md5 unless content_md5.nil?
|
85
|
-
Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
|
86
|
-
commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
|
87
|
-
rescue
|
88
|
-
# Abort the upload & reraise
|
89
|
-
begin
|
90
|
-
delete_blob(container_name, blob_name)
|
91
|
-
rescue => ex
|
92
|
-
Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
|
93
|
-
end
|
94
|
-
raise
|
95
|
-
end
|
96
|
-
|
97
|
-
Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
|
98
|
-
true
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
# This class provides the mock implementation for unit tests.
|
103
|
-
class Mock
|
104
|
-
def multipart_save_block_blob(*)
|
105
|
-
true
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
@@ -1,105 +0,0 @@
|
|
1
|
-
require File.expand_path '../../test_helper', __dir__
|
2
|
-
|
3
|
-
# Storage Blob Class
|
4
|
-
class TestMultipartSaveBlockBlob < Minitest::Test
|
5
|
-
# This class posesses the test cases for the requests of saving storage block blob with multiple parts.
|
6
|
-
def setup
|
7
|
-
Fog.mock!
|
8
|
-
@mock_service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
9
|
-
Fog.unmock!
|
10
|
-
@mocked_response = mocked_storage_http_error
|
11
|
-
|
12
|
-
@service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_multipart_save_block_blob_success
|
16
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
17
|
-
|
18
|
-
@service.stub :create_block_blob, true do
|
19
|
-
@service.stub :put_blob_block, true do
|
20
|
-
@service.stub :commit_blob_blocks, true do
|
21
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', body, worker_thread_num: 6, content_md5: 'oafL1+HS78x65+e39PGIIg==')
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def test_multipart_save_block_blob_with_file_handle_success
|
28
|
-
i = 0
|
29
|
-
multiple_values = lambda do |*|
|
30
|
-
i += 1
|
31
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
32
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
33
|
-
return nil
|
34
|
-
end
|
35
|
-
temp_file = '/dev/null'
|
36
|
-
File.open(temp_file, 'r') do |file_handle|
|
37
|
-
file_handle.stub :read, multiple_values do
|
38
|
-
file_handle.stub :rewind, nil do
|
39
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
40
|
-
@service.stub :put_blob_block, true do
|
41
|
-
@service.stub :commit_blob_blocks, true do
|
42
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 0)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def test_multipart_save_block_blob_with_file_handle_do_not_support_rewind_success
|
52
|
-
i = 0
|
53
|
-
multiple_values = lambda do |*|
|
54
|
-
i += 1
|
55
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
56
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
57
|
-
return nil
|
58
|
-
end
|
59
|
-
temp_file = '/dev/null'
|
60
|
-
exception = ->(*) { raise 'do not support rewind' }
|
61
|
-
File.open(temp_file, 'r') do |file_handle|
|
62
|
-
file_handle.stub :read, multiple_values do
|
63
|
-
file_handle.stub :rewind, exception do
|
64
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
65
|
-
@service.stub :put_blob_block, true do
|
66
|
-
@service.stub :commit_blob_blocks, true do
|
67
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 'invalid')
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def test_multipart_save_block_blob_http_exception
|
77
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
78
|
-
|
79
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
80
|
-
@service.stub :create_block_blob, http_exception do
|
81
|
-
@service.stub :delete_blob, true do
|
82
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
83
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
84
|
-
end
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
def test_multipart_save_block_blob_fail_when_delete_blob_http_exception
|
90
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
91
|
-
|
92
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
93
|
-
@service.stub :create_block_blob, http_exception do
|
94
|
-
@service.stub :delete_blob, http_exception do
|
95
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
96
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
def test_multipart_save_block_blob_mock
|
103
|
-
assert @mock_service.multipart_save_block_blob('test_container', 'test_blob', 'content', {})
|
104
|
-
end
|
105
|
-
end
|