gitlab-fog-azure-rm 0.9.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -1
- data/CHANGELOG.md +20 -0
- data/lib/fog/azurerm/models/storage/file.rb +10 -7
- data/lib/fog/azurerm/models/storage/files.rb +4 -4
- data/lib/fog/azurerm/requests/storage/create_block_blob.rb +3 -31
- data/lib/fog/azurerm/requests/storage/delete_blob_https_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_blob_http_url.rb +5 -3
- data/lib/fog/azurerm/requests/storage/get_blob_https_url.rb +5 -3
- data/lib/fog/azurerm/requests/storage/get_blob_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_container_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_object_url.rb +4 -4
- data/lib/fog/azurerm/requests/storage/put_blob_https_url.rb +1 -1
- data/lib/fog/azurerm/storage.rb +0 -1
- data/lib/fog/azurerm/utilities/general.rb +4 -0
- data/lib/fog/azurerm/version.rb +1 -1
- data/test/models/storage/test_file.rb +15 -5
- data/test/requests/storage/test_create_block_blob.rb +0 -9
- data/test/requests/storage/test_get_blob_http_url.rb +37 -0
- data/test/requests/storage/test_get_blob_https_url.rb +49 -1
- data/test/requests/storage/test_get_blob_url.rb +6 -1
- metadata +6 -8
- data/lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb +0 -110
- data/test/requests/storage/test_multipart_save_block_blob.rb +0 -105
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9cc556f2028cd1a063691e4556dc1421014476690f700eb31fea81bcac574b41
|
4
|
+
data.tar.gz: e6e7d3c355e63cf6a0501d2bc85cbfd994ae54349c950657f6619244fecc9d02
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c1ffb76c0ba61d39f675c90612afdf3ae06f0b0835c8f6162a874b73a637e58e4c2c07c0b193ede34ef84601c040920e7a93c2ade997abcdd3c39a64f45d9ff9
|
7
|
+
data.tar.gz: bae99d6843a450a5072a9a8e5a2e94de43a6849565fa38c2b288d3c14fbeb70066be960b08c0172a5a9eb61314119d1e78dfb890f90fa4b4af7f7784b028fcdd
|
data/.rubocop_todo.yml
CHANGED
@@ -132,7 +132,6 @@ Naming/RescuedExceptionsVariableName:
|
|
132
132
|
# SupportedStyles: separated, grouped
|
133
133
|
Style/AccessorGrouping:
|
134
134
|
Exclude:
|
135
|
-
- 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb'
|
136
135
|
- 'lib/fog/azurerm/requests/storage/save_page_blob.rb'
|
137
136
|
- 'test/test_helper.rb'
|
138
137
|
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,25 @@
|
|
1
1
|
## Unreleased
|
2
2
|
|
3
|
+
## 1.2.0
|
4
|
+
|
5
|
+
- Support setting returned Content-Disposition in GetBlob requests !25
|
6
|
+
|
7
|
+
## 1.1.1
|
8
|
+
|
9
|
+
- Remove trailing periods per path segment when retrieving blob URLs !23
|
10
|
+
|
11
|
+
## 1.1.0
|
12
|
+
|
13
|
+
- Yanked
|
14
|
+
|
15
|
+
## 1.0.1
|
16
|
+
|
17
|
+
- Fix URL generation for files with UTF-8 characters !21
|
18
|
+
|
19
|
+
## 1.0.0
|
20
|
+
|
21
|
+
- Fix multipart uploads not working and drop multipart_save_block_blob !19
|
22
|
+
|
3
23
|
## 0.9.0
|
4
24
|
|
5
25
|
- Fix TimeoutError deprecation !17
|
@@ -240,20 +240,23 @@ module Fog
|
|
240
240
|
#
|
241
241
|
def url(expires, options = {})
|
242
242
|
requires :key
|
243
|
-
collection.get_url(key, expires, options)
|
243
|
+
collection.get_url(key, expires, normalize_options(options))
|
244
244
|
end
|
245
245
|
|
246
246
|
private
|
247
247
|
|
248
|
+
def normalize_options(options)
|
249
|
+
# AWS S3 and Google Cloud Storage pass response-content-disposition
|
250
|
+
# as a query string, while Azure needs the content_disposition parameter
|
251
|
+
# to generate a SAS token.
|
252
|
+
options[:content_disposition] ||= options.dig(:query, 'response-content-disposition')
|
253
|
+
options
|
254
|
+
end
|
255
|
+
|
248
256
|
# Upload blob
|
249
257
|
def save_blob(options)
|
250
258
|
if options[:blob_type].nil? || options[:blob_type] == 'BlockBlob'
|
251
|
-
|
252
|
-
service.create_block_blob(directory.key, key, body, options)
|
253
|
-
else
|
254
|
-
service.multipart_save_block_blob(directory.key, key, body, options)
|
255
|
-
service.get_blob_properties(directory.key, key)
|
256
|
-
end
|
259
|
+
service.create_block_blob(directory.key, key, body, options)
|
257
260
|
else
|
258
261
|
service.save_page_blob(directory.key, key, body, options)
|
259
262
|
service.get_blob_properties(directory.key, key)
|
@@ -123,10 +123,10 @@ module Fog
|
|
123
123
|
#
|
124
124
|
# @return [String] A http URL.
|
125
125
|
#
|
126
|
-
def get_http_url(key, expires,
|
126
|
+
def get_http_url(key, expires, options = {})
|
127
127
|
requires :directory
|
128
128
|
|
129
|
-
service.get_blob_http_url(directory.key, key, expires)
|
129
|
+
service.get_blob_http_url(directory.key, key, expires, options)
|
130
130
|
end
|
131
131
|
|
132
132
|
# Get the https URL of the file(blob) with the given name.
|
@@ -139,10 +139,10 @@ module Fog
|
|
139
139
|
#
|
140
140
|
# @return [String] A https URL.
|
141
141
|
#
|
142
|
-
def get_https_url(key, expires,
|
142
|
+
def get_https_url(key, expires, options = {})
|
143
143
|
requires :directory
|
144
144
|
|
145
|
-
service.get_blob_https_url(directory.key, key, expires)
|
145
|
+
service.get_blob_https_url(directory.key, key, expires, options)
|
146
146
|
end
|
147
147
|
|
148
148
|
# Get the file(blob) without content with the given name.
|
@@ -8,37 +8,9 @@ module Fog
|
|
8
8
|
msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
|
9
9
|
Fog::Logger.debug msg
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
elsif body.respond_to?(:read)
|
15
|
-
if body.respond_to?(:rewind)
|
16
|
-
begin
|
17
|
-
body.rewind
|
18
|
-
rescue
|
19
|
-
nil
|
20
|
-
end
|
21
|
-
end
|
22
|
-
data = body.read
|
23
|
-
else
|
24
|
-
data = Fog::Storage.parse_data(body)
|
25
|
-
options[:content_length] = data[:headers]['Content-Length']
|
26
|
-
options[:content_type] = data[:headers]['Content-Type']
|
27
|
-
data = data[:body]
|
28
|
-
end
|
29
|
-
|
30
|
-
raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
|
31
|
-
blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
|
32
|
-
rescue Azure::Core::Http::HTTPError => ex
|
33
|
-
raise_azure_exception(ex, msg)
|
34
|
-
end
|
35
|
-
|
36
|
-
if data.nil?
|
37
|
-
Fog::Logger.debug "Create a block blob #{blob_name} successfully."
|
38
|
-
else
|
39
|
-
Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
|
40
|
-
end
|
41
|
-
blob
|
11
|
+
@blob_client.create_block_blob(container_name, blob_name, body, options)
|
12
|
+
rescue Azure::Core::Http::HTTPError => ex
|
13
|
+
raise_azure_exception(ex, msg)
|
42
14
|
end
|
43
15
|
end
|
44
16
|
|
@@ -23,7 +23,7 @@ module Fog
|
|
23
23
|
protocol: 'https'
|
24
24
|
}
|
25
25
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
26
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
27
|
"#{uri}?#{token}"
|
28
28
|
end
|
29
29
|
end
|
@@ -13,16 +13,18 @@ module Fog
|
|
13
13
|
#
|
14
14
|
# @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
|
15
15
|
#
|
16
|
-
def get_blob_http_url(container_name, blob_name, expires)
|
16
|
+
def get_blob_http_url(container_name, blob_name, expires, options = {})
|
17
17
|
relative_path = "#{container_name}/#{blob_name}"
|
18
|
+
relative_path = remove_trailing_periods_from_path_segments(relative_path)
|
18
19
|
params = {
|
19
20
|
service: 'b',
|
20
21
|
resource: 'b',
|
21
22
|
permissions: 'r',
|
22
|
-
expiry: expires.utc.iso8601
|
23
|
+
expiry: expires.utc.iso8601,
|
24
|
+
content_disposition: options[:content_disposition]
|
23
25
|
}
|
24
26
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
25
|
-
uri = @blob_client.generate_uri(relative_path)
|
27
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
26
28
|
url = "#{uri}?#{token}"
|
27
29
|
url.gsub('https:', 'http:')
|
28
30
|
end
|
@@ -13,17 +13,19 @@ module Fog
|
|
13
13
|
#
|
14
14
|
# @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
|
15
15
|
#
|
16
|
-
def get_blob_https_url(container_name, blob_name, expires)
|
16
|
+
def get_blob_https_url(container_name, blob_name, expires, options = {})
|
17
17
|
relative_path = "#{container_name}/#{blob_name}"
|
18
|
+
relative_path = remove_trailing_periods_from_path_segments(relative_path)
|
18
19
|
params = {
|
19
20
|
service: 'b',
|
20
21
|
resource: 'b',
|
21
22
|
permissions: 'r',
|
22
23
|
expiry: expires.utc.iso8601,
|
23
|
-
protocol: 'https'
|
24
|
+
protocol: 'https',
|
25
|
+
content_disposition: options[:content_disposition]
|
24
26
|
}
|
25
27
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
28
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
29
|
"#{uri}?#{token}"
|
28
30
|
end
|
29
31
|
end
|
@@ -5,7 +5,7 @@ module Fog
|
|
5
5
|
class Real
|
6
6
|
# Get a public blob url from Azure blob storage
|
7
7
|
def get_blob_url(container_name, blob_name, options = {})
|
8
|
-
uri = @blob_client.generate_uri("#{container_name}/#{blob_name}")
|
8
|
+
uri = @blob_client.generate_uri("#{container_name}/#{blob_name}", {}, { encode: true })
|
9
9
|
|
10
10
|
if options[:scheme] == 'http'
|
11
11
|
uri.to_s.gsub('https:', 'http:')
|
@@ -11,7 +11,7 @@ module Fog
|
|
11
11
|
#
|
12
12
|
def get_container_url(container_name, options = {})
|
13
13
|
query = { 'comp' => 'list', 'restype' => 'container' }
|
14
|
-
uri = @blob_client.generate_uri(container_name, query)
|
14
|
+
uri = @blob_client.generate_uri(container_name, query, { encode: true })
|
15
15
|
|
16
16
|
if options[:scheme] == 'http'
|
17
17
|
uri.to_s.gsub('https:', 'http:')
|
@@ -5,15 +5,15 @@ module Fog
|
|
5
5
|
class Real
|
6
6
|
# Get a public blob url from Azure blob storage
|
7
7
|
# This is to make this library compatible with CarrierWave.
|
8
|
-
def get_object_url(container_name, blob_name, expires)
|
9
|
-
get_blob_https_url(container_name, blob_name, expires)
|
8
|
+
def get_object_url(container_name, blob_name, expires, options = {})
|
9
|
+
get_blob_https_url(container_name, blob_name, expires, options)
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
13
13
|
# This class provides the mock implementation for unit tests.
|
14
14
|
class Mock
|
15
|
-
def get_object_url(container_name, blob_name, expires)
|
16
|
-
get_blob_https_url(container_name, blob_name, expires)
|
15
|
+
def get_object_url(container_name, blob_name, expires, options = {})
|
16
|
+
get_blob_https_url(container_name, blob_name, expires, options)
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
@@ -23,7 +23,7 @@ module Fog
|
|
23
23
|
protocol: 'https'
|
24
24
|
}
|
25
25
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
26
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
27
|
"#{uri}?#{token}"
|
28
28
|
end
|
29
29
|
end
|
data/lib/fog/azurerm/storage.rb
CHANGED
data/lib/fog/azurerm/version.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require File.expand_path '../../test_helper', __dir__
|
2
2
|
|
3
3
|
# Test class for Storage Container Model
|
4
|
-
class TestFile < Minitest::Test
|
4
|
+
class TestFile < Minitest::Test # rubocop:disable Metrics/ClassLength
|
5
5
|
def setup
|
6
6
|
@service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
7
7
|
@directory = directory(@service)
|
@@ -76,10 +76,8 @@ class TestFile < Minitest::Test
|
|
76
76
|
def test_save_method_with_large_block_blob_success
|
77
77
|
@file.body = 'd' * (32 * 1024 * 1024 + 1) # SINGLE_BLOB_PUT_THRESHOLD is 32 * 1024 * 1024
|
78
78
|
|
79
|
-
@service.stub :
|
80
|
-
@
|
81
|
-
assert @file.save
|
82
|
-
end
|
79
|
+
@service.stub :create_block_blob, @raw_cloud_blob do
|
80
|
+
assert @file.save
|
83
81
|
end
|
84
82
|
end
|
85
83
|
|
@@ -285,6 +283,18 @@ class TestFile < Minitest::Test
|
|
285
283
|
end
|
286
284
|
end
|
287
285
|
|
286
|
+
def test_url_method_with_content_disposition
|
287
|
+
@file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
|
288
|
+
assert @file.url(Time.now + 3600, content_disposition: 'attachment'), @blob_https_url
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
def test_url_method_with_response_content_disposition
|
293
|
+
@file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
|
294
|
+
assert @file.url(Time.now + 3600, { query: { 'response-content-disposition' => 'attachment' } }), @blob_https_url
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
288
298
|
def test_url_method_without_key_exception
|
289
299
|
assert_raises(ArgumentError) do
|
290
300
|
@file.attributes.delete(:key)
|
@@ -52,15 +52,6 @@ class TestCreateBlockBlob < Minitest::Test
|
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
-
def test_create_block_blob_exceed_max_body_size
|
56
|
-
data = []
|
57
|
-
data.stub :size, 64 * 1024 * 1024 + 1 do
|
58
|
-
assert_raises(ArgumentError) do
|
59
|
-
@service.create_block_blob('test_container', 'test_blob', data)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
55
|
def test_create_block_blob_http_exception
|
65
56
|
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
66
57
|
@blob_client.stub :create_block_blob, http_exception do
|
@@ -24,6 +24,43 @@ class TestGetBlobHttpUrl < Minitest::Test
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
+
def test_get_blob_http_url_with_content_disposition
|
28
|
+
mock_generate_uri = Minitest::Mock.new
|
29
|
+
mock_generate_service_token = Minitest::Mock.new
|
30
|
+
url_params = { content_disposition: 'attachment' }
|
31
|
+
|
32
|
+
mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
|
33
|
+
mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
|
34
|
+
params[:service] == 'b' &&
|
35
|
+
params[:resource] == 'b' &&
|
36
|
+
params[:permissions] == 'r' &&
|
37
|
+
params[:protocol].nil? &&
|
38
|
+
params[:content_disposition] == url_params[:content_disposition]
|
39
|
+
end
|
40
|
+
|
41
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
42
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
43
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_get_url_remove_trailing_periods_from_path_segments
|
49
|
+
mock_generate_uri = Minitest::Mock.new
|
50
|
+
mock_generate_service_token = Minitest::Mock.new
|
51
|
+
|
52
|
+
mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
|
53
|
+
mock_generate_service_token.expect(:call, @token) do |relative_path, _|
|
54
|
+
relative_path == '.test0/..test1/...test2'
|
55
|
+
end
|
56
|
+
|
57
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
58
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
59
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
27
64
|
def test_get_blob_http_url_mock
|
28
65
|
assert_equal "#{@url}?#{@token}", @mock_service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600)
|
29
66
|
end
|
@@ -17,7 +17,13 @@ class TestGetBlobHttpsUrl < Minitest::Test
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def test_get_blob_https_url_success
|
20
|
-
|
20
|
+
mock_generate_uri = Minitest::Mock.new
|
21
|
+
|
22
|
+
2.times do
|
23
|
+
mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
|
24
|
+
end
|
25
|
+
|
26
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
21
27
|
@signature_client.stub :generate_service_sas_token, @token do
|
22
28
|
assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600)
|
23
29
|
assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600)
|
@@ -25,6 +31,48 @@ class TestGetBlobHttpsUrl < Minitest::Test
|
|
25
31
|
end
|
26
32
|
end
|
27
33
|
|
34
|
+
def test_get_blob_https_url_with_content_disposition
|
35
|
+
mock_generate_uri = Minitest::Mock.new
|
36
|
+
mock_generate_service_token = Minitest::Mock.new
|
37
|
+
url_params = { content_disposition: 'attachment' }
|
38
|
+
|
39
|
+
2.times do
|
40
|
+
mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
|
41
|
+
mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
|
42
|
+
params[:service] == 'b' &&
|
43
|
+
params[:resource] == 'b' &&
|
44
|
+
params[:permissions] == 'r' &&
|
45
|
+
params[:content_disposition] == url_params[:content_disposition]
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
50
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
51
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
|
52
|
+
assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_get_url_remove_trailing_periods_from_path_segments
|
58
|
+
mock_generate_uri = Minitest::Mock.new
|
59
|
+
mock_generate_service_token = Minitest::Mock.new
|
60
|
+
|
61
|
+
2.times do
|
62
|
+
mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
|
63
|
+
mock_generate_service_token.expect(:call, @token) do |relative_path, _|
|
64
|
+
relative_path == '.test0/..test1/...test2'
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
69
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
70
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
|
71
|
+
assert_equal "#{@url}?#{@token}", @service.get_object_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
28
76
|
def test_get_blob_https_url_with_domain_success
|
29
77
|
service = Fog::Storage::AzureRM.new(storage_account_credentials_with_domain)
|
30
78
|
blob_client = service.instance_variable_get(:@blob_client)
|
@@ -12,9 +12,14 @@ class TestGetBlobUrl < Minitest::Test
|
|
12
12
|
def test_get_blob_url_success
|
13
13
|
service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
14
14
|
blob_client = service.instance_variable_get(:@blob_client)
|
15
|
+
|
16
|
+
mock_generate_uri = Minitest::Mock.new
|
15
17
|
url = ApiStub::Requests::Storage::File.blob_https_url
|
18
|
+
2.times do
|
19
|
+
mock_generate_uri.expect(:call, url, ['test_container/test_blob', {}, { encode: true }])
|
20
|
+
end
|
16
21
|
|
17
|
-
blob_client.stub :generate_uri,
|
22
|
+
blob_client.stub :generate_uri, mock_generate_uri do
|
18
23
|
assert_equal url, service.get_blob_url('test_container', 'test_blob')
|
19
24
|
|
20
25
|
options = { scheme: 'http' }
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-fog-azure-rm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shaffan Chaudhry
|
@@ -15,10 +15,10 @@ authors:
|
|
15
15
|
- Azeem Sajid
|
16
16
|
- Maham Nazir
|
17
17
|
- Abbas Sheikh
|
18
|
-
autorequire:
|
18
|
+
autorequire:
|
19
19
|
bindir: bin
|
20
20
|
cert_chain: []
|
21
|
-
date:
|
21
|
+
date: 2021-09-20 00:00:00.000000000 Z
|
22
22
|
dependencies:
|
23
23
|
- !ruby/object:Gem::Dependency
|
24
24
|
name: codeclimate-test-reporter
|
@@ -176,7 +176,7 @@ dependencies:
|
|
176
176
|
version: 0.12.0
|
177
177
|
description: This is a stripped-down fork of fog-azure-rm that enables Azure Blob
|
178
178
|
Storage to be used with CarrierWave and Fog.
|
179
|
-
email:
|
179
|
+
email:
|
180
180
|
executables: []
|
181
181
|
extensions: []
|
182
182
|
extra_rdoc_files:
|
@@ -249,7 +249,6 @@ files:
|
|
249
249
|
- lib/fog/azurerm/requests/storage/get_object_url.rb
|
250
250
|
- lib/fog/azurerm/requests/storage/list_blobs.rb
|
251
251
|
- lib/fog/azurerm/requests/storage/list_containers.rb
|
252
|
-
- lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb
|
253
252
|
- lib/fog/azurerm/requests/storage/put_blob_block.rb
|
254
253
|
- lib/fog/azurerm/requests/storage/put_blob_https_url.rb
|
255
254
|
- lib/fog/azurerm/requests/storage/put_blob_metadata.rb
|
@@ -305,7 +304,6 @@ files:
|
|
305
304
|
- test/requests/storage/test_get_container_url.rb
|
306
305
|
- test/requests/storage/test_list_blobs.rb
|
307
306
|
- test/requests/storage/test_list_containers.rb
|
308
|
-
- test/requests/storage/test_multipart_save_block_blob.rb
|
309
307
|
- test/requests/storage/test_put_blob_block.rb
|
310
308
|
- test/requests/storage/test_put_blob_https_url.rb
|
311
309
|
- test/requests/storage/test_put_blob_metadata.rb
|
@@ -340,8 +338,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
340
338
|
- !ruby/object:Gem::Version
|
341
339
|
version: '0'
|
342
340
|
requirements: []
|
343
|
-
rubygems_version: 3.
|
344
|
-
signing_key:
|
341
|
+
rubygems_version: 3.1.6
|
342
|
+
signing_key:
|
345
343
|
specification_version: 4
|
346
344
|
summary: Module for the 'fog' gem to support Azure Blob Storage with CarrierWave and
|
347
345
|
Fog.
|
@@ -1,110 +0,0 @@
|
|
1
|
-
module Fog
|
2
|
-
module Storage
|
3
|
-
class AzureRM
|
4
|
-
# This class provides the actual implementation for service calls.
|
5
|
-
class Real
|
6
|
-
# This class is used to store chunk data for block blob before uploading.
|
7
|
-
class BlockChunk
|
8
|
-
attr_reader :id # For debug
|
9
|
-
attr_reader :block_id
|
10
|
-
attr_reader :data
|
11
|
-
|
12
|
-
def initialize(id, block_id, data)
|
13
|
-
@id = id
|
14
|
-
@block_id = block_id
|
15
|
-
@data = data
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
# This class is a stream to read chunk data.
|
20
|
-
class BlockFileStream
|
21
|
-
attr_reader :blocks
|
22
|
-
|
23
|
-
def initialize(body)
|
24
|
-
if body.respond_to?(:read)
|
25
|
-
if body.respond_to?(:rewind)
|
26
|
-
begin
|
27
|
-
body.rewind
|
28
|
-
rescue => ex
|
29
|
-
Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
|
30
|
-
end
|
31
|
-
end
|
32
|
-
@stream = body
|
33
|
-
else
|
34
|
-
@stream = StringIO.new(body)
|
35
|
-
end
|
36
|
-
@mutex = Mutex.new
|
37
|
-
@blocks = []
|
38
|
-
end
|
39
|
-
|
40
|
-
def read(size)
|
41
|
-
block_id = Base64.strict_encode64(random_string(32))
|
42
|
-
data = nil
|
43
|
-
id = 0
|
44
|
-
@mutex.synchronize do
|
45
|
-
data = @stream.read(size)
|
46
|
-
return nil if data.nil?
|
47
|
-
@blocks << [block_id]
|
48
|
-
id = @blocks.size
|
49
|
-
end
|
50
|
-
BlockChunk.new(id, block_id, data)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def multipart_save_block_blob(container_name, blob_name, body, options)
|
55
|
-
threads_num = options.delete(:worker_thread_num)
|
56
|
-
threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
|
57
|
-
|
58
|
-
begin
|
59
|
-
# Initiate the upload
|
60
|
-
Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
|
61
|
-
content_md5 = options.delete(:content_md5)
|
62
|
-
create_block_blob(container_name, blob_name, nil, options)
|
63
|
-
|
64
|
-
# Uploading parts
|
65
|
-
Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
|
66
|
-
iostream = BlockFileStream.new(body)
|
67
|
-
|
68
|
-
threads = []
|
69
|
-
threads_num.times do |id|
|
70
|
-
thread = Thread.new do
|
71
|
-
Fog::Logger.debug "Created upload thread #{id}."
|
72
|
-
while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
|
73
|
-
Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
|
74
|
-
put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
|
75
|
-
end
|
76
|
-
Fog::Logger.debug "Upload thread #{id} finished."
|
77
|
-
end
|
78
|
-
thread.abort_on_exception = true
|
79
|
-
threads << thread
|
80
|
-
end
|
81
|
-
|
82
|
-
threads.each(&:join)
|
83
|
-
# Complete the upload
|
84
|
-
options[:content_md5] = content_md5 unless content_md5.nil?
|
85
|
-
Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
|
86
|
-
commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
|
87
|
-
rescue
|
88
|
-
# Abort the upload & reraise
|
89
|
-
begin
|
90
|
-
delete_blob(container_name, blob_name)
|
91
|
-
rescue => ex
|
92
|
-
Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
|
93
|
-
end
|
94
|
-
raise
|
95
|
-
end
|
96
|
-
|
97
|
-
Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
|
98
|
-
true
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
# This class provides the mock implementation for unit tests.
|
103
|
-
class Mock
|
104
|
-
def multipart_save_block_blob(*)
|
105
|
-
true
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
@@ -1,105 +0,0 @@
|
|
1
|
-
require File.expand_path '../../test_helper', __dir__
|
2
|
-
|
3
|
-
# Storage Blob Class
|
4
|
-
class TestMultipartSaveBlockBlob < Minitest::Test
|
5
|
-
# This class posesses the test cases for the requests of saving storage block blob with multiple parts.
|
6
|
-
def setup
|
7
|
-
Fog.mock!
|
8
|
-
@mock_service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
9
|
-
Fog.unmock!
|
10
|
-
@mocked_response = mocked_storage_http_error
|
11
|
-
|
12
|
-
@service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_multipart_save_block_blob_success
|
16
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
17
|
-
|
18
|
-
@service.stub :create_block_blob, true do
|
19
|
-
@service.stub :put_blob_block, true do
|
20
|
-
@service.stub :commit_blob_blocks, true do
|
21
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', body, worker_thread_num: 6, content_md5: 'oafL1+HS78x65+e39PGIIg==')
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def test_multipart_save_block_blob_with_file_handle_success
|
28
|
-
i = 0
|
29
|
-
multiple_values = lambda do |*|
|
30
|
-
i += 1
|
31
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
32
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
33
|
-
return nil
|
34
|
-
end
|
35
|
-
temp_file = '/dev/null'
|
36
|
-
File.open(temp_file, 'r') do |file_handle|
|
37
|
-
file_handle.stub :read, multiple_values do
|
38
|
-
file_handle.stub :rewind, nil do
|
39
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
40
|
-
@service.stub :put_blob_block, true do
|
41
|
-
@service.stub :commit_blob_blocks, true do
|
42
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 0)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def test_multipart_save_block_blob_with_file_handle_do_not_support_rewind_success
|
52
|
-
i = 0
|
53
|
-
multiple_values = lambda do |*|
|
54
|
-
i += 1
|
55
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
56
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
57
|
-
return nil
|
58
|
-
end
|
59
|
-
temp_file = '/dev/null'
|
60
|
-
exception = ->(*) { raise 'do not support rewind' }
|
61
|
-
File.open(temp_file, 'r') do |file_handle|
|
62
|
-
file_handle.stub :read, multiple_values do
|
63
|
-
file_handle.stub :rewind, exception do
|
64
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
65
|
-
@service.stub :put_blob_block, true do
|
66
|
-
@service.stub :commit_blob_blocks, true do
|
67
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 'invalid')
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def test_multipart_save_block_blob_http_exception
|
77
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
78
|
-
|
79
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
80
|
-
@service.stub :create_block_blob, http_exception do
|
81
|
-
@service.stub :delete_blob, true do
|
82
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
83
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
84
|
-
end
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
def test_multipart_save_block_blob_fail_when_delete_blob_http_exception
|
90
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
91
|
-
|
92
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
93
|
-
@service.stub :create_block_blob, http_exception do
|
94
|
-
@service.stub :delete_blob, http_exception do
|
95
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
96
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
def test_multipart_save_block_blob_mock
|
103
|
-
assert @mock_service.multipart_save_block_blob('test_container', 'test_blob', 'content', {})
|
104
|
-
end
|
105
|
-
end
|