gitlab-fog-azure-rm 0.9.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -1
- data/CHANGELOG.md +20 -0
- data/lib/fog/azurerm/models/storage/file.rb +10 -7
- data/lib/fog/azurerm/models/storage/files.rb +4 -4
- data/lib/fog/azurerm/requests/storage/create_block_blob.rb +3 -31
- data/lib/fog/azurerm/requests/storage/delete_blob_https_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_blob_http_url.rb +5 -3
- data/lib/fog/azurerm/requests/storage/get_blob_https_url.rb +5 -3
- data/lib/fog/azurerm/requests/storage/get_blob_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_container_url.rb +1 -1
- data/lib/fog/azurerm/requests/storage/get_object_url.rb +4 -4
- data/lib/fog/azurerm/requests/storage/put_blob_https_url.rb +1 -1
- data/lib/fog/azurerm/storage.rb +0 -1
- data/lib/fog/azurerm/utilities/general.rb +4 -0
- data/lib/fog/azurerm/version.rb +1 -1
- data/test/models/storage/test_file.rb +15 -5
- data/test/requests/storage/test_create_block_blob.rb +0 -9
- data/test/requests/storage/test_get_blob_http_url.rb +37 -0
- data/test/requests/storage/test_get_blob_https_url.rb +49 -1
- data/test/requests/storage/test_get_blob_url.rb +6 -1
- metadata +6 -8
- data/lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb +0 -110
- data/test/requests/storage/test_multipart_save_block_blob.rb +0 -105
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9cc556f2028cd1a063691e4556dc1421014476690f700eb31fea81bcac574b41
|
4
|
+
data.tar.gz: e6e7d3c355e63cf6a0501d2bc85cbfd994ae54349c950657f6619244fecc9d02
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c1ffb76c0ba61d39f675c90612afdf3ae06f0b0835c8f6162a874b73a637e58e4c2c07c0b193ede34ef84601c040920e7a93c2ade997abcdd3c39a64f45d9ff9
|
7
|
+
data.tar.gz: bae99d6843a450a5072a9a8e5a2e94de43a6849565fa38c2b288d3c14fbeb70066be960b08c0172a5a9eb61314119d1e78dfb890f90fa4b4af7f7784b028fcdd
|
data/.rubocop_todo.yml
CHANGED
@@ -132,7 +132,6 @@ Naming/RescuedExceptionsVariableName:
|
|
132
132
|
# SupportedStyles: separated, grouped
|
133
133
|
Style/AccessorGrouping:
|
134
134
|
Exclude:
|
135
|
-
- 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb'
|
136
135
|
- 'lib/fog/azurerm/requests/storage/save_page_blob.rb'
|
137
136
|
- 'test/test_helper.rb'
|
138
137
|
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,25 @@
|
|
1
1
|
## Unreleased
|
2
2
|
|
3
|
+
## 1.2.0
|
4
|
+
|
5
|
+
- Support setting returned Content-Disposition in GetBlob requests !25
|
6
|
+
|
7
|
+
## 1.1.1
|
8
|
+
|
9
|
+
- Remove trailing periods per path segment when retrieving blob URLs !23
|
10
|
+
|
11
|
+
## 1.1.0
|
12
|
+
|
13
|
+
- Yanked
|
14
|
+
|
15
|
+
## 1.0.1
|
16
|
+
|
17
|
+
- Fix URL generation for files with UTF-8 characters !21
|
18
|
+
|
19
|
+
## 1.0.0
|
20
|
+
|
21
|
+
- Fix multipart uploads not working and drop multipart_save_block_blob !19
|
22
|
+
|
3
23
|
## 0.9.0
|
4
24
|
|
5
25
|
- Fix TimeoutError deprecation !17
|
@@ -240,20 +240,23 @@ module Fog
|
|
240
240
|
#
|
241
241
|
def url(expires, options = {})
|
242
242
|
requires :key
|
243
|
-
collection.get_url(key, expires, options)
|
243
|
+
collection.get_url(key, expires, normalize_options(options))
|
244
244
|
end
|
245
245
|
|
246
246
|
private
|
247
247
|
|
248
|
+
def normalize_options(options)
|
249
|
+
# AWS S3 and Google Cloud Storage pass response-content-disposition
|
250
|
+
# as a query string, while Azure needs the content_disposition parameter
|
251
|
+
# to generate a SAS token.
|
252
|
+
options[:content_disposition] ||= options.dig(:query, 'response-content-disposition')
|
253
|
+
options
|
254
|
+
end
|
255
|
+
|
248
256
|
# Upload blob
|
249
257
|
def save_blob(options)
|
250
258
|
if options[:blob_type].nil? || options[:blob_type] == 'BlockBlob'
|
251
|
-
|
252
|
-
service.create_block_blob(directory.key, key, body, options)
|
253
|
-
else
|
254
|
-
service.multipart_save_block_blob(directory.key, key, body, options)
|
255
|
-
service.get_blob_properties(directory.key, key)
|
256
|
-
end
|
259
|
+
service.create_block_blob(directory.key, key, body, options)
|
257
260
|
else
|
258
261
|
service.save_page_blob(directory.key, key, body, options)
|
259
262
|
service.get_blob_properties(directory.key, key)
|
@@ -123,10 +123,10 @@ module Fog
|
|
123
123
|
#
|
124
124
|
# @return [String] A http URL.
|
125
125
|
#
|
126
|
-
def get_http_url(key, expires,
|
126
|
+
def get_http_url(key, expires, options = {})
|
127
127
|
requires :directory
|
128
128
|
|
129
|
-
service.get_blob_http_url(directory.key, key, expires)
|
129
|
+
service.get_blob_http_url(directory.key, key, expires, options)
|
130
130
|
end
|
131
131
|
|
132
132
|
# Get the https URL of the file(blob) with the given name.
|
@@ -139,10 +139,10 @@ module Fog
|
|
139
139
|
#
|
140
140
|
# @return [String] A https URL.
|
141
141
|
#
|
142
|
-
def get_https_url(key, expires,
|
142
|
+
def get_https_url(key, expires, options = {})
|
143
143
|
requires :directory
|
144
144
|
|
145
|
-
service.get_blob_https_url(directory.key, key, expires)
|
145
|
+
service.get_blob_https_url(directory.key, key, expires, options)
|
146
146
|
end
|
147
147
|
|
148
148
|
# Get the file(blob) without content with the given name.
|
@@ -8,37 +8,9 @@ module Fog
|
|
8
8
|
msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
|
9
9
|
Fog::Logger.debug msg
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
elsif body.respond_to?(:read)
|
15
|
-
if body.respond_to?(:rewind)
|
16
|
-
begin
|
17
|
-
body.rewind
|
18
|
-
rescue
|
19
|
-
nil
|
20
|
-
end
|
21
|
-
end
|
22
|
-
data = body.read
|
23
|
-
else
|
24
|
-
data = Fog::Storage.parse_data(body)
|
25
|
-
options[:content_length] = data[:headers]['Content-Length']
|
26
|
-
options[:content_type] = data[:headers]['Content-Type']
|
27
|
-
data = data[:body]
|
28
|
-
end
|
29
|
-
|
30
|
-
raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
|
31
|
-
blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
|
32
|
-
rescue Azure::Core::Http::HTTPError => ex
|
33
|
-
raise_azure_exception(ex, msg)
|
34
|
-
end
|
35
|
-
|
36
|
-
if data.nil?
|
37
|
-
Fog::Logger.debug "Create a block blob #{blob_name} successfully."
|
38
|
-
else
|
39
|
-
Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
|
40
|
-
end
|
41
|
-
blob
|
11
|
+
@blob_client.create_block_blob(container_name, blob_name, body, options)
|
12
|
+
rescue Azure::Core::Http::HTTPError => ex
|
13
|
+
raise_azure_exception(ex, msg)
|
42
14
|
end
|
43
15
|
end
|
44
16
|
|
@@ -23,7 +23,7 @@ module Fog
|
|
23
23
|
protocol: 'https'
|
24
24
|
}
|
25
25
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
26
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
27
|
"#{uri}?#{token}"
|
28
28
|
end
|
29
29
|
end
|
@@ -13,16 +13,18 @@ module Fog
|
|
13
13
|
#
|
14
14
|
# @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
|
15
15
|
#
|
16
|
-
def get_blob_http_url(container_name, blob_name, expires)
|
16
|
+
def get_blob_http_url(container_name, blob_name, expires, options = {})
|
17
17
|
relative_path = "#{container_name}/#{blob_name}"
|
18
|
+
relative_path = remove_trailing_periods_from_path_segments(relative_path)
|
18
19
|
params = {
|
19
20
|
service: 'b',
|
20
21
|
resource: 'b',
|
21
22
|
permissions: 'r',
|
22
|
-
expiry: expires.utc.iso8601
|
23
|
+
expiry: expires.utc.iso8601,
|
24
|
+
content_disposition: options[:content_disposition]
|
23
25
|
}
|
24
26
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
25
|
-
uri = @blob_client.generate_uri(relative_path)
|
27
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
26
28
|
url = "#{uri}?#{token}"
|
27
29
|
url.gsub('https:', 'http:')
|
28
30
|
end
|
@@ -13,17 +13,19 @@ module Fog
|
|
13
13
|
#
|
14
14
|
# @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
|
15
15
|
#
|
16
|
-
def get_blob_https_url(container_name, blob_name, expires)
|
16
|
+
def get_blob_https_url(container_name, blob_name, expires, options = {})
|
17
17
|
relative_path = "#{container_name}/#{blob_name}"
|
18
|
+
relative_path = remove_trailing_periods_from_path_segments(relative_path)
|
18
19
|
params = {
|
19
20
|
service: 'b',
|
20
21
|
resource: 'b',
|
21
22
|
permissions: 'r',
|
22
23
|
expiry: expires.utc.iso8601,
|
23
|
-
protocol: 'https'
|
24
|
+
protocol: 'https',
|
25
|
+
content_disposition: options[:content_disposition]
|
24
26
|
}
|
25
27
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
28
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
29
|
"#{uri}?#{token}"
|
28
30
|
end
|
29
31
|
end
|
@@ -5,7 +5,7 @@ module Fog
|
|
5
5
|
class Real
|
6
6
|
# Get a public blob url from Azure blob storage
|
7
7
|
def get_blob_url(container_name, blob_name, options = {})
|
8
|
-
uri = @blob_client.generate_uri("#{container_name}/#{blob_name}")
|
8
|
+
uri = @blob_client.generate_uri("#{container_name}/#{blob_name}", {}, { encode: true })
|
9
9
|
|
10
10
|
if options[:scheme] == 'http'
|
11
11
|
uri.to_s.gsub('https:', 'http:')
|
@@ -11,7 +11,7 @@ module Fog
|
|
11
11
|
#
|
12
12
|
def get_container_url(container_name, options = {})
|
13
13
|
query = { 'comp' => 'list', 'restype' => 'container' }
|
14
|
-
uri = @blob_client.generate_uri(container_name, query)
|
14
|
+
uri = @blob_client.generate_uri(container_name, query, { encode: true })
|
15
15
|
|
16
16
|
if options[:scheme] == 'http'
|
17
17
|
uri.to_s.gsub('https:', 'http:')
|
@@ -5,15 +5,15 @@ module Fog
|
|
5
5
|
class Real
|
6
6
|
# Get a public blob url from Azure blob storage
|
7
7
|
# This is to make this library compatible with CarrierWave.
|
8
|
-
def get_object_url(container_name, blob_name, expires)
|
9
|
-
get_blob_https_url(container_name, blob_name, expires)
|
8
|
+
def get_object_url(container_name, blob_name, expires, options = {})
|
9
|
+
get_blob_https_url(container_name, blob_name, expires, options)
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
13
13
|
# This class provides the mock implementation for unit tests.
|
14
14
|
class Mock
|
15
|
-
def get_object_url(container_name, blob_name, expires)
|
16
|
-
get_blob_https_url(container_name, blob_name, expires)
|
15
|
+
def get_object_url(container_name, blob_name, expires, options = {})
|
16
|
+
get_blob_https_url(container_name, blob_name, expires, options)
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
@@ -23,7 +23,7 @@ module Fog
|
|
23
23
|
protocol: 'https'
|
24
24
|
}
|
25
25
|
token = @signature_client.generate_service_sas_token(relative_path, params)
|
26
|
-
uri = @blob_client.generate_uri(relative_path)
|
26
|
+
uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
|
27
27
|
"#{uri}?#{token}"
|
28
28
|
end
|
29
29
|
end
|
data/lib/fog/azurerm/storage.rb
CHANGED
data/lib/fog/azurerm/version.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require File.expand_path '../../test_helper', __dir__
|
2
2
|
|
3
3
|
# Test class for Storage Container Model
|
4
|
-
class TestFile < Minitest::Test
|
4
|
+
class TestFile < Minitest::Test # rubocop:disable Metrics/ClassLength
|
5
5
|
def setup
|
6
6
|
@service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
7
7
|
@directory = directory(@service)
|
@@ -76,10 +76,8 @@ class TestFile < Minitest::Test
|
|
76
76
|
def test_save_method_with_large_block_blob_success
|
77
77
|
@file.body = 'd' * (32 * 1024 * 1024 + 1) # SINGLE_BLOB_PUT_THRESHOLD is 32 * 1024 * 1024
|
78
78
|
|
79
|
-
@service.stub :
|
80
|
-
@
|
81
|
-
assert @file.save
|
82
|
-
end
|
79
|
+
@service.stub :create_block_blob, @raw_cloud_blob do
|
80
|
+
assert @file.save
|
83
81
|
end
|
84
82
|
end
|
85
83
|
|
@@ -285,6 +283,18 @@ class TestFile < Minitest::Test
|
|
285
283
|
end
|
286
284
|
end
|
287
285
|
|
286
|
+
def test_url_method_with_content_disposition
|
287
|
+
@file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
|
288
|
+
assert @file.url(Time.now + 3600, content_disposition: 'attachment'), @blob_https_url
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
def test_url_method_with_response_content_disposition
|
293
|
+
@file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
|
294
|
+
assert @file.url(Time.now + 3600, { query: { 'response-content-disposition' => 'attachment' } }), @blob_https_url
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
288
298
|
def test_url_method_without_key_exception
|
289
299
|
assert_raises(ArgumentError) do
|
290
300
|
@file.attributes.delete(:key)
|
@@ -52,15 +52,6 @@ class TestCreateBlockBlob < Minitest::Test
|
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
-
def test_create_block_blob_exceed_max_body_size
|
56
|
-
data = []
|
57
|
-
data.stub :size, 64 * 1024 * 1024 + 1 do
|
58
|
-
assert_raises(ArgumentError) do
|
59
|
-
@service.create_block_blob('test_container', 'test_blob', data)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
55
|
def test_create_block_blob_http_exception
|
65
56
|
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
66
57
|
@blob_client.stub :create_block_blob, http_exception do
|
@@ -24,6 +24,43 @@ class TestGetBlobHttpUrl < Minitest::Test
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
+
def test_get_blob_http_url_with_content_disposition
|
28
|
+
mock_generate_uri = Minitest::Mock.new
|
29
|
+
mock_generate_service_token = Minitest::Mock.new
|
30
|
+
url_params = { content_disposition: 'attachment' }
|
31
|
+
|
32
|
+
mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
|
33
|
+
mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
|
34
|
+
params[:service] == 'b' &&
|
35
|
+
params[:resource] == 'b' &&
|
36
|
+
params[:permissions] == 'r' &&
|
37
|
+
params[:protocol].nil? &&
|
38
|
+
params[:content_disposition] == url_params[:content_disposition]
|
39
|
+
end
|
40
|
+
|
41
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
42
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
43
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_get_url_remove_trailing_periods_from_path_segments
|
49
|
+
mock_generate_uri = Minitest::Mock.new
|
50
|
+
mock_generate_service_token = Minitest::Mock.new
|
51
|
+
|
52
|
+
mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
|
53
|
+
mock_generate_service_token.expect(:call, @token) do |relative_path, _|
|
54
|
+
relative_path == '.test0/..test1/...test2'
|
55
|
+
end
|
56
|
+
|
57
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
58
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
59
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
27
64
|
def test_get_blob_http_url_mock
|
28
65
|
assert_equal "#{@url}?#{@token}", @mock_service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600)
|
29
66
|
end
|
@@ -17,7 +17,13 @@ class TestGetBlobHttpsUrl < Minitest::Test
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def test_get_blob_https_url_success
|
20
|
-
|
20
|
+
mock_generate_uri = Minitest::Mock.new
|
21
|
+
|
22
|
+
2.times do
|
23
|
+
mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
|
24
|
+
end
|
25
|
+
|
26
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
21
27
|
@signature_client.stub :generate_service_sas_token, @token do
|
22
28
|
assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600)
|
23
29
|
assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600)
|
@@ -25,6 +31,48 @@ class TestGetBlobHttpsUrl < Minitest::Test
|
|
25
31
|
end
|
26
32
|
end
|
27
33
|
|
34
|
+
def test_get_blob_https_url_with_content_disposition
|
35
|
+
mock_generate_uri = Minitest::Mock.new
|
36
|
+
mock_generate_service_token = Minitest::Mock.new
|
37
|
+
url_params = { content_disposition: 'attachment' }
|
38
|
+
|
39
|
+
2.times do
|
40
|
+
mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
|
41
|
+
mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
|
42
|
+
params[:service] == 'b' &&
|
43
|
+
params[:resource] == 'b' &&
|
44
|
+
params[:permissions] == 'r' &&
|
45
|
+
params[:content_disposition] == url_params[:content_disposition]
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
50
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
51
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
|
52
|
+
assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_get_url_remove_trailing_periods_from_path_segments
|
58
|
+
mock_generate_uri = Minitest::Mock.new
|
59
|
+
mock_generate_service_token = Minitest::Mock.new
|
60
|
+
|
61
|
+
2.times do
|
62
|
+
mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
|
63
|
+
mock_generate_service_token.expect(:call, @token) do |relative_path, _|
|
64
|
+
relative_path == '.test0/..test1/...test2'
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
@blob_client.stub :generate_uri, mock_generate_uri do
|
69
|
+
@signature_client.stub :generate_service_sas_token, mock_generate_service_token do
|
70
|
+
assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
|
71
|
+
assert_equal "#{@url}?#{@token}", @service.get_object_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
28
76
|
def test_get_blob_https_url_with_domain_success
|
29
77
|
service = Fog::Storage::AzureRM.new(storage_account_credentials_with_domain)
|
30
78
|
blob_client = service.instance_variable_get(:@blob_client)
|
@@ -12,9 +12,14 @@ class TestGetBlobUrl < Minitest::Test
|
|
12
12
|
def test_get_blob_url_success
|
13
13
|
service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
14
14
|
blob_client = service.instance_variable_get(:@blob_client)
|
15
|
+
|
16
|
+
mock_generate_uri = Minitest::Mock.new
|
15
17
|
url = ApiStub::Requests::Storage::File.blob_https_url
|
18
|
+
2.times do
|
19
|
+
mock_generate_uri.expect(:call, url, ['test_container/test_blob', {}, { encode: true }])
|
20
|
+
end
|
16
21
|
|
17
|
-
blob_client.stub :generate_uri,
|
22
|
+
blob_client.stub :generate_uri, mock_generate_uri do
|
18
23
|
assert_equal url, service.get_blob_url('test_container', 'test_blob')
|
19
24
|
|
20
25
|
options = { scheme: 'http' }
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-fog-azure-rm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shaffan Chaudhry
|
@@ -15,10 +15,10 @@ authors:
|
|
15
15
|
- Azeem Sajid
|
16
16
|
- Maham Nazir
|
17
17
|
- Abbas Sheikh
|
18
|
-
autorequire:
|
18
|
+
autorequire:
|
19
19
|
bindir: bin
|
20
20
|
cert_chain: []
|
21
|
-
date:
|
21
|
+
date: 2021-09-20 00:00:00.000000000 Z
|
22
22
|
dependencies:
|
23
23
|
- !ruby/object:Gem::Dependency
|
24
24
|
name: codeclimate-test-reporter
|
@@ -176,7 +176,7 @@ dependencies:
|
|
176
176
|
version: 0.12.0
|
177
177
|
description: This is a stripped-down fork of fog-azure-rm that enables Azure Blob
|
178
178
|
Storage to be used with CarrierWave and Fog.
|
179
|
-
email:
|
179
|
+
email:
|
180
180
|
executables: []
|
181
181
|
extensions: []
|
182
182
|
extra_rdoc_files:
|
@@ -249,7 +249,6 @@ files:
|
|
249
249
|
- lib/fog/azurerm/requests/storage/get_object_url.rb
|
250
250
|
- lib/fog/azurerm/requests/storage/list_blobs.rb
|
251
251
|
- lib/fog/azurerm/requests/storage/list_containers.rb
|
252
|
-
- lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb
|
253
252
|
- lib/fog/azurerm/requests/storage/put_blob_block.rb
|
254
253
|
- lib/fog/azurerm/requests/storage/put_blob_https_url.rb
|
255
254
|
- lib/fog/azurerm/requests/storage/put_blob_metadata.rb
|
@@ -305,7 +304,6 @@ files:
|
|
305
304
|
- test/requests/storage/test_get_container_url.rb
|
306
305
|
- test/requests/storage/test_list_blobs.rb
|
307
306
|
- test/requests/storage/test_list_containers.rb
|
308
|
-
- test/requests/storage/test_multipart_save_block_blob.rb
|
309
307
|
- test/requests/storage/test_put_blob_block.rb
|
310
308
|
- test/requests/storage/test_put_blob_https_url.rb
|
311
309
|
- test/requests/storage/test_put_blob_metadata.rb
|
@@ -340,8 +338,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
340
338
|
- !ruby/object:Gem::Version
|
341
339
|
version: '0'
|
342
340
|
requirements: []
|
343
|
-
rubygems_version: 3.
|
344
|
-
signing_key:
|
341
|
+
rubygems_version: 3.1.6
|
342
|
+
signing_key:
|
345
343
|
specification_version: 4
|
346
344
|
summary: Module for the 'fog' gem to support Azure Blob Storage with CarrierWave and
|
347
345
|
Fog.
|
@@ -1,110 +0,0 @@
|
|
1
|
-
module Fog
|
2
|
-
module Storage
|
3
|
-
class AzureRM
|
4
|
-
# This class provides the actual implementation for service calls.
|
5
|
-
class Real
|
6
|
-
# This class is used to store chunk data for block blob before uploading.
|
7
|
-
class BlockChunk
|
8
|
-
attr_reader :id # For debug
|
9
|
-
attr_reader :block_id
|
10
|
-
attr_reader :data
|
11
|
-
|
12
|
-
def initialize(id, block_id, data)
|
13
|
-
@id = id
|
14
|
-
@block_id = block_id
|
15
|
-
@data = data
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
# This class is a stream to read chunk data.
|
20
|
-
class BlockFileStream
|
21
|
-
attr_reader :blocks
|
22
|
-
|
23
|
-
def initialize(body)
|
24
|
-
if body.respond_to?(:read)
|
25
|
-
if body.respond_to?(:rewind)
|
26
|
-
begin
|
27
|
-
body.rewind
|
28
|
-
rescue => ex
|
29
|
-
Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
|
30
|
-
end
|
31
|
-
end
|
32
|
-
@stream = body
|
33
|
-
else
|
34
|
-
@stream = StringIO.new(body)
|
35
|
-
end
|
36
|
-
@mutex = Mutex.new
|
37
|
-
@blocks = []
|
38
|
-
end
|
39
|
-
|
40
|
-
def read(size)
|
41
|
-
block_id = Base64.strict_encode64(random_string(32))
|
42
|
-
data = nil
|
43
|
-
id = 0
|
44
|
-
@mutex.synchronize do
|
45
|
-
data = @stream.read(size)
|
46
|
-
return nil if data.nil?
|
47
|
-
@blocks << [block_id]
|
48
|
-
id = @blocks.size
|
49
|
-
end
|
50
|
-
BlockChunk.new(id, block_id, data)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def multipart_save_block_blob(container_name, blob_name, body, options)
|
55
|
-
threads_num = options.delete(:worker_thread_num)
|
56
|
-
threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
|
57
|
-
|
58
|
-
begin
|
59
|
-
# Initiate the upload
|
60
|
-
Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
|
61
|
-
content_md5 = options.delete(:content_md5)
|
62
|
-
create_block_blob(container_name, blob_name, nil, options)
|
63
|
-
|
64
|
-
# Uploading parts
|
65
|
-
Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
|
66
|
-
iostream = BlockFileStream.new(body)
|
67
|
-
|
68
|
-
threads = []
|
69
|
-
threads_num.times do |id|
|
70
|
-
thread = Thread.new do
|
71
|
-
Fog::Logger.debug "Created upload thread #{id}."
|
72
|
-
while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
|
73
|
-
Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
|
74
|
-
put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
|
75
|
-
end
|
76
|
-
Fog::Logger.debug "Upload thread #{id} finished."
|
77
|
-
end
|
78
|
-
thread.abort_on_exception = true
|
79
|
-
threads << thread
|
80
|
-
end
|
81
|
-
|
82
|
-
threads.each(&:join)
|
83
|
-
# Complete the upload
|
84
|
-
options[:content_md5] = content_md5 unless content_md5.nil?
|
85
|
-
Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
|
86
|
-
commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
|
87
|
-
rescue
|
88
|
-
# Abort the upload & reraise
|
89
|
-
begin
|
90
|
-
delete_blob(container_name, blob_name)
|
91
|
-
rescue => ex
|
92
|
-
Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
|
93
|
-
end
|
94
|
-
raise
|
95
|
-
end
|
96
|
-
|
97
|
-
Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
|
98
|
-
true
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
# This class provides the mock implementation for unit tests.
|
103
|
-
class Mock
|
104
|
-
def multipart_save_block_blob(*)
|
105
|
-
true
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
@@ -1,105 +0,0 @@
|
|
1
|
-
require File.expand_path '../../test_helper', __dir__
|
2
|
-
|
3
|
-
# Storage Blob Class
|
4
|
-
class TestMultipartSaveBlockBlob < Minitest::Test
|
5
|
-
# This class posesses the test cases for the requests of saving storage block blob with multiple parts.
|
6
|
-
def setup
|
7
|
-
Fog.mock!
|
8
|
-
@mock_service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
9
|
-
Fog.unmock!
|
10
|
-
@mocked_response = mocked_storage_http_error
|
11
|
-
|
12
|
-
@service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_multipart_save_block_blob_success
|
16
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
17
|
-
|
18
|
-
@service.stub :create_block_blob, true do
|
19
|
-
@service.stub :put_blob_block, true do
|
20
|
-
@service.stub :commit_blob_blocks, true do
|
21
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', body, worker_thread_num: 6, content_md5: 'oafL1+HS78x65+e39PGIIg==')
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def test_multipart_save_block_blob_with_file_handle_success
|
28
|
-
i = 0
|
29
|
-
multiple_values = lambda do |*|
|
30
|
-
i += 1
|
31
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
32
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
33
|
-
return nil
|
34
|
-
end
|
35
|
-
temp_file = '/dev/null'
|
36
|
-
File.open(temp_file, 'r') do |file_handle|
|
37
|
-
file_handle.stub :read, multiple_values do
|
38
|
-
file_handle.stub :rewind, nil do
|
39
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
40
|
-
@service.stub :put_blob_block, true do
|
41
|
-
@service.stub :commit_blob_blocks, true do
|
42
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 0)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def test_multipart_save_block_blob_with_file_handle_do_not_support_rewind_success
|
52
|
-
i = 0
|
53
|
-
multiple_values = lambda do |*|
|
54
|
-
i += 1
|
55
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
56
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
57
|
-
return nil
|
58
|
-
end
|
59
|
-
temp_file = '/dev/null'
|
60
|
-
exception = ->(*) { raise 'do not support rewind' }
|
61
|
-
File.open(temp_file, 'r') do |file_handle|
|
62
|
-
file_handle.stub :read, multiple_values do
|
63
|
-
file_handle.stub :rewind, exception do
|
64
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
65
|
-
@service.stub :put_blob_block, true do
|
66
|
-
@service.stub :commit_blob_blocks, true do
|
67
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 'invalid')
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def test_multipart_save_block_blob_http_exception
|
77
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
78
|
-
|
79
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
80
|
-
@service.stub :create_block_blob, http_exception do
|
81
|
-
@service.stub :delete_blob, true do
|
82
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
83
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
84
|
-
end
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
def test_multipart_save_block_blob_fail_when_delete_blob_http_exception
|
90
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
91
|
-
|
92
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
93
|
-
@service.stub :create_block_blob, http_exception do
|
94
|
-
@service.stub :delete_blob, http_exception do
|
95
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
96
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
def test_multipart_save_block_blob_mock
|
103
|
-
assert @mock_service.multipart_save_block_blob('test_container', 'test_blob', 'content', {})
|
104
|
-
end
|
105
|
-
end
|