gitlab-fog-azure-rm 0.9.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9a3e5613e808f27ef9bcd2be6015eac3ffc43e55fe558b2c78b30b97164c3f78
4
- data.tar.gz: d64eb6be36585d864bb5bb7fdcafdd5191a7bda376a4f84edc7e82c9bd972554
3
+ metadata.gz: 9cc556f2028cd1a063691e4556dc1421014476690f700eb31fea81bcac574b41
4
+ data.tar.gz: e6e7d3c355e63cf6a0501d2bc85cbfd994ae54349c950657f6619244fecc9d02
5
5
  SHA512:
6
- metadata.gz: b5b884c2fc10a00a1c385522f2138492427ab50162df609fdfe1398f380c42184bb8a99bf8eae0fd50ff77521796f5371fd40b3222114a5f794368555c8408c7
7
- data.tar.gz: c1a11715408b8c7eb825915e0ecf1dbfb584f831c85dcbb711973f1b2e22ac54de07f3bbbaf59273c525abe9fa6919385fed04c7e85de400ca727d5bbe19c7b2
6
+ metadata.gz: c1ffb76c0ba61d39f675c90612afdf3ae06f0b0835c8f6162a874b73a637e58e4c2c07c0b193ede34ef84601c040920e7a93c2ade997abcdd3c39a64f45d9ff9
7
+ data.tar.gz: bae99d6843a450a5072a9a8e5a2e94de43a6849565fa38c2b288d3c14fbeb70066be960b08c0172a5a9eb61314119d1e78dfb890f90fa4b4af7f7784b028fcdd
data/.rubocop_todo.yml CHANGED
@@ -132,7 +132,6 @@ Naming/RescuedExceptionsVariableName:
132
132
  # SupportedStyles: separated, grouped
133
133
  Style/AccessorGrouping:
134
134
  Exclude:
135
- - 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb'
136
135
  - 'lib/fog/azurerm/requests/storage/save_page_blob.rb'
137
136
  - 'test/test_helper.rb'
138
137
 
data/CHANGELOG.md CHANGED
@@ -1,5 +1,25 @@
1
1
  ## Unreleased
2
2
 
3
+ ## 1.2.0
4
+
5
+ - Support setting returned Content-Disposition in GetBlob requests !25
6
+
7
+ ## 1.1.1
8
+
9
+ - Remove trailing periods per path segment when retrieving blob URLs !23
10
+
11
+ ## 1.1.0
12
+
13
+ - Yanked
14
+
15
+ ## 1.0.1
16
+
17
+ - Fix URL generation for files with UTF-8 characters !21
18
+
19
+ ## 1.0.0
20
+
21
+ - Fix multipart uploads not working and drop multipart_save_block_blob !19
22
+
3
23
  ## 0.9.0
4
24
 
5
25
  - Fix TimeoutError deprecation !17
@@ -240,20 +240,23 @@ module Fog
240
240
  #
241
241
  def url(expires, options = {})
242
242
  requires :key
243
- collection.get_url(key, expires, options)
243
+ collection.get_url(key, expires, normalize_options(options))
244
244
  end
245
245
 
246
246
  private
247
247
 
248
+ def normalize_options(options)
249
+ # AWS S3 and Google Cloud Storage pass response-content-disposition
250
+ # as a query string, while Azure needs the content_disposition parameter
251
+ # to generate a SAS token.
252
+ options[:content_disposition] ||= options.dig(:query, 'response-content-disposition')
253
+ options
254
+ end
255
+
248
256
  # Upload blob
249
257
  def save_blob(options)
250
258
  if options[:blob_type].nil? || options[:blob_type] == 'BlockBlob'
251
- if Fog::Storage.get_body_size(body) <= SINGLE_BLOB_PUT_THRESHOLD
252
- service.create_block_blob(directory.key, key, body, options)
253
- else
254
- service.multipart_save_block_blob(directory.key, key, body, options)
255
- service.get_blob_properties(directory.key, key)
256
- end
259
+ service.create_block_blob(directory.key, key, body, options)
257
260
  else
258
261
  service.save_page_blob(directory.key, key, body, options)
259
262
  service.get_blob_properties(directory.key, key)
@@ -123,10 +123,10 @@ module Fog
123
123
  #
124
124
  # @return [String] A http URL.
125
125
  #
126
- def get_http_url(key, expires, _options = {})
126
+ def get_http_url(key, expires, options = {})
127
127
  requires :directory
128
128
 
129
- service.get_blob_http_url(directory.key, key, expires)
129
+ service.get_blob_http_url(directory.key, key, expires, options)
130
130
  end
131
131
 
132
132
  # Get the https URL of the file(blob) with the given name.
@@ -139,10 +139,10 @@ module Fog
139
139
  #
140
140
  # @return [String] A https URL.
141
141
  #
142
- def get_https_url(key, expires, _options = {})
142
+ def get_https_url(key, expires, options = {})
143
143
  requires :directory
144
144
 
145
- service.get_blob_https_url(directory.key, key, expires)
145
+ service.get_blob_https_url(directory.key, key, expires, options)
146
146
  end
147
147
 
148
148
  # Get the file(blob) without content with the given name.
@@ -8,37 +8,9 @@ module Fog
8
8
  msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
9
9
  Fog::Logger.debug msg
10
10
 
11
- begin
12
- if body.nil?
13
- data = nil
14
- elsif body.respond_to?(:read)
15
- if body.respond_to?(:rewind)
16
- begin
17
- body.rewind
18
- rescue
19
- nil
20
- end
21
- end
22
- data = body.read
23
- else
24
- data = Fog::Storage.parse_data(body)
25
- options[:content_length] = data[:headers]['Content-Length']
26
- options[:content_type] = data[:headers]['Content-Type']
27
- data = data[:body]
28
- end
29
-
30
- raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
31
- blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
32
- rescue Azure::Core::Http::HTTPError => ex
33
- raise_azure_exception(ex, msg)
34
- end
35
-
36
- if data.nil?
37
- Fog::Logger.debug "Create a block blob #{blob_name} successfully."
38
- else
39
- Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
40
- end
41
- blob
11
+ @blob_client.create_block_blob(container_name, blob_name, body, options)
12
+ rescue Azure::Core::Http::HTTPError => ex
13
+ raise_azure_exception(ex, msg)
42
14
  end
43
15
  end
44
16
 
@@ -23,7 +23,7 @@ module Fog
23
23
  protocol: 'https'
24
24
  }
25
25
  token = @signature_client.generate_service_sas_token(relative_path, params)
26
- uri = @blob_client.generate_uri(relative_path)
26
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
27
27
  "#{uri}?#{token}"
28
28
  end
29
29
  end
@@ -13,16 +13,18 @@ module Fog
13
13
  #
14
14
  # @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
15
15
  #
16
- def get_blob_http_url(container_name, blob_name, expires)
16
+ def get_blob_http_url(container_name, blob_name, expires, options = {})
17
17
  relative_path = "#{container_name}/#{blob_name}"
18
+ relative_path = remove_trailing_periods_from_path_segments(relative_path)
18
19
  params = {
19
20
  service: 'b',
20
21
  resource: 'b',
21
22
  permissions: 'r',
22
- expiry: expires.utc.iso8601
23
+ expiry: expires.utc.iso8601,
24
+ content_disposition: options[:content_disposition]
23
25
  }
24
26
  token = @signature_client.generate_service_sas_token(relative_path, params)
25
- uri = @blob_client.generate_uri(relative_path)
27
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
26
28
  url = "#{uri}?#{token}"
27
29
  url.gsub('https:', 'http:')
28
30
  end
@@ -13,17 +13,19 @@ module Fog
13
13
  #
14
14
  # @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
15
15
  #
16
- def get_blob_https_url(container_name, blob_name, expires)
16
+ def get_blob_https_url(container_name, blob_name, expires, options = {})
17
17
  relative_path = "#{container_name}/#{blob_name}"
18
+ relative_path = remove_trailing_periods_from_path_segments(relative_path)
18
19
  params = {
19
20
  service: 'b',
20
21
  resource: 'b',
21
22
  permissions: 'r',
22
23
  expiry: expires.utc.iso8601,
23
- protocol: 'https'
24
+ protocol: 'https',
25
+ content_disposition: options[:content_disposition]
24
26
  }
25
27
  token = @signature_client.generate_service_sas_token(relative_path, params)
26
- uri = @blob_client.generate_uri(relative_path)
28
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
27
29
  "#{uri}?#{token}"
28
30
  end
29
31
  end
@@ -5,7 +5,7 @@ module Fog
5
5
  class Real
6
6
  # Get a public blob url from Azure blob storage
7
7
  def get_blob_url(container_name, blob_name, options = {})
8
- uri = @blob_client.generate_uri("#{container_name}/#{blob_name}")
8
+ uri = @blob_client.generate_uri("#{container_name}/#{blob_name}", {}, { encode: true })
9
9
 
10
10
  if options[:scheme] == 'http'
11
11
  uri.to_s.gsub('https:', 'http:')
@@ -11,7 +11,7 @@ module Fog
11
11
  #
12
12
  def get_container_url(container_name, options = {})
13
13
  query = { 'comp' => 'list', 'restype' => 'container' }
14
- uri = @blob_client.generate_uri(container_name, query)
14
+ uri = @blob_client.generate_uri(container_name, query, { encode: true })
15
15
 
16
16
  if options[:scheme] == 'http'
17
17
  uri.to_s.gsub('https:', 'http:')
@@ -5,15 +5,15 @@ module Fog
5
5
  class Real
6
6
  # Get a public blob url from Azure blob storage
7
7
  # This is to make this library compatible with CarrierWave.
8
- def get_object_url(container_name, blob_name, expires)
9
- get_blob_https_url(container_name, blob_name, expires)
8
+ def get_object_url(container_name, blob_name, expires, options = {})
9
+ get_blob_https_url(container_name, blob_name, expires, options)
10
10
  end
11
11
  end
12
12
 
13
13
  # This class provides the mock implementation for unit tests.
14
14
  class Mock
15
- def get_object_url(container_name, blob_name, expires)
16
- get_blob_https_url(container_name, blob_name, expires)
15
+ def get_object_url(container_name, blob_name, expires, options = {})
16
+ get_blob_https_url(container_name, blob_name, expires, options)
17
17
  end
18
18
  end
19
19
  end
@@ -23,7 +23,7 @@ module Fog
23
23
  protocol: 'https'
24
24
  }
25
25
  token = @signature_client.generate_service_sas_token(relative_path, params)
26
- uri = @blob_client.generate_uri(relative_path)
26
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
27
27
  "#{uri}?#{token}"
28
28
  end
29
29
  end
@@ -57,7 +57,6 @@ module Fog
57
57
  request :put_blob_pages
58
58
  request :wait_blob_copy_operation_to_finish
59
59
  request :save_page_blob
60
- request :multipart_save_block_blob
61
60
 
62
61
  model_path 'fog/azurerm/models/storage'
63
62
  model :directory
@@ -189,3 +189,7 @@ end
189
189
  def get_subscription_id(id)
190
190
  id.split('/')[2]
191
191
  end
192
+
193
+ def remove_trailing_periods_from_path_segments(path)
194
+ path.split('/').map { |segment| segment.gsub(/\.*$/, '') }.join('/')
195
+ end
@@ -1,5 +1,5 @@
1
1
  module Fog
2
2
  module AzureRM
3
- VERSION = '0.9.0'.freeze
3
+ VERSION = '1.2.0'.freeze
4
4
  end
5
5
  end
@@ -1,7 +1,7 @@
1
1
  require File.expand_path '../../test_helper', __dir__
2
2
 
3
3
  # Test class for Storage Container Model
4
- class TestFile < Minitest::Test
4
+ class TestFile < Minitest::Test # rubocop:disable Metrics/ClassLength
5
5
  def setup
6
6
  @service = Fog::Storage::AzureRM.new(storage_account_credentials)
7
7
  @directory = directory(@service)
@@ -76,10 +76,8 @@ class TestFile < Minitest::Test
76
76
  def test_save_method_with_large_block_blob_success
77
77
  @file.body = 'd' * (32 * 1024 * 1024 + 1) # SINGLE_BLOB_PUT_THRESHOLD is 32 * 1024 * 1024
78
78
 
79
- @service.stub :multipart_save_block_blob, true do
80
- @service.stub :get_blob_properties, @raw_cloud_blob do
81
- assert @file.save
82
- end
79
+ @service.stub :create_block_blob, @raw_cloud_blob do
80
+ assert @file.save
83
81
  end
84
82
  end
85
83
 
@@ -285,6 +283,18 @@ class TestFile < Minitest::Test
285
283
  end
286
284
  end
287
285
 
286
+ def test_url_method_with_content_disposition
287
+ @file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
288
+ assert @file.url(Time.now + 3600, content_disposition: 'attachment'), @blob_https_url
289
+ end
290
+ end
291
+
292
+ def test_url_method_with_response_content_disposition
293
+ @file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
294
+ assert @file.url(Time.now + 3600, { query: { 'response-content-disposition' => 'attachment' } }), @blob_https_url
295
+ end
296
+ end
297
+
288
298
  def test_url_method_without_key_exception
289
299
  assert_raises(ArgumentError) do
290
300
  @file.attributes.delete(:key)
@@ -52,15 +52,6 @@ class TestCreateBlockBlob < Minitest::Test
52
52
  end
53
53
  end
54
54
 
55
- def test_create_block_blob_exceed_max_body_size
56
- data = []
57
- data.stub :size, 64 * 1024 * 1024 + 1 do
58
- assert_raises(ArgumentError) do
59
- @service.create_block_blob('test_container', 'test_blob', data)
60
- end
61
- end
62
- end
63
-
64
55
  def test_create_block_blob_http_exception
65
56
  http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
66
57
  @blob_client.stub :create_block_blob, http_exception do
@@ -24,6 +24,43 @@ class TestGetBlobHttpUrl < Minitest::Test
24
24
  end
25
25
  end
26
26
 
27
+ def test_get_blob_http_url_with_content_disposition
28
+ mock_generate_uri = Minitest::Mock.new
29
+ mock_generate_service_token = Minitest::Mock.new
30
+ url_params = { content_disposition: 'attachment' }
31
+
32
+ mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
33
+ mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
34
+ params[:service] == 'b' &&
35
+ params[:resource] == 'b' &&
36
+ params[:permissions] == 'r' &&
37
+ params[:protocol].nil? &&
38
+ params[:content_disposition] == url_params[:content_disposition]
39
+ end
40
+
41
+ @blob_client.stub :generate_uri, mock_generate_uri do
42
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
43
+ assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
44
+ end
45
+ end
46
+ end
47
+
48
+ def test_get_url_remove_trailing_periods_from_path_segments
49
+ mock_generate_uri = Minitest::Mock.new
50
+ mock_generate_service_token = Minitest::Mock.new
51
+
52
+ mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
53
+ mock_generate_service_token.expect(:call, @token) do |relative_path, _|
54
+ relative_path == '.test0/..test1/...test2'
55
+ end
56
+
57
+ @blob_client.stub :generate_uri, mock_generate_uri do
58
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
59
+ assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
60
+ end
61
+ end
62
+ end
63
+
27
64
  def test_get_blob_http_url_mock
28
65
  assert_equal "#{@url}?#{@token}", @mock_service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600)
29
66
  end
@@ -17,7 +17,13 @@ class TestGetBlobHttpsUrl < Minitest::Test
17
17
  end
18
18
 
19
19
  def test_get_blob_https_url_success
20
- @blob_client.stub :generate_uri, @url do
20
+ mock_generate_uri = Minitest::Mock.new
21
+
22
+ 2.times do
23
+ mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
24
+ end
25
+
26
+ @blob_client.stub :generate_uri, mock_generate_uri do
21
27
  @signature_client.stub :generate_service_sas_token, @token do
22
28
  assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600)
23
29
  assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600)
@@ -25,6 +31,48 @@ class TestGetBlobHttpsUrl < Minitest::Test
25
31
  end
26
32
  end
27
33
 
34
+ def test_get_blob_https_url_with_content_disposition
35
+ mock_generate_uri = Minitest::Mock.new
36
+ mock_generate_service_token = Minitest::Mock.new
37
+ url_params = { content_disposition: 'attachment' }
38
+
39
+ 2.times do
40
+ mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
41
+ mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
42
+ params[:service] == 'b' &&
43
+ params[:resource] == 'b' &&
44
+ params[:permissions] == 'r' &&
45
+ params[:content_disposition] == url_params[:content_disposition]
46
+ end
47
+ end
48
+
49
+ @blob_client.stub :generate_uri, mock_generate_uri do
50
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
51
+ assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
52
+ assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
53
+ end
54
+ end
55
+ end
56
+
57
+ def test_get_url_remove_trailing_periods_from_path_segments
58
+ mock_generate_uri = Minitest::Mock.new
59
+ mock_generate_service_token = Minitest::Mock.new
60
+
61
+ 2.times do
62
+ mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
63
+ mock_generate_service_token.expect(:call, @token) do |relative_path, _|
64
+ relative_path == '.test0/..test1/...test2'
65
+ end
66
+ end
67
+
68
+ @blob_client.stub :generate_uri, mock_generate_uri do
69
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
70
+ assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
71
+ assert_equal "#{@url}?#{@token}", @service.get_object_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
72
+ end
73
+ end
74
+ end
75
+
28
76
  def test_get_blob_https_url_with_domain_success
29
77
  service = Fog::Storage::AzureRM.new(storage_account_credentials_with_domain)
30
78
  blob_client = service.instance_variable_get(:@blob_client)
@@ -12,9 +12,14 @@ class TestGetBlobUrl < Minitest::Test
12
12
  def test_get_blob_url_success
13
13
  service = Fog::Storage::AzureRM.new(storage_account_credentials)
14
14
  blob_client = service.instance_variable_get(:@blob_client)
15
+
16
+ mock_generate_uri = Minitest::Mock.new
15
17
  url = ApiStub::Requests::Storage::File.blob_https_url
18
+ 2.times do
19
+ mock_generate_uri.expect(:call, url, ['test_container/test_blob', {}, { encode: true }])
20
+ end
16
21
 
17
- blob_client.stub :generate_uri, url do
22
+ blob_client.stub :generate_uri, mock_generate_uri do
18
23
  assert_equal url, service.get_blob_url('test_container', 'test_blob')
19
24
 
20
25
  options = { scheme: 'http' }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-fog-azure-rm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shaffan Chaudhry
@@ -15,10 +15,10 @@ authors:
15
15
  - Azeem Sajid
16
16
  - Maham Nazir
17
17
  - Abbas Sheikh
18
- autorequire:
18
+ autorequire:
19
19
  bindir: bin
20
20
  cert_chain: []
21
- date: 2020-09-06 00:00:00.000000000 Z
21
+ date: 2021-09-20 00:00:00.000000000 Z
22
22
  dependencies:
23
23
  - !ruby/object:Gem::Dependency
24
24
  name: codeclimate-test-reporter
@@ -176,7 +176,7 @@ dependencies:
176
176
  version: 0.12.0
177
177
  description: This is a stripped-down fork of fog-azure-rm that enables Azure Blob
178
178
  Storage to be used with CarrierWave and Fog.
179
- email:
179
+ email:
180
180
  executables: []
181
181
  extensions: []
182
182
  extra_rdoc_files:
@@ -249,7 +249,6 @@ files:
249
249
  - lib/fog/azurerm/requests/storage/get_object_url.rb
250
250
  - lib/fog/azurerm/requests/storage/list_blobs.rb
251
251
  - lib/fog/azurerm/requests/storage/list_containers.rb
252
- - lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb
253
252
  - lib/fog/azurerm/requests/storage/put_blob_block.rb
254
253
  - lib/fog/azurerm/requests/storage/put_blob_https_url.rb
255
254
  - lib/fog/azurerm/requests/storage/put_blob_metadata.rb
@@ -305,7 +304,6 @@ files:
305
304
  - test/requests/storage/test_get_container_url.rb
306
305
  - test/requests/storage/test_list_blobs.rb
307
306
  - test/requests/storage/test_list_containers.rb
308
- - test/requests/storage/test_multipart_save_block_blob.rb
309
307
  - test/requests/storage/test_put_blob_block.rb
310
308
  - test/requests/storage/test_put_blob_https_url.rb
311
309
  - test/requests/storage/test_put_blob_metadata.rb
@@ -340,8 +338,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
340
338
  - !ruby/object:Gem::Version
341
339
  version: '0'
342
340
  requirements: []
343
- rubygems_version: 3.0.3
344
- signing_key:
341
+ rubygems_version: 3.1.6
342
+ signing_key:
345
343
  specification_version: 4
346
344
  summary: Module for the 'fog' gem to support Azure Blob Storage with CarrierWave and
347
345
  Fog.
@@ -1,110 +0,0 @@
1
- module Fog
2
- module Storage
3
- class AzureRM
4
- # This class provides the actual implementation for service calls.
5
- class Real
6
- # This class is used to store chunk data for block blob before uploading.
7
- class BlockChunk
8
- attr_reader :id # For debug
9
- attr_reader :block_id
10
- attr_reader :data
11
-
12
- def initialize(id, block_id, data)
13
- @id = id
14
- @block_id = block_id
15
- @data = data
16
- end
17
- end
18
-
19
- # This class is a stream to read chunk data.
20
- class BlockFileStream
21
- attr_reader :blocks
22
-
23
- def initialize(body)
24
- if body.respond_to?(:read)
25
- if body.respond_to?(:rewind)
26
- begin
27
- body.rewind
28
- rescue => ex
29
- Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
30
- end
31
- end
32
- @stream = body
33
- else
34
- @stream = StringIO.new(body)
35
- end
36
- @mutex = Mutex.new
37
- @blocks = []
38
- end
39
-
40
- def read(size)
41
- block_id = Base64.strict_encode64(random_string(32))
42
- data = nil
43
- id = 0
44
- @mutex.synchronize do
45
- data = @stream.read(size)
46
- return nil if data.nil?
47
- @blocks << [block_id]
48
- id = @blocks.size
49
- end
50
- BlockChunk.new(id, block_id, data)
51
- end
52
- end
53
-
54
- def multipart_save_block_blob(container_name, blob_name, body, options)
55
- threads_num = options.delete(:worker_thread_num)
56
- threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
57
-
58
- begin
59
- # Initiate the upload
60
- Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
61
- content_md5 = options.delete(:content_md5)
62
- create_block_blob(container_name, blob_name, nil, options)
63
-
64
- # Uploading parts
65
- Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
66
- iostream = BlockFileStream.new(body)
67
-
68
- threads = []
69
- threads_num.times do |id|
70
- thread = Thread.new do
71
- Fog::Logger.debug "Created upload thread #{id}."
72
- while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
73
- Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
74
- put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
75
- end
76
- Fog::Logger.debug "Upload thread #{id} finished."
77
- end
78
- thread.abort_on_exception = true
79
- threads << thread
80
- end
81
-
82
- threads.each(&:join)
83
- # Complete the upload
84
- options[:content_md5] = content_md5 unless content_md5.nil?
85
- Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
86
- commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
87
- rescue
88
- # Abort the upload & reraise
89
- begin
90
- delete_blob(container_name, blob_name)
91
- rescue => ex
92
- Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
93
- end
94
- raise
95
- end
96
-
97
- Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
98
- true
99
- end
100
- end
101
-
102
- # This class provides the mock implementation for unit tests.
103
- class Mock
104
- def multipart_save_block_blob(*)
105
- true
106
- end
107
- end
108
- end
109
- end
110
- end
@@ -1,105 +0,0 @@
1
- require File.expand_path '../../test_helper', __dir__
2
-
3
- # Storage Blob Class
4
- class TestMultipartSaveBlockBlob < Minitest::Test
5
- # This class posesses the test cases for the requests of saving storage block blob with multiple parts.
6
- def setup
7
- Fog.mock!
8
- @mock_service = Fog::Storage::AzureRM.new(storage_account_credentials)
9
- Fog.unmock!
10
- @mocked_response = mocked_storage_http_error
11
-
12
- @service = Fog::Storage::AzureRM.new(storage_account_credentials)
13
- end
14
-
15
- def test_multipart_save_block_blob_success
16
- body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
17
-
18
- @service.stub :create_block_blob, true do
19
- @service.stub :put_blob_block, true do
20
- @service.stub :commit_blob_blocks, true do
21
- assert @service.multipart_save_block_blob('test_container', 'test_blob', body, worker_thread_num: 6, content_md5: 'oafL1+HS78x65+e39PGIIg==')
22
- end
23
- end
24
- end
25
- end
26
-
27
- def test_multipart_save_block_blob_with_file_handle_success
28
- i = 0
29
- multiple_values = lambda do |*|
30
- i += 1
31
- return 'd' * 4 * 1024 * 1024 if i == 1
32
- return 'd' * (1 * 1024 * 1024 + 1) if i == 2
33
- return nil
34
- end
35
- temp_file = '/dev/null'
36
- File.open(temp_file, 'r') do |file_handle|
37
- file_handle.stub :read, multiple_values do
38
- file_handle.stub :rewind, nil do
39
- @service.stub :create_block_blob, @raw_cloud_blob do
40
- @service.stub :put_blob_block, true do
41
- @service.stub :commit_blob_blocks, true do
42
- assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 0)
43
- end
44
- end
45
- end
46
- end
47
- end
48
- end
49
- end
50
-
51
- def test_multipart_save_block_blob_with_file_handle_do_not_support_rewind_success
52
- i = 0
53
- multiple_values = lambda do |*|
54
- i += 1
55
- return 'd' * 4 * 1024 * 1024 if i == 1
56
- return 'd' * (1 * 1024 * 1024 + 1) if i == 2
57
- return nil
58
- end
59
- temp_file = '/dev/null'
60
- exception = ->(*) { raise 'do not support rewind' }
61
- File.open(temp_file, 'r') do |file_handle|
62
- file_handle.stub :read, multiple_values do
63
- file_handle.stub :rewind, exception do
64
- @service.stub :create_block_blob, @raw_cloud_blob do
65
- @service.stub :put_blob_block, true do
66
- @service.stub :commit_blob_blocks, true do
67
- assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 'invalid')
68
- end
69
- end
70
- end
71
- end
72
- end
73
- end
74
- end
75
-
76
- def test_multipart_save_block_blob_http_exception
77
- body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
78
-
79
- http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
80
- @service.stub :create_block_blob, http_exception do
81
- @service.stub :delete_blob, true do
82
- assert_raises(Azure::Core::Http::HTTPError) do
83
- @service.multipart_save_block_blob('test_container', 'test_blob', body, {})
84
- end
85
- end
86
- end
87
- end
88
-
89
- def test_multipart_save_block_blob_fail_when_delete_blob_http_exception
90
- body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
91
-
92
- http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
93
- @service.stub :create_block_blob, http_exception do
94
- @service.stub :delete_blob, http_exception do
95
- assert_raises(Azure::Core::Http::HTTPError) do
96
- @service.multipart_save_block_blob('test_container', 'test_blob', body, {})
97
- end
98
- end
99
- end
100
- end
101
-
102
- def test_multipart_save_block_blob_mock
103
- assert @mock_service.multipart_save_block_blob('test_container', 'test_blob', 'content', {})
104
- end
105
- end