gitlab-fog-azure-rm 0.9.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9a3e5613e808f27ef9bcd2be6015eac3ffc43e55fe558b2c78b30b97164c3f78
4
- data.tar.gz: d64eb6be36585d864bb5bb7fdcafdd5191a7bda376a4f84edc7e82c9bd972554
3
+ metadata.gz: 9cc556f2028cd1a063691e4556dc1421014476690f700eb31fea81bcac574b41
4
+ data.tar.gz: e6e7d3c355e63cf6a0501d2bc85cbfd994ae54349c950657f6619244fecc9d02
5
5
  SHA512:
6
- metadata.gz: b5b884c2fc10a00a1c385522f2138492427ab50162df609fdfe1398f380c42184bb8a99bf8eae0fd50ff77521796f5371fd40b3222114a5f794368555c8408c7
7
- data.tar.gz: c1a11715408b8c7eb825915e0ecf1dbfb584f831c85dcbb711973f1b2e22ac54de07f3bbbaf59273c525abe9fa6919385fed04c7e85de400ca727d5bbe19c7b2
6
+ metadata.gz: c1ffb76c0ba61d39f675c90612afdf3ae06f0b0835c8f6162a874b73a637e58e4c2c07c0b193ede34ef84601c040920e7a93c2ade997abcdd3c39a64f45d9ff9
7
+ data.tar.gz: bae99d6843a450a5072a9a8e5a2e94de43a6849565fa38c2b288d3c14fbeb70066be960b08c0172a5a9eb61314119d1e78dfb890f90fa4b4af7f7784b028fcdd
data/.rubocop_todo.yml CHANGED
@@ -132,7 +132,6 @@ Naming/RescuedExceptionsVariableName:
132
132
  # SupportedStyles: separated, grouped
133
133
  Style/AccessorGrouping:
134
134
  Exclude:
135
- - 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb'
136
135
  - 'lib/fog/azurerm/requests/storage/save_page_blob.rb'
137
136
  - 'test/test_helper.rb'
138
137
 
data/CHANGELOG.md CHANGED
@@ -1,5 +1,25 @@
1
1
  ## Unreleased
2
2
 
3
+ ## 1.2.0
4
+
5
+ - Support setting returned Content-Disposition in GetBlob requests !25
6
+
7
+ ## 1.1.1
8
+
9
+ - Remove trailing periods per path segment when retrieving blob URLs !23
10
+
11
+ ## 1.1.0
12
+
13
+ - Yanked
14
+
15
+ ## 1.0.1
16
+
17
+ - Fix URL generation for files with UTF-8 characters !21
18
+
19
+ ## 1.0.0
20
+
21
+ - Fix multipart uploads not working and drop multipart_save_block_blob !19
22
+
3
23
  ## 0.9.0
4
24
 
5
25
  - Fix TimeoutError deprecation !17
@@ -240,20 +240,23 @@ module Fog
240
240
  #
241
241
  def url(expires, options = {})
242
242
  requires :key
243
- collection.get_url(key, expires, options)
243
+ collection.get_url(key, expires, normalize_options(options))
244
244
  end
245
245
 
246
246
  private
247
247
 
248
+ def normalize_options(options)
249
+ # AWS S3 and Google Cloud Storage pass response-content-disposition
250
+ # as a query string, while Azure needs the content_disposition parameter
251
+ # to generate a SAS token.
252
+ options[:content_disposition] ||= options.dig(:query, 'response-content-disposition')
253
+ options
254
+ end
255
+
248
256
  # Upload blob
249
257
  def save_blob(options)
250
258
  if options[:blob_type].nil? || options[:blob_type] == 'BlockBlob'
251
- if Fog::Storage.get_body_size(body) <= SINGLE_BLOB_PUT_THRESHOLD
252
- service.create_block_blob(directory.key, key, body, options)
253
- else
254
- service.multipart_save_block_blob(directory.key, key, body, options)
255
- service.get_blob_properties(directory.key, key)
256
- end
259
+ service.create_block_blob(directory.key, key, body, options)
257
260
  else
258
261
  service.save_page_blob(directory.key, key, body, options)
259
262
  service.get_blob_properties(directory.key, key)
@@ -123,10 +123,10 @@ module Fog
123
123
  #
124
124
  # @return [String] A http URL.
125
125
  #
126
- def get_http_url(key, expires, _options = {})
126
+ def get_http_url(key, expires, options = {})
127
127
  requires :directory
128
128
 
129
- service.get_blob_http_url(directory.key, key, expires)
129
+ service.get_blob_http_url(directory.key, key, expires, options)
130
130
  end
131
131
 
132
132
  # Get the https URL of the file(blob) with the given name.
@@ -139,10 +139,10 @@ module Fog
139
139
  #
140
140
  # @return [String] A https URL.
141
141
  #
142
- def get_https_url(key, expires, _options = {})
142
+ def get_https_url(key, expires, options = {})
143
143
  requires :directory
144
144
 
145
- service.get_blob_https_url(directory.key, key, expires)
145
+ service.get_blob_https_url(directory.key, key, expires, options)
146
146
  end
147
147
 
148
148
  # Get the file(blob) without content with the given name.
@@ -8,37 +8,9 @@ module Fog
8
8
  msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
9
9
  Fog::Logger.debug msg
10
10
 
11
- begin
12
- if body.nil?
13
- data = nil
14
- elsif body.respond_to?(:read)
15
- if body.respond_to?(:rewind)
16
- begin
17
- body.rewind
18
- rescue
19
- nil
20
- end
21
- end
22
- data = body.read
23
- else
24
- data = Fog::Storage.parse_data(body)
25
- options[:content_length] = data[:headers]['Content-Length']
26
- options[:content_type] = data[:headers]['Content-Type']
27
- data = data[:body]
28
- end
29
-
30
- raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
31
- blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
32
- rescue Azure::Core::Http::HTTPError => ex
33
- raise_azure_exception(ex, msg)
34
- end
35
-
36
- if data.nil?
37
- Fog::Logger.debug "Create a block blob #{blob_name} successfully."
38
- else
39
- Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
40
- end
41
- blob
11
+ @blob_client.create_block_blob(container_name, blob_name, body, options)
12
+ rescue Azure::Core::Http::HTTPError => ex
13
+ raise_azure_exception(ex, msg)
42
14
  end
43
15
  end
44
16
 
@@ -23,7 +23,7 @@ module Fog
23
23
  protocol: 'https'
24
24
  }
25
25
  token = @signature_client.generate_service_sas_token(relative_path, params)
26
- uri = @blob_client.generate_uri(relative_path)
26
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
27
27
  "#{uri}?#{token}"
28
28
  end
29
29
  end
@@ -13,16 +13,18 @@ module Fog
13
13
  #
14
14
  # @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
15
15
  #
16
- def get_blob_http_url(container_name, blob_name, expires)
16
+ def get_blob_http_url(container_name, blob_name, expires, options = {})
17
17
  relative_path = "#{container_name}/#{blob_name}"
18
+ relative_path = remove_trailing_periods_from_path_segments(relative_path)
18
19
  params = {
19
20
  service: 'b',
20
21
  resource: 'b',
21
22
  permissions: 'r',
22
- expiry: expires.utc.iso8601
23
+ expiry: expires.utc.iso8601,
24
+ content_disposition: options[:content_disposition]
23
25
  }
24
26
  token = @signature_client.generate_service_sas_token(relative_path, params)
25
- uri = @blob_client.generate_uri(relative_path)
27
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
26
28
  url = "#{uri}?#{token}"
27
29
  url.gsub('https:', 'http:')
28
30
  end
@@ -13,17 +13,19 @@ module Fog
13
13
  #
14
14
  # @see https://msdn.microsoft.com/en-us/library/azure/mt584140.aspx
15
15
  #
16
- def get_blob_https_url(container_name, blob_name, expires)
16
+ def get_blob_https_url(container_name, blob_name, expires, options = {})
17
17
  relative_path = "#{container_name}/#{blob_name}"
18
+ relative_path = remove_trailing_periods_from_path_segments(relative_path)
18
19
  params = {
19
20
  service: 'b',
20
21
  resource: 'b',
21
22
  permissions: 'r',
22
23
  expiry: expires.utc.iso8601,
23
- protocol: 'https'
24
+ protocol: 'https',
25
+ content_disposition: options[:content_disposition]
24
26
  }
25
27
  token = @signature_client.generate_service_sas_token(relative_path, params)
26
- uri = @blob_client.generate_uri(relative_path)
28
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
27
29
  "#{uri}?#{token}"
28
30
  end
29
31
  end
@@ -5,7 +5,7 @@ module Fog
5
5
  class Real
6
6
  # Get a public blob url from Azure blob storage
7
7
  def get_blob_url(container_name, blob_name, options = {})
8
- uri = @blob_client.generate_uri("#{container_name}/#{blob_name}")
8
+ uri = @blob_client.generate_uri("#{container_name}/#{blob_name}", {}, { encode: true })
9
9
 
10
10
  if options[:scheme] == 'http'
11
11
  uri.to_s.gsub('https:', 'http:')
@@ -11,7 +11,7 @@ module Fog
11
11
  #
12
12
  def get_container_url(container_name, options = {})
13
13
  query = { 'comp' => 'list', 'restype' => 'container' }
14
- uri = @blob_client.generate_uri(container_name, query)
14
+ uri = @blob_client.generate_uri(container_name, query, { encode: true })
15
15
 
16
16
  if options[:scheme] == 'http'
17
17
  uri.to_s.gsub('https:', 'http:')
@@ -5,15 +5,15 @@ module Fog
5
5
  class Real
6
6
  # Get a public blob url from Azure blob storage
7
7
  # This is to make this library compatible with CarrierWave.
8
- def get_object_url(container_name, blob_name, expires)
9
- get_blob_https_url(container_name, blob_name, expires)
8
+ def get_object_url(container_name, blob_name, expires, options = {})
9
+ get_blob_https_url(container_name, blob_name, expires, options)
10
10
  end
11
11
  end
12
12
 
13
13
  # This class provides the mock implementation for unit tests.
14
14
  class Mock
15
- def get_object_url(container_name, blob_name, expires)
16
- get_blob_https_url(container_name, blob_name, expires)
15
+ def get_object_url(container_name, blob_name, expires, options = {})
16
+ get_blob_https_url(container_name, blob_name, expires, options)
17
17
  end
18
18
  end
19
19
  end
@@ -23,7 +23,7 @@ module Fog
23
23
  protocol: 'https'
24
24
  }
25
25
  token = @signature_client.generate_service_sas_token(relative_path, params)
26
- uri = @blob_client.generate_uri(relative_path)
26
+ uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
27
27
  "#{uri}?#{token}"
28
28
  end
29
29
  end
@@ -57,7 +57,6 @@ module Fog
57
57
  request :put_blob_pages
58
58
  request :wait_blob_copy_operation_to_finish
59
59
  request :save_page_blob
60
- request :multipart_save_block_blob
61
60
 
62
61
  model_path 'fog/azurerm/models/storage'
63
62
  model :directory
@@ -189,3 +189,7 @@ end
189
189
  def get_subscription_id(id)
190
190
  id.split('/')[2]
191
191
  end
192
+
193
+ def remove_trailing_periods_from_path_segments(path)
194
+ path.split('/').map { |segment| segment.gsub(/\.*$/, '') }.join('/')
195
+ end
@@ -1,5 +1,5 @@
1
1
  module Fog
2
2
  module AzureRM
3
- VERSION = '0.9.0'.freeze
3
+ VERSION = '1.2.0'.freeze
4
4
  end
5
5
  end
@@ -1,7 +1,7 @@
1
1
  require File.expand_path '../../test_helper', __dir__
2
2
 
3
3
  # Test class for Storage Container Model
4
- class TestFile < Minitest::Test
4
+ class TestFile < Minitest::Test # rubocop:disable Metrics/ClassLength
5
5
  def setup
6
6
  @service = Fog::Storage::AzureRM.new(storage_account_credentials)
7
7
  @directory = directory(@service)
@@ -76,10 +76,8 @@ class TestFile < Minitest::Test
76
76
  def test_save_method_with_large_block_blob_success
77
77
  @file.body = 'd' * (32 * 1024 * 1024 + 1) # SINGLE_BLOB_PUT_THRESHOLD is 32 * 1024 * 1024
78
78
 
79
- @service.stub :multipart_save_block_blob, true do
80
- @service.stub :get_blob_properties, @raw_cloud_blob do
81
- assert @file.save
82
- end
79
+ @service.stub :create_block_blob, @raw_cloud_blob do
80
+ assert @file.save
83
81
  end
84
82
  end
85
83
 
@@ -285,6 +283,18 @@ class TestFile < Minitest::Test
285
283
  end
286
284
  end
287
285
 
286
+ def test_url_method_with_content_disposition
287
+ @file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
288
+ assert @file.url(Time.now + 3600, content_disposition: 'attachment'), @blob_https_url
289
+ end
290
+ end
291
+
292
+ def test_url_method_with_response_content_disposition
293
+ @file.collection.stub :get_url, @blob_https_url, { content_disposition: 'attachment' } do
294
+ assert @file.url(Time.now + 3600, { query: { 'response-content-disposition' => 'attachment' } }), @blob_https_url
295
+ end
296
+ end
297
+
288
298
  def test_url_method_without_key_exception
289
299
  assert_raises(ArgumentError) do
290
300
  @file.attributes.delete(:key)
@@ -52,15 +52,6 @@ class TestCreateBlockBlob < Minitest::Test
52
52
  end
53
53
  end
54
54
 
55
- def test_create_block_blob_exceed_max_body_size
56
- data = []
57
- data.stub :size, 64 * 1024 * 1024 + 1 do
58
- assert_raises(ArgumentError) do
59
- @service.create_block_blob('test_container', 'test_blob', data)
60
- end
61
- end
62
- end
63
-
64
55
  def test_create_block_blob_http_exception
65
56
  http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
66
57
  @blob_client.stub :create_block_blob, http_exception do
@@ -24,6 +24,43 @@ class TestGetBlobHttpUrl < Minitest::Test
24
24
  end
25
25
  end
26
26
 
27
+ def test_get_blob_http_url_with_content_disposition
28
+ mock_generate_uri = Minitest::Mock.new
29
+ mock_generate_service_token = Minitest::Mock.new
30
+ url_params = { content_disposition: 'attachment' }
31
+
32
+ mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
33
+ mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
34
+ params[:service] == 'b' &&
35
+ params[:resource] == 'b' &&
36
+ params[:permissions] == 'r' &&
37
+ params[:protocol].nil? &&
38
+ params[:content_disposition] == url_params[:content_disposition]
39
+ end
40
+
41
+ @blob_client.stub :generate_uri, mock_generate_uri do
42
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
43
+ assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
44
+ end
45
+ end
46
+ end
47
+
48
+ def test_get_url_remove_trailing_periods_from_path_segments
49
+ mock_generate_uri = Minitest::Mock.new
50
+ mock_generate_service_token = Minitest::Mock.new
51
+
52
+ mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
53
+ mock_generate_service_token.expect(:call, @token) do |relative_path, _|
54
+ relative_path == '.test0/..test1/...test2'
55
+ end
56
+
57
+ @blob_client.stub :generate_uri, mock_generate_uri do
58
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
59
+ assert_equal "#{@url}?#{@token}", @service.get_blob_http_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
60
+ end
61
+ end
62
+ end
63
+
27
64
  def test_get_blob_http_url_mock
28
65
  assert_equal "#{@url}?#{@token}", @mock_service.get_blob_http_url('test_container', 'test_blob', Time.now.utc + 3600)
29
66
  end
@@ -17,7 +17,13 @@ class TestGetBlobHttpsUrl < Minitest::Test
17
17
  end
18
18
 
19
19
  def test_get_blob_https_url_success
20
- @blob_client.stub :generate_uri, @url do
20
+ mock_generate_uri = Minitest::Mock.new
21
+
22
+ 2.times do
23
+ mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
24
+ end
25
+
26
+ @blob_client.stub :generate_uri, mock_generate_uri do
21
27
  @signature_client.stub :generate_service_sas_token, @token do
22
28
  assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600)
23
29
  assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600)
@@ -25,6 +31,48 @@ class TestGetBlobHttpsUrl < Minitest::Test
25
31
  end
26
32
  end
27
33
 
34
+ def test_get_blob_https_url_with_content_disposition
35
+ mock_generate_uri = Minitest::Mock.new
36
+ mock_generate_service_token = Minitest::Mock.new
37
+ url_params = { content_disposition: 'attachment' }
38
+
39
+ 2.times do
40
+ mock_generate_uri.expect(:call, @url, ['test_container/test_blob', {}, { encode: true }])
41
+ mock_generate_service_token.expect(:call, @token) do |_relative_path, params|
42
+ params[:service] == 'b' &&
43
+ params[:resource] == 'b' &&
44
+ params[:permissions] == 'r' &&
45
+ params[:content_disposition] == url_params[:content_disposition]
46
+ end
47
+ end
48
+
49
+ @blob_client.stub :generate_uri, mock_generate_uri do
50
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
51
+ assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
52
+ assert_equal "#{@url}?#{@token}", @service.get_object_url('test_container', 'test_blob', Time.now.utc + 3600, url_params)
53
+ end
54
+ end
55
+ end
56
+
57
+ def test_get_url_remove_trailing_periods_from_path_segments
58
+ mock_generate_uri = Minitest::Mock.new
59
+ mock_generate_service_token = Minitest::Mock.new
60
+
61
+ 2.times do
62
+ mock_generate_uri.expect(:call, @url, ['.test0/..test1/...test2', {}, { encode: true }])
63
+ mock_generate_service_token.expect(:call, @token) do |relative_path, _|
64
+ relative_path == '.test0/..test1/...test2'
65
+ end
66
+ end
67
+
68
+ @blob_client.stub :generate_uri, mock_generate_uri do
69
+ @signature_client.stub :generate_service_sas_token, mock_generate_service_token do
70
+ assert_equal "#{@url}?#{@token}", @service.get_blob_https_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
71
+ assert_equal "#{@url}?#{@token}", @service.get_object_url('.test0.', '..test1../...test2...', Time.now.utc + 3600)
72
+ end
73
+ end
74
+ end
75
+
28
76
  def test_get_blob_https_url_with_domain_success
29
77
  service = Fog::Storage::AzureRM.new(storage_account_credentials_with_domain)
30
78
  blob_client = service.instance_variable_get(:@blob_client)
@@ -12,9 +12,14 @@ class TestGetBlobUrl < Minitest::Test
12
12
  def test_get_blob_url_success
13
13
  service = Fog::Storage::AzureRM.new(storage_account_credentials)
14
14
  blob_client = service.instance_variable_get(:@blob_client)
15
+
16
+ mock_generate_uri = Minitest::Mock.new
15
17
  url = ApiStub::Requests::Storage::File.blob_https_url
18
+ 2.times do
19
+ mock_generate_uri.expect(:call, url, ['test_container/test_blob', {}, { encode: true }])
20
+ end
16
21
 
17
- blob_client.stub :generate_uri, url do
22
+ blob_client.stub :generate_uri, mock_generate_uri do
18
23
  assert_equal url, service.get_blob_url('test_container', 'test_blob')
19
24
 
20
25
  options = { scheme: 'http' }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-fog-azure-rm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shaffan Chaudhry
@@ -15,10 +15,10 @@ authors:
15
15
  - Azeem Sajid
16
16
  - Maham Nazir
17
17
  - Abbas Sheikh
18
- autorequire:
18
+ autorequire:
19
19
  bindir: bin
20
20
  cert_chain: []
21
- date: 2020-09-06 00:00:00.000000000 Z
21
+ date: 2021-09-20 00:00:00.000000000 Z
22
22
  dependencies:
23
23
  - !ruby/object:Gem::Dependency
24
24
  name: codeclimate-test-reporter
@@ -176,7 +176,7 @@ dependencies:
176
176
  version: 0.12.0
177
177
  description: This is a stripped-down fork of fog-azure-rm that enables Azure Blob
178
178
  Storage to be used with CarrierWave and Fog.
179
- email:
179
+ email:
180
180
  executables: []
181
181
  extensions: []
182
182
  extra_rdoc_files:
@@ -249,7 +249,6 @@ files:
249
249
  - lib/fog/azurerm/requests/storage/get_object_url.rb
250
250
  - lib/fog/azurerm/requests/storage/list_blobs.rb
251
251
  - lib/fog/azurerm/requests/storage/list_containers.rb
252
- - lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb
253
252
  - lib/fog/azurerm/requests/storage/put_blob_block.rb
254
253
  - lib/fog/azurerm/requests/storage/put_blob_https_url.rb
255
254
  - lib/fog/azurerm/requests/storage/put_blob_metadata.rb
@@ -305,7 +304,6 @@ files:
305
304
  - test/requests/storage/test_get_container_url.rb
306
305
  - test/requests/storage/test_list_blobs.rb
307
306
  - test/requests/storage/test_list_containers.rb
308
- - test/requests/storage/test_multipart_save_block_blob.rb
309
307
  - test/requests/storage/test_put_blob_block.rb
310
308
  - test/requests/storage/test_put_blob_https_url.rb
311
309
  - test/requests/storage/test_put_blob_metadata.rb
@@ -340,8 +338,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
340
338
  - !ruby/object:Gem::Version
341
339
  version: '0'
342
340
  requirements: []
343
- rubygems_version: 3.0.3
344
- signing_key:
341
+ rubygems_version: 3.1.6
342
+ signing_key:
345
343
  specification_version: 4
346
344
  summary: Module for the 'fog' gem to support Azure Blob Storage with CarrierWave and
347
345
  Fog.
@@ -1,110 +0,0 @@
1
- module Fog
2
- module Storage
3
- class AzureRM
4
- # This class provides the actual implementation for service calls.
5
- class Real
6
- # This class is used to store chunk data for block blob before uploading.
7
- class BlockChunk
8
- attr_reader :id # For debug
9
- attr_reader :block_id
10
- attr_reader :data
11
-
12
- def initialize(id, block_id, data)
13
- @id = id
14
- @block_id = block_id
15
- @data = data
16
- end
17
- end
18
-
19
- # This class is a stream to read chunk data.
20
- class BlockFileStream
21
- attr_reader :blocks
22
-
23
- def initialize(body)
24
- if body.respond_to?(:read)
25
- if body.respond_to?(:rewind)
26
- begin
27
- body.rewind
28
- rescue => ex
29
- Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
30
- end
31
- end
32
- @stream = body
33
- else
34
- @stream = StringIO.new(body)
35
- end
36
- @mutex = Mutex.new
37
- @blocks = []
38
- end
39
-
40
- def read(size)
41
- block_id = Base64.strict_encode64(random_string(32))
42
- data = nil
43
- id = 0
44
- @mutex.synchronize do
45
- data = @stream.read(size)
46
- return nil if data.nil?
47
- @blocks << [block_id]
48
- id = @blocks.size
49
- end
50
- BlockChunk.new(id, block_id, data)
51
- end
52
- end
53
-
54
- def multipart_save_block_blob(container_name, blob_name, body, options)
55
- threads_num = options.delete(:worker_thread_num)
56
- threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
57
-
58
- begin
59
- # Initiate the upload
60
- Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
61
- content_md5 = options.delete(:content_md5)
62
- create_block_blob(container_name, blob_name, nil, options)
63
-
64
- # Uploading parts
65
- Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
66
- iostream = BlockFileStream.new(body)
67
-
68
- threads = []
69
- threads_num.times do |id|
70
- thread = Thread.new do
71
- Fog::Logger.debug "Created upload thread #{id}."
72
- while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
73
- Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
74
- put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
75
- end
76
- Fog::Logger.debug "Upload thread #{id} finished."
77
- end
78
- thread.abort_on_exception = true
79
- threads << thread
80
- end
81
-
82
- threads.each(&:join)
83
- # Complete the upload
84
- options[:content_md5] = content_md5 unless content_md5.nil?
85
- Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
86
- commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
87
- rescue
88
- # Abort the upload & reraise
89
- begin
90
- delete_blob(container_name, blob_name)
91
- rescue => ex
92
- Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
93
- end
94
- raise
95
- end
96
-
97
- Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
98
- true
99
- end
100
- end
101
-
102
- # This class provides the mock implementation for unit tests.
103
- class Mock
104
- def multipart_save_block_blob(*)
105
- true
106
- end
107
- end
108
- end
109
- end
110
- end
@@ -1,105 +0,0 @@
1
- require File.expand_path '../../test_helper', __dir__
2
-
3
- # Storage Blob Class
4
- class TestMultipartSaveBlockBlob < Minitest::Test
5
- # This class posesses the test cases for the requests of saving storage block blob with multiple parts.
6
- def setup
7
- Fog.mock!
8
- @mock_service = Fog::Storage::AzureRM.new(storage_account_credentials)
9
- Fog.unmock!
10
- @mocked_response = mocked_storage_http_error
11
-
12
- @service = Fog::Storage::AzureRM.new(storage_account_credentials)
13
- end
14
-
15
- def test_multipart_save_block_blob_success
16
- body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
17
-
18
- @service.stub :create_block_blob, true do
19
- @service.stub :put_blob_block, true do
20
- @service.stub :commit_blob_blocks, true do
21
- assert @service.multipart_save_block_blob('test_container', 'test_blob', body, worker_thread_num: 6, content_md5: 'oafL1+HS78x65+e39PGIIg==')
22
- end
23
- end
24
- end
25
- end
26
-
27
- def test_multipart_save_block_blob_with_file_handle_success
28
- i = 0
29
- multiple_values = lambda do |*|
30
- i += 1
31
- return 'd' * 4 * 1024 * 1024 if i == 1
32
- return 'd' * (1 * 1024 * 1024 + 1) if i == 2
33
- return nil
34
- end
35
- temp_file = '/dev/null'
36
- File.open(temp_file, 'r') do |file_handle|
37
- file_handle.stub :read, multiple_values do
38
- file_handle.stub :rewind, nil do
39
- @service.stub :create_block_blob, @raw_cloud_blob do
40
- @service.stub :put_blob_block, true do
41
- @service.stub :commit_blob_blocks, true do
42
- assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 0)
43
- end
44
- end
45
- end
46
- end
47
- end
48
- end
49
- end
50
-
51
- def test_multipart_save_block_blob_with_file_handle_do_not_support_rewind_success
52
- i = 0
53
- multiple_values = lambda do |*|
54
- i += 1
55
- return 'd' * 4 * 1024 * 1024 if i == 1
56
- return 'd' * (1 * 1024 * 1024 + 1) if i == 2
57
- return nil
58
- end
59
- temp_file = '/dev/null'
60
- exception = ->(*) { raise 'do not support rewind' }
61
- File.open(temp_file, 'r') do |file_handle|
62
- file_handle.stub :read, multiple_values do
63
- file_handle.stub :rewind, exception do
64
- @service.stub :create_block_blob, @raw_cloud_blob do
65
- @service.stub :put_blob_block, true do
66
- @service.stub :commit_blob_blocks, true do
67
- assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 'invalid')
68
- end
69
- end
70
- end
71
- end
72
- end
73
- end
74
- end
75
-
76
- def test_multipart_save_block_blob_http_exception
77
- body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
78
-
79
- http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
80
- @service.stub :create_block_blob, http_exception do
81
- @service.stub :delete_blob, true do
82
- assert_raises(Azure::Core::Http::HTTPError) do
83
- @service.multipart_save_block_blob('test_container', 'test_blob', body, {})
84
- end
85
- end
86
- end
87
- end
88
-
89
- def test_multipart_save_block_blob_fail_when_delete_blob_http_exception
90
- body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
91
-
92
- http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
93
- @service.stub :create_block_blob, http_exception do
94
- @service.stub :delete_blob, http_exception do
95
- assert_raises(Azure::Core::Http::HTTPError) do
96
- @service.multipart_save_block_blob('test_container', 'test_blob', body, {})
97
- end
98
- end
99
- end
100
- end
101
-
102
- def test_multipart_save_block_blob_mock
103
- assert @mock_service.multipart_save_block_blob('test_container', 'test_blob', 'content', {})
104
- end
105
- end