gitlab-fog-azure-rm 0.9.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +0 -1
- data/CHANGELOG.md +4 -0
- data/lib/fog/azurerm/models/storage/file.rb +1 -6
- data/lib/fog/azurerm/requests/storage/create_block_blob.rb +3 -31
- data/lib/fog/azurerm/storage.rb +0 -1
- data/lib/fog/azurerm/version.rb +1 -1
- data/test/models/storage/test_file.rb +2 -4
- data/test/requests/storage/test_create_block_blob.rb +0 -9
- metadata +2 -4
- data/lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb +0 -110
- data/test/requests/storage/test_multipart_save_block_blob.rb +0 -105
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 177b1f767e814e126cce9905dc11bd5b4384daaafb7292ae379a9f05c435b4ea
|
4
|
+
data.tar.gz: 19d21cd53ffd3a246e195009c4f709df9e4e955ae5c2dace73e4436f82f5ac4d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aa5d061efb46aee21a8c74d949f7ba5c1632019ccda79799cfe70775f40b6a3448d0f4c01fc12b0c82863419c74a673a68992ca9582317a1df45c42d3ffd5dad
|
7
|
+
data.tar.gz: 8d45eb802da36f53c8aa7a67a49f609de9e39be351af967237166da94c52bda5e474cbb7633200f18bc9386341591d8880df9cda8cd7e5a055beb6f0ad09e13c
|
data/.rubocop_todo.yml
CHANGED
@@ -132,7 +132,6 @@ Naming/RescuedExceptionsVariableName:
|
|
132
132
|
# SupportedStyles: separated, grouped
|
133
133
|
Style/AccessorGrouping:
|
134
134
|
Exclude:
|
135
|
-
- 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb'
|
136
135
|
- 'lib/fog/azurerm/requests/storage/save_page_blob.rb'
|
137
136
|
- 'test/test_helper.rb'
|
138
137
|
|
data/CHANGELOG.md
CHANGED
@@ -248,12 +248,7 @@ module Fog
|
|
248
248
|
# Upload blob
|
249
249
|
def save_blob(options)
|
250
250
|
if options[:blob_type].nil? || options[:blob_type] == 'BlockBlob'
|
251
|
-
|
252
|
-
service.create_block_blob(directory.key, key, body, options)
|
253
|
-
else
|
254
|
-
service.multipart_save_block_blob(directory.key, key, body, options)
|
255
|
-
service.get_blob_properties(directory.key, key)
|
256
|
-
end
|
251
|
+
service.create_block_blob(directory.key, key, body, options)
|
257
252
|
else
|
258
253
|
service.save_page_blob(directory.key, key, body, options)
|
259
254
|
service.get_blob_properties(directory.key, key)
|
@@ -8,37 +8,9 @@ module Fog
|
|
8
8
|
msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
|
9
9
|
Fog::Logger.debug msg
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
elsif body.respond_to?(:read)
|
15
|
-
if body.respond_to?(:rewind)
|
16
|
-
begin
|
17
|
-
body.rewind
|
18
|
-
rescue
|
19
|
-
nil
|
20
|
-
end
|
21
|
-
end
|
22
|
-
data = body.read
|
23
|
-
else
|
24
|
-
data = Fog::Storage.parse_data(body)
|
25
|
-
options[:content_length] = data[:headers]['Content-Length']
|
26
|
-
options[:content_type] = data[:headers]['Content-Type']
|
27
|
-
data = data[:body]
|
28
|
-
end
|
29
|
-
|
30
|
-
raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
|
31
|
-
blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
|
32
|
-
rescue Azure::Core::Http::HTTPError => ex
|
33
|
-
raise_azure_exception(ex, msg)
|
34
|
-
end
|
35
|
-
|
36
|
-
if data.nil?
|
37
|
-
Fog::Logger.debug "Create a block blob #{blob_name} successfully."
|
38
|
-
else
|
39
|
-
Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
|
40
|
-
end
|
41
|
-
blob
|
11
|
+
@blob_client.create_block_blob(container_name, blob_name, body, options)
|
12
|
+
rescue Azure::Core::Http::HTTPError => ex
|
13
|
+
raise_azure_exception(ex, msg)
|
42
14
|
end
|
43
15
|
end
|
44
16
|
|
data/lib/fog/azurerm/storage.rb
CHANGED
data/lib/fog/azurerm/version.rb
CHANGED
@@ -76,10 +76,8 @@ class TestFile < Minitest::Test
|
|
76
76
|
def test_save_method_with_large_block_blob_success
|
77
77
|
@file.body = 'd' * (32 * 1024 * 1024 + 1) # SINGLE_BLOB_PUT_THRESHOLD is 32 * 1024 * 1024
|
78
78
|
|
79
|
-
@service.stub :
|
80
|
-
@
|
81
|
-
assert @file.save
|
82
|
-
end
|
79
|
+
@service.stub :create_block_blob, @raw_cloud_blob do
|
80
|
+
assert @file.save
|
83
81
|
end
|
84
82
|
end
|
85
83
|
|
@@ -52,15 +52,6 @@ class TestCreateBlockBlob < Minitest::Test
|
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
-
def test_create_block_blob_exceed_max_body_size
|
56
|
-
data = []
|
57
|
-
data.stub :size, 64 * 1024 * 1024 + 1 do
|
58
|
-
assert_raises(ArgumentError) do
|
59
|
-
@service.create_block_blob('test_container', 'test_blob', data)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
55
|
def test_create_block_blob_http_exception
|
65
56
|
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
66
57
|
@blob_client.stub :create_block_blob, http_exception do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-fog-azure-rm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shaffan Chaudhry
|
@@ -18,7 +18,7 @@ authors:
|
|
18
18
|
autorequire:
|
19
19
|
bindir: bin
|
20
20
|
cert_chain: []
|
21
|
-
date: 2020-
|
21
|
+
date: 2020-10-04 00:00:00.000000000 Z
|
22
22
|
dependencies:
|
23
23
|
- !ruby/object:Gem::Dependency
|
24
24
|
name: codeclimate-test-reporter
|
@@ -249,7 +249,6 @@ files:
|
|
249
249
|
- lib/fog/azurerm/requests/storage/get_object_url.rb
|
250
250
|
- lib/fog/azurerm/requests/storage/list_blobs.rb
|
251
251
|
- lib/fog/azurerm/requests/storage/list_containers.rb
|
252
|
-
- lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb
|
253
252
|
- lib/fog/azurerm/requests/storage/put_blob_block.rb
|
254
253
|
- lib/fog/azurerm/requests/storage/put_blob_https_url.rb
|
255
254
|
- lib/fog/azurerm/requests/storage/put_blob_metadata.rb
|
@@ -305,7 +304,6 @@ files:
|
|
305
304
|
- test/requests/storage/test_get_container_url.rb
|
306
305
|
- test/requests/storage/test_list_blobs.rb
|
307
306
|
- test/requests/storage/test_list_containers.rb
|
308
|
-
- test/requests/storage/test_multipart_save_block_blob.rb
|
309
307
|
- test/requests/storage/test_put_blob_block.rb
|
310
308
|
- test/requests/storage/test_put_blob_https_url.rb
|
311
309
|
- test/requests/storage/test_put_blob_metadata.rb
|
@@ -1,110 +0,0 @@
|
|
1
|
-
module Fog
|
2
|
-
module Storage
|
3
|
-
class AzureRM
|
4
|
-
# This class provides the actual implementation for service calls.
|
5
|
-
class Real
|
6
|
-
# This class is used to store chunk data for block blob before uploading.
|
7
|
-
class BlockChunk
|
8
|
-
attr_reader :id # For debug
|
9
|
-
attr_reader :block_id
|
10
|
-
attr_reader :data
|
11
|
-
|
12
|
-
def initialize(id, block_id, data)
|
13
|
-
@id = id
|
14
|
-
@block_id = block_id
|
15
|
-
@data = data
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
# This class is a stream to read chunk data.
|
20
|
-
class BlockFileStream
|
21
|
-
attr_reader :blocks
|
22
|
-
|
23
|
-
def initialize(body)
|
24
|
-
if body.respond_to?(:read)
|
25
|
-
if body.respond_to?(:rewind)
|
26
|
-
begin
|
27
|
-
body.rewind
|
28
|
-
rescue => ex
|
29
|
-
Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
|
30
|
-
end
|
31
|
-
end
|
32
|
-
@stream = body
|
33
|
-
else
|
34
|
-
@stream = StringIO.new(body)
|
35
|
-
end
|
36
|
-
@mutex = Mutex.new
|
37
|
-
@blocks = []
|
38
|
-
end
|
39
|
-
|
40
|
-
def read(size)
|
41
|
-
block_id = Base64.strict_encode64(random_string(32))
|
42
|
-
data = nil
|
43
|
-
id = 0
|
44
|
-
@mutex.synchronize do
|
45
|
-
data = @stream.read(size)
|
46
|
-
return nil if data.nil?
|
47
|
-
@blocks << [block_id]
|
48
|
-
id = @blocks.size
|
49
|
-
end
|
50
|
-
BlockChunk.new(id, block_id, data)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def multipart_save_block_blob(container_name, blob_name, body, options)
|
55
|
-
threads_num = options.delete(:worker_thread_num)
|
56
|
-
threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
|
57
|
-
|
58
|
-
begin
|
59
|
-
# Initiate the upload
|
60
|
-
Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
|
61
|
-
content_md5 = options.delete(:content_md5)
|
62
|
-
create_block_blob(container_name, blob_name, nil, options)
|
63
|
-
|
64
|
-
# Uploading parts
|
65
|
-
Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
|
66
|
-
iostream = BlockFileStream.new(body)
|
67
|
-
|
68
|
-
threads = []
|
69
|
-
threads_num.times do |id|
|
70
|
-
thread = Thread.new do
|
71
|
-
Fog::Logger.debug "Created upload thread #{id}."
|
72
|
-
while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
|
73
|
-
Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
|
74
|
-
put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
|
75
|
-
end
|
76
|
-
Fog::Logger.debug "Upload thread #{id} finished."
|
77
|
-
end
|
78
|
-
thread.abort_on_exception = true
|
79
|
-
threads << thread
|
80
|
-
end
|
81
|
-
|
82
|
-
threads.each(&:join)
|
83
|
-
# Complete the upload
|
84
|
-
options[:content_md5] = content_md5 unless content_md5.nil?
|
85
|
-
Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
|
86
|
-
commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
|
87
|
-
rescue
|
88
|
-
# Abort the upload & reraise
|
89
|
-
begin
|
90
|
-
delete_blob(container_name, blob_name)
|
91
|
-
rescue => ex
|
92
|
-
Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
|
93
|
-
end
|
94
|
-
raise
|
95
|
-
end
|
96
|
-
|
97
|
-
Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
|
98
|
-
true
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
# This class provides the mock implementation for unit tests.
|
103
|
-
class Mock
|
104
|
-
def multipart_save_block_blob(*)
|
105
|
-
true
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
@@ -1,105 +0,0 @@
|
|
1
|
-
require File.expand_path '../../test_helper', __dir__
|
2
|
-
|
3
|
-
# Storage Blob Class
|
4
|
-
class TestMultipartSaveBlockBlob < Minitest::Test
|
5
|
-
# This class posesses the test cases for the requests of saving storage block blob with multiple parts.
|
6
|
-
def setup
|
7
|
-
Fog.mock!
|
8
|
-
@mock_service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
9
|
-
Fog.unmock!
|
10
|
-
@mocked_response = mocked_storage_http_error
|
11
|
-
|
12
|
-
@service = Fog::Storage::AzureRM.new(storage_account_credentials)
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_multipart_save_block_blob_success
|
16
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
17
|
-
|
18
|
-
@service.stub :create_block_blob, true do
|
19
|
-
@service.stub :put_blob_block, true do
|
20
|
-
@service.stub :commit_blob_blocks, true do
|
21
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', body, worker_thread_num: 6, content_md5: 'oafL1+HS78x65+e39PGIIg==')
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def test_multipart_save_block_blob_with_file_handle_success
|
28
|
-
i = 0
|
29
|
-
multiple_values = lambda do |*|
|
30
|
-
i += 1
|
31
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
32
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
33
|
-
return nil
|
34
|
-
end
|
35
|
-
temp_file = '/dev/null'
|
36
|
-
File.open(temp_file, 'r') do |file_handle|
|
37
|
-
file_handle.stub :read, multiple_values do
|
38
|
-
file_handle.stub :rewind, nil do
|
39
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
40
|
-
@service.stub :put_blob_block, true do
|
41
|
-
@service.stub :commit_blob_blocks, true do
|
42
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 0)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def test_multipart_save_block_blob_with_file_handle_do_not_support_rewind_success
|
52
|
-
i = 0
|
53
|
-
multiple_values = lambda do |*|
|
54
|
-
i += 1
|
55
|
-
return 'd' * 4 * 1024 * 1024 if i == 1
|
56
|
-
return 'd' * (1 * 1024 * 1024 + 1) if i == 2
|
57
|
-
return nil
|
58
|
-
end
|
59
|
-
temp_file = '/dev/null'
|
60
|
-
exception = ->(*) { raise 'do not support rewind' }
|
61
|
-
File.open(temp_file, 'r') do |file_handle|
|
62
|
-
file_handle.stub :read, multiple_values do
|
63
|
-
file_handle.stub :rewind, exception do
|
64
|
-
@service.stub :create_block_blob, @raw_cloud_blob do
|
65
|
-
@service.stub :put_blob_block, true do
|
66
|
-
@service.stub :commit_blob_blocks, true do
|
67
|
-
assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 'invalid')
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def test_multipart_save_block_blob_http_exception
|
77
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
78
|
-
|
79
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
80
|
-
@service.stub :create_block_blob, http_exception do
|
81
|
-
@service.stub :delete_blob, true do
|
82
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
83
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
84
|
-
end
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
def test_multipart_save_block_blob_fail_when_delete_blob_http_exception
|
90
|
-
body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
|
91
|
-
|
92
|
-
http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
|
93
|
-
@service.stub :create_block_blob, http_exception do
|
94
|
-
@service.stub :delete_blob, http_exception do
|
95
|
-
assert_raises(Azure::Core::Http::HTTPError) do
|
96
|
-
@service.multipart_save_block_blob('test_container', 'test_blob', body, {})
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
def test_multipart_save_block_blob_mock
|
103
|
-
assert @mock_service.multipart_save_block_blob('test_container', 'test_blob', 'content', {})
|
104
|
-
end
|
105
|
-
end
|