gitlab-fog-azure-rm 0.9.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a3e5613e808f27ef9bcd2be6015eac3ffc43e55fe558b2c78b30b97164c3f78
-  data.tar.gz: d64eb6be36585d864bb5bb7fdcafdd5191a7bda376a4f84edc7e82c9bd972554
+  metadata.gz: 177b1f767e814e126cce9905dc11bd5b4384daaafb7292ae379a9f05c435b4ea
+  data.tar.gz: 19d21cd53ffd3a246e195009c4f709df9e4e955ae5c2dace73e4436f82f5ac4d
 SHA512:
-  metadata.gz: b5b884c2fc10a00a1c385522f2138492427ab50162df609fdfe1398f380c42184bb8a99bf8eae0fd50ff77521796f5371fd40b3222114a5f794368555c8408c7
-  data.tar.gz: c1a11715408b8c7eb825915e0ecf1dbfb584f831c85dcbb711973f1b2e22ac54de07f3bbbaf59273c525abe9fa6919385fed04c7e85de400ca727d5bbe19c7b2
+  metadata.gz: aa5d061efb46aee21a8c74d949f7ba5c1632019ccda79799cfe70775f40b6a3448d0f4c01fc12b0c82863419c74a673a68992ca9582317a1df45c42d3ffd5dad
+  data.tar.gz: 8d45eb802da36f53c8aa7a67a49f609de9e39be351af967237166da94c52bda5e474cbb7633200f18bc9386341591d8880df9cda8cd7e5a055beb6f0ad09e13c

data/.rubocop_todo.yml

@@ -132,7 +132,6 @@ Naming/RescuedExceptionsVariableName:
 # SupportedStyles: separated, grouped
 Style/AccessorGrouping:
   Exclude:
-    - 'lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb'
     - 'lib/fog/azurerm/requests/storage/save_page_blob.rb'
     - 'test/test_helper.rb'
 

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Unreleased
 
+## 1.0.0
+
+- Fix multipart uploads not working and drop multipart_save_block_blob !19
+
 ## 0.9.0
 
 - Fix TimeoutError deprecation !17

data/lib/fog/azurerm/models/storage/file.rb

@@ -248,12 +248,7 @@ module Fog
         # Upload blob
         def save_blob(options)
           if options[:blob_type].nil? || options[:blob_type] == 'BlockBlob'
-            if Fog::Storage.get_body_size(body) <= SINGLE_BLOB_PUT_THRESHOLD
-              service.create_block_blob(directory.key, key, body, options)
-            else
-              service.multipart_save_block_blob(directory.key, key, body, options)
-              service.get_blob_properties(directory.key, key)
-            end
+            service.create_block_blob(directory.key, key, body, options)
           else
             service.save_page_blob(directory.key, key, body, options)
             service.get_blob_properties(directory.key, key)

data/lib/fog/azurerm/requests/storage/create_block_blob.rb

@@ -8,37 +8,9 @@ module Fog
           msg = "create_block_blob #{blob_name} to the container #{container_name}. options: #{options}"
           Fog::Logger.debug msg
 
-          begin
-            if body.nil?
-              data = nil
-            elsif body.respond_to?(:read)
-              if body.respond_to?(:rewind)
-                begin
-                  body.rewind
-                rescue
-                  nil
-                end
-              end
-              data = body.read
-            else
-              data = Fog::Storage.parse_data(body)
-              options[:content_length] = data[:headers]['Content-Length']
-              options[:content_type] = data[:headers]['Content-Type']
-              data = data[:body]
-            end
-
-            raise ArgumentError.new('The maximum size for a block blob created via create_block_blob is 64 MB.') if !data.nil? && Fog::Storage.get_body_size(data) > 64 * 1024 * 1024
-            blob = @blob_client.create_block_blob(container_name, blob_name, data, options)
-          rescue Azure::Core::Http::HTTPError => ex
-            raise_azure_exception(ex, msg)
-          end
-
-          if data.nil?
-            Fog::Logger.debug "Create a block blob #{blob_name} successfully."
-          else
-            Fog::Logger.debug "Upload a block blob #{blob_name} successfully."
-          end
-          blob
+          @blob_client.create_block_blob(container_name, blob_name, body, options)
+        rescue Azure::Core::Http::HTTPError => ex
+          raise_azure_exception(ex, msg)
         end
       end
 

data/lib/fog/azurerm/storage.rb

@@ -57,7 +57,6 @@ module Fog
       request :put_blob_pages
       request :wait_blob_copy_operation_to_finish
       request :save_page_blob
-      request :multipart_save_block_blob
 
       model_path 'fog/azurerm/models/storage'
       model :directory

data/lib/fog/azurerm/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AzureRM
-    VERSION = '0.9.0'.freeze
+    VERSION = '1.0.0'.freeze
   end
 end

data/test/models/storage/test_file.rb

@@ -76,10 +76,8 @@ class TestFile < Minitest::Test
   def test_save_method_with_large_block_blob_success
     @file.body = 'd' * (32 * 1024 * 1024 + 1) # SINGLE_BLOB_PUT_THRESHOLD is 32 * 1024 * 1024
 
-    @service.stub :multipart_save_block_blob, true do
-      @service.stub :get_blob_properties, @raw_cloud_blob do
-        assert @file.save
-      end
+    @service.stub :create_block_blob, @raw_cloud_blob do
+      assert @file.save
     end
   end
 

data/test/requests/storage/test_create_block_blob.rb

@@ -52,15 +52,6 @@ class TestCreateBlockBlob < Minitest::Test
     end
   end
 
-  def test_create_block_blob_exceed_max_body_size
-    data = []
-    data.stub :size, 64 * 1024 * 1024 + 1 do
-      assert_raises(ArgumentError) do
-        @service.create_block_blob('test_container', 'test_blob', data)
-      end
-    end
-  end
-
   def test_create_block_blob_http_exception
     http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
     @blob_client.stub :create_block_blob, http_exception do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-fog-azure-rm
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Shaffan Chaudhry
@@ -18,7 +18,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-06 00:00:00.000000000 Z
+date: 2020-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: codeclimate-test-reporter
@@ -249,7 +249,6 @@ files:
 - lib/fog/azurerm/requests/storage/get_object_url.rb
 - lib/fog/azurerm/requests/storage/list_blobs.rb
 - lib/fog/azurerm/requests/storage/list_containers.rb
-- lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb
 - lib/fog/azurerm/requests/storage/put_blob_block.rb
 - lib/fog/azurerm/requests/storage/put_blob_https_url.rb
 - lib/fog/azurerm/requests/storage/put_blob_metadata.rb
@@ -305,7 +304,6 @@ files:
 - test/requests/storage/test_get_container_url.rb
 - test/requests/storage/test_list_blobs.rb
 - test/requests/storage/test_list_containers.rb
-- test/requests/storage/test_multipart_save_block_blob.rb
 - test/requests/storage/test_put_blob_block.rb
 - test/requests/storage/test_put_blob_https_url.rb
 - test/requests/storage/test_put_blob_metadata.rb

data/lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb

@@ -1,110 +0,0 @@
-module Fog
-  module Storage
-    class AzureRM
-      # This class provides the actual implementation for service calls.
-      class Real
-        # This class is used to store chunk data for block blob before uploading.
-        class BlockChunk
-          attr_reader :id # For debug
-          attr_reader :block_id
-          attr_reader :data
-
-          def initialize(id, block_id, data)
-            @id = id
-            @block_id = block_id
-            @data = data
-          end
-        end
-
-        # This class is a stream to read chunk data.
-        class BlockFileStream
-          attr_reader :blocks
-
-          def initialize(body)
-            if body.respond_to?(:read)
-              if body.respond_to?(:rewind)
-                begin
-                  body.rewind
-                rescue => ex
-                  Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
-                end
-              end
-              @stream = body
-            else
-              @stream = StringIO.new(body)
-            end
-            @mutex = Mutex.new
-            @blocks = []
-          end
-
-          def read(size)
-            block_id = Base64.strict_encode64(random_string(32))
-            data = nil
-            id = 0
-            @mutex.synchronize do
-              data = @stream.read(size)
-              return nil if data.nil?
-              @blocks << [block_id]
-              id = @blocks.size
-            end
-            BlockChunk.new(id, block_id, data)
-          end
-        end
-
-        def multipart_save_block_blob(container_name, blob_name, body, options)
-          threads_num = options.delete(:worker_thread_num)
-          threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
-
-          begin
-            # Initiate the upload
-            Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
-            content_md5 = options.delete(:content_md5)
-            create_block_blob(container_name, blob_name, nil, options)
-
-            # Uploading parts
-            Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
-            iostream = BlockFileStream.new(body)
-
-            threads = []
-            threads_num.times do |id|
-              thread = Thread.new do
-                Fog::Logger.debug "Created upload thread #{id}."
-                while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
-                  Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
-                  put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
-                end
-                Fog::Logger.debug "Upload thread #{id} finished."
-              end
-              thread.abort_on_exception = true
-              threads << thread
-            end
-
-            threads.each(&:join)
-            # Complete the upload
-            options[:content_md5] = content_md5 unless content_md5.nil?
-            Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
-            commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
-          rescue
-            # Abort the upload & reraise
-            begin
-              delete_blob(container_name, blob_name)
-            rescue => ex
-              Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
-            end
-            raise
-          end
-
-          Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
-          true
-        end
-      end
-
-      # This class provides the mock implementation for unit tests.
-      class Mock
-        def multipart_save_block_blob(*)
-          true
-        end
-      end
-    end
-  end
-end

data/test/requests/storage/test_multipart_save_block_blob.rb

@@ -1,105 +0,0 @@
-require File.expand_path '../../test_helper', __dir__
-
-# Storage Blob Class
-class TestMultipartSaveBlockBlob < Minitest::Test
-  # This class posesses the test cases for the requests of saving storage block blob with multiple parts.
-  def setup
-    Fog.mock!
-    @mock_service = Fog::Storage::AzureRM.new(storage_account_credentials)
-    Fog.unmock!
-    @mocked_response = mocked_storage_http_error
-
-    @service = Fog::Storage::AzureRM.new(storage_account_credentials)
-  end
-
-  def test_multipart_save_block_blob_success
-    body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
-
-    @service.stub :create_block_blob, true do
-      @service.stub :put_blob_block, true do
-        @service.stub :commit_blob_blocks, true do
-          assert @service.multipart_save_block_blob('test_container', 'test_blob', body, worker_thread_num: 6, content_md5: 'oafL1+HS78x65+e39PGIIg==')
-        end
-      end
-    end
-  end
-
-  def test_multipart_save_block_blob_with_file_handle_success
-    i = 0
-    multiple_values = lambda do |*|
-      i += 1
-      return 'd' * 4 * 1024 * 1024 if i == 1
-      return 'd' * (1 * 1024 * 1024 + 1) if i == 2
-      return nil
-    end
-    temp_file = '/dev/null'
-    File.open(temp_file, 'r') do |file_handle|
-      file_handle.stub :read, multiple_values do
-        file_handle.stub :rewind, nil do
-          @service.stub :create_block_blob, @raw_cloud_blob do
-            @service.stub :put_blob_block, true do
-              @service.stub :commit_blob_blocks, true do
-                assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 0)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-
-  def test_multipart_save_block_blob_with_file_handle_do_not_support_rewind_success
-    i = 0
-    multiple_values = lambda do |*|
-      i += 1
-      return 'd' * 4 * 1024 * 1024 if i == 1
-      return 'd' * (1 * 1024 * 1024 + 1) if i == 2
-      return nil
-    end
-    temp_file = '/dev/null'
-    exception = ->(*) { raise 'do not support rewind' }
-    File.open(temp_file, 'r') do |file_handle|
-      file_handle.stub :read, multiple_values do
-        file_handle.stub :rewind, exception do
-          @service.stub :create_block_blob, @raw_cloud_blob do
-            @service.stub :put_blob_block, true do
-              @service.stub :commit_blob_blocks, true do
-                assert @service.multipart_save_block_blob('test_container', 'test_blob', file_handle, worker_thread_num: 'invalid')
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-
-  def test_multipart_save_block_blob_http_exception
-    body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
-
-    http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
-    @service.stub :create_block_blob, http_exception do
-      @service.stub :delete_blob, true do
-        assert_raises(Azure::Core::Http::HTTPError) do
-          @service.multipart_save_block_blob('test_container', 'test_blob', body, {})
-        end
-      end
-    end
-  end
-
-  def test_multipart_save_block_blob_fail_when_delete_blob_http_exception
-    body = 'd' * 5 * 1024 * 1024 # MAXIMUM_CHUNK_SIZE is 4 * 1024 * 1024
-
-    http_exception = ->(*) { raise Azure::Core::Http::HTTPError.new(@mocked_response) }
-    @service.stub :create_block_blob, http_exception do
-      @service.stub :delete_blob, http_exception do
-        assert_raises(Azure::Core::Http::HTTPError) do
-          @service.multipart_save_block_blob('test_container', 'test_blob', body, {})
-        end
-      end
-    end
-  end
-
-  def test_multipart_save_block_blob_mock
-    assert @mock_service.multipart_save_block_blob('test_container', 'test_blob', 'content', {})
-  end
-end