fog-azure-rm 0.1.0 → 0.1.1

data/lib/fog/azurerm/requests/storage/put_blob_block.rb

@@ -0,0 +1,30 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def put_blob_block(container_name, blob_name, block_id, data, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "put_blob_block block_id: #{block_id} / #{blob_name} to the container #{container_name}. options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.put_blob_block(container_name, blob_name, block_id, data, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "block_id #{block_id} is uploaded successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation.
+      class Mock
+        def put_blob_block(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/{set_blob_metadata.rb → put_blob_metadata.rb}

@@ -3,21 +3,25 @@ module Fog
     class AzureRM
       # This class provides the actual implementation for service calls.
       class Real
-        def set_blob_metadata(container_name, name, metadata, options = {})
-          msg = "Setting Blob #{name} metadata in a container #{container_name}."
+        def put_blob_metadata(container_name, name, metadata, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Setting Blob #{name} metadata in a container #{container_name}. options: #{options}"
           Fog::Logger.debug msg
+
           begin
             @blob_client.set_blob_metadata(container_name, name, metadata, options)
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end
+
           Fog::Logger.debug "Setting metadata of blob #{name} successfully."
           true
         end
       end
+
       # This class provides the mock implementation for unit tests.
       class Mock
-        def set_blob_metadata(container_name, name, metadata, _options = {})
+        def put_blob_metadata(container_name, name, metadata, _options = {})
           Fog::Logger.debug "Set Blob #{name} metadata #{metadata} in a container #{container_name} successfully."
           true
         end

data/lib/fog/azurerm/requests/storage/put_blob_pages.rb

@@ -0,0 +1,30 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "put_blob_pages [#{start_range}-#{end_range}] / #{blob_name} to the container #{container_name}. options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.put_blob_pages(container_name, blob_name, start_range, end_range, data, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "[#{start_range}-#{end_range}] / #{blob_name} is uploaded successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation.
+      class Mock
+        def put_blob_pages(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_blob_properties.rb

@@ -0,0 +1,31 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implemention for service calls.
+      class Real
+        def put_blob_properties(container_name, name, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Set Blob #{name} properties #{options} in container #{container_name}."
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.set_blob_properties(container_name, name, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "Setting properties of blob #{name} successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_blob_properties(container_name, name, options = {})
+          Fog::Logger.debug "Set Blob #{name} properties #{options} in a container #{container_name} successfully."
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_container_acl.rb

@@ -0,0 +1,31 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def put_container_acl(name, acl, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Setting Container #{name} acl #{acl.nil? ? 'nil' : acl}. options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.set_container_acl(name, acl, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "Setting acl of container #{name} successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_container_acl(*)
+          Fog::Logger.debug 'Set Container testcontainer1 acl successfully.'
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/{set_container_metadata.rb → put_container_metadata.rb}

@@ -3,21 +3,25 @@ module Fog
     class AzureRM
       # This class provides the actual implementation for service calls.
       class Real
-        def set_container_metadata(name, metadata, options = {})
-          msg = "Setting Container #{name} metadata."
+        def put_container_metadata(name, metadata, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Setting Container #{name} metadata. options: #{options}"
           Fog::Logger.debug msg
+
           begin
             @blob_client.set_container_metadata(name, metadata, options)
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end
+
           Fog::Logger.debug "Setting metadata of container #{name} successfully."
           true
         end
       end
+
       # This class provides the mock implementation for unit tests.
       class Mock
-        def set_container_metadata(*)
+        def put_container_metadata(*)
           Fog::Logger.debug 'Set Container testcontainer1 metadata successfully.'
           true
         end

data/lib/fog/azurerm/requests/storage/release_blob_lease.rb

@@ -4,13 +4,16 @@ module Fog
       # This class provides the actual implementation for service calls.
       class Real
         def release_blob_lease(container_name, name, lease_id, options = {})
-          msg = "Releasing blob: #{name} of container #{container_name} having lease_id #{lease_id}"
+          options[:request_id] = SecureRandom.uuid
+          msg = "Releasing blob: #{name} of container #{container_name} having lease_id #{lease_id} options: #{options}"
           Fog::Logger.debug msg
+
           begin
             @blob_client.release_blob_lease(container_name, name, lease_id, options)
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end
+
           Fog::Logger.debug "Blob #{name} released successfully."
           true
         end

data/lib/fog/azurerm/requests/storage/release_container_lease.rb

@@ -4,13 +4,16 @@ module Fog
       # This class provides the actual implementation for service calls.
       class Real
         def release_container_lease(name, lease_id, options={})
-          msg = "Releasing container: #{name} having lease_id #{lease_id}"
+          options[:request_id] = SecureRandom.uuid
+          msg = "Releasing container: #{name} having lease_id #{lease_id} options: #{options}"
           Fog::Logger.debug msg
+
           begin
             @blob_client.release_container_lease(name, lease_id, options)
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end
+
           Fog::Logger.debug "Container #{name} released successfully."
           true
         end

data/lib/fog/azurerm/requests/storage/save_page_blob.rb

@@ -0,0 +1,111 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        # This class is used to store chunk data for page blob before uploading.
+        class BlobChunk
+          attr_reader :id # For debug
+          attr_reader :start_range
+          attr_reader :data
+
+          def initialize(id, start_range, data)
+            @id = id
+            @start_range = start_range
+            @data = data
+          end
+
+          def end_range
+            @start_range + @data.size - 1
+          end
+        end
+
+        # This class is a stream to read chunk data.
+        class BlobFileStream
+          def initialize(body)
+            if body.respond_to?(:read)
+              if body.respond_to?(:rewind)
+                begin
+                  body.rewind
+                rescue => ex
+                  Fog::Logger.debug "save_page_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
+                end
+              end
+              @stream = body
+            else
+              @stream = StringIO.new(body)
+            end
+            @mutex = Mutex.new
+            @count = 0
+          end
+
+          def read(size)
+            data = nil
+            id = 0
+            start_range = 0
+            @mutex.synchronize do
+              start_range = @stream.pos
+              data = @stream.read(size)
+              return nil if data.nil?
+              @count += 1
+              id = @count
+            end
+            BlobChunk.new(id, start_range, data)
+          end
+        end
+
+        def save_page_blob(container_name, blob_name, body, options)
+          threads_num = options.delete(:worker_thread_num)
+          threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
+
+          begin
+            blob_size = Fog::Storage.get_body_size(body)
+            raise "The page blob size must be aligned to a 512-byte boundary. But the file size is #{blob_size}." if (blob_size % 512).nonzero?
+
+            # Initiate the upload
+            Fog::Logger.debug "Creating the page blob #{container_name}/#{blob_name}. options: #{options}"
+            create_page_blob(container_name, blob_name, blob_size, options)
+            options.delete(:content_md5)
+
+            # Uploading content
+            iostream = BlobFileStream.new(body)
+
+            threads = []
+            threads_num.times do |id|
+              thread = Thread.new do
+                Fog::Logger.debug "Created upload thread #{id}."
+                while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
+                  Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, start_range: #{chunk.start_range}, size: #{chunk.data.size}."
+                  put_blob_pages(container_name, blob_name, chunk.start_range, chunk.end_range, chunk.data, options) if Digest::MD5.hexdigest(chunk.data) != HASH_OF_4MB_EMPTY_CONTENT
+                end
+                Fog::Logger.debug "Upload thread #{id} finished."
+              end
+              thread.abort_on_exception = true
+              threads << thread
+            end
+
+            threads.each(&:join)
+          rescue
+            # Abort the upload & reraise
+            begin
+              delete_blob(container_name, blob_name)
+            rescue => ex
+              Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after save_page_blob failed. #{ex.inspect}"
+            end
+            raise
+          end
+
+          Fog::Logger.debug "Successfully save the page blob: #{container_name}/#{blob_name}."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def save_page_blob(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/start_backup.rb

@@ -16,7 +16,7 @@ module Fog
             return false
           end
 
-          backup_items = get_backup_item(resource_group, name, vm_name)
+          backup_items = get_backup_item(resource_group, name)
           backup_item = backup_items.select { |item| (item['properties']['friendlyName'].eql? vm_name.downcase) && (vm_resource_group.eql? get_resource_group_from_id(item['properties']['virtualMachineId'])) }[0]
 
           resource_url = "#{AZURE_RESOURCE}/subscriptions/#{@subscription_id}/resourceGroups/#{resource_group}/providers/Microsoft.RecoveryServices/vaults/#{name}/backupFabrics/Azure/protectionContainers/iaasvmcontainer;#{backup_item['name']}/protectedItems/vm;#{backup_item['name']}/backup?api-version=2016-05-01"

data/lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb

@@ -0,0 +1,56 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil)
+          begin
+            start_time = Time.new
+            while copy_status == COPY_STATUS[:PENDING]
+              blob = get_blob_properties(container_name, blob_name)
+              blob_props = blob.properties
+              if !copy_id.nil? && blob_props[:copy_id] != copy_id
+                raise "The progress of copying to #{container_name}/#{blob_name} was interrupted by other copy operations."
+              end
+
+              copy_status_description = blob_props[:copy_status_description]
+              copy_status = blob_props[:copy_status]
+              break if copy_status != COPY_STATUS[:PENDING]
+
+              elapse_time = Time.new - start_time
+              raise TimeoutError.new("The copy operation cannot be finished in #{timeout} seconds") if !timeout.nil? && elapse_time >= timeout
+
+              copied_bytes, total_bytes = blob_props[:copy_progress].split('/').map(&:to_i)
+              interval = copied_bytes.zero? ? 5 : (total_bytes - copied_bytes).to_f / copied_bytes * elapse_time
+              interval = 30 if interval > 30
+              interval = 1 if interval < 1
+              sleep(interval)
+            end
+
+            if copy_status != COPY_STATUS[:SUCCESS]
+              raise "Failed to copy to #{container_name}/#{blob_name}: \n\tcopy status: #{copy_status}\n\tcopy description: #{copy_status_description}"
+            end
+          rescue
+            # Abort the copy & reraise
+            begin
+              delete_blob(container_name, blob_name)
+            rescue => ex
+              Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after the copy operation failed. #{ex.inspect}"
+            end
+            raise
+          end
+
+          Fog::Logger.debug "Successfully copied the blob: #{container_name}/#{blob_name}."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def wait_blob_copy_operation_to_finish(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/resources.rb

@@ -43,7 +43,7 @@ module Fog
         end
       end
 
-      # This class provides the actual implemention for service calls.
+      # This class provides the actual implementation for service calls.
       class Real
         def initialize(options)
           begin
@@ -54,7 +54,7 @@ module Fog
           end
 
           credentials = Fog::Credentials::AzureRM.get_credentials(options[:tenant_id], options[:client_id], options[:client_secret])
-          @rmc = ::Azure::ARM::Resources::ResourceManagementClient.new(credentials)
+          @rmc = ::Azure::ARM::Resources::ResourceManagementClient.new(credentials, resource_manager_endpoint_url)
           @rmc.subscription_id = options[:subscription_id]
         end
       end

data/lib/fog/azurerm/storage.rb

@@ -33,25 +33,35 @@ module Fog
       request :acquire_container_lease
       request :delete_container
       request :list_containers
-      request :get_container_metadata
-      request :set_container_metadata
+      request :put_container_metadata
       request :get_container_properties
-      request :get_container_access_control_list
+      request :get_container_acl
+      request :put_container_acl
+      request :get_container_url
       # Azure Storage Blob requests
       request :list_blobs
-      request :set_blob_metadata
-      request :get_blob_metadata
-      request :set_blob_properties
+      request :put_blob_metadata
+      request :put_blob_properties
       request :get_blob_properties
-      request :upload_block_blob_from_file
-      request :download_blob_to_file
       request :copy_blob
       request :copy_blob_from_uri
-      request :compare_blob
-      request :check_blob_exist
+      request :compare_container_blobs
       request :acquire_blob_lease
       request :release_blob_lease
       request :delete_blob
+      request :get_blob
+      request :get_blob_url
+      request :get_blob_http_url
+      request :get_blob_https_url
+      request :create_block_blob
+      request :put_blob_block
+      request :commit_blob_blocks
+      request :create_page_blob
+      request :put_blob_pages
+      request :wait_blob_copy_operation_to_finish
+      request :save_page_blob
+      request :multipart_save_block_blob
+
       # Azure Recovery Vault requests
       request :create_or_update_recovery_vault
       request :get_recovery_vault
@@ -97,6 +107,8 @@ module Fog
           begin
             require 'azure_mgmt_storage'
             require 'azure/storage'
+            require 'securerandom'
+            require 'vhd'
             @debug = ENV['DEBUG'] || options[:debug]
             require 'azure/core/http/debug_filter' if @debug
           rescue LoadError => e
@@ -111,19 +123,21 @@ module Fog
 
           credentials = Fog::Credentials::AzureRM.get_credentials(options[:tenant_id], options[:client_id], options[:client_secret])
           unless credentials.nil?
-            @storage_mgmt_client = ::Azure::ARM::Storage::StorageManagementClient.new(credentials)
+            @storage_mgmt_client = ::Azure::ARM::Storage::StorageManagementClient.new(credentials, resource_manager_endpoint_url)
             @storage_mgmt_client.subscription_id = options[:subscription_id]
           end
 
-          if Fog::Credentials::AzureRM.new_account_credential? options
-            Azure::Storage.setup(storage_account_name: options[:azure_storage_account_name],
-                                 storage_access_key: options[:azure_storage_access_key],
-                                 storage_connection_string: options[:azure_storage_connection_string])
+          return unless Fog::Credentials::AzureRM.new_account_credential?(options)
 
-            @blob_client = Azure::Storage::Blob::BlobService.new
-            @blob_client.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
-            @blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
-          end
+          Azure::Storage.setup(storage_account_name: options[:azure_storage_account_name],
+                               storage_access_key: options[:azure_storage_access_key],
+                               storage_connection_string: options[:azure_storage_connection_string])
+
+          @blob_client = Azure::Storage::Blob::BlobService.new
+          @blob_client.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
+          @blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
+          @signature_client = Azure::Storage::Core::Auth::SharedAccessSignature.new(options[:azure_storage_account_name],
+                                                                                    options[:azure_storage_access_key])
         end
       end
     end