gitlab-fog-azure-rm 0.6.0

data/lib/fog/azurerm/requests/storage/put_blob_pages.rb

@@ -0,0 +1,30 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "put_blob_pages [#{start_range}-#{end_range}] / #{blob_name} to the container #{container_name}. options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.put_blob_pages(container_name, blob_name, start_range, end_range, data, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "[#{start_range}-#{end_range}] / #{blob_name} is uploaded successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation.
+      class Mock
+        def put_blob_pages(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_blob_properties.rb

@@ -0,0 +1,31 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implemention for service calls.
+      class Real
+        def put_blob_properties(container_name, name, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Set Blob #{name} properties #{options} in container #{container_name}."
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.set_blob_properties(container_name, name, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "Setting properties of blob #{name} successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_blob_properties(container_name, name, options = {})
+          Fog::Logger.debug "Set Blob #{name} properties #{options} in a container #{container_name} successfully."
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_container_acl.rb

@@ -0,0 +1,31 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def put_container_acl(name, acl, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Setting Container #{name} acl #{acl.nil? ? 'nil' : acl}. options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.set_container_acl(name, acl, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "Setting acl of container #{name} successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_container_acl(*)
+          Fog::Logger.debug 'Set Container testcontainer1 acl successfully.'
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_container_metadata.rb

@@ -0,0 +1,31 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def put_container_metadata(name, metadata, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Setting Container #{name} metadata. options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.set_container_metadata(name, metadata, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "Setting metadata of container #{name} successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_container_metadata(*)
+          Fog::Logger.debug 'Set Container testcontainer1 metadata successfully.'
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_object_url.rb

@@ -0,0 +1,30 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implemention for service calls.
+      class Real
+        # Generate a pre-signed URL for create an object in an Azure blob storage
+        # This is to make this library compatible with CarrierWave.
+        #
+        # @param container_name [String] Name of container containing blob
+        # @param blob_name [String] Name of blob to get expiring url for
+        # @param expires [Time] An expiry time for this url
+        #
+        # @return [String] - https url for blob
+        #
+        # @see https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob
+        #
+        def put_object_url(bucket_name, object_name, expire_at, _upload_options)
+          put_blob_https_url(bucket_name, object_name, expire_at)
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_object_url(*args)
+          put_blob_https_url(*args)
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/release_blob_lease.rb

@@ -0,0 +1,30 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def release_blob_lease(container_name, name, lease_id, options = {})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Releasing blob: #{name} of container #{container_name} having lease_id #{lease_id} options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.release_blob_lease(container_name, name, lease_id, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "Blob #{name} released successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def release_blob_lease(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/release_container_lease.rb

@@ -0,0 +1,30 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def release_container_lease(name, lease_id, options={})
+          options[:request_id] = SecureRandom.uuid
+          msg = "Releasing container: #{name} having lease_id #{lease_id} options: #{options}"
+          Fog::Logger.debug msg
+
+          begin
+            @blob_client.release_container_lease(name, lease_id, options)
+          rescue Azure::Core::Http::HTTPError => ex
+            raise_azure_exception(ex, msg)
+          end
+
+          Fog::Logger.debug "Container #{name} released successfully."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def release_container_lease(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/save_page_blob.rb

@@ -0,0 +1,111 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        # This class is used to store chunk data for page blob before uploading.
+        class BlobChunk
+          attr_reader :id # For debug
+          attr_reader :start_range
+          attr_reader :data
+
+          def initialize(id, start_range, data)
+            @id = id
+            @start_range = start_range
+            @data = data
+          end
+
+          def end_range
+            @start_range + @data.size - 1
+          end
+        end
+
+        # This class is a stream to read chunk data.
+        class BlobFileStream
+          def initialize(body)
+            if body.respond_to?(:read)
+              if body.respond_to?(:rewind)
+                begin
+                  body.rewind
+                rescue => ex
+                  Fog::Logger.debug "save_page_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
+                end
+              end
+              @stream = body
+            else
+              @stream = StringIO.new(body)
+            end
+            @mutex = Mutex.new
+            @count = 0
+          end
+
+          def read(size)
+            data = nil
+            id = 0
+            start_range = 0
+            @mutex.synchronize do
+              start_range = @stream.pos
+              data = @stream.read(size)
+              return nil if data.nil?
+              @count += 1
+              id = @count
+            end
+            BlobChunk.new(id, start_range, data)
+          end
+        end
+
+        def save_page_blob(container_name, blob_name, body, options)
+          threads_num = options.delete(:worker_thread_num)
+          threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
+
+          begin
+            blob_size = Fog::Storage.get_body_size(body)
+            raise "The page blob size must be aligned to a 512-byte boundary. But the file size is #{blob_size}." if (blob_size % 512).nonzero?
+
+            # Initiate the upload
+            Fog::Logger.debug "Creating the page blob #{container_name}/#{blob_name}. options: #{options}"
+            create_page_blob(container_name, blob_name, blob_size, options)
+            options.delete(:content_md5)
+
+            # Uploading content
+            iostream = BlobFileStream.new(body)
+
+            threads = []
+            threads_num.times do |id|
+              thread = Thread.new do
+                Fog::Logger.debug "Created upload thread #{id}."
+                while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
+                  Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, start_range: #{chunk.start_range}, size: #{chunk.data.size}."
+                  put_blob_pages(container_name, blob_name, chunk.start_range, chunk.end_range, chunk.data, options) if Digest::MD5.hexdigest(chunk.data) != HASH_OF_4MB_EMPTY_CONTENT
+                end
+                Fog::Logger.debug "Upload thread #{id} finished."
+              end
+              thread.abort_on_exception = true
+              threads << thread
+            end
+
+            threads.each(&:join)
+          rescue
+            # Abort the upload & reraise
+            begin
+              delete_blob(container_name, blob_name)
+            rescue => ex
+              Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after save_page_blob failed. #{ex.inspect}"
+            end
+            raise
+          end
+
+          Fog::Logger.debug "Successfully save the page blob: #{container_name}/#{blob_name}."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def save_page_blob(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb

@@ -0,0 +1,56 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        def wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil)
+          begin
+            start_time = Time.new
+            while copy_status == COPY_STATUS[:PENDING]
+              blob = get_blob_properties(container_name, blob_name)
+              blob_props = blob.properties
+              if !copy_id.nil? && blob_props[:copy_id] != copy_id
+                raise "The progress of copying to #{container_name}/#{blob_name} was interrupted by other copy operations."
+              end
+
+              copy_status_description = blob_props[:copy_status_description]
+              copy_status = blob_props[:copy_status]
+              break if copy_status != COPY_STATUS[:PENDING]
+
+              elapse_time = Time.new - start_time
+              raise TimeoutError.new("The copy operation cannot be finished in #{timeout} seconds") if !timeout.nil? && elapse_time >= timeout
+
+              copied_bytes, total_bytes = blob_props[:copy_progress].split('/').map(&:to_i)
+              interval = copied_bytes.zero? ? 5 : (total_bytes - copied_bytes).to_f / copied_bytes * elapse_time
+              interval = 30 if interval > 30
+              interval = 1 if interval < 1
+              sleep(interval)
+            end
+
+            if copy_status != COPY_STATUS[:SUCCESS]
+              raise "Failed to copy to #{container_name}/#{blob_name}: \n\tcopy status: #{copy_status}\n\tcopy description: #{copy_status_description}"
+            end
+          rescue
+            # Abort the copy & reraise
+            begin
+              delete_blob(container_name, blob_name)
+            rescue => ex
+              Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after the copy operation failed. #{ex.inspect}"
+            end
+            raise
+          end
+
+          Fog::Logger.debug "Successfully copied the blob: #{container_name}/#{blob_name}."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def wait_blob_copy_operation_to_finish(*)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/storage.rb

@@ -0,0 +1,126 @@
+module Fog
+  module Storage
+    # This class registers models, requests and collections
+    class AzureRM < Fog::Service
+      # Recognizes when creating management client
+      recognizes :tenant_id
+      recognizes :client_id
+      recognizes :client_secret
+      recognizes :subscription_id
+      recognizes :environment
+
+      # Recognizes when creating data client
+      recognizes :azure_storage_account_name
+      recognizes :azure_storage_access_key
+      recognizes :debug
+
+      request_path 'fog/azurerm/requests/storage'
+      # Azure Storage Disk requests
+      request :delete_disk
+      request :create_disk
+      # Azure Storage Container requests
+      request :create_container
+      request :release_container_lease
+      request :acquire_container_lease
+      request :delete_container
+      request :list_containers
+      request :put_container_metadata
+      request :get_container_properties
+      request :get_container_acl
+      request :put_container_acl
+      request :get_container_url
+      request :check_container_exists
+      # Azure Storage Blob requests
+      request :list_blobs
+      request :put_blob_metadata
+      request :put_blob_properties
+      request :get_blob_properties
+      request :copy_blob
+      request :copy_object
+      request :copy_blob_from_uri
+      request :compare_container_blobs
+      request :acquire_blob_lease
+      request :release_blob_lease
+      request :delete_blob
+      request :get_blob
+      request :get_blob_url
+      request :get_object_url
+      request :get_blob_http_url
+      request :get_blob_https_url
+      request :create_block_blob
+      request :put_blob_block
+      request :put_blob_https_url
+      request :put_object_url
+      request :delete_blob_https_url
+      request :delete_object_url
+      request :commit_blob_blocks
+      request :create_page_blob
+      request :put_blob_pages
+      request :wait_blob_copy_operation_to_finish
+      request :save_page_blob
+      request :multipart_save_block_blob
+
+      model_path 'fog/azurerm/models/storage'
+      model :directory
+      collection :directories
+      model :file
+      collection :files
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def initialize(_options = {})
+          begin
+            require 'azure_mgmt_storage'
+            require 'azure/storage'
+          rescue LoadError => e
+            retry if require('rubygems')
+            raise e.message
+          end
+        end
+      end
+
+      # This class provides the actual implementation for service calls.
+      class Real
+        def initialize(options)
+          begin
+            require 'azure_mgmt_storage'
+            require 'azure/storage'
+            require 'securerandom'
+            require 'vhd'
+            @debug = ENV['DEBUG'] || options[:debug]
+            require 'azure/core/http/debug_filter' if @debug
+          rescue LoadError => e
+            retry if require('rubygems')
+            raise e.message
+          end
+
+          options[:environment] = 'AzureCloud' if options[:environment].nil?
+
+          @tenant_id = options[:tenant_id]
+          @client_id = options[:client_id]
+          @client_secret = options[:client_secret]
+          @subscription_id = options[:subscription_id]
+          @environment = options[:environment]
+
+          credentials = Fog::Credentials::AzureRM.get_credentials(@tenant_id, @client_id, @client_secret, @environment)
+          telemetry = "gitlab-fog-azure-rm/#{Fog::AzureRM::VERSION}"
+
+          return unless @azure_storage_account_name != options[:azure_storage_account_name] ||
+                        @azure_storage_access_key != options[:azure_storage_access_key]
+
+          @azure_storage_account_name = options[:azure_storage_account_name]
+          @azure_storage_access_key = options[:azure_storage_access_key]
+
+          azure_client = Azure::Storage::Client.create(storage_account_name: @azure_storage_account_name,
+                                                       storage_access_key: @azure_storage_access_key)
+          azure_client.storage_blob_host = get_blob_endpoint(@azure_storage_account_name, true, @environment)
+          @blob_client = azure_client.blob_client
+          @blob_client.with_filter(Azure::Storage::Core::Filter::ExponentialRetryPolicyFilter.new)
+          @blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
+          @signature_client = Azure::Storage::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
+                                                                                    @azure_storage_access_key)
+        end
+      end
+    end
+  end
+end