gitlab-fog-azure-rm 1.7.0 → 1.9.0

data/lib/fog/azurerm/requests/storage/get_blob_url.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         # Get a public blob url from Azure blob storage
@@ -18,8 +18,8 @@ module Fog
       # This class provides the mock implementation for unit tests.
       class Mock
         def get_blob_url(_container_name, _blob_name, options = {})
-          url = 'https://sa.blob.core.windows.net/test_container/test_blob'
-          url.gsub!('https:', 'http:') if options[:scheme] == 'http'
+          url = 'https://mockaccount.blob.core.windows.net/test_container/test_blob'
+          url.sub!('https:', 'http:') if options[:scheme] == 'http'
           url
         end
       end

data/lib/fog/azurerm/requests/storage/get_container_acl.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def get_container_acl(container_name, options = {})

data/lib/fog/azurerm/requests/storage/get_container_properties.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def get_container_properties(name, options = {})

data/lib/fog/azurerm/requests/storage/get_container_url.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         # Get a public container url from Azure storage container
@@ -24,8 +24,8 @@ module Fog
       # This class provides the mock implementation for unit tests.
       class Mock
         def get_container_url(_container_name, options = {})
-          url = 'https://sa.blob.core.windows.net/test_container?comp=list&restype=container'
-          url.gsub!('https:', 'http:') if options[:scheme] == 'http'
+          url = 'https://mockaccount.blob.core.windows.net/test_container?comp=list&restype=container'
+          url.sub!('https:', 'http:') if options[:scheme] == 'http'
           url
         end
       end

data/lib/fog/azurerm/requests/storage/get_object.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         def get_object(...)

data/lib/fog/azurerm/requests/storage/get_object_url.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         # Get a public blob url from Azure blob storage

data/lib/fog/azurerm/requests/storage/list_blobs.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       # https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx
       class Real

data/lib/fog/azurerm/requests/storage/list_containers.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       # https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
       class Real

data/lib/fog/azurerm/requests/storage/put_blob_block.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_blob_block(container_name, blob_name, block_id, data, options = {})

data/lib/fog/azurerm/requests/storage/put_blob_http_url.rb

@@ -0,0 +1,39 @@
+module Fog
+  module AzureRM
+    class Storage
+      # This class provides the actual implemention for service calls.
+      class Real
+        # Generate a pre-signed URL for create an object in an Azure blob storage
+        #
+        # @param container_name [String] Name of container containing blob
+        # @param blob_name [String] Name of blob to get expiring url for
+        # @param expires [Time] An expiry time for this url
+        #
+        # @return [String] - https url for blob
+        #
+        # @see https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob
+        #
+        def put_blob_http_url(container_name, blob_name, expires)
+          relative_path = "#{container_name}/#{blob_name}"
+          params = {
+            service: 'b',
+            resource: 'b',
+            permissions: 'c',
+            expiry: expires.utc.iso8601
+          }
+          token = signature_client(expires).generate_service_sas_token(relative_path, params)
+          uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
+          url = "#{uri}?#{token}"
+          url.sub('https:', 'http:')
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_blob_http_url(*)
+          'http://mockaccount.blob.core.windows.net/test_container/test_blob?token'
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_blob_https_url.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         # Generate a pre-signed URL for create an object in an Azure blob storage
@@ -22,7 +22,7 @@ module Fog
             expiry: expires.utc.iso8601,
             protocol: 'https'
           }
-          token = @signature_client.generate_service_sas_token(relative_path, params)
+          token = signature_client(expires).generate_service_sas_token(relative_path, params)
           uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
           "#{uri}?#{token}"
         end
@@ -31,7 +31,7 @@ module Fog
       # This class provides the mock implementation for unit tests.
       class Mock
         def put_blob_https_url(*)
-          'https://sa.blob.core.windows.net/test_container/test_blob?token'
+          'https://mockaccount.blob.core.windows.net/test_container/test_blob?token'
         end
       end
     end

data/lib/fog/azurerm/requests/storage/put_blob_metadata.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_blob_metadata(container_name, name, metadata, options = {})

data/lib/fog/azurerm/requests/storage/put_blob_pages.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {})

data/lib/fog/azurerm/requests/storage/put_blob_properties.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         def put_blob_properties(container_name, name, options = {})

data/lib/fog/azurerm/requests/storage/put_container_acl.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_container_acl(name, acl, options = {})

data/lib/fog/azurerm/requests/storage/put_container_metadata.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_container_metadata(name, metadata, options = {})

data/lib/fog/azurerm/requests/storage/put_object_url.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         # Generate a pre-signed URL for create an object in an Azure blob storage

data/lib/fog/azurerm/requests/storage/release_blob_lease.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def release_blob_lease(container_name, name, lease_id, options = {})

data/lib/fog/azurerm/requests/storage/release_container_lease.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def release_container_lease(name, lease_id, options={})

data/lib/fog/azurerm/requests/storage/save_page_blob.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         # This class is used to store chunk data for page blob before uploading.

data/lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil)

data/lib/fog/azurerm/storage.rb

@@ -1,18 +1,14 @@
 module Fog
-  module Storage
+  module AzureRM
     # This class registers models, requests and collections
-    class AzureRM < Fog::Service
-      # Recognizes when creating management client
-      recognizes :tenant_id
-      recognizes :client_id
-      recognizes :client_secret
-      recognizes :subscription_id
-      recognizes :environment
-
+    class Storage < Fog::Service
       # Recognizes when creating data client
       recognizes :azure_storage_account_name
       recognizes :azure_storage_access_key
+      recognizes :azure_storage_token_signer
+      recognizes :azure_storage_endpoint
       recognizes :azure_storage_domain
+      recognizes :environment
 
       recognizes :debug
 
@@ -48,6 +44,7 @@ module Fog
       request :get_blob_https_url
       request :create_block_blob
       request :put_blob_block
+      request :put_blob_http_url
       request :put_blob_https_url
       request :put_object_url
       request :delete_blob
@@ -90,44 +87,82 @@ module Fog
             require 'securerandom'
             @debug = ENV['DEBUG'] || options[:debug]
             require 'azure/core/http/debug_filter' if @debug
+            require 'fog/azurerm/identity_encoding_filter'
           rescue LoadError => e
             retry if require('rubygems')
             raise e.message
           end
 
-          options[:environment] = 'AzureCloud' if options[:environment].nil?
-
-          @tenant_id = options[:tenant_id]
-          @client_id = options[:client_id]
-          @client_secret = options[:client_secret]
-          @subscription_id = options[:subscription_id]
-          @environment = options[:environment]
-
-          credentials = Fog::Credentials::AzureRM.get_credentials(@tenant_id, @client_id, @client_secret, @environment)
-          telemetry = "gitlab-fog-azure-rm/#{Fog::AzureRM::VERSION}"
-
           return unless @azure_storage_account_name != options[:azure_storage_account_name] ||
-                        @azure_storage_access_key != options[:azure_storage_access_key]
+                        @azure_storage_access_key != options[:azure_storage_access_key] ||
+                        @azure_storage_token_signer != options[:azure_storage_token_signer]
 
           @azure_storage_account_name = options[:azure_storage_account_name]
           @azure_storage_access_key = options[:azure_storage_access_key]
+          @azure_storage_token_signer = options[:azure_storage_token_signer]
+          @azure_storage_endpoint = options[:azure_storage_endpoint]
           @azure_storage_domain = options[:azure_storage_domain]
 
-          domain =
+          options[:environment] = 'AzureCloud' if options[:environment].nil?
+          @environment = options[:environment]
+
+          storage_blob_host =
+            @azure_storage_endpoint ||
             if @azure_storage_domain.nil? || @azure_storage_domain.empty?
               get_blob_endpoint(@azure_storage_account_name, true, @environment)
             else
               get_blob_endpoint_with_domain(@azure_storage_account_name, true, @azure_storage_domain)
             end
 
-          azure_client = Azure::Storage::Common::Client.create(storage_account_name: @azure_storage_account_name,
-                                                               storage_access_key: @azure_storage_access_key,
-                                                               storage_blob_host: domain)
+          azure_client = Azure::Storage::Common::Client.create({
+            storage_account_name: @azure_storage_account_name,
+            storage_access_key: @azure_storage_access_key,
+            signer: @azure_storage_token_signer
+          }.compact)
+          azure_client.storage_blob_host = storage_blob_host
           @blob_client = Azure::Storage::Blob::BlobService.new(client: azure_client)
+          @blob_client.with_filter(Fog::AzureRM::IdentityEncodingFilter.new)
           @blob_client.with_filter(Azure::Storage::Common::Core::Filter::ExponentialRetryPolicyFilter.new)
           @blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
-          @signature_client = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
-                                                                                            @azure_storage_access_key)
+        end
+
+        private
+
+        def signature_client(requested_expiry)
+          access_key = @azure_storage_access_key.to_s
+          user_delegation_key = user_delegation_key(requested_expiry)
+
+          # invalidate cache when the delegation key changes
+          unless @signature_client_delegation_key == user_delegation_key
+            @signature_client_delegation_key = user_delegation_key
+            @signature_client = nil
+          end
+
+          @signature_client ||= Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(
+            @azure_storage_account_name,
+            access_key,
+            user_delegation_key
+          )
+        end
+
+        def user_delegation_key(requested_expiry)
+          return nil unless @azure_storage_token_signer
+
+          @user_delegation_key_mutex ||= Mutex.new
+          @user_delegation_key_mutex.synchronize do
+            if @user_delegation_key_expiry.nil? || @user_delegation_key_expiry < requested_expiry
+              start = Time.now
+              expiry = start + Azure::Storage::Blob::BlobConstants::MAX_USER_DELEGATION_KEY_SECONDS
+
+              @user_delegation_key = @blob_client.get_user_delegation_key(
+                start,
+                expiry
+              )
+              @user_delegation_key_expiry = expiry
+            end
+          end
+
+          @user_delegation_key
         end
       end
     end

data/lib/fog/azurerm/utilities/general.rb

@@ -53,13 +53,9 @@ module Fog
           type.split('/').last
         end
 
-        def raise_azure_exception(exception, msg)
+        def raise_azure_exception(exception, _msg)
           raise Fog::AzureRM::CustomAzureCoreHttpError.new(exception) if exception.is_a?(Azure::Core::Http::HTTPError)
-          raise exception unless exception.is_a?(MsRestAzure::AzureOperationError)
-
-          azure_operation_error = Fog::AzureRM::CustomAzureOperationError.new(msg, exception)
-          azure_operation_error.print_subscription_limits_information if !azure_operation_error.request.nil? && !azure_operation_error.response.nil?
-          raise azure_operation_error
+          raise exception
         end
 
         # Make sure if input_params(Hash) contains all keys present in required_params(Array)
@@ -80,47 +76,15 @@ module Fog
           (0...length).map { ('a'..'z').to_a[rand(26)] }.join
         end
 
-        def active_directory_service_settings(environment = ENVIRONMENT_AZURE_CLOUD)
-          case environment
-          when ENVIRONMENT_AZURE_CHINA_CLOUD
-            MsRestAzure::ActiveDirectoryServiceSettings.get_azure_china_settings
-          when ENVIRONMENT_AZURE_US_GOVERNMENT
-            MsRestAzure::ActiveDirectoryServiceSettings.get_azure_us_government_settings
-          when ENVIRONMENT_AZURE_GERMAN_CLOUD
-            MsRestAzure::ActiveDirectoryServiceSettings.get_azure_german_settings
-          else
-            MsRestAzure::ActiveDirectoryServiceSettings.get_azure_settings
-          end
-        end
-
-        def resource_manager_endpoint_url(environment = ENVIRONMENT_AZURE_CLOUD)
-          case environment
-          when ENVIRONMENT_AZURE_CHINA_CLOUD
-            MsRestAzure::AzureEnvironments::ChinaCloud.resource_manager_endpoint_url
-          when ENVIRONMENT_AZURE_US_GOVERNMENT
-            MsRestAzure::AzureEnvironments::USGovernment.resource_manager_endpoint_url
-          when ENVIRONMENT_AZURE_GERMAN_CLOUD
-            MsRestAzure::AzureEnvironments::GermanCloud.resource_manager_endpoint_url
-          else
-            MsRestAzure::AzureEnvironments::AzureCloud.resource_manager_endpoint_url
-          end
-        end
-
-        # storage_endpoint_suffix is nil in ms_rest_azure 0.6.2
-        # Reference the issue: https://github.com/Azure/azure-sdk-for-ruby/issues/603
         def storage_endpoint_suffix(environment = ENVIRONMENT_AZURE_CLOUD)
           case environment
           when ENVIRONMENT_AZURE_CHINA_CLOUD
-            # MsRestAzure::AzureEnvironments::AzureChina.storage_endpoint_suffix
             '.core.chinacloudapi.cn'
           when ENVIRONMENT_AZURE_US_GOVERNMENT
-            # MsRestAzure::AzureEnvironments::AzureUSGovernment.storage_endpoint_suffix
             '.core.usgovcloudapi.net'
           when ENVIRONMENT_AZURE_GERMAN_CLOUD
-            # MsRestAzure::AzureEnvironments::AzureGermanCloud.storage_endpoint_suffix
             '.core.cloudapi.de'
           else
-            # MsRestAzure::AzureEnvironments::Azure.storage_endpoint_suffix
             '.core.windows.net'
           end
         end
@@ -135,11 +99,6 @@ module Fog
           "#{protocol}://#{storage_account_name}.#{domain}"
         end
 
-        def current_time
-          time = Time.now.to_f.to_s
-          time.split(/\W+/).join
-        end
-
         # Parse storage blob/container to a hash
         def parse_storage_object(object)
           data = {}
@@ -172,21 +131,6 @@ module Fog
           data
         end
 
-        def resource_not_found?(azure_operation_error)
-          is_found = false
-          if azure_operation_error.response.status == HTTP_NOT_FOUND
-            if azure_operation_error.body['code']
-              is_found = azure_operation_error.body['code'] == ERROR_CODE_NOT_FOUND
-            elsif azure_operation_error.body['error']
-              is_found = azure_operation_error.body['error']['code'] == ERROR_CODE_NOT_FOUND ||
-                         azure_operation_error.body['error']['code'] == ERROR_CODE_RESOURCE_GROUP_NOT_FOUND ||
-                         azure_operation_error.body['error']['code'] == ERROR_CODE_RESOURCE_NOT_FOUND ||
-                         azure_operation_error.body['error']['code'] == ERROR_CODE_PARENT_RESOURCE_NOT_FOUND
-            end
-          end
-          is_found
-        end
-
         def get_image_name(id)
           id.split('/').last
         end

data/lib/fog/azurerm/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AzureRM
-    VERSION = '1.7.0'.freeze
+    VERSION = '1.9.0'.freeze
   end
 end

data/lib/fog/azurerm.rb

@@ -1,4 +1,3 @@
-require 'ms_rest_azure'
 require 'azure/core/http/http_error'
 require 'erb'
 require 'fog/azurerm/config'
@@ -12,19 +11,13 @@ require 'fog/azurerm/models/storage/sku_tier'
 require 'fog/azurerm/models/storage/kind'
 
 module Fog
-  # Autoload Module for Credentials
-  module Credentials
-    autoload :AzureRM, File.expand_path('azurerm/credentials', __dir__)
-  end
-
-  # Autoload Module for Storage
-  module Storage
-    autoload :AzureRM, File.expand_path('azurerm/storage', __dir__)
-  end
-
   # Main AzureRM fog Provider Module
   module AzureRM
     extend Fog::Provider
+
+    # Autoload Module for Storage
+    autoload :Storage, File.expand_path('azurerm/storage', __dir__)
+
     service(:storage, 'Storage')
   end
 end

data/rakefile

@@ -10,6 +10,10 @@ task :test do
   Dir.glob('test/requests/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
 end
 
+task :integration do
+  Dir.glob('test/integration/*.rb').each { |file| require File.expand_path file, __dir__ }
+end
+
 desc 'Generates a coverage report for minitest-cases using simple-cov'
 task :coverage do
   ENV['CODECLIMATE_REPO_TOKEN'] = nil
@@ -27,8 +31,3 @@ task :cc_coverage do
   Dir.glob('test/models/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
   Dir.glob('test/requests/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
 end
-
-desc 'Generates a coverage report for integration tests'
-task :integration_tests do
-  Dir.glob('test/smoke_tests/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
-end

data/test/api_stub/models/storage/directory.rb

@@ -22,7 +22,7 @@ module ApiStub
         end
 
         def self.container_https_url
-          'https://sa.blob.core.windows.net/test_container?comp=list&restype=container'
+          'https://mockaccount.blob.core.windows.net/test_container?comp=list&restype=container'
         end
 
         def self.container_list

data/test/api_stub/models/storage/file.rb

@@ -34,7 +34,7 @@ module ApiStub
         end
 
         def self.blob_https_url
-          'https://sa.blob.core.windows.net/test_container/test_blob'
+          'https://mockaccount.blob.core.windows.net/test_container/test_blob'
         end
 
         def self.blob_metadata

data/test/api_stub/requests/storage/directory.rb

@@ -37,7 +37,7 @@ module ApiStub
         end
 
         def self.container_https_url
-          'https://sa.blob.core.windows.net/test_container?comp=list&restype=container'
+          'https://mockaccount.blob.core.windows.net/test_container?comp=list&restype=container'
         end
 
         def self.container_metadata

data/test/api_stub/requests/storage/file.rb

@@ -46,13 +46,13 @@ module ApiStub
         def self.blob_https_url(environment = Fog::AzureRM::ENVIRONMENT_AZURE_CLOUD)
           case environment
           when Fog::AzureRM::ENVIRONMENT_AZURE_CHINA_CLOUD
-            'https://sa.blob.core.chinacloudapi.cn/test_container/test_blob'
+            'https://mockaccount.blob.core.chinacloudapi.cn/test_container/test_blob'
           when Fog::AzureRM::ENVIRONMENT_AZURE_US_GOVERNMENT
-            'https://sa.blob.core.usgovcloudapi.net/test_container/test_blob'
+            'https://mockaccount.blob.core.usgovcloudapi.net/test_container/test_blob'
           when Fog::AzureRM::ENVIRONMENT_AZURE_GERMAN_CLOUD
-            'https://sa.blob.core.cloudapi.de/test_container/test_blob'
+            'https://mockaccount.blob.core.cloudapi.de/test_container/test_blob'
           else
-            'https://sa.blob.core.windows.net/test_container/test_blob'
+            'https://mockaccount.blob.core.windows.net/test_container/test_blob'
           end
         end
 

data/test/integration/README.md

@@ -1,14 +1,12 @@
 # Integration Tests
 
-To run Integration Tests, first enter following information from your Azure subscription in **credentials\azure.yml** file:
+To run Integration Tests, first enter the following information for your test storage account into **credentials\azure.yml** file:
 
-- tenant_id
-- client_id
-- client_secret
-- subscription_id
+- `azure_storage_account_name`
+- `azure_storage_access_key`
 
-Then run **ruby file_name.rb** to run integration test for a specific service e.g. For integration tests of storage account, run:
+Then run **bundle exec ruby file_name.rb** to run integration test for a specific service e.g. For integration tests for blobs, run:
 
-**ruby storage_account.rb**
+**bundle exec ruby test/integration/blob.rb**
 
 Also make sure the **DEBUG** flag is set in your fog environment to see proper logging.