gitlab-fog-azure-rm 1.8.0 → 1.9.0

data/lib/fog/azurerm/requests/storage/list_containers.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       # https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
       class Real

data/lib/fog/azurerm/requests/storage/put_blob_block.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_blob_block(container_name, blob_name, block_id, data, options = {})

data/lib/fog/azurerm/requests/storage/put_blob_http_url.rb

@@ -0,0 +1,39 @@
+module Fog
+  module AzureRM
+    class Storage
+      # This class provides the actual implemention for service calls.
+      class Real
+        # Generate a pre-signed URL for create an object in an Azure blob storage
+        #
+        # @param container_name [String] Name of container containing blob
+        # @param blob_name [String] Name of blob to get expiring url for
+        # @param expires [Time] An expiry time for this url
+        #
+        # @return [String] - https url for blob
+        #
+        # @see https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob
+        #
+        def put_blob_http_url(container_name, blob_name, expires)
+          relative_path = "#{container_name}/#{blob_name}"
+          params = {
+            service: 'b',
+            resource: 'b',
+            permissions: 'c',
+            expiry: expires.utc.iso8601
+          }
+          token = signature_client(expires).generate_service_sas_token(relative_path, params)
+          uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
+          url = "#{uri}?#{token}"
+          url.sub('https:', 'http:')
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def put_blob_http_url(*)
+          'http://mockaccount.blob.core.windows.net/test_container/test_blob?token'
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/put_blob_https_url.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         # Generate a pre-signed URL for create an object in an Azure blob storage
@@ -22,7 +22,7 @@ module Fog
             expiry: expires.utc.iso8601,
             protocol: 'https'
           }
-          token = @signature_client.generate_service_sas_token(relative_path, params)
+          token = signature_client(expires).generate_service_sas_token(relative_path, params)
           uri = @blob_client.generate_uri(relative_path, {}, { encode: true })
           "#{uri}?#{token}"
         end
@@ -31,7 +31,7 @@ module Fog
       # This class provides the mock implementation for unit tests.
       class Mock
         def put_blob_https_url(*)
-          'https://sa.blob.core.windows.net/test_container/test_blob?token'
+          'https://mockaccount.blob.core.windows.net/test_container/test_blob?token'
         end
       end
     end

data/lib/fog/azurerm/requests/storage/put_blob_metadata.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_blob_metadata(container_name, name, metadata, options = {})

data/lib/fog/azurerm/requests/storage/put_blob_pages.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_blob_pages(container_name, blob_name, start_range, end_range, data, options = {})

data/lib/fog/azurerm/requests/storage/put_blob_properties.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         def put_blob_properties(container_name, name, options = {})

data/lib/fog/azurerm/requests/storage/put_container_acl.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_container_acl(name, acl, options = {})

data/lib/fog/azurerm/requests/storage/put_container_metadata.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def put_container_metadata(name, metadata, options = {})

data/lib/fog/azurerm/requests/storage/put_object_url.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implemention for service calls.
       class Real
         # Generate a pre-signed URL for create an object in an Azure blob storage

data/lib/fog/azurerm/requests/storage/release_blob_lease.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def release_blob_lease(container_name, name, lease_id, options = {})

data/lib/fog/azurerm/requests/storage/release_container_lease.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def release_container_lease(name, lease_id, options={})

data/lib/fog/azurerm/requests/storage/save_page_blob.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         # This class is used to store chunk data for page blob before uploading.

data/lib/fog/azurerm/requests/storage/wait_blob_copy_operation_to_finish.rb

@@ -1,6 +1,6 @@
 module Fog
-  module Storage
-    class AzureRM
+  module AzureRM
+    class Storage
       # This class provides the actual implementation for service calls.
       class Real
         def wait_blob_copy_operation_to_finish(container_name, blob_name, copy_id, copy_status, timeout = nil)

data/lib/fog/azurerm/storage.rb

@@ -1,11 +1,14 @@
 module Fog
-  module Storage
+  module AzureRM
     # This class registers models, requests and collections
-    class AzureRM < Fog::Service
+    class Storage < Fog::Service
       # Recognizes when creating data client
       recognizes :azure_storage_account_name
       recognizes :azure_storage_access_key
+      recognizes :azure_storage_token_signer
+      recognizes :azure_storage_endpoint
       recognizes :azure_storage_domain
+      recognizes :environment
 
       recognizes :debug
 
@@ -41,6 +44,7 @@ module Fog
       request :get_blob_https_url
       request :create_block_blob
       request :put_blob_block
+      request :put_blob_http_url
       request :put_blob_https_url
       request :put_object_url
       request :delete_blob
@@ -83,33 +87,82 @@ module Fog
             require 'securerandom'
             @debug = ENV['DEBUG'] || options[:debug]
             require 'azure/core/http/debug_filter' if @debug
+            require 'fog/azurerm/identity_encoding_filter'
           rescue LoadError => e
             retry if require('rubygems')
             raise e.message
           end
 
           return unless @azure_storage_account_name != options[:azure_storage_account_name] ||
-                        @azure_storage_access_key != options[:azure_storage_access_key]
+                        @azure_storage_access_key != options[:azure_storage_access_key] ||
+                        @azure_storage_token_signer != options[:azure_storage_token_signer]
 
           @azure_storage_account_name = options[:azure_storage_account_name]
           @azure_storage_access_key = options[:azure_storage_access_key]
+          @azure_storage_token_signer = options[:azure_storage_token_signer]
+          @azure_storage_endpoint = options[:azure_storage_endpoint]
           @azure_storage_domain = options[:azure_storage_domain]
 
-          domain =
+          options[:environment] = 'AzureCloud' if options[:environment].nil?
+          @environment = options[:environment]
+
+          storage_blob_host =
+            @azure_storage_endpoint ||
             if @azure_storage_domain.nil? || @azure_storage_domain.empty?
               get_blob_endpoint(@azure_storage_account_name, true, @environment)
             else
               get_blob_endpoint_with_domain(@azure_storage_account_name, true, @azure_storage_domain)
             end
 
-          azure_client = Azure::Storage::Common::Client.create(storage_account_name: @azure_storage_account_name,
-                                                               storage_access_key: @azure_storage_access_key,
-                                                               storage_blob_host: domain)
+          azure_client = Azure::Storage::Common::Client.create({
+            storage_account_name: @azure_storage_account_name,
+            storage_access_key: @azure_storage_access_key,
+            signer: @azure_storage_token_signer
+          }.compact)
+          azure_client.storage_blob_host = storage_blob_host
           @blob_client = Azure::Storage::Blob::BlobService.new(client: azure_client)
+          @blob_client.with_filter(Fog::AzureRM::IdentityEncodingFilter.new)
           @blob_client.with_filter(Azure::Storage::Common::Core::Filter::ExponentialRetryPolicyFilter.new)
           @blob_client.with_filter(Azure::Core::Http::DebugFilter.new) if @debug
-          @signature_client = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(@azure_storage_account_name,
-                                                                                            @azure_storage_access_key)
+        end
+
+        private
+
+        def signature_client(requested_expiry)
+          access_key = @azure_storage_access_key.to_s
+          user_delegation_key = user_delegation_key(requested_expiry)
+
+          # invalidate cache when the delegation key changes
+          unless @signature_client_delegation_key == user_delegation_key
+            @signature_client_delegation_key = user_delegation_key
+            @signature_client = nil
+          end
+
+          @signature_client ||= Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(
+            @azure_storage_account_name,
+            access_key,
+            user_delegation_key
+          )
+        end
+
+        def user_delegation_key(requested_expiry)
+          return nil unless @azure_storage_token_signer
+
+          @user_delegation_key_mutex ||= Mutex.new
+          @user_delegation_key_mutex.synchronize do
+            if @user_delegation_key_expiry.nil? || @user_delegation_key_expiry < requested_expiry
+              start = Time.now
+              expiry = start + Azure::Storage::Blob::BlobConstants::MAX_USER_DELEGATION_KEY_SECONDS
+
+              @user_delegation_key = @blob_client.get_user_delegation_key(
+                start,
+                expiry
+              )
+              @user_delegation_key_expiry = expiry
+            end
+          end
+
+          @user_delegation_key
         end
       end
     end

data/lib/fog/azurerm/utilities/general.rb

@@ -99,11 +99,6 @@ module Fog
           "#{protocol}://#{storage_account_name}.#{domain}"
         end
 
-        def current_time
-          time = Time.now.to_f.to_s
-          time.split(/\W+/).join
-        end
-
         # Parse storage blob/container to a hash
         def parse_storage_object(object)
           data = {}

data/lib/fog/azurerm/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AzureRM
-    VERSION = '1.8.0'.freeze
+    VERSION = '1.9.0'.freeze
   end
 end

data/lib/fog/azurerm.rb

@@ -11,14 +11,13 @@ require 'fog/azurerm/models/storage/sku_tier'
 require 'fog/azurerm/models/storage/kind'
 
 module Fog
-  # Autoload Module for Storage
-  module Storage
-    autoload :AzureRM, File.expand_path('azurerm/storage', __dir__)
-  end
-
   # Main AzureRM fog Provider Module
   module AzureRM
     extend Fog::Provider
+
+    # Autoload Module for Storage
+    autoload :Storage, File.expand_path('azurerm/storage', __dir__)
+
     service(:storage, 'Storage')
   end
 end

data/rakefile

@@ -10,6 +10,10 @@ task :test do
   Dir.glob('test/requests/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
 end
 
+task :integration do
+  Dir.glob('test/integration/*.rb').each { |file| require File.expand_path file, __dir__ }
+end
+
 desc 'Generates a coverage report for minitest-cases using simple-cov'
 task :coverage do
   ENV['CODECLIMATE_REPO_TOKEN'] = nil
@@ -27,8 +31,3 @@ task :cc_coverage do
   Dir.glob('test/models/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
   Dir.glob('test/requests/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
 end
-
-desc 'Generates a coverage report for integration tests'
-task :integration_tests do
-  Dir.glob('test/smoke_tests/**/test_*.rb').each { |file| require File.expand_path file, __dir__ }
-end

data/test/api_stub/models/storage/directory.rb

@@ -22,7 +22,7 @@ module ApiStub
         end
 
         def self.container_https_url
-          'https://sa.blob.core.windows.net/test_container?comp=list&restype=container'
+          'https://mockaccount.blob.core.windows.net/test_container?comp=list&restype=container'
         end
 
         def self.container_list

data/test/api_stub/models/storage/file.rb

@@ -34,7 +34,7 @@ module ApiStub
         end
 
         def self.blob_https_url
-          'https://sa.blob.core.windows.net/test_container/test_blob'
+          'https://mockaccount.blob.core.windows.net/test_container/test_blob'
         end
 
         def self.blob_metadata

data/test/api_stub/requests/storage/directory.rb

@@ -37,7 +37,7 @@ module ApiStub
         end
 
         def self.container_https_url
-          'https://sa.blob.core.windows.net/test_container?comp=list&restype=container'
+          'https://mockaccount.blob.core.windows.net/test_container?comp=list&restype=container'
         end
 
         def self.container_metadata

data/test/api_stub/requests/storage/file.rb

@@ -46,13 +46,13 @@ module ApiStub
         def self.blob_https_url(environment = Fog::AzureRM::ENVIRONMENT_AZURE_CLOUD)
           case environment
           when Fog::AzureRM::ENVIRONMENT_AZURE_CHINA_CLOUD
-            'https://sa.blob.core.chinacloudapi.cn/test_container/test_blob'
+            'https://mockaccount.blob.core.chinacloudapi.cn/test_container/test_blob'
           when Fog::AzureRM::ENVIRONMENT_AZURE_US_GOVERNMENT
-            'https://sa.blob.core.usgovcloudapi.net/test_container/test_blob'
+            'https://mockaccount.blob.core.usgovcloudapi.net/test_container/test_blob'
           when Fog::AzureRM::ENVIRONMENT_AZURE_GERMAN_CLOUD
-            'https://sa.blob.core.cloudapi.de/test_container/test_blob'
+            'https://mockaccount.blob.core.cloudapi.de/test_container/test_blob'
           else
-            'https://sa.blob.core.windows.net/test_container/test_blob'
+            'https://mockaccount.blob.core.windows.net/test_container/test_blob'
           end
         end
 

data/test/integration/README.md

@@ -1,14 +1,12 @@
 # Integration Tests
 
-To run Integration Tests, first enter following information from your Azure subscription in **credentials\azure.yml** file:
+To run Integration Tests, first enter the following information for your test storage account into **credentials\azure.yml** file:
 
-- tenant_id
-- client_id
-- client_secret
-- subscription_id
+- `azure_storage_account_name`
+- `azure_storage_access_key`
 
-Then run **ruby file_name.rb** to run integration test for a specific service e.g. For integration tests of storage account, run:
+Then run **bundle exec ruby file_name.rb** to run integration test for a specific service e.g. For integration tests for blobs, run:
 
-**ruby storage_account.rb**
+**bundle exec ruby test/integration/blob.rb**
 
 Also make sure the **DEBUG** flag is set in your fog environment to see proper logging.

data/test/integration/blob.rb

@@ -8,28 +8,11 @@ require 'yaml'
 
 azure_credentials = YAML.load_file(File.expand_path('credentials/azure.yml', __dir__))
 
-rs = Fog::Resources::AzureRM.new(
-  tenant_id: azure_credentials['tenant_id'],
-  client_id: azure_credentials['client_id'],
-  client_secret: azure_credentials['client_secret'],
-  subscription_id: azure_credentials['subscription_id']
-)
-
-storage = Fog::Storage::AzureRM.new(
-  tenant_id: azure_credentials['tenant_id'],
-  client_id: azure_credentials['client_id'],
-  client_secret: azure_credentials['client_secret'],
-  subscription_id: azure_credentials['subscription_id'],
-  environment: azure_credentials['environment']
-)
-
 ########################################################################################################################
 ######################                               Resource names                                #####################
 ########################################################################################################################
 
-time = current_time
-resource_group_name = "Blob-RG-#{time}"
-storage_account_name = "sa#{time}"
+time = Time.now.to_i
 container_name = "con#{time}"
 test_container_name = "tcon#{time}"
 
@@ -38,36 +21,25 @@ test_container_name = "tcon#{time}"
 ########################################################################################################################
 
 begin
-  resource_group = rs.resource_groups.create(
-    name: resource_group_name,
-    location: LOCATION
-  )
-
-  storage_account = storage.storage_accounts.create(
-    name: storage_account_name,
-    location: LOCATION,
-    resource_group: resource_group_name
-  )
-
-  access_key = storage_account.get_access_keys[0].value
-  Fog::Logger.debug access_key.inspect
   storage_data = Fog::Storage.new(
     provider: 'AzureRM',
-    azure_storage_account_name: storage_account.name,
-    azure_storage_access_key: access_key,
+    azure_storage_account_name: azure_credentials['azure_storage_account_name'],
+    azure_storage_access_key: azure_credentials['azure_storage_access_key'],
+    azure_storage_endpoint: azure_credentials['azure_storage_endpoint'],
     environment: azure_credentials['environment']
   )
+  scheme = URI.parse(azure_credentials.fetch('azure_storage_endpoint', 'https:')).scheme
 
   ########################################################################################################################
   ######################                                Create Container                            ######################
   ########################################################################################################################
 
-  storage_data.directories.create(
+  test_container1 = storage_data.directories.create(
     key: container_name,
     public: true
   )
 
-  storage_data.directories.create(
+  test_container2 = storage_data.directories.create(
     key: test_container_name,
     public: false
   )
@@ -270,6 +242,24 @@ begin
 
   puts "Get a http URL with expires: #{test_blob.url(Time.now + 3600, scheme: 'http')}"
 
+  ########################################################################################################################
+  ######################                            Test generated GET and PUT URLs                 ######################
+  ########################################################################################################################
+
+  url_blob_name = 'url-test.txt'
+  url_blob_content = 'Hello world.'
+  get_url = storage_data.public_send("get_blob_#{scheme}_url", container_name, url_blob_name, Time.now + 3600)
+  put_url = storage_data.public_send("put_blob_#{scheme}_url", container_name, url_blob_name, Time.now + 3600)
+
+  response = Faraday.put(put_url, url_blob_content, 'x-ms-blob-type' => 'BlockBlob')
+  raise unless response.success?
+  puts 'Uploaded blob using PUT blob URL'
+
+  response = Faraday.get(get_url)
+  raise unless response.success?
+  raise unless response.body == url_blob_content
+  puts 'Downloaded blob using GET blob URL'
+
   ########################################################################################################################
   ######################                            Lease Blob                                      ######################
   ########################################################################################################################
@@ -292,6 +282,8 @@ begin
   puts "Deleted blob: #{blob.destroy}"
 rescue => ex
   puts "Integration Test for blob is failing: #{ex.inspect}\n#{ex.backtrace.join("\n")}"
+  raise
 ensure
-  resource_group.destroy unless resource_group.nil?
+  test_container1&.destroy
+  test_container2&.destroy
 end

data/test/integration/container.rb

@@ -8,28 +8,11 @@ require 'yaml'
 
 azure_credentials = YAML.load_file(File.expand_path('credentials/azure.yml', __dir__))
 
-rs = Fog::Resources::AzureRM.new(
-  tenant_id: azure_credentials['tenant_id'],
-  client_id: azure_credentials['client_id'],
-  client_secret: azure_credentials['client_secret'],
-  subscription_id: azure_credentials['subscription_id']
-)
-
-storage = Fog::Storage::AzureRM.new(
-  tenant_id: azure_credentials['tenant_id'],
-  client_id: azure_credentials['client_id'],
-  client_secret: azure_credentials['client_secret'],
-  subscription_id: azure_credentials['subscription_id'],
-  environment: azure_credentials['environment']
-)
-
 ########################################################################################################################
 ######################                               Resource names                                #####################
 ########################################################################################################################
 
-time = current_time
-resource_group_name = "Blob-RG-#{time}"
-storage_account_name = "sa#{time}"
+time = Time.now.to_i
 container_name = "con#{time}"
 test_container_name = "tcon#{time}"
 
@@ -38,25 +21,10 @@ test_container_name = "tcon#{time}"
 ########################################################################################################################
 
 begin
-  resource_group = rs.resource_groups.create(
-    name: resource_group_name,
-    location: LOCATION
-  )
-
-  storage_account_name = "sa#{current_time}"
-
-  storage_account = storage.storage_accounts.create(
-    name: storage_account_name,
-    location: LOCATION,
-    resource_group: resource_group_name
-  )
-
-  keys = storage_account.get_access_keys
-  access_key = keys.first.value
-
-  storage_data = Fog::Storage::AzureRM.new(
-    azure_storage_account_name: storage_account.name,
-    azure_storage_access_key: access_key,
+  storage_data = Fog::AzureRM::Storage.new(
+    azure_storage_account_name: azure_credentials['azure_storage_account_name'],
+    azure_storage_access_key: azure_credentials['azure_storage_access_key'],
+    azure_storage_endpoint: azure_credentials['azure_storage_endpoint'],
     environment: azure_credentials['environment']
   )
 
@@ -76,7 +44,7 @@ begin
   )
   puts "Created container: #{container.key}"
 
-  storage_data.directories.create(
+  container2 = storage_data.directories.create(
     key: test_container_name,
     public: true
   )
@@ -155,6 +123,7 @@ begin
   puts "Deleted container: #{container.destroy}"
 rescue => ex
   puts "Integration Test for container is failing: #{ex.inspect}\n#{ex.backtrace.join("\n")}"
+  raise
 ensure
-  resource_group.destroy unless resource_group.nil?
+  container2&.destroy
 end

data/test/integration/credentials/azure.yml

@@ -1,5 +1,4 @@
-client_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-client_secret: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-subscription_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-tenant_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-environment: XXXXXXXX
+azure_storage_account_name: devstoreaccount1
+azure_storage_access_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+azure_storage_endpoint: http://azurite:10000/devstoreaccount1
+environment: AzureCloud