fog-azure-rm 0.1.0 → 0.1.1

data/lib/fog/azurerm/requests/storage/get_container_url.rb

@@ -0,0 +1,34 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implemention for service calls.
+      class Real
+        # Get a public container url from Azure storage container
+        #
+        # @param container_name [String] Name of container
+        #
+        # @return [String] - url for container
+        #
+        def get_container_url(container_name, options = {})
+          query = { 'comp' => 'list', 'restype' => 'container' }
+          uri = @blob_client.generate_uri(container_name, query)
+
+          if options[:scheme] == 'http'
+            uri.to_s.gsub('https:', 'http:')
+          else
+            uri.to_s
+          end
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def get_container_url(_container_name, options = {})
+          url = 'https://sa.blob.core.windows.net/test_container?comp=list&restype=container'
+          url.gsub!('https:', 'http:') if options[:scheme] == 'http'
+          url
+        end
+      end
+    end
+  end
+end

data/lib/fog/azurerm/requests/storage/get_recovery_vault.rb

@@ -19,7 +19,7 @@ module Fog
             raise_azure_exception(e, msg)
           end
           Fog::Logger.debug "Recovery Vault #{name} from Resource Group #{resource_group} retrieved successfully"
-          recovery_vault = JSON.parse(recovery_vault_response)['value']
+          recovery_vault = Fog::JSON.decode(recovery_vault_response)['value']
           recovery_vault.select { |vault| vault['name'].eql? name }[0]
         end
       end
@@ -41,7 +41,7 @@ module Fog
               }
             }]
           }'
-          JSON.parse(body)['value'][0]
+          Fog::JSON.decode(body)['value'][0]
         end
       end
     end

data/lib/fog/azurerm/requests/storage/get_storage_access_keys.rb

@@ -4,12 +4,13 @@ module Fog
       # This class provides the actual implementation for service calls.
       class Real
         def get_storage_access_keys(resource_group, storage_account_name, options = {})
-          msg = "Getting storage access keys for storage account: #{storage_account_name}."
+          options[:request_id] = SecureRandom.uuid
+          msg = "Getting storage access keys for storage account: #{storage_account_name}, #{options}."
           Fog::Logger.debug msg
           begin
             storage_account_keys = @storage_mgmt_client.storage_accounts.list_keys(resource_group, storage_account_name, options)
-          rescue MsRestAzure::AzureOperationError => e
-            raise_azure_exception(e, msg)
+          rescue MsRestAzure::AzureOperationError => ex
+            raise_azure_exception(ex, msg)
           end
           Fog::Logger.debug "Storage access keys for storage account: #{storage_account_name} listed successfully."
           storage_account_keys.keys

data/lib/fog/azurerm/requests/storage/get_storage_account.rb

@@ -8,8 +8,8 @@ module Fog
           Fog::Logger.debug msg
           begin
             storage_account = @storage_mgmt_client.storage_accounts.get_properties(resource_group_name, storage_account_name)
-          rescue MsRestAzure::AzureOperationError => e
-            raise_azure_exception(e, msg)
+          rescue MsRestAzure::AzureOperationError => ex
+            raise_azure_exception(ex, msg)
           end
           Fog::Logger.debug "Getting storage account: #{storage_account_name} successfully."
           storage_account

data/lib/fog/azurerm/requests/storage/list_blobs.rb

@@ -1,17 +1,50 @@
 module Fog
   module Storage
     class AzureRM
-      # This class provides the actual implemention for service calls.
+      # This class provides the actual implementation for service calls.
+      # https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx
       class Real
         def list_blobs(container_name, options = {})
-          Fog::Logger.debug "Listing blobs in container: #{container_name}"
+          options = options.dup
+          options[:metadata] = true
+          next_marker = nil
+          blobs = []
+
+          msg = nil
+
+          max_results = -1
+          max_results = options[:max_results].to_i if options[:max_results]
           begin
-            blobs = @blob_client.list_blobs(container_name, options)
-            Fog::Logger.debug "Listing blobs in container: #{container_name} successfully."
-            blobs
+            loop do
+              options[:request_id] = SecureRandom.uuid
+              msg = "Listing blobs in container: #{container_name}, options: #{options}"
+              Fog::Logger.debug msg
+              temp = @blob_client.list_blobs(container_name, options)
+              # Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
+              raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)
+
+              blobs += temp unless temp.empty?
+              break if temp.continuation_token.nil? || temp.continuation_token.empty?
+              options[:marker] = temp.continuation_token
+
+              next if max_results == -1
+
+              options[:max_results] = max_results - blobs.size
+              if options[:max_results].zero?
+                next_marker = temp.continuation_token
+                break
+              end
+            end
           rescue Azure::Core::Http::HTTPError => ex
-            raise "Exception in listing blobs: #{ex.inspect}"
+            raise 'NotFound' if ex.message.include?('(404)')
+            raise_azure_exception(ex, msg)
           end
+
+          Fog::Logger.debug "Listing blobs in container: #{container_name} successfully."
+          {
+            next_marker: next_marker,
+            blobs: blobs
+          }
         end
       end
 
@@ -19,11 +52,13 @@ module Fog
       class Mock
         def list_blobs(*)
           Fog::Logger.debug 'Listing blobs in container successfully.'
-          [
-            {
-              'name' => 'testblob1',
-              'properties' =>
-                {
+          {
+            next_marker: 'marker',
+            blobs: [
+              {
+                'name' => 'test_blob1',
+                'metadata' => {},
+                'properties' => {
                   'last_modified' => 'Mon, 04 Jul 2016 02:50:20 GMT',
                   'etag' => '0x8D3A3B5F017F52D',
                   'lease_status' => 'unlocked',
@@ -39,18 +74,69 @@ module Fog
                   'blob_type' => 'PageBlob',
                   'copy_id' => '095adc3b-e277-4c3d-97e0-0abca881f60c',
                   'copy_status' => 'success',
-                  'copy_source' => 'https://testaccount.blob.core.windows.net/testblob/testblob1?snapshot=2016-02-04T08%3A35%3A50.3256874Z',
+                  'copy_source' => 'https://testaccount.blob.core.windows.net/test_container/test_blob?snapshot=2016-02-04T08%3A35%3A50.3256874Z',
                   'copy_progress' => '4194304/4194304',
                   'copy_completion_time' => 'Thu, 04 Feb 2016 08:35:52 GMT',
                   'copy_status_description' => nil,
                   'accept_ranges' => 0
-                },
-              'metadata' => {}
-            },
-            {
-              'name' => 'testblob2',
-              'properties' =>
-                {
+                }
+              },
+              {
+                'name' => 'test_blob2',
+                'metadata' => {},
+                'properties' => {
+                  'last_modified' => 'Tue, 04 Aug 2015 06:02:08 GMT',
+                  'etag' => '0x8D29C92173526C8',
+                  'lease_status' => 'unlocked',
+                  'lease_state' => 'available',
+                  'content_length' => 4_194_304,
+                  'content_type' => 'application/octet-stream',
+                  'content_encoding' => nil,
+                  'content_language' => nil,
+                  'content_disposition' => nil,
+                  'content_md5' => 'tXAohIyxuu/t94Lp/ujeRw==',
+                  'cache_control' => nil,
+                  'sequence_number' => 0,
+                  'blob_type' => 'PageBlob',
+                  'copy_id' => '0abcdc3b-4c3d-e277-97e0-0abca881f60c',
+                  'copy_status' => 'success',
+                  'copy_source' => 'https://testaccount.blob.core.windows.net/test_container/test_blob?snapshot=2016-02-04T08%3A35%3A55.3157696Z',
+                  'copy_progress' => '4194304/4194304',
+                  'copy_completion_time' => 'Thu, 04 Feb 2016 08:40:52 GMT',
+                  'copy_status_description' => nil,
+                  'accept_ranges' => 0
+                }
+              },
+              {
+                'name' => 'test_blob3',
+                'metadata' => {},
+                'properties' => {
+                  'last_modified' => 'Tue, 04 Aug 2015 06:02:08 GMT',
+                  'etag' => '0x8D29C92173526C8',
+                  'lease_status' => 'unlocked',
+                  'lease_state' => 'available',
+                  'content_length' => 4_194_304,
+                  'content_type' => 'application/octet-stream',
+                  'content_encoding' => nil,
+                  'content_language' => nil,
+                  'content_disposition' => nil,
+                  'content_md5' => 'tXAohIyxuu/t94Lp/ujeRw==',
+                  'cache_control' => nil,
+                  'sequence_number' => 0,
+                  'blob_type' => 'PageBlob',
+                  'copy_id' => '0abcdc3b-4c3d-e277-97e0-0abca881f60c',
+                  'copy_status' => 'success',
+                  'copy_source' => 'https://testaccount.blob.core.windows.net/test_container/test_blob?snapshot=2016-02-04T08%3A35%3A55.3157696Z',
+                  'copy_progress' => '4194304/4194304',
+                  'copy_completion_time' => 'Thu, 04 Feb 2016 08:40:52 GMT',
+                  'copy_status_description' => nil,
+                  'accept_ranges' => 0
+                }
+              },
+              {
+                'name' => 'test_blob4',
+                'metadata' => {},
+                'properties' => {
                   'last_modified' => 'Tue, 04 Aug 2015 06:02:08 GMT',
                   'etag' => '0x8D29C92173526C8',
                   'lease_status' => 'unlocked',
@@ -66,15 +152,15 @@ module Fog
                   'blob_type' => 'PageBlob',
                   'copy_id' => '0abcdc3b-4c3d-e277-97e0-0abca881f60c',
                   'copy_status' => 'success',
-                  'copy_source' => 'https://testaccount.blob.core.windows.net/testblob/testblob2?snapshot=2016-02-04T08%3A35%3A55.3157696Z',
+                  'copy_source' => 'https://testaccount.blob.core.windows.net/test_container/test_blob?snapshot=2016-02-04T08%3A35%3A55.3157696Z',
                   'copy_progress' => '4194304/4194304',
                   'copy_completion_time' => 'Thu, 04 Feb 2016 08:40:52 GMT',
                   'copy_status_description' => nil,
                   'accept_ranges' => 0
-                },
-              'metadata' => {}
-            }
-          ]
+                }
+              }
+            ]
+          }
         end
       end
     end

data/lib/fog/azurerm/requests/storage/list_containers.rb

@@ -2,15 +2,30 @@ module Fog
   module Storage
     class AzureRM
       # This class provides the actual implementation for service calls.
+      # https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
       class Real
-        def list_containers(options = {})
-          msg = 'Listing containers.'
-          Fog::Logger.debug msg
+        def list_containers
+          options = { metadata: true }
+          containers = []
+          msg = nil
+
           begin
-            containers = @blob_client.list_containers(options)
+            loop do
+              options[:request_id] = SecureRandom.uuid
+              msg = "Listing containers. options: #{options}"
+              Fog::Logger.debug msg
+              temp = @blob_client.list_containers(options)
+              # Workaround for the issue https://github.com/Azure/azure-storage-ruby/issues/37
+              raise temp unless temp.instance_of?(Azure::Service::EnumerationResults)
+
+              containers += temp unless temp.empty?
+              break if temp.continuation_token.nil? || temp.continuation_token.empty?
+              options[:marker] = temp.continuation_token
+            end
           rescue Azure::Core::Http::HTTPError => ex
             raise_azure_exception(ex, msg)
           end
+
           Fog::Logger.debug 'Listing containers successfully.'
           containers
         end
@@ -18,29 +33,37 @@ module Fog
 
       # This class provides the mock implementation for unit tests.
       class Mock
-        def list_containers(*)
+        def list_containers
           [
             {
-              'name' => 'testcontainer1',
-              'properties' =>
-                {
-                  'last_modified' => 'Mon, 04 Jul 2016 02:50:20 GMT',
-                  'etag' => '0x8D3A3B5F017F52D',
-                  'lease_status' => 'unlocked',
-                  'lease_state' => 'available'
-                },
-              'metadata' => {}
+              'name' => 'test_container1',
+              'metadata' => {},
+              'properties' => {
+                'last_modified' => 'Mon, 04 Jul 2016 02:50:20 GMT',
+                'etag' => '0x8D3A3B5F017F52D',
+                'lease_status' => 'unlocked',
+                'lease_state' => 'available'
+              }
+            },
+            {
+              'name' => 'test_container2',
+              'metadata' => {},
+              'properties' => {
+                'last_modified' => 'Tue, 04 Aug 2015 06:01:08 GMT',
+                'etag' => '0x8D29C92176C8352',
+                'lease_status' => 'unlocked',
+                'lease_state' => 'available'
+              }
             },
             {
-              'name' => 'testcontainer2',
-              'properties' =>
-                {
-                  'last_modified' => 'Tue, 04 Aug 2015 06:01:08 GMT',
-                  'etag' => '0x8D29C92176C8352',
-                  'lease_status' => 'unlocked',
-                  'lease_state' => 'available'
-                },
-              'metadata' => {}
+              'name' => 'test_container3',
+              'metadata' => {},
+              'properties' => {
+                'last_modified' => 'Tue, 01 Sep 2015 05:15:36 GMT',
+                'etag' => '0x8D2B28C5EB36458',
+                'lease_status' => 'unlocked',
+                'lease_state' => 'available'
+              }
             }
           ]
         end

data/lib/fog/azurerm/requests/storage/list_recovery_vaults.rb

@@ -19,7 +19,7 @@ module Fog
             raise_azure_exception(e, msg)
           end
           Fog::Logger.debug "Recovery Vaults in Resource Group #{resource_group} listed successfully"
-          JSON.parse(recovery_vaults_response)['value']
+          Fog::JSON.decode(recovery_vaults_response)['value']
         end
       end
 
@@ -40,7 +40,7 @@ module Fog
               }
             }]
           }'
-          JSON.parse(body)['value']
+          Fog::JSON.decode(body)['value']
         end
       end
     end

data/lib/fog/azurerm/requests/storage/list_storage_account_for_rg.rb

@@ -8,8 +8,8 @@ module Fog
           Fog::Logger.debug msg
           begin
             result = @storage_mgmt_client.storage_accounts.list_by_resource_group(resource_group)
-          rescue MsRestAzure::AzureOperationError => e
-            raise_azure_exception(e, msg)
+          rescue MsRestAzure::AzureOperationError => ex
+            raise_azure_exception(ex, msg)
           end
           result.value
         end

data/lib/fog/azurerm/requests/storage/list_storage_accounts.rb

@@ -8,8 +8,8 @@ module Fog
           Fog::Logger.debug msg
           begin
             result = @storage_mgmt_client.storage_accounts.list
-          rescue MsRestAzure::AzureOperationError => e
-            raise_azure_exception(e, msg)
+          rescue MsRestAzure::AzureOperationError => ex
+            raise_azure_exception(ex, msg)
           end
           result.value
         end

data/lib/fog/azurerm/requests/storage/multipart_save_block_blob.rb

@@ -0,0 +1,110 @@
+module Fog
+  module Storage
+    class AzureRM
+      # This class provides the actual implementation for service calls.
+      class Real
+        # This class is used to store chunk data for block blob before uploading.
+        class BlockChunk
+          attr_reader :id # For debug
+          attr_reader :block_id
+          attr_reader :data
+
+          def initialize(id, block_id, data)
+            @id = id
+            @block_id = block_id
+            @data = data
+          end
+        end
+
+        # This class is a stream to read chunk data.
+        class BlockFileStream
+          attr_reader :blocks
+
+          def initialize(body)
+            if body.respond_to?(:read)
+              if body.respond_to?(:rewind)
+                begin
+                  body.rewind
+                rescue => ex
+                  Fog::Logger.debug "multipart_save_block_blob - body responds to :rewind but throws an exception when calling :rewind: #{ex.inspect}"
+                end
+              end
+              @stream = body
+            else
+              @stream = StringIO.new(body)
+            end
+            @mutex = Mutex.new
+            @blocks = []
+          end
+
+          def read(size)
+            block_id = Base64.strict_encode64(random_string(32))
+            data = nil
+            id = 0
+            @mutex.synchronize do
+              data = @stream.read(size)
+              return nil if data.nil?
+              @blocks << [block_id]
+              id = @blocks.size
+            end
+            BlockChunk.new(id, block_id, data)
+          end
+        end
+
+        def multipart_save_block_blob(container_name, blob_name, body, options)
+          threads_num = options.delete(:worker_thread_num)
+          threads_num = UPLOAD_BLOB_WORKER_THREAD_COUNT if threads_num.nil? || !threads_num.is_a?(Integer) || threads_num < 1
+
+          begin
+            # Initiate the upload
+            Fog::Logger.debug "Creating the block blob #{container_name}/#{blob_name}. options: #{options}"
+            content_md5 = options.delete(:content_md5)
+            create_block_blob(container_name, blob_name, nil, options)
+
+            # Uploading parts
+            Fog::Logger.debug "Starting to upload parts for the block blob #{container_name}/#{blob_name}."
+            iostream = BlockFileStream.new(body)
+
+            threads = []
+            threads_num.times do |id|
+              thread = Thread.new do
+                Fog::Logger.debug "Created upload thread #{id}."
+                while (chunk = iostream.read(MAXIMUM_CHUNK_SIZE))
+                  Fog::Logger.debug "Upload thread #{id} is uploading #{chunk.id}, size: #{chunk.data.size}, options: #{options}."
+                  put_blob_block(container_name, blob_name, chunk.block_id, chunk.data, options)
+                end
+                Fog::Logger.debug "Upload thread #{id} finished."
+              end
+              thread.abort_on_exception = true
+              threads << thread
+            end
+
+            threads.each(&:join)
+            # Complete the upload
+            options[:content_md5] = content_md5 unless content_md5.nil?
+            Fog::Logger.debug "Commiting the block blob #{container_name}/#{blob_name}. options: #{options}"
+            commit_blob_blocks(container_name, blob_name, iostream.blocks, options)
+          rescue
+            # Abort the upload & reraise
+            begin
+              delete_blob(container_name, blob_name)
+            rescue => ex
+              Fog::Logger.debug "Cannot delete the blob: #{container_name}/#{blob_name} after multipart_save_block_blob failed. #{ex.inspect}"
+            end
+            raise
+          end
+
+          Fog::Logger.debug "Successfully save the block blob: #{container_name}/#{blob_name}."
+          true
+        end
+      end
+
+      # This class provides the mock implementation for unit tests.
+      class Mock
+        def multipart_save_block_blob(*)
+          true
+        end
+      end
+    end
+  end
+end