azure-storage 0.14.0.preview → 0.15.0.preview

data/lib/azure/storage/configurable.rb

@@ -60,7 +60,11 @@ module Azure::Storage
     attr_writer :storage_table_host,
                 :storage_blob_host,
                 :storage_queue_host,
-                :storage_file_host
+                :storage_file_host,
+                :storage_table_host_secondary,
+                :storage_blob_host_secondary,
+                :storage_queue_host_secondary,
+                :storage_file_host_secondary
 
     attr_reader :signer
 
@@ -76,7 +80,8 @@ module Azure::Storage
           :storage_table_host,
           :storage_blob_host,
           :storage_queue_host,
-          :storage_file_host
+          :storage_file_host,
+          :signer
         ]
       end
     end
@@ -93,12 +98,18 @@ module Azure::Storage
     # Reset configuration options to default values
     def reset_config!(options = {})
       Azure::Storage::Configurable.keys.each do |key|
-        value = if self == Azure::Storage
-          Azure::Storage::Default.options[key]
-                else
-                  Azure::Storage.send(key)
-                end
+        value =
+          if self == Azure::Storage
+            Azure::Storage::Default.options[key]
+          else
+            Azure::Storage.send(key)
+          end
         instance_variable_set(:"@#{key}", options.fetch(key, value))
+
+        # Set the secondary endpoint if the primary one is given
+        if key.to_s.include? "host"
+          instance_variable_set(:"@#{key}_secondary", secondary_endpoint(options.fetch(key, value)))
+        end
       end
       self.send(:reset_agents!) if self.respond_to?(:reset_agents!)
       setup_signer_for_service
@@ -109,32 +120,48 @@ module Azure::Storage
 
     # Storage queue host
     # @return [String]
-    def storage_queue_host
-      @storage_queue_host || default_host(:queue)
+    def storage_queue_host(isSecondary = false)
+      if isSecondary
+        @storage_queue_host_secondary || default_host(:queue, true)
+      else
+        @storage_queue_host || default_host(:queue, false)
+      end
     end
 
     # Storage blob host
     # @return [String]
-    def storage_blob_host
-      @storage_blob_host || default_host(:blob)
+    def storage_blob_host(isSecondary = false)
+      if isSecondary
+        @storage_blob_host_secondary || default_host(:blob, true)
+      else
+        @storage_blob_host || default_host(:blob, false)
+      end
     end
 
     # Storage table host
     # @return [String]
-    def storage_table_host
-      @storage_table_host || default_host(:table)
+    def storage_table_host(isSecondary = false)
+      if isSecondary
+        @storage_table_host_secondary || default_host(:table, true)
+      else
+        @storage_table_host || default_host(:table, false)
+      end
     end
 
     # Storage file host
     # @return [String]
-    def storage_file_host
-      @storage_file_host || default_host(:file)
+    def storage_file_host(isSecondary = false)
+      if isSecondary
+        @storage_file_host_secondary || default_host(:file, true)
+      else
+        @storage_file_host || default_host(:file, false)
+      end
     end
 
     private
 
-      def default_host(service)
-        "https://#{storage_account_name}.#{service}.core.windows.net" if storage_account_name
+      def default_host(service, isSecondary = false)
+        "https://#{storage_account_name}#{isSecondary ? "-secondary" : ""}.#{service}.core.windows.net" if storage_account_name
       end
 
       def setup_options
@@ -152,6 +179,12 @@ module Azure::Storage
         fields[0]
       end
 
+      def secondary_endpoint(primary_endpoint)
+        return nil if primary_endpoint.nil?
+        account_name = account_name_from_endpoint primary_endpoint
+        primary_endpoint.sub account_name, account_name + "-secondary"
+      end
+
       def determine_account_name
         if instance_variable_get(:@storage_account_name).nil?
           hosts = [@storage_blob_host, @storage_table_host, @storage_queue_host, @storage_file_host]

data/lib/azure/storage/core/auth/anonymous_signer.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+#-------------------------------------------------------------------------
+# # Copyright (c) Microsoft and contributors. All rights reserved.
+#
+# The MIT License(MIT)
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files(the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions :
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#--------------------------------------------------------------------------
+
+require "azure/core/auth/signer"
+require "base64"
+
+module Azure::Storage::Core
+  module Auth
+    class AnonymousSigner < Azure::Core::Auth::Signer
+      # Public: Initialize the Anonymous Signer
+      def initialize()
+        # Use mock key to initialize super class
+        super(Base64.strict_encode64("accesskey"))
+      end
+
+      def sign_request(req)
+        # Do nothing.
+      end
+    end
+  end
+end

data/lib/azure/storage/core/filter/retry_filter.rb

@@ -31,6 +31,7 @@ module Azure::Storage::Core::Filter
     def initialize(retry_count = nil, retry_interval = nil)
       @retry_count = retry_count
       @retry_interval = retry_interval
+      @request_options = {}
 
       super &:should_retry?
     end
@@ -44,6 +45,9 @@ module Azure::Storage::Core::Filter
     # response - HttpResponse. The response from the active request
     # retry_data - Hash. Stores stateful retry data
     def should_retry?(response, retry_data)
+      # Fill necessary information
+      init_retry_data retry_data
+
       # Applies the logic when there is subclass overrides it
       apply_retry_policy retry_data
 
@@ -60,7 +64,12 @@ module Azure::Storage::Core::Filter
       should_retry_on_local_error? retry_data
       return false unless should_retry_on_error? response, retry_data
 
-      adjust_retry_parameter retry_data
+      # Determined that it needs to retry.
+      adjust_retry_request retry_data
+
+      wait_for_retry
+
+      retry_data[:retryable]
     end
 
     # Apply the retry policy to determine how the HTTP request should continue retrying
@@ -131,12 +140,82 @@ module Azure::Storage::Core::Filter
         return retry_data[:retryable]
       end
 
+      check_location(response, retry_data)
+
+      check_status_code(retry_data)
+
+      retry_data[:retryable]
+    end
+
+    # Adjust the retry parameter and wait for retry
+    def wait_for_retry
+      sleep @retry_interval
+    end
+
+    # Adjust the retry request
+    #
+    # retry_data - Hash. Stores stateful retry data
+    def adjust_retry_request(retry_data)
+      # Adjust the location first
+      next_location = @request_options[:target_location].nil? ? get_next_location(retry_data) : @request_options[:target_location]
+      retry_data[:current_location] = next_location
+
+      retry_data[:uri] =
+        if next_location == Azure::Storage::StorageLocation::PRIMARY
+          @request_options[:primary_uri]
+        else
+          @request_options[:secondary_uri]
+        end
+
+      # Now is the time to calculate the exact retry interval. ShouldRetry call above already
+      # returned back how long two requests to the same location should be apart from each other.
+      # However, for the reasons explained above, the time spent between the last attempt to
+      # the target location and current time must be subtracted from the total retry interval
+      # that ShouldRetry returned.
+      lastAttemptTime = 
+        if retry_data[:current_location] == Azure::Storage::StorageLocation::PRIMARY
+          retry_data[:last_primary_attempt]
+        else
+          retry_data[:last_secondary_attempt]
+        end
+
+      @retry_interval =
+        if lastAttemptTime.nil?
+          0
+        else
+          since_last_attempt = Time.now - lastAttemptTime
+          retry_data[:interval] - since_last_attempt
+        end
+    end
+
+    # Initialize the retry data
+    #
+    # retry_data - Hash. Stores stateful retry data
+    def init_retry_data(retry_data)
+      @request_options = retry_data[:request_options] unless retry_data[:request_options].nil?
+
+      if retry_data[:current_location].nil?
+        retry_data[:current_location] = Azure::Storage::Service::StorageService.get_location(@request_options[:location_mode], @request_options[:request_location_mode])
+      end
+      
+      if retry_data[:current_location] == Azure::Storage::StorageLocation::PRIMARY
+        retry_data[:last_primary_attempt] = Time.now
+      else
+        retry_data[:last_secondary_attempt] = Time.now
+      end
+
+    end
+
+    # Check the location
+    #
+    # retry_data - Hash. Stores stateful retry data
+    def check_location(response, retry_data)
       # If a request sent to the secondary location fails with 404 (Not Found), it is possible
       # that the resource replication is not finished yet. So, in case of 404 only in the secondary
       # location, the failure should still be retryable.
-      secondary_not_found = (retry_data[:current_location] === Azure::Storage::RequestLocationMode::SECONDARY_ONLY) && response.status_code === 404;
+      retry_data[:secondary_not_found] = (retry_data[:current_location] === Azure::Storage::StorageLocation::SECONDARY) && response.status_code === 404;
 
-      if secondary_not_found
+      if retry_data[:secondary_not_found]
         retry_data[:status_code] = 500
       else
         if (response.status_code)
@@ -145,20 +224,28 @@ module Azure::Storage::Core::Filter
           retry_data[:status_code] = nil
         end
       end
+    end
 
+    # Check the status code
+    #
+    # retry_data - Hash. Stores stateful retry data
+    def check_status_code(retry_data)
       if (retry_data[:status_code] < 400)
         retry_data[:retryable] = false;
-        return false;
       # Non-timeout Cases
       elsif (retry_data[:status_code] != 408)
         # Always no retry on "not implemented" and "version not supported"
         if (retry_data[:status_code] == 501 || retry_data[:status_code] == 505)
           retry_data[:retryable] = false;
-          return false;
+        end
+
+        if (retry_data[:status_code] == 404)
+          retry_data[:retryable] = true;
+          return true;
         end
 
         # When absorb_conditional_errors_on_retry is set (for append blob)
-        if (retry_data[:request_options] && retry_data[:request_options][:absorb_conditional_errors_on_retry])
+        if (@request_options[:absorb_conditional_errors_on_retry])
           if (retry_data[:status_code] == 412)
             # When appending block with precondition failure and their was a server error before, we ignore the error.
             if (retry_data[:last_server_error])
@@ -177,16 +264,40 @@ module Azure::Storage::Core::Filter
           retry_data[:retryable] = false;
         end
       end
-      retry_data[:retryable]
     end
 
-    # Adjust the retry parameter
+    # Get retry request destination
     #
     # retry_data - Hash. Stores stateful retry data
-    def adjust_retry_parameter(retry_data)
-      # TODO: Adjust the retry parameter according to the location and last attempt time
-      sleep retry_data[:interval] if retry_data[:retryable]
-      retry_data[:retryable]
+    def get_next_location(retry_data)
+      # In case of 404 when trying the secondary location, instead of retrying on the
+      # secondary, further requests should be sent only to the primary location, as it most
+      # probably has a higher chance of succeeding there.
+      if retry_data[:secondary_not_found] && @request_options[:location_mode] != Azure::Storage::LocationMode::SECONDARY_ONLY
+        @request_options[:location_mode] = Azure::Storage::LocationMode::PRIMARY_ONLY;
+        return Azure::Storage::StorageLocation::PRIMARY
+      end
+
+      case @request_options[:location_mode]
+      when Azure::Storage::LocationMode::PRIMARY_ONLY
+        Azure::Storage::StorageLocation::PRIMARY
+      when Azure::Storage::LocationMode::SECONDARY_ONLY
+        Azure::Storage::StorageLocation::SECONDARY
+      else
+        # request_location_mode cannot be SECONDARY_ONLY because it will be blocked at the first time
+        if @request_options[:request_location_mode] == Azure::Storage::RequestLocationMode::PRIMARY_ONLY
+          Azure::Storage::StorageLocation::PRIMARY
+        elsif @request_options[:request_location_mode] == Azure::Storage::RequestLocationMode::SECONDARY_ONLY
+          Azure::Storage::StorageLocation::SECONDARY
+        else
+          if retry_data[:current_location] === Azure::Storage::StorageLocation::PRIMARY
+            Azure::Storage::StorageLocation::SECONDARY
+          else
+            Azure::Storage::StorageLocation::PRIMARY
+          end
+        end
+      end
     end
+
   end
 end

data/lib/azure/storage/default.rb

@@ -140,7 +140,16 @@ module Azure::Storage
     FILE = "file"
   end
 
-  # Specifies the location used to indicate which location the operation can be performed against.
+  # Specifies the location mode used to decide which location the request should be sent to.
+  module LocationMode
+    PRIMARY_ONLY = 0
+    PRIMARY_THEN_SECONDARY = 1
+    SECONDARY_ONLY = 2
+    SECONDARY_THEN_PRIMARY = 3
+  end
+
+  # Specifies the location used to indicate which location the operation (REST API) can be performed against.
+  # This is determined by the API and cannot be specified by the users.
   module RequestLocationMode
     PRIMARY_ONLY = 0
     SECONDARY_ONLY = 1

data/lib/azure/storage/file/directory.rb

@@ -75,6 +75,9 @@ module Azure::Storage::File
   # * +:request_id+              - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded
   #                                in the analytics logs when storage analytics logging is enabled.
   #
+  # * +:location_mode+           - LocationMode. Specifies the location mode used to decide 
+  #                                which location the request should be sent to.
+  #
   # See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/list-directories-and-files
   #
   # Returns an Azure::Service::EnumerationResults
@@ -88,7 +91,8 @@ module Azure::Storage::File
       StorageService.with_query query, "prefix", options[:prefix].to_s if options[:prefix]
     end
 
-    uri = directory_uri(share, directory_path, query)
+    options[:request_location_mode] = Azure::Storage::RequestLocationMode::PRIMARY_OR_SECONDARY
+    uri = directory_uri(share, directory_path, query, options)
     response = call(:get, uri, nil, {}, options)
 
     # Result
@@ -156,6 +160,8 @@ module Azure::Storage::File
   # * +:timeout+                  - Integer. A timeout in seconds.
   # * +:request_id+               - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded
   #                                 in the analytics logs when storage analytics logging is enabled.
+  # * +:location_mode+            - LocationMode. Specifies the location mode used to decide 
+  #                                 which location the request should be sent to.
   #
   # See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-directory-properties
   #
@@ -166,7 +172,8 @@ module Azure::Storage::File
     query["timeout"] = options[:timeout].to_s if options[:timeout]
 
     # Call
-    response = call(:get, directory_uri(share, directory_path, query), nil, {}, options)
+    options[:request_location_mode] = Azure::Storage::RequestLocationMode::PRIMARY_OR_SECONDARY
+    response = call(:get, directory_uri(share, directory_path, query, options), nil, {}, options)
 
     # result
     directory = Serialization.directory_from_headers(response.headers)
@@ -218,6 +225,8 @@ module Azure::Storage::File
   # * +:timeout+                  - Integer. A timeout in seconds.
   # * +:request_id+               - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded
   #                                 in the analytics logs when storage analytics logging is enabled.
+  # * +:location_mode+            - LocationMode. Specifies the location mode used to decide 
+  #                                 which location the request should be sent to.
   #
   # See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-directory-metadata
   #
@@ -228,7 +237,8 @@ module Azure::Storage::File
     query["timeout"] = options[:timeout].to_s if options[:timeout]
 
     # Call
-    response = call(:get, directory_uri(share, directory_path, query), nil, {}, options)
+    options[:request_location_mode] = Azure::Storage::RequestLocationMode::PRIMARY_OR_SECONDARY
+    response = call(:get, directory_uri(share, directory_path, query, options), nil, {}, options)
 
     # result
     directory = Serialization.directory_from_headers(response.headers)

data/lib/azure/storage/file/file.rb

@@ -129,6 +129,8 @@ module Azure::Storage::File
   # * +:timeout+                   - Integer. A timeout in seconds.
   # * +:request_id+                - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded
   #                                  in the analytics logs when storage analytics logging is enabled.
+  # * +:location_mode+             - LocationMode. Specifies the location mode used to decide 
+  #                                  which location the request should be sent to.
   #
   # See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file
   #
@@ -136,7 +138,9 @@ module Azure::Storage::File
   def get_file(share, directory_path, file, options = {})
     query = {}
     StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout]
-    uri = file_uri(share, directory_path, file, query)
+
+    options[:request_location_mode] = Azure::Storage::RequestLocationMode::PRIMARY_OR_SECONDARY
+    uri = file_uri(share, directory_path, file, query, options)
 
     headers = StorageService.common_headers
     options[:start_range] = 0 if options[:end_range] && (not options[:start_range])
@@ -167,6 +171,8 @@ module Azure::Storage::File
   # * +:timeout+                   - Integer. A timeout in seconds.
   # * +:request_id+                - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded
   #                                  in the analytics logs when storage analytics logging is enabled.
+  # * +:location_mode+             - LocationMode. Specifies the location mode used to decide 
+  #                                  which location the request should be sent to.
   #
   # See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file-properties
   #
@@ -177,7 +183,8 @@ module Azure::Storage::File
 
     headers = StorageService.common_headers
 
-    uri = file_uri(share, directory_path, file, query)
+    options[:request_location_mode] = Azure::Storage::RequestLocationMode::PRIMARY_OR_SECONDARY
+    uri = file_uri(share, directory_path, file, query, options)
 
     response = call(:head, uri, nil, headers, options)
 
@@ -361,6 +368,8 @@ module Azure::Storage::File
   # * +:timeout+                   - Integer. A timeout in seconds.
   # * +:request_id+                - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded
   #                                  in the analytics logs when storage analytics logging is enabled.
+  # * +:location_mode+             - LocationMode. Specifies the location mode used to decide 
+  #                                  which location the request should be sent to.
   #
   # See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/list-ranges
   #
@@ -372,7 +381,8 @@ module Azure::Storage::File
     query = { "comp" => "rangelist" }
     StorageService.with_query query, "timeout", options[:timeout].to_s if options[:timeout]
 
-    uri = file_uri(share, directory_path, file, query)
+    options[:request_location_mode] = Azure::Storage::RequestLocationMode::PRIMARY_OR_SECONDARY
+    uri = file_uri(share, directory_path, file, query, options)
 
     options[:start_range] = 0 if options[:end_range] && (not options[:start_range])
 
@@ -402,6 +412,8 @@ module Azure::Storage::File
   # * +:timeout+                  - Integer. A timeout in seconds.
   # * +:request_id+               - String. Provides a client-generated, opaque value with a 1 KB character limit that is recorded
   #                                 in the analytics logs when storage analytics logging is enabled.
+  # * +:location_mode+            - LocationMode. Specifies the location mode used to decide 
+  #                                 which location the request should be sent to.
   #
   # See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file-metadata
   #
@@ -412,7 +424,8 @@ module Azure::Storage::File
     query["timeout"] = options[:timeout].to_s if options[:timeout]
 
     # Call
-    response = call(:get, file_uri(share, directory_path, file, query), nil, {}, options)
+    options[:request_location_mode] = Azure::Storage::RequestLocationMode::PRIMARY_OR_SECONDARY
+    response = call(:get, file_uri(share, directory_path, file, query, options), nil, {}, options)
 
     # result
     result = Serialization.file_from_headers(response.headers)