backup-backblaze 0.1.2 → 0.2.0

data/lib/backup/backblaze/retry.rb

@@ -1,77 +1,81 @@
 module Backup
   module Backblaze
     module Retry
-      MAX_RETRIES = 3
+      MAX_RETRIES = 5
 
-      # Use the url and token returned by the next_url_token block, until we get
-      # a reset indicating that we need a new url and token.
-      class TokenProvider
-        def initialize &next_url_token
-          @next_url_token = next_url_token
-          reset
-        end
+      class TooManyRetries < RuntimeError; end
 
-        attr_reader :upload_url, :file_auth_token
+      # This is raised when a an api endpoint needs to be retried in a
+      # complicate way.
+      class RetrySequence < StandardError
+        def initialize retry_sequence, backoff
+          unless retry_sequence.is_a?(Array) && retry_sequence.all?{|s| Symbol === s}
+            raise "provide an array of symbols in #{@retry_sequence.inspect}"
+          end
 
-        def reset
-          @upload_url, @file_auth_token = @next_url_token.call
-          self
+          super retry_sequence.inspect
+          @retry_sequence = retry_sequence
+          @backoff = backoff
         end
-      end
 
-      class TooManyRetries < RuntimeError; end
+        attr_reader :backoff
 
-      # Try up to retries times to call the upload_blk. Recursive.
-      #
-      # Various errors (passed through from Excon) coming out of upload_blk will
-      # be caught. When an error is caught, :reset method called on
-      # token_provider.
-      #
-      # Return whatever upload_blk returns
-      def retry_upload retries, token_provider, &upload_blk
-        raise TooManyRetries, "max retries is #{MAX_RETRIES}" unless retries < MAX_RETRIES
-        sleep retries ** 2 # exponential backoff for retries > 0
-
-        # Called by all the rescue blocks that want to retry.
-        # Mainly so we don't make stoopid errors - like leaving out the +1 for one of the calls :-|
-        retry_lambda = lambda do
-          retry_upload retries + 1, token_provider.reset, &upload_blk
+        def each &blk
+          return enum_for :each unless block_given?
+          @retry_sequence.each &blk
         end
 
-        begin
-          upload_blk.call token_provider, retries
-        rescue Excon::Errors::Error => ex
-          # The most convenient place to log this
-          Backup::Logger.info ex.message
-          raise
-        end
+        include Enumerable
+      end
 
-      # Recoverable errors details sourced from:
-      #   https://www.backblaze.com/b2/docs/integration_checklist.html
-      #   https://www.backblaze.com/b2/docs/uploading.html
+      module_function def call retries, backoff, api_call_name, &blk
+        raise TooManyRetries, "max tries is #{MAX_RETRIES}" unless retries < MAX_RETRIES
 
-      # socket-related, 408, and 429
-      rescue Excon::Errors::SocketError, Excon::Errors::Timeout, Excon::Errors::RequestTimeout, Excon::Errors::TooManyRequests
-        retry_lambda.call
+        # default exponential backoff for retries > 0
+        backoff ||= retries ** 2
 
-      # some 401
-      rescue Excon::Errors::Unauthorized => ex
-        hw = HashWrap.from_json ex.response.body
-        case hw.code
-        when 'bad_auth_token', 'expired_auth_token'
-          retry_lambda.call
+        # minor avoidance of unnecessary work in sleep if there's no backoff needed.
+        if backoff > 0
+          ::Backup::Logger.info "calling #{api_call_name} retry #{retries} after sleep #{backoff}"
+          sleep backoff
         else
-          raise
+          ::Backup::Logger.info "calling #{api_call_name}"
         end
 
-      # 500-599 where the BackBlaze "code" doesn't matter
+        # Finally! Do the call.
+        blk.call
+
       rescue Excon::Errors::HTTPStatusError => ex
-        if (500..599) === ex.response.status
-          retry_lambda.call
+        Backup::Logger.info ex.message
+        # backoff can end up nil, if Retry-After isn't specified.
+        backoff = ex.response.headers['Retry-After']&.to_i
+        ::Backup::Logger.info "server specified Retry-After of #{backoff.inspect}"
+        raise "Retry-After #{backoff} > 60 is too long" if backoff && backoff > 60
+
+        # need to get code from body
+        body_wrap = HashWrap.from_json ex.response.body
+
+        # figure out which retry sequence to use
+        recovery_sequence = RetryLookup.retry_sequence api_call_name, ex.response.status, body_wrap.code
+
+        # There's a sequence of retries, and we don't know how to hook the
+        # return values and parameters together. So make that someone else's
+        # problem.
+        #
+        # TODO possibly just execute the retry sequence here?
+        # That's quite hard cos it will have to have access to the calling self
+        if recovery_sequence.any?
+          ::Backup::Logger.info "recovery sequence of #{recovery_sequence.inspect}"
+          raise RetrySequence.new(recovery_sequence, backoff)
         else
           raise
         end
 
+      rescue Excon::Errors::Error => ex
+        Backup::Logger.info ex.message
+        # Socket errors etc therefore no http status code and no response body.
+        # So just retry with default exponential backoff.
+        call retries + 1, nil, api_call_name, &blk
       end
     end
   end

data/lib/backup/backblaze/retry_lookup.rb

@@ -0,0 +1,112 @@
+require 'set'
+
+module Backup
+  module Backblaze
+    module RetryLookup
+      def (Any = Object.new).=== _other; true end
+
+      module Matcher
+        refine Array do
+          def === other
+            return false unless size == other.size
+            size.times.all? do |idx|
+              self[idx] === other[idx]
+            end
+          end
+        end
+      end
+
+      using Matcher
+
+      # Generated from retry.pl
+      #
+      # Cross-product of all the retry scenarios we know about. This probably
+      # isn't the fastest way to calculate retries. But they're rare, so the
+      # slowdown doesn't matter. There is a more general pattern, but I don't
+      # want to get sucked into implementing unification.
+      module_function def retry_sequence api_call, http_status, code
+        case [api_call.to_sym,        http_status, code.to_sym]
+        when [:b2_upload_part, 401, :expired_auth_token] then [:b2_get_upload_part_url,:b2_upload_part]
+        when [:b2_upload_part, 401, :bad_auth_token] then [:b2_get_upload_part_url,:b2_upload_part]
+        when [:b2_upload_part, 408, Any] then [:b2_get_upload_part_url,:b2_upload_part]
+        when [:b2_upload_part, 500..599, Any] then [:b2_get_upload_part_url,:b2_upload_part]
+        when [:b2_upload_part, 429, Any] then [:b2_upload_part]
+        when [:b2_get_upload_part_url, 401, :expired_auth_token] then [:b2_authorize_account,:b2_get_upload_part_url]
+        when [:b2_get_upload_part_url, 401, :bad_auth_token] then [:b2_authorize_account,:b2_get_upload_part_url]
+        when [:b2_get_upload_part_url, 408, Any] then [:b2_get_upload_part_url]
+        when [:b2_get_upload_part_url, 429, Any] then [:b2_get_upload_part_url]
+        when [:b2_get_upload_part_url, 500..599, Any] then [:b2_get_upload_part_url]
+        when [:b2_get_upload_url, 401, :expired_auth_token] then [:b2_authorize_account,:b2_get_upload_url]
+        when [:b2_get_upload_url, 401, :bad_auth_token] then [:b2_authorize_account,:b2_get_upload_url]
+        when [:b2_get_upload_url, 408, Any] then [:b2_get_upload_url]
+        when [:b2_get_upload_url, 429, Any] then [:b2_get_upload_url]
+        when [:b2_get_upload_url, 500..599, Any] then [:b2_get_upload_url]
+        when [:b2_upload_file, 401, :expired_auth_token] then [:b2_get_upload_url,:b2_upload_file]
+        when [:b2_upload_file, 401, :bad_auth_token] then [:b2_get_upload_url,:b2_upload_file]
+        when [:b2_upload_file, 408, Any] then [:b2_get_upload_url,:b2_upload_file]
+        when [:b2_upload_file, 500..599, Any] then [:b2_get_upload_url,:b2_upload_file]
+        when [:b2_upload_file, 429, Any] then [:b2_upload_file]
+        when [:b2_authorize_account, 408, Any] then [:b2_authorize_account]
+        when [:b2_authorize_account, 429, Any] then [:b2_authorize_account]
+        when [:b2_authorize_account, 500..599, Any] then [:b2_authorize_account]
+        when [:b2_list_buckets, 401, :expired_auth_token] then [:b2_authorize_account,:b2_list_buckets]
+        when [:b2_list_buckets, 401, :bad_auth_token] then [:b2_authorize_account,:b2_list_buckets]
+        when [:b2_list_buckets, 408, Any] then [:b2_list_buckets]
+        when [:b2_list_buckets, 429, Any] then [:b2_list_buckets]
+        when [:b2_list_buckets, 500..599, Any] then [:b2_list_buckets]
+        when [:b2_list_file_names, 401, :expired_auth_token] then [:b2_authorize_account,:b2_list_file_names]
+        when [:b2_list_file_names, 401, :bad_auth_token] then [:b2_authorize_account,:b2_list_file_names]
+        when [:b2_list_file_names, 408, Any] then [:b2_list_file_names]
+        when [:b2_list_file_names, 429, Any] then [:b2_list_file_names]
+        when [:b2_list_file_names, 500..599, Any] then [:b2_list_file_names]
+        when [:b2_delete_file_version, 401, :expired_auth_token] then [:b2_authorize_account,:b2_delete_file_version]
+        when [:b2_delete_file_version, 401, :bad_auth_token] then [:b2_authorize_account,:b2_delete_file_version]
+        when [:b2_delete_file_version, 408, Any] then [:b2_delete_file_version]
+        when [:b2_delete_file_version, 429, Any] then [:b2_delete_file_version]
+        when [:b2_delete_file_version, 500..599, Any] then [:b2_delete_file_version]
+        when [:b2_finish_large_file, 401, :expired_auth_token] then [:b2_authorize_account,:b2_finish_large_file]
+        when [:b2_finish_large_file, 401, :bad_auth_token] then [:b2_authorize_account,:b2_finish_large_file]
+        when [:b2_finish_large_file, 408, Any] then [:b2_finish_large_file]
+        when [:b2_finish_large_file, 429, Any] then [:b2_finish_large_file]
+        when [:b2_finish_large_file, 500..599, Any] then [:b2_finish_large_file]
+        when [:b2_start_large_file, 401, :expired_auth_token] then [:b2_authorize_account,:b2_start_large_file]
+        when [:b2_start_large_file, 401, :bad_auth_token] then [:b2_authorize_account,:b2_start_large_file]
+        when [:b2_start_large_file, 408, Any] then [:b2_start_large_file]
+        when [:b2_start_large_file, 429, Any] then [:b2_start_large_file]
+        when [:b2_start_large_file, 500..599, Any] then [:b2_start_large_file]
+        else [] # No retry. eg 400 and most 401 should just fail immediately
+        end
+      end
+
+      module_function def retry_dependencies
+        @retry_dependencies ||= begin
+          # didn't want to fight with prolog to generate uniq values here, so just let ruby do it.
+          retries = Hash.new{|h,k| h[k] = Set.new}
+          retries[:b2_upload_part].merge([:b2_get_upload_part_url])
+          retries[:b2_upload_part].merge([:b2_get_upload_part_url])
+          retries[:b2_upload_part].merge([:b2_get_upload_part_url])
+          retries[:b2_upload_part].merge([:b2_get_upload_part_url])
+          retries[:b2_get_upload_part_url].merge([:b2_authorize_account])
+          retries[:b2_get_upload_part_url].merge([:b2_authorize_account])
+          retries[:b2_get_upload_url].merge([:b2_authorize_account])
+          retries[:b2_get_upload_url].merge([:b2_authorize_account])
+          retries[:b2_upload_file].merge([:b2_get_upload_url])
+          retries[:b2_upload_file].merge([:b2_get_upload_url])
+          retries[:b2_upload_file].merge([:b2_get_upload_url])
+          retries[:b2_upload_file].merge([:b2_get_upload_url])
+          retries[:b2_list_buckets].merge([:b2_authorize_account])
+          retries[:b2_list_buckets].merge([:b2_authorize_account])
+          retries[:b2_list_file_names].merge([:b2_authorize_account])
+          retries[:b2_list_file_names].merge([:b2_authorize_account])
+          retries[:b2_delete_file_version].merge([:b2_authorize_account])
+          retries[:b2_delete_file_version].merge([:b2_authorize_account])
+          retries[:b2_finish_large_file].merge([:b2_authorize_account])
+          retries[:b2_finish_large_file].merge([:b2_authorize_account])
+          retries[:b2_start_large_file].merge([:b2_authorize_account])
+          retries[:b2_start_large_file].merge([:b2_authorize_account])
+          retries
+        end
+      end
+    end
+  end
+end

data/lib/backup/backblaze/upload_file.rb

@@ -1,6 +1,7 @@
 require 'digest'
 
-require_relative 'retry.rb'
+require_relative 'api_importer'
+require_relative 'url_token'
 
 module Backup
   module Backblaze
@@ -9,14 +10,20 @@ module Backup
     #
     # dst can contain / for namespaces
     class UploadFile
-      def initialize src:, dst:, token_provider:, content_type: nil
+      def initialize account:, src:, bucket_id:, dst:, url_token: nil, content_type: nil
+        @account = account
         @src = src
         @dst = dst
         @content_type = content_type
-        @token_provider = token_provider
+        @bucket_id = bucket_id
+        @url_token = url_token
       end
 
-      attr_reader :src, :dst, :token_provider, :content_type
+      attr_reader :account, :src, :dst, :bucket_id, :content_type
+
+      def url_token
+        @url_token or b2_get_upload_url
+      end
 
       def headers
         # headers all have to be strings, otherwise excon & Net::HTTP choke :-|
@@ -29,7 +36,7 @@ module Backup
           # optional
           'X-Bz-Info-src_last_modified_millis' => last_modified_millis.to_s,
           'X-Bz-Info-b2-content-disposition'   => content_disposition,
-        }.select{|k,v| v}
+        }.merge(TEST_HEADERS).select{|k,v| v}
       end
 
       def content_type
@@ -59,52 +66,34 @@ module Backup
         end
       end
 
-      include Retry
-
-      # upload with incorrect sha1 responds with
-      #
-      # {"code"=>"bad_request", "message"=>"Sha1 did not match data received", "status"=>400}
-      #
-      # Normal response
-      #
-      #{"accountId"=>"d765e276730e",
-      # "action"=>"upload",
-      # "bucketId"=>"dd8786b5eef2c7d66743001e",
-      # "contentLength"=>6144,
-      # "contentSha1"=>"5ba6cf1b3b3a088d73941052f60e78baf05d91fd",
-      # "contentType"=>"application/octet-stream",
-      # "fileId"=>"4_zdd8786b5eef2c7d66743001e_f1096f3027e0b1927_d20180725_m115148_c002_v0001095_t0047",
-      # "fileInfo"=>{"src_last_modified_millis"=>"1532503455580"},
-      # "fileName"=>"test_file",
-      # "uploadTimestamp"=>1532519508000}
-      def call
-        retry_upload 0, token_provider do |token_provider, retries|
-          Backup::Logger.info "#{src} retry #{retries}"
-          rsp = Excon.post \
-            token_provider.upload_url,
-            headers: (headers.merge 'Authorization' => token_provider.file_auth_token),
-            body: (File.read src),
-            expects: 200
-
-          HashWrap.from_json rsp.body
-        end
+      extend ApiImporter
+
+      # needed for retry logic
+      def b2_authorize_account(retries:, backoff:)
+        account.b2_authorize_account retries: retries, backoff: backoff
       end
 
-      # Seems this doesn't work. Fails with
-      #
-      # 400 Missing header: Content-Length
-      #
-      # Probably because chunked encoding doesn't send an initial Content-Length
-      private def excon_stream_upload( upload )
-        File.open src do |io|
-          chunker = lambda do
-            # Excon.defaults[:chunk_size] defaults to 1048576, ie 1MB
-            # to_s will convert the nil received after everything is read to the final empty chunk
-            io.read(Excon.defaults[:chunk_size]).to_s
-          end
-
-          Excon.post url, headers: headers, :request_block => chunker, debug_request: true, debug_response: true, instrumentor: Excon::StandardInstrumentor
-        end
+      # returns [upload_url, auth_token]
+      # Several files can be uploaded to one url.
+      # But uploading files in parallel requires one upload url per thread.
+      import_endpoint :b2_get_upload_url do |fn|
+        headers = {
+          'Authorization' => account.authorization_token,
+        }.merge(TEST_HEADERS)
+        body_wrap = fn[account.api_url, headers, bucket_id]
+
+        # have to set this here for when this gets called by a retry-sequence
+        @url_token = UrlToken.new body_wrap.uploadUrl, body_wrap.authorizationToken
+      end
+
+      import_endpoint :b2_upload_file do |fn|
+        fn[src, headers, url_token]
+      end
+
+      def call
+        Backup::Logger.info "uploading '#{dst}' of #{src.size}"
+        url_token # not necessary, but makes the flow of control more obvious in the logs
+        b2_upload_file
       end
     end
   end

data/lib/backup/backblaze/upload_large_file.rb

@@ -1,6 +1,7 @@
 require 'digest'
-require_relative 'hash_wrap'
-require_relative 'retry'
+
+require_relative 'api_importer'
+require_relative 'url_token'
 
 module Backup
   module Backblaze
@@ -8,22 +9,24 @@ module Backup
     class UploadLargeFile
       # src is a Pathname
       # dst is a String
-      def initialize src:, dst:, authorization_token:, content_type: nil, url:, part_size:, bucket_id:
+      def initialize account:, src:, bucket_id:, dst:, url_token: nil, part_size:, content_type: nil
+        @account = account
         @src = src
         @dst = dst
-        @authorization_token = authorization_token
+        @bucket_id = bucket_id
         @content_type = content_type
-        @url = url
+        @url_token = url_token
         @part_size = part_size
-        @bucket_id = bucket_id
       end
 
-      attr_reader :src, :dst, :authorization_token, :url, :content_type, :part_size, :bucket_id
+      attr_reader :src, :dst, :account, :url, :content_type, :part_size, :bucket_id
 
       # same as account
       def auth_headers
         # only cos the double {{}} is a quite ugly :-p
-        Hash headers: {'Authorization' => authorization_token}
+        Hash headers: {
+          'Authorization' => account.authorization_token,
+        }.merge(TEST_HEADERS)
       end
 
       def content_type
@@ -45,10 +48,14 @@ module Backup
         end
       end
 
-      # https://www.backblaze.com/b2/docs/b2_start_large_file.html
-      # definitely need fileInfo back from this. Maybe also uploadTimestamp not sure yet.
-      def b2_start_large_file
-        # Unlike in UploadFile, it's OK to use symbols here cos to_json converts them to strings
+      extend ApiImporter
+
+      # needed for retry logic
+      def b2_authorize_account(retries:, backoff:)
+        account.b2_authorize_account retries: retries, backoff: backoff
+      end
+
+      import_endpoint :b2_start_large_file do |fn|
         body = {
           bucketId: bucket_id,
           fileName: dst,
@@ -61,107 +68,78 @@ module Backup
             # large_file_sha1: sha1_digest,
           }.select{|k,v| v}
         }
-
-        rsp = Excon.post \
-          "#{url}/b2api/v1/b2_start_large_file",
-          **auth_headers,
-          body: body.to_json,
-          expects: 200
-
-        HashWrap.from_json rsp.body
+        body_wrap = fn[account.api_url, auth_headers, body]
+        @file_id = body_wrap.fileId
       end
 
       def file_id
-        @file_id ||= b2_start_large_file.fileId
+        @file_id or b2_start_large_file
       end
 
-      def b2_get_upload_part_url
-        rsp = Excon.post \
-          "#{url}/b2api/v1/b2_get_upload_part_url",
-          **auth_headers,
-          body: {fileId: file_id}.to_json,
-          expects: 200
+      def url_token
+        @url_token or b2_get_upload_part_url
+      end
+
+      import_endpoint :b2_get_upload_part_url do |fn|
+        body_wrap = fn[account.api_url, auth_headers, file_id]
+        @url_token = UrlToken.new body_wrap.uploadUrl, body_wrap.authorizationToken
+      end
 
-        hash = JSON.parse rsp.body
-        return hash.values_at 'uploadUrl', 'authorizationToken'
+      def part_count
+        @part_count ||= (src.size / part_size.to_r).ceil
       end
 
       # NOTE Is there a way to stream this instead of loading multiple 100M chunks
       # into memory? No, backblaze does not allow parts to use chunked encoding.
-      def b2_upload_part sequence, upload_url, file_auth_token, &log_block
-        # read length, offset
-        bytes = src.read part_size, part_size * sequence
-
-        # return nil if the read comes back as a nil, ie no bytes read
-        return if bytes.nil? || bytes.empty?
-
-        # This is a bit weird. But not so weird that it needs fixing.
-        log_block.call
+      import_endpoint :b2_upload_part do |fn, sequence, bytes, sha|
+        Backup::Logger.info "#{src} trying part #{sequence + 1} of #{part_count}"
 
+        # not the same as the auth_headers value
         headers = {
-          # not the same as the auth_headers value
-          'Authorization' => file_auth_token,
+          'Authorization' => url_token.auth,
           # cos backblaze wants 1-based, but we want 0-based for reading file
           'X-Bz-Part-Number' => sequence + 1,
           'Content-Length' => bytes.length,
-          'X-Bz-Content-Sha1' => (sha = Digest::SHA1.hexdigest bytes),
-        }
-
-        # Yes, this is a different pattern to the other Excon.post calls ¯\_(ツ)_/¯
-        rsp = Excon.post \
-          upload_url,
-          headers: headers,
-          body: bytes,
-          expects: 200
-
-        # 200 response will be
-        # fileId The unique ID for this file.
-        # partNumber Which part this is.
-        # contentLength The number of bytes stored in the part.
-        # contentSha1 The SHA1 of the bytes stored in the part.
+          'X-Bz-Content-Sha1' => sha,
+        }.merge(TEST_HEADERS)
 
-        # return for the sha collection
-        sha
+        fn[url_token.url, headers, bytes]
       end
 
-      def b2_finish_large_file shas
-        rsp = Excon.post \
-          "#{url}/b2api/v1/b2_finish_large_file",
-          **auth_headers,
-          body: {fileId: file_id, partSha1Array: shas }.to_json,
-          expects: 200
-
-        HashWrap.from_json rsp.body
+      import_endpoint :b2_finish_large_file do |fn, shas|
+        fn[account.api_url, auth_headers, file_id, shas]
       end
 
       # 10000 is backblaze specified max number of parts
       MAX_PARTS = 10000
 
-      include Retry
+      def upload_parts
+        (0...MAX_PARTS).each_with_object [] do |sequence, shas|
+          # read length, offset
+          bytes = src.read part_size, part_size * sequence
+
+          if bytes.nil? || bytes.empty?
+            # no more file to send
+            break shas
+          else
+            sha = Digest::SHA1.hexdigest bytes
+            b2_upload_part sequence, bytes, sha
+            Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
+            shas << sha
+          end
+        end
+      end
 
       def call
         if src.size > part_size * MAX_PARTS
           raise Error, "File #{src.to_s} has size #{src.size} which is larger than part_size * MAX_PARTS #{part_size * MAX_PARTS}. Try increasing part_size in model."
         end
 
-        # TODO could have multiple threads here, each would need a separate token_provider
-        token_provider = TokenProvider.new &method(:b2_get_upload_part_url)
-        shas = (0...MAX_PARTS).each_with_object [] do |sequence, shas|
-          sha = retry_upload 0, token_provider do |token_provider, retries|
-            # return sha
-            b2_upload_part sequence, token_provider.upload_url, token_provider.file_auth_token do
-              Backup::Logger.info "#{src} trying part #{sequence + 1} of #{(src.size / part_size.to_r).ceil} retry #{retries}"
-            end
-          end
+        Logger.info "uploading '#{src}' to #{dst}' of #{src.size} in #{part_count} parts"
 
-          # sha will come back as nil once the file is done.
-          if sha
-            shas << sha
-            Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
-          else
-            break shas
-          end
-        end
+        b2_start_large_file # not really necessary, but makes the flow clearer
+        url_token # try to re-use existing url token if there is one
+        shas = upload_parts
 
         # finish up, log and return the response
         hash_wrap = b2_finish_large_file shas