backup-backblaze 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,77 +1,81 @@
1
1
  module Backup
2
2
  module Backblaze
3
3
  module Retry
4
- MAX_RETRIES = 3
4
+ MAX_RETRIES = 5
5
5
 
6
- # Use the url and token returned by the next_url_token block, until we get
7
- # a reset indicating that we need a new url and token.
8
- class TokenProvider
9
- def initialize &next_url_token
10
- @next_url_token = next_url_token
11
- reset
12
- end
6
+ class TooManyRetries < RuntimeError; end
13
7
 
14
- attr_reader :upload_url, :file_auth_token
8
+ # This is raised when a an api endpoint needs to be retried in a
9
+ # complicate way.
10
+ class RetrySequence < StandardError
11
+ def initialize retry_sequence, backoff
12
+ unless retry_sequence.is_a?(Array) && retry_sequence.all?{|s| Symbol === s}
13
+ raise "provide an array of symbols in #{@retry_sequence.inspect}"
14
+ end
15
15
 
16
- def reset
17
- @upload_url, @file_auth_token = @next_url_token.call
18
- self
16
+ super retry_sequence.inspect
17
+ @retry_sequence = retry_sequence
18
+ @backoff = backoff
19
19
  end
20
- end
21
20
 
22
- class TooManyRetries < RuntimeError; end
21
+ attr_reader :backoff
23
22
 
24
- # Try up to retries times to call the upload_blk. Recursive.
25
- #
26
- # Various errors (passed through from Excon) coming out of upload_blk will
27
- # be caught. When an error is caught, :reset method called on
28
- # token_provider.
29
- #
30
- # Return whatever upload_blk returns
31
- def retry_upload retries, token_provider, &upload_blk
32
- raise TooManyRetries, "max retries is #{MAX_RETRIES}" unless retries < MAX_RETRIES
33
- sleep retries ** 2 # exponential backoff for retries > 0
34
-
35
- # Called by all the rescue blocks that want to retry.
36
- # Mainly so we don't make stoopid errors - like leaving out the +1 for one of the calls :-|
37
- retry_lambda = lambda do
38
- retry_upload retries + 1, token_provider.reset, &upload_blk
23
+ def each &blk
24
+ return enum_for :each unless block_given?
25
+ @retry_sequence.each &blk
39
26
  end
40
27
 
41
- begin
42
- upload_blk.call token_provider, retries
43
- rescue Excon::Errors::Error => ex
44
- # The most convenient place to log this
45
- Backup::Logger.info ex.message
46
- raise
47
- end
28
+ include Enumerable
29
+ end
48
30
 
49
- # Recoverable errors details sourced from:
50
- # https://www.backblaze.com/b2/docs/integration_checklist.html
51
- # https://www.backblaze.com/b2/docs/uploading.html
31
+ module_function def call retries, backoff, api_call_name, &blk
32
+ raise TooManyRetries, "max tries is #{MAX_RETRIES}" unless retries < MAX_RETRIES
52
33
 
53
- # socket-related, 408, and 429
54
- rescue Excon::Errors::SocketError, Excon::Errors::Timeout, Excon::Errors::RequestTimeout, Excon::Errors::TooManyRequests
55
- retry_lambda.call
34
+ # default exponential backoff for retries > 0
35
+ backoff ||= retries ** 2
56
36
 
57
- # some 401
58
- rescue Excon::Errors::Unauthorized => ex
59
- hw = HashWrap.from_json ex.response.body
60
- case hw.code
61
- when 'bad_auth_token', 'expired_auth_token'
62
- retry_lambda.call
37
+ # minor avoidance of unnecessary work in sleep if there's no backoff needed.
38
+ if backoff > 0
39
+ ::Backup::Logger.info "calling #{api_call_name} retry #{retries} after sleep #{backoff}"
40
+ sleep backoff
63
41
  else
64
- raise
42
+ ::Backup::Logger.info "calling #{api_call_name}"
65
43
  end
66
44
 
67
- # 500-599 where the BackBlaze "code" doesn't matter
45
+ # Finally! Do the call.
46
+ blk.call
47
+
68
48
  rescue Excon::Errors::HTTPStatusError => ex
69
- if (500..599) === ex.response.status
70
- retry_lambda.call
49
+ Backup::Logger.info ex.message
50
+ # backoff can end up nil, if Retry-After isn't specified.
51
+ backoff = ex.response.headers['Retry-After']&.to_i
52
+ ::Backup::Logger.info "server specified Retry-After of #{backoff.inspect}"
53
+ raise "Retry-After #{backoff} > 60 is too long" if backoff && backoff > 60
54
+
55
+ # need to get code from body
56
+ body_wrap = HashWrap.from_json ex.response.body
57
+
58
+ # figure out which retry sequence to use
59
+ recovery_sequence = RetryLookup.retry_sequence api_call_name, ex.response.status, body_wrap.code
60
+
61
+ # There's a sequence of retries, and we don't know how to hook the
62
+ # return values and parameters together. So make that someone else's
63
+ # problem.
64
+ #
65
+ # TODO possibly just execute the retry sequence here?
66
+ # That's quite hard cos it will have to have access to the calling self
67
+ if recovery_sequence.any?
68
+ ::Backup::Logger.info "recovery sequence of #{recovery_sequence.inspect}"
69
+ raise RetrySequence.new(recovery_sequence, backoff)
71
70
  else
72
71
  raise
73
72
  end
74
73
 
74
+ rescue Excon::Errors::Error => ex
75
+ Backup::Logger.info ex.message
76
+ # Socket errors etc therefore no http status code and no response body.
77
+ # So just retry with default exponential backoff.
78
+ call retries + 1, nil, api_call_name, &blk
75
79
  end
76
80
  end
77
81
  end
@@ -0,0 +1,112 @@
1
+ require 'set'
2
+
3
+ module Backup
4
+ module Backblaze
5
+ module RetryLookup
6
+ def (Any = Object.new).=== _other; true end
7
+
8
+ module Matcher
9
+ refine Array do
10
+ def === other
11
+ return false unless size == other.size
12
+ size.times.all? do |idx|
13
+ self[idx] === other[idx]
14
+ end
15
+ end
16
+ end
17
+ end
18
+
19
+ using Matcher
20
+
21
+ # Generated from retry.pl
22
+ #
23
+ # Cross-product of all the retry scenarios we know about. This probably
24
+ # isn't the fastest way to calculate retries. But they're rare, so the
25
+ # slowdown doesn't matter. There is a more general pattern, but I don't
26
+ # want to get sucked into implementing unification.
27
+ module_function def retry_sequence api_call, http_status, code
28
+ case [api_call.to_sym, http_status, code.to_sym]
29
+ when [:b2_upload_part, 401, :expired_auth_token] then [:b2_get_upload_part_url,:b2_upload_part]
30
+ when [:b2_upload_part, 401, :bad_auth_token] then [:b2_get_upload_part_url,:b2_upload_part]
31
+ when [:b2_upload_part, 408, Any] then [:b2_get_upload_part_url,:b2_upload_part]
32
+ when [:b2_upload_part, 500..599, Any] then [:b2_get_upload_part_url,:b2_upload_part]
33
+ when [:b2_upload_part, 429, Any] then [:b2_upload_part]
34
+ when [:b2_get_upload_part_url, 401, :expired_auth_token] then [:b2_authorize_account,:b2_get_upload_part_url]
35
+ when [:b2_get_upload_part_url, 401, :bad_auth_token] then [:b2_authorize_account,:b2_get_upload_part_url]
36
+ when [:b2_get_upload_part_url, 408, Any] then [:b2_get_upload_part_url]
37
+ when [:b2_get_upload_part_url, 429, Any] then [:b2_get_upload_part_url]
38
+ when [:b2_get_upload_part_url, 500..599, Any] then [:b2_get_upload_part_url]
39
+ when [:b2_get_upload_url, 401, :expired_auth_token] then [:b2_authorize_account,:b2_get_upload_url]
40
+ when [:b2_get_upload_url, 401, :bad_auth_token] then [:b2_authorize_account,:b2_get_upload_url]
41
+ when [:b2_get_upload_url, 408, Any] then [:b2_get_upload_url]
42
+ when [:b2_get_upload_url, 429, Any] then [:b2_get_upload_url]
43
+ when [:b2_get_upload_url, 500..599, Any] then [:b2_get_upload_url]
44
+ when [:b2_upload_file, 401, :expired_auth_token] then [:b2_get_upload_url,:b2_upload_file]
45
+ when [:b2_upload_file, 401, :bad_auth_token] then [:b2_get_upload_url,:b2_upload_file]
46
+ when [:b2_upload_file, 408, Any] then [:b2_get_upload_url,:b2_upload_file]
47
+ when [:b2_upload_file, 500..599, Any] then [:b2_get_upload_url,:b2_upload_file]
48
+ when [:b2_upload_file, 429, Any] then [:b2_upload_file]
49
+ when [:b2_authorize_account, 408, Any] then [:b2_authorize_account]
50
+ when [:b2_authorize_account, 429, Any] then [:b2_authorize_account]
51
+ when [:b2_authorize_account, 500..599, Any] then [:b2_authorize_account]
52
+ when [:b2_list_buckets, 401, :expired_auth_token] then [:b2_authorize_account,:b2_list_buckets]
53
+ when [:b2_list_buckets, 401, :bad_auth_token] then [:b2_authorize_account,:b2_list_buckets]
54
+ when [:b2_list_buckets, 408, Any] then [:b2_list_buckets]
55
+ when [:b2_list_buckets, 429, Any] then [:b2_list_buckets]
56
+ when [:b2_list_buckets, 500..599, Any] then [:b2_list_buckets]
57
+ when [:b2_list_file_names, 401, :expired_auth_token] then [:b2_authorize_account,:b2_list_file_names]
58
+ when [:b2_list_file_names, 401, :bad_auth_token] then [:b2_authorize_account,:b2_list_file_names]
59
+ when [:b2_list_file_names, 408, Any] then [:b2_list_file_names]
60
+ when [:b2_list_file_names, 429, Any] then [:b2_list_file_names]
61
+ when [:b2_list_file_names, 500..599, Any] then [:b2_list_file_names]
62
+ when [:b2_delete_file_version, 401, :expired_auth_token] then [:b2_authorize_account,:b2_delete_file_version]
63
+ when [:b2_delete_file_version, 401, :bad_auth_token] then [:b2_authorize_account,:b2_delete_file_version]
64
+ when [:b2_delete_file_version, 408, Any] then [:b2_delete_file_version]
65
+ when [:b2_delete_file_version, 429, Any] then [:b2_delete_file_version]
66
+ when [:b2_delete_file_version, 500..599, Any] then [:b2_delete_file_version]
67
+ when [:b2_finish_large_file, 401, :expired_auth_token] then [:b2_authorize_account,:b2_finish_large_file]
68
+ when [:b2_finish_large_file, 401, :bad_auth_token] then [:b2_authorize_account,:b2_finish_large_file]
69
+ when [:b2_finish_large_file, 408, Any] then [:b2_finish_large_file]
70
+ when [:b2_finish_large_file, 429, Any] then [:b2_finish_large_file]
71
+ when [:b2_finish_large_file, 500..599, Any] then [:b2_finish_large_file]
72
+ when [:b2_start_large_file, 401, :expired_auth_token] then [:b2_authorize_account,:b2_start_large_file]
73
+ when [:b2_start_large_file, 401, :bad_auth_token] then [:b2_authorize_account,:b2_start_large_file]
74
+ when [:b2_start_large_file, 408, Any] then [:b2_start_large_file]
75
+ when [:b2_start_large_file, 429, Any] then [:b2_start_large_file]
76
+ when [:b2_start_large_file, 500..599, Any] then [:b2_start_large_file]
77
+ else [] # No retry. eg 400 and most 401 should just fail immediately
78
+ end
79
+ end
80
+
81
+ module_function def retry_dependencies
82
+ @retry_dependencies ||= begin
83
+ # didn't want to fight with prolog to generate uniq values here, so just let ruby do it.
84
+ retries = Hash.new{|h,k| h[k] = Set.new}
85
+ retries[:b2_upload_part].merge([:b2_get_upload_part_url])
86
+ retries[:b2_upload_part].merge([:b2_get_upload_part_url])
87
+ retries[:b2_upload_part].merge([:b2_get_upload_part_url])
88
+ retries[:b2_upload_part].merge([:b2_get_upload_part_url])
89
+ retries[:b2_get_upload_part_url].merge([:b2_authorize_account])
90
+ retries[:b2_get_upload_part_url].merge([:b2_authorize_account])
91
+ retries[:b2_get_upload_url].merge([:b2_authorize_account])
92
+ retries[:b2_get_upload_url].merge([:b2_authorize_account])
93
+ retries[:b2_upload_file].merge([:b2_get_upload_url])
94
+ retries[:b2_upload_file].merge([:b2_get_upload_url])
95
+ retries[:b2_upload_file].merge([:b2_get_upload_url])
96
+ retries[:b2_upload_file].merge([:b2_get_upload_url])
97
+ retries[:b2_list_buckets].merge([:b2_authorize_account])
98
+ retries[:b2_list_buckets].merge([:b2_authorize_account])
99
+ retries[:b2_list_file_names].merge([:b2_authorize_account])
100
+ retries[:b2_list_file_names].merge([:b2_authorize_account])
101
+ retries[:b2_delete_file_version].merge([:b2_authorize_account])
102
+ retries[:b2_delete_file_version].merge([:b2_authorize_account])
103
+ retries[:b2_finish_large_file].merge([:b2_authorize_account])
104
+ retries[:b2_finish_large_file].merge([:b2_authorize_account])
105
+ retries[:b2_start_large_file].merge([:b2_authorize_account])
106
+ retries[:b2_start_large_file].merge([:b2_authorize_account])
107
+ retries
108
+ end
109
+ end
110
+ end
111
+ end
112
+ end
@@ -1,6 +1,7 @@
1
1
  require 'digest'
2
2
 
3
- require_relative 'retry.rb'
3
+ require_relative 'api_importer'
4
+ require_relative 'url_token'
4
5
 
5
6
  module Backup
6
7
  module Backblaze
@@ -9,14 +10,20 @@ module Backup
9
10
  #
10
11
  # dst can contain / for namespaces
11
12
  class UploadFile
12
- def initialize src:, dst:, token_provider:, content_type: nil
13
+ def initialize account:, src:, bucket_id:, dst:, url_token: nil, content_type: nil
14
+ @account = account
13
15
  @src = src
14
16
  @dst = dst
15
17
  @content_type = content_type
16
- @token_provider = token_provider
18
+ @bucket_id = bucket_id
19
+ @url_token = url_token
17
20
  end
18
21
 
19
- attr_reader :src, :dst, :token_provider, :content_type
22
+ attr_reader :account, :src, :dst, :bucket_id, :content_type
23
+
24
+ def url_token
25
+ @url_token or b2_get_upload_url
26
+ end
20
27
 
21
28
  def headers
22
29
  # headers all have to be strings, otherwise excon & Net::HTTP choke :-|
@@ -29,7 +36,7 @@ module Backup
29
36
  # optional
30
37
  'X-Bz-Info-src_last_modified_millis' => last_modified_millis.to_s,
31
38
  'X-Bz-Info-b2-content-disposition' => content_disposition,
32
- }.select{|k,v| v}
39
+ }.merge(TEST_HEADERS).select{|k,v| v}
33
40
  end
34
41
 
35
42
  def content_type
@@ -59,52 +66,34 @@ module Backup
59
66
  end
60
67
  end
61
68
 
62
- include Retry
63
-
64
- # upload with incorrect sha1 responds with
65
- #
66
- # {"code"=>"bad_request", "message"=>"Sha1 did not match data received", "status"=>400}
67
- #
68
- # Normal response
69
- #
70
- #{"accountId"=>"d765e276730e",
71
- # "action"=>"upload",
72
- # "bucketId"=>"dd8786b5eef2c7d66743001e",
73
- # "contentLength"=>6144,
74
- # "contentSha1"=>"5ba6cf1b3b3a088d73941052f60e78baf05d91fd",
75
- # "contentType"=>"application/octet-stream",
76
- # "fileId"=>"4_zdd8786b5eef2c7d66743001e_f1096f3027e0b1927_d20180725_m115148_c002_v0001095_t0047",
77
- # "fileInfo"=>{"src_last_modified_millis"=>"1532503455580"},
78
- # "fileName"=>"test_file",
79
- # "uploadTimestamp"=>1532519508000}
80
- def call
81
- retry_upload 0, token_provider do |token_provider, retries|
82
- Backup::Logger.info "#{src} retry #{retries}"
83
- rsp = Excon.post \
84
- token_provider.upload_url,
85
- headers: (headers.merge 'Authorization' => token_provider.file_auth_token),
86
- body: (File.read src),
87
- expects: 200
88
-
89
- HashWrap.from_json rsp.body
90
- end
69
+ extend ApiImporter
70
+
71
+ # needed for retry logic
72
+ def b2_authorize_account(retries:, backoff:)
73
+ account.b2_authorize_account retries: retries, backoff: backoff
91
74
  end
92
75
 
93
- # Seems this doesn't work. Fails with
94
- #
95
- # 400 Missing header: Content-Length
96
- #
97
- # Probably because chunked encoding doesn't send an initial Content-Length
98
- private def excon_stream_upload( upload )
99
- File.open src do |io|
100
- chunker = lambda do
101
- # Excon.defaults[:chunk_size] defaults to 1048576, ie 1MB
102
- # to_s will convert the nil received after everything is read to the final empty chunk
103
- io.read(Excon.defaults[:chunk_size]).to_s
104
- end
105
-
106
- Excon.post url, headers: headers, :request_block => chunker, debug_request: true, debug_response: true, instrumentor: Excon::StandardInstrumentor
107
- end
76
+ # returns [upload_url, auth_token]
77
+ # Several files can be uploaded to one url.
78
+ # But uploading files in parallel requires one upload url per thread.
79
+ import_endpoint :b2_get_upload_url do |fn|
80
+ headers = {
81
+ 'Authorization' => account.authorization_token,
82
+ }.merge(TEST_HEADERS)
83
+ body_wrap = fn[account.api_url, headers, bucket_id]
84
+
85
+ # have to set this here for when this gets called by a retry-sequence
86
+ @url_token = UrlToken.new body_wrap.uploadUrl, body_wrap.authorizationToken
87
+ end
88
+
89
+ import_endpoint :b2_upload_file do |fn|
90
+ fn[src, headers, url_token]
91
+ end
92
+
93
+ def call
94
+ Backup::Logger.info "uploading '#{dst}' of #{src.size}"
95
+ url_token # not necessary, but makes the flow of control more obvious in the logs
96
+ b2_upload_file
108
97
  end
109
98
  end
110
99
  end
@@ -1,6 +1,7 @@
1
1
  require 'digest'
2
- require_relative 'hash_wrap'
3
- require_relative 'retry'
2
+
3
+ require_relative 'api_importer'
4
+ require_relative 'url_token'
4
5
 
5
6
  module Backup
6
7
  module Backblaze
@@ -8,22 +9,24 @@ module Backup
8
9
  class UploadLargeFile
9
10
  # src is a Pathname
10
11
  # dst is a String
11
- def initialize src:, dst:, authorization_token:, content_type: nil, url:, part_size:, bucket_id:
12
+ def initialize account:, src:, bucket_id:, dst:, url_token: nil, part_size:, content_type: nil
13
+ @account = account
12
14
  @src = src
13
15
  @dst = dst
14
- @authorization_token = authorization_token
16
+ @bucket_id = bucket_id
15
17
  @content_type = content_type
16
- @url = url
18
+ @url_token = url_token
17
19
  @part_size = part_size
18
- @bucket_id = bucket_id
19
20
  end
20
21
 
21
- attr_reader :src, :dst, :authorization_token, :url, :content_type, :part_size, :bucket_id
22
+ attr_reader :src, :dst, :account, :url, :content_type, :part_size, :bucket_id
22
23
 
23
24
  # same as account
24
25
  def auth_headers
25
26
  # only cos the double {{}} is a quite ugly :-p
26
- Hash headers: {'Authorization' => authorization_token}
27
+ Hash headers: {
28
+ 'Authorization' => account.authorization_token,
29
+ }.merge(TEST_HEADERS)
27
30
  end
28
31
 
29
32
  def content_type
@@ -45,10 +48,14 @@ module Backup
45
48
  end
46
49
  end
47
50
 
48
- # https://www.backblaze.com/b2/docs/b2_start_large_file.html
49
- # definitely need fileInfo back from this. Maybe also uploadTimestamp not sure yet.
50
- def b2_start_large_file
51
- # Unlike in UploadFile, it's OK to use symbols here cos to_json converts them to strings
51
+ extend ApiImporter
52
+
53
+ # needed for retry logic
54
+ def b2_authorize_account(retries:, backoff:)
55
+ account.b2_authorize_account retries: retries, backoff: backoff
56
+ end
57
+
58
+ import_endpoint :b2_start_large_file do |fn|
52
59
  body = {
53
60
  bucketId: bucket_id,
54
61
  fileName: dst,
@@ -61,107 +68,78 @@ module Backup
61
68
  # large_file_sha1: sha1_digest,
62
69
  }.select{|k,v| v}
63
70
  }
64
-
65
- rsp = Excon.post \
66
- "#{url}/b2api/v1/b2_start_large_file",
67
- **auth_headers,
68
- body: body.to_json,
69
- expects: 200
70
-
71
- HashWrap.from_json rsp.body
71
+ body_wrap = fn[account.api_url, auth_headers, body]
72
+ @file_id = body_wrap.fileId
72
73
  end
73
74
 
74
75
  def file_id
75
- @file_id ||= b2_start_large_file.fileId
76
+ @file_id or b2_start_large_file
76
77
  end
77
78
 
78
- def b2_get_upload_part_url
79
- rsp = Excon.post \
80
- "#{url}/b2api/v1/b2_get_upload_part_url",
81
- **auth_headers,
82
- body: {fileId: file_id}.to_json,
83
- expects: 200
79
+ def url_token
80
+ @url_token or b2_get_upload_part_url
81
+ end
82
+
83
+ import_endpoint :b2_get_upload_part_url do |fn|
84
+ body_wrap = fn[account.api_url, auth_headers, file_id]
85
+ @url_token = UrlToken.new body_wrap.uploadUrl, body_wrap.authorizationToken
86
+ end
84
87
 
85
- hash = JSON.parse rsp.body
86
- return hash.values_at 'uploadUrl', 'authorizationToken'
88
+ def part_count
89
+ @part_count ||= (src.size / part_size.to_r).ceil
87
90
  end
88
91
 
89
92
  # NOTE Is there a way to stream this instead of loading multiple 100M chunks
90
93
  # into memory? No, backblaze does not allow parts to use chunked encoding.
91
- def b2_upload_part sequence, upload_url, file_auth_token, &log_block
92
- # read length, offset
93
- bytes = src.read part_size, part_size * sequence
94
-
95
- # return nil if the read comes back as a nil, ie no bytes read
96
- return if bytes.nil? || bytes.empty?
97
-
98
- # This is a bit weird. But not so weird that it needs fixing.
99
- log_block.call
94
+ import_endpoint :b2_upload_part do |fn, sequence, bytes, sha|
95
+ Backup::Logger.info "#{src} trying part #{sequence + 1} of #{part_count}"
100
96
 
97
+ # not the same as the auth_headers value
101
98
  headers = {
102
- # not the same as the auth_headers value
103
- 'Authorization' => file_auth_token,
99
+ 'Authorization' => url_token.auth,
104
100
  # cos backblaze wants 1-based, but we want 0-based for reading file
105
101
  'X-Bz-Part-Number' => sequence + 1,
106
102
  'Content-Length' => bytes.length,
107
- 'X-Bz-Content-Sha1' => (sha = Digest::SHA1.hexdigest bytes),
108
- }
109
-
110
- # Yes, this is a different pattern to the other Excon.post calls ¯\_(ツ)_/¯
111
- rsp = Excon.post \
112
- upload_url,
113
- headers: headers,
114
- body: bytes,
115
- expects: 200
116
-
117
- # 200 response will be
118
- # fileId The unique ID for this file.
119
- # partNumber Which part this is.
120
- # contentLength The number of bytes stored in the part.
121
- # contentSha1 The SHA1 of the bytes stored in the part.
103
+ 'X-Bz-Content-Sha1' => sha,
104
+ }.merge(TEST_HEADERS)
122
105
 
123
- # return for the sha collection
124
- sha
106
+ fn[url_token.url, headers, bytes]
125
107
  end
126
108
 
127
- def b2_finish_large_file shas
128
- rsp = Excon.post \
129
- "#{url}/b2api/v1/b2_finish_large_file",
130
- **auth_headers,
131
- body: {fileId: file_id, partSha1Array: shas }.to_json,
132
- expects: 200
133
-
134
- HashWrap.from_json rsp.body
109
+ import_endpoint :b2_finish_large_file do |fn, shas|
110
+ fn[account.api_url, auth_headers, file_id, shas]
135
111
  end
136
112
 
137
113
  # 10000 is backblaze specified max number of parts
138
114
  MAX_PARTS = 10000
139
115
 
140
- include Retry
116
+ def upload_parts
117
+ (0...MAX_PARTS).each_with_object [] do |sequence, shas|
118
+ # read length, offset
119
+ bytes = src.read part_size, part_size * sequence
120
+
121
+ if bytes.nil? || bytes.empty?
122
+ # no more file to send
123
+ break shas
124
+ else
125
+ sha = Digest::SHA1.hexdigest bytes
126
+ b2_upload_part sequence, bytes, sha
127
+ Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
128
+ shas << sha
129
+ end
130
+ end
131
+ end
141
132
 
142
133
  def call
143
134
  if src.size > part_size * MAX_PARTS
144
135
  raise Error, "File #{src.to_s} has size #{src.size} which is larger than part_size * MAX_PARTS #{part_size * MAX_PARTS}. Try increasing part_size in model."
145
136
  end
146
137
 
147
- # TODO could have multiple threads here, each would need a separate token_provider
148
- token_provider = TokenProvider.new &method(:b2_get_upload_part_url)
149
- shas = (0...MAX_PARTS).each_with_object [] do |sequence, shas|
150
- sha = retry_upload 0, token_provider do |token_provider, retries|
151
- # return sha
152
- b2_upload_part sequence, token_provider.upload_url, token_provider.file_auth_token do
153
- Backup::Logger.info "#{src} trying part #{sequence + 1} of #{(src.size / part_size.to_r).ceil} retry #{retries}"
154
- end
155
- end
138
+ Logger.info "uploading '#{src}' to #{dst}' of #{src.size} in #{part_count} parts"
156
139
 
157
- # sha will come back as nil once the file is done.
158
- if sha
159
- shas << sha
160
- Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
161
- else
162
- break shas
163
- end
164
- end
140
+ b2_start_large_file # not really necessary, but makes the flow clearer
141
+ url_token # try to re-use existing url token if there is one
142
+ shas = upload_parts
165
143
 
166
144
  # finish up, log and return the response
167
145
  hash_wrap = b2_finish_large_file shas