backup-backblaze 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/Rakefile +25 -1
- data/lib/backup/backblaze.rb +3 -0
- data/lib/backup/backblaze/account.rb +41 -85
- data/lib/backup/backblaze/api_importer.rb +93 -0
- data/lib/backup/backblaze/back_blaze.rb +6 -18
- data/lib/backup/backblaze/hash_wrap.rb +2 -0
- data/lib/backup/backblaze/http.rb +132 -0
- data/lib/backup/backblaze/retry.rb +56 -52
- data/lib/backup/backblaze/retry_lookup.rb +112 -0
- data/lib/backup/backblaze/upload_file.rb +38 -49
- data/lib/backup/backblaze/upload_large_file.rb +61 -83
- data/lib/backup/backblaze/url_token.rb +11 -0
- data/lib/backup/backblaze/version.rb +1 -1
- data/src/retry.pl +157 -0
- data/src/retry_lookup.erb +42 -0
- metadata +8 -2
@@ -1,77 +1,81 @@
|
|
1
1
|
module Backup
|
2
2
|
module Backblaze
|
3
3
|
module Retry
|
4
|
-
MAX_RETRIES =
|
4
|
+
MAX_RETRIES = 5
|
5
5
|
|
6
|
-
|
7
|
-
# a reset indicating that we need a new url and token.
|
8
|
-
class TokenProvider
|
9
|
-
def initialize &next_url_token
|
10
|
-
@next_url_token = next_url_token
|
11
|
-
reset
|
12
|
-
end
|
6
|
+
class TooManyRetries < RuntimeError; end
|
13
7
|
|
14
|
-
|
8
|
+
# This is raised when a an api endpoint needs to be retried in a
|
9
|
+
# complicate way.
|
10
|
+
class RetrySequence < StandardError
|
11
|
+
def initialize retry_sequence, backoff
|
12
|
+
unless retry_sequence.is_a?(Array) && retry_sequence.all?{|s| Symbol === s}
|
13
|
+
raise "provide an array of symbols in #{@retry_sequence.inspect}"
|
14
|
+
end
|
15
15
|
|
16
|
-
|
17
|
-
@
|
18
|
-
|
16
|
+
super retry_sequence.inspect
|
17
|
+
@retry_sequence = retry_sequence
|
18
|
+
@backoff = backoff
|
19
19
|
end
|
20
|
-
end
|
21
20
|
|
22
|
-
|
21
|
+
attr_reader :backoff
|
23
22
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
# be caught. When an error is caught, :reset method called on
|
28
|
-
# token_provider.
|
29
|
-
#
|
30
|
-
# Return whatever upload_blk returns
|
31
|
-
def retry_upload retries, token_provider, &upload_blk
|
32
|
-
raise TooManyRetries, "max retries is #{MAX_RETRIES}" unless retries < MAX_RETRIES
|
33
|
-
sleep retries ** 2 # exponential backoff for retries > 0
|
34
|
-
|
35
|
-
# Called by all the rescue blocks that want to retry.
|
36
|
-
# Mainly so we don't make stoopid errors - like leaving out the +1 for one of the calls :-|
|
37
|
-
retry_lambda = lambda do
|
38
|
-
retry_upload retries + 1, token_provider.reset, &upload_blk
|
23
|
+
def each &blk
|
24
|
+
return enum_for :each unless block_given?
|
25
|
+
@retry_sequence.each &blk
|
39
26
|
end
|
40
27
|
|
41
|
-
|
42
|
-
|
43
|
-
rescue Excon::Errors::Error => ex
|
44
|
-
# The most convenient place to log this
|
45
|
-
Backup::Logger.info ex.message
|
46
|
-
raise
|
47
|
-
end
|
28
|
+
include Enumerable
|
29
|
+
end
|
48
30
|
|
49
|
-
|
50
|
-
|
51
|
-
# https://www.backblaze.com/b2/docs/uploading.html
|
31
|
+
module_function def call retries, backoff, api_call_name, &blk
|
32
|
+
raise TooManyRetries, "max tries is #{MAX_RETRIES}" unless retries < MAX_RETRIES
|
52
33
|
|
53
|
-
|
54
|
-
|
55
|
-
retry_lambda.call
|
34
|
+
# default exponential backoff for retries > 0
|
35
|
+
backoff ||= retries ** 2
|
56
36
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
when 'bad_auth_token', 'expired_auth_token'
|
62
|
-
retry_lambda.call
|
37
|
+
# minor avoidance of unnecessary work in sleep if there's no backoff needed.
|
38
|
+
if backoff > 0
|
39
|
+
::Backup::Logger.info "calling #{api_call_name} retry #{retries} after sleep #{backoff}"
|
40
|
+
sleep backoff
|
63
41
|
else
|
64
|
-
|
42
|
+
::Backup::Logger.info "calling #{api_call_name}"
|
65
43
|
end
|
66
44
|
|
67
|
-
|
45
|
+
# Finally! Do the call.
|
46
|
+
blk.call
|
47
|
+
|
68
48
|
rescue Excon::Errors::HTTPStatusError => ex
|
69
|
-
|
70
|
-
|
49
|
+
Backup::Logger.info ex.message
|
50
|
+
# backoff can end up nil, if Retry-After isn't specified.
|
51
|
+
backoff = ex.response.headers['Retry-After']&.to_i
|
52
|
+
::Backup::Logger.info "server specified Retry-After of #{backoff.inspect}"
|
53
|
+
raise "Retry-After #{backoff} > 60 is too long" if backoff && backoff > 60
|
54
|
+
|
55
|
+
# need to get code from body
|
56
|
+
body_wrap = HashWrap.from_json ex.response.body
|
57
|
+
|
58
|
+
# figure out which retry sequence to use
|
59
|
+
recovery_sequence = RetryLookup.retry_sequence api_call_name, ex.response.status, body_wrap.code
|
60
|
+
|
61
|
+
# There's a sequence of retries, and we don't know how to hook the
|
62
|
+
# return values and parameters together. So make that someone else's
|
63
|
+
# problem.
|
64
|
+
#
|
65
|
+
# TODO possibly just execute the retry sequence here?
|
66
|
+
# That's quite hard cos it will have to have access to the calling self
|
67
|
+
if recovery_sequence.any?
|
68
|
+
::Backup::Logger.info "recovery sequence of #{recovery_sequence.inspect}"
|
69
|
+
raise RetrySequence.new(recovery_sequence, backoff)
|
71
70
|
else
|
72
71
|
raise
|
73
72
|
end
|
74
73
|
|
74
|
+
rescue Excon::Errors::Error => ex
|
75
|
+
Backup::Logger.info ex.message
|
76
|
+
# Socket errors etc therefore no http status code and no response body.
|
77
|
+
# So just retry with default exponential backoff.
|
78
|
+
call retries + 1, nil, api_call_name, &blk
|
75
79
|
end
|
76
80
|
end
|
77
81
|
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'set'
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
module Backblaze
|
5
|
+
module RetryLookup
|
6
|
+
def (Any = Object.new).=== _other; true end
|
7
|
+
|
8
|
+
module Matcher
|
9
|
+
refine Array do
|
10
|
+
def === other
|
11
|
+
return false unless size == other.size
|
12
|
+
size.times.all? do |idx|
|
13
|
+
self[idx] === other[idx]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
using Matcher
|
20
|
+
|
21
|
+
# Generated from retry.pl
|
22
|
+
#
|
23
|
+
# Cross-product of all the retry scenarios we know about. This probably
|
24
|
+
# isn't the fastest way to calculate retries. But they're rare, so the
|
25
|
+
# slowdown doesn't matter. There is a more general pattern, but I don't
|
26
|
+
# want to get sucked into implementing unification.
|
27
|
+
module_function def retry_sequence api_call, http_status, code
|
28
|
+
case [api_call.to_sym, http_status, code.to_sym]
|
29
|
+
when [:b2_upload_part, 401, :expired_auth_token] then [:b2_get_upload_part_url,:b2_upload_part]
|
30
|
+
when [:b2_upload_part, 401, :bad_auth_token] then [:b2_get_upload_part_url,:b2_upload_part]
|
31
|
+
when [:b2_upload_part, 408, Any] then [:b2_get_upload_part_url,:b2_upload_part]
|
32
|
+
when [:b2_upload_part, 500..599, Any] then [:b2_get_upload_part_url,:b2_upload_part]
|
33
|
+
when [:b2_upload_part, 429, Any] then [:b2_upload_part]
|
34
|
+
when [:b2_get_upload_part_url, 401, :expired_auth_token] then [:b2_authorize_account,:b2_get_upload_part_url]
|
35
|
+
when [:b2_get_upload_part_url, 401, :bad_auth_token] then [:b2_authorize_account,:b2_get_upload_part_url]
|
36
|
+
when [:b2_get_upload_part_url, 408, Any] then [:b2_get_upload_part_url]
|
37
|
+
when [:b2_get_upload_part_url, 429, Any] then [:b2_get_upload_part_url]
|
38
|
+
when [:b2_get_upload_part_url, 500..599, Any] then [:b2_get_upload_part_url]
|
39
|
+
when [:b2_get_upload_url, 401, :expired_auth_token] then [:b2_authorize_account,:b2_get_upload_url]
|
40
|
+
when [:b2_get_upload_url, 401, :bad_auth_token] then [:b2_authorize_account,:b2_get_upload_url]
|
41
|
+
when [:b2_get_upload_url, 408, Any] then [:b2_get_upload_url]
|
42
|
+
when [:b2_get_upload_url, 429, Any] then [:b2_get_upload_url]
|
43
|
+
when [:b2_get_upload_url, 500..599, Any] then [:b2_get_upload_url]
|
44
|
+
when [:b2_upload_file, 401, :expired_auth_token] then [:b2_get_upload_url,:b2_upload_file]
|
45
|
+
when [:b2_upload_file, 401, :bad_auth_token] then [:b2_get_upload_url,:b2_upload_file]
|
46
|
+
when [:b2_upload_file, 408, Any] then [:b2_get_upload_url,:b2_upload_file]
|
47
|
+
when [:b2_upload_file, 500..599, Any] then [:b2_get_upload_url,:b2_upload_file]
|
48
|
+
when [:b2_upload_file, 429, Any] then [:b2_upload_file]
|
49
|
+
when [:b2_authorize_account, 408, Any] then [:b2_authorize_account]
|
50
|
+
when [:b2_authorize_account, 429, Any] then [:b2_authorize_account]
|
51
|
+
when [:b2_authorize_account, 500..599, Any] then [:b2_authorize_account]
|
52
|
+
when [:b2_list_buckets, 401, :expired_auth_token] then [:b2_authorize_account,:b2_list_buckets]
|
53
|
+
when [:b2_list_buckets, 401, :bad_auth_token] then [:b2_authorize_account,:b2_list_buckets]
|
54
|
+
when [:b2_list_buckets, 408, Any] then [:b2_list_buckets]
|
55
|
+
when [:b2_list_buckets, 429, Any] then [:b2_list_buckets]
|
56
|
+
when [:b2_list_buckets, 500..599, Any] then [:b2_list_buckets]
|
57
|
+
when [:b2_list_file_names, 401, :expired_auth_token] then [:b2_authorize_account,:b2_list_file_names]
|
58
|
+
when [:b2_list_file_names, 401, :bad_auth_token] then [:b2_authorize_account,:b2_list_file_names]
|
59
|
+
when [:b2_list_file_names, 408, Any] then [:b2_list_file_names]
|
60
|
+
when [:b2_list_file_names, 429, Any] then [:b2_list_file_names]
|
61
|
+
when [:b2_list_file_names, 500..599, Any] then [:b2_list_file_names]
|
62
|
+
when [:b2_delete_file_version, 401, :expired_auth_token] then [:b2_authorize_account,:b2_delete_file_version]
|
63
|
+
when [:b2_delete_file_version, 401, :bad_auth_token] then [:b2_authorize_account,:b2_delete_file_version]
|
64
|
+
when [:b2_delete_file_version, 408, Any] then [:b2_delete_file_version]
|
65
|
+
when [:b2_delete_file_version, 429, Any] then [:b2_delete_file_version]
|
66
|
+
when [:b2_delete_file_version, 500..599, Any] then [:b2_delete_file_version]
|
67
|
+
when [:b2_finish_large_file, 401, :expired_auth_token] then [:b2_authorize_account,:b2_finish_large_file]
|
68
|
+
when [:b2_finish_large_file, 401, :bad_auth_token] then [:b2_authorize_account,:b2_finish_large_file]
|
69
|
+
when [:b2_finish_large_file, 408, Any] then [:b2_finish_large_file]
|
70
|
+
when [:b2_finish_large_file, 429, Any] then [:b2_finish_large_file]
|
71
|
+
when [:b2_finish_large_file, 500..599, Any] then [:b2_finish_large_file]
|
72
|
+
when [:b2_start_large_file, 401, :expired_auth_token] then [:b2_authorize_account,:b2_start_large_file]
|
73
|
+
when [:b2_start_large_file, 401, :bad_auth_token] then [:b2_authorize_account,:b2_start_large_file]
|
74
|
+
when [:b2_start_large_file, 408, Any] then [:b2_start_large_file]
|
75
|
+
when [:b2_start_large_file, 429, Any] then [:b2_start_large_file]
|
76
|
+
when [:b2_start_large_file, 500..599, Any] then [:b2_start_large_file]
|
77
|
+
else [] # No retry. eg 400 and most 401 should just fail immediately
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
module_function def retry_dependencies
|
82
|
+
@retry_dependencies ||= begin
|
83
|
+
# didn't want to fight with prolog to generate uniq values here, so just let ruby do it.
|
84
|
+
retries = Hash.new{|h,k| h[k] = Set.new}
|
85
|
+
retries[:b2_upload_part].merge([:b2_get_upload_part_url])
|
86
|
+
retries[:b2_upload_part].merge([:b2_get_upload_part_url])
|
87
|
+
retries[:b2_upload_part].merge([:b2_get_upload_part_url])
|
88
|
+
retries[:b2_upload_part].merge([:b2_get_upload_part_url])
|
89
|
+
retries[:b2_get_upload_part_url].merge([:b2_authorize_account])
|
90
|
+
retries[:b2_get_upload_part_url].merge([:b2_authorize_account])
|
91
|
+
retries[:b2_get_upload_url].merge([:b2_authorize_account])
|
92
|
+
retries[:b2_get_upload_url].merge([:b2_authorize_account])
|
93
|
+
retries[:b2_upload_file].merge([:b2_get_upload_url])
|
94
|
+
retries[:b2_upload_file].merge([:b2_get_upload_url])
|
95
|
+
retries[:b2_upload_file].merge([:b2_get_upload_url])
|
96
|
+
retries[:b2_upload_file].merge([:b2_get_upload_url])
|
97
|
+
retries[:b2_list_buckets].merge([:b2_authorize_account])
|
98
|
+
retries[:b2_list_buckets].merge([:b2_authorize_account])
|
99
|
+
retries[:b2_list_file_names].merge([:b2_authorize_account])
|
100
|
+
retries[:b2_list_file_names].merge([:b2_authorize_account])
|
101
|
+
retries[:b2_delete_file_version].merge([:b2_authorize_account])
|
102
|
+
retries[:b2_delete_file_version].merge([:b2_authorize_account])
|
103
|
+
retries[:b2_finish_large_file].merge([:b2_authorize_account])
|
104
|
+
retries[:b2_finish_large_file].merge([:b2_authorize_account])
|
105
|
+
retries[:b2_start_large_file].merge([:b2_authorize_account])
|
106
|
+
retries[:b2_start_large_file].merge([:b2_authorize_account])
|
107
|
+
retries
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'digest'
|
2
2
|
|
3
|
-
require_relative '
|
3
|
+
require_relative 'api_importer'
|
4
|
+
require_relative 'url_token'
|
4
5
|
|
5
6
|
module Backup
|
6
7
|
module Backblaze
|
@@ -9,14 +10,20 @@ module Backup
|
|
9
10
|
#
|
10
11
|
# dst can contain / for namespaces
|
11
12
|
class UploadFile
|
12
|
-
def initialize src:,
|
13
|
+
def initialize account:, src:, bucket_id:, dst:, url_token: nil, content_type: nil
|
14
|
+
@account = account
|
13
15
|
@src = src
|
14
16
|
@dst = dst
|
15
17
|
@content_type = content_type
|
16
|
-
@
|
18
|
+
@bucket_id = bucket_id
|
19
|
+
@url_token = url_token
|
17
20
|
end
|
18
21
|
|
19
|
-
attr_reader :src, :dst, :
|
22
|
+
attr_reader :account, :src, :dst, :bucket_id, :content_type
|
23
|
+
|
24
|
+
def url_token
|
25
|
+
@url_token or b2_get_upload_url
|
26
|
+
end
|
20
27
|
|
21
28
|
def headers
|
22
29
|
# headers all have to be strings, otherwise excon & Net::HTTP choke :-|
|
@@ -29,7 +36,7 @@ module Backup
|
|
29
36
|
# optional
|
30
37
|
'X-Bz-Info-src_last_modified_millis' => last_modified_millis.to_s,
|
31
38
|
'X-Bz-Info-b2-content-disposition' => content_disposition,
|
32
|
-
}.select{|k,v| v}
|
39
|
+
}.merge(TEST_HEADERS).select{|k,v| v}
|
33
40
|
end
|
34
41
|
|
35
42
|
def content_type
|
@@ -59,52 +66,34 @@ module Backup
|
|
59
66
|
end
|
60
67
|
end
|
61
68
|
|
62
|
-
|
63
|
-
|
64
|
-
#
|
65
|
-
|
66
|
-
|
67
|
-
#
|
68
|
-
# Normal response
|
69
|
-
#
|
70
|
-
#{"accountId"=>"d765e276730e",
|
71
|
-
# "action"=>"upload",
|
72
|
-
# "bucketId"=>"dd8786b5eef2c7d66743001e",
|
73
|
-
# "contentLength"=>6144,
|
74
|
-
# "contentSha1"=>"5ba6cf1b3b3a088d73941052f60e78baf05d91fd",
|
75
|
-
# "contentType"=>"application/octet-stream",
|
76
|
-
# "fileId"=>"4_zdd8786b5eef2c7d66743001e_f1096f3027e0b1927_d20180725_m115148_c002_v0001095_t0047",
|
77
|
-
# "fileInfo"=>{"src_last_modified_millis"=>"1532503455580"},
|
78
|
-
# "fileName"=>"test_file",
|
79
|
-
# "uploadTimestamp"=>1532519508000}
|
80
|
-
def call
|
81
|
-
retry_upload 0, token_provider do |token_provider, retries|
|
82
|
-
Backup::Logger.info "#{src} retry #{retries}"
|
83
|
-
rsp = Excon.post \
|
84
|
-
token_provider.upload_url,
|
85
|
-
headers: (headers.merge 'Authorization' => token_provider.file_auth_token),
|
86
|
-
body: (File.read src),
|
87
|
-
expects: 200
|
88
|
-
|
89
|
-
HashWrap.from_json rsp.body
|
90
|
-
end
|
69
|
+
extend ApiImporter
|
70
|
+
|
71
|
+
# needed for retry logic
|
72
|
+
def b2_authorize_account(retries:, backoff:)
|
73
|
+
account.b2_authorize_account retries: retries, backoff: backoff
|
91
74
|
end
|
92
75
|
|
93
|
-
#
|
94
|
-
#
|
95
|
-
#
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
76
|
+
# returns [upload_url, auth_token]
|
77
|
+
# Several files can be uploaded to one url.
|
78
|
+
# But uploading files in parallel requires one upload url per thread.
|
79
|
+
import_endpoint :b2_get_upload_url do |fn|
|
80
|
+
headers = {
|
81
|
+
'Authorization' => account.authorization_token,
|
82
|
+
}.merge(TEST_HEADERS)
|
83
|
+
body_wrap = fn[account.api_url, headers, bucket_id]
|
84
|
+
|
85
|
+
# have to set this here for when this gets called by a retry-sequence
|
86
|
+
@url_token = UrlToken.new body_wrap.uploadUrl, body_wrap.authorizationToken
|
87
|
+
end
|
88
|
+
|
89
|
+
import_endpoint :b2_upload_file do |fn|
|
90
|
+
fn[src, headers, url_token]
|
91
|
+
end
|
92
|
+
|
93
|
+
def call
|
94
|
+
Backup::Logger.info "uploading '#{dst}' of #{src.size}"
|
95
|
+
url_token # not necessary, but makes the flow of control more obvious in the logs
|
96
|
+
b2_upload_file
|
108
97
|
end
|
109
98
|
end
|
110
99
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'digest'
|
2
|
-
|
3
|
-
require_relative '
|
2
|
+
|
3
|
+
require_relative 'api_importer'
|
4
|
+
require_relative 'url_token'
|
4
5
|
|
5
6
|
module Backup
|
6
7
|
module Backblaze
|
@@ -8,22 +9,24 @@ module Backup
|
|
8
9
|
class UploadLargeFile
|
9
10
|
# src is a Pathname
|
10
11
|
# dst is a String
|
11
|
-
def initialize src:,
|
12
|
+
def initialize account:, src:, bucket_id:, dst:, url_token: nil, part_size:, content_type: nil
|
13
|
+
@account = account
|
12
14
|
@src = src
|
13
15
|
@dst = dst
|
14
|
-
@
|
16
|
+
@bucket_id = bucket_id
|
15
17
|
@content_type = content_type
|
16
|
-
@
|
18
|
+
@url_token = url_token
|
17
19
|
@part_size = part_size
|
18
|
-
@bucket_id = bucket_id
|
19
20
|
end
|
20
21
|
|
21
|
-
attr_reader :src, :dst, :
|
22
|
+
attr_reader :src, :dst, :account, :url, :content_type, :part_size, :bucket_id
|
22
23
|
|
23
24
|
# same as account
|
24
25
|
def auth_headers
|
25
26
|
# only cos the double {{}} is a quite ugly :-p
|
26
|
-
Hash headers: {
|
27
|
+
Hash headers: {
|
28
|
+
'Authorization' => account.authorization_token,
|
29
|
+
}.merge(TEST_HEADERS)
|
27
30
|
end
|
28
31
|
|
29
32
|
def content_type
|
@@ -45,10 +48,14 @@ module Backup
|
|
45
48
|
end
|
46
49
|
end
|
47
50
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
51
|
+
extend ApiImporter
|
52
|
+
|
53
|
+
# needed for retry logic
|
54
|
+
def b2_authorize_account(retries:, backoff:)
|
55
|
+
account.b2_authorize_account retries: retries, backoff: backoff
|
56
|
+
end
|
57
|
+
|
58
|
+
import_endpoint :b2_start_large_file do |fn|
|
52
59
|
body = {
|
53
60
|
bucketId: bucket_id,
|
54
61
|
fileName: dst,
|
@@ -61,107 +68,78 @@ module Backup
|
|
61
68
|
# large_file_sha1: sha1_digest,
|
62
69
|
}.select{|k,v| v}
|
63
70
|
}
|
64
|
-
|
65
|
-
|
66
|
-
"#{url}/b2api/v1/b2_start_large_file",
|
67
|
-
**auth_headers,
|
68
|
-
body: body.to_json,
|
69
|
-
expects: 200
|
70
|
-
|
71
|
-
HashWrap.from_json rsp.body
|
71
|
+
body_wrap = fn[account.api_url, auth_headers, body]
|
72
|
+
@file_id = body_wrap.fileId
|
72
73
|
end
|
73
74
|
|
74
75
|
def file_id
|
75
|
-
@file_id
|
76
|
+
@file_id or b2_start_large_file
|
76
77
|
end
|
77
78
|
|
78
|
-
def
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
79
|
+
def url_token
|
80
|
+
@url_token or b2_get_upload_part_url
|
81
|
+
end
|
82
|
+
|
83
|
+
import_endpoint :b2_get_upload_part_url do |fn|
|
84
|
+
body_wrap = fn[account.api_url, auth_headers, file_id]
|
85
|
+
@url_token = UrlToken.new body_wrap.uploadUrl, body_wrap.authorizationToken
|
86
|
+
end
|
84
87
|
|
85
|
-
|
86
|
-
|
88
|
+
def part_count
|
89
|
+
@part_count ||= (src.size / part_size.to_r).ceil
|
87
90
|
end
|
88
91
|
|
89
92
|
# NOTE Is there a way to stream this instead of loading multiple 100M chunks
|
90
93
|
# into memory? No, backblaze does not allow parts to use chunked encoding.
|
91
|
-
|
92
|
-
#
|
93
|
-
bytes = src.read part_size, part_size * sequence
|
94
|
-
|
95
|
-
# return nil if the read comes back as a nil, ie no bytes read
|
96
|
-
return if bytes.nil? || bytes.empty?
|
97
|
-
|
98
|
-
# This is a bit weird. But not so weird that it needs fixing.
|
99
|
-
log_block.call
|
94
|
+
import_endpoint :b2_upload_part do |fn, sequence, bytes, sha|
|
95
|
+
Backup::Logger.info "#{src} trying part #{sequence + 1} of #{part_count}"
|
100
96
|
|
97
|
+
# not the same as the auth_headers value
|
101
98
|
headers = {
|
102
|
-
|
103
|
-
'Authorization' => file_auth_token,
|
99
|
+
'Authorization' => url_token.auth,
|
104
100
|
# cos backblaze wants 1-based, but we want 0-based for reading file
|
105
101
|
'X-Bz-Part-Number' => sequence + 1,
|
106
102
|
'Content-Length' => bytes.length,
|
107
|
-
'X-Bz-Content-Sha1' =>
|
108
|
-
}
|
109
|
-
|
110
|
-
# Yes, this is a different pattern to the other Excon.post calls ¯\_(ツ)_/¯
|
111
|
-
rsp = Excon.post \
|
112
|
-
upload_url,
|
113
|
-
headers: headers,
|
114
|
-
body: bytes,
|
115
|
-
expects: 200
|
116
|
-
|
117
|
-
# 200 response will be
|
118
|
-
# fileId The unique ID for this file.
|
119
|
-
# partNumber Which part this is.
|
120
|
-
# contentLength The number of bytes stored in the part.
|
121
|
-
# contentSha1 The SHA1 of the bytes stored in the part.
|
103
|
+
'X-Bz-Content-Sha1' => sha,
|
104
|
+
}.merge(TEST_HEADERS)
|
122
105
|
|
123
|
-
|
124
|
-
sha
|
106
|
+
fn[url_token.url, headers, bytes]
|
125
107
|
end
|
126
108
|
|
127
|
-
|
128
|
-
|
129
|
-
"#{url}/b2api/v1/b2_finish_large_file",
|
130
|
-
**auth_headers,
|
131
|
-
body: {fileId: file_id, partSha1Array: shas }.to_json,
|
132
|
-
expects: 200
|
133
|
-
|
134
|
-
HashWrap.from_json rsp.body
|
109
|
+
import_endpoint :b2_finish_large_file do |fn, shas|
|
110
|
+
fn[account.api_url, auth_headers, file_id, shas]
|
135
111
|
end
|
136
112
|
|
137
113
|
# 10000 is backblaze specified max number of parts
|
138
114
|
MAX_PARTS = 10000
|
139
115
|
|
140
|
-
|
116
|
+
def upload_parts
|
117
|
+
(0...MAX_PARTS).each_with_object [] do |sequence, shas|
|
118
|
+
# read length, offset
|
119
|
+
bytes = src.read part_size, part_size * sequence
|
120
|
+
|
121
|
+
if bytes.nil? || bytes.empty?
|
122
|
+
# no more file to send
|
123
|
+
break shas
|
124
|
+
else
|
125
|
+
sha = Digest::SHA1.hexdigest bytes
|
126
|
+
b2_upload_part sequence, bytes, sha
|
127
|
+
Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
|
128
|
+
shas << sha
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
141
132
|
|
142
133
|
def call
|
143
134
|
if src.size > part_size * MAX_PARTS
|
144
135
|
raise Error, "File #{src.to_s} has size #{src.size} which is larger than part_size * MAX_PARTS #{part_size * MAX_PARTS}. Try increasing part_size in model."
|
145
136
|
end
|
146
137
|
|
147
|
-
|
148
|
-
token_provider = TokenProvider.new &method(:b2_get_upload_part_url)
|
149
|
-
shas = (0...MAX_PARTS).each_with_object [] do |sequence, shas|
|
150
|
-
sha = retry_upload 0, token_provider do |token_provider, retries|
|
151
|
-
# return sha
|
152
|
-
b2_upload_part sequence, token_provider.upload_url, token_provider.file_auth_token do
|
153
|
-
Backup::Logger.info "#{src} trying part #{sequence + 1} of #{(src.size / part_size.to_r).ceil} retry #{retries}"
|
154
|
-
end
|
155
|
-
end
|
138
|
+
Logger.info "uploading '#{src}' to #{dst}' of #{src.size} in #{part_count} parts"
|
156
139
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
Backup::Logger.info "#{src} stored part #{sequence + 1} with #{sha}"
|
161
|
-
else
|
162
|
-
break shas
|
163
|
-
end
|
164
|
-
end
|
140
|
+
b2_start_large_file # not really necessary, but makes the flow clearer
|
141
|
+
url_token # try to re-use existing url token if there is one
|
142
|
+
shas = upload_parts
|
165
143
|
|
166
144
|
# finish up, log and return the response
|
167
145
|
hash_wrap = b2_finish_large_file shas
|