vpsadmin-client 2.2.0 → 2.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +9 -0
- data/lib/vpsadmin/cli.rb +11 -1
- data/lib/vpsadmin/cli/commands/backup_dataset.rb +543 -0
- data/lib/vpsadmin/cli/commands/backup_vps.rb +29 -0
- data/lib/vpsadmin/cli/commands/base_download.rb +31 -0
- data/lib/vpsadmin/cli/commands/snapshot_download.rb +214 -0
- data/lib/vpsadmin/cli/commands/snapshot_send.rb +101 -0
- data/lib/vpsadmin/cli/commands/vps_migrate_many.rb +5 -0
- data/lib/vpsadmin/cli/stream_downloader.rb +236 -0
- data/lib/vpsadmin/client/version.rb +1 -1
- data/vpsadmin-client.gemspec +1 -0
- metadata +22 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0201c799bf6e1d1573fe678f3c55645b0f6e6524
|
4
|
+
data.tar.gz: c7ac0933e44dd445735a44497cf2696426095dfd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5c94af86fdbec2dfd8132e7e5931c371652152d9159fc7d721785188b725764f057ce15a83f8a67f3630d1a3b29d2f596d7685dc1fa9a4ad596c341dc874123c
|
7
|
+
data.tar.gz: 39688e6d85d56d6d1fa2da38060cc719296427af25cb3ddc52dfff2ea097a3820f7fe7e22446f9f095ad8832f29f2fa3c801b1e7dd461bf9885023691b3ccfe9
|
data/CHANGELOG
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
* Mon Mar 21 2016 - version 2.3.0
|
2
|
+
- New commands for downloading snapshots:
|
3
|
+
- snapshot download
|
4
|
+
- snapshot send
|
5
|
+
- backup dataset
|
6
|
+
- backup vps
|
7
|
+
- vps migrate_many has new option --[no-]cleanup-data
|
8
|
+
|
9
|
+
* Fri Feb 26 2016 - version 2.2.0
|
1
10
|
- New command vps migrate_many
|
2
11
|
|
3
12
|
* Sat Jan 23 2016 - version 2.1.0
|
data/lib/vpsadmin/cli.rb
CHANGED
@@ -1,14 +1,24 @@
|
|
1
1
|
require 'haveapi/cli'
|
2
|
+
require 'vpsadmin/client/version'
|
2
3
|
|
3
4
|
module VpsAdmin
|
4
5
|
module CLI
|
5
6
|
module Commands ; end
|
6
7
|
|
7
8
|
class Cli < HaveAPI::CLI::Cli
|
8
|
-
|
9
|
+
def show_version
|
10
|
+
puts "#{VpsAdmin::Client::VERSION} based on haveapi-client "+
|
11
|
+
HaveAPI::Client::VERSION
|
12
|
+
end
|
9
13
|
end
|
10
14
|
end
|
11
15
|
end
|
12
16
|
|
17
|
+
require 'vpsadmin/cli/stream_downloader'
|
18
|
+
require 'vpsadmin/cli/commands/base_download'
|
13
19
|
require 'vpsadmin/cli/commands/vps_remote_console'
|
14
20
|
require 'vpsadmin/cli/commands/vps_migrate_many'
|
21
|
+
require 'vpsadmin/cli/commands/snapshot_download'
|
22
|
+
require 'vpsadmin/cli/commands/snapshot_send'
|
23
|
+
require 'vpsadmin/cli/commands/backup_dataset'
|
24
|
+
require 'vpsadmin/cli/commands/backup_vps'
|
@@ -0,0 +1,543 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
module VpsAdmin::CLI::Commands
|
4
|
+
class BackupDataset < BaseDownload
|
5
|
+
cmd :backup, :dataset
|
6
|
+
args '[DATASET_ID] FILESYSTEM'
|
7
|
+
desc 'Backup dataset locally'
|
8
|
+
|
9
|
+
LocalSnapshot = Struct.new(:name, :hist_id, :creation) do
|
10
|
+
def creation=(c)
|
11
|
+
self[:creation] = c.to_i
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def options(opts)
|
16
|
+
@opts = {
|
17
|
+
rotate: true,
|
18
|
+
min_snapshots: 30,
|
19
|
+
max_snapshots: 45,
|
20
|
+
max_age: 30,
|
21
|
+
attempts: 10,
|
22
|
+
checksum: true,
|
23
|
+
delete_after: true,
|
24
|
+
}
|
25
|
+
|
26
|
+
opts.on('-p', '--pretend', 'Print what would the program do') do
|
27
|
+
@opts[:pretend] = true
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on('-r', '--[no-]rotate', 'Delete old snapshots (enabled)') do |r|
|
31
|
+
@opts[:rotate] = r
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on('-m', '--min-snapshots N', Integer, 'Keep at least N snapshots (30)') do |m|
|
35
|
+
exit_msg('--min-snapshots must be greater than zero') if m <= 0
|
36
|
+
@opts[:min_snapshots] = m
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on('-M', '--max-snapshots N', Integer, 'Keep at most N snapshots (45)') do |m|
|
40
|
+
exit_msg('--max-snapshots must be greater than zero') if m <= 0
|
41
|
+
@opts[:max_snapshots] = m
|
42
|
+
end
|
43
|
+
|
44
|
+
opts.on('-a', '--max-age N', Integer, 'Delete snapshots older then N days (30)') do |m|
|
45
|
+
exit_msg('--max-age must be greater than zero') if m <= 0
|
46
|
+
@opts[:max_age] = m
|
47
|
+
end
|
48
|
+
|
49
|
+
opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
|
50
|
+
exit_msg('--max-rate must be greater than zero') if r <= 0
|
51
|
+
@opts[:max_rate] = r
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on('-q', '--quiet', 'Print only errors') do |q|
|
55
|
+
@opts[:quiet] = q
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on('-s', '--safe-download', 'Download to a temp file (needs 2x disk space)') do |s|
|
59
|
+
@opts[:safe] = s
|
60
|
+
end
|
61
|
+
|
62
|
+
opts.on('--retry-attemps N', Integer, 'Retry N times to recover from download error (10)') do |n|
|
63
|
+
exit_msg('--retry-attempts must be greater than zero') if n <= 0
|
64
|
+
@opts[:attempts] = n
|
65
|
+
end
|
66
|
+
|
67
|
+
opts.on('-i', '--init-snapshots N', Integer, 'Download max N snapshots initially') do |s|
|
68
|
+
exit_msg('--init-snapshots must be greater than zero') if s <= 0
|
69
|
+
@opts[:init_snapshots] = s
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
|
73
|
+
@opts[:checksum] = c
|
74
|
+
end
|
75
|
+
|
76
|
+
opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download (enabled)') do |d|
|
77
|
+
@opts[:delete_after] = d
|
78
|
+
end
|
79
|
+
|
80
|
+
opts.on('--no-snapshots-as-error', 'Consider no snapshots to download as an error') do
|
81
|
+
@opts[:no_snapshots_error] = true
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def exec(args)
|
86
|
+
if args.size == 1 && /^\d+$/ !~ args[0]
|
87
|
+
fs = args[0]
|
88
|
+
|
89
|
+
ds_id = read_dataset_id(fs)
|
90
|
+
|
91
|
+
if ds_id
|
92
|
+
ds = @api.dataset.show(ds_id)
|
93
|
+
else
|
94
|
+
ds = dataset_chooser
|
95
|
+
end
|
96
|
+
|
97
|
+
elsif args.size != 2
|
98
|
+
warn "Provide DATASET_ID and FILESYSTEM arguments"
|
99
|
+
exit(false)
|
100
|
+
|
101
|
+
else
|
102
|
+
ds = @api.dataset.show(args[0].to_i)
|
103
|
+
fs = args[1]
|
104
|
+
end
|
105
|
+
|
106
|
+
check_dataset_id!(ds, fs)
|
107
|
+
snapshots = ds.snapshot.list
|
108
|
+
|
109
|
+
local_state = parse_tree(fs)
|
110
|
+
|
111
|
+
# - Find out current history ID
|
112
|
+
# - If there are snapshots with this ID that are not present locally,
|
113
|
+
# download them
|
114
|
+
# - If the dataset for this history ID does not exist, create it
|
115
|
+
# - If it exists, check what snapshots are there and make an incremental
|
116
|
+
# download
|
117
|
+
|
118
|
+
remote_state = {}
|
119
|
+
|
120
|
+
snapshots.each do |s|
|
121
|
+
remote_state[s.history_id] ||= []
|
122
|
+
remote_state[s.history_id] << s
|
123
|
+
end
|
124
|
+
|
125
|
+
if remote_state[ds.current_history_id].nil? \
|
126
|
+
|| remote_state[ds.current_history_id].empty?
|
127
|
+
exit_msg(
|
128
|
+
"Nothing to transfer: no snapshots with history id #{ds.current_history_id}",
|
129
|
+
error: @opts[:no_snapshots_error]
|
130
|
+
)
|
131
|
+
end
|
132
|
+
|
133
|
+
for_transfer = []
|
134
|
+
|
135
|
+
latest_local_snapshot = local_state[ds.current_history_id] \
|
136
|
+
&& local_state[ds.current_history_id].last
|
137
|
+
found_latest = false
|
138
|
+
|
139
|
+
# This is the first run within this history id, no local snapshots are
|
140
|
+
# present
|
141
|
+
if !latest_local_snapshot && @opts[:init_snapshots]
|
142
|
+
remote_state[ds.current_history_id] = \
|
143
|
+
remote_state[ds.current_history_id].last(@opts[:init_snapshots])
|
144
|
+
end
|
145
|
+
|
146
|
+
remote_state[ds.current_history_id].each do |snap|
|
147
|
+
found = false
|
148
|
+
|
149
|
+
local_state.values.each do |snapshots|
|
150
|
+
found = snapshots.detect { |s| s.name == snap.name }
|
151
|
+
break if found
|
152
|
+
end
|
153
|
+
|
154
|
+
if !found_latest && latest_local_snapshot \
|
155
|
+
&& latest_local_snapshot.name == snap.name
|
156
|
+
found_latest = true
|
157
|
+
|
158
|
+
elsif latest_local_snapshot
|
159
|
+
next unless found_latest
|
160
|
+
end
|
161
|
+
|
162
|
+
for_transfer << snap unless found
|
163
|
+
end
|
164
|
+
|
165
|
+
if for_transfer.empty?
|
166
|
+
if found_latest
|
167
|
+
exit_msg(
|
168
|
+
"Nothing to transfer: all snapshots with history id "+
|
169
|
+
"#{ds.current_history_id} are already present locally",
|
170
|
+
error: @opts[:no_snapshots_error]
|
171
|
+
)
|
172
|
+
|
173
|
+
else
|
174
|
+
exit_msg(<<END
|
175
|
+
Unable to transfer: the common snapshot has not been found
|
176
|
+
|
177
|
+
This can happen when the latest local snapshot was deleted from the server,
|
178
|
+
i.e. you have not backed up this dataset for quite some time.
|
179
|
+
|
180
|
+
You can either rename or destroy the whole current history id:
|
181
|
+
|
182
|
+
zfs rename #{fs}/#{ds.current_history_id} #{fs}/#{ds.current_history_id}.old
|
183
|
+
|
184
|
+
or
|
185
|
+
|
186
|
+
zfs list -r -t all #{fs}/#{ds.current_history_id}
|
187
|
+
zfs destroy -r #{fs}/#{ds.current_history_id}
|
188
|
+
|
189
|
+
which will destroy all snapshots with this history id.
|
190
|
+
|
191
|
+
You can also destroy the local backup completely or backup to another dataset
|
192
|
+
and start anew.
|
193
|
+
END
|
194
|
+
)
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
unless @opts[:quiet]
|
199
|
+
puts "Will download #{for_transfer.size} snapshots:"
|
200
|
+
for_transfer.each { |s| puts " @#{s.name}" }
|
201
|
+
puts
|
202
|
+
end
|
203
|
+
|
204
|
+
if @opts[:pretend]
|
205
|
+
pretend_state = local_state.clone
|
206
|
+
pretend_state[ds.current_history_id] ||= []
|
207
|
+
pretend_state[ds.current_history_id].concat(for_transfer.map do |s|
|
208
|
+
LocalSnapshot.new(s.name, ds.current_history_id, Time.iso8601(s.created_at).to_i)
|
209
|
+
end)
|
210
|
+
|
211
|
+
rotate(fs, pretend: pretend_state) if @opts[:rotate]
|
212
|
+
|
213
|
+
else
|
214
|
+
# Find the common snapshot between server and localhost, so that the transfer
|
215
|
+
# can be incremental.
|
216
|
+
shared_name = local_state[ds.current_history_id] \
|
217
|
+
&& !local_state[ds.current_history_id].empty? \
|
218
|
+
&& local_state[ds.current_history_id].last.name
|
219
|
+
shared = nil
|
220
|
+
|
221
|
+
if shared_name
|
222
|
+
shared = remote_state[ds.current_history_id].detect { |s| s.name == shared_name }
|
223
|
+
|
224
|
+
if shared && !for_transfer.detect { |s| s.id == shared.id }
|
225
|
+
for_transfer.insert(0, shared)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
write_dataset_id!(ds, fs) unless written_dataset_id?
|
230
|
+
transfer(local_state, for_transfer, ds.current_history_id, fs)
|
231
|
+
rotate(fs) if @opts[:rotate]
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
protected
|
236
|
+
def transfer(local_state, snapshots, hist_id, fs)
|
237
|
+
ds = "#{fs}/#{hist_id}"
|
238
|
+
no_local_snapshots = local_state[hist_id].nil? || local_state[hist_id].empty?
|
239
|
+
|
240
|
+
if local_state[hist_id].nil?
|
241
|
+
zfs(:create, nil, ds)
|
242
|
+
end
|
243
|
+
|
244
|
+
if no_local_snapshots
|
245
|
+
msg "Performing a full receive of @#{snapshots.first.name} to #{ds}"
|
246
|
+
|
247
|
+
if @opts[:safe]
|
248
|
+
safe_download(ds, snapshots.first)
|
249
|
+
|
250
|
+
else
|
251
|
+
run_piped(zfs_cmd(:recv, '-F', ds)) do
|
252
|
+
SnapshotSend.new({}, @api).do_exec({
|
253
|
+
snapshot: snapshots.first.id,
|
254
|
+
send_mail: false,
|
255
|
+
delete_after: @opts[:delete_after],
|
256
|
+
max_rate: @opts[:max_rate],
|
257
|
+
checksum: @opts[:checksum],
|
258
|
+
quiet: @opts[:quiet],
|
259
|
+
})
|
260
|
+
end || exit_msg('Receive failed')
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
if !no_local_snapshots || snapshots.size > 1
|
265
|
+
msg "Performing an incremental receive of "+
|
266
|
+
"@#{snapshots.first.name} - @#{snapshots.last.name} to #{ds}"
|
267
|
+
|
268
|
+
if @opts[:safe]
|
269
|
+
safe_download(ds, snapshots.last, snapshots.first)
|
270
|
+
|
271
|
+
else
|
272
|
+
run_piped(zfs_cmd(:recv, '-F', ds)) do
|
273
|
+
SnapshotSend.new({}, @api).do_exec({
|
274
|
+
snapshot: snapshots.last.id,
|
275
|
+
from_snapshot: snapshots.first.id,
|
276
|
+
send_mail: false,
|
277
|
+
delete_after: @opts[:delete_after],
|
278
|
+
max_rate: @opts[:max_rate],
|
279
|
+
checksum: @opts[:checksum],
|
280
|
+
quiet: @opts[:quiet],
|
281
|
+
})
|
282
|
+
end || exit_msg('Receive failed')
|
283
|
+
end
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def safe_download(ds, snapshot, from_snapshot = nil)
|
288
|
+
part, full = snapshot_tmp_file(snapshot, from_snapshot)
|
289
|
+
|
290
|
+
if !File.exists?(full)
|
291
|
+
attempts = 0
|
292
|
+
|
293
|
+
begin
|
294
|
+
SnapshotDownload.new({}, @api).do_exec({
|
295
|
+
snapshot: snapshot.id,
|
296
|
+
from_snapshot: from_snapshot && from_snapshot.id,
|
297
|
+
format: from_snapshot ? :incremental_stream : :stream,
|
298
|
+
file: part,
|
299
|
+
max_rate: @opts[:max_rate],
|
300
|
+
checksum: @opts[:checksum],
|
301
|
+
quiet: @opts[:quiet],
|
302
|
+
resume: true,
|
303
|
+
delete_after: @opts[:delete_after],
|
304
|
+
send_mail: false,
|
305
|
+
})
|
306
|
+
|
307
|
+
rescue Errno::ECONNREFUSED,
|
308
|
+
Errno::ETIMEDOUT,
|
309
|
+
Errno::EHOSTUNREACH,
|
310
|
+
Errno::ECONNRESET => e
|
311
|
+
warn "Connection error: #{e.message}"
|
312
|
+
|
313
|
+
attempts += 1
|
314
|
+
|
315
|
+
if attempts >= @opts[:attempts]
|
316
|
+
warn "Run out of attempts"
|
317
|
+
exit(false)
|
318
|
+
|
319
|
+
else
|
320
|
+
warn "Retry in 60 seconds"
|
321
|
+
sleep(60)
|
322
|
+
retry
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
File.rename(part, full)
|
327
|
+
end
|
328
|
+
|
329
|
+
run_piped(zfs_cmd(:recv, '-F', ds)) do
|
330
|
+
Process.exec("zcat #{full}")
|
331
|
+
end || exit_msg('Receive failed')
|
332
|
+
|
333
|
+
File.delete(full)
|
334
|
+
end
|
335
|
+
|
336
|
+
def rotate(fs, pretend: false)
|
337
|
+
msg "Rotating snapshots"
|
338
|
+
local_state = pretend ? pretend : parse_tree(fs)
|
339
|
+
|
340
|
+
# Order snapshots by date of creation
|
341
|
+
snapshots = local_state.values.flatten.sort do |a, b|
|
342
|
+
a.creation <=> b.creation
|
343
|
+
end
|
344
|
+
|
345
|
+
cnt = local_state.values.inject(0) { |sum, snapshots| sum + snapshots.count }
|
346
|
+
deleted = 0
|
347
|
+
oldest = Time.now.to_i - (@opts[:max_age] * 60 * 60 * 24)
|
348
|
+
|
349
|
+
snapshots.each do |s|
|
350
|
+
ds = "#{fs}/#{s.hist_id}"
|
351
|
+
|
352
|
+
if (cnt - deleted) <= @opts[:min_snapshots] \
|
353
|
+
|| (s.creation > oldest && (cnt - deleted) <= @opts[:max_snapshots])
|
354
|
+
break
|
355
|
+
end
|
356
|
+
|
357
|
+
deleted += 1
|
358
|
+
local_state[s.hist_id].delete(s)
|
359
|
+
|
360
|
+
msg "Destroying #{ds}@#{s.name}"
|
361
|
+
zfs(:destroy, nil, "#{ds}@#{s.name}", pretend: pretend)
|
362
|
+
end
|
363
|
+
|
364
|
+
local_state.each do |hist_id, snapshots|
|
365
|
+
next unless snapshots.empty?
|
366
|
+
|
367
|
+
ds = "#{fs}/#{hist_id}"
|
368
|
+
|
369
|
+
msg "Destroying #{ds}"
|
370
|
+
zfs(:destroy, nil, ds, pretend: pretend)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
def parse_tree(fs)
|
375
|
+
ret = {}
|
376
|
+
|
377
|
+
# This is intentionally done by two zfs commands, because -d2 would include
|
378
|
+
# nested subdatasets, which should not be there, but the user might create
|
379
|
+
# them and it could confuse the program.
|
380
|
+
zfs(:list, '-r -d1 -tfilesystem -H -oname', fs).split("\n")[1..-1].each do |name|
|
381
|
+
last_name = name.split('/').last
|
382
|
+
ret[last_name.to_i] = [] if dataset?(last_name)
|
383
|
+
end
|
384
|
+
|
385
|
+
zfs(
|
386
|
+
:get,
|
387
|
+
'-Hrp -d2 name,creation -tsnapshot -oname,property,value',
|
388
|
+
fs
|
389
|
+
).split("\n").each do |line|
|
390
|
+
name, property, value = line.split
|
391
|
+
ds, snap_name = name.split('@')
|
392
|
+
ds_name = ds.split('/').last
|
393
|
+
next unless dataset?(ds_name)
|
394
|
+
|
395
|
+
hist_id = ds_name.to_i
|
396
|
+
|
397
|
+
if snap = ret[hist_id].detect { |s| s.name == snap_name }
|
398
|
+
snap.send("#{property}=", value)
|
399
|
+
|
400
|
+
else
|
401
|
+
snap = LocalSnapshot.new(snap_name, hist_id)
|
402
|
+
ret[hist_id] << snap
|
403
|
+
end
|
404
|
+
end
|
405
|
+
|
406
|
+
ret
|
407
|
+
end
|
408
|
+
|
409
|
+
def dataset?(name)
|
410
|
+
/^\d+$/ =~ name
|
411
|
+
end
|
412
|
+
|
413
|
+
def read_dataset_id(fs)
|
414
|
+
ds_id = zfs(:get, '-H -ovalue cz.vpsfree.vpsadmin:dataset_id', fs).strip
|
415
|
+
return nil if ds_id == '-'
|
416
|
+
@dataset_id = ds_id.to_i
|
417
|
+
end
|
418
|
+
|
419
|
+
def check_dataset_id!(ds, fs)
|
420
|
+
if @dataset_id && @dataset_id != ds.id
|
421
|
+
warn "Dataset '#{fs}' is used to backup remote dataset with id '#{@dataset_id}', not '#{ds.id}'"
|
422
|
+
exit(false)
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
def written_dataset_id?
|
427
|
+
!@dataset_id.nil?
|
428
|
+
end
|
429
|
+
|
430
|
+
def write_dataset_id!(ds, fs)
|
431
|
+
zfs(:set, "cz.vpsfree.vpsadmin:dataset_id=#{ds.id}", fs)
|
432
|
+
end
|
433
|
+
|
434
|
+
# Run two processes like +block | cmd2+, where block's stdout is piped into
|
435
|
+
# cmd2's stdin.
|
436
|
+
def run_piped(cmd2, &block)
|
437
|
+
r, w = IO.pipe
|
438
|
+
pids = []
|
439
|
+
|
440
|
+
pids << Process.fork do
|
441
|
+
r.close
|
442
|
+
STDOUT.reopen(w)
|
443
|
+
block.call
|
444
|
+
end
|
445
|
+
|
446
|
+
pids << Process.fork do
|
447
|
+
w.close
|
448
|
+
STDIN.reopen(r)
|
449
|
+
Process.exec(cmd2)
|
450
|
+
end
|
451
|
+
|
452
|
+
r.close
|
453
|
+
w.close
|
454
|
+
|
455
|
+
ret = true
|
456
|
+
|
457
|
+
pids.each do |pid|
|
458
|
+
Process.wait(pid)
|
459
|
+
ret = false if $?.exitstatus != 0
|
460
|
+
end
|
461
|
+
|
462
|
+
ret
|
463
|
+
end
|
464
|
+
|
465
|
+
def zfs_cmd(cmd, opts, fs)
|
466
|
+
s = ''
|
467
|
+
s += 'sudo ' if Process.euid != 0
|
468
|
+
s += 'zfs'
|
469
|
+
"#{s} #{cmd} #{opts} #{fs}"
|
470
|
+
end
|
471
|
+
|
472
|
+
def zfs(cmd, opts, fs, pretend: false)
|
473
|
+
cmd = zfs_cmd(cmd, opts, fs)
|
474
|
+
|
475
|
+
if pretend
|
476
|
+
puts "> #{cmd}"
|
477
|
+
return
|
478
|
+
end
|
479
|
+
|
480
|
+
ret = `#{cmd}`
|
481
|
+
exit_msg("#{cmd} failed with exit code #{$?.exitstatus}") if $?.exitstatus != 0
|
482
|
+
ret
|
483
|
+
end
|
484
|
+
|
485
|
+
def dataset_chooser(vps_only: false)
|
486
|
+
user = @api.user.current
|
487
|
+
vpses = @api.vps.list(user: user.id)
|
488
|
+
|
489
|
+
vps_map = {}
|
490
|
+
vpses.each do |vps|
|
491
|
+
vps_map[vps.dataset_id] = vps
|
492
|
+
end
|
493
|
+
|
494
|
+
i = 1
|
495
|
+
ds_map = {}
|
496
|
+
|
497
|
+
@api.dataset.index(user: user.id).each do |ds|
|
498
|
+
if vps = vps_map[ds.id]
|
499
|
+
puts "(#{i}) VPS ##{vps.id}"
|
500
|
+
|
501
|
+
else
|
502
|
+
next if vps_only
|
503
|
+
puts "(#{i}) Dataset #{ds.name}"
|
504
|
+
end
|
505
|
+
|
506
|
+
ds_map[i] = ds
|
507
|
+
i += 1
|
508
|
+
end
|
509
|
+
|
510
|
+
loop do
|
511
|
+
STDOUT.write('Pick a dataset to backup: ')
|
512
|
+
STDOUT.flush
|
513
|
+
|
514
|
+
i = STDIN.readline.strip.to_i
|
515
|
+
next if i <= 0 || ds_map[i].nil?
|
516
|
+
|
517
|
+
return ds_map[i]
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
521
|
+
def snapshot_tmp_file(s, from_s = nil)
|
522
|
+
if from_s
|
523
|
+
base = ".snapshot_#{from_s.id}-#{s.id}.inc.dat.gz"
|
524
|
+
|
525
|
+
else
|
526
|
+
base = ".snapshot_#{s.id}.dat.gz"
|
527
|
+
end
|
528
|
+
|
529
|
+
["#{base}.part", base]
|
530
|
+
end
|
531
|
+
|
532
|
+
def exit_msg(str, error: true)
|
533
|
+
if error
|
534
|
+
warn str
|
535
|
+
exit(1)
|
536
|
+
|
537
|
+
else
|
538
|
+
msg str
|
539
|
+
exit(0)
|
540
|
+
end
|
541
|
+
end
|
542
|
+
end
|
543
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module VpsAdmin::CLI::Commands
|
2
|
+
class BackupVps < BackupDataset
|
3
|
+
cmd :backup, :vps
|
4
|
+
args '[VPS_ID] FILESYSTEM'
|
5
|
+
desc 'Backup VPS locally'
|
6
|
+
|
7
|
+
def exec(args)
|
8
|
+
if args.size == 1 && /^\d+$/ !~ args[0]
|
9
|
+
fs = args[0]
|
10
|
+
|
11
|
+
ds_id = read_dataset_id(fs)
|
12
|
+
|
13
|
+
if ds_id
|
14
|
+
super([ds_id, fs])
|
15
|
+
|
16
|
+
else
|
17
|
+
ds = dataset_chooser(vps_only: true)
|
18
|
+
super([ds.id, fs])
|
19
|
+
end
|
20
|
+
|
21
|
+
elsif args.size == 2
|
22
|
+
super([@api.vps.show(args[0].to_i).dataset_id, args[1]])
|
23
|
+
|
24
|
+
else
|
25
|
+
super(args)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module VpsAdmin::CLI::Commands
|
2
|
+
class BaseDownload < HaveAPI::CLI::Command
|
3
|
+
protected
|
4
|
+
def find_or_create_dl(opts, do_create = true)
|
5
|
+
@api.snapshot_download.index(snapshot: opts[:snapshot]).each do |r|
|
6
|
+
if opts[:from_snapshot] == (r.from_snapshot && r.from_snapshot_id)
|
7
|
+
if r.format != opts[:format].to_s
|
8
|
+
fail "SnapshotDownload id=#{r.id} is in unusable format '#{r.format}' (needs '#{opts[:format]}')"
|
9
|
+
end
|
10
|
+
|
11
|
+
return [r, false]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
if do_create
|
16
|
+
[@api.snapshot_download.create(opts), true]
|
17
|
+
|
18
|
+
else
|
19
|
+
[nil, true]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def msg(str)
|
24
|
+
puts str unless @opts[:quiet]
|
25
|
+
end
|
26
|
+
|
27
|
+
def warn_msg(str)
|
28
|
+
warn str unless @opts[:quiet]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,214 @@
|
|
1
|
+
module VpsAdmin::CLI::Commands
|
2
|
+
class SnapshotDownload < BaseDownload
|
3
|
+
cmd :snapshot, :download
|
4
|
+
args '[SNAPSHOT_ID]'
|
5
|
+
desc 'Download a snapshot as an archive or a stream'
|
6
|
+
|
7
|
+
def options(opts)
|
8
|
+
@opts = {
|
9
|
+
delete_after: true,
|
10
|
+
send_mail: false,
|
11
|
+
checksum: true,
|
12
|
+
format: 'archive',
|
13
|
+
}
|
14
|
+
|
15
|
+
opts.on('-f', '--format FORMAT', 'archive, stream or incremental_stream') do |f|
|
16
|
+
@opts[:format] = f
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
|
20
|
+
@opts[:from_snapshot] = s
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
|
24
|
+
@opts[:delete_after] = d
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on('-F', '--force', 'Overwrite existing files if necessary') do |f|
|
28
|
+
@opts[:force] = f
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on('-o', '--output FILE', 'Save the download to FILE') do |f|
|
32
|
+
@opts[:file] = f
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on('-q', '--quiet', 'Print only errors') do |q|
|
36
|
+
@opts[:quiet] = q
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on('-r', '--resume', 'Resume cancelled download') do |r|
|
40
|
+
@opts[:resume] = r
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
|
44
|
+
@opts[:send_mail] = s
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
|
48
|
+
exit_msg('--max-rate must be greater than zero') if r <= 0
|
49
|
+
@opts[:max_rate] = r
|
50
|
+
end
|
51
|
+
|
52
|
+
opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
|
53
|
+
@opts[:checksum] = c
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def exec(args)
|
58
|
+
if args.size == 0 && STDIN.tty?
|
59
|
+
@opts[:snapshot] = snapshot_chooser
|
60
|
+
|
61
|
+
elsif args.size != 1
|
62
|
+
warn "Provide exactly one SNAPSHOT_ID as an argument"
|
63
|
+
exit(false)
|
64
|
+
|
65
|
+
else
|
66
|
+
@opts[:snapshot] = args.first.to_i
|
67
|
+
end
|
68
|
+
|
69
|
+
do_exec(@opts)
|
70
|
+
end
|
71
|
+
|
72
|
+
def do_exec(opts)
|
73
|
+
@opts = opts
|
74
|
+
f = action = nil
|
75
|
+
pos = 0
|
76
|
+
|
77
|
+
if @opts[:file] == '-'
|
78
|
+
f = STDOUT
|
79
|
+
|
80
|
+
elsif @opts[:file]
|
81
|
+
f, action, pos = open_file(@opts[:file])
|
82
|
+
end
|
83
|
+
|
84
|
+
dl, created = find_or_create_dl(@opts, action != :resume)
|
85
|
+
f, action, pos = open_file(dl.file_name) unless @opts[:file]
|
86
|
+
|
87
|
+
if created
|
88
|
+
if action == :resume
|
89
|
+
warn "Unable to resume the download: the file has been deleted from the server"
|
90
|
+
exit(false)
|
91
|
+
end
|
92
|
+
|
93
|
+
msg "The download is being prepared..."
|
94
|
+
sleep(5)
|
95
|
+
|
96
|
+
else
|
97
|
+
warn "Reusing existing SnapshotDownload (id=#{dl.id})"
|
98
|
+
end
|
99
|
+
|
100
|
+
msg "Downloading to #{f.path}"
|
101
|
+
|
102
|
+
begin
|
103
|
+
VpsAdmin::CLI::StreamDownloader.download(
|
104
|
+
@api,
|
105
|
+
dl,
|
106
|
+
f,
|
107
|
+
progress: !@opts[:quiet] && (f == STDOUT ? STDERR : STDOUT),
|
108
|
+
position: pos,
|
109
|
+
max_rate: @opts[:max_rate],
|
110
|
+
checksum: @opts[:checksum],
|
111
|
+
)
|
112
|
+
|
113
|
+
rescue VpsAdmin::CLI::DownloadError => e
|
114
|
+
warn e.message
|
115
|
+
exit(false)
|
116
|
+
|
117
|
+
ensure
|
118
|
+
f.close
|
119
|
+
end
|
120
|
+
|
121
|
+
@api.snapshot_download.delete(dl.id) if @opts[:delete_after]
|
122
|
+
end
|
123
|
+
|
124
|
+
protected
|
125
|
+
def open_file(path)
|
126
|
+
f = action = nil
|
127
|
+
pos = 0
|
128
|
+
|
129
|
+
if File.exists?(path) && File.size(path) > 0
|
130
|
+
if @opts[:resume]
|
131
|
+
action = :resume
|
132
|
+
|
133
|
+
elsif @opts[:force]
|
134
|
+
action = :overwrite
|
135
|
+
|
136
|
+
elsif STDIN.tty?
|
137
|
+
while action.nil?
|
138
|
+
STDERR.write("'#{path}' already exists. [A]bort, [r]esume or [o]verwrite? [a]: ")
|
139
|
+
STDERR.flush
|
140
|
+
|
141
|
+
action = {
|
142
|
+
'r' => :resume,
|
143
|
+
'o' => :overwrite,
|
144
|
+
'' => false,
|
145
|
+
}[STDIN.readline.strip.downcase]
|
146
|
+
end
|
147
|
+
|
148
|
+
else
|
149
|
+
warn "File '#{path}' already exists"
|
150
|
+
exit(false)
|
151
|
+
end
|
152
|
+
|
153
|
+
case action
|
154
|
+
when :resume
|
155
|
+
mode = 'a+'
|
156
|
+
pos = File.size(path)
|
157
|
+
|
158
|
+
when :overwrite
|
159
|
+
mode = 'w'
|
160
|
+
|
161
|
+
else
|
162
|
+
exit
|
163
|
+
end
|
164
|
+
|
165
|
+
f = File.open(path, mode)
|
166
|
+
else
|
167
|
+
f = File.open(path, 'w')
|
168
|
+
end
|
169
|
+
|
170
|
+
[f, action, pos]
|
171
|
+
end
|
172
|
+
|
173
|
+
def snapshot_chooser
|
174
|
+
user = @api.user.current
|
175
|
+
vpses = @api.vps.list(user: user.id)
|
176
|
+
|
177
|
+
ds_map = {}
|
178
|
+
vpses.each do |vps|
|
179
|
+
ds_map[vps.dataset_id] = vps
|
180
|
+
end
|
181
|
+
|
182
|
+
i = 1
|
183
|
+
snap_map = {}
|
184
|
+
|
185
|
+
@api.dataset.index(user: user.id).each do |ds|
|
186
|
+
snapshots = ds.snapshot.index
|
187
|
+
next if snapshots.empty?
|
188
|
+
|
189
|
+
if vps = ds_map[ds.id]
|
190
|
+
puts "VPS ##{vps.id}"
|
191
|
+
|
192
|
+
else
|
193
|
+
puts "Dataset #{ds.name}"
|
194
|
+
end
|
195
|
+
|
196
|
+
snapshots.each do |s|
|
197
|
+
snap_map[i] = s
|
198
|
+
puts " (#{i}) @#{s.created_at}"
|
199
|
+
i += 1
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
loop do
|
204
|
+
STDOUT.write('Pick a snapshot for download: ')
|
205
|
+
STDOUT.flush
|
206
|
+
|
207
|
+
i = STDIN.readline.strip.to_i
|
208
|
+
next if i <= 0 || snap_map[i].nil?
|
209
|
+
|
210
|
+
return snap_map[i].id
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module VpsAdmin::CLI::Commands
|
4
|
+
class SnapshotSend < BaseDownload
|
5
|
+
cmd :snapshot, :send
|
6
|
+
args 'SNAPSHOT_ID'
|
7
|
+
desc 'Download a snapshot stream and write it on stdout'
|
8
|
+
|
9
|
+
def options(opts)
|
10
|
+
@opts = {
|
11
|
+
delete_after: true,
|
12
|
+
send_mail: false,
|
13
|
+
checksum: true,
|
14
|
+
}
|
15
|
+
|
16
|
+
opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
|
17
|
+
@opts[:from_snapshot] = s
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
|
21
|
+
@opts[:delete_after] = d
|
22
|
+
end
|
23
|
+
|
24
|
+
opts.on('-q', '--quiet', 'Print only errors') do |q|
|
25
|
+
@opts[:quiet] = q
|
26
|
+
end
|
27
|
+
|
28
|
+
opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
|
29
|
+
@opts[:send_mail] = s
|
30
|
+
end
|
31
|
+
|
32
|
+
opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
|
33
|
+
exit_msg('--max-rate must be greater than zero') if r <= 0
|
34
|
+
@opts[:max_rate] = r
|
35
|
+
end
|
36
|
+
|
37
|
+
opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
|
38
|
+
@opts[:checksum] = c
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def exec(args)
|
43
|
+
if args.size != 1
|
44
|
+
warn "Provide exactly one SNAPSHOT_ID as an argument"
|
45
|
+
exit(false)
|
46
|
+
end
|
47
|
+
|
48
|
+
opts = @opts.clone
|
49
|
+
opts[:snapshot] = args.first.to_i
|
50
|
+
|
51
|
+
do_exec(opts)
|
52
|
+
end
|
53
|
+
|
54
|
+
def do_exec(opts)
|
55
|
+
@opts = opts
|
56
|
+
opts[:format] = opts[:from_snapshot] ? :incremental_stream : :stream
|
57
|
+
|
58
|
+
dl, created = find_or_create_dl(opts)
|
59
|
+
|
60
|
+
if created
|
61
|
+
warn_msg "The download is being prepared..."
|
62
|
+
sleep(5)
|
63
|
+
|
64
|
+
else
|
65
|
+
warn_msg "Reusing existing SnapshotDownload (id=#{dl.id})"
|
66
|
+
end
|
67
|
+
|
68
|
+
r, w = IO.pipe
|
69
|
+
|
70
|
+
pid = Process.fork do
|
71
|
+
r.close
|
72
|
+
|
73
|
+
begin
|
74
|
+
VpsAdmin::CLI::StreamDownloader.download(
|
75
|
+
@api,
|
76
|
+
dl,
|
77
|
+
w,
|
78
|
+
progress: !opts[:quiet] && STDERR,
|
79
|
+
max_rate: opts[:max_rate],
|
80
|
+
checksum: opts[:checksum],
|
81
|
+
)
|
82
|
+
|
83
|
+
rescue VpsAdmin::CLI::DownloadError => e
|
84
|
+
warn e.message
|
85
|
+
exit(false)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
w.close
|
90
|
+
|
91
|
+
gz = Zlib::GzipReader.new(r)
|
92
|
+
STDOUT.write(gz.readpartial(16*1024)) while !gz.eof?
|
93
|
+
gz.close
|
94
|
+
|
95
|
+
Process.wait(pid)
|
96
|
+
exit($?.exitstatus) if $?.exitstatus != 0
|
97
|
+
|
98
|
+
@api.snapshot_download.delete(dl.id) if opts[:delete_after]
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
@@ -18,6 +18,10 @@ module VpsAdmin::CLI::Commands
|
|
18
18
|
opts.on('--[no-]outage-window', 'Migrate VPSes inside outage windows') do |w|
|
19
19
|
@opts[:outage_window] = w
|
20
20
|
end
|
21
|
+
|
22
|
+
opts.on('--[no-]cleanup-data', 'Cleanup VPS dataset on the source node') do |c|
|
23
|
+
@opts[:cleanup_data] = c
|
24
|
+
end
|
21
25
|
|
22
26
|
opts.on('--[no-]stop-on-error', 'Cancel the plan if a migration fails') do |s|
|
23
27
|
@opts[:stop_on_error] = s
|
@@ -87,6 +91,7 @@ module VpsAdmin::CLI::Commands
|
|
87
91
|
dst_node: @opts[:dst_node],
|
88
92
|
}
|
89
93
|
params[:outage_window] = @opts[:outage_window] unless @opts[:outage_window].nil?
|
94
|
+
params[:cleanup_data] = @opts[:cleanup_data] unless @opts[:cleanup_data].nil?
|
90
95
|
|
91
96
|
plan.vps_migration.create(params)
|
92
97
|
end
|
@@ -0,0 +1,236 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'net/http'
|
3
|
+
require 'ruby-progressbar'
|
4
|
+
require 'digest'
|
5
|
+
|
6
|
+
module VpsAdmin::CLI
|
7
|
+
class DownloadError < StandardError ; end
|
8
|
+
|
9
|
+
class StreamDownloader
|
10
|
+
def self.download(*args)
|
11
|
+
new(*args)
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(api, dl, io, progress: STDOUT, position: 0, max_rate: nil,
|
15
|
+
checksum: true)
|
16
|
+
downloaded = position
|
17
|
+
uri = URI(dl.url)
|
18
|
+
digest = Digest::SHA256.new
|
19
|
+
dl_check = nil
|
20
|
+
|
21
|
+
if position > 0 && checksum
|
22
|
+
if progress
|
23
|
+
pb = ProgressBar.create(
|
24
|
+
title: 'Calculating checksum',
|
25
|
+
total: position,
|
26
|
+
format: '%E %t: [%B] %p%% %r MB/s',
|
27
|
+
rate_scale: ->(rate) { (rate / 1024.0 / 1024.0).round(2) },
|
28
|
+
throttle_rate: 0.2,
|
29
|
+
output: progress,
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
read = 0
|
34
|
+
step = 1*1024*1024
|
35
|
+
io.seek(0)
|
36
|
+
|
37
|
+
while read < position
|
38
|
+
data = io.read((read + step) > position ? position - read : step)
|
39
|
+
read += data.size
|
40
|
+
|
41
|
+
digest << data
|
42
|
+
pb.progress = read if pb
|
43
|
+
end
|
44
|
+
|
45
|
+
pb.finish if pb
|
46
|
+
end
|
47
|
+
|
48
|
+
if progress
|
49
|
+
self.format = '%t: [%B] %r kB/s'
|
50
|
+
|
51
|
+
@pb = ProgressBar.create(
|
52
|
+
title: 'Downloading',
|
53
|
+
total: nil,
|
54
|
+
format: @format,
|
55
|
+
rate_scale: ->(rate) { (rate / 1024.0).round(2) },
|
56
|
+
throttle_rate: 0.2,
|
57
|
+
starting_at: downloaded,
|
58
|
+
autofinish: false,
|
59
|
+
output: progress,
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
63
|
+
args = [uri.host] + Array.new(5, nil) + [{use_ssl: uri.scheme == 'https'}]
|
64
|
+
|
65
|
+
Net::HTTP.start(*args) do |http|
|
66
|
+
loop do
|
67
|
+
begin
|
68
|
+
dl_check = api.snapshot_download.show(dl.id)
|
69
|
+
|
70
|
+
if @pb && (dl_check.ready || (dl_check.size && dl_check.size > 0))
|
71
|
+
@pb.progress = downloaded
|
72
|
+
|
73
|
+
total = dl_check.size * 1024 * 1024
|
74
|
+
@pb.total = @pb.progress > total ? @pb.progress : total
|
75
|
+
@download_size = (dl_check.size / 1024.0).round(2)
|
76
|
+
|
77
|
+
if dl_check.ready
|
78
|
+
@download_ready = true
|
79
|
+
self.format = "%E %t #{@download_size} GB: [%B] %p%% %r kB/s"
|
80
|
+
|
81
|
+
else
|
82
|
+
self.format = "%E %t ~#{@download_size} GB: [%B] %p%% %r kB/s"
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
rescue HaveAPI::Client::ActionFailed => e
|
87
|
+
# The SnapshotDownload object no longer exists, the transaction
|
88
|
+
# responsible for its creation must have failed.
|
89
|
+
stop
|
90
|
+
raise DownloadError, 'The download has failed due to transaction failure'
|
91
|
+
end
|
92
|
+
|
93
|
+
headers = {}
|
94
|
+
headers['Range'] = "bytes=#{downloaded}-" if downloaded > 0
|
95
|
+
|
96
|
+
http.request_get(uri.path, headers) do |res|
|
97
|
+
case res.code
|
98
|
+
when '404' # Not Found
|
99
|
+
if downloaded > 0
|
100
|
+
# This means that the transaction used for preparing the download
|
101
|
+
# has failed, the file to download does not exist anymore, so fail.
|
102
|
+
raise DownloadError, 'The download has failed, most likely transaction failure'
|
103
|
+
|
104
|
+
else
|
105
|
+
# The file is not available yet, this is normal, the transaction
|
106
|
+
# may be queued and it can take some time before it is processed.
|
107
|
+
pause(10)
|
108
|
+
next
|
109
|
+
end
|
110
|
+
|
111
|
+
when '416' # Range Not Satisfiable
|
112
|
+
if downloaded > position
|
113
|
+
# We have already managed to download something (at this run, if the trasfer
|
114
|
+
# was resumed) and the server cannot provide more data yet. This can be
|
115
|
+
# because the server is busy. Wait and retry.
|
116
|
+
pause(20)
|
117
|
+
next
|
118
|
+
|
119
|
+
else
|
120
|
+
# The file is not ready yet - we ask for range that cannot be provided
|
121
|
+
# This happens when we're resuming a download and the file on the
|
122
|
+
# server was deleted meanwhile. The file might not be exactly the same
|
123
|
+
# as the one before, sha256sum would most likely fail.
|
124
|
+
raise DownloadError, 'Range not satisfiable'
|
125
|
+
end
|
126
|
+
|
127
|
+
when '200', '206' # OK and Partial Content
|
128
|
+
resume
|
129
|
+
|
130
|
+
else
|
131
|
+
raise DownloadError, "Unexpected HTTP status code '#{res.code}'"
|
132
|
+
end
|
133
|
+
|
134
|
+
t1 = Time.now
|
135
|
+
data_counter = 0
|
136
|
+
|
137
|
+
res.read_body do |fragment|
|
138
|
+
size = fragment.size
|
139
|
+
|
140
|
+
data_counter += size
|
141
|
+
downloaded += size
|
142
|
+
|
143
|
+
begin
|
144
|
+
if @pb && (@pb.total.nil? || @pb.progress < @pb.total)
|
145
|
+
@pb.progress += size
|
146
|
+
end
|
147
|
+
|
148
|
+
rescue ProgressBar::InvalidProgressError
|
149
|
+
# The total value is in MB, it is not precise, so the actual
|
150
|
+
# size may be a little bit bigger.
|
151
|
+
@pb.progress = @pb.total
|
152
|
+
end
|
153
|
+
|
154
|
+
digest.update(fragment) if checksum
|
155
|
+
|
156
|
+
if max_rate && max_rate > 0
|
157
|
+
t2 = Time.now
|
158
|
+
diff = t2 - t1
|
159
|
+
|
160
|
+
if diff > 0.005
|
161
|
+
# Current and expected rates in kB per interval +diff+
|
162
|
+
current_rate = data_counter / 1024
|
163
|
+
expected_rate = max_rate * diff
|
164
|
+
|
165
|
+
if current_rate > expected_rate
|
166
|
+
delay = diff / (expected_rate / (current_rate - expected_rate))
|
167
|
+
sleep(delay)
|
168
|
+
end
|
169
|
+
|
170
|
+
data_counter = 0
|
171
|
+
t1 = Time.now
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
io.write(fragment)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
# This was the last download, the transfer is complete.
|
180
|
+
break if dl_check.ready
|
181
|
+
|
182
|
+
# Give the server time to prepare additional data
|
183
|
+
pause(15)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
@pb.finish if @pb
|
188
|
+
|
189
|
+
# Verify the checksum
|
190
|
+
if checksum && digest.hexdigest != dl_check.sha256sum
|
191
|
+
raise DownloadError, 'The sha256sum does not match, retry the download'
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
protected
|
196
|
+
def pause(secs)
|
197
|
+
@paused = true
|
198
|
+
|
199
|
+
if @pb
|
200
|
+
secs.times do |i|
|
201
|
+
if @download_size
|
202
|
+
if @download_ready
|
203
|
+
@pb.format("%t #{@download_size} GB: [%B] waiting #{secs - i}")
|
204
|
+
|
205
|
+
else
|
206
|
+
@pb.format("%t ~#{@download_size} GB: [%B] waiting #{secs - i}")
|
207
|
+
end
|
208
|
+
|
209
|
+
else
|
210
|
+
@pb.format("%t: [%B] waiting #{secs - i}")
|
211
|
+
end
|
212
|
+
|
213
|
+
@pb.refresh(force: true)
|
214
|
+
sleep(1)
|
215
|
+
end
|
216
|
+
|
217
|
+
else
|
218
|
+
sleep(secs)
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def resume
|
223
|
+
@pb.format(@format) if @pb && @paused
|
224
|
+
@paused = false
|
225
|
+
end
|
226
|
+
|
227
|
+
def stop
|
228
|
+
@pb.stop if @pb
|
229
|
+
end
|
230
|
+
|
231
|
+
def format=(fmt)
|
232
|
+
@format = fmt
|
233
|
+
@pb.format(@format) if @pb
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
data/vpsadmin-client.gemspec
CHANGED
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_runtime_dependency 'eventmachine', '~> 1.0.3'
|
26
26
|
spec.add_runtime_dependency 'em-http-request', '~> 1.1.3'
|
27
27
|
spec.add_runtime_dependency 'json', '~> 1.8.3'
|
28
|
+
spec.add_runtime_dependency 'ruby-progressbar', '~> 1.7.5'
|
28
29
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: vpsadmin-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jakub Skokan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ~>
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 1.8.3
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: ruby-progressbar
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ~>
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 1.7.5
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ~>
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.7.5
|
97
111
|
description: Ruby API and CLI for vpsAdmin API
|
98
112
|
email:
|
99
113
|
- jakub.skokan@vpsfree.cz
|
@@ -111,8 +125,14 @@ files:
|
|
111
125
|
- bin/vpsadminctl
|
112
126
|
- lib/terminal-size.rb
|
113
127
|
- lib/vpsadmin/cli.rb
|
128
|
+
- lib/vpsadmin/cli/commands/backup_dataset.rb
|
129
|
+
- lib/vpsadmin/cli/commands/backup_vps.rb
|
130
|
+
- lib/vpsadmin/cli/commands/base_download.rb
|
131
|
+
- lib/vpsadmin/cli/commands/snapshot_download.rb
|
132
|
+
- lib/vpsadmin/cli/commands/snapshot_send.rb
|
114
133
|
- lib/vpsadmin/cli/commands/vps_migrate_many.rb
|
115
134
|
- lib/vpsadmin/cli/commands/vps_remote_console.rb
|
135
|
+
- lib/vpsadmin/cli/stream_downloader.rb
|
116
136
|
- lib/vpsadmin/client.rb
|
117
137
|
- lib/vpsadmin/client/version.rb
|
118
138
|
- vpsadmin-client.gemspec
|