vpsadmin-client 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG +9 -0
- data/lib/vpsadmin/cli.rb +11 -1
- data/lib/vpsadmin/cli/commands/backup_dataset.rb +543 -0
- data/lib/vpsadmin/cli/commands/backup_vps.rb +29 -0
- data/lib/vpsadmin/cli/commands/base_download.rb +31 -0
- data/lib/vpsadmin/cli/commands/snapshot_download.rb +214 -0
- data/lib/vpsadmin/cli/commands/snapshot_send.rb +101 -0
- data/lib/vpsadmin/cli/commands/vps_migrate_many.rb +5 -0
- data/lib/vpsadmin/cli/stream_downloader.rb +236 -0
- data/lib/vpsadmin/client/version.rb +1 -1
- data/vpsadmin-client.gemspec +1 -0
- metadata +22 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0201c799bf6e1d1573fe678f3c55645b0f6e6524
|
4
|
+
data.tar.gz: c7ac0933e44dd445735a44497cf2696426095dfd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5c94af86fdbec2dfd8132e7e5931c371652152d9159fc7d721785188b725764f057ce15a83f8a67f3630d1a3b29d2f596d7685dc1fa9a4ad596c341dc874123c
|
7
|
+
data.tar.gz: 39688e6d85d56d6d1fa2da38060cc719296427af25cb3ddc52dfff2ea097a3820f7fe7e22446f9f095ad8832f29f2fa3c801b1e7dd461bf9885023691b3ccfe9
|
data/CHANGELOG
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
* Mon Mar 21 2016 - version 2.3.0
|
2
|
+
- New commands for downloading snapshots:
|
3
|
+
- snapshot download
|
4
|
+
- snapshot send
|
5
|
+
- backup dataset
|
6
|
+
- backup vps
|
7
|
+
- vps migrate_many has new option --[no-]cleanup-data
|
8
|
+
|
9
|
+
* Fri Feb 26 2016 - version 2.2.0
|
1
10
|
- New command vps migrate_many
|
2
11
|
|
3
12
|
* Sat Jan 23 2016 - version 2.1.0
|
data/lib/vpsadmin/cli.rb
CHANGED
@@ -1,14 +1,24 @@
|
|
1
1
|
require 'haveapi/cli'
|
2
|
+
require 'vpsadmin/client/version'
|
2
3
|
|
3
4
|
module VpsAdmin
|
4
5
|
module CLI
|
5
6
|
module Commands ; end
|
6
7
|
|
7
8
|
class Cli < HaveAPI::CLI::Cli
|
8
|
-
|
9
|
+
def show_version
|
10
|
+
puts "#{VpsAdmin::Client::VERSION} based on haveapi-client "+
|
11
|
+
HaveAPI::Client::VERSION
|
12
|
+
end
|
9
13
|
end
|
10
14
|
end
|
11
15
|
end
|
12
16
|
|
17
|
+
require 'vpsadmin/cli/stream_downloader'
|
18
|
+
require 'vpsadmin/cli/commands/base_download'
|
13
19
|
require 'vpsadmin/cli/commands/vps_remote_console'
|
14
20
|
require 'vpsadmin/cli/commands/vps_migrate_many'
|
21
|
+
require 'vpsadmin/cli/commands/snapshot_download'
|
22
|
+
require 'vpsadmin/cli/commands/snapshot_send'
|
23
|
+
require 'vpsadmin/cli/commands/backup_dataset'
|
24
|
+
require 'vpsadmin/cli/commands/backup_vps'
|
@@ -0,0 +1,543 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
module VpsAdmin::CLI::Commands
|
4
|
+
class BackupDataset < BaseDownload
|
5
|
+
cmd :backup, :dataset
|
6
|
+
args '[DATASET_ID] FILESYSTEM'
|
7
|
+
desc 'Backup dataset locally'
|
8
|
+
|
9
|
+
LocalSnapshot = Struct.new(:name, :hist_id, :creation) do
|
10
|
+
def creation=(c)
|
11
|
+
self[:creation] = c.to_i
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def options(opts)
|
16
|
+
@opts = {
|
17
|
+
rotate: true,
|
18
|
+
min_snapshots: 30,
|
19
|
+
max_snapshots: 45,
|
20
|
+
max_age: 30,
|
21
|
+
attempts: 10,
|
22
|
+
checksum: true,
|
23
|
+
delete_after: true,
|
24
|
+
}
|
25
|
+
|
26
|
+
opts.on('-p', '--pretend', 'Print what would the program do') do
|
27
|
+
@opts[:pretend] = true
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on('-r', '--[no-]rotate', 'Delete old snapshots (enabled)') do |r|
|
31
|
+
@opts[:rotate] = r
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on('-m', '--min-snapshots N', Integer, 'Keep at least N snapshots (30)') do |m|
|
35
|
+
exit_msg('--min-snapshots must be greater than zero') if m <= 0
|
36
|
+
@opts[:min_snapshots] = m
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on('-M', '--max-snapshots N', Integer, 'Keep at most N snapshots (45)') do |m|
|
40
|
+
exit_msg('--max-snapshots must be greater than zero') if m <= 0
|
41
|
+
@opts[:max_snapshots] = m
|
42
|
+
end
|
43
|
+
|
44
|
+
opts.on('-a', '--max-age N', Integer, 'Delete snapshots older then N days (30)') do |m|
|
45
|
+
exit_msg('--max-age must be greater than zero') if m <= 0
|
46
|
+
@opts[:max_age] = m
|
47
|
+
end
|
48
|
+
|
49
|
+
opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
|
50
|
+
exit_msg('--max-rate must be greater than zero') if r <= 0
|
51
|
+
@opts[:max_rate] = r
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on('-q', '--quiet', 'Print only errors') do |q|
|
55
|
+
@opts[:quiet] = q
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on('-s', '--safe-download', 'Download to a temp file (needs 2x disk space)') do |s|
|
59
|
+
@opts[:safe] = s
|
60
|
+
end
|
61
|
+
|
62
|
+
opts.on('--retry-attemps N', Integer, 'Retry N times to recover from download error (10)') do |n|
|
63
|
+
exit_msg('--retry-attempts must be greater than zero') if n <= 0
|
64
|
+
@opts[:attempts] = n
|
65
|
+
end
|
66
|
+
|
67
|
+
opts.on('-i', '--init-snapshots N', Integer, 'Download max N snapshots initially') do |s|
|
68
|
+
exit_msg('--init-snapshots must be greater than zero') if s <= 0
|
69
|
+
@opts[:init_snapshots] = s
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
|
73
|
+
@opts[:checksum] = c
|
74
|
+
end
|
75
|
+
|
76
|
+
opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download (enabled)') do |d|
|
77
|
+
@opts[:delete_after] = d
|
78
|
+
end
|
79
|
+
|
80
|
+
opts.on('--no-snapshots-as-error', 'Consider no snapshots to download as an error') do
|
81
|
+
@opts[:no_snapshots_error] = true
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def exec(args)
|
86
|
+
if args.size == 1 && /^\d+$/ !~ args[0]
|
87
|
+
fs = args[0]
|
88
|
+
|
89
|
+
ds_id = read_dataset_id(fs)
|
90
|
+
|
91
|
+
if ds_id
|
92
|
+
ds = @api.dataset.show(ds_id)
|
93
|
+
else
|
94
|
+
ds = dataset_chooser
|
95
|
+
end
|
96
|
+
|
97
|
+
elsif args.size != 2
|
98
|
+
warn "Provide DATASET_ID and FILESYSTEM arguments"
|
99
|
+
exit(false)
|
100
|
+
|
101
|
+
else
|
102
|
+
ds = @api.dataset.show(args[0].to_i)
|
103
|
+
fs = args[1]
|
104
|
+
end
|
105
|
+
|
106
|
+
check_dataset_id!(ds, fs)
|
107
|
+
snapshots = ds.snapshot.list
|
108
|
+
|
109
|
+
local_state = parse_tree(fs)
|
110
|
+
|
111
|
+
# - Find out current history ID
|
112
|
+
# - If there are snapshots with this ID that are not present locally,
|
113
|
+
# download them
|
114
|
+
# - If the dataset for this history ID does not exist, create it
|
115
|
+
# - If it exists, check what snapshots are there and make an incremental
|
116
|
+
# download
|
117
|
+
|
118
|
+
remote_state = {}
|
119
|
+
|
120
|
+
snapshots.each do |s|
|
121
|
+
remote_state[s.history_id] ||= []
|
122
|
+
remote_state[s.history_id] << s
|
123
|
+
end
|
124
|
+
|
125
|
+
if remote_state[ds.current_history_id].nil? \
|
126
|
+
|| remote_state[ds.current_history_id].empty?
|
127
|
+
exit_msg(
|
128
|
+
"Nothing to transfer: no snapshots with history id #{ds.current_history_id}",
|
129
|
+
error: @opts[:no_snapshots_error]
|
130
|
+
)
|
131
|
+
end
|
132
|
+
|
133
|
+
for_transfer = []
|
134
|
+
|
135
|
+
latest_local_snapshot = local_state[ds.current_history_id] \
|
136
|
+
&& local_state[ds.current_history_id].last
|
137
|
+
found_latest = false
|
138
|
+
|
139
|
+
# This is the first run within this history id, no local snapshots are
|
140
|
+
# present
|
141
|
+
if !latest_local_snapshot && @opts[:init_snapshots]
|
142
|
+
remote_state[ds.current_history_id] = \
|
143
|
+
remote_state[ds.current_history_id].last(@opts[:init_snapshots])
|
144
|
+
end
|
145
|
+
|
146
|
+
remote_state[ds.current_history_id].each do |snap|
|
147
|
+
found = false
|
148
|
+
|
149
|
+
local_state.values.each do |snapshots|
|
150
|
+
found = snapshots.detect { |s| s.name == snap.name }
|
151
|
+
break if found
|
152
|
+
end
|
153
|
+
|
154
|
+
if !found_latest && latest_local_snapshot \
|
155
|
+
&& latest_local_snapshot.name == snap.name
|
156
|
+
found_latest = true
|
157
|
+
|
158
|
+
elsif latest_local_snapshot
|
159
|
+
next unless found_latest
|
160
|
+
end
|
161
|
+
|
162
|
+
for_transfer << snap unless found
|
163
|
+
end
|
164
|
+
|
165
|
+
if for_transfer.empty?
|
166
|
+
if found_latest
|
167
|
+
exit_msg(
|
168
|
+
"Nothing to transfer: all snapshots with history id "+
|
169
|
+
"#{ds.current_history_id} are already present locally",
|
170
|
+
error: @opts[:no_snapshots_error]
|
171
|
+
)
|
172
|
+
|
173
|
+
else
|
174
|
+
exit_msg(<<END
|
175
|
+
Unable to transfer: the common snapshot has not been found
|
176
|
+
|
177
|
+
This can happen when the latest local snapshot was deleted from the server,
|
178
|
+
i.e. you have not backed up this dataset for quite some time.
|
179
|
+
|
180
|
+
You can either rename or destroy the whole current history id:
|
181
|
+
|
182
|
+
zfs rename #{fs}/#{ds.current_history_id} #{fs}/#{ds.current_history_id}.old
|
183
|
+
|
184
|
+
or
|
185
|
+
|
186
|
+
zfs list -r -t all #{fs}/#{ds.current_history_id}
|
187
|
+
zfs destroy -r #{fs}/#{ds.current_history_id}
|
188
|
+
|
189
|
+
which will destroy all snapshots with this history id.
|
190
|
+
|
191
|
+
You can also destroy the local backup completely or backup to another dataset
|
192
|
+
and start anew.
|
193
|
+
END
|
194
|
+
)
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
unless @opts[:quiet]
|
199
|
+
puts "Will download #{for_transfer.size} snapshots:"
|
200
|
+
for_transfer.each { |s| puts " @#{s.name}" }
|
201
|
+
puts
|
202
|
+
end
|
203
|
+
|
204
|
+
if @opts[:pretend]
|
205
|
+
pretend_state = local_state.clone
|
206
|
+
pretend_state[ds.current_history_id] ||= []
|
207
|
+
pretend_state[ds.current_history_id].concat(for_transfer.map do |s|
|
208
|
+
LocalSnapshot.new(s.name, ds.current_history_id, Time.iso8601(s.created_at).to_i)
|
209
|
+
end)
|
210
|
+
|
211
|
+
rotate(fs, pretend: pretend_state) if @opts[:rotate]
|
212
|
+
|
213
|
+
else
|
214
|
+
# Find the common snapshot between server and localhost, so that the transfer
|
215
|
+
# can be incremental.
|
216
|
+
shared_name = local_state[ds.current_history_id] \
|
217
|
+
&& !local_state[ds.current_history_id].empty? \
|
218
|
+
&& local_state[ds.current_history_id].last.name
|
219
|
+
shared = nil
|
220
|
+
|
221
|
+
if shared_name
|
222
|
+
shared = remote_state[ds.current_history_id].detect { |s| s.name == shared_name }
|
223
|
+
|
224
|
+
if shared && !for_transfer.detect { |s| s.id == shared.id }
|
225
|
+
for_transfer.insert(0, shared)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
write_dataset_id!(ds, fs) unless written_dataset_id?
|
230
|
+
transfer(local_state, for_transfer, ds.current_history_id, fs)
|
231
|
+
rotate(fs) if @opts[:rotate]
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
protected
|
236
|
+
def transfer(local_state, snapshots, hist_id, fs)
|
237
|
+
ds = "#{fs}/#{hist_id}"
|
238
|
+
no_local_snapshots = local_state[hist_id].nil? || local_state[hist_id].empty?
|
239
|
+
|
240
|
+
if local_state[hist_id].nil?
|
241
|
+
zfs(:create, nil, ds)
|
242
|
+
end
|
243
|
+
|
244
|
+
if no_local_snapshots
|
245
|
+
msg "Performing a full receive of @#{snapshots.first.name} to #{ds}"
|
246
|
+
|
247
|
+
if @opts[:safe]
|
248
|
+
safe_download(ds, snapshots.first)
|
249
|
+
|
250
|
+
else
|
251
|
+
run_piped(zfs_cmd(:recv, '-F', ds)) do
|
252
|
+
SnapshotSend.new({}, @api).do_exec({
|
253
|
+
snapshot: snapshots.first.id,
|
254
|
+
send_mail: false,
|
255
|
+
delete_after: @opts[:delete_after],
|
256
|
+
max_rate: @opts[:max_rate],
|
257
|
+
checksum: @opts[:checksum],
|
258
|
+
quiet: @opts[:quiet],
|
259
|
+
})
|
260
|
+
end || exit_msg('Receive failed')
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
if !no_local_snapshots || snapshots.size > 1
|
265
|
+
msg "Performing an incremental receive of "+
|
266
|
+
"@#{snapshots.first.name} - @#{snapshots.last.name} to #{ds}"
|
267
|
+
|
268
|
+
if @opts[:safe]
|
269
|
+
safe_download(ds, snapshots.last, snapshots.first)
|
270
|
+
|
271
|
+
else
|
272
|
+
run_piped(zfs_cmd(:recv, '-F', ds)) do
|
273
|
+
SnapshotSend.new({}, @api).do_exec({
|
274
|
+
snapshot: snapshots.last.id,
|
275
|
+
from_snapshot: snapshots.first.id,
|
276
|
+
send_mail: false,
|
277
|
+
delete_after: @opts[:delete_after],
|
278
|
+
max_rate: @opts[:max_rate],
|
279
|
+
checksum: @opts[:checksum],
|
280
|
+
quiet: @opts[:quiet],
|
281
|
+
})
|
282
|
+
end || exit_msg('Receive failed')
|
283
|
+
end
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def safe_download(ds, snapshot, from_snapshot = nil)
|
288
|
+
part, full = snapshot_tmp_file(snapshot, from_snapshot)
|
289
|
+
|
290
|
+
if !File.exists?(full)
|
291
|
+
attempts = 0
|
292
|
+
|
293
|
+
begin
|
294
|
+
SnapshotDownload.new({}, @api).do_exec({
|
295
|
+
snapshot: snapshot.id,
|
296
|
+
from_snapshot: from_snapshot && from_snapshot.id,
|
297
|
+
format: from_snapshot ? :incremental_stream : :stream,
|
298
|
+
file: part,
|
299
|
+
max_rate: @opts[:max_rate],
|
300
|
+
checksum: @opts[:checksum],
|
301
|
+
quiet: @opts[:quiet],
|
302
|
+
resume: true,
|
303
|
+
delete_after: @opts[:delete_after],
|
304
|
+
send_mail: false,
|
305
|
+
})
|
306
|
+
|
307
|
+
rescue Errno::ECONNREFUSED,
|
308
|
+
Errno::ETIMEDOUT,
|
309
|
+
Errno::EHOSTUNREACH,
|
310
|
+
Errno::ECONNRESET => e
|
311
|
+
warn "Connection error: #{e.message}"
|
312
|
+
|
313
|
+
attempts += 1
|
314
|
+
|
315
|
+
if attempts >= @opts[:attempts]
|
316
|
+
warn "Run out of attempts"
|
317
|
+
exit(false)
|
318
|
+
|
319
|
+
else
|
320
|
+
warn "Retry in 60 seconds"
|
321
|
+
sleep(60)
|
322
|
+
retry
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
File.rename(part, full)
|
327
|
+
end
|
328
|
+
|
329
|
+
run_piped(zfs_cmd(:recv, '-F', ds)) do
|
330
|
+
Process.exec("zcat #{full}")
|
331
|
+
end || exit_msg('Receive failed')
|
332
|
+
|
333
|
+
File.delete(full)
|
334
|
+
end
|
335
|
+
|
336
|
+
def rotate(fs, pretend: false)
|
337
|
+
msg "Rotating snapshots"
|
338
|
+
local_state = pretend ? pretend : parse_tree(fs)
|
339
|
+
|
340
|
+
# Order snapshots by date of creation
|
341
|
+
snapshots = local_state.values.flatten.sort do |a, b|
|
342
|
+
a.creation <=> b.creation
|
343
|
+
end
|
344
|
+
|
345
|
+
cnt = local_state.values.inject(0) { |sum, snapshots| sum + snapshots.count }
|
346
|
+
deleted = 0
|
347
|
+
oldest = Time.now.to_i - (@opts[:max_age] * 60 * 60 * 24)
|
348
|
+
|
349
|
+
snapshots.each do |s|
|
350
|
+
ds = "#{fs}/#{s.hist_id}"
|
351
|
+
|
352
|
+
if (cnt - deleted) <= @opts[:min_snapshots] \
|
353
|
+
|| (s.creation > oldest && (cnt - deleted) <= @opts[:max_snapshots])
|
354
|
+
break
|
355
|
+
end
|
356
|
+
|
357
|
+
deleted += 1
|
358
|
+
local_state[s.hist_id].delete(s)
|
359
|
+
|
360
|
+
msg "Destroying #{ds}@#{s.name}"
|
361
|
+
zfs(:destroy, nil, "#{ds}@#{s.name}", pretend: pretend)
|
362
|
+
end
|
363
|
+
|
364
|
+
local_state.each do |hist_id, snapshots|
|
365
|
+
next unless snapshots.empty?
|
366
|
+
|
367
|
+
ds = "#{fs}/#{hist_id}"
|
368
|
+
|
369
|
+
msg "Destroying #{ds}"
|
370
|
+
zfs(:destroy, nil, ds, pretend: pretend)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
def parse_tree(fs)
|
375
|
+
ret = {}
|
376
|
+
|
377
|
+
# This is intentionally done by two zfs commands, because -d2 would include
|
378
|
+
# nested subdatasets, which should not be there, but the user might create
|
379
|
+
# them and it could confuse the program.
|
380
|
+
zfs(:list, '-r -d1 -tfilesystem -H -oname', fs).split("\n")[1..-1].each do |name|
|
381
|
+
last_name = name.split('/').last
|
382
|
+
ret[last_name.to_i] = [] if dataset?(last_name)
|
383
|
+
end
|
384
|
+
|
385
|
+
zfs(
|
386
|
+
:get,
|
387
|
+
'-Hrp -d2 name,creation -tsnapshot -oname,property,value',
|
388
|
+
fs
|
389
|
+
).split("\n").each do |line|
|
390
|
+
name, property, value = line.split
|
391
|
+
ds, snap_name = name.split('@')
|
392
|
+
ds_name = ds.split('/').last
|
393
|
+
next unless dataset?(ds_name)
|
394
|
+
|
395
|
+
hist_id = ds_name.to_i
|
396
|
+
|
397
|
+
if snap = ret[hist_id].detect { |s| s.name == snap_name }
|
398
|
+
snap.send("#{property}=", value)
|
399
|
+
|
400
|
+
else
|
401
|
+
snap = LocalSnapshot.new(snap_name, hist_id)
|
402
|
+
ret[hist_id] << snap
|
403
|
+
end
|
404
|
+
end
|
405
|
+
|
406
|
+
ret
|
407
|
+
end
|
408
|
+
|
409
|
+
def dataset?(name)
|
410
|
+
/^\d+$/ =~ name
|
411
|
+
end
|
412
|
+
|
413
|
+
def read_dataset_id(fs)
|
414
|
+
ds_id = zfs(:get, '-H -ovalue cz.vpsfree.vpsadmin:dataset_id', fs).strip
|
415
|
+
return nil if ds_id == '-'
|
416
|
+
@dataset_id = ds_id.to_i
|
417
|
+
end
|
418
|
+
|
419
|
+
def check_dataset_id!(ds, fs)
|
420
|
+
if @dataset_id && @dataset_id != ds.id
|
421
|
+
warn "Dataset '#{fs}' is used to backup remote dataset with id '#{@dataset_id}', not '#{ds.id}'"
|
422
|
+
exit(false)
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
def written_dataset_id?
|
427
|
+
!@dataset_id.nil?
|
428
|
+
end
|
429
|
+
|
430
|
+
def write_dataset_id!(ds, fs)
|
431
|
+
zfs(:set, "cz.vpsfree.vpsadmin:dataset_id=#{ds.id}", fs)
|
432
|
+
end
|
433
|
+
|
434
|
+
# Run two processes like +block | cmd2+, where block's stdout is piped into
|
435
|
+
# cmd2's stdin.
|
436
|
+
def run_piped(cmd2, &block)
|
437
|
+
r, w = IO.pipe
|
438
|
+
pids = []
|
439
|
+
|
440
|
+
pids << Process.fork do
|
441
|
+
r.close
|
442
|
+
STDOUT.reopen(w)
|
443
|
+
block.call
|
444
|
+
end
|
445
|
+
|
446
|
+
pids << Process.fork do
|
447
|
+
w.close
|
448
|
+
STDIN.reopen(r)
|
449
|
+
Process.exec(cmd2)
|
450
|
+
end
|
451
|
+
|
452
|
+
r.close
|
453
|
+
w.close
|
454
|
+
|
455
|
+
ret = true
|
456
|
+
|
457
|
+
pids.each do |pid|
|
458
|
+
Process.wait(pid)
|
459
|
+
ret = false if $?.exitstatus != 0
|
460
|
+
end
|
461
|
+
|
462
|
+
ret
|
463
|
+
end
|
464
|
+
|
465
|
+
def zfs_cmd(cmd, opts, fs)
|
466
|
+
s = ''
|
467
|
+
s += 'sudo ' if Process.euid != 0
|
468
|
+
s += 'zfs'
|
469
|
+
"#{s} #{cmd} #{opts} #{fs}"
|
470
|
+
end
|
471
|
+
|
472
|
+
def zfs(cmd, opts, fs, pretend: false)
|
473
|
+
cmd = zfs_cmd(cmd, opts, fs)
|
474
|
+
|
475
|
+
if pretend
|
476
|
+
puts "> #{cmd}"
|
477
|
+
return
|
478
|
+
end
|
479
|
+
|
480
|
+
ret = `#{cmd}`
|
481
|
+
exit_msg("#{cmd} failed with exit code #{$?.exitstatus}") if $?.exitstatus != 0
|
482
|
+
ret
|
483
|
+
end
|
484
|
+
|
485
|
+
def dataset_chooser(vps_only: false)
|
486
|
+
user = @api.user.current
|
487
|
+
vpses = @api.vps.list(user: user.id)
|
488
|
+
|
489
|
+
vps_map = {}
|
490
|
+
vpses.each do |vps|
|
491
|
+
vps_map[vps.dataset_id] = vps
|
492
|
+
end
|
493
|
+
|
494
|
+
i = 1
|
495
|
+
ds_map = {}
|
496
|
+
|
497
|
+
@api.dataset.index(user: user.id).each do |ds|
|
498
|
+
if vps = vps_map[ds.id]
|
499
|
+
puts "(#{i}) VPS ##{vps.id}"
|
500
|
+
|
501
|
+
else
|
502
|
+
next if vps_only
|
503
|
+
puts "(#{i}) Dataset #{ds.name}"
|
504
|
+
end
|
505
|
+
|
506
|
+
ds_map[i] = ds
|
507
|
+
i += 1
|
508
|
+
end
|
509
|
+
|
510
|
+
loop do
|
511
|
+
STDOUT.write('Pick a dataset to backup: ')
|
512
|
+
STDOUT.flush
|
513
|
+
|
514
|
+
i = STDIN.readline.strip.to_i
|
515
|
+
next if i <= 0 || ds_map[i].nil?
|
516
|
+
|
517
|
+
return ds_map[i]
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
521
|
+
def snapshot_tmp_file(s, from_s = nil)
|
522
|
+
if from_s
|
523
|
+
base = ".snapshot_#{from_s.id}-#{s.id}.inc.dat.gz"
|
524
|
+
|
525
|
+
else
|
526
|
+
base = ".snapshot_#{s.id}.dat.gz"
|
527
|
+
end
|
528
|
+
|
529
|
+
["#{base}.part", base]
|
530
|
+
end
|
531
|
+
|
532
|
+
def exit_msg(str, error: true)
|
533
|
+
if error
|
534
|
+
warn str
|
535
|
+
exit(1)
|
536
|
+
|
537
|
+
else
|
538
|
+
msg str
|
539
|
+
exit(0)
|
540
|
+
end
|
541
|
+
end
|
542
|
+
end
|
543
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module VpsAdmin::CLI::Commands
|
2
|
+
class BackupVps < BackupDataset
|
3
|
+
cmd :backup, :vps
|
4
|
+
args '[VPS_ID] FILESYSTEM'
|
5
|
+
desc 'Backup VPS locally'
|
6
|
+
|
7
|
+
def exec(args)
|
8
|
+
if args.size == 1 && /^\d+$/ !~ args[0]
|
9
|
+
fs = args[0]
|
10
|
+
|
11
|
+
ds_id = read_dataset_id(fs)
|
12
|
+
|
13
|
+
if ds_id
|
14
|
+
super([ds_id, fs])
|
15
|
+
|
16
|
+
else
|
17
|
+
ds = dataset_chooser(vps_only: true)
|
18
|
+
super([ds.id, fs])
|
19
|
+
end
|
20
|
+
|
21
|
+
elsif args.size == 2
|
22
|
+
super([@api.vps.show(args[0].to_i).dataset_id, args[1]])
|
23
|
+
|
24
|
+
else
|
25
|
+
super(args)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module VpsAdmin::CLI::Commands
|
2
|
+
class BaseDownload < HaveAPI::CLI::Command
|
3
|
+
protected
|
4
|
+
def find_or_create_dl(opts, do_create = true)
|
5
|
+
@api.snapshot_download.index(snapshot: opts[:snapshot]).each do |r|
|
6
|
+
if opts[:from_snapshot] == (r.from_snapshot && r.from_snapshot_id)
|
7
|
+
if r.format != opts[:format].to_s
|
8
|
+
fail "SnapshotDownload id=#{r.id} is in unusable format '#{r.format}' (needs '#{opts[:format]}')"
|
9
|
+
end
|
10
|
+
|
11
|
+
return [r, false]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
if do_create
|
16
|
+
[@api.snapshot_download.create(opts), true]
|
17
|
+
|
18
|
+
else
|
19
|
+
[nil, true]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def msg(str)
|
24
|
+
puts str unless @opts[:quiet]
|
25
|
+
end
|
26
|
+
|
27
|
+
def warn_msg(str)
|
28
|
+
warn str unless @opts[:quiet]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,214 @@
|
|
1
|
+
module VpsAdmin::CLI::Commands
|
2
|
+
class SnapshotDownload < BaseDownload
|
3
|
+
cmd :snapshot, :download
|
4
|
+
args '[SNAPSHOT_ID]'
|
5
|
+
desc 'Download a snapshot as an archive or a stream'
|
6
|
+
|
7
|
+
def options(opts)
|
8
|
+
@opts = {
|
9
|
+
delete_after: true,
|
10
|
+
send_mail: false,
|
11
|
+
checksum: true,
|
12
|
+
format: 'archive',
|
13
|
+
}
|
14
|
+
|
15
|
+
opts.on('-f', '--format FORMAT', 'archive, stream or incremental_stream') do |f|
|
16
|
+
@opts[:format] = f
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
|
20
|
+
@opts[:from_snapshot] = s
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
|
24
|
+
@opts[:delete_after] = d
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on('-F', '--force', 'Overwrite existing files if necessary') do |f|
|
28
|
+
@opts[:force] = f
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on('-o', '--output FILE', 'Save the download to FILE') do |f|
|
32
|
+
@opts[:file] = f
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on('-q', '--quiet', 'Print only errors') do |q|
|
36
|
+
@opts[:quiet] = q
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on('-r', '--resume', 'Resume cancelled download') do |r|
|
40
|
+
@opts[:resume] = r
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
|
44
|
+
@opts[:send_mail] = s
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
|
48
|
+
exit_msg('--max-rate must be greater than zero') if r <= 0
|
49
|
+
@opts[:max_rate] = r
|
50
|
+
end
|
51
|
+
|
52
|
+
opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
|
53
|
+
@opts[:checksum] = c
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def exec(args)
|
58
|
+
if args.size == 0 && STDIN.tty?
|
59
|
+
@opts[:snapshot] = snapshot_chooser
|
60
|
+
|
61
|
+
elsif args.size != 1
|
62
|
+
warn "Provide exactly one SNAPSHOT_ID as an argument"
|
63
|
+
exit(false)
|
64
|
+
|
65
|
+
else
|
66
|
+
@opts[:snapshot] = args.first.to_i
|
67
|
+
end
|
68
|
+
|
69
|
+
do_exec(@opts)
|
70
|
+
end
|
71
|
+
|
72
|
+
def do_exec(opts)
|
73
|
+
@opts = opts
|
74
|
+
f = action = nil
|
75
|
+
pos = 0
|
76
|
+
|
77
|
+
if @opts[:file] == '-'
|
78
|
+
f = STDOUT
|
79
|
+
|
80
|
+
elsif @opts[:file]
|
81
|
+
f, action, pos = open_file(@opts[:file])
|
82
|
+
end
|
83
|
+
|
84
|
+
dl, created = find_or_create_dl(@opts, action != :resume)
|
85
|
+
f, action, pos = open_file(dl.file_name) unless @opts[:file]
|
86
|
+
|
87
|
+
if created
|
88
|
+
if action == :resume
|
89
|
+
warn "Unable to resume the download: the file has been deleted from the server"
|
90
|
+
exit(false)
|
91
|
+
end
|
92
|
+
|
93
|
+
msg "The download is being prepared..."
|
94
|
+
sleep(5)
|
95
|
+
|
96
|
+
else
|
97
|
+
warn "Reusing existing SnapshotDownload (id=#{dl.id})"
|
98
|
+
end
|
99
|
+
|
100
|
+
msg "Downloading to #{f.path}"
|
101
|
+
|
102
|
+
begin
|
103
|
+
VpsAdmin::CLI::StreamDownloader.download(
|
104
|
+
@api,
|
105
|
+
dl,
|
106
|
+
f,
|
107
|
+
progress: !@opts[:quiet] && (f == STDOUT ? STDERR : STDOUT),
|
108
|
+
position: pos,
|
109
|
+
max_rate: @opts[:max_rate],
|
110
|
+
checksum: @opts[:checksum],
|
111
|
+
)
|
112
|
+
|
113
|
+
rescue VpsAdmin::CLI::DownloadError => e
|
114
|
+
warn e.message
|
115
|
+
exit(false)
|
116
|
+
|
117
|
+
ensure
|
118
|
+
f.close
|
119
|
+
end
|
120
|
+
|
121
|
+
@api.snapshot_download.delete(dl.id) if @opts[:delete_after]
|
122
|
+
end
|
123
|
+
|
124
|
+
protected
|
125
|
+
def open_file(path)
|
126
|
+
f = action = nil
|
127
|
+
pos = 0
|
128
|
+
|
129
|
+
if File.exists?(path) && File.size(path) > 0
|
130
|
+
if @opts[:resume]
|
131
|
+
action = :resume
|
132
|
+
|
133
|
+
elsif @opts[:force]
|
134
|
+
action = :overwrite
|
135
|
+
|
136
|
+
elsif STDIN.tty?
|
137
|
+
while action.nil?
|
138
|
+
STDERR.write("'#{path}' already exists. [A]bort, [r]esume or [o]verwrite? [a]: ")
|
139
|
+
STDERR.flush
|
140
|
+
|
141
|
+
action = {
|
142
|
+
'r' => :resume,
|
143
|
+
'o' => :overwrite,
|
144
|
+
'' => false,
|
145
|
+
}[STDIN.readline.strip.downcase]
|
146
|
+
end
|
147
|
+
|
148
|
+
else
|
149
|
+
warn "File '#{path}' already exists"
|
150
|
+
exit(false)
|
151
|
+
end
|
152
|
+
|
153
|
+
case action
|
154
|
+
when :resume
|
155
|
+
mode = 'a+'
|
156
|
+
pos = File.size(path)
|
157
|
+
|
158
|
+
when :overwrite
|
159
|
+
mode = 'w'
|
160
|
+
|
161
|
+
else
|
162
|
+
exit
|
163
|
+
end
|
164
|
+
|
165
|
+
f = File.open(path, mode)
|
166
|
+
else
|
167
|
+
f = File.open(path, 'w')
|
168
|
+
end
|
169
|
+
|
170
|
+
[f, action, pos]
|
171
|
+
end
|
172
|
+
|
173
|
+
def snapshot_chooser
|
174
|
+
user = @api.user.current
|
175
|
+
vpses = @api.vps.list(user: user.id)
|
176
|
+
|
177
|
+
ds_map = {}
|
178
|
+
vpses.each do |vps|
|
179
|
+
ds_map[vps.dataset_id] = vps
|
180
|
+
end
|
181
|
+
|
182
|
+
i = 1
|
183
|
+
snap_map = {}
|
184
|
+
|
185
|
+
@api.dataset.index(user: user.id).each do |ds|
|
186
|
+
snapshots = ds.snapshot.index
|
187
|
+
next if snapshots.empty?
|
188
|
+
|
189
|
+
if vps = ds_map[ds.id]
|
190
|
+
puts "VPS ##{vps.id}"
|
191
|
+
|
192
|
+
else
|
193
|
+
puts "Dataset #{ds.name}"
|
194
|
+
end
|
195
|
+
|
196
|
+
snapshots.each do |s|
|
197
|
+
snap_map[i] = s
|
198
|
+
puts " (#{i}) @#{s.created_at}"
|
199
|
+
i += 1
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
loop do
|
204
|
+
STDOUT.write('Pick a snapshot for download: ')
|
205
|
+
STDOUT.flush
|
206
|
+
|
207
|
+
i = STDIN.readline.strip.to_i
|
208
|
+
next if i <= 0 || snap_map[i].nil?
|
209
|
+
|
210
|
+
return snap_map[i].id
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
module VpsAdmin::CLI::Commands
|
4
|
+
class SnapshotSend < BaseDownload
|
5
|
+
cmd :snapshot, :send
|
6
|
+
args 'SNAPSHOT_ID'
|
7
|
+
desc 'Download a snapshot stream and write it on stdout'
|
8
|
+
|
9
|
+
def options(opts)
|
10
|
+
@opts = {
|
11
|
+
delete_after: true,
|
12
|
+
send_mail: false,
|
13
|
+
checksum: true,
|
14
|
+
}
|
15
|
+
|
16
|
+
opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
|
17
|
+
@opts[:from_snapshot] = s
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
|
21
|
+
@opts[:delete_after] = d
|
22
|
+
end
|
23
|
+
|
24
|
+
opts.on('-q', '--quiet', 'Print only errors') do |q|
|
25
|
+
@opts[:quiet] = q
|
26
|
+
end
|
27
|
+
|
28
|
+
opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
|
29
|
+
@opts[:send_mail] = s
|
30
|
+
end
|
31
|
+
|
32
|
+
opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
|
33
|
+
exit_msg('--max-rate must be greater than zero') if r <= 0
|
34
|
+
@opts[:max_rate] = r
|
35
|
+
end
|
36
|
+
|
37
|
+
opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
|
38
|
+
@opts[:checksum] = c
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def exec(args)
|
43
|
+
if args.size != 1
|
44
|
+
warn "Provide exactly one SNAPSHOT_ID as an argument"
|
45
|
+
exit(false)
|
46
|
+
end
|
47
|
+
|
48
|
+
opts = @opts.clone
|
49
|
+
opts[:snapshot] = args.first.to_i
|
50
|
+
|
51
|
+
do_exec(opts)
|
52
|
+
end
|
53
|
+
|
54
|
+
def do_exec(opts)
|
55
|
+
@opts = opts
|
56
|
+
opts[:format] = opts[:from_snapshot] ? :incremental_stream : :stream
|
57
|
+
|
58
|
+
dl, created = find_or_create_dl(opts)
|
59
|
+
|
60
|
+
if created
|
61
|
+
warn_msg "The download is being prepared..."
|
62
|
+
sleep(5)
|
63
|
+
|
64
|
+
else
|
65
|
+
warn_msg "Reusing existing SnapshotDownload (id=#{dl.id})"
|
66
|
+
end
|
67
|
+
|
68
|
+
r, w = IO.pipe
|
69
|
+
|
70
|
+
pid = Process.fork do
|
71
|
+
r.close
|
72
|
+
|
73
|
+
begin
|
74
|
+
VpsAdmin::CLI::StreamDownloader.download(
|
75
|
+
@api,
|
76
|
+
dl,
|
77
|
+
w,
|
78
|
+
progress: !opts[:quiet] && STDERR,
|
79
|
+
max_rate: opts[:max_rate],
|
80
|
+
checksum: opts[:checksum],
|
81
|
+
)
|
82
|
+
|
83
|
+
rescue VpsAdmin::CLI::DownloadError => e
|
84
|
+
warn e.message
|
85
|
+
exit(false)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
w.close
|
90
|
+
|
91
|
+
gz = Zlib::GzipReader.new(r)
|
92
|
+
STDOUT.write(gz.readpartial(16*1024)) while !gz.eof?
|
93
|
+
gz.close
|
94
|
+
|
95
|
+
Process.wait(pid)
|
96
|
+
exit($?.exitstatus) if $?.exitstatus != 0
|
97
|
+
|
98
|
+
@api.snapshot_download.delete(dl.id) if opts[:delete_after]
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
@@ -18,6 +18,10 @@ module VpsAdmin::CLI::Commands
|
|
18
18
|
opts.on('--[no-]outage-window', 'Migrate VPSes inside outage windows') do |w|
|
19
19
|
@opts[:outage_window] = w
|
20
20
|
end
|
21
|
+
|
22
|
+
opts.on('--[no-]cleanup-data', 'Cleanup VPS dataset on the source node') do |c|
|
23
|
+
@opts[:cleanup_data] = c
|
24
|
+
end
|
21
25
|
|
22
26
|
opts.on('--[no-]stop-on-error', 'Cancel the plan if a migration fails') do |s|
|
23
27
|
@opts[:stop_on_error] = s
|
@@ -87,6 +91,7 @@ module VpsAdmin::CLI::Commands
|
|
87
91
|
dst_node: @opts[:dst_node],
|
88
92
|
}
|
89
93
|
params[:outage_window] = @opts[:outage_window] unless @opts[:outage_window].nil?
|
94
|
+
params[:cleanup_data] = @opts[:cleanup_data] unless @opts[:cleanup_data].nil?
|
90
95
|
|
91
96
|
plan.vps_migration.create(params)
|
92
97
|
end
|
@@ -0,0 +1,236 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'net/http'
|
3
|
+
require 'ruby-progressbar'
|
4
|
+
require 'digest'
|
5
|
+
|
6
|
+
module VpsAdmin::CLI
|
7
|
+
class DownloadError < StandardError ; end
|
8
|
+
|
9
|
+
class StreamDownloader
|
10
|
+
def self.download(*args)
|
11
|
+
new(*args)
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(api, dl, io, progress: STDOUT, position: 0, max_rate: nil,
|
15
|
+
checksum: true)
|
16
|
+
downloaded = position
|
17
|
+
uri = URI(dl.url)
|
18
|
+
digest = Digest::SHA256.new
|
19
|
+
dl_check = nil
|
20
|
+
|
21
|
+
if position > 0 && checksum
|
22
|
+
if progress
|
23
|
+
pb = ProgressBar.create(
|
24
|
+
title: 'Calculating checksum',
|
25
|
+
total: position,
|
26
|
+
format: '%E %t: [%B] %p%% %r MB/s',
|
27
|
+
rate_scale: ->(rate) { (rate / 1024.0 / 1024.0).round(2) },
|
28
|
+
throttle_rate: 0.2,
|
29
|
+
output: progress,
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
read = 0
|
34
|
+
step = 1*1024*1024
|
35
|
+
io.seek(0)
|
36
|
+
|
37
|
+
while read < position
|
38
|
+
data = io.read((read + step) > position ? position - read : step)
|
39
|
+
read += data.size
|
40
|
+
|
41
|
+
digest << data
|
42
|
+
pb.progress = read if pb
|
43
|
+
end
|
44
|
+
|
45
|
+
pb.finish if pb
|
46
|
+
end
|
47
|
+
|
48
|
+
if progress
|
49
|
+
self.format = '%t: [%B] %r kB/s'
|
50
|
+
|
51
|
+
@pb = ProgressBar.create(
|
52
|
+
title: 'Downloading',
|
53
|
+
total: nil,
|
54
|
+
format: @format,
|
55
|
+
rate_scale: ->(rate) { (rate / 1024.0).round(2) },
|
56
|
+
throttle_rate: 0.2,
|
57
|
+
starting_at: downloaded,
|
58
|
+
autofinish: false,
|
59
|
+
output: progress,
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
63
|
+
args = [uri.host] + Array.new(5, nil) + [{use_ssl: uri.scheme == 'https'}]
|
64
|
+
|
65
|
+
Net::HTTP.start(*args) do |http|
|
66
|
+
loop do
|
67
|
+
begin
|
68
|
+
dl_check = api.snapshot_download.show(dl.id)
|
69
|
+
|
70
|
+
if @pb && (dl_check.ready || (dl_check.size && dl_check.size > 0))
|
71
|
+
@pb.progress = downloaded
|
72
|
+
|
73
|
+
total = dl_check.size * 1024 * 1024
|
74
|
+
@pb.total = @pb.progress > total ? @pb.progress : total
|
75
|
+
@download_size = (dl_check.size / 1024.0).round(2)
|
76
|
+
|
77
|
+
if dl_check.ready
|
78
|
+
@download_ready = true
|
79
|
+
self.format = "%E %t #{@download_size} GB: [%B] %p%% %r kB/s"
|
80
|
+
|
81
|
+
else
|
82
|
+
self.format = "%E %t ~#{@download_size} GB: [%B] %p%% %r kB/s"
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
rescue HaveAPI::Client::ActionFailed => e
|
87
|
+
# The SnapshotDownload object no longer exists, the transaction
|
88
|
+
# responsible for its creation must have failed.
|
89
|
+
stop
|
90
|
+
raise DownloadError, 'The download has failed due to transaction failure'
|
91
|
+
end
|
92
|
+
|
93
|
+
headers = {}
|
94
|
+
headers['Range'] = "bytes=#{downloaded}-" if downloaded > 0
|
95
|
+
|
96
|
+
http.request_get(uri.path, headers) do |res|
|
97
|
+
case res.code
|
98
|
+
when '404' # Not Found
|
99
|
+
if downloaded > 0
|
100
|
+
# This means that the transaction used for preparing the download
|
101
|
+
# has failed, the file to download does not exist anymore, so fail.
|
102
|
+
raise DownloadError, 'The download has failed, most likely transaction failure'
|
103
|
+
|
104
|
+
else
|
105
|
+
# The file is not available yet, this is normal, the transaction
|
106
|
+
# may be queued and it can take some time before it is processed.
|
107
|
+
pause(10)
|
108
|
+
next
|
109
|
+
end
|
110
|
+
|
111
|
+
when '416' # Range Not Satisfiable
|
112
|
+
if downloaded > position
|
113
|
+
# We have already managed to download something (at this run, if the trasfer
|
114
|
+
# was resumed) and the server cannot provide more data yet. This can be
|
115
|
+
# because the server is busy. Wait and retry.
|
116
|
+
pause(20)
|
117
|
+
next
|
118
|
+
|
119
|
+
else
|
120
|
+
# The file is not ready yet - we ask for range that cannot be provided
|
121
|
+
# This happens when we're resuming a download and the file on the
|
122
|
+
# server was deleted meanwhile. The file might not be exactly the same
|
123
|
+
# as the one before, sha256sum would most likely fail.
|
124
|
+
raise DownloadError, 'Range not satisfiable'
|
125
|
+
end
|
126
|
+
|
127
|
+
when '200', '206' # OK and Partial Content
|
128
|
+
resume
|
129
|
+
|
130
|
+
else
|
131
|
+
raise DownloadError, "Unexpected HTTP status code '#{res.code}'"
|
132
|
+
end
|
133
|
+
|
134
|
+
t1 = Time.now
|
135
|
+
data_counter = 0
|
136
|
+
|
137
|
+
res.read_body do |fragment|
|
138
|
+
size = fragment.size
|
139
|
+
|
140
|
+
data_counter += size
|
141
|
+
downloaded += size
|
142
|
+
|
143
|
+
begin
|
144
|
+
if @pb && (@pb.total.nil? || @pb.progress < @pb.total)
|
145
|
+
@pb.progress += size
|
146
|
+
end
|
147
|
+
|
148
|
+
rescue ProgressBar::InvalidProgressError
|
149
|
+
# The total value is in MB, it is not precise, so the actual
|
150
|
+
# size may be a little bit bigger.
|
151
|
+
@pb.progress = @pb.total
|
152
|
+
end
|
153
|
+
|
154
|
+
digest.update(fragment) if checksum
|
155
|
+
|
156
|
+
if max_rate && max_rate > 0
|
157
|
+
t2 = Time.now
|
158
|
+
diff = t2 - t1
|
159
|
+
|
160
|
+
if diff > 0.005
|
161
|
+
# Current and expected rates in kB per interval +diff+
|
162
|
+
current_rate = data_counter / 1024
|
163
|
+
expected_rate = max_rate * diff
|
164
|
+
|
165
|
+
if current_rate > expected_rate
|
166
|
+
delay = diff / (expected_rate / (current_rate - expected_rate))
|
167
|
+
sleep(delay)
|
168
|
+
end
|
169
|
+
|
170
|
+
data_counter = 0
|
171
|
+
t1 = Time.now
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
io.write(fragment)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
# This was the last download, the transfer is complete.
|
180
|
+
break if dl_check.ready
|
181
|
+
|
182
|
+
# Give the server time to prepare additional data
|
183
|
+
pause(15)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
@pb.finish if @pb
|
188
|
+
|
189
|
+
# Verify the checksum
|
190
|
+
if checksum && digest.hexdigest != dl_check.sha256sum
|
191
|
+
raise DownloadError, 'The sha256sum does not match, retry the download'
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
protected
|
196
|
+
def pause(secs)
|
197
|
+
@paused = true
|
198
|
+
|
199
|
+
if @pb
|
200
|
+
secs.times do |i|
|
201
|
+
if @download_size
|
202
|
+
if @download_ready
|
203
|
+
@pb.format("%t #{@download_size} GB: [%B] waiting #{secs - i}")
|
204
|
+
|
205
|
+
else
|
206
|
+
@pb.format("%t ~#{@download_size} GB: [%B] waiting #{secs - i}")
|
207
|
+
end
|
208
|
+
|
209
|
+
else
|
210
|
+
@pb.format("%t: [%B] waiting #{secs - i}")
|
211
|
+
end
|
212
|
+
|
213
|
+
@pb.refresh(force: true)
|
214
|
+
sleep(1)
|
215
|
+
end
|
216
|
+
|
217
|
+
else
|
218
|
+
sleep(secs)
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def resume
|
223
|
+
@pb.format(@format) if @pb && @paused
|
224
|
+
@paused = false
|
225
|
+
end
|
226
|
+
|
227
|
+
def stop
|
228
|
+
@pb.stop if @pb
|
229
|
+
end
|
230
|
+
|
231
|
+
def format=(fmt)
|
232
|
+
@format = fmt
|
233
|
+
@pb.format(@format) if @pb
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
data/vpsadmin-client.gemspec
CHANGED
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_runtime_dependency 'eventmachine', '~> 1.0.3'
|
26
26
|
spec.add_runtime_dependency 'em-http-request', '~> 1.1.3'
|
27
27
|
spec.add_runtime_dependency 'json', '~> 1.8.3'
|
28
|
+
spec.add_runtime_dependency 'ruby-progressbar', '~> 1.7.5'
|
28
29
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: vpsadmin-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jakub Skokan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ~>
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 1.8.3
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: ruby-progressbar
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ~>
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 1.7.5
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ~>
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.7.5
|
97
111
|
description: Ruby API and CLI for vpsAdmin API
|
98
112
|
email:
|
99
113
|
- jakub.skokan@vpsfree.cz
|
@@ -111,8 +125,14 @@ files:
|
|
111
125
|
- bin/vpsadminctl
|
112
126
|
- lib/terminal-size.rb
|
113
127
|
- lib/vpsadmin/cli.rb
|
128
|
+
- lib/vpsadmin/cli/commands/backup_dataset.rb
|
129
|
+
- lib/vpsadmin/cli/commands/backup_vps.rb
|
130
|
+
- lib/vpsadmin/cli/commands/base_download.rb
|
131
|
+
- lib/vpsadmin/cli/commands/snapshot_download.rb
|
132
|
+
- lib/vpsadmin/cli/commands/snapshot_send.rb
|
114
133
|
- lib/vpsadmin/cli/commands/vps_migrate_many.rb
|
115
134
|
- lib/vpsadmin/cli/commands/vps_remote_console.rb
|
135
|
+
- lib/vpsadmin/cli/stream_downloader.rb
|
116
136
|
- lib/vpsadmin/client.rb
|
117
137
|
- lib/vpsadmin/client/version.rb
|
118
138
|
- vpsadmin-client.gemspec
|