vpsadmin-client 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 90de46caa07d6a1e3b79cc71f322d6b5deea4cc2
4
- data.tar.gz: 95cd27f3867fe42945542f0965be82ff47e5d6d0
3
+ metadata.gz: 0201c799bf6e1d1573fe678f3c55645b0f6e6524
4
+ data.tar.gz: c7ac0933e44dd445735a44497cf2696426095dfd
5
5
  SHA512:
6
- metadata.gz: daff91e634cf9b40cbd113423874a08feec9fd8b46dc0a64fa6c843ee4f0735b1ebab09607d3ffb3382a8c6053cdc169ee167503f7bcd5477721c75f53e68a42
7
- data.tar.gz: d67762ed6357b3c3463c05253a500b8f50fcb7b76933a24844f9cd31ffdfad05805306b0545937b303df71e0fedf4a23f0ca00a93bdc1f92e4605f171cd61714
6
+ metadata.gz: 5c94af86fdbec2dfd8132e7e5931c371652152d9159fc7d721785188b725764f057ce15a83f8a67f3630d1a3b29d2f596d7685dc1fa9a4ad596c341dc874123c
7
+ data.tar.gz: 39688e6d85d56d6d1fa2da38060cc719296427af25cb3ddc52dfff2ea097a3820f7fe7e22446f9f095ad8832f29f2fa3c801b1e7dd461bf9885023691b3ccfe9
data/CHANGELOG CHANGED
@@ -1,3 +1,12 @@
1
+ * Mon Mar 21 2016 - version 2.3.0
2
+ - New commands for downloading snapshots:
3
+ - snapshot download
4
+ - snapshot send
5
+ - backup dataset
6
+ - backup vps
7
+ - vps migrate_many has new option --[no-]cleanup-data
8
+
9
+ * Fri Feb 26 2016 - version 2.2.0
1
10
  - New command vps migrate_many
2
11
 
3
12
  * Sat Jan 23 2016 - version 2.1.0
data/lib/vpsadmin/cli.rb CHANGED
@@ -1,14 +1,24 @@
1
1
  require 'haveapi/cli'
2
+ require 'vpsadmin/client/version'
2
3
 
3
4
  module VpsAdmin
4
5
  module CLI
5
6
  module Commands ; end
6
7
 
7
8
  class Cli < HaveAPI::CLI::Cli
8
-
9
+ def show_version
10
+ puts "#{VpsAdmin::Client::VERSION} based on haveapi-client "+
11
+ HaveAPI::Client::VERSION
12
+ end
9
13
  end
10
14
  end
11
15
  end
12
16
 
17
+ require 'vpsadmin/cli/stream_downloader'
18
+ require 'vpsadmin/cli/commands/base_download'
13
19
  require 'vpsadmin/cli/commands/vps_remote_console'
14
20
  require 'vpsadmin/cli/commands/vps_migrate_many'
21
+ require 'vpsadmin/cli/commands/snapshot_download'
22
+ require 'vpsadmin/cli/commands/snapshot_send'
23
+ require 'vpsadmin/cli/commands/backup_dataset'
24
+ require 'vpsadmin/cli/commands/backup_vps'
@@ -0,0 +1,543 @@
1
+ require 'time'
2
+
3
+ module VpsAdmin::CLI::Commands
4
+ class BackupDataset < BaseDownload
5
+ cmd :backup, :dataset
6
+ args '[DATASET_ID] FILESYSTEM'
7
+ desc 'Backup dataset locally'
8
+
9
+ LocalSnapshot = Struct.new(:name, :hist_id, :creation) do
10
+ def creation=(c)
11
+ self[:creation] = c.to_i
12
+ end
13
+ end
14
+
15
+ def options(opts)
16
+ @opts = {
17
+ rotate: true,
18
+ min_snapshots: 30,
19
+ max_snapshots: 45,
20
+ max_age: 30,
21
+ attempts: 10,
22
+ checksum: true,
23
+ delete_after: true,
24
+ }
25
+
26
+ opts.on('-p', '--pretend', 'Print what would the program do') do
27
+ @opts[:pretend] = true
28
+ end
29
+
30
+ opts.on('-r', '--[no-]rotate', 'Delete old snapshots (enabled)') do |r|
31
+ @opts[:rotate] = r
32
+ end
33
+
34
+ opts.on('-m', '--min-snapshots N', Integer, 'Keep at least N snapshots (30)') do |m|
35
+ exit_msg('--min-snapshots must be greater than zero') if m <= 0
36
+ @opts[:min_snapshots] = m
37
+ end
38
+
39
+ opts.on('-M', '--max-snapshots N', Integer, 'Keep at most N snapshots (45)') do |m|
40
+ exit_msg('--max-snapshots must be greater than zero') if m <= 0
41
+ @opts[:max_snapshots] = m
42
+ end
43
+
44
+ opts.on('-a', '--max-age N', Integer, 'Delete snapshots older then N days (30)') do |m|
45
+ exit_msg('--max-age must be greater than zero') if m <= 0
46
+ @opts[:max_age] = m
47
+ end
48
+
49
+ opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
50
+ exit_msg('--max-rate must be greater than zero') if r <= 0
51
+ @opts[:max_rate] = r
52
+ end
53
+
54
+ opts.on('-q', '--quiet', 'Print only errors') do |q|
55
+ @opts[:quiet] = q
56
+ end
57
+
58
+ opts.on('-s', '--safe-download', 'Download to a temp file (needs 2x disk space)') do |s|
59
+ @opts[:safe] = s
60
+ end
61
+
62
+ opts.on('--retry-attemps N', Integer, 'Retry N times to recover from download error (10)') do |n|
63
+ exit_msg('--retry-attempts must be greater than zero') if n <= 0
64
+ @opts[:attempts] = n
65
+ end
66
+
67
+ opts.on('-i', '--init-snapshots N', Integer, 'Download max N snapshots initially') do |s|
68
+ exit_msg('--init-snapshots must be greater than zero') if s <= 0
69
+ @opts[:init_snapshots] = s
70
+ end
71
+
72
+ opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
73
+ @opts[:checksum] = c
74
+ end
75
+
76
+ opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download (enabled)') do |d|
77
+ @opts[:delete_after] = d
78
+ end
79
+
80
+ opts.on('--no-snapshots-as-error', 'Consider no snapshots to download as an error') do
81
+ @opts[:no_snapshots_error] = true
82
+ end
83
+ end
84
+
85
+ def exec(args)
86
+ if args.size == 1 && /^\d+$/ !~ args[0]
87
+ fs = args[0]
88
+
89
+ ds_id = read_dataset_id(fs)
90
+
91
+ if ds_id
92
+ ds = @api.dataset.show(ds_id)
93
+ else
94
+ ds = dataset_chooser
95
+ end
96
+
97
+ elsif args.size != 2
98
+ warn "Provide DATASET_ID and FILESYSTEM arguments"
99
+ exit(false)
100
+
101
+ else
102
+ ds = @api.dataset.show(args[0].to_i)
103
+ fs = args[1]
104
+ end
105
+
106
+ check_dataset_id!(ds, fs)
107
+ snapshots = ds.snapshot.list
108
+
109
+ local_state = parse_tree(fs)
110
+
111
+ # - Find out current history ID
112
+ # - If there are snapshots with this ID that are not present locally,
113
+ # download them
114
+ # - If the dataset for this history ID does not exist, create it
115
+ # - If it exists, check what snapshots are there and make an incremental
116
+ # download
117
+
118
+ remote_state = {}
119
+
120
+ snapshots.each do |s|
121
+ remote_state[s.history_id] ||= []
122
+ remote_state[s.history_id] << s
123
+ end
124
+
125
+ if remote_state[ds.current_history_id].nil? \
126
+ || remote_state[ds.current_history_id].empty?
127
+ exit_msg(
128
+ "Nothing to transfer: no snapshots with history id #{ds.current_history_id}",
129
+ error: @opts[:no_snapshots_error]
130
+ )
131
+ end
132
+
133
+ for_transfer = []
134
+
135
+ latest_local_snapshot = local_state[ds.current_history_id] \
136
+ && local_state[ds.current_history_id].last
137
+ found_latest = false
138
+
139
+ # This is the first run within this history id, no local snapshots are
140
+ # present
141
+ if !latest_local_snapshot && @opts[:init_snapshots]
142
+ remote_state[ds.current_history_id] = \
143
+ remote_state[ds.current_history_id].last(@opts[:init_snapshots])
144
+ end
145
+
146
+ remote_state[ds.current_history_id].each do |snap|
147
+ found = false
148
+
149
+ local_state.values.each do |snapshots|
150
+ found = snapshots.detect { |s| s.name == snap.name }
151
+ break if found
152
+ end
153
+
154
+ if !found_latest && latest_local_snapshot \
155
+ && latest_local_snapshot.name == snap.name
156
+ found_latest = true
157
+
158
+ elsif latest_local_snapshot
159
+ next unless found_latest
160
+ end
161
+
162
+ for_transfer << snap unless found
163
+ end
164
+
165
+ if for_transfer.empty?
166
+ if found_latest
167
+ exit_msg(
168
+ "Nothing to transfer: all snapshots with history id "+
169
+ "#{ds.current_history_id} are already present locally",
170
+ error: @opts[:no_snapshots_error]
171
+ )
172
+
173
+ else
174
+ exit_msg(<<END
175
+ Unable to transfer: the common snapshot has not been found
176
+
177
+ This can happen when the latest local snapshot was deleted from the server,
178
+ i.e. you have not backed up this dataset for quite some time.
179
+
180
+ You can either rename or destroy the whole current history id:
181
+
182
+ zfs rename #{fs}/#{ds.current_history_id} #{fs}/#{ds.current_history_id}.old
183
+
184
+ or
185
+
186
+ zfs list -r -t all #{fs}/#{ds.current_history_id}
187
+ zfs destroy -r #{fs}/#{ds.current_history_id}
188
+
189
+ which will destroy all snapshots with this history id.
190
+
191
+ You can also destroy the local backup completely or backup to another dataset
192
+ and start anew.
193
+ END
194
+ )
195
+ end
196
+ end
197
+
198
+ unless @opts[:quiet]
199
+ puts "Will download #{for_transfer.size} snapshots:"
200
+ for_transfer.each { |s| puts " @#{s.name}" }
201
+ puts
202
+ end
203
+
204
+ if @opts[:pretend]
205
+ pretend_state = local_state.clone
206
+ pretend_state[ds.current_history_id] ||= []
207
+ pretend_state[ds.current_history_id].concat(for_transfer.map do |s|
208
+ LocalSnapshot.new(s.name, ds.current_history_id, Time.iso8601(s.created_at).to_i)
209
+ end)
210
+
211
+ rotate(fs, pretend: pretend_state) if @opts[:rotate]
212
+
213
+ else
214
+ # Find the common snapshot between server and localhost, so that the transfer
215
+ # can be incremental.
216
+ shared_name = local_state[ds.current_history_id] \
217
+ && !local_state[ds.current_history_id].empty? \
218
+ && local_state[ds.current_history_id].last.name
219
+ shared = nil
220
+
221
+ if shared_name
222
+ shared = remote_state[ds.current_history_id].detect { |s| s.name == shared_name }
223
+
224
+ if shared && !for_transfer.detect { |s| s.id == shared.id }
225
+ for_transfer.insert(0, shared)
226
+ end
227
+ end
228
+
229
+ write_dataset_id!(ds, fs) unless written_dataset_id?
230
+ transfer(local_state, for_transfer, ds.current_history_id, fs)
231
+ rotate(fs) if @opts[:rotate]
232
+ end
233
+ end
234
+
235
+ protected
236
+ def transfer(local_state, snapshots, hist_id, fs)
237
+ ds = "#{fs}/#{hist_id}"
238
+ no_local_snapshots = local_state[hist_id].nil? || local_state[hist_id].empty?
239
+
240
+ if local_state[hist_id].nil?
241
+ zfs(:create, nil, ds)
242
+ end
243
+
244
+ if no_local_snapshots
245
+ msg "Performing a full receive of @#{snapshots.first.name} to #{ds}"
246
+
247
+ if @opts[:safe]
248
+ safe_download(ds, snapshots.first)
249
+
250
+ else
251
+ run_piped(zfs_cmd(:recv, '-F', ds)) do
252
+ SnapshotSend.new({}, @api).do_exec({
253
+ snapshot: snapshots.first.id,
254
+ send_mail: false,
255
+ delete_after: @opts[:delete_after],
256
+ max_rate: @opts[:max_rate],
257
+ checksum: @opts[:checksum],
258
+ quiet: @opts[:quiet],
259
+ })
260
+ end || exit_msg('Receive failed')
261
+ end
262
+ end
263
+
264
+ if !no_local_snapshots || snapshots.size > 1
265
+ msg "Performing an incremental receive of "+
266
+ "@#{snapshots.first.name} - @#{snapshots.last.name} to #{ds}"
267
+
268
+ if @opts[:safe]
269
+ safe_download(ds, snapshots.last, snapshots.first)
270
+
271
+ else
272
+ run_piped(zfs_cmd(:recv, '-F', ds)) do
273
+ SnapshotSend.new({}, @api).do_exec({
274
+ snapshot: snapshots.last.id,
275
+ from_snapshot: snapshots.first.id,
276
+ send_mail: false,
277
+ delete_after: @opts[:delete_after],
278
+ max_rate: @opts[:max_rate],
279
+ checksum: @opts[:checksum],
280
+ quiet: @opts[:quiet],
281
+ })
282
+ end || exit_msg('Receive failed')
283
+ end
284
+ end
285
+ end
286
+
287
+ def safe_download(ds, snapshot, from_snapshot = nil)
288
+ part, full = snapshot_tmp_file(snapshot, from_snapshot)
289
+
290
+ if !File.exists?(full)
291
+ attempts = 0
292
+
293
+ begin
294
+ SnapshotDownload.new({}, @api).do_exec({
295
+ snapshot: snapshot.id,
296
+ from_snapshot: from_snapshot && from_snapshot.id,
297
+ format: from_snapshot ? :incremental_stream : :stream,
298
+ file: part,
299
+ max_rate: @opts[:max_rate],
300
+ checksum: @opts[:checksum],
301
+ quiet: @opts[:quiet],
302
+ resume: true,
303
+ delete_after: @opts[:delete_after],
304
+ send_mail: false,
305
+ })
306
+
307
+ rescue Errno::ECONNREFUSED,
308
+ Errno::ETIMEDOUT,
309
+ Errno::EHOSTUNREACH,
310
+ Errno::ECONNRESET => e
311
+ warn "Connection error: #{e.message}"
312
+
313
+ attempts += 1
314
+
315
+ if attempts >= @opts[:attempts]
316
+ warn "Run out of attempts"
317
+ exit(false)
318
+
319
+ else
320
+ warn "Retry in 60 seconds"
321
+ sleep(60)
322
+ retry
323
+ end
324
+ end
325
+
326
+ File.rename(part, full)
327
+ end
328
+
329
+ run_piped(zfs_cmd(:recv, '-F', ds)) do
330
+ Process.exec("zcat #{full}")
331
+ end || exit_msg('Receive failed')
332
+
333
+ File.delete(full)
334
+ end
335
+
336
+ def rotate(fs, pretend: false)
337
+ msg "Rotating snapshots"
338
+ local_state = pretend ? pretend : parse_tree(fs)
339
+
340
+ # Order snapshots by date of creation
341
+ snapshots = local_state.values.flatten.sort do |a, b|
342
+ a.creation <=> b.creation
343
+ end
344
+
345
+ cnt = local_state.values.inject(0) { |sum, snapshots| sum + snapshots.count }
346
+ deleted = 0
347
+ oldest = Time.now.to_i - (@opts[:max_age] * 60 * 60 * 24)
348
+
349
+ snapshots.each do |s|
350
+ ds = "#{fs}/#{s.hist_id}"
351
+
352
+ if (cnt - deleted) <= @opts[:min_snapshots] \
353
+ || (s.creation > oldest && (cnt - deleted) <= @opts[:max_snapshots])
354
+ break
355
+ end
356
+
357
+ deleted += 1
358
+ local_state[s.hist_id].delete(s)
359
+
360
+ msg "Destroying #{ds}@#{s.name}"
361
+ zfs(:destroy, nil, "#{ds}@#{s.name}", pretend: pretend)
362
+ end
363
+
364
+ local_state.each do |hist_id, snapshots|
365
+ next unless snapshots.empty?
366
+
367
+ ds = "#{fs}/#{hist_id}"
368
+
369
+ msg "Destroying #{ds}"
370
+ zfs(:destroy, nil, ds, pretend: pretend)
371
+ end
372
+ end
373
+
374
+ def parse_tree(fs)
375
+ ret = {}
376
+
377
+ # This is intentionally done by two zfs commands, because -d2 would include
378
+ # nested subdatasets, which should not be there, but the user might create
379
+ # them and it could confuse the program.
380
+ zfs(:list, '-r -d1 -tfilesystem -H -oname', fs).split("\n")[1..-1].each do |name|
381
+ last_name = name.split('/').last
382
+ ret[last_name.to_i] = [] if dataset?(last_name)
383
+ end
384
+
385
+ zfs(
386
+ :get,
387
+ '-Hrp -d2 name,creation -tsnapshot -oname,property,value',
388
+ fs
389
+ ).split("\n").each do |line|
390
+ name, property, value = line.split
391
+ ds, snap_name = name.split('@')
392
+ ds_name = ds.split('/').last
393
+ next unless dataset?(ds_name)
394
+
395
+ hist_id = ds_name.to_i
396
+
397
+ if snap = ret[hist_id].detect { |s| s.name == snap_name }
398
+ snap.send("#{property}=", value)
399
+
400
+ else
401
+ snap = LocalSnapshot.new(snap_name, hist_id)
402
+ ret[hist_id] << snap
403
+ end
404
+ end
405
+
406
+ ret
407
+ end
408
+
409
+ def dataset?(name)
410
+ /^\d+$/ =~ name
411
+ end
412
+
413
+ def read_dataset_id(fs)
414
+ ds_id = zfs(:get, '-H -ovalue cz.vpsfree.vpsadmin:dataset_id', fs).strip
415
+ return nil if ds_id == '-'
416
+ @dataset_id = ds_id.to_i
417
+ end
418
+
419
+ def check_dataset_id!(ds, fs)
420
+ if @dataset_id && @dataset_id != ds.id
421
+ warn "Dataset '#{fs}' is used to backup remote dataset with id '#{@dataset_id}', not '#{ds.id}'"
422
+ exit(false)
423
+ end
424
+ end
425
+
426
+ def written_dataset_id?
427
+ !@dataset_id.nil?
428
+ end
429
+
430
+ def write_dataset_id!(ds, fs)
431
+ zfs(:set, "cz.vpsfree.vpsadmin:dataset_id=#{ds.id}", fs)
432
+ end
433
+
434
+ # Run two processes like +block | cmd2+, where block's stdout is piped into
435
+ # cmd2's stdin.
436
+ def run_piped(cmd2, &block)
437
+ r, w = IO.pipe
438
+ pids = []
439
+
440
+ pids << Process.fork do
441
+ r.close
442
+ STDOUT.reopen(w)
443
+ block.call
444
+ end
445
+
446
+ pids << Process.fork do
447
+ w.close
448
+ STDIN.reopen(r)
449
+ Process.exec(cmd2)
450
+ end
451
+
452
+ r.close
453
+ w.close
454
+
455
+ ret = true
456
+
457
+ pids.each do |pid|
458
+ Process.wait(pid)
459
+ ret = false if $?.exitstatus != 0
460
+ end
461
+
462
+ ret
463
+ end
464
+
465
+ def zfs_cmd(cmd, opts, fs)
466
+ s = ''
467
+ s += 'sudo ' if Process.euid != 0
468
+ s += 'zfs'
469
+ "#{s} #{cmd} #{opts} #{fs}"
470
+ end
471
+
472
+ def zfs(cmd, opts, fs, pretend: false)
473
+ cmd = zfs_cmd(cmd, opts, fs)
474
+
475
+ if pretend
476
+ puts "> #{cmd}"
477
+ return
478
+ end
479
+
480
+ ret = `#{cmd}`
481
+ exit_msg("#{cmd} failed with exit code #{$?.exitstatus}") if $?.exitstatus != 0
482
+ ret
483
+ end
484
+
485
+ def dataset_chooser(vps_only: false)
486
+ user = @api.user.current
487
+ vpses = @api.vps.list(user: user.id)
488
+
489
+ vps_map = {}
490
+ vpses.each do |vps|
491
+ vps_map[vps.dataset_id] = vps
492
+ end
493
+
494
+ i = 1
495
+ ds_map = {}
496
+
497
+ @api.dataset.index(user: user.id).each do |ds|
498
+ if vps = vps_map[ds.id]
499
+ puts "(#{i}) VPS ##{vps.id}"
500
+
501
+ else
502
+ next if vps_only
503
+ puts "(#{i}) Dataset #{ds.name}"
504
+ end
505
+
506
+ ds_map[i] = ds
507
+ i += 1
508
+ end
509
+
510
+ loop do
511
+ STDOUT.write('Pick a dataset to backup: ')
512
+ STDOUT.flush
513
+
514
+ i = STDIN.readline.strip.to_i
515
+ next if i <= 0 || ds_map[i].nil?
516
+
517
+ return ds_map[i]
518
+ end
519
+ end
520
+
521
+ def snapshot_tmp_file(s, from_s = nil)
522
+ if from_s
523
+ base = ".snapshot_#{from_s.id}-#{s.id}.inc.dat.gz"
524
+
525
+ else
526
+ base = ".snapshot_#{s.id}.dat.gz"
527
+ end
528
+
529
+ ["#{base}.part", base]
530
+ end
531
+
532
+ def exit_msg(str, error: true)
533
+ if error
534
+ warn str
535
+ exit(1)
536
+
537
+ else
538
+ msg str
539
+ exit(0)
540
+ end
541
+ end
542
+ end
543
+ end
@@ -0,0 +1,29 @@
1
+ module VpsAdmin::CLI::Commands
2
+ class BackupVps < BackupDataset
3
+ cmd :backup, :vps
4
+ args '[VPS_ID] FILESYSTEM'
5
+ desc 'Backup VPS locally'
6
+
7
+ def exec(args)
8
+ if args.size == 1 && /^\d+$/ !~ args[0]
9
+ fs = args[0]
10
+
11
+ ds_id = read_dataset_id(fs)
12
+
13
+ if ds_id
14
+ super([ds_id, fs])
15
+
16
+ else
17
+ ds = dataset_chooser(vps_only: true)
18
+ super([ds.id, fs])
19
+ end
20
+
21
+ elsif args.size == 2
22
+ super([@api.vps.show(args[0].to_i).dataset_id, args[1]])
23
+
24
+ else
25
+ super(args)
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,31 @@
1
+ module VpsAdmin::CLI::Commands
2
+ class BaseDownload < HaveAPI::CLI::Command
3
+ protected
4
+ def find_or_create_dl(opts, do_create = true)
5
+ @api.snapshot_download.index(snapshot: opts[:snapshot]).each do |r|
6
+ if opts[:from_snapshot] == (r.from_snapshot && r.from_snapshot_id)
7
+ if r.format != opts[:format].to_s
8
+ fail "SnapshotDownload id=#{r.id} is in unusable format '#{r.format}' (needs '#{opts[:format]}')"
9
+ end
10
+
11
+ return [r, false]
12
+ end
13
+ end
14
+
15
+ if do_create
16
+ [@api.snapshot_download.create(opts), true]
17
+
18
+ else
19
+ [nil, true]
20
+ end
21
+ end
22
+
23
+ def msg(str)
24
+ puts str unless @opts[:quiet]
25
+ end
26
+
27
+ def warn_msg(str)
28
+ warn str unless @opts[:quiet]
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,214 @@
1
+ module VpsAdmin::CLI::Commands
2
+ class SnapshotDownload < BaseDownload
3
+ cmd :snapshot, :download
4
+ args '[SNAPSHOT_ID]'
5
+ desc 'Download a snapshot as an archive or a stream'
6
+
7
+ def options(opts)
8
+ @opts = {
9
+ delete_after: true,
10
+ send_mail: false,
11
+ checksum: true,
12
+ format: 'archive',
13
+ }
14
+
15
+ opts.on('-f', '--format FORMAT', 'archive, stream or incremental_stream') do |f|
16
+ @opts[:format] = f
17
+ end
18
+
19
+ opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
20
+ @opts[:from_snapshot] = s
21
+ end
22
+
23
+ opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
24
+ @opts[:delete_after] = d
25
+ end
26
+
27
+ opts.on('-F', '--force', 'Overwrite existing files if necessary') do |f|
28
+ @opts[:force] = f
29
+ end
30
+
31
+ opts.on('-o', '--output FILE', 'Save the download to FILE') do |f|
32
+ @opts[:file] = f
33
+ end
34
+
35
+ opts.on('-q', '--quiet', 'Print only errors') do |q|
36
+ @opts[:quiet] = q
37
+ end
38
+
39
+ opts.on('-r', '--resume', 'Resume cancelled download') do |r|
40
+ @opts[:resume] = r
41
+ end
42
+
43
+ opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
44
+ @opts[:send_mail] = s
45
+ end
46
+
47
+ opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
48
+ exit_msg('--max-rate must be greater than zero') if r <= 0
49
+ @opts[:max_rate] = r
50
+ end
51
+
52
+ opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
53
+ @opts[:checksum] = c
54
+ end
55
+ end
56
+
57
+ def exec(args)
58
+ if args.size == 0 && STDIN.tty?
59
+ @opts[:snapshot] = snapshot_chooser
60
+
61
+ elsif args.size != 1
62
+ warn "Provide exactly one SNAPSHOT_ID as an argument"
63
+ exit(false)
64
+
65
+ else
66
+ @opts[:snapshot] = args.first.to_i
67
+ end
68
+
69
+ do_exec(@opts)
70
+ end
71
+
72
+ def do_exec(opts)
73
+ @opts = opts
74
+ f = action = nil
75
+ pos = 0
76
+
77
+ if @opts[:file] == '-'
78
+ f = STDOUT
79
+
80
+ elsif @opts[:file]
81
+ f, action, pos = open_file(@opts[:file])
82
+ end
83
+
84
+ dl, created = find_or_create_dl(@opts, action != :resume)
85
+ f, action, pos = open_file(dl.file_name) unless @opts[:file]
86
+
87
+ if created
88
+ if action == :resume
89
+ warn "Unable to resume the download: the file has been deleted from the server"
90
+ exit(false)
91
+ end
92
+
93
+ msg "The download is being prepared..."
94
+ sleep(5)
95
+
96
+ else
97
+ warn "Reusing existing SnapshotDownload (id=#{dl.id})"
98
+ end
99
+
100
+ msg "Downloading to #{f.path}"
101
+
102
+ begin
103
+ VpsAdmin::CLI::StreamDownloader.download(
104
+ @api,
105
+ dl,
106
+ f,
107
+ progress: !@opts[:quiet] && (f == STDOUT ? STDERR : STDOUT),
108
+ position: pos,
109
+ max_rate: @opts[:max_rate],
110
+ checksum: @opts[:checksum],
111
+ )
112
+
113
+ rescue VpsAdmin::CLI::DownloadError => e
114
+ warn e.message
115
+ exit(false)
116
+
117
+ ensure
118
+ f.close
119
+ end
120
+
121
+ @api.snapshot_download.delete(dl.id) if @opts[:delete_after]
122
+ end
123
+
124
+ protected
125
+ def open_file(path)
126
+ f = action = nil
127
+ pos = 0
128
+
129
+ if File.exists?(path) && File.size(path) > 0
130
+ if @opts[:resume]
131
+ action = :resume
132
+
133
+ elsif @opts[:force]
134
+ action = :overwrite
135
+
136
+ elsif STDIN.tty?
137
+ while action.nil?
138
+ STDERR.write("'#{path}' already exists. [A]bort, [r]esume or [o]verwrite? [a]: ")
139
+ STDERR.flush
140
+
141
+ action = {
142
+ 'r' => :resume,
143
+ 'o' => :overwrite,
144
+ '' => false,
145
+ }[STDIN.readline.strip.downcase]
146
+ end
147
+
148
+ else
149
+ warn "File '#{path}' already exists"
150
+ exit(false)
151
+ end
152
+
153
+ case action
154
+ when :resume
155
+ mode = 'a+'
156
+ pos = File.size(path)
157
+
158
+ when :overwrite
159
+ mode = 'w'
160
+
161
+ else
162
+ exit
163
+ end
164
+
165
+ f = File.open(path, mode)
166
+ else
167
+ f = File.open(path, 'w')
168
+ end
169
+
170
+ [f, action, pos]
171
+ end
172
+
173
+ def snapshot_chooser
174
+ user = @api.user.current
175
+ vpses = @api.vps.list(user: user.id)
176
+
177
+ ds_map = {}
178
+ vpses.each do |vps|
179
+ ds_map[vps.dataset_id] = vps
180
+ end
181
+
182
+ i = 1
183
+ snap_map = {}
184
+
185
+ @api.dataset.index(user: user.id).each do |ds|
186
+ snapshots = ds.snapshot.index
187
+ next if snapshots.empty?
188
+
189
+ if vps = ds_map[ds.id]
190
+ puts "VPS ##{vps.id}"
191
+
192
+ else
193
+ puts "Dataset #{ds.name}"
194
+ end
195
+
196
+ snapshots.each do |s|
197
+ snap_map[i] = s
198
+ puts " (#{i}) @#{s.created_at}"
199
+ i += 1
200
+ end
201
+ end
202
+
203
+ loop do
204
+ STDOUT.write('Pick a snapshot for download: ')
205
+ STDOUT.flush
206
+
207
+ i = STDIN.readline.strip.to_i
208
+ next if i <= 0 || snap_map[i].nil?
209
+
210
+ return snap_map[i].id
211
+ end
212
+ end
213
+ end
214
+ end
@@ -0,0 +1,101 @@
1
+ require 'zlib'
2
+
3
+ module VpsAdmin::CLI::Commands
4
+ class SnapshotSend < BaseDownload
5
+ cmd :snapshot, :send
6
+ args 'SNAPSHOT_ID'
7
+ desc 'Download a snapshot stream and write it on stdout'
8
+
9
+ def options(opts)
10
+ @opts = {
11
+ delete_after: true,
12
+ send_mail: false,
13
+ checksum: true,
14
+ }
15
+
16
+ opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
17
+ @opts[:from_snapshot] = s
18
+ end
19
+
20
+ opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
21
+ @opts[:delete_after] = d
22
+ end
23
+
24
+ opts.on('-q', '--quiet', 'Print only errors') do |q|
25
+ @opts[:quiet] = q
26
+ end
27
+
28
+ opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
29
+ @opts[:send_mail] = s
30
+ end
31
+
32
+ opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
33
+ exit_msg('--max-rate must be greater than zero') if r <= 0
34
+ @opts[:max_rate] = r
35
+ end
36
+
37
+ opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
38
+ @opts[:checksum] = c
39
+ end
40
+ end
41
+
42
+ def exec(args)
43
+ if args.size != 1
44
+ warn "Provide exactly one SNAPSHOT_ID as an argument"
45
+ exit(false)
46
+ end
47
+
48
+ opts = @opts.clone
49
+ opts[:snapshot] = args.first.to_i
50
+
51
+ do_exec(opts)
52
+ end
53
+
54
+ def do_exec(opts)
55
+ @opts = opts
56
+ opts[:format] = opts[:from_snapshot] ? :incremental_stream : :stream
57
+
58
+ dl, created = find_or_create_dl(opts)
59
+
60
+ if created
61
+ warn_msg "The download is being prepared..."
62
+ sleep(5)
63
+
64
+ else
65
+ warn_msg "Reusing existing SnapshotDownload (id=#{dl.id})"
66
+ end
67
+
68
+ r, w = IO.pipe
69
+
70
+ pid = Process.fork do
71
+ r.close
72
+
73
+ begin
74
+ VpsAdmin::CLI::StreamDownloader.download(
75
+ @api,
76
+ dl,
77
+ w,
78
+ progress: !opts[:quiet] && STDERR,
79
+ max_rate: opts[:max_rate],
80
+ checksum: opts[:checksum],
81
+ )
82
+
83
+ rescue VpsAdmin::CLI::DownloadError => e
84
+ warn e.message
85
+ exit(false)
86
+ end
87
+ end
88
+
89
+ w.close
90
+
91
+ gz = Zlib::GzipReader.new(r)
92
+ STDOUT.write(gz.readpartial(16*1024)) while !gz.eof?
93
+ gz.close
94
+
95
+ Process.wait(pid)
96
+ exit($?.exitstatus) if $?.exitstatus != 0
97
+
98
+ @api.snapshot_download.delete(dl.id) if opts[:delete_after]
99
+ end
100
+ end
101
+ end
@@ -18,6 +18,10 @@ module VpsAdmin::CLI::Commands
18
18
  opts.on('--[no-]outage-window', 'Migrate VPSes inside outage windows') do |w|
19
19
  @opts[:outage_window] = w
20
20
  end
21
+
22
+ opts.on('--[no-]cleanup-data', 'Cleanup VPS dataset on the source node') do |c|
23
+ @opts[:cleanup_data] = c
24
+ end
21
25
 
22
26
  opts.on('--[no-]stop-on-error', 'Cancel the plan if a migration fails') do |s|
23
27
  @opts[:stop_on_error] = s
@@ -87,6 +91,7 @@ module VpsAdmin::CLI::Commands
87
91
  dst_node: @opts[:dst_node],
88
92
  }
89
93
  params[:outage_window] = @opts[:outage_window] unless @opts[:outage_window].nil?
94
+ params[:cleanup_data] = @opts[:cleanup_data] unless @opts[:cleanup_data].nil?
90
95
 
91
96
  plan.vps_migration.create(params)
92
97
  end
@@ -0,0 +1,236 @@
1
+ require 'uri'
2
+ require 'net/http'
3
+ require 'ruby-progressbar'
4
+ require 'digest'
5
+
6
+ module VpsAdmin::CLI
7
+ class DownloadError < StandardError ; end
8
+
9
+ class StreamDownloader
10
+ def self.download(*args)
11
+ new(*args)
12
+ end
13
+
14
+ def initialize(api, dl, io, progress: STDOUT, position: 0, max_rate: nil,
15
+ checksum: true)
16
+ downloaded = position
17
+ uri = URI(dl.url)
18
+ digest = Digest::SHA256.new
19
+ dl_check = nil
20
+
21
+ if position > 0 && checksum
22
+ if progress
23
+ pb = ProgressBar.create(
24
+ title: 'Calculating checksum',
25
+ total: position,
26
+ format: '%E %t: [%B] %p%% %r MB/s',
27
+ rate_scale: ->(rate) { (rate / 1024.0 / 1024.0).round(2) },
28
+ throttle_rate: 0.2,
29
+ output: progress,
30
+ )
31
+ end
32
+
33
+ read = 0
34
+ step = 1*1024*1024
35
+ io.seek(0)
36
+
37
+ while read < position
38
+ data = io.read((read + step) > position ? position - read : step)
39
+ read += data.size
40
+
41
+ digest << data
42
+ pb.progress = read if pb
43
+ end
44
+
45
+ pb.finish if pb
46
+ end
47
+
48
+ if progress
49
+ self.format = '%t: [%B] %r kB/s'
50
+
51
+ @pb = ProgressBar.create(
52
+ title: 'Downloading',
53
+ total: nil,
54
+ format: @format,
55
+ rate_scale: ->(rate) { (rate / 1024.0).round(2) },
56
+ throttle_rate: 0.2,
57
+ starting_at: downloaded,
58
+ autofinish: false,
59
+ output: progress,
60
+ )
61
+ end
62
+
63
+ args = [uri.host] + Array.new(5, nil) + [{use_ssl: uri.scheme == 'https'}]
64
+
65
+ Net::HTTP.start(*args) do |http|
66
+ loop do
67
+ begin
68
+ dl_check = api.snapshot_download.show(dl.id)
69
+
70
+ if @pb && (dl_check.ready || (dl_check.size && dl_check.size > 0))
71
+ @pb.progress = downloaded
72
+
73
+ total = dl_check.size * 1024 * 1024
74
+ @pb.total = @pb.progress > total ? @pb.progress : total
75
+ @download_size = (dl_check.size / 1024.0).round(2)
76
+
77
+ if dl_check.ready
78
+ @download_ready = true
79
+ self.format = "%E %t #{@download_size} GB: [%B] %p%% %r kB/s"
80
+
81
+ else
82
+ self.format = "%E %t ~#{@download_size} GB: [%B] %p%% %r kB/s"
83
+ end
84
+ end
85
+
86
+ rescue HaveAPI::Client::ActionFailed => e
87
+ # The SnapshotDownload object no longer exists, the transaction
88
+ # responsible for its creation must have failed.
89
+ stop
90
+ raise DownloadError, 'The download has failed due to transaction failure'
91
+ end
92
+
93
+ headers = {}
94
+ headers['Range'] = "bytes=#{downloaded}-" if downloaded > 0
95
+
96
+ http.request_get(uri.path, headers) do |res|
97
+ case res.code
98
+ when '404' # Not Found
99
+ if downloaded > 0
100
+ # This means that the transaction used for preparing the download
101
+ # has failed, the file to download does not exist anymore, so fail.
102
+ raise DownloadError, 'The download has failed, most likely transaction failure'
103
+
104
+ else
105
+ # The file is not available yet, this is normal, the transaction
106
+ # may be queued and it can take some time before it is processed.
107
+ pause(10)
108
+ next
109
+ end
110
+
111
+ when '416' # Range Not Satisfiable
112
+ if downloaded > position
113
+ # We have already managed to download something (at this run, if the trasfer
114
+ # was resumed) and the server cannot provide more data yet. This can be
115
+ # because the server is busy. Wait and retry.
116
+ pause(20)
117
+ next
118
+
119
+ else
120
+ # The file is not ready yet - we ask for range that cannot be provided
121
+ # This happens when we're resuming a download and the file on the
122
+ # server was deleted meanwhile. The file might not be exactly the same
123
+ # as the one before, sha256sum would most likely fail.
124
+ raise DownloadError, 'Range not satisfiable'
125
+ end
126
+
127
+ when '200', '206' # OK and Partial Content
128
+ resume
129
+
130
+ else
131
+ raise DownloadError, "Unexpected HTTP status code '#{res.code}'"
132
+ end
133
+
134
+ t1 = Time.now
135
+ data_counter = 0
136
+
137
+ res.read_body do |fragment|
138
+ size = fragment.size
139
+
140
+ data_counter += size
141
+ downloaded += size
142
+
143
+ begin
144
+ if @pb && (@pb.total.nil? || @pb.progress < @pb.total)
145
+ @pb.progress += size
146
+ end
147
+
148
+ rescue ProgressBar::InvalidProgressError
149
+ # The total value is in MB, it is not precise, so the actual
150
+ # size may be a little bit bigger.
151
+ @pb.progress = @pb.total
152
+ end
153
+
154
+ digest.update(fragment) if checksum
155
+
156
+ if max_rate && max_rate > 0
157
+ t2 = Time.now
158
+ diff = t2 - t1
159
+
160
+ if diff > 0.005
161
+ # Current and expected rates in kB per interval +diff+
162
+ current_rate = data_counter / 1024
163
+ expected_rate = max_rate * diff
164
+
165
+ if current_rate > expected_rate
166
+ delay = diff / (expected_rate / (current_rate - expected_rate))
167
+ sleep(delay)
168
+ end
169
+
170
+ data_counter = 0
171
+ t1 = Time.now
172
+ end
173
+ end
174
+
175
+ io.write(fragment)
176
+ end
177
+ end
178
+
179
+ # This was the last download, the transfer is complete.
180
+ break if dl_check.ready
181
+
182
+ # Give the server time to prepare additional data
183
+ pause(15)
184
+ end
185
+ end
186
+
187
+ @pb.finish if @pb
188
+
189
+ # Verify the checksum
190
+ if checksum && digest.hexdigest != dl_check.sha256sum
191
+ raise DownloadError, 'The sha256sum does not match, retry the download'
192
+ end
193
+ end
194
+
195
+ protected
196
+ def pause(secs)
197
+ @paused = true
198
+
199
+ if @pb
200
+ secs.times do |i|
201
+ if @download_size
202
+ if @download_ready
203
+ @pb.format("%t #{@download_size} GB: [%B] waiting #{secs - i}")
204
+
205
+ else
206
+ @pb.format("%t ~#{@download_size} GB: [%B] waiting #{secs - i}")
207
+ end
208
+
209
+ else
210
+ @pb.format("%t: [%B] waiting #{secs - i}")
211
+ end
212
+
213
+ @pb.refresh(force: true)
214
+ sleep(1)
215
+ end
216
+
217
+ else
218
+ sleep(secs)
219
+ end
220
+ end
221
+
222
+ def resume
223
+ @pb.format(@format) if @pb && @paused
224
+ @paused = false
225
+ end
226
+
227
+ def stop
228
+ @pb.stop if @pb
229
+ end
230
+
231
+ def format=(fmt)
232
+ @format = fmt
233
+ @pb.format(@format) if @pb
234
+ end
235
+ end
236
+ end
@@ -1,5 +1,5 @@
1
1
  module VpsAdmin
2
2
  module Client
3
- VERSION = '2.2.0'
3
+ VERSION = '2.3.0'
4
4
  end
5
5
  end
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
25
25
  spec.add_runtime_dependency 'eventmachine', '~> 1.0.3'
26
26
  spec.add_runtime_dependency 'em-http-request', '~> 1.1.3'
27
27
  spec.add_runtime_dependency 'json', '~> 1.8.3'
28
+ spec.add_runtime_dependency 'ruby-progressbar', '~> 1.7.5'
28
29
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: vpsadmin-client
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0
4
+ version: 2.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jakub Skokan
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-09 00:00:00.000000000 Z
11
+ date: 2016-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -94,6 +94,20 @@ dependencies:
94
94
  - - ~>
95
95
  - !ruby/object:Gem::Version
96
96
  version: 1.8.3
97
+ - !ruby/object:Gem::Dependency
98
+ name: ruby-progressbar
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ~>
102
+ - !ruby/object:Gem::Version
103
+ version: 1.7.5
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ~>
109
+ - !ruby/object:Gem::Version
110
+ version: 1.7.5
97
111
  description: Ruby API and CLI for vpsAdmin API
98
112
  email:
99
113
  - jakub.skokan@vpsfree.cz
@@ -111,8 +125,14 @@ files:
111
125
  - bin/vpsadminctl
112
126
  - lib/terminal-size.rb
113
127
  - lib/vpsadmin/cli.rb
128
+ - lib/vpsadmin/cli/commands/backup_dataset.rb
129
+ - lib/vpsadmin/cli/commands/backup_vps.rb
130
+ - lib/vpsadmin/cli/commands/base_download.rb
131
+ - lib/vpsadmin/cli/commands/snapshot_download.rb
132
+ - lib/vpsadmin/cli/commands/snapshot_send.rb
114
133
  - lib/vpsadmin/cli/commands/vps_migrate_many.rb
115
134
  - lib/vpsadmin/cli/commands/vps_remote_console.rb
135
+ - lib/vpsadmin/cli/stream_downloader.rb
116
136
  - lib/vpsadmin/client.rb
117
137
  - lib/vpsadmin/client/version.rb
118
138
  - vpsadmin-client.gemspec