vpsadmin-client 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90de46caa07d6a1e3b79cc71f322d6b5deea4cc2
-  data.tar.gz: 95cd27f3867fe42945542f0965be82ff47e5d6d0
+  metadata.gz: 0201c799bf6e1d1573fe678f3c55645b0f6e6524
+  data.tar.gz: c7ac0933e44dd445735a44497cf2696426095dfd
 SHA512:
-  metadata.gz: daff91e634cf9b40cbd113423874a08feec9fd8b46dc0a64fa6c843ee4f0735b1ebab09607d3ffb3382a8c6053cdc169ee167503f7bcd5477721c75f53e68a42
-  data.tar.gz: d67762ed6357b3c3463c05253a500b8f50fcb7b76933a24844f9cd31ffdfad05805306b0545937b303df71e0fedf4a23f0ca00a93bdc1f92e4605f171cd61714
+  metadata.gz: 5c94af86fdbec2dfd8132e7e5931c371652152d9159fc7d721785188b725764f057ce15a83f8a67f3630d1a3b29d2f596d7685dc1fa9a4ad596c341dc874123c
+  data.tar.gz: 39688e6d85d56d6d1fa2da38060cc719296427af25cb3ddc52dfff2ea097a3820f7fe7e22446f9f095ad8832f29f2fa3c801b1e7dd461bf9885023691b3ccfe9

data/CHANGELOG

@@ -1,3 +1,12 @@
+* Mon Mar 21 2016 - version 2.3.0
+- New commands for downloading snapshots:
+  - snapshot download
+  - snapshot send
+  - backup dataset
+  - backup vps
+- vps migrate_many has new option --[no-]cleanup-data
+
+* Fri Feb 26 2016 - version 2.2.0
 - New command vps migrate_many
 
 * Sat Jan 23 2016 - version 2.1.0

data/lib/vpsadmin/cli.rb

@@ -1,14 +1,24 @@
 require 'haveapi/cli'
+require 'vpsadmin/client/version'
 
 module VpsAdmin
   module CLI
     module Commands ; end
 
     class Cli < HaveAPI::CLI::Cli
-
+      def show_version
+        puts "#{VpsAdmin::Client::VERSION} based on haveapi-client "+
+             HaveAPI::Client::VERSION
+      end
     end
   end
 end
 
+require 'vpsadmin/cli/stream_downloader'
+require 'vpsadmin/cli/commands/base_download'
 require 'vpsadmin/cli/commands/vps_remote_console'
 require 'vpsadmin/cli/commands/vps_migrate_many'
+require 'vpsadmin/cli/commands/snapshot_download'
+require 'vpsadmin/cli/commands/snapshot_send'
+require 'vpsadmin/cli/commands/backup_dataset'
+require 'vpsadmin/cli/commands/backup_vps'

data/lib/vpsadmin/cli/commands/backup_dataset.rb

@@ -0,0 +1,543 @@
+require 'time'
+
+module VpsAdmin::CLI::Commands
+  class BackupDataset < BaseDownload
+    cmd :backup, :dataset
+    args '[DATASET_ID] FILESYSTEM'
+    desc 'Backup dataset locally'
+
+    LocalSnapshot = Struct.new(:name, :hist_id, :creation) do
+      def creation=(c)
+        self[:creation] = c.to_i
+      end
+    end
+
+    def options(opts)
+      @opts = {
+          rotate: true,
+          min_snapshots: 30,
+          max_snapshots: 45,
+          max_age: 30,
+          attempts: 10,
+          checksum: true,
+          delete_after: true,
+      }
+
+      opts.on('-p', '--pretend', 'Print what would the program do') do
+        @opts[:pretend] = true
+      end
+
+      opts.on('-r', '--[no-]rotate', 'Delete old snapshots (enabled)') do |r|
+        @opts[:rotate] = r
+      end
+
+      opts.on('-m', '--min-snapshots N', Integer, 'Keep at least N snapshots (30)') do |m|
+        exit_msg('--min-snapshots must be greater than zero') if m <= 0
+        @opts[:min_snapshots] = m
+      end
+
+      opts.on('-M', '--max-snapshots N', Integer, 'Keep at most N snapshots (45)') do |m|
+        exit_msg('--max-snapshots must be greater than zero') if m <= 0
+        @opts[:max_snapshots] = m
+      end
+
+      opts.on('-a', '--max-age N', Integer, 'Delete snapshots older then N days (30)') do |m|
+        exit_msg('--max-age must be greater than zero') if m <= 0
+        @opts[:max_age] = m
+      end
+
+      opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
+        exit_msg('--max-rate must be greater than zero') if r <= 0
+        @opts[:max_rate] = r
+      end
+      
+      opts.on('-q', '--quiet', 'Print only errors') do |q|
+        @opts[:quiet] = q
+      end
+
+      opts.on('-s', '--safe-download', 'Download to a temp file (needs 2x disk space)') do |s|
+        @opts[:safe] = s
+      end
+
+      opts.on('--retry-attemps N', Integer, 'Retry N times to recover from download error (10)') do |n|
+        exit_msg('--retry-attempts must be greater than zero') if n <= 0
+        @opts[:attempts] = n
+      end
+
+      opts.on('-i', '--init-snapshots N', Integer, 'Download max N snapshots initially') do |s|
+        exit_msg('--init-snapshots must be greater than zero') if s <= 0
+        @opts[:init_snapshots] = s
+      end
+
+      opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
+        @opts[:checksum] = c
+      end
+
+      opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download (enabled)') do |d|
+        @opts[:delete_after] = d
+      end
+
+      opts.on('--no-snapshots-as-error', 'Consider no snapshots to download as an error') do
+        @opts[:no_snapshots_error] = true
+      end
+    end
+
+    def exec(args)
+      if args.size == 1 && /^\d+$/ !~ args[0]
+        fs = args[0]
+        
+        ds_id = read_dataset_id(fs)
+        
+        if ds_id
+          ds = @api.dataset.show(ds_id)
+        else
+          ds = dataset_chooser
+        end
+
+      elsif args.size != 2
+        warn "Provide DATASET_ID and FILESYSTEM arguments"
+        exit(false)
+
+      else
+        ds = @api.dataset.show(args[0].to_i)
+        fs = args[1]
+      end
+
+      check_dataset_id!(ds, fs)
+      snapshots = ds.snapshot.list
+
+      local_state = parse_tree(fs)
+
+      # - Find out current history ID
+      # - If there are snapshots with this ID that are not present locally,
+      #   download them
+      #   - If the dataset for this history ID does not exist, create it
+      #   - If it exists, check what snapshots are there and make an incremental
+      #     download
+
+      remote_state = {}
+
+      snapshots.each do |s|
+        remote_state[s.history_id] ||= []
+        remote_state[s.history_id] << s
+      end
+
+      if remote_state[ds.current_history_id].nil? \
+         || remote_state[ds.current_history_id].empty?
+        exit_msg(
+            "Nothing to transfer: no snapshots with history id #{ds.current_history_id}",
+            error: @opts[:no_snapshots_error]
+        )
+      end
+
+      for_transfer = []
+
+      latest_local_snapshot = local_state[ds.current_history_id] \
+                              && local_state[ds.current_history_id].last
+      found_latest = false
+
+      # This is the first run within this history id, no local snapshots are
+      # present
+      if !latest_local_snapshot && @opts[:init_snapshots]
+        remote_state[ds.current_history_id] = \
+          remote_state[ds.current_history_id].last(@opts[:init_snapshots])
+      end
+
+      remote_state[ds.current_history_id].each do |snap|
+        found = false
+
+        local_state.values.each do |snapshots|
+          found = snapshots.detect { |s| s.name == snap.name }
+          break if found
+        end
+
+        if !found_latest && latest_local_snapshot \
+           && latest_local_snapshot.name == snap.name
+          found_latest = true
+
+        elsif latest_local_snapshot
+          next unless found_latest
+        end
+
+        for_transfer << snap unless found
+      end
+
+      if for_transfer.empty?
+        if found_latest
+          exit_msg(
+              "Nothing to transfer: all snapshots with history id "+
+              "#{ds.current_history_id} are already present locally",
+              error: @opts[:no_snapshots_error]
+          )
+
+        else
+          exit_msg(<<END
+Unable to transfer: the common snapshot has not been found
+
+This can happen when the latest local snapshot was deleted from the server,
+i.e. you have not backed up this dataset for quite some time.
+
+You can either rename or destroy the whole current history id:
+
+  zfs rename #{fs}/#{ds.current_history_id} #{fs}/#{ds.current_history_id}.old
+
+or
+
+  zfs list -r -t all #{fs}/#{ds.current_history_id}
+  zfs destroy -r #{fs}/#{ds.current_history_id}
+
+which will destroy all snapshots with this history id.
+
+You can also destroy the local backup completely or backup to another dataset
+and start anew.
+END
+          )
+        end
+      end
+
+      unless @opts[:quiet]
+        puts "Will download #{for_transfer.size} snapshots:"
+        for_transfer.each { |s| puts "  @#{s.name}" }
+        puts
+      end
+    
+      if @opts[:pretend]
+        pretend_state = local_state.clone
+        pretend_state[ds.current_history_id] ||= []
+        pretend_state[ds.current_history_id].concat(for_transfer.map do |s|
+          LocalSnapshot.new(s.name, ds.current_history_id, Time.iso8601(s.created_at).to_i)
+        end)
+
+        rotate(fs, pretend: pretend_state) if @opts[:rotate]
+
+      else
+        # Find the common snapshot between server and localhost, so that the transfer
+        # can be incremental.
+        shared_name = local_state[ds.current_history_id] \
+                      && !local_state[ds.current_history_id].empty? \
+                      && local_state[ds.current_history_id].last.name
+        shared = nil
+
+        if shared_name
+          shared = remote_state[ds.current_history_id].detect { |s| s.name == shared_name }
+
+          if shared && !for_transfer.detect { |s| s.id == shared.id }
+            for_transfer.insert(0, shared)
+          end
+        end
+
+        write_dataset_id!(ds, fs) unless written_dataset_id?
+        transfer(local_state, for_transfer, ds.current_history_id, fs)
+        rotate(fs) if @opts[:rotate]
+      end
+    end
+
+    protected
+    def transfer(local_state, snapshots, hist_id, fs)
+      ds = "#{fs}/#{hist_id}"
+      no_local_snapshots = local_state[hist_id].nil? || local_state[hist_id].empty?
+
+      if local_state[hist_id].nil?
+        zfs(:create, nil, ds)
+      end
+      
+      if no_local_snapshots
+        msg "Performing a full receive of @#{snapshots.first.name} to #{ds}"
+
+        if @opts[:safe]
+          safe_download(ds, snapshots.first)
+
+        else
+          run_piped(zfs_cmd(:recv, '-F', ds)) do
+            SnapshotSend.new({}, @api).do_exec({
+                snapshot: snapshots.first.id,
+                send_mail: false,
+                delete_after: @opts[:delete_after],
+                max_rate: @opts[:max_rate],
+                checksum: @opts[:checksum],
+                quiet: @opts[:quiet],
+            })
+          end || exit_msg('Receive failed')
+        end
+      end
+
+      if !no_local_snapshots || snapshots.size > 1
+        msg "Performing an incremental receive of "+
+            "@#{snapshots.first.name} - @#{snapshots.last.name} to #{ds}"
+
+        if @opts[:safe]
+          safe_download(ds, snapshots.last, snapshots.first)
+
+        else
+          run_piped(zfs_cmd(:recv, '-F', ds)) do
+            SnapshotSend.new({}, @api).do_exec({
+                snapshot: snapshots.last.id,
+                from_snapshot: snapshots.first.id,
+                send_mail: false,
+                delete_after: @opts[:delete_after],
+                max_rate: @opts[:max_rate],
+                checksum: @opts[:checksum],
+                quiet: @opts[:quiet],
+            })
+          end || exit_msg('Receive failed')
+        end
+      end
+    end
+
+    def safe_download(ds, snapshot, from_snapshot = nil)
+      part, full = snapshot_tmp_file(snapshot, from_snapshot)
+
+      if !File.exists?(full)
+        attempts = 0
+
+        begin
+          SnapshotDownload.new({}, @api).do_exec({
+              snapshot: snapshot.id,
+              from_snapshot: from_snapshot && from_snapshot.id,
+              format: from_snapshot ? :incremental_stream : :stream,
+              file: part,
+              max_rate: @opts[:max_rate],
+              checksum: @opts[:checksum],
+              quiet: @opts[:quiet],
+              resume: true,
+              delete_after: @opts[:delete_after],
+              send_mail: false,
+          })
+
+        rescue Errno::ECONNREFUSED,
+               Errno::ETIMEDOUT,
+               Errno::EHOSTUNREACH,
+               Errno::ECONNRESET => e
+          warn "Connection error: #{e.message}"
+
+          attempts += 1
+
+          if attempts >= @opts[:attempts]
+            warn "Run out of attempts"
+            exit(false)
+
+          else
+            warn "Retry in 60 seconds"
+            sleep(60)
+            retry
+          end
+        end
+
+        File.rename(part, full)
+      end
+
+      run_piped(zfs_cmd(:recv, '-F', ds)) do
+        Process.exec("zcat #{full}")
+      end || exit_msg('Receive failed')
+
+      File.delete(full)
+    end
+
+    def rotate(fs, pretend: false)
+      msg "Rotating snapshots"
+      local_state = pretend ? pretend : parse_tree(fs)
+      
+      # Order snapshots by date of creation
+      snapshots = local_state.values.flatten.sort do |a, b|
+        a.creation <=> b.creation
+      end
+
+      cnt = local_state.values.inject(0) { |sum, snapshots| sum + snapshots.count }
+      deleted = 0
+      oldest = Time.now.to_i - (@opts[:max_age] * 60 * 60 * 24)
+
+      snapshots.each do |s|
+        ds = "#{fs}/#{s.hist_id}"
+
+        if (cnt - deleted) <= @opts[:min_snapshots] \
+            || (s.creation > oldest && (cnt - deleted) <= @opts[:max_snapshots])
+          break
+        end
+
+        deleted += 1
+        local_state[s.hist_id].delete(s)
+
+        msg "Destroying #{ds}@#{s.name}"
+        zfs(:destroy, nil, "#{ds}@#{s.name}", pretend: pretend)
+      end
+
+      local_state.each do |hist_id, snapshots|
+        next unless snapshots.empty?
+        
+        ds = "#{fs}/#{hist_id}"
+
+        msg "Destroying #{ds}"
+        zfs(:destroy, nil, ds, pretend: pretend)
+      end
+    end
+
+    def parse_tree(fs)
+      ret = {}
+
+      # This is intentionally done by two zfs commands, because -d2 would include
+      # nested subdatasets, which should not be there, but the user might create
+      # them and it could confuse the program.
+      zfs(:list, '-r -d1 -tfilesystem -H -oname', fs).split("\n")[1..-1].each do |name|
+        last_name = name.split('/').last
+        ret[last_name.to_i] = [] if dataset?(last_name)
+      end
+      
+      zfs(
+          :get,
+          '-Hrp -d2 name,creation -tsnapshot -oname,property,value',
+          fs
+      ).split("\n").each do |line|
+        name, property, value = line.split
+        ds, snap_name = name.split('@')
+        ds_name = ds.split('/').last
+        next unless dataset?(ds_name)
+
+        hist_id = ds_name.to_i
+
+        if snap = ret[hist_id].detect { |s| s.name == snap_name }
+          snap.send("#{property}=", value)
+
+        else
+          snap = LocalSnapshot.new(snap_name, hist_id)
+          ret[hist_id] << snap
+        end
+      end
+
+      ret
+    end
+
+    def dataset?(name)
+      /^\d+$/ =~ name
+    end
+
+    def read_dataset_id(fs)
+      ds_id = zfs(:get, '-H -ovalue cz.vpsfree.vpsadmin:dataset_id', fs).strip
+      return nil if ds_id == '-'
+      @dataset_id = ds_id.to_i
+    end
+
+    def check_dataset_id!(ds, fs)
+      if @dataset_id && @dataset_id != ds.id
+        warn "Dataset '#{fs}' is used to backup remote dataset with id '#{@dataset_id}', not '#{ds.id}'"
+        exit(false)
+      end
+    end
+
+    def written_dataset_id?
+      !@dataset_id.nil?
+    end
+
+    def write_dataset_id!(ds, fs)
+      zfs(:set, "cz.vpsfree.vpsadmin:dataset_id=#{ds.id}", fs)
+    end
+
+    # Run two processes like +block | cmd2+, where block's stdout is piped into
+    # cmd2's stdin.
+    def run_piped(cmd2, &block)
+      r, w = IO.pipe
+      pids = []
+
+      pids << Process.fork do
+        r.close
+        STDOUT.reopen(w)
+        block.call
+      end
+
+      pids << Process.fork do
+        w.close
+        STDIN.reopen(r)
+        Process.exec(cmd2)
+      end
+
+      r.close
+      w.close
+
+      ret = true
+
+      pids.each do |pid|
+        Process.wait(pid)
+        ret = false if $?.exitstatus != 0
+      end
+
+      ret
+    end
+
+    def zfs_cmd(cmd, opts, fs)
+      s = ''
+      s += 'sudo ' if Process.euid != 0
+      s += 'zfs'
+      "#{s} #{cmd} #{opts} #{fs}"
+    end
+
+    def zfs(cmd, opts, fs, pretend: false)
+      cmd = zfs_cmd(cmd, opts, fs)
+
+      if pretend
+        puts "> #{cmd}"
+        return
+      end
+
+      ret = `#{cmd}`
+      exit_msg("#{cmd} failed with exit code #{$?.exitstatus}") if $?.exitstatus != 0
+      ret
+    end
+
+    def dataset_chooser(vps_only: false)
+      user = @api.user.current
+      vpses = @api.vps.list(user: user.id)
+
+      vps_map = {}
+      vpses.each do |vps|
+        vps_map[vps.dataset_id] = vps
+      end
+
+      i = 1
+      ds_map = {}
+
+      @api.dataset.index(user: user.id).each do |ds|
+        if vps = vps_map[ds.id]
+          puts "(#{i}) VPS ##{vps.id}"
+
+        else
+          next if vps_only
+          puts "(#{i}) Dataset #{ds.name}"
+        end
+
+        ds_map[i] = ds
+        i += 1
+      end
+
+      loop do
+        STDOUT.write('Pick a dataset to backup: ')
+        STDOUT.flush
+
+        i = STDIN.readline.strip.to_i
+        next if i <= 0 || ds_map[i].nil?
+
+        return ds_map[i]
+      end
+    end
+
+    def snapshot_tmp_file(s, from_s = nil)
+      if from_s
+        base = ".snapshot_#{from_s.id}-#{s.id}.inc.dat.gz"
+
+      else
+        base = ".snapshot_#{s.id}.dat.gz"
+      end
+
+      ["#{base}.part", base]
+    end
+
+    def exit_msg(str, error: true)
+      if error
+        warn str
+        exit(1)
+
+      else
+        msg str
+        exit(0)
+      end
+    end
+  end
+end

data/lib/vpsadmin/cli/commands/backup_vps.rb

@@ -0,0 +1,29 @@
+module VpsAdmin::CLI::Commands
+  class BackupVps < BackupDataset
+    cmd :backup, :vps
+    args '[VPS_ID] FILESYSTEM'
+    desc 'Backup VPS locally'
+
+    def exec(args)
+      if args.size == 1 && /^\d+$/ !~ args[0]
+        fs = args[0]
+
+        ds_id = read_dataset_id(fs)
+
+        if ds_id
+          super([ds_id, fs])
+
+        else
+          ds = dataset_chooser(vps_only: true)
+          super([ds.id, fs])
+        end
+
+      elsif args.size == 2
+        super([@api.vps.show(args[0].to_i).dataset_id, args[1]])
+
+      else
+        super(args)
+      end
+    end
+  end
+end

data/lib/vpsadmin/cli/commands/base_download.rb

@@ -0,0 +1,31 @@
+module VpsAdmin::CLI::Commands
+  class BaseDownload < HaveAPI::CLI::Command
+    protected
+    def find_or_create_dl(opts, do_create = true)
+      @api.snapshot_download.index(snapshot: opts[:snapshot]).each do |r|
+        if opts[:from_snapshot] == (r.from_snapshot && r.from_snapshot_id)
+          if r.format != opts[:format].to_s
+            fail "SnapshotDownload id=#{r.id} is in unusable format '#{r.format}' (needs '#{opts[:format]}')"
+          end
+
+          return [r, false]
+        end
+      end
+
+      if do_create
+        [@api.snapshot_download.create(opts), true]
+
+      else
+        [nil, true]
+      end
+    end
+
+    def msg(str)
+      puts str unless @opts[:quiet]
+    end
+
+    def warn_msg(str)
+      warn str unless @opts[:quiet]
+    end
+  end
+end

data/lib/vpsadmin/cli/commands/snapshot_download.rb

@@ -0,0 +1,214 @@
+module VpsAdmin::CLI::Commands
+  class SnapshotDownload < BaseDownload
+    cmd :snapshot, :download
+    args '[SNAPSHOT_ID]'
+    desc 'Download a snapshot as an archive or a stream'
+
+    def options(opts)
+      @opts = {
+          delete_after: true,
+          send_mail: false,
+          checksum: true,
+          format: 'archive',
+      }
+
+      opts.on('-f', '--format FORMAT', 'archive, stream or incremental_stream') do |f|
+        @opts[:format] = f
+      end
+      
+      opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
+        @opts[:from_snapshot] = s
+      end
+
+      opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
+        @opts[:delete_after] = d
+      end
+
+      opts.on('-F', '--force', 'Overwrite existing files if necessary') do |f|
+        @opts[:force] = f
+      end
+
+      opts.on('-o', '--output FILE', 'Save the download to FILE') do |f|
+        @opts[:file] = f
+      end
+
+      opts.on('-q', '--quiet', 'Print only errors') do |q|
+        @opts[:quiet] = q
+      end
+
+      opts.on('-r', '--resume', 'Resume cancelled download') do |r|
+        @opts[:resume] = r
+      end
+
+      opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
+        @opts[:send_mail] = s
+      end
+
+      opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
+        exit_msg('--max-rate must be greater than zero') if r <= 0
+        @opts[:max_rate] = r
+      end
+
+      opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
+        @opts[:checksum] = c
+      end
+    end
+
+    def exec(args)
+      if args.size == 0 && STDIN.tty?
+        @opts[:snapshot] = snapshot_chooser
+
+      elsif args.size != 1
+        warn "Provide exactly one SNAPSHOT_ID as an argument"
+        exit(false)
+
+      else
+        @opts[:snapshot] = args.first.to_i
+      end
+      
+      do_exec(@opts)
+    end
+
+    def do_exec(opts)
+      @opts = opts
+      f = action = nil
+      pos = 0
+
+      if @opts[:file] == '-'
+        f = STDOUT
+
+      elsif @opts[:file]
+        f, action, pos = open_file(@opts[:file])
+      end
+
+      dl, created = find_or_create_dl(@opts, action != :resume)
+      f, action, pos = open_file(dl.file_name) unless @opts[:file]
+
+      if created
+        if action == :resume
+          warn "Unable to resume the download: the file has been deleted from the server"
+          exit(false)
+        end
+
+        msg "The download is being prepared..."
+        sleep(5)
+
+      else
+        warn "Reusing existing SnapshotDownload (id=#{dl.id})"
+      end
+
+      msg "Downloading to #{f.path}"
+
+      begin
+        VpsAdmin::CLI::StreamDownloader.download(
+            @api,
+            dl,
+            f,
+            progress: !@opts[:quiet] && (f == STDOUT ? STDERR : STDOUT),
+            position: pos,
+            max_rate: @opts[:max_rate],
+            checksum: @opts[:checksum],
+        )
+
+      rescue VpsAdmin::CLI::DownloadError => e
+        warn e.message
+        exit(false)
+        
+      ensure
+        f.close
+      end
+
+      @api.snapshot_download.delete(dl.id) if @opts[:delete_after]
+    end
+
+    protected
+    def open_file(path)
+      f = action = nil
+      pos = 0
+
+      if File.exists?(path) && File.size(path) > 0
+        if @opts[:resume]
+          action = :resume
+
+        elsif @opts[:force]
+          action = :overwrite
+
+        elsif STDIN.tty?
+          while action.nil?
+            STDERR.write("'#{path}' already exists. [A]bort, [r]esume or [o]verwrite? [a]: ")
+            STDERR.flush
+
+            action = {
+                'r' => :resume,
+                'o' => :overwrite,
+                '' => false,
+            }[STDIN.readline.strip.downcase]
+          end
+
+        else
+          warn "File '#{path}' already exists"
+          exit(false)
+        end
+        
+        case action
+        when :resume
+          mode = 'a+'
+          pos = File.size(path)
+
+        when :overwrite
+          mode = 'w'
+
+        else
+          exit
+        end
+
+        f = File.open(path, mode)
+      else
+        f = File.open(path, 'w')
+      end
+
+      [f, action, pos]
+    end
+
+    def snapshot_chooser
+      user = @api.user.current
+      vpses = @api.vps.list(user: user.id)
+
+      ds_map = {}
+      vpses.each do |vps|
+        ds_map[vps.dataset_id] = vps
+      end
+
+      i = 1
+      snap_map = {}
+
+      @api.dataset.index(user: user.id).each do |ds|
+        snapshots = ds.snapshot.index
+        next if snapshots.empty?
+
+        if vps = ds_map[ds.id]
+          puts "VPS ##{vps.id}"
+
+        else
+          puts "Dataset #{ds.name}"
+        end
+
+        snapshots.each do |s|
+          snap_map[i] = s
+          puts "  (#{i}) @#{s.created_at}"
+          i += 1
+        end
+      end
+
+      loop do
+        STDOUT.write('Pick a snapshot for download: ')
+        STDOUT.flush
+
+        i = STDIN.readline.strip.to_i
+        next if i <= 0 || snap_map[i].nil?
+
+        return snap_map[i].id
+      end
+    end
+  end
+end 

data/lib/vpsadmin/cli/commands/snapshot_send.rb

@@ -0,0 +1,101 @@
+require 'zlib'
+
+module VpsAdmin::CLI::Commands
+  class SnapshotSend < BaseDownload
+    cmd :snapshot, :send
+    args 'SNAPSHOT_ID'
+    desc 'Download a snapshot stream and write it on stdout'
+
+    def options(opts)
+      @opts = {
+          delete_after: true,
+          send_mail: false,
+          checksum: true,
+      }
+      
+      opts.on('-I', '--from-snapshot SNAPSHOT_ID', Integer, 'Download snapshot incrementally from SNAPSHOT_ID') do |s|
+        @opts[:from_snapshot] = s
+      end
+
+      opts.on('-d', '--[no-]delete-after', 'Delete the file from the server after successful download') do |d|
+        @opts[:delete_after] = d
+      end
+      
+      opts.on('-q', '--quiet', 'Print only errors') do |q|
+        @opts[:quiet] = q
+      end
+
+      opts.on('-s', '--[no-]send-mail', 'Send mail after the file for download is completed') do |s|
+        @opts[:send_mail] = s
+      end
+
+      opts.on('-x', '--max-rate N', Integer, 'Maximum download speed in kB/s') do |r|
+        exit_msg('--max-rate must be greater than zero') if r <= 0
+        @opts[:max_rate] = r
+      end
+
+      opts.on('--[no-]checksum', 'Verify checksum of the downloaded data (enabled)') do |c|
+        @opts[:checksum] = c
+      end
+    end
+
+    def exec(args)
+      if args.size != 1
+        warn "Provide exactly one SNAPSHOT_ID as an argument"
+        exit(false)
+      end
+
+      opts = @opts.clone
+      opts[:snapshot] = args.first.to_i
+
+      do_exec(opts)
+    end
+
+    def do_exec(opts)
+      @opts = opts
+      opts[:format] = opts[:from_snapshot] ? :incremental_stream : :stream
+
+      dl, created = find_or_create_dl(opts)
+
+      if created
+        warn_msg "The download is being prepared..."
+        sleep(5)
+
+      else
+        warn_msg "Reusing existing SnapshotDownload (id=#{dl.id})"
+      end
+
+      r, w = IO.pipe
+
+      pid = Process.fork do
+        r.close
+
+        begin
+          VpsAdmin::CLI::StreamDownloader.download(
+              @api,
+              dl,
+              w,
+              progress: !opts[:quiet] && STDERR,
+              max_rate: opts[:max_rate],
+              checksum: opts[:checksum],
+          )
+
+        rescue VpsAdmin::CLI::DownloadError => e
+          warn e.message
+          exit(false)
+        end
+      end
+
+      w.close
+
+      gz = Zlib::GzipReader.new(r)
+      STDOUT.write(gz.readpartial(16*1024)) while !gz.eof?
+      gz.close
+
+      Process.wait(pid)
+      exit($?.exitstatus) if $?.exitstatus != 0
+
+      @api.snapshot_download.delete(dl.id) if opts[:delete_after]
+    end
+  end
+end

data/lib/vpsadmin/cli/commands/vps_migrate_many.rb

@@ -18,6 +18,10 @@ module VpsAdmin::CLI::Commands
       opts.on('--[no-]outage-window', 'Migrate VPSes inside outage windows') do |w|
         @opts[:outage_window] = w
       end
+      
+      opts.on('--[no-]cleanup-data', 'Cleanup VPS dataset on the source node') do |c|
+        @opts[:cleanup_data] = c
+      end
 
       opts.on('--[no-]stop-on-error', 'Cancel the plan if a migration fails') do |s|
         @opts[:stop_on_error] = s
@@ -87,6 +91,7 @@ module VpsAdmin::CLI::Commands
               dst_node: @opts[:dst_node],
           }
           params[:outage_window] = @opts[:outage_window] unless @opts[:outage_window].nil?
+          params[:cleanup_data] = @opts[:cleanup_data] unless @opts[:cleanup_data].nil?
 
           plan.vps_migration.create(params)
         end

data/lib/vpsadmin/cli/stream_downloader.rb

@@ -0,0 +1,236 @@
+require 'uri'
+require 'net/http'
+require 'ruby-progressbar'
+require 'digest'
+
+module VpsAdmin::CLI
+  class DownloadError < StandardError ; end
+
+  class StreamDownloader
+    def self.download(*args)
+      new(*args)
+    end
+
+    def initialize(api, dl, io, progress: STDOUT, position: 0, max_rate: nil,
+                  checksum: true)
+      downloaded = position
+      uri = URI(dl.url)
+      digest = Digest::SHA256.new
+      dl_check = nil
+
+      if position > 0 && checksum
+        if progress
+          pb = ProgressBar.create(
+              title: 'Calculating checksum',
+              total: position,
+              format: '%E %t: [%B] %p%% %r MB/s',
+              rate_scale: ->(rate) { (rate / 1024.0 / 1024.0).round(2) },
+              throttle_rate: 0.2,
+              output: progress,
+          )
+        end
+
+        read = 0
+        step = 1*1024*1024
+        io.seek(0)
+
+        while read < position
+          data = io.read((read + step) > position ? position - read : step)
+          read += data.size
+
+          digest << data
+          pb.progress = read if pb
+        end
+
+        pb.finish if pb
+      end
+
+      if progress
+        self.format = '%t: [%B] %r kB/s'
+
+        @pb = ProgressBar.create(
+            title: 'Downloading',
+            total: nil,
+            format: @format,
+            rate_scale: ->(rate) { (rate / 1024.0).round(2) },
+            throttle_rate: 0.2,
+            starting_at: downloaded,
+            autofinish: false,
+            output: progress,
+        )
+      end
+
+      args = [uri.host] + Array.new(5, nil) + [{use_ssl: uri.scheme == 'https'}]
+
+      Net::HTTP.start(*args) do |http|
+        loop do
+          begin
+            dl_check = api.snapshot_download.show(dl.id)
+
+            if @pb && (dl_check.ready || (dl_check.size && dl_check.size > 0))
+              @pb.progress = downloaded
+
+              total = dl_check.size * 1024 * 1024
+              @pb.total = @pb.progress > total ? @pb.progress : total
+              @download_size = (dl_check.size / 1024.0).round(2)
+
+              if dl_check.ready
+                @download_ready = true
+                self.format = "%E %t #{@download_size} GB: [%B] %p%% %r kB/s"
+
+              else
+                self.format = "%E %t ~#{@download_size} GB: [%B] %p%% %r kB/s"
+              end
+            end
+
+          rescue HaveAPI::Client::ActionFailed => e
+            # The SnapshotDownload object no longer exists, the transaction
+            # responsible for its creation must have failed.
+            stop
+            raise DownloadError, 'The download has failed due to transaction failure'
+          end
+
+          headers = {}
+          headers['Range'] = "bytes=#{downloaded}-" if downloaded > 0
+
+          http.request_get(uri.path, headers) do |res|
+            case res.code
+            when '404'  # Not Found
+              if downloaded > 0
+                # This means that the transaction used for preparing the download
+                # has failed, the file to download does not exist anymore, so fail.
+                raise DownloadError, 'The download has failed, most likely transaction failure'
+
+              else
+                # The file is not available yet, this is normal, the transaction
+                # may be queued and it can take some time before it is processed.
+                pause(10)
+                next
+              end
+
+            when '416'  # Range Not Satisfiable
+              if downloaded > position
+                # We have already managed to download something (at this run, if the trasfer
+                # was resumed) and the server cannot provide more data yet. This can be
+                # because the server is busy. Wait and retry.
+                pause(20)
+                next
+
+              else
+                # The file is not ready yet - we ask for range that cannot be provided
+                # This happens when we're resuming a download and the file on the
+                # server was deleted meanwhile. The file might not be exactly the same
+                # as the one before, sha256sum would most likely fail.
+                raise DownloadError, 'Range not satisfiable'
+              end
+
+            when '200', '206'  # OK and Partial Content
+              resume
+
+            else
+              raise DownloadError, "Unexpected HTTP status code '#{res.code}'"
+            end
+           
+            t1 = Time.now
+            data_counter = 0
+
+            res.read_body do |fragment|
+              size = fragment.size
+
+              data_counter += size
+              downloaded += size
+
+              begin
+                if @pb && (@pb.total.nil? || @pb.progress < @pb.total)
+                  @pb.progress += size
+                end
+
+              rescue ProgressBar::InvalidProgressError
+                # The total value is in MB, it is not precise, so the actual
+                # size may be a little bit bigger.
+                @pb.progress = @pb.total
+              end
+
+              digest.update(fragment) if checksum
+
+              if max_rate && max_rate > 0
+                t2 = Time.now
+                diff = t2 - t1
+
+                if diff > 0.005
+                  # Current and expected rates in kB per interval +diff+
+                  current_rate = data_counter / 1024
+                  expected_rate = max_rate * diff
+
+                  if current_rate > expected_rate
+                    delay = diff / (expected_rate / (current_rate - expected_rate))
+                    sleep(delay)
+                  end
+
+                  data_counter = 0
+                  t1 = Time.now
+                end
+              end
+            
+              io.write(fragment)
+            end
+          end
+
+          # This was the last download, the transfer is complete.
+          break if dl_check.ready
+
+          # Give the server time to prepare additional data
+          pause(15)
+        end
+      end
+
+      @pb.finish if @pb
+
+      # Verify the checksum
+      if checksum && digest.hexdigest != dl_check.sha256sum
+        raise DownloadError, 'The sha256sum does not match, retry the download'
+      end
+    end
+
+    protected
+    def pause(secs)
+      @paused = true
+      
+      if @pb
+        secs.times do |i|
+          if @download_size
+            if @download_ready
+              @pb.format("%t #{@download_size} GB: [%B] waiting #{secs - i}")
+
+            else
+              @pb.format("%t ~#{@download_size} GB: [%B] waiting #{secs - i}")
+            end
+
+          else
+            @pb.format("%t: [%B] waiting #{secs - i}")
+          end
+
+          @pb.refresh(force: true)
+          sleep(1)
+        end
+
+      else
+        sleep(secs)
+      end
+    end
+
+    def resume
+      @pb.format(@format) if @pb && @paused
+      @paused = false
+    end
+
+    def stop
+      @pb.stop if @pb
+    end
+
+    def format=(fmt)
+      @format = fmt
+      @pb.format(@format) if @pb
+    end
+  end
+end

data/lib/vpsadmin/client/version.rb

@@ -1,5 +1,5 @@
 module VpsAdmin
   module Client
-    VERSION = '2.2.0'
+    VERSION = '2.3.0'
   end
 end

data/vpsadmin-client.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'eventmachine', '~> 1.0.3'
   spec.add_runtime_dependency 'em-http-request', '~> 1.1.3'
   spec.add_runtime_dependency 'json', '~> 1.8.3'
+  spec.add_runtime_dependency 'ruby-progressbar', '~> 1.7.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vpsadmin-client
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Jakub Skokan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-09 00:00:00.000000000 Z
+date: 2016-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.8.3
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.5
 description: Ruby API and CLI for vpsAdmin API
 email:
 - jakub.skokan@vpsfree.cz
@@ -111,8 +125,14 @@ files:
 - bin/vpsadminctl
 - lib/terminal-size.rb
 - lib/vpsadmin/cli.rb
+- lib/vpsadmin/cli/commands/backup_dataset.rb
+- lib/vpsadmin/cli/commands/backup_vps.rb
+- lib/vpsadmin/cli/commands/base_download.rb
+- lib/vpsadmin/cli/commands/snapshot_download.rb
+- lib/vpsadmin/cli/commands/snapshot_send.rb
 - lib/vpsadmin/cli/commands/vps_migrate_many.rb
 - lib/vpsadmin/cli/commands/vps_remote_console.rb
+- lib/vpsadmin/cli/stream_downloader.rb
 - lib/vpsadmin/client.rb
 - lib/vpsadmin/client/version.rb
 - vpsadmin-client.gemspec