backup-remote 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0aa4fc4049cb34499a58fe72e4ebe283b950063
-  data.tar.gz: c6b09225ac5faf3de22f73cf63fc801400cdfa3d
+  metadata.gz: c2fc3569f37e204350b31fc06bf5edde0f15a89e
+  data.tar.gz: dd9db17395b314a876850f35aecc9bea56ea5c0d
 SHA512:
-  metadata.gz: 9c7162a70d86ae53879c7733a479985c34fcaea433dc2fcf1ed2ed2db26c51bfece25f6970fc10166d4ce0b5966a267089e13b473c66c8168aff8c5c7565e017
-  data.tar.gz: 52b99fefd11d57b4a03b6dca9744c7d26e605fc8f8db5b30dd4ea988a2735d43359659c2af5025861f9724f60edc6f32794fa513c469201e9f3855e21db240f0
+  metadata.gz: 7e6bdac05bd47da6f855508e369576f59011058ea55cccad9b856b0514158786730a6207d1f0be31371c9fd7055913c1a8d7e5688c0fce0f74fdecec050b8a99
+  data.tar.gz: a40f18efe432ad1e45297e04549070435a6d487e55bc673e44fdb00ca3cf17a143c031bdbf95bf0c1a780f0140d0d7e48e58b9233ff7f515b4f5f87093456609

data/README.md

@@ -12,6 +12,12 @@ Backup is a system utility for Linux and Mac OS X, distributed as a RubyGem, tha
 operations. It provides an elegant DSL in Ruby for _modeling_ your backups. 
 Backup has built-in support for various databases, storage protocols/services, syncers, compressors, encryptors and notifiers which you can mix and match. 
 
+The gem adds the following model components:
+* Remote Archive
+* Remote MySQL database
+* Remote data
+
+
 
 
 # How it works
@@ -67,11 +73,34 @@ Options for SSH connection:
 
 ## Archive files on a remote server
 
-* Use RemoteArchive
+* Use remote_archive in your model
+
 
 ```
+
+Model.new(:my_server_files_backup, 'Backup files') do
+
+  remote_archive :files do |archive|
+    archive.server_host = "myserver.com"
+    archive.server_ssh_user = "user"
+    archive.server_ssh_password = "pwd"
+
+
+    # archive options - the same options as for archive
+    # see  http://backup.github.io/backup/v4/archives/
+
+
+  end
+  
+  ...
+  
+end  
+    
 ```
 
+Options:
+* server_command - command to create archive file
+
 
 # Databases
 
@@ -80,6 +109,7 @@ Options for SSH connection:
 * Now it is implemented the following databases:
 * RemoteMySQL
 
+
 ### RemoteMySQL
 
 ```
@@ -97,13 +127,45 @@ Model.new(:my_backup, 'My Backup') do
     ...
   end
   ..
-end  
-    
+end
+
 ````
 
-# Custom backup command
 
-* Run custom command to create a backup archive on a remote server
+
+# Custom data on remote server
+
+* Run custom command on the remote server to create a backup archive
+
+* Specify command to run to generate archive file on the remote server
+
+* This command should create an archive file with filename specified in server_path option.
+
+
+```
+Model.new(:my_server_data_backup, 'Backup data') do
+
+  remote_data :mydata do |archive|
+    archive.server_host = "myserver.com"
+    archive.server_ssh_user = "user"
+    archive.server_ssh_password = "pwd"
+    
+    
+    archive.command = "--any command to generate backup archive file--"
+    # archive.command = "echo '1' > /tmp/backup.txt"
+    
+    archive.server_path = "/path/to/archive.tar.gz"
+    # archive.command = "/tmp/backup.txt"
+
+
+    
+  end
+  
+  ...
+  
+end  
+    
+```
 
 
 # Backup gem

data/lib/backup.rb

@@ -149,7 +149,13 @@ module Backup
     pipeline
     splitter
     template
+
+    remote_archive
+    remote_data
+
     version
-  }.each {|lib| require File.join(LIBRARY_PATH, lib) }
+  }.each do |lib|
+    require File.join(LIBRARY_PATH, lib)
+  end
 
 end

data/lib/backup/model.rb

@@ -139,6 +139,19 @@ module Backup
       @archives << Archive.new(self, name, &block)
     end
 
+    ##
+    # Adds a Remote Archive. Multiple archives may be added to the model.
+    def remote_archive(name, &block)
+      @archives << RemoteArchive.new(self, name, &block)
+    end
+
+    ##
+    # Adds a Remote data Archive
+    def remote_data(name, &block)
+      @archives << RemoteData.new(self, name, &block)
+    end
+
+
     ##
     # Adds an Database. Multiple Databases may be added to the model.
     def database(name, database_id = nil, &block)

data/lib/backup/remote/command.rb

@@ -77,6 +77,39 @@ module Backup
         }
       end
 
+  def ssh_upload_file(hostname, ssh_user, ssh_pass, source_file, dest_file, handler=nil)
+    host = SSHKit::Host.new("#{ssh_user}@#{hostname}")
+    host.password = ssh_pass
+
+    # scp
+    f_temp = "/tmp/#{SecureRandom.uuid}"
+
+    # sshkit
+    on host do |host|
+      as(user: ssh_user) do
+
+      end
+
+      # NOT WORK with sudo
+      #upload! source_file, dest_file
+
+      # upload to temp file
+      upload! source_file, f_temp
+
+      # upload to dest
+      execute("cp #{f_temp} #{dest_file}", interaction_handler: handler)
+
+    end
+
+    #
+    return     {res: 1, output: ""}
+  rescue => e
+    {
+        res: 0,
+        error: e.message
+    }
+  end
+
 end
 end
 end

data/lib/backup/remote_archive.rb

@@ -0,0 +1,266 @@
+# encoding: utf-8
+
+require 'net/ssh'
+
+require 'sshkit'
+require 'sshkit/dsl'
+require 'sshkit/sudo'
+
+
+module Backup
+  class RemoteArchive < Archive
+    class Error < Backup::Error; end
+
+    include Utilities::Helpers
+    #attr_reader :name, :options
+
+    include SSHKit::DSL
+
+    # server options
+    attr_accessor :server_host
+    attr_accessor :server_ssh_user
+    attr_accessor :server_ssh_password
+    attr_accessor :server_ssh_key
+    attr_accessor :server_backup_path
+
+
+
+    ##
+    # Adds a new Archive to a Backup Model.
+    #
+    #     Backup::Model.new(:my_backup, 'My Backup') do
+    #       archive :my_archive do |archive|
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # All paths added using `add` or `exclude` will be expanded to their
+    # full paths from the root of the filesystem. Files will be added to
+    # the tar archive using these full paths, and their leading `/` will
+    # be preserved (using tar's `-P` option).
+    #
+    #     /path/to/pwd/path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # When a `root` path is given, paths to add/exclude are taken as
+    # relative to the `root` path, unless given as absolute paths.
+    #
+    #     Backup::Model.new(:my_backup, 'My Backup') do
+    #       archive :my_archive do |archive|
+    #         archive.root '~/my_data'
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # This directs `tar` to change directories to the `root` path to create
+    # the archive. Unless paths were given as absolute, the paths within the
+    # archive will be relative to the `root` path.
+    #
+    #     path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # For absolute paths added to this archive, the leading `/` will be
+    # preserved. Take note that when archives are extracted, leading `/` are
+    # stripped by default, so care must be taken when extracting archives with
+    # mixed relative/absolute paths.
+    def initialize(model, name, &block)
+      @model   = model
+      @name    = name.to_s
+      @options = {
+        :sudo        => false,
+        :root        => false,
+        :paths       => [],
+        :excludes    => [],
+        :tar_options => ''
+      }
+
+      DSL.new(@options).instance_eval(&block)
+
+      #
+      self.server_host = @options[:server_host]
+      self.server_ssh_user = @options[:server_ssh_user]
+      self.server_ssh_password = @options[:server_ssh_password]
+    end
+
+    def perform!
+      Logger.info "Creating Archive '#{ name }'..."
+
+      #
+      path = File.join(Config.tmp_path, @model.trigger, 'archives')
+      FileUtils.mkdir_p(path)
+
+
+      #
+      remote = Backup::Remote::Command.new
+
+      pipeline = Pipeline.new
+      with_files_from(paths_to_package) do |files_from|
+        # upload to server
+        res_upload = remote.ssh_upload_file(server_host, server_ssh_user, server_ssh_password, files_from, files_from)
+
+        if res_upload[:res]==0
+          raise 'Cannot upload file from server - #{files_from}'
+        end
+
+        #
+        pipeline.add(
+          "#{ tar_command } #{ tar_options } -cPf -#{ tar_root } " +
+          "#{ paths_to_exclude } -T '#{ files_from }'",
+          tar_success_codes
+        )
+
+        extension = 'tar'
+        @model.compressor.compress_with do |command, ext|
+          pipeline << command
+          extension << ext
+        end if @model.compressor
+
+        #
+        archive_file = File.join(path, "#{ name }.#{ extension }")
+        remote_archive_file = File.join('/tmp', "#{ name }.#{ extension }")
+        pipeline << "#{ utility(:cat) } > '#{ remote_archive_file }'"
+
+
+        #pipeline.run
+
+        # generate backup on remote server
+        cmd_remote = pipeline.commands.join(" | ")
+
+        #puts "remote cmd: #{cmd_remote}"
+        #exit
+
+
+        res_generate = remote.run_ssh_cmd(server_host, server_ssh_user, server_ssh_password, cmd_remote)
+
+        if res_generate[:res]==0
+          raise 'Cannot create backup on server'
+        end
+
+        # download backup
+        res_download = remote.ssh_download_file(server_host, server_ssh_user, server_ssh_password, remote_archive_file, archive_file)
+
+        #puts "res: #{res_download}"
+
+        if res_download[:res]==0
+          raise 'Cannot download file from server'
+        end
+
+        # delete archive on server
+        res_delete = remote.run_ssh_cmd(server_host, server_ssh_user, server_ssh_password, "rm #{remote_archive_file}")
+
+      end
+
+      Logger.info "Archive '#{ name }' Complete!"
+
+      #if pipeline.success?
+      #  Logger.info "Archive '#{ name }' Complete!"
+      #else
+      #  raise Error, "Failed to Create Archive '#{ name }'\n" + pipeline.error_messages
+      #end
+    end
+
+    private
+
+    def tar_command
+      tar = utility(:tar)
+      options[:sudo] ? "#{ utility(:sudo) } -n #{ tar }" : tar
+    end
+
+    def tar_root
+      options[:root] ? " -C '#{ File.expand_path(options[:root]) }'" : ''
+    end
+
+    def paths_to_package
+      options[:paths].map {|path| prepare_path(path) }
+    end
+
+    def with_files_from(paths)
+      tmpfile = Tempfile.new('backup-archive-paths')
+      paths.each {|path| tmpfile.puts path }
+      tmpfile.close
+
+      puts "tmpfile #{tmpfile.path}"
+
+      puts "content: #{File.read(tmpfile.path)}"
+      #yield "-T '#{ tmpfile.path }'"
+      yield "#{ tmpfile.path }"
+    ensure
+
+      puts "delete file #{tmpfile.path}"
+      tmpfile.delete
+    end
+
+    def paths_to_exclude
+      options[:excludes].map {|path|
+        "--exclude='#{ prepare_path(path) }'"
+      }.join(' ')
+    end
+
+    def prepare_path(path)
+
+      res = options[:root] ? path : File.expand_path(path)
+
+      puts "path #{path} ===> #{res}"
+
+      res
+    end
+
+    def tar_options
+      args = options[:tar_options]
+      gnu_tar? ? "--ignore-failed-read #{ args }".strip : args
+    end
+
+    def tar_success_codes
+      gnu_tar? ? [0, 1] : [0]
+    end
+
+
+    ### DSL for RemoteArchive
+    class DSL
+      def initialize(options)
+        @options = options
+      end
+
+
+      ### remote server
+      def server_host=(val = true)
+        @options[:server_host] = val
+      end
+
+      def server_ssh_user=(val = true)
+        @options[:server_ssh_user] = val
+      end
+      def server_ssh_password=(val = true)
+        @options[:server_ssh_password] = val
+      end
+
+      ###
+      def use_sudo(val = true)
+        @options[:sudo] = val
+      end
+
+      def root(path)
+        @options[:root] = path
+      end
+
+      def add(path)
+        @options[:paths] << path
+      end
+
+      def exclude(path)
+        @options[:excludes] << path
+      end
+
+      def tar_options(opts)
+        @options[:tar_options] = opts
+      end
+    end
+
+  end
+end

data/lib/backup/remote_data.rb

@@ -0,0 +1,237 @@
+# encoding: utf-8
+
+require 'net/ssh'
+
+require 'sshkit'
+require 'sshkit/dsl'
+require 'sshkit/sudo'
+
+
+module Backup
+  class RemoteData < Archive
+    class Error < Backup::Error; end
+
+    include Utilities::Helpers
+    #attr_reader :name, :options
+
+    include SSHKit::DSL
+
+    # server options
+    attr_accessor :server_host
+    attr_accessor :server_ssh_user
+    attr_accessor :server_ssh_password
+    attr_accessor :server_ssh_key
+    attr_accessor :server_path
+    attr_accessor :server_command
+
+
+
+
+
+    def initialize(model, name, &block)
+      @model   = model
+      @name    = name.to_s
+      @options = {
+        :sudo        => false,
+        :root        => false,
+        :paths       => [],
+        :excludes    => [],
+        :tar_options => ''
+      }
+
+      DSL.new(@options).instance_eval(&block)
+
+      #
+      self.server_host = @options[:server_host]
+      self.server_ssh_user = @options[:server_ssh_user]
+      self.server_ssh_password = @options[:server_ssh_password]
+      self.server_path = @options[:server_path]
+      self.server_command = @options[:server_command]
+    end
+
+    def perform!
+      Logger.info "Creating Archive '#{ name }'..."
+
+      # local archive
+      path = File.join(Config.tmp_path, @model.trigger, 'archives')
+      FileUtils.mkdir_p(path)
+
+      extension = 'tar'
+      #temp_archive_file = File.join(path, "#{ name }.#{ extension }")
+
+      remote = Backup::Remote::Command.new
+
+
+      Dir.mktmpdir do |temp_dir|
+        temp_local_file = File.join("#{temp_dir}", File.basename(server_path))
+        #temp_local_file = File.join(path, File.basename(server_path))
+        #temp_local_file = Tempfile.new("").path+"."+File.extname(server_path)
+
+        remote_archive_file = server_path
+
+        # generate backup on remote server
+        cmd_remote = server_command
+        res_generate = remote.run_ssh_cmd(
+            server_host, server_ssh_user, server_ssh_password,
+            cmd_remote)
+
+        if res_generate[:res]==0
+          raise 'Cannot create backup on server'
+        end
+
+        # download backup
+        res_download = remote.ssh_download_file(
+            server_host, server_ssh_user, server_ssh_password,
+            remote_archive_file, temp_local_file)
+
+        if res_download[:res]==0
+          raise 'Cannot download file from server'
+        end
+
+        # delete archive on server
+        res_delete = remote.run_ssh_cmd(
+            server_host, server_ssh_user, server_ssh_password,
+            "rm #{remote_archive_file}")
+
+
+        # process archive locally
+
+        pipeline = Pipeline.new
+
+        #temp_tar_root= tar_root
+        temp_tar_root= temp_dir
+        pipeline.add(
+            "#{ tar_command } #{ tar_options } -cPf - -C #{temp_tar_root } #{ File.basename(temp_local_file) }",
+            tar_success_codes
+        )
+
+        extension = 'tar'
+        @model.compressor.compress_with do |command, ext|
+          pipeline << command
+          extension << ext
+        end if @model.compressor
+
+        pipeline << "#{ utility(:cat) } > '#{ File.join(path, "#{ name }.#{ extension }") }'"
+
+        #puts "commands: #{pipeline.commands}"
+        #exit
+
+        pipeline.run
+
+
+        if pipeline.success?
+          Logger.info "Archive '#{ name }' Complete!"
+        else
+          raise Error, "Failed to Create Archive '#{ name }'\n" +
+              pipeline.error_messages
+        end
+
+      end
+    end
+
+    private
+
+    def tar_command
+      tar = utility(:tar)
+      options[:sudo] ? "#{ utility(:sudo) } -n #{ tar }" : tar
+    end
+
+    def tar_root
+      options[:root] ? " -C '#{ File.expand_path(options[:root]) }'" : ''
+    end
+
+    def paths_to_package
+      options[:paths].map {|path| prepare_path(path) }
+    end
+
+    def with_files_from(paths)
+      tmpfile = Tempfile.new('backup-archive-paths')
+      paths.each {|path| tmpfile.puts path }
+      tmpfile.close
+
+      puts "tmpfile #{tmpfile.path}"
+
+      puts "content: #{File.read(tmpfile.path)}"
+      #yield "-T '#{ tmpfile.path }'"
+      yield "#{ tmpfile.path }"
+    ensure
+
+      puts "delete file #{tmpfile.path}"
+      tmpfile.delete
+    end
+
+    def paths_to_exclude
+      options[:excludes].map {|path|
+        "--exclude='#{ prepare_path(path) }'"
+      }.join(' ')
+    end
+
+    def prepare_path(path)
+
+      res = options[:root] ? path : File.expand_path(path)
+
+      puts "path #{path} ===> #{res}"
+
+      res
+    end
+
+    def tar_options
+      args = options[:tar_options]
+      gnu_tar? ? "--ignore-failed-read #{ args }".strip : args
+    end
+
+    def tar_success_codes
+      gnu_tar? ? [0, 1] : [0]
+    end
+
+
+    ### DSL for RemoteArchive
+    class DSL
+      def initialize(options)
+        @options = options
+      end
+
+
+      ### remote server
+      def server_host=(val = true)
+        @options[:server_host] = val
+      end
+
+      def server_ssh_user=(val = true)
+        @options[:server_ssh_user] = val
+      end
+      def server_ssh_password=(val = true)
+        @options[:server_ssh_password] = val
+      end
+
+      def server_command=(val = true)
+        @options[:server_command] = val
+      end
+      def server_path=(val = true)
+        @options[:server_path] = val
+      end
+
+      ###
+      def use_sudo(val = true)
+        @options[:sudo] = val
+      end
+
+      def root(path)
+        @options[:root] = path
+      end
+
+      def add(path)
+        @options[:paths] << path
+      end
+
+      def exclude(path)
+        @options[:excludes] << path
+      end
+
+      def tar_options(opts)
+        @options[:tar_options] = opts
+      end
+    end
+
+  end
+end

data/lib/backup/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Backup
-  VERSION = '0.0.3'
+  VERSION = '0.0.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backup-remote
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Max Ivak, Michael van Rooijen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-14 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: CFPropertyList
@@ -842,14 +842,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.1
+        version: 0.19.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.1
+        version: 0.19.1
 - !ruby/object:Gem::Dependency
   name: thread_safe
   requirement: !ruby/object:Gem::Requirement
@@ -1017,6 +1017,8 @@ files:
 - lib/backup/packager.rb
 - lib/backup/pipeline.rb
 - lib/backup/remote/command.rb
+- lib/backup/remote_archive.rb
+- lib/backup/remote_data.rb
 - lib/backup/splitter.rb
 - lib/backup/storage/base.rb
 - lib/backup/storage/cloud_files.rb