backup-remote 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6348ea6aeaeb00662de0ab77653a06d502bda873
+  data.tar.gz: a3b3dc874abda57381f51cce9304a4c3ce7cec02
+SHA512:
+  metadata.gz: 59d8e1ff63b2282972f5d9f84b128e4a84c6599437977aa49f94540a627bffbae85887e821d49313fca9c50250472ca7293d4e89e6aee2d9550e1ad8a85e31b9
+  data.tar.gz: 0ef397f317bcb405cd93716e7a207d30137c085a475e1f11bad4606730cc402c95e8437c4a8d2fe1ff4771aa42b49b9e888e5cbe410ed62ee9a49930a84e1785

data/README.md

@@ -0,0 +1,112 @@
+Backup Remote
+===========
+Backup-remote gem extends [Backup gem](https://github.com/backup/backup) to perform backups on remote servers.
+
+If you use backup gem you should run `backup perform` on the machine where resources (files, databases) are located.
+
+This gem adds support for models to perform backups on a remote server.
+
+
+
+Backup is a system utility for Linux and Mac OS X, distributed as a RubyGem, that allows you to easily perform backup
+operations. It provides an elegant DSL in Ruby for _modeling_ your backups. 
+Backup has built-in support for various databases, storage protocols/services, syncers, compressors, encryptors and notifiers which you can mix and match. 
+
+
+
+# How it works
+
+Backup process:
+
+* specify server connection options in your model
+
+```
+Model.new(:my_backup, 'My Backup') do
+  database RemoteMySQL do |db|
+    # options for server
+    db.server_host = "server1.com"
+    db.server_ssh_user = "username"
+    db.server_ssh_password = "mypwd"
+    db.server_ssh_key = "/path/to/ssh/key"
+    
+    # other options for resource
+    ...
+    
+  end
+  ...    
+end
+````
+
+* perform backup - run script `backup peform` from the backup server
+```
+backup-remote perform -t my_backup
+```
+
+* it will connect to the remote server by SSH and run command remotely which creates a backup file
+* then it downloads the archive file from the remote server to the backup server
+* finally, it performs all operations with backup file, like storing file to storages, etc. as gem backup does.
+
+
+
+
+It uses SSHKit to connect to server by SSH.
+
+
+
+# Options
+
+Options for SSH connection:
+* server_host - host name or IP
+* server_ssh_user - user name to connect by SSH
+* server_ssh_password - not used if server_ssh-key is provided
+* server_ssh_key - (optional) - path to ssh key
+
+
+
+# Archives
+
+## Archive files on a remote server
+
+* Use RemoteArchive
+
+```
+```
+
+
+# Databases
+
+## Backup database on a remote server
+
+* Now it is implemented the following databases:
+* RemoteMySQL
+
+### RemoteMySQL
+
+```
+Model.new(:my_backup, 'My Backup') do
+  database RemoteMySQL do |db|
+    # options for server
+    db.server_host = "server1.com"
+    db.server_ssh_user = "username"
+    db.server_ssh_password = "mypwd"
+    db.server_ssh_key = "/path/to/ssh/key"
+    
+    ### options for MySQL 
+    # see http://backup.github.io/backup/v4/database-mysql/
+    
+    ...
+  end
+  ..
+end  
+    
+````
+
+# Custom backup command
+
+* Run custom command to create a backup archive on a remote server
+
+
+# Backup gem
+[Installation]:  http://backup.github.io/backup/v4/installation
+[Release Notes]: http://backup.github.io/backup/v4/release-notes
+[Documentation]: http://backup.github.io/backup/v4

data/bin/backup-remote

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require File.expand_path("../../lib/backup", __FILE__)
+Backup::CLI.start

data/lib/backup.rb

@@ -0,0 +1,155 @@
+# encoding: utf-8
+
+# Load Ruby Core Libraries
+require 'time'
+require 'fileutils'
+require 'tempfile'
+require 'syslog'
+require 'yaml'
+require 'etc'
+require 'forwardable'
+require 'thread'
+
+require 'open4'
+require 'thor'
+require 'shellwords'
+
+require 'excon'
+# Include response.inspect in error messages.
+Excon.defaults[:debug_response] = true
+# Excon should not retry failed requests. We handle that.
+Excon.defaults[:middlewares].delete(Excon::Middleware::Idempotent)
+
+##
+# The Backup Ruby Gem
+module Backup
+
+  ##
+  # Backup's internal paths
+  LIBRARY_PATH       = File.join(File.dirname(__FILE__), 'backup')
+  STORAGE_PATH       = File.join(LIBRARY_PATH, 'storage')
+  SYNCER_PATH        = File.join(LIBRARY_PATH, 'syncer')
+  DATABASE_PATH      = File.join(LIBRARY_PATH, 'database')
+  COMPRESSOR_PATH    = File.join(LIBRARY_PATH, 'compressor')
+  ENCRYPTOR_PATH     = File.join(LIBRARY_PATH, 'encryptor')
+  NOTIFIER_PATH      = File.join(LIBRARY_PATH, 'notifier')
+  TEMPLATE_PATH      = File.expand_path('../../templates', __FILE__)
+
+  ##
+  # Autoload Backup storage files
+  module Storage
+    autoload :Base,       File.join(STORAGE_PATH, 'base')
+    autoload :Cycler,     File.join(STORAGE_PATH, 'cycler')
+    autoload :S3,         File.join(STORAGE_PATH, 's3')
+    autoload :CloudFiles, File.join(STORAGE_PATH, 'cloud_files')
+    autoload :Ninefold,   File.join(STORAGE_PATH, 'ninefold')
+    autoload :Dropbox,    File.join(STORAGE_PATH, 'dropbox')
+    autoload :FTP,        File.join(STORAGE_PATH, 'ftp')
+    autoload :SFTP,       File.join(STORAGE_PATH, 'sftp')
+    autoload :SCP,        File.join(STORAGE_PATH, 'scp')
+    autoload :RSync,      File.join(STORAGE_PATH, 'rsync')
+    autoload :Local,      File.join(STORAGE_PATH, 'local')
+    autoload :Qiniu,      File.join(STORAGE_PATH, 'qiniu')
+  end
+
+  ##
+  # Autoload Backup syncer files
+  module Syncer
+    autoload :Base, File.join(SYNCER_PATH, 'base')
+    module Cloud
+      autoload :Base,       File.join(SYNCER_PATH, 'cloud', 'base')
+      autoload :LocalFile,  File.join(SYNCER_PATH, 'cloud', 'local_file')
+      autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud', 'cloud_files')
+      autoload :S3,         File.join(SYNCER_PATH, 'cloud', 's3')
+    end
+    module RSync
+      autoload :Base,  File.join(SYNCER_PATH, 'rsync', 'base')
+      autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
+      autoload :Push,  File.join(SYNCER_PATH, 'rsync', 'push')
+      autoload :Pull,  File.join(SYNCER_PATH, 'rsync', 'pull')
+    end
+  end
+
+  ##
+  # Autoload Backup database files
+  module Database
+    autoload :Base,       File.join(DATABASE_PATH, 'base')
+    autoload :MySQL,      File.join(DATABASE_PATH, 'mysql')
+    autoload :PostgreSQL, File.join(DATABASE_PATH, 'postgresql')
+    autoload :MongoDB,    File.join(DATABASE_PATH, 'mongodb')
+    autoload :Redis,      File.join(DATABASE_PATH, 'redis')
+    autoload :Riak,       File.join(DATABASE_PATH, 'riak')
+    autoload :OpenLDAP,   File.join(DATABASE_PATH, 'openldap')
+    autoload :SQLite,     File.join(DATABASE_PATH, 'sqlite')
+
+    autoload :RemoteMySQL,      File.join(DATABASE_PATH, 'remote_mysql')
+  end
+
+  ##
+  # Autoload compressor files
+  module Compressor
+    autoload :Base,   File.join(COMPRESSOR_PATH, 'base')
+    autoload :Gzip,   File.join(COMPRESSOR_PATH, 'gzip')
+    autoload :Bzip2,  File.join(COMPRESSOR_PATH, 'bzip2')
+    autoload :Custom, File.join(COMPRESSOR_PATH, 'custom')
+  end
+
+  ##
+  # Autoload encryptor files
+  module Encryptor
+    autoload :Base,    File.join(ENCRYPTOR_PATH, 'base')
+    autoload :OpenSSL, File.join(ENCRYPTOR_PATH, 'open_ssl')
+    autoload :GPG,     File.join(ENCRYPTOR_PATH, 'gpg')
+  end
+
+  ##
+  # Autoload notification files
+  module Notifier
+    autoload :Base,      File.join(NOTIFIER_PATH, 'base')
+    autoload :Mail,      File.join(NOTIFIER_PATH, 'mail')
+    autoload :Twitter,   File.join(NOTIFIER_PATH, 'twitter')
+    autoload :Campfire,  File.join(NOTIFIER_PATH, 'campfire')
+    autoload :Prowl,     File.join(NOTIFIER_PATH, 'prowl')
+    autoload :Hipchat,   File.join(NOTIFIER_PATH, 'hipchat')
+    autoload :PagerDuty, File.join(NOTIFIER_PATH, 'pagerduty')
+    autoload :Pushover,  File.join(NOTIFIER_PATH, 'pushover')
+    autoload :Slack,     File.join(NOTIFIER_PATH, 'slack')
+    autoload :HttpPost,  File.join(NOTIFIER_PATH, 'http_post')
+    autoload :Nagios,    File.join(NOTIFIER_PATH, 'nagios')
+    autoload :FlowDock,  File.join(NOTIFIER_PATH, 'flowdock')
+    autoload :Zabbix,    File.join(NOTIFIER_PATH, 'zabbix')
+    autoload :DataDog,   File.join(NOTIFIER_PATH, 'datadog')
+    autoload :Ses,       File.join(NOTIFIER_PATH, 'ses')
+    autoload :Command,   File.join(NOTIFIER_PATH, 'command')
+  end
+
+  ##
+  # Autoload Remote files
+  LIB_REMOTE_PATH      = File.join(LIBRARY_PATH, 'remote')
+
+  module Remote
+    autoload :Command,      File.join(LIB_REMOTE_PATH, 'command')
+  end
+
+
+  ##
+  # Require Backup base files
+  %w{
+    errors
+    logger
+    utilities
+    archive
+    binder
+    cleaner
+    model
+    config
+    cli
+    package
+    packager
+    pipeline
+    splitter
+    template
+    version
+  }.each {|lib| require File.join(LIBRARY_PATH, lib) }
+
+end

data/lib/backup/archive.rb

@@ -0,0 +1,170 @@
+# encoding: utf-8
+
+module Backup
+  class Archive
+    class Error < Backup::Error; end
+
+    include Utilities::Helpers
+    attr_reader :name, :options
+
+    ##
+    # Adds a new Archive to a Backup Model.
+    #
+    #     Backup::Model.new(:my_backup, 'My Backup') do
+    #       archive :my_archive do |archive|
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # All paths added using `add` or `exclude` will be expanded to their
+    # full paths from the root of the filesystem. Files will be added to
+    # the tar archive using these full paths, and their leading `/` will
+    # be preserved (using tar's `-P` option).
+    #
+    #     /path/to/pwd/path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # When a `root` path is given, paths to add/exclude are taken as
+    # relative to the `root` path, unless given as absolute paths.
+    #
+    #     Backup::Model.new(:my_backup, 'My Backup') do
+    #       archive :my_archive do |archive|
+    #         archive.root '~/my_data'
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # This directs `tar` to change directories to the `root` path to create
+    # the archive. Unless paths were given as absolute, the paths within the
+    # archive will be relative to the `root` path.
+    #
+    #     path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # For absolute paths added to this archive, the leading `/` will be
+    # preserved. Take note that when archives are extracted, leading `/` are
+    # stripped by default, so care must be taken when extracting archives with
+    # mixed relative/absolute paths.
+    def initialize(model, name, &block)
+      @model   = model
+      @name    = name.to_s
+      @options = {
+        :sudo        => false,
+        :root        => false,
+        :paths       => [],
+        :excludes    => [],
+        :tar_options => ''
+      }
+      DSL.new(@options).instance_eval(&block)
+    end
+
+    def perform!
+      Logger.info "Creating Archive '#{ name }'..."
+
+      path = File.join(Config.tmp_path, @model.trigger, 'archives')
+      FileUtils.mkdir_p(path)
+
+      pipeline = Pipeline.new
+      with_files_from(paths_to_package) do |files_from|
+        pipeline.add(
+          "#{ tar_command } #{ tar_options } -cPf -#{ tar_root } " +
+          "#{ paths_to_exclude } #{ files_from }",
+          tar_success_codes
+        )
+
+        extension = 'tar'
+        @model.compressor.compress_with do |command, ext|
+          pipeline << command
+          extension << ext
+        end if @model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(path, "#{ name }.#{ extension }") }'"
+        pipeline.run
+      end
+
+      if pipeline.success?
+        Logger.info "Archive '#{ name }' Complete!"
+      else
+        raise Error, "Failed to Create Archive '#{ name }'\n" +
+            pipeline.error_messages
+      end
+    end
+
+    private
+
+    def tar_command
+      tar = utility(:tar)
+      options[:sudo] ? "#{ utility(:sudo) } -n #{ tar }" : tar
+    end
+
+    def tar_root
+      options[:root] ? " -C '#{ File.expand_path(options[:root]) }'" : ''
+    end
+
+    def paths_to_package
+      options[:paths].map {|path| prepare_path(path) }
+    end
+
+    def with_files_from(paths)
+      tmpfile = Tempfile.new('backup-archive-paths')
+      paths.each {|path| tmpfile.puts path }
+      tmpfile.close
+      yield "-T '#{ tmpfile.path }'"
+    ensure
+      tmpfile.delete
+    end
+
+    def paths_to_exclude
+      options[:excludes].map {|path|
+        "--exclude='#{ prepare_path(path) }'"
+      }.join(' ')
+    end
+
+    def prepare_path(path)
+      options[:root] ? path : File.expand_path(path)
+    end
+
+    def tar_options
+      args = options[:tar_options]
+      gnu_tar? ? "--ignore-failed-read #{ args }".strip : args
+    end
+
+    def tar_success_codes
+      gnu_tar? ? [0, 1] : [0]
+    end
+
+    class DSL
+      def initialize(options)
+        @options = options
+      end
+
+      def use_sudo(val = true)
+        @options[:sudo] = val
+      end
+
+      def root(path)
+        @options[:root] = path
+      end
+
+      def add(path)
+        @options[:paths] << path
+      end
+
+      def exclude(path)
+        @options[:excludes] << path
+      end
+
+      def tar_options(opts)
+        @options[:tar_options] = opts
+      end
+    end
+
+  end
+end