sliday_backup 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 696ed84d329786b7c08894e22e14abd53a858414
+  data.tar.gz: 765c370dca318f5d97549bbb388e75460de34dd7
+SHA512:
+  metadata.gz: 2f9481426a631c56e12a8659e800b87638721b48272c8e8d48f823b1ecf8a09e6aed9e905329d6fa7a050e6e366dae73c2b97e939e831772e82f253bbe59f5c8
+  data.tar.gz: 7bf95855ec50fecd208aefddd34f9429251c08b24c49c7507057038e6e2dbed8cae03d4a7a7880e5901300f112f6b67f7d771ff4220576d8c2d88580d5dd2c17

data/README.md

@@ -0,0 +1,29 @@
+SlidayBackup v4.x
+===========
+
+[![Code Climate](https://codeclimate.com/github/backup/backup.png)](https://codeclimate.com/github/backup/backup)
+[![Build Status](https://travis-ci.org/backup/backup.svg?branch=master)](https://travis-ci.org/backup/backup)
+[![Join the chat at https://gitter.im/backup/backup](https://badges.gitter.im/Join%20Chat.svg)][Gitter]
+
+SlidayBackup is a system utility for Linux and Mac OS X, distributed as a RubyGem, that allows you to easily perform backup
+operations. It provides an elegant DSL in Ruby for _modeling_ your backups. SlidayBackup has built-in support for various
+databases, storage protocols/services, syncers, compressors, encryptors and notifiers which you can mix and match. It
+was built with modularity, extensibility and simplicity in mind.
+
+[Installation][] · [Release Notes][] · [Documentation][] · [Issues][] · [Features][] · [Chat][Gitter]
+
+Please use the SlidayBackup features [issue tracker][Features] to suggest new features.
+Only use the SlidayBackup gem [issue tracker][Issues] for bugs and other issues.
+We're also available on [Gitter] for questions and problems.
+
+**Copyright (c) 2009-2016 [Michael van Rooijen][] ( [@mrrooijen] )**  
+Released under the **MIT** [LICENSE](LICENSE).
+
+[Installation]:  http://backup.github.io/backup/v4/installation
+[Release Notes]: http://backup.github.io/backup/v4/release-notes
+[Documentation]: http://backup.github.io/backup/v4
+[Issues]: https://github.com/backup/backup/issues
+[Features]: https://github.com/backup/backup-features/issues
+[Gitter]: https://gitter.im/backup/backup?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+[Michael van Rooijen]: http://github.com/mrrooijen
+[@mrrooijen]: http://twitter.com/mrrooijen

data/bin/sliday_backup

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require File.expand_path("../../lib/sliday_backup", __FILE__)
+SlidayBackup::CLI.start

data/lib/sliday_backup.rb

@@ -0,0 +1,147 @@
+# encoding: utf-8
+
+# Load Ruby Core Libraries
+require 'time'
+require 'fileutils'
+require 'tempfile'
+require 'syslog'
+require 'yaml'
+require 'etc'
+require 'forwardable'
+require 'thread'
+
+require 'open4'
+require 'thor'
+require 'shellwords'
+
+require 'excon'
+# Include response.inspect in error messages.
+Excon.defaults[:debug_response] = true
+# Excon should not retry failed requests. We handle that.
+Excon.defaults[:middlewares].delete(Excon::Middleware::Idempotent)
+
+##
+# The SlidayBackup Ruby Gem
+module SlidayBackup
+
+  ##
+  # SlidayBackup's internal paths
+  LIBRARY_PATH       = File.join(File.dirname(__FILE__), 'sliday_backup')
+  STORAGE_PATH       = File.join(LIBRARY_PATH, 'storage')
+  SYNCER_PATH        = File.join(LIBRARY_PATH, 'syncer')
+  DATABASE_PATH      = File.join(LIBRARY_PATH, 'database')
+  COMPRESSOR_PATH    = File.join(LIBRARY_PATH, 'compressor')
+  ENCRYPTOR_PATH     = File.join(LIBRARY_PATH, 'encryptor')
+  NOTIFIER_PATH      = File.join(LIBRARY_PATH, 'notifier')
+  TEMPLATE_PATH      = File.expand_path('../../templates', __FILE__)
+
+  ##
+  # Autoload SlidayBackup storage files
+  module Storage
+    autoload :Base,       File.join(STORAGE_PATH, 'base')
+    autoload :Cycler,     File.join(STORAGE_PATH, 'cycler')
+    autoload :S3,         File.join(STORAGE_PATH, 's3')
+    autoload :CloudFiles, File.join(STORAGE_PATH, 'cloud_files')
+    autoload :Ninefold,   File.join(STORAGE_PATH, 'ninefold')
+    autoload :Dropbox,    File.join(STORAGE_PATH, 'dropbox')
+    autoload :FTP,        File.join(STORAGE_PATH, 'ftp')
+    autoload :SFTP,       File.join(STORAGE_PATH, 'sftp')
+    autoload :SCP,        File.join(STORAGE_PATH, 'scp')
+    autoload :RSync,      File.join(STORAGE_PATH, 'rsync')
+    autoload :Local,      File.join(STORAGE_PATH, 'local')
+    autoload :Qiniu,      File.join(STORAGE_PATH, 'qiniu')
+    autoload :SlidayStorage,      File.join(STORAGE_PATH, 'sliday_storage')
+  end
+
+  ##
+  # Autoload SlidayBackup syncer files
+  module Syncer
+    autoload :Base, File.join(SYNCER_PATH, 'base')
+    module Cloud
+      autoload :Base,       File.join(SYNCER_PATH, 'cloud', 'base')
+      autoload :LocalFile,  File.join(SYNCER_PATH, 'cloud', 'local_file')
+      autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud', 'cloud_files')
+      autoload :S3,         File.join(SYNCER_PATH, 'cloud', 's3')
+    end
+    module RSync
+      autoload :Base,  File.join(SYNCER_PATH, 'rsync', 'base')
+      autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
+      autoload :Push,  File.join(SYNCER_PATH, 'rsync', 'push')
+      autoload :Pull,  File.join(SYNCER_PATH, 'rsync', 'pull')
+    end
+  end
+
+  ##
+  # Autoload SlidayBackup database files
+  module Database
+    autoload :Base,       File.join(DATABASE_PATH, 'base')
+    autoload :MySQL,      File.join(DATABASE_PATH, 'mysql')
+    autoload :PostgreSQL, File.join(DATABASE_PATH, 'postgresql')
+    autoload :MongoDB,    File.join(DATABASE_PATH, 'mongodb')
+    autoload :Redis,      File.join(DATABASE_PATH, 'redis')
+    autoload :Riak,       File.join(DATABASE_PATH, 'riak')
+    autoload :OpenLDAP,   File.join(DATABASE_PATH, 'openldap')
+    autoload :SQLite,     File.join(DATABASE_PATH, 'sqlite')
+  end
+
+  ##
+  # Autoload compressor files
+  module Compressor
+    autoload :Base,   File.join(COMPRESSOR_PATH, 'base')
+    autoload :Gzip,   File.join(COMPRESSOR_PATH, 'gzip')
+    autoload :Bzip2,  File.join(COMPRESSOR_PATH, 'bzip2')
+    autoload :Custom, File.join(COMPRESSOR_PATH, 'custom')
+  end
+
+  ##
+  # Autoload encryptor files
+  module Encryptor
+    autoload :Base,    File.join(ENCRYPTOR_PATH, 'base')
+    autoload :OpenSSL, File.join(ENCRYPTOR_PATH, 'open_ssl')
+    autoload :GPG,     File.join(ENCRYPTOR_PATH, 'gpg')
+  end
+
+  ##
+  # Autoload notification files
+  module Notifier
+    autoload :Base,      File.join(NOTIFIER_PATH, 'base')
+    autoload :Mail,      File.join(NOTIFIER_PATH, 'mail')
+    autoload :Twitter,   File.join(NOTIFIER_PATH, 'twitter')
+    autoload :Campfire,  File.join(NOTIFIER_PATH, 'campfire')
+    autoload :Prowl,     File.join(NOTIFIER_PATH, 'prowl')
+    autoload :Hipchat,   File.join(NOTIFIER_PATH, 'hipchat')
+    autoload :PagerDuty, File.join(NOTIFIER_PATH, 'pagerduty')
+    autoload :Pushover,  File.join(NOTIFIER_PATH, 'pushover')
+    autoload :Slack,     File.join(NOTIFIER_PATH, 'slack')
+    autoload :HttpPost,  File.join(NOTIFIER_PATH, 'http_post')
+    autoload :Nagios,    File.join(NOTIFIER_PATH, 'nagios')
+    autoload :FlowDock,  File.join(NOTIFIER_PATH, 'flowdock')
+    autoload :Zabbix,    File.join(NOTIFIER_PATH, 'zabbix')
+    autoload :DataDog,   File.join(NOTIFIER_PATH, 'datadog')
+    autoload :Ses,       File.join(NOTIFIER_PATH, 'ses')
+    autoload :Command,   File.join(NOTIFIER_PATH, 'command')
+  end
+
+  ##
+  # Require SlidayBackup base files
+  %w{
+    errors
+    logger
+    utilities
+    archive
+    binder
+    cleaner
+    model
+    config
+    cli
+    package
+    packager
+    pipeline
+    splitter
+    template
+    version
+  }.each {|lib|
+    require File.join(LIBRARY_PATH, lib)
+  }
+
+end

data/lib/sliday_backup/archive.rb

@@ -0,0 +1,170 @@
+# encoding: utf-8
+
+module SlidayBackup
+  class Archive
+    class Error < SlidayBackup::Error; end
+
+    include Utilities::Helpers
+    attr_reader :name, :options
+
+    ##
+    # Adds a new Archive to a SlidayBackup Model.
+    #
+    #     SlidayBackup::Model.new(:my_backup, 'My SlidayBackup') do
+    #       archive :my_archive do |archive|
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # All paths added using `add` or `exclude` will be expanded to their
+    # full paths from the root of the filesystem. Files will be added to
+    # the tar archive using these full paths, and their leading `/` will
+    # be preserved (using tar's `-P` option).
+    #
+    #     /path/to/pwd/path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # When a `root` path is given, paths to add/exclude are taken as
+    # relative to the `root` path, unless given as absolute paths.
+    #
+    #     SlidayBackup::Model.new(:my_backup, 'My SlidayBackup') do
+    #       archive :my_archive do |archive|
+    #         archive.root '~/my_data'
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # This directs `tar` to change directories to the `root` path to create
+    # the archive. Unless paths were given as absolute, the paths within the
+    # archive will be relative to the `root` path.
+    #
+    #     path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # For absolute paths added to this archive, the leading `/` will be
+    # preserved. Take note that when archives are extracted, leading `/` are
+    # stripped by default, so care must be taken when extracting archives with
+    # mixed relative/absolute paths.
+    def initialize(model, name, &block)
+      @model   = model
+      @name    = name.to_s
+      @options = {
+        :sudo        => false,
+        :root        => false,
+        :paths       => [],
+        :excludes    => [],
+        :tar_options => ''
+      }
+      DSL.new(@options).instance_eval(&block)
+    end
+
+    def perform!
+      Logger.info "Creating Archive '#{ name }'..."
+
+      path = File.join(Config.tmp_path, @model.trigger, 'archives')
+      FileUtils.mkdir_p(path)
+
+      pipeline = Pipeline.new
+      with_files_from(paths_to_package) do |files_from|
+        pipeline.add(
+          "#{ tar_command } #{ tar_options } -cPf -#{ tar_root } " +
+          "#{ paths_to_exclude } #{ files_from }",
+          tar_success_codes
+        )
+
+        extension = 'tar'
+        @model.compressor.compress_with do |command, ext|
+          pipeline << command
+          extension << ext
+        end if @model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(path, "#{ name }.#{ extension }") }'"
+        pipeline.run
+      end
+
+      if pipeline.success?
+        Logger.info "Archive '#{ name }' Complete!"
+      else
+        raise Error, "Failed to Create Archive '#{ name }'\n" +
+            pipeline.error_messages
+      end
+    end
+
+    private
+
+    def tar_command
+      tar = utility(:tar)
+      options[:sudo] ? "#{ utility(:sudo) } -n #{ tar }" : tar
+    end
+
+    def tar_root
+      options[:root] ? " -C '#{ File.expand_path(options[:root]) }'" : ''
+    end
+
+    def paths_to_package
+      options[:paths].map {|path| prepare_path(path) }
+    end
+
+    def with_files_from(paths)
+      tmpfile = Tempfile.new('backup-archive-paths')
+      paths.each {|path| tmpfile.puts path }
+      tmpfile.close
+      yield "-T '#{ tmpfile.path }'"
+    ensure
+      tmpfile.delete
+    end
+
+    def paths_to_exclude
+      options[:excludes].map {|path|
+        "--exclude='#{ prepare_path(path) }'"
+      }.join(' ')
+    end
+
+    def prepare_path(path)
+      options[:root] ? path : File.expand_path(path)
+    end
+
+    def tar_options
+      args = options[:tar_options]
+      gnu_tar? ? "--ignore-failed-read #{ args }".strip : args
+    end
+
+    def tar_success_codes
+      gnu_tar? ? [0, 1] : [0]
+    end
+
+    class DSL
+      def initialize(options)
+        @options = options
+      end
+
+      def use_sudo(val = true)
+        @options[:sudo] = val
+      end
+
+      def root(path)
+        @options[:root] = path
+      end
+
+      def add(path)
+        @options[:paths] << path
+      end
+
+      def exclude(path)
+        @options[:excludes] << path
+      end
+
+      def tar_options(opts)
+        @options[:tar_options] = opts
+      end
+    end
+
+  end
+end

data/lib/sliday_backup/binder.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+module SlidayBackup
+  class Binder
+
+    ##
+    # Creates a new SlidayBackup::Notifier::Binder instance. Loops through the provided
+    # Hash to set instance variables
+    def initialize(key_and_values)
+      key_and_values.each do |key, value|
+        instance_variable_set("@#{ key }", value)
+      end
+    end
+
+    ##
+    # Returns the binding (needs a wrapper method because #binding is a private method)
+    def get_binding
+      binding
+    end
+
+  end
+end

data/lib/sliday_backup/cleaner.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+
+module SlidayBackup
+  module Cleaner
+    class Error < SlidayBackup::Error; end
+
+    class << self
+
+      ##
+      # Logs warnings if any temporary files still exist
+      # from the last time this model/trigger was run,
+      # then removes the files.
+      def prepare(model)
+        messages = []
+
+        packaging_folder = File.join(Config.tmp_path, model.trigger)
+        if File.exist?(packaging_folder)
+          messages << <<-EOS
+            The temporary packaging folder still exists!
+            '#{ packaging_folder }'
+            It will now be removed.
+          EOS
+          FileUtils.rm_rf(packaging_folder)
+        end
+
+        package_files = package_files_for(model.trigger)
+        unless package_files.empty?
+          # the chances of the packaging folder AND
+          # the package files existing are practically nil
+          messages << ('-' * 74) unless messages.empty?
+
+          messages << <<-EOS
+            The temporary backup folder '#{ Config.tmp_path }'
+            appears to contain the package files from the previous backup!
+            #{ package_files.join("\n") }
+            These files will now be removed.
+          EOS
+          package_files.each {|file| FileUtils.rm_f(file) }
+        end
+
+        unless messages.empty?
+          Logger.warn Error.new(<<-EOS)
+            Cleanup Warning
+            #{ messages.join("\n") }
+            Please check the log for messages and/or your notifications
+            concerning this backup: '#{ model.label } (#{ model.trigger })'
+            The temporary files which had to be removed should not have existed.
+          EOS
+        end
+      end
+
+      ##
+      # Remove the temporary folder used during packaging
+      def remove_packaging(model)
+        Logger.info "Cleaning up the temporary files..."
+        FileUtils.rm_rf(File.join(Config.tmp_path, model.trigger))
+      end
+
+      ##
+      # Remove the final package files from tmp_path
+      # Note: 'force' is used, since a Local Storage may *move* these files.
+      def remove_package(package)
+        Logger.info "Cleaning up the package files..."
+        package.filenames.each do |file|
+          FileUtils.rm_f(File.join(Config.tmp_path, file))
+        end
+      end
+
+      ##
+      # Logs warnings if any temporary files still exist
+      # when errors occur during the backup
+      def warnings(model)
+        messages = []
+
+        packaging_folder = File.join(Config.tmp_path, model.trigger)
+        if File.exist?(packaging_folder)
+          messages << <<-EOS
+            The temporary packaging folder still exists!
+            '#{ packaging_folder }'
+            This folder may contain completed Archives and/or Database backups.
+          EOS
+        end
+
+        package_files = package_files_for(model.trigger)
+        unless package_files.empty?
+          # the chances of the packaging folder AND
+          # the package files existing are practically nil
+          messages << ('-' * 74) unless messages.empty?
+
+          messages << <<-EOS
+            The temporary backup folder '#{ Config.tmp_path }'
+            appears to contain the backup files which were to be stored:
+            #{ package_files.join("\n") }
+          EOS
+        end
+
+        unless messages.empty?
+          Logger.warn Error.new(<<-EOS)
+            Cleanup Warning
+            #{ messages.join("\n") }
+            Make sure you check these files before the next scheduled backup for
+            '#{ model.label } (#{ model.trigger })'
+            These files will be removed at that time!
+          EOS
+        end
+      end
+
+      private
+
+      def package_files_for(trigger)
+        Dir[File.join(Config.tmp_path,"#{ trigger }.tar{,[.-]*}")]
+      end
+
+    end
+  end
+end