ext_backup 5.0.0.beta.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9af003c3024a1c1a8cfa1a4ab31a4dce2fc76d0ff86d1c48add68ce4902bf4ea
+  data.tar.gz: a7876332cefc3ded8aced38e9f9a868b35eda17347a478d920d01409cf09ce3d
+SHA512:
+  metadata.gz: 913b6b7ed4ace64232889ccd791320b818dd7decebd44952c454e675819c42e4416a0e465d2f192bed8d77e6d5cf8b21f18cc09cf36faf0b252a36f0e3aaa862
+  data.tar.gz: 3ab0d83a7c43295a2cd4d35ed009101809afddab0adbe877078b8ee376c39b920a7f69ac1b3b076c369849fdb950332d722b5eba31c0472a23b67251cb59a0cb

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2009-2017 Michael van Rooijen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+Backup
+======
+
+[![Code Climate](https://codeclimate.com/github/backup/backup.svg)](https://codeclimate.com/github/backup/backup)
+[![Build Status](https://travis-ci.org/backup/backup.svg?branch=master)](https://travis-ci.org/backup/backup)
+[![Join the chat at https://gitter.im/backup/backup](https://badges.gitter.im/Join%20Chat.svg)][Gitter]
+
+> This project is now in maintenance. No new features are planned.
+
+Backup is a system utility for Linux and Mac OS X, distributed as a RubyGem, that allows you to easily perform backup
+operations. It provides an elegant DSL in Ruby for _modeling_ your backups. Backup has built-in support for various
+databases, storage protocols/services, syncers, compressors, encryptors and notifiers which you can mix and match. It
+was built with modularity, extensibility and simplicity in mind.
+
+[Installation][] · [Release Notes][] · [Documentation][] · [Issues][] ·  [Chat][Gitter]
+
+## Project Status: Maintenance-Only ##
+
+This project is not under active development, although we will continue to provide support for current users, and at least one more maintenance release: version 5.0. The version 5.0 release will include support for Ruby 2.4, and various other fixes. Future releases of Backup will only include bug fixes.
+
+If you use this project and would like to develop it further, please introduce yourself on the [maintainers wanted][Maintainers wanted] ticket.
+
+**Copyright (c) 2009-2017 [Michael van Rooijen][] ( [@mrrooijen] )**  
+Released under the **MIT** [LICENSE](LICENSE).
+
+[Installation]:  http://backup.github.io/backup/v4/installation
+[Release Notes]: http://backup.github.io/backup/v4/release-notes
+[Documentation]: http://backup.github.io/backup/v4
+[Issues]: https://github.com/backup/backup/issues
+[Gitter]: https://gitter.im/backup/backup?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+[Maintainers wanted]: https://github.com/backup/backup/issues/803
+[Michael van Rooijen]: http://github.com/mrrooijen
+[@mrrooijen]: http://twitter.com/mrrooijen

data/bin/backup

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require File.expand_path("../../lib/backup", __FILE__)
+Backup::CLI.start

data/bin/docker_test

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -eu
+
+export BUNDLE_PATH=/tmp/docker/bundle
+export BUNDLE_JOBS=4
+
+case "${1-}" in
+prepare)
+  bundle check || bundle install
+  ;;
+rspec)
+  bundle exec rspec spec
+  ;;
+integration)
+  bundle exec rspec integration/acceptance/
+  ;;
+console)
+  bash
+  ;;
+*)
+  echo "Task not found"
+  exit 1
+  ;;
+esac

data/lib/backup.rb

@@ -0,0 +1,140 @@
+# Load Ruby Core Libraries
+require "time"
+require "fileutils"
+require "tempfile"
+require "syslog"
+require "yaml"
+require "etc"
+require "forwardable"
+require "thread"
+
+require "open4"
+require "thor"
+require "shellwords"
+
+require "excon"
+# Include response.inspect in error messages.
+Excon.defaults[:debug_response] = true
+# Excon should not retry failed requests. We handle that.
+Excon.defaults[:middlewares].delete(Excon::Middleware::Idempotent)
+
+##
+# The Backup Ruby Gem
+module Backup
+  ##
+  # Backup's internal paths
+  LIBRARY_PATH       = File.join(File.dirname(__FILE__), "backup")
+  STORAGE_PATH       = File.join(LIBRARY_PATH, "storage")
+  SYNCER_PATH        = File.join(LIBRARY_PATH, "syncer")
+  DATABASE_PATH      = File.join(LIBRARY_PATH, "database")
+  COMPRESSOR_PATH    = File.join(LIBRARY_PATH, "compressor")
+  ENCRYPTOR_PATH     = File.join(LIBRARY_PATH, "encryptor")
+  NOTIFIER_PATH      = File.join(LIBRARY_PATH, "notifier")
+  TEMPLATE_PATH      = File.expand_path("../../templates", __FILE__)
+
+  ##
+  # Autoload Backup storage files
+  module Storage
+    autoload :Base,       File.join(STORAGE_PATH, "base")
+    autoload :Cycler,     File.join(STORAGE_PATH, "cycler")
+    autoload :S3,         File.join(STORAGE_PATH, "s3")
+    autoload :CloudFiles, File.join(STORAGE_PATH, "cloud_files")
+    autoload :Ninefold,   File.join(STORAGE_PATH, "ninefold")
+    autoload :Dropbox,    File.join(STORAGE_PATH, "dropbox")
+    autoload :FTP,        File.join(STORAGE_PATH, "ftp")
+    autoload :SFTP,       File.join(STORAGE_PATH, "sftp")
+    autoload :SCP,        File.join(STORAGE_PATH, "scp")
+    autoload :RSync,      File.join(STORAGE_PATH, "rsync")
+    autoload :Local,      File.join(STORAGE_PATH, "local")
+    autoload :Qiniu,      File.join(STORAGE_PATH, "qiniu")
+  end
+
+  ##
+  # Autoload Backup syncer files
+  module Syncer
+    autoload :Base, File.join(SYNCER_PATH, "base")
+    module Cloud
+      autoload :Base,       File.join(SYNCER_PATH, "cloud", "base")
+      autoload :LocalFile,  File.join(SYNCER_PATH, "cloud", "local_file")
+      autoload :CloudFiles, File.join(SYNCER_PATH, "cloud", "cloud_files")
+      autoload :S3,         File.join(SYNCER_PATH, "cloud", "s3")
+    end
+    module RSync
+      autoload :Base,  File.join(SYNCER_PATH, "rsync", "base")
+      autoload :Local, File.join(SYNCER_PATH, "rsync", "local")
+      autoload :Push,  File.join(SYNCER_PATH, "rsync", "push")
+      autoload :Pull,  File.join(SYNCER_PATH, "rsync", "pull")
+    end
+  end
+
+  ##
+  # Autoload Backup database files
+  module Database
+    autoload :Base,       File.join(DATABASE_PATH, "base")
+    autoload :MySQL,      File.join(DATABASE_PATH, "mysql")
+    autoload :PostgreSQL, File.join(DATABASE_PATH, "postgresql")
+    autoload :MongoDB,    File.join(DATABASE_PATH, "mongodb")
+    autoload :Redis,      File.join(DATABASE_PATH, "redis")
+    autoload :Riak,       File.join(DATABASE_PATH, "riak")
+    autoload :OpenLDAP,   File.join(DATABASE_PATH, "openldap")
+    autoload :SQLite,     File.join(DATABASE_PATH, "sqlite")
+  end
+
+  ##
+  # Autoload compressor files
+  module Compressor
+    autoload :Base,   File.join(COMPRESSOR_PATH, "base")
+    autoload :Gzip,   File.join(COMPRESSOR_PATH, "gzip")
+    autoload :Bzip2,  File.join(COMPRESSOR_PATH, "bzip2")
+    autoload :Custom, File.join(COMPRESSOR_PATH, "custom")
+  end
+
+  ##
+  # Autoload encryptor files
+  module Encryptor
+    autoload :Base,    File.join(ENCRYPTOR_PATH, "base")
+    autoload :OpenSSL, File.join(ENCRYPTOR_PATH, "open_ssl")
+    autoload :GPG,     File.join(ENCRYPTOR_PATH, "gpg")
+  end
+
+  ##
+  # Autoload notification files
+  module Notifier
+    autoload :Base,      File.join(NOTIFIER_PATH, "base")
+    autoload :Mail,      File.join(NOTIFIER_PATH, "mail")
+    autoload :Twitter,   File.join(NOTIFIER_PATH, "twitter")
+    autoload :Campfire,  File.join(NOTIFIER_PATH, "campfire")
+    autoload :Prowl,     File.join(NOTIFIER_PATH, "prowl")
+    autoload :Hipchat,   File.join(NOTIFIER_PATH, "hipchat")
+    autoload :PagerDuty, File.join(NOTIFIER_PATH, "pagerduty")
+    autoload :Pushover,  File.join(NOTIFIER_PATH, "pushover")
+    autoload :Slack,     File.join(NOTIFIER_PATH, "slack")
+    autoload :HttpPost,  File.join(NOTIFIER_PATH, "http_post")
+    autoload :Nagios,    File.join(NOTIFIER_PATH, "nagios")
+    autoload :FlowDock,  File.join(NOTIFIER_PATH, "flowdock")
+    autoload :Zabbix,    File.join(NOTIFIER_PATH, "zabbix")
+    autoload :DataDog,   File.join(NOTIFIER_PATH, "datadog")
+    autoload :Ses,       File.join(NOTIFIER_PATH, "ses")
+    autoload :Command,   File.join(NOTIFIER_PATH, "command")
+  end
+
+  ##
+  # Require Backup base files
+  %w[
+    errors
+    logger
+    utilities
+    archive
+    binder
+    cleaner
+    model
+    config
+    cli
+    package
+    packager
+    pipeline
+    splitter
+    template
+    version
+  ].each { |lib| require File.join(LIBRARY_PATH, lib) }
+end

data/lib/backup/archive.rb

@@ -0,0 +1,169 @@
+module Backup
+  class Archive
+    class Error < Backup::Error; end
+
+    include Utilities::Helpers
+    attr_reader :name, :options
+
+    ##
+    # Adds a new Archive to a Backup Model.
+    #
+    #     Backup::Model.new(:my_backup, 'My Backup') do
+    #       archive :my_archive do |archive|
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # All paths added using `add` or `exclude` will be expanded to their
+    # full paths from the root of the filesystem. Files will be added to
+    # the tar archive using these full paths, and their leading `/` will
+    # be preserved (using tar's `-P` option).
+    #
+    #     /path/to/pwd/path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # When a `root` path is given, paths to add/exclude are taken as
+    # relative to the `root` path, unless given as absolute paths.
+    #
+    #     Backup::Model.new(:my_backup, 'My Backup') do
+    #       archive :my_archive do |archive|
+    #         archive.root '~/my_data'
+    #         archive.add 'path/to/archive'
+    #         archive.add '/another/path/to/archive'
+    #         archive.exclude 'path/to/exclude'
+    #         archive.exclude '/another/path/to/exclude'
+    #       end
+    #     end
+    #
+    # This directs `tar` to change directories to the `root` path to create
+    # the archive. Unless paths were given as absolute, the paths within the
+    # archive will be relative to the `root` path.
+    #
+    #     path/to/archive/...
+    #     /another/path/to/archive/...
+    #
+    # For absolute paths added to this archive, the leading `/` will be
+    # preserved. Take note that when archives are extracted, leading `/` are
+    # stripped by default, so care must be taken when extracting archives with
+    # mixed relative/absolute paths.
+    def initialize(model, name, &block)
+      @model   = model
+      @name    = name.to_s
+      @options = {
+        sudo: false,
+        root: false,
+        paths: [],
+        excludes: [],
+        tar_options: ""
+      }
+      DSL.new(@options).instance_eval(&block)
+    end
+
+    def perform!
+      Logger.info "Creating Archive '#{name}'..."
+
+      path = File.join(Config.tmp_path, @model.trigger, "archives")
+      FileUtils.mkdir_p(path)
+
+      pipeline = Pipeline.new
+      with_files_from(paths_to_package) do |files_from|
+        pipeline.add(
+          "#{tar_command} #{tar_options} -cPf -#{tar_root} " \
+            "#{paths_to_exclude} #{files_from}",
+          tar_success_codes
+        )
+
+        extension = "tar"
+        if @model.compressor
+          @model.compressor.compress_with do |command, ext|
+            pipeline << command
+            extension << ext
+          end
+        end
+
+        pipeline << "#{utility(:cat)} > " \
+          "'#{File.join(path, "#{name}.#{extension}")}'"
+        pipeline.run
+      end
+
+      if pipeline.success?
+        Logger.info "Archive '#{name}' Complete!"
+      else
+        raise Error, "Failed to Create Archive '#{name}'\n" +
+          pipeline.error_messages
+      end
+    end
+
+    private
+
+    def tar_command
+      tar = utility(:tar)
+      options[:sudo] ? "#{utility(:sudo)} -n #{tar}" : tar
+    end
+
+    def tar_root
+      options[:root] ? " -C '#{File.expand_path(options[:root])}'" : ""
+    end
+
+    def paths_to_package
+      options[:paths].map { |path| prepare_path(path) }
+    end
+
+    def with_files_from(paths)
+      tmpfile = Tempfile.new("backup-archive-paths")
+      paths.each { |path| tmpfile.puts path }
+      tmpfile.close
+      yield "-T '#{tmpfile.path}'"
+    ensure
+      tmpfile.delete
+    end
+
+    def paths_to_exclude
+      options[:excludes].map do |path|
+        "--exclude='#{prepare_path(path)}'"
+      end.join(" ")
+    end
+
+    def prepare_path(path)
+      options[:root] ? path : File.expand_path(path)
+    end
+
+    def tar_options
+      args = options[:tar_options]
+      gnu_tar? ? "--ignore-failed-read #{args}".strip : args
+    end
+
+    def tar_success_codes
+      gnu_tar? ? [0, 1] : [0]
+    end
+
+    class DSL
+      def initialize(options)
+        @options = options
+      end
+
+      def use_sudo(val = true)
+        @options[:sudo] = val
+      end
+
+      def root(path)
+        @options[:root] = path
+      end
+
+      def add(path)
+        @options[:paths] << path
+      end
+
+      def exclude(path)
+        @options[:excludes] << path
+      end
+
+      def tar_options(opts)
+        @options[:tar_options] = opts
+      end
+    end
+  end
+end

data/lib/backup/binder.rb

@@ -0,0 +1,18 @@
+module Backup
+  class Binder
+    ##
+    # Creates a new Backup::Notifier::Binder instance. Loops through the provided
+    # Hash to set instance variables
+    def initialize(key_and_values)
+      key_and_values.each do |key, value|
+        instance_variable_set("@#{key}", value)
+      end
+    end
+
+    ##
+    # Returns the binding (needs a wrapper method because #binding is a private method)
+    def get_binding
+      binding
+    end
+  end
+end

data/lib/backup/cleaner.rb

@@ -0,0 +1,112 @@
+module Backup
+  module Cleaner
+    class Error < Backup::Error; end
+
+    class << self
+      ##
+      # Logs warnings if any temporary files still exist
+      # from the last time this model/trigger was run,
+      # then removes the files.
+      def prepare(model)
+        messages = []
+
+        packaging_folder = File.join(Config.tmp_path, model.trigger)
+        if File.exist?(packaging_folder)
+          messages << <<-EOS
+            The temporary packaging folder still exists!
+            '#{packaging_folder}'
+            It will now be removed.
+          EOS
+          FileUtils.rm_rf(packaging_folder)
+        end
+
+        package_files = package_files_for(model.trigger)
+        unless package_files.empty?
+          # the chances of the packaging folder AND
+          # the package files existing are practically nil
+          messages << ("-" * 74) unless messages.empty?
+
+          messages << <<-EOS
+            The temporary backup folder '#{Config.tmp_path}'
+            appears to contain the package files from the previous backup!
+            #{package_files.join("\n")}
+            These files will now be removed.
+          EOS
+          package_files.each { |file| FileUtils.rm_f(file) }
+        end
+
+        unless messages.empty?
+          Logger.warn Error.new(<<-EOS)
+            Cleanup Warning
+            #{messages.join("\n")}
+            Please check the log for messages and/or your notifications
+            concerning this backup: '#{model.label} (#{model.trigger})'
+            The temporary files which had to be removed should not have existed.
+          EOS
+        end
+      end
+
+      ##
+      # Remove the temporary folder used during packaging
+      def remove_packaging(model)
+        Logger.info "Cleaning up the temporary files..."
+        FileUtils.rm_rf(File.join(Config.tmp_path, model.trigger))
+      end
+
+      ##
+      # Remove the final package files from tmp_path
+      # Note: 'force' is used, since a Local Storage may *move* these files.
+      def remove_package(package)
+        Logger.info "Cleaning up the package files..."
+        package.filenames.each do |file|
+          FileUtils.rm_f(File.join(Config.tmp_path, file))
+        end
+      end
+
+      ##
+      # Logs warnings if any temporary files still exist
+      # when errors occur during the backup
+      def warnings(model)
+        messages = []
+
+        packaging_folder = File.join(Config.tmp_path, model.trigger)
+        if File.exist?(packaging_folder)
+          messages << <<-EOS
+            The temporary packaging folder still exists!
+            '#{packaging_folder}'
+            This folder may contain completed Archives and/or Database backups.
+          EOS
+        end
+
+        package_files = package_files_for(model.trigger)
+        unless package_files.empty?
+          # the chances of the packaging folder AND
+          # the package files existing are practically nil
+          messages << ("-" * 74) unless messages.empty?
+
+          messages << <<-EOS
+            The temporary backup folder '#{Config.tmp_path}'
+            appears to contain the backup files which were to be stored:
+            #{package_files.join("\n")}
+          EOS
+        end
+
+        unless messages.empty?
+          Logger.warn Error.new(<<-EOS)
+            Cleanup Warning
+            #{messages.join("\n")}
+            Make sure you check these files before the next scheduled backup for
+            '#{model.label} (#{model.trigger})'
+            These files will be removed at that time!
+          EOS
+        end
+      end
+
+      private
+
+      def package_files_for(trigger)
+        Dir[File.join(Config.tmp_path, "#{trigger}.tar{,[.-]*}")]
+      end
+    end
+  end
+end