rtbackup 0.1.21

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/backup_conf.sample.yml

@@ -0,0 +1,83 @@
+###############################################################################
+#
+#                  Sample Backup Configuration
+#
+#
+#
+###############################################################################
+
+logging:
+    # DEBUG, INFO, WARN, ERROR, FATAL
+    log_threshold: 'DEBUG'
+    # If set to true, logs to STDOUT and all the subsequent settings in this
+    # section will be ignored.
+    use_stdout: true
+    logfile_path: '/tmp/server_backup.log'
+    keep_files: 1
+    file_size:
+        kilobytes: 100
+
+# Optional.  Defaults to /var/www
+file_location: 'spec/fixtures/www'
+# The time zone that you'd like backups to be kept in.
+# Backup files will be named with a timestamp located in the given timezone.
+time_zone: 'Singapore'
+# Optional.  Will attempt to determine from operating system
+# defaults to the value of the `time_zone` property above.
+system_time_zone: 'Asia/Ho_Chi_Minh'
+
+slack:
+  # Set up a new webhook at https://slack.com/apps/A0F7XDUAZ-incoming-webhooks
+  # Once configured, put the address here.
+  webhook: https://hooks.slack.com/services/asdasdasdasdasdasdasdasdasdasd
+  # If you want to babysit your backups, set this true.
+  notify_on_success: true
+  # A list of user ids of persons to @mention when posting success messages.
+  mention_users_on_failure:
+    - U9ASDFASD
+    # - U8SXDFG34
+    # - U23SDU3LD
+    # - U3234ASD4
+  # Leave empty for none.
+  mention_users_on_success:
+    - U9ASDFASD
+
+retention:
+  incremental:
+    hours: 144
+  daily:
+    days: 6
+  weekly:
+    weeks: 3
+  monthly:
+    months: 11
+
+mysql:
+  host: 127.0.0.1
+  # Database to back up
+  database:
+  # MySql credentials
+  user: root
+  password:
+  # Path (with trailing slash) to mysql commands e.g. mysqldump
+  bin_path: /usr/local/bin/
+  # If you are using MySql binary logging:
+  # Path to the binary logs, should match the log_bin option in your my.cnf.
+  # Comment out if you are not using mysql binary logging
+  bin_log: /var/tmp/mysql-bin
+
+s3:
+  # S3 credentials
+  access_key_id: ASDFASDFASDFASDFASDF
+  secret_access_key: asdfasdfasdfasdfasdfasdfasdfasdfasdfasdf
+  # Bucket in which to store your backups
+  # # Bucket in which to store your backups
+  bucket: myapp-backup-test
+
+  # The prefix under which to store and retrieve backups for this server
+  #  e.g. my_app_name
+  prefix: my_test_app
+
+  # AWS region your bucket lives in.
+  # (I suspect you only need to specify this when your 'location' is in a different region.)
+  region: ap-southeast-1

data/bin/rtbackup

@@ -0,0 +1,169 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'fileutils'
+
+lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
+$LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
+
+require 'server_backups'
+require 'main'
+require 'tmpdir'
+
+def find_time_zone(name)
+    ActiveSupport::TimeZone.all.select do |zone|
+        name.casecmp(zone.name).zero? || name.casecmp(zone.tzinfo.name).zero?
+    end.first
+end
+
+ERRORS = []
+def backup_thread
+    Thread.new do
+        begin
+            dir = File.join(Dir.getwd, rand.to_s)
+            FileUtils.mkdir_p(dir)
+            yield dir
+        rescue StandardError => error
+            ERRORS << error
+        ensure
+            FileUtils.rm_rf dir
+        end
+    end
+end
+
+def handle_errors(config_path)
+    notifier = ServerBackups::Notifier.new(config_path)
+    if ERRORS.empty?
+        notifier.notify_success
+        puts "That's the jam"
+    else
+        notifier.notify_failure(ERRORS)
+        ERRORS.each do |error|
+            puts error.message
+            puts "\n\n    " + error.backtrace.join("\n    ")
+        end
+    end
+end
+
+def run_backups(backup_type, config_file, database, db_only, files_only)
+    begin
+        tmp_dir = File.join(Dir.getwd, rand.to_s)
+        FileUtils.mkdir_p tmp_dir
+        unless db_only
+            ServerBackups::WebsiteBackup.new(config_file,
+                                            tmp_dir, backup_type).do_backup
+        end
+        unless files_only
+            ServerBackups::MysqlBackup.send(backup_type,
+                                            config_file,
+                                            tmp_dir, database).do_backup
+        end
+    rescue StandardError => error
+        puts error.message
+        puts "\n\n    " + error.backtrace.join("\n    ")
+        exit(2)
+    ensure
+        FileUtils.rm_rf tmp_dir
+    end
+end
+
+def run_restore(config_file, database, up_to, tz, db_only, files_only)
+    Chronic.time_class = find_time_zone(tz) if tz
+    up_to = Chronic.parse(up_to)
+    tmp_dir = File.join(Dir.getwd, rand.to_s)
+    begin
+        FileUtils.mkdir_p tmp_dir
+        unless db_only
+            ServerBackups::WebsiteRestore.new(config_file,
+                tmp_dir, up_to).do_restore
+        end
+        unless files_only
+            ServerBackups::MysqlRestore.restore(config_file,
+                tmp_dir,
+                up_to, database)
+        end
+    rescue StandardError => error
+        puts error.message
+        puts "\n\n    " + error.backtrace.join("\n    ")
+        exit(2)
+    end
+end
+
+Main do
+    option 'config', 'c' do
+        argument :required
+        description 'load configuration from YAML file'
+        defaults '~/.backup_conf.yml'
+    end
+
+    option 'database', 'd' do
+        argument :required
+        description 'Which database to back up, defaults to all non-system databases.'
+        defaults 'all'
+    end
+
+    option 'db_only', 'b' do
+        argument :optional
+        description 'Only work with database(s).'
+    end
+
+    option 'files_only', 'f' do
+        argument :optional
+        description 'Only work with files.'
+    end
+
+    mode 'restore' do
+        params[:backup_type].ignore!
+        option 'up_to', 'u' do
+            argument :required
+            description 'The point in time to restore to.  See ' \
+                        'https://github.com/mojombo/chronic for examples.'
+        end
+        option 'time_zone', 'z' do
+            argument :required
+            description 'Time zone that <up_to> is given in. Default: time_zone from config file.'
+        end
+        def run
+            options = params.to_options
+            run_restore(*options.slice('config',
+                                       'database',
+                                       'up_to',
+                                       'time_zone',
+                                       'db_only',
+                                       'files_only').values)
+        end
+    end
+
+    mode 'zones' do
+        params[:backup_type].ignore!
+        argument 'search' do
+            argument :required
+            description "Add a regex pattern to search for, e.g. '*america*'"
+            defaults '.*'
+        end
+
+        def run
+            pattern = Regexp.new(params['search'].value, 'i')
+            ActiveSupport::TimeZone.all.sort_by(&:name).each do |timezone|
+                if pattern =~ timezone.name || pattern =~ timezone.tzinfo.name
+                    puts timezone.name + "\t\t" + timezone.tzinfo.name
+                end
+            end
+        end
+    end
+
+    argument 'backup_type' do
+        cast :symbol
+        validate { |command| ServerBackups::BackupBase::BACKUP_TYPES.include? command }
+        description 'specifies the backup type to perform [incremental | daily | weekly | monthly]'
+    end
+
+    def run
+        options = params.to_options
+        run_backups(*options.slice('backup_type',
+                                   'config',
+                                   'database',
+                                   'db_only',
+                                   'files_only').values)
+    end
+end

data/crontab.txt

@@ -0,0 +1,4 @@
+5 0 1 * * server_backup monthly
+5 0 2-31 * 0 server_backup weekly
+5 0 2-31 * 1-6 server_backup daily
+5 1-23 * * * server_backup incremental

data/lib/server_backups.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext'
+require 'chronic'
+
+require 'server_backups/version'
+require 'server_backups/errors'
+require 'server_backups/config'
+require 'server_backups/s3'
+require 'server_backups/backup_base'
+require 'server_backups/website_backup'
+require 'server_backups/mysql_backup'
+require 'server_backups/mysql_incremental_backup'
+require 'server_backups/ordered_backup_file_collection'
+require 'server_backups/restore_base'
+require 'server_backups/website_restore'
+require 'server_backups/mysql_restore'
+require 'server_backups/notifier'
+
+module ServerBackups
+end

data/lib/server_backups/backup_base.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'English'
+
+module ServerBackups
+    class BackupBase
+        attr_reader :working_directory, :config, :logger, :s3, :backup_type
+
+        BACKUP_TYPES = %i[incremental daily weekly monthly].freeze
+
+        TIMESTAMP_FORMAT = '%Y-%m-%dT%H00.UTC%z'
+
+        def initialize(config_file, working_directory, backup_type)
+            @working_directory = working_directory
+            @config = Config.new(config_file)
+            Time.zone = config.time_zone
+            @logger = config.logger
+            @backup_type = backup_type.to_sym
+            logger.debug "Initialized #{backup_type} #{self.class.name.demodulize.titleize}, " \
+                         "prefix: '#{s3_prefix}'"
+        end
+
+        def incremental?
+            backup_type == :incremental
+        end
+
+        class << self
+            def daily(config_file, working_directory)
+                new(config_file, working_directory, :daily)
+            end
+
+            def weekly(config_file, working_directory)
+                new(config_file, working_directory, :weekly)
+            end
+
+            def monthly(config_file, working_directory)
+                new(config_file, working_directory, :monthly)
+            end
+
+            def incremental(config_file, working_directory)
+                new(config_file, working_directory, :incremental)
+            end
+        end
+
+        def backup_filename
+            "#{self.class.name.demodulize.underscore}.#{backup_type}.#{timestamp}.tgz"
+        end
+
+        def title
+            self.class.name.demodulize.titleize
+        end
+
+        def take_backup
+            logger.info "Creating #{backup_type} #{title} #{create_archive_command}"
+
+            system create_archive_command
+            unless last_command_succeeded?
+                raise BackupCreationError.new("Received #{$CHILD_STATUS} from tar command.",
+                                              self.class, backup_type)
+            end
+            logger.debug "Backup exited with #{$CHILD_STATUS}"
+        end
+
+        def s3_prefix
+            File.join(config.prefix, self.class.name.demodulize.underscore,
+                      backup_type.to_s, '/')
+        end
+
+        def backup_s3_key
+            File.join(s3_prefix, File.basename(backup_filename))
+        end
+
+        def store_backup
+            logger.info 'Upload file'
+            @uploaded_file = s3.save backup_path, backup_s3_key
+            logger.info 'Finished uploading file.'
+        end
+
+        def remove_old_backups
+            s3.delete_files_not_newer_than s3_prefix, config.get_retention_threshold(backup_type)
+        end
+
+        def backup_path
+            File.join(working_directory, backup_filename)
+        end
+
+        def do_backup
+            load_resources
+            take_backup
+            store_backup
+            # verify_backup
+            remove_old_backups
+        end
+
+        def timestamp
+            Time.zone.now.strftime(TIMESTAMP_FORMAT)
+        end
+
+        def load_resources
+            # @mysql = Mys3ql::Mysql.new self.config
+            @s3 = S3.new config
+        end
+
+        private
+
+        def last_command_succeeded?
+            $CHILD_STATUS.exitstatus.zero?
+        end
+    end
+end

data/lib/server_backups/config.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+# lifted from https://github.com/airblade/mys3ql/blob/master/lib/mys3ql/config.rb
+
+require 'yaml'
+require 'logger'
+
+module ServerBackups
+    class Config
+        attr_reader :logger, :logging, :config_file
+        cattr_accessor :config
+
+        DEFAULT_LOGFILE_SIZE = 1.megabytes
+        DEFAULT_LOGFILE_COUNT = 7
+
+        def initialize(config_file = nil)
+            if config_file == '-'
+                @config_file = config_file
+                @config = Config::config ||= YAML::safe_load(STDIN.read())
+            else
+                @config_file = config_file || default_config_file
+                @config = YAML.load_file File.expand_path(config_file)
+            end
+
+            @logging = @config['logging']
+            @logger = Logger.new(log_device, logfile_count, logfile_size)
+        rescue Errno::ENOENT => e
+            puts e.backtrace
+            warn "missing config file #{config_file}"
+            exit 1
+        end
+
+        class << self
+            def get_time_zone(name)
+                ActiveSupport::TimeZone.all.find do |time_zone|
+                    time_zone.name.casecmp(name).zero? || time_zone.tzinfo.name.casecmp(name).zero?
+                end
+            end
+        end
+
+        def slack_webhook
+            @config.fetch('slack', nil)&.fetch('webhook')
+        end
+
+        def notify_on_success
+            @config.fetch('slack', nil)&.fetch('notify_on_success', false)
+        end
+
+        def slack_mention_on_failure
+            @config.fetch('slack', nil)&.fetch('mention_users_on_failure', [])
+        end
+
+        def slack_mention_on_success
+            @config.fetch('slack', nil)&.fetch('mention_users_on_success', [])
+        end
+
+        #
+        # General
+        #
+        FILE_LOCATION = '/var/www'
+
+        def web_root
+            File.absolute_path(@config['file_location'] || FILE_LOCATION)
+        end
+
+        def time_zone
+            @config['time_zone'] || 'UTC'
+        end
+
+        def system_time_zone
+            tz = if @config['system_time_zone']
+                     @config['system_time_zone']
+                 elsif File.exist?('/etc/timezone')
+                     File.read('/etc/timezone')
+                 elsif File.exist?('/etc/localtime')
+                     %r{([\w_]+\/[\w_]+)$}.match(`ls -l /etc/localtime`).captures.first
+                 else
+                     'UTC'
+                 end
+            self.class.get_time_zone(tz)
+        end
+
+        def log_device
+            logging['use_stdout'] == true ? STDOUT : logging['logfile_path']
+        end
+
+        def logfile_size
+            if logging['file_size']
+                unit, quantity = @logging['file_size'].first
+                quantity = quantity.to_i
+                quantity.send(unit)
+            else
+                DEFAULT_LOGFILE_SIZE
+            end
+        end
+
+        def logfile_count
+            if logging['keep_files']
+                logging['keep_files'].to_i
+            else
+                DEFAULT_LOGFILE_COUNT
+            end
+        end
+
+        def get_retention_threshold(backup_type)
+            interval, quantity = @config['retention'][backup_type.to_s].first
+            quantity = quantity.to_i + 1
+            quantity.send(interval).ago
+        end
+
+        def retain_dailies_after
+            get_retention_threshold(:daily)
+        end
+
+        def retain_incrementals_after
+            get_retention_threshold(:incremental)
+        end
+
+        def retain_monthlies_after
+            get_retention_threshold(:monthly)
+        end
+
+        def retain_weeklies_after
+            get_retention_threshold(:weekly)
+        end
+
+        #
+        # MySQL
+        #
+
+        def db_host
+            mysql['host'] || '127.0.0.1'
+        end
+
+        def user
+            mysql['user']
+        end
+
+        def password
+            mysql['password']
+        end
+
+        def database
+            mysql['database']
+        end
+
+        def bin_path
+            mysql['bin_path']
+        end
+
+        def bin_log
+            mysql['bin_log']
+        end
+
+        MYSQLDUMP = 'mysqldump'
+        MYSQL = 'mysql'
+        MYSQLBINLOG = 'mysqlbinlog'
+
+        def mysqldump_bin
+            File.join(bin_path, MYSQLDUMP)
+        end
+
+        def mysqlbinlog_bin
+            File.join(bin_path, MYSQLBINLOG)
+        end
+
+        def mysql_bin
+            File.join(bin_path, MYSQL)
+        end
+
+        #
+        # S3
+        #
+
+        def access_key_id
+            s3['access_key_id']
+        end
+
+        def secret_access_key
+            s3['secret_access_key']
+        end
+
+        def bucket
+            s3['bucket']
+        end
+
+        def region
+            s3['region']
+        end
+
+        def prefix
+            s3['prefix']
+        end
+
+        private
+
+        def mysql
+            @config['mysql']
+        end
+
+        def s3
+            @config['s3']
+        end
+
+        def default_config_file
+            File.join (ENV['HOME']).to_s, '.mys3ql'
+        end
+    end
+end