server_backups 0.1.8

data/lib/server_backups/mysql_restore.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+
+module ServerBackups
+    class MysqlRestore < RestoreBase
+        ALL_DATABASES = 'all'
+
+        def initialize(config_file, working_dir, restore_point, database)
+            @database = database
+            super config_file, working_dir, restore_point
+        end
+
+        def full_backup_file
+            full_backup_prefix = File.join(config.prefix, 'mysql_backup', database)
+            s3.get_ordered_collection(full_backup_prefix).full_backup_for(restore_point)
+        end
+
+        def incremental_backups
+            incr_backup_prefix = File.join(config.prefix, 'mysql_backup')
+            s3.get_ordered_collection(incr_backup_prefix).incremental_backups_for(restore_point)
+        end
+
+        def restore_script_path
+            File.join(working_dir, "#{database}.sql")
+        end
+
+        ETC_TIMEZONE = '/etc/timezone'
+
+        def formatted_restore_point_in_system_time_zone
+            restore_point.in_time_zone(config.system_time_zone) \
+                         .strftime('%Y-%m-%d %H:%M:%S')
+        end
+
+        def do_restore
+            full_backup_file.get response_target: (restore_script_path + '.gz')
+            system "gunzip #{restore_script_path}.gz"
+
+            incremental_backups.each do |s3object|
+                file = Tempfile.new('foo')
+                begin
+                    s3object.get response_target: file
+                    file.close
+                    system config.mysqlbinlog_bin + ' ' + file.path + \
+                           " --stop-datetime='#{formatted_restore_point_in_system_time_zone}'" \
+                           " --database=#{database} >> " + restore_script_path
+                ensure
+                    file.close
+                    file.unlink # deletes the temp file
+                end
+            end
+
+            execute_script restore_script_path
+        end
+        # #{@config.bin_path}mysqlbinlog --database=#{@config.database} #{file}
+        class << self
+            def restore(config_file, working_dir, restore_point, database)
+                return new(config_file, working_dir, restore_point, database).do_restore \
+                    if database != ALL_DATABASES
+
+                all_databases(config_file, working_dir).each do |db_name|
+                    new(config_file, working_dir, restore_point, db_name).do_restore
+                end
+            end
+
+            def all_databases(config_file, working_dir)
+                MysqlBackup.new(config_file, working_dir, 'daily', 'mysql').all_databases
+            end
+        end
+
+        private
+
+        def cli_options
+            cmd = config.password.blank? ? '' : " -p'#{config.password}' "
+            cmd + " -u'#{config.user}' " + database
+        end
+
+        def execute_script(path)
+            cmd = "#{config.mysql_bin} --silent --skip-column-names #{cli_options}"
+            logger.debug "Executing raw SQL against #{ database}\n#{cmd}"
+            output = `#{cmd} < #{path}`
+            logger.debug "Returned #{$CHILD_STATUS.inspect}. STDOUT was:\n#{output}"
+            output.split("\n") unless output.blank?
+        end
+    end
+end

data/lib/server_backups/notifier.rb

@@ -0,0 +1,33 @@
+require 'slack-notifier'
+
+module ServerBackups
+    class Notifier
+        attr_reader :config
+
+        def initialize(config_path)
+            @config = Config.new(config_path)
+        end
+
+        def notify_success
+            return unless config.slack_webhook && config.notify_on_success
+
+            notifier = Slack::Notifier.new config.slack_webhook
+            message = "Backups at `#{config.prefix}` succeeded. "
+            message += config.slack_mention_on_success.map{|t| "<@#{t}>"}.to_sentence
+            notifier.post text: message, icon_emoji: ':100:'
+        end
+
+        def notify_failure(errors)
+            return unless config.slack_webhook
+
+            notifier = Slack::Notifier.new config.slack_webhook
+            message = "Backups at `#{config.prefix}` failed. "
+            message += config.slack_mention_on_failure.map{|t| "<@#{t}>"}.to_sentence
+            attachments = []
+            for error in errors do
+                attachments << {text: error.message + "\n" + error.backtrace.join("\n")}
+            end
+            notifier.post text: message, icon_emoji: ':bomb:', attachments: attachments
+        end
+    end
+end

data/lib/server_backups/ordered_backup_file_collection.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ServerBackups
+    class OrderedBackupFileCollection
+        attr_reader :s3_collection
+        def initialize(s3_collection)
+            @s3_collection = s3_collection
+        end
+
+        def full_backup_for(restore_point)
+            sorted(full_backups).reverse.find do |file|
+                get_timestamp_from_s3_object(file) <= restore_point
+            end
+        end
+
+        def incremental_backups_for(restore_point)
+            sorted eligible_incremental_backups(restore_point)
+        end
+
+        INCREMENTAL = /incremental/i
+        def full_backups
+            s3_collection.reject { |file| INCREMENTAL =~ file.key }
+        end
+
+        def incremental_backups
+            @incremental_backups ||=
+                sorted(s3_collection.select { |file| INCREMENTAL =~ file.key }).to_a
+        end
+
+        private
+
+        TIMESTAMP_REGEXP = /(\d{4})-(\d{2})-(\d{2})T(\d{2})00\.UTC([+-]\d{4})/
+        def get_timestamp_from_s3_object(file)
+            time_parts = TIMESTAMP_REGEXP.match(file.key).captures
+            time_parts[-1].insert(3, ':')
+            # Add in hours and seconds arguments
+            # https://ruby-doc.org/core-2.2.0/Time.html#method-c-new
+            time_parts.insert(4, 0, 0)
+            Time.new(*time_parts)
+        end
+
+        def sorted(coll)
+            coll.sort_by { |file| get_timestamp_from_s3_object file }
+        end
+
+        def eligible_incremental_backups(restore_point)
+            full_backup_timestamp = get_timestamp_from_s3_object full_backup_for(restore_point)
+            incremental_backups.select do |file|
+                get_timestamp_from_s3_object(file) > full_backup_timestamp &&
+                    get_timestamp_from_s3_object(file) <= restore_point
+            end
+        end
+    end
+end

data/lib/server_backups/restore_base.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module ServerBackups
+    class RestoreBase
+        attr_reader :config, :s3, :working_dir, :restore_point, :database
+
+        def initialize(config_file, working_dir, restore_point)
+            @working_dir = working_dir
+            @config = Config.new(config_file)
+            Time.zone = config.time_zone
+            @restore_point = if restore_point.present?
+                                 restore_point
+                             else
+                                 Time.zone.now
+                             end
+            @s3 = S3.new(config)
+            logger.debug "Initialized #{title}."
+        end
+
+        private
+
+        def title
+            self.class.name.demodulize.titleize
+        end
+
+        TIMESTAMP_REGEXP = /(\d{4})-(\d{2})-(\d{2})T(\d{2})00\.UTC([+-]\d{4})/
+        def extract_backup_time_from_filename(filename)
+            time_parts = TIMESTAMP_REGEXP.match(filename).captures
+            # Add in hours and seconds arguments
+            # https://ruby-doc.org/core-2.2.0/Time.html#method-c-new
+            time_parts.insert(4, 0, 0)
+            Time.new(*time_parts)
+        end
+
+        def all_files
+            @all_files ||= s3.get_ordered_collection(s3_prefix)
+        end
+
+        def logger
+            config.logger
+        end
+
+        def last_command_succeeded?
+            $CHILD_STATUS.exitstatus.zero?
+        end
+    end
+end

data/lib/server_backups/s3.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+
+module ServerBackups
+    class S3
+        PROVIDER = 'AWS'
+        attr_reader :config, :logger
+        def initialize(config)
+            @config = config
+            @logger = config.logger
+        end
+
+        def client
+            @client ||= begin
+                Aws.config[:credentials] = Aws::Credentials.new(
+                    config.access_key_id, config.secret_access_key
+                )
+                Aws::S3::Client.new region: config.region
+            end
+        end
+
+        def bucket
+            @bucket ||= Aws::S3::Bucket.new(config.bucket, client: client)
+        end
+
+        def get_ordered_collection(prefix)
+            OrderedBackupFileCollection.new bucket.objects(prefix: prefix)
+        end
+
+        def delete_files_not_newer_than(key, age)
+            bucket.objects(prefix: key).each do |file|
+                destroy key, true unless file.last_modified.to_datetime > age
+            end
+        end
+
+        def exists?(path)
+            logger.debug "Exists? #{config.bucket}  #{path}"
+            !bucket.objects(prefix: path).to_a.empty?
+            # !!client.head_object(bucket: config.bucket, key: path)
+        end
+
+        def destroy(key, existence_known = false)
+            return unless existence_known || exists?(key)
+            client.delete_object bucket: config.bucket, key: key
+        end
+
+        def save(local_file_name, s3_key)
+            full_path = if s3_key[-1] == '/'
+                            File.join(s3_key, File.basename(local_file_name))
+                        else
+                            s3_key
+                        end
+
+            return if exists?(full_path)
+            file = Aws::S3::Object.new(config.bucket, full_path, client: client)
+            file.put(
+                acl: 'private',
+                body: File.open(local_file_name, 'rb'),
+                content_md5: md5of(local_file_name),
+                storage_class: 'STANDARD_IA'
+            )
+        end
+
+        private
+
+        def md5of(local_file_name)
+            Digest::MD5.base64digest(File.read(local_file_name))
+        end
+    end
+end

data/lib/server_backups/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ServerBackups
+    VERSION = '0.1.8'
+end

data/lib/server_backups/website_backup.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ServerBackups
+    class WebsiteBackup < BackupBase
+        SNAPSHOT_PATH = Pathname('~/.var.www.backup.tar.snapshot.bin').expand_path
+
+        def restore(time); end
+
+        def backup_filename
+            "#{self.class.name.demodulize.underscore}.#{backup_type}.#{timestamp}.tgz"
+        end
+
+        def create_archive_command
+            <<-CMD
+                tar --create -C '#{config.web_root}' --listed-incremental=#{SNAPSHOT_PATH} --gzip \
+                --no-check-device \
+                #{'--level=0' unless incremental?} \
+                --file=#{backup_path} #{config.web_root}
+            CMD
+        end
+    end
+end

data/lib/server_backups/website_restore.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'English'
+
+module ServerBackups
+    class WebsiteRestore < RestoreBase
+        def files_to_restore
+            [all_files.full_backup_for(restore_point),
+             *all_files.incremental_backups_for(restore_point)]
+        end
+
+        def do_restore
+            logger.warn "Moving old #{config.web_root} ---->>> #{config.web_root}.backup\n" \
+                        'You will have to delete it yourself.'
+            make_copy_of_existing_web_root
+            files_to_restore.each_with_index { |file, index| restore_file(file, index) }
+        end
+
+        private
+
+        def make_copy_of_existing_web_root
+            FileUtils.mv config.web_root, "#{config.web_root}.backup" \
+                if File.exist?(config.web_root)
+            FileUtils.mkdir config.web_root
+        end
+
+        def s3_prefix
+            File.join(config.prefix, 'website_backup', '/')
+        end
+
+        def restore_file(s3object, ordinal)
+            local_filename = File.join(working_dir, "#{ordinal}.tgz")
+            s3object.get response_target: local_filename
+            system untar_command(local_filename)
+            unless last_command_succeeded?
+                raise BackupCreationError.new("Received #{$CHILD_STATUS} from tar command.",
+                                              self.class, 'restore')
+            end
+            logger.info "Restored #{File.basename(s3object.key)}"
+        end
+
+        def untar_command(local_filename)
+            <<-CMD
+                tar zxp -C '/' --listed-incremental=/dev/null \
+                --no-check-device \
+                --file=#{local_filename}
+            CMD
+        end
+    end
+end

data/lib/server_backups.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext'
+require 'chronic'
+
+require 'server_backups/version'
+require 'server_backups/errors'
+require 'server_backups/config'
+require 'server_backups/s3'
+require 'server_backups/backup_base'
+require 'server_backups/website_backup'
+require 'server_backups/mysql_backup'
+require 'server_backups/mysql_incremental_backup'
+require 'server_backups/ordered_backup_file_collection'
+require 'server_backups/restore_base'
+require 'server_backups/website_restore'
+require 'server_backups/mysql_restore'
+require 'server_backups/notifier'
+
+module ServerBackups
+end

data/server_backups.gemspec

@@ -0,0 +1,45 @@
+
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'server_backups/version'
+
+Gem::Specification.new do |spec|
+    spec.name = 'server_backups'
+    spec.version       = ServerBackups::VERSION
+    spec.authors       = ['Tyler Gannon']
+    spec.email         = ['tgannon@gmail.com']
+
+    spec.summary       = 'For taking backups of servers.'
+    spec.description   = 'For taking backups of servers.'
+    spec.homepage      = 'https://github.com/tylergannon/server_backups'
+    spec.license       = 'MIT'
+
+    # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+    # to allow pushing to a single host or delete this section to allow pushing to any host.
+    if spec.respond_to?(:metadata)
+        spec.metadata['allowed_push_host'] = "https://rubygems.org"
+    else
+        raise 'RubyGems 2.0 or newer is required to protect against ' \
+          'public gem pushes.'
+    end
+
+    spec.add_dependency 'activesupport', '~> 5.1'
+    spec.add_dependency 'aws-sdk-s3', '~> 1.8'
+    spec.add_dependency 'main', '~> 6.2'
+    spec.add_dependency 'slack-notifier', '~> 2.3'
+
+    spec.files = `git ls-files -z`.split("\x0").reject do |f|
+        f.match(%r{^(test|spec|features)/})
+    end
+    spec.bindir        = 'bin'
+    spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+    spec.require_paths = ['lib']
+
+    spec.add_development_dependency 'bundler', '~> 1.16'
+    spec.add_development_dependency 'rake', '~> 10.5'
+    spec.add_development_dependency 'rspec', '~> 3.0'
+    spec.add_development_dependency 'vcr', '~> 4.0'
+    spec.add_development_dependency 'webmock', '~> 3.3'
+end

metadata

@@ -0,0 +1,207 @@
+--- !ruby/object:Gem::Specification
+name: server_backups
+version: !ruby/object:Gem::Version
+  version: 0.1.8
+platform: ruby
+authors:
+- Tyler Gannon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: main
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: slack-notifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.5'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+description: For taking backups of servers.
+email:
+- tgannon@gmail.com
+executables:
+- console
+- server_backup
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".ruby-version"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- backup_conf.sample.yml
+- bin/console
+- bin/server_backup
+- bin/setup
+- crontab.txt
+- incremental_snapshot
+- lib/server_backups.rb
+- lib/server_backups/backup_base.rb
+- lib/server_backups/config.rb
+- lib/server_backups/errors.rb
+- lib/server_backups/mysql_backup.rb
+- lib/server_backups/mysql_incremental_backup.rb
+- lib/server_backups/mysql_restore.rb
+- lib/server_backups/notifier.rb
+- lib/server_backups/ordered_backup_file_collection.rb
+- lib/server_backups/restore_base.rb
+- lib/server_backups/s3.rb
+- lib/server_backups/version.rb
+- lib/server_backups/website_backup.rb
+- lib/server_backups/website_restore.rb
+- server_backups.gemspec
+homepage: https://github.com/tylergannon/server_backups
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: For taking backups of servers.
+test_files: []