backup-agent 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 30d19db4d2b7b5ccd2d5b22ba4984b12cbf2d91b
+  data.tar.gz: eca7fd9183ba78f8b1bee805167ffcb30c967548
+SHA512:
+  metadata.gz: 580cb22de5ecfdeb372e1b2fa163d979123f2099da0f4dd4b0d17105f8acd739627e5b6645db613fdae6d3a5ec8e624897a377edde1421924a1999506e09f293
+  data.tar.gz: 5656f4668372fff2b3d1be5dedeb644aa97a26d672a9d90c52d21e5897512125ac2ef36ac077883a212ea138b5590d5a09d54b0f381f0c9f7e960c660c81df56

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+.idea

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in confo.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Yaroslav Konoplov
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,16 @@
+```ruby
+BackupAgent.configure_s3 do
+  set :access_key_id, 'xxx'
+  set :secret_access_key, 'yyy'
+  set :region, 'eu-central-1'
+end
+
+BackupAgent.perform_backup do
+  set :s3_bucket, 'my-backups'
+end
+```
+
+## Gemfile
+```ruby
+gem 'backup-agent', github: 'yivo/backup-agent'
+```

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/backup-agent.gemspec

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+Gem::Specification.new do |s|
+  s.name            = 'backup-agent'
+  s.version         = '1.0.0'
+  s.authors         = ['Yaroslav Konoplov']
+  s.email           = ['yaroslav@inbox.com']
+  s.summary         = 'Easy AWS S3 backup'
+  s.description     = 'Easy AWS S3 backup'
+  s.homepage        = 'http://github.com/yivo/backup-agent'
+  s.license         = 'MIT'
+
+  s.executables     = `git ls-files -z -- bin/*`.split("\x0").map{ |f| File.basename(f) }
+  s.files           = `git ls-files -z`.split("\x0")
+  s.test_files      = `git ls-files -z -- {test,spec,features}/*`.split("\x0")
+  s.require_paths   = ['lib']
+
+  s.add_dependency 'aws-sdk'
+  s.add_dependency 'confo-config', '>= 1.0.0'
+  s.add_dependency 'activesupport', '>= 3.2.0'
+end

data/lib/backup-agent.rb

@@ -0,0 +1,19 @@
+puts "Ruby version #{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}"
+
+%w(rubygems aws-sdk fileutils confo-config shellwords).each { |lib| require lib }
+
+require_relative 'backup-agent/s3_config'
+require_relative 'backup-agent/task_config'
+require_relative 'backup-agent/service'
+require_relative 'backup-agent/features'
+require_relative 'backup-agent/s3'
+
+module BackupAgent
+  class << self
+    def perform_backup(&block)
+      task_config = TaskConfig.new
+      task_config.configure(&block) if block
+      Service.new(task_config).perform_backup
+    end
+  end
+end

data/lib/backup-agent/features.rb

@@ -0,0 +1,32 @@
+module BackupAgent
+  class << self
+    def check_mysql
+      if @mysql_check.nil?
+        @mysql_check = system('/usr/bin/env mysql --version') ? true : (puts('MySQL is not installed'); false)
+      end
+      @mysql_check
+    end
+
+    def check_mongodb
+      if @mongodb_check.nil?
+        @mongodb_check = system('/usr/bin/env mongod --version') ? true : (puts('MongoDB is not installed'))
+      end
+      @mongodb_check
+    end
+
+    def check_features
+      check_mysql
+      check_mongodb
+    end
+
+    def mysql_installed?
+      check_features
+      !!@mysql_check
+    end
+
+    def mongodb_installed?
+      check_features
+      !!@mongodb_check
+    end
+  end
+end

data/lib/backup-agent/s3.rb

@@ -0,0 +1,22 @@
+module BackupAgent
+  class << self
+    def s3_config
+      @s3_config ||= S3Config.new
+    end
+
+    def configure_s3(&block)
+      @s3 = nil
+      s3_config.configure(&block)
+    end
+
+    def s3
+      @s3 ||= begin
+        Aws.config.update(
+          region: s3_config.region,
+          credentials: Aws::Credentials.new(s3_config.access_key_id, s3_config.secret_access_key)
+        )
+        Aws::S3::Resource.new
+      end
+    end
+  end
+end

data/lib/backup-agent/s3_config.rb

@@ -0,0 +1,10 @@
+module BackupAgent
+  class S3Config < Confo::Config
+    def initialize(*)
+      super
+      set :access_key_id, nil
+      set :secret_access_key, nil
+      set :region, nil
+    end
+  end
+end

data/lib/backup-agent/service.rb

@@ -0,0 +1,109 @@
+module BackupAgent
+  class Service
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def perform_backup
+      @started_at = Time.now.utc
+      @timestamp = @started_at.strftime('%s - %A %d %B %Y %H:%M')
+      make_tmp_dir
+      backup_mysql if BackupAgent.mysql_installed?
+      backup_mongodb if BackupAgent.mongodb_installed?
+      backup_directories
+      backup_files
+      cleanup_old_backups
+    ensure
+      remove_tmp_dir
+    end
+
+  protected
+
+    def backup_directories
+      config.get(:directories).each do |name, dir|
+        dir_filename = "dir-#{name}.tar.gz"
+        cmd = "cd #{dir} && /usr/bin/env tar -czf #{tmp_path}/#{dir_filename} ."
+        puts "Exec #{cmd}"
+        system(cmd)
+
+        obj = BackupAgent.s3.bucket(config.s3_bucket).object("#{@timestamp}/#{dir_filename}")
+        obj.upload_file("#{tmp_path}/#{dir_filename}")
+      end
+    end
+
+    def backup_files
+      config.get(:files).each do |name, files|
+        begin
+          files_tmp_path  = File.join(tmp_path, "#{name}-tmp")
+          file_bunch_name = "files-#{name}.tar.gz"
+
+          FileUtils.mkdir_p(files_tmp_path)
+          FileUtils.cp(files.select { |el| File.exists?(el) }, files_tmp_path)
+
+          cmd = "cd #{files_tmp_path} && /usr/bin/env tar -czf #{tmp_path}/#{file_bunch_name} ."
+          system(cmd)
+
+          obj = BackupAgent.s3.bucket(config.s3_bucket).object("#{@timestamp}/#{file_bunch_name}")
+          obj.upload_file("#{tmp_path}/#{file_bunch_name}")
+        ensure
+          FileUtils.remove_dir(files_tmp_path)
+        end
+      end
+    end
+
+    def backup_mysql
+      config.get(:mysql_databases).each do |db|
+        db_filename = "mysql-#{db}.gz"
+        dump = "/usr/bin/env mysqldump #{config.get(:mysql_connect)} #{config.get(:mysqldump_options).join(' ')} #{db}"
+        gzip = "/usr/bin/env gzip -5 -c > #{tmp_path}/#{db_filename}"
+
+        puts "Exec #{dump} | #{gzip}"
+        system "#{dump} | #{gzip}"
+
+        obj = BackupAgent.s3.bucket(config.s3_bucket).object("#{@timestamp}/#{db_filename}")
+        obj.upload_file("#{tmp_path}/#{db_filename}")
+      end
+    end
+
+    def backup_mongodb
+      mongo_dump_dir = File.join(tmp_path, 'mongo')
+      FileUtils.mkdir_p(mongo_dump_dir)
+
+      config.get(:mongo_databases).each do |db|
+        db_filename = "mongo-#{db}.tgz"
+        dump = "/usr/bin/env mongodump #{config.get(:mongo_connect)} -d #{db} -o #{mongo_dump_dir}"
+        cd   = "cd #{mongo_dump_dir}/#{db}"
+        tar  = "/usr/bin/env tar -czf #{tmp_path}/#{db_filename} ."
+
+        puts "Exec #{dump} && #{cd} && #{tar}"
+        system "#{dump} && #{cd} && #{tar}"
+
+        obj = BackupAgent.s3.bucket(config.s3_bucket).object("#{@timestamp}/#{db_filename}")
+        obj.upload_file("#{tmp_path}/#{db_filename}")
+      end
+    ensure
+      FileUtils.remove_dir(mongo_dump_dir)
+    end
+
+    def cleanup_old_backups
+      cutoff_date = Time.now.utc.to_i - (config.get(:days_to_keep_backups) * 86400)
+      BackupAgent.s3.bucket(config.s3_bucket).objects.each do |o|
+        o.delete if o.last_modified.to_i < cutoff_date
+      end
+    end
+
+    def make_tmp_dir
+      FileUtils.mkdir_p(tmp_path)
+    end
+
+    def remove_tmp_dir
+      FileUtils.remove_dir(tmp_path)
+    end
+
+    def tmp_path
+      "/tmp/backup-agent-#{@started_at.strftime('%d-%m-%Y-%H:%M')}"
+    end
+  end
+end

data/lib/backup-agent/task_config.rb

@@ -0,0 +1,55 @@
+module BackupAgent
+  class TaskConfig < Confo::Config
+    def initialize(*)
+      super
+      set :mysql_user, 'root'
+      set :mysql_password, 'root'
+      set :mysql_host, 'localhost'
+      set :mysql_databases, -> do
+        `/usr/bin/env mysql #{get(:mysql_connect)} -e "SHOW DATABASES;"`
+          .split("\n")
+          .reject { |el| el =~ /Database|information_schema|mysql|performance_schema|test|phpmyadmin/ }
+      end
+
+      set :mysqldump_options, %w(
+        --single-transaction
+        --add-drop-table
+        --add-locks
+        --create-options
+        --disable-keys
+        --extended-insert
+        --quick)
+
+      set :mysql_connect, -> do
+        pass       = get(:mysql_password)
+        pass_param = pass && !pass.empty? ? "-p#{pass}" : ''
+        "-u #{get(:mysql_user)} #{pass_param} -h #{get(:mysql_host)}"
+      end
+
+      set :mongo_databases, -> do
+        if `/usr/bin/env mongo --eval "db.getMongo().getDBNames()"` =~ /connecting to: (.*)/m
+          $1.split(/[\n,]/).reject(&:empty?)
+        else
+          []
+        end
+      end
+
+      set :mongo_host, 'localhost'
+      set :mongo_connect, -> { "-h #{get(:mongo_host)}" }
+
+      set :directories, -> {
+        Dir['/var/www/*'].each_with_object({}) do |el, memo|
+          if Dir.exists?(File.join(el, 'current/public/uploads'))
+            memo["#{File.basename(el)}-uploads"] = File.join(el, 'current/public/uploads')
+          end
+        end
+      }
+
+      set :files, {}
+
+      set :s3_bucket, nil
+
+      set :days_to_keep_backups, 30
+    end
+  end
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: backup-agent
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Yaroslav Konoplov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: confo-config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+description: Easy AWS S3 backup
+email:
+- yaroslav@inbox.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- backup-agent.gemspec
+- lib/backup-agent.rb
+- lib/backup-agent/features.rb
+- lib/backup-agent/s3.rb
+- lib/backup-agent/s3_config.rb
+- lib/backup-agent/service.rb
+- lib/backup-agent/task_config.rb
+homepage: http://github.com/yivo/backup-agent
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Easy AWS S3 backup
+test_files: []