backup_mongo_s3 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6818188f3168c9c81bb4872cb8879696dd6bc659
+  data.tar.gz: 7e64a7949927fc492e5f37b7f1da076bfbda157b
+SHA512:
+  metadata.gz: b37923f6e3c23e4b62322db449e8452989f6488b22598aa7c5497301ef018d8c2bb92659783e77fb93f9468287b37801def3de49723de1ee2949bdc3a6256909
+  data.tar.gz: c1af757aae23d51381d98d5071fc440e7fd07845dd324958e407a306b2ee66e180170ff45509fd14a7b18683ff7a536e6eca59b3a7b52e383a0684121bef6406

data/.gitignore

@@ -0,0 +1,24 @@
+*.gem
+*.rbc
+*.idea
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.yml

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'aws-sdk'

data/README.md

@@ -0,0 +1,17 @@
+# BackupMongoS3
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'backup_mongo_s3'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install backup_mongo_s3

data/backup_mongo_s3.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |spec|
+  spec.name          = 'backup_mongo_s3'
+  spec.version       = '0.0.2'
+  spec.authors       = ['Yakupov Dima']
+  spec.email         = ['yakupov.dima@mail.ru']
+  spec.summary       = "Some summary"
+  spec.description   = "Some description"
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+end

data/bin/backup_mongo_s3

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/backup_mongo_s3'
+
+begin
+  BackupMongoS3::Application.new(ARGV).run
+rescue SystemExit
+  # do nothing
+rescue Exception => err
+  abort "[abort] #{err.message}"
+end

data/lib/backup_mongo_s3/application.rb

@@ -0,0 +1,284 @@
+module BackupMongoS3
+  class Application
+
+    def initialize(argv)
+      @parser = OptionParser.new
+
+      @params = parse_options(argv)
+      @config = parse_config(@params[:config])
+
+      @db      = Db.new(@config[:mongo])
+      @storage = Storage.new(@config[:s3])
+    end
+
+    public
+
+    def run
+
+      case
+        when @params[:backups_list] # BACKUPS_LIST
+          show_backups_list(@params[:backups_list])
+
+        when @params[:backup_all] # BACKUP_ALL
+          dbs_str = @config[:backup][:dbs]
+
+          if dbs_str.nil? || dbs_str.empty?
+            raise 'config.yml::backup.dbs is empty'
+          end
+
+          dbs_name = dbs_str.split(',').each { |db_name| db_name.strip! }
+
+          backup(dbs_name)
+
+        when @params[:backup] # BACKUP
+          backup([@params[:backup]])
+
+        when @params[:restore] # RESTORE
+          if @params[:backup_date].nil?
+            raise 'param --date BACKUP_DATE is not specified'
+          end
+
+          restore(@params[:restore], @params[:backup_date])
+
+        when @params[:cron_update] || @params[:cron_clear] # CRON
+          cron_options =
+              {
+                  update: @params[:cron_update],
+                  clear:  @params[:cron_clear],
+                  config: @params[:config],
+                  time:   @config[:backup][:cron_time],
+              }
+
+          Scheduler.new(cron_options).execute
+
+        else
+          puts "\n#{@parser}\n"
+          exit
+      end
+
+    end
+
+    private
+
+    def show_backups_list(db_name)
+
+      backups = @storage.get_backups_list("#{db_name}")
+
+      if backups.empty?
+        puts 'Backups not found'
+      else
+
+        puts sprintf('%-30s %-15s %-20s', 'name', 'size, MB', 'last_modified')
+
+        backups.each do |backup|
+
+          backup_name          = File.join(File.dirname(backup.key), File.basename(backup.key, '.backup'))
+          backup_size          = backup.content_length / 1024 / 1024
+          backup_last_modified = backup.last_modified
+
+          puts sprintf('%-30s %-15s %-20s', backup_name, backup_size, backup_last_modified)
+        end
+
+      end
+    end
+
+    def backup(dbs_name)
+
+      history_days_str = @config[:backup][:history_days]
+
+      begin
+        history_days = history_days_str.to_i
+      rescue
+        raise 'config.yml::backup.history_days is not integer'
+      end
+
+      dbs_name.each do |db_name|
+
+        puts "backup db #{db_name.upcase}:"
+
+        tmp_dir = get_temp_dir
+
+        begin
+
+          dump_path = File.join(tmp_dir, db_name)
+
+          puts "\t dump db..."
+          @db.dump(db_name, tmp_dir)
+
+          if Dir["#{dump_path}/*"].empty?
+            puts "\t [skip] db is empty"
+
+          else
+
+            puts "\t compress..."
+            system("zip -6 -r '#{dump_path}.zip' '#{dump_path}/' -j > /dev/null")
+            raise 'Error zip' unless $?.exitstatus.zero?
+
+            FileUtils.rm_rf(dump_path)
+
+            zip_file_path = "#{dump_path}.zip"
+
+            if File.exists?(zip_file_path)
+              puts "\t upload backup to s3..."
+              @storage.upload(db_name, zip_file_path)
+
+              puts "\t delete old backups from s3..."
+              @storage.delete_old_backups(db_name, history_days)
+
+              File.delete(zip_file_path)
+            end
+
+          end
+
+        ensure
+          FileUtils.remove_entry_secure(tmp_dir)
+        end
+
+      end
+
+      puts '[done] backup'
+    end
+
+
+    def restore(db_name, date)
+
+      puts "restore db #{db_name.upcase}:"
+
+      tmp_dir = get_temp_dir
+
+      begin
+
+        dump_path = File.join(tmp_dir, db_name)
+
+        zip_file_path = "#{dump_path}.zip"
+
+        puts "\t download file from s3..."
+        @storage.download(db_name, date, zip_file_path)
+
+        if File.exists?(zip_file_path)
+          puts "\t uncompress..."
+          system("unzip '#{zip_file_path}' -d '#{dump_path}' > /dev/null")
+          raise 'Error unzip' unless $?.exitstatus.zero?
+
+          File.delete(zip_file_path)
+
+          puts "\t restore db..."
+          @db.restore(db_name, dump_path)
+        end
+
+      ensure
+        FileUtils.remove_entry_secure(tmp_dir)
+      end
+
+      puts '[done] restore'
+    end
+
+    def create_config(path)
+
+      path = '.' if path.nil? || path == ''
+
+      file = File.join(path, 'config.yml')
+
+      if File.exists?(file)
+        raise "create_config: '#{file}' already exists"
+      elsif File.exists?(file.downcase)
+        raise "create_config: '#{file.downcase}' exists, which could conflict with '#{file}'"
+      elsif !File.exists?(File.dirname(file))
+        raise "create_config: directory '#{File.dirname(file)}' does not exist"
+      else
+        file_template = File.join(BackupMongoS3.root_path, 'lib/helpers/config.yml.template')
+
+        FileUtils.cp file_template, file
+      end
+
+      puts "[done] file #{file} was created"
+    end
+
+    def parse_options(argv)
+      params = {}
+
+      @parser.on('--backup_all', 'Backup databases specified in config.yml and upload to S3 bucket') do
+        params[:backup_all] = true
+      end
+      @parser.on('--backup DB_NAME', String, 'Backup database and upload to S3 bucket') do |db_name|
+        params[:backup] = db_name
+      end
+      @parser.on('-r', '--restore DB_NAME', String, 'Restore database from BACKUP_DATE backup') do |db_name|
+        params[:restore] = db_name
+      end
+      @parser.on('-d', '--date BACKUP_DATE', String, 'Restore date YYYYMMDD') do |backup_date|
+        params[:backup_date] = backup_date
+      end
+      @parser.on('-l', '--list_backups [DB_NAME]', String, 'Show list of available backups') do |db_name|
+        params[:backups_list] = db_name || ''
+      end
+      @parser.on('--write_cron', 'Add/update backup_all job') do
+        params[:cron_update] = true
+      end
+      @parser.on('--clear_cron', 'Clear backup_all job') do
+        params[:cron_clear] = true
+      end
+      @parser.on('-c', '--config PATH', String, 'Path to config *.yml. Default ./config.yml') do |path|
+        params[:config] = path || ''
+      end
+      @parser.on('--create_config [PATH]', String, 'Create template config.yml in current/PATH directory') do |path|
+        create_config(path)
+        exit
+      end
+      @parser.on('-h', '--help', 'Show help') do
+        puts "\n#{@parser}\n"
+        exit
+      end
+
+      begin
+        @parser.parse!(argv)
+
+      rescue OptionParser::ParseError => err
+        puts "#{err.message}\n\n#{@parser}"
+        exit
+      end
+
+      if [params[:backup_all], params[:backup], params[:restore], params[:backups_list], params[:cron_update], params[:cron_clear]].compact.length > 1
+        raise 'Can only backup_all, backup, restore, backups_list, cron_update or cron_clear. Choose one.'
+      end
+
+      if params[:config].nil? || params[:config] == ''
+        params[:config] = './config.yml'
+      end
+
+      params[:config] = File.absolute_path(params[:config])
+
+      params
+    end
+
+    def parse_config(config_path)
+
+      begin
+        config = YAML.load(File.read(config_path))
+      rescue Errno::ENOENT
+        raise "Could not find config file '#{config_path}'"
+      rescue ArgumentError => err
+        raise "Could not parse config file '#{config_path}' - #{err}"
+      end
+
+      config.deep_symbolize_keys!
+
+      raise 'config.yml. Section <backup> not found' if config[:backup].nil?
+      raise 'config.yml. Section <mongo> not found' if config[:mongo].nil?
+      raise 'config.yml. Section <s3> not found' if config[:s3].nil?
+
+      config
+    end
+
+    def get_temp_dir
+      temp_dir = @config[:backup][:temp_directory]
+
+      if temp_dir.nil? || temp_dir == ''
+        temp_dir = Dir.tmpdir
+      end
+
+      Dir.mktmpdir(nil, temp_dir)
+    end
+
+  end
+end

data/lib/backup_mongo_s3/db.rb

@@ -0,0 +1,45 @@
+module BackupMongoS3
+  class Db
+
+    def initialize(options)
+      @connection_options = connection(options)
+    end
+
+    private
+    def connection(options)
+
+      host     = (options[:host].nil? || options[:host].empty?) ? 'localhost' : options[:host]
+      port     = options[:port].nil? ? 27017 : options[:port]
+      username = options[:username]
+      password = options[:password]
+
+      auth_options = ''
+
+      unless username.nil? || username.empty? || password.nil? || password.empty?
+        auth_options = "-u '#{username}' -p '#{password}'"
+      end
+
+      "--host '#{host}' --port '#{port}' #{auth_options}"
+    end
+
+    public
+    def dump(db_name, backup_path)
+      command = "mongodump --dumpDbUsersAndRoles #{@connection_options} --db '#{db_name}' --out '#{backup_path}'"
+      command << ' > /dev/null'
+
+      system(command)
+      raise "Error mongodump '#{db_name}'" unless $?.exitstatus.zero?
+    end
+
+    public
+    def restore(db_name, backup_path)
+      command = "mongorestore --restoreDbUsersAndRoles #{@connection_options} --db '#{db_name}' '#{backup_path}'"
+      command << ' > /dev/null'
+
+      system(command)
+      raise "Error mongodump '#{db_name}'" unless $?.exitstatus.zero?
+    end
+
+
+  end
+end

data/lib/backup_mongo_s3/scheduler.rb

@@ -0,0 +1,102 @@
+module BackupMongoS3
+  class Scheduler
+
+    def initialize(options = {})
+      @options = options
+
+      if [@options[:write], @options[:clear]].compact.length > 1
+        raise 'cron: Can only write or clear. Choose one.'
+      end
+
+      unless @options[:time] =~ /\A[*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-6]+\z/
+        raise 'config.yml: cron_time is not valid'
+      end
+    end
+
+    def execute
+      write_crontab(updated_crontab)
+    end
+
+    private
+
+    def read_crontab
+      return @read_crontab if @read_crontab
+
+      command = 'crontab -l'
+
+      command_results = %x[#{command} 2> /dev/null]
+
+      @read_crontab = $?.exitstatus.zero? ? prepare(command_results) : ''
+    end
+
+    def prepare(contents)
+      # Some cron implementations require all non-comment lines to be newline-
+      # terminated. (issue #95) Strip all newlines and replace with the default
+      # platform record seperator ($/)
+      contents.gsub!(/\s+$/, $/)
+    end
+
+    def write_crontab(contents)
+      command = 'crontab -'
+
+      IO.popen(command, 'r+') do |crontab|
+        crontab.write(contents)
+        crontab.close_write
+      end
+
+      success = $?.exitstatus.zero?
+
+      if success
+        action = @options[:update] ? 'updated' : 'cleared'
+        puts "[done] crontab file #{action}"
+        exit(0)
+      else
+        raise "Couldn't write crontab"
+      end
+    end
+
+    def updated_crontab
+      # Check for unopened or unclosed identifier blocks
+      if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && (read_crontab =~ Regexp.new("^#{comment_close}\s*$")).nil?
+        raise "Unclosed indentifier; Your crontab file contains '#{comment_open}', but no '#{comment_close}'"
+      elsif (read_crontab =~ Regexp.new("^#{comment_open}\s*$")).nil? && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
+        raise "Unopened indentifier; Your crontab file contains '#{comment_close}', but no '#{comment_open}'"
+      end
+
+      # If an existing identier block is found, replace it with the new cron entries
+      if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
+        # If the existing crontab file contains backslashes they get lost going through gsub.
+        # .gsub('\\', '\\\\\\') preserves them. Go figure.
+        read_crontab.gsub(Regexp.new("^#{comment_open}\s*$.+^#{comment_close}\s*$", Regexp::MULTILINE), crontab_task.chomp.gsub('\\', '\\\\\\'))
+      else # Otherwise, append the new cron entries after any existing ones
+        [read_crontab, crontab_task].join("\n\n")
+      end.gsub(/\n{3,}/, "\n\n") # More than two newlines becomes just two.
+    end
+
+    def crontab_task
+      return '' if @options[:clear]
+      [comment_open, crontab_job, comment_close].compact.join("\n") + "\n"
+    end
+
+    def crontab_job
+      job = [@options[:time]]
+      job << BackupMongoS3.name
+      job << '--backup_all'
+      job << "--config #{@options[:config]}"
+
+      job.join(' ')
+    end
+
+    def comment_base
+      "#{BackupMongoS3.name} task"
+    end
+
+    def comment_open
+      "# Begin #{comment_base}"
+    end
+
+    def comment_close
+      "# End #{comment_base}"
+    end
+  end
+end

data/lib/backup_mongo_s3/storage.rb

@@ -0,0 +1,110 @@
+module BackupMongoS3
+  class Storage
+
+    def initialize(options)
+      @s3     = AWS::S3.new({access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key]})
+      @bucket = get_bucket(options[:bucket])
+    end
+
+    private
+    def get_bucket(bucket_name)
+      bucket = @s3.buckets[bucket_name]
+
+      unless bucket.exists?
+        raise "Bucket #{bucket_name} doesn't not exists. Please create it"
+      end
+
+      bucket
+    end
+
+    public
+    def upload(storage_path, file_name)
+      key = File.join(storage_path, Time.now.utc.strftime('%Y%m%d') + '.backup')
+
+      checksum = get_signature(file_name)
+
+      begin
+
+        file = File.open(file_name, 'rb')
+
+        obj = @bucket.objects[key]
+
+        obj.write(:content_length => file.size, metadata: {checksum: checksum}) do |buffer, bytes|
+          buffer.write(file.read(bytes))
+        end
+
+        file.close
+
+      rescue Exception => err
+        raise "Error upload file <#{file_name}> to s3 <#{key}>: #{err.message}"
+      end
+
+    end
+
+    public
+    def download(storage_path, storage_file_name, file_name)
+      key = File.join(storage_path, storage_file_name + '.backup')
+
+      begin
+
+        file = File.open(file_name, 'wb')
+
+        obj = @bucket.objects[key]
+
+        response = obj.read do |chunk|
+          file.write(chunk)
+        end
+
+        file.close
+
+        checksum = get_signature(file_name)
+
+        if checksum != response[:meta]['checksum']
+          raise 'Backup signature is not valid'
+        end
+
+      rescue Exception => err
+        raise "Error download file <#{key}> from s3 to <#{file_name}>: #{err.message}"
+      end
+
+    end
+
+    public
+    def get_backups_list(prefix = '', limit = 100)
+      result =[]
+
+      @bucket.objects.with_prefix(prefix).each(:limit => limit) do |object|
+        if File.extname(object.key) == '.backup'
+          result << object
+        end
+      end
+
+      result
+    end
+
+    public
+    def delete_old_backups(prefix, history_days)
+
+      old_backups = []
+
+      backups_time_limit = Time.now.utc.midnight - history_days.days
+
+      backups = get_backups_list(prefix)
+
+      backups.each do |backup|
+        if backup.last_modified.utc < backups_time_limit
+          old_backups << backup
+        end
+      end
+
+      @bucket.delete(old_backups) unless old_backups.empty?
+
+    end
+
+    private
+    def get_signature(file_name)
+      Digest::MD5.hexdigest(File.size(file_name).to_s)
+    end
+
+  end
+end

data/lib/backup_mongo_s3.rb

@@ -0,0 +1,28 @@
+require 'optparse'
+require 'aws-sdk'
+require 'fileutils'
+require 'digest/md5'
+require 'tmpdir'
+
+require_relative 'backup_mongo_s3/application'
+require_relative 'backup_mongo_s3/db'
+require_relative 'backup_mongo_s3/storage'
+require_relative 'backup_mongo_s3/scheduler'
+
+require_relative 'helpers/fixnum'
+require_relative 'helpers/hash'
+require_relative 'helpers/time'
+
+module BackupMongoS3
+
+  def self.name
+    File.basename( __FILE__, '.rb')
+  end
+
+  def self.root_path
+    @root_path if @root_path
+    spec = Gem::Specification.find_by_name(self.name)
+    @root_path = spec.gem_dir
+  end
+end
+

data/lib/helpers/config.yml.template

@@ -0,0 +1,23 @@
+backup:
+  # List dbs for backup when run backup_all
+  dbs: db1, db2
+
+  # All backups older than [today - history_days] will be deleted when run backup/backup_all
+  history_days: 5
+
+  # Temporary directory for dump files. Default system temp directory
+  temp_directory:
+
+  # [minute] [hour] [day] [month] [weekday]
+  cron_time: 0 0 * * *
+
+mongo:
+  host: 'localhost'
+  port: 27017
+  username:
+  password:
+
+s3:
+  access_key_id: 'access_key_id'
+  secret_access_key: 'secret_access_key'
+  bucket: 'backup_mongo_s3-backups'

data/lib/helpers/fixnum.rb

@@ -0,0 +1,11 @@
+class Fixnum
+  SECONDS_IN_DAY = 24 * 60 * 60
+
+  def days
+    self * SECONDS_IN_DAY
+  end
+
+  def ago
+    Time.now - self
+  end
+end

data/lib/helpers/hash.rb

@@ -0,0 +1,26 @@
+class Hash
+
+  def symbolize_keys!
+    transform_keys! { |key| key.to_sym rescue key }
+  end
+
+  def deep_symbolize_keys!
+    deep_transform_keys!{ |key| key.to_sym rescue key }
+  end
+
+  def transform_keys!
+    keys.each do |key|
+      self[yield(key)] = delete(key)
+    end
+    self
+  end
+
+  def deep_transform_keys!(&block)
+    keys.each do |key|
+      value = delete(key)
+      self[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys!(&block) : value
+    end
+    self
+  end
+
+end

data/lib/helpers/time.rb

@@ -0,0 +1,25 @@
+class Time
+
+  def midnight
+    change(:hour => 0)
+  end
+
+  def change(options)
+    new_year  = options.fetch(:year, year)
+    new_month = options.fetch(:month, month)
+    new_day   = options.fetch(:day, day)
+    new_hour  = options.fetch(:hour, hour)
+    new_min   = options.fetch(:min, options[:hour] ? 0 : min)
+    new_sec   = options.fetch(:sec, (options[:hour] || options[:min]) ? 0 : sec)
+    new_usec  = options.fetch(:usec, (options[:hour] || options[:min] || options[:sec]) ? 0 : Rational(nsec, 1000))
+
+    if utc?
+      ::Time.utc(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
+    elsif zone
+      ::Time.local(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
+    else
+      ::Time.new(new_year, new_month, new_day, new_hour, new_min, new_sec + (new_usec.to_r / 1000000), utc_offset)
+    end
+  end
+
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: backup_mongo_s3
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Yakupov Dima
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: Some description
+email:
+- yakupov.dima@mail.ru
+executables:
+- backup_mongo_s3
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- backup_mongo_s3.gemspec
+- bin/backup_mongo_s3
+- lib/backup_mongo_s3.rb
+- lib/backup_mongo_s3/application.rb
+- lib/backup_mongo_s3/db.rb
+- lib/backup_mongo_s3/scheduler.rb
+- lib/backup_mongo_s3/storage.rb
+- lib/helpers/config.yml.template
+- lib/helpers/fixnum.rb
+- lib/helpers/hash.rb
+- lib/helpers/time.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: Some summary
+test_files: []