dbget 0.2.2

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+config/*.yml

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in dbget.gemspec
+gemspec

data/README

@@ -0,0 +1 @@
+initial commit for dbget

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/dbget-adm

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join(File.dirname(__FILE__), '../lib/dbget'))
+
+DBGet::Admin.start

data/bin/dbget-serve

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join(File.dirname(__FILE__), '../lib/dbget'))
+exit if ENV['SSH_ORIGINAL_COMMAND'].nil?
+
+options = ENV['SSH_ORIGINAL_COMMAND'].split(" ")
+user = ARGV[0]
+
+runner = DBGet::Runner.boot(user, options)

data/config/dbget.yml.sample

@@ -0,0 +1,28 @@
+backups:
+  path: /opt/backups
+  cache: /tmp/dbget_backups
+
+openssl:
+  passphrase: xxx
+
+databases:
+  lmedi:
+    mongo: lmedi_production
+  clinic:
+    mysql: clinic_production
+  test:
+    mysql: test
+
+users: ['poy', 'jan']
+
+mysql:
+  host: localhost
+  port: 3306
+  username: root
+  password:
+
+mongo:
+  host: localhost
+  port: 27017
+  username: root
+  password:

data/config/mask.yml

@@ -0,0 +1,16 @@
+people:
+  columns:
+    email: 'masked@caresharing.eu'
+    house_number: '01'
+    postal_code: '1234XX'
+    mobile_number: '123-4567'
+    phone_number: '123-4567'
+    firstname:
+    lastname:
+    street_name:
+    region:
+    birthplace:
+    city:
+patient_list_displays:
+  columns:
+    name:

data/dbget.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'dbget/version'
+
+Gem::Specification.new do |s|
+  s.name        = "dbget"
+  s.version     = DBGet::VERSION
+  s.authors     = ["Jan Mendoza"]
+  s.email       = ["poymode@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Get an encrypted and compressed backup of a MySQL or mongoDB}
+  s.description = %q{This serves the backup}
+
+  s.rubyforge_project = "dbget"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'thor'
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/dbget/admin.rb

@@ -0,0 +1,30 @@
+require 'thor'
+
+module DBGet
+  class Admin < Thor
+    desc "generate_authorized_keys", "generate authorized keys for access"
+    def generate_authorized_keys
+      if File.exist? File.join(Dir.pwd, 'keydir')
+        key_dir = File.join(Dir.pwd, 'keydir')
+      elsif ENV.include?('DBGET_KEYDIR_PATH')
+        key_dir = ENV['DBGET_KEYDIR_PATH']
+      else
+        key_dir = ''
+      end
+
+      raise "Key directory 'keydir' not found." unless File.exists?(key_dir) and File.directory?(key_dir)
+
+      puts "### Generated by dbget, DO NOT EDIT"
+
+      keys = Dir["#{key_dir}/*.pub"]
+      keys.each do |pub_key|
+        bin_path = 'dbget-serve'
+        key_name = File.basename(pub_key, '.pub')
+        key_data = File.read(pub_key)
+
+        puts "command=\"#{bin_path} #{key_name}\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty #{key_data}"
+      end
+    end
+  end
+
+end

data/lib/dbget/binaries.rb

@@ -0,0 +1,23 @@
+module DBGet
+  module Binaries
+    def self.get_binary(bin)
+      `which #{bin}`.chomp
+    end
+
+    def self.mysql_cmd
+      get_binary('mysql')
+    end
+
+    def self.mongorestore_cmd
+      get_binary('mongorestore')
+    end
+
+    def self.find_cmd
+      get_binary('find')
+    end
+
+    def self.tar_cmd
+      get_binary('tar')
+    end
+  end
+end

data/lib/dbget/config.rb

@@ -0,0 +1,17 @@
+require 'singleton'
+
+module DBGet
+  class Config < Hash
+    include Singleton
+
+    def self.load_from_yml(yaml_path)
+      config = self.instance
+      config.clear
+      config.merge!(YAML.load_file(yaml_path))
+    end
+
+    def self.database(name)
+      self.instance['databases'][name]
+    end
+  end
+end

data/lib/dbget/constants.rb

@@ -0,0 +1,7 @@
+module DBGet
+  module Constants
+    MONGO_FILE_EXT = '.bson'
+    MONGO_INDEX_FILE = 'system.indexes'
+    CONFIG_PATH = 'config/dbget.yml'
+  end
+end

data/lib/dbget/controller.rb

@@ -0,0 +1,61 @@
+module DBGet
+  class Controller
+    def self.boot(options)
+      self.new(options).run!
+    end
+
+    def initialize(options)
+      @dump = DBGet::Dump.new(options)
+      @config = DBGet::Config.instance
+    end
+
+    def run!
+      user_allowed?
+      @dump.prepare
+      finalize_dump
+      load_dump
+      status_report
+    end
+
+    def user_allowed?
+      unless @config['users'].include?(@dump.user)
+        raise "User not allowed to dump"
+        exit 1
+      end
+    end
+
+    def finalize_dump
+      if @dump.in_cache? @dump.cache_file
+        message = "Found cached file, no need to decrypt."
+      else
+        message = "Decrypting #{@dump.encrypted_dump}..."
+      end
+
+      Utils.say_with_time message do
+        @dump.decrypted_dump = @dump.decrypt_dump
+      end
+
+      @dump.set_final_db
+    end
+
+    def load_dump
+      Utils.say_with_time "Dumping, this may take a while" do
+        case @dump.db_type
+        when 'mysql'
+          DBGet::Loaders::MySql.boot(@dump, @config).load!
+        when 'mongo'
+          DBGet::Loaders::Mongo.boot(@dump, @config).load!
+        else
+          raise "Dump is not supported!"
+        end
+      end
+
+    end
+
+    def status_report
+      Utils.say "Dump for #{@dump.db} done!"
+      Utils.say "Source: #{@dump.decrypted_dump}"
+      Utils.say "Target: #{@dump.target_db}"
+    end
+  end
+end

data/lib/dbget/dump.rb

@@ -0,0 +1,150 @@
+require 'fileutils'
+
+module DBGet
+  class Dump
+    include Constants
+    include Utils
+
+    attr_reader :db_type, :user, :db
+    attr_reader :encrypted_dump
+    attr_reader :date, :clean, :verbose, :append_date
+    attr_accessor :backup_name, :collections, :target_db, :decrypted_dump
+
+    def initialize(opts)
+      @db = opts['db']
+      @db_type = opts['db_type']
+      @user = opts['user']
+      @server = opts['server']
+      @custom_name = opts['custom_name']
+
+      @collections = opts['collections']
+      @date = opts['date']
+      @clean = opts['clean']
+      @verbose = opts['verbose']
+      @append_date = opts['append_date']
+
+      @storage_path = File.join(DBGet.base_backups_path, @server, @db_type)
+    end
+
+    def prepare
+      @backup_name = get_backup_name
+      @encrypted_dump = get_encrypted_dump
+    end
+
+    def set_final_db
+      @target_db = @custom_name || @backup_name
+    end
+
+    def form_db_name
+      if !date.nil? and append_date
+        self.target_db = "#{self.user}_#{self.target_db}_#{Utils.get_converted_date(self.date)}"
+      else
+        self.target_db = "#{self.user}_#{self.target_db}"
+      end
+    end
+
+    def clean?
+      @clean
+    end
+
+    def decrypt_dump
+      unless @encrypted_dump.nil?
+        if in_cache? cache_file
+          file_path = cache_file
+        elsif File.exists? @encrypted_dump
+          file_path = Utils.decompress_file(decrypt_file(copy_to_cache(@encrypted_dump)))
+        end
+
+        file_path
+      end
+    end
+
+    def in_cache?(file)
+      File.exists? file
+    end
+
+    def cache_file
+      file = File.join(DBGet.cache_path, decrypted_file_name)
+      file.concat('.tar') if @db_type.eql? 'mongo'
+
+      file
+    end
+
+    protected
+
+    def latest?
+      @date.nil?
+    end
+
+    def decrypted_file_name
+      File.basename(@encrypted_dump.split('.').first)
+    end
+
+    def get_backup_name
+      if !@db.nil? and !@db_type.nil?
+        db = DBGet::Config.database(@db)
+
+        if db.nil?
+          raise "Database \'#{@db}\' not found in config!"
+        end
+
+        db[@db_type]
+      end
+    end
+
+    def get_encrypted_dump
+      monthly_dir_path = File.join(@storage_path, @backup_name)
+
+      unless File.exist?(monthly_dir_path)
+        raise "Database \'#{@backup_name}\' can't be found in #{@storage_path}!"
+      end
+
+      monthly_dirs = Utils.get_files(monthly_dir_path).sort
+
+      if !monthly_dirs.empty?
+        db_month = get_db_month(monthly_dirs)
+        db_dumps_path = File.join(monthly_dir_path.to_s, db_month.to_s)
+        db_dumps = Utils.get_files(db_dumps_path)
+
+        @encrypted_dump = File.join(db_dumps_path, get_db_file(db_dumps))
+      end
+    end
+
+    def get_db_month(monthly_dirs)
+      if latest?
+        File.basename(monthly_dirs.last)
+      else
+        @date.strftime("%Y%m")
+      end
+    end
+
+    def get_db_file(db_dumps)
+      if latest?
+        db_file = db_dumps.last
+      else
+        db_dumps.each do |filename|
+          if filename.include? "-#{@date.strftime("%Y%m%d")}"
+            db_file = filename
+            break
+          end
+        end
+      end
+
+      db_file
+    end
+
+    def copy_to_cache(file_path)
+      FileUtils.mkdir_p(DBGet.cache_path)
+      FileUtils.cp(file_path, DBGet.cache_path)
+
+      File.join(DBGet.cache_path, File.basename(file_path))
+    end
+
+    def decrypt_file(file_path)
+      if File.extname(file_path) == ".enc"
+        Utils.decode_file(file_path)
+      end
+    end
+
+  end
+end

data/lib/dbget/loaders/mongo.rb

@@ -0,0 +1,77 @@
+module DBGet
+  module Loaders
+    class Mongo
+      include Constants
+
+      def self.boot(dump, config)
+        self.new(dump, config)
+      end
+
+      def initialize(dump, config)
+        @dump = dump
+        @config = config
+      end
+
+      def load!
+        temp_path = get_temp_path
+
+        FileUtils.mkdir(temp_path) if !File.exists?(temp_path)
+
+        @dump.form_db_name
+        extract_mongo_dump(temp_path)
+        prepare_bson_files(temp_path)
+
+        dump_files = Dir["#{temp_path}/*#{MONGO_FILE_EXT}"]
+        dump_files = specify_collections(dump_files, temp_path)
+        mongo_restore(dump_files)
+        remove_temp_path(temp_path)
+      end
+
+      def mongo_restore(dump_files)
+        dump_files.each do |file|
+          unless index?(file)
+            mongo_restore = Binaries.mongorestore_cmd
+
+            system "#{mongo_restore} -d #{@dump.target_db} #{file} --drop"
+          end
+        end
+      end
+
+      def get_temp_path
+        random = Utils.randomize(16)
+        decrypted_basename = File.basename(@dump.decrypted_dump, '.tar')
+        temp_path = File.join(DBGet.cache_path, "#{decrypted_basename}_#{random}")
+      end
+
+      def remove_temp_path(path)
+        FileUtils.rm_rf(path)
+      end
+
+      def index?(file)
+        File.basename(file) == "#{MONGO_INDEX_FILE}#{MONGO_FILE_EXT}"
+      end
+
+      def specify_collections(dump_files, temp_path)
+        if !@dump.collections.empty?
+          @dump.collections = @dump.collections.collect do |c|
+            File.join(temp_path, c.concat(MONGO_FILE_EXT))
+          end
+
+          dump_files &= @dump.collections
+        end
+
+        dump_files
+      end
+
+      def extract_mongo_dump(temp_path)
+        `#{Binaries.tar_cmd} -C #{temp_path} -xf #{@dump.decrypted_dump} 2> /dev/null`
+      end
+
+      def prepare_bson_files(temp_path)
+        `#{Binaries.find_cmd} #{temp_path} -name '*#{MONGO_FILE_EXT}'`.each_line do |l|
+          FileUtils.mv(l.chomp!, File.join(temp_path, File.basename(l)))
+        end
+      end
+    end
+  end
+end

data/lib/dbget/loaders/mysql.rb

@@ -0,0 +1,55 @@
+module DBGet
+  module Loaders
+    class MySql
+      include Constants
+      include Binaries
+
+      def self.boot(dump, config)
+        self.new(dump, config)
+      end
+
+      def initialize(dump, config)
+        @dump = dump
+        @config = config
+        @date = @dump.date
+        @dump_command = form_dump_command
+      end
+
+      def load!
+        @dump.form_db_name
+        clean_dump if @dump.clean?
+        create_db_if_not_exist
+        dump_mysql
+      end
+
+      def form_dump_command
+        @dump_command = "#{Binaries.mysql_cmd} "
+        @dump_command += "-h#{@config['mysql']['host']} "
+        @dump_command += "-P#{@config['mysql']['port']} "
+        @dump_command += "-u#{@config['mysql']['username']} "
+        @dump_command += "-p#{@config['mysql']['password']} " if @config['mysql']['password']
+        @dump_command
+      end
+
+      def get_converted_date
+        @dump.date.to_s.delete('-')
+      end
+
+      def clean_dump
+        puts "Dropping database..."
+        system "echo \"DROP DATABASE IF EXISTS #{@dump.target_db}\" | #{@dump_command}"
+      end
+
+      def create_db_if_not_exist
+        system "echo \"CREATE DATABASE IF NOT EXISTS #{@dump.target_db}\" | #{@dump_command}"
+      end
+
+      def dump_mysql
+        @dump_command += " #{@dump.target_db} "
+
+        out = system("#{@dump_command}< #{@dump.decrypted_dump}")
+        exit 1 unless out
+      end
+    end
+  end
+end

data/lib/dbget/loaders.rb

@@ -0,0 +1,6 @@
+module DBGet
+  module Loaders
+    autoload :MySql, File.join(File.dirname(__FILE__), 'loaders/mysql')
+    autoload :Mongo, File.join(File.dirname(__FILE__), 'loaders/mongo')
+  end
+end

data/lib/dbget/runner.rb

@@ -0,0 +1,67 @@
+module DBGet
+  class Runner
+    def self.boot(user, options)
+      self.new(parse_options(user, options)).run!
+    end
+
+    def self.parse_options(user, options)
+      opts = {}
+      opts['user'] = user
+
+      options.each do |o|
+        k, v = o.split('=')
+        opts[k] = v
+      end
+
+      opts
+    end
+
+    def initialize(options)
+      @options = options
+    end
+
+    def run!
+      init_config(get_dbget_path)
+      prepare_dump_options
+      dump
+    end
+
+    def prepare_dump_options
+      @options['collections'] = get_final_collection(@options['collections'])
+      @options['date'] = format_date(@options['date'])
+      @options['clean'] = Utils.to_bool(@options['clean'])
+      @options['verbose'] = Utils.to_bool(@options['verbose'])
+      @options['append_date'] = Utils.to_bool(@options['append_date'])
+    end
+
+    def get_final_collection(collection)
+      unless collection == 'EMPTY'
+        collection.split(',')
+      else
+        []
+      end
+    end
+
+    def format_date(date)
+      unless date == 'xxxx-xx-xx'
+        Date.strptime(date, "%Y-%m-%d")
+      end
+    end
+
+    def dump
+      Controller.boot(@options)
+    end
+
+    def get_dbget_path
+      if ENV.include?('DBGET_PATH')
+        ENV['DBGET_PATH']
+      else
+        raise "DBGET_PATH not defined"
+      end
+    end
+
+    def init_config(dbget_path)
+      DBGet.read_config(dbget_path)
+    end
+  end
+end

data/lib/dbget/utils.rb

@@ -0,0 +1,65 @@
+require 'csv'
+require 'benchmark'
+
+module DBGet
+  module Utils
+
+    def self.get_files(path)
+      Dir.new(path).entries - %w{ . .. }
+    end
+
+    def self.decode_file(input_file)
+      output_file = input_file.sub(/\.enc$/, "")
+      pass_phrase = DBGet::Config.instance['openssl']['passphrase']
+
+      command =  "openssl enc -d -aes-256-cbc -in #{input_file}"
+      command += " -pass pass:#{pass_phrase}"
+      command += " -out #{output_file}; rm #{input_file}"
+      system command
+
+      output_file
+    end
+
+    def self.decompress_file(input_file)
+      command = "gzip -df #{input_file}"
+      system command
+
+      input_file.sub(/\.gz$/, "")
+    end
+
+    def self.compress_file(input_file)
+      command = "gzip #{input_file}"
+      system command
+
+      input_file.concat('.gz')
+    end
+
+    def self.say(message, subitem = false)
+      puts "#{subitem ? "   ->" : "--"} #{message}"
+    end
+
+    def self.say_with_time(message)
+      say(message)
+      result = nil
+      time = Benchmark.measure { result = yield }
+      say "%.4fs" % time.real, :subitem
+      say("#{result} rows", :subitem) if result.is_a?(Integer)
+      result
+    end
+
+    def self.randomize(size)
+      chars = ('a'..'z').to_a + ('A'..'Z').to_a
+      (0...size).collect { chars[Kernel.rand(chars.length)] }.join
+    end
+
+    def self.to_bool(string)
+      return true if string == true || string =~ (/(true|t|yes|y|1)$/i)
+      return false if string == false || string.empty? || string =~ (/(false|f|no|n|0)$/i)
+      raise ArgumentError.new("invalid value for Boolean: \"#{string}\"")
+    end
+
+    def self.get_converted_date(date)
+      date.to_s.delete('-')
+    end
+  end
+end

data/lib/dbget/version.rb

@@ -0,0 +1,3 @@
+module DBGet
+  VERSION = '0.2.2'
+end

data/lib/dbget.rb

@@ -0,0 +1,26 @@
+require 'yaml'
+
+module DBGet
+  autoload :Admin, File.join(File.dirname(__FILE__), 'dbget/admin')
+  autoload :Binaries, File.join(File.dirname(__FILE__), 'dbget/binaries')
+  autoload :Config, File.join(File.dirname(__FILE__), 'dbget/config')
+  autoload :Constants, File.join(File.dirname(__FILE__), 'dbget/constants')
+  autoload :Dump, File.join(File.dirname(__FILE__), 'dbget/dump')
+  autoload :Controller, File.join(File.dirname(__FILE__), 'dbget/controller')
+  autoload :Loaders, File.join(File.dirname(__FILE__), 'dbget/loaders')
+  autoload :Runner, File.join(File.dirname(__FILE__), 'dbget/runner')
+  autoload :Utils, File.join(File.dirname(__FILE__), 'dbget/utils')
+
+  def self.read_config(dbget_path)
+    config_file = File.join(dbget_path, Constants::CONFIG_PATH)
+    DBGet::Config.load_from_yml(config_file)
+  end
+
+  def self.base_backups_path
+    DBGet::Config.instance['backups']['path']
+  end
+
+  def self.cache_path
+    DBGet::Config.instance['backups']['cache']
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: dbget
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+  prerelease: 
+platform: ruby
+authors:
+- Jan Mendoza
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-27 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &15915760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *15915760
+description: This serves the backup
+email:
+- poymode@gmail.com
+executables:
+- dbget-adm
+- dbget-serve
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README
+- Rakefile
+- bin/dbget-adm
+- bin/dbget-serve
+- config/dbget.yml.sample
+- config/mask.yml
+- dbget.gemspec
+- lib/dbget.rb
+- lib/dbget/admin.rb
+- lib/dbget/binaries.rb
+- lib/dbget/config.rb
+- lib/dbget/constants.rb
+- lib/dbget/controller.rb
+- lib/dbget/dump.rb
+- lib/dbget/loaders.rb
+- lib/dbget/loaders/mongo.rb
+- lib/dbget/loaders/mysql.rb
+- lib/dbget/runner.rb
+- lib/dbget/utils.rb
+- lib/dbget/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: dbget
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Get an encrypted and compressed backup of a MySQL or mongoDB
+test_files: []