dbget_client 0.1.4

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in dbget_client.gemspec
+gemspec

data/README

@@ -0,0 +1 @@
+dbget client

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/dbget

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+DBGET_LIB_PATH = File.join(File.dirname(__FILE__), '../lib')
+require File.join(DBGET_LIB_PATH, 'dbget')
+
+DBGet::Runner.boot(ARGV, 'mysql')

data/bin/mongoget

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+DBGET_LIB_PATH = File.join(File.dirname(__FILE__), '../lib')
+require File.join(DBGET_LIB_PATH, 'dbget')
+
+DBGet::Runner.boot(ARGV, 'mongo')

data/dbget_client.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'dbget/version'
+
+Gem::Specification.new do |s|
+  s.name        = "dbget_client"
+  s.version     = DBGet::VERSION
+  s.authors     = ["Jan Mendoza"]
+  s.email       = ["poymode@gmail.com"]
+  s.homepage    = "https://github.com/poymode/dbget_client"
+  s.summary     = %q{Client for dbget server}
+  s.description = %q{Decrypts and uncompress MySQL and MongoDB backups from dbget server}
+
+  s.rubyforge_project = "dbget_client"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/dbget.rb

@@ -0,0 +1,17 @@
+require 'fileutils'
+require 'yaml'
+require 'zlib'
+require 'optparse'
+
+DBGET_LIB_ROOT = File.expand_path('../dbget', __FILE__)
+require File.join(DBGET_LIB_ROOT, 'version')
+
+module DBGet
+  autoload :Initializer, File.join(DBGET_LIB_ROOT, 'initializer')
+  autoload :DBDump, File.join(DBGET_LIB_ROOT, 'db_dump')
+  autoload :Loaders, File.join(DBGET_LIB_ROOT, 'loaders')
+  autoload :Constants, File.join(DBGET_LIB_ROOT, 'constants')
+  autoload :ConfigLoader, File.join(DBGET_LIB_ROOT, 'config_loader')
+  autoload :Runner, File.join(DBGET_LIB_ROOT, 'runner')
+  autoload :Utils, File.join(DBGET_LIB_ROOT, 'utils')
+end

data/lib/dbget/config_loader.rb

@@ -0,0 +1,19 @@
+module DBGet
+  class ConfigLoader
+
+    def initialize(dbget_config, options = {})
+      @dbget_config = dbget_config
+      @options = options
+    end
+
+    def get_config(db_dump)
+      @db_dump = db_dump
+
+      if @dbget_config['mapping'].include?(db_dump.db)
+        @dbget_config
+      else
+        raise "#{db_dump.db} is not found in your dbget.yml!"
+      end
+    end
+  end
+end

data/lib/dbget/constants.rb

@@ -0,0 +1,6 @@
+module DBGet
+  module Constants
+    DBGET_CONFIG_FILE = 'dbget.yml'
+    MONGO_FILE_EXT = '.bson'
+  end
+end

data/lib/dbget/db_dump.rb

@@ -0,0 +1,124 @@
+module DBGet
+  class DBDump
+    include Utils
+    include Constants
+
+    attr_accessor :dump_path, :collections
+    attr_reader :db, :database, :db_path
+    attr_reader :host, :port, :username, :password
+    attr_reader :clean, :verbose
+
+    def initialize(options)
+      @db = options[:db]
+      @date = options[:date]
+      @key = options[:key]
+      @dbtype = options[:dbtype]
+      @server = options[:server]
+      @opt_db_name = options[:opt_db_name]
+      @verbose = options[:verbose]
+      @append_date = options[:append_date]
+      @clean = options[:clean]
+      @collections = options[:collections] || []
+      @header = {}
+    end
+
+    def load!(db_config)
+      @server ||= db_config['source']['server']
+      @key ||= db_config['key']
+      @database = @opt_db_name || db_config['mapping'][@db][@dbtype]
+      @host = db_config['target'][@dbtype]['host']
+      @port = db_config['target'][@dbtype]['port']
+      @username = db_config['target'][@dbtype]['username']
+      @password = db_config['target'][@dbtype]['password']
+      @dump_path = db_config['source']['dump_path']
+      @db_path = File.join(@dump_path, "#{@db}")
+
+      if @append_date
+        @database = @database.concat("_#{@date.delete('-')}")
+      end
+
+      get_dump(db_config)
+
+      if @header[:status] == 'SUCCESS'
+        if !@database.nil?
+          Utils.say "Dump info of #{@db} to #{@dbtype} using connection: \n" +
+          "  host: #{@host}\n" +
+          "  port: #{@port}\n" +
+          "  user: #{@username}\n" +
+          "  database: #{@database}\n"
+
+          if @dbtype == 'mysql'
+            Loaders::MySQL.new(self).load!
+          else
+            Loaders::MongoDB.new(self).load!
+          end
+        else
+          raise "Database #{@db} for #{@dbtype} is not found on #{DBGET_CONFIG_FILE}!"
+        end
+      else
+        raise "There was a problem fetching the database from the server!"
+      end
+    end
+
+    protected
+
+    def get_dump(db_config)
+      user = db_config['source']['user']
+      host = db_config['source']['host']
+
+      unless File.exist? @dump_path
+        FileUtils.mkdir_p(@dump_path)
+      end
+
+      ssh_params = "#{user}@#{host} db=#{@db} date=#{@date} " +
+                   "dbtype=#{@dbtype} server=#{@server}"
+
+      if !@key.nil?
+        ssh_cmd = "#{SSH_CMD} -i #{@key} #{ssh_params}"
+      else
+        ssh_cmd = "#{SSH_CMD} #{ssh_params}"
+      end
+
+      Utils.say "Fetching..."
+
+      io_handle = IO.popen(ssh_cmd)
+
+      parse_header(io_handle)
+
+      if !io_handle.eof?
+        File.open(@db_path, "w+") do |f|
+          gz = Zlib::GzipReader.new(io_handle)
+
+          while !io_handle.eof?
+            g = gz.read(16*1024)
+            f.write(g)
+          end
+
+          f.write(gz.read)
+
+          gz.close
+          f.close
+        end
+      end
+    end
+
+    def parse_header(io_handle)
+      while (s = io_handle.readline) != "\r\n"
+        s = s.split(': ')
+        @header[s.first.to_sym] = s.last.chomp
+      end
+
+      print_header if @verbose
+
+      if @header[:status] != "SUCCESS"
+        raise "Server returned #{@header[:status]}!"
+      end
+    end
+
+    def print_header
+      @header.each do |k, v|
+        puts ">> #{k.capitalize}: #{v}"
+      end
+    end
+  end
+end

data/lib/dbget/initializer.rb

@@ -0,0 +1,53 @@
+module DBGet
+  class Initializer
+    include Constants
+
+    def self.boot(options)
+      self.new(options).init
+    end
+
+    def initialize(options)
+      @options = options
+    end
+
+    def init
+      load_dbget_config
+
+      @options[:databases].each do |d|
+        @options[:db] = d
+
+        db_dump = DBGet::DBDump.new(@options)
+
+        load_db_dumps(db_dump)
+      end
+    end
+
+    protected
+
+    def load_dbget_config
+      config_path = File.join(@options[:dbget_path], DBGET_CONFIG_FILE)
+
+      if File.exists?(config_path)
+        @dbget_config = YAML.load_file(config_path)
+      else
+        raise "Cannot find #{config_path}!"
+      end
+    end
+
+    def load_db_dumps(db_dump)
+      if !@dbget_config.empty?
+        config_loader = DBGet::ConfigLoader.new(@dbget_config, @options)
+      else
+        raise "Your dbget.yml is empty!"
+      end
+
+      db_dump.load!(config_loader.get_config(db_dump))
+    end
+
+    def cleanup_db_dumps(db_dumps)
+      db_dumps.each do |db_dump|
+        db_dump.clean_up!
+      end
+    end
+  end
+end

data/lib/dbget/loaders.rb

@@ -0,0 +1,6 @@
+module DBGet
+  module Loaders
+    autoload :MongoDB, File.join(DBGET_LIB_ROOT, 'loaders/mongodb')
+    autoload :MySQL, File.join(DBGET_LIB_ROOT, 'loaders/mysql')
+  end
+end

data/lib/dbget/loaders/mongodb.rb

@@ -0,0 +1,69 @@
+module DBGet
+  module Loaders
+    class MongoDB
+      include Utils
+      include Constants
+
+      def self.boot(dump)
+        # some boot here
+
+        self.new(dump)
+      end
+
+      def initialize(dump)
+        @dump = dump
+      end
+
+      def load!
+        temp_path = File.join(@dump.dump_path, "#{@dump.db}_#{Utils.randomize(16)}")
+
+        if !File.exists?(temp_path)
+          FileUtils.mkdir(temp_path)
+
+          Utils.say_with_time "Extracting archive..." do
+            `#{TAR_CMD} -C #{temp_path} -xf #{File.join(@dump.dump_path, @dump.db)} 2> /dev/null`
+          end
+
+          Utils.say_with_time "Moving mongo files..." do
+            `#{FIND_CMD} #{temp_path} -name '*.#{MONGO_FILE_EXT}'`.each_line do |l|
+                FileUtils.mv(l.chomp!, File.join(temp_path, File.basename(l)))
+            end
+          end
+        end
+
+        dump_files = Dir["#{temp_path}/*#{MONGO_FILE_EXT}"]
+
+        if !@dump.collections.empty?
+          @dump.collections = @dump.collections.collect do |c|
+            File.join(temp_path, c.concat(MONGO_FILE_EXT))
+          end
+
+          dump_files &= @dump.collections
+        end
+
+        dump_files.each do |d|
+          # do not include indexes
+          if File.basename(d) != "system.indexes#{MONGO_FILE_EXT}"
+            Utils.say_with_time "Dumping #{d}..." do
+              if !@dump.verbose
+                `#{MONGORESTORE_CMD} -d #{@dump.database} #{d} --drop`
+              else
+                system "#{MONGORESTORE_CMD} -d #{@dump.database} #{d} --drop"
+              end
+            end
+          end
+        end
+
+        Utils.say "Hooray! Dump for #{@dump.db} done!"
+
+        if FileUtils.rm_rf(File.join(@dump.dump_path, @dump.db))
+         Utils.say "Dump file removed!"
+        end
+
+        if FileUtils.rm_rf(temp_path)
+          Utils.say "Temp directory removed!"
+        end
+      end
+    end
+  end
+end

data/lib/dbget/loaders/mysql.rb

@@ -0,0 +1,50 @@
+module DBGet
+  module Loaders
+    class MySQL
+      include Utils
+      include Constants
+
+      def self.boot(dump)
+        # some boot here
+
+        self.new(dump)
+      end
+
+      def initialize(dump)
+        @dump = dump
+      end
+
+      def load!
+        command = "#{MYSQL_CMD} "
+        command += "-h#{@dump.host} "
+        command += "-P#{@dump.port} "
+        command += "-u#{@dump.username} "
+        command += "-p#{@dump.password} " if @dump.password
+
+        if @dump.clean
+          Utils.say_with_time "Dropping database..." do
+            system "echo \"DROP DATABASE IF EXISTS #{@dump.database}\" | #{command}"
+          end
+        end
+
+        system "echo \"CREATE DATABASE IF NOT EXISTS #{@dump.database}\" | #{command}"
+
+        if File.exist?(@dump.db_path) and !File.size?(@dump.db_path).nil?
+          command += " #{@dump.database} "
+
+          Utils.say_with_time "Dumping #{@dump.db}..." do
+            system "#{command}< #{File.join(@dump.dump_path, @dump.db)}"
+          end
+
+          if FileUtils.rm_rf(File.join(@dump.dump_path, @dump.db))
+            Utils.say "Cleaned temporary file!"
+          end
+        else
+          raise "Dump for #{@dump.db} not found!"
+        end
+
+        Utils.say "Hooray! Dump for #{@dump.db} done!"
+      end
+    end
+  end
+end

data/lib/dbget/runner.rb

@@ -0,0 +1,102 @@
+module DBGet
+  class Runner
+    def self.boot(args, dbtype)
+      DBGet::Constants.const_set("MYSQL_CMD", `which mysql 2>/dev/null`.strip)
+      DBGet::Constants.const_set("SSH_CMD", `which ssh 2>/dev/null`.strip)
+      DBGet::Constants.const_set("MONGORESTORE_CMD", `which mongorestore 2>/dev/null`.strip)
+      DBGet::Constants.const_set("FIND_CMD", `which find 2>/dev/null`.strip)
+      DBGet::Constants.const_set("TAR_CMD", `which tar 2>/dev/null`.strip)
+
+      self.new(args, dbtype)
+    end
+
+    def initialize(args, dbtype)
+      options = {}
+      options[:append_date] = false
+      options[:verbose] = false
+
+      optparse = OptionParser.new do |opts|
+        opts.banner = "Usage: dbget [options] db1 db2 ...\n"
+        opts.separator "Options:"
+
+        opts.on('-d', '--date DATE', 'Date of database dump (yyyy-mm-dd).') do |date|
+          options[:date] = date
+        end
+
+        opts.on('-c', '--clean', 'Drops the database before dumping') do
+          options[:clean] = true
+        end
+
+        opts.on('-f', '--force', 'Force the use of dbget in production.') do
+          options[:force] = true
+        end
+
+        opts.on('-n', '--name NAME', 'Explicitly declare a database name for dumping.') do |name|
+          options[:opt_db_name] = name
+        end
+
+        opts.on('-i', '--key KEY', 'Specify ssh connection key') do |key|
+          options[:key] = key
+        end
+
+        opts.on('-s', '--server SERVER', 'Specify the server that contained the database.') do |server|
+          options[:server] = server
+        end
+
+        opts.on('-a', '--append-date', 'Append the given date as suffix') do
+          options[:append_date] = true
+        end
+
+        opts.on('--collections COLLECTION', Array, 'Only dump specific mongodb collections separated by comma') do |collection|
+          options[:collections] = collection
+        end
+
+        opts.on('-v', '--verbose', 'Execute NERD mode!') do
+          options[:verbose] = true
+        end
+
+        opts.on('-V', '--version', 'Version') do
+          puts DBGet::VERSION
+          exit
+        end
+
+        opts.on('-h', '--help', 'Display this screen') do
+          puts opts
+          exit
+        end
+      end
+
+      optparse.parse!
+
+      if args.length == 0
+        puts optparse.help
+      else
+        options[:dbtype] = dbtype
+        options[:databases] = args.dup
+
+        run(options)
+      end
+    end
+
+    def run(opts)
+      if ENV.include?('DBGET_PATH')
+        opts[:dbget_path] = ENV['DBGET_PATH']
+      else
+        raise "Cannot find DBGET_PATH!"
+      end
+
+      # lets turn off opt_db_name if we have more arguments
+      # this feature is not supported at the moment
+      if opts[:databases].count > 1 and opts.include?(:opt_db_name)
+        raise "You cannot use -n with multiple databases!"
+      end
+
+      # check if -a switch is give but no -d
+      if opts[:append_date] and !opts.include?(:date)
+        raise "You cannnot use -a without -d!"
+      end
+
+      DBGet::Initializer.boot(opts)
+    end
+  end
+end

data/lib/dbget/utils.rb

@@ -0,0 +1,23 @@
+require 'benchmark'
+
+module DBGet
+  module Utils
+    def self.say(message, subitem = false)
+      puts "#{subitem ? "   ->" : "--"} #{message}"
+    end
+
+    def self.say_with_time(message)
+      say(message)
+      result = nil
+      time = Benchmark.measure { result = yield }
+      say "%.4fs" % time.real, :subitem
+      say("#{result} rows", :subitem) if result.is_a?(Integer)
+      result
+    end
+
+    def self.randomize(size)
+      chars = ('a'..'z').to_a + ('A'..'Z').to_a
+      (0...size).collect { chars[Kernel.rand(chars.length)] }.join
+    end
+  end
+end

data/lib/dbget/version.rb

@@ -0,0 +1,3 @@
+module DBGet
+  VERSION = "0.1.4"
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: dbget_client
+version: !ruby/object:Gem::Version
+  version: 0.1.4
+  prerelease: 
+platform: ruby
+authors:
+- Jan Mendoza
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-20 00:00:00.000000000Z
+dependencies: []
+description: Decrypts and uncompress MySQL and MongoDB backups from dbget server
+email:
+- poymode@gmail.com
+executables:
+- dbget
+- mongoget
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README
+- Rakefile
+- bin/dbget
+- bin/mongoget
+- dbget_client.gemspec
+- lib/dbget.rb
+- lib/dbget/config_loader.rb
+- lib/dbget/constants.rb
+- lib/dbget/db_dump.rb
+- lib/dbget/initializer.rb
+- lib/dbget/loaders.rb
+- lib/dbget/loaders/mongodb.rb
+- lib/dbget/loaders/mysql.rb
+- lib/dbget/runner.rb
+- lib/dbget/utils.rb
+- lib/dbget/version.rb
+homepage: https://github.com/poymode/dbget_client
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: dbget_client
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Client for dbget server
+test_files: []