dbget_client 0.1.4 → 0.2.2

data/bin/dbget

@@ -3,4 +3,4 @@
 DBGET_LIB_PATH = File.join(File.dirname(__FILE__), '../lib')
 require File.join(DBGET_LIB_PATH, 'dbget')
 
-DBGet::Runner.boot(ARGV, 'mysql')
+DBGet::Runner.new.run!(ARGV, 'mysql')

data/bin/mongoget

@@ -3,4 +3,4 @@
 DBGET_LIB_PATH = File.join(File.dirname(__FILE__), '../lib')
 require File.join(DBGET_LIB_PATH, 'dbget')
 
-DBGet::Runner.boot(ARGV, 'mongo')
+DBGet::Runner.new.run!(ARGV, 'mongo')

data/lib/dbget.rb

@@ -1,17 +1,15 @@
 require 'fileutils'
 require 'yaml'
-require 'zlib'
 require 'optparse'
 
 DBGET_LIB_ROOT = File.expand_path('../dbget', __FILE__)
 require File.join(DBGET_LIB_ROOT, 'version')
 
 module DBGet
-  autoload :Initializer, File.join(DBGET_LIB_ROOT, 'initializer')
-  autoload :DBDump, File.join(DBGET_LIB_ROOT, 'db_dump')
-  autoload :Loaders, File.join(DBGET_LIB_ROOT, 'loaders')
+  autoload :Config, File.join(DBGET_LIB_ROOT, 'config')
+  autoload :Connector, File.join(DBGET_LIB_ROOT, 'connector')
   autoload :Constants, File.join(DBGET_LIB_ROOT, 'constants')
-  autoload :ConfigLoader, File.join(DBGET_LIB_ROOT, 'config_loader')
+  autoload :Controller, File.join(DBGET_LIB_ROOT, 'controller')
   autoload :Runner, File.join(DBGET_LIB_ROOT, 'runner')
   autoload :Utils, File.join(DBGET_LIB_ROOT, 'utils')
 end

data/lib/dbget/config.rb

@@ -0,0 +1,14 @@
+require 'singleton'
+
+module DBGet
+  class Config < Hash
+    include Singleton
+
+    def self.load_from_yaml(yaml_path)
+      config = self.instance
+      config.clear
+      config.merge!(YAML.load_file(yaml_path))
+    end
+  end
+end
+

data/lib/dbget/connector.rb

@@ -0,0 +1,57 @@
+require 'net/ssh'
+
+module DBGet
+  class Connector
+    include Utils
+    include Constants
+
+    attr_accessor :collections
+    attr_reader :db, :db_path
+    attr_reader :clean, :verbose
+
+    def initialize(database, config, options)
+      @db = database
+      @options = Utils.stringify(options)
+      @config = config.merge!(@options)
+      @login = @config['login']
+      @db_type = @options['db_type']
+      @server = @config['server']
+      @date = @config['date']
+      @clean = @config['clean']
+      @verbose = @config['verbose']
+      @collections = @config['collections']
+      @custom_name = @config['name']
+      @append_date = @config['append_date']
+      @ssh_params = init_ssh_params
+    end
+
+    def send_data!
+      user, host = @config['login'].split('@')
+
+      Net::SSH.start(host, user) do |ssh|
+
+        channel = ssh.open_channel do |ch|
+          ch.exec(@ssh_params) do |ch, success|
+            raise "SSH connection failure." unless success
+
+            ch.on_data do |c,data|
+              $stdout.puts data
+            end
+
+            ch.on_extended_data do |c, type, data|
+              $stderr.print data
+            end
+
+          end
+          channel.wait
+
+        end
+      end
+    end
+
+    def init_ssh_params
+      "%s db=%s db_type=%s server=%s date=%s clean=%s verbose=%s collections=%s custom_name=%s append_date=%s" %
+        [@login, @db, @db_type, @server, @date, @clean, @verbose, @collections, @custom_name, @append_date]
+    end
+  end
+end

data/lib/dbget/constants.rb

@@ -1,6 +1,5 @@
 module DBGet
   module Constants
     DBGET_CONFIG_FILE = 'dbget.yml'
-    MONGO_FILE_EXT = '.bson'
   end
 end

data/lib/dbget/controller.rb

@@ -0,0 +1,43 @@
+module DBGet
+  class Controller
+    include Constants
+
+    attr_reader :connector, :config, :databases, :options
+
+    def initialize(databases, options)
+      @databases = databases
+      @options = options
+      @connections = []
+    end
+
+    def boot
+      load_dbget_config
+      @config = DBGet::Config.instance
+      store_connections
+    end
+
+    def store_connections
+      @databases.each do |database|
+        @connections << Connector.new(database, @config, @options)
+      end
+    end
+
+    def send_data!
+      @connections.each do |connection|
+        connection.send_data!
+      end
+    end
+
+    protected
+
+    def load_dbget_config
+      config_path = File.join(@options[:dbget_path], DBGET_CONFIG_FILE)
+
+      if File.exists?(config_path)
+        DBGet::Config.load_from_yaml(config_path)
+      else
+        raise "Cannot find #{config_path}!"
+      end
+    end
+  end
+end

data/lib/dbget/runner.rb

@@ -1,58 +1,47 @@
 module DBGet
   class Runner
-    def self.boot(args, dbtype)
-      DBGet::Constants.const_set("MYSQL_CMD", `which mysql 2>/dev/null`.strip)
-      DBGet::Constants.const_set("SSH_CMD", `which ssh 2>/dev/null`.strip)
-      DBGet::Constants.const_set("MONGORESTORE_CMD", `which mongorestore 2>/dev/null`.strip)
-      DBGet::Constants.const_set("FIND_CMD", `which find 2>/dev/null`.strip)
-      DBGet::Constants.const_set("TAR_CMD", `which tar 2>/dev/null`.strip)
-
-      self.new(args, dbtype)
+    def initialize
+      @options = {}
+      set_default_options
+      @optparse = init_option_parser
+      @args = nil
     end
 
-    def initialize(args, dbtype)
-      options = {}
-      options[:append_date] = false
-      options[:verbose] = false
-
+    def init_option_parser
       optparse = OptionParser.new do |opts|
         opts.banner = "Usage: dbget [options] db1 db2 ...\n"
         opts.separator "Options:"
 
         opts.on('-d', '--date DATE', 'Date of database dump (yyyy-mm-dd).') do |date|
-          options[:date] = date
+          @options[:date] = date
         end
 
         opts.on('-c', '--clean', 'Drops the database before dumping') do
-          options[:clean] = true
-        end
-
-        opts.on('-f', '--force', 'Force the use of dbget in production.') do
-          options[:force] = true
+          @options[:clean] = true
         end
 
-        opts.on('-n', '--name NAME', 'Explicitly declare a database name for dumping.') do |name|
-          options[:opt_db_name] = name
-        end
-
-        opts.on('-i', '--key KEY', 'Specify ssh connection key') do |key|
-          options[:key] = key
+        opts.on('-k', '--key KEY', 'Specify ssh connection key') do |key|
+          @options[:key] = key
         end
 
         opts.on('-s', '--server SERVER', 'Specify the server that contained the database.') do |server|
-          options[:server] = server
+          @options[:server] = server
         end
 
         opts.on('-a', '--append-date', 'Append the given date as suffix') do
-          options[:append_date] = true
+          @options[:append_date] = true
         end
 
-        opts.on('--collections COLLECTION', Array, 'Only dump specific mongodb collections separated by comma') do |collection|
-          options[:collections] = collection
+        opts.on('--collections COLLECTION', 'Only dump specific mongodb collections separated by comma') do |collection|
+          @options[:collections] = collection
+        end
+
+        opts.on('--n', '--name NAME', 'Specify a custom name/key for databases') do |name|
+          @options[:name] = name
         end
 
         opts.on('-v', '--verbose', 'Execute NERD mode!') do
-          options[:verbose] = true
+          @options[:verbose] = true
         end
 
         opts.on('-V', '--version', 'Version') do
@@ -66,37 +55,47 @@ module DBGet
         end
       end
 
-      optparse.parse!
+      optparse
+    end
 
-      if args.length == 0
-        puts optparse.help
-      else
-        options[:dbtype] = dbtype
-        options[:databases] = args.dup
+    def set_default_options
+      @options[:append_date] = false
+      @options[:verbose] = false
+      @options[:clean] = false
+      @options[:date] = 'xxxx-xx-xx'
+      @options[:collections] = 'EMPTY'
+    end
+
+    def run!(args, type)
+      check_path
+      @args = args
+      has_arguments?
 
-        run(options)
+      @optparse.parse!
+      @options[:db_type] = type
+
+      controller
+    end
+
+    def controller
+      controller = DBGet::Controller.new(@args, @options)
+      controller.boot
+      controller.send_data!
+    end
+
+    def has_arguments?
+      if @args.length == 0
+        puts @optparse.help
+        exit
       end
     end
 
-    def run(opts)
+    def check_path
       if ENV.include?('DBGET_PATH')
-        opts[:dbget_path] = ENV['DBGET_PATH']
+        @options[:dbget_path] = ENV['DBGET_PATH']
       else
         raise "Cannot find DBGET_PATH!"
       end
-
-      # lets turn off opt_db_name if we have more arguments
-      # this feature is not supported at the moment
-      if opts[:databases].count > 1 and opts.include?(:opt_db_name)
-        raise "You cannot use -n with multiple databases!"
-      end
-
-      # check if -a switch is give but no -d
-      if opts[:append_date] and !opts.include?(:date)
-        raise "You cannnot use -a without -d!"
-      end
-
-      DBGet::Initializer.boot(opts)
     end
   end
 end

data/lib/dbget/utils.rb

@@ -19,5 +19,12 @@ module DBGet
       chars = ('a'..'z').to_a + ('A'..'Z').to_a
       (0...size).collect { chars[Kernel.rand(chars.length)] }.join
     end
+
+    def self.stringify(hash)
+      hash.inject({}) do |options, (key, value)|
+        options[key.to_s] = value.to_s
+      options
+      end
+    end
   end
 end

data/lib/dbget/version.rb

@@ -1,3 +1,3 @@
 module DBGet
-  VERSION = "0.1.4"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dbget_client
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-20 00:00:00.000000000Z
+date: 2012-08-27 00:00:00.000000000Z
 dependencies: []
 description: Decrypts and uncompress MySQL and MongoDB backups from dbget server
 email:
@@ -28,13 +28,10 @@ files:
 - bin/mongoget
 - dbget_client.gemspec
 - lib/dbget.rb
-- lib/dbget/config_loader.rb
+- lib/dbget/config.rb
+- lib/dbget/connector.rb
 - lib/dbget/constants.rb
-- lib/dbget/db_dump.rb
-- lib/dbget/initializer.rb
-- lib/dbget/loaders.rb
-- lib/dbget/loaders/mongodb.rb
-- lib/dbget/loaders/mysql.rb
+- lib/dbget/controller.rb
 - lib/dbget/runner.rb
 - lib/dbget/utils.rb
 - lib/dbget/version.rb

data/lib/dbget/config_loader.rb

@@ -1,19 +0,0 @@
-module DBGet
-  class ConfigLoader
-
-    def initialize(dbget_config, options = {})
-      @dbget_config = dbget_config
-      @options = options
-    end
-
-    def get_config(db_dump)
-      @db_dump = db_dump
-
-      if @dbget_config['mapping'].include?(db_dump.db)
-        @dbget_config
-      else
-        raise "#{db_dump.db} is not found in your dbget.yml!"
-      end
-    end
-  end
-end

data/lib/dbget/db_dump.rb

@@ -1,124 +0,0 @@
-module DBGet
-  class DBDump
-    include Utils
-    include Constants
-
-    attr_accessor :dump_path, :collections
-    attr_reader :db, :database, :db_path
-    attr_reader :host, :port, :username, :password
-    attr_reader :clean, :verbose
-
-    def initialize(options)
-      @db = options[:db]
-      @date = options[:date]
-      @key = options[:key]
-      @dbtype = options[:dbtype]
-      @server = options[:server]
-      @opt_db_name = options[:opt_db_name]
-      @verbose = options[:verbose]
-      @append_date = options[:append_date]
-      @clean = options[:clean]
-      @collections = options[:collections] || []
-      @header = {}
-    end
-
-    def load!(db_config)
-      @server ||= db_config['source']['server']
-      @key ||= db_config['key']
-      @database = @opt_db_name || db_config['mapping'][@db][@dbtype]
-      @host = db_config['target'][@dbtype]['host']
-      @port = db_config['target'][@dbtype]['port']
-      @username = db_config['target'][@dbtype]['username']
-      @password = db_config['target'][@dbtype]['password']
-      @dump_path = db_config['source']['dump_path']
-      @db_path = File.join(@dump_path, "#{@db}")
-
-      if @append_date
-        @database = @database.concat("_#{@date.delete('-')}")
-      end
-
-      get_dump(db_config)
-
-      if @header[:status] == 'SUCCESS'
-        if !@database.nil?
-          Utils.say "Dump info of #{@db} to #{@dbtype} using connection: \n" +
-          "  host: #{@host}\n" +
-          "  port: #{@port}\n" +
-          "  user: #{@username}\n" +
-          "  database: #{@database}\n"
-
-          if @dbtype == 'mysql'
-            Loaders::MySQL.new(self).load!
-          else
-            Loaders::MongoDB.new(self).load!
-          end
-        else
-          raise "Database #{@db} for #{@dbtype} is not found on #{DBGET_CONFIG_FILE}!"
-        end
-      else
-        raise "There was a problem fetching the database from the server!"
-      end
-    end
-
-    protected
-
-    def get_dump(db_config)
-      user = db_config['source']['user']
-      host = db_config['source']['host']
-
-      unless File.exist? @dump_path
-        FileUtils.mkdir_p(@dump_path)
-      end
-
-      ssh_params = "#{user}@#{host} db=#{@db} date=#{@date} " +
-                   "dbtype=#{@dbtype} server=#{@server}"
-
-      if !@key.nil?
-        ssh_cmd = "#{SSH_CMD} -i #{@key} #{ssh_params}"
-      else
-        ssh_cmd = "#{SSH_CMD} #{ssh_params}"
-      end
-
-      Utils.say "Fetching..."
-
-      io_handle = IO.popen(ssh_cmd)
-
-      parse_header(io_handle)
-
-      if !io_handle.eof?
-        File.open(@db_path, "w+") do |f|
-          gz = Zlib::GzipReader.new(io_handle)
-
-          while !io_handle.eof?
-            g = gz.read(16*1024)
-            f.write(g)
-          end
-
-          f.write(gz.read)
-
-          gz.close
-          f.close
-        end
-      end
-    end
-
-    def parse_header(io_handle)
-      while (s = io_handle.readline) != "\r\n"
-        s = s.split(': ')
-        @header[s.first.to_sym] = s.last.chomp
-      end
-
-      print_header if @verbose
-
-      if @header[:status] != "SUCCESS"
-        raise "Server returned #{@header[:status]}!"
-      end
-    end
-
-    def print_header
-      @header.each do |k, v|
-        puts ">> #{k.capitalize}: #{v}"
-      end
-    end
-  end
-end

data/lib/dbget/initializer.rb

@@ -1,53 +0,0 @@
-module DBGet
-  class Initializer
-    include Constants
-
-    def self.boot(options)
-      self.new(options).init
-    end
-
-    def initialize(options)
-      @options = options
-    end
-
-    def init
-      load_dbget_config
-
-      @options[:databases].each do |d|
-        @options[:db] = d
-
-        db_dump = DBGet::DBDump.new(@options)
-
-        load_db_dumps(db_dump)
-      end
-    end
-
-    protected
-
-    def load_dbget_config
-      config_path = File.join(@options[:dbget_path], DBGET_CONFIG_FILE)
-
-      if File.exists?(config_path)
-        @dbget_config = YAML.load_file(config_path)
-      else
-        raise "Cannot find #{config_path}!"
-      end
-    end
-
-    def load_db_dumps(db_dump)
-      if !@dbget_config.empty?
-        config_loader = DBGet::ConfigLoader.new(@dbget_config, @options)
-      else
-        raise "Your dbget.yml is empty!"
-      end
-
-      db_dump.load!(config_loader.get_config(db_dump))
-    end
-
-    def cleanup_db_dumps(db_dumps)
-      db_dumps.each do |db_dump|
-        db_dump.clean_up!
-      end
-    end
-  end
-end

data/lib/dbget/loaders.rb

@@ -1,6 +0,0 @@
-module DBGet
-  module Loaders
-    autoload :MongoDB, File.join(DBGET_LIB_ROOT, 'loaders/mongodb')
-    autoload :MySQL, File.join(DBGET_LIB_ROOT, 'loaders/mysql')
-  end
-end

data/lib/dbget/loaders/mongodb.rb

@@ -1,69 +0,0 @@
-module DBGet
-  module Loaders
-    class MongoDB
-      include Utils
-      include Constants
-
-      def self.boot(dump)
-        # some boot here
-
-        self.new(dump)
-      end
-
-      def initialize(dump)
-        @dump = dump
-      end
-
-      def load!
-        temp_path = File.join(@dump.dump_path, "#{@dump.db}_#{Utils.randomize(16)}")
-
-        if !File.exists?(temp_path)
-          FileUtils.mkdir(temp_path)
-
-          Utils.say_with_time "Extracting archive..." do
-            `#{TAR_CMD} -C #{temp_path} -xf #{File.join(@dump.dump_path, @dump.db)} 2> /dev/null`
-          end
-
-          Utils.say_with_time "Moving mongo files..." do
-            `#{FIND_CMD} #{temp_path} -name '*.#{MONGO_FILE_EXT}'`.each_line do |l|
-                FileUtils.mv(l.chomp!, File.join(temp_path, File.basename(l)))
-            end
-          end
-        end
-
-        dump_files = Dir["#{temp_path}/*#{MONGO_FILE_EXT}"]
-
-        if !@dump.collections.empty?
-          @dump.collections = @dump.collections.collect do |c|
-            File.join(temp_path, c.concat(MONGO_FILE_EXT))
-          end
-
-          dump_files &= @dump.collections
-        end
-
-        dump_files.each do |d|
-          # do not include indexes
-          if File.basename(d) != "system.indexes#{MONGO_FILE_EXT}"
-            Utils.say_with_time "Dumping #{d}..." do
-              if !@dump.verbose
-                `#{MONGORESTORE_CMD} -d #{@dump.database} #{d} --drop`
-              else
-                system "#{MONGORESTORE_CMD} -d #{@dump.database} #{d} --drop"
-              end
-            end
-          end
-        end
-
-        Utils.say "Hooray! Dump for #{@dump.db} done!"
-
-        if FileUtils.rm_rf(File.join(@dump.dump_path, @dump.db))
-         Utils.say "Dump file removed!"
-        end
-
-        if FileUtils.rm_rf(temp_path)
-          Utils.say "Temp directory removed!"
-        end
-      end
-    end
-  end
-end

data/lib/dbget/loaders/mysql.rb

@@ -1,50 +0,0 @@
-module DBGet
-  module Loaders
-    class MySQL
-      include Utils
-      include Constants
-
-      def self.boot(dump)
-        # some boot here
-
-        self.new(dump)
-      end
-
-      def initialize(dump)
-        @dump = dump
-      end
-
-      def load!
-        command = "#{MYSQL_CMD} "
-        command += "-h#{@dump.host} "
-        command += "-P#{@dump.port} "
-        command += "-u#{@dump.username} "
-        command += "-p#{@dump.password} " if @dump.password
-
-        if @dump.clean
-          Utils.say_with_time "Dropping database..." do
-            system "echo \"DROP DATABASE IF EXISTS #{@dump.database}\" | #{command}"
-          end
-        end
-
-        system "echo \"CREATE DATABASE IF NOT EXISTS #{@dump.database}\" | #{command}"
-
-        if File.exist?(@dump.db_path) and !File.size?(@dump.db_path).nil?
-          command += " #{@dump.database} "
-
-          Utils.say_with_time "Dumping #{@dump.db}..." do
-            system "#{command}< #{File.join(@dump.dump_path, @dump.db)}"
-          end
-
-          if FileUtils.rm_rf(File.join(@dump.dump_path, @dump.db))
-            Utils.say "Cleaned temporary file!"
-          end
-        else
-          raise "Dump for #{@dump.db} not found!"
-        end
-
-        Utils.say "Hooray! Dump for #{@dump.db} done!"
-      end
-    end
-  end
-end