mongo-db-utils 0.0.5 → 0.0.6

data/README.md

@@ -12,9 +12,6 @@ It saves your database urls so any task is just a few clicks.
 * backup a database and deploy it to Amazon S3
 * copy a database from one server to another (whilst backing up locally the target db if it exists)
 
-
-
-
 ## Installation
 
 You need to have *mongodump* and *mongorestore* on your path.
@@ -34,7 +31,13 @@ Only works on ruby 1.9.3 (to_yaml is throwing an error in earlier versions)
 Once you launch the console it'll provide you with a set of options - pretty self explanatory.
 When it does backups it stores them in ````~/.mongo-db-utils/backups/````. The naming convention is ````${server}_${port}/${database_name}/${timestamp}/db````
 
+## Testing
+    
+    bundle exec rspec spec
 
+    #cucumber can't handle interactive CLIs so need to wait on this.
+    #bundle exec cucumber features
+    
 ## Building source
 
     #run console

data/lib/mongo-db-utils/cli.rb

@@ -27,16 +27,10 @@ module MongoDbUtils
     desc "backup_s3 MONGO_URI BUCKET ACCESS_KEY SECRET_ACCESS_KEY", "backup a db to Amason s3 with a mongo uri eg: mongodb://user:pass@server:port/dbname"
     def backup_s3(mongo_uri, bucket_name, access_key_id, secret_access_key)
       @config = MongoDbUtils::ConfigLoader.load
+      backup_folder = @config.backup_folder
       db = MongoDbUtils::Model::Db.from_uri(mongo_uri)
       raise "can't parse uri" if db.nil?
-      tar_file = MongoDbUtils::Cmd.backup(db, @config.backup_folder)
-
-      name = tar_file.gsub(File.expand_path(@config.backup_folder), "")
-
-      MongoDbUtils::S3::put_file(tar_file, name, bucket_name, access_key_id, secret_access_key)
-      file = File.basename(tar_file)
-      folder = tar_file.gsub(file, "")
-      `rm -fr #{folder}`
+      MongoDbUtils::Cmd.backup_s3(backup_folder, db, bucket_name, access_key_id, secret_access_key)
     end
 
   end

data/lib/mongo-db-utils/cmd/mongotools.rb

@@ -3,49 +3,35 @@ require 'aws/s3'
 module MongoDbUtils
   module Commands
 
-    class Option
-      attr_accessor :key, :value
-
-      def initialize(key,value)
-        @key = key
-        @value = value
-      end
-
-      def empty?
-        @value.nil? || @value.empty?
-      end
-    end
-
     class MongoTools
 
+      # wrapper for monogdump shell command
       def self.dump(host,port,db,output,username = "", password = "")
 
         options = []
-        options << Option.new("-h", "#{host}:#{port}")
-        options << Option.new("-db", db)
-        options << Option.new("-o", output)
-        options << Option.new("-u", username)
-        options << Option.new("-p", password)
+        options << o("-h", "#{host}:#{port}")
+        options << o("-db", db)
+        options << o("-o", output)
+        options << o("-u", username)
+        options << o("-p", password)
 
         cmd = "mongodump "
 
         options.each do |o|
           cmd << "#{o.key} #{o.value} " unless o.empty?
         end
-        puts "cmd:"
-        puts cmd
         `#{cmd}`
-
       end
 
 
+      # wrapper for mongorestore shell command
       def self.restore(host,port,db,source_folder,username = "", password = "")
 
         options = []
-        options << Option.new("-h", "#{host}:#{port}")
-        options << Option.new("-db", db)
-        options << Option.new("-u", username)
-        options << Option.new("-p", password)
+        options << o("-h", "#{host}:#{port}")
+        options << o("-db", db)
+        options << o("-u", username)
+        options << o("-p", password)
 
         cmd = "mongorestore "
 
@@ -53,25 +39,28 @@ module MongoDbUtils
           cmd << "#{o.key} #{o.value} " unless o.empty?
         end
         cmd << "#{source_folder}"
-        puts "cmd:"
-        puts cmd
         `#{cmd}`
+      end
 
+      private 
+      def self.o(key,value)
+        Option.new(key,value)
       end
 
     end
 
-    class S3
+    class Option
+      attr_accessor :key, :value
 
-      def self.put_file(file, bucket_name, access_key_id, secret_access_key)
-        AWS::S3::Base.establish_connection!(
-          :access_key_id     => access_key_id,
-          :secret_access_key => secret_access_key
-        )
+      def initialize(key,value)
+        @key = key
+        @value = value
+      end
 
-        Service.buckets.create(bucket_name) if Service.buckets.find(bucket_name).nil?
-        S3Object.store(file, open(file), bucket_name)
+      def empty?
+        @value.nil? || @value.empty?
       end
     end
+
   end
 end

data/lib/mongo-db-utils/cmd.rb

@@ -6,19 +6,15 @@ module MongoDbUtils
   class Cmd
 
     def self.backup(db, folder, final_path = nil, tar_it = true)
-      puts "--"
       puts ">> Backing up: #{db}, #{folder}, #{final_path}"
       unless( db_exists?(db) )
         return false
       end
 
-      t = Time.new
-      timestamp = t.strftime("%Y.%m.%d__%H.%M")
-
       if( final_path.nil? )
         out_path = "#{folder}/#{db.host}_#{db.port}/#{db.name}/#{timestamp}"
       else
-        puts "final path not nil out_path: #{out_path}"
+        out_path = "#{folder}/#{final_path}"
       end
 
       full_path = File.expand_path(out_path)
@@ -38,9 +34,10 @@ module MongoDbUtils
         Dir.chdir(full_path)
         `tar cvf #{db.name}.tar #{db.name}`
         `rm -fr #{full_path}/#{db.name}`
+        "#{full_path}/#{db.name}.tar"
+      else
+        "#{full_path}/#{db.name}"
       end
-
-      "#{full_path}/#{db.name}.tar"
     end
 
     # With remote dbs you can't do a copy_database if you're not an admin.
@@ -52,11 +49,9 @@ module MongoDbUtils
     # end
     # host = "#{source.host}:#{source.port}"
     # connection.copy_database(source.name, destination.name, host, source.username, source.password)
-    #
-    #
     def self.copy(path, source, destination, halt_on_no_backup = true)
 
-      backup_made = self.backup(destination, path)
+      backup_made = backup(destination, path)
 
       if( !backup_made && halt_on_no_backup)
         puts "aborting - no backup was made"
@@ -68,53 +63,64 @@ module MongoDbUtils
       FileUtils.mkdir_p(tmp_path)
 
       puts "backup to: #{tmp_path}/#{source.name}"
-      backup(source,tmp_path, source.name, false)
+      tmp_dump_path = backup(source,tmp_path, source.name, false)
+
+      username = destination.username
+      password = destination.password
+
+      # if the destination db doesn't exist
+      # we assume that the user has admin control of the server
+      # eg its a local server (this needs to be thought through a bit more)
+      if( !db_exists?(destination) )
+        username = ""
+        password = ""
+      end
 
       MongoDbUtils::Commands::MongoTools.restore(
         destination.host,
         destination.port,
         destination.name,
-        "#{tmp_path}/#{source.name}/#{source.name}",
-        destination.username,
-      destination.password)
+        "#{tmp_dump_path}",
+        username,
+        password)
 
       `rm -fr #{tmp_path}`
     end
 
+     def self.backup_s3(backup_folder, db, bucket_name, access_key_id, secret_access_key)
+      tar_file = MongoDbUtils::Cmd.backup(db, backup_folder)
+      name = tar_file.gsub(File.expand_path(backup_folder), "")
+      MongoDbUtils::S3::put_file(tar_file, name, bucket_name, access_key_id, secret_access_key)
+      file = File.basename(tar_file)
+      folder = tar_file.gsub(file, "")
+      `rm -fr #{folder}`
+    end
+
 
     private
-    def self.db_exists?(db)
 
-      puts "DB exists? #{db.to_s}"
+    def self.timestamp
+      t = Time.new
+      t.strftime("%Y.%m.%d__%H.%M")
+    end
 
+    def self.db_exists?(db)
+      puts "DB exists? #{db.to_s}"
       connection = Mongo::Connection.from_uri(db.to_s)
-
-      puts "db.name: #{db.name}"
-      mongo_db = connection[db.name]
-
-      exists = !mongo_db.nil?
-
-      puts "mongo_db: #{mongo_db}"
-      if( db.authentication_required? && exists )
-        login_result = mongo_db.authenticate(db.username, db.password)
-        exists = !login_result.nil?
-      else
+      exists = true
+      begin
+        connection.ping
+      rescue Mongo::AuthenticationError => e
+        exists = false
       end
       connection.close
       exists
     end
+=begin
+       connection.database_names 
+      rescue Mongo::OperationFailure => e
+=end
 
-    def self.remove_db(db)
-      if( db_exists?(db))
-        connection = Mongo::Connection.from_uri(db.to_s)
-        if( db.authentication_required? )
-          mongo_db = connection[db.name]
-          mongo_db.authenticate(db.username, db.password)
-        end
 
-        connection.drop_database(db.name)
-        connection.close
-      end
-    end
   end
 end

data/lib/mongo-db-utils/console.rb

@@ -6,10 +6,10 @@ module MongoDbUtils
   class Console
 
     HEADER = <<-eos
-    ===================================
-      Mongo Db Utils - Version: #{MongoDbUtils::VERSION}
-      ===================================
-      eos
+=====================================
+|| Mongo Db Utils - Version: #{MongoDbUtils::VERSION} ||
+=====================================
+eos
 
 
     def initialize(config, cmd)
@@ -25,16 +25,16 @@ module MongoDbUtils
 
     private
     def main_menu
-      say("\nWhat do you want to do?")
-      choose do |menu|
-        prep_menu(menu)
+
+      my_menu("What do you want to do?") do |menu|
         menu.choice "copy a db" do copy_a_db end
-        menu.choice "backup a db" do do_backup end
+        menu.choice "backup a db locally" do do_backup end
+        menu.choice "backup a db to an amazon s3 bucket" do backup_to_s3 end
         menu.choice "remove config" do remove_config end
         menu.choice "show config" do show_config end
         menu.choice "add server to config" do add_config end
+        menu.choice "add Amazon bucket to config" do add_bucket_to_config end
         menu.choice "remove server from config" do remove_server_from_config end
-        menu.choice "exit" do say("goodbye") end
       end
     end
 
@@ -43,15 +43,60 @@ module MongoDbUtils
       main_menu
     end
 
+    #
     def do_backup
-
       if @config.empty?
         get_config
       else
-        list_dbs
+        db_list_menu("Choose a DB:") do |db|
+          @cmd.backup(db, @config.backup_folder)
+        end
+      end
+    end
+
+
+    def backup_to_s3
+
+      plan = Hash.new
+
+      db_list_menu("Choose a DB:") do |db|
+        plan[:db] = db
+      end
+
+      if @config.has_buckets?
+        list_buckets("Choose a Bucket:") do |bucket|
+          plan[:bucket] = bucket
+        end
+      else
+        say("You don't have any buckets yet - add one..")
+        add_bucket_to_config
+        return
       end
+
+      @cmd.backup_s3(
+        @config.backup_folder,
+        plan[:db],
+        plan[:bucket].name,
+        plan[:bucket].access_key,
+        plan[:bucket].secret_key)
+    end
+
+
+    def add_bucket_to_config
+      say("add an amazon bucket:")
+      bucket = MongoDbUtils::Model::Bucket.new
+
+      bucket.name = ask("Name")
+      bucket.access_key = ask("Access Key")
+      bucket.secret_key = ask("Secret Key")
+
+      @config.add_bucket(bucket)
+
+      say("Bucket added")
+      my_menu("")
     end
 
+
     def get_config
       say("You don't have any servers configured, please add one:")
       add_config
@@ -63,18 +108,23 @@ module MongoDbUtils
       say("dbs:")
       say("--------------------")
       @config.dbs.sort.each do |db|
-        say("#{db.to_s_simple}")
+        say(db.to_s_simple)
       end
       say("--------------------")
-      say("")
+      say("Amazon S3 Buckets")
+      if( @config.buckets.nil?)
+        say("no buckets")
+      else
+        @config.buckets.sort.each do |bucket|
+          say(bucket.to_s)
+        end
+      end
+
       say("backups folder:")
       say("--------------------")
       say("#{@config.backup_folder}")
       say("--------------------")
-      choose do |menu|
-        prep_menu(menu)
-        menu.choice "back" do main_menu end
-      end
+      my_menu("")
     end
 
     def add_config
@@ -83,76 +133,29 @@ module MongoDbUtils
       new_uri = entry[:mongo_uri].gsub(" ", "")
       successful = @config.add_db_from_uri(new_uri)
 
-      if successful
-        say("added server")
-        choose do |menu|
-          prep_menu(menu)
-          menu.choice "add another?" do add_config end
-          menu.choice "done" do main_menu end
-        end
-
-      else
-        say("couldn't add uri")
-        add_config
-      end
-    end
+      say("bad uri!") unless successful
 
+      label = successful ? "add another?" : "try again?"
 
-    def list_dbs
-      say("Which db?")
-      choose do |menu|
-        prep_menu(menu)
-        @config.dbs.each do |db|
-          menu.choice "#{db.to_s}" do backup(db) end
-        end
-        menu.choice "back" do main_menu end
+      my_menu("") do |menu|
+        menu.choice label do add_config end
       end
     end
 
-
     def remove_server_from_config
-      say("remove server from config...")
-      choose do |menu|
-        prep_menu(menu)
-        @config.dbs.sort.each do |db|
-          menu.choice "#{db.to_s}" do
-            @config.remove_db(db)
-            remove_server_from_config
-          end
-        end
-        menu.choice "back" do main_menu end
+      db_list_menu("Remove server from config:") do |db|
+        @config.remove_db(db)
+        remove_server_from_config
       end
     end
 
     def copy_a_db
-
       copy_plan = Hash.new
-
-      say("Choose db to copy:")
-      choose do |menu|
-        prep_menu(menu)
-        @config.dbs.sort.each do |db|
-          menu.choice "#{db.to_s}" do
-            copy_plan[:source] = db
-          end
-
-        end
-        menu.choice "add server to config" do add_config end
-        menu.choice "back" do
-          main_menu
-          return
-        end
+      db_list_menu("Choose db to copy:") do |db|
+        copy_plan[:source] = db
       end
-
-      say("Choose db destination:")
-      choose do |menu|
-        prep_menu(menu)
-        @config.dbs.sort.each do |db|
-          menu.choice "#{db.to_s}" do
-            copy_plan[:destination] = db
-          end unless db == copy_plan[:source]
-        end
-        menu.choice "add server to config" do add_config end
+      db_list_menu("Choose a destination:") do |db|
+        copy_plan[:destination] = db
       end
       show_copy_plan(copy_plan)
     end
@@ -161,31 +164,49 @@ module MongoDbUtils
       say("Copy: (we'll backup the destination before we copy)")
       say("#{plan[:source].to_s} --> #{plan[:destination].to_s}")
 
-      choose do |menu|
-        prep_menu(menu)
-        menu.choice "Begin" do begin_copy(plan) end
+      my_menu("") do |menu|
+        menu.choice "Do it!" do begin_copy(plan) end
         menu.choice "Reverse" do
           show_copy_plan( {:source => plan[:destination], :destination => plan[:source]})
         end
-        menu.choice "Back" do main_menu end
       end
     end
 
     def begin_copy(plan)
-      say("doing copy...")
       @cmd.copy(@config.backup_folder, plan[:source], plan[:destination], false)
     end
 
-    private
-    def backup(db)
-      puts ">> ..backing up #{db}"
-      @cmd.backup(db, @config.backup_folder)
+    def my_menu(prompt, show_defaults = true)
+      say(prompt)
+      choose do |menu|
+        menu.shell  = true
+        menu.index = :number
+        menu.prompt = "\n#{prompt}\n"
+        menu.index_suffix = ") "
+        menu.prompt = "?"
+        if( show_defaults)
+          menu.choice "Back" do main_menu end
+          menu.choice "Exit" do say("Goodbye"); abort("--"); end
+        end
+        say(" ")
+        yield menu if block_given?
+      end
+    end
+
+    def db_list_menu(prompt)
+      my_menu(prompt, false) do |menu|
+        @config.dbs.sort.each do |db|
+          menu.choice "#{db.to_s_simple}" do yield db if block_given? end
+        end
+      end
     end
 
-    def prep_menu(menu)
-      menu.index = :number
-      menu.index_suffix = ") "
-      menu.prompt = "?"
+    def list_buckets(prompt)
+      my_menu(prompt) do |menu|
+        @config.buckets.sort.each do |bucket|
+          menu.choice "#{bucket}" do yield bucket if block_given? end
+        end
+      end
     end
 
   end

data/lib/mongo-db-utils/models.rb

@@ -2,18 +2,24 @@ module MongoDbUtils
 
   module Model
     class Config
-      attr_reader :dbs
+      attr_reader :dbs, :buckets
       attr_writer :writer
       attr_accessor :backup_folder
 
       def initialize
         @dbs = []
+        @buckets = []
       end
       
       def empty?
         @dbs.nil? || @dbs.empty?
       end
 
+      def has_buckets?
+        !@buckets.nil? && !@buckets.empty?
+      end
+
+
       def flush
         @dbs = []
         @writer.flush
@@ -27,7 +33,7 @@ module MongoDbUtils
       def add_db_from_uri(uri)
         @dbs = [] if @dbs.nil?
         db = Db.from_uri(uri)
-        unless db.nil?
+        unless db.nil? || already_contains(db)
           @dbs << db
           @dbs.sort!
           @writer.save(self)
@@ -35,6 +41,41 @@ module MongoDbUtils
         !db.nil?
       end
 
+      def already_contains(db)
+        @dbs.each do |existing|
+          if( existing.to_s == db.to_s)
+            return true
+          end
+
+          if( existing.host == db.host && existing.name == db.name)
+            return true
+          end
+
+        end
+        return false
+      end
+
+      # because we are serializing the config - the bucket may be nil
+      # at this point
+      def add_bucket(bucket)
+        @buckets = [] if @buckets.nil?
+        unless bucket.nil? || already_contains_bucket?(bucket)
+          @buckets << bucket
+          @writer.save(self)
+        end
+      end
+
+      def already_contains_bucket?(bucket)
+        puts "@buckets: #{@buckets}"
+        @buckets.each do |b|
+          if( b.to_s == bucket.to_s )
+            return true
+          end
+        end
+        return false
+      end
+
+
       def save
         @writer.save(self)
       end
@@ -44,6 +85,14 @@ module MongoDbUtils
       end
     end
 
+    class Bucket
+      attr_accessor :name, :access_key, :secret_key
+
+      def to_s
+        "#{name} | #{access_key} | #{secret_key}"
+      end
+        
+    end
 
     # A Db stored in the config
     class Db
@@ -96,7 +145,7 @@ module MongoDbUtils
       end
 
       def to_s_simple
-        "#{@host}:#{@port}/#{@name}"
+        "#{@name} on #{@host}:#{@port} - (#{@username}:#{@password})"
       end
 
       def <=>(other)

data/lib/mongo-db-utils/version.rb

@@ -1,4 +1,4 @@
 module MongoDbUtils
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
   READY_FOR_USE = "(Alpha version - not for production use!!)"
 end

data/mongo-db-utils.gemspec

@@ -13,6 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "mongo-db-utils"
   gem.require_paths = ["lib"]
+  gem.required_ruby_version = '>= 1.9.3'
   gem.version       = MongoDbUtils::VERSION
   gem.add_development_dependency "rspec", "~> 2.6"
   gem.add_development_dependency "cucumber"

data/spec/mongo_db_utils_spec.rb

@@ -47,13 +47,18 @@ describe MongoDbUtils::Model do
   end
 
 
-  it "config should add dbs" do
+  it "config should add dbs only if they are different" do
     config = MongoDbUtils::Model::Config.new
     config.writer = MockWriter.new
 
-    config.add_db_from_uri("mongodb://blah:3333/blah")
+    config.add_db_from_uri("mongodb://blah:3333@server:123/blah")
     config.dbs.length.should eql(1)
-    config.add_db_from_uri("mongodb://blah:3333/blah")
+    config.add_db_from_uri("mongodb://blah:3333@server:123/blah")
+    config.dbs.length.should eql(1)
+    # still the same server + name
+    config.add_db_from_uri("mongodb://server:123/blah")
+    config.dbs.length.should eql(1)
+     config.add_db_from_uri("mongodb://server2:123/blah")
     config.dbs.length.should eql(2)
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongo-db-utils
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -164,14 +164,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
-- .mongo-db-utils/config.yml
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - bin/mongo-db-utils
-- features/mongo-db-utils.feature
-- features/support/setup.rb
 - images/sample.png
 - install_rdebug.sh
 - lib/mongo-db-utils/cli.rb
@@ -196,7 +193,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -210,7 +207,5 @@ signing_key:
 specification_version: 3
 summary: some utilities for managing your mongod dbs
 test_files:
-- features/mongo-db-utils.feature
-- features/support/setup.rb
 - spec/config-loader-spec.rb
 - spec/mongo_db_utils_spec.rb

data/.mongo-db-utils/config.yml

@@ -1,2 +0,0 @@
---- !ruby/object:MongoDbUtils::Model::Config
-name: This is the config

data/features/mongo-db-utils.feature

@@ -1,9 +0,0 @@
-Feature: MongoDbUtils
-  In order to say hello
-  As a CLI
-  I want to be as objective as possible
-
-  Scenario: Hello Ed
-    When I run `mongo-db-utils`
-    Then the output should contain "hello"
-

data/features/support/setup.rb

@@ -1 +0,0 @@
-require 'aruba/cucumber'