mongo-db-utils 0.0.9 → 0.0.9.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 60f9d5dc43d84806e0284ab99d3018d0f7a6013f
+  data.tar.gz: 41d85fc792b941ebe3ff901a2add2f09643bfbbc
+SHA512:
+  metadata.gz: dac4c457c4f95c22b32f5122b91c4f920f11f72687bc41c8702f3e1a96e107a64e82cc6dca0ffaea88b4d7a3a3b6d7c133290c7484dacf86dfb0dcc55ec4dc78
+  data.tar.gz: ebad5fca30541d643d4ddfcfc2311ddc300dcefe215cfc0e28ec7d77c50628b98fb79707aa54ecb18799e87af2b6f15044db04fe712a82271b725cb0cb5ecd99

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+.tmp_path

data/Gemfile

@@ -1,9 +1,4 @@
 source 'https://rubygems.org'
-
 # Specify your gem's dependencies in mongo-db-utils.gemspec
 gemspec
-#RUBY_PATCHLEVEL = "194"
-#gem 'linecache19', '0.5.13', :path => "~/.rvm/gems/ruby-1.9.3-p#{RUBY_PATCHLEVEL}/gems/linecache19-0.5.13/"
-#gem 'ruby-debug-base19', '0.11.26', :path => "~/.rvm/gems/ruby-1.9.3-p#{RUBY_PATCHLEVEL}/gems/ruby-debug-base19-0.11.26/"
-#gem 'ruby-debug19', :require => 'ruby-debug'
 

data/README.md

@@ -1,5 +1,7 @@
 # MongoDbUtils
 
+### !Current version 0.0.9.2 is in Beta - for a safer version use 0.0.9
+
 A little gem that simplifies backing up and copying your mongo dbs.
 
 You can run as a script (eg for cron jobs, or in interactive mode):
@@ -32,18 +34,31 @@ Once you launch the console it'll provide you with a set of options - pretty sel
 When it does backups it stores them in ````~/.mongo-db-utils/backups/````. The naming convention is ````${server}_${port}/${database_name}/${timestamp}/db````
 
 ## Testing
-    
+
     bundle exec rspec spec
 
     #cucumber can't handle interactive CLIs so need to wait on this.
     #bundle exec cucumber features
-    
+
 ## Building source
 
     #run console
-    bundle exec bin/mongo-db-utils console
+    bundle exec bin/mongo-db-utils console path_to/config.yml (optional)
 
     #install the gem locally
     rake build
     gem install pkg/mongo-db-utils.gem
 
+
+
+## Release Notes
+
+* 0.0.9.2 - BETA!
+  - Added support for Replica Sets
+  - console can be run pointing to any config file: `console myconfig.yml`
+  - More specs
+  - Added local mongo environment to simplify testing @see: integration-test-env
+
+* 0.0.9 - First release
+  - Copy mongo dbs
+  - Back up to S3

data/integration-test-env/.gitignore

@@ -0,0 +1,4 @@
+.processes
+dbs
+logs
+rs_config

data/integration-test-env/README.md

@@ -0,0 +1,29 @@
+# Integration Test Environment
+
+This folder contains utilities for setting up a mongo environment and running the console against it.
+This is only really useful if you are working on the source code. If you're just using the gem you can ignore this stuff.
+
+## Available Environments
+
+### Standalone DB
+
+* init the environment `create_two_standalone_dbs` - this creates 2 mongod instances on 27018/27019
+* To run the console go:
+
+    bundle exec bin/mongo-db-utils console integration-test-env/standalone/config.yml
+
+* kill the environment `kill_processes standalone/.processes`
+
+
+### Replica Set DBs
+
+* `create_two_replica_sets`
+* `replica_sets/rs_config` - you'll probably have to run this a few times
+* `seed_replica_sets`
+* run the console:
+
+    bundle exec bin/mongo-db-utils console integration-test-env/replica_sets/config.yml
+
+* kill the environment `kill_processes replica_sets/.processes`
+
+....

data/integration-test-env/create_two_replica_sets

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+require_relative './helper'
+require 'yaml'
+require 'fileutils'
+require 'mongo'
+=begin
+mkdir -p /srv/mongodb/rs0-0 /srv/mongodb/rs0-1 /srv/mongodb/rs0-2
+Issue the following commands, each in a distinct screen window:
+mongod --port 27017 --dbpath /srv/mongodb/rs0-0 --replSet rs0 --smallfiles --oplogSize 128
+mongod --port 27018 --dbpath /srv/mongodb/rs0-1 --replSet rs0 --smallfiles --oplogSize 128
+mongod --port 27019 --dbpath /srv/mongodb/rs0-2 --replSet rs0 --smallfiles --oplogSize 128
+=end
+
+ROOT_FOLDER = "./replica_sets"
+maker = MongoEnvMaker.new(ROOT_FOLDER)
+
+puts "Creating 2 replica sets: 27020-27022 & localhost:27023-27025"
+puts "create the directories"
+
+maker.spawn(27020, "rs0")
+maker.spawn(27021, "rs0")
+maker.spawn(27022, "rs0")
+
+maker.spawn(27023, "rs1")
+maker.spawn(27024, "rs1")
+maker.spawn(27025, "rs1")
+
+puts ">>>>>>>>>>> pid: #{maker.pids}"
+
+File.open( File.expand_path("./#{ROOT_FOLDER}/.processes"), 'w' ) do |out|
+  YAML.dump( maker.pids, out )
+end
+
+def initiate_conf(host, set_name, port)
+  "rs.initiate({_id: '#{set_name}', members: [{_id: 0, host: '#{host}:#{port}'} ] });"
+end
+
+def add_member(host, port)
+"rs.add('#{host}:#{port}')"
+end
+
+def shellscript( base_port, commands )
+  sleep_between_add_members = 4
+  evals = commands.map{ |cmd|  "mongo --port #{base_port} --eval \"#{cmd}\"\nsleep #{sleep_between_add_members}\n"}
+  out = <<-eos
+#{evals.join("\n")}
+eos
+  out
+end
+
+def wrap(s)
+  <<-eos
+#!/usr/bin/env bash
+#{s}
+eos
+end
+
+set_one = shellscript( 27020,
+  [
+    initiate_conf("localhost", "rs0", 27020),
+    add_member("localhost", 27021),
+    add_member("localhost", 27022) ])
+
+set_two = shellscript( 27023,
+    [initiate_conf("localhost", "rs1", 27023),
+    add_member("localhost", 27024),
+    add_member("localhost", 27025)] )
+
+path = "#{ROOT_FOLDER}/rs_config"
+
+File.open(path, 'w'){ |f| f.write( wrap("#{set_one}\n#{set_two}"))}
+
+FileUtils.chmod 0755, path
+
+puts "Done --- Important - you now need to run: #{path} - note you may have to run it a couple of times"

data/integration-test-env/create_two_standalone_dbs

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require_relative './helper'
+require 'yaml'
+require 'fileutils'
+require 'mongo'
+
+ROOT_FOLDER = "./standalone"
+maker = MongoEnvMaker.new(ROOT_FOLDER)
+
+puts "Creating 2 standalone dbs: localhost:27018 & localhost:27019"
+puts "create the directories"
+
+maker.spawn(27018)
+maker.spawn(27019)
+
+puts ">> pids: #{maker.pids}"
+
+File.open( File.expand_path("./#{ROOT_FOLDER}/.processes"), 'w' ) do |out|
+  YAML.dump( maker.pids, out )
+end
+
+sleep 2
+
+puts "Seed 27018 with some data..."
+DummyData.seed("localhost", 27018)
+

data/integration-test-env/helper.rb

@@ -0,0 +1,50 @@
+require 'yaml'
+require 'fileutils'
+require 'mongo'
+
+class MongoEnvMaker
+
+  attr_reader :pids
+
+  def initialize(root_folder)
+    @root_folder = root_folder
+    @pids = []
+  end
+
+
+  def spawn(port, rs_name = nil)
+    `mkdir -p #{@root_folder}/logs/#{port}`
+    out_log = "#{@root_folder}/logs/#{port}/out.log"
+    err_log = "#{@root_folder}/logs/#{port}/err.log"
+    pid = Process.spawn mongod(port, rs_name), :out=> out_log, :err => err_log
+    @pids << pid
+  end
+
+  private
+  def mongod(port, set_name = nil)
+    path = mk_db_dir(port)
+    out = "mongod --port #{port} --dbpath #{path}  --smallfiles --oplogSize 128"
+    out << " --replSet #{set_name}" unless set_name.nil?
+    out
+  end
+
+  def mk_db_dir(port)
+    path = "#{@root_folder}/dbs/#{port}"
+    FileUtils.rm_rf path
+    `mkdir -p #{path}`
+    path
+  end
+end
+
+
+class DummyData
+  def self.seed(host, port)
+    puts "[DummyData] Seed #{host}:#{port} with some dummy data"
+    include Mongo
+    mongo_client = MongoClient.new(host, port)
+    db = mongo_client.db("dummy")
+    coll = db.collection("some_collection")
+    doc = {"name" => "MongoDB", "type" => "database", "count" => 1, "info" => {"x" => 203, "y" => '102'}}
+    id = coll.insert(doc)
+  end
+end

data/integration-test-env/kill_processes

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'fileutils'
+
+path = ARGV[0]
+
+puts "kill processes: #{path}"
+
+full_path = File.expand_path(path)
+
+if File.exist?(full_path) && YAML.load(File.open(full_path))
+  pids = YAML.load(File.open(full_path))
+  puts "found pids: #{pids}"
+  pids.each{ |pid| `kill -9 #{pid}` }
+  FileUtils.rm full_path
+else
+  puts "couldn't find path"
+end

data/integration-test-env/replica_sets/config.yml

@@ -0,0 +1,21 @@
+--- &1 !ruby/object:MongoDbUtils::Model::Config
+dbs:
+- !ruby/object:MongoDbUtils::Model::ReplicaSetDb
+  host_port: localhost:27020,localhost:27021,localhost:27022
+  name: dummy
+  username: ''
+  password: ''
+  uri: mongodb://localhost:27020,localhost:27021,localhost:27022/dummy
+  set_name: rs0
+- !ruby/object:MongoDbUtils::Model::ReplicaSetDb
+  host_port: localhost:27023,localhost:27024,localhost:27025
+  name: dummy
+  username: ''
+  password: ''
+  uri: mongodb://localhost:27023,localhost:27024,localhost:27025/dummy
+  set_name: rs1
+buckets: []
+writer: !ruby/object:MongoDbUtils::ConfigLoader
+  config_path: integration-test-env/replica_sets/config.yml
+  config: *1
+backup_folder: ~/.mongo-db-utils/backups

data/integration-test-env/seed_replica_sets

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'mongo'
+
+include Mongo
+
+dbs = (27020..27022).to_a.map{ |n| "localhost:#{n}"}
+mongo_client = MongoReplicaSetClient.new(dbs)
+
+db = mongo_client.db("dummy")
+coll = db.collection("some_collection")
+doc = {"name" => "MongoDB", "type" => "database", "count" => 1, "info" => {"x" => 203, "y" => '102'}}
+id = coll.insert(doc)
+
+puts "successfully inserted: #{id}"

data/integration-test-env/standalone/config.yml

@@ -0,0 +1,19 @@
+--- &1 !ruby/object:MongoDbUtils::Model::Config
+dbs:
+- !ruby/object:MongoDbUtils::Model::Db
+  host_port: localhost:27018
+  name: dummy
+  username: ''
+  password: ''
+  uri: mongodb://localhost:27018/dummy
+- !ruby/object:MongoDbUtils::Model::Db
+  host_port: localhost:27019
+  name: dummy
+  username: ''
+  password: ''
+  uri: mongodb://localhost:27019/dummy
+buckets: []
+writer: !ruby/object:MongoDbUtils::ConfigLoader
+  config_path: integration-test-env/standalone/config.yml
+  config: *1
+backup_folder: ~/.mongo-db-utils/backups

data/lib/mongo-db-utils/cli.rb

@@ -2,36 +2,53 @@ require 'thor'
 require 'mongo-db-utils/config-loader'
 require 'mongo-db-utils/cmd'
 require 'mongo-db-utils/console'
-require 'mongo-db-utils/models'
+
+Dir['lib/mongo-db-utils/models/*.rb'].each {|file| require file.gsub("lib/", "") }
 require 'mongo-db-utils/s3'
 
 module MongoDbUtils
   class CLI < Thor
 
-    desc "console", "run the interactive console"
-    def console
-      @config = MongoDbUtils::ConfigLoader.load
-      console = MongoDbUtils::Console.new(@config, MongoDbUtils::Cmd)
-      console.run
+    desc "console", "run the interactive console @param path - path to config file"
+    def console(path = MongoDbUtils::ConfigLoader::CONFIG_LOCATION)
+
+      if File.extname(path) != ".yml"
+        puts "Error: You must use a yaml file as your config file location"
+      else
+        @loader = MongoDbUtils::ConfigLoader.new(path)
+        @config = @loader.config
+        console = MongoDbUtils::Console.new(@config, MongoDbUtils::Cmd)
+        console.run
+      end
     end
 
     desc "backup MONGO_URI", "backup a db with a mongo uri eg: mongodb://user:pass@server:port/dbname"
-    def backup(mongo_uri)
+    def backup(mongo_uri, replica_set_name = nil)
       @config = MongoDbUtils::ConfigLoader.load
-      db = MongoDbUtils::Model::Db.from_uri(mongo_uri)
+
+      db = get_db(mongo_uri, replica_set_name)
       raise "can't parse uri" if db.nil?
       MongoDbUtils::Cmd.backup(db, @config.backup_folder)
     end
 
 
     desc "backup_s3 MONGO_URI BUCKET ACCESS_KEY SECRET_ACCESS_KEY", "backup a db to Amason s3 with a mongo uri eg: mongodb://user:pass@server:port/dbname"
-    def backup_s3(mongo_uri, bucket_name, access_key_id, secret_access_key)
+    def backup_s3(mongo_uri, bucket_name, access_key_id, secret_access_key, replica_set_name = nil)
       @config = MongoDbUtils::ConfigLoader.load
       backup_folder = @config.backup_folder
-      db = MongoDbUtils::Model::Db.from_uri(mongo_uri)
+      db = get_db(mongo_uri, replica_set_name)
       raise "can't parse uri" if db.nil?
       MongoDbUtils::Cmd.backup_s3(backup_folder, db, bucket_name, access_key_id, secret_access_key)
     end
 
+    private
+    def get_db(uri, name = nil)
+      if(name.nil?)
+        MongoDbUtils::Model::Db.new(uri)
+      else
+        MongoDbUtils::Models::ReplicaSetDb.new(uri, name)
+      end
+    end
+
   end
 end

data/lib/mongo-db-utils/cmd.rb

@@ -1,6 +1,5 @@
-require 'mongo-db-utils/cmd/mongotools'
+require 'mongo-db-utils/tools/commands'
 require 'mongo'
-require 'mongo/connection'
 
 module MongoDbUtils
   class Cmd
@@ -12,7 +11,7 @@ module MongoDbUtils
       end
 
       if( final_path.nil? )
-        out_path = "#{folder}/#{db.host}_#{db.port}/#{db.name}/#{timestamp}"
+        out_path = "#{folder}/#{db.to_host_s}/#{db.name}/#{timestamp}"
       else
         out_path = "#{folder}/#{final_path}"
       end
@@ -22,13 +21,12 @@ module MongoDbUtils
       puts ">> final backup path: #{full_path}"
 
       FileUtils.mkdir_p(full_path)
-      MongoDbUtils::Commands::MongoTools.dump(
-        db.host,
-        db.port,
+      MongoDbUtils::Tools::Dump.run(
+        db.to_host_s,
         db.name,
         full_path,
         db.username,
-      db.password)
+        db.password)
 
       if( tar_it )
         Dir.chdir(full_path)
@@ -41,14 +39,6 @@ module MongoDbUtils
     end
 
     # With remote dbs you can't do a copy_database if you're not an admin.
-    # so using restore instead
-    #  connection = Mongo::Connection.from_uri(destination.to_s)
-    # mongo_db = connection[destination.name]
-    # if( destination.authentication_required? )
-    #  login_result = mongo_db.authenticate(destination.username, destination.password)
-    # end
-    # host = "#{source.host}:#{source.port}"
-    # connection.copy_database(source.name, destination.name, host, source.username, source.password)
     def self.copy(path, source, destination, halt_on_no_backup = true)
 
       backup_made = backup(destination, path)
@@ -76,9 +66,8 @@ module MongoDbUtils
         password = ""
       end
 
-      MongoDbUtils::Commands::MongoTools.restore(
-        destination.host,
-        destination.port,
+      MongoDbUtils::Tools::Restore.run(
+        destination.to_host_s,
         destination.name,
         "#{tmp_dump_path}",
         username,
@@ -106,7 +95,7 @@ module MongoDbUtils
 
     def self.db_exists?(db)
       puts "DB exists? #{db.to_s}"
-      connection = Mongo::Connection.from_uri(db.to_s)
+      connection = Mongo::Connection.from_uri(db.uri)
       exists = true
       begin
         connection.ping
@@ -116,11 +105,6 @@ module MongoDbUtils
       connection.close
       exists
     end
-=begin
-       connection.database_names 
-      rescue Mongo::OperationFailure => e
-=end
-
 
   end
 end