mongo-db-utils 0.0.9 → 0.0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 60f9d5dc43d84806e0284ab99d3018d0f7a6013f
4
+ data.tar.gz: 41d85fc792b941ebe3ff901a2add2f09643bfbbc
5
+ SHA512:
6
+ metadata.gz: dac4c457c4f95c22b32f5122b91c4f920f11f72687bc41c8702f3e1a96e107a64e82cc6dca0ffaea88b4d7a3a3b6d7c133290c7484dacf86dfb0dcc55ec4dc78
7
+ data.tar.gz: ebad5fca30541d643d4ddfcfc2311ddc300dcefe215cfc0e28ec7d77c50628b98fb79707aa54ecb18799e87af2b6f15044db04fe712a82271b725cb0cb5ecd99
data/.gitignore CHANGED
@@ -15,3 +15,4 @@ spec/reports
15
15
  test/tmp
16
16
  test/version_tmp
17
17
  tmp
18
+ .tmp_path
data/Gemfile CHANGED
@@ -1,9 +1,4 @@
1
1
  source 'https://rubygems.org'
2
-
3
2
  # Specify your gem's dependencies in mongo-db-utils.gemspec
4
3
  gemspec
5
- #RUBY_PATCHLEVEL = "194"
6
- #gem 'linecache19', '0.5.13', :path => "~/.rvm/gems/ruby-1.9.3-p#{RUBY_PATCHLEVEL}/gems/linecache19-0.5.13/"
7
- #gem 'ruby-debug-base19', '0.11.26', :path => "~/.rvm/gems/ruby-1.9.3-p#{RUBY_PATCHLEVEL}/gems/ruby-debug-base19-0.11.26/"
8
- #gem 'ruby-debug19', :require => 'ruby-debug'
9
4
 
data/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # MongoDbUtils
2
2
 
3
+ ### !Current version 0.0.9.2 is in Beta - for a safer version use 0.0.9
4
+
3
5
  A little gem that simplifies backing up and copying your mongo dbs.
4
6
 
5
7
  You can run as a script (eg for cron jobs, or in interactive mode):
@@ -32,18 +34,31 @@ Once you launch the console it'll provide you with a set of options - pretty sel
32
34
  When it does backups it stores them in ````~/.mongo-db-utils/backups/````. The naming convention is ````${server}_${port}/${database_name}/${timestamp}/db````
33
35
 
34
36
  ## Testing
35
-
37
+
36
38
  bundle exec rspec spec
37
39
 
38
40
  #cucumber can't handle interactive CLIs so need to wait on this.
39
41
  #bundle exec cucumber features
40
-
42
+
41
43
  ## Building source
42
44
 
43
45
  #run console
44
- bundle exec bin/mongo-db-utils console
46
+ bundle exec bin/mongo-db-utils console path_to/config.yml (optional)
45
47
 
46
48
  #install the gem locally
47
49
  rake build
48
50
  gem install pkg/mongo-db-utils.gem
49
51
 
52
+
53
+
54
+ ## Release Notes
55
+
56
+ * 0.0.9.2 - BETA!
57
+ - Added support for Replica Sets
58
+ - console can be run pointing to any config file: `console myconfig.yml`
59
+ - More specs
60
+ - Added local mongo environment to simplify testing @see: integration-test-env
61
+
62
+ * 0.0.9 - First release
63
+ - Copy mongo dbs
64
+ - Back up to S3
@@ -0,0 +1,4 @@
1
+ .processes
2
+ dbs
3
+ logs
4
+ rs_config
@@ -0,0 +1,29 @@
1
+ # Integration Test Environment
2
+
3
+ This folder contains utilities for setting up a mongo environment and running the console against it.
4
+ This is only really useful if you are working on the source code. If you're just using the gem you can ignore this stuff.
5
+
6
+ ## Available Environments
7
+
8
+ ### Standalone DB
9
+
10
+ * init the environment `create_two_standalone_dbs` - this creates 2 mongod instances on 27018/27019
11
+ * To run the console go:
12
+
13
+ bundle exec bin/mongo-db-utils console integration-test-env/standalone/config.yml
14
+
15
+ * kill the environment `kill_processes standalone/.processes`
16
+
17
+
18
+ ### Replica Set DBs
19
+
20
+ * `create_two_replica_sets`
21
+ * `replica_sets/rs_config` - you'll probably have to run this a few times
22
+ * `seed_replica_sets`
23
+ * run the console:
24
+
25
+ bundle exec bin/mongo-db-utils console integration-test-env/replica_sets/config.yml
26
+
27
+ * kill the environment `kill_processes replica_sets/.processes`
28
+
29
+ ....
@@ -0,0 +1,76 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative './helper'
4
+ require 'yaml'
5
+ require 'fileutils'
6
+ require 'mongo'
7
+ =begin
8
+ mkdir -p /srv/mongodb/rs0-0 /srv/mongodb/rs0-1 /srv/mongodb/rs0-2
9
+ Issue the following commands, each in a distinct screen window:
10
+ mongod --port 27017 --dbpath /srv/mongodb/rs0-0 --replSet rs0 --smallfiles --oplogSize 128
11
+ mongod --port 27018 --dbpath /srv/mongodb/rs0-1 --replSet rs0 --smallfiles --oplogSize 128
12
+ mongod --port 27019 --dbpath /srv/mongodb/rs0-2 --replSet rs0 --smallfiles --oplogSize 128
13
+ =end
14
+
15
+ ROOT_FOLDER = "./replica_sets"
16
+ maker = MongoEnvMaker.new(ROOT_FOLDER)
17
+
18
+ puts "Creating 2 replica sets: 27020-27022 & localhost:27023-27025"
19
+ puts "create the directories"
20
+
21
+ maker.spawn(27020, "rs0")
22
+ maker.spawn(27021, "rs0")
23
+ maker.spawn(27022, "rs0")
24
+
25
+ maker.spawn(27023, "rs1")
26
+ maker.spawn(27024, "rs1")
27
+ maker.spawn(27025, "rs1")
28
+
29
+ puts ">>>>>>>>>>> pid: #{maker.pids}"
30
+
31
+ File.open( File.expand_path("./#{ROOT_FOLDER}/.processes"), 'w' ) do |out|
32
+ YAML.dump( maker.pids, out )
33
+ end
34
+
35
+ def initiate_conf(host, set_name, port)
36
+ "rs.initiate({_id: '#{set_name}', members: [{_id: 0, host: '#{host}:#{port}'} ] });"
37
+ end
38
+
39
+ def add_member(host, port)
40
+ "rs.add('#{host}:#{port}')"
41
+ end
42
+
43
+ def shellscript( base_port, commands )
44
+ sleep_between_add_members = 4
45
+ evals = commands.map{ |cmd| "mongo --port #{base_port} --eval \"#{cmd}\"\nsleep #{sleep_between_add_members}\n"}
46
+ out = <<-eos
47
+ #{evals.join("\n")}
48
+ eos
49
+ out
50
+ end
51
+
52
+ def wrap(s)
53
+ <<-eos
54
+ #!/usr/bin/env bash
55
+ #{s}
56
+ eos
57
+ end
58
+
59
+ set_one = shellscript( 27020,
60
+ [
61
+ initiate_conf("localhost", "rs0", 27020),
62
+ add_member("localhost", 27021),
63
+ add_member("localhost", 27022) ])
64
+
65
+ set_two = shellscript( 27023,
66
+ [initiate_conf("localhost", "rs1", 27023),
67
+ add_member("localhost", 27024),
68
+ add_member("localhost", 27025)] )
69
+
70
+ path = "#{ROOT_FOLDER}/rs_config"
71
+
72
+ File.open(path, 'w'){ |f| f.write( wrap("#{set_one}\n#{set_two}"))}
73
+
74
+ FileUtils.chmod 0755, path
75
+
76
+ puts "Done --- Important - you now need to run: #{path} - note you may have to run it a couple of times"
@@ -0,0 +1,27 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative './helper'
4
+ require 'yaml'
5
+ require 'fileutils'
6
+ require 'mongo'
7
+
8
+ ROOT_FOLDER = "./standalone"
9
+ maker = MongoEnvMaker.new(ROOT_FOLDER)
10
+
11
+ puts "Creating 2 standalone dbs: localhost:27018 & localhost:27019"
12
+ puts "create the directories"
13
+
14
+ maker.spawn(27018)
15
+ maker.spawn(27019)
16
+
17
+ puts ">> pids: #{maker.pids}"
18
+
19
+ File.open( File.expand_path("./#{ROOT_FOLDER}/.processes"), 'w' ) do |out|
20
+ YAML.dump( maker.pids, out )
21
+ end
22
+
23
+ sleep 2
24
+
25
+ puts "Seed 27018 with some data..."
26
+ DummyData.seed("localhost", 27018)
27
+
@@ -0,0 +1,50 @@
1
+ require 'yaml'
2
+ require 'fileutils'
3
+ require 'mongo'
4
+
5
+ class MongoEnvMaker
6
+
7
+ attr_reader :pids
8
+
9
+ def initialize(root_folder)
10
+ @root_folder = root_folder
11
+ @pids = []
12
+ end
13
+
14
+
15
+ def spawn(port, rs_name = nil)
16
+ `mkdir -p #{@root_folder}/logs/#{port}`
17
+ out_log = "#{@root_folder}/logs/#{port}/out.log"
18
+ err_log = "#{@root_folder}/logs/#{port}/err.log"
19
+ pid = Process.spawn mongod(port, rs_name), :out=> out_log, :err => err_log
20
+ @pids << pid
21
+ end
22
+
23
+ private
24
+ def mongod(port, set_name = nil)
25
+ path = mk_db_dir(port)
26
+ out = "mongod --port #{port} --dbpath #{path} --smallfiles --oplogSize 128"
27
+ out << " --replSet #{set_name}" unless set_name.nil?
28
+ out
29
+ end
30
+
31
+ def mk_db_dir(port)
32
+ path = "#{@root_folder}/dbs/#{port}"
33
+ FileUtils.rm_rf path
34
+ `mkdir -p #{path}`
35
+ path
36
+ end
37
+ end
38
+
39
+
40
+ class DummyData
41
+ def self.seed(host, port)
42
+ puts "[DummyData] Seed #{host}:#{port} with some dummy data"
43
+ include Mongo
44
+ mongo_client = MongoClient.new(host, port)
45
+ db = mongo_client.db("dummy")
46
+ coll = db.collection("some_collection")
47
+ doc = {"name" => "MongoDB", "type" => "database", "count" => 1, "info" => {"x" => 203, "y" => '102'}}
48
+ id = coll.insert(doc)
49
+ end
50
+ end
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'yaml'
4
+ require 'fileutils'
5
+
6
+ path = ARGV[0]
7
+
8
+ puts "kill processes: #{path}"
9
+
10
+ full_path = File.expand_path(path)
11
+
12
+ if File.exist?(full_path) && YAML.load(File.open(full_path))
13
+ pids = YAML.load(File.open(full_path))
14
+ puts "found pids: #{pids}"
15
+ pids.each{ |pid| `kill -9 #{pid}` }
16
+ FileUtils.rm full_path
17
+ else
18
+ puts "couldn't find path"
19
+ end
@@ -0,0 +1,21 @@
1
+ --- &1 !ruby/object:MongoDbUtils::Model::Config
2
+ dbs:
3
+ - !ruby/object:MongoDbUtils::Model::ReplicaSetDb
4
+ host_port: localhost:27020,localhost:27021,localhost:27022
5
+ name: dummy
6
+ username: ''
7
+ password: ''
8
+ uri: mongodb://localhost:27020,localhost:27021,localhost:27022/dummy
9
+ set_name: rs0
10
+ - !ruby/object:MongoDbUtils::Model::ReplicaSetDb
11
+ host_port: localhost:27023,localhost:27024,localhost:27025
12
+ name: dummy
13
+ username: ''
14
+ password: ''
15
+ uri: mongodb://localhost:27023,localhost:27024,localhost:27025/dummy
16
+ set_name: rs1
17
+ buckets: []
18
+ writer: !ruby/object:MongoDbUtils::ConfigLoader
19
+ config_path: integration-test-env/replica_sets/config.yml
20
+ config: *1
21
+ backup_folder: ~/.mongo-db-utils/backups
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'mongo'
4
+
5
+ include Mongo
6
+
7
+ dbs = (27020..27022).to_a.map{ |n| "localhost:#{n}"}
8
+ mongo_client = MongoReplicaSetClient.new(dbs)
9
+
10
+ db = mongo_client.db("dummy")
11
+ coll = db.collection("some_collection")
12
+ doc = {"name" => "MongoDB", "type" => "database", "count" => 1, "info" => {"x" => 203, "y" => '102'}}
13
+ id = coll.insert(doc)
14
+
15
+ puts "successfully inserted: #{id}"
@@ -0,0 +1,19 @@
1
+ --- &1 !ruby/object:MongoDbUtils::Model::Config
2
+ dbs:
3
+ - !ruby/object:MongoDbUtils::Model::Db
4
+ host_port: localhost:27018
5
+ name: dummy
6
+ username: ''
7
+ password: ''
8
+ uri: mongodb://localhost:27018/dummy
9
+ - !ruby/object:MongoDbUtils::Model::Db
10
+ host_port: localhost:27019
11
+ name: dummy
12
+ username: ''
13
+ password: ''
14
+ uri: mongodb://localhost:27019/dummy
15
+ buckets: []
16
+ writer: !ruby/object:MongoDbUtils::ConfigLoader
17
+ config_path: integration-test-env/standalone/config.yml
18
+ config: *1
19
+ backup_folder: ~/.mongo-db-utils/backups
@@ -2,36 +2,53 @@ require 'thor'
2
2
  require 'mongo-db-utils/config-loader'
3
3
  require 'mongo-db-utils/cmd'
4
4
  require 'mongo-db-utils/console'
5
- require 'mongo-db-utils/models'
5
+
6
+ Dir['lib/mongo-db-utils/models/*.rb'].each {|file| require file.gsub("lib/", "") }
6
7
  require 'mongo-db-utils/s3'
7
8
 
8
9
  module MongoDbUtils
9
10
  class CLI < Thor
10
11
 
11
- desc "console", "run the interactive console"
12
- def console
13
- @config = MongoDbUtils::ConfigLoader.load
14
- console = MongoDbUtils::Console.new(@config, MongoDbUtils::Cmd)
15
- console.run
12
+ desc "console", "run the interactive console @param path - path to config file"
13
+ def console(path = MongoDbUtils::ConfigLoader::CONFIG_LOCATION)
14
+
15
+ if File.extname(path) != ".yml"
16
+ puts "Error: You must use a yaml file as your config file location"
17
+ else
18
+ @loader = MongoDbUtils::ConfigLoader.new(path)
19
+ @config = @loader.config
20
+ console = MongoDbUtils::Console.new(@config, MongoDbUtils::Cmd)
21
+ console.run
22
+ end
16
23
  end
17
24
 
18
25
  desc "backup MONGO_URI", "backup a db with a mongo uri eg: mongodb://user:pass@server:port/dbname"
19
- def backup(mongo_uri)
26
+ def backup(mongo_uri, replica_set_name = nil)
20
27
  @config = MongoDbUtils::ConfigLoader.load
21
- db = MongoDbUtils::Model::Db.from_uri(mongo_uri)
28
+
29
+ db = get_db(mongo_uri, replica_set_name)
22
30
  raise "can't parse uri" if db.nil?
23
31
  MongoDbUtils::Cmd.backup(db, @config.backup_folder)
24
32
  end
25
33
 
26
34
 
27
35
  desc "backup_s3 MONGO_URI BUCKET ACCESS_KEY SECRET_ACCESS_KEY", "backup a db to Amason s3 with a mongo uri eg: mongodb://user:pass@server:port/dbname"
28
- def backup_s3(mongo_uri, bucket_name, access_key_id, secret_access_key)
36
+ def backup_s3(mongo_uri, bucket_name, access_key_id, secret_access_key, replica_set_name = nil)
29
37
  @config = MongoDbUtils::ConfigLoader.load
30
38
  backup_folder = @config.backup_folder
31
- db = MongoDbUtils::Model::Db.from_uri(mongo_uri)
39
+ db = get_db(mongo_uri, replica_set_name)
32
40
  raise "can't parse uri" if db.nil?
33
41
  MongoDbUtils::Cmd.backup_s3(backup_folder, db, bucket_name, access_key_id, secret_access_key)
34
42
  end
35
43
 
44
+ private
45
+ def get_db(uri, name = nil)
46
+ if(name.nil?)
47
+ MongoDbUtils::Model::Db.new(uri)
48
+ else
49
+ MongoDbUtils::Models::ReplicaSetDb.new(uri, name)
50
+ end
51
+ end
52
+
36
53
  end
37
54
  end
@@ -1,6 +1,5 @@
1
- require 'mongo-db-utils/cmd/mongotools'
1
+ require 'mongo-db-utils/tools/commands'
2
2
  require 'mongo'
3
- require 'mongo/connection'
4
3
 
5
4
  module MongoDbUtils
6
5
  class Cmd
@@ -12,7 +11,7 @@ module MongoDbUtils
12
11
  end
13
12
 
14
13
  if( final_path.nil? )
15
- out_path = "#{folder}/#{db.host}_#{db.port}/#{db.name}/#{timestamp}"
14
+ out_path = "#{folder}/#{db.to_host_s}/#{db.name}/#{timestamp}"
16
15
  else
17
16
  out_path = "#{folder}/#{final_path}"
18
17
  end
@@ -22,13 +21,12 @@ module MongoDbUtils
22
21
  puts ">> final backup path: #{full_path}"
23
22
 
24
23
  FileUtils.mkdir_p(full_path)
25
- MongoDbUtils::Commands::MongoTools.dump(
26
- db.host,
27
- db.port,
24
+ MongoDbUtils::Tools::Dump.run(
25
+ db.to_host_s,
28
26
  db.name,
29
27
  full_path,
30
28
  db.username,
31
- db.password)
29
+ db.password)
32
30
 
33
31
  if( tar_it )
34
32
  Dir.chdir(full_path)
@@ -41,14 +39,6 @@ module MongoDbUtils
41
39
  end
42
40
 
43
41
  # With remote dbs you can't do a copy_database if you're not an admin.
44
- # so using restore instead
45
- # connection = Mongo::Connection.from_uri(destination.to_s)
46
- # mongo_db = connection[destination.name]
47
- # if( destination.authentication_required? )
48
- # login_result = mongo_db.authenticate(destination.username, destination.password)
49
- # end
50
- # host = "#{source.host}:#{source.port}"
51
- # connection.copy_database(source.name, destination.name, host, source.username, source.password)
52
42
  def self.copy(path, source, destination, halt_on_no_backup = true)
53
43
 
54
44
  backup_made = backup(destination, path)
@@ -76,9 +66,8 @@ module MongoDbUtils
76
66
  password = ""
77
67
  end
78
68
 
79
- MongoDbUtils::Commands::MongoTools.restore(
80
- destination.host,
81
- destination.port,
69
+ MongoDbUtils::Tools::Restore.run(
70
+ destination.to_host_s,
82
71
  destination.name,
83
72
  "#{tmp_dump_path}",
84
73
  username,
@@ -106,7 +95,7 @@ module MongoDbUtils
106
95
 
107
96
  def self.db_exists?(db)
108
97
  puts "DB exists? #{db.to_s}"
109
- connection = Mongo::Connection.from_uri(db.to_s)
98
+ connection = Mongo::Connection.from_uri(db.uri)
110
99
  exists = true
111
100
  begin
112
101
  connection.ping
@@ -116,11 +105,6 @@ module MongoDbUtils
116
105
  connection.close
117
106
  exists
118
107
  end
119
- =begin
120
- connection.database_names
121
- rescue Mongo::OperationFailure => e
122
- =end
123
-
124
108
 
125
109
  end
126
110
  end