mongo-db-utils 0.1.6 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/mongo-db-utils/cli.rb +11 -0
- data/lib/mongo-db-utils/cmd.rb +131 -27
- data/lib/mongo-db-utils/config-loader.rb +4 -3
- data/lib/mongo-db-utils/console.rb +226 -50
- data/lib/mongo-db-utils/s3.rb +43 -4
- data/lib/mongo-db-utils/tools/commands.rb +40 -38
- data/lib/mongo-db-utils/version.rb +1 -1
- data/spec/mongo_tools_cmd_spec.rb +11 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 64d078336bb048c54737b7b64e60fcf47c4f3d0e
|
4
|
+
data.tar.gz: 58354687f31dccf5167fd7b0b29137169a28d3b6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8aaf333d8260db8be954322607dbd274dc618996c0dda3af22ec2e15682bc1b3c61cddb7bae22feecc351468afafb01d22e0ef59928d9dabb83ba64c11a781f5
|
7
|
+
data.tar.gz: a2aeaf7c4bca99df2cc66c356f05a151791aff770587e6aad751b4d0012d6c1db1150c1ef90cfd0748bf01cde71ff15979916e9c9e3f73a2a82497523e0feef7
|
data/lib/mongo-db-utils/cli.rb
CHANGED
@@ -40,6 +40,17 @@ module MongoDbUtils
|
|
40
40
|
Cmd.backup_s3(backup_folder, db, bucket_name, access_key_id, secret_access_key)
|
41
41
|
end
|
42
42
|
|
43
|
+
desc "restore_from_s3 MONGO_URI BUCKET ACCESS_KEY SECRET_ACCESS_KEY SOURCE_DB [REPLICA_SET_NAME]", "restore a db from Amason s3 with a mongo uri eg: mongodb://user:pass@server:port/dbname"
|
44
|
+
def restore_from_s3(mongo_uri, bucket_name, access_key_id, secret_access_key, source_db, replica_set_name = nil)
|
45
|
+
config = get_config
|
46
|
+
db = get_db(mongo_uri, replica_set_name)
|
47
|
+
raise "can't parse uri" if db.nil?
|
48
|
+
Dir.mktmpdir do |tmp_dir|
|
49
|
+
backup = Cmd.download_backup_from_s3(tmp_dir, 'latest', bucket_name, access_key_id, secret_access_key)
|
50
|
+
Cmd.restore_from_backup(tmp_dir, db, backup, source_db)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
43
54
|
private
|
44
55
|
|
45
56
|
def get_config(path = ConfigLoader::CONFIG_LOCATION)
|
data/lib/mongo-db-utils/cmd.rb
CHANGED
@@ -8,14 +8,14 @@ module MongoDbUtils
|
|
8
8
|
|
9
9
|
def self.backup(db, folder, final_path = nil, tar_it = true)
|
10
10
|
puts ">> Backing up: #{db}, #{folder}, #{final_path}"
|
11
|
-
unless(
|
11
|
+
unless (db_exists?(db))
|
12
12
|
return false
|
13
13
|
end
|
14
14
|
|
15
|
-
if(
|
16
|
-
out_path =
|
15
|
+
if (final_path.nil?)
|
16
|
+
out_path = File.join(folder, db.to_host_s, db.name, timestamp)
|
17
17
|
else
|
18
|
-
out_path =
|
18
|
+
out_path = File.join(folder, final_path)
|
19
19
|
end
|
20
20
|
|
21
21
|
full_path = File.expand_path(out_path)
|
@@ -23,6 +23,7 @@ module MongoDbUtils
|
|
23
23
|
puts ">> final backup path: #{full_path}"
|
24
24
|
|
25
25
|
FileUtils.mkdir_p(full_path)
|
26
|
+
|
26
27
|
Dump.new(
|
27
28
|
db.to_host_s,
|
28
29
|
db.name,
|
@@ -30,42 +31,48 @@ module MongoDbUtils
|
|
30
31
|
db.username,
|
31
32
|
db.password).run
|
32
33
|
|
33
|
-
|
34
|
+
full_db_path = File.join(full_path, db.name)
|
35
|
+
|
36
|
+
if (tar_it)
|
34
37
|
Dir.chdir(full_path)
|
35
|
-
`tar
|
36
|
-
|
37
|
-
"#{
|
38
|
+
`tar --create --verbose --file=#{db.name}.tar #{db.name}`
|
39
|
+
delete_folder full_db_path
|
40
|
+
"#{full_db_path}.tar"
|
38
41
|
else
|
39
|
-
|
42
|
+
full_db_path
|
40
43
|
end
|
41
44
|
end
|
42
45
|
|
43
46
|
# With remote dbs you can't do a copy_database if you're not an admin.
|
44
47
|
def self.copy(path, source, destination, halt_on_no_backup = true, skip_backup = false)
|
45
48
|
|
46
|
-
backup_made = if skip_backup
|
49
|
+
backup_made = if skip_backup
|
50
|
+
true
|
51
|
+
else
|
52
|
+
backup(destination, path)
|
53
|
+
end
|
47
54
|
|
48
|
-
if
|
49
|
-
puts
|
55
|
+
if !backup_made && halt_on_no_backup
|
56
|
+
puts 'aborting - no backup was made'
|
50
57
|
return
|
51
58
|
end
|
52
59
|
|
53
|
-
tmp_path =
|
60
|
+
tmp_path = File.join(ConfigLoader::CONFIG_LOCATION, 'tmp')
|
54
61
|
|
55
62
|
FileUtils.mkdir_p(tmp_path)
|
56
63
|
|
57
|
-
puts "backup to: #{tmp_path
|
58
|
-
tmp_dump_path = backup(source,tmp_path, source.name, false)
|
59
|
-
|
60
|
-
username = destination.username
|
61
|
-
password = destination.password
|
64
|
+
puts "backup to: #{File.join(tmp_path, source.name)}"
|
65
|
+
tmp_dump_path = backup(source, tmp_path, source.name, false)
|
62
66
|
|
63
67
|
# if the destination db doesn't exist
|
64
68
|
# we assume that the user has admin control of the server
|
65
69
|
# eg its a local server (this needs to be thought through a bit more)
|
66
|
-
|
67
|
-
|
68
|
-
|
70
|
+
username = ''
|
71
|
+
password = ''
|
72
|
+
|
73
|
+
if db_exists? destination
|
74
|
+
username = destination.username
|
75
|
+
password = destination.password
|
69
76
|
end
|
70
77
|
|
71
78
|
Restore.new(
|
@@ -75,16 +82,62 @@ module MongoDbUtils
|
|
75
82
|
username,
|
76
83
|
password).run
|
77
84
|
|
78
|
-
|
85
|
+
delete_folder tmp_path
|
79
86
|
end
|
80
87
|
|
81
|
-
|
88
|
+
def self.backup_s3(backup_folder, db, bucket_name, access_key_id, secret_access_key)
|
82
89
|
tar_file = MongoDbUtils::Cmd.backup(db, backup_folder)
|
83
|
-
name = tar_file.gsub(File.expand_path(backup_folder),
|
90
|
+
name = tar_file.gsub(File.expand_path(backup_folder), '')
|
84
91
|
MongoDbUtils::S3::put_file(tar_file, name, bucket_name, access_key_id, secret_access_key)
|
85
92
|
file = File.basename(tar_file)
|
86
|
-
folder = tar_file.gsub(file,
|
87
|
-
|
93
|
+
folder = tar_file.gsub(file, '')
|
94
|
+
delete_folder folder
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.list_backups(bucket_name, access_key_id, secret_access_key)
|
98
|
+
MongoDbUtils::S3::list_bucket(bucket_name, access_key_id, secret_access_key)
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.list_downloaded_backups(backup_folder)
|
102
|
+
FileUtils.mkdir_p(backup_folder)
|
103
|
+
Dir.entries(backup_folder).select { |a| a.end_with? '.tgz' }
|
104
|
+
end
|
105
|
+
|
106
|
+
def self.list_backup_folders(backup_folder, backup)
|
107
|
+
puts "reading backup folders from #{backup_folder} #{backup}"
|
108
|
+
folders = list_archive_folders(backup_folder, backup)
|
109
|
+
folders.map { |a| a.slice! File.basename(backup, '.tgz'); a.delete '/' }.select { |a| a != '' }
|
110
|
+
end
|
111
|
+
|
112
|
+
def self.download_backup_from_s3(backup_folder, backup, bucket_name, access_key_id, secret_access_key)
|
113
|
+
if backup == 'latest'
|
114
|
+
backup = get_latest_backup(bucket_name, access_key_id, secret_access_key)
|
115
|
+
end
|
116
|
+
backup_file = File.join(backup_folder, File.basename(backup))
|
117
|
+
|
118
|
+
puts "download_backup_from_s3 #{backup_file}"
|
119
|
+
if File.exists? backup_file
|
120
|
+
puts "File downloaded already #{backup_file}"
|
121
|
+
else
|
122
|
+
FileUtils.mkdir_p(backup_folder)
|
123
|
+
puts "Downloading #{backup_file}"
|
124
|
+
download_backup(backup_file, backup, bucket_name, access_key_id, secret_access_key)
|
125
|
+
end
|
126
|
+
backup
|
127
|
+
end
|
128
|
+
|
129
|
+
def self.restore_from_backup(backup_folder, db, backup, source_db)
|
130
|
+
Dir.mktmpdir do |dir|
|
131
|
+
puts "Unzipping archive ..."
|
132
|
+
unzip(File.join(backup_folder, backup), dir)
|
133
|
+
puts "Restoring db ..."
|
134
|
+
restore_db(db, File.join(dir, source_db ))
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
|
139
|
+
def self.delete_backup(backup_folder, backup)
|
140
|
+
delete_folder File.join(backup_folder, backup)
|
88
141
|
end
|
89
142
|
|
90
143
|
|
@@ -92,7 +145,7 @@ module MongoDbUtils
|
|
92
145
|
|
93
146
|
def self.timestamp
|
94
147
|
t = Time.new
|
95
|
-
t.strftime(
|
148
|
+
t.strftime('%Y.%m.%d__%H.%M')
|
96
149
|
end
|
97
150
|
|
98
151
|
def self.db_exists?(db)
|
@@ -108,5 +161,56 @@ module MongoDbUtils
|
|
108
161
|
exists
|
109
162
|
end
|
110
163
|
|
164
|
+
def self.download_backup(download_file_name, backup, bucket_name, access_key_id, secret_access_key)
|
165
|
+
puts "downloading backup to #{download_file_name} from #{bucket_name} key #{backup}"
|
166
|
+
MongoDbUtils::S3::get_file(download_file_name, backup, bucket_name, access_key_id, secret_access_key)
|
167
|
+
end
|
168
|
+
|
169
|
+
def self.restore_db(destination, backup)
|
170
|
+
puts "restoring db #{destination} from #{backup}"
|
171
|
+
|
172
|
+
# if the destination db doesn't exist
|
173
|
+
# we assume that the user has admin control of the server
|
174
|
+
# eg its a local server (this needs to be thought through a bit more)
|
175
|
+
username = ''
|
176
|
+
password = ''
|
177
|
+
|
178
|
+
if db_exists? destination
|
179
|
+
username = destination.username
|
180
|
+
password = destination.password
|
181
|
+
end
|
182
|
+
|
183
|
+
Restore.new(
|
184
|
+
destination.to_host_s,
|
185
|
+
destination.name,
|
186
|
+
backup,
|
187
|
+
username,
|
188
|
+
password).run
|
189
|
+
end
|
190
|
+
|
191
|
+
def self.unzip(archive, dir)
|
192
|
+
command = "tar --extract --gzip --file=#{archive} --directory=#{dir} --strip 1"
|
193
|
+
output = `#{command} 2>&1`
|
194
|
+
raise "#{output} cmd <#{command}>" unless $?.to_i == 0
|
195
|
+
end
|
196
|
+
|
197
|
+
def self.list_archive_folders(folder, archive)
|
198
|
+
command = "tar --list --file=#{File.join(folder, archive)} | grep '/$'"
|
199
|
+
output = `#{command} 2>&1`
|
200
|
+
raise "#{output} cmd <#{command}>" unless $?.to_i == 0
|
201
|
+
output.split
|
202
|
+
end
|
203
|
+
|
204
|
+
def self.get_latest_backup(bucket_name, access_key_id, secret_access_key)
|
205
|
+
list_backups(
|
206
|
+
bucket_name,
|
207
|
+
access_key_id,
|
208
|
+
secret_access_key).select { |a| a.end_with? ".tgz" }.sort.pop
|
209
|
+
end
|
210
|
+
|
211
|
+
def self.delete_folder(path)
|
212
|
+
puts "deleting folder #{path}"
|
213
|
+
`rm --force --recursive #{path}`
|
214
|
+
end
|
111
215
|
end
|
112
216
|
end
|
@@ -8,8 +8,8 @@ module MongoDbUtils
|
|
8
8
|
|
9
9
|
class ConfigLoader
|
10
10
|
|
11
|
-
ROOT_FOLDER = "
|
12
|
-
CONFIG_LOCATION = "
|
11
|
+
ROOT_FOLDER = File.join("~",".mongo-db-utils")
|
12
|
+
CONFIG_LOCATION = File.join(ROOT_FOLDER, "config.yml")
|
13
13
|
|
14
14
|
attr_reader :config
|
15
15
|
|
@@ -33,6 +33,7 @@ module MongoDbUtils
|
|
33
33
|
end
|
34
34
|
|
35
35
|
private
|
36
|
+
|
36
37
|
def load
|
37
38
|
full_path = File.expand_path(@config_path)
|
38
39
|
puts "loading config from #{full_path}"
|
@@ -49,7 +50,7 @@ module MongoDbUtils
|
|
49
50
|
def create_fresh_install_config(full_path)
|
50
51
|
config = Model::Config.new
|
51
52
|
config.writer = self
|
52
|
-
config.backup_folder = "
|
53
|
+
config.backup_folder = File.join(ROOT_FOLDER, "backups")
|
53
54
|
initialize_files(full_path)
|
54
55
|
File.open( full_path, 'w' ) do |out|
|
55
56
|
YAML.dump( config, out )
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'highline/import'
|
2
2
|
require 'mongo-db-utils/version'
|
3
|
-
Dir['lib/mongo-db-utils/models/*.rb'].each {|file| require file.gsub("lib/", "") }
|
3
|
+
Dir['lib/mongo-db-utils/models/*.rb'].each { |file| require file.gsub("lib/", "") }
|
4
4
|
|
5
5
|
|
6
6
|
module MongoDbUtils
|
@@ -16,27 +16,27 @@ module MongoDbUtils
|
|
16
16
|
end
|
17
17
|
|
18
18
|
protected
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
19
|
+
def method_missing(name, *args, &block)
|
20
|
+
cleaned_args = args.map { |a| trim(clean(a)) }
|
21
|
+
cleaned_args.each { |a| puts "#{a} -> #{a.class}" }
|
22
|
+
@config.send(name, *cleaned_args, &block)
|
23
|
+
end
|
24
24
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
25
|
+
def clean(a)
|
26
|
+
if a.class.to_s == "HighLine::String"
|
27
|
+
a.to_s
|
28
|
+
else
|
29
|
+
a
|
30
|
+
end
|
31
|
+
end
|
32
32
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
33
|
+
def trim(s)
|
34
|
+
if (s.class.to_s == "String")
|
35
|
+
s.strip
|
36
|
+
else
|
37
|
+
s
|
38
|
+
end
|
39
|
+
end
|
40
40
|
end
|
41
41
|
|
42
42
|
|
@@ -46,7 +46,7 @@ module MongoDbUtils
|
|
46
46
|
=====================================
|
47
47
|
|| Mongo Db Utils - Version: #{MongoDbUtils::VERSION} ||
|
48
48
|
=====================================
|
49
|
-
eos
|
49
|
+
eos
|
50
50
|
|
51
51
|
|
52
52
|
def initialize(config, cmd)
|
@@ -64,14 +64,39 @@ eos
|
|
64
64
|
def main_menu
|
65
65
|
|
66
66
|
my_menu("What do you want to do?") do |menu|
|
67
|
-
menu.choice "copy a db" do
|
68
|
-
|
69
|
-
|
70
|
-
menu.choice "
|
71
|
-
|
72
|
-
|
73
|
-
menu.choice "
|
74
|
-
|
67
|
+
menu.choice "copy a db" do
|
68
|
+
copy_a_db
|
69
|
+
end
|
70
|
+
menu.choice "backup a db locally" do
|
71
|
+
do_backup
|
72
|
+
end
|
73
|
+
menu.choice "backup a db to an amazon s3 bucket" do
|
74
|
+
backup_to_s3
|
75
|
+
end
|
76
|
+
menu.choice "download a backup from an amazon s3 bucket" do
|
77
|
+
download_backup_from_s3
|
78
|
+
end
|
79
|
+
menu.choice "delete a backup from the downloads" do
|
80
|
+
delete_downloaded_backup
|
81
|
+
end
|
82
|
+
menu.choice "restore a db from a downloaded backup" do
|
83
|
+
restore_db_from_backup
|
84
|
+
end
|
85
|
+
menu.choice "remove config" do
|
86
|
+
remove_config
|
87
|
+
end
|
88
|
+
menu.choice "show config" do
|
89
|
+
show_config
|
90
|
+
end
|
91
|
+
menu.choice "add db server to config" do
|
92
|
+
add_config
|
93
|
+
end
|
94
|
+
menu.choice "add Amazon bucket to config" do
|
95
|
+
add_bucket_to_config
|
96
|
+
end
|
97
|
+
menu.choice "remove db server from config" do
|
98
|
+
remove_server_from_config
|
99
|
+
end
|
75
100
|
end
|
76
101
|
end
|
77
102
|
|
@@ -93,7 +118,6 @@ eos
|
|
93
118
|
|
94
119
|
|
95
120
|
def backup_to_s3
|
96
|
-
|
97
121
|
plan = Hash.new
|
98
122
|
|
99
123
|
db_list_menu("Choose a DB:") do |db|
|
@@ -101,7 +125,7 @@ eos
|
|
101
125
|
end
|
102
126
|
|
103
127
|
if @config.has_buckets?
|
104
|
-
|
128
|
+
bucket_list_menu("Choose a Bucket:") do |bucket|
|
105
129
|
plan[:bucket] = bucket
|
106
130
|
end
|
107
131
|
else
|
@@ -119,6 +143,91 @@ eos
|
|
119
143
|
end
|
120
144
|
|
121
145
|
|
146
|
+
def download_backup_from_s3
|
147
|
+
plan = Hash.new
|
148
|
+
|
149
|
+
if @config.has_buckets?
|
150
|
+
bucket_list_menu("Choose a source bucket:", true) do |bucket|
|
151
|
+
if bucket == 'back'
|
152
|
+
main_menu
|
153
|
+
return
|
154
|
+
end
|
155
|
+
plan[:bucket] = bucket
|
156
|
+
end
|
157
|
+
else
|
158
|
+
say("You don't have any buckets yet - add one..")
|
159
|
+
add_bucket_to_config
|
160
|
+
return
|
161
|
+
end
|
162
|
+
|
163
|
+
backups = @cmd.list_backups(
|
164
|
+
plan[:bucket].name,
|
165
|
+
plan[:bucket].access_key,
|
166
|
+
plan[:bucket].secret_key).select { |a| a.end_with? ".tgz" }
|
167
|
+
|
168
|
+
backup_list_menu("Choose a backup:", backups, true) do |backup|
|
169
|
+
if backup == 'back'
|
170
|
+
main_menu
|
171
|
+
else
|
172
|
+
plan[:backup] = backup
|
173
|
+
@cmd.download_backup_from_s3(
|
174
|
+
@config.backup_folder,
|
175
|
+
plan[:backup],
|
176
|
+
plan[:bucket].name,
|
177
|
+
plan[:bucket].access_key,
|
178
|
+
plan[:bucket].secret_key)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
|
184
|
+
def delete_downloaded_backup
|
185
|
+
plan = Hash.new
|
186
|
+
|
187
|
+
downloaded_backups_list_menu("Choose a backup:", @cmd.list_downloaded_backups(@config.backup_folder), true) do |backup|
|
188
|
+
if backup == 'back'
|
189
|
+
main_menu
|
190
|
+
return
|
191
|
+
end
|
192
|
+
if ask("Delete backup #{backup}? (Y|n) [n]") != 'Y'
|
193
|
+
main_menu
|
194
|
+
return
|
195
|
+
end
|
196
|
+
plan[:backup] = backup
|
197
|
+
@cmd.delete_backup(
|
198
|
+
@config.backup_folder,
|
199
|
+
plan[:backup])
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
|
204
|
+
def restore_db_from_backup
|
205
|
+
plan = Hash.new
|
206
|
+
|
207
|
+
db_list_menu("Choose a target DB:") do |db|
|
208
|
+
plan[:db] = db
|
209
|
+
end
|
210
|
+
|
211
|
+
downloaded_backups_list_menu("Choose a backup:", @cmd.list_downloaded_backups(@config.backup_folder)) do |backup|
|
212
|
+
plan[:backup] = backup
|
213
|
+
end
|
214
|
+
|
215
|
+
source_db_list_menu("Choose a source_db in the backup:", @cmd.list_backup_folders(@config.backup_folder, plan[:backup])) do |folder|
|
216
|
+
plan[:source_db] = folder
|
217
|
+
end
|
218
|
+
|
219
|
+
if ask("Restore db #{db}? (Y|n) [n]") != 'Y'
|
220
|
+
main_menu
|
221
|
+
return
|
222
|
+
end
|
223
|
+
|
224
|
+
@cmd.restore_from_backup(
|
225
|
+
@config.backup_folder,
|
226
|
+
plan[:db],
|
227
|
+
plan[:backup],
|
228
|
+
plan[:source_db])
|
229
|
+
end
|
230
|
+
|
122
231
|
def add_bucket_to_config
|
123
232
|
say("add an amazon bucket:")
|
124
233
|
bucket = MongoDbUtils::Model::Bucket.new
|
@@ -133,7 +242,6 @@ eos
|
|
133
242
|
my_menu("")
|
134
243
|
end
|
135
244
|
|
136
|
-
|
137
245
|
def get_config
|
138
246
|
say("You don't have any servers configured, please add one:")
|
139
247
|
add_config
|
@@ -149,7 +257,7 @@ eos
|
|
149
257
|
end
|
150
258
|
say("--------------------")
|
151
259
|
say("Amazon S3 Buckets")
|
152
|
-
if(
|
260
|
+
if (@config.buckets.nil?)
|
153
261
|
say("no buckets")
|
154
262
|
else
|
155
263
|
@config.buckets.sort.each do |bucket|
|
@@ -166,8 +274,12 @@ eos
|
|
166
274
|
|
167
275
|
def add_config
|
168
276
|
my_menu("Single db or replica set?") do |menu|
|
169
|
-
menu.choice "single db" do
|
170
|
-
|
277
|
+
menu.choice "single db" do
|
278
|
+
add_single_db
|
279
|
+
end
|
280
|
+
menu.choice "replica set db" do
|
281
|
+
add_replica_set
|
282
|
+
end
|
171
283
|
end
|
172
284
|
end
|
173
285
|
|
@@ -181,7 +293,9 @@ eos
|
|
181
293
|
label = successful ? "add another?" : "try again?"
|
182
294
|
|
183
295
|
my_menu("") do |menu|
|
184
|
-
menu.choice label do
|
296
|
+
menu.choice label do
|
297
|
+
add_config
|
298
|
+
end
|
185
299
|
end
|
186
300
|
end
|
187
301
|
|
@@ -195,11 +309,12 @@ eos
|
|
195
309
|
say("bad replica set uri") unless successful
|
196
310
|
|
197
311
|
my_menu("") do |menu|
|
198
|
-
menu.choice (successful ? "add another" : "try again?") do
|
312
|
+
menu.choice (successful ? "add another" : "try again?") do
|
313
|
+
add_config
|
314
|
+
end
|
199
315
|
end
|
200
316
|
end
|
201
317
|
|
202
|
-
|
203
318
|
def remove_server_from_config
|
204
319
|
db_list_menu("Remove server from config:") do |db|
|
205
320
|
@config.remove_db(db)
|
@@ -223,10 +338,14 @@ eos
|
|
223
338
|
say("#{plan[:source].to_s} --> #{plan[:destination].to_s}")
|
224
339
|
|
225
340
|
my_menu("") do |menu|
|
226
|
-
menu.choice "Do it!" do
|
227
|
-
|
341
|
+
menu.choice "Do it!" do
|
342
|
+
begin_copy(plan, false)
|
343
|
+
end
|
344
|
+
menu.choice "Do it - skip backup" do
|
345
|
+
begin_copy(plan, true)
|
346
|
+
end
|
228
347
|
menu.choice "Reverse" do
|
229
|
-
show_copy_plan(
|
348
|
+
show_copy_plan({:source => plan[:destination], :destination => plan[:source]})
|
230
349
|
end
|
231
350
|
end
|
232
351
|
end
|
@@ -238,16 +357,20 @@ eos
|
|
238
357
|
def my_menu(prompt, show_defaults = true)
|
239
358
|
say(prompt)
|
240
359
|
choose do |menu|
|
241
|
-
menu.shell
|
360
|
+
menu.shell = true
|
242
361
|
menu.index = :number
|
243
362
|
menu.prompt = "\n#{prompt}\n"
|
244
363
|
menu.index_suffix = ") "
|
245
364
|
menu.prompt = "?"
|
246
|
-
if(
|
247
|
-
menu.choice "Back" do
|
248
|
-
|
365
|
+
if (show_defaults)
|
366
|
+
menu.choice "Back" do
|
367
|
+
main_menu
|
368
|
+
end
|
369
|
+
menu.choice "Exit" do
|
370
|
+
say("Goodbye"); abort("--");
|
371
|
+
end
|
249
372
|
end
|
250
|
-
say("
|
373
|
+
say("")
|
251
374
|
yield menu if block_given?
|
252
375
|
end
|
253
376
|
end
|
@@ -255,19 +378,72 @@ eos
|
|
255
378
|
def db_list_menu(prompt)
|
256
379
|
my_menu(prompt, false) do |menu|
|
257
380
|
@config.dbs.sort.each do |db|
|
258
|
-
menu.choice "#{db.to_s_simple}" do
|
381
|
+
menu.choice "#{db.to_s_simple}" do
|
382
|
+
yield db if block_given?
|
383
|
+
end
|
259
384
|
end
|
260
385
|
end
|
261
386
|
end
|
262
387
|
|
263
|
-
def
|
264
|
-
my_menu(prompt) do |menu|
|
388
|
+
def bucket_list_menu(prompt, show_defaults = false)
|
389
|
+
my_menu(prompt, false) do |menu|
|
390
|
+
if show_defaults
|
391
|
+
menu.choice 'Back' do
|
392
|
+
yield 'back' if block_given?
|
393
|
+
end
|
394
|
+
end
|
265
395
|
@config.buckets.sort.each do |bucket|
|
266
|
-
menu.choice "#{bucket}" do
|
396
|
+
menu.choice "#{bucket}" do
|
397
|
+
yield bucket if block_given?
|
398
|
+
end
|
399
|
+
end
|
400
|
+
end
|
401
|
+
end
|
402
|
+
|
403
|
+
def backup_list_menu(prompt, backups, show_defaults = false)
|
404
|
+
my_menu(prompt, false) do |menu|
|
405
|
+
if show_defaults
|
406
|
+
menu.choice 'Back' do
|
407
|
+
yield 'back' if block_given?
|
408
|
+
end
|
409
|
+
end
|
410
|
+
menu.choice 'latest' do
|
411
|
+
yield 'latest' if block_given?
|
412
|
+
end
|
413
|
+
backups.sort.each do |backup|
|
414
|
+
menu.choice "#{backup}" do
|
415
|
+
yield backup if block_given?
|
416
|
+
end
|
417
|
+
end
|
418
|
+
end
|
419
|
+
end
|
420
|
+
|
421
|
+
def source_db_list_menu(prompt, folders)
|
422
|
+
my_menu(prompt, false) do |menu|
|
423
|
+
folders.sort.each do |item|
|
424
|
+
menu.choice "#{item}" do
|
425
|
+
yield item if block_given?
|
426
|
+
end
|
267
427
|
end
|
268
428
|
end
|
269
429
|
end
|
270
430
|
|
431
|
+
def downloaded_backups_list_menu(prompt, backups, show_defaults = false)
|
432
|
+
my_menu(prompt, false) do |menu|
|
433
|
+
if show_defaults
|
434
|
+
menu.choice 'Back' do
|
435
|
+
yield 'back' if block_given?
|
436
|
+
end
|
437
|
+
end
|
438
|
+
backups.sort.each do |item|
|
439
|
+
menu.choice "#{item}" do
|
440
|
+
yield item if block_given?
|
441
|
+
end
|
442
|
+
end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
|
446
|
+
|
271
447
|
private
|
272
448
|
def clean(s)
|
273
449
|
s.to_s.strip
|
data/lib/mongo-db-utils/s3.rb
CHANGED
@@ -6,10 +6,8 @@ module MongoDbUtils
|
|
6
6
|
|
7
7
|
def self.put_file(file, name, bucket_name, access_key_id, secret_access_key)
|
8
8
|
puts "putting file to Amazon S3"
|
9
|
-
|
10
|
-
|
11
|
-
:secret_access_key => secret_access_key
|
12
|
-
)
|
9
|
+
|
10
|
+
self.s3connect(access_key_id, secret_access_key)
|
13
11
|
|
14
12
|
begin
|
15
13
|
AWS::S3::Bucket.find(bucket_name)
|
@@ -21,5 +19,46 @@ module MongoDbUtils
|
|
21
19
|
end
|
22
20
|
AWS::S3::S3Object.store(name, open(file), bucket_name)
|
23
21
|
end
|
22
|
+
|
23
|
+
|
24
|
+
def self.get_file(filename, key, bucket_name, access_key_id, secret_access_key)
|
25
|
+
puts "getting file from Amazon S3"
|
26
|
+
|
27
|
+
self.s3connect(access_key_id, secret_access_key)
|
28
|
+
|
29
|
+
File.open(filename, 'wb') do |file|
|
30
|
+
AWS::S3::S3Object.stream(key, bucket_name) do |chunk|
|
31
|
+
file.write chunk
|
32
|
+
end
|
33
|
+
file.close
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
def self.list_bucket(bucket_name, access_key_id, secret_access_key)
|
39
|
+
puts "getting list of bucket keys from Amazon S3"
|
40
|
+
|
41
|
+
self.s3connect(access_key_id, secret_access_key)
|
42
|
+
|
43
|
+
begin
|
44
|
+
AWS::S3::Bucket.find(bucket_name).objects.collect(&:key)
|
45
|
+
rescue AWS::S3::NoSuchBucket
|
46
|
+
puts "Error:: Bucket does not exist: #{bucket_name}"
|
47
|
+
return nil
|
48
|
+
rescue AWS::S3::AllAccessDisabled
|
49
|
+
puts "Error:: You cannot access this bucket: #{bucket_name}"
|
50
|
+
return nil
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def self.s3connect(access_key_id, secret_access_key)
|
57
|
+
AWS::S3::Base.establish_connection!(
|
58
|
+
:access_key_id => access_key_id,
|
59
|
+
:secret_access_key => secret_access_key
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
24
63
|
end
|
25
64
|
end
|
@@ -6,39 +6,41 @@ module MongoDbUtils
|
|
6
6
|
|
7
7
|
class ToolsException < RuntimeError
|
8
8
|
attr :cmd, :output
|
9
|
-
def initialize(cmd, output)
|
9
|
+
def initialize(cmd, output, message)
|
10
|
+
super(message)
|
10
11
|
@cmd = cmd
|
11
|
-
@output =output
|
12
|
+
@output = output
|
12
13
|
end
|
14
|
+
|
13
15
|
end
|
14
16
|
|
15
17
|
class BaseCmd
|
16
18
|
|
17
|
-
def initialize(cmd_name, host_and_port, db, username =
|
18
|
-
@
|
19
|
-
@
|
19
|
+
def initialize(cmd_name, host_and_port, db, username = '', password = '')
|
20
|
+
@unsafe_options = build_base_options(host_and_port, db, username, password)
|
21
|
+
@options_without_credentials = build_base_options(host_and_port, db, username.empty? ? '':'****', password.empty? ? '':'****')
|
20
22
|
@cmd_name = cmd_name
|
21
23
|
end
|
22
24
|
|
23
25
|
def run
|
24
26
|
puts "[#{self.class}] run: #{cmd}"
|
25
|
-
output = `#{
|
26
|
-
raise ToolsException.new("#{cmd}", output) unless $?.to_i == 0
|
27
|
+
output = `#{executable_cmd} 2>&1`
|
28
|
+
raise ToolsException.new("#{cmd}", output, "Error in #{cmd_name}:: #{output} cmd #{cmd}") unless $?.to_i == 0
|
27
29
|
end
|
28
30
|
|
29
31
|
def cmd
|
30
|
-
"#{@cmd_name} #{options_string(@
|
32
|
+
"#{@cmd_name} #{options_string(@options_without_credentials)}"
|
31
33
|
end
|
32
34
|
|
33
|
-
def
|
34
|
-
"#{@cmd_name} #{options_string(@
|
35
|
+
def executable_cmd
|
36
|
+
"#{@cmd_name} #{options_string(@unsafe_options)}"
|
35
37
|
end
|
36
38
|
|
37
39
|
private
|
38
40
|
|
39
|
-
def
|
40
|
-
@
|
41
|
-
@
|
41
|
+
def add_option(option)
|
42
|
+
@unsafe_options << option
|
43
|
+
@options_without_credentials << option
|
42
44
|
end
|
43
45
|
|
44
46
|
def o(key,value)
|
@@ -46,33 +48,33 @@ module MongoDbUtils
|
|
46
48
|
end
|
47
49
|
|
48
50
|
# options common to all commands
|
49
|
-
def build_base_options(host_and_port,db,username=
|
51
|
+
def build_base_options(host_and_port,db,username='',password='')
|
50
52
|
options = []
|
51
|
-
options << o(
|
52
|
-
options << o(
|
53
|
-
options << o(
|
54
|
-
options << o(
|
53
|
+
options << o('-h', host_and_port)
|
54
|
+
options << o('-db', db)
|
55
|
+
options << o('-u', username)
|
56
|
+
options << o('-p', password)
|
55
57
|
options
|
56
58
|
end
|
57
59
|
|
58
60
|
# given an array of options build a string of those options unless the option is empty
|
59
61
|
def options_string(opts)
|
60
62
|
opt_strings = opts.reject{ |o| o.empty? }.map { |o| o.to_s }
|
61
|
-
opt_strings.join(
|
63
|
+
opt_strings.join(' ').strip
|
62
64
|
end
|
63
65
|
end
|
64
66
|
|
65
67
|
class Dump < BaseCmd
|
66
|
-
def initialize(host_and_port,db,output,username =
|
67
|
-
super(
|
68
|
-
|
68
|
+
def initialize(host_and_port,db,output,username = '', password = '')
|
69
|
+
super('mongodump', host_and_port, db, username, password)
|
70
|
+
add_option(o('-o', output))
|
69
71
|
end
|
70
72
|
end
|
71
73
|
|
72
74
|
class Restore < BaseCmd
|
73
|
-
def initialize(host_and_port,db,source_folder,username =
|
74
|
-
super(
|
75
|
-
|
75
|
+
def initialize(host_and_port,db,source_folder,username = '', password = '')
|
76
|
+
super('mongorestore', host_and_port, db, username, password)
|
77
|
+
add_option('--drop')
|
76
78
|
@source_folder = source_folder
|
77
79
|
end
|
78
80
|
|
@@ -80,29 +82,29 @@ module MongoDbUtils
|
|
80
82
|
"#{super} #{@source_folder}"
|
81
83
|
end
|
82
84
|
|
83
|
-
def
|
85
|
+
def executable_cmd
|
84
86
|
"#{super} #{@source_folder}"
|
85
87
|
end
|
86
88
|
end
|
87
89
|
|
88
90
|
class Import < BaseCmd
|
89
|
-
def initialize(host_and_port, db, collection, file, username =
|
90
|
-
super(
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
91
|
+
def initialize(host_and_port, db, collection, file, username = '', password = '', opts = {})
|
92
|
+
super('mongoimport', host_and_port, db, username, password)
|
93
|
+
add_option(o('-c', collection))
|
94
|
+
add_option(o('--file', file))
|
95
|
+
add_option('--jsonArray') if opts[:json_array]
|
96
|
+
add_option('--drop') if opts[:drop]
|
95
97
|
end
|
96
98
|
end
|
97
99
|
|
98
100
|
class Export < BaseCmd
|
99
101
|
|
100
|
-
def initialize(host_and_port, db, collection, query, output, username =
|
101
|
-
super(
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
102
|
+
def initialize(host_and_port, db, collection, query, output, username = '', password = '', opts = {})
|
103
|
+
super('mongoexport', host_and_port, db, username, password)
|
104
|
+
add_option(o('-c', collection))
|
105
|
+
add_option(o('-o', output))
|
106
|
+
add_option(o('--query', "'#{query}'"))
|
107
|
+
add_option('--jsonArray') if opts[:json_array]
|
106
108
|
end
|
107
109
|
end
|
108
110
|
|
@@ -9,11 +9,11 @@ describe Dump do
|
|
9
9
|
end
|
10
10
|
|
11
11
|
it "should work with single uris" do
|
12
|
-
Dump.new("host:port", "db", "out", "user", "pass").
|
12
|
+
Dump.new("host:port", "db", "out", "user", "pass").executable_cmd.should eql "mongodump -h host:port -db db -u user -p pass -o out"
|
13
13
|
end
|
14
14
|
|
15
15
|
it "should work with single uris - no user/pass" do
|
16
|
-
Dump.new("host:port", "db", "out").
|
16
|
+
Dump.new("host:port", "db", "out").executable_cmd.should eql "mongodump -h host:port -db db -o out"
|
17
17
|
end
|
18
18
|
|
19
19
|
it "should safely work with replica set uris" do
|
@@ -23,7 +23,7 @@ describe Dump do
|
|
23
23
|
|
24
24
|
it "should work with replica set uris" do
|
25
25
|
expected = "mongodump -h setname/host1:port1,host2:port2 -db db -u user -p pass -o out"
|
26
|
-
Dump.new("setname/host1:port1,host2:port2","db", "out", "user", "pass").
|
26
|
+
Dump.new("setname/host1:port1,host2:port2","db", "out", "user", "pass").executable_cmd.should eql expected
|
27
27
|
end
|
28
28
|
|
29
29
|
end
|
@@ -35,7 +35,7 @@ describe Restore do
|
|
35
35
|
end
|
36
36
|
|
37
37
|
it "should work with single uris" do
|
38
|
-
Restore.new("host:port", "db", "source", "user", "pass").
|
38
|
+
Restore.new("host:port", "db", "source", "user", "pass").executable_cmd.should eql "mongorestore -h host:port -db db -u user -p pass --drop source"
|
39
39
|
end
|
40
40
|
|
41
41
|
it "should safely work with replica set uris" do
|
@@ -45,7 +45,7 @@ describe Restore do
|
|
45
45
|
|
46
46
|
it "should work with replica set uris" do
|
47
47
|
expected = "mongorestore -h setname/host1:port1,host2:port2 -db db -u user -p pass --drop source"
|
48
|
-
Restore.new("setname/host1:port1,host2:port2","db", "source", "user", "pass").
|
48
|
+
Restore.new("setname/host1:port1,host2:port2","db", "source", "user", "pass").executable_cmd.should eql expected
|
49
49
|
end
|
50
50
|
end
|
51
51
|
|
@@ -56,11 +56,11 @@ describe Import do
|
|
56
56
|
end
|
57
57
|
|
58
58
|
it "should work with single uris - with user/pass" do
|
59
|
-
Import.new("host:port", "db", "coll", "myfile.json", "user", "pass").
|
59
|
+
Import.new("host:port", "db", "coll", "myfile.json", "user", "pass").executable_cmd.should eql "mongoimport -h host:port -db db -u user -p pass -c coll --file myfile.json"
|
60
60
|
end
|
61
61
|
|
62
62
|
it "should work with single uris - no user/pass" do
|
63
|
-
Import.new("host:port", "db", "coll", "myfile.json").
|
63
|
+
Import.new("host:port", "db", "coll", "myfile.json").executable_cmd.should eql "mongoimport -h host:port -db db -c coll --file myfile.json"
|
64
64
|
end
|
65
65
|
|
66
66
|
it "should safely work with replica set uris" do
|
@@ -70,7 +70,7 @@ describe Import do
|
|
70
70
|
|
71
71
|
it "should work with replica set uris" do
|
72
72
|
expected = "mongoimport -h setname/host1:port1,host2:port2 -db db -u user -p pass -c coll --file myfile.json --jsonArray"
|
73
|
-
Import.new("setname/host1:port1,host2:port2","db", "coll", "myfile.json", "user", "pass", { :json_array => true} ).
|
73
|
+
Import.new("setname/host1:port1,host2:port2","db", "coll", "myfile.json", "user", "pass", { :json_array => true} ).executable_cmd.should eql expected
|
74
74
|
end
|
75
75
|
end
|
76
76
|
|
@@ -81,11 +81,11 @@ describe Export do
|
|
81
81
|
end
|
82
82
|
|
83
83
|
it "should work with single uris - with user/pass" do
|
84
|
-
Export.new("host:port", "db", "coll", "{query}", "myfile.json", "user", "pass").
|
84
|
+
Export.new("host:port", "db", "coll", "{query}", "myfile.json", "user", "pass").executable_cmd.should eql "mongoexport -h host:port -db db -u user -p pass -c coll -o myfile.json --query '{query}'"
|
85
85
|
end
|
86
86
|
|
87
87
|
it "should work with single uris - no user/pass" do
|
88
|
-
Export.new("host:port", "db", "coll", "{query}", "myfile.json").
|
88
|
+
Export.new("host:port", "db", "coll", "{query}", "myfile.json").executable_cmd.should eql "mongoexport -h host:port -db db -c coll -o myfile.json --query '{query}'"
|
89
89
|
end
|
90
90
|
|
91
91
|
it "should safely work with replica set uris" do
|
@@ -95,7 +95,7 @@ describe Export do
|
|
95
95
|
|
96
96
|
it "should work with replica set uris" do
|
97
97
|
expected = "mongoexport -h setname/host1:port1,host2:port2 -db db -u user -p pass -c coll -o myfile.json --query '{query}' --jsonArray"
|
98
|
-
Export.new("setname/host1:port1,host2:port2","db", "coll", "{query}", "myfile.json", "user", "pass", { :json_array => true} ).
|
98
|
+
Export.new("setname/host1:port1,host2:port2","db", "coll", "{query}", "myfile.json", "user", "pass", { :json_array => true} ).executable_cmd.should eql expected
|
99
99
|
end
|
100
100
|
end
|
101
101
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo-db-utils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- edeustace
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-10-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|